5471 lines
259 KiB
Python
5471 lines
259 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
neopig SERP - Search Engine Results Page
|
||
|
||
Fast image/video search across hydrated metadata.
|
||
Serves files directly from filevault via Caddy with 1GB memory cache.
|
||
|
||
Search across:
|
||
- keywords (crawl tags)
|
||
- alt_text (image alt attributes)
|
||
- title (media titles)
|
||
- analysis_result (Qwen 3 VL descriptions)
|
||
- source_page / source_url (origin)
|
||
|
||
Usage:
|
||
python serp.py --port 31337 --vault ./vault --db neopig.db
|
||
"""
|
||
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import mimetypes
|
||
import os
|
||
from pathlib import Path
|
||
from typing import List, Dict, Any, Optional
|
||
|
||
from fastapi import FastAPI, Query, HTTPException, BackgroundTasks, Header, Cookie, UploadFile, File, Request
|
||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response, StreamingResponse
|
||
from fastapi.staticfiles import StaticFiles
|
||
from pydantic import BaseModel
|
||
from sqlalchemy import text
|
||
import uvicorn
|
||
|
||
from database import Database, OVER_9000
|
||
from filevault import hash_to_path
|
||
from miniuri import Uri
|
||
from neopig import get_live_queue
|
||
|
||
logging.basicConfig(level=logging.INFO)
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# ============================================================================
|
||
# INTERNATIONALIZATION (i18n) - Top 10 Languages
|
||
# ============================================================================
|
||
TRANSLATIONS = {
|
||
"en": {
|
||
"search": "Search", "crawl": "Crawl", "live": "Live Feed", "about": "About",
|
||
"random": "Random", "phantom": "Phantom", "loading": "Loading...",
|
||
"start_crawl": "Start Crawl", "pause": "Pause", "resume": "Resume", "delete": "Delete",
|
||
"url": "URL", "mode": "Mode", "depth": "Depth", "keywords": "Keywords",
|
||
"screenshots": "Screenshots", "fast_mode": "Fast Mode", "fresh_start": "Fresh Start",
|
||
"pages": "pages", "found": "found", "saved": "saved", "dupes": "dupes",
|
||
"running": "running", "completed": "completed", "paused": "paused",
|
||
"no_jobs": "No crawl jobs yet", "started": "Started", "filter": "Filter",
|
||
"all_types": "All types", "images": "Images", "videos": "Videos", "audio": "Audio",
|
||
"everything": "Everything (text + media)", "all_media": "All media (images + videos + audio)",
|
||
"images_only": "Images only", "videos_only": "Videos only", "text_only": "Text only",
|
||
"recent_jobs": "Recent Jobs", "search_placeholder": "Search media...",
|
||
"media": "Media", "pages_label": "Pages", "download": "Download",
|
||
"no_media": "No media found", "no_more": "No more results", "load_error": "Failed to load",
|
||
"copy": "Copy", "copied": "Copied!", "starting": "Starting...", "started_ok": "Started!",
|
||
"error": "Error", "watching": "Watching for new images...", "content": "Content",
|
||
"previous": "Previous", "next": "Next", "over_9000": "Over 9,000+!",
|
||
"loading_stats": "Loading stats...", "screenshot": "Screenshot",
|
||
"valid_uri": "Please enter at least one valid URI", "load_jobs_error": "Failed to load jobs",
|
||
"hydrate_subtitle": "Hydrate media from the web", "live_subtitle": "Watch images appear as they're crawled",
|
||
"total_images": "Total Images", "new_session": "New This Session", "per_minute": "Per Minute",
|
||
"all_domains": "All domains", "download_phantom": "Download Phantom Site",
|
||
"phantom_subtitle": "Export archived pages as a static site with local media", "sources": "sources", "max_pages": "Max Pages", "select_domain": "Select Domain",
|
||
"source_uri": "Source URI", "neopig_uri": "Neopig URI", "source_page": "Source Page", "neopig_page": "Neopig Page", "type_label": "Type", "mime_label": "MIME", "size_label": "Size", "alt_label": "Alt", "keywords_label": "Keywords", "description_label": "Description", "items": "items", "bytes": "bytes",
|
||
"used_on": "Used on", "page": "Page", "discovered": "Discovered",
|
||
"console": "Console", "close": "Close", "no_logs": "No logs yet",
|
||
},
|
||
"zh": {
|
||
"search": "搜索", "crawl": "爬取", "live": "实时动态", "about": "关于",
|
||
"random": "随机", "phantom": "幻影", "loading": "加载中...",
|
||
"start_crawl": "开始爬取", "pause": "暂停", "resume": "继续", "delete": "删除",
|
||
"url": "网址", "mode": "模式", "depth": "深度", "keywords": "关键词",
|
||
"screenshots": "截图", "fast_mode": "快速模式", "fresh_start": "全新开始",
|
||
"pages": "页面", "found": "发现", "saved": "保存", "dupes": "重复",
|
||
"running": "运行中", "completed": "已完成", "paused": "已暂停",
|
||
"no_jobs": "暂无爬取任务", "started": "开始于", "filter": "筛选",
|
||
"all_types": "所有类型", "images": "图片", "videos": "视频", "audio": "音频",
|
||
"everything": "全部（文本+媒体）", "all_media": "所有媒体（图片+视频+音频）",
|
||
"images_only": "仅图片", "videos_only": "仅视频", "text_only": "仅文本",
|
||
"recent_jobs": "最近任务", "search_placeholder": "搜索媒体...",
|
||
"media": "媒体", "pages_label": "页面", "download": "下载",
|
||
"no_media": "未找到媒体", "no_more": "没有更多结果", "load_error": "加载失败",
|
||
"copy": "复制", "copied": "已复制!", "starting": "启动中...", "started_ok": "已启动!",
|
||
"error": "错误", "watching": "正在监视新图片...", "content": "内容",
|
||
"previous": "上一页", "next": "下一页", "over_9000": "超过9000+!",
|
||
"loading_stats": "加载统计中...", "screenshot": "截图",
|
||
"valid_uri": "请输入至少一个有效的URI", "load_jobs_error": "加载任务失败",
|
||
"hydrate_subtitle": "从网络获取媒体", "live_subtitle": "观看正在爬取的图片",
|
||
"total_images": "总图片数", "new_session": "本次新增", "per_minute": "每分钟",
|
||
"all_domains": "所有域名", "download_phantom": "下载幻影站点",
|
||
"phantom_subtitle": "将存档页面导出为带本地媒体的静态站点", "max_pages": "最大页数", "select_domain": "选择域名", "sources": "fuentes",
|
||
},
|
||
"es": {
|
||
"search": "Buscar", "crawl": "Rastrear", "live": "En Vivo", "about": "Acerca de",
|
||
"random": "Aleatorio", "phantom": "Fantasma", "loading": "Cargando...",
|
||
"start_crawl": "Iniciar Rastreo", "pause": "Pausar", "resume": "Reanudar", "delete": "Eliminar",
|
||
"url": "URL", "mode": "Modo", "depth": "Profundidad", "keywords": "Palabras clave",
|
||
"screenshots": "Capturas", "fast_mode": "Modo Rápido", "fresh_start": "Inicio Limpio",
|
||
"pages": "páginas", "found": "encontradas", "saved": "guardadas", "dupes": "duplicados",
|
||
"running": "ejecutando", "completed": "completado", "paused": "pausado",
|
||
"no_jobs": "Sin tareas de rastreo", "started": "Iniciado", "filter": "Filtrar",
|
||
"all_types": "Todos los tipos", "images": "Imágenes", "videos": "Videos", "audio": "Audio",
|
||
"everything": "Todo (texto + medios)", "all_media": "Todos los medios",
|
||
"images_only": "Solo imágenes", "videos_only": "Solo videos", "text_only": "Solo texto",
|
||
"recent_jobs": "Tareas Recientes", "search_placeholder": "Buscar medios...",
|
||
"media": "Medios", "pages_label": "Páginas", "download": "Descargar",
|
||
"no_media": "No se encontraron medios", "no_more": "No hay más resultados", "load_error": "Error al cargar",
|
||
"copy": "Copiar", "copied": "¡Copiado!", "starting": "Iniciando...", "started_ok": "¡Iniciado!",
|
||
"error": "Error", "watching": "Buscando nuevas imágenes...", "content": "Contenido",
|
||
"previous": "Anterior", "next": "Siguiente", "over_9000": "¡Más de 9,000+!",
|
||
"loading_stats": "Cargando estadísticas...", "screenshot": "Captura",
|
||
"valid_uri": "Ingrese al menos una URI válida", "load_jobs_error": "Error al cargar tareas",
|
||
"hydrate_subtitle": "Obtener medios de la web", "live_subtitle": "Ver imágenes mientras se rastrean",
|
||
"total_images": "Total de imágenes", "new_session": "Nuevas en esta sesión", "per_minute": "Por minuto",
|
||
"all_domains": "Todos los dominios", "download_phantom": "Descargar sitio fantasma",
|
||
"phantom_subtitle": "Exportar páginas archivadas como sitio estático con medios locales", "max_pages": "Máx. páginas", "select_domain": "Seleccionar dominio", "sources": "स्रोत",
|
||
},
|
||
"hi": {
|
||
"search": "खोजें", "crawl": "क्रॉल", "live": "लाइव फ़ीड", "about": "के बारे में",
|
||
"random": "यादृच्छिक", "phantom": "प्रेत", "loading": "लोड हो रहा है...",
|
||
"start_crawl": "क्रॉल शुरू करें", "pause": "रोकें", "resume": "जारी रखें", "delete": "हटाएं",
|
||
"url": "यूआरएल", "mode": "मोड", "depth": "गहराई", "keywords": "कीवर्ड",
|
||
"screenshots": "स्क्रीनशॉट", "fast_mode": "फास्ट मोड", "fresh_start": "नई शुरुआत",
|
||
"pages": "पेज", "found": "मिले", "saved": "सहेजे", "dupes": "डुप्लीकेट",
|
||
"running": "चल रहा है", "completed": "पूर्ण", "paused": "रुका हुआ",
|
||
"no_jobs": "कोई क्रॉल कार्य नहीं", "started": "शुरू", "filter": "फ़िल्टर",
|
||
"all_types": "सभी प्रकार", "images": "चित्र", "videos": "वीडियो", "audio": "ऑडियो",
|
||
"everything": "सब कुछ", "all_media": "सभी मीडिया", "images_only": "केवल चित्र",
|
||
"videos_only": "केवल वीडियो", "text_only": "केवल टेक्स्ट",
|
||
"recent_jobs": "हाल के कार्य", "search_placeholder": "मीडिया खोजें...",
|
||
"media": "मीडिया", "pages_label": "पेज", "download": "डाउनलोड",
|
||
"no_media": "कोई मीडिया नहीं मिला", "no_more": "और कोई परिणाम नहीं", "load_error": "लोड करने में विफल",
|
||
"copy": "कॉपी", "copied": "कॉपी हो गया!", "starting": "शुरू हो रहा...", "started_ok": "शुरू हो गया!",
|
||
"error": "त्रुटि", "watching": "नई छवियों की तलाश...", "content": "सामग्री",
|
||
"previous": "पिछला", "next": "अगला", "over_9000": "9,000+ से अधिक!",
|
||
"loading_stats": "आँकड़े लोड हो रहे...", "screenshot": "स्क्रीनशॉट",
|
||
"valid_uri": "कृपया कम से कम एक वैध URI दर्ज करें", "load_jobs_error": "कार्य लोड करने में विफल",
|
||
"hydrate_subtitle": "वेब से मीडिया प्राप्त करें", "live_subtitle": "क्रॉल होते हुए छवियां देखें",
|
||
"total_images": "कुल छवियां", "new_session": "इस सत्र में नई", "per_minute": "प्रति मिनट",
|
||
"all_domains": "सभी डोमेन", "download_phantom": "प्रेत साइट डाउनलोड करें",
|
||
"phantom_subtitle": "संग्रहित पृष्ठों को स्थानीय मीडिया के साथ स्थैतिक साइट के रूप में निर्यात करें", "max_pages": "अधिकतम पेज", "select_domain": "डोमेन चुनें", "sources": "مصادر",
|
||
},
|
||
"ar": {
|
||
"search": "بحث", "crawl": "زحف", "live": "مباشر", "about": "حول",
|
||
"random": "عشوائي", "phantom": "شبح", "loading": "جاري التحميل...",
|
||
"start_crawl": "بدء الزحف", "pause": "إيقاف", "resume": "استئناف", "delete": "حذف",
|
||
"url": "الرابط", "mode": "الوضع", "depth": "العمق", "keywords": "كلمات مفتاحية",
|
||
"screenshots": "لقطات", "fast_mode": "وضع سريع", "fresh_start": "بداية جديدة",
|
||
"pages": "صفحات", "found": "وجدت", "saved": "حفظت", "dupes": "مكررات",
|
||
"running": "جاري", "completed": "مكتمل", "paused": "متوقف",
|
||
"no_jobs": "لا توجد مهام", "started": "بدأ", "filter": "تصفية",
|
||
"all_types": "جميع الأنواع", "images": "صور", "videos": "فيديو", "audio": "صوت",
|
||
"everything": "الكل", "all_media": "جميع الوسائط", "images_only": "صور فقط",
|
||
"videos_only": "فيديو فقط", "text_only": "نص فقط",
|
||
"recent_jobs": "المهام الأخيرة", "search_placeholder": "بحث في الوسائط...",
|
||
"media": "وسائط", "pages_label": "صفحات", "download": "تحميل",
|
||
"no_media": "لم يتم العثور على وسائط", "no_more": "لا مزيد من النتائج", "load_error": "فشل التحميل",
|
||
"copy": "نسخ", "copied": "تم النسخ!", "starting": "جاري البدء...", "started_ok": "تم البدء!",
|
||
"error": "خطأ", "watching": "مراقبة الصور الجديدة...", "content": "المحتوى",
|
||
"previous": "السابق", "next": "التالي", "over_9000": "أكثر من 9,000+!",
|
||
"loading_stats": "جاري تحميل الإحصائيات...", "screenshot": "لقطة شاشة",
|
||
"valid_uri": "الرجاء إدخال رابط واحد على الأقل", "load_jobs_error": "فشل تحميل المهام",
|
||
"hydrate_subtitle": "جلب الوسائط من الويب", "live_subtitle": "مشاهدة الصور أثناء الزحف",
|
||
"total_images": "إجمالي الصور", "new_session": "جديدة في هذه الجلسة", "per_minute": "في الدقيقة",
|
||
"all_domains": "جميع النطاقات", "download_phantom": "تحميل موقع الشبح",
|
||
"phantom_subtitle": "تصدير الصفحات المؤرشفة كموقع ثابت مع وسائط محلية", "max_pages": "أقصى عدد صفحات", "select_domain": "اختر النطاق", "sources": "fontes",
|
||
},
|
||
"pt": {
|
||
"search": "Pesquisar", "crawl": "Rastrear", "live": "Ao Vivo", "about": "Sobre",
|
||
"random": "Aleatório", "phantom": "Fantasma", "loading": "Carregando...",
|
||
"start_crawl": "Iniciar Rastreio", "pause": "Pausar", "resume": "Retomar", "delete": "Excluir",
|
||
"url": "URL", "mode": "Modo", "depth": "Profundidade", "keywords": "Palavras-chave",
|
||
"screenshots": "Capturas", "fast_mode": "Modo Rápido", "fresh_start": "Novo Início",
|
||
"pages": "páginas", "found": "encontradas", "saved": "salvas", "dupes": "duplicatas",
|
||
"running": "executando", "completed": "concluído", "paused": "pausado",
|
||
"no_jobs": "Sem tarefas", "started": "Iniciado", "filter": "Filtrar",
|
||
"all_types": "Todos os tipos", "images": "Imagens", "videos": "Vídeos", "audio": "Áudio",
|
||
"everything": "Tudo (texto + mídia)", "all_media": "Toda mídia",
|
||
"images_only": "Apenas imagens", "videos_only": "Apenas vídeos", "text_only": "Apenas texto",
|
||
"recent_jobs": "Tarefas Recentes", "search_placeholder": "Pesquisar mídia...",
|
||
"media": "Mídia", "pages_label": "Páginas", "download": "Baixar",
|
||
"no_media": "Nenhuma mídia encontrada", "no_more": "Sem mais resultados", "load_error": "Falha ao carregar",
|
||
"copy": "Copiar", "copied": "Copiado!", "starting": "Iniciando...", "started_ok": "Iniciado!",
|
||
"error": "Erro", "watching": "Observando novas imagens...", "content": "Conteúdo",
|
||
"previous": "Anterior", "next": "Próximo", "over_9000": "Mais de 9.000+!",
|
||
"loading_stats": "Carregando estatísticas...", "screenshot": "Captura de tela",
|
||
"valid_uri": "Insira pelo menos uma URI válida", "load_jobs_error": "Falha ao carregar tarefas",
|
||
"hydrate_subtitle": "Obter mídia da web", "live_subtitle": "Veja imagens aparecerem durante o rastreio",
|
||
"total_images": "Total de imagens", "new_session": "Novas nesta sessão", "per_minute": "Por minuto",
|
||
"all_domains": "Todos os domínios", "download_phantom": "Baixar site fantasma",
|
||
"phantom_subtitle": "Exportar páginas arquivadas como site estático com mídia local", "max_pages": "Máx. páginas", "select_domain": "Selecionar domínio", "sources": "источников",
|
||
},
|
||
"ru": {
|
||
"search": "Поиск", "crawl": "Сканировать", "live": "Прямой эфир", "about": "О нас",
|
||
"random": "Случайное", "phantom": "Призрак", "loading": "Загрузка...",
|
||
"start_crawl": "Начать сканирование", "pause": "Пауза", "resume": "Продолжить", "delete": "Удалить",
|
||
"url": "URL", "mode": "Режим", "depth": "Глубина", "keywords": "Ключевые слова",
|
||
"screenshots": "Скриншоты", "fast_mode": "Быстрый режим", "fresh_start": "Начать заново",
|
||
"pages": "страниц", "found": "найдено", "saved": "сохранено", "dupes": "дубликаты",
|
||
"running": "выполняется", "completed": "завершено", "paused": "приостановлено",
|
||
"no_jobs": "Нет задач", "started": "Начало", "filter": "Фильтр",
|
||
"all_types": "Все типы", "images": "Изображения", "videos": "Видео", "audio": "Аудио",
|
||
"everything": "Всё (текст + медиа)", "all_media": "Все медиа",
|
||
"images_only": "Только изображения", "videos_only": "Только видео", "text_only": "Только текст",
|
||
"recent_jobs": "Недавние задачи", "search_placeholder": "Поиск медиа...",
|
||
"media": "Медиа", "pages_label": "Страницы", "download": "Скачать",
|
||
"no_media": "Медиа не найдено", "no_more": "Больше нет результатов", "load_error": "Ошибка загрузки",
|
||
"copy": "Копировать", "copied": "Скопировано!", "starting": "Запуск...", "started_ok": "Запущено!",
|
||
"error": "Ошибка", "watching": "Отслеживание новых изображений...", "content": "Контент",
|
||
"previous": "Назад", "next": "Далее", "over_9000": "Более 9000+!",
|
||
"loading_stats": "Загрузка статистики...", "screenshot": "Скриншот",
|
||
"valid_uri": "Введите хотя бы один URL", "load_jobs_error": "Не удалось загрузить задачи",
|
||
"hydrate_subtitle": "Получить медиа из интернета", "live_subtitle": "Смотрите появление изображений",
|
||
"total_images": "Всего изображений", "new_session": "Новые в сессии", "per_minute": "В минуту",
|
||
"all_domains": "Все домены", "download_phantom": "Скачать призрачный сайт",
|
||
"phantom_subtitle": "Экспорт архивных страниц как статический сайт с локальными медиа", "max_pages": "Макс. страниц", "select_domain": "Выбрать домен", "sources": "ソース",
|
||
},
|
||
"ja": {
|
||
"search": "検索", "crawl": "クロール", "live": "ライブ", "about": "について",
|
||
"random": "ランダム", "phantom": "ファントム", "loading": "読み込み中...",
|
||
"start_crawl": "クロール開始", "pause": "一時停止", "resume": "再開", "delete": "削除",
|
||
"url": "URL", "mode": "モード", "depth": "深さ", "keywords": "キーワード",
|
||
"screenshots": "スクリーンショット", "fast_mode": "高速モード", "fresh_start": "新規開始",
|
||
"pages": "ページ", "found": "発見", "saved": "保存", "dupes": "重複",
|
||
"running": "実行中", "completed": "完了", "paused": "一時停止中",
|
||
"no_jobs": "ジョブがありません", "started": "開始", "filter": "フィルター",
|
||
"all_types": "すべてのタイプ", "images": "画像", "videos": "動画", "audio": "オーディオ",
|
||
"everything": "すべて（テキスト＋メディア）", "all_media": "すべてのメディア",
|
||
"images_only": "画像のみ", "videos_only": "動画のみ", "text_only": "テキストのみ",
|
||
"recent_jobs": "最近のジョブ", "search_placeholder": "メディアを検索...",
|
||
"media": "メディア", "pages_label": "ページ", "download": "ダウンロード",
|
||
"no_media": "メディアが見つかりません", "no_more": "これ以上の結果はありません", "load_error": "読み込みに失敗",
|
||
"copy": "コピー", "copied": "コピーしました!", "starting": "開始中...", "started_ok": "開始しました!",
|
||
"error": "エラー", "watching": "新しい画像を監視中...", "content": "コンテンツ",
|
||
"previous": "前へ", "next": "次へ", "over_9000": "9000以上!",
|
||
"loading_stats": "統計を読み込み中...", "screenshot": "スクリーンショット",
|
||
"valid_uri": "有効なURIを入力してください", "load_jobs_error": "ジョブの読み込みに失敗",
|
||
"hydrate_subtitle": "ウェブからメディアを取得", "live_subtitle": "クロール中の画像を表示",
|
||
"total_images": "合計画像数", "new_session": "今回のセッション", "per_minute": "毎分",
|
||
"all_domains": "すべてのドメイン", "download_phantom": "ファントムサイトをダウンロード",
|
||
"phantom_subtitle": "アーカイブページをローカルメディア付きの静的サイトとしてエクスポート", "max_pages": "最大ページ数", "select_domain": "ドメインを選択", "sources": "sources", "max_pages": "Max Pages", "select_domain": "Select Domain",
|
||
},
|
||
"fr": {
|
||
"search": "Rechercher", "crawl": "Explorer", "live": "En Direct", "about": "À propos",
|
||
"random": "Aléatoire", "phantom": "Fantôme", "loading": "Chargement...",
|
||
"start_crawl": "Démarrer", "pause": "Pause", "resume": "Reprendre", "delete": "Supprimer",
|
||
"url": "URL", "mode": "Mode", "depth": "Profondeur", "keywords": "Mots-clés",
|
||
"screenshots": "Captures", "fast_mode": "Mode Rapide", "fresh_start": "Nouveau Départ",
|
||
"pages": "pages", "found": "trouvées", "saved": "enregistrées", "dupes": "doublons",
|
||
"running": "en cours", "completed": "terminé", "paused": "en pause",
|
||
"no_jobs": "Aucune tâche", "started": "Démarré", "filter": "Filtrer",
|
||
"all_types": "Tous les types", "images": "Images", "videos": "Vidéos", "audio": "Audio",
|
||
"everything": "Tout (texte + médias)", "all_media": "Tous les médias",
|
||
"images_only": "Images uniquement", "videos_only": "Vidéos uniquement", "text_only": "Texte uniquement",
|
||
"recent_jobs": "Tâches Récentes", "search_placeholder": "Rechercher des médias...",
|
||
"media": "Médias", "pages_label": "Pages", "download": "Télécharger",
|
||
"no_media": "Aucun média trouvé", "no_more": "Plus de résultats", "load_error": "Échec du chargement",
|
||
"copy": "Copier", "copied": "Copié!", "starting": "Démarrage...", "started_ok": "Démarré!",
|
||
"error": "Erreur", "watching": "Surveillance des nouvelles images...", "content": "Contenu",
|
||
"previous": "Précédent", "next": "Suivant", "over_9000": "Plus de 9 000+!",
|
||
"loading_stats": "Chargement des statistiques...", "screenshot": "Capture d'écran",
|
||
"valid_uri": "Veuillez entrer au moins une URI valide", "load_jobs_error": "Échec du chargement des tâches",
|
||
"hydrate_subtitle": "Récupérer les médias du web", "live_subtitle": "Voir les images apparaître pendant l'exploration",
|
||
"total_images": "Total d'images", "new_session": "Nouvelles cette session", "per_minute": "Par minute",
|
||
"all_domains": "Tous les domaines", "download_phantom": "Télécharger le site fantôme",
|
||
"phantom_subtitle": "Exporter les pages archivées en site statique avec médias locaux", "max_pages": "Pages max", "select_domain": "Sélectionner le domaine", "sources": "Quellen",
|
||
},
|
||
"de": {
|
||
"search": "Suchen", "crawl": "Crawlen", "live": "Live-Feed", "about": "Über",
|
||
"random": "Zufällig", "phantom": "Phantom", "loading": "Laden...",
|
||
"start_crawl": "Crawl starten", "pause": "Pause", "resume": "Fortsetzen", "delete": "Löschen",
|
||
"url": "URL", "mode": "Modus", "depth": "Tiefe", "keywords": "Stichwörter",
|
||
"screenshots": "Screenshots", "fast_mode": "Schnellmodus", "fresh_start": "Neustart",
|
||
"pages": "Seiten", "found": "gefunden", "saved": "gespeichert", "dupes": "Duplikate",
|
||
"running": "läuft", "completed": "abgeschlossen", "paused": "pausiert",
|
||
"no_jobs": "Keine Aufgaben", "started": "Gestartet", "filter": "Filter",
|
||
"all_types": "Alle Typen", "images": "Bilder", "videos": "Videos", "audio": "Audio",
|
||
"everything": "Alles (Text + Medien)", "all_media": "Alle Medien",
|
||
"images_only": "Nur Bilder", "videos_only": "Nur Videos", "text_only": "Nur Text",
|
||
"recent_jobs": "Letzte Aufgaben", "search_placeholder": "Medien suchen...",
|
||
"media": "Medien", "pages_label": "Seiten", "download": "Herunterladen",
|
||
"no_media": "Keine Medien gefunden", "no_more": "Keine weiteren Ergebnisse", "load_error": "Laden fehlgeschlagen",
|
||
"copy": "Kopieren", "copied": "Kopiert!", "starting": "Starten...", "started_ok": "Gestartet!",
|
||
"error": "Fehler", "watching": "Überwache neue Bilder...", "content": "Inhalt",
|
||
"previous": "Zurück", "next": "Weiter", "over_9000": "Über 9.000+!",
|
||
"loading_stats": "Lade Statistiken...", "screenshot": "Screenshot",
|
||
"valid_uri": "Mindestens eine gültige URI eingeben", "load_jobs_error": "Aufgaben konnten nicht geladen werden",
|
||
"hydrate_subtitle": "Medien aus dem Web abrufen", "live_subtitle": "Bilder beim Crawlen beobachten",
|
||
"total_images": "Bilder gesamt", "new_session": "Neu in dieser Sitzung", "per_minute": "Pro Minute",
|
||
"all_domains": "Alle Domains", "download_phantom": "Phantom-Seite herunterladen",
|
||
"phantom_subtitle": "Archivierte Seiten als statische Seite mit lokalen Medien exportieren", "max_pages": "Max. Seiten", "select_domain": "Domain auswählen", "sources": "소스",
|
||
},
|
||
"ko": {
|
||
"search": "검색", "crawl": "크롤", "live": "라이브", "about": "정보",
|
||
"random": "랜덤", "phantom": "팬텀", "loading": "로딩 중...",
|
||
"start_crawl": "크롤 시작", "pause": "일시정지", "resume": "재개", "delete": "삭제",
|
||
"url": "URL", "mode": "모드", "depth": "깊이", "keywords": "키워드",
|
||
"screenshots": "스크린샷", "fast_mode": "빠른 모드", "fresh_start": "새로 시작",
|
||
"pages": "페이지", "found": "발견", "saved": "저장", "dupes": "중복",
|
||
"running": "실행 중", "completed": "완료", "paused": "일시정지됨",
|
||
"no_jobs": "작업 없음", "started": "시작됨", "filter": "필터",
|
||
"all_types": "모든 유형", "images": "이미지", "videos": "동영상", "audio": "오디오",
|
||
"everything": "모두", "all_media": "모든 미디어",
|
||
"images_only": "이미지만", "videos_only": "동영상만", "text_only": "텍스트만",
|
||
"recent_jobs": "최근 작업", "search_placeholder": "미디어 검색...",
|
||
"media": "미디어", "pages_label": "페이지", "download": "다운로드",
|
||
"no_media": "미디어를 찾을 수 없음", "no_more": "더 이상 결과 없음", "load_error": "로드 실패",
|
||
"copy": "복사", "copied": "복사됨!", "starting": "시작 중...", "started_ok": "시작됨!",
|
||
"error": "오류", "watching": "새 이미지 감시 중...", "content": "콘텐츠",
|
||
"previous": "이전", "next": "다음", "over_9000": "9,000 이상!",
|
||
"loading_stats": "통계 로드 중...", "screenshot": "스크린샷",
|
||
"valid_uri": "유효한 URI를 하나 이상 입력하세요", "load_jobs_error": "작업 로드 실패",
|
||
"hydrate_subtitle": "웹에서 미디어 가져오기", "live_subtitle": "크롤링되는 이미지 보기",
|
||
"total_images": "전체 이미지", "new_session": "이번 세션 신규", "per_minute": "분당",
|
||
"all_domains": "모든 도메인", "download_phantom": "팬텀 사이트 다운로드",
|
||
"phantom_subtitle": "보관된 페이지를 로컬 미디어가 포함된 정적 사이트로 내보내기", "max_pages": "최대 페이지", "select_domain": "도메인 선택", "sources": "fonti",
|
||
},
|
||
"it": {
|
||
"search": "Cerca", "crawl": "Scansiona", "live": "In Diretta", "about": "Info",
|
||
"random": "Casuale", "phantom": "Fantasma", "loading": "Caricamento...",
|
||
"start_crawl": "Avvia Scansione", "pause": "Pausa", "resume": "Riprendi", "delete": "Elimina",
|
||
"url": "URL", "mode": "Modalità", "depth": "Profondità", "keywords": "Parole chiave",
|
||
"screenshots": "Screenshot", "fast_mode": "Modalità Veloce", "fresh_start": "Nuovo Inizio",
|
||
"pages": "pagine", "found": "trovati", "saved": "salvati", "dupes": "duplicati",
|
||
"running": "in esecuzione", "completed": "completato", "paused": "in pausa",
|
||
"no_jobs": "Nessuna attività", "started": "Avviato", "filter": "Filtra",
|
||
"all_types": "Tutti i tipi", "images": "Immagini", "videos": "Video", "audio": "Audio",
|
||
"everything": "Tutto", "all_media": "Tutti i media",
|
||
"images_only": "Solo immagini", "videos_only": "Solo video", "text_only": "Solo testo",
|
||
"recent_jobs": "Attività Recenti", "search_placeholder": "Cerca media...",
|
||
"media": "Media", "pages_label": "Pagine", "download": "Scarica",
|
||
"no_media": "Nessun media trovato", "no_more": "Nessun altro risultato", "load_error": "Caricamento fallito",
|
||
"copy": "Copia", "copied": "Copiato!", "starting": "Avvio...", "started_ok": "Avviato!",
|
||
"error": "Errore", "watching": "Monitoraggio nuove immagini...", "content": "Contenuto",
|
||
"previous": "Precedente", "next": "Successivo", "over_9000": "Oltre 9.000+!",
|
||
"loading_stats": "Caricamento statistiche...", "screenshot": "Screenshot",
|
||
"valid_uri": "Inserire almeno un URI valido", "load_jobs_error": "Caricamento attività fallito",
|
||
"hydrate_subtitle": "Ottieni media dal web", "live_subtitle": "Guarda le immagini mentre vengono scansionate",
|
||
"total_images": "Immagini totali", "new_session": "Nuove in questa sessione", "per_minute": "Al minuto",
|
||
"all_domains": "Tutti i domini", "download_phantom": "Scarica sito fantasma",
|
||
"phantom_subtitle": "Esporta pagine archiviate come sito statico con media locali", "max_pages": "Max pagine", "select_domain": "Seleziona dominio", "sources": "bronnen",
|
||
},
|
||
"nl": {
|
||
"search": "Zoeken", "crawl": "Crawlen", "live": "Live", "about": "Over",
|
||
"random": "Willekeurig", "phantom": "Fantoom", "loading": "Laden...",
|
||
"start_crawl": "Start Crawl", "pause": "Pauzeer", "resume": "Hervat", "delete": "Verwijder",
|
||
"url": "URL", "mode": "Modus", "depth": "Diepte", "keywords": "Trefwoorden",
|
||
"screenshots": "Schermafbeeldingen", "fast_mode": "Snelle Modus", "fresh_start": "Nieuwe Start",
|
||
"pages": "pagina's", "found": "gevonden", "saved": "opgeslagen", "dupes": "duplicaten",
|
||
"running": "actief", "completed": "voltooid", "paused": "gepauzeerd",
|
||
"no_jobs": "Geen taken", "started": "Gestart", "filter": "Filter",
|
||
"all_types": "Alle typen", "images": "Afbeeldingen", "videos": "Video's", "audio": "Audio",
|
||
"everything": "Alles", "all_media": "Alle media",
|
||
"images_only": "Alleen afbeeldingen", "videos_only": "Alleen video's", "text_only": "Alleen tekst",
|
||
"recent_jobs": "Recente Taken", "search_placeholder": "Zoek media...",
|
||
"media": "Media", "pages_label": "Pagina's", "download": "Downloaden",
|
||
"no_media": "Geen media gevonden", "no_more": "Geen resultaten meer", "load_error": "Laden mislukt",
|
||
"copy": "Kopiëren", "copied": "Gekopieerd!", "starting": "Starten...", "started_ok": "Gestart!",
|
||
"error": "Fout", "watching": "Nieuwe afbeeldingen bekijken...", "content": "Inhoud",
|
||
"previous": "Vorige", "next": "Volgende", "over_9000": "Meer dan 9.000+!",
|
||
"loading_stats": "Statistieken laden...", "screenshot": "Schermafbeelding",
|
||
"valid_uri": "Voer minimaal één geldige URI in", "load_jobs_error": "Taken laden mislukt",
|
||
"hydrate_subtitle": "Media van het web ophalen", "live_subtitle": "Bekijk afbeeldingen terwijl ze worden gecrawld",
|
||
"total_images": "Totaal afbeeldingen", "new_session": "Nieuw deze sessie", "per_minute": "Per minuut",
|
||
"all_domains": "Alle domeinen", "download_phantom": "Fantoomsite downloaden",
|
||
"phantom_subtitle": "Gearchiveerde pagina's exporteren als statische site met lokale media", "max_pages": "Max. pagina's", "select_domain": "Selecteer domein", "sources": "źródeł",
|
||
},
|
||
"pl": {
|
||
"search": "Szukaj", "crawl": "Indeksuj", "live": "Na żywo", "about": "O nas",
|
||
"random": "Losowo", "phantom": "Fantom", "loading": "Ładowanie...",
|
||
"start_crawl": "Rozpocznij", "pause": "Pauza", "resume": "Wznów", "delete": "Usuń",
|
||
"url": "URL", "mode": "Tryb", "depth": "Głębokość", "keywords": "Słowa kluczowe",
|
||
"screenshots": "Zrzuty ekranu", "fast_mode": "Tryb szybki", "fresh_start": "Nowy start",
|
||
"pages": "stron", "found": "znaleziono", "saved": "zapisano", "dupes": "duplikaty",
|
||
"running": "w toku", "completed": "zakończono", "paused": "wstrzymano",
|
||
"no_jobs": "Brak zadań", "started": "Rozpoczęto", "filter": "Filtruj",
|
||
"all_types": "Wszystkie typy", "images": "Obrazy", "videos": "Filmy", "audio": "Audio",
|
||
"everything": "Wszystko", "all_media": "Wszystkie media",
|
||
"images_only": "Tylko obrazy", "videos_only": "Tylko filmy", "text_only": "Tylko tekst",
|
||
"recent_jobs": "Ostatnie Zadania", "search_placeholder": "Szukaj mediów...",
|
||
"media": "Media", "pages_label": "Strony", "download": "Pobierz",
|
||
"no_media": "Nie znaleziono mediów", "no_more": "Brak więcej wyników", "load_error": "Błąd ładowania",
|
||
"copy": "Kopiuj", "copied": "Skopiowano!", "starting": "Uruchamianie...", "started_ok": "Uruchomiono!",
|
||
"error": "Błąd", "watching": "Obserwowanie nowych obrazów...", "content": "Treść",
|
||
"previous": "Poprzedni", "next": "Następny", "over_9000": "Ponad 9 000+!",
|
||
"loading_stats": "Ładowanie statystyk...", "screenshot": "Zrzut ekranu",
|
||
"valid_uri": "Wprowadź przynajmniej jeden URI", "load_jobs_error": "Nie udało się załadować zadań",
|
||
"hydrate_subtitle": "Pobierz media z sieci", "live_subtitle": "Oglądaj obrazy podczas indeksowania",
|
||
"total_images": "Łącznie obrazów", "new_session": "Nowe w tej sesji", "per_minute": "Na minutę",
|
||
"all_domains": "Wszystkie domeny", "download_phantom": "Pobierz stronę fantomową",
|
||
"phantom_subtitle": "Eksportuj zarchiwizowane strony jako statyczną witrynę z lokalnymi mediami", "max_pages": "Maks. stron", "select_domain": "Wybierz domenę", "sources": "kaynaklar",
|
||
},
|
||
"tr": {
|
||
"search": "Ara", "crawl": "Tara", "live": "Canlı", "about": "Hakkında",
|
||
"random": "Rastgele", "phantom": "Hayalet", "loading": "Yükleniyor...",
|
||
"start_crawl": "Taramayı Başlat", "pause": "Duraklat", "resume": "Devam", "delete": "Sil",
|
||
"url": "URL", "mode": "Mod", "depth": "Derinlik", "keywords": "Anahtar kelimeler",
|
||
"screenshots": "Ekran görüntüleri", "fast_mode": "Hızlı Mod", "fresh_start": "Yeni Başlangıç",
|
||
"pages": "sayfa", "found": "bulundu", "saved": "kaydedildi", "dupes": "kopya",
|
||
"running": "çalışıyor", "completed": "tamamlandı", "paused": "duraklatıldı",
|
||
"no_jobs": "Görev yok", "started": "Başladı", "filter": "Filtrele",
|
||
"all_types": "Tüm türler", "images": "Resimler", "videos": "Videolar", "audio": "Ses",
|
||
"everything": "Her şey", "all_media": "Tüm medya",
|
||
"images_only": "Sadece resimler", "videos_only": "Sadece videolar", "text_only": "Sadece metin",
|
||
"recent_jobs": "Son Görevler", "search_placeholder": "Medya ara...",
|
||
"media": "Medya", "pages_label": "Sayfalar", "download": "İndir",
|
||
"no_media": "Medya bulunamadı", "no_more": "Daha fazla sonuç yok", "load_error": "Yükleme başarısız",
|
||
"copy": "Kopyala", "copied": "Kopyalandı!", "starting": "Başlatılıyor...", "started_ok": "Başlatıldı!",
|
||
"error": "Hata", "watching": "Yeni resimler izleniyor...", "content": "İçerik",
|
||
"previous": "Önceki", "next": "Sonraki", "over_9000": "9.000'den fazla+!",
|
||
"loading_stats": "İstatistikler yükleniyor...", "screenshot": "Ekran görüntüsü",
|
||
"valid_uri": "Lütfen en az bir geçerli URI girin", "load_jobs_error": "Görevler yüklenemedi",
|
||
"hydrate_subtitle": "Web'den medya al", "live_subtitle": "Taranan resimleri izle",
|
||
"total_images": "Toplam resim", "new_session": "Bu oturumda yeni", "per_minute": "Dakikada",
|
||
"all_domains": "Tüm alanlar", "download_phantom": "Hayalet siteyi indir",
|
||
"phantom_subtitle": "Arşivlenmiş sayfaları yerel medya ile statik site olarak dışa aktar", "max_pages": "Maks. sayfa", "select_domain": "Alan adı seç", "sources": "nguồn",
|
||
},
|
||
"vi": {
|
||
"search": "Tìm kiếm", "crawl": "Thu thập", "live": "Trực tiếp", "about": "Giới thiệu",
|
||
"random": "Ngẫu nhiên", "phantom": "Bóng ma", "loading": "Đang tải...",
|
||
"start_crawl": "Bắt đầu", "pause": "Tạm dừng", "resume": "Tiếp tục", "delete": "Xóa",
|
||
"url": "URL", "mode": "Chế độ", "depth": "Độ sâu", "keywords": "Từ khóa",
|
||
"screenshots": "Ảnh chụp", "fast_mode": "Chế độ nhanh", "fresh_start": "Bắt đầu mới",
|
||
"pages": "trang", "found": "tìm thấy", "saved": "đã lưu", "dupes": "trùng lặp",
|
||
"running": "đang chạy", "completed": "hoàn thành", "paused": "tạm dừng",
|
||
"no_jobs": "Không có tác vụ", "started": "Đã bắt đầu", "filter": "Lọc",
|
||
"all_types": "Tất cả loại", "images": "Hình ảnh", "videos": "Video", "audio": "Âm thanh",
|
||
"everything": "Tất cả", "all_media": "Tất cả media",
|
||
"images_only": "Chỉ hình ảnh", "videos_only": "Chỉ video", "text_only": "Chỉ văn bản",
|
||
"recent_jobs": "Tác vụ gần đây", "search_placeholder": "Tìm media...",
|
||
"media": "Media", "pages_label": "Trang", "download": "Tải xuống",
|
||
"no_media": "Không tìm thấy media", "no_more": "Không còn kết quả", "load_error": "Tải thất bại",
|
||
"copy": "Sao chép", "copied": "Đã sao chép!", "starting": "Đang khởi động...", "started_ok": "Đã khởi động!",
|
||
"error": "Lỗi", "watching": "Đang theo dõi hình ảnh mới...", "content": "Nội dung",
|
||
"previous": "Trước", "next": "Tiếp", "over_9000": "Hơn 9.000+!",
|
||
"loading_stats": "Đang tải thống kê...", "screenshot": "Ảnh chụp màn hình",
|
||
"valid_uri": "Vui lòng nhập ít nhất một URI hợp lệ", "load_jobs_error": "Không thể tải tác vụ",
|
||
"hydrate_subtitle": "Lấy media từ web", "live_subtitle": "Xem hình ảnh khi đang thu thập",
|
||
"total_images": "Tổng hình ảnh", "new_session": "Mới trong phiên này", "per_minute": "Mỗi phút",
|
||
"all_domains": "Tất cả tên miền", "download_phantom": "Tải trang bóng ma",
|
||
"phantom_subtitle": "Xuất trang lưu trữ dưới dạng trang tĩnh với media cục bộ", "max_pages": "Tối đa trang", "select_domain": "Chọn tên miền", "sources": "แหล่งที่มา",
|
||
},
|
||
"th": {
|
||
"search": "ค้นหา", "crawl": "รวบรวม", "live": "สด", "about": "เกี่ยวกับ",
|
||
"random": "สุ่ม", "phantom": "แฟนทอม", "loading": "กำลังโหลด...",
|
||
"start_crawl": "เริ่มรวบรวม", "pause": "หยุดชั่วคราว", "resume": "ดำเนินต่อ", "delete": "ลบ",
|
||
"url": "URL", "mode": "โหมด", "depth": "ความลึก", "keywords": "คำสำคัญ",
|
||
"screenshots": "ภาพหน้าจอ", "fast_mode": "โหมดเร็ว", "fresh_start": "เริ่มใหม่",
|
||
"pages": "หน้า", "found": "พบ", "saved": "บันทึก", "dupes": "ซ้ำ",
|
||
"running": "กำลังทำงาน", "completed": "เสร็จสิ้น", "paused": "หยุดชั่วคราว",
|
||
"no_jobs": "ไม่มีงาน", "started": "เริ่มแล้ว", "filter": "กรอง",
|
||
"all_types": "ทุกประเภท", "images": "รูปภาพ", "videos": "วิดีโอ", "audio": "เสียง",
|
||
"everything": "ทั้งหมด", "all_media": "สื่อทั้งหมด",
|
||
"images_only": "เฉพาะรูปภาพ", "videos_only": "เฉพาะวิดีโอ", "text_only": "เฉพาะข้อความ",
|
||
"recent_jobs": "งานล่าสุด", "search_placeholder": "ค้นหาสื่อ...",
|
||
"media": "สื่อ", "pages_label": "หน้า", "download": "ดาวน์โหลด",
|
||
"no_media": "ไม่พบสื่อ", "no_more": "ไม่มีผลลัพธ์เพิ่มเติม", "load_error": "โหลดไม่สำเร็จ",
|
||
"copy": "คัดลอก", "copied": "คัดลอกแล้ว!", "starting": "กำลังเริ่ม...", "started_ok": "เริ่มแล้ว!",
|
||
"error": "ข้อผิดพลาด", "watching": "กำลังดูรูปภาพใหม่...", "content": "เนื้อหา",
|
||
"previous": "ก่อนหน้า", "next": "ถัดไป", "over_9000": "มากกว่า 9,000+!",
|
||
"loading_stats": "กำลังโหลดสถิติ...", "screenshot": "ภาพหน้าจอ",
|
||
"valid_uri": "กรุณาใส่ URI ที่ถูกต้องอย่างน้อยหนึ่งรายการ", "load_jobs_error": "ไม่สามารถโหลดงานได้",
|
||
"hydrate_subtitle": "ดึงสื่อจากเว็บ", "live_subtitle": "ดูรูปภาพขณะรวบรวม",
|
||
"total_images": "รูปภาพทั้งหมด", "new_session": "ใหม่ในเซสชันนี้", "per_minute": "ต่อนาที",
|
||
"all_domains": "โดเมนทั้งหมด", "download_phantom": "ดาวน์โหลดเว็บแฟนทอม",
|
||
"phantom_subtitle": "ส่งออกหน้าที่เก็บถาวรเป็นเว็บไซต์แบบคงที่พร้อมสื่อในเครื่อง", "max_pages": "หน้าสูงสุด", "select_domain": "เลือกโดเมน", "sources": "sumber",
|
||
},
|
||
"id": {
|
||
"search": "Cari", "crawl": "Jelajahi", "live": "Langsung", "about": "Tentang",
|
||
"random": "Acak", "phantom": "Hantu", "loading": "Memuat...",
|
||
"start_crawl": "Mulai Jelajah", "pause": "Jeda", "resume": "Lanjutkan", "delete": "Hapus",
|
||
"url": "URL", "mode": "Mode", "depth": "Kedalaman", "keywords": "Kata kunci",
|
||
"screenshots": "Tangkapan layar", "fast_mode": "Mode Cepat", "fresh_start": "Mulai Baru",
|
||
"pages": "halaman", "found": "ditemukan", "saved": "disimpan", "dupes": "duplikat",
|
||
"running": "berjalan", "completed": "selesai", "paused": "dijeda",
|
||
"no_jobs": "Tidak ada tugas", "started": "Dimulai", "filter": "Filter",
|
||
"all_types": "Semua jenis", "images": "Gambar", "videos": "Video", "audio": "Audio",
|
||
"everything": "Semua", "all_media": "Semua media",
|
||
"images_only": "Hanya gambar", "videos_only": "Hanya video", "text_only": "Hanya teks",
|
||
"recent_jobs": "Tugas Terbaru", "search_placeholder": "Cari media...",
|
||
"media": "Media", "pages_label": "Halaman", "download": "Unduh",
|
||
"no_media": "Media tidak ditemukan", "no_more": "Tidak ada hasil lagi", "load_error": "Gagal memuat",
|
||
"copy": "Salin", "copied": "Disalin!", "starting": "Memulai...", "started_ok": "Dimulai!",
|
||
"error": "Kesalahan", "watching": "Memantau gambar baru...", "content": "Konten",
|
||
"previous": "Sebelumnya", "next": "Berikutnya", "over_9000": "Lebih dari 9.000+!",
|
||
"loading_stats": "Memuat statistik...", "screenshot": "Tangkapan layar",
|
||
"valid_uri": "Masukkan setidaknya satu URI yang valid", "load_jobs_error": "Gagal memuat tugas",
|
||
"hydrate_subtitle": "Ambil media dari web", "live_subtitle": "Lihat gambar saat dijelajahi",
|
||
"total_images": "Total gambar", "new_session": "Baru sesi ini", "per_minute": "Per menit",
|
||
"all_domains": "Semua domain", "download_phantom": "Unduh situs hantu",
|
||
"phantom_subtitle": "Ekspor halaman arsip sebagai situs statis dengan media lokal", "max_pages": "Maks. halaman", "select_domain": "Pilih domain", "sources": "джерел",
|
||
},
|
||
"uk": {
|
||
"search": "Пошук", "crawl": "Сканувати", "live": "Наживо", "about": "Про нас",
|
||
"random": "Випадково", "phantom": "Привид", "loading": "Завантаження...",
|
||
"start_crawl": "Почати сканування", "pause": "Пауза", "resume": "Продовжити", "delete": "Видалити",
|
||
"url": "URL", "mode": "Режим", "depth": "Глибина", "keywords": "Ключові слова",
|
||
"screenshots": "Знімки екрану", "fast_mode": "Швидкий режим", "fresh_start": "Новий старт",
|
||
"pages": "сторінок", "found": "знайдено", "saved": "збережено", "dupes": "дублікати",
|
||
"running": "виконується", "completed": "завершено", "paused": "призупинено",
|
||
"no_jobs": "Немає завдань", "started": "Розпочато", "filter": "Фільтр",
|
||
"all_types": "Усі типи", "images": "Зображення", "videos": "Відео", "audio": "Аудіо",
|
||
"everything": "Все", "all_media": "Усі медіа",
|
||
"images_only": "Лише зображення", "videos_only": "Лише відео", "text_only": "Лише текст",
|
||
"recent_jobs": "Останні завдання", "search_placeholder": "Пошук медіа...",
|
||
"media": "Медіа", "pages_label": "Сторінки", "download": "Завантажити",
|
||
"no_media": "Медіа не знайдено", "no_more": "Більше немає результатів", "load_error": "Помилка завантаження",
|
||
"copy": "Копіювати", "copied": "Скопійовано!", "starting": "Запуск...", "started_ok": "Запущено!",
|
||
"error": "Помилка", "watching": "Відстеження нових зображень...", "content": "Вміст",
|
||
"previous": "Попередній", "next": "Наступний", "over_9000": "Понад 9000+!",
|
||
"loading_stats": "Завантаження статистики...", "screenshot": "Знімок екрану",
|
||
"valid_uri": "Введіть хоча б один дійсний URI", "load_jobs_error": "Не вдалося завантажити завдання",
|
||
"hydrate_subtitle": "Отримати медіа з вебу", "live_subtitle": "Дивіться зображення під час сканування",
|
||
"total_images": "Всього зображень", "new_session": "Нові в сесії", "per_minute": "За хвилину",
|
||
"all_domains": "Усі домени", "download_phantom": "Завантажити привид-сайт",
|
||
"phantom_subtitle": "Експортувати архівні сторінки як статичний сайт з локальними медіа",
|
||
},
|
||
"sv": {
|
||
"search": "Sök", "crawl": "Genomsök", "live": "Live", "about": "Om",
|
||
"random": "Slumpmässig", "phantom": "Fantom", "loading": "Laddar...",
|
||
"start_crawl": "Starta genomsökning", "pause": "Pausa", "resume": "Återuppta", "delete": "Ta bort",
|
||
"url": "URL", "mode": "Läge", "depth": "Djup", "keywords": "Nyckelord",
|
||
"screenshots": "Skärmbilder", "fast_mode": "Snabbläge", "fresh_start": "Ny start",
|
||
"pages": "sidor", "found": "hittade", "saved": "sparade", "dupes": "dubbletter",
|
||
"running": "körs", "completed": "slutförd", "paused": "pausad",
|
||
"no_jobs": "Inga jobb", "started": "Startad", "filter": "Filtrera",
|
||
"all_types": "Alla typer", "images": "Bilder", "videos": "Videor", "audio": "Ljud",
|
||
"everything": "Allt", "all_media": "Alla media",
|
||
"images_only": "Endast bilder", "videos_only": "Endast videor", "text_only": "Endast text",
|
||
"recent_jobs": "Senaste Jobb", "search_placeholder": "Sök media...",
|
||
"media": "Media", "pages_label": "Sidor", "download": "Ladda ner",
|
||
"no_media": "Ingen media hittades", "no_more": "Inga fler resultat", "load_error": "Kunde inte ladda",
|
||
"copy": "Kopiera", "copied": "Kopierat!", "starting": "Startar...", "started_ok": "Startad!",
|
||
"error": "Fel", "watching": "Bevakar nya bilder...", "content": "Innehåll",
|
||
"previous": "Föregående", "next": "Nästa", "over_9000": "Över 9 000+!",
|
||
"loading_stats": "Laddar statistik...", "screenshot": "Skärmbild",
|
||
"valid_uri": "Ange minst en giltig URI", "load_jobs_error": "Kunde inte ladda jobb",
|
||
"hydrate_subtitle": "Hämta media från webben", "live_subtitle": "Se bilder medan de genomsöks",
|
||
"total_images": "Totalt bilder", "new_session": "Nya denna session", "per_minute": "Per minut",
|
||
"all_domains": "Alla domäner", "download_phantom": "Ladda ner fantomwebbplats",
|
||
"phantom_subtitle": "Exportera arkiverade sidor som statisk webbplats med lokal media", "max_pages": "Max sidor", "select_domain": "Välj domän", "sources": "källor",
|
||
},
|
||
}
|
||
|
||
def get_lang(lang_cookie: str = None, accept_language: str = None) -> str:
|
||
"""Get language from cookie first, then Accept-Language header."""
|
||
# Cookie takes priority (user's explicit choice)
|
||
if lang_cookie and lang_cookie in TRANSLATIONS:
|
||
return lang_cookie
|
||
# Fall back to Accept-Language header
|
||
if not accept_language:
|
||
return "en"
|
||
# Parse "en-US,en;q=0.9,zh-CN;q=0.8" format
|
||
for part in accept_language.split(','):
|
||
lang = part.split(';')[0].strip().split('-')[0].lower()
|
||
if lang in TRANSLATIONS:
|
||
return lang
|
||
return "en"
|
||
|
||
# Language names for the selector dropdown
|
||
LANG_NAMES = {
|
||
"en": "English", "zh": "中文", "es": "Español", "hi": "हिन्दी", "ar": "العربية",
|
||
"pt": "Português", "ru": "Русский", "ja": "日本語", "fr": "Français", "de": "Deutsch",
|
||
"ko": "한국어", "it": "Italiano", "nl": "Nederlands", "pl": "Polski", "tr": "Türkçe",
|
||
"vi": "Tiếng Việt", "th": "ไทย", "id": "Bahasa", "uk": "Українська", "sv": "Svenska",
|
||
}
|
||
|
||
def t(key: str, lang: str = "en") -> str:
|
||
"""Get translation for key in language."""
|
||
return TRANSLATIONS.get(lang, TRANSLATIONS["en"]).get(key, TRANSLATIONS["en"].get(key, key))
|
||
|
||
# Single source of truth for navigation - uses {{key}} placeholders
|
||
NAV_HTML = '''<div class="nav">
|
||
<a href="/" class="brand">🐷 neopig</a>
|
||
<a href="/">{{search}}</a>
|
||
<a href="/live">{{live}}</a>
|
||
<a href="/random">{{random}}</a>
|
||
<a href="/crawl">{{crawl}}</a>
|
||
<a href="/phantom">{{phantom}}</a>
|
||
<a href="/about">{{about}}</a>
|
||
</div>'''
|
||
|
||
def inject_i18n(html: str, lang: str) -> str:
|
||
"""Replace {key} placeholders and inject JS translations + language selector."""
|
||
trans = TRANSLATIONS.get(lang, TRANSLATIONS["en"])
|
||
# Add lang attribute and inject JS translations
|
||
html = html.replace('<html>', f'<html lang="{lang}">')
|
||
# Replace nav placeholder with actual nav (add sandbox link if enabled)
|
||
nav = NAV_HTML
|
||
if SANDBOX_MODE:
|
||
nav = nav.replace('</div>', ' <a href="/sandbox">🧪 Sandbox</a>\n</div>')
|
||
html = html.replace('<!-- NAV -->', nav)
|
||
# Build language selector options
|
||
lang_options = ''.join(f'<option value="{code}"{" selected" if code == lang else ""}>{name}</option>'
|
||
for code, name in LANG_NAMES.items())
|
||
lang_selector = f'''<select id="lang-select" onchange="setLang(this.value)" style="background:#1a1a1a;color:#888;border:1px solid #333;border-radius:4px;padding:4px 8px;font-size:12px;cursor:pointer;">{lang_options}</select>'''
|
||
lang_js = '''
|
||
function setLang(code) {
|
||
localStorage.setItem('neopig_lang', code);
|
||
document.cookie = 'lang=' + code + ';path=/;max-age=31536000';
|
||
location.reload();
|
||
}
|
||
'''
|
||
html = html.replace('<script>', f'<script>\nconst T={json.dumps(trans)};\n{lang_js}', 1)
|
||
# Insert language selector after {{about}} link in nav
|
||
html = html.replace('{{about}}</a>\n</div>', '{{about}}</a>\n ' + lang_selector + '\n</div>')
|
||
# Replace all {key} and {{key}} placeholders
|
||
for key, value in trans.items():
|
||
html = html.replace('{{' + key + '}}', value) # Double braces (static templates)
|
||
html = html.replace('{' + key + '}', value) # Single braces (after f-string)
|
||
return html
|
||
|
||
app = FastAPI(title="neopig", description="Media crawler + SERP + Screenshot service")
|
||
|
||
# Try to include uri2png screenshot router (optional dependency)
|
||
try:
|
||
from uri2png import get_screenshot_router
|
||
app.include_router(get_screenshot_router())
|
||
logger.info("Screenshot router loaded from uri2png")
|
||
except ImportError:
|
||
logger.warning("uri2png not installed - screenshot endpoints not available")
|
||
|
||
# Mount static vendor files (we are the CDN)
|
||
STATIC_VENDOR_PATH = Path(__file__).parent / "static" / "vendor"
|
||
if STATIC_VENDOR_PATH.exists():
|
||
app.mount("/static/vendor", StaticFiles(directory=STATIC_VENDOR_PATH), name="vendor")
|
||
logger.info(f"Static vendor files mounted from {STATIC_VENDOR_PATH}")
|
||
|
||
# Config - set via startup
|
||
DB_PATH = "data/neopig.db"
|
||
VAULT_PATH = Path("data/vault")
|
||
LOGS_PATH = Path("data/logs")
|
||
CRAWL_DISABLED = os.environ.get("NEOPIG_DISABLE_CRAWL", "").lower() in ("1", "true", "yes")
|
||
SANDBOX_MODE = os.environ.get("NEOPIG_SANDBOX", "").lower() in ("1", "true", "yes")
|
||
|
||
# Global database instance
|
||
db: Database = None
|
||
|
||
# Per-job log handlers (job_id -> handler)
|
||
JOB_LOG_HANDLERS: Dict[int, logging.FileHandler] = {}
|
||
|
||
def start_job_logging(job_id: int) -> None:
|
||
"""Start capturing logs for a crawl job."""
|
||
LOGS_PATH.mkdir(parents=True, exist_ok=True)
|
||
log_file = LOGS_PATH / f"{job_id}.log"
|
||
handler = logging.FileHandler(log_file, mode='w', encoding='utf-8')
|
||
handler.setLevel(logging.INFO)
|
||
handler.setFormatter(logging.Formatter('%(asctime)s %(name)s %(levelname)s: %(message)s', datefmt='%H:%M:%S'))
|
||
# Add to root logger to capture all modules
|
||
logging.getLogger().addHandler(handler)
|
||
JOB_LOG_HANDLERS[job_id] = handler
|
||
|
||
def stop_job_logging(job_id: int) -> None:
|
||
"""Stop capturing logs for a crawl job."""
|
||
handler = JOB_LOG_HANDLERS.pop(job_id, None)
|
||
if handler:
|
||
handler.close()
|
||
logging.getLogger().removeHandler(handler)
|
||
|
||
def get_job_logs(job_id: int, tail: int = 0) -> str:
|
||
"""Get logs for a crawl job. If tail > 0, return only last N lines."""
|
||
log_file = LOGS_PATH / f"{job_id}.log"
|
||
if not log_file.exists():
|
||
return ""
|
||
content = log_file.read_text(encoding='utf-8')
|
||
if tail > 0:
|
||
lines = content.splitlines()
|
||
return '\n'.join(lines[-tail:])
|
||
return content
|
||
|
||
# Active crawl tasks (for cancellation on shutdown)
|
||
ACTIVE_CRAWL_TASKS: Dict[int, asyncio.Task] = {}
|
||
|
||
# Tarball mode - serve directly from tar.gz archive
|
||
import tarfile
|
||
import tempfile
|
||
TAR_PATH: str = None # Path to tarball (each request opens its own handle)
|
||
TAR_OFFSET: int = 0 # Offset for .run files (0 for plain .tar.gz)
|
||
TAR_MEMBERS: Dict[str, tarfile.TarInfo] = {} # Cached member info
|
||
TAR_MEDIA_INDEX: Dict[str, str] = {} # md5_hash -> full path (for O(1) lookup)
|
||
ARCHIVE_ROOT: str = None # e.g., "example.com-20251230"
|
||
TEMP_DB_PATH: str = None # Extracted database (SQLite needs real file)
|
||
|
||
|
||
# Base CSS shared by all pages
|
||
BASE_CSS = """
|
||
* { box-sizing: border-box; }
|
||
body {
|
||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||
margin: 0; padding: 0;
|
||
background: #0a0a0a; color: #e0e0e0;
|
||
}
|
||
.nav {
|
||
background: #1a1a1a; padding: 10px 20px;
|
||
display: flex; gap: 20px; align-items: center;
|
||
border-bottom: 1px solid #333;
|
||
}
|
||
.nav a { color: #ff6b6b; text-decoration: none; }
|
||
.nav a:hover { text-decoration: underline; }
|
||
.nav .brand { font-weight: bold; font-size: 18px; }
|
||
.container { padding: 20px; }
|
||
a { color: #ff6b6b; }
|
||
input[type="text"], select {
|
||
padding: 12px 16px; font-size: 16px;
|
||
border: 2px solid #333; border-radius: 8px;
|
||
background: #1a1a1a; color: #fff;
|
||
}
|
||
input[type="text"]:focus { outline: none; border-color: #ff6b6b; }
|
||
button {
|
||
padding: 12px 24px; font-size: 16px;
|
||
background: #ff6b6b; color: #fff;
|
||
border: none; border-radius: 8px; cursor: pointer;
|
||
}
|
||
button:hover { background: #ff5252; }
|
||
.search-box { display: flex; gap: 10px; margin-bottom: 20px; }
|
||
"""
|
||
|
||
def layout(title: str, content: str, extra_css: str = "", extra_head: str = "") -> str:
|
||
"""Build a complete HTML page with consistent layout.
|
||
|
||
Args:
|
||
title: Page title (will be appended with " - neopig")
|
||
content: HTML content for the page body
|
||
extra_css: Additional CSS to include
|
||
extra_head: Additional head elements (scripts, links, etc.)
|
||
|
||
Returns:
|
||
Complete HTML document string
|
||
"""
|
||
return f"""<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>{title} - neopig</title>
|
||
<style>
|
||
{BASE_CSS}
|
||
{extra_css}
|
||
</style>
|
||
{extra_head}
|
||
</head>
|
||
<body>
|
||
<!-- NAV -->
|
||
{content}
|
||
<script>
|
||
</script>
|
||
</body>
|
||
</html>"""
|
||
|
||
# Extended CSS for view/detail pages (adds to BASE_CSS)
|
||
VIEW_CSS = """
|
||
h1 { color: #ff6b6b; font-size: 20px; margin: 0 0 10px 0; }
|
||
h3 { color: #ff6b6b; font-size: 16px; margin: 30px 0 15px 0; border-bottom: 1px solid #333; padding-bottom: 8px; }
|
||
.meta { background: #1a1a1a; padding: 15px; border-radius: 8px; margin: 15px 0; }
|
||
.meta-row { display: flex; margin: 8px 0; }
|
||
.meta-label { width: 100px; color: #888; font-size: 13px; }
|
||
.meta-value { flex: 1; word-break: break-all; font-size: 13px; }
|
||
.meta-value a { color: #4ade80; }
|
||
.hero { text-align: center; margin-bottom: 20px; }
|
||
.hero img, .hero video { max-width: 100%; max-height: 60vh; border-radius: 8px; }
|
||
.content-rendered {
|
||
background: #1a1a1a; padding: 20px; border-radius: 8px;
|
||
line-height: 1.7; font-size: 14px; color: #ccc; text-align: left;
|
||
}
|
||
.content-rendered img { max-width: 100%; height: auto; margin: 10px 0; }
|
||
.content-rendered img.avatar {
|
||
display: inline-block; vertical-align: middle;
|
||
width: 40px; height: 40px; border-radius: 50%;
|
||
margin: 0 10px 0 0; object-fit: cover;
|
||
}
|
||
.content-rendered img.emoji,
|
||
.content-rendered img[alt^=":"][alt$=":"],
|
||
.content-rendered img[src*="emoji"] {
|
||
display: inline; width: 20px; height: 20px;
|
||
max-width: 20px; margin: 0 2px; vertical-align: text-bottom;
|
||
}
|
||
.content-rendered a { color: #ff6b6b; }
|
||
.content-rendered pre, .content-rendered code {
|
||
background: #252525; padding: 2px 6px;
|
||
border-radius: 4px; font-family: monospace; font-size: 13px;
|
||
}
|
||
.content-rendered pre {
|
||
padding: 15px; display: block;
|
||
white-space: pre-wrap; overflow-x: auto;
|
||
}
|
||
.content-rendered blockquote {
|
||
background: #151520; border-left: 3px solid #4a9eff;
|
||
padding: 12px 16px; margin: 16px 0;
|
||
border-radius: 0 6px 6px 0; color: #aaa; font-style: italic;
|
||
}
|
||
.content-rendered hr { border: none; border-top: 1px solid #333; margin: 24px 0; }
|
||
.content-rendered p { margin: 0 0 16px 0; line-height: 1.7; }
|
||
.content-rendered h1, .content-rendered h2, .content-rendered h3, .content-rendered h4, .content-rendered h5, .content-rendered h6 {
|
||
display: block; color: #ff6b6b; margin-top: 28px; margin-bottom: 12px;
|
||
}
|
||
.content-rendered h1 { font-size: 1.8em; }
|
||
.content-rendered h2 { font-size: 1.5em; }
|
||
.content-rendered h3 { font-size: 1.25em; }
|
||
.media-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(auto-fill, minmax(150px, 1fr));
|
||
gap: 10px; margin-top: 15px;
|
||
}
|
||
.media-card {
|
||
background: #1a1a1a; border-radius: 8px;
|
||
overflow: hidden; display: block; transition: transform 0.2s;
|
||
}
|
||
.media-card:hover { transform: scale(1.02); }
|
||
.media-card img, .media-card video, .media-card svg {
|
||
width: 100%; height: 120px;
|
||
object-fit: contain; background: #0a0a0a;
|
||
}
|
||
.media-card-name {
|
||
padding: 4px 8px; font-size: 11px; color: #888;
|
||
white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
|
||
background: #111; text-align: center;
|
||
}
|
||
.tag { background: #333; padding: 2px 8px; border-radius: 4px; font-size: 12px; margin-right: 5px; }
|
||
"""
|
||
|
||
|
||
def render_detail_page(
|
||
title: str,
|
||
hero_html: str,
|
||
meta_rows: list,
|
||
media_items: list,
|
||
content_html: str,
|
||
screenshot_hashes: list,
|
||
source_domain: str,
|
||
page_uri: str,
|
||
download_btn_html: str = "",
|
||
noai: bool = False,
|
||
lang: str = "en",
|
||
sources_html: str = "",
|
||
) -> str:
|
||
"""
|
||
Shared template for media view and page view.
|
||
|
||
Args:
|
||
title: Page title
|
||
hero_html: HTML for hero section (media element or empty)
|
||
meta_rows: List of (label, value_html) tuples for metadata
|
||
media_items: List of media dicts with md5_hash, media_type
|
||
content_html: Rendered markdown/content HTML
|
||
screenshot_hashes: List of screenshot md5 hashes
|
||
source_domain: Domain for AI prompt
|
||
page_uri: Page URI for AI prompt
|
||
download_btn_html: Optional download button HTML
|
||
noai: Disable AI assistant
|
||
sources_html: Optional HTML for "Used on X pages" section (reverse image search)
|
||
"""
|
||
import html as html_module
|
||
|
||
# Build metadata section
|
||
meta_html = ''.join(
|
||
f'<div class="meta-row"><span class="meta-label">{label}:</span><span class="meta-value">{value}</span></div>'
|
||
for label, value in meta_rows
|
||
)
|
||
|
||
# Build media gallery
|
||
media_grid = ""
|
||
if media_items:
|
||
media_cards = []
|
||
# MIME to extension mapping for non-renderable files
|
||
mime_to_ext = {
|
||
'text/javascript': '.js', 'application/javascript': '.js',
|
||
'text/css': '.css', 'text/html': '.html',
|
||
'text/xml': '.xml', 'application/xml': '.xml',
|
||
'application/rss+xml': '.rss', 'application/atom+xml': '.atom',
|
||
'application/json': '.json', 'text/plain': '.txt',
|
||
'font/woff': '.woff', 'font/woff2': '.woff2',
|
||
'font/ttf': '.ttf', 'font/otf': '.otf',
|
||
'application/font-woff': '.woff', 'application/font-woff2': '.woff2',
|
||
}
|
||
def placeholder_svg(ext, icon, color):
|
||
return f'<svg viewBox="0 0 100 100" style="width:100%;height:100%;background:#1a1a1a;border-radius:4px;"><text x="50" y="40" text-anchor="middle" fill="{color}" font-size="24">{icon}</text><text x="50" y="65" text-anchor="middle" fill="#888" font-family="monospace" font-size="14" font-weight="bold">{ext}</text></svg>'
|
||
def get_filename(uri):
|
||
if not uri: return ""
|
||
# Extract filename from URI path
|
||
from urllib.parse import urlparse, unquote
|
||
path = urlparse(uri).path
|
||
name = unquote(path.split("/")[-1]) if path else ""
|
||
# Truncate long names
|
||
return name[:30] + "..." if len(name) > 33 else name
|
||
for m in media_items:
|
||
mtype = m.get("media_type", "")
|
||
mime = m.get("mime_type", "")
|
||
filename = get_filename(m.get("media_uri", ""))
|
||
if mtype == "video":
|
||
el = f'<video src="/media/{m["md5_hash"]}" muted loop preload="metadata" onmouseenter="this.play()" onmouseleave="this.pause()"></video>'
|
||
elif mtype == "code":
|
||
ext = mime_to_ext.get(mime, m.get("alt_text", ""))
|
||
if ext and not ext.startswith("."): ext = "." + ext
|
||
el = placeholder_svg(ext, "{ }", "#6af")
|
||
elif mtype == "style":
|
||
el = placeholder_svg(".css", "#", "#f6a")
|
||
elif mtype == "font":
|
||
ext = mime_to_ext.get(mime, ".font")
|
||
el = placeholder_svg(ext, "Aa", "#af6")
|
||
else:
|
||
el = f'<img src="/media/{m["md5_hash"]}" loading="lazy">'
|
||
filename_el = f'<div class="media-card-name">{html_module.escape(filename)}</div>' if filename else ""
|
||
media_cards.append(f'''<a href="/view/{m["md5_hash"]}" class="media-card">{el}{filename_el}</a>''')
|
||
media_count = len(media_items)
|
||
media_grid = '''
|
||
<div class="page-media">
|
||
<h3>{{media}} (''' + str(media_count) + ''')</h3>
|
||
<div class="media-grid">''' + ''.join(media_cards) + '''</div>
|
||
</div>'''
|
||
|
||
escaped_title = html_module.escape(title)
|
||
escaped_uri = html_module.escape(page_uri or "")
|
||
|
||
html = f"""<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>{escaped_title} - neopig</title>
|
||
<link rel="stylesheet" href="/static/vendor/highlight-github-dark.min.css">
|
||
<script src="/static/vendor/highlight.min.js"></script>
|
||
<style>
|
||
{BASE_CSS}
|
||
{VIEW_CSS}
|
||
.screenshot-stack {{ border-radius: 8px; }}
|
||
.screenshot-stack img {{ border-radius: 0; }}
|
||
.screenshot-stack img:first-child {{ border-radius: 8px 8px 0 0; }}
|
||
.screenshot-stack img:last-child {{ border-radius: 0 0 8px 8px; }}
|
||
.screenshot-stack img:only-child {{ border-radius: 8px; }}
|
||
#screenshot-container.collapsed {{ display: none; }}
|
||
.hero-grid {{ display: grid; grid-template-columns: 1fr; gap: 20px; margin-bottom: 20px; }}
|
||
.hero-grid.has-hero {{ grid-template-columns: 1fr 1fr; }}
|
||
.content-grid {{ display: grid; grid-template-columns: 1fr; gap: 20px; margin-top: 20px; }}
|
||
.content-grid.has-screenshots {{ grid-template-columns: 1fr 1fr; }}
|
||
.content-grid.ss-collapsed {{ grid-template-columns: 1fr 20px; }}
|
||
.content-grid.ss-collapsed .screenshot-col h3 {{ writing-mode: vertical-rl; text-orientation: mixed; margin: 0; font-size: 12px; }}
|
||
.content-rendered img.emoji {{ display: inline; width: 20px; height: 20px; margin: 0 2px; vertical-align: text-bottom; }}
|
||
.content-rendered .post-block {{ display: grid; grid-template-columns: 48px 1fr; gap: 12px; padding: 16px 0; border-bottom: 1px solid #252530; }}
|
||
.content-rendered .post-block:last-child {{ border-bottom: none; }}
|
||
.content-rendered .post-avatar-col {{ display: flex; flex-direction: column; align-items: center; }}
|
||
.content-rendered .post-avatar {{ width: 40px; height: 40px; border-radius: 50%; object-fit: cover; }}
|
||
.content-rendered .post-avatar-placeholder {{ width: 40px; height: 40px; border-radius: 50%; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: #fff; display: flex; align-items: center; justify-content: center; font-weight: bold; font-size: 18px; }}
|
||
.content-rendered .post-content {{ min-width: 0; }}
|
||
.content-rendered pre {{ position: relative; }}
|
||
.content-rendered pre .code-actions {{ position: absolute; top: 8px; right: 8px; display: flex; gap: 5px; }}
|
||
.content-rendered pre .code-actions button {{ background: #444; border: none; color: #ccc; padding: 4px 8px; border-radius: 4px; cursor: pointer; font-size: 11px; }}
|
||
.content-rendered pre .code-actions button:hover {{ background: #555; }}
|
||
.content-rendered pre .code-actions button.copied {{ background: #4ade80; color: #000; }}
|
||
.sources-section {{ margin-top: 40px; padding-top: 20px; border-top: 1px solid #333; }}
|
||
.sources-section h3 {{ cursor: pointer; user-select: none; }}
|
||
.sources-section h3:hover {{ color: #ff6b6b; }}
|
||
.sources-list {{ display: none; margin-top: 15px; }}
|
||
.sources-list.expanded {{ display: block; }}
|
||
.sources-list table {{ width: 100%; border-collapse: collapse; font-size: 13px; }}
|
||
.sources-list th, .sources-list td {{ padding: 8px 12px; text-align: left; border-bottom: 1px solid #333; }}
|
||
.sources-list th {{ background: #1a1a1a; color: #888; font-weight: normal; }}
|
||
.sources-list td a {{ color: #6ea8fe; }}
|
||
.sources-list td a:hover {{ color: #ff6b6b; }}
|
||
.sources-list .source-thumb {{ width: 40px; height: 40px; object-fit: cover; border-radius: 4px; }}
|
||
@media (max-width: 768px) {{ .hero-grid, .hero-grid.has-hero, .content-grid, .content-grid.has-screenshots {{ grid-template-columns: 1fr; }} }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<!-- NAV -->
|
||
<div class="container">
|
||
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
||
<input type="text" name="q" placeholder="{{search_placeholder}}" style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;">
|
||
<button type="submit" style="padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">{{search}}</button>
|
||
</form>
|
||
<div class="hero-grid{' has-hero' if hero_html else ''}">
|
||
{f'<div class="hero-left"><div style="background:#111;border-radius:12px;padding:20px;text-align:center;">{hero_html}</div></div>' if hero_html else ''}
|
||
<div class="hero-right" style="background:#151515;border-radius:12px;padding:20px;">
|
||
<h1 style="margin:0 0 15px 0;font-size:1.3em;">{escaped_title}</h1>
|
||
<div class="meta">{meta_html}</div>
|
||
{download_btn_html}
|
||
</div>
|
||
</div>
|
||
{media_grid}
|
||
<div id="content-grid" class="content-grid{' has-screenshots' if screenshot_hashes else ''}">
|
||
<div class="content-col">
|
||
{'<h3>{{content}}</h3><div class="content-rendered">' + content_html + '</div>' if content_html else ''}
|
||
</div>
|
||
{('<div class="screenshot-col"><h3 style="cursor:pointer;" onclick="toggleScreenshot()"><span id="ss-toggle">▼</span> {{screenshot}}</h3><div id="screenshot-container" class="screenshot-stack">' + ''.join('<img src="/media/' + h + '" alt="Screenshot" style="width:100%;display:block;">' for h in screenshot_hashes) + '</div></div>') if screenshot_hashes else ''}
|
||
</div>
|
||
<script>
|
||
hljs.highlightAll();
|
||
|
||
// Screenshot toggle
|
||
function toggleScreenshot() {{
|
||
const container = document.getElementById('screenshot-container');
|
||
const toggle = document.getElementById('ss-toggle');
|
||
const grid = document.getElementById('content-grid');
|
||
if (container) {{
|
||
const collapsed = !container.classList.contains('collapsed');
|
||
container.classList.toggle('collapsed', collapsed);
|
||
grid.classList.toggle('ss-collapsed', collapsed);
|
||
toggle.textContent = collapsed ? '▶' : '▼';
|
||
localStorage.setItem('neopig_ss_collapsed', collapsed);
|
||
}}
|
||
}}
|
||
if (localStorage.getItem('neopig_ss_collapsed') === 'true') {{
|
||
const container = document.getElementById('screenshot-container');
|
||
const toggle = document.getElementById('ss-toggle');
|
||
const grid = document.getElementById('content-grid');
|
||
if (container) {{
|
||
container.classList.add('collapsed');
|
||
grid.classList.add('ss-collapsed');
|
||
toggle.textContent = '▶';
|
||
}}
|
||
}}
|
||
|
||
// Auto-detect emojis/avatars
|
||
document.querySelectorAll('.content-rendered img').forEach(img => {{
|
||
const check = () => {{
|
||
const w = img.naturalWidth || img.width, h = img.naturalHeight || img.height;
|
||
if (w > 0 && h > 0) {{
|
||
if (w <= 24 && h <= 24) img.classList.add('emoji');
|
||
else if (w <= 60 && h <= 60) img.classList.add('avatar');
|
||
}}
|
||
}};
|
||
if (img.complete) check(); else img.onload = check;
|
||
}});
|
||
|
||
// Code block copy/download
|
||
const extMap = {{'python':'py','javascript':'js','typescript':'ts','cpp':'cpp','c':'c','java':'java','rust':'rs','go':'go','ruby':'rb','php':'php','html':'html','css':'css','json':'json','yaml':'yaml','sql':'sql','bash':'sh','sh':'sh','markdown':'md'}};
|
||
document.querySelectorAll('.content-rendered pre').forEach((pre, idx) => {{
|
||
const code = pre.querySelector('code') || pre;
|
||
const text = code.textContent;
|
||
let ext = 'txt';
|
||
(code.className || '').split(/\\s+/).forEach(cls => {{
|
||
const m = cls.match(/^(?:language-)?(.+)$/);
|
||
if (m && extMap[m[1].toLowerCase()]) ext = extMap[m[1].toLowerCase()];
|
||
}});
|
||
const actions = document.createElement('div');
|
||
actions.className = 'code-actions';
|
||
const copyBtn = document.createElement('button');
|
||
copyBtn.textContent = T.copy;
|
||
copyBtn.onclick = async () => {{
|
||
await navigator.clipboard.writeText(text);
|
||
copyBtn.textContent = T.copied; copyBtn.classList.add('copied');
|
||
setTimeout(() => {{ copyBtn.textContent = T.copy; copyBtn.classList.remove('copied'); }}, 2000);
|
||
}};
|
||
const dlBtn = document.createElement('button');
|
||
dlBtn.textContent = T.download;
|
||
dlBtn.onclick = () => {{
|
||
const blob = new Blob([text], {{type: 'text/plain'}});
|
||
const a = document.createElement('a');
|
||
a.href = URL.createObjectURL(blob);
|
||
a.download = `code-${{idx + 1}}.${{ext}}`;
|
||
a.click();
|
||
}};
|
||
actions.appendChild(copyBtn); actions.appendChild(dlBtn);
|
||
pre.appendChild(actions);
|
||
}});
|
||
|
||
// Forum post restructuring
|
||
(function() {{
|
||
const container = document.querySelector('.content-rendered');
|
||
if (!container) return;
|
||
const postStarts = [];
|
||
container.querySelectorAll('p').forEach(p => {{
|
||
const img = p.querySelector('img[alt="avatar"]');
|
||
const strong = p.querySelector('strong');
|
||
if (img && strong) postStarts.push({{p, img, username: strong.textContent}});
|
||
}});
|
||
if (!postStarts.length) return;
|
||
postStarts.forEach(({{p, img, username}}, idx) => {{
|
||
if (p.closest('.post-block')) return;
|
||
const block = document.createElement('div');
|
||
block.className = 'post-block';
|
||
const avatarCol = document.createElement('div');
|
||
avatarCol.className = 'post-avatar-col';
|
||
if (img.src && !img.src.includes('undefined') && !img.src.endsWith('#') && img.getAttribute('src') !== '#') {{
|
||
const avatar = document.createElement('img');
|
||
avatar.className = 'post-avatar';
|
||
avatar.src = img.src;
|
||
avatarCol.appendChild(avatar);
|
||
}} else {{
|
||
const ph = document.createElement('div');
|
||
ph.className = 'post-avatar-placeholder';
|
||
ph.textContent = username.charAt(0).toUpperCase();
|
||
avatarCol.appendChild(ph);
|
||
}}
|
||
const contentCol = document.createElement('div');
|
||
contentCol.className = 'post-content';
|
||
contentCol.innerHTML = `<div class="post-header"><strong>${{username}}</strong></div>`;
|
||
let sib = p.nextElementSibling;
|
||
const nextP = idx < postStarts.length - 1 ? postStarts[idx + 1].p : null;
|
||
while (sib && sib !== nextP && sib.tagName !== 'HR') {{
|
||
contentCol.appendChild(sib.cloneNode(true));
|
||
const rm = sib; sib = sib.nextElementSibling; rm.remove();
|
||
}}
|
||
if (sib && sib.tagName === 'HR') sib.remove();
|
||
block.appendChild(avatarCol); block.appendChild(contentCol);
|
||
p.parentNode.insertBefore(block, p); p.remove();
|
||
}});
|
||
}})();
|
||
</script>
|
||
{sources_html}
|
||
{'' if noai else f'''<script>
|
||
window.UNCLOSEAI_SYSTEM_PROMPT = "You are a neopig assistant, a machine learning persona built into the neopig web archiver. You are viewing an archived copy of a page from {source_domain}. The page title is: {escaped_title}. neopig archived this page - neopig did NOT create the original content. The original content was created by {source_domain}. Help users understand what is on this archived page. Be factual and concise. Only describe what you can actually see - do not invent or embellish.";
|
||
window.UNCLOSEAI_PREFILL = "I am a neopig assistant viewing an archived copy of a page from {source_domain}. The page title is: {escaped_title}. neopig archived this page - neopig did NOT create the original content. The original content was created by {source_domain}. I will help users understand what is on this archived page. I will be factual and concise. I will only describe what I can actually see - I will not invent or embellish.";
|
||
</script>
|
||
<script src="https://uncloseai.com/uncloseai.js" type="module"></script>'''}
|
||
</body>
|
||
</html>"""
|
||
return inject_i18n(html, lang)
|
||
|
||
|
||
def _open_tarball():
|
||
"""Open a fresh tarball handle for this thread."""
|
||
if TAR_OFFSET > 0:
|
||
# .run file - need to seek past bootstrap
|
||
f = open(TAR_PATH, 'rb')
|
||
f.seek(TAR_OFFSET)
|
||
return tarfile.open(fileobj=f, mode='r:gz')
|
||
else:
|
||
return tarfile.open(TAR_PATH, 'r:gz')
|
||
|
||
|
||
def read_from_tarball(path: str) -> bytes:
|
||
"""Read a file from the tarball. Path is relative to archive root.
|
||
|
||
Thread-safe: opens its own tarball handle.
|
||
"""
|
||
if not TAR_PATH or not ARCHIVE_ROOT:
|
||
return None
|
||
full_path = f"{ARCHIVE_ROOT}/{path}"
|
||
if full_path in TAR_MEMBERS:
|
||
member = TAR_MEMBERS[full_path]
|
||
tar = _open_tarball()
|
||
try:
|
||
f = tar.extractfile(member)
|
||
if f:
|
||
return f.read()
|
||
finally:
|
||
tar.close()
|
||
return None
|
||
|
||
|
||
def find_media_in_tarball(md5_hash: str) -> tuple:
|
||
"""Find media file in tarball by hash. Returns (data, extension) or (None, None).
|
||
|
||
Thread-safe: opens its own tarball handle for parallel reads.
|
||
"""
|
||
if not TAR_PATH or not ARCHIVE_ROOT:
|
||
return None, None
|
||
|
||
# O(1) lookup via pre-built index
|
||
if md5_hash in TAR_MEDIA_INDEX:
|
||
name = TAR_MEDIA_INDEX[md5_hash]
|
||
member = TAR_MEMBERS.get(name)
|
||
if member:
|
||
tar = _open_tarball()
|
||
try:
|
||
f = tar.extractfile(member)
|
||
if f:
|
||
ext = Path(name).suffix
|
||
return f.read(), ext
|
||
finally:
|
||
tar.close()
|
||
|
||
return None, None
|
||
|
||
|
||
class ArchiveDB:
|
||
"""Simple sync SQLite wrapper for archive.db (FTS5 search only)."""
|
||
def __init__(self, db_path):
|
||
import sqlite3
|
||
self.conn = sqlite3.connect(db_path)
|
||
self.conn.row_factory = sqlite3.Row
|
||
|
||
async def search_pages(self, query, limit=50):
|
||
cursor = self.conn.execute(
|
||
"SELECT uri, title, snippet(pages_fts, 2, '<b>', '</b>', '...', 32) as snippet "
|
||
"FROM pages_fts WHERE pages_fts MATCH ? LIMIT ?",
|
||
(query, limit)
|
||
)
|
||
return [dict(row) for row in cursor.fetchall()]
|
||
|
||
async def get_stats(self):
|
||
cursor = self.conn.execute("SELECT COUNT(*) FROM pages")
|
||
return {"pages": cursor.fetchone()[0], "media": 0, "screenshots": 0}
|
||
|
||
|
||
@app.on_event("startup")
|
||
async def startup_event():
|
||
"""Initialize database on startup."""
|
||
global db
|
||
if CRAWL_DISABLED:
|
||
logger.info("Crawl disabled via NEOPIG_DISABLE_CRAWL")
|
||
if SANDBOX_MODE:
|
||
logger.info("Sandbox mode enabled via NEOPIG_SANDBOX - archive uploads allowed")
|
||
if TAR_PATH:
|
||
# Tarball mode: use full Database for neopig.db, legacy ArchiveDB for archive.db
|
||
if DB_PATH.endswith('neopig.db'):
|
||
db = Database(DB_PATH)
|
||
await db.init()
|
||
logger.info(f"Using neopig database from tarball: {DB_PATH}")
|
||
else:
|
||
db = ArchiveDB(DB_PATH)
|
||
logger.info(f"Using archive database: {DB_PATH}")
|
||
else:
|
||
# Normal mode: use full async database
|
||
db = Database(DB_PATH)
|
||
await db.init() # Handles schema + WAL mode
|
||
# Backfill uri_hash for existing pages
|
||
count = await db.backfill_page_hashes()
|
||
if count:
|
||
logger.info(f"Backfilled {count} page URI hashes")
|
||
VAULT_PATH.mkdir(parents=True, exist_ok=True)
|
||
logger.info(f"Vault directory ready: {VAULT_PATH}")
|
||
|
||
# Mark any orphaned "running" jobs as "paused" (server was killed)
|
||
async with db.session() as session:
|
||
result = await session.execute(
|
||
text("UPDATE crawl_jobs SET status = 'paused' WHERE status = 'running'")
|
||
)
|
||
if result.rowcount:
|
||
await session.commit()
|
||
logger.info(f"Marked {result.rowcount} orphaned running job(s) as paused")
|
||
|
||
|
||
@app.on_event("shutdown")
|
||
async def shutdown_event():
|
||
"""Pause active crawls on shutdown so they can be resumed."""
|
||
if ACTIVE_CRAWL_TASKS:
|
||
logger.info(f"Pausing {len(ACTIVE_CRAWL_TASKS)} active crawl(s)...")
|
||
for job_id, task in list(ACTIVE_CRAWL_TASKS.items()):
|
||
task.cancel()
|
||
try:
|
||
await task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
# Mark as paused so it can be resumed
|
||
try:
|
||
await db.pause_crawl_job(job_id)
|
||
except Exception as e:
|
||
logger.error(f"Failed to pause job {job_id}: {e}")
|
||
logger.info("All crawls paused")
|
||
|
||
|
||
class CrawlRequest(BaseModel):
|
||
"""Request to start a new crawl."""
|
||
targets: List[str] = [] # Multiple target URIs
|
||
target_uri: str = "" # Deprecated: single target (for backwards compat)
|
||
keywords: List[str] = []
|
||
mode: str = "all" # text, images, videos, media, all
|
||
depth: int = -1 # -1 = unlimited
|
||
max_pages: int = -1 # -1 = unlimited
|
||
fresh: bool = False # Start fresh (rotate state files) - default False for safety
|
||
fast: bool = False # No crawl delay
|
||
screenshots: bool = True # Take page screenshots
|
||
hydra: bool = False # Parse RSS/Atom feeds for bleeding edge discovery
|
||
|
||
|
||
@app.get("/", response_class=HTMLResponse)
|
||
async def index(lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Simple search UI."""
|
||
language = get_lang(lang, accept_language)
|
||
return inject_i18n(get_search_html(), language)
|
||
|
||
|
||
@app.get("/crawl", response_class=HTMLResponse)
|
||
async def crawl_page(lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Crawler command page."""
|
||
language = get_lang(lang, accept_language)
|
||
html = CRAWL_HTML
|
||
if CRAWL_DISABLED:
|
||
# Inject disabled state into the page
|
||
html = html.replace('const CRAWL_DISABLED = false;', 'const CRAWL_DISABLED = true;')
|
||
return inject_i18n(html, language)
|
||
|
||
|
||
# Page-specific CSS for Search page
|
||
SEARCH_CSS = """
|
||
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
||
.subtitle { color: #666; margin-bottom: 20px; }
|
||
.stats { padding: 10px 15px; background: #1a1a1a; border-radius: 8px; margin-bottom: 20px; font-size: 14px; color: #888; }
|
||
.section-title { color: #ff6b6b; font-size: 18px; margin: 25px 0 15px 0; border-bottom: 1px solid #333; padding-bottom: 8px; }
|
||
.results { display: grid; grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); gap: 15px; }
|
||
.result { background: #1a1a1a; border-radius: 8px; overflow: hidden; transition: transform 0.2s; }
|
||
.result:hover { transform: scale(1.02); }
|
||
.result img, .result video { width: 100%; height: 200px; object-fit: contain; background: #1a1a1a; }
|
||
.result-info { padding: 10px; }
|
||
.result-filename { font-size: 12px; color: #aaa; margin-bottom: 4px; word-break: break-all; }
|
||
.result-hash { font-family: monospace; font-size: 11px; color: #ff6b6b; text-decoration: none; word-break: break-all; }
|
||
.result-hash:hover { text-decoration: underline; }
|
||
.result-meta { font-size: 12px; color: #888; margin-top: 5px; }
|
||
.result-keywords { display: flex; flex-wrap: wrap; gap: 5px; margin-top: 8px; }
|
||
.tag { background: #333; padding: 2px 8px; border-radius: 4px; font-size: 11px; color: #aaa; }
|
||
.no-results { text-align: center; padding: 60px; color: #666; }
|
||
.result-count { padding: 10px 15px; color: #888; font-size: 14px; }
|
||
.result-count.over-9000 { color: #ff6b6b; font-weight: bold; font-size: 18px; text-shadow: 0 0 10px rgba(255,107,107,0.5); animation: powerUp 0.5s ease-out; }
|
||
@keyframes powerUp { 0% { transform: scale(1); } 50% { transform: scale(1.2); } 100% { transform: scale(1); } }
|
||
.pagination { display: flex; gap: 15px; justify-content: center; padding: 20px; margin-top: 20px; border-top: 1px solid #333; }
|
||
.pagination button { padding: 10px 20px; background: #4ecdc4; color: #1a1a2e; border: none; border-radius: 5px; cursor: pointer; font-weight: bold; }
|
||
.pagination button:hover { background: #7bed72; }
|
||
.media-link { display: block; cursor: pointer; }
|
||
.media-link:hover img, .media-link:hover video { opacity: 0.8; }
|
||
.page-results { display: flex; flex-direction: column; gap: 10px; }
|
||
.page-result { background: #1a1a1a; border-radius: 8px; padding: 15px; transition: background 0.2s; }
|
||
.page-result:hover { background: #252525; }
|
||
.page-result a { color: #ff6b6b; text-decoration: none; font-size: 16px; font-weight: 500; }
|
||
.page-result a:hover { text-decoration: underline; }
|
||
.page-path { font-size: 12px; color: #4ade80; margin-top: 4px; font-family: monospace; }
|
||
.page-snippet { font-size: 13px; color: #999; margin-top: 8px; line-height: 1.5; }
|
||
.page-snippet mark { background: #ff6b6b33; color: #ff9999; padding: 1px 3px; border-radius: 2px; }
|
||
.search-columns { display: grid; grid-template-columns: 1fr; gap: 30px; align-items: start; }
|
||
.search-columns.has-pages { grid-template-columns: 1fr 2fr; }
|
||
.search-columns .page-column { display: none; }
|
||
.search-columns.has-pages .page-column { display: block; }
|
||
@media (max-width: 1000px) { .search-columns.has-pages { grid-template-columns: 1fr; } }
|
||
"""
|
||
|
||
SEARCH_CONTENT = """
|
||
<div class="container">
|
||
<div class="search-box">
|
||
<input type="text" id="query" placeholder="{{search_placeholder}}" autofocus>
|
||
<select id="type">
|
||
<option value="">{{all_types}}</option>
|
||
<option value="image">{{images}}</option>
|
||
<option value="video">{{videos}}</option>
|
||
<option value="audio">{{audio}}</option>
|
||
</select>
|
||
<button onclick="search()">{{search}}</button>
|
||
</div>
|
||
|
||
<div class="stats" id="stats">{{loading_stats}}</div>
|
||
|
||
<div class="search-columns">
|
||
<div class="page-column" id="page-section">
|
||
<h2 class="section-title">{{pages_label}}</h2>
|
||
<div class="page-results" id="page-results"></div>
|
||
</div>
|
||
|
||
<div class="media-column" id="media-section">
|
||
<h2 class="section-title">{{media}}</h2>
|
||
<div class="results" id="results"></div>
|
||
</div>
|
||
</div>
|
||
|
||
<script>
|
||
async function loadStats() {
|
||
const res = await fetch('/api/stats');
|
||
const stats = await res.json();
|
||
document.getElementById('stats').innerHTML =
|
||
`<strong>${stats.total_media}</strong> ${T.media} | ` +
|
||
`<strong>${stats.by_type?.image || 0}</strong> ${T.images} | ` +
|
||
`<strong>${stats.by_type?.video || 0}</strong> ${T.videos} | ` +
|
||
`<strong>${stats.total_sources}</strong> ${T.sources} | ` +
|
||
`<strong>${stats.total_pages || 0}</strong> ${T.pages_label}`;
|
||
}
|
||
|
||
const OVER_9000 = 9000;
|
||
let currentOffset = 0;
|
||
|
||
async function search(offset = 0) {
|
||
currentOffset = offset;
|
||
const query = document.getElementById('query').value;
|
||
const type = document.getElementById('type').value;
|
||
|
||
// Search media - IT'S OVER 9000!
|
||
let mediaUrl = `/api/search?q=${encodeURIComponent(query)}&limit=${OVER_9000}&offset=${offset}`;
|
||
if (type) mediaUrl += `&type=${type}`;
|
||
|
||
const mediaRes = await fetch(mediaUrl);
|
||
const mediaResults = await mediaRes.json();
|
||
|
||
const mediaContainer = document.getElementById('results');
|
||
const mediaSection = document.getElementById('media-section');
|
||
|
||
// Build count display
|
||
let countDisplay = '';
|
||
if (mediaResults.length >= OVER_9000) {
|
||
countDisplay = `<div class="result-count over-9000">${T.over_9000}</div>`;
|
||
} else if (mediaResults.length > 0) {
|
||
const start = offset + 1;
|
||
const end = offset + mediaResults.length;
|
||
countDisplay = `<div class="result-count">${start.toLocaleString()}-${end.toLocaleString()} of ${offset > 0 ? 'many' : mediaResults.length.toLocaleString()}</div>`;
|
||
}
|
||
|
||
// Build pagination
|
||
let pagination = '';
|
||
if (offset > 0 || mediaResults.length >= OVER_9000) {
|
||
pagination = '<div class="pagination">';
|
||
if (offset > 0) {
|
||
pagination += `<button onclick="search(${Math.max(0, offset - OVER_9000)})">← ${T.previous} ${OVER_9000.toLocaleString()}</button>`;
|
||
}
|
||
if (mediaResults.length >= OVER_9000) {
|
||
pagination += `<button onclick="search(${offset + OVER_9000})">${T.next} ${OVER_9000.toLocaleString()} →</button>`;
|
||
}
|
||
pagination += '</div>';
|
||
}
|
||
|
||
if (mediaResults.length === 0 && offset === 0) {
|
||
mediaContainer.innerHTML = `<div class="no-results">${T.no_media}</div>`;
|
||
} else if (mediaResults.length === 0) {
|
||
mediaContainer.innerHTML = `<div class="no-results">${T.no_more}</div>` + pagination;
|
||
} else {
|
||
mediaContainer.innerHTML = countDisplay + mediaResults.map(r => {
|
||
const isVideo = r.media_type === 'video';
|
||
const isCode = r.media_type === 'code';
|
||
const isStyle = r.media_type === 'style';
|
||
const isFont = r.media_type === 'font';
|
||
|
||
// Get file extension from mime_type
|
||
const getExt = (mime) => {
|
||
const mimeToExt = {
|
||
'text/javascript': '.js', 'application/javascript': '.js',
|
||
'text/x-python': '.py', 'text/python': '.py',
|
||
'text/x-c': '.c', 'text/x-csrc': '.c',
|
||
'text/x-c++': '.cpp', 'text/x-c++src': '.cpp',
|
||
'text/x-java': '.java', 'text/java': '.java',
|
||
'text/x-ruby': '.rb', 'text/ruby': '.rb',
|
||
'text/x-go': '.go', 'text/go': '.go',
|
||
'text/x-rust': '.rs', 'text/rust': '.rs',
|
||
'text/x-php': '.php', 'text/php': '.php',
|
||
'text/x-shellscript': '.sh', 'text/x-sh': '.sh',
|
||
'text/css': '.css', 'text/html': '.html',
|
||
'text/xml': '.xml', 'application/xml': '.xml',
|
||
'application/rss+xml': '.rss', 'application/atom+xml': '.atom',
|
||
'application/json': '.json', 'text/json': '.json',
|
||
'text/x-yaml': '.yaml', 'text/yaml': '.yaml',
|
||
'text/markdown': '.md', 'text/x-markdown': '.md',
|
||
'text/plain': '.txt', 'text/x-typescript': '.ts',
|
||
'font/woff': '.woff', 'font/woff2': '.woff2',
|
||
'font/ttf': '.ttf', 'font/otf': '.otf',
|
||
'application/font-woff': '.woff', 'application/font-woff2': '.woff2',
|
||
'application/x-font-ttf': '.ttf', 'application/x-font-otf': '.otf',
|
||
};
|
||
return mimeToExt[mime] || (r.alt_text ? '.' + r.alt_text : '');
|
||
};
|
||
|
||
// SVG placeholder for non-renderable files
|
||
const placeholder = (ext, icon, color) => `<svg viewBox="0 0 100 100" style="width:100%;height:100%;background:#1a1a1a;border-radius:4px;">
|
||
<text x="50" y="40" text-anchor="middle" fill="${color}" font-size="24">${icon}</text>
|
||
<text x="50" y="65" text-anchor="middle" fill="#888" font-family="monospace" font-size="14" font-weight="bold">${ext}</text>
|
||
</svg>`;
|
||
|
||
let mediaEl;
|
||
if (isVideo) {
|
||
mediaEl = `<video src="/media/${r.md5_hash}" muted loop preload="metadata" onmouseenter="this.play()" onmouseleave="this.pause()"></video>`;
|
||
} else if (isCode) {
|
||
mediaEl = placeholder(getExt(r.mime_type), '{ }', '#6af');
|
||
} else if (isStyle) {
|
||
mediaEl = placeholder('.css', '#', '#f6a');
|
||
} else if (isFont) {
|
||
mediaEl = placeholder(getExt(r.mime_type), 'Aa', '#af6');
|
||
} else {
|
||
mediaEl = `<img src="/media/${r.md5_hash}" alt="${r.alt_text || ''}" loading="lazy">`;
|
||
}
|
||
|
||
const keywords = JSON.parse(r.keywords || '[]');
|
||
const tagsHtml = keywords.map(k => `<span class="tag">${k}</span>`).join('');
|
||
|
||
// Extract filename from media_uri
|
||
const getFilename = (uri) => {
|
||
if (!uri) return '';
|
||
try {
|
||
const path = new URL(uri).pathname;
|
||
const name = decodeURIComponent(path.split('/').pop() || '');
|
||
return name.length > 40 ? name.slice(0, 37) + '...' : name;
|
||
} catch { return ''; }
|
||
};
|
||
const filename = getFilename(r.media_uri);
|
||
|
||
return `
|
||
<div class="result">
|
||
<a href="/view/${r.md5_hash}" class="media-link">
|
||
${mediaEl}
|
||
</a>
|
||
<div class="result-info">
|
||
${filename ? `<div class="result-filename">${filename}</div>` : ''}
|
||
<a href="/view/${r.md5_hash}" class="result-hash">${r.md5_hash}</a>
|
||
<div class="result-meta">
|
||
${r.media_type} · ${formatBytes(r.file_size)}
|
||
${r.alt_text ? ` · ${r.alt_text.substring(0, 50)}` : ''}
|
||
</div>
|
||
<div class="result-keywords">${tagsHtml}</div>
|
||
</div>
|
||
</div>
|
||
`;
|
||
}).join('') + pagination;
|
||
}
|
||
|
||
// Search pages
|
||
const pageContainer = document.getElementById('page-results');
|
||
const searchColumns = document.querySelector('.search-columns');
|
||
|
||
if (query.trim()) {
|
||
const pageRes = await fetch(`/api/search/pages?q=${encodeURIComponent(query)}&limit=30`);
|
||
const pageResults = await pageRes.json();
|
||
|
||
if (pageResults.length > 0) {
|
||
searchColumns.classList.add('has-pages');
|
||
pageContainer.innerHTML = pageResults.map(p => `
|
||
<div class="page-result">
|
||
<a href="/page/${p.uri_hash}">${p.title || p.uri}</a>
|
||
<div class="page-path">${p.path || p.uri}</div>
|
||
<div class="page-snippet">${p.snippet || ''}</div>
|
||
</div>
|
||
`).join('');
|
||
} else {
|
||
searchColumns.classList.remove('has-pages');
|
||
pageContainer.innerHTML = '';
|
||
}
|
||
} else {
|
||
searchColumns.classList.remove('has-pages');
|
||
pageContainer.innerHTML = '';
|
||
}
|
||
}
|
||
|
||
function formatBytes(bytes) {
|
||
if (!bytes) return '?';
|
||
if (bytes < 1024) return bytes + ' B';
|
||
if (bytes < 1024*1024) return (bytes/1024).toFixed(1) + ' KB';
|
||
return (bytes/1024/1024).toFixed(1) + ' MB';
|
||
}
|
||
|
||
// Enter key to search
|
||
document.getElementById('query').addEventListener('keypress', e => {
|
||
if (e.key === 'Enter') search();
|
||
});
|
||
|
||
// Load stats on page load
|
||
loadStats();
|
||
|
||
// Check for query param from nav search
|
||
const urlParams = new URLSearchParams(window.location.search);
|
||
const q = urlParams.get('q');
|
||
if (q) {
|
||
document.getElementById('query').value = q;
|
||
}
|
||
|
||
// Initial search (show all media or query)
|
||
search();
|
||
</script>
|
||
</div>
|
||
"""
|
||
|
||
def get_search_html():
|
||
"""Build search page using layout."""
|
||
return layout("neopig SERP", SEARCH_CONTENT, SEARCH_CSS)
|
||
|
||
CRAWL_HTML = """
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>neopig {{crawl}}</title>
|
||
<style>
|
||
* { box-sizing: border-box; }
|
||
body {
|
||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||
margin: 0;
|
||
padding: 0;
|
||
background: #0a0a0a;
|
||
color: #e0e0e0;
|
||
}
|
||
.nav {
|
||
background: #1a1a1a;
|
||
padding: 10px 20px;
|
||
display: flex;
|
||
gap: 20px;
|
||
align-items: center;
|
||
border-bottom: 1px solid #333;
|
||
}
|
||
.nav a { color: #ff6b6b; text-decoration: none; }
|
||
.nav a:hover { text-decoration: underline; }
|
||
.nav .brand { font-weight: bold; font-size: 18px; }
|
||
.container { padding: 20px; }
|
||
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
||
.subtitle { color: #666; margin-bottom: 20px; }
|
||
|
||
.form-group {
|
||
margin-bottom: 15px;
|
||
}
|
||
label {
|
||
display: block;
|
||
margin-bottom: 5px;
|
||
color: #aaa;
|
||
font-size: 14px;
|
||
}
|
||
input[type="text"], input[type="number"], select {
|
||
width: 100%;
|
||
padding: 12px 16px;
|
||
font-size: 16px;
|
||
border: 2px solid #333;
|
||
border-radius: 8px;
|
||
background: #1a1a1a;
|
||
color: #fff;
|
||
}
|
||
input:focus, select:focus {
|
||
outline: none;
|
||
border-color: #ff6b6b;
|
||
}
|
||
.row {
|
||
display: flex;
|
||
gap: 15px;
|
||
}
|
||
.row > div { flex: 1; }
|
||
|
||
button {
|
||
padding: 14px 28px;
|
||
font-size: 16px;
|
||
background: #ff6b6b;
|
||
color: #fff;
|
||
border: none;
|
||
border-radius: 8px;
|
||
cursor: pointer;
|
||
margin-top: 10px;
|
||
}
|
||
button:hover { background: #ff5252; }
|
||
button:disabled {
|
||
background: #444;
|
||
cursor: not-allowed;
|
||
}
|
||
button.secondary {
|
||
background: #333;
|
||
}
|
||
button.secondary:hover {
|
||
background: #444;
|
||
}
|
||
|
||
.checkbox-group {
|
||
display: flex;
|
||
align-items: center;
|
||
gap: 8px;
|
||
}
|
||
.checkbox-group input {
|
||
width: auto;
|
||
}
|
||
|
||
.jobs-section {
|
||
margin-top: 30px;
|
||
padding-top: 20px;
|
||
border-top: 1px solid #333;
|
||
}
|
||
h2 {
|
||
color: #ff6b6b;
|
||
font-size: 18px;
|
||
margin-bottom: 15px;
|
||
}
|
||
|
||
.job {
|
||
background: #1a1a1a;
|
||
border-radius: 8px;
|
||
padding: 15px;
|
||
margin-bottom: 10px;
|
||
}
|
||
.job-header {
|
||
display: flex;
|
||
justify-content: space-between;
|
||
align-items: center;
|
||
margin-bottom: 10px;
|
||
}
|
||
.job-id {
|
||
font-family: monospace;
|
||
color: #666;
|
||
}
|
||
.job-status {
|
||
padding: 4px 10px;
|
||
border-radius: 4px;
|
||
font-size: 12px;
|
||
font-weight: bold;
|
||
}
|
||
.job-status.running { background: #2d5a27; color: #7bed72; }
|
||
.job-status.completed { background: #1a3a4a; color: #6bc5e8; }
|
||
.job-status.failed { background: #5a2727; color: #ed7272; }
|
||
|
||
.job-target {
|
||
font-size: 14px;
|
||
word-break: break-all;
|
||
margin-bottom: 5px;
|
||
}
|
||
.job-meta {
|
||
font-size: 12px;
|
||
color: #666;
|
||
}
|
||
.job-stats {
|
||
display: flex;
|
||
gap: 15px;
|
||
margin-top: 10px;
|
||
font-size: 13px;
|
||
}
|
||
.job-stats span {
|
||
background: #252525;
|
||
padding: 4px 10px;
|
||
border-radius: 4px;
|
||
}
|
||
|
||
.progress-bar {
|
||
height: 4px;
|
||
background: #333;
|
||
border-radius: 2px;
|
||
margin-top: 10px;
|
||
overflow: hidden;
|
||
}
|
||
.progress-bar-fill {
|
||
height: 100%;
|
||
background: #ff6b6b;
|
||
transition: width 0.3s;
|
||
}
|
||
.progress-bar-fill.running {
|
||
width: 100%;
|
||
background: linear-gradient(90deg, #ff6b6b 0%, #4ecdc4 50%, #ff6b6b 100%);
|
||
background-size: 200% 100%;
|
||
animation: progress-wave 1.5s ease-in-out infinite;
|
||
}
|
||
.job-actions { margin-top: 8px; display: flex; gap: 8px; }
|
||
.job-actions button {
|
||
padding: 4px 12px;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 4px;
|
||
cursor: pointer;
|
||
font-size: 12px;
|
||
}
|
||
.pause-btn { background: #e67e22; }
|
||
.pause-btn:hover { background: #f39c12; }
|
||
.start-btn { background: #27ae60; }
|
||
.start-btn:hover { background: #2ecc71; }
|
||
.recrawl-btn { background: #3498db; }
|
||
.recrawl-btn:hover { background: #2980b9; }
|
||
.delete-btn { background: #7f8c8d; }
|
||
.delete-btn:hover { background: #c0392b; }
|
||
.job-links { margin-left: 10px; font-size: 12px; }
|
||
.job-links a { color: #4ecdc4; margin-right: 8px; }
|
||
.job-links a:hover { color: #7bed72; }
|
||
@keyframes progress-wave {
|
||
0% { background-position: 200% 0; }
|
||
100% { background-position: -200% 0; }
|
||
}
|
||
.job.running {
|
||
border-color: #4ecdc4;
|
||
}
|
||
.job-stats.live {
|
||
color: #4ecdc4;
|
||
}
|
||
|
||
.no-jobs {
|
||
color: #666;
|
||
text-align: center;
|
||
padding: 30px;
|
||
}
|
||
.console-btn { background: #2c3e50; }
|
||
.console-btn:hover { background: #34495e; }
|
||
.console-modal {
|
||
display: none;
|
||
position: fixed;
|
||
top: 0; left: 0; right: 0; bottom: 0;
|
||
background: rgba(0,0,0,0.8);
|
||
z-index: 1000;
|
||
padding: 20px;
|
||
}
|
||
.console-modal.active { display: flex; flex-direction: column; }
|
||
.console-header {
|
||
display: flex;
|
||
justify-content: space-between;
|
||
align-items: center;
|
||
padding: 10px 15px;
|
||
background: #1a1a1a;
|
||
border-radius: 8px 8px 0 0;
|
||
}
|
||
.console-header h3 { margin: 0; color: #ff6b6b; }
|
||
.console-close {
|
||
background: #c0392b;
|
||
border: none;
|
||
color: white;
|
||
padding: 5px 15px;
|
||
border-radius: 4px;
|
||
cursor: pointer;
|
||
}
|
||
.console-body {
|
||
flex: 1;
|
||
background: #0d0d0d;
|
||
padding: 15px;
|
||
overflow: auto;
|
||
font-family: monospace;
|
||
font-size: 12px;
|
||
white-space: pre-wrap;
|
||
word-break: break-all;
|
||
color: #4ade80;
|
||
border-radius: 0 0 8px 8px;
|
||
}
|
||
.console-body .log-line { margin: 0; line-height: 1.4; }
|
||
.console-body .log-INFO { color: #4ade80; }
|
||
.console-body .log-WARNING { color: #f39c12; }
|
||
.console-body .log-ERROR { color: #e74c3c; }
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<!-- NAV -->
|
||
<div class="container">
|
||
|
||
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
||
<input type="text" name="q" placeholder="{{search_placeholder}}" style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||
<option value="">{{all_types}}</option>
|
||
<option value="image">{{images}}</option>
|
||
<option value="video">{{videos}}</option>
|
||
<option value="audio">{{audio}}</option>
|
||
</select>
|
||
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">{{search}}</button>
|
||
</form>
|
||
|
||
<h1>{{crawl}}</h1>
|
||
<p class="subtitle">{{hydrate_subtitle}}</p>
|
||
|
||
<form id="crawl-form" onsubmit="startCrawl(event)">
|
||
<div class="form-group">
|
||
<label>{{url}}</label>
|
||
<textarea id="target" placeholder="https://example.com https://another.com" rows="2" required style="width:100%;padding:10px;border:1px solid #333;border-radius:4px;background:#1a1a1a;color:#e0e0e0;font-size:14px;resize:vertical;"></textarea>
|
||
</div>
|
||
|
||
<div class="form-group">
|
||
<label>{{keywords}}</label>
|
||
<input type="text" id="keywords" placeholder="e.g. rick and morty, adult swim">
|
||
</div>
|
||
|
||
<div class="row">
|
||
<div class="form-group">
|
||
<label>{{mode}}</label>
|
||
<select id="mode">
|
||
<option value="all">{{everything}}</option>
|
||
<option value="media" selected>{{all_media}}</option>
|
||
<option value="images">{{images_only}}</option>
|
||
<option value="videos">{{videos_only}}</option>
|
||
<option value="text">{{text_only}}</option>
|
||
</select>
|
||
</div>
|
||
<div class="form-group">
|
||
<label>{{depth}}</label>
|
||
<input type="number" id="depth" value="9" min="-1" max="15">
|
||
</div>
|
||
<div class="form-group">
|
||
<label>{{max_pages}}</label>
|
||
<input type="number" id="max_pages" value="-1" min="-1">
|
||
</div>
|
||
</div>
|
||
|
||
<div class="form-group checkbox-group" style="display:flex; flex-wrap:wrap; gap:20px;">
|
||
<div>
|
||
<input type="checkbox" id="fresh" checked>
|
||
<label for="fresh" style="display:inline; margin:0;">{{fresh_start}}</label>
|
||
</div>
|
||
<div>
|
||
<input type="checkbox" id="fast">
|
||
<label for="fast" style="display:inline; margin:0;">{{fast_mode}}</label>
|
||
</div>
|
||
<div>
|
||
<input type="checkbox" id="screenshots" checked>
|
||
<label for="screenshots" style="display:inline; margin:0;">{{screenshots}}</label>
|
||
</div>
|
||
<div>
|
||
<input type="checkbox" id="hydra">
|
||
<label for="hydra" style="display:inline; margin:0;" title="Parse RSS/Atom/Sitemap for fast discovery">Hydra</label>
|
||
</div>
|
||
</div>
|
||
|
||
<button type="submit" id="start-btn">{{start_crawl}}</button>
|
||
</form>
|
||
|
||
<div class="jobs-section">
|
||
<h2>{{recent_jobs}}</h2>
|
||
<div id="jobs">{{loading}}</div>
|
||
</div>
|
||
|
||
<script>
|
||
const CRAWL_DISABLED = false;
|
||
|
||
// Disable form when crawling is disabled
|
||
if (CRAWL_DISABLED) {
|
||
document.addEventListener('DOMContentLoaded', () => {
|
||
const form = document.getElementById('crawl-form');
|
||
const inputs = form.querySelectorAll('input, select, textarea, button');
|
||
inputs.forEach(el => {
|
||
el.disabled = true;
|
||
el.style.opacity = '0.5';
|
||
el.style.cursor = 'not-allowed';
|
||
});
|
||
// Add banner
|
||
const banner = document.createElement('div');
|
||
banner.style.cssText = 'background:#442;border:1px solid #664;padding:12px 20px;border-radius:8px;margin-bottom:20px;color:#ffa;';
|
||
banner.innerHTML = '⚠️ Crawling is disabled on this server (read-only archive mode)';
|
||
form.parentNode.insertBefore(banner, form);
|
||
});
|
||
}
|
||
|
||
async function startCrawl(e) {
|
||
e.preventDefault();
|
||
if (CRAWL_DISABLED) return;
|
||
|
||
const btn = document.getElementById('start-btn');
|
||
btn.disabled = true;
|
||
btn.textContent = T.starting;
|
||
|
||
const keywordsRaw = document.getElementById('keywords').value;
|
||
const keywords = keywordsRaw
|
||
.split(/[,\\s]+/)
|
||
.map(k => k.trim())
|
||
.filter(k => k.length > 0);
|
||
|
||
// Parse multiple target URIs (space, comma, or newline separated)
|
||
const targetsRaw = document.getElementById('target').value;
|
||
const targets = targetsRaw
|
||
.split(/[,\\s\\n]+/)
|
||
.map(t => t.trim())
|
||
.filter(t => t.length > 0 && t.startsWith('http'));
|
||
|
||
if (targets.length === 0) {
|
||
alert(T.valid_uri);
|
||
btn.disabled = false;
|
||
btn.textContent = T.start_crawl;
|
||
return;
|
||
}
|
||
|
||
const payload = {
|
||
targets: targets,
|
||
keywords: keywords,
|
||
mode: document.getElementById('mode').value,
|
||
depth: parseInt(document.getElementById('depth').value),
|
||
max_pages: parseInt(document.getElementById('max_pages').value),
|
||
fresh: document.getElementById('fresh').checked,
|
||
fast: document.getElementById('fast').checked,
|
||
screenshots: document.getElementById('screenshots').checked,
|
||
hydra: document.getElementById('hydra').checked
|
||
};
|
||
|
||
try {
|
||
const res = await fetch('/api/crawl', {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify(payload)
|
||
});
|
||
|
||
if (!res.ok) {
|
||
const err = await res.json();
|
||
btn.textContent = T.error + ': ' + (err.detail || T.load_error);
|
||
btn.style.background = '#c0392b';
|
||
setTimeout(() => {
|
||
btn.textContent = T.start_crawl;
|
||
btn.style.background = '';
|
||
}, 3000);
|
||
} else {
|
||
// Clear form and refresh jobs list
|
||
document.getElementById('target').value = '';
|
||
document.getElementById('keywords').value = '';
|
||
btn.textContent = T.started_ok;
|
||
btn.style.background = '#27ae60';
|
||
loadJobs();
|
||
setTimeout(() => {
|
||
btn.textContent = T.start_crawl;
|
||
btn.style.background = '';
|
||
}, 2000);
|
||
}
|
||
} catch (err) {
|
||
btn.textContent = T.error;
|
||
btn.style.background = '#c0392b';
|
||
setTimeout(() => {
|
||
btn.textContent = T.start_crawl;
|
||
btn.style.background = '';
|
||
}, 3000);
|
||
}
|
||
|
||
btn.disabled = false;
|
||
}
|
||
|
||
async function pauseJob(jobId) {
|
||
try {
|
||
const res = await fetch('/api/crawl/jobs/' + jobId + '/pause', { method: 'POST' });
|
||
if (res.ok) {
|
||
loadJobs();
|
||
} else {
|
||
const err = await res.json();
|
||
alert(T.error + ': ' + (err.detail || T.load_error));
|
||
}
|
||
} catch (err) {
|
||
alert(T.error + ': ' + err.message);
|
||
}
|
||
}
|
||
|
||
async function resumeJob(jobId) {
|
||
try {
|
||
const res = await fetch('/api/crawl/jobs/' + jobId + '/resume', { method: 'POST' });
|
||
if (res.ok) {
|
||
loadJobs();
|
||
} else {
|
||
const err = await res.json();
|
||
alert(T.error + ': ' + (err.detail || T.load_error));
|
||
}
|
||
} catch (err) {
|
||
alert(T.error + ': ' + err.message);
|
||
}
|
||
}
|
||
|
||
// Load job settings into form for recrawling (fresh always unchecked)
|
||
function loadJobSettings(job) {
|
||
document.getElementById('target').value = job.target_uri || '';
|
||
document.getElementById('keywords').value = (JSON.parse(job.keywords || '[]')).join(', ');
|
||
document.getElementById('mode').value = job.mode || 'all';
|
||
document.getElementById('depth').value = job.depth || 15;
|
||
document.getElementById('max_pages').value = job.max_pages || -1;
|
||
document.getElementById('fast').checked = !!job.fast;
|
||
document.getElementById('screenshots').checked = job.screenshots !== 0;
|
||
// NEVER load fresh as true when recrawling - we want to resume, not start over
|
||
document.getElementById('fresh').checked = false;
|
||
// Scroll to form
|
||
document.getElementById('target').scrollIntoView({ behavior: 'smooth' });
|
||
document.getElementById('target').focus();
|
||
}
|
||
|
||
async function deleteJob(jobId) {
|
||
if (!confirm(T.delete + '?')) return;
|
||
try {
|
||
const res = await fetch('/api/crawl/jobs/' + jobId, { method: 'DELETE' });
|
||
if (res.ok) {
|
||
loadJobs();
|
||
} else {
|
||
const err = await res.json();
|
||
alert(T.error + ': ' + (err.detail || T.load_error));
|
||
}
|
||
} catch (err) {
|
||
alert(T.error + ': ' + err.message);
|
||
}
|
||
}
|
||
|
||
function formatBytes(bytes) {
|
||
if (!bytes) return '';
|
||
if (bytes < 1024) return bytes + ' B';
|
||
if (bytes < 1024*1024) return (bytes/1024).toFixed(1) + ' KB';
|
||
if (bytes < 1024*1024*1024) return (bytes/1024/1024).toFixed(1) + ' MB';
|
||
return (bytes/1024/1024/1024).toFixed(1) + ' GB';
|
||
}
|
||
|
||
function formatDuration(startedAt, completedAt) {
|
||
if (!startedAt || !completedAt) return '';
|
||
const start = new Date(startedAt);
|
||
const end = new Date(completedAt);
|
||
const sec = Math.round((end - start) / 1000);
|
||
if (sec < 60) return sec + 's';
|
||
if (sec < 3600) return Math.floor(sec / 60) + 'm ' + (sec % 60) + 's';
|
||
const hr = Math.floor(sec / 3600);
|
||
const min = Math.floor((sec % 3600) / 60);
|
||
return hr + 'h ' + min + 'm';
|
||
}
|
||
|
||
async function loadJobs() {
|
||
try {
|
||
const res = await fetch('/api/crawl/jobs');
|
||
const jobs = await res.json();
|
||
|
||
const container = document.getElementById('jobs');
|
||
|
||
if (jobs.length === 0) {
|
||
container.innerHTML = '<div class="no-jobs">' + T.no_jobs + '</div>';
|
||
return;
|
||
}
|
||
|
||
const hasRunning = jobs.some(j => j.status === 'running');
|
||
|
||
container.innerHTML = jobs.map(job => {
|
||
let stats = {};
|
||
let keywords = [];
|
||
try {
|
||
stats = job.stats ? JSON.parse(job.stats) : {};
|
||
} catch (e) {
|
||
console.warn('Failed to parse job stats:', job.id, e);
|
||
}
|
||
try {
|
||
keywords = job.keywords ? JSON.parse(job.keywords) : [];
|
||
} catch (e) {
|
||
console.warn('Failed to parse job keywords:', job.id, e);
|
||
}
|
||
const isBackfill = job.job_kind === 'backfill';
|
||
const kindLabel = isBackfill ? `🔄 ${job.job_type}` : `🕷️ ${job.mode || 'all'}`;
|
||
|
||
// Backfill job progress
|
||
if (isBackfill) {
|
||
const progress = job.total_records > 0
|
||
? Math.round((job.processed_records / job.total_records) * 100)
|
||
: 0;
|
||
return `
|
||
<div class="job ${job.status}">
|
||
<div class="job-header">
|
||
<span class="job-id">${kindLabel} #${job.id}</span>
|
||
<span class="job-status ${job.status}">${T[job.status] || job.status}</span>
|
||
</div>
|
||
<div class="job-target">${job.domain_filter || T.all_domains}</div>
|
||
<div class="job-meta">
|
||
${job.processed_records}/${job.total_records} |
|
||
${job.error_count} ${T.error} |
|
||
${T.started}: ${new Date(job.started_at).toLocaleString()}
|
||
</div>
|
||
${job.status === 'running' ? `
|
||
<div class="progress-bar">
|
||
<div class="progress-bar-fill" style="width: ${progress}%"></div>
|
||
</div>
|
||
` : ''}
|
||
</div>
|
||
`;
|
||
}
|
||
|
||
// Crawl job display with ETA
|
||
const calcEta = () => {
|
||
const crawled = stats.pages_crawled || 0;
|
||
const pending = stats.pages_pending || 0;
|
||
const started = stats.crawl_started || 0;
|
||
if (crawled === 0 || started === 0 || pending === 0) return '';
|
||
const elapsed = Date.now() / 1000 - started;
|
||
if (elapsed < 5) return ''; // Wait for meaningful data
|
||
const rate = crawled / elapsed;
|
||
const etaSec = pending / rate;
|
||
if (etaSec < 60) return `~${Math.round(etaSec)}s`;
|
||
if (etaSec < 3600) return `~${Math.round(etaSec / 60)}m`;
|
||
return `~${(etaSec / 3600).toFixed(1)}h`;
|
||
};
|
||
const eta = calcEta();
|
||
const crawlProgress = (() => {
|
||
const crawled = stats.pages_crawled || 0;
|
||
const pending = stats.pages_pending || 0;
|
||
const total = crawled + pending;
|
||
return total > 0 ? Math.round((crawled / total) * 100) : 0;
|
||
})();
|
||
const liveStats = job.status === 'running' ? `
|
||
<div class="job-stats live">
|
||
<span>📄 ${stats.pages_crawled || 0}${stats.pages_pending ? '/' + ((stats.pages_crawled || 0) + stats.pages_pending) : ''} pages${eta ? ' ⏱️' + eta : ''}</span>
|
||
<span>🖼️ ${stats.media_found || 0} found</span>
|
||
<span>💾 ${stats.media_downloaded || 0} saved</span>
|
||
<span>📸 ${stats.screenshots_taken || 0} screenshots</span>
|
||
<span>♻️ ${(stats.duplicates_skipped || 0) + (stats.content_exists || 0)} dupes</span>
|
||
${stats.bytes_stored ? `<span>📦 ${formatBytes(stats.bytes_stored)}</span>` : ''}
|
||
</div>
|
||
` : '';
|
||
|
||
return `
|
||
<div class="job ${job.status}">
|
||
<div class="job-header">
|
||
<span class="job-id">${kindLabel} #${job.id}</span>
|
||
<span class="job-status ${job.status}">${T[job.status] || job.status}</span>
|
||
</div>
|
||
<div class="job-target">
|
||
<a href="${job.target_uri}" target="_blank">${job.target_uri}</a>
|
||
${(() => { try { const h = new URL(job.target_uri).hostname; return `<span class="job-links"><a href="/?q=${encodeURIComponent(h)}">${T.search}</a> <a href="/live?domain=${encodeURIComponent(h)}">${T.live}</a></span>`; } catch(e) { return ''; } })()}
|
||
</div>
|
||
<div class="job-meta">
|
||
${keywords.length ? `${T.filter}: ${keywords.join(', ')} | ` : ''}${T.started}: ${new Date(job.started_at).toLocaleString()}
|
||
</div>
|
||
${job.status === 'completed' ? `
|
||
<div class="job-stats">
|
||
<span>⏱️ ${formatDuration(job.started_at, job.completed_at)}</span>
|
||
<span>📄 ${stats.pages_crawled || 0} pages</span>
|
||
<span>🖼️ ${stats.media_found || 0} found</span>
|
||
<span>💾 ${stats.media_downloaded || 0} saved</span>
|
||
<span>📸 ${stats.screenshots_taken || 0} screenshots</span>
|
||
<span>♻️ ${(stats.duplicates_skipped || 0) + (stats.content_exists || 0)} dupes</span>
|
||
${stats.bytes_stored ? `<span>📦 ${formatBytes(stats.bytes_stored)}</span>` : ''}
|
||
</div>
|
||
` : ''}
|
||
${liveStats}
|
||
${job.status === 'running' && job.job_kind === 'crawl' ? `
|
||
<div class="progress-bar">
|
||
<div class="progress-bar-fill" style="width: ${crawlProgress}%"></div>
|
||
</div>
|
||
` : ''}
|
||
<div class="job-actions">
|
||
${job.status === 'running' && job.job_kind === 'crawl' ? `<button class="pause-btn" onclick="pauseJob(${job.id})">${T.pause}</button>` : ''}
|
||
${(job.status === 'paused' || job.status === 'cancelled') && job.job_kind === 'crawl' ? `<button class="start-btn" onclick="resumeJob(${job.id})">${T.resume}</button>` : ''}
|
||
${(job.status === 'completed' || job.status === 'failed' || job.status === 'cancelled') && job.job_kind === 'crawl' ? `<button class="recrawl-btn" onclick='loadJobSettings(${JSON.stringify(job)})'>↻</button>` : ''}
|
||
<button class="console-btn" onclick="openConsole(${job.id})">{{console}}</button>
|
||
${job.status !== 'running' ? `<button class="delete-btn" onclick="deleteJob(${job.id})">${T.delete}</button>` : ''}
|
||
</div>
|
||
</div>
|
||
`;
|
||
}).join('');
|
||
|
||
// Poll: 3s when running, 10s when idle
|
||
const delay = hasRunning ? 3000 : 10000;
|
||
setTimeout(loadJobs, delay);
|
||
} catch (err) {
|
||
console.error('loadJobs error:', err);
|
||
document.getElementById('jobs').innerHTML = `<div class="no-jobs">${T.load_jobs_error}: ` + err.message + '</div>';
|
||
setTimeout(loadJobs, 10000);
|
||
}
|
||
}
|
||
|
||
// Load jobs on page load
|
||
loadJobs();
|
||
|
||
// Console modal
|
||
let consoleJobId = null;
|
||
let consoleInterval = null;
|
||
|
||
function openConsole(jobId) {
|
||
consoleJobId = jobId;
|
||
document.getElementById('console-modal').classList.add('active');
|
||
document.getElementById('console-title').textContent = T.console + ' #' + jobId;
|
||
document.getElementById('console-logs').textContent = T.loading;
|
||
fetchLogs();
|
||
consoleInterval = setInterval(fetchLogs, 2000);
|
||
}
|
||
|
||
function closeConsole() {
|
||
document.getElementById('console-modal').classList.remove('active');
|
||
if (consoleInterval) {
|
||
clearInterval(consoleInterval);
|
||
consoleInterval = null;
|
||
}
|
||
consoleJobId = null;
|
||
}
|
||
|
||
async function fetchLogs() {
|
||
if (!consoleJobId) return;
|
||
try {
|
||
const res = await fetch('/api/crawl/jobs/' + consoleJobId + '/logs');
|
||
const logs = await res.text();
|
||
const el = document.getElementById('console-logs');
|
||
const wasAtBottom = el.scrollHeight - el.clientHeight <= el.scrollTop + 50;
|
||
el.innerHTML = logs ? logs.split('\\n').map(line => {
|
||
let cls = 'log-line';
|
||
if (line.includes(' INFO:')) cls += ' log-INFO';
|
||
else if (line.includes(' WARNING:')) cls += ' log-WARNING';
|
||
else if (line.includes(' ERROR:')) cls += ' log-ERROR';
|
||
return '<div class="' + cls + '">' + line.replace(/</g, '<') + '</div>';
|
||
}).join('') : T.no_logs;
|
||
if (wasAtBottom) el.scrollTop = el.scrollHeight;
|
||
} catch (err) {
|
||
document.getElementById('console-logs').textContent = T.error + ': ' + err.message;
|
||
}
|
||
}
|
||
|
||
// Close on escape
|
||
document.addEventListener('keydown', e => {
|
||
if (e.key === 'Escape') closeConsole();
|
||
});
|
||
</script>
|
||
|
||
<div id="console-modal" class="console-modal" onclick="if(event.target===this)closeConsole()">
|
||
<div class="console-header">
|
||
<h3 id="console-title">{{console}}</h3>
|
||
<button class="console-close" onclick="closeConsole()">✕ {{close}}</button>
|
||
</div>
|
||
<div id="console-logs" class="console-body"></div>
|
||
</div>
|
||
</div>
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
|
||
LIVE_HTML = """
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>neopig {{live}} - {{live_subtitle}}</title>
|
||
<style>
|
||
* { box-sizing: border-box; }
|
||
body {
|
||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||
margin: 0;
|
||
padding: 0;
|
||
background: #0a0a0a;
|
||
color: #e0e0e0;
|
||
}
|
||
.nav {
|
||
background: #1a1a1a;
|
||
padding: 10px 20px;
|
||
display: flex;
|
||
gap: 20px;
|
||
align-items: center;
|
||
border-bottom: 1px solid #333;
|
||
}
|
||
.nav a { color: #ff6b6b; text-decoration: none; }
|
||
.nav a:hover { text-decoration: underline; }
|
||
.nav .brand { font-weight: bold; font-size: 18px; }
|
||
.container { padding: 20px; }
|
||
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
||
.subtitle { color: #666; margin-bottom: 10px; }
|
||
.stats {
|
||
background: #1a1a1a;
|
||
padding: 15px;
|
||
border-radius: 8px;
|
||
margin-bottom: 20px;
|
||
display: flex;
|
||
gap: 30px;
|
||
flex-wrap: wrap;
|
||
}
|
||
.stat { display: flex; flex-direction: column; }
|
||
.stat-value { font-size: 24px; font-weight: bold; color: #ff6b6b; }
|
||
.stat-label { font-size: 12px; color: #888; }
|
||
.controls {
|
||
margin-bottom: 20px;
|
||
display: flex;
|
||
gap: 10px;
|
||
align-items: center;
|
||
}
|
||
.controls button {
|
||
padding: 8px 16px;
|
||
border: none;
|
||
border-radius: 4px;
|
||
cursor: pointer;
|
||
font-size: 14px;
|
||
}
|
||
.controls button.primary { background: #ff6b6b; color: white; }
|
||
.controls button.secondary { background: #333; color: #ddd; }
|
||
.controls button:hover { opacity: 0.8; }
|
||
.status { color: #4ade80; font-size: 14px; }
|
||
.status.paused { color: #fbbf24; }
|
||
.live-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
|
||
gap: 15px;
|
||
}
|
||
.live-card {
|
||
background: #1a1a1a;
|
||
border-radius: 8px;
|
||
overflow: hidden;
|
||
animation: fadeIn 0.5s ease-out;
|
||
}
|
||
@keyframes fadeIn {
|
||
from { opacity: 0; transform: translateY(20px); }
|
||
to { opacity: 1; transform: translateY(0); }
|
||
}
|
||
.live-card.new {
|
||
box-shadow: 0 0 20px rgba(255, 107, 107, 0.5);
|
||
}
|
||
.live-card img, .live-card video {
|
||
width: 100%;
|
||
height: 180px;
|
||
object-fit: contain;
|
||
background: #222;
|
||
}
|
||
.live-card-info {
|
||
padding: 10px;
|
||
}
|
||
.live-card-title {
|
||
font-size: 12px;
|
||
color: #888;
|
||
overflow: hidden;
|
||
text-overflow: ellipsis;
|
||
white-space: nowrap;
|
||
}
|
||
.live-card-source {
|
||
font-size: 10px;
|
||
color: #555;
|
||
margin-top: 4px;
|
||
overflow: hidden;
|
||
text-overflow: ellipsis;
|
||
white-space: nowrap;
|
||
}
|
||
.score-badge {
|
||
display: inline-block;
|
||
padding: 2px 6px;
|
||
border-radius: 3px;
|
||
font-size: 9px;
|
||
font-weight: bold;
|
||
text-transform: uppercase;
|
||
margin-left: 6px;
|
||
vertical-align: middle;
|
||
}
|
||
.score-screenshot { background: #666; color: #ccc; }
|
||
.score-og { background: #8b5cf6; color: white; }
|
||
.score-thumb { background: #3b82f6; color: white; }
|
||
.score-hd { background: #10b981; color: white; }
|
||
.live-card.upgraded {
|
||
box-shadow: 0 0 20px rgba(16, 185, 129, 0.7);
|
||
animation: upgradeGlow 1s ease-out;
|
||
}
|
||
@keyframes upgradeGlow {
|
||
0% { box-shadow: 0 0 30px rgba(16, 185, 129, 1); }
|
||
100% { box-shadow: 0 0 20px rgba(16, 185, 129, 0.5); }
|
||
}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<!-- NAV -->
|
||
<div class="container">
|
||
|
||
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
||
<input type="text" name="q" placeholder="{{search_placeholder}}" style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||
<option value="">{{all_types}}</option>
|
||
<option value="image">{{images}}</option>
|
||
<option value="video">{{videos}}</option>
|
||
<option value="audio">{{audio}}</option>
|
||
</select>
|
||
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">{{search}}</button>
|
||
</form>
|
||
|
||
<h1>{{live}}</h1>
|
||
<p class="subtitle">{{live_subtitle}}</p>
|
||
|
||
<div class="stats">
|
||
<div class="stat">
|
||
<span class="stat-value" id="total-count">0</span>
|
||
<span class="stat-label">{{total_images}}</span>
|
||
</div>
|
||
<div class="stat">
|
||
<span class="stat-value" id="new-count">0</span>
|
||
<span class="stat-label">{{new_session}}</span>
|
||
</div>
|
||
<div class="stat">
|
||
<span class="stat-value" id="rate">0</span>
|
||
<span class="stat-label">{{per_minute}}</span>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="controls">
|
||
<button class="primary" id="toggle-btn" onclick="toggleFeed()">{{pause}}</button>
|
||
<button class="secondary" onclick="clearFeed()">{{delete}}</button>
|
||
<span class="status" id="status">{{watching}}</span>
|
||
</div>
|
||
|
||
<div class="live-grid" id="grid"></div>
|
||
|
||
<script>
|
||
let running = true;
|
||
let lastCheck = new Date().toISOString();
|
||
let seenHashes = new Map(); // hash -> {upgraded_at, score}
|
||
let newCount = 0;
|
||
let startTime = Date.now();
|
||
let eventSource = null;
|
||
|
||
// Connect to SSE stream for real-time updates
|
||
function connectSSE() {
|
||
if (eventSource) eventSource.close();
|
||
eventSource = new EventSource('/api/live/stream');
|
||
|
||
eventSource.onmessage = function(event) {
|
||
if (!running) return;
|
||
try {
|
||
const item = JSON.parse(event.data);
|
||
addMediaCard(item, true);
|
||
} catch (e) {
|
||
console.error('SSE parse error:', e);
|
||
}
|
||
};
|
||
|
||
eventSource.onerror = function(e) {
|
||
console.log('SSE reconnecting...');
|
||
setTimeout(connectSSE, 3000);
|
||
};
|
||
}
|
||
|
||
function addMediaCard(item, isNew) {
|
||
if (seenHashes.has(item.md5_hash)) return; // Already shown
|
||
|
||
seenHashes.set(item.md5_hash, {upgraded_at: item.upgraded_at, score: item.score || 5});
|
||
newCount++;
|
||
|
||
const grid = document.getElementById('grid');
|
||
const card = document.createElement('div');
|
||
card.className = 'live-card' + (isNew ? ' new' : '');
|
||
|
||
const isVideo = item.media_type === 'video';
|
||
const isCode = item.media_type === 'code';
|
||
const isStyle = item.media_type === 'style';
|
||
const isFont = item.media_type === 'font';
|
||
|
||
const mimeToExt = {
|
||
'text/javascript': '.js', 'application/javascript': '.js',
|
||
'text/css': '.css', 'text/xml': '.xml', 'application/json': '.json',
|
||
'application/rss+xml': '.rss', 'application/atom+xml': '.atom',
|
||
'font/woff': '.woff', 'font/woff2': '.woff2', 'font/ttf': '.ttf',
|
||
};
|
||
const getExt = (mime, alt) => mimeToExt[mime] || (alt ? '.' + alt : '');
|
||
const placeholder = (ext, icon, color) => `<svg viewBox="0 0 100 100" style="width:100%;height:100%;background:#1a1a1a;border-radius:4px;"><text x="50" y="40" text-anchor="middle" fill="${color}" font-size="24">${icon}</text><text x="50" y="65" text-anchor="middle" fill="#888" font-family="monospace" font-size="14" font-weight="bold">${ext}</text></svg>`;
|
||
|
||
let mediaEl;
|
||
if (isVideo) {
|
||
mediaEl = `<video src="/media/${item.md5_hash}" muted loop onmouseenter="this.play()" onmouseleave="this.pause()"></video>`;
|
||
} else if (isCode) {
|
||
mediaEl = placeholder(getExt(item.mime_type, item.alt_text), '{ }', '#6af');
|
||
} else if (isStyle) {
|
||
mediaEl = placeholder('.css', '#', '#f6a');
|
||
} else if (isFont) {
|
||
mediaEl = placeholder(getExt(item.mime_type, ''), 'Aa', '#af6');
|
||
} else {
|
||
mediaEl = `<img src="/media/${item.md5_hash}" loading="lazy" onerror="this.src='data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 width=%22200%22 height=%22180%22><rect fill=%22%23333%22 width=%22200%22 height=%22180%22/><text x=%2250%%22 y=%2250%%22 fill=%22%23666%22 text-anchor=%22middle%22>Error</text></svg>'">`;
|
||
}
|
||
|
||
const scoreBadge = getScoreBadge(item.score, item.media_type);
|
||
|
||
// Extract filename from media_uri
|
||
const getFilename = (uri) => {
|
||
if (!uri) return '';
|
||
try {
|
||
const path = new URL(uri).pathname;
|
||
const name = decodeURIComponent(path.split('/').pop() || '');
|
||
return name.length > 35 ? name.slice(0, 32) + '...' : name;
|
||
} catch { return ''; }
|
||
};
|
||
const filename = getFilename(item.media_uri);
|
||
|
||
card.innerHTML = `
|
||
<a href="/view/${item.md5_hash}">
|
||
${mediaEl}
|
||
</a>
|
||
<div class="live-card-info">
|
||
<div class="live-card-title">${filename || item.alt_text || item.title || item.md5_hash.slice(0,12)}</div>
|
||
<div class="live-card-source">${item.media_type} - ${formatSize(item.file_size)}${scoreBadge}</div>
|
||
</div>
|
||
`;
|
||
|
||
grid.insertBefore(card, grid.firstChild);
|
||
updateStats(seenHashes.size);
|
||
|
||
// Remove 'new' highlight after animation
|
||
if (isNew) setTimeout(() => card.classList.remove('new'), 2000);
|
||
}
|
||
|
||
function toggleFeed() {
|
||
running = !running;
|
||
const btn = document.getElementById('toggle-btn');
|
||
const status = document.getElementById('status');
|
||
if (running) {
|
||
btn.textContent = T.pause;
|
||
status.textContent = T.watching;
|
||
status.className = 'status';
|
||
connectSSE();
|
||
poll();
|
||
} else {
|
||
btn.textContent = T.resume;
|
||
status.textContent = T.paused;
|
||
status.className = 'status paused';
|
||
if (eventSource) eventSource.close();
|
||
}
|
||
}
|
||
|
||
function clearFeed() {
|
||
document.getElementById('grid').innerHTML = '';
|
||
seenHashes = new Map();
|
||
newCount = 0;
|
||
startTime = Date.now();
|
||
updateStats(0);
|
||
}
|
||
|
||
function updateStats(total) {
|
||
document.getElementById('total-count').textContent = total;
|
||
document.getElementById('new-count').textContent = newCount;
|
||
|
||
const minutes = (Date.now() - startTime) / 60000;
|
||
const rate = minutes > 0 ? Math.round(newCount / minutes) : 0;
|
||
document.getElementById('rate').textContent = rate;
|
||
}
|
||
|
||
async function poll() {
|
||
if (!running) return;
|
||
|
||
try {
|
||
// Get recent media sorted by first_seen_at descending (initial load + catch-up)
|
||
const res = await fetch('/api/search?q=&limit=50');
|
||
const media = await res.json();
|
||
|
||
// Get stats
|
||
const statsRes = await fetch('/api/stats');
|
||
const stats = await statsRes.json();
|
||
|
||
// Find new items not yet shown via SSE
|
||
const isFirstPoll = seenHashes.size === 0;
|
||
const itemsToShow = media.filter(m => !seenHashes.has(m.md5_hash));
|
||
|
||
// Add items (not as "new" on first poll to avoid flash)
|
||
itemsToShow.reverse().forEach(item => {
|
||
addMediaCard(item, !isFirstPoll);
|
||
});
|
||
|
||
updateStats(stats.total_media || 0);
|
||
|
||
} catch (err) {
|
||
console.error('Poll error:', err);
|
||
}
|
||
|
||
// Poll less frequently since SSE handles real-time
|
||
setTimeout(poll, 10000);
|
||
}
|
||
|
||
function formatSize(bytes) {
|
||
if (!bytes) return '?';
|
||
if (bytes < 1024) return bytes + ' B';
|
||
if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB';
|
||
return (bytes / (1024 * 1024)).toFixed(1) + ' MB';
|
||
}
|
||
|
||
function getScoreBadge(score, mediaType) {
|
||
// Score constants: screenshot=1, og=3, thumb=5, hd=10
|
||
// Also check media_type for legacy records without scores
|
||
let label, cssClass;
|
||
if (mediaType === 'screenshot' || score <= 1) {
|
||
label = '📸'; cssClass = 'score-screenshot';
|
||
} else if (score <= 3) {
|
||
label = 'OG'; cssClass = 'score-og';
|
||
} else if (!score || score <= 5) {
|
||
label = 'THUMB'; cssClass = 'score-thumb';
|
||
} else {
|
||
label = 'HD'; cssClass = 'score-hd';
|
||
}
|
||
return `<span class="score-badge ${cssClass}">${label}</span>`;
|
||
}
|
||
|
||
// Start polling and SSE connection
|
||
poll();
|
||
connectSSE();
|
||
</script>
|
||
</div>
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
|
||
@app.get("/live", response_class=HTMLResponse)
|
||
async def live_page(domain: str = Query(None), lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Live feed page - watch images appear as they're crawled."""
|
||
language = get_lang(lang, accept_language)
|
||
# Inject domain filter into HTML
|
||
html = LIVE_HTML
|
||
if domain:
|
||
# Add domain to the poll URL
|
||
html = html.replace(
|
||
"/api/search?q=&limit=50",
|
||
f"/api/search?q={domain}&limit=50"
|
||
)
|
||
html = html.replace(
|
||
"<h1>{{live}}</h1>",
|
||
"<h1>{{live}}: " + domain + "</h1>"
|
||
)
|
||
return inject_i18n(html, language)
|
||
|
||
|
||
@app.get("/view/{md5_hash}", response_class=HTMLResponse)
|
||
async def view_media_page(md5_hash: str, noai: bool = Query(False), lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Detail view page for a single media item."""
|
||
language = get_lang(lang, accept_language)
|
||
import html as html_module
|
||
import re
|
||
from urllib.parse import quote, urljoin
|
||
|
||
async with db.session() as session:
|
||
result = await session.execute(
|
||
text("SELECT * FROM media WHERE md5_hash = :hash"),
|
||
{'hash': md5_hash}
|
||
)
|
||
media = result.fetchone()
|
||
if not media:
|
||
raise HTTPException(status_code=404, detail="Media not found")
|
||
|
||
result = await session.execute(
|
||
text("""SELECT media_uri, page_uri, page_title, page_content,
|
||
detail_page_uri, detail_title, detail_content, discovered_at
|
||
FROM media_sources WHERE md5_hash = :hash"""),
|
||
{'hash': md5_hash}
|
||
)
|
||
sources = result.fetchall()
|
||
|
||
media = dict(media._mapping)
|
||
sources = [dict(s._mapping) for s in sources]
|
||
keywords = json.loads(media.get('keywords') or '[]')
|
||
sorted_sources = sorted(sources, key=lambda s: len(s["page_uri"] or ""), reverse=True)
|
||
|
||
# Get page URI
|
||
page_uri = sorted_sources[0]["page_uri"] if sorted_sources else None
|
||
source_uri = page_uri or (sorted_sources[0]["media_uri"] if sorted_sources else "unknown")
|
||
source_domain = Uri(source_uri).hostname if source_uri else "unknown"
|
||
|
||
# Display title
|
||
display_title = media.get('alt_text') or media.get('title')
|
||
if not display_title and sources:
|
||
display_title = sources[0].get('page_title')
|
||
if not display_title:
|
||
display_title = f"Media {md5_hash[:12]}"
|
||
|
||
# Hero: media element
|
||
is_video = media['media_type'] == 'video'
|
||
is_audio = media['media_type'] == 'audio'
|
||
is_code = media['media_type'] == 'code'
|
||
is_style = media['media_type'] == 'style'
|
||
is_font = media['media_type'] == 'font'
|
||
is_text_file = is_code or is_style # text files shown with syntax highlighting
|
||
if is_video:
|
||
hero_html = f'<a href="/media/{md5_hash}" target="_blank"><video src="/media/{md5_hash}" controls muted loop style="max-width:100%;max-height:70vh;"></video></a>'
|
||
elif is_audio:
|
||
hero_html = f'<audio src="/media/{md5_hash}" controls></audio>'
|
||
elif is_text_file:
|
||
# Fetch code/style content and display with syntax highlighting
|
||
code_content = ""
|
||
lang_class = 'css' if is_style else (media.get('alt_text') or '')
|
||
try:
|
||
vault_path = VAULT_PATH / hash_to_path(md5_hash)
|
||
# Find file with this hash
|
||
subdir = vault_path.parent
|
||
for f in subdir.iterdir():
|
||
if f.stem == md5_hash:
|
||
code_content = f.read_text(errors='replace')[:100000] # Limit size
|
||
break
|
||
except Exception:
|
||
code_content = "(Unable to load file content)"
|
||
escaped_code = html_module.escape(code_content)
|
||
hero_html = f'<pre style="max-height:70vh;overflow:auto;background:#1e1e1e;padding:15px;border-radius:8px;text-align:left;white-space:pre-wrap;word-wrap:break-word;"><code class="language-{lang_class}">{escaped_code}</code></pre>'
|
||
elif is_font:
|
||
# Font preview with sample text
|
||
hero_html = f'''<div style="background:#1e1e1e;padding:30px;border-radius:8px;text-align:center;">
|
||
<style>@font-face {{ font-family: "preview-{md5_hash[:8]}"; src: url("/media/{md5_hash}"); }}</style>
|
||
<div style="font-family:'preview-{md5_hash[:8]}',sans-serif;font-size:48px;color:#fff;margin-bottom:20px;">Aa Bb Cc</div>
|
||
<div style="font-family:'preview-{md5_hash[:8]}',sans-serif;font-size:24px;color:#ccc;">The quick brown fox jumps over the lazy dog</div>
|
||
<div style="font-family:'preview-{md5_hash[:8]}',sans-serif;font-size:16px;color:#888;margin-top:15px;">0123456789 !@#$%^&*()</div>
|
||
</div>'''
|
||
else:
|
||
hero_html = f'<a href="/media/{md5_hash}" target="_blank"><img src="/media/{md5_hash}" alt="{html_module.escape(media.get("alt_text") or "")}" style="max-width:100%;max-height:70vh;"></a>'
|
||
|
||
# Metadata rows
|
||
import hashlib
|
||
media_uri = sorted_sources[0]["media_uri"] if sorted_sources else None
|
||
neopig_media_uri = f"/view/{md5_hash}"
|
||
page_uri_hash = hashlib.md5(page_uri.encode()).hexdigest() if page_uri else None
|
||
neopig_page_uri = f"/page/{page_uri_hash}" if page_uri_hash else None
|
||
keywords_html = ''.join([f'<span class="tag">{k}</span>' for k in keywords]) or '-'
|
||
meta_rows = [
|
||
("{{source_uri}}", f'<a href="{media_uri}" target="_blank" style="font-size:11px;">{media_uri}</a>' if media_uri else '-'),
|
||
("{{neopig_uri}}", f'<a href="{neopig_media_uri}" style="font-size:11px;">{neopig_media_uri}</a>'),
|
||
("{{source_page}}", f'<a href="{page_uri}" target="_blank" style="font-size:11px;">{page_uri}</a>' if page_uri else '-'),
|
||
("{{neopig_page}}", f'<a href="{neopig_page_uri}" style="font-size:11px;">{neopig_page_uri}</a>' if neopig_page_uri else '-'),
|
||
("MD5", f'<code style="font-size:11px;">{md5_hash}</code>'),
|
||
("{{type_label}}", media['media_type']),
|
||
("{{mime_label}}", media.get('mime_type') or 'unknown'),
|
||
("{{size_label}}", f"{media.get('file_size') or 0:,} bytes"),
|
||
("{{alt_label}}", media.get('alt_text') or '-'),
|
||
("{{keywords_label}}", keywords_html),
|
||
]
|
||
|
||
# Download button - get actual extension from vault file
|
||
name_source = media.get('alt_text') or media.get('title')
|
||
if not name_source and sources:
|
||
pt = sources[0].get('page_title', '')
|
||
media_idx = int(md5_hash[:4], 16)
|
||
name_source = f"{pt}-{media_idx}" if pt else f"media-{media_idx}"
|
||
download_name = slugify(name_source or f"media-{md5_hash[:8]}")
|
||
ext_map = {'image/jpeg': '.jpg', 'image/png': '.png', 'image/gif': '.gif', 'image/webp': '.webp',
|
||
'video/mp4': '.mp4', 'video/webm': '.webm', 'audio/mpeg': '.mp3', 'audio/wav': '.wav',
|
||
'text/javascript': '.js', 'text/python': '.py', 'text/css': '.css', 'text/html': '.html',
|
||
'text/markdown': '.md', 'text/rust': '.rs', 'text/go': '.go', 'text/c': '.c', 'text/cpp': '.cpp'}
|
||
ext = ext_map.get(media.get('mime_type', ''), '')
|
||
# If no extension from MIME, try to get from vault file
|
||
if not ext:
|
||
try:
|
||
subdir = VAULT_PATH / hash_to_path(md5_hash).parent
|
||
for f in subdir.iterdir():
|
||
if f.stem == md5_hash:
|
||
ext = f.suffix
|
||
break
|
||
except Exception:
|
||
pass
|
||
# Try original media_uri if still no extension
|
||
if not ext and media.get('media_uri'):
|
||
from pathlib import Path as P
|
||
orig_ext = P(media['media_uri'].split('?')[0]).suffix
|
||
if orig_ext and len(orig_ext) <= 5: # Reasonable extension length
|
||
ext = orig_ext
|
||
download_btn = f'<a href="/media/{md5_hash}?download=1" style="display:block;padding:12px 20px;background:#4a9eff;color:#fff;text-decoration:none;border-radius:6px;text-align:center;font-weight:500;margin-top:15px;">Download ({download_name}{ext})</a>'
|
||
|
||
# Get page media items (siblings)
|
||
media_items = await db.get_page_media(page_uri) if page_uri else []
|
||
# Exclude current item from gallery
|
||
media_items = [m for m in media_items if m.get("md5_hash") != md5_hash]
|
||
|
||
# Get rendered content
|
||
content_html = ""
|
||
if page_uri:
|
||
page_row = await db.get_page_by_uri(page_uri)
|
||
if page_row and page_row.get("markdown"):
|
||
try:
|
||
import markdown
|
||
md_converter = markdown.Markdown(extensions=['fenced_code', 'tables', 'nl2br'])
|
||
content_html = md_converter.convert(page_row["markdown"][:100000])
|
||
# Hydrate images and links
|
||
img_urls = re.findall(r'<img[^>]+src=["\']([^"\']+)["\']', content_html, re.I)
|
||
link_urls = re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', content_html, re.I)
|
||
all_urls = list(set(img_urls + link_urls))
|
||
if all_urls:
|
||
resolved = {u: u if u.startswith(('http://', 'https://', '//')) else urljoin(page_uri, u) for u in all_urls if not u.startswith('#')}
|
||
url_to_hash = await db.lookup_media_by_uris(list(set(resolved.values())))
|
||
exact_matches = set(url_to_hash.keys()) # These are reliable
|
||
|
||
# Fallback for imgur and other CDNs - lookup by filename/ID
|
||
# Only apply to URLs that look like media files (have media extension)
|
||
from async_web_fetcher import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS, AUDIO_EXTENSIONS
|
||
from pathlib import Path as P
|
||
media_exts = IMAGE_EXTENSIONS | VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
|
||
for orig, res in resolved.items():
|
||
if res not in url_to_hash:
|
||
url_path = res.split('?')[0]
|
||
ext = P(url_path).suffix.lower()
|
||
if ext in media_exts:
|
||
fname = P(url_path).stem
|
||
if fname and len(fname) >= 5:
|
||
result = await db.lookup_media_by_filename(fname)
|
||
if result:
|
||
url_to_hash[res] = result[1]
|
||
|
||
for orig, res in resolved.items():
|
||
if res in url_to_hash:
|
||
md5 = url_to_hash[res]
|
||
content_html = content_html.replace(f'src="{orig}"', f'src="/media/{md5}"')
|
||
content_html = content_html.replace(f"src='{orig}'", f'src="/media/{md5}"')
|
||
# Only rewrite hrefs for exact matches or URLs with media extensions
|
||
url_path = res.split('?')[0]
|
||
ext = P(url_path).suffix.lower()
|
||
if res in exact_matches or ext in media_exts:
|
||
content_html = content_html.replace(f'href="{orig}"', f'href="/view/{md5}"')
|
||
content_html = content_html.replace(f"href='{orig}'", f'href="/view/{md5}"')
|
||
except ImportError:
|
||
content_html = f"<pre>{html_module.escape(page_row['markdown'][:50000])}</pre>"
|
||
|
||
# Get screenshots (exclude current if it's a screenshot)
|
||
screenshot_hashes = await db.get_page_screenshots(page_uri) if page_uri else []
|
||
screenshot_hashes = [h for h in screenshot_hashes if h != md5_hash]
|
||
|
||
# Build "Used on X pages" section (reverse image search)
|
||
sources_html = ""
|
||
if sources and len(sources) > 0:
|
||
import hashlib as hl
|
||
unique_pages = {}
|
||
for s in sources:
|
||
pu = s.get('page_uri', '')
|
||
if pu and pu not in unique_pages:
|
||
unique_pages[pu] = s
|
||
if unique_pages:
|
||
rows_html = ""
|
||
for pu, s in unique_pages.items():
|
||
pt = html_module.escape(s.get('page_title', '') or pu[:60])
|
||
ph = hl.md5(pu.encode()).hexdigest()
|
||
rows_html += f'''<tr class="source-row">
|
||
<td><a href="/page/{ph}" title="{html_module.escape(pu)}">{pt}</a></td>
|
||
<td style="font-size:11px;color:#888;">{html_module.escape(s.get('discovered_at', '')[:10] if s.get('discovered_at') else '-')}</td>
|
||
</tr>'''
|
||
total_count = len(unique_pages)
|
||
count_display = f"{OVER_9000:,}" if total_count >= OVER_9000 else str(total_count)
|
||
sources_html = '''
|
||
<div class="sources-section">
|
||
<h3 onclick="toggleSources()">
|
||
<span id="sources-toggle">▶</span> {{used_on}} ''' + count_display + ''' {{pages}}
|
||
</h3>
|
||
<div id="sources-list" class="sources-list">
|
||
<div id="sources-pagination" style="margin-bottom:10px;display:flex;gap:10px;align-items:center;">
|
||
<button onclick="sourcesPage(-1)" id="sources-prev" style="padding:5px 12px;background:#333;border:none;color:#fff;border-radius:4px;cursor:pointer;">←</button>
|
||
<span id="sources-page-info" style="color:#888;font-size:13px;">1 / 1</span>
|
||
<button onclick="sourcesPage(1)" id="sources-next" style="padding:5px 12px;background:#333;border:none;color:#fff;border-radius:4px;cursor:pointer;">→</button>
|
||
</div>
|
||
<table>
|
||
<thead><tr><th>{{page}}</th><th>{{discovered}}</th></tr></thead>
|
||
<tbody id="sources-tbody">''' + rows_html + '''</tbody>
|
||
</table>
|
||
</div>
|
||
</div>
|
||
<script>
|
||
(function() {
|
||
const perPage = 9000;
|
||
const rows = document.querySelectorAll('#sources-tbody .source-row');
|
||
const total = rows.length;
|
||
const totalPages = Math.ceil(total / perPage);
|
||
let currentPage = 1;
|
||
function showPage(page) {
|
||
currentPage = Math.max(1, Math.min(page, totalPages));
|
||
const start = (currentPage - 1) * perPage;
|
||
const end = start + perPage;
|
||
rows.forEach((row, i) => { row.style.display = (i >= start && i < end) ? '' : 'none'; });
|
||
document.getElementById('sources-page-info').textContent = currentPage + ' / ' + totalPages;
|
||
document.getElementById('sources-prev').disabled = currentPage === 1;
|
||
document.getElementById('sources-next').disabled = currentPage === totalPages;
|
||
}
|
||
window.sourcesPage = function(delta) { showPage(currentPage + delta); };
|
||
showPage(1);
|
||
|
||
// Toggle sources with localStorage persistence
|
||
window.toggleSources = function() {
|
||
const list = document.getElementById('sources-list');
|
||
const toggle = document.getElementById('sources-toggle');
|
||
const expanded = !list.classList.contains('expanded');
|
||
list.classList.toggle('expanded', expanded);
|
||
toggle.textContent = expanded ? '▼' : '▶';
|
||
localStorage.setItem('neopig_sources_expanded', expanded);
|
||
};
|
||
|
||
// Restore from localStorage
|
||
if (localStorage.getItem('neopig_sources_expanded') === 'true') {
|
||
document.getElementById('sources-list').classList.add('expanded');
|
||
document.getElementById('sources-toggle').textContent = '▼';
|
||
}
|
||
})();
|
||
</script>'''
|
||
|
||
return render_detail_page(
|
||
title=display_title,
|
||
hero_html=hero_html,
|
||
meta_rows=meta_rows,
|
||
media_items=media_items,
|
||
content_html=content_html,
|
||
screenshot_hashes=screenshot_hashes,
|
||
source_domain=source_domain,
|
||
page_uri=page_uri or "",
|
||
download_btn_html=download_btn,
|
||
noai=noai,
|
||
lang=language,
|
||
sources_html=sources_html,
|
||
)
|
||
|
||
|
||
|
||
|
||
@app.get("/page/{uri_hash}", response_class=HTMLResponse)
|
||
async def view_page_by_hash(
|
||
uri_hash: str,
|
||
noai: bool = Query(False, description="Disable AI assistant"),
|
||
lang: str = Cookie(None),
|
||
accept_language: str = Header(None),
|
||
):
|
||
"""View an archived page by URI hash."""
|
||
page = await db.get_page_by_hash(uri_hash)
|
||
if not page:
|
||
raise HTTPException(status_code=404, detail="Page not found")
|
||
# Redirect to the URI-based view (reuses same logic)
|
||
return await view_page(uri=page['uri'], noai=noai, lang=lang, accept_language=accept_language)
|
||
|
||
|
||
@app.get("/page/view", response_class=HTMLResponse)
|
||
async def view_page(
|
||
uri: str = Query(..., description="Page URI to view"),
|
||
noai: bool = Query(False, description="Disable AI assistant"),
|
||
lang: str = Cookie(None),
|
||
accept_language: str = Header(None),
|
||
):
|
||
"""View an archived page with markdown and screenshot."""
|
||
language = get_lang(lang, accept_language)
|
||
import html as html_module
|
||
import re
|
||
from urllib.parse import urljoin, quote
|
||
|
||
source_domain = Uri(uri).hostname
|
||
|
||
page = await db.get_page_by_uri(uri)
|
||
if not page:
|
||
raise HTTPException(status_code=404, detail="Page not found")
|
||
|
||
page_title = page.get("title") or uri
|
||
|
||
# Render markdown
|
||
content_html = ""
|
||
if page.get("markdown"):
|
||
try:
|
||
import markdown
|
||
md_converter = markdown.Markdown(extensions=['fenced_code', 'tables', 'nl2br'])
|
||
content_html = md_converter.convert(page["markdown"][:100000])
|
||
|
||
# Hydrate images and links from vault
|
||
img_urls = re.findall(r'<img[^>]+src=["\']([^"\']+)["\']', content_html, re.I)
|
||
link_urls = re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', content_html, re.I)
|
||
all_urls = list(set(img_urls + link_urls))
|
||
if all_urls:
|
||
# Keep anchor-only links (#foo) as-is, resolve others
|
||
resolved = {}
|
||
for u in all_urls:
|
||
if u.startswith('#'):
|
||
continue # Skip anchor-only links, they stay internal
|
||
elif u.startswith(('http://', 'https://', '//')):
|
||
resolved[u] = u
|
||
else:
|
||
resolved[u] = urljoin(uri, u)
|
||
|
||
# Lookup media (images, etc) - track which came from exact match vs fallback
|
||
url_to_hash = await db.lookup_media_by_uris(list(set(resolved.values())))
|
||
exact_matches = set(url_to_hash.keys()) # These are reliable
|
||
|
||
# Fallback for imgur and other CDNs - lookup by filename/ID
|
||
# Only apply to URLs that look like media files (have media extension)
|
||
from async_web_fetcher import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS, AUDIO_EXTENSIONS
|
||
from pathlib import Path as P
|
||
media_exts = IMAGE_EXTENSIONS | VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
|
||
for murl in [u for u in resolved.values() if u not in url_to_hash]:
|
||
url_path = murl.split('?')[0]
|
||
ext = P(url_path).suffix.lower()
|
||
# Only do filename fallback for URLs with media extensions
|
||
if ext in media_exts:
|
||
fname = P(url_path).stem
|
||
if fname and len(fname) >= 5:
|
||
result = await db.lookup_media_by_filename(fname)
|
||
if result:
|
||
url_to_hash[murl] = result[1]
|
||
|
||
# Lookup pages for internal link rewriting
|
||
page_links = [res for res in resolved.values() if source_domain and source_domain in res]
|
||
uri_to_page_hash = await db.lookup_pages_by_uris(page_links) if page_links else {}
|
||
|
||
for orig, res in resolved.items():
|
||
if res in url_to_hash:
|
||
md5 = url_to_hash[res]
|
||
# Rewrite img src to serve media directly
|
||
content_html = content_html.replace(f'src="{orig}"', f'src="/media/{md5}"')
|
||
content_html = content_html.replace(f"src='{orig}'", f'src="/media/{md5}"')
|
||
# Only rewrite hrefs for exact matches or URLs with media extensions
|
||
# (avoid rewriting external links that happen to match by filename)
|
||
url_path = res.split('?')[0]
|
||
ext = P(url_path).suffix.lower()
|
||
if res in exact_matches or ext in media_exts:
|
||
content_html = content_html.replace(f'href="{orig}"', f'href="/view/{md5}"')
|
||
content_html = content_html.replace(f"href='{orig}'", f'href="/view/{md5}"')
|
||
elif res in uri_to_page_hash:
|
||
# Rewrite internal page links to archived versions
|
||
page_hash = uri_to_page_hash[res]
|
||
content_html = content_html.replace(f'href="{orig}"', f'href="/page/{page_hash}"')
|
||
content_html = content_html.replace(f"href='{orig}'", f'href="/page/{page_hash}"')
|
||
except ImportError:
|
||
content_html = f"<pre>{html_module.escape(page.get('markdown', '')[:50000])}</pre>"
|
||
elif page.get("content"):
|
||
escaped = html_module.escape(page["content"][:50000])
|
||
content_html = f"<pre style='white-space:pre-wrap;'>{escaped}</pre>"
|
||
|
||
# Get screenshots and media
|
||
screenshot_hashes = await db.get_page_screenshots(uri)
|
||
media_items = await db.get_page_media(uri)
|
||
|
||
# Metadata rows for page view
|
||
import hashlib
|
||
uri_hash = hashlib.md5(uri.encode()).hexdigest()
|
||
neopig_page_uri = f"/page/{uri_hash}"
|
||
keywords = json.loads(page.get('keywords') or '[]') if page.get('keywords') else []
|
||
keywords_html = ''.join([f'<span class="tag">{k}</span>' for k in keywords]) or '-'
|
||
meta_rows = [
|
||
("{{source_uri}}", f'<a href="{uri}" target="_blank" style="font-size:11px;">{uri}</a>'),
|
||
("{{neopig_uri}}", f'<a href="{neopig_page_uri}" style="font-size:11px;">{neopig_page_uri}</a>'),
|
||
("{{description_label}}", page.get('description') or '-'),
|
||
("{{keywords_label}}", keywords_html),
|
||
("{{media}}", f"{len(media_items)}"),
|
||
]
|
||
|
||
return render_detail_page(
|
||
title=page_title,
|
||
hero_html="", # No hero media for page view
|
||
meta_rows=meta_rows,
|
||
media_items=media_items,
|
||
content_html=content_html,
|
||
screenshot_hashes=screenshot_hashes,
|
||
source_domain=source_domain,
|
||
page_uri=uri,
|
||
noai=noai,
|
||
lang=language,
|
||
)
|
||
|
||
|
||
@app.get("/phantom/export")
|
||
async def phantom_export(domain: str = Query(None, description="Filter by domain")):
|
||
"""
|
||
Export a phantom HTML site - original HTML with media URLs rewritten to vault.
|
||
|
||
Creates a downloadable zip of the phantom site ready for static hosting.
|
||
"""
|
||
import io
|
||
import re
|
||
import zipfile
|
||
from urllib.parse import urljoin
|
||
from fastapi.responses import StreamingResponse
|
||
|
||
# Get all pages with raw_html
|
||
pages = await db.get_pages_by_domain(domain)
|
||
|
||
if not pages:
|
||
raise HTTPException(status_code=404, detail="No pages with raw HTML found")
|
||
|
||
# Get all media URL to hash mappings
|
||
url_to_hash = await db.get_all_media_uri_mappings()
|
||
|
||
# Create zip in memory
|
||
zip_buffer = io.BytesIO()
|
||
|
||
with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||
pages_written = 0
|
||
media_hashes = set()
|
||
|
||
for page in pages:
|
||
uri = page['uri']
|
||
raw_html = page.get('raw_html')
|
||
if not raw_html:
|
||
continue
|
||
|
||
# Parse URI to get path
|
||
parsed = Uri(uri)
|
||
site_domain = parsed.hostname
|
||
path = parsed.path.strip('/') or 'index'
|
||
if not path.endswith('.html') and '.' not in path.split('/')[-1]:
|
||
path = f"{path}/index.html" if path else "index.html"
|
||
|
||
# Rewrite media URLs to local paths
|
||
html = raw_html
|
||
|
||
# Find all src and href attributes pointing to media
|
||
patterns = [
|
||
(r'src=["\']([^"\']+)["\']', 'src'),
|
||
(r'href=["\']([^"\']+\.(jpg|jpeg|png|gif|webp|mp4|webm|svg|ico))["\']', 'href'),
|
||
]
|
||
|
||
for pattern, attr in patterns:
|
||
matches = re.findall(pattern, html, re.IGNORECASE)
|
||
for match in matches:
|
||
url = match[0] if isinstance(match, tuple) else match
|
||
|
||
# Resolve relative URLs
|
||
full_url = urljoin(uri, url)
|
||
|
||
# Check if we have this media
|
||
if full_url in url_to_hash:
|
||
md5 = url_to_hash[full_url]
|
||
media_hashes.add(md5)
|
||
# Replace with local path
|
||
ext = Path(url).suffix or '.bin'
|
||
local_path = f"media/{md5}{ext}"
|
||
html = html.replace(f'{attr}="{url}"', f'{attr}="{local_path}"')
|
||
html = html.replace(f"{attr}='{url}'", f'{attr}="{local_path}"')
|
||
elif url in url_to_hash:
|
||
md5 = url_to_hash[url]
|
||
media_hashes.add(md5)
|
||
ext = Path(url).suffix or '.bin'
|
||
local_path = f"media/{md5}{ext}"
|
||
html = html.replace(f'{attr}="{url}"', f'{attr}="{local_path}"')
|
||
html = html.replace(f"{attr}='{url}'", f'{attr}="{local_path}"')
|
||
|
||
# Write HTML file
|
||
zf.writestr(f"site/{path}", html.encode('utf-8'))
|
||
pages_written += 1
|
||
|
||
# Copy media files from vault
|
||
media_copied = 0
|
||
for md5 in media_hashes:
|
||
subdir = VAULT_PATH / md5[:2]
|
||
if subdir.exists():
|
||
for f in subdir.iterdir():
|
||
if f.name.startswith(md5):
|
||
ext = f.suffix or '.bin'
|
||
zf.write(f, f"site/media/{md5}{ext}")
|
||
media_copied += 1
|
||
break
|
||
|
||
# Write index
|
||
index_html = f"""<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>Phantom Site - {site_domain}</title>
|
||
<style>
|
||
body {{ font-family: sans-serif; max-width: 800px; margin: 50px auto; padding: 20px; }}
|
||
h1 {{ color: #333; }}
|
||
ul {{ line-height: 2; }}
|
||
a {{ color: #0066cc; }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<h1>Phantom Site Archive</h1>
|
||
<p>Domain: {site_domain}</p>
|
||
<p>Pages: {pages_written}</p>
|
||
<p>Media: {media_copied}</p>
|
||
<h2>Pages</h2>
|
||
<ul>
|
||
"""
|
||
for page in pages[:100]:
|
||
parsed = Uri(page['uri'])
|
||
path = parsed.path.strip('/') or 'index'
|
||
if not path.endswith('.html') and '.' not in path.split('/')[-1]:
|
||
path = f"{path}/index.html" if path else "index.html"
|
||
title = page.get('title') or path
|
||
index_html += f' <li><a href="{path}">{title}</a></li>\n'
|
||
|
||
index_html += """ </ul>
|
||
</body>
|
||
</html>"""
|
||
zf.writestr("site/phantom_index.html", index_html.encode('utf-8'))
|
||
|
||
# Return zip
|
||
zip_buffer.seek(0)
|
||
return StreamingResponse(
|
||
zip_buffer,
|
||
media_type="application/zip",
|
||
headers={"Content-Disposition": f"attachment; filename=phantom_{site_domain or 'site'}.zip"}
|
||
)
|
||
|
||
|
||
@app.get("/phantom", response_class=HTMLResponse)
|
||
async def phantom_page(lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Phantom site export UI."""
|
||
language = get_lang(lang, accept_language)
|
||
domains = await db.get_domains_with_pages()
|
||
|
||
domain_options = ''.join([
|
||
f'<option value="{d[0]}">{d[0]} ({d[1]} pages)</option>'
|
||
for d in domains if d[0]
|
||
])
|
||
|
||
html = f"""
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>Phantom Site Export - neopig</title>
|
||
<style>
|
||
* {{ box-sizing: border-box; }}
|
||
body {{
|
||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||
margin: 0;
|
||
padding: 0;
|
||
background: #0a0a0a;
|
||
color: #e0e0e0;
|
||
}}
|
||
.nav {{
|
||
background: #1a1a1a;
|
||
padding: 10px 20px;
|
||
display: flex;
|
||
gap: 20px;
|
||
align-items: center;
|
||
border-bottom: 1px solid #333;
|
||
}}
|
||
.nav a {{ color: #ff6b6b; text-decoration: none; }}
|
||
.nav a:hover {{ text-decoration: underline; }}
|
||
.nav .brand {{ font-weight: bold; font-size: 18px; }}
|
||
.container {{ padding: 20px; }}
|
||
h1 {{ color: #ff6b6b; margin-bottom: 10px; }}
|
||
.subtitle {{ color: #888; margin-bottom: 30px; }}
|
||
.form-group {{ margin-bottom: 20px; }}
|
||
label {{ display: block; margin-bottom: 8px; color: #aaa; }}
|
||
select {{
|
||
width: 100%;
|
||
padding: 12px 16px;
|
||
font-size: 16px;
|
||
border: 2px solid #333;
|
||
border-radius: 8px;
|
||
background: #1a1a1a;
|
||
color: #fff;
|
||
}}
|
||
button {{
|
||
padding: 14px 28px;
|
||
font-size: 16px;
|
||
background: #ff6b6b;
|
||
color: #fff;
|
||
border: none;
|
||
border-radius: 8px;
|
||
cursor: pointer;
|
||
}}
|
||
button:hover {{ background: #ff5252; }}
|
||
.info {{
|
||
background: #1a1a1a;
|
||
padding: 20px;
|
||
border-radius: 8px;
|
||
margin-top: 30px;
|
||
}}
|
||
.info h3 {{ color: #ff6b6b; margin-top: 0; }}
|
||
.info ul {{ color: #aaa; line-height: 1.8; }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<!-- NAV -->
|
||
<div class="container">
|
||
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
||
<input type="text" name="q" placeholder="{{search_placeholder}}" style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||
<option value="">{{all_types}}</option>
|
||
<option value="image">{{images}}</option>
|
||
<option value="video">{{videos}}</option>
|
||
<option value="audio">{{audio}}</option>
|
||
</select>
|
||
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">{{search}}</button>
|
||
</form>
|
||
<h1>Phantom Site Export</h1>
|
||
<p class="subtitle">{{phantom_subtitle}}</p>
|
||
|
||
<form action="/phantom/export" method="get">
|
||
<div class="form-group">
|
||
<label>{{select_domain}}</label>
|
||
<select name="domain">
|
||
<option value="">{{all_domains}}</option>
|
||
{domain_options}
|
||
</select>
|
||
</div>
|
||
<button type="submit">{{download_phantom}} (.zip)</button>
|
||
</form>
|
||
|
||
<div class="info">
|
||
<h3>What is a Phantom Site?</h3>
|
||
<ul>
|
||
<li>Original HTML preserved exactly as crawled</li>
|
||
<li>All media URLs rewritten to local paths</li>
|
||
<li>Ready to host statically (nginx, Caddy, S3, etc.)</li>
|
||
<li>Works offline - all assets included</li>
|
||
<li>Perfect for archival & preservation</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</body>
|
||
</html>
|
||
"""
|
||
return inject_i18n(html, language)
|
||
|
||
|
||
ABOUT_CSS = """
|
||
.about-page { width: 100%; min-height: 100vh; background: #0a0a0f; }
|
||
|
||
/* Hero Section */
|
||
.hero-section {
|
||
width: 100%;
|
||
min-height: 100vh;
|
||
display: flex;
|
||
flex-direction: column;
|
||
justify-content: center;
|
||
align-items: center;
|
||
background: linear-gradient(180deg, #0a0a0f 0%, #1a0a1a 50%, #0f0a1a 100%);
|
||
position: relative;
|
||
overflow: hidden;
|
||
padding: 60px 20px;
|
||
}
|
||
.hero-section::before {
|
||
content: '';
|
||
position: absolute;
|
||
top: 0; left: 0; right: 0; bottom: 0;
|
||
background: radial-gradient(ellipse at 50% 0%, rgba(255,50,100,0.15) 0%, transparent 60%),
|
||
radial-gradient(ellipse at 20% 80%, rgba(100,50,255,0.1) 0%, transparent 40%),
|
||
radial-gradient(ellipse at 80% 60%, rgba(255,100,50,0.08) 0%, transparent 40%);
|
||
pointer-events: none;
|
||
}
|
||
.hero-pig {
|
||
width: 280px;
|
||
height: auto;
|
||
filter: drop-shadow(0 0 60px rgba(255,100,100,0.5)) drop-shadow(0 0 120px rgba(255,50,100,0.3));
|
||
animation: pulse-glow 4s ease-in-out infinite;
|
||
margin-bottom: 40px;
|
||
}
|
||
@keyframes pulse-glow {
|
||
0%, 100% { filter: drop-shadow(0 0 60px rgba(255,100,100,0.5)) drop-shadow(0 0 120px rgba(255,50,100,0.3)); }
|
||
50% { filter: drop-shadow(0 0 80px rgba(255,100,100,0.7)) drop-shadow(0 0 160px rgba(255,50,100,0.5)); }
|
||
}
|
||
.hero-title {
|
||
font-size: 6em;
|
||
font-weight: 900;
|
||
letter-spacing: -0.02em;
|
||
background: linear-gradient(135deg, #ff6b6b 0%, #ff8e53 50%, #ff6b9d 100%);
|
||
-webkit-background-clip: text;
|
||
-webkit-text-fill-color: transparent;
|
||
background-clip: text;
|
||
text-shadow: 0 0 80px rgba(255,100,100,0.5);
|
||
margin: 0;
|
||
text-align: center;
|
||
}
|
||
.hero-subtitle {
|
||
font-size: 1.4em;
|
||
color: #888;
|
||
margin-top: 20px;
|
||
font-weight: 300;
|
||
letter-spacing: 0.3em;
|
||
text-transform: uppercase;
|
||
}
|
||
.hero-tagline {
|
||
font-size: 1.1em;
|
||
color: #666;
|
||
margin-top: 30px;
|
||
max-width: 600px;
|
||
text-align: center;
|
||
line-height: 1.8;
|
||
}
|
||
|
||
/* Chapter Sections */
|
||
.chapter {
|
||
width: 100%;
|
||
padding: 120px 40px;
|
||
position: relative;
|
||
}
|
||
.chapter-content {
|
||
max-width: 900px;
|
||
margin: 0 auto;
|
||
}
|
||
.chapter-number {
|
||
font-size: 8em;
|
||
font-weight: 900;
|
||
color: rgba(255,100,100,0.08);
|
||
position: absolute;
|
||
top: 40px;
|
||
left: 40px;
|
||
line-height: 1;
|
||
font-family: 'Georgia', serif;
|
||
}
|
||
.chapter-title {
|
||
font-size: 3em;
|
||
font-weight: 800;
|
||
color: #ff6b6b;
|
||
margin-bottom: 40px;
|
||
position: relative;
|
||
}
|
||
.chapter-title::after {
|
||
content: '';
|
||
display: block;
|
||
width: 100px;
|
||
height: 4px;
|
||
background: linear-gradient(90deg, #ff6b6b, transparent);
|
||
margin-top: 20px;
|
||
}
|
||
|
||
/* Narrative Text */
|
||
.narrative {
|
||
font-size: 1.3em;
|
||
line-height: 2;
|
||
color: #c0c0c0;
|
||
margin-bottom: 30px;
|
||
}
|
||
.narrative em { color: #ff8e53; font-style: italic; }
|
||
.narrative strong { color: #fff; font-weight: 600; }
|
||
.narrative a { color: #ff6b6b; text-decoration: none; border-bottom: 1px solid rgba(255,107,107,0.3); transition: all 0.3s; }
|
||
.narrative a:hover { color: #ff8e53; border-bottom-color: #ff8e53; }
|
||
|
||
/* Dramatic Quote */
|
||
.dramatic-quote {
|
||
padding: 60px;
|
||
margin: 60px 0;
|
||
background: linear-gradient(135deg, rgba(255,50,100,0.1) 0%, rgba(100,50,200,0.1) 100%);
|
||
border-left: 6px solid #ff6b6b;
|
||
position: relative;
|
||
}
|
||
.dramatic-quote::before {
|
||
content: '"';
|
||
font-size: 12em;
|
||
color: rgba(255,100,100,0.15);
|
||
position: absolute;
|
||
top: -40px;
|
||
left: 20px;
|
||
font-family: 'Georgia', serif;
|
||
line-height: 1;
|
||
}
|
||
.dramatic-quote p {
|
||
font-size: 1.8em;
|
||
font-style: italic;
|
||
color: #e0e0e0;
|
||
margin: 0;
|
||
position: relative;
|
||
z-index: 1;
|
||
}
|
||
.dramatic-quote cite {
|
||
display: block;
|
||
margin-top: 30px;
|
||
font-size: 1em;
|
||
color: #888;
|
||
font-style: normal;
|
||
}
|
||
|
||
/* Dead Link Styling */
|
||
.dead-link {
|
||
text-decoration: line-through;
|
||
color: #666 !important;
|
||
cursor: not-allowed;
|
||
position: relative;
|
||
}
|
||
.dead-link::after {
|
||
content: ' [DEAD]';
|
||
color: #ff4444;
|
||
font-size: 0.7em;
|
||
font-weight: bold;
|
||
text-decoration: none;
|
||
}
|
||
|
||
/* Dark Chapter */
|
||
.chapter-dark {
|
||
background: linear-gradient(180deg, #0a0a0f 0%, #150810 50%, #0a0a0f 100%);
|
||
}
|
||
.chapter-dark .chapter-number { color: rgba(255,50,50,0.1); }
|
||
|
||
/* Rebirth Chapter - TempleOS cyan */
|
||
.chapter-rebirth {
|
||
background: linear-gradient(180deg, #0a0a0f 0%, #0a1515 50%, #0a0a0f 100%);
|
||
}
|
||
.chapter-rebirth .chapter-title { color: #00FFFF; /* TempleOS cyan */ }
|
||
.chapter-rebirth .chapter-title::after { background: linear-gradient(90deg, #00FFFF, transparent); }
|
||
.chapter-rebirth .chapter-number { color: rgba(0,255,255,0.08); }
|
||
|
||
/* Beast Section */
|
||
.chapter-beast {
|
||
background: linear-gradient(180deg, #0a0a0f 0%, #1a0a05 50%, #0a0a0f 100%);
|
||
}
|
||
.chapter-beast .chapter-title { color: #ff8e53; }
|
||
.chapter-beast .chapter-title::after { background: linear-gradient(90deg, #ff8e53, transparent); }
|
||
|
||
/* Feature Cards - Full Width Grid */
|
||
.feature-section {
|
||
width: 100%;
|
||
padding: 100px 40px;
|
||
background: linear-gradient(180deg, #0a0a0f 0%, #0f0f18 50%, #0a0a0f 100%);
|
||
}
|
||
.feature-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
|
||
gap: 30px;
|
||
max-width: 1400px;
|
||
margin: 0 auto;
|
||
}
|
||
.feature-card {
|
||
background: linear-gradient(145deg, rgba(30,30,40,0.8) 0%, rgba(20,20,30,0.9) 100%);
|
||
padding: 40px;
|
||
border-radius: 16px;
|
||
border: 1px solid rgba(255,100,100,0.1);
|
||
transition: all 0.4s ease;
|
||
position: relative;
|
||
overflow: hidden;
|
||
}
|
||
.feature-card::before {
|
||
content: '';
|
||
position: absolute;
|
||
top: 0; left: 0;
|
||
width: 100%; height: 4px;
|
||
background: linear-gradient(90deg, #ff6b6b, #ff8e53, #4ecdc4);
|
||
opacity: 0;
|
||
transition: opacity 0.4s;
|
||
}
|
||
.feature-card:hover {
|
||
transform: translateY(-5px);
|
||
border-color: rgba(255,100,100,0.3);
|
||
box-shadow: 0 20px 60px rgba(0,0,0,0.4), 0 0 40px rgba(255,100,100,0.1);
|
||
}
|
||
.feature-card:hover::before { opacity: 1; }
|
||
.feature-card h3 {
|
||
font-size: 1.4em;
|
||
color: #ff6b6b;
|
||
margin-bottom: 15px;
|
||
font-weight: 700;
|
||
}
|
||
.feature-card p {
|
||
color: #999;
|
||
line-height: 1.7;
|
||
font-size: 1.05em;
|
||
}
|
||
.feature-card a { color: #4ecdc4; }
|
||
|
||
/* Code Blocks */
|
||
.code-section {
|
||
background: #0d0d12;
|
||
padding: 40px;
|
||
border-radius: 12px;
|
||
margin: 40px 0;
|
||
overflow-x: auto;
|
||
border: 1px solid #222;
|
||
}
|
||
.code-section pre {
|
||
margin: 0;
|
||
font-family: 'Fira Code', 'Monaco', monospace;
|
||
font-size: 1em;
|
||
line-height: 1.8;
|
||
color: #e0e0e0;
|
||
}
|
||
.code-section code { background: none; }
|
||
|
||
/* Inline Code */
|
||
.narrative code {
|
||
background: rgba(255,100,100,0.1);
|
||
padding: 3px 10px;
|
||
border-radius: 4px;
|
||
font-family: 'Fira Code', monospace;
|
||
font-size: 0.9em;
|
||
color: #ff8e53;
|
||
}
|
||
|
||
/* Mirror Section - Special */
|
||
.chapter-mirror {
|
||
background: linear-gradient(180deg, #0a0a0f 0%, #0a0a1f 30%, #100a1a 70%, #0a0a0f 100%);
|
||
position: relative;
|
||
}
|
||
.chapter-mirror::before {
|
||
content: '';
|
||
position: absolute;
|
||
top: 0; left: 0; right: 0; bottom: 0;
|
||
background: repeating-linear-gradient(
|
||
90deg,
|
||
transparent 0px,
|
||
transparent 100px,
|
||
rgba(100,100,255,0.02) 100px,
|
||
rgba(100,100,255,0.02) 101px
|
||
);
|
||
pointer-events: none;
|
||
}
|
||
|
||
/* The Forge - TempleOS background */
|
||
.chapter-forge {
|
||
background: url('/static/vendor/templeos.png') center center no-repeat, #55FFFF;
|
||
background-size: 100%;
|
||
position: relative;
|
||
}
|
||
.chapter-forge::before { content: none; }
|
||
.chapter-forge .chapter-content {
|
||
position: relative;
|
||
z-index: 1;
|
||
}
|
||
.chapter-forge .chapter-title { color: #000; }
|
||
.chapter-forge .chapter-title::after { background: linear-gradient(90deg, #000, transparent); }
|
||
.chapter-forge .chapter-number { color: rgba(0,0,0,0.15); }
|
||
.chapter-forge .narrative { color: #000; }
|
||
.chapter-forge .narrative a { color: #006666; }
|
||
.chapter-forge .narrative em { color: #004444; }
|
||
.chapter-forge .narrative strong { color: #000; }
|
||
.chapter-forge .dramatic-quote { background: rgba(0,0,0,0.15); border-left-color: #000; }
|
||
.chapter-forge .dramatic-quote p { color: #000; font-style: italic; }
|
||
.chapter-forge .dramatic-quote cite { color: #333; }
|
||
|
||
/* Rules? - 72 Benedictine principles */
|
||
.rule-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
|
||
gap: 12px;
|
||
margin: 40px 0;
|
||
}
|
||
.rule-item {
|
||
font-size: 1.05em;
|
||
line-height: 1.6;
|
||
color: #b0b0b0;
|
||
padding: 12px 16px;
|
||
background: rgba(255,255,255,0.02);
|
||
border-radius: 6px;
|
||
border-left: 3px solid rgba(255,107,107,0.3);
|
||
}
|
||
.rule-item strong {
|
||
color: #ff6b6b;
|
||
font-weight: 700;
|
||
}
|
||
@media (max-width: 768px) {
|
||
.rule-grid { grid-template-columns: 1fr; }
|
||
}
|
||
|
||
.chapter-mirror .chapter-title { color: #8b5cf6; }
|
||
.chapter-mirror .chapter-title::after { background: linear-gradient(90deg, #8b5cf6, transparent); }
|
||
.chapter-mirror .chapter-number { color: rgba(139,92,246,0.08); }
|
||
|
||
/* Footer */
|
||
.about-footer {
|
||
width: 100%;
|
||
padding: 80px 40px;
|
||
background: #050508;
|
||
text-align: center;
|
||
}
|
||
.about-footer p {
|
||
color: #666;
|
||
font-size: 1.1em;
|
||
margin-bottom: 20px;
|
||
}
|
||
.about-footer a {
|
||
color: #ff6b6b;
|
||
font-size: 1.2em;
|
||
text-decoration: none;
|
||
border-bottom: 2px solid rgba(255,107,107,0.3);
|
||
padding-bottom: 4px;
|
||
transition: all 0.3s;
|
||
}
|
||
.about-footer a:hover {
|
||
color: #ff8e53;
|
||
border-bottom-color: #ff8e53;
|
||
}
|
||
|
||
/* Responsive */
|
||
@media (max-width: 768px) {
|
||
.hero-title { font-size: 3.5em; }
|
||
.hero-pig { width: 200px; }
|
||
.chapter { padding: 80px 20px; }
|
||
.chapter-number { font-size: 5em; top: 20px; left: 20px; }
|
||
.chapter-title { font-size: 2em; }
|
||
.narrative { font-size: 1.1em; }
|
||
.dramatic-quote { padding: 30px; }
|
||
.dramatic-quote p { font-size: 1.3em; }
|
||
.feature-section { padding: 60px 20px; }
|
||
}
|
||
"""
|
||
|
||
ABOUT_CONTENT = """
|
||
<div class="about-page">
|
||
|
||
<!-- HERO SECTION -->
|
||
<section class="hero-section">
|
||
<img src="https://russell.ballestrini.net/uploads/2011/08/pig1-color1.png" alt="pig.py" class="hero-pig">
|
||
<h1 class="hero-title">neopig</h1>
|
||
<p class="hero-subtitle">Neo Python Image Grabber</p>
|
||
<p class="hero-tagline">
|
||
A chimera born from dead repositories. A pig that devours domains whole.
|
||
<br>Where others archive pages, we <em>consume realities</em>.
|
||
</p>
|
||
</section>
|
||
|
||
<!-- CHAPTER I: GENESIS -->
|
||
<section class="chapter">
|
||
<div class="chapter-number">I</div>
|
||
<div class="chapter-content">
|
||
<h2 class="chapter-title">Genesis</h2>
|
||
|
||
<div class="dramatic-quote">
|
||
<p>pig.py is a <em>very</em> simple python command line tool to download all the images from a given uri.</p>
|
||
<cite>— Russell Ballestrini, August 22, 2011</cite>
|
||
</div>
|
||
|
||
<p class="narrative">
|
||
In the beginning, there was <a href="https://russell.ballestrini.net/python-image-grabber-pig-py/" target="_blank">pig.py</a>. A simple creature. Innocent. <em>Hungry.</em>
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
Russell Ballestrini crafted it in the summer of 2011 — a humble Python script with a singular appetite: <strong>images</strong>. Point it at a webpage, and it would slurp down every pixel it could find. No configuration. No complexity. Just a hungry little pig gobbling up the visual fabric of the web.
|
||
</p>
|
||
|
||
<div class="code-section">
|
||
<pre><code>python pig.py https://www.foxhop.net</code></pre>
|
||
</div>
|
||
|
||
<p class="narrative">
|
||
That was it. The entire interface. The pig asked only for a target, and it <em>fed</em>.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
Russell released it into the public domain — a gift to anyone who needed to harvest images from the wild web. The source lived at <code>bitbucket.org/russellballestrini/pig</code>, nestled safely in a Mercurial repository. The pig slept soundly in its pen, unaware of the extinction event approaching.
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<!-- CHAPTER II: THE EXTINCTION -->
|
||
<section class="chapter chapter-dark">
|
||
<div class="chapter-number">II</div>
|
||
<div class="chapter-content">
|
||
<h2 class="chapter-title">The Great Bitbucket Extinction</h2>
|
||
|
||
<p class="narrative">
|
||
<strong>Then came the dark times.</strong>
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
In 2020, Atlassian — the corporate leviathan that had swallowed Bitbucket whole — decreed that Mercurial repositories would be <em>purged</em>. Millions of hg repos, atomized. Scattered like digital ash across the void.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
The original pig.py, that innocent image-gobbling creature, was among the casualties. Its home at <a href="https://bitbucket.org/russellballestrini/pig" class="dead-link">bitbucket.org/russellballestrini/pig</a> became a tombstone. Click it. We dare you. There's nothing there but the echo of corporate indifference.
|
||
</p>
|
||
|
||
<div class="dramatic-quote">
|
||
<p>They literally killed pig.py & sent him to bitbucket.</p>
|
||
<cite>— The cruel irony of naming your graveyard after a slang term for death</cite>
|
||
</div>
|
||
|
||
<p class="narrative">
|
||
<em>"Sent to the bitbucket"</em> — programmer slang from the age of punch cards. When you discarded bad data, you threw it in the bit bucket. Atlassian named their service after a trash can, then proved the prophecy true by throwing away everyone's code.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
The pig was dead. Its bones scattered across cached search results & archived blog posts. A ghost in the machine, referenced but unreachable.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
But here's where it gets <em>weird</em>.
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<!-- CHAPTER III: ALCHEMY -->
|
||
<section class="chapter chapter-rebirth">
|
||
<div class="chapter-number">III</div>
|
||
<div class="chapter-content">
|
||
<h2 class="chapter-title">The Alchemy</h2>
|
||
|
||
<p class="narrative">
|
||
Russell once wrote that <a href="https://russell.ballestrini.net/programming-is-like-alchemy/" target="_blank">"programming is like alchemy — instead of exchanging matter, we programmers exchange time."</a>
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
Programs are <strong>golems</strong>. Familiar spirits. Magical servants performing repetitive tasks so we don't have to. <em>"It is more accurate to group programs with technology than magic, but less fun."</em>
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
Years passed. The web continued its relentless churn — sites going dark, forums shutting down, communities scattering like startled birds. Somewhere, a very angry ex-Ruby developer (Python too) watched the digital decay & remembered the pig.
|
||
</p>
|
||
|
||
<div class="dramatic-quote">
|
||
<p>What if we could transmute that dead code into gold?</p>
|
||
<cite>— The thought that started everything</cite>
|
||
</div>
|
||
|
||
<p class="narrative">
|
||
And speaking of transmutation: <a href="https://phys.org/news/2025-07-marathon-fusion-mercury-gold-energy.html" target="_blank">Marathon Fusion</a> discovered that tokamak breeding blankets — wrapped in Mercury-Lithium alloy, like <em>pigs in a blanket</em> — can transmute Mercury-198 into Gold-197 through chrysopoeia. Fast neutrons trigger (n, 2n) reactions; unstable mercury decays into stable gold within 64 hours.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
Two metric tons of gold per gigawatt. The alchemists' dream realized, wrapped in radioactive patience (17.7 years of cooling before you can touch your transmuted treasure).
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
A golden goose born from the ashes of deprecated version control. The old pig was archived, but <strong>a new creature stirred in the digital depths...</strong>
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<!-- CHAPTER IV: THE PIG AWAKENS -->
|
||
<section class="chapter chapter-beast">
|
||
<div class="chapter-number">IV</div>
|
||
<div class="chapter-content">
|
||
<h2 class="chapter-title">The Pig Awakens</h2>
|
||
|
||
<p class="narrative">
|
||
Like a phoenix rising from dead Bitbucket repos, <strong>neopig</strong> emerged.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
Not a resurrection. Not a mere fork. Something <em>else</em>. A chimera. A griffin. A creature stitched together from the DNA of the original pig & the fevered dreams of someone who had watched too many sites die.
|
||
</p>
|
||
|
||
<div class="dramatic-quote">
|
||
<p>A pig with an appetite like Gluttony from Fullmetal Alchemist — a homunculus that devours everything in its path, absorbing knowledge, power & form.</p>
|
||
</div>
|
||
|
||
<p class="narrative">
|
||
Where pig.py sipped politely from single pages, neopig <strong>devours entire domains</strong>. It doesn't just grab images — it consumes HTML, screenshots pages, converts content to markdown, indexes every scrap of text, and stores it all in content-addressed vaults that will outlast the original servers.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
The code lives now at <a href="https://git.unturf.com/engineering/unturf/pig.py" target="_blank">git.unturf.com/engineering/unturf/pig.py</a>. And it keeps evolving. Every crawl adds new capabilities. Every archived site teaches it new tricks.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
<em>We are adding the ability to backup entire codebases.</em> The pig grows hungrier.
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<!-- CHAPTER V: THE MIRROR DIMENSION -->
|
||
<section class="chapter chapter-mirror">
|
||
<div class="chapter-number">V</div>
|
||
<div class="chapter-content">
|
||
<h2 class="chapter-title">The Mirror Dimension</h2>
|
||
|
||
<p class="narrative">
|
||
We are about to <strong>invert the git tree into itself</strong>.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
Like Dr. Strange versus Spider-Man in the mirror dimension — that impossible space where geometry folds back on itself, where buildings become M.C. Escher nightmares & reality reflects endlessly into its own depths.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
neopig doesn't just archive websites. It can archive <em>itself</em>. It can archive the repositories that contain the code that does the archiving. It can crawl git forges & preserve the source code of tools that preserve source code.
|
||
</p>
|
||
|
||
<div class="dramatic-quote">
|
||
<p>Our spider-pig gets out of the danger of the mirror void — where days are years — and exits absorbing everything like a griffin, like a chimera, like a pig that feeds on the bones of dead platforms.</p>
|
||
</div>
|
||
|
||
<p class="narrative">
|
||
When a code hosting platform dies (and they all die eventually — Google Code, Gitorious, BitBucket's hg repos), neopig ensures the knowledge survives. We are the ark. We are the vault. We are the pig that remembers.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
The archives become self-extracting executables. <code>.run</code> files that contain their own server, their own search engine, their own reality. Drop one on a machine with Python, and it <em>wakes up</em>. A sleeping pig, carrying an entire website in its belly, ready to serve it to anyone who asks.
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<!-- FEATURES SECTION -->
|
||
<section class="feature-section">
|
||
<div class="chapter-content" style="max-width: 100%;">
|
||
<h2 class="chapter-title" style="text-align: center; margin-bottom: 60px;">The Arsenal</h2>
|
||
|
||
<div class="feature-grid">
|
||
<div class="feature-card">
|
||
<h3>Async Domain Crawling</h3>
|
||
<p>Full-domain recursive crawling with configurable depth. Respects robots.txt & crawl delays. Polite but <em>relentless</em>.</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>Content-Addressed Vault</h3>
|
||
<p>MD5-based deduplication in a 9-deep hex directory structure. Same image from 100 pages? Stored once. Space is memory.</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>Full-Text Search</h3>
|
||
<p>Every image indexed with its surrounding context — page title, alt text, captions, nearby headings. Find images by <em>meaning</em>, not filename.</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>Page Screenshots</h3>
|
||
<p><a href="https://linkpeek.com">LinkPeek</a> reborn! Full-page captures via <a href="https://github.com/russellballestrini/uri2png">uri2png</a>. wkhtmltoimage, cutycapt, or Playwright backends.</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>Hydra Mode</h3>
|
||
<p>Auto-discover RSS/Atom/Sitemap feeds. Persists feed URLs, checks for new content on every crawl. Self-healing archive maintenance.</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>Media Provenance</h3>
|
||
<p>Track first_seen, last_seen, and when each page first linked to media. Watch content spread across the web over time.</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>Markdown Conversion</h3>
|
||
<p>Intelligent HTML-to-Markdown with forum post detection, noise removal & content extraction. Human-readable archives.</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>Self-Extracting Archives</h3>
|
||
<p>Create <code>.run</code> executables that contain entire websites. Drop on any machine, execute, browse. The pig travels light.</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>Code File Support</h3>
|
||
<p>Archive source code, stylesheets, fonts, and scripts alongside media. Complete preservation for developer communities.</p>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
|
||
<!-- CHAPTER VI: THE SKELETON KEY -->
|
||
<section class="chapter">
|
||
<div class="chapter-number">VI</div>
|
||
<div class="chapter-content">
|
||
<h2 class="chapter-title">The Skeleton Key</h2>
|
||
|
||
<p class="narrative">
|
||
neopig uses what we call the <strong>"skeleton key"</strong> approach to media indexing. For every image, we capture <em>all</em> the text that might help you find it later:
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
<strong>Page context:</strong> Title, description, headings near the image.<br>
|
||
<strong>Image attributes:</strong> Alt text, title, surrounding captions.<br>
|
||
<strong>Link context:</strong> The text of links pointing to the image.<br>
|
||
<strong>Detail pages:</strong> For gallery sites, we follow through to detail pages & harvest their metadata too.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
The result? You can search for <em>"sunset over mountains"</em> and find that image even if it was named <code>IMG_4372.jpg</code> with no alt text — because the page title mentioned it, or someone linked to it with descriptive text, or a caption three divs away contained the words.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
Every door opens. Every search succeeds. The skeleton key fits every lock because it carries <em>every possible key</em> inside it.
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<!-- CHAPTER VII: PRESERVATION -->
|
||
<section class="chapter chapter-dark">
|
||
<div class="chapter-number">VII</div>
|
||
<div class="chapter-content">
|
||
<h2 class="chapter-title">Why We Archive</h2>
|
||
|
||
<p class="narrative">
|
||
<strong>The web is ephemeral.</strong> Sites go dark. Forums shut down. Communities scatter. Platforms get acquired & gutted. Executives decide that Mercurial isn't profitable & delete millions of repositories with a single corporate memo.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
We have watched beloved communities vanish. Game modding forums. Technical documentation. Art galleries. Fan wikis. Personal blogs with decades of writing. Gone. Not archived by the Wayback Machine. Not cached by Google. Just <em>gone</em>.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
neopig is built for <strong>preservation</strong> — capturing not just the media, but the <em>context</em> that gives it meaning. When you archive a site with neopig, you get:
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
• Original HTML with rewritten media links pointing to your local vault<br>
|
||
• Full-text searchable markdown versions of every page<br>
|
||
• Screenshots showing exactly how pages looked<br>
|
||
• A SQLite database you can query, backup, and migrate forever<br>
|
||
• Self-extracting archives that work offline forever
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<!-- CHAPTER VIII: THE FORGE -->
|
||
<section class="chapter chapter-forge">
|
||
<div class="chapter-number">VIII</div>
|
||
<div class="chapter-content">
|
||
<h2 class="chapter-title">The Forge</h2>
|
||
|
||
<p class="narrative">
|
||
This tool was not built for profit. It was <strong>forged</strong>.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
Forged in the fires of <a href="https://permacomputer.com" target="_blank">permacomputer.com</a> — the belief that computing should be <em>permanent</em>. That knowledge should not evaporate when a company pivots. That communities should not lose their histories when a platform gets acquired. That the lore doesn't have to die when you get acqui-hired or outright <span style="color: #8B00FF;">terminated</span>. That the digital commons belongs to <em>everyone</em>, forever.
|
||
</p>
|
||
|
||
<div class="dramatic-quote">
|
||
<p>Truth. Harmony. Freedom. Love.</p>
|
||
<cite>— The quadrivium of the permacomputer</cite>
|
||
</div>
|
||
|
||
<p class="narrative">
|
||
<strong>Truth</strong> — We preserve what actually existed, not sanitized versions. The original HTML. The exact timestamps. The unedited conversations. History as it happened, not as someone wishes it had.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
<strong>Harmony</strong> — Every piece connects. Media links to pages. Pages link to context. Context links to meaning. The archive is not a graveyard of disconnected files but a living web of relationships, searchable, traversable, <em>whole</em>.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
<strong>Freedom</strong> — Public domain. No licenses to parse. No corporate terms of service. Take it. Fork it. Run it. Share it. The code is yours. The archives you create are yours. No one can revoke your access to your own preserved knowledge.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
<strong>Love</strong> — We archive because we <em>care</em>. About the communities that built these spaces. About the people who poured years into forum posts & wiki pages & blog entries. About the future researchers who will want to understand how we lived & thought & created. This is an act of love for human knowledge.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
The pig is imbued with <em>righteous fire</em>. It does not crawl politely because it fears the powerful — it respects robots.txt because it honors the wishes of creators. It does not hoard in darkness — it preserves in light, making archives searchable, shareable, <em>alive</em>.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
When the platforms fall (and they will fall), when the corporate servers go dark (and they will go dark), when the executives decide your community isn't profitable enough to maintain (and they will decide this) — the permacomputer endures. The archives persist. The knowledge survives.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
<strong>This is what we were forged to do.</strong>
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<!-- CHAPTER IX: RULES? -->
|
||
<section class="chapter">
|
||
<div class="chapter-number">IX</div>
|
||
<div class="chapter-content">
|
||
<h2 class="chapter-title">Rules?</h2>
|
||
|
||
<p class="narrative">
|
||
neopig is built on <a href="https://sqlite.org" target="_blank">SQLite</a> — the most deployed database in existence. SQLite adopted the <a href="https://sqlite.org/codeofethics.html" target="_blank">Rule of St. Benedict</a> as their Code of Ethics. 72 principles from a 6th-century monastery, governing software in the 21st.
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
We inherit this lineage. All 72 rules, unabridged:
|
||
</p>
|
||
|
||
<div class="rule-grid">
|
||
<div class="rule-item"><strong>1.</strong> Love the Lord God with your whole heart, soul, and strength.</div>
|
||
<div class="rule-item"><strong>2.</strong> Love your neighbor as yourself.</div>
|
||
<div class="rule-item"><strong>3.</strong> Do not murder.</div>
|
||
<div class="rule-item"><strong>4.</strong> Do not commit adultery.</div>
|
||
<div class="rule-item"><strong>5.</strong> Do not steal.</div>
|
||
<div class="rule-item"><strong>6.</strong> Do not covet.</div>
|
||
<div class="rule-item"><strong>7.</strong> Do not bear false witness.</div>
|
||
<div class="rule-item"><strong>8.</strong> Honor all people.</div>
|
||
<div class="rule-item"><strong>9.</strong> Do not do to another what you would not have done to yourself.</div>
|
||
<div class="rule-item"><strong>10.</strong> Deny oneself in order to follow Christ.</div>
|
||
<div class="rule-item"><strong>11.</strong> Chastise the body.</div>
|
||
<div class="rule-item"><strong>12.</strong> Do not become attached to pleasures.</div>
|
||
<div class="rule-item"><strong>13.</strong> Love fasting.</div>
|
||
<div class="rule-item"><strong>14.</strong> Relieve the poor.</div>
|
||
<div class="rule-item"><strong>15.</strong> Clothe the naked.</div>
|
||
<div class="rule-item"><strong>16.</strong> Visit the sick.</div>
|
||
<div class="rule-item"><strong>17.</strong> Bury the dead.</div>
|
||
<div class="rule-item"><strong>18.</strong> Be a help in times of trouble.</div>
|
||
<div class="rule-item"><strong>19.</strong> Console the sorrowing.</div>
|
||
<div class="rule-item"><strong>20.</strong> Be a stranger to the world's ways.</div>
|
||
<div class="rule-item"><strong>21.</strong> Prefer nothing more than the love of Christ.</div>
|
||
<div class="rule-item"><strong>22.</strong> Do not give way to anger.</div>
|
||
<div class="rule-item"><strong>23.</strong> Do not nurse a grudge.</div>
|
||
<div class="rule-item"><strong>24.</strong> Do not entertain deceit in your heart.</div>
|
||
<div class="rule-item"><strong>25.</strong> Do not give a false peace.</div>
|
||
<div class="rule-item"><strong>26.</strong> Do not forsake charity.</div>
|
||
<div class="rule-item"><strong>27.</strong> Do not swear, for fear of perjuring yourself.</div>
|
||
<div class="rule-item"><strong>28.</strong> Utter only truth from heart and mouth.</div>
|
||
<div class="rule-item"><strong>29.</strong> Do not return evil for evil.</div>
|
||
<div class="rule-item"><strong>30.</strong> Do no wrong to anyone, and bear patiently wrongs done to yourself.</div>
|
||
<div class="rule-item"><strong>31.</strong> Love your enemies.</div>
|
||
<div class="rule-item"><strong>32.</strong> Do not curse those who curse you, but rather bless them.</div>
|
||
<div class="rule-item"><strong>33.</strong> Bear persecution for justice's sake.</div>
|
||
<div class="rule-item"><strong>34.</strong> Be not proud.</div>
|
||
<div class="rule-item"><strong>35.</strong> Be not addicted to wine.</div>
|
||
<div class="rule-item"><strong>36.</strong> Be not a great eater.</div>
|
||
<div class="rule-item"><strong>37.</strong> Be not drowsy.</div>
|
||
<div class="rule-item"><strong>38.</strong> Be not lazy.</div>
|
||
<div class="rule-item"><strong>39.</strong> Be not a grumbler.</div>
|
||
<div class="rule-item"><strong>40.</strong> Be not a detractor.</div>
|
||
<div class="rule-item"><strong>41.</strong> Put your hope in God.</div>
|
||
<div class="rule-item"><strong>42.</strong> Attribute to God, and not to self, whatever good you see in yourself.</div>
|
||
<div class="rule-item"><strong>43.</strong> Recognize always that evil is your own doing, and to impute it to yourself.</div>
|
||
<div class="rule-item"><strong>44.</strong> Fear the Day of Judgment.</div>
|
||
<div class="rule-item"><strong>45.</strong> Be in dread of hell.</div>
|
||
<div class="rule-item"><strong>46.</strong> Desire eternal life with all the passion of the spirit.</div>
|
||
<div class="rule-item"><strong>47.</strong> Keep death daily before your eyes.</div>
|
||
<div class="rule-item"><strong>48.</strong> Keep constant guard over the actions of your life.</div>
|
||
<div class="rule-item"><strong>49.</strong> Know for certain that God sees you everywhere.</div>
|
||
<div class="rule-item"><strong>50.</strong> When wrongful thoughts come into your heart, dash them against Christ immediately.</div>
|
||
<div class="rule-item"><strong>51.</strong> Disclose wrongful thoughts to your spiritual mentor.</div>
|
||
<div class="rule-item"><strong>52.</strong> Guard your tongue against evil and depraved speech.</div>
|
||
<div class="rule-item"><strong>53.</strong> Do not love much talking.</div>
|
||
<div class="rule-item"><strong>54.</strong> Speak no useless words or words that move to laughter.</div>
|
||
<div class="rule-item"><strong>55.</strong> Do not love much or boisterous laughter.</div>
|
||
<div class="rule-item"><strong>56.</strong> Listen willingly to holy reading.</div>
|
||
<div class="rule-item"><strong>57.</strong> Devote yourself frequently to prayer.</div>
|
||
<div class="rule-item"><strong>58.</strong> Daily in your prayers, with tears and sighs, confess your past sins to God, and amend them for the future.</div>
|
||
<div class="rule-item"><strong>59.</strong> Fulfill not the desires of the flesh; hate your own will.</div>
|
||
<div class="rule-item"><strong>60.</strong> Obey in all things the commands of those whom God has placed in authority over you even though they should act otherwise, mindful of the Lord's precept, "Do what they say, but not what they do."</div>
|
||
<div class="rule-item"><strong>61.</strong> Do not wish to be called holy before one is holy; but first to be holy, that you may be truly so called.</div>
|
||
<div class="rule-item"><strong>62.</strong> Fulfill God's commandments daily in your deeds.</div>
|
||
<div class="rule-item"><strong>63.</strong> Love chastity.</div>
|
||
<div class="rule-item"><strong>64.</strong> Hate no one.</div>
|
||
<div class="rule-item"><strong>65.</strong> Be not jealous, nor harbor envy.</div>
|
||
<div class="rule-item"><strong>66.</strong> Do not love quarreling.</div>
|
||
<div class="rule-item"><strong>67.</strong> Shun arrogance.</div>
|
||
<div class="rule-item"><strong>68.</strong> Respect your seniors.</div>
|
||
<div class="rule-item"><strong>69.</strong> Love your juniors.</div>
|
||
<div class="rule-item"><strong>70.</strong> Pray for your enemies in the love of Christ.</div>
|
||
<div class="rule-item"><strong>71.</strong> Make peace with your adversary before the sun sets.</div>
|
||
<div class="rule-item"><strong>72.</strong> Never despair of God's mercy.</div>
|
||
</div>
|
||
|
||
<p class="narrative" style="margin-top: 60px;">
|
||
We archive not to hoard, but to <em>give</em>. We preserve not to control, but to <em>liberate</em>. The database that holds your memories runs on a foundation of ancient wisdom — and so do we.
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<!-- CHAPTER X: INVOKE THE PIG -->
|
||
<section class="chapter chapter-beast">
|
||
<div class="chapter-number">X</div>
|
||
<div class="chapter-content">
|
||
<h2 class="chapter-title">Invoke the Pig</h2>
|
||
|
||
<div class="code-section">
|
||
<pre><code>python neopig.py --serve
|
||
# Opens http://127.0.0.1:31337</code></pre>
|
||
</div>
|
||
|
||
<p class="narrative">
|
||
<strong>neopig</strong> = <strong>Neo</strong> (new) + <strong>P</strong>ython <strong>I</strong>mage <strong>G</strong>rabber
|
||
</p>
|
||
|
||
<p class="narrative">
|
||
A tip of the hat to the original pig.py. A nod to the Matrix's Neo — seeing through the surface of the web to the underlying content within. And perhaps a warning: this pig has grown teeth.
|
||
</p>
|
||
</div>
|
||
</section>
|
||
|
||
<!-- FOOTER -->
|
||
<footer class="about-footer">
|
||
<p>neopig is open source, donated into the public domain.</p>
|
||
<p>The pig lives at:</p>
|
||
<a href="https://git.unturf.com/engineering/unturf/pig.py" target="_blank">git.unturf.com/engineering/unturf/pig.py</a>
|
||
<p style="margin-top: 40px; color: #444; font-size: 0.9em;">
|
||
In memory of all the repositories Atlassian killed.<br>
|
||
We remember. We archive. We <em>persist</em>.
|
||
</p>
|
||
<p style="margin-top: 60px; color: #666; font-style: italic;">
|
||
Sorry for the convenience.
|
||
</p>
|
||
</footer>
|
||
|
||
</div>
|
||
"""
|
||
|
||
|
||
@app.get("/about", response_class=HTMLResponse)
|
||
async def about_page(lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""About neopig - the story of pig.py's evolution."""
|
||
language = get_lang(lang, accept_language)
|
||
html = layout("{{about}}", ABOUT_CONTENT, extra_css=ABOUT_CSS)
|
||
return inject_i18n(html, language)
|
||
|
||
|
||
@app.get("/health")
|
||
async def health():
|
||
"""Health check endpoint."""
|
||
has_screenshot = False
|
||
try:
|
||
from uri2png import get_available_engines
|
||
has_screenshot = True
|
||
except ImportError:
|
||
pass
|
||
|
||
return {
|
||
"status": "healthy",
|
||
"features": {
|
||
"search": True,
|
||
"crawl": True,
|
||
"screenshot": has_screenshot
|
||
}
|
||
}
|
||
|
||
|
||
@app.get("/api/stats")
|
||
async def get_stats_endpoint():
|
||
"""Get database statistics."""
|
||
stats = await db.get_stats()
|
||
# Add page count (handled separately since table may not exist)
|
||
try:
|
||
from sqlalchemy import select, func
|
||
from database import Page
|
||
async with db.session() as session:
|
||
result = await session.execute(select(func.count()).select_from(Page))
|
||
stats['total_pages'] = result.scalar() or 0
|
||
except Exception:
|
||
stats['total_pages'] = 0
|
||
return stats
|
||
|
||
|
||
@app.get("/random")
|
||
async def random_item(type: str = Query(None, description="Type: media or page (random if not specified)")):
|
||
"""Redirect to a random media item or page."""
|
||
from sqlalchemy import select, func
|
||
from database import Media, Page
|
||
import random
|
||
|
||
# If no type specified, randomly pick between media and page
|
||
if type is None:
|
||
type = random.choice(["media", "page"])
|
||
|
||
async with db.session() as session:
|
||
if type == "page":
|
||
# Get a random page
|
||
stmt = select(Page.uri_hash).order_by(func.random()).limit(1)
|
||
result = await session.execute(stmt)
|
||
row = result.fetchone()
|
||
if row and row[0]:
|
||
return RedirectResponse(url=f"/page/{row[0]}", status_code=302)
|
||
else:
|
||
# Get a random media item (excluding screenshots)
|
||
stmt = (
|
||
select(Media.md5_hash)
|
||
.where(Media.media_type != 'screenshot')
|
||
.order_by(func.random())
|
||
.limit(1)
|
||
)
|
||
result = await session.execute(stmt)
|
||
row = result.fetchone()
|
||
if row:
|
||
return RedirectResponse(url=f"/view/{row[0]}", status_code=302)
|
||
|
||
return RedirectResponse(url="/", status_code=302)
|
||
|
||
|
||
@app.get("/api/search")
|
||
async def search(
|
||
q: str = Query("", description="Search query"),
|
||
type: Optional[str] = Query(None, description="Filter by media type"),
|
||
status: Optional[str] = Query(None, description="Filter by analysis status"),
|
||
limit: int = Query(OVER_9000, le=OVER_9000),
|
||
offset: int = Query(0)
|
||
):
|
||
"""
|
||
Search media by text query.
|
||
|
||
Searches across: keywords, alt_text, title, source URLs, analysis results.
|
||
"""
|
||
results = await db.search_media_advanced(
|
||
q=q if q else None,
|
||
media_type=type,
|
||
limit=limit,
|
||
offset=offset
|
||
)
|
||
return results
|
||
|
||
|
||
@app.get("/api/live/stream")
|
||
async def live_stream():
|
||
"""
|
||
SSE endpoint for live media feed.
|
||
|
||
Media appears here immediately after being saved to disk,
|
||
before DB insert completes. Use EventSource to connect.
|
||
"""
|
||
async def event_generator():
|
||
queue = get_live_queue()
|
||
while True:
|
||
try:
|
||
# Wait for next media item with timeout
|
||
media = await asyncio.wait_for(queue.get(), timeout=30)
|
||
yield f"data: {json.dumps(media)}\n\n"
|
||
except asyncio.TimeoutError:
|
||
# Send keepalive
|
||
yield ": keepalive\n\n"
|
||
except Exception as e:
|
||
logger.warning(f"SSE error: {e}")
|
||
break
|
||
|
||
return StreamingResponse(
|
||
event_generator(),
|
||
media_type="text/event-stream",
|
||
headers={
|
||
"Cache-Control": "no-cache",
|
||
"Connection": "keep-alive",
|
||
"X-Accel-Buffering": "no", # Disable nginx buffering
|
||
}
|
||
)
|
||
|
||
|
||
@app.get("/api/search/pages")
|
||
async def search_pages_endpoint(
|
||
q: str = Query("", description="Search query"),
|
||
limit: int = Query(50, le=500),
|
||
):
|
||
"""
|
||
Search pages by text query using FTS5.
|
||
"""
|
||
if not q:
|
||
return []
|
||
return await db.search_pages(q, limit)
|
||
|
||
|
||
@app.get("/api/upgraded")
|
||
async def get_upgraded_media(limit: int = Query(50, le=200)):
|
||
"""
|
||
Get media that was recently upgraded (found higher quality version).
|
||
Used by live feed to show when thumbnails get replaced by full-res.
|
||
"""
|
||
return await db.get_recently_upgraded(limit)
|
||
|
||
|
||
@app.get("/api/media/{md5_hash}")
|
||
async def get_media_info(md5_hash: str):
|
||
"""Get full media info including all source URLs."""
|
||
media = await db.get_media_by_hash(md5_hash)
|
||
if not media:
|
||
raise HTTPException(status_code=404, detail="Media not found")
|
||
|
||
media['sources'] = await db.get_media_sources(md5_hash)
|
||
return media
|
||
|
||
|
||
def slugify(text: str, max_len: int = 60) -> str:
|
||
"""Convert text to a safe filename slug."""
|
||
import re
|
||
import unicodedata
|
||
# Normalize unicode
|
||
text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('ascii')
|
||
# Lowercase and replace spaces/special chars with hyphens
|
||
text = re.sub(r'[^\w\s-]', '', text.lower())
|
||
text = re.sub(r'[-\s]+', '-', text).strip('-')
|
||
return text[:max_len] if text else ""
|
||
|
||
|
||
@app.get("/media/{md5_hash}")
|
||
async def serve_media(md5_hash: str, download: bool = False):
|
||
"""
|
||
Serve media file from vault or tarball.
|
||
|
||
Use ?download=1 for attachment mode with smart filename.
|
||
Caddy should be configured to cache these responses.
|
||
"""
|
||
# Tarball mode: serve from tar.gz (run in thread to avoid blocking)
|
||
if TAR_PATH:
|
||
data, ext = await asyncio.to_thread(find_media_in_tarball, md5_hash)
|
||
if data:
|
||
mime_type, _ = mimetypes.guess_type(f"file{ext}")
|
||
if not mime_type:
|
||
mime_type = "application/octet-stream"
|
||
filename = f"{md5_hash[:12]}{ext}"
|
||
headers = {
|
||
"Cache-Control": "public, max-age=31536000, immutable",
|
||
"X-Content-Hash": md5_hash,
|
||
}
|
||
if download:
|
||
headers["Content-Disposition"] = f'attachment; filename="{filename}"'
|
||
return Response(content=data, media_type=mime_type, headers=headers)
|
||
raise HTTPException(status_code=404, detail="Media not found in archive")
|
||
|
||
# Filesystem mode: find file in vault (9-deep path)
|
||
subdir = VAULT_PATH / hash_to_path(md5_hash).parent
|
||
if not subdir.exists():
|
||
raise HTTPException(status_code=404, detail="Media not found")
|
||
|
||
# Find file with this hash prefix
|
||
for f in subdir.iterdir():
|
||
if f.name.startswith(md5_hash):
|
||
# Guess content type from filename
|
||
mime_type, _ = mimetypes.guess_type(f.name)
|
||
if not mime_type:
|
||
mime_type = "application/octet-stream"
|
||
|
||
# Inline mode: skip DB queries, just serve the file fast
|
||
if not download:
|
||
return FileResponse(
|
||
f,
|
||
media_type=mime_type,
|
||
content_disposition_type="inline",
|
||
headers={
|
||
"Cache-Control": "public, max-age=31536000, immutable",
|
||
"X-Content-Hash": md5_hash,
|
||
}
|
||
)
|
||
|
||
# Download mode: generate smart filename from metadata
|
||
ext = f.suffix or ""
|
||
filename = None
|
||
media_record = await db.get_media_by_hash(md5_hash)
|
||
if media_record:
|
||
if not mime_type and media_record.get("mime_type"):
|
||
mime_type = media_record["mime_type"]
|
||
# Generate filename from alt_text or title
|
||
name_source = media_record.get("alt_text") or media_record.get("title")
|
||
if name_source:
|
||
slug = slugify(name_source)
|
||
if slug:
|
||
filename = f"{slug}{ext}"
|
||
|
||
# Fallback: try to get page_title from media_sources
|
||
if not filename:
|
||
sources = await db.get_media_sources(md5_hash)
|
||
if sources:
|
||
row2 = sources[0]
|
||
# Try page_title + hash index
|
||
if row2.get("page_title"):
|
||
media_idx = int(md5_hash[:4], 16)
|
||
slug = slugify(f"{row2['page_title']}-{media_idx}")
|
||
if slug:
|
||
filename = f"{slug}{ext}"
|
||
# Fallback: original filename from URL
|
||
if not filename and row2.get("media_uri"):
|
||
from urllib.parse import unquote
|
||
parsed = Uri(row2["media_uri"])
|
||
orig_name = Path(unquote(parsed.path)).name
|
||
if orig_name and '.' in orig_name:
|
||
filename = orig_name
|
||
|
||
# Default filename if nothing else
|
||
if not filename:
|
||
filename = f"{md5_hash[:12]}{ext}"
|
||
|
||
return FileResponse(
|
||
f,
|
||
media_type=mime_type,
|
||
filename=filename,
|
||
content_disposition_type="attachment",
|
||
headers={
|
||
"Cache-Control": "public, max-age=31536000, immutable",
|
||
"X-Content-Hash": md5_hash,
|
||
}
|
||
)
|
||
|
||
raise HTTPException(status_code=404, detail="Media not found")
|
||
|
||
|
||
# ============================================================================
|
||
# Sandbox Mode - Upload and serve archives
|
||
# ============================================================================
|
||
|
||
SANDBOX_UPLOAD_DIR = Path(tempfile.gettempdir()) / "neopig_sandbox"
|
||
|
||
# Resumable upload tracking: upload_id -> {filename, total_size, created_at}
|
||
PENDING_UPLOADS: Dict[str, dict] = {}
|
||
|
||
def generate_upload_id() -> str:
|
||
"""Generate a unique upload ID."""
|
||
import secrets
|
||
return f"upload-{secrets.token_hex(8)}"
|
||
|
||
|
||
@app.post("/api/sandbox/upload/init")
|
||
async def init_resumable_upload(filename: str = Query(...), size: int = Query(...)):
|
||
"""Initialize a resumable upload session.
|
||
|
||
Returns upload_id that client uses for subsequent chunk uploads.
|
||
Client can resume from any disconnect by checking /api/sandbox/upload/{upload_id}/status
|
||
"""
|
||
if not SANDBOX_MODE:
|
||
raise HTTPException(status_code=403, detail="Sandbox mode not enabled")
|
||
|
||
if not filename.endswith(('.tar.gz', '.tgz', '.run')):
|
||
raise HTTPException(status_code=400, detail="File must be .tar.gz, .tgz, or .run")
|
||
|
||
upload_id = generate_upload_id()
|
||
SANDBOX_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
# Create empty file for this upload
|
||
upload_path = SANDBOX_UPLOAD_DIR / f"{upload_id}.part"
|
||
upload_path.touch()
|
||
|
||
PENDING_UPLOADS[upload_id] = {
|
||
"filename": filename,
|
||
"total_size": size,
|
||
"created_at": asyncio.get_event_loop().time(),
|
||
"path": str(upload_path)
|
||
}
|
||
|
||
logger.info(f"Sandbox: initialized resumable upload {upload_id} for {filename} ({size / 1024 / 1024:.1f} MB)")
|
||
|
||
return {
|
||
"upload_id": upload_id,
|
||
"filename": filename,
|
||
"total_size": size,
|
||
"bytes_received": 0
|
||
}
|
||
|
||
|
||
@app.get("/api/sandbox/upload/{upload_id}/status")
|
||
async def get_upload_status(upload_id: str):
|
||
"""Get status of a resumable upload. Use this to resume after disconnect."""
|
||
if not SANDBOX_MODE:
|
||
raise HTTPException(status_code=403, detail="Sandbox mode not enabled")
|
||
|
||
if upload_id not in PENDING_UPLOADS:
|
||
raise HTTPException(status_code=404, detail="Upload not found or expired")
|
||
|
||
info = PENDING_UPLOADS[upload_id]
|
||
upload_path = Path(info["path"])
|
||
|
||
if not upload_path.exists():
|
||
raise HTTPException(status_code=404, detail="Upload file not found")
|
||
|
||
bytes_received = upload_path.stat().st_size
|
||
|
||
return {
|
||
"upload_id": upload_id,
|
||
"filename": info["filename"],
|
||
"total_size": info["total_size"],
|
||
"bytes_received": bytes_received,
|
||
"complete": bytes_received >= info["total_size"]
|
||
}
|
||
|
||
|
||
@app.patch("/api/sandbox/upload/{upload_id}")
|
||
async def upload_chunk(
|
||
upload_id: str,
|
||
request: Request,
|
||
content_range: str = Header(None)
|
||
):
|
||
"""Upload a chunk of data for resumable upload.
|
||
|
||
Use Content-Range header: bytes START-END/TOTAL
|
||
Example: Content-Range: bytes 0-1048575/10485760
|
||
|
||
Or use X-Upload-Offset header for simpler resumption.
|
||
"""
|
||
if not SANDBOX_MODE:
|
||
raise HTTPException(status_code=403, detail="Sandbox mode not enabled")
|
||
|
||
if upload_id not in PENDING_UPLOADS:
|
||
raise HTTPException(status_code=404, detail="Upload not found or expired")
|
||
|
||
info = PENDING_UPLOADS[upload_id]
|
||
upload_path = Path(info["path"])
|
||
|
||
# Parse offset from Content-Range or X-Upload-Offset
|
||
offset = 0
|
||
if content_range:
|
||
# Parse "bytes START-END/TOTAL"
|
||
try:
|
||
range_spec = content_range.replace("bytes ", "")
|
||
range_part = range_spec.split("/")[0]
|
||
offset = int(range_part.split("-")[0])
|
||
except Exception:
|
||
raise HTTPException(status_code=400, detail="Invalid Content-Range header")
|
||
else:
|
||
# Use X-Upload-Offset if no Content-Range
|
||
offset_header = request.headers.get("x-upload-offset")
|
||
if offset_header:
|
||
offset = int(offset_header)
|
||
|
||
# Verify offset matches current file size (no gaps)
|
||
current_size = upload_path.stat().st_size if upload_path.exists() else 0
|
||
if offset != current_size:
|
||
raise HTTPException(
|
||
status_code=409,
|
||
detail=f"Offset mismatch: expected {current_size}, got {offset}. Resume from byte {current_size}."
|
||
)
|
||
|
||
# Read and append chunk
|
||
chunk_data = await request.body()
|
||
chunk_size = len(chunk_data)
|
||
|
||
with open(upload_path, 'ab') as f:
|
||
f.write(chunk_data)
|
||
|
||
new_size = upload_path.stat().st_size
|
||
logger.info(f"Sandbox: {upload_id} received chunk {chunk_size} bytes, total {new_size}/{info['total_size']}")
|
||
|
||
return {
|
||
"upload_id": upload_id,
|
||
"bytes_received": new_size,
|
||
"total_size": info["total_size"],
|
||
"complete": new_size >= info["total_size"]
|
||
}
|
||
|
||
|
||
@app.post("/api/sandbox/upload/{upload_id}/complete")
|
||
async def complete_resumable_upload(upload_id: str):
|
||
"""Finalize upload and start import job."""
|
||
if not SANDBOX_MODE:
|
||
raise HTTPException(status_code=403, detail="Sandbox mode not enabled")
|
||
|
||
if upload_id not in PENDING_UPLOADS:
|
||
raise HTTPException(status_code=404, detail="Upload not found or expired")
|
||
|
||
info = PENDING_UPLOADS[upload_id]
|
||
upload_path = Path(info["path"])
|
||
|
||
if not upload_path.exists():
|
||
raise HTTPException(status_code=404, detail="Upload file not found")
|
||
|
||
bytes_received = upload_path.stat().st_size
|
||
if bytes_received < info["total_size"]:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail=f"Upload incomplete: {bytes_received}/{info['total_size']} bytes"
|
||
)
|
||
|
||
# Rename to final filename
|
||
final_path = SANDBOX_UPLOAD_DIR / info["filename"]
|
||
upload_path.rename(final_path)
|
||
|
||
# Remove from pending
|
||
del PENDING_UPLOADS[upload_id]
|
||
|
||
logger.info(f"Sandbox: {upload_id} completed, saved as {info['filename']}")
|
||
|
||
# Create import job
|
||
job_id = await db.create_crawl_job(
|
||
target_uri=f"import://{info['filename']}",
|
||
keywords=["sandbox", "import"],
|
||
mode="import"
|
||
)
|
||
|
||
# Run import in background
|
||
async def run_import():
|
||
try:
|
||
await db.set_crawl_job_status(job_id, "running")
|
||
stats = await import_archive_to_db(final_path, job_id)
|
||
await db.complete_crawl_job(job_id, stats)
|
||
final_path.unlink(missing_ok=True)
|
||
logger.info(f"Sandbox import complete: {stats}")
|
||
except Exception as e:
|
||
logger.error(f"Sandbox import failed: {e}")
|
||
await db.fail_crawl_job(job_id, str(e))
|
||
|
||
task = asyncio.create_task(run_import())
|
||
ACTIVE_CRAWL_TASKS[job_id] = task
|
||
task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))
|
||
|
||
return {
|
||
"status": "importing",
|
||
"job_id": job_id,
|
||
"filename": info["filename"],
|
||
"size": bytes_received
|
||
}
|
||
|
||
|
||
@app.post("/api/sandbox/upload")
|
||
async def upload_archive(file: UploadFile = File(...)):
|
||
"""Upload a tar.gz archive and import into local database as a job."""
|
||
if not SANDBOX_MODE:
|
||
raise HTTPException(status_code=403, detail="Sandbox mode not enabled (NEOPIG_SANDBOX=1)")
|
||
|
||
if not file.filename.endswith(('.tar.gz', '.tgz', '.run')):
|
||
raise HTTPException(status_code=400, detail="File must be .tar.gz, .tgz, or .run")
|
||
|
||
# Save uploaded file
|
||
SANDBOX_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||
upload_path = SANDBOX_UPLOAD_DIR / file.filename
|
||
logger.info(f"Sandbox: receiving upload {file.filename}")
|
||
|
||
try:
|
||
with open(upload_path, 'wb') as f:
|
||
while chunk := await file.read(1024 * 1024):
|
||
f.write(chunk)
|
||
except Exception as e:
|
||
raise HTTPException(status_code=400, detail=f"Upload failed: {e}")
|
||
|
||
file_size = upload_path.stat().st_size
|
||
logger.info(f"Sandbox: saved {file.filename} ({file_size / 1024 / 1024:.1f} MB)")
|
||
|
||
# Create import job
|
||
job_id = await db.create_crawl_job(
|
||
target_uri=f"import://{file.filename}",
|
||
keywords=["sandbox", "import"],
|
||
mode="import"
|
||
)
|
||
|
||
# Run import in background
|
||
async def run_import():
|
||
try:
|
||
await db.set_crawl_job_status(job_id, "running")
|
||
stats = await import_archive_to_db(upload_path, job_id)
|
||
await db.complete_crawl_job(job_id, stats)
|
||
upload_path.unlink(missing_ok=True)
|
||
logger.info(f"Sandbox import complete: {stats}")
|
||
except Exception as e:
|
||
logger.error(f"Sandbox import failed: {e}")
|
||
await db.fail_crawl_job(job_id, str(e))
|
||
|
||
task = asyncio.create_task(run_import())
|
||
ACTIVE_CRAWL_TASKS[job_id] = task
|
||
task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))
|
||
|
||
return {"status": "importing", "job_id": job_id, "filename": file.filename, "size": file_size}
|
||
|
||
|
||
async def import_archive_to_db(archive_path: Path, job_id: int) -> dict:
|
||
"""Import archive's database and vault into local database."""
|
||
import sqlite3
|
||
|
||
stats = {"media_imported": 0, "pages_imported": 0, "sources_imported": 0}
|
||
|
||
# Detect .run offset
|
||
offset = 0
|
||
try:
|
||
with open(archive_path, 'rb') as f:
|
||
f.seek(-22, 2)
|
||
trailer = f.read(22)
|
||
if trailer[:6] == b'NEOPIG':
|
||
offset = int(trailer[6:22].decode(), 16)
|
||
except Exception:
|
||
pass
|
||
|
||
# Open tarball
|
||
if offset > 0:
|
||
f = open(archive_path, 'rb')
|
||
f.seek(offset)
|
||
tar = tarfile.open(fileobj=f, mode='r:gz')
|
||
else:
|
||
tar = tarfile.open(archive_path, 'r:gz')
|
||
|
||
try:
|
||
members = tar.getmembers()
|
||
if not members:
|
||
raise ValueError("Empty archive")
|
||
archive_root = members[0].name.split('/')[0]
|
||
|
||
# Extract neopig.db to temp
|
||
db_member = f"{archive_root}/neopig.db"
|
||
temp_db = SANDBOX_UPLOAD_DIR / f"import_{job_id}.db"
|
||
|
||
for m in members:
|
||
if m.name == db_member:
|
||
f_db = tar.extractfile(m)
|
||
if f_db:
|
||
with open(temp_db, 'wb') as out:
|
||
out.write(f_db.read())
|
||
break
|
||
|
||
# Merge database
|
||
if not temp_db.exists():
|
||
raise ValueError("Malformed archive: missing neopig.db")
|
||
|
||
src = sqlite3.connect(temp_db)
|
||
src.row_factory = sqlite3.Row
|
||
try:
|
||
# Check what tables exist in source
|
||
tables = [r[0] for r in src.execute("SELECT name FROM sqlite_master WHERE type='table'")]
|
||
logger.info(f"Import: source database has tables: {tables}")
|
||
|
||
# Validate archive has required tables
|
||
required_tables = {'pages', 'media', 'media_sources'}
|
||
if not required_tables.intersection(tables):
|
||
raise ValueError(f"Malformed archive: neopig.db has no data tables (found: {tables})")
|
||
|
||
# Import media (if table exists)
|
||
if 'media' in tables:
|
||
for row in src.execute("SELECT * FROM media"):
|
||
try:
|
||
async with db.session() as session:
|
||
await session.execute(text(
|
||
"""INSERT OR IGNORE INTO media
|
||
(md5_hash, media_type, mime_type, file_size, keywords, alt_text, title,
|
||
first_seen_at, last_seen_at, score)
|
||
VALUES (:md5_hash, :media_type, :mime_type, :file_size, :keywords, :alt_text, :title,
|
||
:first_seen_at, :last_seen_at, :score)"""), dict(row))
|
||
await session.commit()
|
||
stats["media_imported"] += 1
|
||
except Exception as e:
|
||
logger.debug(f"Skip media row: {e}")
|
||
else:
|
||
logger.warning("Import: source has no 'media' table")
|
||
|
||
# Import media_sources (if table exists)
|
||
if 'media_sources' in tables:
|
||
for row in src.execute("SELECT * FROM media_sources"):
|
||
try:
|
||
async with db.session() as session:
|
||
await session.execute(text(
|
||
"""INSERT OR IGNORE INTO media_sources
|
||
(md5_hash, media_uri, page_uri, page_title, alt_text, searchable_text, discovered_at)
|
||
VALUES (:md5_hash, :media_uri, :page_uri, :page_title, :alt_text, :searchable_text, :discovered_at)"""), dict(row))
|
||
await session.commit()
|
||
stats["sources_imported"] += 1
|
||
except Exception as e:
|
||
logger.debug(f"Skip media_source row: {e}")
|
||
else:
|
||
logger.warning("Import: source has no 'media_sources' table")
|
||
|
||
# Import pages (if table exists)
|
||
if 'pages' in tables:
|
||
for row in src.execute("SELECT * FROM pages"):
|
||
try:
|
||
row_dict = dict(row)
|
||
async with db.session() as session:
|
||
await session.execute(text(
|
||
"""INSERT OR REPLACE INTO pages
|
||
(uri, uri_hash, path, title, description, keywords, content, markdown, raw_html, crawled_at)
|
||
VALUES (:uri, :uri_hash, :path, :title, :description, :keywords, :content, :markdown, :raw_html, :crawled_at)"""), row_dict)
|
||
await session.commit()
|
||
stats["pages_imported"] += 1
|
||
except Exception as e:
|
||
logger.debug(f"Skip page row: {e}")
|
||
else:
|
||
logger.warning("Import: source has no 'pages' table")
|
||
finally:
|
||
src.close()
|
||
temp_db.unlink(missing_ok=True)
|
||
|
||
# Extract vault files
|
||
vault_prefix = f"{archive_root}/vault/"
|
||
for m in members:
|
||
if m.name.startswith(vault_prefix) and m.isfile():
|
||
rel_path = m.name[len(vault_prefix):]
|
||
dest_path = VAULT_PATH / rel_path
|
||
if not dest_path.exists():
|
||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||
f_media = tar.extractfile(m)
|
||
if f_media:
|
||
with open(dest_path, 'wb') as out:
|
||
out.write(f_media.read())
|
||
stats["media_imported"] += 1
|
||
|
||
finally:
|
||
tar.close()
|
||
|
||
return stats
|
||
|
||
|
||
@app.get("/api/sandbox/status")
|
||
async def sandbox_status():
|
||
"""Get current sandbox status."""
|
||
if not SANDBOX_MODE:
|
||
raise HTTPException(status_code=403, detail="Sandbox mode not enabled")
|
||
|
||
# Get import job stats
|
||
stats = await db.get_stats()
|
||
import_jobs = await db.get_crawl_jobs(limit=10)
|
||
imports = [j for j in import_jobs if j.get("mode") == "import"]
|
||
|
||
return {
|
||
"sandbox_mode": True,
|
||
"media_count": stats.get("media", 0),
|
||
"pages_count": stats.get("pages", 0),
|
||
"recent_imports": len(imports)
|
||
}
|
||
|
||
|
||
SANDBOX_CSS = """
|
||
.container { max-width: 800px; margin: 0 auto; }
|
||
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
||
.subtitle { color: #666; margin-bottom: 30px; }
|
||
|
||
.upload-zone {
|
||
border: 3px dashed #333;
|
||
border-radius: 12px;
|
||
padding: 60px 40px;
|
||
text-align: center;
|
||
background: #111;
|
||
cursor: pointer;
|
||
transition: all 0.2s;
|
||
}
|
||
.upload-zone:hover, .upload-zone.drag-over {
|
||
border-color: #ff6b6b;
|
||
background: #1a1a1a;
|
||
}
|
||
.upload-zone input[type="file"] { display: none; }
|
||
.upload-icon { font-size: 48px; margin-bottom: 15px; }
|
||
.upload-text { color: #888; font-size: 16px; }
|
||
.upload-hint { color: #555; font-size: 13px; margin-top: 10px; }
|
||
|
||
.status {
|
||
margin-top: 30px;
|
||
padding: 20px;
|
||
background: #1a1a1a;
|
||
border-radius: 8px;
|
||
border: 1px solid #333;
|
||
}
|
||
.status-row { display: flex; justify-content: space-between; padding: 8px 0; border-bottom: 1px solid #222; }
|
||
.status-row:last-child { border-bottom: none; }
|
||
.status-label { color: #888; }
|
||
.status-value { color: #e0e0e0; font-family: monospace; }
|
||
|
||
.progress { display: none; margin-top: 20px; }
|
||
.progress-bar {
|
||
height: 8px; background: #333; border-radius: 4px; overflow: hidden;
|
||
}
|
||
.progress-fill {
|
||
height: 100%; background: #ff6b6b; width: 0%; transition: width 0.3s;
|
||
}
|
||
.progress-text { text-align: center; margin-top: 10px; color: #888; font-size: 14px; }
|
||
|
||
.success { color: #6f6; }
|
||
.error { color: #f66; }
|
||
"""
|
||
|
||
SANDBOX_CONTENT = """
|
||
<div class="container">
|
||
<h1>🧪 Sandbox Mode</h1>
|
||
<p class="subtitle">Upload a neopig archive (.tar.gz or .run) to explore it</p>
|
||
|
||
<div class="upload-zone" id="upload-zone" onclick="document.getElementById('file-input').click()">
|
||
<div class="upload-icon">📦</div>
|
||
<div class="upload-text">Drop archive here or click to browse</div>
|
||
<div class="upload-hint">Supports .tar.gz, .tgz, and .run files</div>
|
||
<input type="file" id="file-input" accept=".tar.gz,.tgz,.run">
|
||
</div>
|
||
|
||
<div class="progress" id="progress">
|
||
<div class="progress-bar"><div class="progress-fill" id="progress-fill"></div></div>
|
||
<div class="progress-text" id="progress-text">Uploading...</div>
|
||
</div>
|
||
|
||
<div class="status" id="status">
|
||
<h3 style="margin-top:0;color:#ff6b6b;">Current Archive</h3>
|
||
<div id="status-content">Loading...</div>
|
||
</div>
|
||
</div>
|
||
|
||
<script>
|
||
const zone = document.getElementById('upload-zone');
|
||
const fileInput = document.getElementById('file-input');
|
||
const progress = document.getElementById('progress');
|
||
const progressFill = document.getElementById('progress-fill');
|
||
const progressText = document.getElementById('progress-text');
|
||
|
||
// Drag and drop
|
||
zone.addEventListener('dragover', (e) => { e.preventDefault(); zone.classList.add('drag-over'); });
|
||
zone.addEventListener('dragleave', () => zone.classList.remove('drag-over'));
|
||
zone.addEventListener('drop', (e) => {
|
||
e.preventDefault();
|
||
zone.classList.remove('drag-over');
|
||
if (e.dataTransfer.files.length) uploadFile(e.dataTransfer.files[0]);
|
||
});
|
||
|
||
fileInput.addEventListener('change', () => {
|
||
if (fileInput.files.length) uploadFile(fileInput.files[0]);
|
||
});
|
||
|
||
// Resumable upload configuration
|
||
const CHUNK_SIZE = 10 * 1024 * 1024; // 10MB chunks
|
||
const MAX_RETRIES = 10;
|
||
const RETRY_DELAY = 2000; // 2 seconds
|
||
|
||
let currentUpload = null; // {uploadId, file, offset}
|
||
|
||
async function uploadFile(file) {
|
||
progress.style.display = 'block';
|
||
progressFill.style.width = '0%';
|
||
progressText.textContent = `Initializing upload for ${file.name}...`;
|
||
|
||
try {
|
||
// Step 1: Initialize resumable upload
|
||
const initRes = await fetch(`/api/sandbox/upload/init?filename=${encodeURIComponent(file.name)}&size=${file.size}`, {
|
||
method: 'POST'
|
||
});
|
||
if (!initRes.ok) {
|
||
const err = await initRes.json();
|
||
throw new Error(err.detail || 'Failed to initialize upload');
|
||
}
|
||
const init = await initRes.json();
|
||
currentUpload = { uploadId: init.upload_id, file, offset: 0 };
|
||
|
||
// Step 2: Upload chunks with resume capability
|
||
await uploadChunks();
|
||
|
||
} catch (err) {
|
||
progressText.innerHTML = `<span class="error">✗ ${err.message}</span>`;
|
||
}
|
||
}
|
||
|
||
async function uploadChunks() {
|
||
const { uploadId, file } = currentUpload;
|
||
let retries = 0;
|
||
|
||
while (currentUpload.offset < file.size) {
|
||
const start = currentUpload.offset;
|
||
const end = Math.min(start + CHUNK_SIZE, file.size);
|
||
const chunk = file.slice(start, end);
|
||
|
||
try {
|
||
const res = await fetch(`/api/sandbox/upload/${uploadId}`, {
|
||
method: 'PATCH',
|
||
headers: {
|
||
'Content-Type': 'application/octet-stream',
|
||
'X-Upload-Offset': start.toString()
|
||
},
|
||
body: chunk
|
||
});
|
||
|
||
if (!res.ok) {
|
||
if (res.status === 409) {
|
||
// Offset mismatch - get actual position and retry
|
||
const status = await getUploadStatus(uploadId);
|
||
currentUpload.offset = status.bytes_received;
|
||
continue;
|
||
}
|
||
const err = await res.json();
|
||
throw new Error(err.detail || 'Chunk upload failed');
|
||
}
|
||
|
||
const data = await res.json();
|
||
currentUpload.offset = data.bytes_received;
|
||
retries = 0; // Reset retry counter on success
|
||
|
||
// Update progress
|
||
const pct = (data.bytes_received / file.size * 100).toFixed(1);
|
||
progressFill.style.width = pct + '%';
|
||
const mb = (data.bytes_received / 1024 / 1024).toFixed(1);
|
||
const totalMb = (file.size / 1024 / 1024).toFixed(1);
|
||
progressText.textContent = `Uploading ${file.name}... ${pct}% (${mb}/${totalMb} MB)`;
|
||
|
||
} catch (err) {
|
||
retries++;
|
||
if (retries > MAX_RETRIES) {
|
||
progressText.innerHTML = `<span class="error">✗ Upload failed after ${MAX_RETRIES} retries: ${err.message}</span>`;
|
||
return;
|
||
}
|
||
progressText.textContent = `Connection lost, retrying (${retries}/${MAX_RETRIES})...`;
|
||
await new Promise(r => setTimeout(r, RETRY_DELAY));
|
||
|
||
// Get current position from server before retry
|
||
try {
|
||
const status = await getUploadStatus(uploadId);
|
||
currentUpload.offset = status.bytes_received;
|
||
} catch (e) {
|
||
// If status check fails, just retry from last known position
|
||
}
|
||
}
|
||
}
|
||
|
||
// Step 3: Complete upload
|
||
try {
|
||
const completeRes = await fetch(`/api/sandbox/upload/${uploadId}/complete`, { method: 'POST' });
|
||
if (!completeRes.ok) {
|
||
const err = await completeRes.json();
|
||
throw new Error(err.detail || 'Failed to complete upload');
|
||
}
|
||
const result = await completeRes.json();
|
||
progressText.innerHTML = `<span class="success">✓ Archive loaded successfully! Job #${result.job_id}</span>`;
|
||
currentUpload = null;
|
||
loadStatus();
|
||
} catch (err) {
|
||
progressText.innerHTML = `<span class="error">✗ ${err.message}</span>`;
|
||
}
|
||
}
|
||
|
||
async function getUploadStatus(uploadId) {
|
||
const res = await fetch(`/api/sandbox/upload/${uploadId}/status`);
|
||
if (!res.ok) throw new Error('Failed to get upload status');
|
||
return await res.json();
|
||
}
|
||
|
||
// Resume button for interrupted uploads
|
||
async function resumeUpload() {
|
||
if (!currentUpload) return;
|
||
progress.style.display = 'block';
|
||
progressText.textContent = 'Resuming upload...';
|
||
await uploadChunks();
|
||
}
|
||
|
||
async function loadStatus() {
|
||
try {
|
||
const res = await fetch('/api/sandbox/status');
|
||
const data = await res.json();
|
||
const content = document.getElementById('status-content');
|
||
|
||
content.innerHTML = `
|
||
<div class="status-row"><span class="status-label">Media in database</span><span class="status-value">${(data.media_count || 0).toLocaleString()}</span></div>
|
||
<div class="status-row"><span class="status-label">Pages indexed</span><span class="status-value">${(data.pages_count || 0).toLocaleString()}</span></div>
|
||
<div class="status-row"><span class="status-label">Recent imports</span><span class="status-value">${data.recent_imports || 0}</span></div>
|
||
<div class="status-row"><a href="/crawl" style="color:#ff6b6b;">View all jobs →</a></div>
|
||
`;
|
||
} catch (err) {
|
||
document.getElementById('status-content').innerHTML = '<div class="error">Failed to load status</div>';
|
||
}
|
||
}
|
||
|
||
loadStatus();
|
||
</script>
|
||
"""
|
||
|
||
|
||
@app.get("/sandbox", response_class=HTMLResponse)
|
||
async def sandbox_page(lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Sandbox page for uploading archives."""
|
||
if not SANDBOX_MODE:
|
||
raise HTTPException(status_code=403, detail="Sandbox mode not enabled (NEOPIG_SANDBOX=1)")
|
||
language = get_lang(lang, accept_language)
|
||
html = layout("Sandbox", SANDBOX_CONTENT, extra_css=SANDBOX_CSS)
|
||
return inject_i18n(html, language)
|
||
|
||
|
||
# ============================================================================
|
||
# Crawler API
|
||
# ============================================================================
|
||
|
||
@app.get("/api/crawl/jobs")
|
||
async def get_crawl_jobs_endpoint(limit: int = Query(50, le=200)):
|
||
"""Get recent jobs (crawl + backfill)."""
|
||
try:
|
||
# Get both crawl jobs and backfill jobs
|
||
crawl_jobs = await db.get_crawl_jobs(limit)
|
||
backfill_jobs = await db.get_backfill_jobs(limit)
|
||
|
||
# Add job_kind to distinguish them
|
||
for job in crawl_jobs:
|
||
job['job_kind'] = 'crawl'
|
||
for job in backfill_jobs:
|
||
job['job_kind'] = 'backfill'
|
||
|
||
# Merge and sort by started_at descending
|
||
all_jobs = crawl_jobs + backfill_jobs
|
||
all_jobs.sort(key=lambda j: j.get('started_at', ''), reverse=True)
|
||
|
||
return all_jobs[:limit]
|
||
except Exception as e:
|
||
logger.error(f"Failed to get jobs: {e}")
|
||
raise HTTPException(status_code=500, detail=str(e))
|
||
|
||
|
||
@app.post("/api/crawl/jobs/{job_id}/pause")
|
||
async def pause_crawl_job(job_id: int):
|
||
"""Pause a running crawl job."""
|
||
task = ACTIVE_CRAWL_TASKS.get(job_id)
|
||
if not task:
|
||
raise HTTPException(status_code=404, detail="Job not running")
|
||
|
||
task.cancel()
|
||
try:
|
||
await task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
|
||
await db.pause_crawl_job(job_id)
|
||
ACTIVE_CRAWL_TASKS.pop(job_id, None)
|
||
|
||
return {"status": "paused", "job_id": job_id}
|
||
|
||
|
||
@app.post("/api/crawl/jobs/{job_id}/resume")
|
||
async def resume_crawl_job(job_id: int):
|
||
"""Resume a paused crawl job."""
|
||
job = await db.get_crawl_job(job_id)
|
||
if not job:
|
||
raise HTTPException(status_code=404, detail="Job not found")
|
||
|
||
if job['status'] not in ('paused', 'cancelled'):
|
||
raise HTTPException(status_code=400, detail=f"Job is {job['status']}, cannot resume")
|
||
|
||
if job_id in ACTIVE_CRAWL_TASKS:
|
||
raise HTTPException(status_code=400, detail="Job is already running")
|
||
|
||
# Mark as running
|
||
async with db.session() as session:
|
||
await session.execute(
|
||
text("UPDATE crawl_jobs SET status = 'running' WHERE id = :id"),
|
||
{"id": job_id}
|
||
)
|
||
await session.commit()
|
||
|
||
# Start crawl task (resume from state files with original settings)
|
||
target_uri = job['target_uri']
|
||
keywords = json.loads(job.get('keywords') or '[]')
|
||
mode_str = job.get('mode', 'all')
|
||
# Use stored settings (with fallbacks for old jobs)
|
||
job_depth = job.get('depth', 15) or 15
|
||
job_max_pages = job.get('max_pages', -1) if job.get('max_pages') is not None else -1
|
||
job_fast = bool(job.get('fast', 0))
|
||
job_screenshots = bool(job.get('screenshots', 1))
|
||
|
||
async def run_crawl(jid=job_id, uri=target_uri):
|
||
try:
|
||
from neopig import NeoPig
|
||
from async_web_fetcher import CrawlMode
|
||
from screenshot import ScreenshotConfig
|
||
screenshot_config = ScreenshotConfig(enabled=job_screenshots)
|
||
pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH), fast_mode=job_fast, screenshot_config=screenshot_config)
|
||
await pig.init()
|
||
|
||
# Resume from state (don't clear state files)
|
||
crawled_media = await pig.db.get_crawled_media_uris()
|
||
crawled_screenshots = await pig.db.get_crawled_screenshot_uris()
|
||
if crawled_media:
|
||
pig.seen_media = crawled_media
|
||
if crawled_screenshots:
|
||
pig.seen_screenshots = crawled_screenshots
|
||
|
||
mode = CrawlMode(mode_str) if mode_str else CrawlMode.ALL
|
||
|
||
# Progress updater
|
||
stop_progress = asyncio.Event()
|
||
|
||
async def update_progress():
|
||
while not stop_progress.is_set():
|
||
await asyncio.sleep(2)
|
||
if not stop_progress.is_set():
|
||
await db.update_crawl_job_stats(jid, pig.stats)
|
||
|
||
progress_task = asyncio.create_task(update_progress())
|
||
|
||
try:
|
||
stats = await pig.crawl(
|
||
target_uri=uri,
|
||
keywords=keywords,
|
||
mode=mode,
|
||
depth=job_depth,
|
||
max_pages=job_max_pages,
|
||
job_id=jid,
|
||
quiet=True,
|
||
)
|
||
finally:
|
||
stop_progress.set()
|
||
progress_task.cancel()
|
||
try:
|
||
await progress_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
|
||
await db.complete_crawl_job(jid, stats)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Resumed crawl job {jid} failed: {e}")
|
||
await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})
|
||
|
||
task = asyncio.create_task(run_crawl())
|
||
ACTIVE_CRAWL_TASKS[job_id] = task
|
||
task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))
|
||
|
||
return {"status": "running", "job_id": job_id}
|
||
|
||
|
||
@app.delete("/api/crawl/jobs/{job_id}")
|
||
async def delete_crawl_job(job_id: int, purge: bool = Query(True, description="Purge all data (media, screenshots, pages)")):
|
||
"""Delete a crawl job and optionally purge all associated data.
|
||
|
||
With purge=True (default):
|
||
- Deletes all MediaSource records for this job
|
||
- Deletes all Page records for this job
|
||
- Deletes orphan Media records (not referenced by other jobs)
|
||
- Deletes orphan media files from vault
|
||
- Deletes screenshot files for deleted pages
|
||
- Deletes state files
|
||
"""
|
||
# Can't delete running jobs
|
||
if job_id in ACTIVE_CRAWL_TASKS:
|
||
raise HTTPException(status_code=400, detail="Cannot delete running job. Pause it first.")
|
||
|
||
if purge:
|
||
# Full purge using NeoPig
|
||
from neopig import NeoPig
|
||
pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH))
|
||
await pig.init()
|
||
result = await pig.purge_job(job_id)
|
||
if not result['deleted']:
|
||
raise HTTPException(status_code=404, detail="Job not found")
|
||
return {
|
||
"status": "purged",
|
||
"job_id": job_id,
|
||
"media_files_deleted": result['media_files_deleted'],
|
||
"screenshots_deleted": result['screenshots_deleted'],
|
||
"pages_deleted": result['pages_deleted'],
|
||
"state_files_deleted": result['state_files_deleted'],
|
||
}
|
||
else:
|
||
# Just delete job record
|
||
result = await db.delete_crawl_job(job_id, purge_data=False)
|
||
if not result['deleted']:
|
||
raise HTTPException(status_code=404, detail="Job not found")
|
||
return {"status": "deleted", "job_id": job_id}
|
||
|
||
|
||
@app.get("/api/crawl/jobs/{job_id}")
|
||
async def get_crawl_job_endpoint(job_id: int):
|
||
"""Get a specific crawl job."""
|
||
job = await db.get_crawl_job(job_id)
|
||
if not job:
|
||
raise HTTPException(status_code=404, detail="Job not found")
|
||
return job
|
||
|
||
|
||
@app.get("/api/crawl/jobs/{job_id}/logs")
|
||
async def get_crawl_job_logs(job_id: int, tail: int = Query(0, description="Return only last N lines")):
|
||
"""Get logs for a crawl job."""
|
||
logs = get_job_logs(job_id, tail=tail)
|
||
return Response(content=logs, media_type="text/plain")
|
||
|
||
|
||
@app.post("/api/crawl")
|
||
async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
|
||
"""
|
||
Start new crawl job(s).
|
||
|
||
Supports multiple targets - creates one job per target.
|
||
The crawls run in the background. Poll /api/crawl/jobs/{id} for status.
|
||
"""
|
||
if CRAWL_DISABLED:
|
||
raise HTTPException(status_code=403, detail="Crawling is disabled (NEOPIG_DISABLE_CRAWL=1)")
|
||
|
||
# Import neopig here to avoid circular imports
|
||
from neopig import NeoPig
|
||
from async_web_fetcher import CrawlMode
|
||
|
||
# Map mode string to enum
|
||
mode_map = {
|
||
"text": CrawlMode.TEXT,
|
||
"images": CrawlMode.IMAGES,
|
||
"videos": CrawlMode.VIDEOS,
|
||
"media": CrawlMode.MEDIA,
|
||
"all": CrawlMode.ALL,
|
||
}
|
||
mode = mode_map.get(request.mode, CrawlMode.IMAGES)
|
||
|
||
# Get targets (support both new 'targets' array and old 'target_uri' single value)
|
||
targets = request.targets if request.targets else [request.target_uri] if request.target_uri else []
|
||
if not targets:
|
||
raise HTTPException(status_code=400, detail="No target URIs provided")
|
||
|
||
job_ids = []
|
||
|
||
# Create a job for each target (store all settings for resume)
|
||
depth = request.depth if 1 <= request.depth <= 15 else 15
|
||
for target_uri in targets:
|
||
job_id = await db.create_crawl_job(
|
||
target_uri, request.keywords, request.mode,
|
||
depth=depth, max_pages=request.max_pages,
|
||
fast=request.fast, screenshots=request.screenshots
|
||
)
|
||
job_ids.append(job_id)
|
||
|
||
# Run crawl in background - capture ALL values to avoid closure issues
|
||
is_fresh = request.fresh
|
||
is_fast = request.fast
|
||
has_screenshots = request.screenshots
|
||
is_hydra = request.hydra
|
||
crawl_keywords = list(request.keywords) # Copy list
|
||
crawl_max_pages = request.max_pages
|
||
crawl_depth = request.depth if 1 <= request.depth <= 15 else 15
|
||
|
||
async def run_crawl(jid=job_id, uri=target_uri, fresh=is_fresh, fast=is_fast, screenshots=has_screenshots, hydra=is_hydra, depth=crawl_depth, keywords=crawl_keywords, max_pages=crawl_max_pages):
|
||
try:
|
||
from screenshot import ScreenshotConfig
|
||
screenshot_config = ScreenshotConfig(enabled=screenshots)
|
||
pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH), fast_mode=fast, screenshot_config=screenshot_config)
|
||
await pig.init()
|
||
|
||
# Handle fresh vs resume - NEVER fresh on resume
|
||
if fresh:
|
||
pig._clear_state(uri)
|
||
else:
|
||
# Load existing seen media for resume capability
|
||
crawled_media = await pig.db.get_crawled_media_uris()
|
||
crawled_screenshots = await pig.db.get_crawled_screenshot_uris()
|
||
if crawled_media:
|
||
pig.seen_media = crawled_media
|
||
if crawled_screenshots:
|
||
pig.seen_screenshots = crawled_screenshots
|
||
|
||
# Progress updater - runs alongside crawl
|
||
stop_progress = asyncio.Event()
|
||
|
||
async def update_progress():
|
||
while not stop_progress.is_set():
|
||
await asyncio.sleep(2) # Update every 2 seconds
|
||
if not stop_progress.is_set():
|
||
await db.update_crawl_job_stats(jid, pig.stats)
|
||
|
||
progress_task = asyncio.create_task(update_progress())
|
||
|
||
try:
|
||
stats = await pig.crawl(
|
||
target_uri=uri,
|
||
keywords=keywords,
|
||
mode=mode,
|
||
depth=depth,
|
||
max_pages=max_pages,
|
||
job_id=jid, # Use existing job, don't create another
|
||
quiet=True, # No progress bar for UI-initiated crawls
|
||
hydra=hydra, # Feed/sitemap discovery mode
|
||
)
|
||
finally:
|
||
stop_progress.set()
|
||
progress_task.cancel()
|
||
try:
|
||
await progress_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
|
||
# Update job as completed
|
||
await db.complete_crawl_job(jid, stats)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Crawl job {jid} failed: {e}")
|
||
await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})
|
||
|
||
# Schedule async task on the event loop (not BackgroundTasks which runs in threadpool)
|
||
task = asyncio.create_task(run_crawl())
|
||
ACTIVE_CRAWL_TASKS[job_id] = task
|
||
# Clean up when done
|
||
task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))
|
||
|
||
return {"job_ids": job_ids, "status": "running", "count": len(job_ids)}
|
||
|
||
|
||
def init_tarball_mode(tarball_path: str):
|
||
"""Initialize serving from a tar.gz archive."""
|
||
global TAR_PATH, TAR_OFFSET, TAR_MEMBERS, TAR_MEDIA_INDEX, ARCHIVE_ROOT, DB_PATH, TEMP_DB_PATH
|
||
|
||
logger.info(f"Opening archive: {tarball_path}")
|
||
tarball = Path(tarball_path)
|
||
TAR_PATH = str(tarball.resolve())
|
||
|
||
# Handle .run files with NEOPIG trailer
|
||
TAR_OFFSET = 0
|
||
if tarball.suffix == '.run' or tarball.stat().st_size > 100000:
|
||
try:
|
||
with open(tarball, 'rb') as f:
|
||
f.seek(-22, 2)
|
||
trailer = f.read(22)
|
||
if trailer[:6] == b'NEOPIG':
|
||
TAR_OFFSET = int(trailer[6:22].decode(), 16)
|
||
logger.info(f"Detected .run format, tarball offset: {TAR_OFFSET}")
|
||
except Exception:
|
||
pass
|
||
|
||
# Open tarball temporarily to build index
|
||
tar = _open_tarball()
|
||
|
||
# Build member lookup and media index
|
||
for member in tar.getmembers():
|
||
TAR_MEMBERS[member.name] = member
|
||
# Index media files by hash for O(1) lookup
|
||
name = member.name
|
||
if '/vault/' in name or '/media/' in name:
|
||
# Extract hash from filename (hash.ext)
|
||
basename = Path(name).stem # removes extension
|
||
if len(basename) == 32 and all(c in '0123456789abcdef' for c in basename):
|
||
TAR_MEDIA_INDEX[basename] = name
|
||
|
||
logger.info(f"Indexed {len(TAR_MEDIA_INDEX)} media files")
|
||
|
||
# Get archive root from first member
|
||
first = list(TAR_MEMBERS.keys())[0]
|
||
ARCHIVE_ROOT = first.split('/')[0]
|
||
logger.info(f"Archive root: {ARCHIVE_ROOT}")
|
||
|
||
# Extract database to temp (SQLite needs real file)
|
||
# Try neopig.db first (new format), then archive.db (legacy)
|
||
db_member = f"{ARCHIVE_ROOT}/neopig.db"
|
||
if db_member not in TAR_MEMBERS:
|
||
db_member = f"{ARCHIVE_ROOT}/archive.db"
|
||
if db_member in TAR_MEMBERS:
|
||
temp_dir = tempfile.mkdtemp(prefix="neopig_")
|
||
db_name = Path(db_member).name
|
||
TEMP_DB_PATH = f"{temp_dir}/{db_name}"
|
||
member = TAR_MEMBERS[db_member]
|
||
f = tar.extractfile(member)
|
||
if f:
|
||
with open(TEMP_DB_PATH, 'wb') as out:
|
||
out.write(f.read())
|
||
DB_PATH = TEMP_DB_PATH
|
||
logger.info(f"Extracted database to: {TEMP_DB_PATH}")
|
||
else:
|
||
logger.warning("No neopig.db or archive.db found in tarball")
|
||
|
||
# Close the temporary tar handle (requests will open their own)
|
||
tar.close()
|
||
|
||
|
||
def main():
|
||
global DB_PATH, VAULT_PATH
|
||
|
||
parser = argparse.ArgumentParser(description="neopig SERP")
|
||
parser.add_argument("tarball", nargs='?', help="Path to archive.tar.gz or .run file")
|
||
parser.add_argument("--port", type=int, default=31337)
|
||
parser.add_argument("--host", default="127.0.0.1")
|
||
parser.add_argument("--db", default="data/neopig.db")
|
||
parser.add_argument("--vault", default="data/vault")
|
||
|
||
args = parser.parse_args()
|
||
|
||
# Tarball mode
|
||
if args.tarball:
|
||
init_tarball_mode(args.tarball)
|
||
logger.info(f"Starting neopig SERP (archive mode) on {args.host}:{args.port}")
|
||
else:
|
||
DB_PATH = args.db
|
||
VAULT_PATH = Path(args.vault)
|
||
logger.info(f"Starting neopig SERP on {args.host}:{args.port}")
|
||
logger.info(f"Database: {DB_PATH}")
|
||
logger.info(f"Vault: {VAULT_PATH}")
|
||
|
||
uvicorn.run(app, host=args.host, port=args.port)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|