3631 lines
176 KiB
Python
3631 lines
176 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
neopig SERP - Search Engine Results Page
|
||
|
||
Fast image/video search across hydrated metadata.
|
||
Serves files directly from filevault via Caddy with 1GB memory cache.
|
||
|
||
Search across:
|
||
- keywords (crawl tags)
|
||
- alt_text (image alt attributes)
|
||
- title (media titles)
|
||
- analysis_result (Qwen 3 VL descriptions)
|
||
- source_page / source_url (origin)
|
||
|
||
Usage:
|
||
python serp.py --port 8000 --vault ./vault --db neopig.db
|
||
"""
|
||
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import mimetypes
|
||
from pathlib import Path
|
||
from typing import List, Dict, Any, Optional
|
||
|
||
from fastapi import FastAPI, Query, HTTPException, BackgroundTasks, Header, Cookie
|
||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response
|
||
from fastapi.staticfiles import StaticFiles
|
||
from pydantic import BaseModel
|
||
from sqlalchemy import text
|
||
import uvicorn
|
||
|
||
from database import Database, OVER_9000
|
||
from filevault import hash_to_path
|
||
from miniuri import Uri
|
||
|
||
logging.basicConfig(level=logging.INFO)
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# ============================================================================
|
||
# INTERNATIONALIZATION (i18n) - Top 10 Languages
|
||
# ============================================================================
|
||
TRANSLATIONS = {
|
||
"en": {
|
||
"search": "Search", "crawl": "Crawl", "live": "Live Feed", "about": "About",
|
||
"random": "Random", "phantom": "Phantom", "loading": "Loading...",
|
||
"start_crawl": "Start Crawl", "pause": "Pause", "resume": "Resume", "delete": "Delete",
|
||
"url": "URL", "mode": "Mode", "depth": "Depth", "keywords": "Keywords",
|
||
"screenshots": "Screenshots", "fast_mode": "Fast Mode", "fresh_start": "Fresh Start",
|
||
"pages": "pages", "found": "found", "saved": "saved", "dupes": "dupes",
|
||
"running": "running", "completed": "completed", "paused": "paused",
|
||
"no_jobs": "No crawl jobs yet", "started": "Started", "filter": "Filter",
|
||
"all_types": "All types", "images": "Images", "videos": "Videos", "audio": "Audio",
|
||
"everything": "Everything (text + media)", "all_media": "All media (images + videos + audio)",
|
||
"images_only": "Images only", "videos_only": "Videos only", "text_only": "Text only",
|
||
"recent_jobs": "Recent Jobs", "search_placeholder": "Search media...",
|
||
"media": "Media", "pages_label": "Pages", "download": "Download",
|
||
"no_media": "No media found", "no_more": "No more results", "load_error": "Failed to load",
|
||
"copy": "Copy", "copied": "Copied!", "starting": "Starting...", "started_ok": "Started!",
|
||
"error": "Error", "watching": "Watching for new images...", "content": "Content",
|
||
"previous": "Previous", "next": "Next", "over_9000": "Over 9,000+!",
|
||
"loading_stats": "Loading stats...", "screenshot": "Screenshot",
|
||
"valid_uri": "Please enter at least one valid URI", "load_jobs_error": "Failed to load jobs",
|
||
"hydrate_subtitle": "Hydrate media from the web", "live_subtitle": "Watch images appear as they're crawled",
|
||
"total_images": "Total Images", "new_session": "New This Session", "per_minute": "Per Minute",
|
||
"all_domains": "All domains", "download_phantom": "Download Phantom Site",
|
||
"phantom_subtitle": "Export archived pages as a static site with local media", "sources": "sources", "max_pages": "Max Pages", "select_domain": "Select Domain",
|
||
"source_uri": "Source URI", "neopig_uri": "Neopig URI", "source_page": "Source Page", "neopig_page": "Neopig Page", "type_label": "Type", "mime_label": "MIME", "size_label": "Size", "alt_label": "Alt", "keywords_label": "Keywords", "description_label": "Description", "items": "items", "bytes": "bytes",
|
||
"used_on": "Used on", "page": "Page", "discovered": "Discovered",
|
||
},
|
||
"zh": {
|
||
"search": "搜索", "crawl": "爬取", "live": "实时动态", "about": "关于",
|
||
"random": "随机", "phantom": "幻影", "loading": "加载中...",
|
||
"start_crawl": "开始爬取", "pause": "暂停", "resume": "继续", "delete": "删除",
|
||
"url": "网址", "mode": "模式", "depth": "深度", "keywords": "关键词",
|
||
"screenshots": "截图", "fast_mode": "快速模式", "fresh_start": "全新开始",
|
||
"pages": "页面", "found": "发现", "saved": "保存", "dupes": "重复",
|
||
"running": "运行中", "completed": "已完成", "paused": "已暂停",
|
||
"no_jobs": "暂无爬取任务", "started": "开始于", "filter": "筛选",
|
||
"all_types": "所有类型", "images": "图片", "videos": "视频", "audio": "音频",
|
||
"everything": "全部（文本+媒体）", "all_media": "所有媒体（图片+视频+音频）",
|
||
"images_only": "仅图片", "videos_only": "仅视频", "text_only": "仅文本",
|
||
"recent_jobs": "最近任务", "search_placeholder": "搜索媒体...",
|
||
"media": "媒体", "pages_label": "页面", "download": "下载",
|
||
"no_media": "未找到媒体", "no_more": "没有更多结果", "load_error": "加载失败",
|
||
"copy": "复制", "copied": "已复制!", "starting": "启动中...", "started_ok": "已启动!",
|
||
"error": "错误", "watching": "正在监视新图片...", "content": "内容",
|
||
"previous": "上一页", "next": "下一页", "over_9000": "超过9000+!",
|
||
"loading_stats": "加载统计中...", "screenshot": "截图",
|
||
"valid_uri": "请输入至少一个有效的URI", "load_jobs_error": "加载任务失败",
|
||
"hydrate_subtitle": "从网络获取媒体", "live_subtitle": "观看正在爬取的图片",
|
||
"total_images": "总图片数", "new_session": "本次新增", "per_minute": "每分钟",
|
||
"all_domains": "所有域名", "download_phantom": "下载幻影站点",
|
||
"phantom_subtitle": "将存档页面导出为带本地媒体的静态站点", "max_pages": "最大页数", "select_domain": "选择域名", "sources": "fuentes",
|
||
},
|
||
"es": {
|
||
"search": "Buscar", "crawl": "Rastrear", "live": "En Vivo", "about": "Acerca de",
|
||
"random": "Aleatorio", "phantom": "Fantasma", "loading": "Cargando...",
|
||
"start_crawl": "Iniciar Rastreo", "pause": "Pausar", "resume": "Reanudar", "delete": "Eliminar",
|
||
"url": "URL", "mode": "Modo", "depth": "Profundidad", "keywords": "Palabras clave",
|
||
"screenshots": "Capturas", "fast_mode": "Modo Rápido", "fresh_start": "Inicio Limpio",
|
||
"pages": "páginas", "found": "encontradas", "saved": "guardadas", "dupes": "duplicados",
|
||
"running": "ejecutando", "completed": "completado", "paused": "pausado",
|
||
"no_jobs": "Sin tareas de rastreo", "started": "Iniciado", "filter": "Filtrar",
|
||
"all_types": "Todos los tipos", "images": "Imágenes", "videos": "Videos", "audio": "Audio",
|
||
"everything": "Todo (texto + medios)", "all_media": "Todos los medios",
|
||
"images_only": "Solo imágenes", "videos_only": "Solo videos", "text_only": "Solo texto",
|
||
"recent_jobs": "Tareas Recientes", "search_placeholder": "Buscar medios...",
|
||
"media": "Medios", "pages_label": "Páginas", "download": "Descargar",
|
||
"no_media": "No se encontraron medios", "no_more": "No hay más resultados", "load_error": "Error al cargar",
|
||
"copy": "Copiar", "copied": "¡Copiado!", "starting": "Iniciando...", "started_ok": "¡Iniciado!",
|
||
"error": "Error", "watching": "Buscando nuevas imágenes...", "content": "Contenido",
|
||
"previous": "Anterior", "next": "Siguiente", "over_9000": "¡Más de 9,000+!",
|
||
"loading_stats": "Cargando estadísticas...", "screenshot": "Captura",
|
||
"valid_uri": "Ingrese al menos una URI válida", "load_jobs_error": "Error al cargar tareas",
|
||
"hydrate_subtitle": "Obtener medios de la web", "live_subtitle": "Ver imágenes mientras se rastrean",
|
||
"total_images": "Total de imágenes", "new_session": "Nuevas en esta sesión", "per_minute": "Por minuto",
|
||
"all_domains": "Todos los dominios", "download_phantom": "Descargar sitio fantasma",
|
||
"phantom_subtitle": "Exportar páginas archivadas como sitio estático con medios locales", "max_pages": "Máx. páginas", "select_domain": "Seleccionar dominio", "sources": "स्रोत",
|
||
},
|
||
"hi": {
|
||
"search": "खोजें", "crawl": "क्रॉल", "live": "लाइव फ़ीड", "about": "के बारे में",
|
||
"random": "यादृच्छिक", "phantom": "प्रेत", "loading": "लोड हो रहा है...",
|
||
"start_crawl": "क्रॉल शुरू करें", "pause": "रोकें", "resume": "जारी रखें", "delete": "हटाएं",
|
||
"url": "यूआरएल", "mode": "मोड", "depth": "गहराई", "keywords": "कीवर्ड",
|
||
"screenshots": "स्क्रीनशॉट", "fast_mode": "फास्ट मोड", "fresh_start": "नई शुरुआत",
|
||
"pages": "पेज", "found": "मिले", "saved": "सहेजे", "dupes": "डुप्लीकेट",
|
||
"running": "चल रहा है", "completed": "पूर्ण", "paused": "रुका हुआ",
|
||
"no_jobs": "कोई क्रॉल कार्य नहीं", "started": "शुरू", "filter": "फ़िल्टर",
|
||
"all_types": "सभी प्रकार", "images": "चित्र", "videos": "वीडियो", "audio": "ऑडियो",
|
||
"everything": "सब कुछ", "all_media": "सभी मीडिया", "images_only": "केवल चित्र",
|
||
"videos_only": "केवल वीडियो", "text_only": "केवल टेक्स्ट",
|
||
"recent_jobs": "हाल के कार्य", "search_placeholder": "मीडिया खोजें...",
|
||
"media": "मीडिया", "pages_label": "पेज", "download": "डाउनलोड",
|
||
"no_media": "कोई मीडिया नहीं मिला", "no_more": "और कोई परिणाम नहीं", "load_error": "लोड करने में विफल",
|
||
"copy": "कॉपी", "copied": "कॉपी हो गया!", "starting": "शुरू हो रहा...", "started_ok": "शुरू हो गया!",
|
||
"error": "त्रुटि", "watching": "नई छवियों की तलाश...", "content": "सामग्री",
|
||
"previous": "पिछला", "next": "अगला", "over_9000": "9,000+ से अधिक!",
|
||
"loading_stats": "आँकड़े लोड हो रहे...", "screenshot": "स्क्रीनशॉट",
|
||
"valid_uri": "कृपया कम से कम एक वैध URI दर्ज करें", "load_jobs_error": "कार्य लोड करने में विफल",
|
||
"hydrate_subtitle": "वेब से मीडिया प्राप्त करें", "live_subtitle": "क्रॉल होते हुए छवियां देखें",
|
||
"total_images": "कुल छवियां", "new_session": "इस सत्र में नई", "per_minute": "प्रति मिनट",
|
||
"all_domains": "सभी डोमेन", "download_phantom": "प्रेत साइट डाउनलोड करें",
|
||
"phantom_subtitle": "संग्रहित पृष्ठों को स्थानीय मीडिया के साथ स्थैतिक साइट के रूप में निर्यात करें", "max_pages": "अधिकतम पेज", "select_domain": "डोमेन चुनें", "sources": "مصادر",
|
||
},
|
||
"ar": {
|
||
"search": "بحث", "crawl": "زحف", "live": "مباشر", "about": "حول",
|
||
"random": "عشوائي", "phantom": "شبح", "loading": "جاري التحميل...",
|
||
"start_crawl": "بدء الزحف", "pause": "إيقاف", "resume": "استئناف", "delete": "حذف",
|
||
"url": "الرابط", "mode": "الوضع", "depth": "العمق", "keywords": "كلمات مفتاحية",
|
||
"screenshots": "لقطات", "fast_mode": "وضع سريع", "fresh_start": "بداية جديدة",
|
||
"pages": "صفحات", "found": "وجدت", "saved": "حفظت", "dupes": "مكررات",
|
||
"running": "جاري", "completed": "مكتمل", "paused": "متوقف",
|
||
"no_jobs": "لا توجد مهام", "started": "بدأ", "filter": "تصفية",
|
||
"all_types": "جميع الأنواع", "images": "صور", "videos": "فيديو", "audio": "صوت",
|
||
"everything": "الكل", "all_media": "جميع الوسائط", "images_only": "صور فقط",
|
||
"videos_only": "فيديو فقط", "text_only": "نص فقط",
|
||
"recent_jobs": "المهام الأخيرة", "search_placeholder": "بحث في الوسائط...",
|
||
"media": "وسائط", "pages_label": "صفحات", "download": "تحميل",
|
||
"no_media": "لم يتم العثور على وسائط", "no_more": "لا مزيد من النتائج", "load_error": "فشل التحميل",
|
||
"copy": "نسخ", "copied": "تم النسخ!", "starting": "جاري البدء...", "started_ok": "تم البدء!",
|
||
"error": "خطأ", "watching": "مراقبة الصور الجديدة...", "content": "المحتوى",
|
||
"previous": "السابق", "next": "التالي", "over_9000": "أكثر من 9,000+!",
|
||
"loading_stats": "جاري تحميل الإحصائيات...", "screenshot": "لقطة شاشة",
|
||
"valid_uri": "الرجاء إدخال رابط واحد على الأقل", "load_jobs_error": "فشل تحميل المهام",
|
||
"hydrate_subtitle": "جلب الوسائط من الويب", "live_subtitle": "مشاهدة الصور أثناء الزحف",
|
||
"total_images": "إجمالي الصور", "new_session": "جديدة في هذه الجلسة", "per_minute": "في الدقيقة",
|
||
"all_domains": "جميع النطاقات", "download_phantom": "تحميل موقع الشبح",
|
||
"phantom_subtitle": "تصدير الصفحات المؤرشفة كموقع ثابت مع وسائط محلية", "max_pages": "أقصى عدد صفحات", "select_domain": "اختر النطاق", "sources": "fontes",
|
||
},
|
||
"pt": {
|
||
"search": "Pesquisar", "crawl": "Rastrear", "live": "Ao Vivo", "about": "Sobre",
|
||
"random": "Aleatório", "phantom": "Fantasma", "loading": "Carregando...",
|
||
"start_crawl": "Iniciar Rastreio", "pause": "Pausar", "resume": "Retomar", "delete": "Excluir",
|
||
"url": "URL", "mode": "Modo", "depth": "Profundidade", "keywords": "Palavras-chave",
|
||
"screenshots": "Capturas", "fast_mode": "Modo Rápido", "fresh_start": "Novo Início",
|
||
"pages": "páginas", "found": "encontradas", "saved": "salvas", "dupes": "duplicatas",
|
||
"running": "executando", "completed": "concluído", "paused": "pausado",
|
||
"no_jobs": "Sem tarefas", "started": "Iniciado", "filter": "Filtrar",
|
||
"all_types": "Todos os tipos", "images": "Imagens", "videos": "Vídeos", "audio": "Áudio",
|
||
"everything": "Tudo (texto + mídia)", "all_media": "Toda mídia",
|
||
"images_only": "Apenas imagens", "videos_only": "Apenas vídeos", "text_only": "Apenas texto",
|
||
"recent_jobs": "Tarefas Recentes", "search_placeholder": "Pesquisar mídia...",
|
||
"media": "Mídia", "pages_label": "Páginas", "download": "Baixar",
|
||
"no_media": "Nenhuma mídia encontrada", "no_more": "Sem mais resultados", "load_error": "Falha ao carregar",
|
||
"copy": "Copiar", "copied": "Copiado!", "starting": "Iniciando...", "started_ok": "Iniciado!",
|
||
"error": "Erro", "watching": "Observando novas imagens...", "content": "Conteúdo",
|
||
"previous": "Anterior", "next": "Próximo", "over_9000": "Mais de 9.000+!",
|
||
"loading_stats": "Carregando estatísticas...", "screenshot": "Captura de tela",
|
||
"valid_uri": "Insira pelo menos uma URI válida", "load_jobs_error": "Falha ao carregar tarefas",
|
||
"hydrate_subtitle": "Obter mídia da web", "live_subtitle": "Veja imagens aparecerem durante o rastreio",
|
||
"total_images": "Total de imagens", "new_session": "Novas nesta sessão", "per_minute": "Por minuto",
|
||
"all_domains": "Todos os domínios", "download_phantom": "Baixar site fantasma",
|
||
"phantom_subtitle": "Exportar páginas arquivadas como site estático com mídia local", "max_pages": "Máx. páginas", "select_domain": "Selecionar domínio", "sources": "источников",
|
||
},
|
||
"ru": {
|
||
"search": "Поиск", "crawl": "Сканировать", "live": "Прямой эфир", "about": "О нас",
|
||
"random": "Случайное", "phantom": "Призрак", "loading": "Загрузка...",
|
||
"start_crawl": "Начать сканирование", "pause": "Пауза", "resume": "Продолжить", "delete": "Удалить",
|
||
"url": "URL", "mode": "Режим", "depth": "Глубина", "keywords": "Ключевые слова",
|
||
"screenshots": "Скриншоты", "fast_mode": "Быстрый режим", "fresh_start": "Начать заново",
|
||
"pages": "страниц", "found": "найдено", "saved": "сохранено", "dupes": "дубликаты",
|
||
"running": "выполняется", "completed": "завершено", "paused": "приостановлено",
|
||
"no_jobs": "Нет задач", "started": "Начало", "filter": "Фильтр",
|
||
"all_types": "Все типы", "images": "Изображения", "videos": "Видео", "audio": "Аудио",
|
||
"everything": "Всё (текст + медиа)", "all_media": "Все медиа",
|
||
"images_only": "Только изображения", "videos_only": "Только видео", "text_only": "Только текст",
|
||
"recent_jobs": "Недавние задачи", "search_placeholder": "Поиск медиа...",
|
||
"media": "Медиа", "pages_label": "Страницы", "download": "Скачать",
|
||
"no_media": "Медиа не найдено", "no_more": "Больше нет результатов", "load_error": "Ошибка загрузки",
|
||
"copy": "Копировать", "copied": "Скопировано!", "starting": "Запуск...", "started_ok": "Запущено!",
|
||
"error": "Ошибка", "watching": "Отслеживание новых изображений...", "content": "Контент",
|
||
"previous": "Назад", "next": "Далее", "over_9000": "Более 9000+!",
|
||
"loading_stats": "Загрузка статистики...", "screenshot": "Скриншот",
|
||
"valid_uri": "Введите хотя бы один URL", "load_jobs_error": "Не удалось загрузить задачи",
|
||
"hydrate_subtitle": "Получить медиа из интернета", "live_subtitle": "Смотрите появление изображений",
|
||
"total_images": "Всего изображений", "new_session": "Новые в сессии", "per_minute": "В минуту",
|
||
"all_domains": "Все домены", "download_phantom": "Скачать призрачный сайт",
|
||
"phantom_subtitle": "Экспорт архивных страниц как статический сайт с локальными медиа", "max_pages": "Макс. страниц", "select_domain": "Выбрать домен", "sources": "ソース",
|
||
},
|
||
"ja": {
|
||
"search": "検索", "crawl": "クロール", "live": "ライブ", "about": "について",
|
||
"random": "ランダム", "phantom": "ファントム", "loading": "読み込み中...",
|
||
"start_crawl": "クロール開始", "pause": "一時停止", "resume": "再開", "delete": "削除",
|
||
"url": "URL", "mode": "モード", "depth": "深さ", "keywords": "キーワード",
|
||
"screenshots": "スクリーンショット", "fast_mode": "高速モード", "fresh_start": "新規開始",
|
||
"pages": "ページ", "found": "発見", "saved": "保存", "dupes": "重複",
|
||
"running": "実行中", "completed": "完了", "paused": "一時停止中",
|
||
"no_jobs": "ジョブがありません", "started": "開始", "filter": "フィルター",
|
||
"all_types": "すべてのタイプ", "images": "画像", "videos": "動画", "audio": "オーディオ",
|
||
"everything": "すべて（テキスト＋メディア）", "all_media": "すべてのメディア",
|
||
"images_only": "画像のみ", "videos_only": "動画のみ", "text_only": "テキストのみ",
|
||
"recent_jobs": "最近のジョブ", "search_placeholder": "メディアを検索...",
|
||
"media": "メディア", "pages_label": "ページ", "download": "ダウンロード",
|
||
"no_media": "メディアが見つかりません", "no_more": "これ以上の結果はありません", "load_error": "読み込みに失敗",
|
||
"copy": "コピー", "copied": "コピーしました!", "starting": "開始中...", "started_ok": "開始しました!",
|
||
"error": "エラー", "watching": "新しい画像を監視中...", "content": "コンテンツ",
|
||
"previous": "前へ", "next": "次へ", "over_9000": "9000以上!",
|
||
"loading_stats": "統計を読み込み中...", "screenshot": "スクリーンショット",
|
||
"valid_uri": "有効なURIを入力してください", "load_jobs_error": "ジョブの読み込みに失敗",
|
||
"hydrate_subtitle": "ウェブからメディアを取得", "live_subtitle": "クロール中の画像を表示",
|
||
"total_images": "合計画像数", "new_session": "今回のセッション", "per_minute": "毎分",
|
||
"all_domains": "すべてのドメイン", "download_phantom": "ファントムサイトをダウンロード",
|
||
"phantom_subtitle": "アーカイブページをローカルメディア付きの静的サイトとしてエクスポート", "max_pages": "最大ページ数", "select_domain": "ドメインを選択", "sources": "sources", "max_pages": "Max Pages", "select_domain": "Select Domain",
|
||
},
|
||
"fr": {
|
||
"search": "Rechercher", "crawl": "Explorer", "live": "En Direct", "about": "À propos",
|
||
"random": "Aléatoire", "phantom": "Fantôme", "loading": "Chargement...",
|
||
"start_crawl": "Démarrer", "pause": "Pause", "resume": "Reprendre", "delete": "Supprimer",
|
||
"url": "URL", "mode": "Mode", "depth": "Profondeur", "keywords": "Mots-clés",
|
||
"screenshots": "Captures", "fast_mode": "Mode Rapide", "fresh_start": "Nouveau Départ",
|
||
"pages": "pages", "found": "trouvées", "saved": "enregistrées", "dupes": "doublons",
|
||
"running": "en cours", "completed": "terminé", "paused": "en pause",
|
||
"no_jobs": "Aucune tâche", "started": "Démarré", "filter": "Filtrer",
|
||
"all_types": "Tous les types", "images": "Images", "videos": "Vidéos", "audio": "Audio",
|
||
"everything": "Tout (texte + médias)", "all_media": "Tous les médias",
|
||
"images_only": "Images uniquement", "videos_only": "Vidéos uniquement", "text_only": "Texte uniquement",
|
||
"recent_jobs": "Tâches Récentes", "search_placeholder": "Rechercher des médias...",
|
||
"media": "Médias", "pages_label": "Pages", "download": "Télécharger",
|
||
"no_media": "Aucun média trouvé", "no_more": "Plus de résultats", "load_error": "Échec du chargement",
|
||
"copy": "Copier", "copied": "Copié!", "starting": "Démarrage...", "started_ok": "Démarré!",
|
||
"error": "Erreur", "watching": "Surveillance des nouvelles images...", "content": "Contenu",
|
||
"previous": "Précédent", "next": "Suivant", "over_9000": "Plus de 9 000+!",
|
||
"loading_stats": "Chargement des statistiques...", "screenshot": "Capture d'écran",
|
||
"valid_uri": "Veuillez entrer au moins une URI valide", "load_jobs_error": "Échec du chargement des tâches",
|
||
"hydrate_subtitle": "Récupérer les médias du web", "live_subtitle": "Voir les images apparaître pendant l'exploration",
|
||
"total_images": "Total d'images", "new_session": "Nouvelles cette session", "per_minute": "Par minute",
|
||
"all_domains": "Tous les domaines", "download_phantom": "Télécharger le site fantôme",
|
||
"phantom_subtitle": "Exporter les pages archivées en site statique avec médias locaux", "max_pages": "Pages max", "select_domain": "Sélectionner le domaine", "sources": "Quellen",
|
||
},
|
||
"de": {
|
||
"search": "Suchen", "crawl": "Crawlen", "live": "Live-Feed", "about": "Über",
|
||
"random": "Zufällig", "phantom": "Phantom", "loading": "Laden...",
|
||
"start_crawl": "Crawl starten", "pause": "Pause", "resume": "Fortsetzen", "delete": "Löschen",
|
||
"url": "URL", "mode": "Modus", "depth": "Tiefe", "keywords": "Stichwörter",
|
||
"screenshots": "Screenshots", "fast_mode": "Schnellmodus", "fresh_start": "Neustart",
|
||
"pages": "Seiten", "found": "gefunden", "saved": "gespeichert", "dupes": "Duplikate",
|
||
"running": "läuft", "completed": "abgeschlossen", "paused": "pausiert",
|
||
"no_jobs": "Keine Aufgaben", "started": "Gestartet", "filter": "Filter",
|
||
"all_types": "Alle Typen", "images": "Bilder", "videos": "Videos", "audio": "Audio",
|
||
"everything": "Alles (Text + Medien)", "all_media": "Alle Medien",
|
||
"images_only": "Nur Bilder", "videos_only": "Nur Videos", "text_only": "Nur Text",
|
||
"recent_jobs": "Letzte Aufgaben", "search_placeholder": "Medien suchen...",
|
||
"media": "Medien", "pages_label": "Seiten", "download": "Herunterladen",
|
||
"no_media": "Keine Medien gefunden", "no_more": "Keine weiteren Ergebnisse", "load_error": "Laden fehlgeschlagen",
|
||
"copy": "Kopieren", "copied": "Kopiert!", "starting": "Starten...", "started_ok": "Gestartet!",
|
||
"error": "Fehler", "watching": "Überwache neue Bilder...", "content": "Inhalt",
|
||
"previous": "Zurück", "next": "Weiter", "over_9000": "Über 9.000+!",
|
||
"loading_stats": "Lade Statistiken...", "screenshot": "Screenshot",
|
||
"valid_uri": "Mindestens eine gültige URI eingeben", "load_jobs_error": "Aufgaben konnten nicht geladen werden",
|
||
"hydrate_subtitle": "Medien aus dem Web abrufen", "live_subtitle": "Bilder beim Crawlen beobachten",
|
||
"total_images": "Bilder gesamt", "new_session": "Neu in dieser Sitzung", "per_minute": "Pro Minute",
|
||
"all_domains": "Alle Domains", "download_phantom": "Phantom-Seite herunterladen",
|
||
"phantom_subtitle": "Archivierte Seiten als statische Seite mit lokalen Medien exportieren", "max_pages": "Max. Seiten", "select_domain": "Domain auswählen", "sources": "소스",
|
||
},
|
||
"ko": {
|
||
"search": "검색", "crawl": "크롤", "live": "라이브", "about": "정보",
|
||
"random": "랜덤", "phantom": "팬텀", "loading": "로딩 중...",
|
||
"start_crawl": "크롤 시작", "pause": "일시정지", "resume": "재개", "delete": "삭제",
|
||
"url": "URL", "mode": "모드", "depth": "깊이", "keywords": "키워드",
|
||
"screenshots": "스크린샷", "fast_mode": "빠른 모드", "fresh_start": "새로 시작",
|
||
"pages": "페이지", "found": "발견", "saved": "저장", "dupes": "중복",
|
||
"running": "실행 중", "completed": "완료", "paused": "일시정지됨",
|
||
"no_jobs": "작업 없음", "started": "시작됨", "filter": "필터",
|
||
"all_types": "모든 유형", "images": "이미지", "videos": "동영상", "audio": "오디오",
|
||
"everything": "모두", "all_media": "모든 미디어",
|
||
"images_only": "이미지만", "videos_only": "동영상만", "text_only": "텍스트만",
|
||
"recent_jobs": "최근 작업", "search_placeholder": "미디어 검색...",
|
||
"media": "미디어", "pages_label": "페이지", "download": "다운로드",
|
||
"no_media": "미디어를 찾을 수 없음", "no_more": "더 이상 결과 없음", "load_error": "로드 실패",
|
||
"copy": "복사", "copied": "복사됨!", "starting": "시작 중...", "started_ok": "시작됨!",
|
||
"error": "오류", "watching": "새 이미지 감시 중...", "content": "콘텐츠",
|
||
"previous": "이전", "next": "다음", "over_9000": "9,000 이상!",
|
||
"loading_stats": "통계 로드 중...", "screenshot": "스크린샷",
|
||
"valid_uri": "유효한 URI를 하나 이상 입력하세요", "load_jobs_error": "작업 로드 실패",
|
||
"hydrate_subtitle": "웹에서 미디어 가져오기", "live_subtitle": "크롤링되는 이미지 보기",
|
||
"total_images": "전체 이미지", "new_session": "이번 세션 신규", "per_minute": "분당",
|
||
"all_domains": "모든 도메인", "download_phantom": "팬텀 사이트 다운로드",
|
||
"phantom_subtitle": "보관된 페이지를 로컬 미디어가 포함된 정적 사이트로 내보내기", "max_pages": "최대 페이지", "select_domain": "도메인 선택", "sources": "fonti",
|
||
},
|
||
"it": {
|
||
"search": "Cerca", "crawl": "Scansiona", "live": "In Diretta", "about": "Info",
|
||
"random": "Casuale", "phantom": "Fantasma", "loading": "Caricamento...",
|
||
"start_crawl": "Avvia Scansione", "pause": "Pausa", "resume": "Riprendi", "delete": "Elimina",
|
||
"url": "URL", "mode": "Modalità", "depth": "Profondità", "keywords": "Parole chiave",
|
||
"screenshots": "Screenshot", "fast_mode": "Modalità Veloce", "fresh_start": "Nuovo Inizio",
|
||
"pages": "pagine", "found": "trovati", "saved": "salvati", "dupes": "duplicati",
|
||
"running": "in esecuzione", "completed": "completato", "paused": "in pausa",
|
||
"no_jobs": "Nessuna attività", "started": "Avviato", "filter": "Filtra",
|
||
"all_types": "Tutti i tipi", "images": "Immagini", "videos": "Video", "audio": "Audio",
|
||
"everything": "Tutto", "all_media": "Tutti i media",
|
||
"images_only": "Solo immagini", "videos_only": "Solo video", "text_only": "Solo testo",
|
||
"recent_jobs": "Attività Recenti", "search_placeholder": "Cerca media...",
|
||
"media": "Media", "pages_label": "Pagine", "download": "Scarica",
|
||
"no_media": "Nessun media trovato", "no_more": "Nessun altro risultato", "load_error": "Caricamento fallito",
|
||
"copy": "Copia", "copied": "Copiato!", "starting": "Avvio...", "started_ok": "Avviato!",
|
||
"error": "Errore", "watching": "Monitoraggio nuove immagini...", "content": "Contenuto",
|
||
"previous": "Precedente", "next": "Successivo", "over_9000": "Oltre 9.000+!",
|
||
"loading_stats": "Caricamento statistiche...", "screenshot": "Screenshot",
|
||
"valid_uri": "Inserire almeno un URI valido", "load_jobs_error": "Caricamento attività fallito",
|
||
"hydrate_subtitle": "Ottieni media dal web", "live_subtitle": "Guarda le immagini mentre vengono scansionate",
|
||
"total_images": "Immagini totali", "new_session": "Nuove in questa sessione", "per_minute": "Al minuto",
|
||
"all_domains": "Tutti i domini", "download_phantom": "Scarica sito fantasma",
|
||
"phantom_subtitle": "Esporta pagine archiviate come sito statico con media locali", "max_pages": "Max pagine", "select_domain": "Seleziona dominio", "sources": "bronnen",
|
||
},
|
||
"nl": {
|
||
"search": "Zoeken", "crawl": "Crawlen", "live": "Live", "about": "Over",
|
||
"random": "Willekeurig", "phantom": "Fantoom", "loading": "Laden...",
|
||
"start_crawl": "Start Crawl", "pause": "Pauzeer", "resume": "Hervat", "delete": "Verwijder",
|
||
"url": "URL", "mode": "Modus", "depth": "Diepte", "keywords": "Trefwoorden",
|
||
"screenshots": "Schermafbeeldingen", "fast_mode": "Snelle Modus", "fresh_start": "Nieuwe Start",
|
||
"pages": "pagina's", "found": "gevonden", "saved": "opgeslagen", "dupes": "duplicaten",
|
||
"running": "actief", "completed": "voltooid", "paused": "gepauzeerd",
|
||
"no_jobs": "Geen taken", "started": "Gestart", "filter": "Filter",
|
||
"all_types": "Alle typen", "images": "Afbeeldingen", "videos": "Video's", "audio": "Audio",
|
||
"everything": "Alles", "all_media": "Alle media",
|
||
"images_only": "Alleen afbeeldingen", "videos_only": "Alleen video's", "text_only": "Alleen tekst",
|
||
"recent_jobs": "Recente Taken", "search_placeholder": "Zoek media...",
|
||
"media": "Media", "pages_label": "Pagina's", "download": "Downloaden",
|
||
"no_media": "Geen media gevonden", "no_more": "Geen resultaten meer", "load_error": "Laden mislukt",
|
||
"copy": "Kopiëren", "copied": "Gekopieerd!", "starting": "Starten...", "started_ok": "Gestart!",
|
||
"error": "Fout", "watching": "Nieuwe afbeeldingen bekijken...", "content": "Inhoud",
|
||
"previous": "Vorige", "next": "Volgende", "over_9000": "Meer dan 9.000+!",
|
||
"loading_stats": "Statistieken laden...", "screenshot": "Schermafbeelding",
|
||
"valid_uri": "Voer minimaal één geldige URI in", "load_jobs_error": "Taken laden mislukt",
|
||
"hydrate_subtitle": "Media van het web ophalen", "live_subtitle": "Bekijk afbeeldingen terwijl ze worden gecrawld",
|
||
"total_images": "Totaal afbeeldingen", "new_session": "Nieuw deze sessie", "per_minute": "Per minuut",
|
||
"all_domains": "Alle domeinen", "download_phantom": "Fantoomsite downloaden",
|
||
"phantom_subtitle": "Gearchiveerde pagina's exporteren als statische site met lokale media", "max_pages": "Max. pagina's", "select_domain": "Selecteer domein", "sources": "źródeł",
|
||
},
|
||
"pl": {
|
||
"search": "Szukaj", "crawl": "Indeksuj", "live": "Na żywo", "about": "O nas",
|
||
"random": "Losowo", "phantom": "Fantom", "loading": "Ładowanie...",
|
||
"start_crawl": "Rozpocznij", "pause": "Pauza", "resume": "Wznów", "delete": "Usuń",
|
||
"url": "URL", "mode": "Tryb", "depth": "Głębokość", "keywords": "Słowa kluczowe",
|
||
"screenshots": "Zrzuty ekranu", "fast_mode": "Tryb szybki", "fresh_start": "Nowy start",
|
||
"pages": "stron", "found": "znaleziono", "saved": "zapisano", "dupes": "duplikaty",
|
||
"running": "w toku", "completed": "zakończono", "paused": "wstrzymano",
|
||
"no_jobs": "Brak zadań", "started": "Rozpoczęto", "filter": "Filtruj",
|
||
"all_types": "Wszystkie typy", "images": "Obrazy", "videos": "Filmy", "audio": "Audio",
|
||
"everything": "Wszystko", "all_media": "Wszystkie media",
|
||
"images_only": "Tylko obrazy", "videos_only": "Tylko filmy", "text_only": "Tylko tekst",
|
||
"recent_jobs": "Ostatnie Zadania", "search_placeholder": "Szukaj mediów...",
|
||
"media": "Media", "pages_label": "Strony", "download": "Pobierz",
|
||
"no_media": "Nie znaleziono mediów", "no_more": "Brak więcej wyników", "load_error": "Błąd ładowania",
|
||
"copy": "Kopiuj", "copied": "Skopiowano!", "starting": "Uruchamianie...", "started_ok": "Uruchomiono!",
|
||
"error": "Błąd", "watching": "Obserwowanie nowych obrazów...", "content": "Treść",
|
||
"previous": "Poprzedni", "next": "Następny", "over_9000": "Ponad 9 000+!",
|
||
"loading_stats": "Ładowanie statystyk...", "screenshot": "Zrzut ekranu",
|
||
"valid_uri": "Wprowadź przynajmniej jeden URI", "load_jobs_error": "Nie udało się załadować zadań",
|
||
"hydrate_subtitle": "Pobierz media z sieci", "live_subtitle": "Oglądaj obrazy podczas indeksowania",
|
||
"total_images": "Łącznie obrazów", "new_session": "Nowe w tej sesji", "per_minute": "Na minutę",
|
||
"all_domains": "Wszystkie domeny", "download_phantom": "Pobierz stronę fantomową",
|
||
"phantom_subtitle": "Eksportuj zarchiwizowane strony jako statyczną witrynę z lokalnymi mediami", "max_pages": "Maks. stron", "select_domain": "Wybierz domenę", "sources": "kaynaklar",
|
||
},
|
||
"tr": {
|
||
"search": "Ara", "crawl": "Tara", "live": "Canlı", "about": "Hakkında",
|
||
"random": "Rastgele", "phantom": "Hayalet", "loading": "Yükleniyor...",
|
||
"start_crawl": "Taramayı Başlat", "pause": "Duraklat", "resume": "Devam", "delete": "Sil",
|
||
"url": "URL", "mode": "Mod", "depth": "Derinlik", "keywords": "Anahtar kelimeler",
|
||
"screenshots": "Ekran görüntüleri", "fast_mode": "Hızlı Mod", "fresh_start": "Yeni Başlangıç",
|
||
"pages": "sayfa", "found": "bulundu", "saved": "kaydedildi", "dupes": "kopya",
|
||
"running": "çalışıyor", "completed": "tamamlandı", "paused": "duraklatıldı",
|
||
"no_jobs": "Görev yok", "started": "Başladı", "filter": "Filtrele",
|
||
"all_types": "Tüm türler", "images": "Resimler", "videos": "Videolar", "audio": "Ses",
|
||
"everything": "Her şey", "all_media": "Tüm medya",
|
||
"images_only": "Sadece resimler", "videos_only": "Sadece videolar", "text_only": "Sadece metin",
|
||
"recent_jobs": "Son Görevler", "search_placeholder": "Medya ara...",
|
||
"media": "Medya", "pages_label": "Sayfalar", "download": "İndir",
|
||
"no_media": "Medya bulunamadı", "no_more": "Daha fazla sonuç yok", "load_error": "Yükleme başarısız",
|
||
"copy": "Kopyala", "copied": "Kopyalandı!", "starting": "Başlatılıyor...", "started_ok": "Başlatıldı!",
|
||
"error": "Hata", "watching": "Yeni resimler izleniyor...", "content": "İçerik",
|
||
"previous": "Önceki", "next": "Sonraki", "over_9000": "9.000'den fazla+!",
|
||
"loading_stats": "İstatistikler yükleniyor...", "screenshot": "Ekran görüntüsü",
|
||
"valid_uri": "Lütfen en az bir geçerli URI girin", "load_jobs_error": "Görevler yüklenemedi",
|
||
"hydrate_subtitle": "Web'den medya al", "live_subtitle": "Taranan resimleri izle",
|
||
"total_images": "Toplam resim", "new_session": "Bu oturumda yeni", "per_minute": "Dakikada",
|
||
"all_domains": "Tüm alanlar", "download_phantom": "Hayalet siteyi indir",
|
||
"phantom_subtitle": "Arşivlenmiş sayfaları yerel medya ile statik site olarak dışa aktar", "max_pages": "Maks. sayfa", "select_domain": "Alan adı seç", "sources": "nguồn",
|
||
},
|
||
"vi": {
|
||
"search": "Tìm kiếm", "crawl": "Thu thập", "live": "Trực tiếp", "about": "Giới thiệu",
|
||
"random": "Ngẫu nhiên", "phantom": "Bóng ma", "loading": "Đang tải...",
|
||
"start_crawl": "Bắt đầu", "pause": "Tạm dừng", "resume": "Tiếp tục", "delete": "Xóa",
|
||
"url": "URL", "mode": "Chế độ", "depth": "Độ sâu", "keywords": "Từ khóa",
|
||
"screenshots": "Ảnh chụp", "fast_mode": "Chế độ nhanh", "fresh_start": "Bắt đầu mới",
|
||
"pages": "trang", "found": "tìm thấy", "saved": "đã lưu", "dupes": "trùng lặp",
|
||
"running": "đang chạy", "completed": "hoàn thành", "paused": "tạm dừng",
|
||
"no_jobs": "Không có tác vụ", "started": "Đã bắt đầu", "filter": "Lọc",
|
||
"all_types": "Tất cả loại", "images": "Hình ảnh", "videos": "Video", "audio": "Âm thanh",
|
||
"everything": "Tất cả", "all_media": "Tất cả media",
|
||
"images_only": "Chỉ hình ảnh", "videos_only": "Chỉ video", "text_only": "Chỉ văn bản",
|
||
"recent_jobs": "Tác vụ gần đây", "search_placeholder": "Tìm media...",
|
||
"media": "Media", "pages_label": "Trang", "download": "Tải xuống",
|
||
"no_media": "Không tìm thấy media", "no_more": "Không còn kết quả", "load_error": "Tải thất bại",
|
||
"copy": "Sao chép", "copied": "Đã sao chép!", "starting": "Đang khởi động...", "started_ok": "Đã khởi động!",
|
||
"error": "Lỗi", "watching": "Đang theo dõi hình ảnh mới...", "content": "Nội dung",
|
||
"previous": "Trước", "next": "Tiếp", "over_9000": "Hơn 9.000+!",
|
||
"loading_stats": "Đang tải thống kê...", "screenshot": "Ảnh chụp màn hình",
|
||
"valid_uri": "Vui lòng nhập ít nhất một URI hợp lệ", "load_jobs_error": "Không thể tải tác vụ",
|
||
"hydrate_subtitle": "Lấy media từ web", "live_subtitle": "Xem hình ảnh khi đang thu thập",
|
||
"total_images": "Tổng hình ảnh", "new_session": "Mới trong phiên này", "per_minute": "Mỗi phút",
|
||
"all_domains": "Tất cả tên miền", "download_phantom": "Tải trang bóng ma",
|
||
"phantom_subtitle": "Xuất trang lưu trữ dưới dạng trang tĩnh với media cục bộ", "max_pages": "Tối đa trang", "select_domain": "Chọn tên miền", "sources": "แหล่งที่มา",
|
||
},
|
||
"th": {
|
||
"search": "ค้นหา", "crawl": "รวบรวม", "live": "สด", "about": "เกี่ยวกับ",
|
||
"random": "สุ่ม", "phantom": "แฟนทอม", "loading": "กำลังโหลด...",
|
||
"start_crawl": "เริ่มรวบรวม", "pause": "หยุดชั่วคราว", "resume": "ดำเนินต่อ", "delete": "ลบ",
|
||
"url": "URL", "mode": "โหมด", "depth": "ความลึก", "keywords": "คำสำคัญ",
|
||
"screenshots": "ภาพหน้าจอ", "fast_mode": "โหมดเร็ว", "fresh_start": "เริ่มใหม่",
|
||
"pages": "หน้า", "found": "พบ", "saved": "บันทึก", "dupes": "ซ้ำ",
|
||
"running": "กำลังทำงาน", "completed": "เสร็จสิ้น", "paused": "หยุดชั่วคราว",
|
||
"no_jobs": "ไม่มีงาน", "started": "เริ่มแล้ว", "filter": "กรอง",
|
||
"all_types": "ทุกประเภท", "images": "รูปภาพ", "videos": "วิดีโอ", "audio": "เสียง",
|
||
"everything": "ทั้งหมด", "all_media": "สื่อทั้งหมด",
|
||
"images_only": "เฉพาะรูปภาพ", "videos_only": "เฉพาะวิดีโอ", "text_only": "เฉพาะข้อความ",
|
||
"recent_jobs": "งานล่าสุด", "search_placeholder": "ค้นหาสื่อ...",
|
||
"media": "สื่อ", "pages_label": "หน้า", "download": "ดาวน์โหลด",
|
||
"no_media": "ไม่พบสื่อ", "no_more": "ไม่มีผลลัพธ์เพิ่มเติม", "load_error": "โหลดไม่สำเร็จ",
|
||
"copy": "คัดลอก", "copied": "คัดลอกแล้ว!", "starting": "กำลังเริ่ม...", "started_ok": "เริ่มแล้ว!",
|
||
"error": "ข้อผิดพลาด", "watching": "กำลังดูรูปภาพใหม่...", "content": "เนื้อหา",
|
||
"previous": "ก่อนหน้า", "next": "ถัดไป", "over_9000": "มากกว่า 9,000+!",
|
||
"loading_stats": "กำลังโหลดสถิติ...", "screenshot": "ภาพหน้าจอ",
|
||
"valid_uri": "กรุณาใส่ URI ที่ถูกต้องอย่างน้อยหนึ่งรายการ", "load_jobs_error": "ไม่สามารถโหลดงานได้",
|
||
"hydrate_subtitle": "ดึงสื่อจากเว็บ", "live_subtitle": "ดูรูปภาพขณะรวบรวม",
|
||
"total_images": "รูปภาพทั้งหมด", "new_session": "ใหม่ในเซสชันนี้", "per_minute": "ต่อนาที",
|
||
"all_domains": "โดเมนทั้งหมด", "download_phantom": "ดาวน์โหลดเว็บแฟนทอม",
|
||
"phantom_subtitle": "ส่งออกหน้าที่เก็บถาวรเป็นเว็บไซต์แบบคงที่พร้อมสื่อในเครื่อง", "max_pages": "หน้าสูงสุด", "select_domain": "เลือกโดเมน", "sources": "sumber",
|
||
},
|
||
"id": {
|
||
"search": "Cari", "crawl": "Jelajahi", "live": "Langsung", "about": "Tentang",
|
||
"random": "Acak", "phantom": "Hantu", "loading": "Memuat...",
|
||
"start_crawl": "Mulai Jelajah", "pause": "Jeda", "resume": "Lanjutkan", "delete": "Hapus",
|
||
"url": "URL", "mode": "Mode", "depth": "Kedalaman", "keywords": "Kata kunci",
|
||
"screenshots": "Tangkapan layar", "fast_mode": "Mode Cepat", "fresh_start": "Mulai Baru",
|
||
"pages": "halaman", "found": "ditemukan", "saved": "disimpan", "dupes": "duplikat",
|
||
"running": "berjalan", "completed": "selesai", "paused": "dijeda",
|
||
"no_jobs": "Tidak ada tugas", "started": "Dimulai", "filter": "Filter",
|
||
"all_types": "Semua jenis", "images": "Gambar", "videos": "Video", "audio": "Audio",
|
||
"everything": "Semua", "all_media": "Semua media",
|
||
"images_only": "Hanya gambar", "videos_only": "Hanya video", "text_only": "Hanya teks",
|
||
"recent_jobs": "Tugas Terbaru", "search_placeholder": "Cari media...",
|
||
"media": "Media", "pages_label": "Halaman", "download": "Unduh",
|
||
"no_media": "Media tidak ditemukan", "no_more": "Tidak ada hasil lagi", "load_error": "Gagal memuat",
|
||
"copy": "Salin", "copied": "Disalin!", "starting": "Memulai...", "started_ok": "Dimulai!",
|
||
"error": "Kesalahan", "watching": "Memantau gambar baru...", "content": "Konten",
|
||
"previous": "Sebelumnya", "next": "Berikutnya", "over_9000": "Lebih dari 9.000+!",
|
||
"loading_stats": "Memuat statistik...", "screenshot": "Tangkapan layar",
|
||
"valid_uri": "Masukkan setidaknya satu URI yang valid", "load_jobs_error": "Gagal memuat tugas",
|
||
"hydrate_subtitle": "Ambil media dari web", "live_subtitle": "Lihat gambar saat dijelajahi",
|
||
"total_images": "Total gambar", "new_session": "Baru sesi ini", "per_minute": "Per menit",
|
||
"all_domains": "Semua domain", "download_phantom": "Unduh situs hantu",
|
||
"phantom_subtitle": "Ekspor halaman arsip sebagai situs statis dengan media lokal", "max_pages": "Maks. halaman", "select_domain": "Pilih domain", "sources": "джерел",
|
||
},
|
||
"uk": {
|
||
"search": "Пошук", "crawl": "Сканувати", "live": "Наживо", "about": "Про нас",
|
||
"random": "Випадково", "phantom": "Привид", "loading": "Завантаження...",
|
||
"start_crawl": "Почати сканування", "pause": "Пауза", "resume": "Продовжити", "delete": "Видалити",
|
||
"url": "URL", "mode": "Режим", "depth": "Глибина", "keywords": "Ключові слова",
|
||
"screenshots": "Знімки екрану", "fast_mode": "Швидкий режим", "fresh_start": "Новий старт",
|
||
"pages": "сторінок", "found": "знайдено", "saved": "збережено", "dupes": "дублікати",
|
||
"running": "виконується", "completed": "завершено", "paused": "призупинено",
|
||
"no_jobs": "Немає завдань", "started": "Розпочато", "filter": "Фільтр",
|
||
"all_types": "Усі типи", "images": "Зображення", "videos": "Відео", "audio": "Аудіо",
|
||
"everything": "Все", "all_media": "Усі медіа",
|
||
"images_only": "Лише зображення", "videos_only": "Лише відео", "text_only": "Лише текст",
|
||
"recent_jobs": "Останні завдання", "search_placeholder": "Пошук медіа...",
|
||
"media": "Медіа", "pages_label": "Сторінки", "download": "Завантажити",
|
||
"no_media": "Медіа не знайдено", "no_more": "Більше немає результатів", "load_error": "Помилка завантаження",
|
||
"copy": "Копіювати", "copied": "Скопійовано!", "starting": "Запуск...", "started_ok": "Запущено!",
|
||
"error": "Помилка", "watching": "Відстеження нових зображень...", "content": "Вміст",
|
||
"previous": "Попередній", "next": "Наступний", "over_9000": "Понад 9000+!",
|
||
"loading_stats": "Завантаження статистики...", "screenshot": "Знімок екрану",
|
||
"valid_uri": "Введіть хоча б один дійсний URI", "load_jobs_error": "Не вдалося завантажити завдання",
|
||
"hydrate_subtitle": "Отримати медіа з вебу", "live_subtitle": "Дивіться зображення під час сканування",
|
||
"total_images": "Всього зображень", "new_session": "Нові в сесії", "per_minute": "За хвилину",
|
||
"all_domains": "Усі домени", "download_phantom": "Завантажити привид-сайт",
|
||
"phantom_subtitle": "Експортувати архівні сторінки як статичний сайт з локальними медіа",
|
||
},
|
||
"sv": {
|
||
"search": "Sök", "crawl": "Genomsök", "live": "Live", "about": "Om",
|
||
"random": "Slumpmässig", "phantom": "Fantom", "loading": "Laddar...",
|
||
"start_crawl": "Starta genomsökning", "pause": "Pausa", "resume": "Återuppta", "delete": "Ta bort",
|
||
"url": "URL", "mode": "Läge", "depth": "Djup", "keywords": "Nyckelord",
|
||
"screenshots": "Skärmbilder", "fast_mode": "Snabbläge", "fresh_start": "Ny start",
|
||
"pages": "sidor", "found": "hittade", "saved": "sparade", "dupes": "dubbletter",
|
||
"running": "körs", "completed": "slutförd", "paused": "pausad",
|
||
"no_jobs": "Inga jobb", "started": "Startad", "filter": "Filtrera",
|
||
"all_types": "Alla typer", "images": "Bilder", "videos": "Videor", "audio": "Ljud",
|
||
"everything": "Allt", "all_media": "Alla media",
|
||
"images_only": "Endast bilder", "videos_only": "Endast videor", "text_only": "Endast text",
|
||
"recent_jobs": "Senaste Jobb", "search_placeholder": "Sök media...",
|
||
"media": "Media", "pages_label": "Sidor", "download": "Ladda ner",
|
||
"no_media": "Ingen media hittades", "no_more": "Inga fler resultat", "load_error": "Kunde inte ladda",
|
||
"copy": "Kopiera", "copied": "Kopierat!", "starting": "Startar...", "started_ok": "Startad!",
|
||
"error": "Fel", "watching": "Bevakar nya bilder...", "content": "Innehåll",
|
||
"previous": "Föregående", "next": "Nästa", "over_9000": "Över 9 000+!",
|
||
"loading_stats": "Laddar statistik...", "screenshot": "Skärmbild",
|
||
"valid_uri": "Ange minst en giltig URI", "load_jobs_error": "Kunde inte ladda jobb",
|
||
"hydrate_subtitle": "Hämta media från webben", "live_subtitle": "Se bilder medan de genomsöks",
|
||
"total_images": "Totalt bilder", "new_session": "Nya denna session", "per_minute": "Per minut",
|
||
"all_domains": "Alla domäner", "download_phantom": "Ladda ner fantomwebbplats",
|
||
"phantom_subtitle": "Exportera arkiverade sidor som statisk webbplats med lokal media", "max_pages": "Max sidor", "select_domain": "Välj domän", "sources": "källor",
|
||
},
|
||
}
|
||
|
||
def get_lang(lang_cookie: str = None, accept_language: str = None) -> str:
|
||
"""Get language from cookie first, then Accept-Language header."""
|
||
# Cookie takes priority (user's explicit choice)
|
||
if lang_cookie and lang_cookie in TRANSLATIONS:
|
||
return lang_cookie
|
||
# Fall back to Accept-Language header
|
||
if not accept_language:
|
||
return "en"
|
||
# Parse "en-US,en;q=0.9,zh-CN;q=0.8" format
|
||
for part in accept_language.split(','):
|
||
lang = part.split(';')[0].strip().split('-')[0].lower()
|
||
if lang in TRANSLATIONS:
|
||
return lang
|
||
return "en"
|
||
|
||
# Language names for the selector dropdown
|
||
LANG_NAMES = {
|
||
"en": "English", "zh": "中文", "es": "Español", "hi": "हिन्दी", "ar": "العربية",
|
||
"pt": "Português", "ru": "Русский", "ja": "日本語", "fr": "Français", "de": "Deutsch",
|
||
"ko": "한국어", "it": "Italiano", "nl": "Nederlands", "pl": "Polski", "tr": "Türkçe",
|
||
"vi": "Tiếng Việt", "th": "ไทย", "id": "Bahasa", "uk": "Українська", "sv": "Svenska",
|
||
}
|
||
|
||
def t(key: str, lang: str = "en") -> str:
|
||
"""Get translation for key in language."""
|
||
return TRANSLATIONS.get(lang, TRANSLATIONS["en"]).get(key, TRANSLATIONS["en"].get(key, key))
|
||
|
||
# Single source of truth for navigation - uses {{key}} placeholders
|
||
NAV_HTML = '''<div class="nav">
|
||
<a href="/" class="brand">🐷 neopig</a>
|
||
<a href="/">{{search}}</a>
|
||
<a href="/live">{{live}}</a>
|
||
<a href="/random">{{random}}</a>
|
||
<a href="/crawl">{{crawl}}</a>
|
||
<a href="/phantom">{{phantom}}</a>
|
||
<a href="/about">{{about}}</a>
|
||
</div>'''
|
||
|
||
def inject_i18n(html: str, lang: str) -> str:
|
||
"""Replace {key} placeholders and inject JS translations + language selector."""
|
||
trans = TRANSLATIONS.get(lang, TRANSLATIONS["en"])
|
||
# Add lang attribute and inject JS translations
|
||
html = html.replace('<html>', f'<html lang="{lang}">')
|
||
# Replace nav placeholder with actual nav
|
||
html = html.replace('<!-- NAV -->', NAV_HTML)
|
||
# Build language selector options
|
||
lang_options = ''.join(f'<option value="{code}"{" selected" if code == lang else ""}>{name}</option>'
|
||
for code, name in LANG_NAMES.items())
|
||
lang_selector = f'''<select id="lang-select" onchange="setLang(this.value)" style="background:#1a1a1a;color:#888;border:1px solid #333;border-radius:4px;padding:4px 8px;font-size:12px;cursor:pointer;">{lang_options}</select>'''
|
||
lang_js = '''
|
||
function setLang(code) {
|
||
localStorage.setItem('neopig_lang', code);
|
||
document.cookie = 'lang=' + code + ';path=/;max-age=31536000';
|
||
location.reload();
|
||
}
|
||
'''
|
||
html = html.replace('<script>', f'<script>\nconst T={json.dumps(trans)};\n{lang_js}', 1)
|
||
# Insert language selector after {{about}} link in nav
|
||
html = html.replace('{{about}}</a>\n</div>', '{{about}}</a>\n ' + lang_selector + '\n</div>')
|
||
# Replace all {key} and {{key}} placeholders
|
||
for key, value in trans.items():
|
||
html = html.replace('{{' + key + '}}', value) # Double braces (static templates)
|
||
html = html.replace('{' + key + '}', value) # Single braces (after f-string)
|
||
return html
|
||
|
||
app = FastAPI(title="neopig", description="Media crawler + SERP + Screenshot service")
|
||
|
||
# Try to include uri2png screenshot router (optional dependency)
|
||
try:
|
||
from uri2png import get_screenshot_router
|
||
app.include_router(get_screenshot_router())
|
||
logger.info("Screenshot router loaded from uri2png")
|
||
except ImportError:
|
||
logger.warning("uri2png not installed - screenshot endpoints not available")
|
||
|
||
# Config - set via startup
|
||
DB_PATH = "data/neopig.db"
|
||
VAULT_PATH = Path("data/vault")
|
||
|
||
# Global database instance
|
||
db: Database = None
|
||
|
||
# Active crawl tasks (for cancellation on shutdown)
|
||
ACTIVE_CRAWL_TASKS: Dict[int, asyncio.Task] = {}
|
||
|
||
# Tarball mode - serve directly from tar.gz archive
|
||
import tarfile
|
||
import tempfile
|
||
TAR_FILE: tarfile.TarFile = None
|
||
TAR_MEMBERS: Dict[str, tarfile.TarInfo] = {}
|
||
ARCHIVE_ROOT: str = None # e.g., "example.com-20251230"
|
||
TEMP_DB_PATH: str = None # Extracted database (SQLite needs real file)
|
||
|
||
|
||
# Base CSS shared by all pages
|
||
BASE_CSS = """
|
||
* { box-sizing: border-box; }
|
||
body {
|
||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||
margin: 0; padding: 0;
|
||
background: #0a0a0a; color: #e0e0e0;
|
||
}
|
||
.nav {
|
||
background: #1a1a1a; padding: 10px 20px;
|
||
display: flex; gap: 20px; align-items: center;
|
||
border-bottom: 1px solid #333;
|
||
}
|
||
.nav a { color: #ff6b6b; text-decoration: none; }
|
||
.nav a:hover { text-decoration: underline; }
|
||
.nav .brand { font-weight: bold; font-size: 18px; }
|
||
.container { padding: 20px; }
|
||
a { color: #ff6b6b; }
|
||
input[type="text"], select {
|
||
padding: 12px 16px; font-size: 16px;
|
||
border: 2px solid #333; border-radius: 8px;
|
||
background: #1a1a1a; color: #fff;
|
||
}
|
||
input[type="text"]:focus { outline: none; border-color: #ff6b6b; }
|
||
button {
|
||
padding: 12px 24px; font-size: 16px;
|
||
background: #ff6b6b; color: #fff;
|
||
border: none; border-radius: 8px; cursor: pointer;
|
||
}
|
||
button:hover { background: #ff5252; }
|
||
.search-box { display: flex; gap: 10px; margin-bottom: 20px; }
|
||
"""
|
||
|
||
def layout(title: str, content: str, extra_css: str = "", extra_head: str = "") -> str:
|
||
"""Build a complete HTML page with consistent layout.
|
||
|
||
Args:
|
||
title: Page title (will be appended with " - neopig")
|
||
content: HTML content for the page body
|
||
extra_css: Additional CSS to include
|
||
extra_head: Additional head elements (scripts, links, etc.)
|
||
|
||
Returns:
|
||
Complete HTML document string
|
||
"""
|
||
return f"""<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>{title} - neopig</title>
|
||
<style>
|
||
{BASE_CSS}
|
||
{extra_css}
|
||
</style>
|
||
{extra_head}
|
||
</head>
|
||
<body>
|
||
<!-- NAV -->
|
||
{content}
|
||
<script>
|
||
</script>
|
||
</body>
|
||
</html>"""
|
||
|
||
# Extended CSS for view/detail pages (adds to BASE_CSS)
|
||
VIEW_CSS = """
|
||
h1 { color: #ff6b6b; font-size: 20px; margin: 0 0 10px 0; }
|
||
h3 { color: #ff6b6b; font-size: 16px; margin: 30px 0 15px 0; border-bottom: 1px solid #333; padding-bottom: 8px; }
|
||
.meta { background: #1a1a1a; padding: 15px; border-radius: 8px; margin: 15px 0; }
|
||
.meta-row { display: flex; margin: 8px 0; }
|
||
.meta-label { width: 100px; color: #888; font-size: 13px; }
|
||
.meta-value { flex: 1; word-break: break-all; font-size: 13px; }
|
||
.meta-value a { color: #4ade80; }
|
||
.hero { text-align: center; margin-bottom: 20px; }
|
||
.hero img, .hero video { max-width: 100%; max-height: 60vh; border-radius: 8px; }
|
||
.content-rendered {
|
||
background: #1a1a1a; padding: 20px; border-radius: 8px;
|
||
line-height: 1.7; font-size: 14px; color: #ccc;
|
||
}
|
||
.content-rendered img { max-width: 100%; height: auto; margin: 10px 0; }
|
||
.content-rendered img.avatar {
|
||
display: inline-block; vertical-align: middle;
|
||
width: 40px; height: 40px; border-radius: 50%;
|
||
margin: 0 10px 0 0; object-fit: cover;
|
||
}
|
||
.content-rendered img.emoji,
|
||
.content-rendered img[alt^=":"][alt$=":"],
|
||
.content-rendered img[src*="emoji"] {
|
||
display: inline; width: 20px; height: 20px;
|
||
max-width: 20px; margin: 0 2px; vertical-align: text-bottom;
|
||
}
|
||
.content-rendered a { color: #ff6b6b; }
|
||
.content-rendered pre, .content-rendered code {
|
||
background: #252525; padding: 2px 6px;
|
||
border-radius: 4px; font-family: monospace; font-size: 13px;
|
||
}
|
||
.content-rendered pre {
|
||
padding: 15px; display: block;
|
||
white-space: pre-wrap; overflow-x: auto;
|
||
}
|
||
.content-rendered blockquote {
|
||
background: #151520; border-left: 3px solid #4a9eff;
|
||
padding: 12px 16px; margin: 16px 0;
|
||
border-radius: 0 6px 6px 0; color: #aaa; font-style: italic;
|
||
}
|
||
.content-rendered hr { border: none; border-top: 1px solid #333; margin: 24px 0; }
|
||
.content-rendered p { margin: 0 0 16px 0; line-height: 1.7; }
|
||
.content-rendered h1, .content-rendered h2, .content-rendered h3, .content-rendered h4, .content-rendered h5, .content-rendered h6 {
|
||
display: block; color: #ff6b6b; margin-top: 28px; margin-bottom: 12px;
|
||
}
|
||
.content-rendered h1 { font-size: 1.8em; }
|
||
.content-rendered h2 { font-size: 1.5em; }
|
||
.content-rendered h3 { font-size: 1.25em; }
|
||
.media-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(auto-fill, minmax(150px, 1fr));
|
||
gap: 10px; margin-top: 15px;
|
||
}
|
||
.media-card {
|
||
background: #1a1a1a; border-radius: 8px;
|
||
overflow: hidden; display: block; transition: transform 0.2s;
|
||
}
|
||
.media-card:hover { transform: scale(1.02); }
|
||
.media-card img, .media-card video {
|
||
width: 100%; height: 120px;
|
||
object-fit: contain; background: #0a0a0a;
|
||
}
|
||
.tag { background: #333; padding: 2px 8px; border-radius: 4px; font-size: 12px; margin-right: 5px; }
|
||
"""
|
||
|
||
|
||
def render_detail_page(
|
||
title: str,
|
||
hero_html: str,
|
||
meta_rows: list,
|
||
media_items: list,
|
||
content_html: str,
|
||
screenshot_hashes: list,
|
||
source_domain: str,
|
||
page_uri: str,
|
||
download_btn_html: str = "",
|
||
noai: bool = False,
|
||
lang: str = "en",
|
||
sources_html: str = "",
|
||
) -> str:
|
||
"""
|
||
Shared template for media view and page view.
|
||
|
||
Args:
|
||
title: Page title
|
||
hero_html: HTML for hero section (media element or empty)
|
||
meta_rows: List of (label, value_html) tuples for metadata
|
||
media_items: List of media dicts with md5_hash, media_type
|
||
content_html: Rendered markdown/content HTML
|
||
screenshot_hashes: List of screenshot md5 hashes
|
||
source_domain: Domain for AI prompt
|
||
page_uri: Page URI for AI prompt
|
||
download_btn_html: Optional download button HTML
|
||
noai: Disable AI assistant
|
||
sources_html: Optional HTML for "Used on X pages" section (reverse image search)
|
||
"""
|
||
import html as html_module
|
||
|
||
# Build metadata section
|
||
meta_html = ''.join(
|
||
f'<div class="meta-row"><span class="meta-label">{label}:</span><span class="meta-value">{value}</span></div>'
|
||
for label, value in meta_rows
|
||
)
|
||
|
||
# Build media gallery
|
||
media_grid = ""
|
||
if media_items:
|
||
media_cards = []
|
||
for m in media_items:
|
||
is_video = m.get("media_type") == "video"
|
||
if is_video:
|
||
el = f'<video src="/media/{m["md5_hash"]}" muted loop preload="metadata" onmouseenter="this.play()" onmouseleave="this.pause()"></video>'
|
||
else:
|
||
el = f'<img src="/media/{m["md5_hash"]}" loading="lazy">'
|
||
media_cards.append(f'''<a href="/view/{m["md5_hash"]}" class="media-card">{el}</a>''')
|
||
media_count = len(media_items)
|
||
media_grid = '''
|
||
<div class="page-media">
|
||
<h3>{{media}} (''' + str(media_count) + ''')</h3>
|
||
<div class="media-grid">''' + ''.join(media_cards) + '''</div>
|
||
</div>'''
|
||
|
||
escaped_title = html_module.escape(title)
|
||
escaped_uri = html_module.escape(page_uri or "")
|
||
|
||
html = f"""<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>{escaped_title} - neopig</title>
|
||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css">
|
||
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
|
||
<style>
|
||
{BASE_CSS}
|
||
{VIEW_CSS}
|
||
.screenshot-stack {{ border-radius: 8px; }}
|
||
.screenshot-stack img {{ border-radius: 0; }}
|
||
.screenshot-stack img:first-child {{ border-radius: 8px 8px 0 0; }}
|
||
.screenshot-stack img:last-child {{ border-radius: 0 0 8px 8px; }}
|
||
.screenshot-stack img:only-child {{ border-radius: 8px; }}
|
||
#screenshot-container.collapsed {{ display: none; }}
|
||
.hero-grid {{ display: grid; grid-template-columns: 1fr; gap: 20px; margin-bottom: 20px; }}
|
||
.hero-grid.has-hero {{ grid-template-columns: 1fr 1fr; }}
|
||
.content-grid {{ display: grid; grid-template-columns: 1fr; gap: 20px; margin-top: 20px; }}
|
||
.content-grid.has-screenshots {{ grid-template-columns: 1fr 1fr; }}
|
||
.content-grid.ss-collapsed {{ grid-template-columns: 1fr 20px; }}
|
||
.content-grid.ss-collapsed .screenshot-col h3 {{ writing-mode: vertical-rl; text-orientation: mixed; margin: 0; font-size: 12px; }}
|
||
.content-rendered img.emoji {{ display: inline; width: 20px; height: 20px; margin: 0 2px; vertical-align: text-bottom; }}
|
||
.content-rendered .post-block {{ display: grid; grid-template-columns: 48px 1fr; gap: 12px; padding: 16px 0; border-bottom: 1px solid #252530; }}
|
||
.content-rendered .post-block:last-child {{ border-bottom: none; }}
|
||
.content-rendered .post-avatar-col {{ display: flex; flex-direction: column; align-items: center; }}
|
||
.content-rendered .post-avatar {{ width: 40px; height: 40px; border-radius: 50%; object-fit: cover; }}
|
||
.content-rendered .post-avatar-placeholder {{ width: 40px; height: 40px; border-radius: 50%; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: #fff; display: flex; align-items: center; justify-content: center; font-weight: bold; font-size: 18px; }}
|
||
.content-rendered .post-content {{ min-width: 0; }}
|
||
.content-rendered pre {{ position: relative; }}
|
||
.content-rendered pre .code-actions {{ position: absolute; top: 8px; right: 8px; display: flex; gap: 5px; }}
|
||
.content-rendered pre .code-actions button {{ background: #444; border: none; color: #ccc; padding: 4px 8px; border-radius: 4px; cursor: pointer; font-size: 11px; }}
|
||
.content-rendered pre .code-actions button:hover {{ background: #555; }}
|
||
.content-rendered pre .code-actions button.copied {{ background: #4ade80; color: #000; }}
|
||
.sources-section {{ margin-top: 40px; padding-top: 20px; border-top: 1px solid #333; }}
|
||
.sources-section h3 {{ cursor: pointer; user-select: none; }}
|
||
.sources-section h3:hover {{ color: #ff6b6b; }}
|
||
.sources-list {{ display: none; margin-top: 15px; }}
|
||
.sources-list.expanded {{ display: block; }}
|
||
.sources-list table {{ width: 100%; border-collapse: collapse; font-size: 13px; }}
|
||
.sources-list th, .sources-list td {{ padding: 8px 12px; text-align: left; border-bottom: 1px solid #333; }}
|
||
.sources-list th {{ background: #1a1a1a; color: #888; font-weight: normal; }}
|
||
.sources-list td a {{ color: #6ea8fe; }}
|
||
.sources-list td a:hover {{ color: #ff6b6b; }}
|
||
.sources-list .source-thumb {{ width: 40px; height: 40px; object-fit: cover; border-radius: 4px; }}
|
||
@media (max-width: 768px) {{ .hero-grid, .hero-grid.has-hero, .content-grid, .content-grid.has-screenshots {{ grid-template-columns: 1fr; }} }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<!-- NAV -->
|
||
<div class="container">
|
||
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
||
<input type="text" name="q" placeholder="{{search_placeholder}}" style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;">
|
||
<button type="submit" style="padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">{{search}}</button>
|
||
</form>
|
||
<div class="hero-grid{' has-hero' if hero_html else ''}">
|
||
{f'<div class="hero-left"><div style="background:#111;border-radius:12px;padding:20px;text-align:center;">{hero_html}</div></div>' if hero_html else ''}
|
||
<div class="hero-right" style="background:#151515;border-radius:12px;padding:20px;">
|
||
<h1 style="margin:0 0 15px 0;font-size:1.3em;">{escaped_title}</h1>
|
||
<div class="meta">{meta_html}</div>
|
||
{download_btn_html}
|
||
</div>
|
||
</div>
|
||
{media_grid}
|
||
<div id="content-grid" class="content-grid{' has-screenshots' if screenshot_hashes else ''}">
|
||
<div class="content-col">
|
||
{'<h3>{{content}}</h3><div class="content-rendered">' + content_html + '</div>' if content_html else ''}
|
||
</div>
|
||
{('<div class="screenshot-col"><h3 style="cursor:pointer;" onclick="toggleScreenshot()"><span id="ss-toggle">▼</span> {{screenshot}}</h3><div id="screenshot-container" class="screenshot-stack">' + ''.join('<img src="/media/' + h + '" alt="Screenshot" style="width:100%;display:block;">' for h in screenshot_hashes) + '</div></div>') if screenshot_hashes else ''}
|
||
</div>
|
||
<script>
|
||
hljs.highlightAll();
|
||
|
||
// Screenshot toggle
|
||
function toggleScreenshot() {{
|
||
const container = document.getElementById('screenshot-container');
|
||
const toggle = document.getElementById('ss-toggle');
|
||
const grid = document.getElementById('content-grid');
|
||
if (container) {{
|
||
const collapsed = !container.classList.contains('collapsed');
|
||
container.classList.toggle('collapsed', collapsed);
|
||
grid.classList.toggle('ss-collapsed', collapsed);
|
||
toggle.textContent = collapsed ? '▶' : '▼';
|
||
localStorage.setItem('neopig_ss_collapsed', collapsed);
|
||
}}
|
||
}}
|
||
if (localStorage.getItem('neopig_ss_collapsed') === 'true') {{
|
||
const container = document.getElementById('screenshot-container');
|
||
const toggle = document.getElementById('ss-toggle');
|
||
const grid = document.getElementById('content-grid');
|
||
if (container) {{
|
||
container.classList.add('collapsed');
|
||
grid.classList.add('ss-collapsed');
|
||
toggle.textContent = '▶';
|
||
}}
|
||
}}
|
||
|
||
// Auto-detect emojis/avatars
|
||
document.querySelectorAll('.content-rendered img').forEach(img => {{
|
||
const check = () => {{
|
||
const w = img.naturalWidth || img.width, h = img.naturalHeight || img.height;
|
||
if (w > 0 && h > 0) {{
|
||
if (w <= 24 && h <= 24) img.classList.add('emoji');
|
||
else if (w <= 60 && h <= 60) img.classList.add('avatar');
|
||
}}
|
||
}};
|
||
if (img.complete) check(); else img.onload = check;
|
||
}});
|
||
|
||
// Code block copy/download
|
||
const extMap = {{'python':'py','javascript':'js','typescript':'ts','cpp':'cpp','c':'c','java':'java','rust':'rs','go':'go','ruby':'rb','php':'php','html':'html','css':'css','json':'json','yaml':'yaml','sql':'sql','bash':'sh','sh':'sh','markdown':'md'}};
|
||
document.querySelectorAll('.content-rendered pre').forEach((pre, idx) => {{
|
||
const code = pre.querySelector('code') || pre;
|
||
const text = code.textContent;
|
||
let ext = 'txt';
|
||
(code.className || '').split(/\\s+/).forEach(cls => {{
|
||
const m = cls.match(/^(?:language-)?(.+)$/);
|
||
if (m && extMap[m[1].toLowerCase()]) ext = extMap[m[1].toLowerCase()];
|
||
}});
|
||
const actions = document.createElement('div');
|
||
actions.className = 'code-actions';
|
||
const copyBtn = document.createElement('button');
|
||
copyBtn.textContent = T.copy;
|
||
copyBtn.onclick = async () => {{
|
||
await navigator.clipboard.writeText(text);
|
||
copyBtn.textContent = T.copied; copyBtn.classList.add('copied');
|
||
setTimeout(() => {{ copyBtn.textContent = T.copy; copyBtn.classList.remove('copied'); }}, 2000);
|
||
}};
|
||
const dlBtn = document.createElement('button');
|
||
dlBtn.textContent = T.download;
|
||
dlBtn.onclick = () => {{
|
||
const blob = new Blob([text], {{type: 'text/plain'}});
|
||
const a = document.createElement('a');
|
||
a.href = URL.createObjectURL(blob);
|
||
a.download = `code-${{idx + 1}}.${{ext}}`;
|
||
a.click();
|
||
}};
|
||
actions.appendChild(copyBtn); actions.appendChild(dlBtn);
|
||
pre.appendChild(actions);
|
||
}});
|
||
|
||
// Forum post restructuring
|
||
(function() {{
|
||
const container = document.querySelector('.content-rendered');
|
||
if (!container) return;
|
||
const postStarts = [];
|
||
container.querySelectorAll('p').forEach(p => {{
|
||
const img = p.querySelector('img[alt="avatar"]');
|
||
const strong = p.querySelector('strong');
|
||
if (img && strong) postStarts.push({{p, img, username: strong.textContent}});
|
||
}});
|
||
if (!postStarts.length) return;
|
||
postStarts.forEach(({{p, img, username}}, idx) => {{
|
||
if (p.closest('.post-block')) return;
|
||
const block = document.createElement('div');
|
||
block.className = 'post-block';
|
||
const avatarCol = document.createElement('div');
|
||
avatarCol.className = 'post-avatar-col';
|
||
if (img.src && !img.src.includes('undefined')) {{
|
||
const avatar = document.createElement('img');
|
||
avatar.className = 'post-avatar';
|
||
avatar.src = img.src;
|
||
avatarCol.appendChild(avatar);
|
||
}} else {{
|
||
const ph = document.createElement('div');
|
||
ph.className = 'post-avatar-placeholder';
|
||
ph.textContent = username.charAt(0).toUpperCase();
|
||
avatarCol.appendChild(ph);
|
||
}}
|
||
const contentCol = document.createElement('div');
|
||
contentCol.className = 'post-content';
|
||
contentCol.innerHTML = `<div class="post-header"><strong>${{username}}</strong></div>`;
|
||
let sib = p.nextElementSibling;
|
||
const nextP = idx < postStarts.length - 1 ? postStarts[idx + 1].p : null;
|
||
while (sib && sib !== nextP && sib.tagName !== 'HR') {{
|
||
contentCol.appendChild(sib.cloneNode(true));
|
||
const rm = sib; sib = sib.nextElementSibling; rm.remove();
|
||
}}
|
||
if (sib && sib.tagName === 'HR') sib.remove();
|
||
block.appendChild(avatarCol); block.appendChild(contentCol);
|
||
p.parentNode.insertBefore(block, p); p.remove();
|
||
}});
|
||
}})();
|
||
</script>
|
||
{sources_html}
|
||
{'' if noai else f'''<script>
|
||
window.UNCLOSEAI_SYSTEM_PROMPT = "You are a neopig assistant, a machine learning persona built into the neopig web archiver. You are viewing an archived copy of a page from {source_domain}. The page title is: {escaped_title}. neopig archived this page - neopig did NOT create the original content. The original content was created by {source_domain}. Help users understand what is on this archived page. Be factual and concise. Only describe what you can actually see - do not invent or embellish.";
|
||
</script>
|
||
<script src="https://uncloseai.com/uncloseai.js" type="module"></script>'''}
|
||
</body>
|
||
</html>"""
|
||
return inject_i18n(html, lang)
|
||
|
||
|
||
def read_from_tarball(path: str) -> bytes:
|
||
"""Read a file from the tarball. Path is relative to archive root."""
|
||
if not TAR_FILE or not ARCHIVE_ROOT:
|
||
return None
|
||
full_path = f"{ARCHIVE_ROOT}/{path}"
|
||
if full_path in TAR_MEMBERS:
|
||
member = TAR_MEMBERS[full_path]
|
||
f = TAR_FILE.extractfile(member)
|
||
if f:
|
||
return f.read()
|
||
return None
|
||
|
||
|
||
def find_media_in_tarball(md5_hash: str) -> tuple:
|
||
"""Find media file in tarball by hash. Returns (data, extension) or (None, None)."""
|
||
if not TAR_FILE or not ARCHIVE_ROOT:
|
||
return None, None
|
||
prefix = f"{ARCHIVE_ROOT}/media/{md5_hash}"
|
||
for name, member in TAR_MEMBERS.items():
|
||
if name.startswith(prefix):
|
||
f = TAR_FILE.extractfile(member)
|
||
if f:
|
||
ext = Path(name).suffix
|
||
return f.read(), ext
|
||
return None, None
|
||
|
||
|
||
class ArchiveDB:
|
||
"""Simple sync SQLite wrapper for archive.db (FTS5 search only)."""
|
||
def __init__(self, db_path):
|
||
import sqlite3
|
||
self.conn = sqlite3.connect(db_path)
|
||
self.conn.row_factory = sqlite3.Row
|
||
|
||
async def search_pages(self, query, limit=50):
|
||
cursor = self.conn.execute(
|
||
"SELECT uri, title, snippet(pages_fts, 2, '<b>', '</b>', '...', 32) as snippet "
|
||
"FROM pages_fts WHERE pages_fts MATCH ? LIMIT ?",
|
||
(query, limit)
|
||
)
|
||
return [dict(row) for row in cursor.fetchall()]
|
||
|
||
async def get_stats(self):
|
||
cursor = self.conn.execute("SELECT COUNT(*) FROM pages")
|
||
return {"pages": cursor.fetchone()[0], "media": 0, "screenshots": 0}
|
||
|
||
|
||
@app.on_event("startup")
|
||
async def startup_event():
|
||
"""Initialize database on startup."""
|
||
global db
|
||
if TAR_FILE:
|
||
# Tarball mode: use simple archive DB
|
||
db = ArchiveDB(DB_PATH)
|
||
logger.info(f"Using archive database: {DB_PATH}")
|
||
else:
|
||
# Normal mode: use full async database
|
||
db = Database(DB_PATH)
|
||
await db.init() # Handles schema + WAL mode
|
||
# Backfill uri_hash for existing pages
|
||
count = await db.backfill_page_hashes()
|
||
if count:
|
||
logger.info(f"Backfilled {count} page URI hashes")
|
||
VAULT_PATH.mkdir(parents=True, exist_ok=True)
|
||
logger.info(f"Vault directory ready: {VAULT_PATH}")
|
||
|
||
# Mark any orphaned "running" jobs as "paused" (server was killed)
|
||
async with db.session() as session:
|
||
result = await session.execute(
|
||
text("UPDATE crawl_jobs SET status = 'paused' WHERE status = 'running'")
|
||
)
|
||
if result.rowcount:
|
||
await session.commit()
|
||
logger.info(f"Marked {result.rowcount} orphaned running job(s) as paused")
|
||
|
||
|
||
@app.on_event("shutdown")
|
||
async def shutdown_event():
|
||
"""Pause active crawls on shutdown so they can be resumed."""
|
||
if ACTIVE_CRAWL_TASKS:
|
||
logger.info(f"Pausing {len(ACTIVE_CRAWL_TASKS)} active crawl(s)...")
|
||
for job_id, task in list(ACTIVE_CRAWL_TASKS.items()):
|
||
task.cancel()
|
||
try:
|
||
await task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
# Mark as paused so it can be resumed
|
||
try:
|
||
await db.pause_crawl_job(job_id)
|
||
except Exception as e:
|
||
logger.error(f"Failed to pause job {job_id}: {e}")
|
||
logger.info("All crawls paused")
|
||
|
||
|
||
class CrawlRequest(BaseModel):
|
||
"""Request to start a new crawl."""
|
||
targets: List[str] = [] # Multiple target URIs
|
||
target_uri: str = "" # Deprecated: single target (for backwards compat)
|
||
keywords: List[str] = []
|
||
mode: str = "all" # text, images, videos, media, all
|
||
depth: int = -1 # -1 = unlimited
|
||
max_pages: int = -1 # -1 = unlimited
|
||
fresh: bool = True # Start fresh (rotate state files)
|
||
fast: bool = False # No crawl delay
|
||
screenshots: bool = True # Take page screenshots
|
||
|
||
|
||
@app.get("/", response_class=HTMLResponse)
|
||
async def index(lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Simple search UI."""
|
||
language = get_lang(lang, accept_language)
|
||
return inject_i18n(get_search_html(), language)
|
||
|
||
|
||
@app.get("/crawl", response_class=HTMLResponse)
|
||
async def crawl_page(lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Crawler command page."""
|
||
language = get_lang(lang, accept_language)
|
||
return inject_i18n(CRAWL_HTML, language)
|
||
|
||
|
||
# Page-specific CSS for Search page
|
||
SEARCH_CSS = """
|
||
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
||
.subtitle { color: #666; margin-bottom: 20px; }
|
||
.stats { padding: 10px 15px; background: #1a1a1a; border-radius: 8px; margin-bottom: 20px; font-size: 14px; color: #888; }
|
||
.section-title { color: #ff6b6b; font-size: 18px; margin: 25px 0 15px 0; border-bottom: 1px solid #333; padding-bottom: 8px; }
|
||
.results { display: grid; grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); gap: 15px; }
|
||
.result { background: #1a1a1a; border-radius: 8px; overflow: hidden; transition: transform 0.2s; }
|
||
.result:hover { transform: scale(1.02); }
|
||
.result img, .result video { width: 100%; height: 200px; object-fit: contain; background: #1a1a1a; }
|
||
.result-info { padding: 10px; }
|
||
.result-hash { font-family: monospace; font-size: 11px; color: #ff6b6b; text-decoration: none; word-break: break-all; }
|
||
.result-hash:hover { text-decoration: underline; }
|
||
.result-meta { font-size: 12px; color: #888; margin-top: 5px; }
|
||
.result-keywords { display: flex; flex-wrap: wrap; gap: 5px; margin-top: 8px; }
|
||
.tag { background: #333; padding: 2px 8px; border-radius: 4px; font-size: 11px; color: #aaa; }
|
||
.no-results { text-align: center; padding: 60px; color: #666; }
|
||
.result-count { padding: 10px 15px; color: #888; font-size: 14px; }
|
||
.result-count.over-9000 { color: #ff6b6b; font-weight: bold; font-size: 18px; text-shadow: 0 0 10px rgba(255,107,107,0.5); animation: powerUp 0.5s ease-out; }
|
||
@keyframes powerUp { 0% { transform: scale(1); } 50% { transform: scale(1.2); } 100% { transform: scale(1); } }
|
||
.pagination { display: flex; gap: 15px; justify-content: center; padding: 20px; margin-top: 20px; border-top: 1px solid #333; }
|
||
.pagination button { padding: 10px 20px; background: #4ecdc4; color: #1a1a2e; border: none; border-radius: 5px; cursor: pointer; font-weight: bold; }
|
||
.pagination button:hover { background: #7bed72; }
|
||
.media-link { display: block; cursor: pointer; }
|
||
.media-link:hover img, .media-link:hover video { opacity: 0.8; }
|
||
.page-results { display: flex; flex-direction: column; gap: 10px; }
|
||
.page-result { background: #1a1a1a; border-radius: 8px; padding: 15px; transition: background 0.2s; }
|
||
.page-result:hover { background: #252525; }
|
||
.page-result a { color: #ff6b6b; text-decoration: none; font-size: 16px; font-weight: 500; }
|
||
.page-result a:hover { text-decoration: underline; }
|
||
.page-path { font-size: 12px; color: #4ade80; margin-top: 4px; font-family: monospace; }
|
||
.page-snippet { font-size: 13px; color: #999; margin-top: 8px; line-height: 1.5; }
|
||
.page-snippet mark { background: #ff6b6b33; color: #ff9999; padding: 1px 3px; border-radius: 2px; }
|
||
.search-columns { display: grid; grid-template-columns: 1fr; gap: 30px; align-items: start; }
|
||
.search-columns.has-pages { grid-template-columns: 1fr 2fr; }
|
||
.search-columns .page-column { display: none; }
|
||
.search-columns.has-pages .page-column { display: block; }
|
||
@media (max-width: 1000px) { .search-columns.has-pages { grid-template-columns: 1fr; } }
|
||
"""
|
||
|
||
SEARCH_CONTENT = """
|
||
<div class="container">
|
||
<div class="search-box">
|
||
<input type="text" id="query" placeholder="{{search_placeholder}}" autofocus>
|
||
<select id="type">
|
||
<option value="">{{all_types}}</option>
|
||
<option value="image">{{images}}</option>
|
||
<option value="video">{{videos}}</option>
|
||
<option value="audio">{{audio}}</option>
|
||
</select>
|
||
<button onclick="search()">{{search}}</button>
|
||
</div>
|
||
|
||
<div class="stats" id="stats">{{loading_stats}}</div>
|
||
|
||
<div class="search-columns">
|
||
<div class="page-column" id="page-section">
|
||
<h2 class="section-title">{{pages_label}}</h2>
|
||
<div class="page-results" id="page-results"></div>
|
||
</div>
|
||
|
||
<div class="media-column" id="media-section">
|
||
<h2 class="section-title">{{media}}</h2>
|
||
<div class="results" id="results"></div>
|
||
</div>
|
||
</div>
|
||
|
||
<script>
|
||
async function loadStats() {
|
||
const res = await fetch('/api/stats');
|
||
const stats = await res.json();
|
||
document.getElementById('stats').innerHTML =
|
||
`<strong>${stats.total_media}</strong> ${T.media} | ` +
|
||
`<strong>${stats.by_type?.image || 0}</strong> ${T.images} | ` +
|
||
`<strong>${stats.by_type?.video || 0}</strong> ${T.videos} | ` +
|
||
`<strong>${stats.total_sources}</strong> ${T.sources} | ` +
|
||
`<strong>${stats.total_pages || 0}</strong> ${T.pages_label}`;
|
||
}
|
||
|
||
const OVER_9000 = 9000;
|
||
let currentOffset = 0;
|
||
|
||
async function search(offset = 0) {
|
||
currentOffset = offset;
|
||
const query = document.getElementById('query').value;
|
||
const type = document.getElementById('type').value;
|
||
|
||
// Search media - IT'S OVER 9000!
|
||
let mediaUrl = `/api/search?q=${encodeURIComponent(query)}&limit=${OVER_9000}&offset=${offset}`;
|
||
if (type) mediaUrl += `&type=${type}`;
|
||
|
||
const mediaRes = await fetch(mediaUrl);
|
||
const mediaResults = await mediaRes.json();
|
||
|
||
const mediaContainer = document.getElementById('results');
|
||
const mediaSection = document.getElementById('media-section');
|
||
|
||
// Build count display
|
||
let countDisplay = '';
|
||
if (mediaResults.length >= OVER_9000) {
|
||
countDisplay = `<div class="result-count over-9000">${T.over_9000}</div>`;
|
||
} else if (mediaResults.length > 0) {
|
||
const start = offset + 1;
|
||
const end = offset + mediaResults.length;
|
||
countDisplay = `<div class="result-count">${start.toLocaleString()}-${end.toLocaleString()} of ${offset > 0 ? 'many' : mediaResults.length.toLocaleString()}</div>`;
|
||
}
|
||
|
||
// Build pagination
|
||
let pagination = '';
|
||
if (offset > 0 || mediaResults.length >= OVER_9000) {
|
||
pagination = '<div class="pagination">';
|
||
if (offset > 0) {
|
||
pagination += `<button onclick="search(${Math.max(0, offset - OVER_9000)})">← ${T.previous} ${OVER_9000.toLocaleString()}</button>`;
|
||
}
|
||
if (mediaResults.length >= OVER_9000) {
|
||
pagination += `<button onclick="search(${offset + OVER_9000})">${T.next} ${OVER_9000.toLocaleString()} →</button>`;
|
||
}
|
||
pagination += '</div>';
|
||
}
|
||
|
||
if (mediaResults.length === 0 && offset === 0) {
|
||
mediaContainer.innerHTML = `<div class="no-results">${T.no_media}</div>`;
|
||
} else if (mediaResults.length === 0) {
|
||
mediaContainer.innerHTML = `<div class="no-results">${T.no_more}</div>` + pagination;
|
||
} else {
|
||
mediaContainer.innerHTML = countDisplay + mediaResults.map(r => {
|
||
const isVideo = r.media_type === 'video';
|
||
const mediaEl = isVideo
|
||
? `<video src="/media/${r.md5_hash}" muted loop preload="metadata" onmouseenter="this.play()" onmouseleave="this.pause()"></video>`
|
||
: `<img src="/media/${r.md5_hash}" alt="${r.alt_text || ''}" loading="lazy">`;
|
||
|
||
const keywords = JSON.parse(r.keywords || '[]');
|
||
const tagsHtml = keywords.map(k => `<span class="tag">${k}</span>`).join('');
|
||
|
||
return `
|
||
<div class="result">
|
||
<a href="/view/${r.md5_hash}" class="media-link">
|
||
${mediaEl}
|
||
</a>
|
||
<div class="result-info">
|
||
<a href="/view/${r.md5_hash}" class="result-hash">${r.md5_hash}</a>
|
||
<div class="result-meta">
|
||
${r.media_type} · ${formatBytes(r.file_size)}
|
||
${r.alt_text ? ` · ${r.alt_text.substring(0, 50)}` : ''}
|
||
</div>
|
||
<div class="result-keywords">${tagsHtml}</div>
|
||
</div>
|
||
</div>
|
||
`;
|
||
}).join('') + pagination;
|
||
}
|
||
|
||
// Search pages
|
||
const pageContainer = document.getElementById('page-results');
|
||
const searchColumns = document.querySelector('.search-columns');
|
||
|
||
if (query.trim()) {
|
||
const pageRes = await fetch(`/api/search/pages?q=${encodeURIComponent(query)}&limit=30`);
|
||
const pageResults = await pageRes.json();
|
||
|
||
if (pageResults.length > 0) {
|
||
searchColumns.classList.add('has-pages');
|
||
pageContainer.innerHTML = pageResults.map(p => `
|
||
<div class="page-result">
|
||
<a href="/page/${p.uri_hash}">${p.title || p.uri}</a>
|
||
<div class="page-path">${p.path || p.uri}</div>
|
||
<div class="page-snippet">${p.snippet || ''}</div>
|
||
</div>
|
||
`).join('');
|
||
} else {
|
||
searchColumns.classList.remove('has-pages');
|
||
pageContainer.innerHTML = '';
|
||
}
|
||
} else {
|
||
searchColumns.classList.remove('has-pages');
|
||
pageContainer.innerHTML = '';
|
||
}
|
||
}
|
||
|
||
function formatBytes(bytes) {
|
||
if (!bytes) return '?';
|
||
if (bytes < 1024) return bytes + ' B';
|
||
if (bytes < 1024*1024) return (bytes/1024).toFixed(1) + ' KB';
|
||
return (bytes/1024/1024).toFixed(1) + ' MB';
|
||
}
|
||
|
||
// Enter key to search
|
||
document.getElementById('query').addEventListener('keypress', e => {
|
||
if (e.key === 'Enter') search();
|
||
});
|
||
|
||
// Load stats on page load
|
||
loadStats();
|
||
|
||
// Check for query param from nav search
|
||
const urlParams = new URLSearchParams(window.location.search);
|
||
const q = urlParams.get('q');
|
||
if (q) {
|
||
document.getElementById('query').value = q;
|
||
}
|
||
|
||
// Initial search (show all media or query)
|
||
search();
|
||
</script>
|
||
</div>
|
||
"""
|
||
|
||
def get_search_html():
|
||
"""Build search page using layout."""
|
||
return layout("neopig SERP", SEARCH_CONTENT, SEARCH_CSS)
|
||
|
||
CRAWL_HTML = """
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>neopig {{crawl}}</title>
|
||
<style>
|
||
* { box-sizing: border-box; }
|
||
body {
|
||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||
margin: 0;
|
||
padding: 0;
|
||
background: #0a0a0a;
|
||
color: #e0e0e0;
|
||
}
|
||
.nav {
|
||
background: #1a1a1a;
|
||
padding: 10px 20px;
|
||
display: flex;
|
||
gap: 20px;
|
||
align-items: center;
|
||
border-bottom: 1px solid #333;
|
||
}
|
||
.nav a { color: #ff6b6b; text-decoration: none; }
|
||
.nav a:hover { text-decoration: underline; }
|
||
.nav .brand { font-weight: bold; font-size: 18px; }
|
||
.container { padding: 20px; }
|
||
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
||
.subtitle { color: #666; margin-bottom: 20px; }
|
||
|
||
.form-group {
|
||
margin-bottom: 15px;
|
||
}
|
||
label {
|
||
display: block;
|
||
margin-bottom: 5px;
|
||
color: #aaa;
|
||
font-size: 14px;
|
||
}
|
||
input[type="text"], input[type="number"], select {
|
||
width: 100%;
|
||
padding: 12px 16px;
|
||
font-size: 16px;
|
||
border: 2px solid #333;
|
||
border-radius: 8px;
|
||
background: #1a1a1a;
|
||
color: #fff;
|
||
}
|
||
input:focus, select:focus {
|
||
outline: none;
|
||
border-color: #ff6b6b;
|
||
}
|
||
.row {
|
||
display: flex;
|
||
gap: 15px;
|
||
}
|
||
.row > div { flex: 1; }
|
||
|
||
button {
|
||
padding: 14px 28px;
|
||
font-size: 16px;
|
||
background: #ff6b6b;
|
||
color: #fff;
|
||
border: none;
|
||
border-radius: 8px;
|
||
cursor: pointer;
|
||
margin-top: 10px;
|
||
}
|
||
button:hover { background: #ff5252; }
|
||
button:disabled {
|
||
background: #444;
|
||
cursor: not-allowed;
|
||
}
|
||
button.secondary {
|
||
background: #333;
|
||
}
|
||
button.secondary:hover {
|
||
background: #444;
|
||
}
|
||
|
||
.checkbox-group {
|
||
display: flex;
|
||
align-items: center;
|
||
gap: 8px;
|
||
}
|
||
.checkbox-group input {
|
||
width: auto;
|
||
}
|
||
|
||
.jobs-section {
|
||
margin-top: 30px;
|
||
padding-top: 20px;
|
||
border-top: 1px solid #333;
|
||
}
|
||
h2 {
|
||
color: #ff6b6b;
|
||
font-size: 18px;
|
||
margin-bottom: 15px;
|
||
}
|
||
|
||
.job {
|
||
background: #1a1a1a;
|
||
border-radius: 8px;
|
||
padding: 15px;
|
||
margin-bottom: 10px;
|
||
}
|
||
.job-header {
|
||
display: flex;
|
||
justify-content: space-between;
|
||
align-items: center;
|
||
margin-bottom: 10px;
|
||
}
|
||
.job-id {
|
||
font-family: monospace;
|
||
color: #666;
|
||
}
|
||
.job-status {
|
||
padding: 4px 10px;
|
||
border-radius: 4px;
|
||
font-size: 12px;
|
||
font-weight: bold;
|
||
}
|
||
.job-status.running { background: #2d5a27; color: #7bed72; }
|
||
.job-status.completed { background: #1a3a4a; color: #6bc5e8; }
|
||
.job-status.failed { background: #5a2727; color: #ed7272; }
|
||
|
||
.job-target {
|
||
font-size: 14px;
|
||
word-break: break-all;
|
||
margin-bottom: 5px;
|
||
}
|
||
.job-meta {
|
||
font-size: 12px;
|
||
color: #666;
|
||
}
|
||
.job-stats {
|
||
display: flex;
|
||
gap: 15px;
|
||
margin-top: 10px;
|
||
font-size: 13px;
|
||
}
|
||
.job-stats span {
|
||
background: #252525;
|
||
padding: 4px 10px;
|
||
border-radius: 4px;
|
||
}
|
||
|
||
.progress-bar {
|
||
height: 4px;
|
||
background: #333;
|
||
border-radius: 2px;
|
||
margin-top: 10px;
|
||
overflow: hidden;
|
||
}
|
||
.progress-bar-fill {
|
||
height: 100%;
|
||
background: #ff6b6b;
|
||
transition: width 0.3s;
|
||
}
|
||
.progress-bar-fill.running {
|
||
width: 100%;
|
||
background: linear-gradient(90deg, #ff6b6b 0%, #4ecdc4 50%, #ff6b6b 100%);
|
||
background-size: 200% 100%;
|
||
animation: progress-wave 1.5s ease-in-out infinite;
|
||
}
|
||
.job-actions { margin-top: 8px; display: flex; gap: 8px; }
|
||
.job-actions button {
|
||
padding: 4px 12px;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 4px;
|
||
cursor: pointer;
|
||
font-size: 12px;
|
||
}
|
||
.pause-btn { background: #e67e22; }
|
||
.pause-btn:hover { background: #f39c12; }
|
||
.start-btn { background: #27ae60; }
|
||
.start-btn:hover { background: #2ecc71; }
|
||
.delete-btn { background: #7f8c8d; }
|
||
.delete-btn:hover { background: #c0392b; }
|
||
.job-links { margin-left: 10px; font-size: 12px; }
|
||
.job-links a { color: #4ecdc4; margin-right: 8px; }
|
||
.job-links a:hover { color: #7bed72; }
|
||
@keyframes progress-wave {
|
||
0% { background-position: 200% 0; }
|
||
100% { background-position: -200% 0; }
|
||
}
|
||
.job.running {
|
||
border-color: #4ecdc4;
|
||
}
|
||
.job-stats.live {
|
||
color: #4ecdc4;
|
||
}
|
||
|
||
.no-jobs {
|
||
color: #666;
|
||
text-align: center;
|
||
padding: 30px;
|
||
}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<!-- NAV -->
|
||
<div class="container">
|
||
|
||
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
||
<input type="text" name="q" placeholder="{{search_placeholder}}" style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||
<option value="">{{all_types}}</option>
|
||
<option value="image">{{images}}</option>
|
||
<option value="video">{{videos}}</option>
|
||
<option value="audio">{{audio}}</option>
|
||
</select>
|
||
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">{{search}}</button>
|
||
</form>
|
||
|
||
<h1>{{crawl}}</h1>
|
||
<p class="subtitle">{{hydrate_subtitle}}</p>
|
||
|
||
<form id="crawl-form" onsubmit="startCrawl(event)">
|
||
<div class="form-group">
|
||
<label>{{url}}</label>
|
||
<textarea id="target" placeholder="https://example.com https://another.com" rows="2" required style="width:100%;padding:10px;border:1px solid #333;border-radius:4px;background:#1a1a1a;color:#e0e0e0;font-size:14px;resize:vertical;"></textarea>
|
||
</div>
|
||
|
||
<div class="form-group">
|
||
<label>{{keywords}}</label>
|
||
<input type="text" id="keywords" placeholder="e.g. rick and morty, adult swim">
|
||
</div>
|
||
|
||
<div class="row">
|
||
<div class="form-group">
|
||
<label>{{mode}}</label>
|
||
<select id="mode">
|
||
<option value="all">{{everything}}</option>
|
||
<option value="media" selected>{{all_media}}</option>
|
||
<option value="images">{{images_only}}</option>
|
||
<option value="videos">{{videos_only}}</option>
|
||
<option value="text">{{text_only}}</option>
|
||
</select>
|
||
</div>
|
||
<div class="form-group">
|
||
<label>{{depth}}</label>
|
||
<input type="number" id="depth" value="9" min="1" max="15">
|
||
</div>
|
||
<div class="form-group">
|
||
<label>{{max_pages}}</label>
|
||
<input type="number" id="max_pages" value="-1" min="-1">
|
||
</div>
|
||
</div>
|
||
|
||
<div class="form-group checkbox-group" style="display:flex; flex-wrap:wrap; gap:20px;">
|
||
<div>
|
||
<input type="checkbox" id="fresh" checked>
|
||
<label for="fresh" style="display:inline; margin:0;">{{fresh_start}}</label>
|
||
</div>
|
||
<div>
|
||
<input type="checkbox" id="fast">
|
||
<label for="fast" style="display:inline; margin:0;">{{fast_mode}}</label>
|
||
</div>
|
||
<div>
|
||
<input type="checkbox" id="screenshots" checked>
|
||
<label for="screenshots" style="display:inline; margin:0;">{{screenshots}}</label>
|
||
</div>
|
||
</div>
|
||
|
||
<button type="submit" id="start-btn">{{start_crawl}}</button>
|
||
</form>
|
||
|
||
<div class="jobs-section">
|
||
<h2>{{recent_jobs}}</h2>
|
||
<div id="jobs">{{loading}}</div>
|
||
</div>
|
||
|
||
<script>
|
||
async function startCrawl(e) {
|
||
e.preventDefault();
|
||
|
||
const btn = document.getElementById('start-btn');
|
||
btn.disabled = true;
|
||
btn.textContent = T.starting;
|
||
|
||
const keywordsRaw = document.getElementById('keywords').value;
|
||
const keywords = keywordsRaw
|
||
.split(/[,\\s]+/)
|
||
.map(k => k.trim())
|
||
.filter(k => k.length > 0);
|
||
|
||
// Parse multiple target URIs (space, comma, or newline separated)
|
||
const targetsRaw = document.getElementById('target').value;
|
||
const targets = targetsRaw
|
||
.split(/[,\\s\\n]+/)
|
||
.map(t => t.trim())
|
||
.filter(t => t.length > 0 && t.startsWith('http'));
|
||
|
||
if (targets.length === 0) {
|
||
alert(T.valid_uri);
|
||
btn.disabled = false;
|
||
btn.textContent = T.start_crawl;
|
||
return;
|
||
}
|
||
|
||
const payload = {
|
||
targets: targets,
|
||
keywords: keywords,
|
||
mode: document.getElementById('mode').value,
|
||
depth: parseInt(document.getElementById('depth').value),
|
||
max_pages: parseInt(document.getElementById('max_pages').value),
|
||
fresh: document.getElementById('fresh').checked,
|
||
fast: document.getElementById('fast').checked,
|
||
screenshots: document.getElementById('screenshots').checked
|
||
};
|
||
|
||
try {
|
||
const res = await fetch('/api/crawl', {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify(payload)
|
||
});
|
||
|
||
if (!res.ok) {
|
||
const err = await res.json();
|
||
btn.textContent = T.error + ': ' + (err.detail || T.load_error);
|
||
btn.style.background = '#c0392b';
|
||
setTimeout(() => {
|
||
btn.textContent = T.start_crawl;
|
||
btn.style.background = '';
|
||
}, 3000);
|
||
} else {
|
||
// Clear form and refresh jobs list
|
||
document.getElementById('target').value = '';
|
||
document.getElementById('keywords').value = '';
|
||
btn.textContent = T.started_ok;
|
||
btn.style.background = '#27ae60';
|
||
loadJobs();
|
||
setTimeout(() => {
|
||
btn.textContent = T.start_crawl;
|
||
btn.style.background = '';
|
||
}, 2000);
|
||
}
|
||
} catch (err) {
|
||
btn.textContent = T.error;
|
||
btn.style.background = '#c0392b';
|
||
setTimeout(() => {
|
||
btn.textContent = T.start_crawl;
|
||
btn.style.background = '';
|
||
}, 3000);
|
||
}
|
||
|
||
btn.disabled = false;
|
||
}
|
||
|
||
async function pauseJob(jobId) {
|
||
try {
|
||
const res = await fetch('/api/crawl/jobs/' + jobId + '/pause', { method: 'POST' });
|
||
if (res.ok) {
|
||
loadJobs();
|
||
} else {
|
||
const err = await res.json();
|
||
alert(T.error + ': ' + (err.detail || T.load_error));
|
||
}
|
||
} catch (err) {
|
||
alert(T.error + ': ' + err.message);
|
||
}
|
||
}
|
||
|
||
async function resumeJob(jobId) {
|
||
try {
|
||
const res = await fetch('/api/crawl/jobs/' + jobId + '/resume', { method: 'POST' });
|
||
if (res.ok) {
|
||
loadJobs();
|
||
} else {
|
||
const err = await res.json();
|
||
alert(T.error + ': ' + (err.detail || T.load_error));
|
||
}
|
||
} catch (err) {
|
||
alert(T.error + ': ' + err.message);
|
||
}
|
||
}
|
||
|
||
async function deleteJob(jobId) {
|
||
if (!confirm(T.delete + '?')) return;
|
||
try {
|
||
const res = await fetch('/api/crawl/jobs/' + jobId, { method: 'DELETE' });
|
||
if (res.ok) {
|
||
loadJobs();
|
||
} else {
|
||
const err = await res.json();
|
||
alert(T.error + ': ' + (err.detail || T.load_error));
|
||
}
|
||
} catch (err) {
|
||
alert(T.error + ': ' + err.message);
|
||
}
|
||
}
|
||
|
||
function formatBytes(bytes) {
|
||
if (!bytes) return '';
|
||
if (bytes < 1024) return bytes + ' B';
|
||
if (bytes < 1024*1024) return (bytes/1024).toFixed(1) + ' KB';
|
||
if (bytes < 1024*1024*1024) return (bytes/1024/1024).toFixed(1) + ' MB';
|
||
return (bytes/1024/1024/1024).toFixed(1) + ' GB';
|
||
}
|
||
|
||
function formatDuration(startedAt, completedAt) {
|
||
if (!startedAt || !completedAt) return '';
|
||
const start = new Date(startedAt);
|
||
const end = new Date(completedAt);
|
||
const sec = Math.round((end - start) / 1000);
|
||
if (sec < 60) return sec + 's';
|
||
if (sec < 3600) return Math.floor(sec / 60) + 'm ' + (sec % 60) + 's';
|
||
const hr = Math.floor(sec / 3600);
|
||
const min = Math.floor((sec % 3600) / 60);
|
||
return hr + 'h ' + min + 'm';
|
||
}
|
||
|
||
async function loadJobs() {
|
||
try {
|
||
const res = await fetch('/api/crawl/jobs');
|
||
const jobs = await res.json();
|
||
|
||
const container = document.getElementById('jobs');
|
||
|
||
if (jobs.length === 0) {
|
||
container.innerHTML = '<div class="no-jobs">' + T.no_jobs + '</div>';
|
||
return;
|
||
}
|
||
|
||
const hasRunning = jobs.some(j => j.status === 'running');
|
||
|
||
container.innerHTML = jobs.map(job => {
|
||
let stats = {};
|
||
let keywords = [];
|
||
try {
|
||
stats = job.stats ? JSON.parse(job.stats) : {};
|
||
} catch (e) {
|
||
console.warn('Failed to parse job stats:', job.id, e);
|
||
}
|
||
try {
|
||
keywords = job.keywords ? JSON.parse(job.keywords) : [];
|
||
} catch (e) {
|
||
console.warn('Failed to parse job keywords:', job.id, e);
|
||
}
|
||
const isBackfill = job.job_kind === 'backfill';
|
||
const kindLabel = isBackfill ? `🔄 ${job.job_type}` : `🕷️ ${job.mode || 'all'}`;
|
||
|
||
// Backfill job progress
|
||
if (isBackfill) {
|
||
const progress = job.total_records > 0
|
||
? Math.round((job.processed_records / job.total_records) * 100)
|
||
: 0;
|
||
return `
|
||
<div class="job ${job.status}">
|
||
<div class="job-header">
|
||
<span class="job-id">${kindLabel} #${job.id}</span>
|
||
<span class="job-status ${job.status}">${T[job.status] || job.status}</span>
|
||
</div>
|
||
<div class="job-target">${job.domain_filter || T.all_domains}</div>
|
||
<div class="job-meta">
|
||
${job.processed_records}/${job.total_records} |
|
||
${job.error_count} ${T.error} |
|
||
${T.started}: ${new Date(job.started_at).toLocaleString()}
|
||
</div>
|
||
${job.status === 'running' ? `
|
||
<div class="progress-bar">
|
||
<div class="progress-bar-fill" style="width: ${progress}%"></div>
|
||
</div>
|
||
` : ''}
|
||
</div>
|
||
`;
|
||
}
|
||
|
||
// Crawl job display with ETA
|
||
const calcEta = () => {
|
||
const crawled = stats.pages_crawled || 0;
|
||
const pending = stats.pages_pending || 0;
|
||
const started = stats.crawl_started || 0;
|
||
if (crawled === 0 || started === 0 || pending === 0) return '';
|
||
const elapsed = Date.now() / 1000 - started;
|
||
if (elapsed < 5) return ''; // Wait for meaningful data
|
||
const rate = crawled / elapsed;
|
||
const etaSec = pending / rate;
|
||
if (etaSec < 60) return `~${Math.round(etaSec)}s`;
|
||
if (etaSec < 3600) return `~${Math.round(etaSec / 60)}m`;
|
||
return `~${(etaSec / 3600).toFixed(1)}h`;
|
||
};
|
||
const eta = calcEta();
|
||
const liveStats = job.status === 'running' ? `
|
||
<div class="job-stats live">
|
||
<span>📄 ${stats.pages_crawled || 0}${stats.pages_pending ? '/' + ((stats.pages_crawled || 0) + stats.pages_pending) : ''} pages${eta ? ' ⏱️' + eta : ''}</span>
|
||
<span>🖼️ ${stats.media_found || 0} found</span>
|
||
<span>💾 ${stats.media_downloaded || 0} saved</span>
|
||
<span>📸 ${stats.screenshots_taken || 0} screenshots</span>
|
||
<span>♻️ ${(stats.duplicates_skipped || 0) + (stats.content_exists || 0)} dupes</span>
|
||
${stats.bytes_stored ? `<span>📦 ${formatBytes(stats.bytes_stored)}</span>` : ''}
|
||
</div>
|
||
` : '';
|
||
|
||
return `
|
||
<div class="job ${job.status}">
|
||
<div class="job-header">
|
||
<span class="job-id">${kindLabel} #${job.id}</span>
|
||
<span class="job-status ${job.status}">${T[job.status] || job.status}</span>
|
||
</div>
|
||
<div class="job-target">
|
||
<a href="${job.target_uri}" target="_blank">${job.target_uri}</a>
|
||
${(() => { try { const h = new URL(job.target_uri).hostname; return `<span class="job-links"><a href="/?q=${encodeURIComponent(h)}">${T.search}</a> <a href="/live?domain=${encodeURIComponent(h)}">${T.live}</a></span>`; } catch(e) { return ''; } })()}
|
||
</div>
|
||
<div class="job-meta">
|
||
${keywords.length ? `${T.filter}: ${keywords.join(', ')} | ` : ''}${T.started}: ${new Date(job.started_at).toLocaleString()}
|
||
</div>
|
||
${job.status === 'completed' ? `
|
||
<div class="job-stats">
|
||
<span>⏱️ ${formatDuration(job.started_at, job.completed_at)}</span>
|
||
<span>📄 ${stats.pages_crawled || 0} pages</span>
|
||
<span>🖼️ ${stats.media_found || 0} found</span>
|
||
<span>💾 ${stats.media_downloaded || 0} saved</span>
|
||
<span>📸 ${stats.screenshots_taken || 0} screenshots</span>
|
||
<span>♻️ ${(stats.duplicates_skipped || 0) + (stats.content_exists || 0)} dupes</span>
|
||
${stats.bytes_stored ? `<span>📦 ${formatBytes(stats.bytes_stored)}</span>` : ''}
|
||
</div>
|
||
` : ''}
|
||
${liveStats}
|
||
${job.status === 'running' && job.job_kind === 'crawl' ? `
|
||
<div class="progress-bar">
|
||
<div class="progress-bar-fill running"></div>
|
||
</div>
|
||
` : ''}
|
||
<div class="job-actions">
|
||
${job.status === 'running' && job.job_kind === 'crawl' ? `<button class="pause-btn" onclick="pauseJob(${job.id})">${T.pause}</button>` : ''}
|
||
${(job.status === 'paused' || job.status === 'cancelled') && job.job_kind === 'crawl' ? `<button class="start-btn" onclick="resumeJob(${job.id})">${T.resume}</button>` : ''}
|
||
${job.status !== 'running' ? `<button class="delete-btn" onclick="deleteJob(${job.id})">${T.delete}</button>` : ''}
|
||
</div>
|
||
</div>
|
||
`;
|
||
}).join('');
|
||
|
||
// Poll: 3s when running, 10s when idle
|
||
const delay = hasRunning ? 3000 : 10000;
|
||
setTimeout(loadJobs, delay);
|
||
} catch (err) {
|
||
console.error('loadJobs error:', err);
|
||
document.getElementById('jobs').innerHTML = `<div class="no-jobs">${T.load_jobs_error}: ` + err.message + '</div>';
|
||
setTimeout(loadJobs, 10000);
|
||
}
|
||
}
|
||
|
||
// Load jobs on page load
|
||
loadJobs();
|
||
</script>
|
||
</div>
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
|
||
LIVE_HTML = """
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>neopig {{live}} - {{live_subtitle}}</title>
|
||
<style>
|
||
* { box-sizing: border-box; }
|
||
body {
|
||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||
margin: 0;
|
||
padding: 0;
|
||
background: #0a0a0a;
|
||
color: #e0e0e0;
|
||
}
|
||
.nav {
|
||
background: #1a1a1a;
|
||
padding: 10px 20px;
|
||
display: flex;
|
||
gap: 20px;
|
||
align-items: center;
|
||
border-bottom: 1px solid #333;
|
||
}
|
||
.nav a { color: #ff6b6b; text-decoration: none; }
|
||
.nav a:hover { text-decoration: underline; }
|
||
.nav .brand { font-weight: bold; font-size: 18px; }
|
||
.container { padding: 20px; }
|
||
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
||
.subtitle { color: #666; margin-bottom: 10px; }
|
||
.stats {
|
||
background: #1a1a1a;
|
||
padding: 15px;
|
||
border-radius: 8px;
|
||
margin-bottom: 20px;
|
||
display: flex;
|
||
gap: 30px;
|
||
flex-wrap: wrap;
|
||
}
|
||
.stat { display: flex; flex-direction: column; }
|
||
.stat-value { font-size: 24px; font-weight: bold; color: #ff6b6b; }
|
||
.stat-label { font-size: 12px; color: #888; }
|
||
.controls {
|
||
margin-bottom: 20px;
|
||
display: flex;
|
||
gap: 10px;
|
||
align-items: center;
|
||
}
|
||
.controls button {
|
||
padding: 8px 16px;
|
||
border: none;
|
||
border-radius: 4px;
|
||
cursor: pointer;
|
||
font-size: 14px;
|
||
}
|
||
.controls button.primary { background: #ff6b6b; color: white; }
|
||
.controls button.secondary { background: #333; color: #ddd; }
|
||
.controls button:hover { opacity: 0.8; }
|
||
.status { color: #4ade80; font-size: 14px; }
|
||
.status.paused { color: #fbbf24; }
|
||
.live-grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
|
||
gap: 15px;
|
||
}
|
||
.live-card {
|
||
background: #1a1a1a;
|
||
border-radius: 8px;
|
||
overflow: hidden;
|
||
animation: fadeIn 0.5s ease-out;
|
||
}
|
||
@keyframes fadeIn {
|
||
from { opacity: 0; transform: translateY(20px); }
|
||
to { opacity: 1; transform: translateY(0); }
|
||
}
|
||
.live-card.new {
|
||
box-shadow: 0 0 20px rgba(255, 107, 107, 0.5);
|
||
}
|
||
.live-card img, .live-card video {
|
||
width: 100%;
|
||
height: 180px;
|
||
object-fit: contain;
|
||
background: #222;
|
||
}
|
||
.live-card-info {
|
||
padding: 10px;
|
||
}
|
||
.live-card-title {
|
||
font-size: 12px;
|
||
color: #888;
|
||
overflow: hidden;
|
||
text-overflow: ellipsis;
|
||
white-space: nowrap;
|
||
}
|
||
.live-card-source {
|
||
font-size: 10px;
|
||
color: #555;
|
||
margin-top: 4px;
|
||
overflow: hidden;
|
||
text-overflow: ellipsis;
|
||
white-space: nowrap;
|
||
}
|
||
.score-badge {
|
||
display: inline-block;
|
||
padding: 2px 6px;
|
||
border-radius: 3px;
|
||
font-size: 9px;
|
||
font-weight: bold;
|
||
text-transform: uppercase;
|
||
margin-left: 6px;
|
||
vertical-align: middle;
|
||
}
|
||
.score-screenshot { background: #666; color: #ccc; }
|
||
.score-og { background: #8b5cf6; color: white; }
|
||
.score-thumb { background: #3b82f6; color: white; }
|
||
.score-hd { background: #10b981; color: white; }
|
||
.live-card.upgraded {
|
||
box-shadow: 0 0 20px rgba(16, 185, 129, 0.7);
|
||
animation: upgradeGlow 1s ease-out;
|
||
}
|
||
@keyframes upgradeGlow {
|
||
0% { box-shadow: 0 0 30px rgba(16, 185, 129, 1); }
|
||
100% { box-shadow: 0 0 20px rgba(16, 185, 129, 0.5); }
|
||
}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<!-- NAV -->
|
||
<div class="container">
|
||
|
||
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
||
<input type="text" name="q" placeholder="{{search_placeholder}}" style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||
<option value="">{{all_types}}</option>
|
||
<option value="image">{{images}}</option>
|
||
<option value="video">{{videos}}</option>
|
||
<option value="audio">{{audio}}</option>
|
||
</select>
|
||
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">{{search}}</button>
|
||
</form>
|
||
|
||
<h1>{{live}}</h1>
|
||
<p class="subtitle">{{live_subtitle}}</p>
|
||
|
||
<div class="stats">
|
||
<div class="stat">
|
||
<span class="stat-value" id="total-count">0</span>
|
||
<span class="stat-label">{{total_images}}</span>
|
||
</div>
|
||
<div class="stat">
|
||
<span class="stat-value" id="new-count">0</span>
|
||
<span class="stat-label">{{new_session}}</span>
|
||
</div>
|
||
<div class="stat">
|
||
<span class="stat-value" id="rate">0</span>
|
||
<span class="stat-label">{{per_minute}}</span>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="controls">
|
||
<button class="primary" id="toggle-btn" onclick="toggleFeed()">{{pause}}</button>
|
||
<button class="secondary" onclick="clearFeed()">{{delete}}</button>
|
||
<span class="status" id="status">{{watching}}</span>
|
||
</div>
|
||
|
||
<div class="live-grid" id="grid"></div>
|
||
|
||
<script>
|
||
let running = true;
|
||
let lastCheck = new Date().toISOString();
|
||
let seenHashes = new Map(); // hash -> {upgraded_at, score}
|
||
let newCount = 0;
|
||
let startTime = Date.now();
|
||
|
||
function toggleFeed() {
|
||
running = !running;
|
||
const btn = document.getElementById('toggle-btn');
|
||
const status = document.getElementById('status');
|
||
if (running) {
|
||
btn.textContent = T.pause;
|
||
status.textContent = T.watching;
|
||
status.className = 'status';
|
||
poll();
|
||
} else {
|
||
btn.textContent = T.resume;
|
||
status.textContent = T.paused;
|
||
status.className = 'status paused';
|
||
}
|
||
}
|
||
|
||
function clearFeed() {
|
||
document.getElementById('grid').innerHTML = '';
|
||
seenHashes = new Map();
|
||
newCount = 0;
|
||
startTime = Date.now();
|
||
updateStats(0);
|
||
}
|
||
|
||
function updateStats(total) {
|
||
document.getElementById('total-count').textContent = total;
|
||
document.getElementById('new-count').textContent = newCount;
|
||
|
||
const minutes = (Date.now() - startTime) / 60000;
|
||
const rate = minutes > 0 ? Math.round(newCount / minutes) : 0;
|
||
document.getElementById('rate').textContent = rate;
|
||
}
|
||
|
||
async function poll() {
|
||
if (!running) return;
|
||
|
||
try {
|
||
// Get recent media sorted by first_seen_at descending
|
||
const res = await fetch('/api/search?q=&limit=50');
|
||
const media = await res.json();
|
||
|
||
// Get stats
|
||
const statsRes = await fetch('/api/stats');
|
||
const stats = await statsRes.json();
|
||
updateStats(stats.total_media || 0);
|
||
|
||
const grid = document.getElementById('grid');
|
||
|
||
// Find new items OR upgraded items (score/upgraded_at changed)
|
||
const isFirstPoll = seenHashes.size === 0;
|
||
const itemsToShow = media.filter(m => {
|
||
const seen = seenHashes.get(m.md5_hash);
|
||
if (!seen) return true; // New item
|
||
// Check if upgraded (score increased or upgraded_at changed)
|
||
if (m.upgraded_at && m.upgraded_at !== seen.upgraded_at) return true;
|
||
if (m.score && m.score > seen.score) return true;
|
||
return false;
|
||
});
|
||
|
||
// Add items to the top
|
||
itemsToShow.reverse().forEach(item => {
|
||
const wasUpgraded = seenHashes.has(item.md5_hash);
|
||
seenHashes.set(item.md5_hash, {upgraded_at: item.upgraded_at, score: item.score || 5});
|
||
if (!wasUpgraded) newCount++;
|
||
|
||
const card = document.createElement('div');
|
||
card.className = 'live-card new' + (wasUpgraded ? ' upgraded' : '');
|
||
|
||
const isVideo = item.media_type === 'video';
|
||
const mediaEl = isVideo
|
||
? `<video src="/media/${item.md5_hash}" muted loop onmouseenter="this.play()" onmouseleave="this.pause()"></video>`
|
||
: `<img src="/media/${item.md5_hash}" loading="lazy" onerror="this.src='data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 width=%22200%22 height=%22180%22><rect fill=%22%23333%22 width=%22200%22 height=%22180%22/><text x=%2250%%22 y=%2250%%22 fill=%22%23666%22 text-anchor=%22middle%22>Error</text></svg>'">`;
|
||
|
||
const scoreBadge = getScoreBadge(item.score, item.media_type);
|
||
|
||
card.innerHTML = `
|
||
<a href="/view/${item.md5_hash}">
|
||
${mediaEl}
|
||
</a>
|
||
<div class="live-card-info">
|
||
<div class="live-card-title">${item.alt_text || item.title || item.md5_hash.slice(0,12)}</div>
|
||
<div class="live-card-source">${item.media_type} - ${formatSize(item.file_size)}${scoreBadge}</div>
|
||
</div>
|
||
`;
|
||
|
||
grid.insertBefore(card, grid.firstChild);
|
||
|
||
// Remove 'new' highlight after animation
|
||
setTimeout(() => card.classList.remove('new'), 2000);
|
||
});
|
||
|
||
// Update stats
|
||
updateStats(stats.total_media || 0);
|
||
|
||
} catch (err) {
|
||
console.error('Poll error:', err);
|
||
}
|
||
|
||
// Poll again
|
||
setTimeout(poll, 2000);
|
||
}
|
||
|
||
function formatSize(bytes) {
|
||
if (!bytes) return '?';
|
||
if (bytes < 1024) return bytes + ' B';
|
||
if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB';
|
||
return (bytes / (1024 * 1024)).toFixed(1) + ' MB';
|
||
}
|
||
|
||
function getScoreBadge(score, mediaType) {
|
||
// Score constants: screenshot=1, og=3, thumb=5, hd=10
|
||
// Also check media_type for legacy records without scores
|
||
let label, cssClass;
|
||
if (mediaType === 'screenshot' || score <= 1) {
|
||
label = '📸'; cssClass = 'score-screenshot';
|
||
} else if (score <= 3) {
|
||
label = 'OG'; cssClass = 'score-og';
|
||
} else if (!score || score <= 5) {
|
||
label = 'THUMB'; cssClass = 'score-thumb';
|
||
} else {
|
||
label = 'HD'; cssClass = 'score-hd';
|
||
}
|
||
return `<span class="score-badge ${cssClass}">${label}</span>`;
|
||
}
|
||
|
||
// Start polling
|
||
poll();
|
||
</script>
|
||
</div>
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
|
||
@app.get("/live", response_class=HTMLResponse)
|
||
async def live_page(domain: str = Query(None), lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Live feed page - watch images appear as they're crawled."""
|
||
language = get_lang(lang, accept_language)
|
||
# Inject domain filter into HTML
|
||
html = LIVE_HTML
|
||
if domain:
|
||
# Add domain to the poll URL
|
||
html = html.replace(
|
||
"/api/search?q=&limit=50",
|
||
f"/api/search?q={domain}&limit=50"
|
||
)
|
||
html = html.replace(
|
||
"<h1>{{live}}</h1>",
|
||
"<h1>{{live}}: " + domain + "</h1>"
|
||
)
|
||
return inject_i18n(html, language)
|
||
|
||
|
||
@app.get("/view/{md5_hash}", response_class=HTMLResponse)
|
||
async def view_media_page(md5_hash: str, noai: bool = Query(False), lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Detail view page for a single media item."""
|
||
language = get_lang(lang, accept_language)
|
||
import html as html_module
|
||
import re
|
||
from urllib.parse import quote, urljoin
|
||
|
||
async with db.session() as session:
|
||
result = await session.execute(
|
||
text("SELECT * FROM media WHERE md5_hash = :hash"),
|
||
{'hash': md5_hash}
|
||
)
|
||
media = result.fetchone()
|
||
if not media:
|
||
raise HTTPException(status_code=404, detail="Media not found")
|
||
|
||
result = await session.execute(
|
||
text("""SELECT media_uri, page_uri, page_title, page_content,
|
||
detail_page_uri, detail_title, detail_content, discovered_at
|
||
FROM media_sources WHERE md5_hash = :hash"""),
|
||
{'hash': md5_hash}
|
||
)
|
||
sources = result.fetchall()
|
||
|
||
media = dict(media._mapping)
|
||
sources = [dict(s._mapping) for s in sources]
|
||
keywords = json.loads(media.get('keywords') or '[]')
|
||
sorted_sources = sorted(sources, key=lambda s: len(s["page_uri"] or ""), reverse=True)
|
||
|
||
# Get page URI
|
||
page_uri = sorted_sources[0]["page_uri"] if sorted_sources else None
|
||
source_uri = page_uri or (sorted_sources[0]["media_uri"] if sorted_sources else "unknown")
|
||
source_domain = Uri(source_uri).hostname if source_uri else "unknown"
|
||
|
||
# Display title
|
||
display_title = media.get('alt_text') or media.get('title')
|
||
if not display_title and sources:
|
||
display_title = sources[0].get('page_title')
|
||
if not display_title:
|
||
display_title = f"Media {md5_hash[:12]}"
|
||
|
||
# Hero: media element
|
||
is_video = media['media_type'] == 'video'
|
||
is_audio = media['media_type'] == 'audio'
|
||
if is_video:
|
||
hero_html = f'<a href="/media/{md5_hash}" target="_blank"><video src="/media/{md5_hash}" controls muted loop style="max-width:100%;max-height:70vh;"></video></a>'
|
||
elif is_audio:
|
||
hero_html = f'<audio src="/media/{md5_hash}" controls></audio>'
|
||
else:
|
||
hero_html = f'<a href="/media/{md5_hash}" target="_blank"><img src="/media/{md5_hash}" alt="{html_module.escape(media.get("alt_text") or "")}" style="max-width:100%;max-height:70vh;"></a>'
|
||
|
||
# Metadata rows
|
||
import hashlib
|
||
media_uri = sorted_sources[0]["media_uri"] if sorted_sources else None
|
||
neopig_media_uri = f"/view/{md5_hash}"
|
||
page_uri_hash = hashlib.md5(page_uri.encode()).hexdigest() if page_uri else None
|
||
neopig_page_uri = f"/page/{page_uri_hash}" if page_uri_hash else None
|
||
keywords_html = ''.join([f'<span class="tag">{k}</span>' for k in keywords]) or '-'
|
||
meta_rows = [
|
||
("{{source_uri}}", f'<a href="{media_uri}" target="_blank" style="font-size:11px;">{media_uri}</a>' if media_uri else '-'),
|
||
("{{neopig_uri}}", f'<a href="{neopig_media_uri}" style="font-size:11px;">{neopig_media_uri}</a>'),
|
||
("{{source_page}}", f'<a href="{page_uri}" target="_blank" style="font-size:11px;">{page_uri}</a>' if page_uri else '-'),
|
||
("{{neopig_page}}", f'<a href="{neopig_page_uri}" style="font-size:11px;">{neopig_page_uri}</a>' if neopig_page_uri else '-'),
|
||
("MD5", f'<code style="font-size:11px;">{md5_hash}</code>'),
|
||
("{{type_label}}", media['media_type']),
|
||
("{{mime_label}}", media.get('mime_type') or 'unknown'),
|
||
("{{size_label}}", f"{media.get('file_size') or 0:,} bytes"),
|
||
("{{alt_label}}", media.get('alt_text') or '-'),
|
||
("{{keywords_label}}", keywords_html),
|
||
]
|
||
|
||
# Download button
|
||
name_source = media.get('alt_text') or media.get('title')
|
||
if not name_source and sources:
|
||
pt = sources[0].get('page_title', '')
|
||
media_idx = int(md5_hash[:4], 16)
|
||
name_source = f"{pt}-{media_idx}" if pt else f"media-{media_idx}"
|
||
download_name = slugify(name_source or f"media-{md5_hash[:8]}")
|
||
ext_map = {'image/jpeg': '.jpg', 'image/png': '.png', 'image/gif': '.gif', 'image/webp': '.webp',
|
||
'video/mp4': '.mp4', 'video/webm': '.webm', 'audio/mpeg': '.mp3', 'audio/wav': '.wav'}
|
||
ext = ext_map.get(media.get('mime_type', ''), '.bin')
|
||
download_btn = f'<a href="/media/{md5_hash}?download=1" style="display:block;padding:12px 20px;background:#4a9eff;color:#fff;text-decoration:none;border-radius:6px;text-align:center;font-weight:500;margin-top:15px;">Download ({download_name}{ext})</a>'
|
||
|
||
# Get page media items (siblings)
|
||
media_items = await db.get_page_media(page_uri) if page_uri else []
|
||
# Exclude current item from gallery
|
||
media_items = [m for m in media_items if m.get("md5_hash") != md5_hash]
|
||
|
||
# Get rendered content
|
||
content_html = ""
|
||
if page_uri:
|
||
page_row = await db.get_page_by_uri(page_uri)
|
||
if page_row and page_row.get("markdown"):
|
||
try:
|
||
import markdown
|
||
md_converter = markdown.Markdown(extensions=['fenced_code', 'tables', 'nl2br'])
|
||
content_html = md_converter.convert(page_row["markdown"][:100000])
|
||
# Hydrate images and links
|
||
img_urls = re.findall(r'<img[^>]+src=["\']([^"\']+)["\']', content_html, re.I)
|
||
link_urls = re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', content_html, re.I)
|
||
all_urls = list(set(img_urls + link_urls))
|
||
if all_urls:
|
||
resolved = {u: u if u.startswith(('http://', 'https://', '//')) else urljoin(page_uri, u) for u in all_urls if not u.startswith('#')}
|
||
url_to_hash = await db.lookup_media_by_uris(list(set(resolved.values())))
|
||
exact_matches = set(url_to_hash.keys()) # These are reliable
|
||
|
||
# Fallback for imgur and other CDNs - lookup by filename/ID
|
||
# Only apply to URLs that look like media files (have media extension)
|
||
from async_web_fetcher import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS, AUDIO_EXTENSIONS
|
||
from pathlib import Path as P
|
||
media_exts = IMAGE_EXTENSIONS | VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
|
||
for orig, res in resolved.items():
|
||
if res not in url_to_hash:
|
||
url_path = res.split('?')[0]
|
||
ext = P(url_path).suffix.lower()
|
||
if ext in media_exts:
|
||
fname = P(url_path).stem
|
||
if fname and len(fname) >= 5:
|
||
result = await db.lookup_media_by_filename(fname)
|
||
if result:
|
||
url_to_hash[res] = result[1]
|
||
|
||
for orig, res in resolved.items():
|
||
if res in url_to_hash:
|
||
md5 = url_to_hash[res]
|
||
content_html = content_html.replace(f'src="{orig}"', f'src="/media/{md5}"')
|
||
content_html = content_html.replace(f"src='{orig}'", f'src="/media/{md5}"')
|
||
# Only rewrite hrefs for exact matches or URLs with media extensions
|
||
url_path = res.split('?')[0]
|
||
ext = P(url_path).suffix.lower()
|
||
if res in exact_matches or ext in media_exts:
|
||
content_html = content_html.replace(f'href="{orig}"', f'href="/view/{md5}"')
|
||
content_html = content_html.replace(f"href='{orig}'", f'href="/view/{md5}"')
|
||
except ImportError:
|
||
content_html = f"<pre>{html_module.escape(page_row['markdown'][:50000])}</pre>"
|
||
|
||
# Get screenshots (exclude current if it's a screenshot)
|
||
screenshot_hashes = await db.get_page_screenshots(page_uri) if page_uri else []
|
||
screenshot_hashes = [h for h in screenshot_hashes if h != md5_hash]
|
||
|
||
# Build "Used on X pages" section (reverse image search)
|
||
sources_html = ""
|
||
if sources and len(sources) > 0:
|
||
import hashlib as hl
|
||
unique_pages = {}
|
||
for s in sources:
|
||
pu = s.get('page_uri', '')
|
||
if pu and pu not in unique_pages:
|
||
unique_pages[pu] = s
|
||
if unique_pages:
|
||
rows_html = ""
|
||
for pu, s in list(unique_pages.items())[:100]: # Limit to 100
|
||
pt = html_module.escape(s.get('page_title', '') or pu[:60])
|
||
ph = hl.md5(pu.encode()).hexdigest()
|
||
rows_html += f'''<tr>
|
||
<td><a href="/page/{ph}" title="{html_module.escape(pu)}">{pt}</a></td>
|
||
<td style="font-size:11px;color:#888;">{html_module.escape(s.get('discovered_at', '')[:10] if s.get('discovered_at') else '-')}</td>
|
||
</tr>'''
|
||
sources_html = '''
|
||
<div class="sources-section">
|
||
<h3 onclick="document.getElementById('sources-list').classList.toggle('expanded'); this.querySelector('span').textContent = document.getElementById('sources-list').classList.contains('expanded') ? '▼' : '▶';">
|
||
<span>▶</span> {{used_on}} ''' + str(len(unique_pages)) + ''' {{pages}}
|
||
</h3>
|
||
<div id="sources-list" class="sources-list">
|
||
<table>
|
||
<thead><tr><th>{{page}}</th><th>{{discovered}}</th></tr></thead>
|
||
<tbody>''' + rows_html + '''</tbody>
|
||
</table>
|
||
</div>
|
||
</div>'''
|
||
|
||
return render_detail_page(
|
||
title=display_title,
|
||
hero_html=hero_html,
|
||
meta_rows=meta_rows,
|
||
media_items=media_items,
|
||
content_html=content_html,
|
||
screenshot_hashes=screenshot_hashes,
|
||
source_domain=source_domain,
|
||
page_uri=page_uri or "",
|
||
download_btn_html=download_btn,
|
||
noai=noai,
|
||
lang=language,
|
||
sources_html=sources_html,
|
||
)
|
||
|
||
|
||
|
||
|
||
@app.get("/page/{uri_hash}", response_class=HTMLResponse)
|
||
async def view_page_by_hash(
|
||
uri_hash: str,
|
||
noai: bool = Query(False, description="Disable AI assistant"),
|
||
lang: str = Cookie(None),
|
||
accept_language: str = Header(None),
|
||
):
|
||
"""View an archived page by URI hash."""
|
||
page = await db.get_page_by_hash(uri_hash)
|
||
if not page:
|
||
raise HTTPException(status_code=404, detail="Page not found")
|
||
# Redirect to the URI-based view (reuses same logic)
|
||
return await view_page(uri=page['uri'], noai=noai, lang=lang, accept_language=accept_language)
|
||
|
||
|
||
@app.get("/page/view", response_class=HTMLResponse)
|
||
async def view_page(
|
||
uri: str = Query(..., description="Page URI to view"),
|
||
noai: bool = Query(False, description="Disable AI assistant"),
|
||
lang: str = Cookie(None),
|
||
accept_language: str = Header(None),
|
||
):
|
||
"""View an archived page with markdown and screenshot."""
|
||
language = get_lang(lang, accept_language)
|
||
import html as html_module
|
||
import re
|
||
from urllib.parse import urljoin, quote
|
||
|
||
source_domain = Uri(uri).hostname
|
||
|
||
page = await db.get_page_by_uri(uri)
|
||
if not page:
|
||
raise HTTPException(status_code=404, detail="Page not found")
|
||
|
||
page_title = page.get("title") or uri
|
||
|
||
# Render markdown
|
||
content_html = ""
|
||
if page.get("markdown"):
|
||
try:
|
||
import markdown
|
||
md_converter = markdown.Markdown(extensions=['fenced_code', 'tables', 'nl2br'])
|
||
content_html = md_converter.convert(page["markdown"][:100000])
|
||
|
||
# Hydrate images and links from vault
|
||
img_urls = re.findall(r'<img[^>]+src=["\']([^"\']+)["\']', content_html, re.I)
|
||
link_urls = re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', content_html, re.I)
|
||
all_urls = list(set(img_urls + link_urls))
|
||
if all_urls:
|
||
# Keep anchor-only links (#foo) as-is, resolve others
|
||
resolved = {}
|
||
for u in all_urls:
|
||
if u.startswith('#'):
|
||
continue # Skip anchor-only links, they stay internal
|
||
elif u.startswith(('http://', 'https://', '//')):
|
||
resolved[u] = u
|
||
else:
|
||
resolved[u] = urljoin(uri, u)
|
||
|
||
# Lookup media (images, etc) - track which came from exact match vs fallback
|
||
url_to_hash = await db.lookup_media_by_uris(list(set(resolved.values())))
|
||
exact_matches = set(url_to_hash.keys()) # These are reliable
|
||
|
||
# Fallback for imgur and other CDNs - lookup by filename/ID
|
||
# Only apply to URLs that look like media files (have media extension)
|
||
from async_web_fetcher import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS, AUDIO_EXTENSIONS
|
||
media_exts = IMAGE_EXTENSIONS | VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
|
||
for murl in [u for u in resolved.values() if u not in url_to_hash]:
|
||
from pathlib import Path as P
|
||
url_path = murl.split('?')[0]
|
||
ext = P(url_path).suffix.lower()
|
||
# Only do filename fallback for URLs with media extensions
|
||
if ext in media_exts:
|
||
fname = P(url_path).stem
|
||
if fname and len(fname) >= 5:
|
||
result = await db.lookup_media_by_filename(fname)
|
||
if result:
|
||
url_to_hash[murl] = result[1]
|
||
|
||
# Lookup pages for internal link rewriting
|
||
page_links = [res for res in resolved.values() if source_domain and source_domain in res]
|
||
uri_to_page_hash = await db.lookup_pages_by_uris(page_links) if page_links else {}
|
||
|
||
for orig, res in resolved.items():
|
||
if res in url_to_hash:
|
||
md5 = url_to_hash[res]
|
||
# Rewrite img src to serve media directly
|
||
content_html = content_html.replace(f'src="{orig}"', f'src="/media/{md5}"')
|
||
content_html = content_html.replace(f"src='{orig}'", f'src="/media/{md5}"')
|
||
# Only rewrite hrefs for exact matches or URLs with media extensions
|
||
# (avoid rewriting external links that happen to match by filename)
|
||
url_path = res.split('?')[0]
|
||
ext = P(url_path).suffix.lower()
|
||
if res in exact_matches or ext in media_exts:
|
||
content_html = content_html.replace(f'href="{orig}"', f'href="/view/{md5}"')
|
||
content_html = content_html.replace(f"href='{orig}'", f'href="/view/{md5}"')
|
||
elif res in uri_to_page_hash:
|
||
# Rewrite internal page links to archived versions
|
||
page_hash = uri_to_page_hash[res]
|
||
content_html = content_html.replace(f'href="{orig}"', f'href="/page/{page_hash}"')
|
||
content_html = content_html.replace(f"href='{orig}'", f'href="/page/{page_hash}"')
|
||
except ImportError:
|
||
content_html = f"<pre>{html_module.escape(page.get('markdown', '')[:50000])}</pre>"
|
||
elif page.get("content"):
|
||
escaped = html_module.escape(page["content"][:50000])
|
||
content_html = f"<pre style='white-space:pre-wrap;'>{escaped}</pre>"
|
||
|
||
# Get screenshots and media
|
||
screenshot_hashes = await db.get_page_screenshots(uri)
|
||
media_items = await db.get_page_media(uri)
|
||
|
||
# Metadata rows for page view
|
||
import hashlib
|
||
uri_hash = hashlib.md5(uri.encode()).hexdigest()
|
||
neopig_page_uri = f"/page/{uri_hash}"
|
||
keywords = json.loads(page.get('keywords') or '[]') if page.get('keywords') else []
|
||
keywords_html = ''.join([f'<span class="tag">{k}</span>' for k in keywords]) or '-'
|
||
meta_rows = [
|
||
("{{source_uri}}", f'<a href="{uri}" target="_blank" style="font-size:11px;">{uri}</a>'),
|
||
("{{neopig_uri}}", f'<a href="{neopig_page_uri}" style="font-size:11px;">{neopig_page_uri}</a>'),
|
||
("{{description_label}}", page.get('description') or '-'),
|
||
("{{keywords_label}}", keywords_html),
|
||
("{{media}}", f"{len(media_items)}"),
|
||
]
|
||
|
||
return render_detail_page(
|
||
title=page_title,
|
||
hero_html="", # No hero media for page view
|
||
meta_rows=meta_rows,
|
||
media_items=media_items,
|
||
content_html=content_html,
|
||
screenshot_hashes=screenshot_hashes,
|
||
source_domain=source_domain,
|
||
page_uri=uri,
|
||
noai=noai,
|
||
lang=language,
|
||
)
|
||
|
||
|
||
@app.get("/phantom/export")
|
||
async def phantom_export(domain: str = Query(None, description="Filter by domain")):
|
||
"""
|
||
Export a phantom HTML site - original HTML with media URLs rewritten to vault.
|
||
|
||
Creates a downloadable zip of the phantom site ready for static hosting.
|
||
"""
|
||
import io
|
||
import re
|
||
import zipfile
|
||
from urllib.parse import urljoin
|
||
from fastapi.responses import StreamingResponse
|
||
|
||
# Get all pages with raw_html
|
||
pages = await db.get_pages_by_domain(domain)
|
||
|
||
if not pages:
|
||
raise HTTPException(status_code=404, detail="No pages with raw HTML found")
|
||
|
||
# Get all media URL to hash mappings
|
||
url_to_hash = await db.get_all_media_uri_mappings()
|
||
|
||
# Create zip in memory
|
||
zip_buffer = io.BytesIO()
|
||
|
||
with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||
pages_written = 0
|
||
media_hashes = set()
|
||
|
||
for page in pages:
|
||
uri = page['uri']
|
||
raw_html = page.get('raw_html')
|
||
if not raw_html:
|
||
continue
|
||
|
||
# Parse URI to get path
|
||
parsed = Uri(uri)
|
||
site_domain = parsed.hostname
|
||
path = parsed.path.strip('/') or 'index'
|
||
if not path.endswith('.html') and '.' not in path.split('/')[-1]:
|
||
path = f"{path}/index.html" if path else "index.html"
|
||
|
||
# Rewrite media URLs to local paths
|
||
html = raw_html
|
||
|
||
# Find all src and href attributes pointing to media
|
||
patterns = [
|
||
(r'src=["\']([^"\']+)["\']', 'src'),
|
||
(r'href=["\']([^"\']+\.(jpg|jpeg|png|gif|webp|mp4|webm|svg|ico))["\']', 'href'),
|
||
]
|
||
|
||
for pattern, attr in patterns:
|
||
matches = re.findall(pattern, html, re.IGNORECASE)
|
||
for match in matches:
|
||
url = match[0] if isinstance(match, tuple) else match
|
||
|
||
# Resolve relative URLs
|
||
full_url = urljoin(uri, url)
|
||
|
||
# Check if we have this media
|
||
if full_url in url_to_hash:
|
||
md5 = url_to_hash[full_url]
|
||
media_hashes.add(md5)
|
||
# Replace with local path
|
||
ext = Path(url).suffix or '.bin'
|
||
local_path = f"media/{md5}{ext}"
|
||
html = html.replace(f'{attr}="{url}"', f'{attr}="{local_path}"')
|
||
html = html.replace(f"{attr}='{url}'", f'{attr}="{local_path}"')
|
||
elif url in url_to_hash:
|
||
md5 = url_to_hash[url]
|
||
media_hashes.add(md5)
|
||
ext = Path(url).suffix or '.bin'
|
||
local_path = f"media/{md5}{ext}"
|
||
html = html.replace(f'{attr}="{url}"', f'{attr}="{local_path}"')
|
||
html = html.replace(f"{attr}='{url}'", f'{attr}="{local_path}"')
|
||
|
||
# Write HTML file
|
||
zf.writestr(f"site/{path}", html.encode('utf-8'))
|
||
pages_written += 1
|
||
|
||
# Copy media files from vault
|
||
media_copied = 0
|
||
for md5 in media_hashes:
|
||
subdir = VAULT_PATH / md5[:2]
|
||
if subdir.exists():
|
||
for f in subdir.iterdir():
|
||
if f.name.startswith(md5):
|
||
ext = f.suffix or '.bin'
|
||
zf.write(f, f"site/media/{md5}{ext}")
|
||
media_copied += 1
|
||
break
|
||
|
||
# Write index
|
||
index_html = f"""<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>Phantom Site - {site_domain}</title>
|
||
<style>
|
||
body {{ font-family: sans-serif; max-width: 800px; margin: 50px auto; padding: 20px; }}
|
||
h1 {{ color: #333; }}
|
||
ul {{ line-height: 2; }}
|
||
a {{ color: #0066cc; }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<h1>Phantom Site Archive</h1>
|
||
<p>Domain: {site_domain}</p>
|
||
<p>Pages: {pages_written}</p>
|
||
<p>Media: {media_copied}</p>
|
||
<h2>Pages</h2>
|
||
<ul>
|
||
"""
|
||
for page in pages[:100]:
|
||
parsed = Uri(page['uri'])
|
||
path = parsed.path.strip('/') or 'index'
|
||
if not path.endswith('.html') and '.' not in path.split('/')[-1]:
|
||
path = f"{path}/index.html" if path else "index.html"
|
||
title = page.get('title') or path
|
||
index_html += f' <li><a href="{path}">{title}</a></li>\n'
|
||
|
||
index_html += """ </ul>
|
||
</body>
|
||
</html>"""
|
||
zf.writestr("site/phantom_index.html", index_html.encode('utf-8'))
|
||
|
||
# Return zip
|
||
zip_buffer.seek(0)
|
||
return StreamingResponse(
|
||
zip_buffer,
|
||
media_type="application/zip",
|
||
headers={"Content-Disposition": f"attachment; filename=phantom_{site_domain or 'site'}.zip"}
|
||
)
|
||
|
||
|
||
@app.get("/phantom", response_class=HTMLResponse)
|
||
async def phantom_page(lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Phantom site export UI."""
|
||
language = get_lang(lang, accept_language)
|
||
domains = await db.get_domains_with_pages()
|
||
|
||
domain_options = ''.join([
|
||
f'<option value="{d[0]}">{d[0]} ({d[1]} pages)</option>'
|
||
for d in domains if d[0]
|
||
])
|
||
|
||
html = f"""
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<title>Phantom Site Export - neopig</title>
|
||
<style>
|
||
* {{ box-sizing: border-box; }}
|
||
body {{
|
||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||
margin: 0;
|
||
padding: 0;
|
||
background: #0a0a0a;
|
||
color: #e0e0e0;
|
||
}}
|
||
.nav {{
|
||
background: #1a1a1a;
|
||
padding: 10px 20px;
|
||
display: flex;
|
||
gap: 20px;
|
||
align-items: center;
|
||
border-bottom: 1px solid #333;
|
||
}}
|
||
.nav a {{ color: #ff6b6b; text-decoration: none; }}
|
||
.nav a:hover {{ text-decoration: underline; }}
|
||
.nav .brand {{ font-weight: bold; font-size: 18px; }}
|
||
.container {{ padding: 20px; }}
|
||
h1 {{ color: #ff6b6b; margin-bottom: 10px; }}
|
||
.subtitle {{ color: #888; margin-bottom: 30px; }}
|
||
.form-group {{ margin-bottom: 20px; }}
|
||
label {{ display: block; margin-bottom: 8px; color: #aaa; }}
|
||
select {{
|
||
width: 100%;
|
||
padding: 12px 16px;
|
||
font-size: 16px;
|
||
border: 2px solid #333;
|
||
border-radius: 8px;
|
||
background: #1a1a1a;
|
||
color: #fff;
|
||
}}
|
||
button {{
|
||
padding: 14px 28px;
|
||
font-size: 16px;
|
||
background: #ff6b6b;
|
||
color: #fff;
|
||
border: none;
|
||
border-radius: 8px;
|
||
cursor: pointer;
|
||
}}
|
||
button:hover {{ background: #ff5252; }}
|
||
.info {{
|
||
background: #1a1a1a;
|
||
padding: 20px;
|
||
border-radius: 8px;
|
||
margin-top: 30px;
|
||
}}
|
||
.info h3 {{ color: #ff6b6b; margin-top: 0; }}
|
||
.info ul {{ color: #aaa; line-height: 1.8; }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<!-- NAV -->
|
||
<div class="container">
|
||
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
||
<input type="text" name="q" placeholder="{{search_placeholder}}" style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||
<option value="">{{all_types}}</option>
|
||
<option value="image">{{images}}</option>
|
||
<option value="video">{{videos}}</option>
|
||
<option value="audio">{{audio}}</option>
|
||
</select>
|
||
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">{{search}}</button>
|
||
</form>
|
||
<h1>Phantom Site Export</h1>
|
||
<p class="subtitle">{{phantom_subtitle}}</p>
|
||
|
||
<form action="/phantom/export" method="get">
|
||
<div class="form-group">
|
||
<label>{{select_domain}}</label>
|
||
<select name="domain">
|
||
<option value="">{{all_domains}}</option>
|
||
{domain_options}
|
||
</select>
|
||
</div>
|
||
<button type="submit">{{download_phantom}} (.zip)</button>
|
||
</form>
|
||
|
||
<div class="info">
|
||
<h3>What is a Phantom Site?</h3>
|
||
<ul>
|
||
<li>Original HTML preserved exactly as crawled</li>
|
||
<li>All media URLs rewritten to local paths</li>
|
||
<li>Ready to host statically (nginx, Caddy, S3, etc.)</li>
|
||
<li>Works offline - all assets included</li>
|
||
<li>Perfect for archival and preservation</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</body>
|
||
</html>
|
||
"""
|
||
return inject_i18n(html, language)
|
||
|
||
|
||
@app.get("/about", response_class=HTMLResponse)
|
||
async def about_page(lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""About neopig - the story of pig.py's evolution."""
|
||
language = get_lang(lang, accept_language)
|
||
html = f"""
|
||
<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<title>{{about}} neopig</title>
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<style>
|
||
{VIEW_CSS}
|
||
.about-content {{
|
||
max-width: 800px;
|
||
margin: 0 auto;
|
||
padding: 40px 20px;
|
||
line-height: 1.8;
|
||
font-size: 17px;
|
||
}}
|
||
.about-content h1 {{
|
||
font-size: 2.5em;
|
||
margin-bottom: 10px;
|
||
color: #ff6b6b;
|
||
}}
|
||
.about-content h2 {{
|
||
font-size: 1.6em;
|
||
margin-top: 40px;
|
||
margin-bottom: 15px;
|
||
color: #4ecdc4;
|
||
border-bottom: 2px solid #333;
|
||
padding-bottom: 10px;
|
||
}}
|
||
.about-content p {{
|
||
margin-bottom: 20px;
|
||
color: #e0e0e0;
|
||
}}
|
||
.about-content a {{
|
||
color: #ff6b6b;
|
||
text-decoration: none;
|
||
}}
|
||
.about-content a:hover {{
|
||
text-decoration: underline;
|
||
}}
|
||
.about-content code {{
|
||
background: #2a2a2a;
|
||
padding: 2px 8px;
|
||
border-radius: 4px;
|
||
font-family: 'Fira Code', monospace;
|
||
font-size: 0.9em;
|
||
}}
|
||
.about-content pre {{
|
||
background: #1a1a1a;
|
||
padding: 20px;
|
||
border-radius: 8px;
|
||
overflow-x: auto;
|
||
border: 1px solid #333;
|
||
}}
|
||
.about-content pre code {{
|
||
background: none;
|
||
padding: 0;
|
||
}}
|
||
.about-content ul, .about-content ol {{
|
||
margin-bottom: 20px;
|
||
padding-left: 30px;
|
||
}}
|
||
.about-content li {{
|
||
margin-bottom: 10px;
|
||
color: #e0e0e0;
|
||
}}
|
||
.feature-grid {{
|
||
display: grid;
|
||
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
|
||
gap: 20px;
|
||
margin: 30px 0;
|
||
}}
|
||
.feature-card {{
|
||
background: #1e1e24;
|
||
padding: 20px;
|
||
border-radius: 12px;
|
||
border: 1px solid #333;
|
||
}}
|
||
.feature-card h3 {{
|
||
color: #ff6b6b;
|
||
margin-bottom: 10px;
|
||
font-size: 1.1em;
|
||
}}
|
||
.feature-card p {{
|
||
color: #aaa;
|
||
font-size: 0.95em;
|
||
margin: 0;
|
||
}}
|
||
.origin-quote {{
|
||
background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%);
|
||
border-left: 4px solid #ff6b6b;
|
||
padding: 20px 25px;
|
||
margin: 30px 0;
|
||
border-radius: 0 8px 8px 0;
|
||
font-style: italic;
|
||
}}
|
||
.origin-quote cite {{
|
||
display: block;
|
||
margin-top: 10px;
|
||
color: #888;
|
||
font-style: normal;
|
||
}}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<!-- NAV -->
|
||
<div class="about-content">
|
||
<h1>🐷 neopig</h1>
|
||
<p><strong>Neo Python Image Grabber</strong> — A full-domain async media crawler with content-addressable storage, full-text search, and page screenshot capture.</p>
|
||
|
||
<h2>The Origin Story</h2>
|
||
|
||
<div class="origin-quote">
|
||
"pig.py is a <em>very</em> simple python command line tool to download all the images from a given uri."
|
||
<cite>— Russell Ballestrini, August 22, 2011</cite>
|
||
</div>
|
||
|
||
<p>Back in 2011, <a href="https://russell.ballestrini.net/python-image-grabber-pig-py/" target="_blank">Russell Ballestrini created pig.py</a> — a delightfully simple Python script that did one thing well: download all the images from a webpage. It was placed in the public domain, a gift to anyone who needed to grab images from the web.</p>
|
||
|
||
<p>The original pig.py was elegant in its simplicity:</p>
|
||
|
||
<pre><code>python pig.py https://www.foxhop.net</code></pre>
|
||
|
||
<p>That was it. Point it at a URL, and it would slurp down every image it could find. No configuration, no complexity — just a hungry little pig gobbling up pixels.</p>
|
||
|
||
<h2>The Evolution to neopig</h2>
|
||
|
||
<p>Over a decade later, neopig emerged as a spiritual successor — keeping the original's spirit of simplicity while adding the features needed for serious web archival:</p>
|
||
|
||
<div class="feature-grid">
|
||
<div class="feature-card">
|
||
<h3>🔄 Async Crawling</h3>
|
||
<p>Full-domain recursive crawling with configurable depth, respecting robots.txt and crawl delays.</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>💾 Content-Addressed Storage</h3>
|
||
<p>MD5-based deduplication in a vault system. Same image from 100 pages? Stored once.</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>🔍 Full-Text Search</h3>
|
||
<p>Every image indexed with its surrounding context — page title, alt text, captions, nearby headings.</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>📸 Page Screenshots</h3>
|
||
<p><a href="https://linkpeek.com">LinkPeek</a> reborn! Full-page captures via <a href="https://github.com/russellballestrini/uri2png">uri2png</a> (wkhtmltoimage, Playwright, etc).</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>📝 Markdown Conversion</h3>
|
||
<p>Intelligent HTML-to-Markdown with forum post detection, noise removal, and content extraction.</p>
|
||
</div>
|
||
<div class="feature-card">
|
||
<h3>📦 Distributable Archives</h3>
|
||
<p>Create self-contained tar.gz packages with embedded search servers for offline browsing.</p>
|
||
</div>
|
||
</div>
|
||
|
||
<h2>The Skeleton Key Approach</h2>
|
||
|
||
<p>neopig uses what we call the "skeleton key" approach to media indexing. For every image, we capture <em>all</em> the text that might help you find it later:</p>
|
||
|
||
<ul>
|
||
<li><strong>Page context:</strong> Title, description, headings near the image</li>
|
||
<li><strong>Image attributes:</strong> Alt text, title, surrounding captions</li>
|
||
<li><strong>Link context:</strong> The text of links pointing to the image</li>
|
||
<li><strong>Detail pages:</strong> For gallery sites, we follow through to detail pages</li>
|
||
</ul>
|
||
|
||
<p>The result? You can search for "sunset over mountains" and find that image even if it was named <code>IMG_4372.jpg</code> with no alt text — because the page title mentioned it, or someone linked to it with descriptive text.</p>
|
||
|
||
<h2>Preserving the Web</h2>
|
||
|
||
<p>The web is ephemeral. Sites go dark. Forums shut down. Communities scatter. neopig is built for preservation — capturing not just the media, but the <em>context</em> that gives it meaning.</p>
|
||
|
||
<p>When you archive a site with neopig, you get:</p>
|
||
|
||
<ul>
|
||
<li>Original HTML with rewritten media links pointing to your local vault</li>
|
||
<li>Full-text searchable markdown versions of every page</li>
|
||
<li>Screenshots showing exactly how pages looked</li>
|
||
<li>A SQLite database you can query, backup, and migrate</li>
|
||
<li>Self-extracting archives that work offline forever</li>
|
||
</ul>
|
||
|
||
<h2>The Name</h2>
|
||
|
||
<p><strong>neopig</strong> = <strong>Neo</strong> (new) + <strong>P</strong>ython <strong>I</strong>mage <strong>G</strong>rabber</p>
|
||
|
||
<p>A tip of the hat to the original pig.py, with a nod to the Matrix's Neo — seeing through the surface of the web to the underlying content within.</p>
|
||
|
||
<h2>Get Started</h2>
|
||
|
||
<pre><code># Archive a site and serve it live as you crawl
|
||
python neopig.py https://discourse-urho3d.github.io --archive --serve
|
||
|
||
# Fast mode for static sites (no crawl delay)
|
||
python neopig.py https://russell.ballestrini.net --archive --fast -o ./archives/
|
||
|
||
# Re-run with fresh markdown conversion
|
||
python neopig.py --fresh --backfill-markdown example.com
|
||
|
||
# Upgrade an existing archive with latest neopig/serp
|
||
python neopig.py --upgrade-neopig ./archives/example.com-20251231.tar.gz</code></pre>
|
||
|
||
<p style="margin-top: 40px; padding-top: 20px; border-top: 1px solid #333; color: #888; font-size: 0.9em;">
|
||
neopig is open source. The original pig.py was placed in the public domain by Russell Ballestrini.
|
||
This project continues that tradition of building useful tools for the community.
|
||
</p>
|
||
</div>
|
||
</body>
|
||
</html>
|
||
"""
|
||
return inject_i18n(html, language)
|
||
|
||
|
||
@app.get("/health")
|
||
async def health():
|
||
"""Health check endpoint."""
|
||
has_screenshot = False
|
||
try:
|
||
from uri2png import get_available_engines
|
||
has_screenshot = True
|
||
except ImportError:
|
||
pass
|
||
|
||
return {
|
||
"status": "healthy",
|
||
"features": {
|
||
"search": True,
|
||
"crawl": True,
|
||
"screenshot": has_screenshot
|
||
}
|
||
}
|
||
|
||
|
||
@app.get("/api/stats")
|
||
async def get_stats_endpoint():
|
||
"""Get database statistics."""
|
||
stats = await db.get_stats()
|
||
# Add page count (handled separately since table may not exist)
|
||
try:
|
||
from sqlalchemy import select, func
|
||
from database import Page
|
||
async with db.session() as session:
|
||
result = await session.execute(select(func.count()).select_from(Page))
|
||
stats['total_pages'] = result.scalar() or 0
|
||
except Exception:
|
||
stats['total_pages'] = 0
|
||
return stats
|
||
|
||
|
||
@app.get("/random")
|
||
async def random_item(type: str = Query(None, description="Type: media or page (random if not specified)")):
|
||
"""Redirect to a random media item or page."""
|
||
from sqlalchemy import select, func
|
||
from database import Media, Page
|
||
import random
|
||
|
||
# If no type specified, randomly pick between media and page
|
||
if type is None:
|
||
type = random.choice(["media", "page"])
|
||
|
||
async with db.session() as session:
|
||
if type == "page":
|
||
# Get a random page
|
||
stmt = select(Page.uri_hash).order_by(func.random()).limit(1)
|
||
result = await session.execute(stmt)
|
||
row = result.fetchone()
|
||
if row and row[0]:
|
||
return RedirectResponse(url=f"/page/{row[0]}", status_code=302)
|
||
else:
|
||
# Get a random media item (excluding screenshots)
|
||
stmt = (
|
||
select(Media.md5_hash)
|
||
.where(Media.media_type != 'screenshot')
|
||
.order_by(func.random())
|
||
.limit(1)
|
||
)
|
||
result = await session.execute(stmt)
|
||
row = result.fetchone()
|
||
if row:
|
||
return RedirectResponse(url=f"/view/{row[0]}", status_code=302)
|
||
|
||
return RedirectResponse(url="/", status_code=302)
|
||
|
||
|
||
@app.get("/api/search")
|
||
async def search(
|
||
q: str = Query("", description="Search query"),
|
||
type: Optional[str] = Query(None, description="Filter by media type"),
|
||
status: Optional[str] = Query(None, description="Filter by analysis status"),
|
||
limit: int = Query(OVER_9000, le=OVER_9000),
|
||
offset: int = Query(0)
|
||
):
|
||
"""
|
||
Search media by text query.
|
||
|
||
Searches across: keywords, alt_text, title, source URLs, analysis results.
|
||
"""
|
||
results = await db.search_media_advanced(
|
||
q=q if q else None,
|
||
media_type=type,
|
||
limit=limit,
|
||
offset=offset
|
||
)
|
||
return results
|
||
|
||
|
||
@app.get("/api/search/pages")
|
||
async def search_pages_endpoint(
|
||
q: str = Query("", description="Search query"),
|
||
limit: int = Query(50, le=500),
|
||
):
|
||
"""
|
||
Search pages by text query using FTS5.
|
||
"""
|
||
if not q:
|
||
return []
|
||
return await db.search_pages(q, limit)
|
||
|
||
|
||
@app.get("/api/upgraded")
|
||
async def get_upgraded_media(limit: int = Query(50, le=200)):
|
||
"""
|
||
Get media that was recently upgraded (found higher quality version).
|
||
Used by live feed to show when thumbnails get replaced by full-res.
|
||
"""
|
||
return await db.get_recently_upgraded(limit)
|
||
|
||
|
||
@app.get("/api/media/{md5_hash}")
|
||
async def get_media_info(md5_hash: str):
|
||
"""Get full media info including all source URLs."""
|
||
media = await db.get_media_by_hash(md5_hash)
|
||
if not media:
|
||
raise HTTPException(status_code=404, detail="Media not found")
|
||
|
||
media['sources'] = await db.get_media_sources(md5_hash)
|
||
return media
|
||
|
||
|
||
def slugify(text: str, max_len: int = 60) -> str:
|
||
"""Convert text to a safe filename slug."""
|
||
import re
|
||
import unicodedata
|
||
# Normalize unicode
|
||
text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('ascii')
|
||
# Lowercase and replace spaces/special chars with hyphens
|
||
text = re.sub(r'[^\w\s-]', '', text.lower())
|
||
text = re.sub(r'[-\s]+', '-', text).strip('-')
|
||
return text[:max_len] if text else ""
|
||
|
||
|
||
@app.get("/media/{md5_hash}")
|
||
async def serve_media(md5_hash: str, download: bool = False):
|
||
"""
|
||
Serve media file from vault or tarball.
|
||
|
||
Use ?download=1 for attachment mode with smart filename.
|
||
Caddy should be configured to cache these responses.
|
||
"""
|
||
# Tarball mode: serve from tar.gz
|
||
if TAR_FILE:
|
||
data, ext = find_media_in_tarball(md5_hash)
|
||
if data:
|
||
mime_type, _ = mimetypes.guess_type(f"file{ext}")
|
||
if not mime_type:
|
||
mime_type = "application/octet-stream"
|
||
filename = f"{md5_hash[:12]}{ext}"
|
||
headers = {
|
||
"Cache-Control": "public, max-age=31536000, immutable",
|
||
"X-Content-Hash": md5_hash,
|
||
}
|
||
if download:
|
||
headers["Content-Disposition"] = f'attachment; filename="{filename}"'
|
||
return Response(content=data, media_type=mime_type, headers=headers)
|
||
raise HTTPException(status_code=404, detail="Media not found in archive")
|
||
|
||
# Filesystem mode: find file in vault (9-deep path)
|
||
subdir = VAULT_PATH / hash_to_path(md5_hash).parent
|
||
if not subdir.exists():
|
||
raise HTTPException(status_code=404, detail="Media not found")
|
||
|
||
# Find file with this hash prefix
|
||
for f in subdir.iterdir():
|
||
if f.name.startswith(md5_hash):
|
||
# Guess content type and get extension
|
||
mime_type, _ = mimetypes.guess_type(f.name)
|
||
ext = f.suffix or ""
|
||
|
||
# Get metadata for filename generation
|
||
filename = None
|
||
media_record = await db.get_media_by_hash(md5_hash)
|
||
if media_record:
|
||
if not mime_type and media_record.get("mime_type"):
|
||
mime_type = media_record["mime_type"]
|
||
# Generate filename from alt_text or title
|
||
name_source = media_record.get("alt_text") or media_record.get("title")
|
||
if name_source:
|
||
slug = slugify(name_source)
|
||
if slug:
|
||
filename = f"{slug}{ext}"
|
||
|
||
# Fallback: try to get page_title from media_sources
|
||
if not filename:
|
||
sources = await db.get_media_sources(md5_hash)
|
||
if sources:
|
||
row2 = sources[0]
|
||
# Try page_title + hash index
|
||
if row2.get("page_title"):
|
||
media_idx = int(md5_hash[:4], 16)
|
||
slug = slugify(f"{row2['page_title']}-{media_idx}")
|
||
if slug:
|
||
filename = f"{slug}{ext}"
|
||
# Fallback: original filename from URL
|
||
if not filename and row2.get("media_uri"):
|
||
from urllib.parse import unquote
|
||
parsed = Uri(row2["media_uri"])
|
||
orig_name = Path(unquote(parsed.path)).name
|
||
if orig_name and '.' in orig_name:
|
||
filename = orig_name
|
||
|
||
if not mime_type:
|
||
mime_type = "application/octet-stream"
|
||
|
||
# Default filename if nothing else
|
||
if not filename:
|
||
filename = f"{md5_hash[:12]}{ext}"
|
||
|
||
# Download mode: attachment with smart filename
|
||
# Inline mode: no filename header, browser shows inline
|
||
if download:
|
||
return FileResponse(
|
||
f,
|
||
media_type=mime_type,
|
||
filename=filename,
|
||
content_disposition_type="attachment",
|
||
headers={
|
||
"Cache-Control": "public, max-age=31536000, immutable",
|
||
"X-Content-Hash": md5_hash,
|
||
}
|
||
)
|
||
else:
|
||
return FileResponse(
|
||
f,
|
||
media_type=mime_type,
|
||
content_disposition_type="inline",
|
||
headers={
|
||
"Cache-Control": "public, max-age=31536000, immutable",
|
||
"X-Content-Hash": md5_hash,
|
||
}
|
||
)
|
||
|
||
raise HTTPException(status_code=404, detail="Media not found")
|
||
|
||
|
||
# ============================================================================
|
||
# Crawler API
|
||
# ============================================================================
|
||
|
||
@app.get("/api/crawl/jobs")
|
||
async def get_crawl_jobs_endpoint(limit: int = Query(50, le=200)):
|
||
"""Get recent jobs (crawl + backfill)."""
|
||
try:
|
||
# Get both crawl jobs and backfill jobs
|
||
crawl_jobs = await db.get_crawl_jobs(limit)
|
||
backfill_jobs = await db.get_backfill_jobs(limit)
|
||
|
||
# Add job_kind to distinguish them
|
||
for job in crawl_jobs:
|
||
job['job_kind'] = 'crawl'
|
||
for job in backfill_jobs:
|
||
job['job_kind'] = 'backfill'
|
||
|
||
# Merge and sort by started_at descending
|
||
all_jobs = crawl_jobs + backfill_jobs
|
||
all_jobs.sort(key=lambda j: j.get('started_at', ''), reverse=True)
|
||
|
||
return all_jobs[:limit]
|
||
except Exception as e:
|
||
logger.error(f"Failed to get jobs: {e}")
|
||
raise HTTPException(status_code=500, detail=str(e))
|
||
|
||
|
||
@app.post("/api/crawl/jobs/{job_id}/pause")
|
||
async def pause_crawl_job(job_id: int):
|
||
"""Pause a running crawl job."""
|
||
task = ACTIVE_CRAWL_TASKS.get(job_id)
|
||
if not task:
|
||
raise HTTPException(status_code=404, detail="Job not running")
|
||
|
||
task.cancel()
|
||
try:
|
||
await task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
|
||
await db.pause_crawl_job(job_id)
|
||
ACTIVE_CRAWL_TASKS.pop(job_id, None)
|
||
|
||
return {"status": "paused", "job_id": job_id}
|
||
|
||
|
||
@app.post("/api/crawl/jobs/{job_id}/resume")
|
||
async def resume_crawl_job(job_id: int):
|
||
"""Resume a paused crawl job."""
|
||
job = await db.get_crawl_job(job_id)
|
||
if not job:
|
||
raise HTTPException(status_code=404, detail="Job not found")
|
||
|
||
if job['status'] not in ('paused', 'cancelled'):
|
||
raise HTTPException(status_code=400, detail=f"Job is {job['status']}, cannot resume")
|
||
|
||
if job_id in ACTIVE_CRAWL_TASKS:
|
||
raise HTTPException(status_code=400, detail="Job is already running")
|
||
|
||
# Mark as running
|
||
async with db.session() as session:
|
||
await session.execute(
|
||
text("UPDATE crawl_jobs SET status = 'running' WHERE id = :id"),
|
||
{"id": job_id}
|
||
)
|
||
await session.commit()
|
||
|
||
# Start crawl task (resume from state files)
|
||
target_uri = job['target_uri']
|
||
keywords = json.loads(job.get('keywords') or '[]')
|
||
mode_str = job.get('mode', 'all')
|
||
|
||
async def run_crawl(jid=job_id, uri=target_uri):
|
||
try:
|
||
from neopig import NeoPig
|
||
from async_web_fetcher import CrawlMode
|
||
from screenshot import ScreenshotConfig
|
||
screenshot_config = ScreenshotConfig(enabled=True)
|
||
pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH), screenshot_config=screenshot_config)
|
||
await pig.init()
|
||
|
||
# Resume from state (don't clear state files)
|
||
crawled_media = await pig.db.get_crawled_media_uris()
|
||
crawled_screenshots = await pig.db.get_crawled_screenshot_uris()
|
||
if crawled_media:
|
||
pig.seen_media = crawled_media
|
||
if crawled_screenshots:
|
||
pig.seen_screenshots = crawled_screenshots
|
||
|
||
mode = CrawlMode(mode_str) if mode_str else CrawlMode.ALL
|
||
|
||
# Progress updater
|
||
stop_progress = asyncio.Event()
|
||
|
||
async def update_progress():
|
||
while not stop_progress.is_set():
|
||
await asyncio.sleep(2)
|
||
if not stop_progress.is_set():
|
||
await db.update_crawl_job_stats(jid, pig.stats)
|
||
|
||
progress_task = asyncio.create_task(update_progress())
|
||
|
||
try:
|
||
stats = await pig.crawl(
|
||
target_uri=uri,
|
||
keywords=keywords,
|
||
mode=mode,
|
||
depth=15,
|
||
max_pages=-1,
|
||
job_id=jid,
|
||
quiet=True,
|
||
)
|
||
finally:
|
||
stop_progress.set()
|
||
progress_task.cancel()
|
||
try:
|
||
await progress_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
|
||
await db.complete_crawl_job(jid, stats)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Resumed crawl job {jid} failed: {e}")
|
||
await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})
|
||
|
||
task = asyncio.create_task(run_crawl())
|
||
ACTIVE_CRAWL_TASKS[job_id] = task
|
||
task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))
|
||
|
||
return {"status": "running", "job_id": job_id}
|
||
|
||
|
||
@app.delete("/api/crawl/jobs/{job_id}")
|
||
async def delete_crawl_job(job_id: int):
|
||
"""Delete a crawl job."""
|
||
# Can't delete running jobs
|
||
if job_id in ACTIVE_CRAWL_TASKS:
|
||
raise HTTPException(status_code=400, detail="Cannot delete running job. Pause it first.")
|
||
|
||
deleted = await db.delete_crawl_job(job_id)
|
||
if not deleted:
|
||
raise HTTPException(status_code=404, detail="Job not found")
|
||
|
||
return {"status": "deleted", "job_id": job_id}
|
||
|
||
|
||
@app.get("/api/crawl/jobs/{job_id}")
|
||
async def get_crawl_job_endpoint(job_id: int):
|
||
"""Get a specific crawl job."""
|
||
job = await db.get_crawl_job(job_id)
|
||
if not job:
|
||
raise HTTPException(status_code=404, detail="Job not found")
|
||
return job
|
||
|
||
|
||
@app.post("/api/crawl")
|
||
async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
|
||
"""
|
||
Start new crawl job(s).
|
||
|
||
Supports multiple targets - creates one job per target.
|
||
The crawls run in the background. Poll /api/crawl/jobs/{id} for status.
|
||
"""
|
||
# Import neopig here to avoid circular imports
|
||
from neopig import NeoPig
|
||
from async_web_fetcher import CrawlMode
|
||
|
||
# Map mode string to enum
|
||
mode_map = {
|
||
"text": CrawlMode.TEXT,
|
||
"images": CrawlMode.IMAGES,
|
||
"videos": CrawlMode.VIDEOS,
|
||
"media": CrawlMode.MEDIA,
|
||
"all": CrawlMode.ALL,
|
||
}
|
||
mode = mode_map.get(request.mode, CrawlMode.IMAGES)
|
||
|
||
# Get targets (support both new 'targets' array and old 'target_uri' single value)
|
||
targets = request.targets if request.targets else [request.target_uri] if request.target_uri else []
|
||
if not targets:
|
||
raise HTTPException(status_code=400, detail="No target URIs provided")
|
||
|
||
job_ids = []
|
||
|
||
# Create a job for each target
|
||
for target_uri in targets:
|
||
job_id = await db.create_crawl_job(target_uri, request.keywords, request.mode)
|
||
job_ids.append(job_id)
|
||
|
||
# Run crawl in background (closure captures job_id and target_uri)
|
||
async def run_crawl(jid=job_id, uri=target_uri):
|
||
try:
|
||
from screenshot import ScreenshotConfig
|
||
screenshot_config = ScreenshotConfig(enabled=request.screenshots)
|
||
pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH), fast_mode=request.fast, screenshot_config=screenshot_config)
|
||
await pig.init()
|
||
|
||
# Handle fresh vs resume
|
||
if request.fresh:
|
||
pig._clear_state(uri)
|
||
else:
|
||
# Load existing seen media for resume capability
|
||
crawled_media = await pig.db.get_crawled_media_uris()
|
||
crawled_screenshots = await pig.db.get_crawled_screenshot_uris()
|
||
if crawled_media:
|
||
pig.seen_media = crawled_media
|
||
if crawled_screenshots:
|
||
pig.seen_screenshots = crawled_screenshots
|
||
|
||
# Cap depth at 15 (convert -1 or values > 15 to 15)
|
||
depth = request.depth if 1 <= request.depth <= 15 else 15
|
||
|
||
# Progress updater - runs alongside crawl
|
||
stop_progress = asyncio.Event()
|
||
|
||
async def update_progress():
|
||
while not stop_progress.is_set():
|
||
await asyncio.sleep(2) # Update every 2 seconds
|
||
if not stop_progress.is_set():
|
||
await db.update_crawl_job_stats(jid, pig.stats)
|
||
|
||
progress_task = asyncio.create_task(update_progress())
|
||
|
||
try:
|
||
stats = await pig.crawl(
|
||
target_uri=uri,
|
||
keywords=request.keywords,
|
||
mode=mode,
|
||
depth=depth,
|
||
max_pages=request.max_pages,
|
||
job_id=jid, # Use existing job, don't create another
|
||
quiet=True, # No progress bar for UI-initiated crawls
|
||
)
|
||
finally:
|
||
stop_progress.set()
|
||
progress_task.cancel()
|
||
try:
|
||
await progress_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
|
||
# Update job as completed
|
||
await db.complete_crawl_job(jid, stats)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Crawl job {jid} failed: {e}")
|
||
await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})
|
||
|
||
# Schedule async task on the event loop (not BackgroundTasks which runs in threadpool)
|
||
task = asyncio.create_task(run_crawl())
|
||
ACTIVE_CRAWL_TASKS[job_id] = task
|
||
# Clean up when done
|
||
task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))
|
||
|
||
return {"job_ids": job_ids, "status": "running", "count": len(job_ids)}
|
||
|
||
|
||
def init_tarball_mode(tarball_path: str):
|
||
"""Initialize serving from a tar.gz archive."""
|
||
global TAR_FILE, TAR_MEMBERS, ARCHIVE_ROOT, DB_PATH, TEMP_DB_PATH
|
||
|
||
logger.info(f"Opening archive: {tarball_path}")
|
||
tarball = Path(tarball_path)
|
||
|
||
# Handle .run files with NEOPIG trailer
|
||
offset = 0
|
||
if tarball.suffix == '.run' or tarball.stat().st_size > 100000:
|
||
try:
|
||
with open(tarball, 'rb') as f:
|
||
f.seek(-22, 2)
|
||
trailer = f.read(22)
|
||
if trailer[:6] == b'NEOPIG':
|
||
offset = int(trailer[6:22].decode(), 16)
|
||
logger.info(f"Detected .run format, tarball offset: {offset}")
|
||
except Exception:
|
||
pass
|
||
|
||
# Open tarball
|
||
if offset > 0:
|
||
# Create a wrapper that presents just the tarball portion
|
||
class OffsetFile:
|
||
"""File wrapper that starts reading from an offset."""
|
||
def __init__(self, path, offset):
|
||
self._f = open(path, 'rb')
|
||
self._offset = offset
|
||
self._f.seek(offset)
|
||
def read(self, size=-1):
|
||
return self._f.read(size)
|
||
def seek(self, pos, whence=0):
|
||
if whence == 0: # SEEK_SET
|
||
return self._f.seek(self._offset + pos)
|
||
elif whence == 1: # SEEK_CUR
|
||
return self._f.seek(pos, 1)
|
||
else: # SEEK_END
|
||
return self._f.seek(pos, 2)
|
||
def tell(self):
|
||
return self._f.tell() - self._offset
|
||
def close(self):
|
||
self._f.close()
|
||
|
||
TAR_FILE = tarfile.open(fileobj=OffsetFile(tarball, offset), mode='r:gz')
|
||
else:
|
||
TAR_FILE = tarfile.open(tarball_path, 'r:gz')
|
||
|
||
# Build member lookup
|
||
for member in TAR_FILE.getmembers():
|
||
TAR_MEMBERS[member.name] = member
|
||
|
||
# Get archive root from first member
|
||
first = list(TAR_MEMBERS.keys())[0]
|
||
ARCHIVE_ROOT = first.split('/')[0]
|
||
logger.info(f"Archive root: {ARCHIVE_ROOT}")
|
||
|
||
# Extract database to temp (SQLite needs real file)
|
||
db_member = f"{ARCHIVE_ROOT}/archive.db"
|
||
if db_member in TAR_MEMBERS:
|
||
temp_dir = tempfile.mkdtemp(prefix="neopig_")
|
||
TEMP_DB_PATH = f"{temp_dir}/archive.db"
|
||
member = TAR_MEMBERS[db_member]
|
||
f = TAR_FILE.extractfile(member)
|
||
if f:
|
||
with open(TEMP_DB_PATH, 'wb') as out:
|
||
out.write(f.read())
|
||
DB_PATH = TEMP_DB_PATH
|
||
logger.info(f"Extracted database to: {TEMP_DB_PATH}")
|
||
else:
|
||
logger.warning("No archive.db found in tarball")
|
||
|
||
|
||
def main():
|
||
global DB_PATH, VAULT_PATH
|
||
|
||
parser = argparse.ArgumentParser(description="neopig SERP")
|
||
parser.add_argument("tarball", nargs='?', help="Path to archive.tar.gz or .run file")
|
||
parser.add_argument("--port", type=int, default=8000)
|
||
parser.add_argument("--host", default="127.0.0.1")
|
||
parser.add_argument("--db", default="data/neopig.db")
|
||
parser.add_argument("--vault", default="data/vault")
|
||
|
||
args = parser.parse_args()
|
||
|
||
# Tarball mode
|
||
if args.tarball:
|
||
init_tarball_mode(args.tarball)
|
||
logger.info(f"Starting neopig SERP (archive mode) on {args.host}:{args.port}")
|
||
else:
|
||
DB_PATH = args.db
|
||
VAULT_PATH = Path(args.vault)
|
||
logger.info(f"Starting neopig SERP on {args.host}:{args.port}")
|
||
logger.info(f"Database: {DB_PATH}")
|
||
logger.info(f"Vault: {VAULT_PATH}")
|
||
|
||
uvicorn.run(app, host=args.host, port=args.port)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|