#!/usr/bin/env python3
"""
neopig SERP - Search Engine Results Page

Fast image/video search across hydrated metadata.
Serves files directly from filevault via Caddy with 1GB memory cache.

Search across:
- keywords (crawl tags)
- alt_text (image alt attributes)
- title (media titles)
- analysis_result (Qwen 3 VL descriptions)
- source_page / source_url (origin)

Usage:
    python serp.py --port 31337 --vault ./vault --db neopig.db
"""

import argparse
import asyncio
import json
import logging
import mimetypes
import os
from pathlib import Path
from typing import List, Dict, Any, Optional

from fastapi import FastAPI, Query, HTTPException, BackgroundTasks, Header, Cookie, UploadFile, File, Request
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response, StreamingResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
from sqlalchemy import text
import uvicorn

from database import Database, OVER_9000
from filevault import hash_to_path
from miniuri import Uri
from neopig import get_live_queue

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# ============================================================================
# INTERNATIONALIZATION (i18n) - Top 10 Languages
# ============================================================================
TRANSLATIONS = {
    "en": {
        "search": "Search", "crawl": "Crawl", "live": "Live Feed", "about": "About",
        "random": "Random", "phantom": "Phantom", "loading": "Loading...",
        "start_crawl": "Start Crawl", "pause": "Pause", "resume": "Resume", "delete": "Delete",
        "url": "URL", "mode": "Mode", "depth": "Depth", "keywords": "Keywords",
        "screenshots": "Screenshots", "fast_mode": "Fast Mode", "fresh_start": "Fresh Start",
        "pages": "pages", "found": "found", "saved": "saved", "dupes": "dupes",
        "running": "running", "completed": "completed", "paused": "paused",
        "no_jobs": "No crawl jobs yet", "started": "Started", "filter": "Filter",
        "all_types": "All types", "images": "Images", "videos": "Videos", "audio": "Audio",
        "everything": "Everything (text + media)", "all_media": "All media (images + videos + audio)",
        "images_only": "Images only", "videos_only": "Videos only", "text_only": "Text only",
        "recent_jobs": "Recent Jobs", "search_placeholder": "Search media...",
        "media": "Media", "pages_label": "Pages", "download": "Download",
        "no_media": "No media found", "no_more": "No more results", "load_error": "Failed to load",
        "copy": "Copy", "copied": "Copied!", "starting": "Starting...", "started_ok": "Started!",
        "error": "Error", "watching": "Watching for new images...", "content": "Content",
        "previous": "Previous", "next": "Next", "over_9000": "Over 9,000+!",
        "loading_stats": "Loading stats...", "screenshot": "Screenshot",
        "valid_uri": "Please enter at least one valid URI", "load_jobs_error": "Failed to load jobs",
        "hydrate_subtitle": "Hydrate media from the web", "live_subtitle": "Watch images appear as they're crawled",
        "total_images": "Total Images", "new_session": "New This Session", "per_minute": "Per Minute",
        "all_domains": "All domains", "download_phantom": "Download Phantom Site",
        "phantom_subtitle": "Export archived pages as a static site with local media", "sources": "sources", "max_pages": "Max Pages", "select_domain": "Select Domain",
        "source_uri": "Source URI", "neopig_uri": "Neopig URI", "source_page": "Source Page", "neopig_page": "Neopig Page", "type_label": "Type", "mime_label": "MIME", "size_label": "Size", "alt_label": "Alt", "keywords_label": "Keywords", "description_label": "Description", "items": "items", "bytes": "bytes",
        "used_on": "Used on", "page": "Page", "discovered": "Discovered",
        "console": "Console", "close": "Close", "no_logs": "No logs yet",
    },
    "zh": {
        "search": "搜索", "crawl": "爬取", "live": "实时动态", "about": "关于",
        "random": "随机", "phantom": "幻影", "loading": "加载中...",
        "start_crawl": "开始爬取", "pause": "暂停", "resume": "继续", "delete": "删除",
        "url": "网址", "mode": "模式", "depth": "深度", "keywords": "关键词",
        "screenshots": "截图", "fast_mode": "快速模式", "fresh_start": "全新开始",
        "pages": "页面", "found": "发现", "saved": "保存", "dupes": "重复",
        "running": "运行中", "completed": "已完成", "paused": "已暂停",
        "no_jobs": "暂无爬取任务", "started": "开始于", "filter": "筛选",
        "all_types": "所有类型", "images": "图片", "videos": "视频", "audio": "音频",
        "everything": "全部（文本+媒体）", "all_media": "所有媒体（图片+视频+音频）",
        "images_only": "仅图片", "videos_only": "仅视频", "text_only": "仅文本",
        "recent_jobs": "最近任务", "search_placeholder": "搜索媒体...",
        "media": "媒体", "pages_label": "页面", "download": "下载",
        "no_media": "未找到媒体", "no_more": "没有更多结果", "load_error": "加载失败",
        "copy": "复制", "copied": "已复制!", "starting": "启动中...", "started_ok": "已启动!",
        "error": "错误", "watching": "正在监视新图片...", "content": "内容",
        "previous": "上一页", "next": "下一页", "over_9000": "超过9000+!",
        "loading_stats": "加载统计中...", "screenshot": "截图",
        "valid_uri": "请输入至少一个有效的URI", "load_jobs_error": "加载任务失败",
        "hydrate_subtitle": "从网络获取媒体", "live_subtitle": "观看正在爬取的图片",
        "total_images": "总图片数", "new_session": "本次新增", "per_minute": "每分钟",
        "all_domains": "所有域名", "download_phantom": "下载幻影站点",
        "phantom_subtitle": "将存档页面导出为带本地媒体的静态站点", "max_pages": "最大页数", "select_domain": "选择域名", "sources": "fuentes",
    },
    "es": {
        "search": "Buscar", "crawl": "Rastrear", "live": "En Vivo", "about": "Acerca de",
        "random": "Aleatorio", "phantom": "Fantasma", "loading": "Cargando...",
        "start_crawl": "Iniciar Rastreo", "pause": "Pausar", "resume": "Reanudar", "delete": "Eliminar",
        "url": "URL", "mode": "Modo", "depth": "Profundidad", "keywords": "Palabras clave",
        "screenshots": "Capturas", "fast_mode": "Modo Rápido", "fresh_start": "Inicio Limpio",
        "pages": "páginas", "found": "encontradas", "saved": "guardadas", "dupes": "duplicados",
        "running": "ejecutando", "completed": "completado", "paused": "pausado",
        "no_jobs": "Sin tareas de rastreo", "started": "Iniciado", "filter": "Filtrar",
        "all_types": "Todos los tipos", "images": "Imágenes", "videos": "Videos", "audio": "Audio",
        "everything": "Todo (texto + medios)", "all_media": "Todos los medios",
        "images_only": "Solo imágenes", "videos_only": "Solo videos", "text_only": "Solo texto",
        "recent_jobs": "Tareas Recientes", "search_placeholder": "Buscar medios...",
        "media": "Medios", "pages_label": "Páginas", "download": "Descargar",
        "no_media": "No se encontraron medios", "no_more": "No hay más resultados", "load_error": "Error al cargar",
        "copy": "Copiar", "copied": "¡Copiado!", "starting": "Iniciando...", "started_ok": "¡Iniciado!",
        "error": "Error", "watching": "Buscando nuevas imágenes...", "content": "Contenido",
        "previous": "Anterior", "next": "Siguiente", "over_9000": "¡Más de 9,000+!",
        "loading_stats": "Cargando estadísticas...", "screenshot": "Captura",
        "valid_uri": "Ingrese al menos una URI válida", "load_jobs_error": "Error al cargar tareas",
        "hydrate_subtitle": "Obtener medios de la web", "live_subtitle": "Ver imágenes mientras se rastrean",
        "total_images": "Total de imágenes", "new_session": "Nuevas en esta sesión", "per_minute": "Por minuto",
        "all_domains": "Todos los dominios", "download_phantom": "Descargar sitio fantasma",
        "phantom_subtitle": "Exportar páginas archivadas como sitio estático con medios locales", "max_pages": "Máx. páginas", "select_domain": "Seleccionar dominio", "sources": "स्रोत",
    },
    "hi": {
        "search": "खोजें", "crawl": "क्रॉल", "live": "लाइव फ़ीड", "about": "के बारे में",
        "random": "यादृच्छिक", "phantom": "प्रेत", "loading": "लोड हो रहा है...",
        "start_crawl": "क्रॉल शुरू करें", "pause": "रोकें", "resume": "जारी रखें", "delete": "हटाएं",
        "url": "यूआरएल", "mode": "मोड", "depth": "गहराई", "keywords": "कीवर्ड",
        "screenshots": "स्क्रीनशॉट", "fast_mode": "फास्ट मोड", "fresh_start": "नई शुरुआत",
        "pages": "पेज", "found": "मिले", "saved": "सहेजे", "dupes": "डुप्लीकेट",
        "running": "चल रहा है", "completed": "पूर्ण", "paused": "रुका हुआ",
        "no_jobs": "कोई क्रॉल कार्य नहीं", "started": "शुरू", "filter": "फ़िल्टर",
        "all_types": "सभी प्रकार", "images": "चित्र", "videos": "वीडियो", "audio": "ऑडियो",
        "everything": "सब कुछ", "all_media": "सभी मीडिया", "images_only": "केवल चित्र",
        "videos_only": "केवल वीडियो", "text_only": "केवल टेक्स्ट",
        "recent_jobs": "हाल के कार्य", "search_placeholder": "मीडिया खोजें...",
        "media": "मीडिया", "pages_label": "पेज", "download": "डाउनलोड",
        "no_media": "कोई मीडिया नहीं मिला", "no_more": "और कोई परिणाम नहीं", "load_error": "लोड करने में विफल",
        "copy": "कॉपी", "copied": "कॉपी हो गया!", "starting": "शुरू हो रहा...", "started_ok": "शुरू हो गया!",
        "error": "त्रुटि", "watching": "नई छवियों की तलाश...", "content": "सामग्री",
        "previous": "पिछला", "next": "अगला", "over_9000": "9,000+ से अधिक!",
        "loading_stats": "आँकड़े लोड हो रहे...", "screenshot": "स्क्रीनशॉट",
        "valid_uri": "कृपया कम से कम एक वैध URI दर्ज करें", "load_jobs_error": "कार्य लोड करने में विफल",
        "hydrate_subtitle": "वेब से मीडिया प्राप्त करें", "live_subtitle": "क्रॉल होते हुए छवियां देखें",
        "total_images": "कुल छवियां", "new_session": "इस सत्र में नई", "per_minute": "प्रति मिनट",
        "all_domains": "सभी डोमेन", "download_phantom": "प्रेत साइट डाउनलोड करें",
        "phantom_subtitle": "संग्रहित पृष्ठों को स्थानीय मीडिया के साथ स्थैतिक साइट के रूप में निर्यात करें", "max_pages": "अधिकतम पेज", "select_domain": "डोमेन चुनें", "sources": "مصادر",
    },
    "ar": {
        "search": "بحث", "crawl": "زحف", "live": "مباشر", "about": "حول",
        "random": "عشوائي", "phantom": "شبح", "loading": "جاري التحميل...",
        "start_crawl": "بدء الزحف", "pause": "إيقاف", "resume": "استئناف", "delete": "حذف",
        "url": "الرابط", "mode": "الوضع", "depth": "العمق", "keywords": "كلمات مفتاحية",
        "screenshots": "لقطات", "fast_mode": "وضع سريع", "fresh_start": "بداية جديدة",
        "pages": "صفحات", "found": "وجدت", "saved": "حفظت", "dupes": "مكررات",
        "running": "جاري", "completed": "مكتمل", "paused": "متوقف",
        "no_jobs": "لا توجد مهام", "started": "بدأ", "filter": "تصفية",
        "all_types": "جميع الأنواع", "images": "صور", "videos": "فيديو", "audio": "صوت",
        "everything": "الكل", "all_media": "جميع الوسائط", "images_only": "صور فقط",
        "videos_only": "فيديو فقط", "text_only": "نص فقط",
        "recent_jobs": "المهام الأخيرة", "search_placeholder": "بحث في الوسائط...",
        "media": "وسائط", "pages_label": "صفحات", "download": "تحميل",
        "no_media": "لم يتم العثور على وسائط", "no_more": "لا مزيد من النتائج", "load_error": "فشل التحميل",
        "copy": "نسخ", "copied": "تم النسخ!", "starting": "جاري البدء...", "started_ok": "تم البدء!",
        "error": "خطأ", "watching": "مراقبة الصور الجديدة...", "content": "المحتوى",
        "previous": "السابق", "next": "التالي", "over_9000": "أكثر من 9,000+!",
        "loading_stats": "جاري تحميل الإحصائيات...", "screenshot": "لقطة شاشة",
        "valid_uri": "الرجاء إدخال رابط واحد على الأقل", "load_jobs_error": "فشل تحميل المهام",
        "hydrate_subtitle": "جلب الوسائط من الويب", "live_subtitle": "مشاهدة الصور أثناء الزحف",
        "total_images": "إجمالي الصور", "new_session": "جديدة في هذه الجلسة", "per_minute": "في الدقيقة",
        "all_domains": "جميع النطاقات", "download_phantom": "تحميل موقع الشبح",
        "phantom_subtitle": "تصدير الصفحات المؤرشفة كموقع ثابت مع وسائط محلية", "max_pages": "أقصى عدد صفحات", "select_domain": "اختر النطاق", "sources": "fontes",
    },
    "pt": {
        "search": "Pesquisar", "crawl": "Rastrear", "live": "Ao Vivo", "about": "Sobre",
        "random": "Aleatório", "phantom": "Fantasma", "loading": "Carregando...",
        "start_crawl": "Iniciar Rastreio", "pause": "Pausar", "resume": "Retomar", "delete": "Excluir",
        "url": "URL", "mode": "Modo", "depth": "Profundidade", "keywords": "Palavras-chave",
        "screenshots": "Capturas", "fast_mode": "Modo Rápido", "fresh_start": "Novo Início",
        "pages": "páginas", "found": "encontradas", "saved": "salvas", "dupes": "duplicatas",
        "running": "executando", "completed": "concluído", "paused": "pausado",
        "no_jobs": "Sem tarefas", "started": "Iniciado", "filter": "Filtrar",
        "all_types": "Todos os tipos", "images": "Imagens", "videos": "Vídeos", "audio": "Áudio",
        "everything": "Tudo (texto + mídia)", "all_media": "Toda mídia",
        "images_only": "Apenas imagens", "videos_only": "Apenas vídeos", "text_only": "Apenas texto",
        "recent_jobs": "Tarefas Recentes", "search_placeholder": "Pesquisar mídia...",
        "media": "Mídia", "pages_label": "Páginas", "download": "Baixar",
        "no_media": "Nenhuma mídia encontrada", "no_more": "Sem mais resultados", "load_error": "Falha ao carregar",
        "copy": "Copiar", "copied": "Copiado!", "starting": "Iniciando...", "started_ok": "Iniciado!",
        "error": "Erro", "watching": "Observando novas imagens...", "content": "Conteúdo",
        "previous": "Anterior", "next": "Próximo", "over_9000": "Mais de 9.000+!",
        "loading_stats": "Carregando estatísticas...", "screenshot": "Captura de tela",
        "valid_uri": "Insira pelo menos uma URI válida", "load_jobs_error": "Falha ao carregar tarefas",
        "hydrate_subtitle": "Obter mídia da web", "live_subtitle": "Veja imagens aparecerem durante o rastreio",
        "total_images": "Total de imagens", "new_session": "Novas nesta sessão", "per_minute": "Por minuto",
        "all_domains": "Todos os domínios", "download_phantom": "Baixar site fantasma",
        "phantom_subtitle": "Exportar páginas arquivadas como site estático com mídia local", "max_pages": "Máx. páginas", "select_domain": "Selecionar domínio", "sources": "источников",
    },
    "ru": {
        "search": "Поиск", "crawl": "Сканировать", "live": "Прямой эфир", "about": "О нас",
        "random": "Случайное", "phantom": "Призрак", "loading": "Загрузка...",
        "start_crawl": "Начать сканирование", "pause": "Пауза", "resume": "Продолжить", "delete": "Удалить",
        "url": "URL", "mode": "Режим", "depth": "Глубина", "keywords": "Ключевые слова",
        "screenshots": "Скриншоты", "fast_mode": "Быстрый режим", "fresh_start": "Начать заново",
        "pages": "страниц", "found": "найдено", "saved": "сохранено", "dupes": "дубликаты",
        "running": "выполняется", "completed": "завершено", "paused": "приостановлено",
        "no_jobs": "Нет задач", "started": "Начало", "filter": "Фильтр",
        "all_types": "Все типы", "images": "Изображения", "videos": "Видео", "audio": "Аудио",
        "everything": "Всё (текст + медиа)", "all_media": "Все медиа",
        "images_only": "Только изображения", "videos_only": "Только видео", "text_only": "Только текст",
        "recent_jobs": "Недавние задачи", "search_placeholder": "Поиск медиа...",
        "media": "Медиа", "pages_label": "Страницы", "download": "Скачать",
        "no_media": "Медиа не найдено", "no_more": "Больше нет результатов", "load_error": "Ошибка загрузки",
        "copy": "Копировать", "copied": "Скопировано!", "starting": "Запуск...", "started_ok": "Запущено!",
        "error": "Ошибка", "watching": "Отслеживание новых изображений...", "content": "Контент",
        "previous": "Назад", "next": "Далее", "over_9000": "Более 9000+!",
        "loading_stats": "Загрузка статистики...", "screenshot": "Скриншот",
        "valid_uri": "Введите хотя бы один URL", "load_jobs_error": "Не удалось загрузить задачи",
        "hydrate_subtitle": "Получить медиа из интернета", "live_subtitle": "Смотрите появление изображений",
        "total_images": "Всего изображений", "new_session": "Новые в сессии", "per_minute": "В минуту",
        "all_domains": "Все домены", "download_phantom": "Скачать призрачный сайт",
        "phantom_subtitle": "Экспорт архивных страниц как статический сайт с локальными медиа", "max_pages": "Макс. страниц", "select_domain": "Выбрать домен", "sources": "ソース",
    },
    "ja": {
        "search": "検索", "crawl": "クロール", "live": "ライブ", "about": "について",
        "random": "ランダム", "phantom": "ファントム", "loading": "読み込み中...",
        "start_crawl": "クロール開始", "pause": "一時停止", "resume": "再開", "delete": "削除",
        "url": "URL", "mode": "モード", "depth": "深さ", "keywords": "キーワード",
        "screenshots": "スクリーンショット", "fast_mode": "高速モード", "fresh_start": "新規開始",
        "pages": "ページ", "found": "発見", "saved": "保存", "dupes": "重複",
        "running": "実行中", "completed": "完了", "paused": "一時停止中",
        "no_jobs": "ジョブがありません", "started": "開始", "filter": "フィルター",
        "all_types": "すべてのタイプ", "images": "画像", "videos": "動画", "audio": "オーディオ",
        "everything": "すべて（テキスト＋メディア）", "all_media": "すべてのメディア",
        "images_only": "画像のみ", "videos_only": "動画のみ", "text_only": "テキストのみ",
        "recent_jobs": "最近のジョブ", "search_placeholder": "メディアを検索...",
        "media": "メディア", "pages_label": "ページ", "download": "ダウンロード",
        "no_media": "メディアが見つかりません", "no_more": "これ以上の結果はありません", "load_error": "読み込みに失敗",
        "copy": "コピー", "copied": "コピーしました!", "starting": "開始中...", "started_ok": "開始しました!",
        "error": "エラー", "watching": "新しい画像を監視中...", "content": "コンテンツ",
        "previous": "前へ", "next": "次へ", "over_9000": "9000以上!",
        "loading_stats": "統計を読み込み中...", "screenshot": "スクリーンショット",
        "valid_uri": "有効なURIを入力してください", "load_jobs_error": "ジョブの読み込みに失敗",
        "hydrate_subtitle": "ウェブからメディアを取得", "live_subtitle": "クロール中の画像を表示",
        "total_images": "合計画像数", "new_session": "今回のセッション", "per_minute": "毎分",
        "all_domains": "すべてのドメイン", "download_phantom": "ファントムサイトをダウンロード",
        "phantom_subtitle": "アーカイブページをローカルメディア付きの静的サイトとしてエクスポート", "max_pages": "最大ページ数", "select_domain": "ドメインを選択", "sources": "sources", "max_pages": "Max Pages", "select_domain": "Select Domain",
    },
    "fr": {
        "search": "Rechercher", "crawl": "Explorer", "live": "En Direct", "about": "À propos",
        "random": "Aléatoire", "phantom": "Fantôme", "loading": "Chargement...",
        "start_crawl": "Démarrer", "pause": "Pause", "resume": "Reprendre", "delete": "Supprimer",
        "url": "URL", "mode": "Mode", "depth": "Profondeur", "keywords": "Mots-clés",
        "screenshots": "Captures", "fast_mode": "Mode Rapide", "fresh_start": "Nouveau Départ",
        "pages": "pages", "found": "trouvées", "saved": "enregistrées", "dupes": "doublons",
        "running": "en cours", "completed": "terminé", "paused": "en pause",
        "no_jobs": "Aucune tâche", "started": "Démarré", "filter": "Filtrer",
        "all_types": "Tous les types", "images": "Images", "videos": "Vidéos", "audio": "Audio",
        "everything": "Tout (texte + médias)", "all_media": "Tous les médias",
        "images_only": "Images uniquement", "videos_only": "Vidéos uniquement", "text_only": "Texte uniquement",
        "recent_jobs": "Tâches Récentes", "search_placeholder": "Rechercher des médias...",
        "media": "Médias", "pages_label": "Pages", "download": "Télécharger",
        "no_media": "Aucun média trouvé", "no_more": "Plus de résultats", "load_error": "Échec du chargement",
        "copy": "Copier", "copied": "Copié!", "starting": "Démarrage...", "started_ok": "Démarré!",
        "error": "Erreur", "watching": "Surveillance des nouvelles images...", "content": "Contenu",
        "previous": "Précédent", "next": "Suivant", "over_9000": "Plus de 9 000+!",
        "loading_stats": "Chargement des statistiques...", "screenshot": "Capture d'écran",
        "valid_uri": "Veuillez entrer au moins une URI valide", "load_jobs_error": "Échec du chargement des tâches",
        "hydrate_subtitle": "Récupérer les médias du web", "live_subtitle": "Voir les images apparaître pendant l'exploration",
        "total_images": "Total d'images", "new_session": "Nouvelles cette session", "per_minute": "Par minute",
        "all_domains": "Tous les domaines", "download_phantom": "Télécharger le site fantôme",
        "phantom_subtitle": "Exporter les pages archivées en site statique avec médias locaux", "max_pages": "Pages max", "select_domain": "Sélectionner le domaine", "sources": "Quellen",
    },
    "de": {
        "search": "Suchen", "crawl": "Crawlen", "live": "Live-Feed", "about": "Über",
        "random": "Zufällig", "phantom": "Phantom", "loading": "Laden...",
        "start_crawl": "Crawl starten", "pause": "Pause", "resume": "Fortsetzen", "delete": "Löschen",
        "url": "URL", "mode": "Modus", "depth": "Tiefe", "keywords": "Stichwörter",
        "screenshots": "Screenshots", "fast_mode": "Schnellmodus", "fresh_start": "Neustart",
        "pages": "Seiten", "found": "gefunden", "saved": "gespeichert", "dupes": "Duplikate",
        "running": "läuft", "completed": "abgeschlossen", "paused": "pausiert",
        "no_jobs": "Keine Aufgaben", "started": "Gestartet", "filter": "Filter",
        "all_types": "Alle Typen", "images": "Bilder", "videos": "Videos", "audio": "Audio",
        "everything": "Alles (Text + Medien)", "all_media": "Alle Medien",
        "images_only": "Nur Bilder", "videos_only": "Nur Videos", "text_only": "Nur Text",
        "recent_jobs": "Letzte Aufgaben", "search_placeholder": "Medien suchen...",
        "media": "Medien", "pages_label": "Seiten", "download": "Herunterladen",
        "no_media": "Keine Medien gefunden", "no_more": "Keine weiteren Ergebnisse", "load_error": "Laden fehlgeschlagen",
        "copy": "Kopieren", "copied": "Kopiert!", "starting": "Starten...", "started_ok": "Gestartet!",
        "error": "Fehler", "watching": "Überwache neue Bilder...", "content": "Inhalt",
        "previous": "Zurück", "next": "Weiter", "over_9000": "Über 9.000+!",
        "loading_stats": "Lade Statistiken...", "screenshot": "Screenshot",
        "valid_uri": "Mindestens eine gültige URI eingeben", "load_jobs_error": "Aufgaben konnten nicht geladen werden",
        "hydrate_subtitle": "Medien aus dem Web abrufen", "live_subtitle": "Bilder beim Crawlen beobachten",
        "total_images": "Bilder gesamt", "new_session": "Neu in dieser Sitzung", "per_minute": "Pro Minute",
        "all_domains": "Alle Domains", "download_phantom": "Phantom-Seite herunterladen",
        "phantom_subtitle": "Archivierte Seiten als statische Seite mit lokalen Medien exportieren", "max_pages": "Max. Seiten", "select_domain": "Domain auswählen", "sources": "소스",
    },
    "ko": {
        "search": "검색", "crawl": "크롤", "live": "라이브", "about": "정보",
        "random": "랜덤", "phantom": "팬텀", "loading": "로딩 중...",
        "start_crawl": "크롤 시작", "pause": "일시정지", "resume": "재개", "delete": "삭제",
        "url": "URL", "mode": "모드", "depth": "깊이", "keywords": "키워드",
        "screenshots": "스크린샷", "fast_mode": "빠른 모드", "fresh_start": "새로 시작",
        "pages": "페이지", "found": "발견", "saved": "저장", "dupes": "중복",
        "running": "실행 중", "completed": "완료", "paused": "일시정지됨",
        "no_jobs": "작업 없음", "started": "시작됨", "filter": "필터",
        "all_types": "모든 유형", "images": "이미지", "videos": "동영상", "audio": "오디오",
        "everything": "모두", "all_media": "모든 미디어",
        "images_only": "이미지만", "videos_only": "동영상만", "text_only": "텍스트만",
        "recent_jobs": "최근 작업", "search_placeholder": "미디어 검색...",
        "media": "미디어", "pages_label": "페이지", "download": "다운로드",
        "no_media": "미디어를 찾을 수 없음", "no_more": "더 이상 결과 없음", "load_error": "로드 실패",
        "copy": "복사", "copied": "복사됨!", "starting": "시작 중...", "started_ok": "시작됨!",
        "error": "오류", "watching": "새 이미지 감시 중...", "content": "콘텐츠",
        "previous": "이전", "next": "다음", "over_9000": "9,000 이상!",
        "loading_stats": "통계 로드 중...", "screenshot": "스크린샷",
        "valid_uri": "유효한 URI를 하나 이상 입력하세요", "load_jobs_error": "작업 로드 실패",
        "hydrate_subtitle": "웹에서 미디어 가져오기", "live_subtitle": "크롤링되는 이미지 보기",
        "total_images": "전체 이미지", "new_session": "이번 세션 신규", "per_minute": "분당",
        "all_domains": "모든 도메인", "download_phantom": "팬텀 사이트 다운로드",
        "phantom_subtitle": "보관된 페이지를 로컬 미디어가 포함된 정적 사이트로 내보내기", "max_pages": "최대 페이지", "select_domain": "도메인 선택", "sources": "fonti",
    },
    "it": {
        "search": "Cerca", "crawl": "Scansiona", "live": "In Diretta", "about": "Info",
        "random": "Casuale", "phantom": "Fantasma", "loading": "Caricamento...",
        "start_crawl": "Avvia Scansione", "pause": "Pausa", "resume": "Riprendi", "delete": "Elimina",
        "url": "URL", "mode": "Modalità", "depth": "Profondità", "keywords": "Parole chiave",
        "screenshots": "Screenshot", "fast_mode": "Modalità Veloce", "fresh_start": "Nuovo Inizio",
        "pages": "pagine", "found": "trovati", "saved": "salvati", "dupes": "duplicati",
        "running": "in esecuzione", "completed": "completato", "paused": "in pausa",
        "no_jobs": "Nessuna attività", "started": "Avviato", "filter": "Filtra",
        "all_types": "Tutti i tipi", "images": "Immagini", "videos": "Video", "audio": "Audio",
        "everything": "Tutto", "all_media": "Tutti i media",
        "images_only": "Solo immagini", "videos_only": "Solo video", "text_only": "Solo testo",
        "recent_jobs": "Attività Recenti", "search_placeholder": "Cerca media...",
        "media": "Media", "pages_label": "Pagine", "download": "Scarica",
        "no_media": "Nessun media trovato", "no_more": "Nessun altro risultato", "load_error": "Caricamento fallito",
        "copy": "Copia", "copied": "Copiato!", "starting": "Avvio...", "started_ok": "Avviato!",
        "error": "Errore", "watching": "Monitoraggio nuove immagini...", "content": "Contenuto",
        "previous": "Precedente", "next": "Successivo", "over_9000": "Oltre 9.000+!",
        "loading_stats": "Caricamento statistiche...", "screenshot": "Screenshot",
        "valid_uri": "Inserire almeno un URI valido", "load_jobs_error": "Caricamento attività fallito",
        "hydrate_subtitle": "Ottieni media dal web", "live_subtitle": "Guarda le immagini mentre vengono scansionate",
        "total_images": "Immagini totali", "new_session": "Nuove in questa sessione", "per_minute": "Al minuto",
        "all_domains": "Tutti i domini", "download_phantom": "Scarica sito fantasma",
        "phantom_subtitle": "Esporta pagine archiviate come sito statico con media locali", "max_pages": "Max pagine", "select_domain": "Seleziona dominio", "sources": "bronnen",
    },
    "nl": {
        "search": "Zoeken", "crawl": "Crawlen", "live": "Live", "about": "Over",
        "random": "Willekeurig", "phantom": "Fantoom", "loading": "Laden...",
        "start_crawl": "Start Crawl", "pause": "Pauzeer", "resume": "Hervat", "delete": "Verwijder",
        "url": "URL", "mode": "Modus", "depth": "Diepte", "keywords": "Trefwoorden",
        "screenshots": "Schermafbeeldingen", "fast_mode": "Snelle Modus", "fresh_start": "Nieuwe Start",
        "pages": "pagina's", "found": "gevonden", "saved": "opgeslagen", "dupes": "duplicaten",
        "running": "actief", "completed": "voltooid", "paused": "gepauzeerd",
        "no_jobs": "Geen taken", "started": "Gestart", "filter": "Filter",
        "all_types": "Alle typen", "images": "Afbeeldingen", "videos": "Video's", "audio": "Audio",
        "everything": "Alles", "all_media": "Alle media",
        "images_only": "Alleen afbeeldingen", "videos_only": "Alleen video's", "text_only": "Alleen tekst",
        "recent_jobs": "Recente Taken", "search_placeholder": "Zoek media...",
        "media": "Media", "pages_label": "Pagina's", "download": "Downloaden",
        "no_media": "Geen media gevonden", "no_more": "Geen resultaten meer", "load_error": "Laden mislukt",
        "copy": "Kopiëren", "copied": "Gekopieerd!", "starting": "Starten...", "started_ok": "Gestart!",
        "error": "Fout", "watching": "Nieuwe afbeeldingen bekijken...", "content": "Inhoud",
        "previous": "Vorige", "next": "Volgende", "over_9000": "Meer dan 9.000+!",
        "loading_stats": "Statistieken laden...", "screenshot": "Schermafbeelding",
        "valid_uri": "Voer minimaal één geldige URI in", "load_jobs_error": "Taken laden mislukt",
        "hydrate_subtitle": "Media van het web ophalen", "live_subtitle": "Bekijk afbeeldingen terwijl ze worden gecrawld",
        "total_images": "Totaal afbeeldingen", "new_session": "Nieuw deze sessie", "per_minute": "Per minuut",
        "all_domains": "Alle domeinen", "download_phantom": "Fantoomsite downloaden",
        "phantom_subtitle": "Gearchiveerde pagina's exporteren als statische site met lokale media", "max_pages": "Max. pagina's", "select_domain": "Selecteer domein", "sources": "źródeł",
    },
    "pl": {
        "search": "Szukaj", "crawl": "Indeksuj", "live": "Na żywo", "about": "O nas",
        "random": "Losowo", "phantom": "Fantom", "loading": "Ładowanie...",
        "start_crawl": "Rozpocznij", "pause": "Pauza", "resume": "Wznów", "delete": "Usuń",
        "url": "URL", "mode": "Tryb", "depth": "Głębokość", "keywords": "Słowa kluczowe",
        "screenshots": "Zrzuty ekranu", "fast_mode": "Tryb szybki", "fresh_start": "Nowy start",
        "pages": "stron", "found": "znaleziono", "saved": "zapisano", "dupes": "duplikaty",
        "running": "w toku", "completed": "zakończono", "paused": "wstrzymano",
        "no_jobs": "Brak zadań", "started": "Rozpoczęto", "filter": "Filtruj",
        "all_types": "Wszystkie typy", "images": "Obrazy", "videos": "Filmy", "audio": "Audio",
        "everything": "Wszystko", "all_media": "Wszystkie media",
        "images_only": "Tylko obrazy", "videos_only": "Tylko filmy", "text_only": "Tylko tekst",
        "recent_jobs": "Ostatnie Zadania", "search_placeholder": "Szukaj mediów...",
        "media": "Media", "pages_label": "Strony", "download": "Pobierz",
        "no_media": "Nie znaleziono mediów", "no_more": "Brak więcej wyników", "load_error": "Błąd ładowania",
        "copy": "Kopiuj", "copied": "Skopiowano!", "starting": "Uruchamianie...", "started_ok": "Uruchomiono!",
        "error": "Błąd", "watching": "Obserwowanie nowych obrazów...", "content": "Treść",
        "previous": "Poprzedni", "next": "Następny", "over_9000": "Ponad 9 000+!",
        "loading_stats": "Ładowanie statystyk...", "screenshot": "Zrzut ekranu",
        "valid_uri": "Wprowadź przynajmniej jeden URI", "load_jobs_error": "Nie udało się załadować zadań",
        "hydrate_subtitle": "Pobierz media z sieci", "live_subtitle": "Oglądaj obrazy podczas indeksowania",
        "total_images": "Łącznie obrazów", "new_session": "Nowe w tej sesji", "per_minute": "Na minutę",
        "all_domains": "Wszystkie domeny", "download_phantom": "Pobierz stronę fantomową",
        "phantom_subtitle": "Eksportuj zarchiwizowane strony jako statyczną witrynę z lokalnymi mediami", "max_pages": "Maks. stron", "select_domain": "Wybierz domenę", "sources": "kaynaklar",
    },
    "tr": {
        "search": "Ara", "crawl": "Tara", "live": "Canlı", "about": "Hakkında",
        "random": "Rastgele", "phantom": "Hayalet", "loading": "Yükleniyor...",
        "start_crawl": "Taramayı Başlat", "pause": "Duraklat", "resume": "Devam", "delete": "Sil",
        "url": "URL", "mode": "Mod", "depth": "Derinlik", "keywords": "Anahtar kelimeler",
        "screenshots": "Ekran görüntüleri", "fast_mode": "Hızlı Mod", "fresh_start": "Yeni Başlangıç",
        "pages": "sayfa", "found": "bulundu", "saved": "kaydedildi", "dupes": "kopya",
        "running": "çalışıyor", "completed": "tamamlandı", "paused": "duraklatıldı",
        "no_jobs": "Görev yok", "started": "Başladı", "filter": "Filtrele",
        "all_types": "Tüm türler", "images": "Resimler", "videos": "Videolar", "audio": "Ses",
        "everything": "Her şey", "all_media": "Tüm medya",
        "images_only": "Sadece resimler", "videos_only": "Sadece videolar", "text_only": "Sadece metin",
        "recent_jobs": "Son Görevler", "search_placeholder": "Medya ara...",
        "media": "Medya", "pages_label": "Sayfalar", "download": "İndir",
        "no_media": "Medya bulunamadı", "no_more": "Daha fazla sonuç yok", "load_error": "Yükleme başarısız",
        "copy": "Kopyala", "copied": "Kopyalandı!", "starting": "Başlatılıyor...", "started_ok": "Başlatıldı!",
        "error": "Hata", "watching": "Yeni resimler izleniyor...", "content": "İçerik",
        "previous": "Önceki", "next": "Sonraki", "over_9000": "9.000'den fazla+!",
        "loading_stats": "İstatistikler yükleniyor...", "screenshot": "Ekran görüntüsü",
        "valid_uri": "Lütfen en az bir geçerli URI girin", "load_jobs_error": "Görevler yüklenemedi",
        "hydrate_subtitle": "Web'den medya al", "live_subtitle": "Taranan resimleri izle",
        "total_images": "Toplam resim", "new_session": "Bu oturumda yeni", "per_minute": "Dakikada",
        "all_domains": "Tüm alanlar", "download_phantom": "Hayalet siteyi indir",
        "phantom_subtitle": "Arşivlenmiş sayfaları yerel medya ile statik site olarak dışa aktar", "max_pages": "Maks. sayfa", "select_domain": "Alan adı seç", "sources": "nguồn",
    },
    "vi": {
        "search": "Tìm kiếm", "crawl": "Thu thập", "live": "Trực tiếp", "about": "Giới thiệu",
        "random": "Ngẫu nhiên", "phantom": "Bóng ma", "loading": "Đang tải...",
        "start_crawl": "Bắt đầu", "pause": "Tạm dừng", "resume": "Tiếp tục", "delete": "Xóa",
        "url": "URL", "mode": "Chế độ", "depth": "Độ sâu", "keywords": "Từ khóa",
        "screenshots": "Ảnh chụp", "fast_mode": "Chế độ nhanh", "fresh_start": "Bắt đầu mới",
        "pages": "trang", "found": "tìm thấy", "saved": "đã lưu", "dupes": "trùng lặp",
        "running": "đang chạy", "completed": "hoàn thành", "paused": "tạm dừng",
        "no_jobs": "Không có tác vụ", "started": "Đã bắt đầu", "filter": "Lọc",
        "all_types": "Tất cả loại", "images": "Hình ảnh", "videos": "Video", "audio": "Âm thanh",
        "everything": "Tất cả", "all_media": "Tất cả media",
        "images_only": "Chỉ hình ảnh", "videos_only": "Chỉ video", "text_only": "Chỉ văn bản",
        "recent_jobs": "Tác vụ gần đây", "search_placeholder": "Tìm media...",
        "media": "Media", "pages_label": "Trang", "download": "Tải xuống",
        "no_media": "Không tìm thấy media", "no_more": "Không còn kết quả", "load_error": "Tải thất bại",
        "copy": "Sao chép", "copied": "Đã sao chép!", "starting": "Đang khởi động...", "started_ok": "Đã khởi động!",
        "error": "Lỗi", "watching": "Đang theo dõi hình ảnh mới...", "content": "Nội dung",
        "previous": "Trước", "next": "Tiếp", "over_9000": "Hơn 9.000+!",
        "loading_stats": "Đang tải thống kê...", "screenshot": "Ảnh chụp màn hình",
        "valid_uri": "Vui lòng nhập ít nhất một URI hợp lệ", "load_jobs_error": "Không thể tải tác vụ",
        "hydrate_subtitle": "Lấy media từ web", "live_subtitle": "Xem hình ảnh khi đang thu thập",
        "total_images": "Tổng hình ảnh", "new_session": "Mới trong phiên này", "per_minute": "Mỗi phút",
        "all_domains": "Tất cả tên miền", "download_phantom": "Tải trang bóng ma",
        "phantom_subtitle": "Xuất trang lưu trữ dưới dạng trang tĩnh với media cục bộ", "max_pages": "Tối đa trang", "select_domain": "Chọn tên miền", "sources": "แหล่งที่มา",
    },
    "th": {
        "search": "ค้นหา", "crawl": "รวบรวม", "live": "สด", "about": "เกี่ยวกับ",
        "random": "สุ่ม", "phantom": "แฟนทอม", "loading": "กำลังโหลด...",
        "start_crawl": "เริ่มรวบรวม", "pause": "หยุดชั่วคราว", "resume": "ดำเนินต่อ", "delete": "ลบ",
        "url": "URL", "mode": "โหมด", "depth": "ความลึก", "keywords": "คำสำคัญ",
        "screenshots": "ภาพหน้าจอ", "fast_mode": "โหมดเร็ว", "fresh_start": "เริ่มใหม่",
        "pages": "หน้า", "found": "พบ", "saved": "บันทึก", "dupes": "ซ้ำ",
        "running": "กำลังทำงาน", "completed": "เสร็จสิ้น", "paused": "หยุดชั่วคราว",
        "no_jobs": "ไม่มีงาน", "started": "เริ่มแล้ว", "filter": "กรอง",
        "all_types": "ทุกประเภท", "images": "รูปภาพ", "videos": "วิดีโอ", "audio": "เสียง",
        "everything": "ทั้งหมด", "all_media": "สื่อทั้งหมด",
        "images_only": "เฉพาะรูปภาพ", "videos_only": "เฉพาะวิดีโอ", "text_only": "เฉพาะข้อความ",
        "recent_jobs": "งานล่าสุด", "search_placeholder": "ค้นหาสื่อ...",
        "media": "สื่อ", "pages_label": "หน้า", "download": "ดาวน์โหลด",
        "no_media": "ไม่พบสื่อ", "no_more": "ไม่มีผลลัพธ์เพิ่มเติม", "load_error": "โหลดไม่สำเร็จ",
        "copy": "คัดลอก", "copied": "คัดลอกแล้ว!", "starting": "กำลังเริ่ม...", "started_ok": "เริ่มแล้ว!",
        "error": "ข้อผิดพลาด", "watching": "กำลังดูรูปภาพใหม่...", "content": "เนื้อหา",
        "previous": "ก่อนหน้า", "next": "ถัดไป", "over_9000": "มากกว่า 9,000+!",
        "loading_stats": "กำลังโหลดสถิติ...", "screenshot": "ภาพหน้าจอ",
        "valid_uri": "กรุณาใส่ URI ที่ถูกต้องอย่างน้อยหนึ่งรายการ", "load_jobs_error": "ไม่สามารถโหลดงานได้",
        "hydrate_subtitle": "ดึงสื่อจากเว็บ", "live_subtitle": "ดูรูปภาพขณะรวบรวม",
        "total_images": "รูปภาพทั้งหมด", "new_session": "ใหม่ในเซสชันนี้", "per_minute": "ต่อนาที",
        "all_domains": "โดเมนทั้งหมด", "download_phantom": "ดาวน์โหลดเว็บแฟนทอม",
        "phantom_subtitle": "ส่งออกหน้าที่เก็บถาวรเป็นเว็บไซต์แบบคงที่พร้อมสื่อในเครื่อง", "max_pages": "หน้าสูงสุด", "select_domain": "เลือกโดเมน", "sources": "sumber",
    },
    "id": {
        "search": "Cari", "crawl": "Jelajahi", "live": "Langsung", "about": "Tentang",
        "random": "Acak", "phantom": "Hantu", "loading": "Memuat...",
        "start_crawl": "Mulai Jelajah", "pause": "Jeda", "resume": "Lanjutkan", "delete": "Hapus",
        "url": "URL", "mode": "Mode", "depth": "Kedalaman", "keywords": "Kata kunci",
        "screenshots": "Tangkapan layar", "fast_mode": "Mode Cepat", "fresh_start": "Mulai Baru",
        "pages": "halaman", "found": "ditemukan", "saved": "disimpan", "dupes": "duplikat",
        "running": "berjalan", "completed": "selesai", "paused": "dijeda",
        "no_jobs": "Tidak ada tugas", "started": "Dimulai", "filter": "Filter",
        "all_types": "Semua jenis", "images": "Gambar", "videos": "Video", "audio": "Audio",
        "everything": "Semua", "all_media": "Semua media",
        "images_only": "Hanya gambar", "videos_only": "Hanya video", "text_only": "Hanya teks",
        "recent_jobs": "Tugas Terbaru", "search_placeholder": "Cari media...",
        "media": "Media", "pages_label": "Halaman", "download": "Unduh",
        "no_media": "Media tidak ditemukan", "no_more": "Tidak ada hasil lagi", "load_error": "Gagal memuat",
        "copy": "Salin", "copied": "Disalin!", "starting": "Memulai...", "started_ok": "Dimulai!",
        "error": "Kesalahan", "watching": "Memantau gambar baru...", "content": "Konten",
        "previous": "Sebelumnya", "next": "Berikutnya", "over_9000": "Lebih dari 9.000+!",
        "loading_stats": "Memuat statistik...", "screenshot": "Tangkapan layar",
        "valid_uri": "Masukkan setidaknya satu URI yang valid", "load_jobs_error": "Gagal memuat tugas",
        "hydrate_subtitle": "Ambil media dari web", "live_subtitle": "Lihat gambar saat dijelajahi",
        "total_images": "Total gambar", "new_session": "Baru sesi ini", "per_minute": "Per menit",
        "all_domains": "Semua domain", "download_phantom": "Unduh situs hantu",
        "phantom_subtitle": "Ekspor halaman arsip sebagai situs statis dengan media lokal", "max_pages": "Maks. halaman", "select_domain": "Pilih domain", "sources": "джерел",
    },
    "uk": {
        "search": "Пошук", "crawl": "Сканувати", "live": "Наживо", "about": "Про нас",
        "random": "Випадково", "phantom": "Привид", "loading": "Завантаження...",
        "start_crawl": "Почати сканування", "pause": "Пауза", "resume": "Продовжити", "delete": "Видалити",
        "url": "URL", "mode": "Режим", "depth": "Глибина", "keywords": "Ключові слова",
        "screenshots": "Знімки екрану", "fast_mode": "Швидкий режим", "fresh_start": "Новий старт",
        "pages": "сторінок", "found": "знайдено", "saved": "збережено", "dupes": "дублікати",
        "running": "виконується", "completed": "завершено", "paused": "призупинено",
        "no_jobs": "Немає завдань", "started": "Розпочато", "filter": "Фільтр",
        "all_types": "Усі типи", "images": "Зображення", "videos": "Відео", "audio": "Аудіо",
        "everything": "Все", "all_media": "Усі медіа",
        "images_only": "Лише зображення", "videos_only": "Лише відео", "text_only": "Лише текст",
        "recent_jobs": "Останні завдання", "search_placeholder": "Пошук медіа...",
        "media": "Медіа", "pages_label": "Сторінки", "download": "Завантажити",
        "no_media": "Медіа не знайдено", "no_more": "Більше немає результатів", "load_error": "Помилка завантаження",
        "copy": "Копіювати", "copied": "Скопійовано!", "starting": "Запуск...", "started_ok": "Запущено!",
        "error": "Помилка", "watching": "Відстеження нових зображень...", "content": "Вміст",
        "previous": "Попередній", "next": "Наступний", "over_9000": "Понад 9000+!",
        "loading_stats": "Завантаження статистики...", "screenshot": "Знімок екрану",
        "valid_uri": "Введіть хоча б один дійсний URI", "load_jobs_error": "Не вдалося завантажити завдання",
        "hydrate_subtitle": "Отримати медіа з вебу", "live_subtitle": "Дивіться зображення під час сканування",
        "total_images": "Всього зображень", "new_session": "Нові в сесії", "per_minute": "За хвилину",
        "all_domains": "Усі домени", "download_phantom": "Завантажити привид-сайт",
        "phantom_subtitle": "Експортувати архівні сторінки як статичний сайт з локальними медіа",
    },
    "sv": {
        "search": "Sök", "crawl": "Genomsök", "live": "Live", "about": "Om",
        "random": "Slumpmässig", "phantom": "Fantom", "loading": "Laddar...",
        "start_crawl": "Starta genomsökning", "pause": "Pausa", "resume": "Återuppta", "delete": "Ta bort",
        "url": "URL", "mode": "Läge", "depth": "Djup", "keywords": "Nyckelord",
        "screenshots": "Skärmbilder", "fast_mode": "Snabbläge", "fresh_start": "Ny start",
        "pages": "sidor", "found": "hittade", "saved": "sparade", "dupes": "dubbletter",
        "running": "körs", "completed": "slutförd", "paused": "pausad",
        "no_jobs": "Inga jobb", "started": "Startad", "filter": "Filtrera",
        "all_types": "Alla typer", "images": "Bilder", "videos": "Videor", "audio": "Ljud",
        "everything": "Allt", "all_media": "Alla media",
        "images_only": "Endast bilder", "videos_only": "Endast videor", "text_only": "Endast text",
        "recent_jobs": "Senaste Jobb", "search_placeholder": "Sök media...",
        "media": "Media", "pages_label": "Sidor", "download": "Ladda ner",
        "no_media": "Ingen media hittades", "no_more": "Inga fler resultat", "load_error": "Kunde inte ladda",
        "copy": "Kopiera", "copied": "Kopierat!", "starting": "Startar...", "started_ok": "Startad!",
        "error": "Fel", "watching": "Bevakar nya bilder...", "content": "Innehåll",
        "previous": "Föregående", "next": "Nästa", "over_9000": "Över 9 000+!",
        "loading_stats": "Laddar statistik...", "screenshot": "Skärmbild",
        "valid_uri": "Ange minst en giltig URI", "load_jobs_error": "Kunde inte ladda jobb",
        "hydrate_subtitle": "Hämta media från webben", "live_subtitle": "Se bilder medan de genomsöks",
        "total_images": "Totalt bilder", "new_session": "Nya denna session", "per_minute": "Per minut",
        "all_domains": "Alla domäner", "download_phantom": "Ladda ner fantomwebbplats",
        "phantom_subtitle": "Exportera arkiverade sidor som statisk webbplats med lokal media", "max_pages": "Max sidor", "select_domain": "Välj domän", "sources": "källor",
    },
}

def get_lang(lang_cookie: str = None, accept_language: str = None) -> str:
    """Get language from cookie first, then Accept-Language header."""
    # Cookie takes priority (user's explicit choice)
    if lang_cookie and lang_cookie in TRANSLATIONS:
        return lang_cookie
    # Fall back to Accept-Language header
    if not accept_language:
        return "en"
    # Parse "en-US,en;q=0.9,zh-CN;q=0.8" format
    for part in accept_language.split(','):
        lang = part.split(';')[0].strip().split('-')[0].lower()
        if lang in TRANSLATIONS:
            return lang
    return "en"

# Language names for the selector dropdown
LANG_NAMES = {
    "en": "English", "zh": "中文", "es": "Español", "hi": "हिन्दी", "ar": "العربية",
    "pt": "Português", "ru": "Русский", "ja": "日本語", "fr": "Français", "de": "Deutsch",
    "ko": "한국어", "it": "Italiano", "nl": "Nederlands", "pl": "Polski", "tr": "Türkçe",
    "vi": "Tiếng Việt", "th": "ไทย", "id": "Bahasa", "uk": "Українська", "sv": "Svenska",
}

def t(key: str, lang: str = "en") -> str:
    """Get translation for key in language."""
    return TRANSLATIONS.get(lang, TRANSLATIONS["en"]).get(key, TRANSLATIONS["en"].get(key, key))

# Single source of truth for navigation - uses {{key}} placeholders
NAV_HTML = '''<div class="nav">
    <a href="/" class="brand">🐷 neopig</a>
    <a href="/">{{search}}</a>
    <a href="/live">{{live}}</a>
    <a href="/random">{{random}}</a>
    <a href="/crawl">{{crawl}}</a>
    <a href="/about">{{about}}</a>
</div>'''

def inject_i18n(html: str, lang: str) -> str:
    """Replace {key} placeholders and inject JS translations + language selector."""
    trans = TRANSLATIONS.get(lang, TRANSLATIONS["en"])
    # Add lang attribute and inject JS translations
    html = html.replace('<html>', f'<html lang="{lang}">')
    # Replace nav placeholder with actual nav (add import link if enabled)
    nav = NAV_HTML
    if IMPORT_MODE:
        nav = nav.replace('</div>', '    <a href="/import">Import 🧪</a>\n</div>')
    html = html.replace('<!-- NAV -->', nav)
    # Build language selector options
    lang_options = ''.join(f'<option value="{code}"{" selected" if code == lang else ""}>{name}</option>'
                          for code, name in LANG_NAMES.items())
    lang_selector = f'''<select id="lang-select" onchange="setLang(this.value)" style="background:#1a1a1a;color:#888;border:1px solid #333;border-radius:4px;padding:4px 8px;font-size:12px;cursor:pointer;">{lang_options}</select>'''
    lang_js = '''
function setLang(code) {
    localStorage.setItem('neopig_lang', code);
    document.cookie = 'lang=' + code + ';path=/;max-age=31536000';
    location.reload();
}
'''
    html = html.replace('<script>', f'<script>\nconst T={json.dumps(trans)};\n{lang_js}', 1)
    # Insert language selector after {{about}} link in nav
    html = html.replace('{{about}}</a>\n</div>', '{{about}}</a>\n    ' + lang_selector + '\n</div>')
    # Replace all {key} and {{key}} placeholders
    for key, value in trans.items():
        html = html.replace('{{' + key + '}}', value)  # Double braces (static templates)
        html = html.replace('{' + key + '}', value)    # Single braces (after f-string)
    return html

app = FastAPI(title="neopig", description="Media crawler + SERP + Screenshot service")

# Try to include uri2png screenshot router (optional dependency)
try:
    from uri2png import get_screenshot_router
    app.include_router(get_screenshot_router())
    logger.info("Screenshot router loaded from uri2png")
except ImportError:
    logger.warning("uri2png not installed - screenshot endpoints not available")

# Mount static vendor files (we are the CDN)
STATIC_VENDOR_PATH = Path(__file__).parent / "static" / "vendor"
if STATIC_VENDOR_PATH.exists():
    app.mount("/static/vendor", StaticFiles(directory=STATIC_VENDOR_PATH), name="vendor")
    logger.info(f"Static vendor files mounted from {STATIC_VENDOR_PATH}")

# Mount static images
STATIC_IMAGES_PATH = Path(__file__).parent / "static" / "images"
if STATIC_IMAGES_PATH.exists():
    app.mount("/static/images", StaticFiles(directory=STATIC_IMAGES_PATH), name="images")
    logger.info(f"Static images mounted from {STATIC_IMAGES_PATH}")

# Config - set via startup
DB_PATH = "data/neopig.db"
VAULT_PATH = Path("data/vault")
LOGS_PATH = Path("data/logs")
CRAWL_DISABLED = os.environ.get("NEOPIG_DISABLE_CRAWL", "").lower() in ("1", "true", "yes")
IMPORT_MODE = os.environ.get("NEOPIG_IMPORT", "").lower() in ("1", "true", "yes")

# Global database instance
db: Database = None

# Per-job log handlers (job_id -> handler)
JOB_LOG_HANDLERS: Dict[int, logging.FileHandler] = {}

def start_job_logging(job_id: int) -> None:
    """Start capturing logs for a crawl job."""
    LOGS_PATH.mkdir(parents=True, exist_ok=True)
    log_file = LOGS_PATH / f"{job_id}.log"
    handler = logging.FileHandler(log_file, mode='w', encoding='utf-8')
    handler.setLevel(logging.INFO)
    handler.setFormatter(logging.Formatter('%(asctime)s %(name)s %(levelname)s: %(message)s', datefmt='%H:%M:%S'))
    # Add to root logger to capture all modules
    logging.getLogger().addHandler(handler)
    JOB_LOG_HANDLERS[job_id] = handler

def stop_job_logging(job_id: int) -> None:
    """Stop capturing logs for a crawl job."""
    handler = JOB_LOG_HANDLERS.pop(job_id, None)
    if handler:
        handler.close()
        logging.getLogger().removeHandler(handler)

def get_job_logs(job_id: int, tail: int = 0) -> str:
    """Get logs for a crawl job. If tail > 0, return only last N lines."""
    log_file = LOGS_PATH / f"{job_id}.log"
    if not log_file.exists():
        return ""
    content = log_file.read_text(encoding='utf-8')
    if tail > 0:
        lines = content.splitlines()
        return '\n'.join(lines[-tail:])
    return content

# Active crawl tasks (for cancellation on shutdown)
ACTIVE_CRAWL_TASKS: Dict[int, asyncio.Task] = {}

# Tarball mode - serve directly from tar.gz archive
import tarfile
import tempfile
TAR_PATH: str = None  # Path to tarball (each request opens its own handle)
TAR_OFFSET: int = 0  # Offset for .run files (0 for plain .tar.gz)
TAR_MEMBERS: Dict[str, tarfile.TarInfo] = {}  # Cached member info
TAR_MEDIA_INDEX: Dict[str, str] = {}  # md5_hash -> full path (for O(1) lookup)
ARCHIVE_ROOT: str = None  # e.g., "example.com-20251230"
TEMP_DB_PATH: str = None  # Extracted database (SQLite needs real file)


# Base CSS shared by all pages
BASE_CSS = """
* { box-sizing: border-box; }
body {
    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
    margin: 0; padding: 0;
    background: #0a0a0a; color: #e0e0e0;
}
.nav {
    background: #1a1a1a; padding: 10px 20px;
    display: flex; gap: 20px; align-items: center;
    border-bottom: 1px solid #333;
}
.nav a { color: #ff6b6b; text-decoration: none; }
.nav a:hover { text-decoration: underline; }
.nav .brand { font-weight: bold; font-size: 18px; }
.container { padding: 20px; }
a { color: #ff6b6b; }
input[type="text"], select {
    padding: 12px 16px; font-size: 16px;
    border: 2px solid #333; border-radius: 8px;
    background: #1a1a1a; color: #fff;
}
input[type="text"]:focus { outline: none; border-color: #ff6b6b; }
button {
    padding: 12px 24px; font-size: 16px;
    background: #ff6b6b; color: #fff;
    border: none; border-radius: 8px; cursor: pointer;
}
button:hover { background: #ff5252; }
.search-box { display: flex; gap: 10px; margin-bottom: 20px; }
"""

def layout(title: str, content: str, extra_css: str = "", extra_head: str = "") -> str:
    """Build a complete HTML page with consistent layout.

    Args:
        title: Page title (will be appended with " - neopig")
        content: HTML content for the page body
        extra_css: Additional CSS to include
        extra_head: Additional head elements (scripts, links, etc.)

    Returns:
        Complete HTML document string
    """
    return f"""<!DOCTYPE html>
<html>
<head>
    <title>{title} - neopig</title>
    <style>
        {BASE_CSS}
        {extra_css}
    </style>
    {extra_head}
</head>
<body>
    <!-- NAV -->
    {content}
    <script>
    </script>
</body>
</html>"""

# Extended CSS for view/detail pages (adds to BASE_CSS)
VIEW_CSS = """
h1 { color: #ff6b6b; font-size: 20px; margin: 0 0 10px 0; }
h3 { color: #ff6b6b; font-size: 16px; margin: 30px 0 15px 0; border-bottom: 1px solid #333; padding-bottom: 8px; }
.meta { background: #1a1a1a; padding: 15px; border-radius: 8px; margin: 15px 0; }
.meta-row { display: flex; margin: 8px 0; }
.meta-label { width: 100px; color: #888; font-size: 13px; }
.meta-value { flex: 1; word-break: break-all; font-size: 13px; }
.meta-value a { color: #4ade80; }
.hero { text-align: center; margin-bottom: 20px; }
.hero img, .hero video { max-width: 100%; max-height: 60vh; border-radius: 8px; }
.content-rendered {
    background: #1a1a1a; padding: 20px; border-radius: 8px;
    line-height: 1.7; font-size: 14px; color: #ccc; text-align: left;
}
.content-rendered img { max-width: 100%; height: auto; margin: 10px 0; }
.content-rendered img.avatar,
.content-rendered img[alt="avatar"] {
    display: inline-block; vertical-align: middle;
    width: 40px; height: 40px; border-radius: 50%;
    margin: 0 10px 0 0; object-fit: cover;
}
.content-rendered img.emoji,
.content-rendered img[alt^=":"][alt$=":"],
.content-rendered img[src*="emoji"] {
    display: inline; width: 20px; height: 20px;
    max-width: 20px; margin: 0 2px; vertical-align: text-bottom;
}
.content-rendered a { color: #ff6b6b; }
.content-rendered pre, .content-rendered code {
    background: #252525; padding: 2px 6px;
    border-radius: 4px; font-family: monospace; font-size: 13px;
}
.content-rendered pre {
    padding: 15px; display: block;
    white-space: pre-wrap; overflow-x: auto;
}
.content-rendered blockquote {
    background: #151520; border-left: 3px solid #4a9eff;
    padding: 12px 16px; margin: 16px 0;
    border-radius: 0 6px 6px 0; color: #aaa; font-style: italic;
}
.content-rendered hr { border: none; border-top: 1px solid #333; margin: 24px 0; }
.content-rendered p { margin: 0 0 16px 0; line-height: 1.7; }
.content-rendered h1, .content-rendered h2, .content-rendered h3, .content-rendered h4, .content-rendered h5, .content-rendered h6 {
    display: block; color: #ff6b6b; margin-top: 28px; margin-bottom: 12px;
}
.content-rendered h1 { font-size: 1.8em; }
.content-rendered h2 { font-size: 1.5em; }
.content-rendered h3 { font-size: 1.25em; }
.media-grid {
    display: grid;
    grid-template-columns: repeat(auto-fill, minmax(150px, 1fr));
    gap: 10px; margin-top: 15px;
}
.media-card {
    background: #1a1a1a; border-radius: 8px;
    overflow: hidden; display: block; transition: transform 0.2s;
    text-decoration: none;
}
.media-card:hover { transform: scale(1.02); }
.media-card img, .media-card video, .media-card svg {
    width: 100%; height: 120px;
    object-fit: contain; background: #0a0a0a;
}
.media-card-name {
    padding: 4px 8px; font-size: 11px; color: #888;
    white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
    background: #111; text-align: left; text-decoration: none;
}
.tag { background: #333; padding: 2px 8px; border-radius: 4px; font-size: 12px; margin-right: 5px; }
"""


def render_detail_page(
    title: str,
    hero_html: str,
    meta_rows: list,
    media_items: list,
    content_html: str,
    screenshot_hashes: list,
    source_domain: str,
    page_uri: str,
    download_btn_html: str = "",
    noai: bool = False,
    lang: str = "en",
    sources_html: str = "",
) -> str:
    """
    Shared template for media view and page view.

    Args:
        title: Page title
        hero_html: HTML for hero section (media element or empty)
        meta_rows: List of (label, value_html) tuples for metadata
        media_items: List of media dicts with md5_hash, media_type
        content_html: Rendered markdown/content HTML
        screenshot_hashes: List of screenshot md5 hashes
        source_domain: Domain for AI prompt
        page_uri: Page URI for AI prompt
        download_btn_html: Optional download button HTML
        noai: Disable AI assistant
        sources_html: Optional HTML for "Used on X pages" section (reverse image search)
    """
    import html as html_module

    # Build metadata section
    meta_html = ''.join(
        f'<div class="meta-row"><span class="meta-label">{label}:</span><span class="meta-value">{value}</span></div>'
        for label, value in meta_rows
    )

    # Build media gallery
    media_grid = ""
    if media_items:
        media_cards = []
        # MIME to extension mapping for non-renderable files
        mime_to_ext = {
            'text/javascript': '.js', 'application/javascript': '.js',
            'text/css': '.css', 'text/html': '.html',
            'text/xml': '.xml', 'application/xml': '.xml',
            'application/rss+xml': '.rss', 'application/atom+xml': '.atom',
            'application/json': '.json', 'text/plain': '.txt',
            'font/woff': '.woff', 'font/woff2': '.woff2',
            'font/ttf': '.ttf', 'font/otf': '.otf',
            'application/font-woff': '.woff', 'application/font-woff2': '.woff2',
        }
        def placeholder_svg(ext, icon, color):
            return f'<svg viewBox="0 0 100 100" style="width:100%;height:100%;background:#1a1a1a;border-radius:4px;"><text x="50" y="40" text-anchor="middle" fill="{color}" font-size="24">{icon}</text><text x="50" y="65" text-anchor="middle" fill="#888" font-family="monospace" font-size="14" font-weight="bold">{ext}</text></svg>'
        def get_filename(uri):
            if not uri: return ""
            # Extract filename from URI path
            from urllib.parse import urlparse, unquote
            path = urlparse(uri).path
            name = unquote(path.split("/")[-1]) if path else ""
            # Truncate long names
            return name[:30] + "..." if len(name) > 33 else name
        for m in media_items:
            mtype = m.get("media_type", "")
            mime = m.get("mime_type", "")
            filename = get_filename(m.get("media_uri", ""))
            # For placeholder, use filename if available, otherwise extension
            display_name = filename[:20] if filename else mime_to_ext.get(mime, "")
            if mtype == "video":
                el = f'<video src="/media/{m["md5_hash"]}" muted loop preload="metadata" onmouseenter="this.play()" onmouseleave="this.pause()"></video>'
            elif mtype == "code":
                ext = mime_to_ext.get(mime, m.get("alt_text", ""))
                if ext and not ext.startswith("."): ext = "." + ext
                el = placeholder_svg(display_name or ext, "{ }", "#6af")
            elif mtype == "style":
                el = placeholder_svg(display_name or ".css", "#", "#f6a")
            elif mtype == "font":
                ext = mime_to_ext.get(mime, ".font")
                el = placeholder_svg(display_name or ext, "Aa", "#af6")
            else:
                el = f'<img src="/media/{m["md5_hash"]}" loading="lazy">'
            filename_el = f'<div class="media-card-name">{html_module.escape(filename)}</div>' if filename else ""
            media_cards.append(f'''<a href="/view/{m["md5_hash"]}" class="media-card">{el}{filename_el}</a>''')
        media_count = len(media_items)
        media_grid = '''
        <div class="page-media">
            <h3>{{media}} (''' + str(media_count) + ''')</h3>
            <div class="media-grid">''' + ''.join(media_cards) + '''</div>
        </div>'''

    escaped_title = html_module.escape(title)
    escaped_uri = html_module.escape(page_uri or "")

    html = f"""<!DOCTYPE html>
<html>
<head>
    <title>{escaped_title} - neopig</title>
    <link rel="stylesheet" href="/static/vendor/highlight-github-dark.min.css">
    <script src="/static/vendor/highlight.min.js"></script>
    <style>
        {BASE_CSS}
        {VIEW_CSS}
        .screenshot-stack {{ border-radius: 8px; }}
        .screenshot-stack img {{ border-radius: 0; }}
        .screenshot-stack img:first-child {{ border-radius: 8px 8px 0 0; }}
        .screenshot-stack img:last-child {{ border-radius: 0 0 8px 8px; }}
        .screenshot-stack img:only-child {{ border-radius: 8px; }}
        #screenshot-container.collapsed {{ display: none; }}
        .hero-grid {{ display: grid; grid-template-columns: 1fr; gap: 20px; margin-bottom: 20px; }}
        .hero-grid.has-hero {{ grid-template-columns: 1fr 1fr; }}
        .content-grid {{ display: grid; grid-template-columns: 1fr; gap: 20px; margin-top: 20px; }}
        .content-grid.has-screenshots {{ grid-template-columns: 1fr 1fr; }}
        .content-grid.ss-collapsed {{ grid-template-columns: 1fr 20px; }}
        .content-grid.ss-collapsed .screenshot-col h3 {{ writing-mode: vertical-rl; text-orientation: mixed; margin: 0; font-size: 12px; }}
        .content-rendered img.emoji {{ display: inline; width: 20px; height: 20px; margin: 0 2px; vertical-align: text-bottom; }}
        .content-rendered .post-block {{ display: grid; grid-template-columns: 48px 1fr; gap: 12px; padding: 16px 0; border-bottom: 1px solid #252530; }}
        .content-rendered .post-block:last-child {{ border-bottom: none; }}
        .content-rendered .post-avatar-col {{ grid-column: 1; display: flex; flex-direction: column; align-items: center; width: 48px; }}
        .content-rendered .post-avatar {{ width: 40px; height: 40px; border-radius: 50%; object-fit: cover; }}
        .content-rendered .post-avatar-placeholder {{ width: 40px; height: 40px; border-radius: 50%; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: #fff; display: flex; align-items: center; justify-content: center; font-weight: bold; font-size: 18px; }}
        .content-rendered .post-content {{ grid-column: 2; min-width: 0; }}
        .content-rendered pre {{ position: relative; }}
        .content-rendered pre .code-actions {{ position: absolute; top: 8px; right: 8px; display: flex; gap: 5px; }}
        .content-rendered pre .code-actions button {{ background: #444; border: none; color: #ccc; padding: 4px 8px; border-radius: 4px; cursor: pointer; font-size: 11px; }}
        .content-rendered pre .code-actions button:hover {{ background: #555; }}
        .content-rendered pre .code-actions button.copied {{ background: #4ade80; color: #000; }}
        .sources-section {{ margin-top: 40px; padding-top: 20px; border-top: 1px solid #333; }}
        .sources-section h3 {{ cursor: pointer; user-select: none; }}
        .sources-section h3:hover {{ color: #ff6b6b; }}
        .sources-list {{ display: none; margin-top: 15px; }}
        .sources-list.expanded {{ display: block; }}
        .sources-list table {{ width: 100%; border-collapse: collapse; font-size: 13px; }}
        .sources-list th, .sources-list td {{ padding: 8px 12px; text-align: left; border-bottom: 1px solid #333; }}
        .sources-list th {{ background: #1a1a1a; color: #888; font-weight: normal; }}
        .sources-list td a {{ color: #6ea8fe; }}
        .sources-list td a:hover {{ color: #ff6b6b; }}
        .sources-list .source-thumb {{ width: 40px; height: 40px; object-fit: cover; border-radius: 4px; }}
        @media (max-width: 768px) {{ .hero-grid, .hero-grid.has-hero, .content-grid, .content-grid.has-screenshots {{ grid-template-columns: 1fr; }} }}
    </style>
</head>
<body>
    <!-- NAV -->
    <div class="container">
    <form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
        <input type="text" name="q" placeholder="{{search_placeholder}}" style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;">
        <button type="submit" style="padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">{{search}}</button>
    </form>
    <div class="hero-grid{' has-hero' if hero_html else ''}">
        {f'<div class="hero-left"><div style="background:#111;border-radius:12px;padding:20px;text-align:center;">{hero_html}</div></div>' if hero_html else ''}
        <div class="hero-right" style="background:#151515;border-radius:12px;padding:20px;">
            <h1 style="margin:0 0 15px 0;font-size:1.3em;">{escaped_title}</h1>
            <div class="meta">{meta_html}</div>
            {download_btn_html}
        </div>
    </div>
    {media_grid}
    <div id="content-grid" class="content-grid{' has-screenshots' if screenshot_hashes else ''}">
        <div class="content-col">
            {'<h3>{{content}}</h3><div class="content-rendered">' + content_html + '</div>' if content_html else ''}
        </div>
        {('<div class="screenshot-col"><h3 style="cursor:pointer;" onclick="toggleScreenshot()"><span id="ss-toggle">▼</span> {{screenshot}}</h3><div id="screenshot-container" class="screenshot-stack">' + ''.join('<img src="/media/' + h + '" alt="Screenshot" style="width:100%;display:block;">' for h in screenshot_hashes) + '</div></div>') if screenshot_hashes else ''}
    </div>
    <script>
        hljs.highlightAll();

        // Screenshot toggle
        function toggleScreenshot() {{
            const container = document.getElementById('screenshot-container');
            const toggle = document.getElementById('ss-toggle');
            const grid = document.getElementById('content-grid');
            if (container) {{
                const collapsed = !container.classList.contains('collapsed');
                container.classList.toggle('collapsed', collapsed);
                grid.classList.toggle('ss-collapsed', collapsed);
                toggle.textContent = collapsed ? '▶' : '▼';
                localStorage.setItem('neopig_ss_collapsed', collapsed);
            }}
        }}
        if (localStorage.getItem('neopig_ss_collapsed') === 'true') {{
            const container = document.getElementById('screenshot-container');
            const toggle = document.getElementById('ss-toggle');
            const grid = document.getElementById('content-grid');
            if (container) {{
                container.classList.add('collapsed');
                grid.classList.add('ss-collapsed');
                toggle.textContent = '▶';
            }}
        }}

        // Auto-detect emojis/avatars
        document.querySelectorAll('.content-rendered img').forEach(img => {{
            const check = () => {{
                const w = img.naturalWidth || img.width, h = img.naturalHeight || img.height;
                if (w > 0 && h > 0) {{
                    if (w <= 24 && h <= 24) img.classList.add('emoji');
                    else if (w <= 60 && h <= 60) img.classList.add('avatar');
                }}
            }};
            if (img.complete) check(); else img.onload = check;
        }});

        // Code block copy/download
        const extMap = {{'python':'py','javascript':'js','typescript':'ts','cpp':'cpp','c':'c','java':'java','rust':'rs','go':'go','ruby':'rb','php':'php','html':'html','css':'css','json':'json','yaml':'yaml','sql':'sql','bash':'sh','sh':'sh','markdown':'md'}};
        document.querySelectorAll('.content-rendered pre').forEach((pre, idx) => {{
            const code = pre.querySelector('code') || pre;
            const text = code.textContent;
            let ext = 'txt';
            (code.className || '').split(/\\s+/).forEach(cls => {{
                const m = cls.match(/^(?:language-)?(.+)$/);
                if (m && extMap[m[1].toLowerCase()]) ext = extMap[m[1].toLowerCase()];
            }});
            const actions = document.createElement('div');
            actions.className = 'code-actions';
            const copyBtn = document.createElement('button');
            copyBtn.textContent = T.copy;
            copyBtn.onclick = async () => {{
                await navigator.clipboard.writeText(text);
                copyBtn.textContent = T.copied; copyBtn.classList.add('copied');
                setTimeout(() => {{ copyBtn.textContent = T.copy; copyBtn.classList.remove('copied'); }}, 2000);
            }};
            const dlBtn = document.createElement('button');
            dlBtn.textContent = T.download;
            dlBtn.onclick = () => {{
                const blob = new Blob([text], {{type: 'text/plain'}});
                const a = document.createElement('a');
                a.href = URL.createObjectURL(blob);
                a.download = `code-${{idx + 1}}.${{ext}}`;
                a.click();
            }};
            actions.appendChild(copyBtn); actions.appendChild(dlBtn);
            pre.appendChild(actions);
        }});

        // Forum post restructuring
        (function() {{
            const container = document.querySelector('.content-rendered');
            if (!container) return;
            const postStarts = [];
            container.querySelectorAll('p').forEach(p => {{
                // Pattern 1: <img alt="avatar"> + <strong>username</strong>
                const img = p.querySelector('img[alt="avatar"]');
                const strong = p.querySelector('strong');
                if (img && strong) {{
                    postStarts.push({{p, img, username: strong.textContent}});
                    return;
                }}
                // Pattern 2: "avatar **Username**" as text (no img element)
                const text = p.textContent.trim();
                if (text.startsWith('avatar ') && strong) {{
                    postStarts.push({{p, img: null, username: strong.textContent}});
                    return;
                }}
                // Pattern 3: Just check for img.avatar class followed by strong
                const avatarImg = p.querySelector('img.avatar');
                if (avatarImg && strong) {{
                    postStarts.push({{p, img: avatarImg, username: strong.textContent}});
                }}
            }});
            if (!postStarts.length) return;
            postStarts.forEach(({{p, img, username}}, idx) => {{
                if (p.closest('.post-block')) return;
                const block = document.createElement('div');
                block.className = 'post-block';
                const avatarCol = document.createElement('div');
                avatarCol.className = 'post-avatar-col';
                if (img && img.src && !img.src.includes('undefined') && !img.src.endsWith('#') && img.getAttribute('src') !== '#') {{
                    const avatar = document.createElement('img');
                    avatar.className = 'post-avatar';
                    avatar.src = img.src;
                    avatarCol.appendChild(avatar);
                }} else {{
                    const ph = document.createElement('div');
                    ph.className = 'post-avatar-placeholder';
                    ph.textContent = username.charAt(0).toUpperCase();
                    avatarCol.appendChild(ph);
                }}
                const contentCol = document.createElement('div');
                contentCol.className = 'post-content';
                contentCol.innerHTML = `<div class="post-header"><strong>${{username}}</strong></div>`;
                let sib = p.nextElementSibling;
                const nextP = idx < postStarts.length - 1 ? postStarts[idx + 1].p : null;
                while (sib && sib !== nextP && sib.tagName !== 'HR') {{
                    contentCol.appendChild(sib.cloneNode(true));
                    const rm = sib; sib = sib.nextElementSibling; rm.remove();
                }}
                if (sib && sib.tagName === 'HR') sib.remove();
                block.appendChild(avatarCol); block.appendChild(contentCol);
                p.parentNode.insertBefore(block, p); p.remove();
            }});
        }})();
    </script>
    {sources_html}
    {'' if noai else f'''<script>
    window.UNCLOSEAI_SYSTEM_PROMPT = "You are a neopig assistant, a machine learning persona built into the neopig web archiver. You are viewing an archived copy of a page from {source_domain}. The page title is: {escaped_title}. neopig archived this page - neopig did NOT create the original content. The original content was created by {source_domain}. Help users understand what is on this archived page. Be factual and concise. Only describe what you can actually see - do not invent or embellish.";
    window.UNCLOSEAI_PREFILL = "I am a neopig assistant viewing an archived copy of a page from {source_domain}. The page title is: {escaped_title}. neopig archived this page - neopig did NOT create the original content. The original content was created by {source_domain}. I will help users understand what is on this archived page. I will be factual and concise. I will only describe what I can actually see - I will not invent or embellish.";
    </script>
    <script src="https://uncloseai.com/uncloseai.js" type="module"></script>'''}
</body>
</html>"""
    return inject_i18n(html, lang)


def _open_tarball():
    """Open a fresh tarball handle for this thread."""
    if TAR_OFFSET > 0:
        # .run file - need to seek past bootstrap
        f = open(TAR_PATH, 'rb')
        f.seek(TAR_OFFSET)
        return tarfile.open(fileobj=f, mode='r:gz')
    else:
        return tarfile.open(TAR_PATH, 'r:gz')


def read_from_tarball(path: str) -> bytes:
    """Read a file from the tarball. Path is relative to archive root.

    Thread-safe: opens its own tarball handle.
    """
    if not TAR_PATH or not ARCHIVE_ROOT:
        return None
    full_path = f"{ARCHIVE_ROOT}/{path}"
    if full_path in TAR_MEMBERS:
        member = TAR_MEMBERS[full_path]
        tar = _open_tarball()
        try:
            f = tar.extractfile(member)
            if f:
                return f.read()
        finally:
            tar.close()
    return None


def find_media_in_tarball(md5_hash: str) -> tuple:
    """Find media file in tarball by hash. Returns (data, extension) or (None, None).

    Thread-safe: opens its own tarball handle for parallel reads.
    """
    if not TAR_PATH or not ARCHIVE_ROOT:
        return None, None

    # O(1) lookup via pre-built index
    if md5_hash in TAR_MEDIA_INDEX:
        name = TAR_MEDIA_INDEX[md5_hash]
        member = TAR_MEMBERS.get(name)
        if member:
            tar = _open_tarball()
            try:
                f = tar.extractfile(member)
                if f:
                    ext = Path(name).suffix
                    return f.read(), ext
            finally:
                tar.close()

    return None, None


class ArchiveDB:
    """Simple sync SQLite wrapper for archive.db (FTS5 search only)."""
    def __init__(self, db_path):
        import sqlite3
        self.conn = sqlite3.connect(db_path)
        self.conn.row_factory = sqlite3.Row

    async def search_pages(self, query, limit=50):
        cursor = self.conn.execute(
            "SELECT uri, title, snippet(pages_fts, 2, '<b>', '</b>', '...', 32) as snippet "
            "FROM pages_fts WHERE pages_fts MATCH ? LIMIT ?",
            (query, limit)
        )
        return [dict(row) for row in cursor.fetchall()]

    async def get_stats(self):
        cursor = self.conn.execute("SELECT COUNT(*) FROM pages")
        return {"pages": cursor.fetchone()[0], "media": 0, "screenshots": 0}


@app.on_event("startup")
async def startup_event():
    """Initialize database on startup."""
    global db
    if CRAWL_DISABLED:
        logger.info("Crawl disabled via NEOPIG_DISABLE_CRAWL")
    if IMPORT_MODE:
        logger.info("Import mode enabled via NEOPIG_IMPORT - archive uploads allowed")
    if TAR_PATH:
        # Tarball mode: use full Database for neopig.db, legacy ArchiveDB for archive.db
        if DB_PATH.endswith('neopig.db'):
            db = Database(DB_PATH)
            await db.init()
            logger.info(f"Using neopig database from tarball: {DB_PATH}")
        else:
            db = ArchiveDB(DB_PATH)
            logger.info(f"Using archive database: {DB_PATH}")
    else:
        # Normal mode: use full async database
        db = Database(DB_PATH)
        await db.init()  # Handles schema + WAL mode
        # Backfill uri_hash for existing pages
        count = await db.backfill_page_hashes()
        if count:
            logger.info(f"Backfilled {count} page URI hashes")
        VAULT_PATH.mkdir(parents=True, exist_ok=True)
        logger.info(f"Vault directory ready: {VAULT_PATH}")

        # Mark any orphaned "running" jobs as "paused" (server was killed)
        async with db.session() as session:
            result = await session.execute(
                text("UPDATE crawl_jobs SET status = 'paused' WHERE status = 'running'")
            )
            if result.rowcount:
                await session.commit()
                logger.info(f"Marked {result.rowcount} orphaned running job(s) as paused")


@app.on_event("shutdown")
async def shutdown_event():
    """Pause active crawls on shutdown so they can be resumed."""
    if ACTIVE_CRAWL_TASKS:
        logger.info(f"Pausing {len(ACTIVE_CRAWL_TASKS)} active crawl(s)...")
        for job_id, task in list(ACTIVE_CRAWL_TASKS.items()):
            task.cancel()
            try:
                await task
            except asyncio.CancelledError:
                pass
            # Mark as paused so it can be resumed
            try:
                await db.pause_crawl_job(job_id)
            except Exception as e:
                logger.error(f"Failed to pause job {job_id}: {e}")
        logger.info("All crawls paused")


class CrawlRequest(BaseModel):
    """Request to start a new crawl."""
    targets: List[str] = []  # Multiple target URIs
    target_uri: str = ""  # Deprecated: single target (for backwards compat)
    keywords: List[str] = []
    mode: str = "all"  # text, images, videos, media, all
    depth: int = -1  # -1 = unlimited
    max_pages: int = -1  # -1 = unlimited
    fresh: bool = False  # Start fresh (rotate state files) - default False for safety
    fast: bool = False  # No crawl delay
    screenshots: bool = True  # Take page screenshots
    hydra: bool = False  # Parse RSS/Atom feeds for bleeding edge discovery


@app.get("/", response_class=HTMLResponse)
async def index(lang: str = Cookie(None), accept_language: str = Header(None)):
    """Simple search UI."""
    language = get_lang(lang, accept_language)
    return inject_i18n(get_search_html(), language)


@app.get("/crawl", response_class=HTMLResponse)
async def crawl_page(lang: str = Cookie(None), accept_language: str = Header(None)):
    """Crawler command page."""
    language = get_lang(lang, accept_language)
    html = CRAWL_HTML
    if CRAWL_DISABLED:
        # Inject disabled state into the page
        html = html.replace('const CRAWL_DISABLED = false;', 'const CRAWL_DISABLED = true;')
    return inject_i18n(html, language)


# Page-specific CSS for Search page
SEARCH_CSS = """
h1 { color: #ff6b6b; margin-bottom: 5px; }
.subtitle { color: #666; margin-bottom: 20px; }
.stats { padding: 10px 15px; background: #1a1a1a; border-radius: 8px; margin-bottom: 20px; font-size: 14px; color: #888; }
.section-title { color: #ff6b6b; font-size: 18px; margin: 25px 0 15px 0; border-bottom: 1px solid #333; padding-bottom: 8px; }
.results { display: grid; grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); gap: 15px; }
.result { background: #1a1a1a; border-radius: 8px; overflow: hidden; transition: transform 0.2s; }
.result:hover { transform: scale(1.02); }
.result img, .result video { width: 100%; height: 200px; object-fit: contain; background: #1a1a1a; }
.result-info { padding: 10px; }
.result-filename { font-size: 12px; color: #aaa; margin-bottom: 4px; word-break: break-all; }
.result-hash { font-family: monospace; font-size: 11px; color: #ff6b6b; text-decoration: none; word-break: break-all; }
.result-hash:hover { text-decoration: underline; }
.result-meta { font-size: 12px; color: #888; margin-top: 5px; }
.result-keywords { display: flex; flex-wrap: wrap; gap: 5px; margin-top: 8px; }
.tag { background: #333; padding: 2px 8px; border-radius: 4px; font-size: 11px; color: #aaa; }
.no-results { text-align: center; padding: 60px; color: #666; }
.result-count { padding: 10px 15px; color: #888; font-size: 14px; }
.result-count.over-9000 { color: #ff6b6b; font-weight: bold; font-size: 18px; text-shadow: 0 0 10px rgba(255,107,107,0.5); animation: powerUp 0.5s ease-out; }
@keyframes powerUp { 0% { transform: scale(1); } 50% { transform: scale(1.2); } 100% { transform: scale(1); } }
.pagination { display: flex; gap: 15px; justify-content: center; padding: 20px; margin-top: 20px; border-top: 1px solid #333; }
.pagination button { padding: 10px 20px; background: #4ecdc4; color: #1a1a2e; border: none; border-radius: 5px; cursor: pointer; font-weight: bold; }
.pagination button:hover { background: #7bed72; }
.media-link { display: block; cursor: pointer; }
.media-link:hover img, .media-link:hover video { opacity: 0.8; }
.page-results { display: flex; flex-direction: column; gap: 10px; }
.page-result { background: #1a1a1a; border-radius: 8px; padding: 15px; transition: background 0.2s; }
.page-result:hover { background: #252525; }
.page-result a { color: #ff6b6b; text-decoration: none; font-size: 16px; font-weight: 500; }
.page-result a:hover { text-decoration: underline; }
.page-path { font-size: 12px; color: #4ade80; margin-top: 4px; font-family: monospace; }
.page-snippet { font-size: 13px; color: #999; margin-top: 8px; line-height: 1.5; }
.page-snippet mark { background: #ff6b6b33; color: #ff9999; padding: 1px 3px; border-radius: 2px; }
.search-columns { display: grid; grid-template-columns: 1fr; gap: 30px; align-items: start; }
.search-columns.has-pages { grid-template-columns: 1fr 2fr; }
.search-columns .page-column { display: none; }
.search-columns.has-pages .page-column { display: block; }
@media (max-width: 1000px) { .search-columns.has-pages { grid-template-columns: 1fr; } }
"""

SEARCH_CONTENT = """
    <div class="container">
    <div class="search-box">
        <input type="text" id="query" placeholder="{{search_placeholder}}" autofocus>
        <select id="type">
            <option value="">{{all_types}}</option>
            <option value="image">{{images}}</option>
            <option value="video">{{videos}}</option>
            <option value="audio">{{audio}}</option>
        </select>
        <button onclick="search()">{{search}}</button>
    </div>

    <div class="stats" id="stats">{{loading_stats}}</div>

    <div class="search-columns">
        <div class="page-column" id="page-section">
            <h2 class="section-title">{{pages_label}}</h2>
            <div class="page-results" id="page-results"></div>
        </div>

        <div class="media-column" id="media-section">
            <h2 class="section-title">{{media}}</h2>
            <div class="results" id="results"></div>
        </div>
    </div>

    <script>
        async function loadStats() {
            const res = await fetch('/api/stats');
            const stats = await res.json();
            document.getElementById('stats').innerHTML =
                `<strong>${stats.total_media}</strong> ${T.media} | ` +
                `<strong>${stats.by_type?.image || 0}</strong> ${T.images} | ` +
                `<strong>${stats.by_type?.video || 0}</strong> ${T.videos} | ` +
                `<strong>${stats.total_sources}</strong> ${T.sources} | ` +
                `<strong>${stats.total_pages || 0}</strong> ${T.pages_label}`;
        }

        const OVER_9000 = 9000;
        let currentOffset = 0;

        async function search(offset = 0) {
            currentOffset = offset;
            const query = document.getElementById('query').value;
            const type = document.getElementById('type').value;

            // Search media - IT'S OVER 9000!
            let mediaUrl = `/api/search?q=${encodeURIComponent(query)}&limit=${OVER_9000}&offset=${offset}`;
            if (type) mediaUrl += `&type=${type}`;

            const mediaRes = await fetch(mediaUrl);
            const mediaResults = await mediaRes.json();

            const mediaContainer = document.getElementById('results');
            const mediaSection = document.getElementById('media-section');

            // Build count display
            let countDisplay = '';
            if (mediaResults.length >= OVER_9000) {
                countDisplay = `<div class="result-count over-9000">${T.over_9000}</div>`;
            } else if (mediaResults.length > 0) {
                const start = offset + 1;
                const end = offset + mediaResults.length;
                countDisplay = `<div class="result-count">${start.toLocaleString()}-${end.toLocaleString()} of ${offset > 0 ? 'many' : mediaResults.length.toLocaleString()}</div>`;
            }

            // Build pagination
            let pagination = '';
            if (offset > 0 || mediaResults.length >= OVER_9000) {
                pagination = '<div class="pagination">';
                if (offset > 0) {
                    pagination += `<button onclick="search(${Math.max(0, offset - OVER_9000)})">← ${T.previous} ${OVER_9000.toLocaleString()}</button>`;
                }
                if (mediaResults.length >= OVER_9000) {
                    pagination += `<button onclick="search(${offset + OVER_9000})">${T.next} ${OVER_9000.toLocaleString()} →</button>`;
                }
                pagination += '</div>';
            }

            if (mediaResults.length === 0 && offset === 0) {
                mediaContainer.innerHTML = `<div class="no-results">${T.no_media}</div>`;
            } else if (mediaResults.length === 0) {
                mediaContainer.innerHTML = `<div class="no-results">${T.no_more}</div>` + pagination;
            } else {
                mediaContainer.innerHTML = countDisplay + mediaResults.map(r => {
                    const isVideo = r.media_type === 'video';
                    const isCode = r.media_type === 'code';
                    const isStyle = r.media_type === 'style';
                    const isFont = r.media_type === 'font';

                    // Get file extension from mime_type
                    const getExt = (mime) => {
                        const mimeToExt = {
                            'text/javascript': '.js', 'application/javascript': '.js',
                            'text/x-python': '.py', 'text/python': '.py',
                            'text/x-c': '.c', 'text/x-csrc': '.c',
                            'text/x-c++': '.cpp', 'text/x-c++src': '.cpp',
                            'text/x-java': '.java', 'text/java': '.java',
                            'text/x-ruby': '.rb', 'text/ruby': '.rb',
                            'text/x-go': '.go', 'text/go': '.go',
                            'text/x-rust': '.rs', 'text/rust': '.rs',
                            'text/x-php': '.php', 'text/php': '.php',
                            'text/x-shellscript': '.sh', 'text/x-sh': '.sh',
                            'text/css': '.css', 'text/html': '.html',
                            'text/xml': '.xml', 'application/xml': '.xml',
                            'application/rss+xml': '.rss', 'application/atom+xml': '.atom',
                            'application/json': '.json', 'text/json': '.json',
                            'text/x-yaml': '.yaml', 'text/yaml': '.yaml',
                            'text/markdown': '.md', 'text/x-markdown': '.md',
                            'text/plain': '.txt', 'text/x-typescript': '.ts',
                            'font/woff': '.woff', 'font/woff2': '.woff2',
                            'font/ttf': '.ttf', 'font/otf': '.otf',
                            'application/font-woff': '.woff', 'application/font-woff2': '.woff2',
                            'application/x-font-ttf': '.ttf', 'application/x-font-otf': '.otf',
                        };
                        return mimeToExt[mime] || (r.alt_text ? '.' + r.alt_text : '');
                    };

                    // SVG placeholder for non-renderable files
                    const placeholder = (ext, icon, color) => `<svg viewBox="0 0 100 100" style="width:100%;height:100%;background:#1a1a1a;border-radius:4px;">
                        <text x="50" y="40" text-anchor="middle" fill="${color}" font-size="24">${icon}</text>
                        <text x="50" y="65" text-anchor="middle" fill="#888" font-family="monospace" font-size="14" font-weight="bold">${ext}</text>
                    </svg>`;

                    // Extract filename from media_uri
                    const getFilename = (uri) => {
                        if (!uri) return '';
                        try {
                            const path = new URL(uri).pathname;
                            const name = decodeURIComponent(path.split('/').pop() || '');
                            return name.length > 40 ? name.slice(0, 37) + '...' : name;
                        } catch { return ''; }
                    };
                    const filename = getFilename(r.media_uri);
                    const displayName = filename ? filename.slice(0, 20) : getExt(r.mime_type);

                    let mediaEl;
                    if (isVideo) {
                        mediaEl = `<video src="/media/${r.md5_hash}" muted loop preload="metadata" onmouseenter="this.play()" onmouseleave="this.pause()"></video>`;
                    } else if (isCode) {
                        mediaEl = placeholder(displayName || getExt(r.mime_type), '{ }', '#6af');
                    } else if (isStyle) {
                        mediaEl = placeholder(displayName || '.css', '#', '#f6a');
                    } else if (isFont) {
                        mediaEl = placeholder(displayName || getExt(r.mime_type), 'Aa', '#af6');
                    } else {
                        mediaEl = `<img src="/media/${r.md5_hash}" alt="${r.alt_text || ''}" loading="lazy">`;
                    }

                    const keywords = JSON.parse(r.keywords || '[]');
                    const tagsHtml = keywords.map(k => `<span class="tag">${k}</span>`).join('');

                    return `
                        <div class="result">
                            <a href="/view/${r.md5_hash}" class="media-link">
                                ${mediaEl}
                            </a>
                            <div class="result-info">
                                ${filename ? `<div class="result-filename">${filename}</div>` : ''}
                                <a href="/view/${r.md5_hash}" class="result-hash">${r.md5_hash}</a>
                                <div class="result-meta">
                                    ${r.media_type} · ${formatBytes(r.file_size)}
                                    ${r.alt_text ? ` · ${r.alt_text.substring(0, 50)}` : ''}
                                </div>
                                <div class="result-keywords">${tagsHtml}</div>
                            </div>
                        </div>
                    `;
                }).join('') + pagination;
            }

            // Search pages
            const pageContainer = document.getElementById('page-results');
            const searchColumns = document.querySelector('.search-columns');

            if (query.trim()) {
                const pageRes = await fetch(`/api/search/pages?q=${encodeURIComponent(query)}&limit=30`);
                const pageResults = await pageRes.json();

                if (pageResults.length > 0) {
                    searchColumns.classList.add('has-pages');
                    pageContainer.innerHTML = pageResults.map(p => `
                        <div class="page-result">
                            <a href="/page/${p.uri_hash}">${p.title || p.uri}</a>
                            <div class="page-path">${p.path || p.uri}</div>
                            <div class="page-snippet">${p.snippet || ''}</div>
                        </div>
                    `).join('');
                } else {
                    searchColumns.classList.remove('has-pages');
                    pageContainer.innerHTML = '';
                }
            } else {
                searchColumns.classList.remove('has-pages');
                pageContainer.innerHTML = '';
            }
        }

        function formatBytes(bytes) {
            if (!bytes) return '?';
            if (bytes < 1024) return bytes + ' B';
            if (bytes < 1024*1024) return (bytes/1024).toFixed(1) + ' KB';
            return (bytes/1024/1024).toFixed(1) + ' MB';
        }

        // Enter key to search
        document.getElementById('query').addEventListener('keypress', e => {
            if (e.key === 'Enter') search();
        });

        // Load stats on page load
        loadStats();

        // Check for query param from nav search
        const urlParams = new URLSearchParams(window.location.search);
        const q = urlParams.get('q');
        if (q) {
            document.getElementById('query').value = q;
        }

        // Initial search (show all media or query)
        search();
    </script>
    </div>
"""

def get_search_html():
    """Build search page using layout."""
    return layout("neopig SERP", SEARCH_CONTENT, SEARCH_CSS)

CRAWL_HTML = """
<!DOCTYPE html>
<html>
<head>
    <title>neopig {{crawl}}</title>
    <style>
        * { box-sizing: border-box; }
        body {
            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
            margin: 0;
            padding: 0;
            background: #0a0a0a;
            color: #e0e0e0;
        }
        .nav {
            background: #1a1a1a;
            padding: 10px 20px;
            display: flex;
            gap: 20px;
            align-items: center;
            border-bottom: 1px solid #333;
        }
        .nav a { color: #ff6b6b; text-decoration: none; }
        .nav a:hover { text-decoration: underline; }
        .nav .brand { font-weight: bold; font-size: 18px; }
        .container { padding: 20px; }
        h1 { color: #ff6b6b; margin-bottom: 5px; }
        .subtitle { color: #666; margin-bottom: 20px; }

        .form-group {
            margin-bottom: 15px;
        }
        label {
            display: block;
            margin-bottom: 5px;
            color: #aaa;
            font-size: 14px;
        }
        input[type="text"], input[type="number"], select {
            width: 100%;
            padding: 12px 16px;
            font-size: 16px;
            border: 2px solid #333;
            border-radius: 8px;
            background: #1a1a1a;
            color: #fff;
        }
        input:focus, select:focus {
            outline: none;
            border-color: #ff6b6b;
        }
        .row {
            display: flex;
            gap: 15px;
        }
        .row > div { flex: 1; }

        button {
            padding: 14px 28px;
            font-size: 16px;
            background: #ff6b6b;
            color: #fff;
            border: none;
            border-radius: 8px;
            cursor: pointer;
            margin-top: 10px;
        }
        button:hover { background: #ff5252; }
        button:disabled {
            background: #444;
            cursor: not-allowed;
        }
        button.secondary {
            background: #333;
        }
        button.secondary:hover {
            background: #444;
        }

        .checkbox-group {
            display: flex;
            align-items: center;
            gap: 8px;
        }
        .checkbox-group input {
            width: auto;
        }

        .jobs-section {
            margin-top: 30px;
            padding-top: 20px;
            border-top: 1px solid #333;
        }
        h2 {
            color: #ff6b6b;
            font-size: 18px;
            margin-bottom: 15px;
        }

        .job {
            background: #1a1a1a;
            border-radius: 8px;
            padding: 15px;
            margin-bottom: 10px;
        }
        .job-header {
            display: flex;
            justify-content: space-between;
            align-items: center;
            margin-bottom: 10px;
        }
        .job-id {
            font-family: monospace;
            color: #666;
        }
        .job-status {
            padding: 4px 10px;
            border-radius: 4px;
            font-size: 12px;
            font-weight: bold;
        }
        .job-status.running { background: #2d5a27; color: #7bed72; }
        .job-status.completed { background: #1a3a4a; color: #6bc5e8; }
        .job-status.failed { background: #5a2727; color: #ed7272; }

        .job-target {
            font-size: 14px;
            word-break: break-all;
            margin-bottom: 5px;
        }
        .job-meta {
            font-size: 12px;
            color: #666;
        }
        .job-stats {
            display: flex;
            gap: 15px;
            margin-top: 10px;
            font-size: 13px;
        }
        .job-stats span {
            background: #252525;
            padding: 4px 10px;
            border-radius: 4px;
        }

        .progress-bar {
            height: 4px;
            background: #333;
            border-radius: 2px;
            margin-top: 10px;
            overflow: hidden;
        }
        .progress-bar-fill {
            height: 100%;
            background: #ff6b6b;
            transition: width 0.3s;
        }
        .progress-bar-fill.running {
            width: 100%;
            background: linear-gradient(90deg, #ff6b6b 0%, #4ecdc4 50%, #ff6b6b 100%);
            background-size: 200% 100%;
            animation: progress-wave 1.5s ease-in-out infinite;
        }
        .job-actions { margin-top: 8px; display: flex; gap: 8px; }
        .job-actions button {
            padding: 4px 12px;
            color: white;
            border: none;
            border-radius: 4px;
            cursor: pointer;
            font-size: 12px;
        }
        .pause-btn { background: #e67e22; }
        .pause-btn:hover { background: #f39c12; }
        .start-btn { background: #27ae60; }
        .start-btn:hover { background: #2ecc71; }
        .recrawl-btn { background: #3498db; }
        .recrawl-btn:hover { background: #2980b9; }
        .delete-btn { background: #7f8c8d; }
        .delete-btn:hover { background: #c0392b; }
        .job-links { margin-left: 10px; font-size: 12px; }
        .job-links a { color: #4ecdc4; margin-right: 8px; }
        .job-links a:hover { color: #7bed72; }
        @keyframes progress-wave {
            0% { background-position: 200% 0; }
            100% { background-position: -200% 0; }
        }
        .job.running {
            border-color: #4ecdc4;
        }
        .job-stats.live {
            color: #4ecdc4;
        }

        .no-jobs {
            color: #666;
            text-align: center;
            padding: 30px;
        }
        .console-btn { background: #2c3e50; }
        .console-btn:hover { background: #34495e; }
        .console-modal {
            display: none;
            position: fixed;
            top: 0; left: 0; right: 0; bottom: 0;
            background: rgba(0,0,0,0.8);
            z-index: 1000;
            padding: 20px;
        }
        .console-modal.active { display: flex; flex-direction: column; }
        .console-header {
            display: flex;
            justify-content: space-between;
            align-items: center;
            padding: 10px 15px;
            background: #1a1a1a;
            border-radius: 8px 8px 0 0;
        }
        .console-header h3 { margin: 0; color: #ff6b6b; }
        .console-close {
            background: #c0392b;
            border: none;
            color: white;
            padding: 5px 15px;
            border-radius: 4px;
            cursor: pointer;
        }
        .console-body {
            flex: 1;
            background: #0d0d0d;
            padding: 15px;
            overflow: auto;
            font-family: monospace;
            font-size: 12px;
            white-space: pre-wrap;
            word-break: break-all;
            color: #4ade80;
            border-radius: 0 0 8px 8px;
        }
        .console-body .log-line { margin: 0; line-height: 1.4; }
        .console-body .log-INFO { color: #4ade80; }
        .console-body .log-WARNING { color: #f39c12; }
        .console-body .log-ERROR { color: #e74c3c; }
    </style>
</head>
<body>
    <!-- NAV -->
    <div class="container">

    <form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
        <input type="text" name="q" placeholder="{{search_placeholder}}" style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
        <select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
            <option value="">{{all_types}}</option>
            <option value="image">{{images}}</option>
            <option value="video">{{videos}}</option>
            <option value="audio">{{audio}}</option>
        </select>
        <button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">{{search}}</button>
    </form>

    <h1>{{crawl}}</h1>
    <p class="subtitle">{{hydrate_subtitle}}</p>

    <form id="crawl-form" onsubmit="startCrawl(event)">
        <div class="form-group">
            <label>{{url}}</label>
            <textarea id="target" placeholder="https://example.com https://another.com" rows="2" required style="width:100%;padding:10px;border:1px solid #333;border-radius:4px;background:#1a1a1a;color:#e0e0e0;font-size:14px;resize:vertical;"></textarea>
        </div>

        <div class="form-group">
            <label>{{keywords}}</label>
            <input type="text" id="keywords" placeholder="e.g. rick and morty, adult swim">
        </div>

        <div class="row">
            <div class="form-group">
                <label>{{mode}}</label>
                <select id="mode">
                    <option value="all">{{everything}}</option>
                    <option value="media" selected>{{all_media}}</option>
                    <option value="images">{{images_only}}</option>
                    <option value="videos">{{videos_only}}</option>
                    <option value="text">{{text_only}}</option>
                </select>
            </div>
            <div class="form-group">
                <label>{{depth}}</label>
                <input type="number" id="depth" value="9" min="-1" max="15">
            </div>
            <div class="form-group">
                <label>{{max_pages}}</label>
                <input type="number" id="max_pages" value="-1" min="-1">
            </div>
        </div>

        <div class="form-group checkbox-group" style="display:flex; flex-wrap:wrap; gap:20px;">
            <div>
                <input type="checkbox" id="fresh" checked>
                <label for="fresh" style="display:inline; margin:0;">{{fresh_start}}</label>
            </div>
            <div>
                <input type="checkbox" id="fast">
                <label for="fast" style="display:inline; margin:0;">{{fast_mode}}</label>
            </div>
            <div>
                <input type="checkbox" id="screenshots" checked>
                <label for="screenshots" style="display:inline; margin:0;">{{screenshots}}</label>
            </div>
            <div>
                <input type="checkbox" id="hydra">
                <label for="hydra" style="display:inline; margin:0;" title="Parse RSS/Atom/Sitemap for fast discovery">Hydra</label>
            </div>
        </div>

        <button type="submit" id="start-btn">{{start_crawl}}</button>
    </form>

    <div class="jobs-section">
        <h2>{{recent_jobs}}</h2>
        <div id="jobs">{{loading}}</div>
    </div>

    <script>
        const CRAWL_DISABLED = false;

        // Disable form when crawling is disabled
        if (CRAWL_DISABLED) {
            document.addEventListener('DOMContentLoaded', () => {
                const form = document.getElementById('crawl-form');
                const inputs = form.querySelectorAll('input, select, textarea, button');
                inputs.forEach(el => {
                    el.disabled = true;
                    el.style.opacity = '0.5';
                    el.style.cursor = 'not-allowed';
                });
                // Add banner
                const banner = document.createElement('div');
                banner.style.cssText = 'background:#442;border:1px solid #664;padding:12px 20px;border-radius:8px;margin-bottom:20px;color:#ffa;';
                banner.innerHTML = '⚠️ Crawling is disabled on this server (read-only archive mode)';
                form.parentNode.insertBefore(banner, form);
            });
        }

        async function startCrawl(e) {
            e.preventDefault();
            if (CRAWL_DISABLED) return;

            const btn = document.getElementById('start-btn');
            btn.disabled = true;
            btn.textContent = T.starting;

            const keywordsRaw = document.getElementById('keywords').value;
            const keywords = keywordsRaw
                .split(/[,\\s]+/)
                .map(k => k.trim())
                .filter(k => k.length > 0);

            // Parse multiple target URIs (space, comma, or newline separated)
            const targetsRaw = document.getElementById('target').value;
            const targets = targetsRaw
                .split(/[,\\s\\n]+/)
                .map(t => t.trim())
                .filter(t => t.length > 0 && t.startsWith('http'));

            if (targets.length === 0) {
                alert(T.valid_uri);
                btn.disabled = false;
                btn.textContent = T.start_crawl;
                return;
            }

            const payload = {
                targets: targets,
                keywords: keywords,
                mode: document.getElementById('mode').value,
                depth: parseInt(document.getElementById('depth').value),
                max_pages: parseInt(document.getElementById('max_pages').value),
                fresh: document.getElementById('fresh').checked,
                fast: document.getElementById('fast').checked,
                screenshots: document.getElementById('screenshots').checked,
                hydra: document.getElementById('hydra').checked
            };

            try {
                const res = await fetch('/api/crawl', {
                    method: 'POST',
                    headers: { 'Content-Type': 'application/json' },
                    body: JSON.stringify(payload)
                });

                if (!res.ok) {
                    const err = await res.json();
                    btn.textContent = T.error + ': ' + (err.detail || T.load_error);
                    btn.style.background = '#c0392b';
                    setTimeout(() => {
                        btn.textContent = T.start_crawl;
                        btn.style.background = '';
                    }, 3000);
                } else {
                    // Clear form and refresh jobs list
                    document.getElementById('target').value = '';
                    document.getElementById('keywords').value = '';
                    btn.textContent = T.started_ok;
                    btn.style.background = '#27ae60';
                    loadJobs();
                    setTimeout(() => {
                        btn.textContent = T.start_crawl;
                        btn.style.background = '';
                    }, 2000);
                }
            } catch (err) {
                btn.textContent = T.error;
                btn.style.background = '#c0392b';
                setTimeout(() => {
                    btn.textContent = T.start_crawl;
                    btn.style.background = '';
                }, 3000);
            }

            btn.disabled = false;
        }

        async function pauseJob(jobId) {
            try {
                const res = await fetch('/api/crawl/jobs/' + jobId + '/pause', { method: 'POST' });
                if (res.ok) {
                    loadJobs();
                } else {
                    const err = await res.json();
                    alert(T.error + ': ' + (err.detail || T.load_error));
                }
            } catch (err) {
                alert(T.error + ': ' + err.message);
            }
        }

        async function resumeJob(jobId) {
            try {
                const res = await fetch('/api/crawl/jobs/' + jobId + '/resume', { method: 'POST' });
                if (res.ok) {
                    loadJobs();
                } else {
                    const err = await res.json();
                    alert(T.error + ': ' + (err.detail || T.load_error));
                }
            } catch (err) {
                alert(T.error + ': ' + err.message);
            }
        }

        // Load job settings into form for recrawling (fresh always unchecked)
        function loadJobSettings(job) {
            document.getElementById('target').value = job.target_uri || '';
            document.getElementById('keywords').value = (JSON.parse(job.keywords || '[]')).join(', ');
            document.getElementById('mode').value = job.mode || 'all';
            document.getElementById('depth').value = job.depth || 15;
            document.getElementById('max_pages').value = job.max_pages || -1;
            document.getElementById('fast').checked = !!job.fast;
            document.getElementById('screenshots').checked = job.screenshots !== 0;
            // NEVER load fresh as true when recrawling - we want to resume, not start over
            document.getElementById('fresh').checked = false;
            // Scroll to form
            document.getElementById('target').scrollIntoView({ behavior: 'smooth' });
            document.getElementById('target').focus();
        }

        async function deleteJob(jobId) {
            if (!confirm(T.delete + '?')) return;
            try {
                const res = await fetch('/api/crawl/jobs/' + jobId, { method: 'DELETE' });
                if (res.ok) {
                    loadJobs();
                } else {
                    const err = await res.json();
                    alert(T.error + ': ' + (err.detail || T.load_error));
                }
            } catch (err) {
                alert(T.error + ': ' + err.message);
            }
        }

        function formatBytes(bytes) {
            if (!bytes) return '';
            if (bytes < 1024) return bytes + ' B';
            if (bytes < 1024*1024) return (bytes/1024).toFixed(1) + ' KB';
            if (bytes < 1024*1024*1024) return (bytes/1024/1024).toFixed(1) + ' MB';
            return (bytes/1024/1024/1024).toFixed(1) + ' GB';
        }

        function formatDuration(startedAt, completedAt) {
            if (!startedAt || !completedAt) return '';
            const start = new Date(startedAt);
            const end = new Date(completedAt);
            const sec = Math.round((end - start) / 1000);
            if (sec < 60) return sec + 's';
            if (sec < 3600) return Math.floor(sec / 60) + 'm ' + (sec % 60) + 's';
            const hr = Math.floor(sec / 3600);
            const min = Math.floor((sec % 3600) / 60);
            return hr + 'h ' + min + 'm';
        }

        async function loadJobs() {
            try {
                const res = await fetch('/api/crawl/jobs');
                const jobs = await res.json();

                const container = document.getElementById('jobs');

                if (jobs.length === 0) {
                    container.innerHTML = '<div class="no-jobs">' + T.no_jobs + '</div>';
                    return;
                }

                const hasRunning = jobs.some(j => j.status === 'running');

                container.innerHTML = jobs.map(job => {
                    let stats = {};
                    let keywords = [];
                    try {
                        stats = job.stats ? JSON.parse(job.stats) : {};
                    } catch (e) {
                        console.warn('Failed to parse job stats:', job.id, e);
                    }
                    try {
                        keywords = job.keywords ? JSON.parse(job.keywords) : [];
                    } catch (e) {
                        console.warn('Failed to parse job keywords:', job.id, e);
                    }
                    const isBackfill = job.job_kind === 'backfill';
                    const kindLabel = isBackfill ? `🔄 ${job.job_type}` : `🕷️ ${job.mode || 'all'}`;

                    // Backfill job progress
                    if (isBackfill) {
                        const progress = job.total_records > 0
                            ? Math.round((job.processed_records / job.total_records) * 100)
                            : 0;
                        return `
                            <div class="job ${job.status}">
                                <div class="job-header">
                                    <span class="job-id">${kindLabel} #${job.id}</span>
                                    <span class="job-status ${job.status}">${T[job.status] || job.status}</span>
                                </div>
                                <div class="job-target">${job.domain_filter || T.all_domains}</div>
                                <div class="job-meta">
                                    ${job.processed_records}/${job.total_records} |
                                    ${job.error_count} ${T.error} |
                                    ${T.started}: ${new Date(job.started_at).toLocaleString()}
                                </div>
                                ${job.status === 'running' ? `
                                    <div class="progress-bar">
                                        <div class="progress-bar-fill" style="width: ${progress}%"></div>
                                    </div>
                                ` : ''}
                            </div>
                        `;
                    }

                    // Crawl job display with ETA
                    const calcEta = () => {
                        const crawled = stats.pages_crawled || 0;
                        const pending = stats.pages_pending || 0;
                        const started = stats.crawl_started || 0;
                        if (crawled === 0 || started === 0 || pending === 0) return '';
                        const elapsed = Date.now() / 1000 - started;
                        if (elapsed < 5) return '';  // Wait for meaningful data
                        const rate = crawled / elapsed;
                        const etaSec = pending / rate;
                        if (etaSec < 60) return `~${Math.round(etaSec)}s`;
                        if (etaSec < 3600) return `~${Math.round(etaSec / 60)}m`;
                        return `~${(etaSec / 3600).toFixed(1)}h`;
                    };
                    const eta = calcEta();
                    const crawlProgress = (() => {
                        const crawled = stats.pages_crawled || 0;
                        const pending = stats.pages_pending || 0;
                        const total = crawled + pending;
                        return total > 0 ? Math.round((crawled / total) * 100) : 0;
                    })();
                    const liveStats = job.status === 'running' ? `
                        <div class="job-stats live">
                            <span>📄 ${stats.pages_crawled || 0}${stats.pages_pending ? '/' + ((stats.pages_crawled || 0) + stats.pages_pending) : ''} pages${eta ? ' ⏱️' + eta : ''}</span>
                            <span>🖼️ ${stats.media_found || 0} found</span>
                            <span>💾 ${stats.media_downloaded || 0} saved</span>
                            <span>📸 ${stats.screenshots_taken || 0} screenshots</span>
                            <span>♻️ ${(stats.duplicates_skipped || 0) + (stats.content_exists || 0)} dupes</span>
                            ${stats.bytes_stored ? `<span>📦 ${formatBytes(stats.bytes_stored)}</span>` : ''}
                        </div>
                    ` : '';

                    return `
                        <div class="job ${job.status}">
                            <div class="job-header">
                                <span class="job-id">${kindLabel} #${job.id}</span>
                                <span class="job-status ${job.status}">${T[job.status] || job.status}</span>
                            </div>
                            <div class="job-target">
                                <a href="${job.target_uri}" target="_blank">${job.target_uri}</a>
                                ${(() => { try { const h = new URL(job.target_uri).hostname; return `<span class="job-links"><a href="/?q=${encodeURIComponent(h)}">${T.search}</a> <a href="/live?domain=${encodeURIComponent(h)}">${T.live}</a></span>`; } catch(e) { return ''; } })()}
                            </div>
                            <div class="job-meta">
                                ${keywords.length ? `${T.filter}: ${keywords.join(', ')} | ` : ''}${T.started}: ${new Date(job.started_at).toLocaleString()}
                            </div>
                            ${job.status === 'completed' ? `
                                <div class="job-stats">
                                    <span>⏱️ ${formatDuration(job.started_at, job.completed_at)}</span>
                                    <span>📄 ${stats.pages_crawled || 0} pages</span>
                                    <span>🖼️ ${stats.media_found || 0} found</span>
                                    <span>💾 ${stats.media_downloaded || 0} saved</span>
                                    <span>📸 ${stats.screenshots_taken || 0} screenshots</span>
                                    <span>♻️ ${(stats.duplicates_skipped || 0) + (stats.content_exists || 0)} dupes</span>
                                    ${stats.bytes_stored ? `<span>📦 ${formatBytes(stats.bytes_stored)}</span>` : ''}
                                </div>
                            ` : ''}
                            ${liveStats}
                            ${job.status === 'running' && job.job_kind === 'crawl' ? `
                                <div class="progress-bar">
                                    <div class="progress-bar-fill" style="width: ${crawlProgress}%"></div>
                                </div>
                            ` : ''}
                            <div class="job-actions">
                                ${job.status === 'running' && job.job_kind === 'crawl' ? `<button class="pause-btn" onclick="pauseJob(${job.id})">${T.pause}</button>` : ''}
                                ${(job.status === 'paused' || job.status === 'cancelled') && job.job_kind === 'crawl' ? `<button class="start-btn" onclick="resumeJob(${job.id})">${T.resume}</button>` : ''}
                                ${(job.status === 'completed' || job.status === 'failed' || job.status === 'cancelled') && job.job_kind === 'crawl' ? `<button class="recrawl-btn" onclick='loadJobSettings(${JSON.stringify(job)})'>↻</button>` : ''}
                                <button class="console-btn" onclick="openConsole(${job.id})">{{console}}</button>
                                ${job.status !== 'running' ? `<button class="delete-btn" onclick="deleteJob(${job.id})">${T.delete}</button>` : ''}
                            </div>
                        </div>
                    `;
                }).join('');

                // Poll: 3s when running, 10s when idle
                const delay = hasRunning ? 3000 : 10000;
                setTimeout(loadJobs, delay);
            } catch (err) {
                console.error('loadJobs error:', err);
                document.getElementById('jobs').innerHTML = `<div class="no-jobs">${T.load_jobs_error}: ` + err.message + '</div>';
                setTimeout(loadJobs, 10000);
            }
        }

        // Load jobs on page load
        loadJobs();

        // Console modal
        let consoleJobId = null;
        let consoleInterval = null;

        function openConsole(jobId) {
            consoleJobId = jobId;
            document.getElementById('console-modal').classList.add('active');
            document.getElementById('console-title').textContent = T.console + ' #' + jobId;
            document.getElementById('console-logs').textContent = T.loading;
            fetchLogs();
            consoleInterval = setInterval(fetchLogs, 2000);
        }

        function closeConsole() {
            document.getElementById('console-modal').classList.remove('active');
            if (consoleInterval) {
                clearInterval(consoleInterval);
                consoleInterval = null;
            }
            consoleJobId = null;
        }

        async function fetchLogs() {
            if (!consoleJobId) return;
            try {
                const res = await fetch('/api/crawl/jobs/' + consoleJobId + '/logs');
                const logs = await res.text();
                const el = document.getElementById('console-logs');
                const wasAtBottom = el.scrollHeight - el.clientHeight <= el.scrollTop + 50;
                el.innerHTML = logs ? logs.split('\\n').map(line => {
                    let cls = 'log-line';
                    if (line.includes(' INFO:')) cls += ' log-INFO';
                    else if (line.includes(' WARNING:')) cls += ' log-WARNING';
                    else if (line.includes(' ERROR:')) cls += ' log-ERROR';
                    return '<div class="' + cls + '">' + line.replace(/</g, '&lt;') + '</div>';
                }).join('') : T.no_logs;
                if (wasAtBottom) el.scrollTop = el.scrollHeight;
            } catch (err) {
                document.getElementById('console-logs').textContent = T.error + ': ' + err.message;
            }
        }

        // Close on escape
        document.addEventListener('keydown', e => {
            if (e.key === 'Escape') closeConsole();
        });
    </script>

    <div id="console-modal" class="console-modal" onclick="if(event.target===this)closeConsole()">
        <div class="console-header">
            <h3 id="console-title">{{console}}</h3>
            <button class="console-close" onclick="closeConsole()">✕ {{close}}</button>
        </div>
        <div id="console-logs" class="console-body"></div>
    </div>
    </div>
</body>
</html>
"""


LIVE_HTML = """
<!DOCTYPE html>
<html>
<head>
    <title>neopig {{live}} - {{live_subtitle}}</title>
    <style>
        * { box-sizing: border-box; }
        body {
            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
            margin: 0;
            padding: 0;
            background: #0a0a0a;
            color: #e0e0e0;
        }
        .nav {
            background: #1a1a1a;
            padding: 10px 20px;
            display: flex;
            gap: 20px;
            align-items: center;
            border-bottom: 1px solid #333;
        }
        .nav a { color: #ff6b6b; text-decoration: none; }
        .nav a:hover { text-decoration: underline; }
        .nav .brand { font-weight: bold; font-size: 18px; }
        .container { padding: 20px; }
        h1 { color: #ff6b6b; margin-bottom: 5px; }
        .subtitle { color: #666; margin-bottom: 10px; }
        .stats {
            background: #1a1a1a;
            padding: 15px;
            border-radius: 8px;
            margin-bottom: 20px;
            display: flex;
            gap: 30px;
            flex-wrap: wrap;
        }
        .stat { display: flex; flex-direction: column; }
        .stat-value { font-size: 24px; font-weight: bold; color: #ff6b6b; }
        .stat-label { font-size: 12px; color: #888; }
        .controls {
            margin-bottom: 20px;
            display: flex;
            gap: 10px;
            align-items: center;
        }
        .controls button {
            padding: 8px 16px;
            border: none;
            border-radius: 4px;
            cursor: pointer;
            font-size: 14px;
        }
        .controls button.primary { background: #ff6b6b; color: white; }
        .controls button.secondary { background: #333; color: #ddd; }
        .controls button:hover { opacity: 0.8; }
        .status { color: #4ade80; font-size: 14px; }
        .status.paused { color: #fbbf24; }
        .live-grid {
            display: grid;
            grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
            gap: 15px;
        }
        .live-card {
            background: #1a1a1a;
            border-radius: 8px;
            overflow: hidden;
            animation: fadeIn 0.5s ease-out;
        }
        @keyframes fadeIn {
            from { opacity: 0; transform: translateY(20px); }
            to { opacity: 1; transform: translateY(0); }
        }
        .live-card.new {
            box-shadow: 0 0 20px rgba(255, 107, 107, 0.5);
        }
        .live-card img, .live-card video {
            width: 100%;
            height: 180px;
            object-fit: contain;
            background: #222;
        }
        .live-card-info {
            padding: 10px;
        }
        .live-card-title {
            font-size: 12px;
            color: #888;
            overflow: hidden;
            text-overflow: ellipsis;
            white-space: nowrap;
        }
        .live-card-source {
            font-size: 10px;
            color: #555;
            margin-top: 4px;
            overflow: hidden;
            text-overflow: ellipsis;
            white-space: nowrap;
        }
        .score-badge {
            display: inline-block;
            padding: 2px 6px;
            border-radius: 3px;
            font-size: 9px;
            font-weight: bold;
            text-transform: uppercase;
            margin-left: 6px;
            vertical-align: middle;
        }
        .score-screenshot { background: #666; color: #ccc; }
        .score-og { background: #8b5cf6; color: white; }
        .score-thumb { background: #3b82f6; color: white; }
        .score-hd { background: #10b981; color: white; }
        .live-card.upgraded {
            box-shadow: 0 0 20px rgba(16, 185, 129, 0.7);
            animation: upgradeGlow 1s ease-out;
        }
        @keyframes upgradeGlow {
            0% { box-shadow: 0 0 30px rgba(16, 185, 129, 1); }
            100% { box-shadow: 0 0 20px rgba(16, 185, 129, 0.5); }
        }
    </style>
</head>
<body>
    <!-- NAV -->
    <div class="container">

    <form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
        <input type="text" name="q" placeholder="{{search_placeholder}}" style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
        <select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
            <option value="">{{all_types}}</option>
            <option value="image">{{images}}</option>
            <option value="video">{{videos}}</option>
            <option value="audio">{{audio}}</option>
        </select>
        <button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">{{search}}</button>
    </form>

    <h1>{{live}}</h1>
    <p class="subtitle">{{live_subtitle}}</p>

    <div class="stats">
        <div class="stat">
            <span class="stat-value" id="total-count">0</span>
            <span class="stat-label">{{total_images}}</span>
        </div>
        <div class="stat">
            <span class="stat-value" id="new-count">0</span>
            <span class="stat-label">{{new_session}}</span>
        </div>
        <div class="stat">
            <span class="stat-value" id="rate">0</span>
            <span class="stat-label">{{per_minute}}</span>
        </div>
    </div>

    <div class="controls">
        <button class="primary" id="toggle-btn" onclick="toggleFeed()">{{pause}}</button>
        <button class="secondary" onclick="clearFeed()">{{delete}}</button>
        <span class="status" id="status">{{watching}}</span>
    </div>

    <div class="live-grid" id="grid"></div>

    <script>
        let running = true;
        let lastCheck = new Date().toISOString();
        let seenHashes = new Map();  // hash -> {upgraded_at, score}
        let newCount = 0;
        let startTime = Date.now();
        let eventSource = null;

        // Connect to SSE stream for real-time updates
        function connectSSE() {
            if (eventSource) eventSource.close();
            eventSource = new EventSource('/api/live/stream');

            eventSource.onmessage = function(event) {
                if (!running) return;
                try {
                    const item = JSON.parse(event.data);
                    addMediaCard(item, true);
                } catch (e) {
                    console.error('SSE parse error:', e);
                }
            };

            eventSource.onerror = function(e) {
                console.log('SSE reconnecting...');
                setTimeout(connectSSE, 3000);
            };
        }

        function addMediaCard(item, isNew) {
            if (seenHashes.has(item.md5_hash)) return;  // Already shown

            seenHashes.set(item.md5_hash, {upgraded_at: item.upgraded_at, score: item.score || 5});
            newCount++;

            const grid = document.getElementById('grid');
            const card = document.createElement('div');
            card.className = 'live-card' + (isNew ? ' new' : '');

            const isVideo = item.media_type === 'video';
            const isCode = item.media_type === 'code';
            const isStyle = item.media_type === 'style';
            const isFont = item.media_type === 'font';

            const mimeToExt = {
                'text/javascript': '.js', 'application/javascript': '.js',
                'text/css': '.css', 'text/xml': '.xml', 'application/json': '.json',
                'application/rss+xml': '.rss', 'application/atom+xml': '.atom',
                'font/woff': '.woff', 'font/woff2': '.woff2', 'font/ttf': '.ttf',
            };
            const getExt = (mime, alt) => mimeToExt[mime] || (alt ? '.' + alt : '');
            const placeholder = (ext, icon, color) => `<svg viewBox="0 0 100 100" style="width:100%;height:100%;background:#1a1a1a;border-radius:4px;"><text x="50" y="40" text-anchor="middle" fill="${color}" font-size="24">${icon}</text><text x="50" y="65" text-anchor="middle" fill="#888" font-family="monospace" font-size="14" font-weight="bold">${ext}</text></svg>`;

            let mediaEl;
            if (isVideo) {
                mediaEl = `<video src="/media/${item.md5_hash}" muted loop onmouseenter="this.play()" onmouseleave="this.pause()"></video>`;
            } else if (isCode) {
                mediaEl = placeholder(getExt(item.mime_type, item.alt_text), '{ }', '#6af');
            } else if (isStyle) {
                mediaEl = placeholder('.css', '#', '#f6a');
            } else if (isFont) {
                mediaEl = placeholder(getExt(item.mime_type, ''), 'Aa', '#af6');
            } else {
                mediaEl = `<img src="/media/${item.md5_hash}" loading="lazy" onerror="this.src='data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 width=%22200%22 height=%22180%22><rect fill=%22%23333%22 width=%22200%22 height=%22180%22/><text x=%2250%%22 y=%2250%%22 fill=%22%23666%22 text-anchor=%22middle%22>Error</text></svg>'">`;
            }

            const scoreBadge = getScoreBadge(item.score, item.media_type);

            // Extract filename from media_uri
            const getFilename = (uri) => {
                if (!uri) return '';
                try {
                    const path = new URL(uri).pathname;
                    const name = decodeURIComponent(path.split('/').pop() || '');
                    return name.length > 35 ? name.slice(0, 32) + '...' : name;
                } catch { return ''; }
            };
            const filename = getFilename(item.media_uri);

            card.innerHTML = `
                <a href="/view/${item.md5_hash}">
                    ${mediaEl}
                </a>
                <div class="live-card-info">
                    <div class="live-card-title">${filename || item.alt_text || item.title || item.md5_hash.slice(0,12)}</div>
                    <div class="live-card-source">${item.media_type} - ${formatSize(item.file_size)}${scoreBadge}</div>
                </div>
            `;

            grid.insertBefore(card, grid.firstChild);
            updateStats(seenHashes.size);

            // Remove 'new' highlight after animation
            if (isNew) setTimeout(() => card.classList.remove('new'), 2000);
        }

        function toggleFeed() {
            running = !running;
            const btn = document.getElementById('toggle-btn');
            const status = document.getElementById('status');
            if (running) {
                btn.textContent = T.pause;
                status.textContent = T.watching;
                status.className = 'status';
                connectSSE();
                poll();
            } else {
                btn.textContent = T.resume;
                status.textContent = T.paused;
                status.className = 'status paused';
                if (eventSource) eventSource.close();
            }
        }

        function clearFeed() {
            document.getElementById('grid').innerHTML = '';
            seenHashes = new Map();
            newCount = 0;
            startTime = Date.now();
            updateStats(0);
        }

        function updateStats(total) {
            document.getElementById('total-count').textContent = total;
            document.getElementById('new-count').textContent = newCount;

            const minutes = (Date.now() - startTime) / 60000;
            const rate = minutes > 0 ? Math.round(newCount / minutes) : 0;
            document.getElementById('rate').textContent = rate;
        }

        async function poll() {
            if (!running) return;

            try {
                // Get recent media sorted by first_seen_at descending (initial load + catch-up)
                const res = await fetch('/api/search?q=&limit=50');
                const media = await res.json();

                // Get stats
                const statsRes = await fetch('/api/stats');
                const stats = await statsRes.json();

                // Find new items not yet shown via SSE
                const isFirstPoll = seenHashes.size === 0;
                const itemsToShow = media.filter(m => !seenHashes.has(m.md5_hash));

                // Add items (not as "new" on first poll to avoid flash)
                itemsToShow.reverse().forEach(item => {
                    addMediaCard(item, !isFirstPoll);
                });

                updateStats(stats.total_media || 0);

            } catch (err) {
                console.error('Poll error:', err);
            }

            // Poll less frequently since SSE handles real-time
            setTimeout(poll, 10000);
        }

        function formatSize(bytes) {
            if (!bytes) return '?';
            if (bytes < 1024) return bytes + ' B';
            if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB';
            return (bytes / (1024 * 1024)).toFixed(1) + ' MB';
        }

        function getScoreBadge(score, mediaType) {
            // Score constants: screenshot=1, og=3, thumb=5, hd=10
            // Also check media_type for legacy records without scores
            let label, cssClass;
            if (mediaType === 'screenshot' || score <= 1) {
                label = '📸'; cssClass = 'score-screenshot';
            } else if (score <= 3) {
                label = 'OG'; cssClass = 'score-og';
            } else if (!score || score <= 5) {
                label = 'THUMB'; cssClass = 'score-thumb';
            } else {
                label = 'HD'; cssClass = 'score-hd';
            }
            return `<span class="score-badge ${cssClass}">${label}</span>`;
        }

        // Start polling and SSE connection
        poll();
        connectSSE();
    </script>
    </div>
</body>
</html>
"""


@app.get("/live", response_class=HTMLResponse)
async def live_page(domain: str = Query(None), lang: str = Cookie(None), accept_language: str = Header(None)):
    """Live feed page - watch images appear as they're crawled."""
    language = get_lang(lang, accept_language)
    # Inject domain filter into HTML
    html = LIVE_HTML
    if domain:
        # Add domain to the poll URL
        html = html.replace(
            "/api/search?q=&limit=50",
            f"/api/search?q={domain}&limit=50"
        )
        html = html.replace(
            "<h1>{{live}}</h1>",
            "<h1>{{live}}: " + domain + "</h1>"
        )
    return inject_i18n(html, language)


@app.get("/view/{md5_hash}", response_class=HTMLResponse)
async def view_media_page(md5_hash: str, noai: bool = Query(False), lang: str = Cookie(None), accept_language: str = Header(None)):
    """Detail view page for a single media item."""
    language = get_lang(lang, accept_language)
    import html as html_module
    import re
    from urllib.parse import quote, urljoin

    async with db.session() as session:
        result = await session.execute(
            text("SELECT * FROM media WHERE md5_hash = :hash"),
            {'hash': md5_hash}
        )
        media = result.fetchone()
        if not media:
            raise HTTPException(status_code=404, detail="Media not found")

        result = await session.execute(
            text("""SELECT media_uri, page_uri, page_title, page_content,
                      detail_page_uri, detail_title, detail_content, discovered_at
               FROM media_sources WHERE md5_hash = :hash"""),
            {'hash': md5_hash}
        )
        sources = result.fetchall()

    media = dict(media._mapping)
    sources = [dict(s._mapping) for s in sources]
    keywords = json.loads(media.get('keywords') or '[]')
    sorted_sources = sorted(sources, key=lambda s: len(s["page_uri"] or ""), reverse=True)

    # Get page URI
    page_uri = sorted_sources[0]["page_uri"] if sorted_sources else None
    source_uri = page_uri or (sorted_sources[0]["media_uri"] if sorted_sources else "unknown")
    source_domain = Uri(source_uri).hostname if source_uri else "unknown"

    # Display title
    display_title = media.get('alt_text') or media.get('title')
    if not display_title and sources:
        display_title = sources[0].get('page_title')
    if not display_title:
        display_title = f"Media {md5_hash[:12]}"

    # Hero: media element
    is_video = media['media_type'] == 'video'
    is_audio = media['media_type'] == 'audio'
    is_code = media['media_type'] == 'code'
    is_style = media['media_type'] == 'style'
    is_font = media['media_type'] == 'font'
    is_text_file = is_code or is_style  # text files shown with syntax highlighting
    if is_video:
        hero_html = f'<a href="/media/{md5_hash}" target="_blank"><video src="/media/{md5_hash}" controls muted loop style="max-width:100%;max-height:70vh;"></video></a>'
    elif is_audio:
        hero_html = f'<audio src="/media/{md5_hash}" controls></audio>'
    elif is_text_file:
        # Fetch code/style content and display with syntax highlighting
        code_content = ""
        lang_class = 'css' if is_style else (media.get('alt_text') or '')
        try:
            vault_path = VAULT_PATH / hash_to_path(md5_hash)
            # Find file with this hash
            subdir = vault_path.parent
            for f in subdir.iterdir():
                if f.stem == md5_hash:
                    code_content = f.read_text(errors='replace')[:100000]  # Limit size
                    break
        except Exception:
            code_content = "(Unable to load file content)"
        escaped_code = html_module.escape(code_content)
        hero_html = f'<pre style="max-height:70vh;overflow:auto;background:#1e1e1e;padding:15px;border-radius:8px;text-align:left;white-space:pre-wrap;word-wrap:break-word;"><code class="language-{lang_class}">{escaped_code}</code></pre>'
    elif is_font:
        # Font preview with sample text
        hero_html = f'''<div style="background:#1e1e1e;padding:30px;border-radius:8px;text-align:center;">
            <style>@font-face {{ font-family: "preview-{md5_hash[:8]}"; src: url("/media/{md5_hash}"); }}</style>
            <div style="font-family:'preview-{md5_hash[:8]}',sans-serif;font-size:48px;color:#fff;margin-bottom:20px;">Aa Bb Cc</div>
            <div style="font-family:'preview-{md5_hash[:8]}',sans-serif;font-size:24px;color:#ccc;">The quick brown fox jumps over the lazy dog</div>
            <div style="font-family:'preview-{md5_hash[:8]}',sans-serif;font-size:16px;color:#888;margin-top:15px;">0123456789 !@#$%^&*()</div>
        </div>'''
    else:
        hero_html = f'<a href="/media/{md5_hash}" target="_blank"><img src="/media/{md5_hash}" alt="{html_module.escape(media.get("alt_text") or "")}" style="max-width:100%;max-height:70vh;"></a>'

    # Metadata rows
    import hashlib
    media_uri = sorted_sources[0]["media_uri"] if sorted_sources else None
    neopig_media_uri = f"/view/{md5_hash}"
    page_uri_hash = hashlib.md5(page_uri.encode()).hexdigest() if page_uri else None
    neopig_page_uri = f"/page/{page_uri_hash}" if page_uri_hash else None
    keywords_html = ''.join([f'<span class="tag">{k}</span>' for k in keywords]) or '-'
    meta_rows = [
        ("{{source_uri}}", f'<a href="{media_uri}" target="_blank" style="font-size:11px;">{media_uri}</a>' if media_uri else '-'),
        ("{{neopig_uri}}", f'<a href="{neopig_media_uri}" style="font-size:11px;">{neopig_media_uri}</a>'),
        ("{{source_page}}", f'<a href="{page_uri}" target="_blank" style="font-size:11px;">{page_uri}</a>' if page_uri else '-'),
        ("{{neopig_page}}", f'<a href="{neopig_page_uri}" style="font-size:11px;">{neopig_page_uri}</a>' if neopig_page_uri else '-'),
        ("MD5", f'<code style="font-size:11px;">{md5_hash}</code>'),
        ("{{type_label}}", media['media_type']),
        ("{{mime_label}}", media.get('mime_type') or 'unknown'),
        ("{{size_label}}", f"{media.get('file_size') or 0:,} bytes"),
        ("{{alt_label}}", media.get('alt_text') or '-'),
        ("{{keywords_label}}", keywords_html),
    ]

    # Download button - prioritize original filename from URL
    download_filename = None
    if sources and sources[0].get('media_uri'):
        from urllib.parse import unquote
        parsed = Uri(sources[0]['media_uri'])
        orig_name = Path(unquote(parsed.path)).name
        if orig_name and '.' in orig_name:
            download_filename = orig_name
    # Fallback to alt_text/title
    if not download_filename:
        name_source = media.get('alt_text') or media.get('title')
        if name_source:
            ext_map = {'image/jpeg': '.jpg', 'image/png': '.png', 'image/gif': '.gif', 'image/webp': '.webp',
                       'video/mp4': '.mp4', 'video/webm': '.webm', 'audio/mpeg': '.mp3', 'audio/wav': '.wav',
                       'text/javascript': '.js', 'text/python': '.py', 'text/css': '.css', 'text/html': '.html',
                       'text/markdown': '.md', 'text/rust': '.rs', 'text/go': '.go', 'text/c': '.c', 'text/cpp': '.cpp'}
            ext = ext_map.get(media.get('mime_type', ''), '')
            if not ext:
                try:
                    subdir = VAULT_PATH / hash_to_path(md5_hash).parent
                    for f in subdir.iterdir():
                        if f.stem == md5_hash:
                            ext = f.suffix
                            break
                except Exception:
                    pass
            download_filename = f"{slugify(name_source)}{ext}"
    # Fallback to page_title + hash
    if not download_filename and sources:
        pt = sources[0].get('page_title', '')
        media_idx = int(md5_hash[:4], 16)
        name_source = f"{pt}-{media_idx}" if pt else f"media-{media_idx}"
        download_filename = f"{slugify(name_source)}.bin"
    # Default
    if not download_filename:
        download_filename = f"{md5_hash[:12]}.bin"
    download_btn = f'<a href="/media/{md5_hash}?download=1" style="display:block;padding:12px 20px;background:#6bff6b;color:#000;text-decoration:none;border-radius:6px;text-align:center;font-weight:500;margin-top:15px;">Download ({download_filename})</a>'

    # Get page media items (siblings)
    media_items = await db.get_page_media(page_uri) if page_uri else []
    # Exclude current item from gallery
    media_items = [m for m in media_items if m.get("md5_hash") != md5_hash]

    # Get rendered content
    content_html = ""
    if page_uri:
        page_row = await db.get_page_by_uri(page_uri)
        if page_row and page_row.get("markdown"):
            try:
                import markdown
                md_converter = markdown.Markdown(extensions=['fenced_code', 'tables', 'nl2br'])
                content_html = md_converter.convert(page_row["markdown"][:100000])
                # Hydrate images and links
                img_urls = re.findall(r'<img[^>]+src=["\']([^"\']+)["\']', content_html, re.I)
                link_urls = re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', content_html, re.I)
                all_urls = list(set(img_urls + link_urls))
                if all_urls:
                    resolved = {u: u if u.startswith(('http://', 'https://', '//')) else urljoin(page_uri, u) for u in all_urls if not u.startswith('#')}
                    url_to_hash = await db.lookup_media_by_uris(list(set(resolved.values())))
                    exact_matches = set(url_to_hash.keys())  # These are reliable

                    # Fallback for imgur and other CDNs - lookup by filename/ID
                    # Only apply to URLs that look like media files (have media extension)
                    from async_web_fetcher import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS, AUDIO_EXTENSIONS
                    from pathlib import Path as P
                    media_exts = IMAGE_EXTENSIONS | VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
                    for orig, res in resolved.items():
                        if res not in url_to_hash:
                            url_path = res.split('?')[0]
                            ext = P(url_path).suffix.lower()
                            if ext in media_exts:
                                fname = P(url_path).stem
                                if fname and len(fname) >= 5:
                                    result = await db.lookup_media_by_filename(fname)
                                    if result:
                                        url_to_hash[res] = result[1]

                    for orig, res in resolved.items():
                        if res in url_to_hash:
                            md5 = url_to_hash[res]
                            content_html = content_html.replace(f'src="{orig}"', f'src="/media/{md5}"')
                            content_html = content_html.replace(f"src='{orig}'", f'src="/media/{md5}"')
                            # Only rewrite hrefs for exact matches or URLs with media extensions
                            url_path = res.split('?')[0]
                            ext = P(url_path).suffix.lower()
                            if res in exact_matches or ext in media_exts:
                                content_html = content_html.replace(f'href="{orig}"', f'href="/view/{md5}"')
                                content_html = content_html.replace(f"href='{orig}'", f'href="/view/{md5}"')
            except ImportError:
                content_html = f"<pre>{html_module.escape(page_row['markdown'][:50000])}</pre>"

    # Get screenshots (exclude current if it's a screenshot)
    screenshot_hashes = await db.get_page_screenshots(page_uri) if page_uri else []
    screenshot_hashes = [h for h in screenshot_hashes if h != md5_hash]

    # Build "Used on X pages" section (reverse image search)
    sources_html = ""
    if sources and len(sources) > 0:
        import hashlib as hl
        unique_pages = {}
        for s in sources:
            pu = s.get('page_uri', '')
            if pu and pu not in unique_pages:
                unique_pages[pu] = s
        if unique_pages:
            rows_html = ""
            for pu, s in unique_pages.items():
                pt = html_module.escape(s.get('page_title', '') or pu[:60])
                ph = hl.md5(pu.encode()).hexdigest()
                rows_html += f'''<tr class="source-row">
                    <td><a href="/page/{ph}" title="{html_module.escape(pu)}">{pt}</a></td>
                    <td style="font-size:11px;color:#888;">{html_module.escape(s.get('discovered_at', '')[:10] if s.get('discovered_at') else '-')}</td>
                </tr>'''
            total_count = len(unique_pages)
            count_display = f"{OVER_9000:,}" if total_count >= OVER_9000 else str(total_count)
            sources_html = '''
            <div class="sources-section">
                <h3 onclick="toggleSources()">
                    <span id="sources-toggle">▶</span> {{used_on}} ''' + count_display + ''' {{pages}}
                </h3>
                <div id="sources-list" class="sources-list">
                    <div id="sources-pagination" style="margin-bottom:10px;display:flex;gap:10px;align-items:center;">
                        <button onclick="sourcesPage(-1)" id="sources-prev" style="padding:5px 12px;background:#333;border:none;color:#fff;border-radius:4px;cursor:pointer;">←</button>
                        <span id="sources-page-info" style="color:#888;font-size:13px;">1 / 1</span>
                        <button onclick="sourcesPage(1)" id="sources-next" style="padding:5px 12px;background:#333;border:none;color:#fff;border-radius:4px;cursor:pointer;">→</button>
                    </div>
                    <table>
                        <thead><tr><th>{{page}}</th><th>{{discovered}}</th></tr></thead>
                        <tbody id="sources-tbody">''' + rows_html + '''</tbody>
                    </table>
                </div>
            </div>
            <script>
            (function() {
                const perPage = 9000;
                const rows = document.querySelectorAll('#sources-tbody .source-row');
                const total = rows.length;
                const totalPages = Math.ceil(total / perPage);
                let currentPage = 1;
                function showPage(page) {
                    currentPage = Math.max(1, Math.min(page, totalPages));
                    const start = (currentPage - 1) * perPage;
                    const end = start + perPage;
                    rows.forEach((row, i) => { row.style.display = (i >= start && i < end) ? '' : 'none'; });
                    document.getElementById('sources-page-info').textContent = currentPage + ' / ' + totalPages;
                    document.getElementById('sources-prev').disabled = currentPage === 1;
                    document.getElementById('sources-next').disabled = currentPage === totalPages;
                }
                window.sourcesPage = function(delta) { showPage(currentPage + delta); };
                showPage(1);

                // Toggle sources with localStorage persistence
                window.toggleSources = function() {
                    const list = document.getElementById('sources-list');
                    const toggle = document.getElementById('sources-toggle');
                    const expanded = !list.classList.contains('expanded');
                    list.classList.toggle('expanded', expanded);
                    toggle.textContent = expanded ? '▼' : '▶';
                    localStorage.setItem('neopig_sources_expanded', expanded);
                };

                // Restore from localStorage
                if (localStorage.getItem('neopig_sources_expanded') === 'true') {
                    document.getElementById('sources-list').classList.add('expanded');
                    document.getElementById('sources-toggle').textContent = '▼';
                }
            })();
            </script>'''

    return render_detail_page(
        title=display_title,
        hero_html=hero_html,
        meta_rows=meta_rows,
        media_items=media_items,
        content_html=content_html,
        screenshot_hashes=screenshot_hashes,
        source_domain=source_domain,
        page_uri=page_uri or "",
        download_btn_html=download_btn,
        noai=noai,
        lang=language,
        sources_html=sources_html,
    )




@app.get("/page/{uri_hash}", response_class=HTMLResponse)
async def view_page_by_hash(
    uri_hash: str,
    noai: bool = Query(False, description="Disable AI assistant"),
    lang: str = Cookie(None),
    accept_language: str = Header(None),
):
    """View an archived page by URI hash."""
    page = await db.get_page_by_hash(uri_hash)
    if not page:
        raise HTTPException(status_code=404, detail="Page not found")
    # Redirect to the URI-based view (reuses same logic)
    return await view_page(uri=page['uri'], noai=noai, lang=lang, accept_language=accept_language)


@app.get("/page/view", response_class=HTMLResponse)
async def view_page(
    uri: str = Query(..., description="Page URI to view"),
    noai: bool = Query(False, description="Disable AI assistant"),
    lang: str = Cookie(None),
    accept_language: str = Header(None),
):
    """View an archived page with markdown and screenshot."""
    language = get_lang(lang, accept_language)
    import html as html_module
    import re
    from urllib.parse import urljoin, quote

    source_domain = Uri(uri).hostname

    page = await db.get_page_by_uri(uri)
    if not page:
        raise HTTPException(status_code=404, detail="Page not found")

    page_title = page.get("title") or uri

    # Render markdown
    content_html = ""
    if page.get("markdown"):
        try:
            import markdown
            md_converter = markdown.Markdown(extensions=['fenced_code', 'tables', 'nl2br'])
            content_html = md_converter.convert(page["markdown"][:100000])

            # Hydrate images and links from vault
            img_urls = re.findall(r'<img[^>]+src=["\']([^"\']+)["\']', content_html, re.I)
            link_urls = re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', content_html, re.I)
            all_urls = list(set(img_urls + link_urls))
            if all_urls:
                # Keep anchor-only links (#foo) as-is, resolve others
                resolved = {}
                for u in all_urls:
                    if u.startswith('#'):
                        continue  # Skip anchor-only links, they stay internal
                    elif u.startswith(('http://', 'https://', '//')):
                        resolved[u] = u
                    else:
                        resolved[u] = urljoin(uri, u)

                # Lookup media (images, etc) - track which came from exact match vs fallback
                url_to_hash = await db.lookup_media_by_uris(list(set(resolved.values())))
                exact_matches = set(url_to_hash.keys())  # These are reliable

                # Fallback for imgur and other CDNs - lookup by filename/ID
                # Only apply to URLs that look like media files (have media extension)
                from async_web_fetcher import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS, AUDIO_EXTENSIONS
                from pathlib import Path as P
                media_exts = IMAGE_EXTENSIONS | VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
                for murl in [u for u in resolved.values() if u not in url_to_hash]:
                    url_path = murl.split('?')[0]
                    ext = P(url_path).suffix.lower()
                    # Only do filename fallback for URLs with media extensions
                    if ext in media_exts:
                        fname = P(url_path).stem
                        if fname and len(fname) >= 5:
                            result = await db.lookup_media_by_filename(fname)
                            if result:
                                url_to_hash[murl] = result[1]

                # Lookup pages for internal link rewriting
                page_links = [res for res in resolved.values() if source_domain and source_domain in res]
                uri_to_page_hash = await db.lookup_pages_by_uris(page_links) if page_links else {}

                for orig, res in resolved.items():
                    if res in url_to_hash:
                        md5 = url_to_hash[res]
                        # Rewrite img src to serve media directly
                        content_html = content_html.replace(f'src="{orig}"', f'src="/media/{md5}"')
                        content_html = content_html.replace(f"src='{orig}'", f'src="/media/{md5}"')
                        # Only rewrite hrefs for exact matches or URLs with media extensions
                        # (avoid rewriting external links that happen to match by filename)
                        url_path = res.split('?')[0]
                        ext = P(url_path).suffix.lower()
                        if res in exact_matches or ext in media_exts:
                            content_html = content_html.replace(f'href="{orig}"', f'href="/view/{md5}"')
                            content_html = content_html.replace(f"href='{orig}'", f'href="/view/{md5}"')
                    elif res in uri_to_page_hash:
                        # Rewrite internal page links to archived versions
                        page_hash = uri_to_page_hash[res]
                        content_html = content_html.replace(f'href="{orig}"', f'href="/page/{page_hash}"')
                        content_html = content_html.replace(f"href='{orig}'", f'href="/page/{page_hash}"')
        except ImportError:
            content_html = f"<pre>{html_module.escape(page.get('markdown', '')[:50000])}</pre>"
    elif page.get("content"):
        escaped = html_module.escape(page["content"][:50000])
        content_html = f"<pre style='white-space:pre-wrap;'>{escaped}</pre>"

    # Get screenshots and media
    screenshot_hashes = await db.get_page_screenshots(uri)
    media_items = await db.get_page_media(uri)

    # Metadata rows for page view
    import hashlib
    uri_hash = hashlib.md5(uri.encode()).hexdigest()
    neopig_page_uri = f"/page/{uri_hash}"
    keywords = json.loads(page.get('keywords') or '[]') if page.get('keywords') else []
    keywords_html = ''.join([f'<span class="tag">{k}</span>' for k in keywords]) or '-'
    meta_rows = [
        ("{{source_uri}}", f'<a href="{uri}" target="_blank" style="font-size:11px;">{uri}</a>'),
        ("{{neopig_uri}}", f'<a href="{neopig_page_uri}" style="font-size:11px;">{neopig_page_uri}</a>'),
        ("{{description_label}}", page.get('description') or '-'),
        ("{{keywords_label}}", keywords_html),
        ("{{media}}", f"{len(media_items)}"),
    ]

    return render_detail_page(
        title=page_title,
        hero_html="",  # No hero media for page view
        meta_rows=meta_rows,
        media_items=media_items,
        content_html=content_html,
        screenshot_hashes=screenshot_hashes,
        source_domain=source_domain,
        page_uri=uri,
        noai=noai,
        lang=language,
    )


@app.get("/phantom/export")
async def phantom_export(domain: str = Query(None, description="Filter by domain")):
    """
    Export a phantom HTML site - original HTML with media URLs rewritten to vault.

    Creates a downloadable zip of the phantom site ready for static hosting.
    """
    import io
    import re
    import zipfile
    from urllib.parse import urljoin
    from fastapi.responses import StreamingResponse

    # Get all pages with raw_html
    pages = await db.get_pages_by_domain(domain)

    if not pages:
        raise HTTPException(status_code=404, detail="No pages with raw HTML found")

    # Get all media URL to hash mappings
    url_to_hash = await db.get_all_media_uri_mappings()

    # Create zip in memory
    zip_buffer = io.BytesIO()

    with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
        pages_written = 0
        media_hashes = set()

        for page in pages:
            uri = page['uri']
            raw_html = page.get('raw_html')
            if not raw_html:
                continue

            # Parse URI to get path
            parsed = Uri(uri)
            site_domain = parsed.hostname
            path = parsed.path.strip('/') or 'index'
            if not path.endswith('.html') and '.' not in path.split('/')[-1]:
                path = f"{path}/index.html" if path else "index.html"

            # Rewrite media URLs to local paths
            html = raw_html

            # Find all src and href attributes pointing to media
            patterns = [
                (r'src=["\']([^"\']+)["\']', 'src'),
                (r'href=["\']([^"\']+\.(jpg|jpeg|png|gif|webp|mp4|webm|svg|ico))["\']', 'href'),
            ]

            for pattern, attr in patterns:
                matches = re.findall(pattern, html, re.IGNORECASE)
                for match in matches:
                    url = match[0] if isinstance(match, tuple) else match

                    # Resolve relative URLs
                    full_url = urljoin(uri, url)

                    # Check if we have this media
                    if full_url in url_to_hash:
                        md5 = url_to_hash[full_url]
                        media_hashes.add(md5)
                        # Replace with local path
                        ext = Path(url).suffix or '.bin'
                        local_path = f"media/{md5}{ext}"
                        html = html.replace(f'{attr}="{url}"', f'{attr}="{local_path}"')
                        html = html.replace(f"{attr}='{url}'", f'{attr}="{local_path}"')
                    elif url in url_to_hash:
                        md5 = url_to_hash[url]
                        media_hashes.add(md5)
                        ext = Path(url).suffix or '.bin'
                        local_path = f"media/{md5}{ext}"
                        html = html.replace(f'{attr}="{url}"', f'{attr}="{local_path}"')
                        html = html.replace(f"{attr}='{url}'", f'{attr}="{local_path}"')

            # Write HTML file
            zf.writestr(f"site/{path}", html.encode('utf-8'))
            pages_written += 1

        # Copy media files from vault
        media_copied = 0
        for md5 in media_hashes:
            subdir = VAULT_PATH / md5[:2]
            if subdir.exists():
                for f in subdir.iterdir():
                    if f.name.startswith(md5):
                        ext = f.suffix or '.bin'
                        zf.write(f, f"site/media/{md5}{ext}")
                        media_copied += 1
                        break

        # Write index
        index_html = f"""<!DOCTYPE html>
<html>
<head>
    <title>Phantom Site - {site_domain}</title>
    <style>
        body {{ font-family: sans-serif; max-width: 800px; margin: 50px auto; padding: 20px; }}
        h1 {{ color: #333; }}
        ul {{ line-height: 2; }}
        a {{ color: #0066cc; }}
    </style>
</head>
<body>
    <h1>Phantom Site Archive</h1>
    <p>Domain: {site_domain}</p>
    <p>Pages: {pages_written}</p>
    <p>Media: {media_copied}</p>
    <h2>Pages</h2>
    <ul>
"""
        for page in pages[:100]:
            parsed = Uri(page['uri'])
            path = parsed.path.strip('/') or 'index'
            if not path.endswith('.html') and '.' not in path.split('/')[-1]:
                path = f"{path}/index.html" if path else "index.html"
            title = page.get('title') or path
            index_html += f'        <li><a href="{path}">{title}</a></li>\n'

        index_html += """    </ul>
</body>
</html>"""
        zf.writestr("site/phantom_index.html", index_html.encode('utf-8'))

    # Return zip
    zip_buffer.seek(0)
    return StreamingResponse(
        zip_buffer,
        media_type="application/zip",
        headers={"Content-Disposition": f"attachment; filename=phantom_{site_domain or 'site'}.zip"}
    )


@app.get("/phantom", response_class=HTMLResponse)
async def phantom_page(lang: str = Cookie(None), accept_language: str = Header(None)):
    """Phantom site export UI."""
    language = get_lang(lang, accept_language)
    domains = await db.get_domains_with_pages()

    domain_options = ''.join([
        f'<option value="{d[0]}">{d[0]} ({d[1]} pages)</option>'
        for d in domains if d[0]
    ])

    html = f"""
<!DOCTYPE html>
<html>
<head>
    <title>Phantom Site Export - neopig</title>
    <style>
        * {{ box-sizing: border-box; }}
        body {{
            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
            margin: 0;
            padding: 0;
            background: #0a0a0a;
            color: #e0e0e0;
        }}
        .nav {{
            background: #1a1a1a;
            padding: 10px 20px;
            display: flex;
            gap: 20px;
            align-items: center;
            border-bottom: 1px solid #333;
        }}
        .nav a {{ color: #ff6b6b; text-decoration: none; }}
        .nav a:hover {{ text-decoration: underline; }}
        .nav .brand {{ font-weight: bold; font-size: 18px; }}
        .container {{ padding: 20px; }}
        h1 {{ color: #ff6b6b; margin-bottom: 10px; }}
        .subtitle {{ color: #888; margin-bottom: 30px; }}
        .form-group {{ margin-bottom: 20px; }}
        label {{ display: block; margin-bottom: 8px; color: #aaa; }}
        select {{
            width: 100%;
            padding: 12px 16px;
            font-size: 16px;
            border: 2px solid #333;
            border-radius: 8px;
            background: #1a1a1a;
            color: #fff;
        }}
        button {{
            padding: 14px 28px;
            font-size: 16px;
            background: #ff6b6b;
            color: #fff;
            border: none;
            border-radius: 8px;
            cursor: pointer;
        }}
        button:hover {{ background: #ff5252; }}
        .info {{
            background: #1a1a1a;
            padding: 20px;
            border-radius: 8px;
            margin-top: 30px;
        }}
        .info h3 {{ color: #ff6b6b; margin-top: 0; }}
        .info ul {{ color: #aaa; line-height: 1.8; }}
    </style>
</head>
<body>
    <!-- NAV -->
    <div class="container">
        <form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
            <input type="text" name="q" placeholder="{{search_placeholder}}" style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
            <select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
                <option value="">{{all_types}}</option>
                <option value="image">{{images}}</option>
                <option value="video">{{videos}}</option>
                <option value="audio">{{audio}}</option>
            </select>
            <button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">{{search}}</button>
        </form>
        <h1>Phantom Site Export</h1>
        <p class="subtitle">{{phantom_subtitle}}</p>

        <form action="/phantom/export" method="get">
            <div class="form-group">
                <label>{{select_domain}}</label>
                <select name="domain">
                    <option value="">{{all_domains}}</option>
                    {domain_options}
                </select>
            </div>
            <button type="submit">{{download_phantom}} (.zip)</button>
        </form>

        <div class="info">
            <h3>What is a Phantom Site?</h3>
            <ul>
                <li>Original HTML preserved exactly as crawled</li>
                <li>All media URLs rewritten to local paths</li>
                <li>Ready to host statically (nginx, Caddy, S3, etc.)</li>
                <li>Works offline - all assets included</li>
                <li>Perfect for archival & preservation</li>
            </ul>
        </div>
    </div>
</body>
</html>
"""
    return inject_i18n(html, language)


ABOUT_CSS = """
.about-page { width: 100%; min-height: 100vh; background: #0a0a0f; }

/* Hero Section */
.hero-section {
    width: 100%;
    min-height: 100vh;
    display: flex;
    flex-direction: column;
    justify-content: center;
    align-items: center;
    background: linear-gradient(180deg, #0a0a0f 0%, #1a0a1a 50%, #0f0a1a 100%);
    position: relative;
    overflow: hidden;
    padding: 60px 20px;
}
.hero-section::before {
    content: '';
    position: absolute;
    top: 0; left: 0; right: 0; bottom: 0;
    background: radial-gradient(ellipse at 50% 0%, rgba(255,50,100,0.15) 0%, transparent 60%),
                radial-gradient(ellipse at 20% 80%, rgba(100,50,255,0.1) 0%, transparent 40%),
                radial-gradient(ellipse at 80% 60%, rgba(255,100,50,0.08) 0%, transparent 40%);
    pointer-events: none;
}
.hero-pig {
    width: 280px;
    height: auto;
    filter: drop-shadow(0 0 60px rgba(255,100,100,0.5)) drop-shadow(0 0 120px rgba(255,50,100,0.3));
    animation: pulse-glow 4s ease-in-out infinite;
    margin-bottom: 40px;
}
@keyframes pulse-glow {
    0%, 100% { filter: drop-shadow(0 0 60px rgba(255,100,100,0.5)) drop-shadow(0 0 120px rgba(255,50,100,0.3)); }
    50% { filter: drop-shadow(0 0 80px rgba(255,100,100,0.7)) drop-shadow(0 0 160px rgba(255,50,100,0.5)); }
}
.hero-title {
    font-size: 6em;
    font-weight: 900;
    letter-spacing: -0.02em;
    background: linear-gradient(135deg, #ff6b6b 0%, #ff8e53 50%, #ff6b9d 100%);
    -webkit-background-clip: text;
    -webkit-text-fill-color: transparent;
    background-clip: text;
    text-shadow: 0 0 80px rgba(255,100,100,0.5);
    margin: 0;
    text-align: center;
}
.hero-subtitle {
    font-size: 1.4em;
    color: #888;
    margin-top: 20px;
    font-weight: 300;
    letter-spacing: 0.3em;
    text-transform: uppercase;
}
.hero-tagline {
    font-size: 1.1em;
    color: #666;
    margin-top: 30px;
    max-width: 600px;
    text-align: center;
    line-height: 1.8;
}

/* Chapter Sections */
.chapter {
    width: 100%;
    padding: 120px 40px;
    position: relative;
}
.chapter-content {
    max-width: 900px;
    margin: 0 auto;
}
.chapter-number {
    font-size: 8em;
    font-weight: 900;
    color: rgba(255,100,100,0.08);
    position: absolute;
    top: 40px;
    left: 40px;
    line-height: 1;
    font-family: 'Georgia', serif;
}
.chapter-title {
    font-size: 3em;
    font-weight: 800;
    color: #ff6b6b;
    margin-bottom: 40px;
    position: relative;
}
.chapter-title::after {
    content: '';
    display: block;
    width: 100px;
    height: 4px;
    background: linear-gradient(90deg, #ff6b6b, transparent);
    margin-top: 20px;
}

/* Narrative Text */
.narrative {
    font-size: 1.3em;
    line-height: 2;
    color: #c0c0c0;
    margin-bottom: 30px;
}
.narrative em { color: #ff8e53; font-style: italic; }
.narrative strong { color: #fff; font-weight: 600; }
.narrative a { color: #ff6b6b; text-decoration: none; border-bottom: 1px solid rgba(255,107,107,0.3); transition: all 0.3s; }
.narrative a:hover { color: #ff8e53; border-bottom-color: #ff8e53; }

/* Dramatic Quote */
.dramatic-quote {
    padding: 60px;
    margin: 60px 0;
    background: linear-gradient(135deg, rgba(255,50,100,0.1) 0%, rgba(100,50,200,0.1) 100%);
    border-left: 6px solid #ff6b6b;
    position: relative;
}
.dramatic-quote::before {
    content: '"';
    font-size: 12em;
    color: rgba(255,100,100,0.15);
    position: absolute;
    top: -40px;
    left: 20px;
    font-family: 'Georgia', serif;
    line-height: 1;
}
.dramatic-quote p {
    font-size: 1.8em;
    font-style: italic;
    color: #e0e0e0;
    margin: 0;
    position: relative;
    z-index: 1;
}
.dramatic-quote cite {
    display: block;
    margin-top: 30px;
    font-size: 1em;
    color: #888;
    font-style: normal;
}

/* Dead Link Styling */
.dead-link {
    text-decoration: line-through;
    color: #666;
    cursor: not-allowed;
    position: relative;
}
.dead-link::after {
    content: ' [DEAD]';
    color: #ff4444;
    font-size: 0.7em;
    font-weight: bold;
    text-decoration: none;
}

/* Dark Chapter */
.chapter-dark {
    background: linear-gradient(180deg, #0a0a0f 0%, #150810 50%, #0a0a0f 100%);
}
.chapter-dark .chapter-number { color: rgba(255,50,50,0.1); }

/* Rebirth Chapter - TempleOS cyan */
.chapter-rebirth {
    background: linear-gradient(180deg, #0a0a0f 0%, #0a1515 50%, #0a0a0f 100%);
}
.chapter-rebirth .chapter-title { color: #00FFFF; /* TempleOS cyan */ }
.chapter-rebirth .chapter-title::after { background: linear-gradient(90deg, #00FFFF, transparent); }
.chapter-rebirth .chapter-number { color: rgba(0,255,255,0.08); }

/* Beast Section */
.chapter-beast {
    background: linear-gradient(180deg, #0a0a0f 0%, #1a0a05 50%, #0a0a0f 100%);
}
.chapter-beast .chapter-title { color: #ff8e53; }
.chapter-beast .chapter-title::after { background: linear-gradient(90deg, #ff8e53, transparent); }

/* Feature Cards - Full Width Grid */
.feature-section {
    width: 100%;
    padding: 100px 40px;
    background: linear-gradient(180deg, #0a0a0f 0%, #0f0f18 50%, #0a0a0f 100%);
    position: relative;
}
.feature-section .chapter-title::after {
    margin-left: auto;
    margin-right: auto;
}
.feature-grid {
    display: grid;
    grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
    gap: 30px;
    max-width: 1400px;
    margin: 0 auto;
}
.feature-card {
    background: linear-gradient(145deg, rgba(30,30,40,0.8) 0%, rgba(20,20,30,0.9) 100%);
    padding: 40px;
    border-radius: 16px;
    border: 1px solid rgba(255,100,100,0.1);
    transition: all 0.4s ease;
    position: relative;
    overflow: hidden;
}
.feature-card::before {
    content: '';
    position: absolute;
    top: 0; left: 0;
    width: 100%; height: 4px;
    background: linear-gradient(90deg, #ff6b6b, #ff8e53, #4ecdc4);
    opacity: 0;
    transition: opacity 0.4s;
}
.feature-card:hover {
    transform: translateY(-5px);
    border-color: rgba(255,100,100,0.3);
    box-shadow: 0 20px 60px rgba(0,0,0,0.4), 0 0 40px rgba(255,100,100,0.1);
}
.feature-card:hover::before { opacity: 1; }
.feature-card h3 {
    font-size: 1.4em;
    color: #ff6b6b;
    margin-bottom: 15px;
    font-weight: 700;
}
.feature-card p {
    color: #999;
    line-height: 1.7;
    font-size: 1.05em;
}
.feature-card a { color: #4ecdc4; }

/* Code Blocks */
.code-section {
    background: #0d0d12;
    padding: 40px;
    border-radius: 12px;
    margin: 40px 0;
    overflow-x: auto;
    border: 1px solid #222;
}
.code-section pre {
    margin: 0;
    font-family: 'Fira Code', 'Monaco', monospace;
    font-size: 2em;
    line-height: 1.8;
    color: #e0e0e0;
}
.code-section code { background: none; }

/* Inline Code */
.narrative code {
    background: rgba(255,100,100,0.1);
    padding: 3px 10px;
    border-radius: 4px;
    font-family: 'Fira Code', monospace;
    font-size: 0.9em;
    color: #ff8e53;
}

/* Mirror Section - Special */
.chapter-mirror {
    background: linear-gradient(180deg, #0a0a0f 0%, #0a0a1f 30%, #100a1a 70%, #0a0a0f 100%);
    position: relative;
}
.chapter-mirror::before {
    content: '';
    position: absolute;
    top: 0; left: 0; right: 0; bottom: 0;
    background: repeating-linear-gradient(
        90deg,
        transparent 0px,
        transparent 100px,
        rgba(100,100,255,0.02) 100px,
        rgba(100,100,255,0.02) 101px
    );
    pointer-events: none;
}

/* Transition into cyan (Why We Archive) */
.chapter-to-cyan {
    background: linear-gradient(180deg, #0a0a0f 0%, #0a0a0f calc(100% - 121px), #55FFFF 100%);
}
.chapter-to-cyan .chapter-number { color: rgba(255,50,50,0.1); }

/* The Forge - TempleOS background */
.chapter-forge {
    background: #55FFFF;
    position: relative;
}
.chapter-forge::before { content: none; }

/* Transition from cyan (Rules?) */
.chapter-from-cyan {
    background: linear-gradient(180deg, #55FFFF 0%, #0a0a0f 121px, #0a0a0f 100%);
    padding-top: 200px;
}
.chapter-from-cyan .chapter-number { color: rgba(255,50,50,0.1); top: 140px; }
.forge-logo {
    display: block;
    max-width: 100%;
    height: auto;
    margin: 0 auto;
}
.chapter-forge .chapter-content {
    position: relative;
    z-index: 1;
}
.chapter-forge .chapter-title { color: #000; }
.chapter-forge .chapter-title::after { background: linear-gradient(90deg, #000, transparent); }
.chapter-forge .chapter-number { color: rgba(0,0,0,0.15); }
.chapter-forge .narrative { color: #000; }
.chapter-forge .narrative a { color: #006666; }
.chapter-forge .narrative em { color: #004444; }
.chapter-forge .narrative strong { color: #000; }
.chapter-forge .dramatic-quote { background: rgba(0,0,0,0.15); border-left-color: #000; }
.chapter-forge .dramatic-quote p { color: #000; font-style: italic; }
.chapter-forge .dramatic-quote cite { color: #333; }

/* Rules? - 72 Benedictine principles */
.rule-grid {
    display: grid;
    grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
    gap: 12px;
    margin: 40px 0;
}
.rule-item {
    font-size: 1.05em;
    line-height: 1.6;
    color: #b0b0b0;
    padding: 12px 16px;
    background: rgba(255,255,255,0.02);
    border-radius: 6px;
    border-left: 3px solid rgba(255,107,107,0.3);
}
.rule-item strong {
    color: #ff6b6b;
    font-weight: 700;
}
@media (max-width: 768px) {
    .rule-grid { grid-template-columns: 1fr; }
}

.chapter-mirror .chapter-title { color: #8b5cf6; }
.chapter-mirror .chapter-title::after { background: linear-gradient(90deg, #8b5cf6, transparent); }
.chapter-mirror .chapter-number { color: rgba(139,92,246,0.08); }

/* Footer */
.about-footer {
    width: 100%;
    padding: 80px 40px;
    background: #050508;
    text-align: center;
}
.about-footer p {
    color: #666;
    font-size: 1.1em;
    margin-bottom: 20px;
}
.about-footer a {
    color: #ff6b6b;
    font-size: 1.2em;
    text-decoration: none;
    border-bottom: 2px solid rgba(255,107,107,0.3);
    padding-bottom: 4px;
    transition: all 0.3s;
}
.about-footer a:hover {
    color: #ff8e53;
    border-bottom-color: #ff8e53;
}

/* Responsive */
@media (max-width: 768px) {
    .hero-title { font-size: 3.5em; }
    .hero-pig { width: 200px; }
    .chapter { padding: 80px 20px; }
    .chapter-number { font-size: 5em; top: 20px; left: 20px; }
    .chapter-title { font-size: 2em; }
    .narrative { font-size: 1.1em; }
    .dramatic-quote { padding: 30px; }
    .dramatic-quote p { font-size: 1.3em; }
    .feature-section { padding: 60px 20px; }
}
"""

ABOUT_CONTENT = """
<div class="about-page">

    <!-- HERO SECTION -->
    <section class="hero-section">
        <img src="https://russell.ballestrini.net/uploads/2011/08/pig1-color1.png" alt="pig.py" class="hero-pig">
        <h1 class="hero-title">neopig</h1>
        <p class="hero-subtitle">Neo Python Image Grabber</p>
        <p class="hero-tagline">
            A phoenix-like chimera grown from the bones of dead repositories &amp; forums. A pig that devours web domains whole. Where others archive pages, neopig <em>consumes all digital files</em>.
        </p>
    </section>

    <!-- CHAPTER I: GENESIS -->
    <section class="chapter">
        <div class="chapter-number">I</div>
        <div class="chapter-content">
            <h2 class="chapter-title">Genesis</h2>

            <div class="dramatic-quote">
                <p>pig.py is a <em>very</em> simple python command line tool to download all the images from a given uri.</p>
                <cite>&mdash; Russell Ballestrini, August 22, 2011</cite>
            </div>

            <p class="narrative">
                In the beginning, there was <a href="https://russell.ballestrini.net/python-image-grabber-pig-py/" target="_blank">pig.py</a>. A simple creature. Innocent. <em>Hungry.</em>
            </p>

            <p class="narrative">
                Russell Ballestrini crafted it in the summer of 2011 &mdash; a humble Python script with a singular appetite: <strong>images</strong>. Point it at a webpage, and it would slurp down every pixel it could find. No configuration. No complexity. Just a hungry little pig gobbling up the visual fabric of the web.
            </p>

            <div class="code-section">
                <pre><code>python pig.py https://www.foxhop.net</code></pre>
            </div>

            <p class="narrative">
                That was it. The entire interface. The pig asked only for a target, and it <em>fed</em>.
            </p>

            <p class="narrative">
                Russell released it into the public domain &mdash; a gift to anyone who needed to harvest images from the wild web. The source lived at <code>bitbucket.org/russellballestrini/pig</code>, nestled safely in a Mercurial repository. The pig slept soundly in its pen, unaware of the extinction event approaching.
            </p>
        </div>
    </section>

    <!-- CHAPTER II: THE EXTINCTION -->
    <section class="chapter chapter-dark">
        <div class="chapter-number">II</div>
        <div class="chapter-content">
            <h2 class="chapter-title">A Great Bitbucket Extinction</h2>

            <p class="narrative">
                <strong>Then came the dark times.</strong>
            </p>

            <p class="narrative">
                In 2020, Atlassian &mdash; the corporate leviathan that had swallowed Bitbucket whole &mdash; decreed that Mercurial repositories would be <em>purged</em>. All of them. Every hg repo on Bitbucket Cloud, atomized. Scattered like digital ash across the void. Atlassian never disclosed how many.
            </p>

            <p class="narrative">
                The original pig.py, that innocent image-gobbling creature, was among the casualties. Its home at <a href="https://bitbucket.org/russellballestrini/pig" class="dead-link">bitbucket.org/russellballestrini/pig</a> became a tombstone. Click it. We dare you. There's nothing there but the echo of corporate indifference.
            </p>

            <div class="dramatic-quote">
                <p>They literally killed pig.py & sent him to bitbucket.</p>
                <cite>&mdash; The cruel irony of naming your graveyard after a slang term for death</cite>
            </div>

            <p class="narrative">
                <em>"Sent to the bitbucket"</em> &mdash; programmer slang from the age of punch cards. When you discarded bad data, you threw it in the bit bucket. Atlassian named their service after a trash can, then proved the prophecy true by throwing away everyone's code.
            </p>

            <p class="narrative">
                The pig was dead. Its bones scattered across cached search results & archived blog posts. A ghost in the machine, referenced but unreachable.
            </p>

            <p class="narrative">
                But here's where it gets <em>weird</em>.
            </p>
        </div>
    </section>

    <!-- CHAPTER III: ALCHEMY -->
    <section class="chapter chapter-rebirth">
        <div class="chapter-number">III</div>
        <div class="chapter-content">
            <h2 class="chapter-title">An Alchemy</h2>

            <p class="narrative">
                Russell once wrote that <a href="https://russell.ballestrini.net/programming-is-like-alchemy/" target="_blank">"programming is like alchemy &mdash; instead of exchanging matter, we programmers exchange time."</a>
            </p>

            <p class="narrative">
                Programs are <strong>golems</strong>. Familiar spirits. Magical servants performing repetitive tasks so we don't have to. <em>"It is more accurate to group programs with technology than magic, but less fun."</em>
            </p>

            <p class="narrative">
                Years passed. The web continued its relentless churn &mdash; sites going dark, forums shutting down, communities scattering like startled birds. Somewhere, a very angry ex-Ruby developer (Python too) watched the digital decay & remembered the pig.
            </p>

            <div class="dramatic-quote">
                <p>What if we could transmute that dead code into gold?</p>
                <cite>&mdash; The thought that started everything</cite>
            </div>

            <p class="narrative">
                And speaking of transmutation: <a href="https://phys.org/news/2025-07-marathon-fusion-mercury-gold-energy.html" target="_blank">Marathon Fusion</a> discovered that tokamak breeding blankets &mdash; wrapped in Mercury-Lithium alloy, like <em>pigs in a blanket</em> &mdash; can transmute Mercury-198 into Gold-197 through chrysopoeia. Fast neutrons trigger (n, 2n) reactions; unstable mercury decays into stable gold within 64 hours.
            </p>

            <p class="narrative">
                Two metric tons of gold per gigawatt. The alchemists' dream realized, wrapped in radioactive patience (17.7 years of cooling before you can touch your transmuted treasure).
            </p>

            <p class="narrative">
                A golden goose born from the ashes of deprecated version control. The old pig was archived, but <strong>a new creature stirred in the digital depths...</strong>
            </p>
        </div>
    </section>

    <!-- CHAPTER IV: THE PIG AWAKENS -->
    <section class="chapter chapter-beast">
        <div class="chapter-number">IV</div>
        <div class="chapter-content">
            <h2 class="chapter-title">A Pig Awakens</h2>

            <p class="narrative">
                Like a phoenix rising from dead Bitbucket repos, <strong>neopig</strong> emerged.
            </p>

            <p class="narrative">
                Not a resurrection. Not a mere fork. Something <em>else</em>. A chimera. A griffin. A creature stitched together from the DNA of the original pig & the fevered dreams of someone who had watched too many sites die.
            </p>

            <div class="dramatic-quote">
                <p>A pig with an appetite like Gluttony from Fullmetal Alchemist &mdash; a homunculus that devours everything in its path, absorbing knowledge, power & form.</p>
            </div>

            <p class="narrative">
                Where pig.py sipped politely from single pages, neopig <strong>devours entire domains</strong>. It doesn't just grab images &mdash; it consumes HTML, screenshots pages, converts content to markdown, indexes every scrap of text, and stores it all in content-addressed vaults that will outlast the original servers.
            </p>

            <p class="narrative">
                The code lives now at <a href="https://git.unturf.com/engineering/unturf/pig.py" target="_blank">git.unturf.com/engineering/unturf/pig.py</a>. And it keeps evolving. Every crawl adds new capabilities. Every archived site teaches it new tricks.
            </p>

            <p class="narrative">
                <em>We are adding the ability to backup entire codebases.</em> Neopig grows hungrier.
            </p>
            <img src="/static/images/hungry-pig.jpg" alt="Neopig grows hungrier" style="width:42%">
        </div>
    </section>

    <!-- CHAPTER V: THE MIRROR DIMENSION -->
    <section class="chapter chapter-mirror">
        <div class="chapter-number">V</div>
        <div class="chapter-content">
            <h2 class="chapter-title">A Mirror Dimension</h2>

            <p class="narrative">
                We are about to <strong>invert the git tree into itself</strong>.
            </p>

            <p class="narrative">
                Like Dr. Strange versus Spider-Man in the mirror dimension &mdash; that impossible space where geometry folds back on itself, where buildings become M.C. Escher nightmares & reality reflects endlessly into its own depths.
            </p>

            <p class="narrative">
                neopig doesn't just archive websites. It can archive <em>itself</em>. It can archive the repositories that contain the code that does the archiving. It can crawl git forges & preserve the source code of tools that preserve source code.
            </p>

            <div class="dramatic-quote">
                <p>Our spider-pig gets out of the danger of the mirror void &mdash; where days are years &mdash; and exits absorbing everything like a griffin, like a chimera, like a pig that feeds on the bones of dead platforms.</p>
            </div>

            <p class="narrative">
                When a code hosting platform dies (and they all die eventually &mdash; Google Code, Gitorious, BitBucket's hg repos), neopig ensures the knowledge survives. We are the ark. We are the vault. We are the pig that remembers.
            </p>

            <p class="narrative">
                The archives become self-extracting executables. <code>.run</code> files that contain their own server, their own search engine, their own reality. Drop one on a machine with Python, and it <em>wakes up</em>. A sleeping pig, carrying an entire website in its belly, ready to serve it to anyone who asks.
            </p>
            <img src="/static/images/sleeping-pig.gif" alt="A sleeping pig" style="width:42%">
        </div>
    </section>

    <!-- FEATURES SECTION -->
    <section class="feature-section">
        <div class="chapter-number">VI</div>
        <div class="chapter-content" style="max-width: 100%;">
            <h2 class="chapter-title" style="text-align: center; margin-bottom: 60px;">The Arsenal</h2>

            <div class="feature-grid">
                <div class="feature-card">
                    <h3>Async Domain Crawling</h3>
                    <p>Full-domain recursive crawling with configurable depth. Respects robots.txt & crawl delays. Polite but <em>relentless</em>.</p>
                </div>
                <div class="feature-card">
                    <h3>Content-Addressed Vault</h3>
                    <p>MD5-based deduplication in a 9-deep hex directory structure. Same image from 100 pages? Stored once. Space is memory.</p>
                </div>
                <div class="feature-card">
                    <h3>Full-Text Search</h3>
                    <p>Every image indexed with its surrounding context &mdash; page title, alt text, captions, nearby headings. Find images by <em>meaning</em>, not filename.</p>
                </div>
                <div class="feature-card">
                    <h3>Page Screenshots</h3>
                    <p><a href="https://linkpeek.com">LinkPeek</a> reborn! Full-page captures via <a href="https://github.com/russellballestrini/uri2png">uri2png</a>. wkhtmltoimage, cutycapt, or Playwright backends.</p>
                </div>
                <div class="feature-card">
                    <h3>Hydra Mode</h3>
                    <p>Auto-discover RSS/Atom/Sitemap feeds. Persists feed URLs, checks for new content on every crawl. Self-healing archive maintenance.</p>
                </div>
                <div class="feature-card">
                    <h3>Media Provenance</h3>
                    <p>Track first_seen, last_seen, and when each page first linked to media. Watch content spread across the web over time.</p>
                </div>
                <div class="feature-card">
                    <h3>Markdown Conversion</h3>
                    <p>Intelligent HTML-to-Markdown with forum post detection, noise removal & content extraction. Human-readable archives.</p>
                </div>
                <div class="feature-card">
                    <h3>Self-Extracting Archives</h3>
                    <p>Create <code>.run</code> executables that contain entire websites. Drop on any machine, execute, browse. The pig travels light.</p>
                </div>
                <div class="feature-card">
                    <h3>Code File Support</h3>
                    <p>Archive source code, stylesheets, fonts, and scripts alongside media. Complete preservation for developer communities.</p>
                </div>
                <div class="feature-card">
                    <h3>VCS Repository Cloning</h3>
                    <p>Auto-detect GitHub, GitLab, Bitbucket & more. Clone repos, symlink files to vault. Deduplicated code across forks.</p>
                </div>
                <div class="feature-card">
                    <h3>Live Feed</h3>
                    <p>Watch media appear in real-time as crawls progress. SSE-powered live updates. The pig hunts while you watch.</p>
                </div>
                <div class="feature-card">
                    <h3>SERP Web Interface</h3>
                    <p>FastAPI-powered search engine. Browse by type, search by context, view media details. Your personal media search engine.</p>
                </div>
            </div>
        </div>
    </section>

    <!-- CHAPTER VI: THE SKELETON KEY -->
    <section class="chapter">
        <div class="chapter-number">VII</div>
        <div class="chapter-content">
            <h2 class="chapter-title">A Skeleton Key 🗝️🔑</h2>

            <p class="narrative">
                neopig uses what we call the <strong>"skeleton key"</strong> approach to media indexing. For every image, we capture <em>all</em> the text that might help you find it later:
            </p>

            <p class="narrative">
                <strong>Page context:</strong> Title, description, headings near the image.<br>
                <strong>Image attributes:</strong> Alt text, title, surrounding captions.<br>
                <strong>Link context:</strong> The text of links pointing to the image.<br>
                <strong>Detail pages:</strong> For gallery sites, we follow through to detail pages & harvest their metadata too.
            </p>

            <p class="narrative">
                The result? You can search for <em>"sunset over mountains"</em> and find that image even if it was named <code>IMG_4372.jpg</code> with no alt text &mdash; because the page title mentioned it, or someone linked to it with descriptive text, or a caption three divs away contained the words.
            </p>

            <p class="narrative">
                Every door opens. Every search succeeds. The skeleton key fits every lock because it carries <em>every possible key</em> inside it.
            </p>
        </div>
    </section>

    <!-- CHAPTER VIII: PRESERVATION -->
    <section class="chapter chapter-to-cyan">
        <div class="chapter-number">VIII</div>
        <div class="chapter-content">
            <h2 class="chapter-title">Why We Archive</h2>

            <p class="narrative">
                <strong>The web is ephemeral.</strong> Sites go dark. Forums shut down. Communities scatter. Platforms get acquired &amp; gutted. Executives decide that Mercurial isn't profitable &amp; delete every repository with a single corporate memo.
            </p>

            <p class="narrative">
                We have watched beloved communities vanish. Game modding forums. Technical documentation. Art galleries. Fan wikis. Personal blogs with decades of writing. Gone. Not archived by the Wayback Machine. Not cached by Google. Just <em>gone</em>.
            </p>

            <p class="narrative">
                neopig is built for <strong>preservation</strong> &mdash; capturing not just the media, but the <em>context</em> that gives it meaning. When you archive a site with neopig, you get:
            </p>

            <p class="narrative">
                &bull; Original HTML with rewritten media links pointing to your local vault<br>
                &bull; Full-text searchable markdown versions of every page<br>
                &bull; Screenshots showing exactly how pages looked<br>
                &bull; A SQLite database you can query, backup, and migrate forever<br>
                &bull; Self-extracting archives that work offline forever
                <br><br><br>
            </p>
        </div>
    </section>

    <!-- CHAPTER IX: THE FORGE -->
    <section class="chapter chapter-forge">
        <div class="chapter-number">IX</div>
        <div class="chapter-content">
            <h2 class="chapter-title">A Forge</h2>

            <p class="narrative">
                This tool was not built for profit. It was <strong>forged</strong>.
            </p>

            <p class="narrative">
                Forged in the fires of <a href="https://permacomputer.com" target="_blank">permacomputer.com</a> &mdash; the belief that computing should be <em>permanent</em>. That knowledge should not evaporate when a company pivots. That communities should not lose their histories when a platform gets acquired. That the lore doesn't have to die when you get acqui-hired or outright <span style="color: #8B00FF;">terminated</span>. That the digital commons belongs to <em>everyone</em>, forever.
            </p>

            <img src="/static/vendor/templeos.png" alt="TempleOS" class="forge-logo">

            <div class="dramatic-quote">
                <p>Truth. Harmony. Freedom. Love.</p>
                <cite>&mdash; The quadrivium of a permacomputer</cite>
            </div>

            <p class="narrative">
                <strong>Truth</strong> &mdash; We preserve what actually existed, not sanitized versions. The original HTML. The exact timestamps. The unedited conversations. History as it happened, not as someone wishes it had.
            </p>

            <p class="narrative">
                <strong>Harmony</strong> &mdash; Every piece connects. Media links to pages. Pages link to context. Context links to meaning. The archive is not a graveyard of disconnected files but a living web of relationships, searchable, traversable, <em>whole</em>.
            </p>

            <p class="narrative">
                <strong>Freedom</strong> &mdash; Public domain. No licenses to parse. No corporate terms of service. Take it. Fork it. Run it. Share it. The code is yours. The archives you create are yours. No one can revoke your access to your own preserved knowledge.
            </p>

            <p class="narrative">
                <strong>Love</strong> &mdash; We archive because we <em>care</em>. About the communities that built these spaces. About the people who poured years into forum posts & wiki pages & blog entries. About the future researchers who will want to understand how we lived & thought & created. This is an act of love for human knowledge.
            </p>

            <p class="narrative">
                The pig is imbued with <em>righteous fire</em>. It does not crawl politely because it fears the powerful &mdash; it respects robots.txt because it honors the wishes of creators. It does not hoard in darkness &mdash; it preserves in light, making archives searchable, shareable, <em>alive</em>.
            </p>

            <p class="narrative">
                When the platforms fall (and they will fall), when the corporate servers go dark (and they will go dark), when the executives decide your community isn't profitable enough to maintain (and they will decide this) &mdash; a permacomputer endures. The archives persist. The knowledge survives.
            </p>

            <p class="narrative">
                <strong>This is what we were forged to do.</strong>
            </p>
        </div>
    </section>

    <!-- CHAPTER X: RULES? -->
    <section class="chapter chapter-from-cyan">
        <div class="chapter-number">X</div>
        <div class="chapter-content">
            <h2 class="chapter-title">Rules?</h2>

            <p class="narrative">
                neopig is built on <a href="https://sqlite.org" target="_blank">SQLite</a> &mdash; the most deployed database in existence. SQLite adopted the <a href="https://sqlite.org/codeofethics.html" target="_blank">Rule of St. Benedict</a> as their Code of Ethics. 72 principles from a 6th-century monastery, governing software in the 21st.
            </p>

            <p class="narrative">
                We inherit this lineage. All 72 rules, unabridged:
            </p>

            <div class="rule-grid">
                <div class="rule-item"><strong>1.</strong> Love the Lord God with your whole heart, soul, and strength.</div>
                <div class="rule-item"><strong>2.</strong> Love your neighbor as yourself.</div>
                <div class="rule-item"><strong>3.</strong> Do not murder.</div>
                <div class="rule-item"><strong>4.</strong> Do not commit adultery.</div>
                <div class="rule-item"><strong>5.</strong> Do not steal.</div>
                <div class="rule-item"><strong>6.</strong> Do not covet.</div>
                <div class="rule-item"><strong>7.</strong> Do not bear false witness.</div>
                <div class="rule-item"><strong>8.</strong> Honor all people.</div>
                <div class="rule-item"><strong>9.</strong> Do not do to another what you would not have done to yourself.</div>
                <div class="rule-item"><strong>10.</strong> Deny oneself in order to follow Christ.</div>
                <div class="rule-item"><strong>11.</strong> Chastise the body.</div>
                <div class="rule-item"><strong>12.</strong> Do not become attached to pleasures.</div>
                <div class="rule-item"><strong>13.</strong> Love fasting.</div>
                <div class="rule-item"><strong>14.</strong> Relieve the poor.</div>
                <div class="rule-item"><strong>15.</strong> Clothe the naked.</div>
                <div class="rule-item"><strong>16.</strong> Visit the sick.</div>
                <div class="rule-item"><strong>17.</strong> Bury the dead.</div>
                <div class="rule-item"><strong>18.</strong> Be a help in times of trouble.</div>
                <div class="rule-item"><strong>19.</strong> Console the sorrowing.</div>
                <div class="rule-item"><strong>20.</strong> Be a stranger to the world's ways.</div>
                <div class="rule-item"><strong>21.</strong> Prefer nothing more than the love of Christ.</div>
                <div class="rule-item"><strong>22.</strong> Do not give way to anger.</div>
                <div class="rule-item"><strong>23.</strong> Do not nurse a grudge.</div>
                <div class="rule-item"><strong>24.</strong> Do not entertain deceit in your heart.</div>
                <div class="rule-item"><strong>25.</strong> Do not give a false peace.</div>
                <div class="rule-item"><strong>26.</strong> Do not forsake charity.</div>
                <div class="rule-item"><strong>27.</strong> Do not swear, for fear of perjuring yourself.</div>
                <div class="rule-item"><strong>28.</strong> Utter only truth from heart and mouth.</div>
                <div class="rule-item"><strong>29.</strong> Do not return evil for evil.</div>
                <div class="rule-item"><strong>30.</strong> Do no wrong to anyone, and bear patiently wrongs done to yourself.</div>
                <div class="rule-item"><strong>31.</strong> Love your enemies.</div>
                <div class="rule-item"><strong>32.</strong> Do not curse those who curse you, but rather bless them.</div>
                <div class="rule-item"><strong>33.</strong> Bear persecution for justice's sake.</div>
                <div class="rule-item"><strong>34.</strong> Be not proud.</div>
                <div class="rule-item"><strong>35.</strong> Be not addicted to wine.</div>
                <div class="rule-item"><strong>36.</strong> Be not a great eater.</div>
                <div class="rule-item"><strong>37.</strong> Be not drowsy.</div>
                <div class="rule-item"><strong>38.</strong> Be not lazy.</div>
                <div class="rule-item"><strong>39.</strong> Be not a grumbler.</div>
                <div class="rule-item"><strong>40.</strong> Be not a detractor.</div>
                <div class="rule-item"><strong>41.</strong> Put your hope in God.</div>
                <div class="rule-item"><strong>42.</strong> Attribute to God, and not to self, whatever good you see in yourself.</div>
                <div class="rule-item"><strong>43.</strong> Recognize always that evil is your own doing, and to impute it to yourself.</div>
                <div class="rule-item"><strong>44.</strong> Fear the Day of Judgment.</div>
                <div class="rule-item"><strong>45.</strong> Be in dread of hell.</div>
                <div class="rule-item"><strong>46.</strong> Desire eternal life with all the passion of the spirit.</div>
                <div class="rule-item"><strong>47.</strong> Keep death daily before your eyes.</div>
                <div class="rule-item"><strong>48.</strong> Keep constant guard over the actions of your life.</div>
                <div class="rule-item"><strong>49.</strong> Know for certain that God sees you everywhere.</div>
                <div class="rule-item"><strong>50.</strong> When wrongful thoughts come into your heart, dash them against Christ immediately.</div>
                <div class="rule-item"><strong>51.</strong> Disclose wrongful thoughts to your spiritual mentor.</div>
                <div class="rule-item"><strong>52.</strong> Guard your tongue against evil and depraved speech.</div>
                <div class="rule-item"><strong>53.</strong> Do not love much talking.</div>
                <div class="rule-item"><strong>54.</strong> Speak no useless words or words that move to laughter.</div>
                <div class="rule-item"><strong>55.</strong> Do not love much or boisterous laughter.</div>
                <div class="rule-item"><strong>56.</strong> Listen willingly to holy reading.</div>
                <div class="rule-item"><strong>57.</strong> Devote yourself frequently to prayer.</div>
                <div class="rule-item"><strong>58.</strong> Daily in your prayers, with tears and sighs, confess your past sins to God, and amend them for the future.</div>
                <div class="rule-item"><strong>59.</strong> Fulfill not the desires of the flesh; hate your own will.</div>
                <div class="rule-item"><strong>60.</strong> Obey in all things the commands of those whom God has placed in authority over you even though they should act otherwise, mindful of the Lord's precept, "Do what they say, but not what they do."</div>
                <div class="rule-item"><strong>61.</strong> Do not wish to be called holy before one is holy; but first to be holy, that you may be truly so called.</div>
                <div class="rule-item"><strong>62.</strong> Fulfill God's commandments daily in your deeds.</div>
                <div class="rule-item"><strong>63.</strong> Love chastity.</div>
                <div class="rule-item"><strong>64.</strong> Hate no one.</div>
                <div class="rule-item"><strong>65.</strong> Be not jealous, nor harbor envy.</div>
                <div class="rule-item"><strong>66.</strong> Do not love quarreling.</div>
                <div class="rule-item"><strong>67.</strong> Shun arrogance.</div>
                <div class="rule-item"><strong>68.</strong> Respect your seniors.</div>
                <div class="rule-item"><strong>69.</strong> Love your juniors.</div>
                <div class="rule-item"><strong>70.</strong> Pray for your enemies in the love of Christ.</div>
                <div class="rule-item"><strong>71.</strong> Make peace with your adversary before the sun sets.</div>
                <div class="rule-item"><strong>72.</strong> Never despair of God's mercy.</div>
            </div>

            <p class="narrative" style="margin-top: 60px;">
                We archive not to hoard, but to <em>give</em>. We preserve not to control, but to <em>liberate</em>. The database that holds your memories runs on a foundation of ancient wisdom &mdash; and so do we.
            </p>
        </div>
    </section>

    <!-- CHAPTER XI: INVOKE THE PIG -->
    <section class="chapter chapter-beast">
        <div class="chapter-number">XI</div>
        <div class="chapter-content">
            <h2 class="chapter-title">Invoke the Pig</h2>

            <div class="code-section">
                <pre><code># open http://127.0.0.1:31337 in browser
python neopig.py --serve</code></pre>
            </div>

            <p class="narrative">
                <strong>neopig</strong> = <strong>Neo</strong> (new) + <strong>P</strong>ython <strong>I</strong>mage <strong>G</strong>rabber
            </p>

            <p class="narrative">
                A tip of the hat to the original pig.py. A nod to the Matrix's Neo &mdash; seeing through the surface of the web to the underlying content within. And perhaps a warning: this pig has grown teeth.
            </p>
        </div>
    </section>

    <!-- FOOTER -->
    <footer class="about-footer">
        <p>neopig is open source, donated into the public domain.</p>
        <p>Neopig lives:</p>
        <a href="https://git.unturf.com/engineering/unturf/pig.py" target="_blank">git.unturf.com/engineering/unturf/pig.py</a>
        <p style="margin-top: 40px; color: #444; font-size: 0.9em;">
            In memory of all the repositories Atlassian killed.<br>
            Neopig remembers. Neopig archives. Neopig <em>persists</em>.
        </p>
        <p style="margin-top: 60px; color: #666; font-style: italic;">
            Sorry for the convenience.
        </p>
    </footer>

</div>
"""


@app.get("/about", response_class=HTMLResponse)
async def about_page(lang: str = Cookie(None), accept_language: str = Header(None)):
    """About neopig - the story of pig.py's evolution."""
    language = get_lang(lang, accept_language)
    html = layout("{{about}}", ABOUT_CONTENT, extra_css=ABOUT_CSS)
    return inject_i18n(html, language)


@app.get("/health")
async def health():
    """Health check endpoint."""
    has_screenshot = False
    try:
        from uri2png import get_available_engines
        has_screenshot = True
    except ImportError:
        pass

    return {
        "status": "healthy",
        "features": {
            "search": True,
            "crawl": True,
            "screenshot": has_screenshot
        }
    }


@app.get("/api/stats")
async def get_stats_endpoint():
    """Get database statistics."""
    stats = await db.get_stats()
    # Add page count (handled separately since table may not exist)
    try:
        from sqlalchemy import select, func
        from database import Page
        async with db.session() as session:
            result = await session.execute(select(func.count()).select_from(Page))
            stats['total_pages'] = result.scalar() or 0
    except Exception:
        stats['total_pages'] = 0
    return stats


@app.get("/random")
async def random_item(type: str = Query(None, description="Type: media or page (random if not specified)")):
    """Redirect to a random media item or page."""
    from sqlalchemy import select, func
    from database import Media, Page
    import random

    # If no type specified, randomly pick between media and page
    if type is None:
        type = random.choice(["media", "page"])

    async with db.session() as session:
        if type == "page":
            # Get a random page
            stmt = select(Page.uri_hash).order_by(func.random()).limit(1)
            result = await session.execute(stmt)
            row = result.fetchone()
            if row and row[0]:
                return RedirectResponse(url=f"/page/{row[0]}", status_code=302)
        else:
            # Get a random media item (excluding screenshots)
            stmt = (
                select(Media.md5_hash)
                .where(Media.media_type != 'screenshot')
                .order_by(func.random())
                .limit(1)
            )
            result = await session.execute(stmt)
            row = result.fetchone()
            if row:
                return RedirectResponse(url=f"/view/{row[0]}", status_code=302)

        return RedirectResponse(url="/", status_code=302)


@app.get("/api/search")
async def search(
    q: str = Query("", description="Search query"),
    type: Optional[str] = Query(None, description="Filter by media type"),
    status: Optional[str] = Query(None, description="Filter by analysis status"),
    limit: int = Query(OVER_9000, le=OVER_9000),
    offset: int = Query(0)
):
    """
    Search media by text query.

    Searches across: keywords, alt_text, title, source URLs, analysis results.
    """
    results = await db.search_media_advanced(
        q=q if q else None,
        media_type=type,
        limit=limit,
        offset=offset
    )
    return results


@app.get("/api/live/stream")
async def live_stream():
    """
    SSE endpoint for live media feed.

    Media appears here immediately after being saved to disk,
    before DB insert completes. Use EventSource to connect.
    """
    async def event_generator():
        queue = get_live_queue()
        while True:
            try:
                # Wait for next media item with timeout
                media = await asyncio.wait_for(queue.get(), timeout=30)
                yield f"data: {json.dumps(media)}\n\n"
            except asyncio.TimeoutError:
                # Send keepalive
                yield ": keepalive\n\n"
            except Exception as e:
                logger.warning(f"SSE error: {e}")
                break

    return StreamingResponse(
        event_generator(),
        media_type="text/event-stream",
        headers={
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "X-Accel-Buffering": "no",  # Disable nginx buffering
        }
    )


@app.get("/api/search/pages")
async def search_pages_endpoint(
    q: str = Query("", description="Search query"),
    limit: int = Query(50, le=500),
):
    """
    Search pages by text query using FTS5.
    """
    if not q:
        return []
    return await db.search_pages(q, limit)


@app.get("/api/upgraded")
async def get_upgraded_media(limit: int = Query(50, le=200)):
    """
    Get media that was recently upgraded (found higher quality version).
    Used by live feed to show when thumbnails get replaced by full-res.
    """
    return await db.get_recently_upgraded(limit)


@app.get("/api/media/{md5_hash}")
async def get_media_info(md5_hash: str):
    """Get full media info including all source URLs."""
    media = await db.get_media_by_hash(md5_hash)
    if not media:
        raise HTTPException(status_code=404, detail="Media not found")

    media['sources'] = await db.get_media_sources(md5_hash)
    return media


def slugify(text: str, max_len: int = 60) -> str:
    """Convert text to a safe filename slug."""
    import re
    import unicodedata
    # Normalize unicode
    text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('ascii')
    # Lowercase and replace spaces/special chars with hyphens
    text = re.sub(r'[^\w\s-]', '', text.lower())
    text = re.sub(r'[-\s]+', '-', text).strip('-')
    return text[:max_len] if text else ""


@app.get("/media/{md5_hash}")
async def serve_media(md5_hash: str, download: bool = False):
    """
    Serve media file from vault or tarball.

    Use ?download=1 for attachment mode with smart filename.
    Caddy should be configured to cache these responses.
    """
    # Tarball mode: serve from tar.gz (run in thread to avoid blocking)
    if TAR_PATH:
        data, ext = await asyncio.to_thread(find_media_in_tarball, md5_hash)
        if data:
            mime_type, _ = mimetypes.guess_type(f"file{ext}")
            if not mime_type:
                mime_type = "application/octet-stream"
            filename = f"{md5_hash[:12]}{ext}"
            headers = {
                "Cache-Control": "public, max-age=31536000, immutable",
                "X-Content-Hash": md5_hash,
            }
            if download:
                headers["Content-Disposition"] = f'attachment; filename="{filename}"'
            return Response(content=data, media_type=mime_type, headers=headers)
        raise HTTPException(status_code=404, detail="Media not found in archive")

    # Filesystem mode: find file in vault (9-deep path)
    subdir = VAULT_PATH / hash_to_path(md5_hash).parent
    if not subdir.exists():
        raise HTTPException(status_code=404, detail="Media not found")

    # Find file with this hash prefix
    for f in subdir.iterdir():
        if f.name.startswith(md5_hash):
            # Guess content type from filename
            mime_type, _ = mimetypes.guess_type(f.name)
            if not mime_type:
                mime_type = "application/octet-stream"

            # Inline mode: skip DB queries, just serve the file fast
            if not download:
                return FileResponse(
                    f,
                    media_type=mime_type,
                    content_disposition_type="inline",
                    headers={
                        "Cache-Control": "public, max-age=31536000, immutable",
                        "X-Content-Hash": md5_hash,
                    }
                )

            # Download mode: generate smart filename from metadata
            ext = f.suffix or ""
            filename = None

            # Priority 1: Original filename from URL
            sources = await db.get_media_sources(md5_hash)
            if sources:
                row2 = sources[0]
                if row2.get("media_uri"):
                    from urllib.parse import unquote
                    parsed = Uri(row2["media_uri"])
                    orig_name = Path(unquote(parsed.path)).name
                    if orig_name and '.' in orig_name:
                        filename = orig_name

            # Priority 2: Generate from alt_text or title
            if not filename:
                media_record = await db.get_media_by_hash(md5_hash)
                if media_record:
                    if not mime_type and media_record.get("mime_type"):
                        mime_type = media_record["mime_type"]
                    name_source = media_record.get("alt_text") or media_record.get("title")
                    if name_source:
                        slug = slugify(name_source)
                        if slug:
                            filename = f"{slug}{ext}"

            # Priority 3: page_title + hash index
            if not filename and sources:
                row2 = sources[0]
                if row2.get("page_title"):
                    media_idx = int(md5_hash[:4], 16)
                    slug = slugify(f"{row2['page_title']}-{media_idx}")
                    if slug:
                        filename = f"{slug}{ext}"

            # Default filename if nothing else
            if not filename:
                filename = f"{md5_hash[:12]}{ext}"

            return FileResponse(
                f,
                media_type=mime_type,
                filename=filename,
                content_disposition_type="attachment",
                headers={
                    "Cache-Control": "public, max-age=31536000, immutable",
                    "X-Content-Hash": md5_hash,
                }
            )

    raise HTTPException(status_code=404, detail="Media not found")


# ============================================================================
# Import Mode - Upload and serve archives
# ============================================================================

IMPORT_UPLOAD_DIR = Path(tempfile.gettempdir()) / "neopig_import"

# Resumable upload tracking: upload_id -> {filename, total_size, created_at}
PENDING_UPLOADS: Dict[str, dict] = {}

def generate_upload_id() -> str:
    """Generate a unique upload ID."""
    import secrets
    return f"upload-{secrets.token_hex(8)}"


@app.post("/api/import/upload/init")
async def init_resumable_upload(filename: str = Query(...), size: int = Query(...)):
    """Initialize a resumable upload session.

    Returns upload_id that client uses for subsequent chunk uploads.
    Client can resume from any disconnect by checking /api/import/upload/{upload_id}/status
    """
    if not IMPORT_MODE:
        raise HTTPException(status_code=403, detail="Import mode not enabled")

    if not filename.endswith(('.tar.gz', '.tgz', '.run')):
        raise HTTPException(status_code=400, detail="File must be .tar.gz, .tgz, or .run")

    upload_id = generate_upload_id()
    IMPORT_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)

    # Create empty file for this upload
    upload_path = IMPORT_UPLOAD_DIR / f"{upload_id}.part"
    upload_path.touch()

    PENDING_UPLOADS[upload_id] = {
        "filename": filename,
        "total_size": size,
        "created_at": asyncio.get_event_loop().time(),
        "path": str(upload_path)
    }

    logger.info(f"Import: initialized resumable upload {upload_id} for {filename} ({size / 1024 / 1024:.1f} MB)")

    return {
        "upload_id": upload_id,
        "filename": filename,
        "total_size": size,
        "bytes_received": 0
    }


@app.get("/api/import/upload/{upload_id}/status")
async def get_upload_status(upload_id: str):
    """Get status of a resumable upload. Use this to resume after disconnect."""
    if not IMPORT_MODE:
        raise HTTPException(status_code=403, detail="Import mode not enabled")

    if upload_id not in PENDING_UPLOADS:
        raise HTTPException(status_code=404, detail="Upload not found or expired")

    info = PENDING_UPLOADS[upload_id]
    upload_path = Path(info["path"])

    if not upload_path.exists():
        raise HTTPException(status_code=404, detail="Upload file not found")

    bytes_received = upload_path.stat().st_size

    return {
        "upload_id": upload_id,
        "filename": info["filename"],
        "total_size": info["total_size"],
        "bytes_received": bytes_received,
        "complete": bytes_received >= info["total_size"]
    }


@app.patch("/api/import/upload/{upload_id}")
async def upload_chunk(
    upload_id: str,
    request: Request,
    content_range: str = Header(None)
):
    """Upload a chunk of data for resumable upload.

    Use Content-Range header: bytes START-END/TOTAL
    Example: Content-Range: bytes 0-1048575/10485760

    Or use X-Upload-Offset header for simpler resumption.
    """
    if not IMPORT_MODE:
        raise HTTPException(status_code=403, detail="Import mode not enabled")

    if upload_id not in PENDING_UPLOADS:
        raise HTTPException(status_code=404, detail="Upload not found or expired")

    info = PENDING_UPLOADS[upload_id]
    upload_path = Path(info["path"])

    # Parse offset from Content-Range or X-Upload-Offset
    offset = 0
    if content_range:
        # Parse "bytes START-END/TOTAL"
        try:
            range_spec = content_range.replace("bytes ", "")
            range_part = range_spec.split("/")[0]
            offset = int(range_part.split("-")[0])
        except Exception:
            raise HTTPException(status_code=400, detail="Invalid Content-Range header")
    else:
        # Use X-Upload-Offset if no Content-Range
        offset_header = request.headers.get("x-upload-offset")
        if offset_header:
            offset = int(offset_header)

    # Verify offset matches current file size (no gaps)
    current_size = upload_path.stat().st_size if upload_path.exists() else 0
    if offset != current_size:
        raise HTTPException(
            status_code=409,
            detail=f"Offset mismatch: expected {current_size}, got {offset}. Resume from byte {current_size}."
        )

    # Read and append chunk
    chunk_data = await request.body()
    chunk_size = len(chunk_data)

    with open(upload_path, 'ab') as f:
        f.write(chunk_data)

    new_size = upload_path.stat().st_size
    logger.info(f"Import: {upload_id} received chunk {chunk_size} bytes, total {new_size}/{info['total_size']}")

    return {
        "upload_id": upload_id,
        "bytes_received": new_size,
        "total_size": info["total_size"],
        "complete": new_size >= info["total_size"]
    }


@app.post("/api/import/upload/{upload_id}/complete")
async def complete_resumable_upload(upload_id: str):
    """Finalize upload and start import job."""
    if not IMPORT_MODE:
        raise HTTPException(status_code=403, detail="Import mode not enabled")

    if upload_id not in PENDING_UPLOADS:
        raise HTTPException(status_code=404, detail="Upload not found or expired")

    info = PENDING_UPLOADS[upload_id]
    upload_path = Path(info["path"])

    if not upload_path.exists():
        raise HTTPException(status_code=404, detail="Upload file not found")

    bytes_received = upload_path.stat().st_size
    if bytes_received < info["total_size"]:
        raise HTTPException(
            status_code=400,
            detail=f"Upload incomplete: {bytes_received}/{info['total_size']} bytes"
        )

    # Rename to final filename
    final_path = IMPORT_UPLOAD_DIR / info["filename"]
    upload_path.rename(final_path)

    # Remove from pending
    del PENDING_UPLOADS[upload_id]

    logger.info(f"Import: {upload_id} completed, saved as {info['filename']}")

    # Create import job
    job_id = await db.create_crawl_job(
        target_uri=f"import://{info['filename']}",
        keywords=["import"],
        mode="import"
    )

    # Run import in background
    async def run_import():
        try:
            await db.set_crawl_job_status(job_id, "running")
            stats = await import_archive_to_db(final_path, job_id)
            await db.complete_crawl_job(job_id, stats)
            final_path.unlink(missing_ok=True)
            logger.info(f"Import complete: {stats}")
        except Exception as e:
            logger.error(f"Import failed: {e}")
            await db.fail_crawl_job(job_id, str(e))

    task = asyncio.create_task(run_import())
    ACTIVE_CRAWL_TASKS[job_id] = task
    task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))

    return {
        "status": "importing",
        "job_id": job_id,
        "filename": info["filename"],
        "size": bytes_received
    }


@app.post("/api/import/upload")
async def upload_archive(file: UploadFile = File(...)):
    """Upload a tar.gz archive and import into local database as a job."""
    if not IMPORT_MODE:
        raise HTTPException(status_code=403, detail="Import mode not enabled (NEOPIG_IMPORT=1)")

    if not file.filename.endswith(('.tar.gz', '.tgz', '.run')):
        raise HTTPException(status_code=400, detail="File must be .tar.gz, .tgz, or .run")

    # Save uploaded file
    IMPORT_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
    upload_path = IMPORT_UPLOAD_DIR / file.filename
    logger.info(f"Import: receiving upload {file.filename}")

    try:
        with open(upload_path, 'wb') as f:
            while chunk := await file.read(1024 * 1024):
                f.write(chunk)
    except Exception as e:
        raise HTTPException(status_code=400, detail=f"Upload failed: {e}")

    file_size = upload_path.stat().st_size
    logger.info(f"Import: saved {file.filename} ({file_size / 1024 / 1024:.1f} MB)")

    # Create import job
    job_id = await db.create_crawl_job(
        target_uri=f"import://{file.filename}",
        keywords=["import"],
        mode="import"
    )

    # Run import in background
    async def run_import():
        try:
            await db.set_crawl_job_status(job_id, "running")
            stats = await import_archive_to_db(upload_path, job_id)
            await db.complete_crawl_job(job_id, stats)
            upload_path.unlink(missing_ok=True)
            logger.info(f"Import complete: {stats}")
        except Exception as e:
            logger.error(f"Import failed: {e}")
            await db.fail_crawl_job(job_id, str(e))

    task = asyncio.create_task(run_import())
    ACTIVE_CRAWL_TASKS[job_id] = task
    task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))

    return {"status": "importing", "job_id": job_id, "filename": file.filename, "size": file_size}


async def import_archive_to_db(archive_path: Path, job_id: int) -> dict:
    """Import archive's database and vault into local database."""
    import sqlite3

    stats = {"media_imported": 0, "pages_imported": 0, "sources_imported": 0}

    # Detect .run offset
    offset = 0
    try:
        with open(archive_path, 'rb') as f:
            f.seek(-22, 2)
            trailer = f.read(22)
            if trailer[:6] == b'NEOPIG':
                offset = int(trailer[6:22].decode(), 16)
    except Exception:
        pass

    # Open tarball
    if offset > 0:
        f = open(archive_path, 'rb')
        f.seek(offset)
        tar = tarfile.open(fileobj=f, mode='r:gz')
    else:
        tar = tarfile.open(archive_path, 'r:gz')

    try:
        members = tar.getmembers()
        if not members:
            raise ValueError("Empty archive")
        archive_root = members[0].name.split('/')[0]

        # Extract neopig.db to temp
        db_member = f"{archive_root}/neopig.db"
        temp_db = IMPORT_UPLOAD_DIR / f"import_{job_id}.db"

        for m in members:
            if m.name == db_member:
                f_db = tar.extractfile(m)
                if f_db:
                    with open(temp_db, 'wb') as out:
                        out.write(f_db.read())
                break

        # Merge database
        if not temp_db.exists():
            raise ValueError("Malformed archive: missing neopig.db")

        src = sqlite3.connect(temp_db)
        src.row_factory = sqlite3.Row
        try:
            # Check what tables exist in source
            tables = [r[0] for r in src.execute("SELECT name FROM sqlite_master WHERE type='table'")]
            logger.info(f"Import: source database has tables: {tables}")

            # Validate archive has required tables
            required_tables = {'pages', 'media', 'media_sources'}
            if not required_tables.intersection(tables):
                raise ValueError(f"Malformed archive: neopig.db has no data tables (found: {tables})")

            # Import media (if table exists)
            if 'media' in tables:
                for row in src.execute("SELECT * FROM media"):
                    try:
                        async with db.session() as session:
                            await session.execute(text(
                                """INSERT OR IGNORE INTO media
                                (md5_hash, media_type, mime_type, file_size, keywords, alt_text, title,
                                 first_seen_at, last_seen_at, score)
                                VALUES (:md5_hash, :media_type, :mime_type, :file_size, :keywords, :alt_text, :title,
                                 :first_seen_at, :last_seen_at, :score)"""), dict(row))
                            await session.commit()
                        stats["media_imported"] += 1
                    except Exception as e:
                        logger.debug(f"Skip media row: {e}")
            else:
                logger.warning("Import: source has no 'media' table")

            # Import media_sources (if table exists)
            if 'media_sources' in tables:
                for row in src.execute("SELECT * FROM media_sources"):
                    try:
                        row_dict = dict(row)
                        row_dict['crawl_job_id'] = job_id  # Link to import job for deletion
                        async with db.session() as session:
                            await session.execute(text(
                                """INSERT OR IGNORE INTO media_sources
                                (md5_hash, media_uri, page_uri, page_title, alt_text, searchable_text, discovered_at, crawl_job_id)
                                VALUES (:md5_hash, :media_uri, :page_uri, :page_title, :alt_text, :searchable_text, :discovered_at, :crawl_job_id)"""), row_dict)
                            await session.commit()
                        stats["sources_imported"] += 1
                    except Exception as e:
                        logger.debug(f"Skip media_source row: {e}")
            else:
                logger.warning("Import: source has no 'media_sources' table")

            # Import pages (if table exists)
            if 'pages' in tables:
                for row in src.execute("SELECT * FROM pages"):
                    try:
                        row_dict = dict(row)
                        row_dict['crawl_job_id'] = job_id  # Link to import job for deletion
                        async with db.session() as session:
                            await session.execute(text(
                                """INSERT OR REPLACE INTO pages
                                (uri, uri_hash, path, title, description, keywords, content, markdown, raw_html, crawled_at, crawl_job_id)
                                VALUES (:uri, :uri_hash, :path, :title, :description, :keywords, :content, :markdown, :raw_html, :crawled_at, :crawl_job_id)"""), row_dict)
                            await session.commit()
                        stats["pages_imported"] += 1
                    except Exception as e:
                        logger.debug(f"Skip page row: {e}")
            else:
                logger.warning("Import: source has no 'pages' table")
        finally:
            src.close()
            temp_db.unlink(missing_ok=True)

        # Extract vault files
        vault_prefix = f"{archive_root}/vault/"
        for m in members:
            if m.name.startswith(vault_prefix) and m.isfile():
                rel_path = m.name[len(vault_prefix):]
                dest_path = VAULT_PATH / rel_path
                if not dest_path.exists():
                    dest_path.parent.mkdir(parents=True, exist_ok=True)
                    f_media = tar.extractfile(m)
                    if f_media:
                        with open(dest_path, 'wb') as out:
                            out.write(f_media.read())
                        stats["media_imported"] += 1

    finally:
        tar.close()

    return stats


@app.get("/api/import/status")
async def import_status():
    """Get current import status."""
    if not IMPORT_MODE:
        raise HTTPException(status_code=403, detail="Import mode not enabled")

    # Get import job stats
    stats = await db.get_stats()
    import_jobs = await db.get_crawl_jobs(limit=10)
    imports = [j for j in import_jobs if j.get("mode") == "import"]

    return {
        "import_mode": True,
        "media_count": stats.get("media", 0),
        "pages_count": stats.get("pages", 0),
        "recent_imports": len(imports)
    }


IMPORT_CSS = """
        .container { max-width: 800px; margin: 0 auto; }
        h1 { color: #ff6b6b; margin-bottom: 5px; }
        .subtitle { color: #666; margin-bottom: 30px; }

        .upload-zone {
            border: 3px dashed #333;
            border-radius: 12px;
            padding: 60px 40px;
            text-align: center;
            background: #111;
            cursor: pointer;
            transition: all 0.2s;
        }
        .upload-zone:hover, .upload-zone.drag-over {
            border-color: #ff6b6b;
            background: #1a1a1a;
        }
        .upload-zone input[type="file"] { display: none; }
        .upload-icon { font-size: 48px; margin-bottom: 15px; }
        .upload-text { color: #888; font-size: 16px; }
        .upload-hint { color: #555; font-size: 13px; margin-top: 10px; }

        .status {
            margin-top: 30px;
            padding: 20px;
            background: #1a1a1a;
            border-radius: 8px;
            border: 1px solid #333;
        }
        .status-row { display: flex; justify-content: space-between; padding: 8px 0; border-bottom: 1px solid #222; }
        .status-row:last-child { border-bottom: none; }
        .status-label { color: #888; }
        .status-value { color: #e0e0e0; font-family: monospace; }

        .progress { display: none; margin-top: 20px; }
        .progress-bar {
            height: 8px; background: #333; border-radius: 4px; overflow: hidden;
        }
        .progress-fill {
            height: 100%; background: #ff6b6b; width: 0%; transition: width 0.3s;
        }
        .progress-text { text-align: center; margin-top: 10px; color: #888; font-size: 14px; }

        .success { color: #6f6; }
        .error { color: #f66; }
"""

IMPORT_CONTENT = """
    <div class="container">
        <h1>Import Mode 🧪</h1>
        <p class="subtitle">Upload a neopig archive (.tar.gz or .run) to explore it</p>

        <div class="upload-zone" id="upload-zone" onclick="document.getElementById('file-input').click()">
            <div class="upload-icon">📦</div>
            <div class="upload-text">Drop archive here or click to browse</div>
            <div class="upload-hint">Supports .tar.gz, .tgz, and .run files</div>
            <input type="file" id="file-input" accept=".tar.gz,.tgz,.run">
        </div>

        <div class="progress" id="progress">
            <div class="progress-bar"><div class="progress-fill" id="progress-fill"></div></div>
            <div class="progress-text" id="progress-text">Uploading...</div>
        </div>

        <div class="status" id="status">
            <h3 style="margin-top:0;color:#ff6b6b;">Current Archive</h3>
            <div id="status-content">Loading...</div>
        </div>
    </div>

    <script>
        const zone = document.getElementById('upload-zone');
        const fileInput = document.getElementById('file-input');
        const progress = document.getElementById('progress');
        const progressFill = document.getElementById('progress-fill');
        const progressText = document.getElementById('progress-text');

        // Drag and drop
        zone.addEventListener('dragover', (e) => { e.preventDefault(); zone.classList.add('drag-over'); });
        zone.addEventListener('dragleave', () => zone.classList.remove('drag-over'));
        zone.addEventListener('drop', (e) => {
            e.preventDefault();
            zone.classList.remove('drag-over');
            if (e.dataTransfer.files.length) uploadFile(e.dataTransfer.files[0]);
        });

        fileInput.addEventListener('change', () => {
            if (fileInput.files.length) uploadFile(fileInput.files[0]);
        });

        // Resumable upload configuration
        const CHUNK_SIZE = 10 * 1024 * 1024;  // 10MB chunks
        const MAX_RETRIES = 10;
        const RETRY_DELAY = 2000;  // 2 seconds

        let currentUpload = null;  // {uploadId, file, offset}

        async function uploadFile(file) {
            progress.style.display = 'block';
            progressFill.style.width = '0%';
            progressText.textContent = `Initializing upload for ${file.name}...`;

            try {
                // Step 1: Initialize resumable upload
                const initRes = await fetch(`/api/import/upload/init?filename=${encodeURIComponent(file.name)}&size=${file.size}`, {
                    method: 'POST'
                });
                if (!initRes.ok) {
                    const err = await initRes.json();
                    throw new Error(err.detail || 'Failed to initialize upload');
                }
                const init = await initRes.json();
                currentUpload = { uploadId: init.upload_id, file, offset: 0 };

                // Step 2: Upload chunks with resume capability
                await uploadChunks();

            } catch (err) {
                progressText.innerHTML = `<span class="error">✗ ${err.message}</span>`;
            }
        }

        async function uploadChunks() {
            const { uploadId, file } = currentUpload;
            let retries = 0;

            while (currentUpload.offset < file.size) {
                const start = currentUpload.offset;
                const end = Math.min(start + CHUNK_SIZE, file.size);
                const chunk = file.slice(start, end);

                try {
                    const res = await fetch(`/api/import/upload/${uploadId}`, {
                        method: 'PATCH',
                        headers: {
                            'Content-Type': 'application/octet-stream',
                            'X-Upload-Offset': start.toString()
                        },
                        body: chunk
                    });

                    if (!res.ok) {
                        if (res.status === 409) {
                            // Offset mismatch - get actual position and retry
                            const status = await getUploadStatus(uploadId);
                            currentUpload.offset = status.bytes_received;
                            continue;
                        }
                        const err = await res.json();
                        throw new Error(err.detail || 'Chunk upload failed');
                    }

                    const data = await res.json();
                    currentUpload.offset = data.bytes_received;
                    retries = 0;  // Reset retry counter on success

                    // Update progress
                    const pct = (data.bytes_received / file.size * 100).toFixed(1);
                    progressFill.style.width = pct + '%';
                    const mb = (data.bytes_received / 1024 / 1024).toFixed(1);
                    const totalMb = (file.size / 1024 / 1024).toFixed(1);
                    progressText.textContent = `Uploading ${file.name}... ${pct}% (${mb}/${totalMb} MB)`;

                } catch (err) {
                    retries++;
                    if (retries > MAX_RETRIES) {
                        progressText.innerHTML = `<span class="error">✗ Upload failed after ${MAX_RETRIES} retries: ${err.message}</span>`;
                        return;
                    }
                    progressText.textContent = `Connection lost, retrying (${retries}/${MAX_RETRIES})...`;
                    await new Promise(r => setTimeout(r, RETRY_DELAY));

                    // Get current position from server before retry
                    try {
                        const status = await getUploadStatus(uploadId);
                        currentUpload.offset = status.bytes_received;
                    } catch (e) {
                        // If status check fails, just retry from last known position
                    }
                }
            }

            // Step 3: Complete upload
            try {
                const completeRes = await fetch(`/api/import/upload/${uploadId}/complete`, { method: 'POST' });
                if (!completeRes.ok) {
                    const err = await completeRes.json();
                    throw new Error(err.detail || 'Failed to complete upload');
                }
                const result = await completeRes.json();
                progressText.innerHTML = `<span class="success">✓ Archive loaded successfully! Job #${result.job_id}</span>`;
                currentUpload = null;
                loadStatus();
            } catch (err) {
                progressText.innerHTML = `<span class="error">✗ ${err.message}</span>`;
            }
        }

        async function getUploadStatus(uploadId) {
            const res = await fetch(`/api/import/upload/${uploadId}/status`);
            if (!res.ok) throw new Error('Failed to get upload status');
            return await res.json();
        }

        // Resume button for interrupted uploads
        async function resumeUpload() {
            if (!currentUpload) return;
            progress.style.display = 'block';
            progressText.textContent = 'Resuming upload...';
            await uploadChunks();
        }

        async function loadStatus() {
            try {
                const res = await fetch('/api/import/status');
                const data = await res.json();
                const content = document.getElementById('status-content');

                content.innerHTML = `
                    <div class="status-row"><span class="status-label">Media in database</span><span class="status-value">${(data.media_count || 0).toLocaleString()}</span></div>
                    <div class="status-row"><span class="status-label">Pages indexed</span><span class="status-value">${(data.pages_count || 0).toLocaleString()}</span></div>
                    <div class="status-row"><span class="status-label">Recent imports</span><span class="status-value">${data.recent_imports || 0}</span></div>
                    <div class="status-row"><a href="/crawl" style="color:#ff6b6b;">View all jobs →</a></div>
                `;
            } catch (err) {
                document.getElementById('status-content').innerHTML = '<div class="error">Failed to load status</div>';
            }
        }

        loadStatus();
    </script>
"""


@app.get("/import", response_class=HTMLResponse)
async def import_page(lang: str = Cookie(None), accept_language: str = Header(None)):
    """Import page for uploading archives."""
    if not IMPORT_MODE:
        raise HTTPException(status_code=403, detail="Import mode not enabled (NEOPIG_IMPORT=1)")
    language = get_lang(lang, accept_language)
    html = layout("Import", IMPORT_CONTENT, extra_css=IMPORT_CSS)
    return inject_i18n(html, language)


# ============================================================================
# Crawler API
# ============================================================================

@app.get("/api/crawl/jobs")
async def get_crawl_jobs_endpoint(limit: int = Query(50, le=200)):
    """Get recent jobs (crawl + backfill)."""
    try:
        # Get both crawl jobs and backfill jobs
        crawl_jobs = await db.get_crawl_jobs(limit)
        backfill_jobs = await db.get_backfill_jobs(limit)

        # Add job_kind to distinguish them
        for job in crawl_jobs:
            job['job_kind'] = 'crawl'
        for job in backfill_jobs:
            job['job_kind'] = 'backfill'

        # Merge and sort by started_at descending
        all_jobs = crawl_jobs + backfill_jobs
        all_jobs.sort(key=lambda j: j.get('started_at', ''), reverse=True)

        return all_jobs[:limit]
    except Exception as e:
        logger.error(f"Failed to get jobs: {e}")
        raise HTTPException(status_code=500, detail=str(e))


@app.post("/api/crawl/jobs/{job_id}/pause")
async def pause_crawl_job(job_id: int):
    """Pause a running crawl job."""
    task = ACTIVE_CRAWL_TASKS.get(job_id)
    if not task:
        raise HTTPException(status_code=404, detail="Job not running")

    task.cancel()
    try:
        await task
    except asyncio.CancelledError:
        pass

    await db.pause_crawl_job(job_id)
    ACTIVE_CRAWL_TASKS.pop(job_id, None)

    return {"status": "paused", "job_id": job_id}


@app.post("/api/crawl/jobs/{job_id}/resume")
async def resume_crawl_job(job_id: int):
    """Resume a paused crawl job."""
    job = await db.get_crawl_job(job_id)
    if not job:
        raise HTTPException(status_code=404, detail="Job not found")

    if job['status'] not in ('paused', 'cancelled'):
        raise HTTPException(status_code=400, detail=f"Job is {job['status']}, cannot resume")

    if job_id in ACTIVE_CRAWL_TASKS:
        raise HTTPException(status_code=400, detail="Job is already running")

    # Mark as running
    async with db.session() as session:
        await session.execute(
            text("UPDATE crawl_jobs SET status = 'running' WHERE id = :id"),
            {"id": job_id}
        )
        await session.commit()

    # Start crawl task (resume from state files with original settings)
    target_uri = job['target_uri']
    keywords = json.loads(job.get('keywords') or '[]')
    mode_str = job.get('mode', 'all')
    # Use stored settings (with fallbacks for old jobs)
    job_depth = job.get('depth', 15) or 15
    job_max_pages = job.get('max_pages', -1) if job.get('max_pages') is not None else -1
    job_fast = bool(job.get('fast', 0))
    job_screenshots = bool(job.get('screenshots', 1))

    async def run_crawl(jid=job_id, uri=target_uri):
        try:
            from neopig import NeoPig
            from async_web_fetcher import CrawlMode
            from screenshot import ScreenshotConfig
            screenshot_config = ScreenshotConfig(enabled=job_screenshots)
            pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH), fast_mode=job_fast, screenshot_config=screenshot_config)
            await pig.init()

            # Resume from state (don't clear state files)
            crawled_media = await pig.db.get_crawled_media_uris()
            crawled_screenshots = await pig.db.get_crawled_screenshot_uris()
            if crawled_media:
                pig.seen_media = crawled_media
            if crawled_screenshots:
                pig.seen_screenshots = crawled_screenshots

            mode = CrawlMode(mode_str) if mode_str else CrawlMode.ALL

            # Progress updater
            stop_progress = asyncio.Event()

            async def update_progress():
                while not stop_progress.is_set():
                    await asyncio.sleep(2)
                    if not stop_progress.is_set():
                        await db.update_crawl_job_stats(jid, pig.stats)

            progress_task = asyncio.create_task(update_progress())

            try:
                stats = await pig.crawl(
                    target_uri=uri,
                    keywords=keywords,
                    mode=mode,
                    depth=job_depth,
                    max_pages=job_max_pages,
                    job_id=jid,
                    quiet=True,
                )
            finally:
                stop_progress.set()
                progress_task.cancel()
                try:
                    await progress_task
                except asyncio.CancelledError:
                    pass

            await db.complete_crawl_job(jid, stats)

        except Exception as e:
            logger.error(f"Resumed crawl job {jid} failed: {e}")
            await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})

    task = asyncio.create_task(run_crawl())
    ACTIVE_CRAWL_TASKS[job_id] = task
    task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))

    return {"status": "running", "job_id": job_id}


@app.delete("/api/crawl/jobs/{job_id}")
async def delete_crawl_job(job_id: int, purge: bool = Query(True, description="Purge all data (media, screenshots, pages)")):
    """Delete a crawl job and optionally purge all associated data.

    With purge=True (default):
    - Deletes all MediaSource records for this job
    - Deletes all Page records for this job
    - Deletes orphan Media records (not referenced by other jobs)
    - Deletes orphan media files from vault
    - Deletes screenshot files for deleted pages
    - Deletes state files
    """
    # Can't delete running jobs
    if job_id in ACTIVE_CRAWL_TASKS:
        raise HTTPException(status_code=400, detail="Cannot delete running job. Pause it first.")

    if purge:
        # Full purge using NeoPig
        from neopig import NeoPig
        pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH))
        await pig.init()
        result = await pig.purge_job(job_id)
        if not result['deleted']:
            raise HTTPException(status_code=404, detail="Job not found")
        return {
            "status": "purged",
            "job_id": job_id,
            "media_files_deleted": result['media_files_deleted'],
            "screenshots_deleted": result['screenshots_deleted'],
            "pages_deleted": result['pages_deleted'],
            "state_files_deleted": result['state_files_deleted'],
        }
    else:
        # Just delete job record
        result = await db.delete_crawl_job(job_id, purge_data=False)
        if not result['deleted']:
            raise HTTPException(status_code=404, detail="Job not found")
        return {"status": "deleted", "job_id": job_id}


@app.get("/api/crawl/jobs/{job_id}")
async def get_crawl_job_endpoint(job_id: int):
    """Get a specific crawl job."""
    job = await db.get_crawl_job(job_id)
    if not job:
        raise HTTPException(status_code=404, detail="Job not found")
    return job


@app.get("/api/crawl/jobs/{job_id}/logs")
async def get_crawl_job_logs(job_id: int, tail: int = Query(0, description="Return only last N lines")):
    """Get logs for a crawl job."""
    logs = get_job_logs(job_id, tail=tail)
    return Response(content=logs, media_type="text/plain")


@app.post("/api/crawl")
async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
    """
    Start new crawl job(s).

    Supports multiple targets - creates one job per target.
    The crawls run in the background. Poll /api/crawl/jobs/{id} for status.
    """
    if CRAWL_DISABLED:
        raise HTTPException(status_code=403, detail="Crawling is disabled (NEOPIG_DISABLE_CRAWL=1)")

    # Import neopig here to avoid circular imports
    from neopig import NeoPig
    from async_web_fetcher import CrawlMode

    # Map mode string to enum
    mode_map = {
        "text": CrawlMode.TEXT,
        "images": CrawlMode.IMAGES,
        "videos": CrawlMode.VIDEOS,
        "media": CrawlMode.MEDIA,
        "all": CrawlMode.ALL,
    }
    mode = mode_map.get(request.mode, CrawlMode.IMAGES)

    # Get targets (support both new 'targets' array and old 'target_uri' single value)
    targets = request.targets if request.targets else [request.target_uri] if request.target_uri else []
    if not targets:
        raise HTTPException(status_code=400, detail="No target URIs provided")

    job_ids = []

    # Create a job for each target (store all settings for resume)
    depth = request.depth if 1 <= request.depth <= 15 else 15
    for target_uri in targets:
        job_id = await db.create_crawl_job(
            target_uri, request.keywords, request.mode,
            depth=depth, max_pages=request.max_pages,
            fast=request.fast, screenshots=request.screenshots
        )
        job_ids.append(job_id)

        # Run crawl in background - capture ALL values to avoid closure issues
        is_fresh = request.fresh
        is_fast = request.fast
        has_screenshots = request.screenshots
        is_hydra = request.hydra
        crawl_keywords = list(request.keywords)  # Copy list
        crawl_max_pages = request.max_pages
        crawl_depth = request.depth if 1 <= request.depth <= 15 else 15

        async def run_crawl(jid=job_id, uri=target_uri, fresh=is_fresh, fast=is_fast, screenshots=has_screenshots, hydra=is_hydra, depth=crawl_depth, keywords=crawl_keywords, max_pages=crawl_max_pages):
            try:
                from screenshot import ScreenshotConfig
                screenshot_config = ScreenshotConfig(enabled=screenshots)
                pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH), fast_mode=fast, screenshot_config=screenshot_config)
                await pig.init()

                # Handle fresh vs resume - NEVER fresh on resume
                if fresh:
                    pig._clear_state(uri)
                else:
                    # Load existing seen media for resume capability
                    crawled_media = await pig.db.get_crawled_media_uris()
                    crawled_screenshots = await pig.db.get_crawled_screenshot_uris()
                    if crawled_media:
                        pig.seen_media = crawled_media
                    if crawled_screenshots:
                        pig.seen_screenshots = crawled_screenshots

                # Progress updater - runs alongside crawl
                stop_progress = asyncio.Event()

                async def update_progress():
                    while not stop_progress.is_set():
                        await asyncio.sleep(2)  # Update every 2 seconds
                        if not stop_progress.is_set():
                            await db.update_crawl_job_stats(jid, pig.stats)

                progress_task = asyncio.create_task(update_progress())

                try:
                    stats = await pig.crawl(
                        target_uri=uri,
                        keywords=keywords,
                        mode=mode,
                        depth=depth,
                        max_pages=max_pages,
                        job_id=jid,  # Use existing job, don't create another
                        quiet=True,  # No progress bar for UI-initiated crawls
                        hydra=hydra,  # Feed/sitemap discovery mode
                    )
                finally:
                    stop_progress.set()
                    progress_task.cancel()
                    try:
                        await progress_task
                    except asyncio.CancelledError:
                        pass

                # Update job as completed
                await db.complete_crawl_job(jid, stats)

            except Exception as e:
                logger.error(f"Crawl job {jid} failed: {e}")
                await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})

        # Schedule async task on the event loop (not BackgroundTasks which runs in threadpool)
        task = asyncio.create_task(run_crawl())
        ACTIVE_CRAWL_TASKS[job_id] = task
        # Clean up when done
        task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))

    return {"job_ids": job_ids, "status": "running", "count": len(job_ids)}


def init_tarball_mode(tarball_path: str):
    """Initialize serving from a tar.gz archive."""
    global TAR_PATH, TAR_OFFSET, TAR_MEMBERS, TAR_MEDIA_INDEX, ARCHIVE_ROOT, DB_PATH, TEMP_DB_PATH

    logger.info(f"Opening archive: {tarball_path}")
    tarball = Path(tarball_path)
    TAR_PATH = str(tarball.resolve())

    # Handle .run files with NEOPIG trailer
    TAR_OFFSET = 0
    if tarball.suffix == '.run' or tarball.stat().st_size > 100000:
        try:
            with open(tarball, 'rb') as f:
                f.seek(-22, 2)
                trailer = f.read(22)
                if trailer[:6] == b'NEOPIG':
                    TAR_OFFSET = int(trailer[6:22].decode(), 16)
                    logger.info(f"Detected .run format, tarball offset: {TAR_OFFSET}")
        except Exception:
            pass

    # Open tarball temporarily to build index
    tar = _open_tarball()

    # Build member lookup and media index
    for member in tar.getmembers():
        TAR_MEMBERS[member.name] = member
        # Index media files by hash for O(1) lookup
        name = member.name
        if '/vault/' in name or '/media/' in name:
            # Extract hash from filename (hash.ext)
            basename = Path(name).stem  # removes extension
            if len(basename) == 32 and all(c in '0123456789abcdef' for c in basename):
                TAR_MEDIA_INDEX[basename] = name

    logger.info(f"Indexed {len(TAR_MEDIA_INDEX)} media files")

    # Get archive root from first member
    first = list(TAR_MEMBERS.keys())[0]
    ARCHIVE_ROOT = first.split('/')[0]
    logger.info(f"Archive root: {ARCHIVE_ROOT}")

    # Extract database to temp (SQLite needs real file)
    # Try neopig.db first (new format), then archive.db (legacy)
    db_member = f"{ARCHIVE_ROOT}/neopig.db"
    if db_member not in TAR_MEMBERS:
        db_member = f"{ARCHIVE_ROOT}/archive.db"
    if db_member in TAR_MEMBERS:
        temp_dir = tempfile.mkdtemp(prefix="neopig_")
        db_name = Path(db_member).name
        TEMP_DB_PATH = f"{temp_dir}/{db_name}"
        member = TAR_MEMBERS[db_member]
        f = tar.extractfile(member)
        if f:
            with open(TEMP_DB_PATH, 'wb') as out:
                out.write(f.read())
            DB_PATH = TEMP_DB_PATH
            logger.info(f"Extracted database to: {TEMP_DB_PATH}")
    else:
        logger.warning("No neopig.db or archive.db found in tarball")

    # Close the temporary tar handle (requests will open their own)
    tar.close()


def main():
    global DB_PATH, VAULT_PATH

    parser = argparse.ArgumentParser(description="neopig SERP")
    parser.add_argument("tarball", nargs='?', help="Path to archive.tar.gz or .run file")
    parser.add_argument("--port", type=int, default=31337)
    parser.add_argument("--host", default="127.0.0.1")
    parser.add_argument("--db", default="data/neopig.db")
    parser.add_argument("--vault", default="data/vault")

    args = parser.parse_args()

    # Tarball mode
    if args.tarball:
        init_tarball_mode(args.tarball)
        logger.info(f"Starting neopig SERP (archive mode) on {args.host}:{args.port}")
    else:
        DB_PATH = args.db
        VAULT_PATH = Path(args.vault)
        logger.info(f"Starting neopig SERP on {args.host}:{args.port}")
        logger.info(f"Database: {DB_PATH}")
        logger.info(f"Vault: {VAULT_PATH}")

    uvicorn.run(app, host=args.host, port=args.port)


if __name__ == "__main__":
    main()
