Russell Ballestrini
11f8fc59ae
modified: serp.py new file: templates/about.html.j2 new file: templates/base.html.j2 new file: templates/crawl.html.j2 new file: templates/import.html.j2 new file: templates/live.html.j2 new file: templates/partials/search_box.html.j2 new file: templates/phantom.html.j2 new file: templates/search.html.j2 new file: templates/view.html.j2
2925 lines
165 KiB
Python
2925 lines
165 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
neopig SERP - Search Engine Results Page
|
||
|
||
Fast image/video search across hydrated metadata.
|
||
Serves files directly from filevault via Caddy with 1GB memory cache.
|
||
|
||
Search across:
|
||
- keywords (crawl tags)
|
||
- alt_text (image alt attributes)
|
||
- title (media titles)
|
||
- analysis_result (Qwen 3 VL descriptions)
|
||
- source_page / source_url (origin)
|
||
|
||
Usage:
|
||
python serp.py --port 31337 --vault ./vault --db neopig.db
|
||
"""
|
||
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import mimetypes
|
||
import os
|
||
from pathlib import Path
|
||
from typing import List, Dict, Any, Optional
|
||
|
||
from fastapi import FastAPI, Query, HTTPException, BackgroundTasks, Header, Cookie, UploadFile, File, Request
|
||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response, StreamingResponse
|
||
from fastapi.staticfiles import StaticFiles
|
||
from fastapi.templating import Jinja2Templates
|
||
from pydantic import BaseModel
|
||
from sqlalchemy import text
|
||
import uvicorn
|
||
|
||
from database import Database, OVER_9000
|
||
from filevault import hash_to_path
|
||
from miniuri import Uri
|
||
from neopig import get_live_queue
|
||
|
||
logging.basicConfig(level=logging.INFO)
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# ============================================================================
|
||
# INTERNATIONALIZATION (i18n) - Top 10 Languages
|
||
# ============================================================================
|
||
TRANSLATIONS = {
|
||
"en": {
|
||
"search": "Search", "crawl": "Crawl", "live": "Live Feed", "about": "About",
|
||
"random": "Random", "phantom": "Phantom", "loading": "Loading...",
|
||
"start_crawl": "Start Crawl", "pause": "Pause", "resume": "Resume", "delete": "Delete",
|
||
"url": "URL", "mode": "Mode", "depth": "Depth", "keywords": "Keywords",
|
||
"screenshots": "Screenshots", "fast_mode": "Fast Mode", "fresh_start": "Fresh Start",
|
||
"pages": "pages", "found": "found", "saved": "saved", "dupes": "dupes",
|
||
"running": "running", "completed": "completed", "paused": "paused",
|
||
"no_jobs": "No crawl jobs yet", "started": "Started", "filter": "Filter",
|
||
"all_types": "All types", "images": "Images", "videos": "Videos", "audio": "Audio",
|
||
"everything": "Everything (text + media)", "all_media": "All media (images + videos + audio)",
|
||
"images_only": "Images only", "videos_only": "Videos only", "text_only": "Text only",
|
||
"recent_jobs": "Recent Jobs", "search_placeholder": "Search media...",
|
||
"media": "Media", "pages_label": "Pages", "download": "Download",
|
||
"no_media": "No media found", "no_more": "No more results", "load_error": "Failed to load",
|
||
"copy": "Copy", "copied": "Copied!", "starting": "Starting...", "started_ok": "Started!",
|
||
"error": "Error", "watching": "Watching for new images...", "content": "Content",
|
||
"previous": "Previous", "next": "Next", "over_9000": "Over 9,000+!",
|
||
"loading_stats": "Loading stats...", "screenshot": "Screenshot",
|
||
"valid_uri": "Please enter at least one valid URI", "load_jobs_error": "Failed to load jobs",
|
||
"hydrate_subtitle": "Hydrate media from the web", "live_subtitle": "Watch images appear as they're crawled",
|
||
"total_images": "Total Images", "new_session": "New This Session", "per_minute": "Per Minute",
|
||
"all_domains": "All domains", "download_phantom": "Download Phantom Site",
|
||
"phantom_subtitle": "Export archived pages as a static site with local media", "sources": "sources", "max_pages": "Max Pages", "select_domain": "Select Domain",
|
||
"source_uri": "Source URI", "neopig_uri": "Neopig URI", "source_page": "Source Page", "neopig_page": "Neopig Page", "type_label": "Type", "mime_label": "MIME", "size_label": "Size", "alt_label": "Alt", "keywords_label": "Keywords", "description_label": "Description", "items": "items", "bytes": "bytes",
|
||
"used_on": "Used on", "page": "Page", "discovered": "Discovered",
|
||
"console": "Console", "close": "Close", "no_logs": "No logs yet",
|
||
},
|
||
"zh": {
|
||
"search": "搜索", "crawl": "爬取", "live": "实时动态", "about": "关于",
|
||
"random": "随机", "phantom": "幻影", "loading": "加载中...",
|
||
"start_crawl": "开始爬取", "pause": "暂停", "resume": "继续", "delete": "删除",
|
||
"url": "网址", "mode": "模式", "depth": "深度", "keywords": "关键词",
|
||
"screenshots": "截图", "fast_mode": "快速模式", "fresh_start": "全新开始",
|
||
"pages": "页面", "found": "发现", "saved": "保存", "dupes": "重复",
|
||
"running": "运行中", "completed": "已完成", "paused": "已暂停",
|
||
"no_jobs": "暂无爬取任务", "started": "开始于", "filter": "筛选",
|
||
"all_types": "所有类型", "images": "图片", "videos": "视频", "audio": "音频",
|
||
"everything": "全部（文本+媒体）", "all_media": "所有媒体（图片+视频+音频）",
|
||
"images_only": "仅图片", "videos_only": "仅视频", "text_only": "仅文本",
|
||
"recent_jobs": "最近任务", "search_placeholder": "搜索媒体...",
|
||
"media": "媒体", "pages_label": "页面", "download": "下载",
|
||
"no_media": "未找到媒体", "no_more": "没有更多结果", "load_error": "加载失败",
|
||
"copy": "复制", "copied": "已复制!", "starting": "启动中...", "started_ok": "已启动!",
|
||
"error": "错误", "watching": "正在监视新图片...", "content": "内容",
|
||
"previous": "上一页", "next": "下一页", "over_9000": "超过9000+!",
|
||
"loading_stats": "加载统计中...", "screenshot": "截图",
|
||
"valid_uri": "请输入至少一个有效的URI", "load_jobs_error": "加载任务失败",
|
||
"hydrate_subtitle": "从网络获取媒体", "live_subtitle": "观看正在爬取的图片",
|
||
"total_images": "总图片数", "new_session": "本次新增", "per_minute": "每分钟",
|
||
"all_domains": "所有域名", "download_phantom": "下载幻影站点",
|
||
"phantom_subtitle": "将存档页面导出为带本地媒体的静态站点", "max_pages": "最大页数", "select_domain": "选择域名", "sources": "来源",
|
||
"source_uri": "源URI", "neopig_uri": "Neopig URI", "source_page": "源页面", "neopig_page": "Neopig页面",
|
||
"type_label": "类型", "mime_label": "MIME", "size_label": "大小", "alt_label": "替代文本",
|
||
"keywords_label": "关键词", "description_label": "描述", "items": "项", "bytes": "字节",
|
||
"used_on": "使用于", "page": "页面", "discovered": "发现时间",
|
||
"console": "控制台", "close": "关闭", "no_logs": "暂无日志",
|
||
},
|
||
"es": {
|
||
"search": "Buscar", "crawl": "Rastrear", "live": "En Vivo", "about": "Acerca de",
|
||
"random": "Aleatorio", "phantom": "Fantasma", "loading": "Cargando...",
|
||
"start_crawl": "Iniciar Rastreo", "pause": "Pausar", "resume": "Reanudar", "delete": "Eliminar",
|
||
"url": "URL", "mode": "Modo", "depth": "Profundidad", "keywords": "Palabras clave",
|
||
"screenshots": "Capturas", "fast_mode": "Modo Rápido", "fresh_start": "Inicio Limpio",
|
||
"pages": "páginas", "found": "encontradas", "saved": "guardadas", "dupes": "duplicados",
|
||
"running": "ejecutando", "completed": "completado", "paused": "pausado",
|
||
"no_jobs": "Sin tareas de rastreo", "started": "Iniciado", "filter": "Filtrar",
|
||
"all_types": "Todos los tipos", "images": "Imágenes", "videos": "Videos", "audio": "Audio",
|
||
"everything": "Todo (texto + medios)", "all_media": "Todos los medios",
|
||
"images_only": "Solo imágenes", "videos_only": "Solo videos", "text_only": "Solo texto",
|
||
"recent_jobs": "Tareas Recientes", "search_placeholder": "Buscar medios...",
|
||
"media": "Medios", "pages_label": "Páginas", "download": "Descargar",
|
||
"no_media": "No se encontraron medios", "no_more": "No hay más resultados", "load_error": "Error al cargar",
|
||
"copy": "Copiar", "copied": "¡Copiado!", "starting": "Iniciando...", "started_ok": "¡Iniciado!",
|
||
"error": "Error", "watching": "Buscando nuevas imágenes...", "content": "Contenido",
|
||
"previous": "Anterior", "next": "Siguiente", "over_9000": "¡Más de 9,000+!",
|
||
"loading_stats": "Cargando estadísticas...", "screenshot": "Captura",
|
||
"valid_uri": "Ingrese al menos una URI válida", "load_jobs_error": "Error al cargar tareas",
|
||
"hydrate_subtitle": "Obtener medios de la web", "live_subtitle": "Ver imágenes mientras se rastrean",
|
||
"total_images": "Total de imágenes", "new_session": "Nuevas en esta sesión", "per_minute": "Por minuto",
|
||
"all_domains": "Todos los dominios", "download_phantom": "Descargar sitio fantasma",
|
||
"phantom_subtitle": "Exportar páginas archivadas como sitio estático con medios locales", "max_pages": "Máx. páginas", "select_domain": "Seleccionar dominio", "sources": "fuentes",
|
||
"source_uri": "URI de origen", "neopig_uri": "URI Neopig", "source_page": "Página de origen", "neopig_page": "Página Neopig",
|
||
"type_label": "Tipo", "mime_label": "MIME", "size_label": "Tamaño", "alt_label": "Alt",
|
||
"keywords_label": "Palabras clave", "description_label": "Descripción", "items": "elementos", "bytes": "bytes",
|
||
"used_on": "Usado en", "page": "Página", "discovered": "Descubierto",
|
||
"console": "Consola", "close": "Cerrar", "no_logs": "Sin registros aún",
|
||
},
|
||
"hi": {
|
||
"search": "खोजें", "crawl": "क्रॉल", "live": "लाइव फ़ीड", "about": "के बारे में",
|
||
"random": "यादृच्छिक", "phantom": "प्रेत", "loading": "लोड हो रहा है...",
|
||
"start_crawl": "क्रॉल शुरू करें", "pause": "रोकें", "resume": "जारी रखें", "delete": "हटाएं",
|
||
"url": "यूआरएल", "mode": "मोड", "depth": "गहराई", "keywords": "कीवर्ड",
|
||
"screenshots": "स्क्रीनशॉट", "fast_mode": "फास्ट मोड", "fresh_start": "नई शुरुआत",
|
||
"pages": "पेज", "found": "मिले", "saved": "सहेजे", "dupes": "डुप्लीकेट",
|
||
"running": "चल रहा है", "completed": "पूर्ण", "paused": "रुका हुआ",
|
||
"no_jobs": "कोई क्रॉल कार्य नहीं", "started": "शुरू", "filter": "फ़िल्टर",
|
||
"all_types": "सभी प्रकार", "images": "चित्र", "videos": "वीडियो", "audio": "ऑडियो",
|
||
"everything": "सब कुछ", "all_media": "सभी मीडिया", "images_only": "केवल चित्र",
|
||
"videos_only": "केवल वीडियो", "text_only": "केवल टेक्स्ट",
|
||
"recent_jobs": "हाल के कार्य", "search_placeholder": "मीडिया खोजें...",
|
||
"media": "मीडिया", "pages_label": "पेज", "download": "डाउनलोड",
|
||
"no_media": "कोई मीडिया नहीं मिला", "no_more": "और कोई परिणाम नहीं", "load_error": "लोड करने में विफल",
|
||
"copy": "कॉपी", "copied": "कॉपी हो गया!", "starting": "शुरू हो रहा...", "started_ok": "शुरू हो गया!",
|
||
"error": "त्रुटि", "watching": "नई छवियों की तलाश...", "content": "सामग्री",
|
||
"previous": "पिछला", "next": "अगला", "over_9000": "9,000+ से अधिक!",
|
||
"loading_stats": "आँकड़े लोड हो रहे...", "screenshot": "स्क्रीनशॉट",
|
||
"valid_uri": "कृपया कम से कम एक वैध URI दर्ज करें", "load_jobs_error": "कार्य लोड करने में विफल",
|
||
"hydrate_subtitle": "वेब से मीडिया प्राप्त करें", "live_subtitle": "क्रॉल होते हुए छवियां देखें",
|
||
"total_images": "कुल छवियां", "new_session": "इस सत्र में नई", "per_minute": "प्रति मिनट",
|
||
"all_domains": "सभी डोमेन", "download_phantom": "प्रेत साइट डाउनलोड करें",
|
||
"phantom_subtitle": "संग्रहित पृष्ठों को स्थानीय मीडिया के साथ स्थैतिक साइट के रूप में निर्यात करें", "max_pages": "अधिकतम पेज", "select_domain": "डोमेन चुनें", "sources": "स्रोत",
|
||
"source_uri": "स्रोत URI", "neopig_uri": "Neopig URI", "source_page": "स्रोत पृष्ठ", "neopig_page": "Neopig पृष्ठ",
|
||
"type_label": "प्रकार", "mime_label": "MIME", "size_label": "आकार", "alt_label": "Alt",
|
||
"keywords_label": "कीवर्ड", "description_label": "विवरण", "items": "आइटम", "bytes": "बाइट्स",
|
||
"used_on": "में उपयोग", "page": "पृष्ठ", "discovered": "खोजा गया",
|
||
"console": "कंसोल", "close": "बंद करें", "no_logs": "अभी तक कोई लॉग नहीं",
|
||
},
|
||
"ar": {
|
||
"search": "بحث", "crawl": "زحف", "live": "مباشر", "about": "حول",
|
||
"random": "عشوائي", "phantom": "شبح", "loading": "جاري التحميل...",
|
||
"start_crawl": "بدء الزحف", "pause": "إيقاف", "resume": "استئناف", "delete": "حذف",
|
||
"url": "الرابط", "mode": "الوضع", "depth": "العمق", "keywords": "كلمات مفتاحية",
|
||
"screenshots": "لقطات", "fast_mode": "وضع سريع", "fresh_start": "بداية جديدة",
|
||
"pages": "صفحات", "found": "وجدت", "saved": "حفظت", "dupes": "مكررات",
|
||
"running": "جاري", "completed": "مكتمل", "paused": "متوقف",
|
||
"no_jobs": "لا توجد مهام", "started": "بدأ", "filter": "تصفية",
|
||
"all_types": "جميع الأنواع", "images": "صور", "videos": "فيديو", "audio": "صوت",
|
||
"everything": "الكل", "all_media": "جميع الوسائط", "images_only": "صور فقط",
|
||
"videos_only": "فيديو فقط", "text_only": "نص فقط",
|
||
"recent_jobs": "المهام الأخيرة", "search_placeholder": "بحث في الوسائط...",
|
||
"media": "وسائط", "pages_label": "صفحات", "download": "تحميل",
|
||
"no_media": "لم يتم العثور على وسائط", "no_more": "لا مزيد من النتائج", "load_error": "فشل التحميل",
|
||
"copy": "نسخ", "copied": "تم النسخ!", "starting": "جاري البدء...", "started_ok": "تم البدء!",
|
||
"error": "خطأ", "watching": "مراقبة الصور الجديدة...", "content": "المحتوى",
|
||
"previous": "السابق", "next": "التالي", "over_9000": "أكثر من 9,000+!",
|
||
"loading_stats": "جاري تحميل الإحصائيات...", "screenshot": "لقطة شاشة",
|
||
"valid_uri": "الرجاء إدخال رابط واحد على الأقل", "load_jobs_error": "فشل تحميل المهام",
|
||
"hydrate_subtitle": "جلب الوسائط من الويب", "live_subtitle": "مشاهدة الصور أثناء الزحف",
|
||
"total_images": "إجمالي الصور", "new_session": "جديدة في هذه الجلسة", "per_minute": "في الدقيقة",
|
||
"all_domains": "جميع النطاقات", "download_phantom": "تحميل موقع الشبح",
|
||
"phantom_subtitle": "تصدير الصفحات المؤرشفة كموقع ثابت مع وسائط محلية", "max_pages": "أقصى عدد صفحات", "select_domain": "اختر النطاق", "sources": "مصادر",
|
||
"source_uri": "URI المصدر", "neopig_uri": "URI Neopig", "source_page": "صفحة المصدر", "neopig_page": "صفحة Neopig",
|
||
"type_label": "النوع", "mime_label": "MIME", "size_label": "الحجم", "alt_label": "Alt",
|
||
"keywords_label": "كلمات مفتاحية", "description_label": "الوصف", "items": "عناصر", "bytes": "بايت",
|
||
"used_on": "مستخدم في", "page": "صفحة", "discovered": "اكتشف",
|
||
"console": "وحدة التحكم", "close": "إغلاق", "no_logs": "لا توجد سجلات بعد",
|
||
},
|
||
"pt": {
|
||
"search": "Pesquisar", "crawl": "Rastrear", "live": "Ao Vivo", "about": "Sobre",
|
||
"random": "Aleatório", "phantom": "Fantasma", "loading": "Carregando...",
|
||
"start_crawl": "Iniciar Rastreio", "pause": "Pausar", "resume": "Retomar", "delete": "Excluir",
|
||
"url": "URL", "mode": "Modo", "depth": "Profundidade", "keywords": "Palavras-chave",
|
||
"screenshots": "Capturas", "fast_mode": "Modo Rápido", "fresh_start": "Novo Início",
|
||
"pages": "páginas", "found": "encontradas", "saved": "salvas", "dupes": "duplicatas",
|
||
"running": "executando", "completed": "concluído", "paused": "pausado",
|
||
"no_jobs": "Sem tarefas", "started": "Iniciado", "filter": "Filtrar",
|
||
"all_types": "Todos os tipos", "images": "Imagens", "videos": "Vídeos", "audio": "Áudio",
|
||
"everything": "Tudo (texto + mídia)", "all_media": "Toda mídia",
|
||
"images_only": "Apenas imagens", "videos_only": "Apenas vídeos", "text_only": "Apenas texto",
|
||
"recent_jobs": "Tarefas Recentes", "search_placeholder": "Pesquisar mídia...",
|
||
"media": "Mídia", "pages_label": "Páginas", "download": "Baixar",
|
||
"no_media": "Nenhuma mídia encontrada", "no_more": "Sem mais resultados", "load_error": "Falha ao carregar",
|
||
"copy": "Copiar", "copied": "Copiado!", "starting": "Iniciando...", "started_ok": "Iniciado!",
|
||
"error": "Erro", "watching": "Observando novas imagens...", "content": "Conteúdo",
|
||
"previous": "Anterior", "next": "Próximo", "over_9000": "Mais de 9.000+!",
|
||
"loading_stats": "Carregando estatísticas...", "screenshot": "Captura de tela",
|
||
"valid_uri": "Insira pelo menos uma URI válida", "load_jobs_error": "Falha ao carregar tarefas",
|
||
"hydrate_subtitle": "Obter mídia da web", "live_subtitle": "Veja imagens aparecerem durante o rastreio",
|
||
"total_images": "Total de imagens", "new_session": "Novas nesta sessão", "per_minute": "Por minuto",
|
||
"all_domains": "Todos os domínios", "download_phantom": "Baixar site fantasma",
|
||
"phantom_subtitle": "Exportar páginas arquivadas como site estático com mídia local", "max_pages": "Máx. páginas", "select_domain": "Selecionar domínio", "sources": "fontes",
|
||
"source_uri": "URI de origem", "neopig_uri": "URI Neopig", "source_page": "Página de origem", "neopig_page": "Página Neopig",
|
||
"type_label": "Tipo", "mime_label": "MIME", "size_label": "Tamanho", "alt_label": "Alt",
|
||
"keywords_label": "Palavras-chave", "description_label": "Descrição", "items": "itens", "bytes": "bytes",
|
||
"used_on": "Usado em", "page": "Página", "discovered": "Descoberto",
|
||
"console": "Console", "close": "Fechar", "no_logs": "Sem logs ainda",
|
||
},
|
||
"ru": {
|
||
"search": "Поиск", "crawl": "Сканировать", "live": "Прямой эфир", "about": "О нас",
|
||
"random": "Случайное", "phantom": "Призрак", "loading": "Загрузка...",
|
||
"start_crawl": "Начать сканирование", "pause": "Пауза", "resume": "Продолжить", "delete": "Удалить",
|
||
"url": "URL", "mode": "Режим", "depth": "Глубина", "keywords": "Ключевые слова",
|
||
"screenshots": "Скриншоты", "fast_mode": "Быстрый режим", "fresh_start": "Начать заново",
|
||
"pages": "страниц", "found": "найдено", "saved": "сохранено", "dupes": "дубликаты",
|
||
"running": "выполняется", "completed": "завершено", "paused": "приостановлено",
|
||
"no_jobs": "Нет задач", "started": "Начало", "filter": "Фильтр",
|
||
"all_types": "Все типы", "images": "Изображения", "videos": "Видео", "audio": "Аудио",
|
||
"everything": "Всё (текст + медиа)", "all_media": "Все медиа",
|
||
"images_only": "Только изображения", "videos_only": "Только видео", "text_only": "Только текст",
|
||
"recent_jobs": "Недавние задачи", "search_placeholder": "Поиск медиа...",
|
||
"media": "Медиа", "pages_label": "Страницы", "download": "Скачать",
|
||
"no_media": "Медиа не найдено", "no_more": "Больше нет результатов", "load_error": "Ошибка загрузки",
|
||
"copy": "Копировать", "copied": "Скопировано!", "starting": "Запуск...", "started_ok": "Запущено!",
|
||
"error": "Ошибка", "watching": "Отслеживание новых изображений...", "content": "Контент",
|
||
"previous": "Назад", "next": "Далее", "over_9000": "Более 9000+!",
|
||
"loading_stats": "Загрузка статистики...", "screenshot": "Скриншот",
|
||
"valid_uri": "Введите хотя бы один URL", "load_jobs_error": "Не удалось загрузить задачи",
|
||
"hydrate_subtitle": "Получить медиа из интернета", "live_subtitle": "Смотрите появление изображений",
|
||
"total_images": "Всего изображений", "new_session": "Новые в сессии", "per_minute": "В минуту",
|
||
"all_domains": "Все домены", "download_phantom": "Скачать призрачный сайт",
|
||
"phantom_subtitle": "Экспорт архивных страниц как статический сайт с локальными медиа", "max_pages": "Макс. страниц", "select_domain": "Выбрать домен", "sources": "источников",
|
||
"source_uri": "URI источника", "neopig_uri": "URI Neopig", "source_page": "Страница источника", "neopig_page": "Страница Neopig",
|
||
"type_label": "Тип", "mime_label": "MIME", "size_label": "Размер", "alt_label": "Alt",
|
||
"keywords_label": "Ключевые слова", "description_label": "Описание", "items": "элементов", "bytes": "байт",
|
||
"used_on": "Используется на", "page": "Страница", "discovered": "Обнаружено",
|
||
"console": "Консоль", "close": "Закрыть", "no_logs": "Пока нет логов",
|
||
},
|
||
"ja": {
|
||
"search": "検索", "crawl": "クロール", "live": "ライブ", "about": "について",
|
||
"random": "ランダム", "phantom": "ファントム", "loading": "読み込み中...",
|
||
"start_crawl": "クロール開始", "pause": "一時停止", "resume": "再開", "delete": "削除",
|
||
"url": "URL", "mode": "モード", "depth": "深さ", "keywords": "キーワード",
|
||
"screenshots": "スクリーンショット", "fast_mode": "高速モード", "fresh_start": "新規開始",
|
||
"pages": "ページ", "found": "発見", "saved": "保存", "dupes": "重複",
|
||
"running": "実行中", "completed": "完了", "paused": "一時停止中",
|
||
"no_jobs": "ジョブがありません", "started": "開始", "filter": "フィルター",
|
||
"all_types": "すべてのタイプ", "images": "画像", "videos": "動画", "audio": "オーディオ",
|
||
"everything": "すべて（テキスト＋メディア）", "all_media": "すべてのメディア",
|
||
"images_only": "画像のみ", "videos_only": "動画のみ", "text_only": "テキストのみ",
|
||
"recent_jobs": "最近のジョブ", "search_placeholder": "メディアを検索...",
|
||
"media": "メディア", "pages_label": "ページ", "download": "ダウンロード",
|
||
"no_media": "メディアが見つかりません", "no_more": "これ以上の結果はありません", "load_error": "読み込みに失敗",
|
||
"copy": "コピー", "copied": "コピーしました!", "starting": "開始中...", "started_ok": "開始しました!",
|
||
"error": "エラー", "watching": "新しい画像を監視中...", "content": "コンテンツ",
|
||
"previous": "前へ", "next": "次へ", "over_9000": "9000以上!",
|
||
"loading_stats": "統計を読み込み中...", "screenshot": "スクリーンショット",
|
||
"valid_uri": "有効なURIを入力してください", "load_jobs_error": "ジョブの読み込みに失敗",
|
||
"hydrate_subtitle": "ウェブからメディアを取得", "live_subtitle": "クロール中の画像を表示",
|
||
"total_images": "合計画像数", "new_session": "今回のセッション", "per_minute": "毎分",
|
||
"all_domains": "すべてのドメイン", "download_phantom": "ファントムサイトをダウンロード",
|
||
"phantom_subtitle": "アーカイブページをローカルメディア付きの静的サイトとしてエクスポート", "max_pages": "最大ページ数", "select_domain": "ドメインを選択", "sources": "ソース",
|
||
"source_uri": "ソースURI", "neopig_uri": "Neopig URI", "source_page": "ソースページ", "neopig_page": "Neopigページ",
|
||
"type_label": "タイプ", "mime_label": "MIME", "size_label": "サイズ", "alt_label": "Alt",
|
||
"keywords_label": "キーワード", "description_label": "説明", "items": "アイテム", "bytes": "バイト",
|
||
"used_on": "使用場所", "page": "ページ", "discovered": "発見日時",
|
||
"console": "コンソール", "close": "閉じる", "no_logs": "ログがありません",
|
||
},
|
||
"fr": {
|
||
"search": "Rechercher", "crawl": "Explorer", "live": "En Direct", "about": "À propos",
|
||
"random": "Aléatoire", "phantom": "Fantôme", "loading": "Chargement...",
|
||
"start_crawl": "Démarrer", "pause": "Pause", "resume": "Reprendre", "delete": "Supprimer",
|
||
"url": "URL", "mode": "Mode", "depth": "Profondeur", "keywords": "Mots-clés",
|
||
"screenshots": "Captures", "fast_mode": "Mode Rapide", "fresh_start": "Nouveau Départ",
|
||
"pages": "pages", "found": "trouvées", "saved": "enregistrées", "dupes": "doublons",
|
||
"running": "en cours", "completed": "terminé", "paused": "en pause",
|
||
"no_jobs": "Aucune tâche", "started": "Démarré", "filter": "Filtrer",
|
||
"all_types": "Tous les types", "images": "Images", "videos": "Vidéos", "audio": "Audio",
|
||
"everything": "Tout (texte + médias)", "all_media": "Tous les médias",
|
||
"images_only": "Images uniquement", "videos_only": "Vidéos uniquement", "text_only": "Texte uniquement",
|
||
"recent_jobs": "Tâches Récentes", "search_placeholder": "Rechercher des médias...",
|
||
"media": "Médias", "pages_label": "Pages", "download": "Télécharger",
|
||
"no_media": "Aucun média trouvé", "no_more": "Plus de résultats", "load_error": "Échec du chargement",
|
||
"copy": "Copier", "copied": "Copié!", "starting": "Démarrage...", "started_ok": "Démarré!",
|
||
"error": "Erreur", "watching": "Surveillance des nouvelles images...", "content": "Contenu",
|
||
"previous": "Précédent", "next": "Suivant", "over_9000": "Plus de 9 000+!",
|
||
"loading_stats": "Chargement des statistiques...", "screenshot": "Capture d'écran",
|
||
"valid_uri": "Veuillez entrer au moins une URI valide", "load_jobs_error": "Échec du chargement des tâches",
|
||
"hydrate_subtitle": "Récupérer les médias du web", "live_subtitle": "Voir les images apparaître pendant l'exploration",
|
||
"total_images": "Total d'images", "new_session": "Nouvelles cette session", "per_minute": "Par minute",
|
||
"all_domains": "Tous les domaines", "download_phantom": "Télécharger le site fantôme",
|
||
"phantom_subtitle": "Exporter les pages archivées en site statique avec médias locaux", "max_pages": "Pages max", "select_domain": "Sélectionner le domaine", "sources": "sources",
|
||
"source_uri": "URI source", "neopig_uri": "URI Neopig", "source_page": "Page source", "neopig_page": "Page Neopig",
|
||
"type_label": "Type", "mime_label": "MIME", "size_label": "Taille", "alt_label": "Alt",
|
||
"keywords_label": "Mots-clés", "description_label": "Description", "items": "éléments", "bytes": "octets",
|
||
"used_on": "Utilisé sur", "page": "Page", "discovered": "Découvert",
|
||
"console": "Console", "close": "Fermer", "no_logs": "Pas encore de logs",
|
||
},
|
||
"de": {
|
||
"search": "Suchen", "crawl": "Crawlen", "live": "Live-Feed", "about": "Über",
|
||
"random": "Zufällig", "phantom": "Phantom", "loading": "Laden...",
|
||
"start_crawl": "Crawl starten", "pause": "Pause", "resume": "Fortsetzen", "delete": "Löschen",
|
||
"url": "URL", "mode": "Modus", "depth": "Tiefe", "keywords": "Stichwörter",
|
||
"screenshots": "Screenshots", "fast_mode": "Schnellmodus", "fresh_start": "Neustart",
|
||
"pages": "Seiten", "found": "gefunden", "saved": "gespeichert", "dupes": "Duplikate",
|
||
"running": "läuft", "completed": "abgeschlossen", "paused": "pausiert",
|
||
"no_jobs": "Keine Aufgaben", "started": "Gestartet", "filter": "Filter",
|
||
"all_types": "Alle Typen", "images": "Bilder", "videos": "Videos", "audio": "Audio",
|
||
"everything": "Alles (Text + Medien)", "all_media": "Alle Medien",
|
||
"images_only": "Nur Bilder", "videos_only": "Nur Videos", "text_only": "Nur Text",
|
||
"recent_jobs": "Letzte Aufgaben", "search_placeholder": "Medien suchen...",
|
||
"media": "Medien", "pages_label": "Seiten", "download": "Herunterladen",
|
||
"no_media": "Keine Medien gefunden", "no_more": "Keine weiteren Ergebnisse", "load_error": "Laden fehlgeschlagen",
|
||
"copy": "Kopieren", "copied": "Kopiert!", "starting": "Starten...", "started_ok": "Gestartet!",
|
||
"error": "Fehler", "watching": "Überwache neue Bilder...", "content": "Inhalt",
|
||
"previous": "Zurück", "next": "Weiter", "over_9000": "Über 9.000+!",
|
||
"loading_stats": "Lade Statistiken...", "screenshot": "Screenshot",
|
||
"valid_uri": "Mindestens eine gültige URI eingeben", "load_jobs_error": "Aufgaben konnten nicht geladen werden",
|
||
"hydrate_subtitle": "Medien aus dem Web abrufen", "live_subtitle": "Bilder beim Crawlen beobachten",
|
||
"total_images": "Bilder gesamt", "new_session": "Neu in dieser Sitzung", "per_minute": "Pro Minute",
|
||
"all_domains": "Alle Domains", "download_phantom": "Phantom-Seite herunterladen",
|
||
"phantom_subtitle": "Archivierte Seiten als statische Seite mit lokalen Medien exportieren", "max_pages": "Max. Seiten", "select_domain": "Domain auswählen", "sources": "Quellen",
|
||
"source_uri": "Quell-URI", "neopig_uri": "Neopig-URI", "source_page": "Quellseite", "neopig_page": "Neopig-Seite",
|
||
"type_label": "Typ", "mime_label": "MIME", "size_label": "Größe", "alt_label": "Alt",
|
||
"keywords_label": "Stichwörter", "description_label": "Beschreibung", "items": "Elemente", "bytes": "Bytes",
|
||
"used_on": "Verwendet auf", "page": "Seite", "discovered": "Entdeckt",
|
||
"console": "Konsole", "close": "Schließen", "no_logs": "Noch keine Logs",
|
||
},
|
||
"ko": {
|
||
"search": "검색", "crawl": "크롤", "live": "라이브", "about": "정보",
|
||
"random": "랜덤", "phantom": "팬텀", "loading": "로딩 중...",
|
||
"start_crawl": "크롤 시작", "pause": "일시정지", "resume": "재개", "delete": "삭제",
|
||
"url": "URL", "mode": "모드", "depth": "깊이", "keywords": "키워드",
|
||
"screenshots": "스크린샷", "fast_mode": "빠른 모드", "fresh_start": "새로 시작",
|
||
"pages": "페이지", "found": "발견", "saved": "저장", "dupes": "중복",
|
||
"running": "실행 중", "completed": "완료", "paused": "일시정지됨",
|
||
"no_jobs": "작업 없음", "started": "시작됨", "filter": "필터",
|
||
"all_types": "모든 유형", "images": "이미지", "videos": "동영상", "audio": "오디오",
|
||
"everything": "모두", "all_media": "모든 미디어",
|
||
"images_only": "이미지만", "videos_only": "동영상만", "text_only": "텍스트만",
|
||
"recent_jobs": "최근 작업", "search_placeholder": "미디어 검색...",
|
||
"media": "미디어", "pages_label": "페이지", "download": "다운로드",
|
||
"no_media": "미디어를 찾을 수 없음", "no_more": "더 이상 결과 없음", "load_error": "로드 실패",
|
||
"copy": "복사", "copied": "복사됨!", "starting": "시작 중...", "started_ok": "시작됨!",
|
||
"error": "오류", "watching": "새 이미지 감시 중...", "content": "콘텐츠",
|
||
"previous": "이전", "next": "다음", "over_9000": "9,000 이상!",
|
||
"loading_stats": "통계 로드 중...", "screenshot": "스크린샷",
|
||
"valid_uri": "유효한 URI를 하나 이상 입력하세요", "load_jobs_error": "작업 로드 실패",
|
||
"hydrate_subtitle": "웹에서 미디어 가져오기", "live_subtitle": "크롤링되는 이미지 보기",
|
||
"total_images": "전체 이미지", "new_session": "이번 세션 신규", "per_minute": "분당",
|
||
"all_domains": "모든 도메인", "download_phantom": "팬텀 사이트 다운로드",
|
||
"phantom_subtitle": "보관된 페이지를 로컬 미디어가 포함된 정적 사이트로 내보내기", "max_pages": "최대 페이지", "select_domain": "도메인 선택", "sources": "소스",
|
||
"source_uri": "소스 URI", "neopig_uri": "Neopig URI", "source_page": "소스 페이지", "neopig_page": "Neopig 페이지",
|
||
"type_label": "유형", "mime_label": "MIME", "size_label": "크기", "alt_label": "Alt",
|
||
"keywords_label": "키워드", "description_label": "설명", "items": "항목", "bytes": "바이트",
|
||
"used_on": "사용처", "page": "페이지", "discovered": "발견됨",
|
||
"console": "콘솔", "close": "닫기", "no_logs": "아직 로그 없음",
|
||
},
|
||
"it": {
|
||
"search": "Cerca", "crawl": "Scansiona", "live": "In Diretta", "about": "Info",
|
||
"random": "Casuale", "phantom": "Fantasma", "loading": "Caricamento...",
|
||
"start_crawl": "Avvia Scansione", "pause": "Pausa", "resume": "Riprendi", "delete": "Elimina",
|
||
"url": "URL", "mode": "Modalità", "depth": "Profondità", "keywords": "Parole chiave",
|
||
"screenshots": "Screenshot", "fast_mode": "Modalità Veloce", "fresh_start": "Nuovo Inizio",
|
||
"pages": "pagine", "found": "trovati", "saved": "salvati", "dupes": "duplicati",
|
||
"running": "in esecuzione", "completed": "completato", "paused": "in pausa",
|
||
"no_jobs": "Nessuna attività", "started": "Avviato", "filter": "Filtra",
|
||
"all_types": "Tutti i tipi", "images": "Immagini", "videos": "Video", "audio": "Audio",
|
||
"everything": "Tutto", "all_media": "Tutti i media",
|
||
"images_only": "Solo immagini", "videos_only": "Solo video", "text_only": "Solo testo",
|
||
"recent_jobs": "Attività Recenti", "search_placeholder": "Cerca media...",
|
||
"media": "Media", "pages_label": "Pagine", "download": "Scarica",
|
||
"no_media": "Nessun media trovato", "no_more": "Nessun altro risultato", "load_error": "Caricamento fallito",
|
||
"copy": "Copia", "copied": "Copiato!", "starting": "Avvio...", "started_ok": "Avviato!",
|
||
"error": "Errore", "watching": "Monitoraggio nuove immagini...", "content": "Contenuto",
|
||
"previous": "Precedente", "next": "Successivo", "over_9000": "Oltre 9.000+!",
|
||
"loading_stats": "Caricamento statistiche...", "screenshot": "Screenshot",
|
||
"valid_uri": "Inserire almeno un URI valido", "load_jobs_error": "Caricamento attività fallito",
|
||
"hydrate_subtitle": "Ottieni media dal web", "live_subtitle": "Guarda le immagini mentre vengono scansionate",
|
||
"total_images": "Immagini totali", "new_session": "Nuove in questa sessione", "per_minute": "Al minuto",
|
||
"all_domains": "Tutti i domini", "download_phantom": "Scarica sito fantasma",
|
||
"phantom_subtitle": "Esporta pagine archiviate come sito statico con media locali", "max_pages": "Max pagine", "select_domain": "Seleziona dominio", "sources": "fonti",
|
||
"source_uri": "URI sorgente", "neopig_uri": "URI Neopig", "source_page": "Pagina sorgente", "neopig_page": "Pagina Neopig",
|
||
"type_label": "Tipo", "mime_label": "MIME", "size_label": "Dimensione", "alt_label": "Alt",
|
||
"keywords_label": "Parole chiave", "description_label": "Descrizione", "items": "elementi", "bytes": "byte",
|
||
"used_on": "Usato su", "page": "Pagina", "discovered": "Scoperto",
|
||
"console": "Console", "close": "Chiudi", "no_logs": "Nessun log ancora",
|
||
},
|
||
"nl": {
|
||
"search": "Zoeken", "crawl": "Crawlen", "live": "Live", "about": "Over",
|
||
"random": "Willekeurig", "phantom": "Fantoom", "loading": "Laden...",
|
||
"start_crawl": "Start Crawl", "pause": "Pauzeer", "resume": "Hervat", "delete": "Verwijder",
|
||
"url": "URL", "mode": "Modus", "depth": "Diepte", "keywords": "Trefwoorden",
|
||
"screenshots": "Schermafbeeldingen", "fast_mode": "Snelle Modus", "fresh_start": "Nieuwe Start",
|
||
"pages": "pagina's", "found": "gevonden", "saved": "opgeslagen", "dupes": "duplicaten",
|
||
"running": "actief", "completed": "voltooid", "paused": "gepauzeerd",
|
||
"no_jobs": "Geen taken", "started": "Gestart", "filter": "Filter",
|
||
"all_types": "Alle typen", "images": "Afbeeldingen", "videos": "Video's", "audio": "Audio",
|
||
"everything": "Alles", "all_media": "Alle media",
|
||
"images_only": "Alleen afbeeldingen", "videos_only": "Alleen video's", "text_only": "Alleen tekst",
|
||
"recent_jobs": "Recente Taken", "search_placeholder": "Zoek media...",
|
||
"media": "Media", "pages_label": "Pagina's", "download": "Downloaden",
|
||
"no_media": "Geen media gevonden", "no_more": "Geen resultaten meer", "load_error": "Laden mislukt",
|
||
"copy": "Kopiëren", "copied": "Gekopieerd!", "starting": "Starten...", "started_ok": "Gestart!",
|
||
"error": "Fout", "watching": "Nieuwe afbeeldingen bekijken...", "content": "Inhoud",
|
||
"previous": "Vorige", "next": "Volgende", "over_9000": "Meer dan 9.000+!",
|
||
"loading_stats": "Statistieken laden...", "screenshot": "Schermafbeelding",
|
||
"valid_uri": "Voer minimaal één geldige URI in", "load_jobs_error": "Taken laden mislukt",
|
||
"hydrate_subtitle": "Media van het web ophalen", "live_subtitle": "Bekijk afbeeldingen terwijl ze worden gecrawld",
|
||
"total_images": "Totaal afbeeldingen", "new_session": "Nieuw deze sessie", "per_minute": "Per minuut",
|
||
"all_domains": "Alle domeinen", "download_phantom": "Fantoomsite downloaden",
|
||
"phantom_subtitle": "Gearchiveerde pagina's exporteren als statische site met lokale media", "max_pages": "Max. pagina's", "select_domain": "Selecteer domein", "sources": "bronnen",
|
||
"source_uri": "Bron-URI", "neopig_uri": "Neopig-URI", "source_page": "Bronpagina", "neopig_page": "Neopig-pagina",
|
||
"type_label": "Type", "mime_label": "MIME", "size_label": "Grootte", "alt_label": "Alt",
|
||
"keywords_label": "Trefwoorden", "description_label": "Beschrijving", "items": "items", "bytes": "bytes",
|
||
"used_on": "Gebruikt op", "page": "Pagina", "discovered": "Ontdekt",
|
||
"console": "Console", "close": "Sluiten", "no_logs": "Nog geen logs",
|
||
},
|
||
"pl": {
|
||
"search": "Szukaj", "crawl": "Indeksuj", "live": "Na żywo", "about": "O nas",
|
||
"random": "Losowo", "phantom": "Fantom", "loading": "Ładowanie...",
|
||
"start_crawl": "Rozpocznij", "pause": "Pauza", "resume": "Wznów", "delete": "Usuń",
|
||
"url": "URL", "mode": "Tryb", "depth": "Głębokość", "keywords": "Słowa kluczowe",
|
||
"screenshots": "Zrzuty ekranu", "fast_mode": "Tryb szybki", "fresh_start": "Nowy start",
|
||
"pages": "stron", "found": "znaleziono", "saved": "zapisano", "dupes": "duplikaty",
|
||
"running": "w toku", "completed": "zakończono", "paused": "wstrzymano",
|
||
"no_jobs": "Brak zadań", "started": "Rozpoczęto", "filter": "Filtruj",
|
||
"all_types": "Wszystkie typy", "images": "Obrazy", "videos": "Filmy", "audio": "Audio",
|
||
"everything": "Wszystko", "all_media": "Wszystkie media",
|
||
"images_only": "Tylko obrazy", "videos_only": "Tylko filmy", "text_only": "Tylko tekst",
|
||
"recent_jobs": "Ostatnie Zadania", "search_placeholder": "Szukaj mediów...",
|
||
"media": "Media", "pages_label": "Strony", "download": "Pobierz",
|
||
"no_media": "Nie znaleziono mediów", "no_more": "Brak więcej wyników", "load_error": "Błąd ładowania",
|
||
"copy": "Kopiuj", "copied": "Skopiowano!", "starting": "Uruchamianie...", "started_ok": "Uruchomiono!",
|
||
"error": "Błąd", "watching": "Obserwowanie nowych obrazów...", "content": "Treść",
|
||
"previous": "Poprzedni", "next": "Następny", "over_9000": "Ponad 9 000+!",
|
||
"loading_stats": "Ładowanie statystyk...", "screenshot": "Zrzut ekranu",
|
||
"valid_uri": "Wprowadź przynajmniej jeden URI", "load_jobs_error": "Nie udało się załadować zadań",
|
||
"hydrate_subtitle": "Pobierz media z sieci", "live_subtitle": "Oglądaj obrazy podczas indeksowania",
|
||
"total_images": "Łącznie obrazów", "new_session": "Nowe w tej sesji", "per_minute": "Na minutę",
|
||
"all_domains": "Wszystkie domeny", "download_phantom": "Pobierz stronę fantomową",
|
||
"phantom_subtitle": "Eksportuj zarchiwizowane strony jako statyczną witrynę z lokalnymi mediami", "max_pages": "Maks. stron", "select_domain": "Wybierz domenę", "sources": "źródeł",
|
||
"source_uri": "URI źródła", "neopig_uri": "URI Neopig", "source_page": "Strona źródłowa", "neopig_page": "Strona Neopig",
|
||
"type_label": "Typ", "mime_label": "MIME", "size_label": "Rozmiar", "alt_label": "Alt",
|
||
"keywords_label": "Słowa kluczowe", "description_label": "Opis", "items": "elementy", "bytes": "bajty",
|
||
"used_on": "Używane na", "page": "Strona", "discovered": "Odkryto",
|
||
"console": "Konsola", "close": "Zamknij", "no_logs": "Brak logów",
|
||
},
|
||
"tr": {
|
||
"search": "Ara", "crawl": "Tara", "live": "Canlı", "about": "Hakkında",
|
||
"random": "Rastgele", "phantom": "Hayalet", "loading": "Yükleniyor...",
|
||
"start_crawl": "Taramayı Başlat", "pause": "Duraklat", "resume": "Devam", "delete": "Sil",
|
||
"url": "URL", "mode": "Mod", "depth": "Derinlik", "keywords": "Anahtar kelimeler",
|
||
"screenshots": "Ekran görüntüleri", "fast_mode": "Hızlı Mod", "fresh_start": "Yeni Başlangıç",
|
||
"pages": "sayfa", "found": "bulundu", "saved": "kaydedildi", "dupes": "kopya",
|
||
"running": "çalışıyor", "completed": "tamamlandı", "paused": "duraklatıldı",
|
||
"no_jobs": "Görev yok", "started": "Başladı", "filter": "Filtrele",
|
||
"all_types": "Tüm türler", "images": "Resimler", "videos": "Videolar", "audio": "Ses",
|
||
"everything": "Her şey", "all_media": "Tüm medya",
|
||
"images_only": "Sadece resimler", "videos_only": "Sadece videolar", "text_only": "Sadece metin",
|
||
"recent_jobs": "Son Görevler", "search_placeholder": "Medya ara...",
|
||
"media": "Medya", "pages_label": "Sayfalar", "download": "İndir",
|
||
"no_media": "Medya bulunamadı", "no_more": "Daha fazla sonuç yok", "load_error": "Yükleme başarısız",
|
||
"copy": "Kopyala", "copied": "Kopyalandı!", "starting": "Başlatılıyor...", "started_ok": "Başlatıldı!",
|
||
"error": "Hata", "watching": "Yeni resimler izleniyor...", "content": "İçerik",
|
||
"previous": "Önceki", "next": "Sonraki", "over_9000": "9.000'den fazla+!",
|
||
"loading_stats": "İstatistikler yükleniyor...", "screenshot": "Ekran görüntüsü",
|
||
"valid_uri": "Lütfen en az bir geçerli URI girin", "load_jobs_error": "Görevler yüklenemedi",
|
||
"hydrate_subtitle": "Web'den medya al", "live_subtitle": "Taranan resimleri izle",
|
||
"total_images": "Toplam resim", "new_session": "Bu oturumda yeni", "per_minute": "Dakikada",
|
||
"all_domains": "Tüm alanlar", "download_phantom": "Hayalet siteyi indir",
|
||
"phantom_subtitle": "Arşivlenmiş sayfaları yerel medya ile statik site olarak dışa aktar", "max_pages": "Maks. sayfa", "select_domain": "Alan adı seç", "sources": "kaynaklar",
|
||
"source_uri": "Kaynak URI", "neopig_uri": "Neopig URI", "source_page": "Kaynak Sayfa", "neopig_page": "Neopig Sayfası",
|
||
"type_label": "Tür", "mime_label": "MIME", "size_label": "Boyut", "alt_label": "Alt",
|
||
"keywords_label": "Anahtar kelimeler", "description_label": "Açıklama", "items": "öğe", "bytes": "bayt",
|
||
"used_on": "Kullanıldığı yer", "page": "Sayfa", "discovered": "Keşfedildi",
|
||
"console": "Konsol", "close": "Kapat", "no_logs": "Henüz log yok",
|
||
},
|
||
"vi": {
|
||
"search": "Tìm kiếm", "crawl": "Thu thập", "live": "Trực tiếp", "about": "Giới thiệu",
|
||
"random": "Ngẫu nhiên", "phantom": "Bóng ma", "loading": "Đang tải...",
|
||
"start_crawl": "Bắt đầu", "pause": "Tạm dừng", "resume": "Tiếp tục", "delete": "Xóa",
|
||
"url": "URL", "mode": "Chế độ", "depth": "Độ sâu", "keywords": "Từ khóa",
|
||
"screenshots": "Ảnh chụp", "fast_mode": "Chế độ nhanh", "fresh_start": "Bắt đầu mới",
|
||
"pages": "trang", "found": "tìm thấy", "saved": "đã lưu", "dupes": "trùng lặp",
|
||
"running": "đang chạy", "completed": "hoàn thành", "paused": "tạm dừng",
|
||
"no_jobs": "Không có tác vụ", "started": "Đã bắt đầu", "filter": "Lọc",
|
||
"all_types": "Tất cả loại", "images": "Hình ảnh", "videos": "Video", "audio": "Âm thanh",
|
||
"everything": "Tất cả", "all_media": "Tất cả media",
|
||
"images_only": "Chỉ hình ảnh", "videos_only": "Chỉ video", "text_only": "Chỉ văn bản",
|
||
"recent_jobs": "Tác vụ gần đây", "search_placeholder": "Tìm media...",
|
||
"media": "Media", "pages_label": "Trang", "download": "Tải xuống",
|
||
"no_media": "Không tìm thấy media", "no_more": "Không còn kết quả", "load_error": "Tải thất bại",
|
||
"copy": "Sao chép", "copied": "Đã sao chép!", "starting": "Đang khởi động...", "started_ok": "Đã khởi động!",
|
||
"error": "Lỗi", "watching": "Đang theo dõi hình ảnh mới...", "content": "Nội dung",
|
||
"previous": "Trước", "next": "Tiếp", "over_9000": "Hơn 9.000+!",
|
||
"loading_stats": "Đang tải thống kê...", "screenshot": "Ảnh chụp màn hình",
|
||
"valid_uri": "Vui lòng nhập ít nhất một URI hợp lệ", "load_jobs_error": "Không thể tải tác vụ",
|
||
"hydrate_subtitle": "Lấy media từ web", "live_subtitle": "Xem hình ảnh khi đang thu thập",
|
||
"total_images": "Tổng hình ảnh", "new_session": "Mới trong phiên này", "per_minute": "Mỗi phút",
|
||
"all_domains": "Tất cả tên miền", "download_phantom": "Tải trang bóng ma",
|
||
"phantom_subtitle": "Xuất trang lưu trữ dưới dạng trang tĩnh với media cục bộ", "max_pages": "Tối đa trang", "select_domain": "Chọn tên miền", "sources": "nguồn",
|
||
"source_uri": "URI nguồn", "neopig_uri": "URI Neopig", "source_page": "Trang nguồn", "neopig_page": "Trang Neopig",
|
||
"type_label": "Loại", "mime_label": "MIME", "size_label": "Kích thước", "alt_label": "Alt",
|
||
"keywords_label": "Từ khóa", "description_label": "Mô tả", "items": "mục", "bytes": "byte",
|
||
"used_on": "Sử dụng tại", "page": "Trang", "discovered": "Phát hiện",
|
||
"console": "Bảng điều khiển", "close": "Đóng", "no_logs": "Chưa có nhật ký",
|
||
},
|
||
"th": {
|
||
"search": "ค้นหา", "crawl": "รวบรวม", "live": "สด", "about": "เกี่ยวกับ",
|
||
"random": "สุ่ม", "phantom": "แฟนทอม", "loading": "กำลังโหลด...",
|
||
"start_crawl": "เริ่มรวบรวม", "pause": "หยุดชั่วคราว", "resume": "ดำเนินต่อ", "delete": "ลบ",
|
||
"url": "URL", "mode": "โหมด", "depth": "ความลึก", "keywords": "คำสำคัญ",
|
||
"screenshots": "ภาพหน้าจอ", "fast_mode": "โหมดเร็ว", "fresh_start": "เริ่มใหม่",
|
||
"pages": "หน้า", "found": "พบ", "saved": "บันทึก", "dupes": "ซ้ำ",
|
||
"running": "กำลังทำงาน", "completed": "เสร็จสิ้น", "paused": "หยุดชั่วคราว",
|
||
"no_jobs": "ไม่มีงาน", "started": "เริ่มแล้ว", "filter": "กรอง",
|
||
"all_types": "ทุกประเภท", "images": "รูปภาพ", "videos": "วิดีโอ", "audio": "เสียง",
|
||
"everything": "ทั้งหมด", "all_media": "สื่อทั้งหมด",
|
||
"images_only": "เฉพาะรูปภาพ", "videos_only": "เฉพาะวิดีโอ", "text_only": "เฉพาะข้อความ",
|
||
"recent_jobs": "งานล่าสุด", "search_placeholder": "ค้นหาสื่อ...",
|
||
"media": "สื่อ", "pages_label": "หน้า", "download": "ดาวน์โหลด",
|
||
"no_media": "ไม่พบสื่อ", "no_more": "ไม่มีผลลัพธ์เพิ่มเติม", "load_error": "โหลดไม่สำเร็จ",
|
||
"copy": "คัดลอก", "copied": "คัดลอกแล้ว!", "starting": "กำลังเริ่ม...", "started_ok": "เริ่มแล้ว!",
|
||
"error": "ข้อผิดพลาด", "watching": "กำลังดูรูปภาพใหม่...", "content": "เนื้อหา",
|
||
"previous": "ก่อนหน้า", "next": "ถัดไป", "over_9000": "มากกว่า 9,000+!",
|
||
"loading_stats": "กำลังโหลดสถิติ...", "screenshot": "ภาพหน้าจอ",
|
||
"valid_uri": "กรุณาใส่ URI ที่ถูกต้องอย่างน้อยหนึ่งรายการ", "load_jobs_error": "ไม่สามารถโหลดงานได้",
|
||
"hydrate_subtitle": "ดึงสื่อจากเว็บ", "live_subtitle": "ดูรูปภาพขณะรวบรวม",
|
||
"total_images": "รูปภาพทั้งหมด", "new_session": "ใหม่ในเซสชันนี้", "per_minute": "ต่อนาที",
|
||
"all_domains": "โดเมนทั้งหมด", "download_phantom": "ดาวน์โหลดเว็บแฟนทอม",
|
||
"phantom_subtitle": "ส่งออกหน้าที่เก็บถาวรเป็นเว็บไซต์แบบคงที่พร้อมสื่อในเครื่อง", "max_pages": "หน้าสูงสุด", "select_domain": "เลือกโดเมน", "sources": "แหล่งที่มา",
|
||
"source_uri": "URI แหล่งที่มา", "neopig_uri": "URI Neopig", "source_page": "หน้าแหล่งที่มา", "neopig_page": "หน้า Neopig",
|
||
"type_label": "ประเภท", "mime_label": "MIME", "size_label": "ขนาด", "alt_label": "Alt",
|
||
"keywords_label": "คำสำคัญ", "description_label": "คำอธิบาย", "items": "รายการ", "bytes": "ไบต์",
|
||
"used_on": "ใช้บน", "page": "หน้า", "discovered": "ค้นพบ",
|
||
"console": "คอนโซล", "close": "ปิด", "no_logs": "ยังไม่มีบันทึก",
|
||
},
|
||
"id": {
|
||
"search": "Cari", "crawl": "Jelajahi", "live": "Langsung", "about": "Tentang",
|
||
"random": "Acak", "phantom": "Hantu", "loading": "Memuat...",
|
||
"start_crawl": "Mulai Jelajah", "pause": "Jeda", "resume": "Lanjutkan", "delete": "Hapus",
|
||
"url": "URL", "mode": "Mode", "depth": "Kedalaman", "keywords": "Kata kunci",
|
||
"screenshots": "Tangkapan layar", "fast_mode": "Mode Cepat", "fresh_start": "Mulai Baru",
|
||
"pages": "halaman", "found": "ditemukan", "saved": "disimpan", "dupes": "duplikat",
|
||
"running": "berjalan", "completed": "selesai", "paused": "dijeda",
|
||
"no_jobs": "Tidak ada tugas", "started": "Dimulai", "filter": "Filter",
|
||
"all_types": "Semua jenis", "images": "Gambar", "videos": "Video", "audio": "Audio",
|
||
"everything": "Semua", "all_media": "Semua media",
|
||
"images_only": "Hanya gambar", "videos_only": "Hanya video", "text_only": "Hanya teks",
|
||
"recent_jobs": "Tugas Terbaru", "search_placeholder": "Cari media...",
|
||
"media": "Media", "pages_label": "Halaman", "download": "Unduh",
|
||
"no_media": "Media tidak ditemukan", "no_more": "Tidak ada hasil lagi", "load_error": "Gagal memuat",
|
||
"copy": "Salin", "copied": "Disalin!", "starting": "Memulai...", "started_ok": "Dimulai!",
|
||
"error": "Kesalahan", "watching": "Memantau gambar baru...", "content": "Konten",
|
||
"previous": "Sebelumnya", "next": "Berikutnya", "over_9000": "Lebih dari 9.000+!",
|
||
"loading_stats": "Memuat statistik...", "screenshot": "Tangkapan layar",
|
||
"valid_uri": "Masukkan setidaknya satu URI yang valid", "load_jobs_error": "Gagal memuat tugas",
|
||
"hydrate_subtitle": "Ambil media dari web", "live_subtitle": "Lihat gambar saat dijelajahi",
|
||
"total_images": "Total gambar", "new_session": "Baru sesi ini", "per_minute": "Per menit",
|
||
"all_domains": "Semua domain", "download_phantom": "Unduh situs hantu",
|
||
"phantom_subtitle": "Ekspor halaman arsip sebagai situs statis dengan media lokal", "max_pages": "Maks. halaman", "select_domain": "Pilih domain", "sources": "sumber",
|
||
"source_uri": "URI sumber", "neopig_uri": "URI Neopig", "source_page": "Halaman sumber", "neopig_page": "Halaman Neopig",
|
||
"type_label": "Jenis", "mime_label": "MIME", "size_label": "Ukuran", "alt_label": "Alt",
|
||
"keywords_label": "Kata kunci", "description_label": "Deskripsi", "items": "item", "bytes": "byte",
|
||
"used_on": "Digunakan di", "page": "Halaman", "discovered": "Ditemukan",
|
||
"console": "Konsol", "close": "Tutup", "no_logs": "Belum ada log",
|
||
},
|
||
"uk": {
|
||
"search": "Пошук", "crawl": "Сканувати", "live": "Наживо", "about": "Про нас",
|
||
"random": "Випадково", "phantom": "Привид", "loading": "Завантаження...",
|
||
"start_crawl": "Почати сканування", "pause": "Пауза", "resume": "Продовжити", "delete": "Видалити",
|
||
"url": "URL", "mode": "Режим", "depth": "Глибина", "keywords": "Ключові слова",
|
||
"screenshots": "Знімки екрану", "fast_mode": "Швидкий режим", "fresh_start": "Новий старт",
|
||
"pages": "сторінок", "found": "знайдено", "saved": "збережено", "dupes": "дублікати",
|
||
"running": "виконується", "completed": "завершено", "paused": "призупинено",
|
||
"no_jobs": "Немає завдань", "started": "Розпочато", "filter": "Фільтр",
|
||
"all_types": "Усі типи", "images": "Зображення", "videos": "Відео", "audio": "Аудіо",
|
||
"everything": "Все", "all_media": "Усі медіа",
|
||
"images_only": "Лише зображення", "videos_only": "Лише відео", "text_only": "Лише текст",
|
||
"recent_jobs": "Останні завдання", "search_placeholder": "Пошук медіа...",
|
||
"media": "Медіа", "pages_label": "Сторінки", "download": "Завантажити",
|
||
"no_media": "Медіа не знайдено", "no_more": "Більше немає результатів", "load_error": "Помилка завантаження",
|
||
"copy": "Копіювати", "copied": "Скопійовано!", "starting": "Запуск...", "started_ok": "Запущено!",
|
||
"error": "Помилка", "watching": "Відстеження нових зображень...", "content": "Вміст",
|
||
"previous": "Попередній", "next": "Наступний", "over_9000": "Понад 9000+!",
|
||
"loading_stats": "Завантаження статистики...", "screenshot": "Знімок екрану",
|
||
"valid_uri": "Введіть хоча б один дійсний URI", "load_jobs_error": "Не вдалося завантажити завдання",
|
||
"hydrate_subtitle": "Отримати медіа з вебу", "live_subtitle": "Дивіться зображення під час сканування",
|
||
"total_images": "Всього зображень", "new_session": "Нові в сесії", "per_minute": "За хвилину",
|
||
"all_domains": "Усі домени", "download_phantom": "Завантажити привид-сайт",
|
||
"phantom_subtitle": "Експортувати архівні сторінки як статичний сайт з локальними медіа", "max_pages": "Макс. сторінок", "select_domain": "Вибрати домен", "sources": "джерел",
|
||
"source_uri": "URI джерела", "neopig_uri": "URI Neopig", "source_page": "Сторінка джерела", "neopig_page": "Сторінка Neopig",
|
||
"type_label": "Тип", "mime_label": "MIME", "size_label": "Розмір", "alt_label": "Alt",
|
||
"keywords_label": "Ключові слова", "description_label": "Опис", "items": "елементів", "bytes": "байт",
|
||
"used_on": "Використовується на", "page": "Сторінка", "discovered": "Виявлено",
|
||
"console": "Консоль", "close": "Закрити", "no_logs": "Ще немає логів",
|
||
},
|
||
"sv": {
|
||
"search": "Sök", "crawl": "Genomsök", "live": "Live", "about": "Om",
|
||
"random": "Slumpmässig", "phantom": "Fantom", "loading": "Laddar...",
|
||
"start_crawl": "Starta genomsökning", "pause": "Pausa", "resume": "Återuppta", "delete": "Ta bort",
|
||
"url": "URL", "mode": "Läge", "depth": "Djup", "keywords": "Nyckelord",
|
||
"screenshots": "Skärmbilder", "fast_mode": "Snabbläge", "fresh_start": "Ny start",
|
||
"pages": "sidor", "found": "hittade", "saved": "sparade", "dupes": "dubbletter",
|
||
"running": "körs", "completed": "slutförd", "paused": "pausad",
|
||
"no_jobs": "Inga jobb", "started": "Startad", "filter": "Filtrera",
|
||
"all_types": "Alla typer", "images": "Bilder", "videos": "Videor", "audio": "Ljud",
|
||
"everything": "Allt", "all_media": "Alla media",
|
||
"images_only": "Endast bilder", "videos_only": "Endast videor", "text_only": "Endast text",
|
||
"recent_jobs": "Senaste Jobb", "search_placeholder": "Sök media...",
|
||
"media": "Media", "pages_label": "Sidor", "download": "Ladda ner",
|
||
"no_media": "Ingen media hittades", "no_more": "Inga fler resultat", "load_error": "Kunde inte ladda",
|
||
"copy": "Kopiera", "copied": "Kopierat!", "starting": "Startar...", "started_ok": "Startad!",
|
||
"error": "Fel", "watching": "Bevakar nya bilder...", "content": "Innehåll",
|
||
"previous": "Föregående", "next": "Nästa", "over_9000": "Över 9 000+!",
|
||
"loading_stats": "Laddar statistik...", "screenshot": "Skärmbild",
|
||
"valid_uri": "Ange minst en giltig URI", "load_jobs_error": "Kunde inte ladda jobb",
|
||
"hydrate_subtitle": "Hämta media från webben", "live_subtitle": "Se bilder medan de genomsöks",
|
||
"total_images": "Totalt bilder", "new_session": "Nya denna session", "per_minute": "Per minut",
|
||
"all_domains": "Alla domäner", "download_phantom": "Ladda ner fantomwebbplats",
|
||
"phantom_subtitle": "Exportera arkiverade sidor som statisk webbplats med lokal media", "max_pages": "Max sidor", "select_domain": "Välj domän", "sources": "källor",
|
||
"source_uri": "Käll-URI", "neopig_uri": "Neopig URI", "source_page": "Källsida", "neopig_page": "Neopig-sida",
|
||
"type_label": "Typ", "mime_label": "MIME", "size_label": "Storlek", "alt_label": "Alt",
|
||
"keywords_label": "Nyckelord", "description_label": "Beskrivning", "items": "objekt", "bytes": "bytes",
|
||
"used_on": "Används på", "page": "Sida", "discovered": "Upptäckt",
|
||
"console": "Konsol", "close": "Stäng", "no_logs": "Inga loggar ännu",
|
||
},
|
||
"zh-tw": {
|
||
"search": "搜尋", "crawl": "爬取", "live": "即時動態", "about": "關於",
|
||
"random": "隨機", "phantom": "幻影", "loading": "載入中...",
|
||
"start_crawl": "開始爬取", "pause": "暫停", "resume": "繼續", "delete": "刪除",
|
||
"url": "網址", "mode": "模式", "depth": "深度", "keywords": "關鍵字",
|
||
"screenshots": "螢幕截圖", "fast_mode": "快速模式", "fresh_start": "全新開始",
|
||
"pages": "頁面", "found": "發現", "saved": "儲存", "dupes": "重複",
|
||
"running": "執行中", "completed": "已完成", "paused": "已暫停",
|
||
"no_jobs": "暫無爬取任務", "started": "開始於", "filter": "篩選",
|
||
"all_types": "所有類型", "images": "圖片", "videos": "影片", "audio": "音訊",
|
||
"everything": "全部（文字+媒體）", "all_media": "所有媒體",
|
||
"images_only": "僅圖片", "videos_only": "僅影片", "text_only": "僅文字",
|
||
"recent_jobs": "最近任務", "search_placeholder": "搜尋媒體...",
|
||
"media": "媒體", "pages_label": "頁面", "download": "下載",
|
||
"no_media": "未找到媒體", "no_more": "沒有更多結果", "load_error": "載入失敗",
|
||
"copy": "複製", "copied": "已複製!", "starting": "啟動中...", "started_ok": "已啟動!",
|
||
"error": "錯誤", "watching": "正在監視新圖片...", "content": "內容",
|
||
"previous": "上一頁", "next": "下一頁", "over_9000": "超過9000+!",
|
||
"loading_stats": "載入統計中...", "screenshot": "螢幕截圖",
|
||
"valid_uri": "請輸入至少一個有效的URI", "load_jobs_error": "載入任務失敗",
|
||
"hydrate_subtitle": "從網路取得媒體", "live_subtitle": "觀看正在爬取的圖片",
|
||
"total_images": "總圖片數", "new_session": "本次新增", "per_minute": "每分鐘",
|
||
"all_domains": "所有網域", "download_phantom": "下載幻影網站",
|
||
"phantom_subtitle": "將存檔頁面匯出為帶本地媒體的靜態網站", "max_pages": "最大頁數", "select_domain": "選擇網域", "sources": "來源",
|
||
"source_uri": "來源URI", "neopig_uri": "Neopig URI", "source_page": "來源頁面", "neopig_page": "Neopig頁面",
|
||
"type_label": "類型", "mime_label": "MIME", "size_label": "大小", "alt_label": "替代文字",
|
||
"keywords_label": "關鍵字", "description_label": "描述", "items": "項", "bytes": "位元組",
|
||
"used_on": "使用於", "page": "頁面", "discovered": "發現時間",
|
||
"console": "主控台", "close": "關閉", "no_logs": "暫無日誌",
|
||
},
|
||
"bn": {
|
||
"search": "অনুসন্ধান", "crawl": "ক্রল", "live": "লাইভ ফিড", "about": "সম্পর্কে",
|
||
"random": "এলোমেলো", "phantom": "ফ্যান্টম", "loading": "লোড হচ্ছে...",
|
||
"start_crawl": "ক্রল শুরু করুন", "pause": "বিরতি", "resume": "পুনরায় শুরু", "delete": "মুছুন",
|
||
"url": "URL", "mode": "মোড", "depth": "গভীরতা", "keywords": "কীওয়ার্ড",
|
||
"screenshots": "স্ক্রিনশট", "fast_mode": "দ্রুত মোড", "fresh_start": "নতুন শুরু",
|
||
"pages": "পৃষ্ঠা", "found": "পাওয়া গেছে", "saved": "সংরক্ষিত", "dupes": "ডুপ্লিকেট",
|
||
"running": "চলছে", "completed": "সম্পন্ন", "paused": "বিরতি",
|
||
"no_jobs": "কোন ক্রল কাজ নেই", "started": "শুরু", "filter": "ফিল্টার",
|
||
"all_types": "সব ধরনের", "images": "ছবি", "videos": "ভিডিও", "audio": "অডিও",
|
||
"everything": "সবকিছু", "all_media": "সব মিডিয়া",
|
||
"images_only": "শুধু ছবি", "videos_only": "শুধু ভিডিও", "text_only": "শুধু টেক্সট",
|
||
"recent_jobs": "সাম্প্রতিক কাজ", "search_placeholder": "মিডিয়া অনুসন্ধান...",
|
||
"media": "মিডিয়া", "pages_label": "পৃষ্ঠা", "download": "ডাউনলোড",
|
||
"no_media": "কোন মিডিয়া পাওয়া যায়নি", "no_more": "আর কোন ফলাফল নেই", "load_error": "লোড ব্যর্থ",
|
||
"copy": "কপি", "copied": "কপি হয়েছে!", "starting": "শুরু হচ্ছে...", "started_ok": "শুরু হয়েছে!",
|
||
"error": "ত্রুটি", "watching": "নতুন ছবি দেখছি...", "content": "বিষয়বস্তু",
|
||
"previous": "পূর্ববর্তী", "next": "পরবর্তী", "over_9000": "৯,০০০+ এর বেশি!",
|
||
"loading_stats": "পরিসংখ্যান লোড হচ্ছে...", "screenshot": "স্ক্রিনশট",
|
||
"valid_uri": "অন্তত একটি বৈধ URI দিন", "load_jobs_error": "কাজ লোড ব্যর্থ",
|
||
"hydrate_subtitle": "ওয়েব থেকে মিডিয়া আনুন", "live_subtitle": "ক্রল হওয়া ছবি দেখুন",
|
||
"total_images": "মোট ছবি", "new_session": "এই সেশনে নতুন", "per_minute": "প্রতি মিনিটে",
|
||
"all_domains": "সব ডোমেইন", "download_phantom": "ফ্যান্টম সাইট ডাউনলোড",
|
||
"phantom_subtitle": "সংরক্ষিত পৃষ্ঠা স্থানীয় মিডিয়া সহ স্ট্যাটিক সাইট হিসেবে রপ্তানি", "max_pages": "সর্বোচ্চ পৃষ্ঠা", "select_domain": "ডোমেইন নির্বাচন", "sources": "সূত্র",
|
||
"source_uri": "উৎস URI", "neopig_uri": "Neopig URI", "source_page": "উৎস পৃষ্ঠা", "neopig_page": "Neopig পৃষ্ঠা",
|
||
"type_label": "ধরন", "mime_label": "MIME", "size_label": "আকার", "alt_label": "Alt",
|
||
"keywords_label": "কীওয়ার্ড", "description_label": "বিবরণ", "items": "আইটেম", "bytes": "বাইট",
|
||
"used_on": "ব্যবহৃত", "page": "পৃষ্ঠা", "discovered": "আবিষ্কৃত",
|
||
"console": "কনসোল", "close": "বন্ধ", "no_logs": "কোন লগ নেই",
|
||
},
|
||
"ur": {
|
||
"search": "تلاش", "crawl": "کرال", "live": "لائیو فیڈ", "about": "کے بارے میں",
|
||
"random": "بے ترتیب", "phantom": "فینٹم", "loading": "لوڈ ہو رہا ہے...",
|
||
"start_crawl": "کرال شروع کریں", "pause": "روکیں", "resume": "جاری رکھیں", "delete": "حذف کریں",
|
||
"url": "یو آر ایل", "mode": "موڈ", "depth": "گہرائی", "keywords": "کلیدی الفاظ",
|
||
"screenshots": "اسکرین شاٹس", "fast_mode": "تیز موڈ", "fresh_start": "نئی شروعات",
|
||
"pages": "صفحات", "found": "ملے", "saved": "محفوظ", "dupes": "ڈپلیکیٹ",
|
||
"running": "چل رہا ہے", "completed": "مکمل", "paused": "روکا ہوا",
|
||
"no_jobs": "کوئی کرال کام نہیں", "started": "شروع", "filter": "فلٹر",
|
||
"all_types": "تمام اقسام", "images": "تصاویر", "videos": "ویڈیوز", "audio": "آڈیو",
|
||
"everything": "سب کچھ", "all_media": "تمام میڈیا",
|
||
"images_only": "صرف تصاویر", "videos_only": "صرف ویڈیوز", "text_only": "صرف متن",
|
||
"recent_jobs": "حالیہ کام", "search_placeholder": "میڈیا تلاش کریں...",
|
||
"media": "میڈیا", "pages_label": "صفحات", "download": "ڈاؤن لوڈ",
|
||
"no_media": "کوئی میڈیا نہیں ملا", "no_more": "مزید نتائج نہیں", "load_error": "لوڈ ناکام",
|
||
"copy": "کاپی", "copied": "کاپی ہو گیا!", "starting": "شروع ہو رہا ہے...", "started_ok": "شروع ہو گیا!",
|
||
"error": "خرابی", "watching": "نئی تصاویر دیکھ رہا ہے...", "content": "مواد",
|
||
"previous": "پچھلا", "next": "اگلا", "over_9000": "9,000+ سے زیادہ!",
|
||
"loading_stats": "اعدادوشمار لوڈ ہو رہے ہیں...", "screenshot": "اسکرین شاٹ",
|
||
"valid_uri": "کم از کم ایک درست URI درج کریں", "load_jobs_error": "کام لوڈ ناکام",
|
||
"hydrate_subtitle": "ویب سے میڈیا حاصل کریں", "live_subtitle": "کرال ہوتی تصاویر دیکھیں",
|
||
"total_images": "کل تصاویر", "new_session": "اس سیشن میں نئی", "per_minute": "فی منٹ",
|
||
"all_domains": "تمام ڈومینز", "download_phantom": "فینٹم سائٹ ڈاؤن لوڈ",
|
||
"phantom_subtitle": "آرکائیو شدہ صفحات کو مقامی میڈیا کے ساتھ جامد سائٹ کے طور پر برآمد کریں", "max_pages": "زیادہ سے زیادہ صفحات", "select_domain": "ڈومین منتخب کریں", "sources": "ذرائع",
|
||
"source_uri": "ماخذ URI", "neopig_uri": "Neopig URI", "source_page": "ماخذ صفحہ", "neopig_page": "Neopig صفحہ",
|
||
"type_label": "قسم", "mime_label": "MIME", "size_label": "سائز", "alt_label": "Alt",
|
||
"keywords_label": "کلیدی الفاظ", "description_label": "تفصیل", "items": "آئٹمز", "bytes": "بائٹس",
|
||
"used_on": "استعمال شدہ", "page": "صفحہ", "discovered": "دریافت",
|
||
"console": "کنسول", "close": "بند کریں", "no_logs": "ابھی کوئی لاگ نہیں",
|
||
},
|
||
"sw": {
|
||
"search": "Tafuta", "crawl": "Tambaa", "live": "Moja kwa Moja", "about": "Kuhusu",
|
||
"random": "Nasibu", "phantom": "Phantom", "loading": "Inapakia...",
|
||
"start_crawl": "Anza Kutambaa", "pause": "Simamisha", "resume": "Endelea", "delete": "Futa",
|
||
"url": "URL", "mode": "Hali", "depth": "Kina", "keywords": "Maneno muhimu",
|
||
"screenshots": "Picha za skrini", "fast_mode": "Hali ya Haraka", "fresh_start": "Mwanzo Mpya",
|
||
"pages": "kurasa", "found": "zilizopatikana", "saved": "zilizohifadhiwa", "dupes": "nakala",
|
||
"running": "inaendelea", "completed": "imekamilika", "paused": "imesimamishwa",
|
||
"no_jobs": "Hakuna kazi", "started": "Ilianza", "filter": "Chuja",
|
||
"all_types": "Aina zote", "images": "Picha", "videos": "Video", "audio": "Sauti",
|
||
"everything": "Kila kitu", "all_media": "Media zote",
|
||
"images_only": "Picha tu", "videos_only": "Video tu", "text_only": "Maandishi tu",
|
||
"recent_jobs": "Kazi za Hivi Karibuni", "search_placeholder": "Tafuta media...",
|
||
"media": "Media", "pages_label": "Kurasa", "download": "Pakua",
|
||
"no_media": "Hakuna media iliyopatikana", "no_more": "Hakuna matokeo zaidi", "load_error": "Imeshindwa kupakia",
|
||
"copy": "Nakili", "copied": "Imenakiliwa!", "starting": "Inaanza...", "started_ok": "Imeanza!",
|
||
"error": "Hitilafu", "watching": "Inatazama picha mpya...", "content": "Maudhui",
|
||
"previous": "Iliyotangulia", "next": "Inayofuata", "over_9000": "Zaidi ya 9,000+!",
|
||
"loading_stats": "Inapakia takwimu...", "screenshot": "Picha ya skrini",
|
||
"valid_uri": "Tafadhali weka URI moja halali", "load_jobs_error": "Imeshindwa kupakia kazi",
|
||
"hydrate_subtitle": "Pata media kutoka wavuti", "live_subtitle": "Tazama picha zikitambaazwa",
|
||
"total_images": "Jumla ya Picha", "new_session": "Mpya Kipindi Hiki", "per_minute": "Kwa Dakika",
|
||
"all_domains": "Vikoa vyote", "download_phantom": "Pakua tovuti ya phantom",
|
||
"phantom_subtitle": "Hamisha kurasa zilizohifadhiwa kama tovuti tuli na media za ndani", "max_pages": "Kurasa za juu", "select_domain": "Chagua Kikoa", "sources": "vyanzo",
|
||
"source_uri": "URI Chanzo", "neopig_uri": "Neopig URI", "source_page": "Ukurasa Chanzo", "neopig_page": "Ukurasa Neopig",
|
||
"type_label": "Aina", "mime_label": "MIME", "size_label": "Ukubwa", "alt_label": "Alt",
|
||
"keywords_label": "Maneno muhimu", "description_label": "Maelezo", "items": "vitu", "bytes": "baiti",
|
||
"used_on": "Imetumika", "page": "Ukurasa", "discovered": "Iligunduliwa",
|
||
"console": "Konsoli", "close": "Funga", "no_logs": "Hakuna kumbukumbu bado",
|
||
},
|
||
"mr": {
|
||
"search": "शोध", "crawl": "क्रॉल", "live": "थेट फीड", "about": "बद्दल",
|
||
"random": "यादृच्छिक", "phantom": "फँटम", "loading": "लोड होत आहे...",
|
||
"start_crawl": "क्रॉल सुरू करा", "pause": "थांबवा", "resume": "पुन्हा सुरू करा", "delete": "हटवा",
|
||
"url": "URL", "mode": "मोड", "depth": "खोली", "keywords": "कीवर्ड",
|
||
"screenshots": "स्क्रीनशॉट", "fast_mode": "जलद मोड", "fresh_start": "नवीन सुरुवात",
|
||
"pages": "पृष्ठे", "found": "सापडले", "saved": "जतन केले", "dupes": "डुप्लिकेट",
|
||
"running": "चालू आहे", "completed": "पूर्ण", "paused": "थांबवले",
|
||
"no_jobs": "कोणतेही क्रॉल काम नाही", "started": "सुरू", "filter": "फिल्टर",
|
||
"all_types": "सर्व प्रकार", "images": "प्रतिमा", "videos": "व्हिडिओ", "audio": "ऑडिओ",
|
||
"everything": "सर्वकाही", "all_media": "सर्व मीडिया",
|
||
"images_only": "फक्त प्रतिमा", "videos_only": "फक्त व्हिडिओ", "text_only": "फक्त मजकूर",
|
||
"recent_jobs": "अलीकडील कामे", "search_placeholder": "मीडिया शोधा...",
|
||
"media": "मीडिया", "pages_label": "पृष्ठे", "download": "डाउनलोड",
|
||
"no_media": "मीडिया सापडला नाही", "no_more": "आणखी परिणाम नाहीत", "load_error": "लोड अयशस्वी",
|
||
"copy": "कॉपी", "copied": "कॉपी झाले!", "starting": "सुरू होत आहे...", "started_ok": "सुरू झाले!",
|
||
"error": "त्रुटी", "watching": "नवीन प्रतिमा पाहत आहे...", "content": "सामग्री",
|
||
"previous": "मागील", "next": "पुढील", "over_9000": "9,000+ पेक्षा जास्त!",
|
||
"loading_stats": "आकडेवारी लोड होत आहे...", "screenshot": "स्क्रीनशॉट",
|
||
"valid_uri": "कृपया किमान एक वैध URI प्रविष्ट करा", "load_jobs_error": "कामे लोड अयशस्वी",
|
||
"hydrate_subtitle": "वेबवरून मीडिया मिळवा", "live_subtitle": "क्रॉल होताना प्रतिमा पहा",
|
||
"total_images": "एकूण प्रतिमा", "new_session": "या सत्रात नवीन", "per_minute": "प्रति मिनिट",
|
||
"all_domains": "सर्व डोमेन", "download_phantom": "फँटम साइट डाउनलोड करा",
|
||
"phantom_subtitle": "संग्रहित पृष्ठे स्थानिक मीडियासह स्थिर साइट म्हणून निर्यात करा", "max_pages": "जास्तीत जास्त पृष्ठे", "select_domain": "डोमेन निवडा", "sources": "स्रोत",
|
||
"source_uri": "स्रोत URI", "neopig_uri": "Neopig URI", "source_page": "स्रोत पृष्ठ", "neopig_page": "Neopig पृष्ठ",
|
||
"type_label": "प्रकार", "mime_label": "MIME", "size_label": "आकार", "alt_label": "Alt",
|
||
"keywords_label": "कीवर्ड", "description_label": "वर्णन", "items": "आयटम", "bytes": "बाइट्स",
|
||
"used_on": "वापरलेले", "page": "पृष्ठ", "discovered": "शोधले",
|
||
"console": "कन्सोल", "close": "बंद करा", "no_logs": "अद्याप कोणतेही लॉग नाहीत",
|
||
},
|
||
"te": {
|
||
"search": "శోధన", "crawl": "క్రాల్", "live": "లైవ్ ఫీడ్", "about": "గురించి",
|
||
"random": "యాదృచ్ఛిక", "phantom": "ఫాంటమ్", "loading": "లోడ్ అవుతోంది...",
|
||
"start_crawl": "క్రాల్ ప్రారంభించు", "pause": "పాజ్", "resume": "పునఃప్రారంభించు", "delete": "తొలగించు",
|
||
"url": "URL", "mode": "మోడ్", "depth": "లోతు", "keywords": "కీవర్డ్లు",
|
||
"screenshots": "స్క్రీన్షాట్లు", "fast_mode": "వేగవంతమైన మోడ్", "fresh_start": "కొత్త ప్రారంభం",
|
||
"pages": "పేజీలు", "found": "కనుగొనబడింది", "saved": "సేవ్ చేయబడింది", "dupes": "నకిలీలు",
|
||
"running": "నడుస్తోంది", "completed": "పూర్తయింది", "paused": "పాజ్ చేయబడింది",
|
||
"no_jobs": "క్రాల్ జాబ్లు లేవు", "started": "ప్రారంభమైంది", "filter": "ఫిల్టర్",
|
||
"all_types": "అన్ని రకాలు", "images": "చిత్రాలు", "videos": "వీడియోలు", "audio": "ఆడియో",
|
||
"everything": "అన్నీ", "all_media": "అన్ని మీడియా",
|
||
"images_only": "చిత్రాలు మాత్రమే", "videos_only": "వీడియోలు మాత్రమే", "text_only": "వచనం మాత్రమే",
|
||
"recent_jobs": "ఇటీవలి జాబ్లు", "search_placeholder": "మీడియా శోధించండి...",
|
||
"media": "మీడియా", "pages_label": "పేజీలు", "download": "డౌన్లోడ్",
|
||
"no_media": "మీడియా కనుగొనబడలేదు", "no_more": "మరిన్ని ఫలితాలు లేవు", "load_error": "లోడ్ విఫలమైంది",
|
||
"copy": "కాపీ", "copied": "కాపీ అయింది!", "starting": "ప్రారంభమవుతోంది...", "started_ok": "ప్రారంభమైంది!",
|
||
"error": "లోపం", "watching": "కొత్త చిత్రాల కోసం చూస్తోంది...", "content": "కంటెంట్",
|
||
"previous": "మునుపటి", "next": "తదుపరి", "over_9000": "9,000+ కంటే ఎక్కువ!",
|
||
"loading_stats": "గణాంకాలు లోడ్ అవుతున్నాయి...", "screenshot": "స్క్రీన్షాట్",
|
||
"valid_uri": "దయచేసి కనీసం ఒక చెల్లుబాటు అయ్యే URI నమోదు చేయండి", "load_jobs_error": "జాబ్లు లోడ్ విఫలమైంది",
|
||
"hydrate_subtitle": "వెబ్ నుండి మీడియా పొందండి", "live_subtitle": "క్రాల్ అవుతున్న చిత్రాలను చూడండి",
|
||
"total_images": "మొత్తం చిత్రాలు", "new_session": "ఈ సెషన్లో కొత్తవి", "per_minute": "నిమిషానికి",
|
||
"all_domains": "అన్ని డొమైన్లు", "download_phantom": "ఫాంటమ్ సైట్ డౌన్లోడ్",
|
||
"phantom_subtitle": "ఆర్కైవ్ చేసిన పేజీలను స్థానిక మీడియాతో స్టాటిక్ సైట్గా ఎగుమతి చేయండి", "max_pages": "గరిష్ట పేజీలు", "select_domain": "డొమైన్ ఎంచుకోండి", "sources": "మూలాలు",
|
||
"source_uri": "మూల URI", "neopig_uri": "Neopig URI", "source_page": "మూల పేజీ", "neopig_page": "Neopig పేజీ",
|
||
"type_label": "రకం", "mime_label": "MIME", "size_label": "పరిమాణం", "alt_label": "Alt",
|
||
"keywords_label": "కీవర్డ్లు", "description_label": "వివరణ", "items": "అంశాలు", "bytes": "బైట్లు",
|
||
"used_on": "ఉపయోగించబడింది", "page": "పేజీ", "discovered": "కనుగొనబడింది",
|
||
"console": "కన్సోల్", "close": "మూసివేయి", "no_logs": "ఇంకా లాగ్లు లేవు",
|
||
},
|
||
}
|
||
|
||
def get_lang(lang_cookie: str = None, accept_language: str = None) -> str:
|
||
"""Get language from cookie first, then Accept-Language header."""
|
||
# Cookie takes priority (user's explicit choice)
|
||
if lang_cookie and lang_cookie in TRANSLATIONS:
|
||
return lang_cookie
|
||
# Fall back to Accept-Language header
|
||
if not accept_language:
|
||
return "en"
|
||
# Parse "en-US,en;q=0.9,zh-CN;q=0.8" format
|
||
for part in accept_language.split(','):
|
||
lang = part.split(';')[0].strip().split('-')[0].lower()
|
||
if lang in TRANSLATIONS:
|
||
return lang
|
||
return "en"
|
||
|
||
# Language names for the selector dropdown
|
||
LANG_NAMES = {
|
||
"en": "English", "zh": "中文", "zh-tw": "繁體中文", "es": "Español", "hi": "हिन्दी", "ar": "العربية",
|
||
"pt": "Português", "ru": "Русский", "ja": "日本語", "fr": "Français", "de": "Deutsch",
|
||
"ko": "한국어", "it": "Italiano", "nl": "Nederlands", "pl": "Polski", "tr": "Türkçe",
|
||
"vi": "Tiếng Việt", "th": "ไทย", "id": "Bahasa", "uk": "Українська", "sv": "Svenska",
|
||
"bn": "বাংলা", "ur": "اردو", "sw": "Kiswahili", "mr": "मराठी", "te": "తెలుగు",
|
||
}
|
||
|
||
def t(key: str, lang: str = "en") -> str:
|
||
"""Get translation for key in language."""
|
||
return TRANSLATIONS.get(lang, TRANSLATIONS["en"]).get(key, TRANSLATIONS["en"].get(key, key))
|
||
|
||
# Single source of truth for navigation - uses {{key}} placeholders
|
||
NAV_HTML = '''<div class="nav">
|
||
<a href="/" class="brand">🐷 neopig</a>
|
||
<a href="/">{{search}}</a>
|
||
<a href="/live">{{live}}</a>
|
||
<a href="/random">{{random}}</a>
|
||
<a href="/crawl">{{crawl}}</a>
|
||
<a href="/about">{{about}}</a>
|
||
</div>'''
|
||
|
||
# Single source of truth for search box
|
||
SEARCH_BOX_HTML = '''<form class="search-box" action="/" method="get">
|
||
<input type="text" name="q" placeholder="{{search_placeholder}}">
|
||
<select name="type">
|
||
<option value="">{{all_types}}</option>
|
||
<option value="image">{{images}}</option>
|
||
<option value="video">{{videos}}</option>
|
||
<option value="audio">{{audio}}</option>
|
||
</select>
|
||
<button type="submit">{{search}}</button>
|
||
</form>'''
|
||
|
||
def inject_i18n(html: str, lang: str) -> str:
|
||
"""Replace {key} placeholders and inject JS translations + language selector."""
|
||
trans = TRANSLATIONS.get(lang, TRANSLATIONS["en"])
|
||
# Add lang attribute and inject JS translations
|
||
html = html.replace('<html>', f'<html lang="{lang}">')
|
||
# Replace nav placeholder with actual nav (add import link if enabled)
|
||
nav = NAV_HTML
|
||
if IMPORT_MODE:
|
||
nav = nav.replace('<a href="/about">', '<a href="/import">Import</a>\n <a href="/about">')
|
||
html = html.replace('<!-- NAV -->', nav)
|
||
# Replace search placeholder with search box
|
||
html = html.replace('<!-- SEARCH -->', SEARCH_BOX_HTML)
|
||
# Build language selector options
|
||
lang_options = ''.join(f'<option value="{code}"{" selected" if code == lang else ""}>{name}</option>'
|
||
for code, name in LANG_NAMES.items())
|
||
lang_selector = f'''<select id="lang-select" onchange="setLang(this.value)" style="background:#1a1a1a;color:#888;border:1px solid #333;border-radius:4px;padding:4px 8px;font-size:12px;cursor:pointer;">{lang_options}</select>'''
|
||
lang_js = '''
|
||
function setLang(code) {
|
||
localStorage.setItem('neopig_lang', code);
|
||
document.cookie = 'lang=' + code + ';path=/;max-age=31536000';
|
||
location.reload();
|
||
}
|
||
'''
|
||
html = html.replace('<script>', f'<script>\nconst T={json.dumps(trans)};\n{lang_js}', 1)
|
||
# Insert language selector after {{about}} link in nav
|
||
html = html.replace('{{about}}</a>\n</div>', '{{about}}</a>\n ' + lang_selector + '\n</div>')
|
||
# Replace all {key} and {{key}} placeholders
|
||
for key, value in trans.items():
|
||
html = html.replace('{{' + key + '}}', value) # Double braces (static templates)
|
||
html = html.replace('{' + key + '}', value) # Single braces (after f-string)
|
||
return html
|
||
|
||
app = FastAPI(title="neopig", description="Media crawler + SERP + Screenshot service")
|
||
|
||
# Set up Jinja2 templates
|
||
TEMPLATES_PATH = Path(__file__).parent / "templates"
|
||
templates = Jinja2Templates(directory=str(TEMPLATES_PATH)) if TEMPLATES_PATH.exists() else None
|
||
|
||
# Try to include uri2png screenshot router (optional dependency)
|
||
try:
|
||
from uri2png import get_screenshot_router
|
||
app.include_router(get_screenshot_router())
|
||
logger.info("Screenshot router loaded from uri2png")
|
||
except ImportError:
|
||
logger.warning("uri2png not installed - screenshot endpoints not available")
|
||
|
||
# Mount static vendor files (we are the CDN)
|
||
STATIC_VENDOR_PATH = Path(__file__).parent / "static" / "vendor"
|
||
if STATIC_VENDOR_PATH.exists():
|
||
app.mount("/static/vendor", StaticFiles(directory=STATIC_VENDOR_PATH), name="vendor")
|
||
logger.info(f"Static vendor files mounted from {STATIC_VENDOR_PATH}")
|
||
|
||
# Mount static images
|
||
STATIC_IMAGES_PATH = Path(__file__).parent / "static" / "images"
|
||
if STATIC_IMAGES_PATH.exists():
|
||
app.mount("/static/images", StaticFiles(directory=STATIC_IMAGES_PATH), name="images")
|
||
logger.info(f"Static images mounted from {STATIC_IMAGES_PATH}")
|
||
|
||
# Config - set via startup
|
||
DB_PATH = "data/neopig.db"
|
||
VAULT_PATH = Path("data/vault")
|
||
LOGS_PATH = Path("data/logs")
|
||
CRAWL_DISABLED = os.environ.get("NEOPIG_DISABLE_CRAWL", "").lower() in ("1", "true", "yes")
|
||
IMPORT_MODE = os.environ.get("NEOPIG_IMPORT", "").lower() in ("1", "true", "yes")
|
||
|
||
# Global database instance
|
||
db: Database = None
|
||
|
||
# Per-job log handlers (job_id -> handler)
|
||
JOB_LOG_HANDLERS: Dict[int, logging.FileHandler] = {}
|
||
|
||
def start_job_logging(job_id: int) -> None:
|
||
"""Start capturing logs for a crawl job."""
|
||
LOGS_PATH.mkdir(parents=True, exist_ok=True)
|
||
log_file = LOGS_PATH / f"{job_id}.log"
|
||
handler = logging.FileHandler(log_file, mode='w', encoding='utf-8')
|
||
handler.setLevel(logging.INFO)
|
||
handler.setFormatter(logging.Formatter('%(asctime)s %(name)s %(levelname)s: %(message)s', datefmt='%H:%M:%S'))
|
||
# Add to root logger to capture all modules
|
||
logging.getLogger().addHandler(handler)
|
||
JOB_LOG_HANDLERS[job_id] = handler
|
||
|
||
def stop_job_logging(job_id: int) -> None:
|
||
"""Stop capturing logs for a crawl job."""
|
||
handler = JOB_LOG_HANDLERS.pop(job_id, None)
|
||
if handler:
|
||
handler.close()
|
||
logging.getLogger().removeHandler(handler)
|
||
|
||
def get_job_logs(job_id: int, tail: int = 0) -> str:
|
||
"""Get logs for a crawl job. If tail > 0, return only last N lines."""
|
||
log_file = LOGS_PATH / f"{job_id}.log"
|
||
if not log_file.exists():
|
||
return ""
|
||
content = log_file.read_text(encoding='utf-8')
|
||
if tail > 0:
|
||
lines = content.splitlines()
|
||
return '\n'.join(lines[-tail:])
|
||
return content
|
||
|
||
# Active crawl tasks (for cancellation on shutdown)
|
||
ACTIVE_CRAWL_TASKS: Dict[int, asyncio.Task] = {}
|
||
|
||
# Tarball mode - serve directly from tar.gz archive
|
||
import tarfile
|
||
import tempfile
|
||
TAR_PATH: str = None # Path to tarball (each request opens its own handle)
|
||
TAR_OFFSET: int = 0 # Offset for .run files (0 for plain .tar.gz)
|
||
TAR_MEMBERS: Dict[str, tarfile.TarInfo] = {} # Cached member info
|
||
TAR_MEDIA_INDEX: Dict[str, str] = {} # md5_hash -> full path (for O(1) lookup)
|
||
ARCHIVE_ROOT: str = None # e.g., "example.com-20251230"
|
||
TEMP_DB_PATH: str = None # Extracted database (SQLite needs real file)
|
||
|
||
|
||
# Base CSS shared by all pages
|
||
def _open_tarball():
|
||
"""Open a fresh tarball handle for this thread."""
|
||
if TAR_OFFSET > 0:
|
||
# .run file - need to seek past bootstrap
|
||
f = open(TAR_PATH, 'rb')
|
||
f.seek(TAR_OFFSET)
|
||
return tarfile.open(fileobj=f, mode='r:gz')
|
||
else:
|
||
return tarfile.open(TAR_PATH, 'r:gz')
|
||
|
||
|
||
def read_from_tarball(path: str) -> bytes:
|
||
"""Read a file from the tarball. Path is relative to archive root.
|
||
|
||
Thread-safe: opens its own tarball handle.
|
||
"""
|
||
if not TAR_PATH or not ARCHIVE_ROOT:
|
||
return None
|
||
full_path = f"{ARCHIVE_ROOT}/{path}"
|
||
if full_path in TAR_MEMBERS:
|
||
member = TAR_MEMBERS[full_path]
|
||
tar = _open_tarball()
|
||
try:
|
||
f = tar.extractfile(member)
|
||
if f:
|
||
return f.read()
|
||
finally:
|
||
tar.close()
|
||
return None
|
||
|
||
|
||
def find_media_in_tarball(md5_hash: str) -> tuple:
|
||
"""Find media file in tarball by hash. Returns (data, extension) or (None, None).
|
||
|
||
Thread-safe: opens its own tarball handle for parallel reads.
|
||
"""
|
||
if not TAR_PATH or not ARCHIVE_ROOT:
|
||
return None, None
|
||
|
||
# O(1) lookup via pre-built index
|
||
if md5_hash in TAR_MEDIA_INDEX:
|
||
name = TAR_MEDIA_INDEX[md5_hash]
|
||
member = TAR_MEMBERS.get(name)
|
||
if member:
|
||
tar = _open_tarball()
|
||
try:
|
||
f = tar.extractfile(member)
|
||
if f:
|
||
ext = Path(name).suffix
|
||
return f.read(), ext
|
||
finally:
|
||
tar.close()
|
||
|
||
return None, None
|
||
|
||
|
||
class ArchiveDB:
|
||
"""Simple sync SQLite wrapper for archive.db (FTS5 search only)."""
|
||
def __init__(self, db_path):
|
||
import sqlite3
|
||
self.conn = sqlite3.connect(db_path)
|
||
self.conn.row_factory = sqlite3.Row
|
||
|
||
async def search_pages(self, query, limit=50):
|
||
cursor = self.conn.execute(
|
||
"SELECT uri, title, snippet(pages_fts, 2, '<b>', '</b>', '...', 32) as snippet "
|
||
"FROM pages_fts WHERE pages_fts MATCH ? LIMIT ?",
|
||
(query, limit)
|
||
)
|
||
return [dict(row) for row in cursor.fetchall()]
|
||
|
||
async def get_stats(self):
|
||
cursor = self.conn.execute("SELECT COUNT(*) FROM pages")
|
||
return {"pages": cursor.fetchone()[0], "media": 0, "screenshots": 0}
|
||
|
||
|
||
@app.on_event("startup")
|
||
async def startup_event():
|
||
"""Initialize database on startup."""
|
||
global db
|
||
if CRAWL_DISABLED:
|
||
logger.info("Crawl disabled via NEOPIG_DISABLE_CRAWL")
|
||
if IMPORT_MODE:
|
||
logger.info("Import mode enabled via NEOPIG_IMPORT - archive uploads allowed")
|
||
if TAR_PATH:
|
||
# Tarball mode: use full Database for neopig.db, legacy ArchiveDB for archive.db
|
||
if DB_PATH.endswith('neopig.db'):
|
||
db = Database(DB_PATH)
|
||
await db.init()
|
||
logger.info(f"Using neopig database from tarball: {DB_PATH}")
|
||
else:
|
||
db = ArchiveDB(DB_PATH)
|
||
logger.info(f"Using archive database: {DB_PATH}")
|
||
else:
|
||
# Normal mode: use full async database
|
||
db = Database(DB_PATH)
|
||
await db.init() # Handles schema + WAL mode
|
||
# Backfill uri_hash for existing pages
|
||
count = await db.backfill_page_hashes()
|
||
if count:
|
||
logger.info(f"Backfilled {count} page URI hashes")
|
||
VAULT_PATH.mkdir(parents=True, exist_ok=True)
|
||
logger.info(f"Vault directory ready: {VAULT_PATH}")
|
||
|
||
# Mark any orphaned "running" jobs as "paused" (server was killed)
|
||
async with db.session() as session:
|
||
result = await session.execute(
|
||
text("UPDATE crawl_jobs SET status = 'paused' WHERE status = 'running'")
|
||
)
|
||
if result.rowcount:
|
||
await session.commit()
|
||
logger.info(f"Marked {result.rowcount} orphaned running job(s) as paused")
|
||
|
||
|
||
@app.on_event("shutdown")
|
||
async def shutdown_event():
|
||
"""Pause active crawls on shutdown so they can be resumed."""
|
||
if ACTIVE_CRAWL_TASKS:
|
||
logger.info(f"Pausing {len(ACTIVE_CRAWL_TASKS)} active crawl(s)...")
|
||
for job_id, task in list(ACTIVE_CRAWL_TASKS.items()):
|
||
task.cancel()
|
||
try:
|
||
await task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
# Mark as paused so it can be resumed
|
||
try:
|
||
await db.pause_crawl_job(job_id)
|
||
except Exception as e:
|
||
logger.error(f"Failed to pause job {job_id}: {e}")
|
||
logger.info("All crawls paused")
|
||
|
||
|
||
class CrawlRequest(BaseModel):
|
||
"""Request to start a new crawl."""
|
||
targets: List[str] = [] # Multiple target URIs
|
||
target_uri: str = "" # Deprecated: single target (for backwards compat)
|
||
keywords: List[str] = []
|
||
mode: str = "all" # text, images, videos, media, all
|
||
depth: int = -1 # -1 = unlimited
|
||
max_pages: int = -1 # -1 = unlimited
|
||
fresh: bool = False # Start fresh (rotate state files) - default False for safety
|
||
fast: bool = False # No crawl delay
|
||
screenshots: bool = True # Take page screenshots
|
||
hydra: bool = False # Parse RSS/Atom feeds for bleeding edge discovery
|
||
|
||
|
||
@app.get("/", response_class=HTMLResponse)
|
||
async def index(request: Request, lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Simple search UI."""
|
||
language = get_lang(lang, accept_language)
|
||
t = TRANSLATIONS.get(language, TRANSLATIONS["en"])
|
||
|
||
if templates:
|
||
return templates.TemplateResponse("search.html.j2", {
|
||
"request": request,
|
||
"t": t,
|
||
"t_json": json.dumps(t),
|
||
"lang": language,
|
||
"langs": LANG_NAMES,
|
||
"import_mode": IMPORT_MODE,
|
||
})
|
||
|
||
raise HTTPException(status_code=500, detail="Templates required")
|
||
|
||
|
||
@app.get("/crawl", response_class=HTMLResponse)
|
||
async def crawl_page(request: Request, lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Crawler command page."""
|
||
language = get_lang(lang, accept_language)
|
||
t = TRANSLATIONS.get(language, TRANSLATIONS["en"])
|
||
|
||
# Use Jinja2 template if available, otherwise fall back to inline HTML
|
||
if templates:
|
||
return templates.TemplateResponse("crawl.html.j2", {
|
||
"request": request,
|
||
"t": t,
|
||
"t_json": json.dumps(t),
|
||
"lang": language,
|
||
"langs": LANG_NAMES,
|
||
"import_mode": IMPORT_MODE,
|
||
"crawl_disabled": CRAWL_DISABLED,
|
||
})
|
||
raise HTTPException(status_code=500, detail="Templates required")
|
||
|
||
|
||
# Page-specific CSS for Search page
|
||
|
||
|
||
|
||
|
||
|
||
@app.get("/live", response_class=HTMLResponse)
|
||
async def live_page(request: Request, domain: str = Query(None), lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Live feed page - watch images appear as they're crawled."""
|
||
language = get_lang(lang, accept_language)
|
||
t = TRANSLATIONS.get(language, TRANSLATIONS["en"])
|
||
|
||
if templates:
|
||
return templates.TemplateResponse("live.html.j2", {
|
||
"request": request,
|
||
"t": t,
|
||
"t_json": json.dumps(t),
|
||
"domain_json": json.dumps(domain or ""),
|
||
"domain": domain,
|
||
"lang": language,
|
||
"langs": LANG_NAMES,
|
||
"import_mode": IMPORT_MODE,
|
||
})
|
||
raise HTTPException(status_code=500, detail="Templates required")
|
||
|
||
|
||
@app.get("/view/{md5_hash}", response_class=HTMLResponse)
|
||
async def view_media_page(request: Request, md5_hash: str, noai: bool = Query(False), lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Detail view page for a single media item."""
|
||
language = get_lang(lang, accept_language)
|
||
import html as html_module
|
||
import re
|
||
from urllib.parse import quote, urljoin
|
||
|
||
async with db.session() as session:
|
||
result = await session.execute(
|
||
text("SELECT * FROM media WHERE md5_hash = :hash"),
|
||
{'hash': md5_hash}
|
||
)
|
||
media = result.fetchone()
|
||
if not media:
|
||
raise HTTPException(status_code=404, detail="Media not found")
|
||
|
||
result = await session.execute(
|
||
text("""SELECT media_uri, page_uri, page_title, page_content,
|
||
detail_page_uri, detail_title, detail_content, discovered_at
|
||
FROM media_sources WHERE md5_hash = :hash"""),
|
||
{'hash': md5_hash}
|
||
)
|
||
sources = result.fetchall()
|
||
|
||
media = dict(media._mapping)
|
||
sources = [dict(s._mapping) for s in sources]
|
||
keywords = json.loads(media.get('keywords') or '[]')
|
||
sorted_sources = sorted(sources, key=lambda s: len(s["page_uri"] or ""), reverse=True)
|
||
|
||
# Get page URI
|
||
page_uri = sorted_sources[0]["page_uri"] if sorted_sources else None
|
||
source_uri = page_uri or (sorted_sources[0]["media_uri"] if sorted_sources else "unknown")
|
||
source_domain = Uri(source_uri).hostname if source_uri else "unknown"
|
||
|
||
# Display title
|
||
display_title = media.get('alt_text') or media.get('title')
|
||
if not display_title and sources:
|
||
display_title = sources[0].get('page_title')
|
||
if not display_title:
|
||
display_title = f"Media {md5_hash[:12]}"
|
||
|
||
# Hero: media element
|
||
is_video = media['media_type'] == 'video'
|
||
is_audio = media['media_type'] == 'audio'
|
||
is_code = media['media_type'] == 'code'
|
||
is_style = media['media_type'] == 'style'
|
||
is_font = media['media_type'] == 'font'
|
||
is_text_file = is_code or is_style # text files shown with syntax highlighting
|
||
if is_video:
|
||
hero_html = f'<a href="/media/{md5_hash}" target="_blank"><video src="/media/{md5_hash}" controls muted loop style="max-width:100%;max-height:70vh;"></video></a>'
|
||
elif is_audio:
|
||
hero_html = f'<audio src="/media/{md5_hash}" controls></audio>'
|
||
elif is_text_file:
|
||
# Fetch code/style content and display with syntax highlighting
|
||
code_content = ""
|
||
lang_class = 'css' if is_style else (media.get('alt_text') or '')
|
||
try:
|
||
vault_path = VAULT_PATH / hash_to_path(md5_hash)
|
||
# Find file with this hash
|
||
subdir = vault_path.parent
|
||
for f in subdir.iterdir():
|
||
if f.stem == md5_hash:
|
||
code_content = f.read_text(errors='replace')[:100000] # Limit size
|
||
break
|
||
except Exception:
|
||
code_content = "(Unable to load file content)"
|
||
escaped_code = html_module.escape(code_content)
|
||
hero_html = f'<pre style="max-height:70vh;overflow:auto;background:#1e1e1e;padding:15px;border-radius:8px;text-align:left;white-space:pre-wrap;word-wrap:break-word;"><code class="language-{lang_class}">{escaped_code}</code></pre>'
|
||
elif is_font:
|
||
# Font preview with sample text
|
||
hero_html = f'''<div style="background:#1e1e1e;padding:30px;border-radius:8px;text-align:center;">
|
||
<style>@font-face {{ font-family: "preview-{md5_hash[:8]}"; src: url("/media/{md5_hash}"); }}</style>
|
||
<div style="font-family:'preview-{md5_hash[:8]}',sans-serif;font-size:48px;color:#fff;margin-bottom:20px;">Aa Bb Cc</div>
|
||
<div style="font-family:'preview-{md5_hash[:8]}',sans-serif;font-size:24px;color:#ccc;">The quick brown fox jumps over the lazy dog</div>
|
||
<div style="font-family:'preview-{md5_hash[:8]}',sans-serif;font-size:16px;color:#888;margin-top:15px;">0123456789 !@#$%^&*()</div>
|
||
</div>'''
|
||
else:
|
||
hero_html = f'<a href="/media/{md5_hash}" target="_blank"><img src="/media/{md5_hash}" alt="{html_module.escape(media.get("alt_text") or "")}" style="max-width:100%;max-height:70vh;"></a>'
|
||
|
||
# Metadata rows - (key, value) where key is looked up via t.get(key, key)
|
||
import hashlib
|
||
media_uri = sorted_sources[0]["media_uri"] if sorted_sources else None
|
||
neopig_media_uri = f"/view/{md5_hash}"
|
||
page_uri_hash = hashlib.md5(page_uri.encode()).hexdigest() if page_uri else None
|
||
neopig_page_uri = f"/page/{page_uri_hash}" if page_uri_hash else None
|
||
keywords_html = ''.join([f'<span class="tag">{k}</span>' for k in keywords]) or '-'
|
||
meta_rows = [
|
||
("source_uri", f'<a href="{media_uri}" target="_blank" style="font-size:11px;">{media_uri}</a>' if media_uri else '-'),
|
||
("neopig_uri", f'<a href="{neopig_media_uri}" style="font-size:11px;">{neopig_media_uri}</a>'),
|
||
("source_page", f'<a href="{page_uri}" target="_blank" style="font-size:11px;">{page_uri}</a>' if page_uri else '-'),
|
||
("neopig_page", f'<a href="{neopig_page_uri}" style="font-size:11px;">{neopig_page_uri}</a>' if neopig_page_uri else '-'),
|
||
("MD5", f'<code style="font-size:11px;">{md5_hash}</code>'),
|
||
("type_label", media['media_type']),
|
||
("mime_label", media.get('mime_type') or 'unknown'),
|
||
("size_label", f"{media.get('file_size') or 0:,} bytes"),
|
||
("alt_label", media.get('alt_text') or '-'),
|
||
("keywords_label", keywords_html),
|
||
]
|
||
|
||
# Download button - prioritize original filename from URL
|
||
download_filename = None
|
||
if sources and sources[0].get('media_uri'):
|
||
from urllib.parse import unquote
|
||
parsed = Uri(sources[0]['media_uri'])
|
||
orig_name = Path(unquote(parsed.path)).name
|
||
if orig_name and '.' in orig_name:
|
||
download_filename = orig_name
|
||
# Fallback to alt_text/title
|
||
if not download_filename:
|
||
name_source = media.get('alt_text') or media.get('title')
|
||
if name_source:
|
||
ext_map = {'image/jpeg': '.jpg', 'image/png': '.png', 'image/gif': '.gif', 'image/webp': '.webp',
|
||
'video/mp4': '.mp4', 'video/webm': '.webm', 'audio/mpeg': '.mp3', 'audio/wav': '.wav',
|
||
'text/javascript': '.js', 'text/python': '.py', 'text/css': '.css', 'text/html': '.html',
|
||
'text/markdown': '.md', 'text/rust': '.rs', 'text/go': '.go', 'text/c': '.c', 'text/cpp': '.cpp'}
|
||
ext = ext_map.get(media.get('mime_type', ''), '')
|
||
if not ext:
|
||
try:
|
||
subdir = VAULT_PATH / hash_to_path(md5_hash).parent
|
||
for f in subdir.iterdir():
|
||
if f.stem == md5_hash:
|
||
ext = f.suffix
|
||
break
|
||
except Exception:
|
||
pass
|
||
download_filename = f"{slugify(name_source)}{ext}"
|
||
# Fallback to page_title + hash
|
||
if not download_filename and sources:
|
||
pt = sources[0].get('page_title', '')
|
||
media_idx = int(md5_hash[:4], 16)
|
||
name_source = f"{pt}-{media_idx}" if pt else f"media-{media_idx}"
|
||
download_filename = f"{slugify(name_source)}.bin"
|
||
# Default
|
||
if not download_filename:
|
||
download_filename = f"{md5_hash[:12]}.bin"
|
||
download_btn = f'<a href="/media/{md5_hash}?download=1" style="display:block;padding:12px 20px;background:#6bff6b;color:#000;text-decoration:none;border-radius:6px;text-align:center;font-weight:500;margin-top:15px;">Download ({download_filename})</a>'
|
||
|
||
# Get page media items (siblings)
|
||
media_items = await db.get_page_media(page_uri) if page_uri else []
|
||
# Exclude current item from gallery
|
||
media_items = [m for m in media_items if m.get("md5_hash") != md5_hash]
|
||
|
||
# Get rendered content
|
||
content_html = ""
|
||
if page_uri:
|
||
page_row = await db.get_page_by_uri(page_uri)
|
||
if page_row and page_row.get("markdown"):
|
||
try:
|
||
import markdown
|
||
md_converter = markdown.Markdown(extensions=['fenced_code', 'tables', 'nl2br'])
|
||
content_html = md_converter.convert(page_row["markdown"][:100000])
|
||
# Hydrate images and links
|
||
img_urls = re.findall(r'<img[^>]+src=["\']([^"\']+)["\']', content_html, re.I)
|
||
link_urls = re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', content_html, re.I)
|
||
all_urls = list(set(img_urls + link_urls))
|
||
if all_urls:
|
||
resolved = {u: u if u.startswith(('http://', 'https://', '//')) else urljoin(page_uri, u) for u in all_urls if not u.startswith('#')}
|
||
url_to_hash = await db.lookup_media_by_uris(list(set(resolved.values())))
|
||
exact_matches = set(url_to_hash.keys()) # These are reliable
|
||
|
||
# Fallback for imgur and other CDNs - lookup by filename/ID
|
||
# Only apply to URLs that look like media files (have media extension)
|
||
from async_web_fetcher import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS, AUDIO_EXTENSIONS
|
||
from pathlib import Path as P
|
||
media_exts = IMAGE_EXTENSIONS | VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
|
||
for orig, res in resolved.items():
|
||
if res not in url_to_hash:
|
||
url_path = res.split('?')[0]
|
||
ext = P(url_path).suffix.lower()
|
||
if ext in media_exts:
|
||
fname = P(url_path).stem
|
||
if fname and len(fname) >= 5:
|
||
result = await db.lookup_media_by_filename(fname)
|
||
if result:
|
||
url_to_hash[res] = result[1]
|
||
|
||
for orig, res in resolved.items():
|
||
if res in url_to_hash:
|
||
md5 = url_to_hash[res]
|
||
content_html = content_html.replace(f'src="{orig}"', f'src="/media/{md5}"')
|
||
content_html = content_html.replace(f"src='{orig}'", f'src="/media/{md5}"')
|
||
# Only rewrite hrefs for exact matches or URLs with media extensions
|
||
url_path = res.split('?')[0]
|
||
ext = P(url_path).suffix.lower()
|
||
if res in exact_matches or ext in media_exts:
|
||
content_html = content_html.replace(f'href="{orig}"', f'href="/view/{md5}"')
|
||
content_html = content_html.replace(f"href='{orig}'", f'href="/view/{md5}"')
|
||
except ImportError:
|
||
content_html = f"<pre>{html_module.escape(page_row['markdown'][:50000])}</pre>"
|
||
|
||
# Get screenshots (exclude current if it's a screenshot)
|
||
screenshot_hashes = await db.get_page_screenshots(page_uri) if page_uri else []
|
||
screenshot_hashes = [h for h in screenshot_hashes if h != md5_hash]
|
||
|
||
# Build "Used on X pages" section (reverse image search)
|
||
t = TRANSLATIONS.get(language, TRANSLATIONS["en"])
|
||
sources_html = ""
|
||
if sources and len(sources) > 0:
|
||
import hashlib as hl
|
||
unique_pages = {}
|
||
for s in sources:
|
||
pu = s.get('page_uri', '')
|
||
if pu and pu not in unique_pages:
|
||
unique_pages[pu] = s
|
||
if unique_pages:
|
||
rows_html = ""
|
||
for pu, s in unique_pages.items():
|
||
pt = html_module.escape(s.get('page_title', '') or pu[:60])
|
||
ph = hl.md5(pu.encode()).hexdigest()
|
||
rows_html += f'''<tr class="source-row">
|
||
<td><a href="/page/{ph}" title="{html_module.escape(pu)}">{pt}</a></td>
|
||
<td style="font-size:11px;color:#888;">{html_module.escape(s.get('discovered_at', '')[:10] if s.get('discovered_at') else '-')}</td>
|
||
</tr>'''
|
||
total_count = len(unique_pages)
|
||
count_display = f"{OVER_9000:,}" if total_count >= OVER_9000 else str(total_count)
|
||
sources_html = f'''
|
||
<div class="sources-section">
|
||
<h3 onclick="toggleSources()">
|
||
<span id="sources-toggle">▶</span> {t["used_on"]} ''' + count_display + f''' {t["pages"]}
|
||
</h3>
|
||
<div id="sources-list" class="sources-list">
|
||
<div id="sources-pagination" style="margin-bottom:10px;display:flex;gap:10px;align-items:center;">
|
||
<button onclick="sourcesPage(-1)" id="sources-prev" style="padding:5px 12px;background:#333;border:none;color:#fff;border-radius:4px;cursor:pointer;">←</button>
|
||
<span id="sources-page-info" style="color:#888;font-size:13px;">1 / 1</span>
|
||
<button onclick="sourcesPage(1)" id="sources-next" style="padding:5px 12px;background:#333;border:none;color:#fff;border-radius:4px;cursor:pointer;">→</button>
|
||
</div>
|
||
<table>
|
||
<thead><tr><th>{t["page"]}</th><th>{t["discovered"]}</th></tr></thead>
|
||
<tbody id="sources-tbody">''' + rows_html + '''</tbody>
|
||
</table>
|
||
</div>
|
||
</div>
|
||
<script>
|
||
(function() {
|
||
const perPage = 9000;
|
||
const rows = document.querySelectorAll('#sources-tbody .source-row');
|
||
const total = rows.length;
|
||
const totalPages = Math.ceil(total / perPage);
|
||
let currentPage = 1;
|
||
function showPage(page) {
|
||
currentPage = Math.max(1, Math.min(page, totalPages));
|
||
const start = (currentPage - 1) * perPage;
|
||
const end = start + perPage;
|
||
rows.forEach((row, i) => { row.style.display = (i >= start && i < end) ? '' : 'none'; });
|
||
document.getElementById('sources-page-info').textContent = currentPage + ' / ' + totalPages;
|
||
document.getElementById('sources-prev').disabled = currentPage === 1;
|
||
document.getElementById('sources-next').disabled = currentPage === totalPages;
|
||
}
|
||
window.sourcesPage = function(delta) { showPage(currentPage + delta); };
|
||
showPage(1);
|
||
|
||
// Toggle sources with localStorage persistence
|
||
window.toggleSources = function() {
|
||
const list = document.getElementById('sources-list');
|
||
const toggle = document.getElementById('sources-toggle');
|
||
const expanded = !list.classList.contains('expanded');
|
||
list.classList.toggle('expanded', expanded);
|
||
toggle.textContent = expanded ? '▼' : '▶';
|
||
localStorage.setItem('neopig_sources_expanded', expanded);
|
||
};
|
||
|
||
// Restore from localStorage
|
||
if (localStorage.getItem('neopig_sources_expanded') === 'true') {
|
||
document.getElementById('sources-list').classList.add('expanded');
|
||
document.getElementById('sources-toggle').textContent = '▼';
|
||
}
|
||
})();
|
||
</script>'''
|
||
|
||
return templates.TemplateResponse("view.html.j2", {
|
||
"request": request,
|
||
"t": TRANSLATIONS.get(language, TRANSLATIONS["en"]),
|
||
"t_json": json.dumps(TRANSLATIONS.get(language, TRANSLATIONS["en"])),
|
||
"title": display_title,
|
||
"hero_html": hero_html,
|
||
"meta_rows": meta_rows,
|
||
"media_items": media_items,
|
||
"content_html": content_html,
|
||
"screenshot_hashes": screenshot_hashes,
|
||
"source_domain": source_domain,
|
||
"page_uri": page_uri or "",
|
||
"download_btn_html": download_btn,
|
||
"noai": noai,
|
||
"sources_html": sources_html,
|
||
"lang": language,
|
||
"langs": LANG_NAMES,
|
||
"import_mode": IMPORT_MODE,
|
||
})
|
||
|
||
|
||
|
||
@app.get("/page/{uri_hash}", response_class=HTMLResponse)
|
||
async def view_page_by_hash(request: Request,
|
||
uri_hash: str,
|
||
noai: bool = Query(False, description="Disable AI assistant"),
|
||
lang: str = Cookie(None),
|
||
accept_language: str = Header(None),
|
||
):
|
||
"""View an archived page by URI hash."""
|
||
page = await db.get_page_by_hash(uri_hash)
|
||
if not page:
|
||
raise HTTPException(status_code=404, detail="Page not found")
|
||
# Redirect to the URI-based view (reuses same logic)
|
||
return await view_page(request=request, uri=page['uri'], noai=noai, lang=lang, accept_language=accept_language)
|
||
|
||
|
||
@app.get("/page/view", response_class=HTMLResponse)
|
||
async def view_page(request: Request,
|
||
uri: str = Query(..., description="Page URI to view"),
|
||
noai: bool = Query(False, description="Disable AI assistant"),
|
||
lang: str = Cookie(None),
|
||
accept_language: str = Header(None),
|
||
):
|
||
"""View an archived page with markdown and screenshot."""
|
||
language = get_lang(lang, accept_language)
|
||
import html as html_module
|
||
import re
|
||
from urllib.parse import urljoin, quote
|
||
|
||
source_domain = Uri(uri).hostname
|
||
|
||
page = await db.get_page_by_uri(uri)
|
||
if not page:
|
||
raise HTTPException(status_code=404, detail="Page not found")
|
||
|
||
page_title = page.get("title") or uri
|
||
|
||
# Render markdown
|
||
content_html = ""
|
||
if page.get("markdown"):
|
||
try:
|
||
import markdown
|
||
md_converter = markdown.Markdown(extensions=['fenced_code', 'tables', 'nl2br'])
|
||
content_html = md_converter.convert(page["markdown"][:100000])
|
||
|
||
# Hydrate images and links from vault
|
||
img_urls = re.findall(r'<img[^>]+src=["\']([^"\']+)["\']', content_html, re.I)
|
||
link_urls = re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', content_html, re.I)
|
||
all_urls = list(set(img_urls + link_urls))
|
||
if all_urls:
|
||
# Keep anchor-only links (#foo) as-is, resolve others
|
||
resolved = {}
|
||
for u in all_urls:
|
||
if u.startswith('#'):
|
||
continue # Skip anchor-only links, they stay internal
|
||
elif u.startswith(('http://', 'https://', '//')):
|
||
resolved[u] = u
|
||
else:
|
||
resolved[u] = urljoin(uri, u)
|
||
|
||
# Lookup media (images, etc) - track which came from exact match vs fallback
|
||
url_to_hash = await db.lookup_media_by_uris(list(set(resolved.values())))
|
||
exact_matches = set(url_to_hash.keys()) # These are reliable
|
||
|
||
# Fallback for imgur and other CDNs - lookup by filename/ID
|
||
# Only apply to URLs that look like media files (have media extension)
|
||
from async_web_fetcher import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS, AUDIO_EXTENSIONS
|
||
from pathlib import Path as P
|
||
media_exts = IMAGE_EXTENSIONS | VIDEO_EXTENSIONS | AUDIO_EXTENSIONS
|
||
for murl in [u for u in resolved.values() if u not in url_to_hash]:
|
||
url_path = murl.split('?')[0]
|
||
ext = P(url_path).suffix.lower()
|
||
# Only do filename fallback for URLs with media extensions
|
||
if ext in media_exts:
|
||
fname = P(url_path).stem
|
||
if fname and len(fname) >= 5:
|
||
result = await db.lookup_media_by_filename(fname)
|
||
if result:
|
||
url_to_hash[murl] = result[1]
|
||
|
||
# Lookup pages for internal link rewriting
|
||
page_links = [res for res in resolved.values() if source_domain and source_domain in res]
|
||
uri_to_page_hash = await db.lookup_pages_by_uris(page_links) if page_links else {}
|
||
|
||
for orig, res in resolved.items():
|
||
if res in url_to_hash:
|
||
md5 = url_to_hash[res]
|
||
# Rewrite img src to serve media directly
|
||
content_html = content_html.replace(f'src="{orig}"', f'src="/media/{md5}"')
|
||
content_html = content_html.replace(f"src='{orig}'", f'src="/media/{md5}"')
|
||
# Only rewrite hrefs for exact matches or URLs with media extensions
|
||
# (avoid rewriting external links that happen to match by filename)
|
||
url_path = res.split('?')[0]
|
||
ext = P(url_path).suffix.lower()
|
||
if res in exact_matches or ext in media_exts:
|
||
content_html = content_html.replace(f'href="{orig}"', f'href="/view/{md5}"')
|
||
content_html = content_html.replace(f"href='{orig}'", f'href="/view/{md5}"')
|
||
elif res in uri_to_page_hash:
|
||
# Rewrite internal page links to archived versions
|
||
page_hash = uri_to_page_hash[res]
|
||
content_html = content_html.replace(f'href="{orig}"', f'href="/page/{page_hash}"')
|
||
content_html = content_html.replace(f"href='{orig}'", f'href="/page/{page_hash}"')
|
||
except ImportError:
|
||
content_html = f"<pre>{html_module.escape(page.get('markdown', '')[:50000])}</pre>"
|
||
elif page.get("content"):
|
||
escaped = html_module.escape(page["content"][:50000])
|
||
content_html = f"<pre style='white-space:pre-wrap;'>{escaped}</pre>"
|
||
|
||
# Get screenshots and media
|
||
screenshot_hashes = await db.get_page_screenshots(uri)
|
||
media_items = await db.get_page_media(uri)
|
||
|
||
# Metadata rows - (key, value) where key is looked up via t.get(key, key)
|
||
import hashlib
|
||
uri_hash = hashlib.md5(uri.encode()).hexdigest()
|
||
neopig_page_uri = f"/page/{uri_hash}"
|
||
keywords = json.loads(page.get('keywords') or '[]') if page.get('keywords') else []
|
||
keywords_html = ''.join([f'<span class="tag">{k}</span>' for k in keywords]) or '-'
|
||
meta_rows = [
|
||
("source_uri", f'<a href="{uri}" target="_blank" style="font-size:11px;">{uri}</a>'),
|
||
("neopig_uri", f'<a href="{neopig_page_uri}" style="font-size:11px;">{neopig_page_uri}</a>'),
|
||
("description_label", page.get('description') or '-'),
|
||
("keywords_label", keywords_html),
|
||
("media", f"{len(media_items)}"),
|
||
]
|
||
|
||
return templates.TemplateResponse("view.html.j2", {
|
||
"request": request,
|
||
"t": TRANSLATIONS.get(language, TRANSLATIONS["en"]),
|
||
"t_json": json.dumps(TRANSLATIONS.get(language, TRANSLATIONS["en"])),
|
||
"title": page_title,
|
||
"hero_html": "",
|
||
"meta_rows": meta_rows,
|
||
"media_items": media_items,
|
||
"content_html": content_html,
|
||
"screenshot_hashes": screenshot_hashes,
|
||
"source_domain": source_domain,
|
||
"page_uri": uri,
|
||
"download_btn_html": "",
|
||
"noai": noai,
|
||
"sources_html": "",
|
||
"lang": language,
|
||
"langs": LANG_NAMES,
|
||
"import_mode": IMPORT_MODE,
|
||
})
|
||
@app.get("/phantom/export")
|
||
async def phantom_export(domain: str = Query(None, description="Filter by domain")):
|
||
"""
|
||
Export a phantom HTML site - original HTML with media URLs rewritten to vault.
|
||
|
||
Creates a downloadable zip of the phantom site ready for static hosting.
|
||
"""
|
||
import io
|
||
import re
|
||
import zipfile
|
||
from urllib.parse import urljoin
|
||
from fastapi.responses import StreamingResponse
|
||
|
||
# Get all pages with raw_html
|
||
pages = await db.get_pages_by_domain(domain)
|
||
|
||
if not pages:
|
||
raise HTTPException(status_code=404, detail="No pages with raw HTML found")
|
||
|
||
# Get all media URL to hash mappings
|
||
url_to_hash = await db.get_all_media_uri_mappings()
|
||
|
||
# Create zip in memory
|
||
zip_buffer = io.BytesIO()
|
||
|
||
with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||
pages_written = 0
|
||
media_hashes = set()
|
||
|
||
for page in pages:
|
||
uri = page['uri']
|
||
raw_html = page.get('raw_html')
|
||
if not raw_html:
|
||
continue
|
||
|
||
# Parse URI to get path
|
||
parsed = Uri(uri)
|
||
site_domain = parsed.hostname
|
||
path = parsed.path.strip('/') or 'index'
|
||
if not path.endswith('.html') and '.' not in path.split('/')[-1]:
|
||
path = f"{path}/index.html" if path else "index.html"
|
||
|
||
# Rewrite media URLs to local paths
|
||
html = raw_html
|
||
|
||
# Find all src and href attributes pointing to media
|
||
patterns = [
|
||
(r'src=["\']([^"\']+)["\']', 'src'),
|
||
(r'href=["\']([^"\']+\.(jpg|jpeg|png|gif|webp|mp4|webm|svg|ico))["\']', 'href'),
|
||
]
|
||
|
||
for pattern, attr in patterns:
|
||
matches = re.findall(pattern, html, re.IGNORECASE)
|
||
for match in matches:
|
||
url = match[0] if isinstance(match, tuple) else match
|
||
|
||
# Resolve relative URLs
|
||
full_url = urljoin(uri, url)
|
||
|
||
# Check if we have this media
|
||
if full_url in url_to_hash:
|
||
md5 = url_to_hash[full_url]
|
||
media_hashes.add(md5)
|
||
# Replace with local path
|
||
ext = Path(url).suffix or '.bin'
|
||
local_path = f"media/{md5}{ext}"
|
||
html = html.replace(f'{attr}="{url}"', f'{attr}="{local_path}"')
|
||
html = html.replace(f"{attr}='{url}'", f'{attr}="{local_path}"')
|
||
elif url in url_to_hash:
|
||
md5 = url_to_hash[url]
|
||
media_hashes.add(md5)
|
||
ext = Path(url).suffix or '.bin'
|
||
local_path = f"media/{md5}{ext}"
|
||
html = html.replace(f'{attr}="{url}"', f'{attr}="{local_path}"')
|
||
html = html.replace(f"{attr}='{url}'", f'{attr}="{local_path}"')
|
||
|
||
# Write HTML file
|
||
zf.writestr(f"site/{path}", html.encode('utf-8'))
|
||
pages_written += 1
|
||
|
||
# Copy media files from vault
|
||
media_copied = 0
|
||
for md5 in media_hashes:
|
||
subdir = VAULT_PATH / md5[:2]
|
||
if subdir.exists():
|
||
for f in subdir.iterdir():
|
||
if f.name.startswith(md5):
|
||
ext = f.suffix or '.bin'
|
||
zf.write(f, f"site/media/{md5}{ext}")
|
||
media_copied += 1
|
||
break
|
||
|
||
# Write index
|
||
index_html = f"""<!DOCTYPE html>
|
||
<html>
|
||
<head>
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<title>Phantom Site - {site_domain}</title>
|
||
<style>
|
||
body {{ font-family: sans-serif; max-width: 800px; margin: 50px auto; padding: 20px; }}
|
||
h1 {{ color: #333; }}
|
||
ul {{ line-height: 2; }}
|
||
a {{ color: #0066cc; }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<h1>Phantom Site Archive</h1>
|
||
<p>Domain: {site_domain}</p>
|
||
<p>Pages: {pages_written}</p>
|
||
<p>Media: {media_copied}</p>
|
||
<h2>Pages</h2>
|
||
<ul>
|
||
"""
|
||
for page in pages[:100]:
|
||
parsed = Uri(page['uri'])
|
||
path = parsed.path.strip('/') or 'index'
|
||
if not path.endswith('.html') and '.' not in path.split('/')[-1]:
|
||
path = f"{path}/index.html" if path else "index.html"
|
||
title = page.get('title') or path
|
||
index_html += f' <li><a href="{path}">{title}</a></li>\n'
|
||
|
||
index_html += """ </ul>
|
||
</body>
|
||
</html>"""
|
||
zf.writestr("site/phantom_index.html", index_html.encode('utf-8'))
|
||
|
||
# Return zip
|
||
zip_buffer.seek(0)
|
||
return StreamingResponse(
|
||
zip_buffer,
|
||
media_type="application/zip",
|
||
headers={"Content-Disposition": f"attachment; filename=phantom_{site_domain or 'site'}.zip"}
|
||
)
|
||
|
||
|
||
@app.get("/phantom", response_class=HTMLResponse)
|
||
async def phantom_page(request: Request, lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Phantom site export UI."""
|
||
language = get_lang(lang, accept_language)
|
||
t = TRANSLATIONS.get(language, TRANSLATIONS["en"])
|
||
domains = await db.get_domains_with_pages()
|
||
|
||
if templates:
|
||
return templates.TemplateResponse("phantom.html.j2", {
|
||
"request": request,
|
||
"t": t,
|
||
"lang": language,
|
||
"langs": LANG_NAMES,
|
||
"import_mode": IMPORT_MODE,
|
||
"domains": domains,
|
||
})
|
||
|
||
raise HTTPException(status_code=500, detail="Templates required")
|
||
|
||
|
||
|
||
@app.get("/about", response_class=HTMLResponse)
|
||
async def about_page(request: Request, lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""About neopig - the story of pig.py's evolution."""
|
||
language = get_lang(lang, accept_language)
|
||
t = TRANSLATIONS.get(language, TRANSLATIONS["en"])
|
||
|
||
if templates:
|
||
return templates.TemplateResponse("about.html.j2", {
|
||
"request": request,
|
||
"t": t,
|
||
"lang": language,
|
||
"langs": LANG_NAMES,
|
||
"import_mode": IMPORT_MODE,
|
||
})
|
||
|
||
raise HTTPException(status_code=500, detail="Templates required")
|
||
|
||
|
||
@app.get("/health")
|
||
async def health():
|
||
"""Health check endpoint."""
|
||
has_screenshot = False
|
||
try:
|
||
from uri2png import get_available_engines
|
||
has_screenshot = True
|
||
except ImportError:
|
||
pass
|
||
|
||
return {
|
||
"status": "healthy",
|
||
"features": {
|
||
"search": True,
|
||
"crawl": True,
|
||
"screenshot": has_screenshot
|
||
}
|
||
}
|
||
|
||
|
||
@app.get("/api/stats")
|
||
async def get_stats_endpoint():
|
||
"""Get database statistics."""
|
||
stats = await db.get_stats()
|
||
# Add page count (handled separately since table may not exist)
|
||
try:
|
||
from sqlalchemy import select, func
|
||
from database import Page
|
||
async with db.session() as session:
|
||
result = await session.execute(select(func.count()).select_from(Page))
|
||
stats['total_pages'] = result.scalar() or 0
|
||
except Exception:
|
||
stats['total_pages'] = 0
|
||
return stats
|
||
|
||
|
||
@app.get("/random")
|
||
async def random_item(type: str = Query(None, description="Type: media or page (random if not specified)")):
|
||
"""Redirect to a random media item or page."""
|
||
from sqlalchemy import select, func
|
||
from database import Media, Page
|
||
import random
|
||
|
||
# If no type specified, randomly pick between media and page
|
||
if type is None:
|
||
type = random.choice(["media", "page"])
|
||
|
||
async with db.session() as session:
|
||
if type == "page":
|
||
# Get a random page
|
||
stmt = select(Page.uri_hash).order_by(func.random()).limit(1)
|
||
result = await session.execute(stmt)
|
||
row = result.fetchone()
|
||
if row and row[0]:
|
||
return RedirectResponse(url=f"/page/{row[0]}", status_code=302)
|
||
else:
|
||
# Get a random media item (excluding screenshots)
|
||
stmt = (
|
||
select(Media.md5_hash)
|
||
.where(Media.media_type != 'screenshot')
|
||
.order_by(func.random())
|
||
.limit(1)
|
||
)
|
||
result = await session.execute(stmt)
|
||
row = result.fetchone()
|
||
if row:
|
||
return RedirectResponse(url=f"/view/{row[0]}", status_code=302)
|
||
|
||
return RedirectResponse(url="/", status_code=302)
|
||
|
||
|
||
@app.get("/api/search")
|
||
async def search(
|
||
q: str = Query("", description="Search query"),
|
||
type: Optional[str] = Query(None, description="Filter by media type"),
|
||
status: Optional[str] = Query(None, description="Filter by analysis status"),
|
||
limit: int = Query(OVER_9000, le=OVER_9000),
|
||
offset: int = Query(0)
|
||
):
|
||
"""
|
||
Search media by text query.
|
||
|
||
Searches across: keywords, alt_text, title, source URLs, analysis results.
|
||
"""
|
||
results = await db.search_media_advanced(
|
||
q=q if q else None,
|
||
media_type=type,
|
||
limit=limit,
|
||
offset=offset
|
||
)
|
||
return results
|
||
|
||
|
||
@app.get("/api/live/stream")
|
||
async def live_stream():
|
||
"""
|
||
SSE endpoint for live media feed.
|
||
|
||
Media appears here immediately after being saved to disk,
|
||
before DB insert completes. Use EventSource to connect.
|
||
"""
|
||
async def event_generator():
|
||
queue = get_live_queue()
|
||
while True:
|
||
try:
|
||
# Wait for next media item with timeout
|
||
media = await asyncio.wait_for(queue.get(), timeout=30)
|
||
yield f"data: {json.dumps(media)}\n\n"
|
||
except asyncio.TimeoutError:
|
||
# Send keepalive
|
||
yield ": keepalive\n\n"
|
||
except Exception as e:
|
||
logger.warning(f"SSE error: {e}")
|
||
break
|
||
|
||
return StreamingResponse(
|
||
event_generator(),
|
||
media_type="text/event-stream",
|
||
headers={
|
||
"Cache-Control": "no-cache",
|
||
"Connection": "keep-alive",
|
||
"X-Accel-Buffering": "no", # Disable nginx buffering
|
||
}
|
||
)
|
||
|
||
|
||
@app.get("/api/search/pages")
|
||
async def search_pages_endpoint(
|
||
q: str = Query("", description="Search query"),
|
||
limit: int = Query(50, le=500),
|
||
):
|
||
"""
|
||
Search pages by text query using FTS5.
|
||
"""
|
||
if not q:
|
||
return []
|
||
return await db.search_pages(q, limit)
|
||
|
||
|
||
@app.get("/api/upgraded")
|
||
async def get_upgraded_media(limit: int = Query(50, le=200)):
|
||
"""
|
||
Get media that was recently upgraded (found higher quality version).
|
||
Used by live feed to show when thumbnails get replaced by full-res.
|
||
"""
|
||
return await db.get_recently_upgraded(limit)
|
||
|
||
|
||
@app.get("/api/media/{md5_hash}")
|
||
async def get_media_info(md5_hash: str):
|
||
"""Get full media info including all source URLs."""
|
||
media = await db.get_media_by_hash(md5_hash)
|
||
if not media:
|
||
raise HTTPException(status_code=404, detail="Media not found")
|
||
|
||
media['sources'] = await db.get_media_sources(md5_hash)
|
||
return media
|
||
|
||
|
||
def slugify(text: str, max_len: int = 60) -> str:
|
||
"""Convert text to a safe filename slug."""
|
||
import re
|
||
import unicodedata
|
||
# Normalize unicode
|
||
text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('ascii')
|
||
# Lowercase and replace spaces/special chars with hyphens
|
||
text = re.sub(r'[^\w\s-]', '', text.lower())
|
||
text = re.sub(r'[-\s]+', '-', text).strip('-')
|
||
return text[:max_len] if text else ""
|
||
|
||
|
||
@app.get("/media/{md5_hash}")
|
||
async def serve_media(md5_hash: str, download: bool = False):
|
||
"""
|
||
Serve media file from vault or tarball.
|
||
|
||
Use ?download=1 for attachment mode with smart filename.
|
||
Caddy should be configured to cache these responses.
|
||
"""
|
||
# Tarball mode: serve from tar.gz (run in thread to avoid blocking)
|
||
if TAR_PATH:
|
||
data, ext = await asyncio.to_thread(find_media_in_tarball, md5_hash)
|
||
if data:
|
||
mime_type, _ = mimetypes.guess_type(f"file{ext}")
|
||
if not mime_type:
|
||
mime_type = "application/octet-stream"
|
||
filename = f"{md5_hash[:12]}{ext}"
|
||
headers = {
|
||
"Cache-Control": "public, max-age=31536000, immutable",
|
||
"X-Content-Hash": md5_hash,
|
||
}
|
||
if download:
|
||
headers["Content-Disposition"] = f'attachment; filename="{filename}"'
|
||
return Response(content=data, media_type=mime_type, headers=headers)
|
||
raise HTTPException(status_code=404, detail="Media not found in archive")
|
||
|
||
# Filesystem mode: find file in vault (9-deep path)
|
||
subdir = VAULT_PATH / hash_to_path(md5_hash).parent
|
||
if not subdir.exists():
|
||
raise HTTPException(status_code=404, detail="Media not found")
|
||
|
||
# Find file with this hash prefix
|
||
for f in subdir.iterdir():
|
||
if f.name.startswith(md5_hash):
|
||
# Guess content type from filename
|
||
mime_type, _ = mimetypes.guess_type(f.name)
|
||
if not mime_type:
|
||
mime_type = "application/octet-stream"
|
||
|
||
# Inline mode: skip DB queries, just serve the file fast
|
||
if not download:
|
||
return FileResponse(
|
||
f,
|
||
media_type=mime_type,
|
||
content_disposition_type="inline",
|
||
headers={
|
||
"Cache-Control": "public, max-age=31536000, immutable",
|
||
"X-Content-Hash": md5_hash,
|
||
}
|
||
)
|
||
|
||
# Download mode: generate smart filename from metadata
|
||
ext = f.suffix or ""
|
||
filename = None
|
||
|
||
# Priority 1: Original filename from URL
|
||
sources = await db.get_media_sources(md5_hash)
|
||
if sources:
|
||
row2 = sources[0]
|
||
if row2.get("media_uri"):
|
||
from urllib.parse import unquote
|
||
parsed = Uri(row2["media_uri"])
|
||
orig_name = Path(unquote(parsed.path)).name
|
||
if orig_name and '.' in orig_name:
|
||
filename = orig_name
|
||
|
||
# Priority 2: Generate from alt_text or title
|
||
if not filename:
|
||
media_record = await db.get_media_by_hash(md5_hash)
|
||
if media_record:
|
||
if not mime_type and media_record.get("mime_type"):
|
||
mime_type = media_record["mime_type"]
|
||
name_source = media_record.get("alt_text") or media_record.get("title")
|
||
if name_source:
|
||
slug = slugify(name_source)
|
||
if slug:
|
||
filename = f"{slug}{ext}"
|
||
|
||
# Priority 3: page_title + hash index
|
||
if not filename and sources:
|
||
row2 = sources[0]
|
||
if row2.get("page_title"):
|
||
media_idx = int(md5_hash[:4], 16)
|
||
slug = slugify(f"{row2['page_title']}-{media_idx}")
|
||
if slug:
|
||
filename = f"{slug}{ext}"
|
||
|
||
# Default filename if nothing else
|
||
if not filename:
|
||
filename = f"{md5_hash[:12]}{ext}"
|
||
|
||
return FileResponse(
|
||
f,
|
||
media_type=mime_type,
|
||
filename=filename,
|
||
content_disposition_type="attachment",
|
||
headers={
|
||
"Cache-Control": "public, max-age=31536000, immutable",
|
||
"X-Content-Hash": md5_hash,
|
||
}
|
||
)
|
||
|
||
raise HTTPException(status_code=404, detail="Media not found")
|
||
|
||
|
||
# ============================================================================
|
||
# Import Mode - Upload and serve archives
|
||
# ============================================================================
|
||
|
||
IMPORT_UPLOAD_DIR = Path(tempfile.gettempdir()) / "neopig_import"
|
||
|
||
# Resumable upload tracking: upload_id -> {filename, total_size, created_at}
|
||
PENDING_UPLOADS: Dict[str, dict] = {}
|
||
|
||
def generate_upload_id() -> str:
|
||
"""Generate a unique upload ID."""
|
||
import secrets
|
||
return f"upload-{secrets.token_hex(8)}"
|
||
|
||
|
||
@app.post("/api/import/upload/init")
|
||
async def init_resumable_upload(filename: str = Query(...), size: int = Query(...)):
|
||
"""Initialize a resumable upload session.
|
||
|
||
Returns upload_id that client uses for subsequent chunk uploads.
|
||
Client can resume from any disconnect by checking /api/import/upload/{upload_id}/status
|
||
"""
|
||
if not IMPORT_MODE:
|
||
raise HTTPException(status_code=403, detail="Import mode not enabled")
|
||
|
||
if not filename.endswith(('.tar.gz', '.tgz', '.run')):
|
||
raise HTTPException(status_code=400, detail="File must be .tar.gz, .tgz, or .run")
|
||
|
||
upload_id = generate_upload_id()
|
||
IMPORT_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||
|
||
# Create empty file for this upload
|
||
upload_path = IMPORT_UPLOAD_DIR / f"{upload_id}.part"
|
||
upload_path.touch()
|
||
|
||
PENDING_UPLOADS[upload_id] = {
|
||
"filename": filename,
|
||
"total_size": size,
|
||
"created_at": asyncio.get_event_loop().time(),
|
||
"path": str(upload_path)
|
||
}
|
||
|
||
logger.info(f"Import: initialized resumable upload {upload_id} for {filename} ({size / 1024 / 1024:.1f} MB)")
|
||
|
||
return {
|
||
"upload_id": upload_id,
|
||
"filename": filename,
|
||
"total_size": size,
|
||
"bytes_received": 0
|
||
}
|
||
|
||
|
||
@app.get("/api/import/upload/{upload_id}/status")
|
||
async def get_upload_status(upload_id: str):
|
||
"""Get status of a resumable upload. Use this to resume after disconnect."""
|
||
if not IMPORT_MODE:
|
||
raise HTTPException(status_code=403, detail="Import mode not enabled")
|
||
|
||
if upload_id not in PENDING_UPLOADS:
|
||
raise HTTPException(status_code=404, detail="Upload not found or expired")
|
||
|
||
info = PENDING_UPLOADS[upload_id]
|
||
upload_path = Path(info["path"])
|
||
|
||
if not upload_path.exists():
|
||
raise HTTPException(status_code=404, detail="Upload file not found")
|
||
|
||
bytes_received = upload_path.stat().st_size
|
||
|
||
return {
|
||
"upload_id": upload_id,
|
||
"filename": info["filename"],
|
||
"total_size": info["total_size"],
|
||
"bytes_received": bytes_received,
|
||
"complete": bytes_received >= info["total_size"]
|
||
}
|
||
|
||
|
||
@app.patch("/api/import/upload/{upload_id}")
|
||
async def upload_chunk(
|
||
upload_id: str,
|
||
request: Request,
|
||
content_range: str = Header(None)
|
||
):
|
||
"""Upload a chunk of data for resumable upload.
|
||
|
||
Use Content-Range header: bytes START-END/TOTAL
|
||
Example: Content-Range: bytes 0-1048575/10485760
|
||
|
||
Or use X-Upload-Offset header for simpler resumption.
|
||
"""
|
||
if not IMPORT_MODE:
|
||
raise HTTPException(status_code=403, detail="Import mode not enabled")
|
||
|
||
if upload_id not in PENDING_UPLOADS:
|
||
raise HTTPException(status_code=404, detail="Upload not found or expired")
|
||
|
||
info = PENDING_UPLOADS[upload_id]
|
||
upload_path = Path(info["path"])
|
||
|
||
# Parse offset from Content-Range or X-Upload-Offset
|
||
offset = 0
|
||
if content_range:
|
||
# Parse "bytes START-END/TOTAL"
|
||
try:
|
||
range_spec = content_range.replace("bytes ", "")
|
||
range_part = range_spec.split("/")[0]
|
||
offset = int(range_part.split("-")[0])
|
||
except Exception:
|
||
raise HTTPException(status_code=400, detail="Invalid Content-Range header")
|
||
else:
|
||
# Use X-Upload-Offset if no Content-Range
|
||
offset_header = request.headers.get("x-upload-offset")
|
||
if offset_header:
|
||
offset = int(offset_header)
|
||
|
||
# Verify offset matches current file size (no gaps)
|
||
current_size = upload_path.stat().st_size if upload_path.exists() else 0
|
||
if offset != current_size:
|
||
raise HTTPException(
|
||
status_code=409,
|
||
detail=f"Offset mismatch: expected {current_size}, got {offset}. Resume from byte {current_size}."
|
||
)
|
||
|
||
# Read and append chunk
|
||
chunk_data = await request.body()
|
||
chunk_size = len(chunk_data)
|
||
|
||
with open(upload_path, 'ab') as f:
|
||
f.write(chunk_data)
|
||
|
||
new_size = upload_path.stat().st_size
|
||
logger.info(f"Import: {upload_id} received chunk {chunk_size} bytes, total {new_size}/{info['total_size']}")
|
||
|
||
return {
|
||
"upload_id": upload_id,
|
||
"bytes_received": new_size,
|
||
"total_size": info["total_size"],
|
||
"complete": new_size >= info["total_size"]
|
||
}
|
||
|
||
|
||
@app.post("/api/import/upload/{upload_id}/complete")
|
||
async def complete_resumable_upload(upload_id: str):
|
||
"""Finalize upload and start import job."""
|
||
if not IMPORT_MODE:
|
||
raise HTTPException(status_code=403, detail="Import mode not enabled")
|
||
|
||
if upload_id not in PENDING_UPLOADS:
|
||
raise HTTPException(status_code=404, detail="Upload not found or expired")
|
||
|
||
info = PENDING_UPLOADS[upload_id]
|
||
upload_path = Path(info["path"])
|
||
|
||
if not upload_path.exists():
|
||
raise HTTPException(status_code=404, detail="Upload file not found")
|
||
|
||
bytes_received = upload_path.stat().st_size
|
||
if bytes_received < info["total_size"]:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail=f"Upload incomplete: {bytes_received}/{info['total_size']} bytes"
|
||
)
|
||
|
||
# Rename to final filename
|
||
final_path = IMPORT_UPLOAD_DIR / info["filename"]
|
||
upload_path.rename(final_path)
|
||
|
||
# Remove from pending
|
||
del PENDING_UPLOADS[upload_id]
|
||
|
||
logger.info(f"Import: {upload_id} completed, saved as {info['filename']}")
|
||
|
||
# Create import job
|
||
job_id = await db.create_crawl_job(
|
||
target_uri=f"import://{info['filename']}",
|
||
keywords=["import"],
|
||
mode="import"
|
||
)
|
||
|
||
# Run import in background
|
||
async def run_import():
|
||
try:
|
||
await db.set_crawl_job_status(job_id, "running")
|
||
stats = await import_archive_to_db(final_path, job_id)
|
||
await db.complete_crawl_job(job_id, stats)
|
||
final_path.unlink(missing_ok=True)
|
||
logger.info(f"Import complete: {stats}")
|
||
except Exception as e:
|
||
logger.error(f"Import failed: {e}")
|
||
await db.fail_crawl_job(job_id, str(e))
|
||
|
||
task = asyncio.create_task(run_import())
|
||
ACTIVE_CRAWL_TASKS[job_id] = task
|
||
task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))
|
||
|
||
return {
|
||
"status": "importing",
|
||
"job_id": job_id,
|
||
"filename": info["filename"],
|
||
"size": bytes_received
|
||
}
|
||
|
||
|
||
@app.post("/api/import/upload")
|
||
async def upload_archive(file: UploadFile = File(...)):
|
||
"""Upload a tar.gz archive and import into local database as a job."""
|
||
if not IMPORT_MODE:
|
||
raise HTTPException(status_code=403, detail="Import mode not enabled (NEOPIG_IMPORT=1)")
|
||
|
||
if not file.filename.endswith(('.tar.gz', '.tgz', '.run')):
|
||
raise HTTPException(status_code=400, detail="File must be .tar.gz, .tgz, or .run")
|
||
|
||
# Save uploaded file
|
||
IMPORT_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||
upload_path = IMPORT_UPLOAD_DIR / file.filename
|
||
logger.info(f"Import: receiving upload {file.filename}")
|
||
|
||
try:
|
||
with open(upload_path, 'wb') as f:
|
||
while chunk := await file.read(1024 * 1024):
|
||
f.write(chunk)
|
||
except Exception as e:
|
||
raise HTTPException(status_code=400, detail=f"Upload failed: {e}")
|
||
|
||
file_size = upload_path.stat().st_size
|
||
logger.info(f"Import: saved {file.filename} ({file_size / 1024 / 1024:.1f} MB)")
|
||
|
||
# Create import job
|
||
job_id = await db.create_crawl_job(
|
||
target_uri=f"import://{file.filename}",
|
||
keywords=["import"],
|
||
mode="import"
|
||
)
|
||
|
||
# Run import in background
|
||
async def run_import():
|
||
try:
|
||
await db.set_crawl_job_status(job_id, "running")
|
||
stats = await import_archive_to_db(upload_path, job_id)
|
||
await db.complete_crawl_job(job_id, stats)
|
||
upload_path.unlink(missing_ok=True)
|
||
logger.info(f"Import complete: {stats}")
|
||
except Exception as e:
|
||
logger.error(f"Import failed: {e}")
|
||
await db.fail_crawl_job(job_id, str(e))
|
||
|
||
task = asyncio.create_task(run_import())
|
||
ACTIVE_CRAWL_TASKS[job_id] = task
|
||
task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))
|
||
|
||
return {"status": "importing", "job_id": job_id, "filename": file.filename, "size": file_size}
|
||
|
||
|
||
async def import_archive_to_db(archive_path: Path, job_id: int) -> dict:
|
||
"""Import archive's database and vault into local database."""
|
||
import sqlite3
|
||
|
||
stats = {"media_imported": 0, "pages_imported": 0, "sources_imported": 0}
|
||
|
||
# Detect .run offset
|
||
offset = 0
|
||
try:
|
||
with open(archive_path, 'rb') as f:
|
||
f.seek(-22, 2)
|
||
trailer = f.read(22)
|
||
if trailer[:6] == b'NEOPIG':
|
||
offset = int(trailer[6:22].decode(), 16)
|
||
except Exception:
|
||
pass
|
||
|
||
# Open tarball
|
||
if offset > 0:
|
||
f = open(archive_path, 'rb')
|
||
f.seek(offset)
|
||
tar = tarfile.open(fileobj=f, mode='r:gz')
|
||
else:
|
||
tar = tarfile.open(archive_path, 'r:gz')
|
||
|
||
try:
|
||
members = tar.getmembers()
|
||
if not members:
|
||
raise ValueError("Empty archive")
|
||
archive_root = members[0].name.split('/')[0]
|
||
|
||
# Extract neopig.db to temp
|
||
db_member = f"{archive_root}/neopig.db"
|
||
temp_db = IMPORT_UPLOAD_DIR / f"import_{job_id}.db"
|
||
|
||
for m in members:
|
||
if m.name == db_member:
|
||
f_db = tar.extractfile(m)
|
||
if f_db:
|
||
with open(temp_db, 'wb') as out:
|
||
out.write(f_db.read())
|
||
break
|
||
|
||
# Merge database
|
||
if not temp_db.exists():
|
||
raise ValueError("Malformed archive: missing neopig.db")
|
||
|
||
src = sqlite3.connect(temp_db)
|
||
src.row_factory = sqlite3.Row
|
||
try:
|
||
# Check what tables exist in source
|
||
tables = [r[0] for r in src.execute("SELECT name FROM sqlite_master WHERE type='table'")]
|
||
logger.info(f"Import: source database has tables: {tables}")
|
||
|
||
# Validate archive has required tables
|
||
required_tables = {'pages', 'media', 'media_sources'}
|
||
if not required_tables.intersection(tables):
|
||
raise ValueError(f"Malformed archive: neopig.db has no data tables (found: {tables})")
|
||
|
||
# Import media (if table exists)
|
||
if 'media' in tables:
|
||
for row in src.execute("SELECT * FROM media"):
|
||
try:
|
||
async with db.session() as session:
|
||
await session.execute(text(
|
||
"""INSERT OR IGNORE INTO media
|
||
(md5_hash, media_type, mime_type, file_size, keywords, alt_text, title,
|
||
first_seen_at, last_seen_at, score)
|
||
VALUES (:md5_hash, :media_type, :mime_type, :file_size, :keywords, :alt_text, :title,
|
||
:first_seen_at, :last_seen_at, :score)"""), dict(row))
|
||
await session.commit()
|
||
stats["media_imported"] += 1
|
||
except Exception as e:
|
||
logger.debug(f"Skip media row: {e}")
|
||
else:
|
||
logger.warning("Import: source has no 'media' table")
|
||
|
||
# Import media_sources (if table exists)
|
||
if 'media_sources' in tables:
|
||
for row in src.execute("SELECT * FROM media_sources"):
|
||
try:
|
||
row_dict = dict(row)
|
||
row_dict['crawl_job_id'] = job_id # Link to import job for deletion
|
||
async with db.session() as session:
|
||
await session.execute(text(
|
||
"""INSERT OR IGNORE INTO media_sources
|
||
(md5_hash, media_uri, page_uri, page_title, alt_text, searchable_text, discovered_at, crawl_job_id)
|
||
VALUES (:md5_hash, :media_uri, :page_uri, :page_title, :alt_text, :searchable_text, :discovered_at, :crawl_job_id)"""), row_dict)
|
||
await session.commit()
|
||
stats["sources_imported"] += 1
|
||
except Exception as e:
|
||
logger.debug(f"Skip media_source row: {e}")
|
||
else:
|
||
logger.warning("Import: source has no 'media_sources' table")
|
||
|
||
# Import pages (if table exists)
|
||
if 'pages' in tables:
|
||
for row in src.execute("SELECT * FROM pages"):
|
||
try:
|
||
row_dict = dict(row)
|
||
row_dict['crawl_job_id'] = job_id # Link to import job for deletion
|
||
async with db.session() as session:
|
||
await session.execute(text(
|
||
"""INSERT OR REPLACE INTO pages
|
||
(uri, uri_hash, path, title, description, keywords, content, markdown, raw_html, crawled_at, crawl_job_id)
|
||
VALUES (:uri, :uri_hash, :path, :title, :description, :keywords, :content, :markdown, :raw_html, :crawled_at, :crawl_job_id)"""), row_dict)
|
||
await session.commit()
|
||
stats["pages_imported"] += 1
|
||
except Exception as e:
|
||
logger.debug(f"Skip page row: {e}")
|
||
else:
|
||
logger.warning("Import: source has no 'pages' table")
|
||
finally:
|
||
src.close()
|
||
temp_db.unlink(missing_ok=True)
|
||
|
||
# Extract vault files
|
||
vault_prefix = f"{archive_root}/vault/"
|
||
for m in members:
|
||
if m.name.startswith(vault_prefix) and m.isfile():
|
||
rel_path = m.name[len(vault_prefix):]
|
||
dest_path = VAULT_PATH / rel_path
|
||
if not dest_path.exists():
|
||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||
f_media = tar.extractfile(m)
|
||
if f_media:
|
||
with open(dest_path, 'wb') as out:
|
||
out.write(f_media.read())
|
||
stats["media_imported"] += 1
|
||
|
||
finally:
|
||
tar.close()
|
||
|
||
return stats
|
||
|
||
|
||
@app.get("/api/import/status")
|
||
async def import_status():
|
||
"""Get current import status."""
|
||
if not IMPORT_MODE:
|
||
raise HTTPException(status_code=403, detail="Import mode not enabled")
|
||
|
||
# Get import job stats
|
||
stats = await db.get_stats()
|
||
import_jobs = await db.get_crawl_jobs(limit=10)
|
||
imports = [j for j in import_jobs if j.get("mode") == "import"]
|
||
|
||
return {
|
||
"import_mode": True,
|
||
"media_count": stats.get("media", 0),
|
||
"pages_count": stats.get("pages", 0),
|
||
"recent_imports": len(imports)
|
||
}
|
||
|
||
|
||
|
||
|
||
@app.get("/import", response_class=HTMLResponse)
|
||
async def import_page(request: Request, lang: str = Cookie(None), accept_language: str = Header(None)):
|
||
"""Import page for uploading archives."""
|
||
if not IMPORT_MODE:
|
||
raise HTTPException(status_code=403, detail="Import mode not enabled (NEOPIG_IMPORT=1)")
|
||
language = get_lang(lang, accept_language)
|
||
t = TRANSLATIONS.get(language, TRANSLATIONS["en"])
|
||
|
||
if templates:
|
||
return templates.TemplateResponse("import.html.j2", {
|
||
"request": request,
|
||
"t": t,
|
||
"t_json": json.dumps(t),
|
||
"lang": language,
|
||
"langs": LANG_NAMES,
|
||
"import_mode": IMPORT_MODE,
|
||
})
|
||
|
||
raise HTTPException(status_code=500, detail="Templates required")
|
||
|
||
|
||
# ============================================================================
|
||
# Crawler API
|
||
# ============================================================================
|
||
|
||
@app.get("/api/crawl/jobs")
|
||
async def get_crawl_jobs_endpoint(limit: int = Query(50, le=200)):
|
||
"""Get recent jobs (crawl + backfill)."""
|
||
try:
|
||
# Get both crawl jobs and backfill jobs
|
||
crawl_jobs = await db.get_crawl_jobs(limit)
|
||
backfill_jobs = await db.get_backfill_jobs(limit)
|
||
|
||
# Add job_kind to distinguish them
|
||
for job in crawl_jobs:
|
||
job['job_kind'] = 'crawl'
|
||
for job in backfill_jobs:
|
||
job['job_kind'] = 'backfill'
|
||
|
||
# Merge and sort by started_at descending
|
||
all_jobs = crawl_jobs + backfill_jobs
|
||
all_jobs.sort(key=lambda j: j.get('started_at', ''), reverse=True)
|
||
|
||
return all_jobs[:limit]
|
||
except Exception as e:
|
||
logger.error(f"Failed to get jobs: {e}")
|
||
raise HTTPException(status_code=500, detail=str(e))
|
||
|
||
|
||
@app.post("/api/crawl/jobs/{job_id}/pause")
|
||
async def pause_crawl_job(job_id: int):
|
||
"""Pause a running crawl job."""
|
||
task = ACTIVE_CRAWL_TASKS.get(job_id)
|
||
if not task:
|
||
raise HTTPException(status_code=404, detail="Job not running")
|
||
|
||
task.cancel()
|
||
try:
|
||
await task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
|
||
await db.pause_crawl_job(job_id)
|
||
ACTIVE_CRAWL_TASKS.pop(job_id, None)
|
||
|
||
return {"status": "paused", "job_id": job_id}
|
||
|
||
|
||
@app.post("/api/crawl/jobs/{job_id}/resume")
|
||
async def resume_crawl_job(job_id: int):
|
||
"""Resume a paused crawl job."""
|
||
job = await db.get_crawl_job(job_id)
|
||
if not job:
|
||
raise HTTPException(status_code=404, detail="Job not found")
|
||
|
||
if job['status'] not in ('paused', 'cancelled'):
|
||
raise HTTPException(status_code=400, detail=f"Job is {job['status']}, cannot resume")
|
||
|
||
if job_id in ACTIVE_CRAWL_TASKS:
|
||
raise HTTPException(status_code=400, detail="Job is already running")
|
||
|
||
# Mark as running
|
||
async with db.session() as session:
|
||
await session.execute(
|
||
text("UPDATE crawl_jobs SET status = 'running' WHERE id = :id"),
|
||
{"id": job_id}
|
||
)
|
||
await session.commit()
|
||
|
||
# Start crawl task (resume from state files with original settings)
|
||
target_uri = job['target_uri']
|
||
keywords = json.loads(job.get('keywords') or '[]')
|
||
mode_str = job.get('mode', 'all')
|
||
# Use stored settings (with fallbacks for old jobs)
|
||
job_depth = job.get('depth', 15) or 15
|
||
job_max_pages = job.get('max_pages', -1) if job.get('max_pages') is not None else -1
|
||
job_fast = bool(job.get('fast', 0))
|
||
job_screenshots = bool(job.get('screenshots', 1))
|
||
|
||
async def run_crawl(jid=job_id, uri=target_uri):
|
||
try:
|
||
from neopig import NeoPig
|
||
from async_web_fetcher import CrawlMode
|
||
from screenshot import ScreenshotConfig
|
||
screenshot_config = ScreenshotConfig(enabled=job_screenshots)
|
||
pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH), fast_mode=job_fast, screenshot_config=screenshot_config)
|
||
await pig.init()
|
||
|
||
# Resume from state (don't clear state files)
|
||
crawled_media = await pig.db.get_crawled_media_uris()
|
||
crawled_screenshots = await pig.db.get_crawled_screenshot_uris()
|
||
if crawled_media:
|
||
pig.seen_media = crawled_media
|
||
if crawled_screenshots:
|
||
pig.seen_screenshots = crawled_screenshots
|
||
|
||
mode = CrawlMode(mode_str) if mode_str else CrawlMode.ALL
|
||
|
||
# Progress updater
|
||
stop_progress = asyncio.Event()
|
||
|
||
async def update_progress():
|
||
while not stop_progress.is_set():
|
||
await asyncio.sleep(2)
|
||
if not stop_progress.is_set():
|
||
await db.update_crawl_job_stats(jid, pig.stats)
|
||
|
||
progress_task = asyncio.create_task(update_progress())
|
||
|
||
try:
|
||
stats = await pig.crawl(
|
||
target_uri=uri,
|
||
keywords=keywords,
|
||
mode=mode,
|
||
depth=job_depth,
|
||
max_pages=job_max_pages,
|
||
job_id=jid,
|
||
quiet=True,
|
||
)
|
||
finally:
|
||
stop_progress.set()
|
||
progress_task.cancel()
|
||
try:
|
||
await progress_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
|
||
await db.complete_crawl_job(jid, stats)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Resumed crawl job {jid} failed: {e}")
|
||
await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})
|
||
|
||
task = asyncio.create_task(run_crawl())
|
||
ACTIVE_CRAWL_TASKS[job_id] = task
|
||
task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))
|
||
|
||
return {"status": "running", "job_id": job_id}
|
||
|
||
|
||
@app.delete("/api/crawl/jobs/{job_id}")
|
||
async def delete_crawl_job(job_id: int, purge: bool = Query(True, description="Purge all data (media, screenshots, pages)")):
|
||
"""Delete a crawl job and optionally purge all associated data.
|
||
|
||
With purge=True (default):
|
||
- Deletes all MediaSource records for this job
|
||
- Deletes all Page records for this job
|
||
- Deletes orphan Media records (not referenced by other jobs)
|
||
- Deletes orphan media files from vault
|
||
- Deletes screenshot files for deleted pages
|
||
- Deletes state files
|
||
"""
|
||
# Can't delete running jobs
|
||
if job_id in ACTIVE_CRAWL_TASKS:
|
||
raise HTTPException(status_code=400, detail="Cannot delete running job. Pause it first.")
|
||
|
||
if purge:
|
||
# Full purge using NeoPig
|
||
from neopig import NeoPig
|
||
pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH))
|
||
await pig.init()
|
||
result = await pig.purge_job(job_id)
|
||
if not result['deleted']:
|
||
raise HTTPException(status_code=404, detail="Job not found")
|
||
return {
|
||
"status": "purged",
|
||
"job_id": job_id,
|
||
"media_files_deleted": result['media_files_deleted'],
|
||
"screenshots_deleted": result['screenshots_deleted'],
|
||
"pages_deleted": result['pages_deleted'],
|
||
"state_files_deleted": result['state_files_deleted'],
|
||
}
|
||
else:
|
||
# Just delete job record
|
||
result = await db.delete_crawl_job(job_id, purge_data=False)
|
||
if not result['deleted']:
|
||
raise HTTPException(status_code=404, detail="Job not found")
|
||
return {"status": "deleted", "job_id": job_id}
|
||
|
||
|
||
@app.get("/api/crawl/jobs/{job_id}")
|
||
async def get_crawl_job_endpoint(job_id: int):
|
||
"""Get a specific crawl job."""
|
||
job = await db.get_crawl_job(job_id)
|
||
if not job:
|
||
raise HTTPException(status_code=404, detail="Job not found")
|
||
return job
|
||
|
||
|
||
@app.get("/api/crawl/jobs/{job_id}/logs")
|
||
async def get_crawl_job_logs(job_id: int, tail: int = Query(0, description="Return only last N lines")):
|
||
"""Get logs for a crawl job."""
|
||
logs = get_job_logs(job_id, tail=tail)
|
||
return Response(content=logs, media_type="text/plain")
|
||
|
||
|
||
@app.post("/api/crawl")
|
||
async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
|
||
"""
|
||
Start new crawl job(s).
|
||
|
||
Supports multiple targets - creates one job per target.
|
||
The crawls run in the background. Poll /api/crawl/jobs/{id} for status.
|
||
"""
|
||
if CRAWL_DISABLED:
|
||
raise HTTPException(status_code=403, detail="Crawling is disabled (NEOPIG_DISABLE_CRAWL=1)")
|
||
|
||
# Import neopig here to avoid circular imports
|
||
from neopig import NeoPig
|
||
from async_web_fetcher import CrawlMode
|
||
|
||
# Map mode string to enum
|
||
mode_map = {
|
||
"text": CrawlMode.TEXT,
|
||
"images": CrawlMode.IMAGES,
|
||
"videos": CrawlMode.VIDEOS,
|
||
"media": CrawlMode.MEDIA,
|
||
"all": CrawlMode.ALL,
|
||
}
|
||
mode = mode_map.get(request.mode, CrawlMode.IMAGES)
|
||
|
||
# Get targets (support both new 'targets' array and old 'target_uri' single value)
|
||
targets = request.targets if request.targets else [request.target_uri] if request.target_uri else []
|
||
if not targets:
|
||
raise HTTPException(status_code=400, detail="No target URIs provided")
|
||
|
||
job_ids = []
|
||
|
||
# Create a job for each target (store all settings for resume)
|
||
depth = request.depth if 1 <= request.depth <= 15 else 15
|
||
for target_uri in targets:
|
||
job_id = await db.create_crawl_job(
|
||
target_uri, request.keywords, request.mode,
|
||
depth=depth, max_pages=request.max_pages,
|
||
fast=request.fast, screenshots=request.screenshots
|
||
)
|
||
job_ids.append(job_id)
|
||
|
||
# Run crawl in background - capture ALL values to avoid closure issues
|
||
is_fresh = request.fresh
|
||
is_fast = request.fast
|
||
has_screenshots = request.screenshots
|
||
is_hydra = request.hydra
|
||
crawl_keywords = list(request.keywords) # Copy list
|
||
crawl_max_pages = request.max_pages
|
||
crawl_depth = request.depth if 1 <= request.depth <= 15 else 15
|
||
|
||
async def run_crawl(jid=job_id, uri=target_uri, fresh=is_fresh, fast=is_fast, screenshots=has_screenshots, hydra=is_hydra, depth=crawl_depth, keywords=crawl_keywords, max_pages=crawl_max_pages):
|
||
try:
|
||
from screenshot import ScreenshotConfig
|
||
screenshot_config = ScreenshotConfig(enabled=screenshots)
|
||
pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH), fast_mode=fast, screenshot_config=screenshot_config)
|
||
await pig.init()
|
||
|
||
# Handle fresh vs resume - NEVER fresh on resume
|
||
if fresh:
|
||
pig._clear_state(uri)
|
||
else:
|
||
# Load existing seen media for resume capability
|
||
crawled_media = await pig.db.get_crawled_media_uris()
|
||
crawled_screenshots = await pig.db.get_crawled_screenshot_uris()
|
||
if crawled_media:
|
||
pig.seen_media = crawled_media
|
||
if crawled_screenshots:
|
||
pig.seen_screenshots = crawled_screenshots
|
||
|
||
# Progress updater - runs alongside crawl
|
||
stop_progress = asyncio.Event()
|
||
|
||
async def update_progress():
|
||
while not stop_progress.is_set():
|
||
await asyncio.sleep(2) # Update every 2 seconds
|
||
if not stop_progress.is_set():
|
||
await db.update_crawl_job_stats(jid, pig.stats)
|
||
|
||
progress_task = asyncio.create_task(update_progress())
|
||
|
||
try:
|
||
stats = await pig.crawl(
|
||
target_uri=uri,
|
||
keywords=keywords,
|
||
mode=mode,
|
||
depth=depth,
|
||
max_pages=max_pages,
|
||
job_id=jid, # Use existing job, don't create another
|
||
quiet=True, # No progress bar for UI-initiated crawls
|
||
hydra=hydra, # Feed/sitemap discovery mode
|
||
)
|
||
finally:
|
||
stop_progress.set()
|
||
progress_task.cancel()
|
||
try:
|
||
await progress_task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
|
||
# Update job as completed
|
||
await db.complete_crawl_job(jid, stats)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Crawl job {jid} failed: {e}")
|
||
await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})
|
||
|
||
# Schedule async task on the event loop (not BackgroundTasks which runs in threadpool)
|
||
task = asyncio.create_task(run_crawl())
|
||
ACTIVE_CRAWL_TASKS[job_id] = task
|
||
# Clean up when done
|
||
task.add_done_callback(lambda t, jid=job_id: ACTIVE_CRAWL_TASKS.pop(jid, None))
|
||
|
||
return {"job_ids": job_ids, "status": "running", "count": len(job_ids)}
|
||
|
||
|
||
def init_tarball_mode(tarball_path: str):
|
||
"""Initialize serving from a tar.gz archive."""
|
||
global TAR_PATH, TAR_OFFSET, TAR_MEMBERS, TAR_MEDIA_INDEX, ARCHIVE_ROOT, DB_PATH, TEMP_DB_PATH
|
||
|
||
logger.info(f"Opening archive: {tarball_path}")
|
||
tarball = Path(tarball_path)
|
||
TAR_PATH = str(tarball.resolve())
|
||
|
||
# Handle .run files with NEOPIG trailer
|
||
TAR_OFFSET = 0
|
||
if tarball.suffix == '.run' or tarball.stat().st_size > 100000:
|
||
try:
|
||
with open(tarball, 'rb') as f:
|
||
f.seek(-22, 2)
|
||
trailer = f.read(22)
|
||
if trailer[:6] == b'NEOPIG':
|
||
TAR_OFFSET = int(trailer[6:22].decode(), 16)
|
||
logger.info(f"Detected .run format, tarball offset: {TAR_OFFSET}")
|
||
except Exception:
|
||
pass
|
||
|
||
# Open tarball temporarily to build index
|
||
tar = _open_tarball()
|
||
|
||
# Build member lookup and media index
|
||
for member in tar.getmembers():
|
||
TAR_MEMBERS[member.name] = member
|
||
# Index media files by hash for O(1) lookup
|
||
name = member.name
|
||
if '/vault/' in name or '/media/' in name:
|
||
# Extract hash from filename (hash.ext)
|
||
basename = Path(name).stem # removes extension
|
||
if len(basename) == 32 and all(c in '0123456789abcdef' for c in basename):
|
||
TAR_MEDIA_INDEX[basename] = name
|
||
|
||
logger.info(f"Indexed {len(TAR_MEDIA_INDEX)} media files")
|
||
|
||
# Get archive root from first member
|
||
first = list(TAR_MEMBERS.keys())[0]
|
||
ARCHIVE_ROOT = first.split('/')[0]
|
||
logger.info(f"Archive root: {ARCHIVE_ROOT}")
|
||
|
||
# Extract database to temp (SQLite needs real file)
|
||
# Try neopig.db first (new format), then archive.db (legacy)
|
||
db_member = f"{ARCHIVE_ROOT}/neopig.db"
|
||
if db_member not in TAR_MEMBERS:
|
||
db_member = f"{ARCHIVE_ROOT}/archive.db"
|
||
if db_member in TAR_MEMBERS:
|
||
temp_dir = tempfile.mkdtemp(prefix="neopig_")
|
||
db_name = Path(db_member).name
|
||
TEMP_DB_PATH = f"{temp_dir}/{db_name}"
|
||
member = TAR_MEMBERS[db_member]
|
||
f = tar.extractfile(member)
|
||
if f:
|
||
with open(TEMP_DB_PATH, 'wb') as out:
|
||
out.write(f.read())
|
||
DB_PATH = TEMP_DB_PATH
|
||
logger.info(f"Extracted database to: {TEMP_DB_PATH}")
|
||
else:
|
||
logger.warning("No neopig.db or archive.db found in tarball")
|
||
|
||
# Close the temporary tar handle (requests will open their own)
|
||
tar.close()
|
||
|
||
|
||
def main():
|
||
global DB_PATH, VAULT_PATH
|
||
|
||
parser = argparse.ArgumentParser(description="neopig SERP")
|
||
parser.add_argument("tarball", nargs='?', help="Path to archive.tar.gz or .run file")
|
||
parser.add_argument("--port", type=int, default=31337)
|
||
parser.add_argument("--host", default="127.0.0.1")
|
||
parser.add_argument("--db", default="data/neopig.db")
|
||
parser.add_argument("--vault", default="data/vault")
|
||
|
||
args = parser.parse_args()
|
||
|
||
# Tarball mode
|
||
if args.tarball:
|
||
init_tarball_mode(args.tarball)
|
||
logger.info(f"Starting neopig SERP (archive mode) on {args.host}:{args.port}")
|
||
else:
|
||
DB_PATH = args.db
|
||
VAULT_PATH = Path(args.vault)
|
||
logger.info(f"Starting neopig SERP on {args.host}:{args.port}")
|
||
logger.info(f"Database: {DB_PATH}")
|
||
logger.info(f"Vault: {VAULT_PATH}")
|
||
|
||
uvicorn.run(app, host=args.host, port=args.port)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|