Extract i18n to module, add Georgian, translate about page body
- Extract 1100+ lines of translations from serp.py to i18n.py - Add Georgian (ka) as 27th supported language - Add 74 about page body text keys (hero, chapters, features, footer) - Update about.html.j2 to use translation placeholders - Full Chinese translations for all body text - English fallback for other 25 languages (can be translated later) - serp.py reduced from 3291 to 2142 lines - i18n.py now 3808 lines with 190 keys per language
This commit is contained in:
parent
1df3e82be9
commit
a46f93aaf1
14 changed files with 6462 additions and 1138 deletions
866
serp.py
866
serp.py
|
|
@ -37,875 +37,11 @@ from database import Database, OVER_9000
|
|||
from filevault import hash_to_path
|
||||
from miniuri import Uri
|
||||
from neopig import get_live_queue
|
||||
from i18n import TRANSLATIONS, LANG_NAMES, get_lang, t, inject_i18n, NAV_HTML, SEARCH_BOX_HTML
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ============================================================================
|
||||
# INTERNATIONALIZATION (i18n) - Top 10 Languages
|
||||
# ============================================================================
|
||||
TRANSLATIONS = {
|
||||
"en": {
|
||||
"search": "Search", "crawl": "Crawl", "live": "Live Feed", "about": "About",
|
||||
"random": "Random", "phantom": "Phantom", "loading": "Loading...",
|
||||
"start_crawl": "Start Crawl", "pause": "Pause", "resume": "Resume", "delete": "Delete",
|
||||
"url": "URL", "mode": "Mode", "depth": "Depth", "keywords": "Keywords",
|
||||
"screenshots": "Screenshots", "fast_mode": "Fast Mode", "fresh_start": "Fresh Start",
|
||||
"pages": "pages", "found": "found", "saved": "saved", "dupes": "dupes",
|
||||
"running": "running", "completed": "completed", "paused": "paused",
|
||||
"no_jobs": "No crawl jobs yet", "started": "Started", "filter": "Filter",
|
||||
"all_types": "All types", "images": "Images", "videos": "Videos", "audio": "Audio",
|
||||
"everything": "Everything (text + media)", "all_media": "All media (images + videos + audio)",
|
||||
"images_only": "Images only", "videos_only": "Videos only", "text_only": "Text only",
|
||||
"recent_jobs": "Recent Jobs", "search_placeholder": "Search media...",
|
||||
"media": "Media", "pages_label": "Pages", "download": "Download",
|
||||
"no_media": "No media found", "no_more": "No more results", "load_error": "Failed to load",
|
||||
"copy": "Copy", "copied": "Copied!", "starting": "Starting...", "started_ok": "Started!",
|
||||
"error": "Error", "watching": "Watching for new images...", "content": "Content",
|
||||
"previous": "Previous", "next": "Next", "over_9000": "Over 9,000+!",
|
||||
"loading_stats": "Loading stats...", "screenshot": "Screenshot",
|
||||
"valid_uri": "Please enter at least one valid URI", "load_jobs_error": "Failed to load jobs",
|
||||
"hydrate_subtitle": "Hydrate media from the web", "live_subtitle": "Watch images appear as they're crawled",
|
||||
"total_images": "Total Images", "new_session": "New This Session", "per_minute": "Per Minute",
|
||||
"all_domains": "All domains", "download_phantom": "Download Phantom Site",
|
||||
"phantom_subtitle": "Export archived pages as a static site with local media", "sources": "sources", "max_pages": "Max Pages", "select_domain": "Select Domain",
|
||||
"source_uri": "Source URI", "neopig_uri": "Neopig URI", "source_page": "Source Page", "neopig_page": "Neopig Page", "type_label": "Type", "mime_label": "MIME", "size_label": "Size", "alt_label": "Alt", "keywords_label": "Keywords", "description_label": "Description", "items": "items", "bytes": "bytes",
|
||||
"used_on": "Used on", "page": "Page", "discovered": "Discovered",
|
||||
"console": "Console", "close": "Close", "no_logs": "No logs yet",
|
||||
},
|
||||
"zh": {
|
||||
"search": "搜索", "crawl": "爬取", "live": "实时动态", "about": "关于",
|
||||
"random": "随机", "phantom": "幻影", "loading": "加载中...",
|
||||
"start_crawl": "开始爬取", "pause": "暂停", "resume": "继续", "delete": "删除",
|
||||
"url": "网址", "mode": "模式", "depth": "深度", "keywords": "关键词",
|
||||
"screenshots": "截图", "fast_mode": "快速模式", "fresh_start": "全新开始",
|
||||
"pages": "页面", "found": "发现", "saved": "保存", "dupes": "重复",
|
||||
"running": "运行中", "completed": "已完成", "paused": "已暂停",
|
||||
"no_jobs": "暂无爬取任务", "started": "开始于", "filter": "筛选",
|
||||
"all_types": "所有类型", "images": "图片", "videos": "视频", "audio": "音频",
|
||||
"everything": "全部（文本+媒体）", "all_media": "所有媒体（图片+视频+音频）",
|
||||
"images_only": "仅图片", "videos_only": "仅视频", "text_only": "仅文本",
|
||||
"recent_jobs": "最近任务", "search_placeholder": "搜索媒体...",
|
||||
"media": "媒体", "pages_label": "页面", "download": "下载",
|
||||
"no_media": "未找到媒体", "no_more": "没有更多结果", "load_error": "加载失败",
|
||||
"copy": "复制", "copied": "已复制!", "starting": "启动中...", "started_ok": "已启动!",
|
||||
"error": "错误", "watching": "正在监视新图片...", "content": "内容",
|
||||
"previous": "上一页", "next": "下一页", "over_9000": "超过9000+!",
|
||||
"loading_stats": "加载统计中...", "screenshot": "截图",
|
||||
"valid_uri": "请输入至少一个有效的URI", "load_jobs_error": "加载任务失败",
|
||||
"hydrate_subtitle": "从网络获取媒体", "live_subtitle": "观看正在爬取的图片",
|
||||
"total_images": "总图片数", "new_session": "本次新增", "per_minute": "每分钟",
|
||||
"all_domains": "所有域名", "download_phantom": "下载幻影站点",
|
||||
"phantom_subtitle": "将存档页面导出为带本地媒体的静态站点", "max_pages": "最大页数", "select_domain": "选择域名", "sources": "来源",
|
||||
"source_uri": "源URI", "neopig_uri": "Neopig URI", "source_page": "源页面", "neopig_page": "Neopig页面",
|
||||
"type_label": "类型", "mime_label": "MIME", "size_label": "大小", "alt_label": "替代文本",
|
||||
"keywords_label": "关键词", "description_label": "描述", "items": "项", "bytes": "字节",
|
||||
"used_on": "使用于", "page": "页面", "discovered": "发现时间",
|
||||
"console": "控制台", "close": "关闭", "no_logs": "暂无日志",
|
||||
},
|
||||
"es": {
|
||||
"search": "Buscar", "crawl": "Rastrear", "live": "En Vivo", "about": "Acerca de",
|
||||
"random": "Aleatorio", "phantom": "Fantasma", "loading": "Cargando...",
|
||||
"start_crawl": "Iniciar Rastreo", "pause": "Pausar", "resume": "Reanudar", "delete": "Eliminar",
|
||||
"url": "URL", "mode": "Modo", "depth": "Profundidad", "keywords": "Palabras clave",
|
||||
"screenshots": "Capturas", "fast_mode": "Modo Rápido", "fresh_start": "Inicio Limpio",
|
||||
"pages": "páginas", "found": "encontradas", "saved": "guardadas", "dupes": "duplicados",
|
||||
"running": "ejecutando", "completed": "completado", "paused": "pausado",
|
||||
"no_jobs": "Sin tareas de rastreo", "started": "Iniciado", "filter": "Filtrar",
|
||||
"all_types": "Todos los tipos", "images": "Imágenes", "videos": "Videos", "audio": "Audio",
|
||||
"everything": "Todo (texto + medios)", "all_media": "Todos los medios",
|
||||
"images_only": "Solo imágenes", "videos_only": "Solo videos", "text_only": "Solo texto",
|
||||
"recent_jobs": "Tareas Recientes", "search_placeholder": "Buscar medios...",
|
||||
"media": "Medios", "pages_label": "Páginas", "download": "Descargar",
|
||||
"no_media": "No se encontraron medios", "no_more": "No hay más resultados", "load_error": "Error al cargar",
|
||||
"copy": "Copiar", "copied": "¡Copiado!", "starting": "Iniciando...", "started_ok": "¡Iniciado!",
|
||||
"error": "Error", "watching": "Buscando nuevas imágenes...", "content": "Contenido",
|
||||
"previous": "Anterior", "next": "Siguiente", "over_9000": "¡Más de 9,000+!",
|
||||
"loading_stats": "Cargando estadísticas...", "screenshot": "Captura",
|
||||
"valid_uri": "Ingrese al menos una URI válida", "load_jobs_error": "Error al cargar tareas",
|
||||
"hydrate_subtitle": "Obtener medios de la web", "live_subtitle": "Ver imágenes mientras se rastrean",
|
||||
"total_images": "Total de imágenes", "new_session": "Nuevas en esta sesión", "per_minute": "Por minuto",
|
||||
"all_domains": "Todos los dominios", "download_phantom": "Descargar sitio fantasma",
|
||||
"phantom_subtitle": "Exportar páginas archivadas como sitio estático con medios locales", "max_pages": "Máx. páginas", "select_domain": "Seleccionar dominio", "sources": "fuentes",
|
||||
"source_uri": "URI de origen", "neopig_uri": "URI Neopig", "source_page": "Página de origen", "neopig_page": "Página Neopig",
|
||||
"type_label": "Tipo", "mime_label": "MIME", "size_label": "Tamaño", "alt_label": "Alt",
|
||||
"keywords_label": "Palabras clave", "description_label": "Descripción", "items": "elementos", "bytes": "bytes",
|
||||
"used_on": "Usado en", "page": "Página", "discovered": "Descubierto",
|
||||
"console": "Consola", "close": "Cerrar", "no_logs": "Sin registros aún",
|
||||
},
|
||||
"hi": {
|
||||
"search": "खोजें", "crawl": "क्रॉल", "live": "लाइव फ़ीड", "about": "के बारे में",
|
||||
"random": "यादृच्छिक", "phantom": "प्रेत", "loading": "लोड हो रहा है...",
|
||||
"start_crawl": "क्रॉल शुरू करें", "pause": "रोकें", "resume": "जारी रखें", "delete": "हटाएं",
|
||||
"url": "यूआरएल", "mode": "मोड", "depth": "गहराई", "keywords": "कीवर्ड",
|
||||
"screenshots": "स्क्रीनशॉट", "fast_mode": "फास्ट मोड", "fresh_start": "नई शुरुआत",
|
||||
"pages": "पेज", "found": "मिले", "saved": "सहेजे", "dupes": "डुप्लीकेट",
|
||||
"running": "चल रहा है", "completed": "पूर्ण", "paused": "रुका हुआ",
|
||||
"no_jobs": "कोई क्रॉल कार्य नहीं", "started": "शुरू", "filter": "फ़िल्टर",
|
||||
"all_types": "सभी प्रकार", "images": "चित्र", "videos": "वीडियो", "audio": "ऑडियो",
|
||||
"everything": "सब कुछ", "all_media": "सभी मीडिया", "images_only": "केवल चित्र",
|
||||
"videos_only": "केवल वीडियो", "text_only": "केवल टेक्स्ट",
|
||||
"recent_jobs": "हाल के कार्य", "search_placeholder": "मीडिया खोजें...",
|
||||
"media": "मीडिया", "pages_label": "पेज", "download": "डाउनलोड",
|
||||
"no_media": "कोई मीडिया नहीं मिला", "no_more": "और कोई परिणाम नहीं", "load_error": "लोड करने में विफल",
|
||||
"copy": "कॉपी", "copied": "कॉपी हो गया!", "starting": "शुरू हो रहा...", "started_ok": "शुरू हो गया!",
|
||||
"error": "त्रुटि", "watching": "नई छवियों की तलाश...", "content": "सामग्री",
|
||||
"previous": "पिछला", "next": "अगला", "over_9000": "9,000+ से अधिक!",
|
||||
"loading_stats": "आँकड़े लोड हो रहे...", "screenshot": "स्क्रीनशॉट",
|
||||
"valid_uri": "कृपया कम से कम एक वैध URI दर्ज करें", "load_jobs_error": "कार्य लोड करने में विफल",
|
||||
"hydrate_subtitle": "वेब से मीडिया प्राप्त करें", "live_subtitle": "क्रॉल होते हुए छवियां देखें",
|
||||
"total_images": "कुल छवियां", "new_session": "इस सत्र में नई", "per_minute": "प्रति मिनट",
|
||||
"all_domains": "सभी डोमेन", "download_phantom": "प्रेत साइट डाउनलोड करें",
|
||||
"phantom_subtitle": "संग्रहित पृष्ठों को स्थानीय मीडिया के साथ स्थैतिक साइट के रूप में निर्यात करें", "max_pages": "अधिकतम पेज", "select_domain": "डोमेन चुनें", "sources": "स्रोत",
|
||||
"source_uri": "स्रोत URI", "neopig_uri": "Neopig URI", "source_page": "स्रोत पृष्ठ", "neopig_page": "Neopig पृष्ठ",
|
||||
"type_label": "प्रकार", "mime_label": "MIME", "size_label": "आकार", "alt_label": "Alt",
|
||||
"keywords_label": "कीवर्ड", "description_label": "विवरण", "items": "आइटम", "bytes": "बाइट्स",
|
||||
"used_on": "में उपयोग", "page": "पृष्ठ", "discovered": "खोजा गया",
|
||||
"console": "कंसोल", "close": "बंद करें", "no_logs": "अभी तक कोई लॉग नहीं",
|
||||
},
|
||||
"ar": {
|
||||
"search": "بحث", "crawl": "زحف", "live": "مباشر", "about": "حول",
|
||||
"random": "عشوائي", "phantom": "شبح", "loading": "جاري التحميل...",
|
||||
"start_crawl": "بدء الزحف", "pause": "إيقاف", "resume": "استئناف", "delete": "حذف",
|
||||
"url": "الرابط", "mode": "الوضع", "depth": "العمق", "keywords": "كلمات مفتاحية",
|
||||
"screenshots": "لقطات", "fast_mode": "وضع سريع", "fresh_start": "بداية جديدة",
|
||||
"pages": "صفحات", "found": "وجدت", "saved": "حفظت", "dupes": "مكررات",
|
||||
"running": "جاري", "completed": "مكتمل", "paused": "متوقف",
|
||||
"no_jobs": "لا توجد مهام", "started": "بدأ", "filter": "تصفية",
|
||||
"all_types": "جميع الأنواع", "images": "صور", "videos": "فيديو", "audio": "صوت",
|
||||
"everything": "الكل", "all_media": "جميع الوسائط", "images_only": "صور فقط",
|
||||
"videos_only": "فيديو فقط", "text_only": "نص فقط",
|
||||
"recent_jobs": "المهام الأخيرة", "search_placeholder": "بحث في الوسائط...",
|
||||
"media": "وسائط", "pages_label": "صفحات", "download": "تحميل",
|
||||
"no_media": "لم يتم العثور على وسائط", "no_more": "لا مزيد من النتائج", "load_error": "فشل التحميل",
|
||||
"copy": "نسخ", "copied": "تم النسخ!", "starting": "جاري البدء...", "started_ok": "تم البدء!",
|
||||
"error": "خطأ", "watching": "مراقبة الصور الجديدة...", "content": "المحتوى",
|
||||
"previous": "السابق", "next": "التالي", "over_9000": "أكثر من 9,000+!",
|
||||
"loading_stats": "جاري تحميل الإحصائيات...", "screenshot": "لقطة شاشة",
|
||||
"valid_uri": "الرجاء إدخال رابط واحد على الأقل", "load_jobs_error": "فشل تحميل المهام",
|
||||
"hydrate_subtitle": "جلب الوسائط من الويب", "live_subtitle": "مشاهدة الصور أثناء الزحف",
|
||||
"total_images": "إجمالي الصور", "new_session": "جديدة في هذه الجلسة", "per_minute": "في الدقيقة",
|
||||
"all_domains": "جميع النطاقات", "download_phantom": "تحميل موقع الشبح",
|
||||
"phantom_subtitle": "تصدير الصفحات المؤرشفة كموقع ثابت مع وسائط محلية", "max_pages": "أقصى عدد صفحات", "select_domain": "اختر النطاق", "sources": "مصادر",
|
||||
"source_uri": "URI المصدر", "neopig_uri": "URI Neopig", "source_page": "صفحة المصدر", "neopig_page": "صفحة Neopig",
|
||||
"type_label": "النوع", "mime_label": "MIME", "size_label": "الحجم", "alt_label": "Alt",
|
||||
"keywords_label": "كلمات مفتاحية", "description_label": "الوصف", "items": "عناصر", "bytes": "بايت",
|
||||
"used_on": "مستخدم في", "page": "صفحة", "discovered": "اكتشف",
|
||||
"console": "وحدة التحكم", "close": "إغلاق", "no_logs": "لا توجد سجلات بعد",
|
||||
},
|
||||
"pt": {
|
||||
"search": "Pesquisar", "crawl": "Rastrear", "live": "Ao Vivo", "about": "Sobre",
|
||||
"random": "Aleatório", "phantom": "Fantasma", "loading": "Carregando...",
|
||||
"start_crawl": "Iniciar Rastreio", "pause": "Pausar", "resume": "Retomar", "delete": "Excluir",
|
||||
"url": "URL", "mode": "Modo", "depth": "Profundidade", "keywords": "Palavras-chave",
|
||||
"screenshots": "Capturas", "fast_mode": "Modo Rápido", "fresh_start": "Novo Início",
|
||||
"pages": "páginas", "found": "encontradas", "saved": "salvas", "dupes": "duplicatas",
|
||||
"running": "executando", "completed": "concluído", "paused": "pausado",
|
||||
"no_jobs": "Sem tarefas", "started": "Iniciado", "filter": "Filtrar",
|
||||
"all_types": "Todos os tipos", "images": "Imagens", "videos": "Vídeos", "audio": "Áudio",
|
||||
"everything": "Tudo (texto + mídia)", "all_media": "Toda mídia",
|
||||
"images_only": "Apenas imagens", "videos_only": "Apenas vídeos", "text_only": "Apenas texto",
|
||||
"recent_jobs": "Tarefas Recentes", "search_placeholder": "Pesquisar mídia...",
|
||||
"media": "Mídia", "pages_label": "Páginas", "download": "Baixar",
|
||||
"no_media": "Nenhuma mídia encontrada", "no_more": "Sem mais resultados", "load_error": "Falha ao carregar",
|
||||
"copy": "Copiar", "copied": "Copiado!", "starting": "Iniciando...", "started_ok": "Iniciado!",
|
||||
"error": "Erro", "watching": "Observando novas imagens...", "content": "Conteúdo",
|
||||
"previous": "Anterior", "next": "Próximo", "over_9000": "Mais de 9.000+!",
|
||||
"loading_stats": "Carregando estatísticas...", "screenshot": "Captura de tela",
|
||||
"valid_uri": "Insira pelo menos uma URI válida", "load_jobs_error": "Falha ao carregar tarefas",
|
||||
"hydrate_subtitle": "Obter mídia da web", "live_subtitle": "Veja imagens aparecerem durante o rastreio",
|
||||
"total_images": "Total de imagens", "new_session": "Novas nesta sessão", "per_minute": "Por minuto",
|
||||
"all_domains": "Todos os domínios", "download_phantom": "Baixar site fantasma",
|
||||
"phantom_subtitle": "Exportar páginas arquivadas como site estático com mídia local", "max_pages": "Máx. páginas", "select_domain": "Selecionar domínio", "sources": "fontes",
|
||||
"source_uri": "URI de origem", "neopig_uri": "URI Neopig", "source_page": "Página de origem", "neopig_page": "Página Neopig",
|
||||
"type_label": "Tipo", "mime_label": "MIME", "size_label": "Tamanho", "alt_label": "Alt",
|
||||
"keywords_label": "Palavras-chave", "description_label": "Descrição", "items": "itens", "bytes": "bytes",
|
||||
"used_on": "Usado em", "page": "Página", "discovered": "Descoberto",
|
||||
"console": "Console", "close": "Fechar", "no_logs": "Sem logs ainda",
|
||||
},
|
||||
"ru": {
|
||||
"search": "Поиск", "crawl": "Сканировать", "live": "Прямой эфир", "about": "О нас",
|
||||
"random": "Случайное", "phantom": "Призрак", "loading": "Загрузка...",
|
||||
"start_crawl": "Начать сканирование", "pause": "Пауза", "resume": "Продолжить", "delete": "Удалить",
|
||||
"url": "URL", "mode": "Режим", "depth": "Глубина", "keywords": "Ключевые слова",
|
||||
"screenshots": "Скриншоты", "fast_mode": "Быстрый режим", "fresh_start": "Начать заново",
|
||||
"pages": "страниц", "found": "найдено", "saved": "сохранено", "dupes": "дубликаты",
|
||||
"running": "выполняется", "completed": "завершено", "paused": "приостановлено",
|
||||
"no_jobs": "Нет задач", "started": "Начало", "filter": "Фильтр",
|
||||
"all_types": "Все типы", "images": "Изображения", "videos": "Видео", "audio": "Аудио",
|
||||
"everything": "Всё (текст + медиа)", "all_media": "Все медиа",
|
||||
"images_only": "Только изображения", "videos_only": "Только видео", "text_only": "Только текст",
|
||||
"recent_jobs": "Недавние задачи", "search_placeholder": "Поиск медиа...",
|
||||
"media": "Медиа", "pages_label": "Страницы", "download": "Скачать",
|
||||
"no_media": "Медиа не найдено", "no_more": "Больше нет результатов", "load_error": "Ошибка загрузки",
|
||||
"copy": "Копировать", "copied": "Скопировано!", "starting": "Запуск...", "started_ok": "Запущено!",
|
||||
"error": "Ошибка", "watching": "Отслеживание новых изображений...", "content": "Контент",
|
||||
"previous": "Назад", "next": "Далее", "over_9000": "Более 9000+!",
|
||||
"loading_stats": "Загрузка статистики...", "screenshot": "Скриншот",
|
||||
"valid_uri": "Введите хотя бы один URL", "load_jobs_error": "Не удалось загрузить задачи",
|
||||
"hydrate_subtitle": "Получить медиа из интернета", "live_subtitle": "Смотрите появление изображений",
|
||||
"total_images": "Всего изображений", "new_session": "Новые в сессии", "per_minute": "В минуту",
|
||||
"all_domains": "Все домены", "download_phantom": "Скачать призрачный сайт",
|
||||
"phantom_subtitle": "Экспорт архивных страниц как статический сайт с локальными медиа", "max_pages": "Макс. страниц", "select_domain": "Выбрать домен", "sources": "источников",
|
||||
"source_uri": "URI источника", "neopig_uri": "URI Neopig", "source_page": "Страница источника", "neopig_page": "Страница Neopig",
|
||||
"type_label": "Тип", "mime_label": "MIME", "size_label": "Размер", "alt_label": "Alt",
|
||||
"keywords_label": "Ключевые слова", "description_label": "Описание", "items": "элементов", "bytes": "байт",
|
||||
"used_on": "Используется на", "page": "Страница", "discovered": "Обнаружено",
|
||||
"console": "Консоль", "close": "Закрыть", "no_logs": "Пока нет логов",
|
||||
},
|
||||
"ja": {
|
||||
"search": "検索", "crawl": "クロール", "live": "ライブ", "about": "について",
|
||||
"random": "ランダム", "phantom": "ファントム", "loading": "読み込み中...",
|
||||
"start_crawl": "クロール開始", "pause": "一時停止", "resume": "再開", "delete": "削除",
|
||||
"url": "URL", "mode": "モード", "depth": "深さ", "keywords": "キーワード",
|
||||
"screenshots": "スクリーンショット", "fast_mode": "高速モード", "fresh_start": "新規開始",
|
||||
"pages": "ページ", "found": "発見", "saved": "保存", "dupes": "重複",
|
||||
"running": "実行中", "completed": "完了", "paused": "一時停止中",
|
||||
"no_jobs": "ジョブがありません", "started": "開始", "filter": "フィルター",
|
||||
"all_types": "すべてのタイプ", "images": "画像", "videos": "動画", "audio": "オーディオ",
|
||||
"everything": "すべて（テキスト＋メディア）", "all_media": "すべてのメディア",
|
||||
"images_only": "画像のみ", "videos_only": "動画のみ", "text_only": "テキストのみ",
|
||||
"recent_jobs": "最近のジョブ", "search_placeholder": "メディアを検索...",
|
||||
"media": "メディア", "pages_label": "ページ", "download": "ダウンロード",
|
||||
"no_media": "メディアが見つかりません", "no_more": "これ以上の結果はありません", "load_error": "読み込みに失敗",
|
||||
"copy": "コピー", "copied": "コピーしました!", "starting": "開始中...", "started_ok": "開始しました!",
|
||||
"error": "エラー", "watching": "新しい画像を監視中...", "content": "コンテンツ",
|
||||
"previous": "前へ", "next": "次へ", "over_9000": "9000以上!",
|
||||
"loading_stats": "統計を読み込み中...", "screenshot": "スクリーンショット",
|
||||
"valid_uri": "有効なURIを入力してください", "load_jobs_error": "ジョブの読み込みに失敗",
|
||||
"hydrate_subtitle": "ウェブからメディアを取得", "live_subtitle": "クロール中の画像を表示",
|
||||
"total_images": "合計画像数", "new_session": "今回のセッション", "per_minute": "毎分",
|
||||
"all_domains": "すべてのドメイン", "download_phantom": "ファントムサイトをダウンロード",
|
||||
"phantom_subtitle": "アーカイブページをローカルメディア付きの静的サイトとしてエクスポート", "max_pages": "最大ページ数", "select_domain": "ドメインを選択", "sources": "ソース",
|
||||
"source_uri": "ソースURI", "neopig_uri": "Neopig URI", "source_page": "ソースページ", "neopig_page": "Neopigページ",
|
||||
"type_label": "タイプ", "mime_label": "MIME", "size_label": "サイズ", "alt_label": "Alt",
|
||||
"keywords_label": "キーワード", "description_label": "説明", "items": "アイテム", "bytes": "バイト",
|
||||
"used_on": "使用場所", "page": "ページ", "discovered": "発見日時",
|
||||
"console": "コンソール", "close": "閉じる", "no_logs": "ログがありません",
|
||||
},
|
||||
"fr": {
|
||||
"search": "Rechercher", "crawl": "Explorer", "live": "En Direct", "about": "À propos",
|
||||
"random": "Aléatoire", "phantom": "Fantôme", "loading": "Chargement...",
|
||||
"start_crawl": "Démarrer", "pause": "Pause", "resume": "Reprendre", "delete": "Supprimer",
|
||||
"url": "URL", "mode": "Mode", "depth": "Profondeur", "keywords": "Mots-clés",
|
||||
"screenshots": "Captures", "fast_mode": "Mode Rapide", "fresh_start": "Nouveau Départ",
|
||||
"pages": "pages", "found": "trouvées", "saved": "enregistrées", "dupes": "doublons",
|
||||
"running": "en cours", "completed": "terminé", "paused": "en pause",
|
||||
"no_jobs": "Aucune tâche", "started": "Démarré", "filter": "Filtrer",
|
||||
"all_types": "Tous les types", "images": "Images", "videos": "Vidéos", "audio": "Audio",
|
||||
"everything": "Tout (texte + médias)", "all_media": "Tous les médias",
|
||||
"images_only": "Images uniquement", "videos_only": "Vidéos uniquement", "text_only": "Texte uniquement",
|
||||
"recent_jobs": "Tâches Récentes", "search_placeholder": "Rechercher des médias...",
|
||||
"media": "Médias", "pages_label": "Pages", "download": "Télécharger",
|
||||
"no_media": "Aucun média trouvé", "no_more": "Plus de résultats", "load_error": "Échec du chargement",
|
||||
"copy": "Copier", "copied": "Copié!", "starting": "Démarrage...", "started_ok": "Démarré!",
|
||||
"error": "Erreur", "watching": "Surveillance des nouvelles images...", "content": "Contenu",
|
||||
"previous": "Précédent", "next": "Suivant", "over_9000": "Plus de 9 000+!",
|
||||
"loading_stats": "Chargement des statistiques...", "screenshot": "Capture d'écran",
|
||||
"valid_uri": "Veuillez entrer au moins une URI valide", "load_jobs_error": "Échec du chargement des tâches",
|
||||
"hydrate_subtitle": "Récupérer les médias du web", "live_subtitle": "Voir les images apparaître pendant l'exploration",
|
||||
"total_images": "Total d'images", "new_session": "Nouvelles cette session", "per_minute": "Par minute",
|
||||
"all_domains": "Tous les domaines", "download_phantom": "Télécharger le site fantôme",
|
||||
"phantom_subtitle": "Exporter les pages archivées en site statique avec médias locaux", "max_pages": "Pages max", "select_domain": "Sélectionner le domaine", "sources": "sources",
|
||||
"source_uri": "URI source", "neopig_uri": "URI Neopig", "source_page": "Page source", "neopig_page": "Page Neopig",
|
||||
"type_label": "Type", "mime_label": "MIME", "size_label": "Taille", "alt_label": "Alt",
|
||||
"keywords_label": "Mots-clés", "description_label": "Description", "items": "éléments", "bytes": "octets",
|
||||
"used_on": "Utilisé sur", "page": "Page", "discovered": "Découvert",
|
||||
"console": "Console", "close": "Fermer", "no_logs": "Pas encore de logs",
|
||||
},
|
||||
"de": {
|
||||
"search": "Suchen", "crawl": "Crawlen", "live": "Live-Feed", "about": "Über",
|
||||
"random": "Zufällig", "phantom": "Phantom", "loading": "Laden...",
|
||||
"start_crawl": "Crawl starten", "pause": "Pause", "resume": "Fortsetzen", "delete": "Löschen",
|
||||
"url": "URL", "mode": "Modus", "depth": "Tiefe", "keywords": "Stichwörter",
|
||||
"screenshots": "Screenshots", "fast_mode": "Schnellmodus", "fresh_start": "Neustart",
|
||||
"pages": "Seiten", "found": "gefunden", "saved": "gespeichert", "dupes": "Duplikate",
|
||||
"running": "läuft", "completed": "abgeschlossen", "paused": "pausiert",
|
||||
"no_jobs": "Keine Aufgaben", "started": "Gestartet", "filter": "Filter",
|
||||
"all_types": "Alle Typen", "images": "Bilder", "videos": "Videos", "audio": "Audio",
|
||||
"everything": "Alles (Text + Medien)", "all_media": "Alle Medien",
|
||||
"images_only": "Nur Bilder", "videos_only": "Nur Videos", "text_only": "Nur Text",
|
||||
"recent_jobs": "Letzte Aufgaben", "search_placeholder": "Medien suchen...",
|
||||
"media": "Medien", "pages_label": "Seiten", "download": "Herunterladen",
|
||||
"no_media": "Keine Medien gefunden", "no_more": "Keine weiteren Ergebnisse", "load_error": "Laden fehlgeschlagen",
|
||||
"copy": "Kopieren", "copied": "Kopiert!", "starting": "Starten...", "started_ok": "Gestartet!",
|
||||
"error": "Fehler", "watching": "Überwache neue Bilder...", "content": "Inhalt",
|
||||
"previous": "Zurück", "next": "Weiter", "over_9000": "Über 9.000+!",
|
||||
"loading_stats": "Lade Statistiken...", "screenshot": "Screenshot",
|
||||
"valid_uri": "Mindestens eine gültige URI eingeben", "load_jobs_error": "Aufgaben konnten nicht geladen werden",
|
||||
"hydrate_subtitle": "Medien aus dem Web abrufen", "live_subtitle": "Bilder beim Crawlen beobachten",
|
||||
"total_images": "Bilder gesamt", "new_session": "Neu in dieser Sitzung", "per_minute": "Pro Minute",
|
||||
"all_domains": "Alle Domains", "download_phantom": "Phantom-Seite herunterladen",
|
||||
"phantom_subtitle": "Archivierte Seiten als statische Seite mit lokalen Medien exportieren", "max_pages": "Max. Seiten", "select_domain": "Domain auswählen", "sources": "Quellen",
|
||||
"source_uri": "Quell-URI", "neopig_uri": "Neopig-URI", "source_page": "Quellseite", "neopig_page": "Neopig-Seite",
|
||||
"type_label": "Typ", "mime_label": "MIME", "size_label": "Größe", "alt_label": "Alt",
|
||||
"keywords_label": "Stichwörter", "description_label": "Beschreibung", "items": "Elemente", "bytes": "Bytes",
|
||||
"used_on": "Verwendet auf", "page": "Seite", "discovered": "Entdeckt",
|
||||
"console": "Konsole", "close": "Schließen", "no_logs": "Noch keine Logs",
|
||||
},
|
||||
"ko": {
|
||||
"search": "검색", "crawl": "크롤", "live": "라이브", "about": "정보",
|
||||
"random": "랜덤", "phantom": "팬텀", "loading": "로딩 중...",
|
||||
"start_crawl": "크롤 시작", "pause": "일시정지", "resume": "재개", "delete": "삭제",
|
||||
"url": "URL", "mode": "모드", "depth": "깊이", "keywords": "키워드",
|
||||
"screenshots": "스크린샷", "fast_mode": "빠른 모드", "fresh_start": "새로 시작",
|
||||
"pages": "페이지", "found": "발견", "saved": "저장", "dupes": "중복",
|
||||
"running": "실행 중", "completed": "완료", "paused": "일시정지됨",
|
||||
"no_jobs": "작업 없음", "started": "시작됨", "filter": "필터",
|
||||
"all_types": "모든 유형", "images": "이미지", "videos": "동영상", "audio": "오디오",
|
||||
"everything": "모두", "all_media": "모든 미디어",
|
||||
"images_only": "이미지만", "videos_only": "동영상만", "text_only": "텍스트만",
|
||||
"recent_jobs": "최근 작업", "search_placeholder": "미디어 검색...",
|
||||
"media": "미디어", "pages_label": "페이지", "download": "다운로드",
|
||||
"no_media": "미디어를 찾을 수 없음", "no_more": "더 이상 결과 없음", "load_error": "로드 실패",
|
||||
"copy": "복사", "copied": "복사됨!", "starting": "시작 중...", "started_ok": "시작됨!",
|
||||
"error": "오류", "watching": "새 이미지 감시 중...", "content": "콘텐츠",
|
||||
"previous": "이전", "next": "다음", "over_9000": "9,000 이상!",
|
||||
"loading_stats": "통계 로드 중...", "screenshot": "스크린샷",
|
||||
"valid_uri": "유효한 URI를 하나 이상 입력하세요", "load_jobs_error": "작업 로드 실패",
|
||||
"hydrate_subtitle": "웹에서 미디어 가져오기", "live_subtitle": "크롤링되는 이미지 보기",
|
||||
"total_images": "전체 이미지", "new_session": "이번 세션 신규", "per_minute": "분당",
|
||||
"all_domains": "모든 도메인", "download_phantom": "팬텀 사이트 다운로드",
|
||||
"phantom_subtitle": "보관된 페이지를 로컬 미디어가 포함된 정적 사이트로 내보내기", "max_pages": "최대 페이지", "select_domain": "도메인 선택", "sources": "소스",
|
||||
"source_uri": "소스 URI", "neopig_uri": "Neopig URI", "source_page": "소스 페이지", "neopig_page": "Neopig 페이지",
|
||||
"type_label": "유형", "mime_label": "MIME", "size_label": "크기", "alt_label": "Alt",
|
||||
"keywords_label": "키워드", "description_label": "설명", "items": "항목", "bytes": "바이트",
|
||||
"used_on": "사용처", "page": "페이지", "discovered": "발견됨",
|
||||
"console": "콘솔", "close": "닫기", "no_logs": "아직 로그 없음",
|
||||
},
|
||||
"it": {
|
||||
"search": "Cerca", "crawl": "Scansiona", "live": "In Diretta", "about": "Info",
|
||||
"random": "Casuale", "phantom": "Fantasma", "loading": "Caricamento...",
|
||||
"start_crawl": "Avvia Scansione", "pause": "Pausa", "resume": "Riprendi", "delete": "Elimina",
|
||||
"url": "URL", "mode": "Modalità", "depth": "Profondità", "keywords": "Parole chiave",
|
||||
"screenshots": "Screenshot", "fast_mode": "Modalità Veloce", "fresh_start": "Nuovo Inizio",
|
||||
"pages": "pagine", "found": "trovati", "saved": "salvati", "dupes": "duplicati",
|
||||
"running": "in esecuzione", "completed": "completato", "paused": "in pausa",
|
||||
"no_jobs": "Nessuna attività", "started": "Avviato", "filter": "Filtra",
|
||||
"all_types": "Tutti i tipi", "images": "Immagini", "videos": "Video", "audio": "Audio",
|
||||
"everything": "Tutto", "all_media": "Tutti i media",
|
||||
"images_only": "Solo immagini", "videos_only": "Solo video", "text_only": "Solo testo",
|
||||
"recent_jobs": "Attività Recenti", "search_placeholder": "Cerca media...",
|
||||
"media": "Media", "pages_label": "Pagine", "download": "Scarica",
|
||||
"no_media": "Nessun media trovato", "no_more": "Nessun altro risultato", "load_error": "Caricamento fallito",
|
||||
"copy": "Copia", "copied": "Copiato!", "starting": "Avvio...", "started_ok": "Avviato!",
|
||||
"error": "Errore", "watching": "Monitoraggio nuove immagini...", "content": "Contenuto",
|
||||
"previous": "Precedente", "next": "Successivo", "over_9000": "Oltre 9.000+!",
|
||||
"loading_stats": "Caricamento statistiche...", "screenshot": "Screenshot",
|
||||
"valid_uri": "Inserire almeno un URI valido", "load_jobs_error": "Caricamento attività fallito",
|
||||
"hydrate_subtitle": "Ottieni media dal web", "live_subtitle": "Guarda le immagini mentre vengono scansionate",
|
||||
"total_images": "Immagini totali", "new_session": "Nuove in questa sessione", "per_minute": "Al minuto",
|
||||
"all_domains": "Tutti i domini", "download_phantom": "Scarica sito fantasma",
|
||||
"phantom_subtitle": "Esporta pagine archiviate come sito statico con media locali", "max_pages": "Max pagine", "select_domain": "Seleziona dominio", "sources": "fonti",
|
||||
"source_uri": "URI sorgente", "neopig_uri": "URI Neopig", "source_page": "Pagina sorgente", "neopig_page": "Pagina Neopig",
|
||||
"type_label": "Tipo", "mime_label": "MIME", "size_label": "Dimensione", "alt_label": "Alt",
|
||||
"keywords_label": "Parole chiave", "description_label": "Descrizione", "items": "elementi", "bytes": "byte",
|
||||
"used_on": "Usato su", "page": "Pagina", "discovered": "Scoperto",
|
||||
"console": "Console", "close": "Chiudi", "no_logs": "Nessun log ancora",
|
||||
},
|
||||
"nl": {
|
||||
"search": "Zoeken", "crawl": "Crawlen", "live": "Live", "about": "Over",
|
||||
"random": "Willekeurig", "phantom": "Fantoom", "loading": "Laden...",
|
||||
"start_crawl": "Start Crawl", "pause": "Pauzeer", "resume": "Hervat", "delete": "Verwijder",
|
||||
"url": "URL", "mode": "Modus", "depth": "Diepte", "keywords": "Trefwoorden",
|
||||
"screenshots": "Schermafbeeldingen", "fast_mode": "Snelle Modus", "fresh_start": "Nieuwe Start",
|
||||
"pages": "pagina's", "found": "gevonden", "saved": "opgeslagen", "dupes": "duplicaten",
|
||||
"running": "actief", "completed": "voltooid", "paused": "gepauzeerd",
|
||||
"no_jobs": "Geen taken", "started": "Gestart", "filter": "Filter",
|
||||
"all_types": "Alle typen", "images": "Afbeeldingen", "videos": "Video's", "audio": "Audio",
|
||||
"everything": "Alles", "all_media": "Alle media",
|
||||
"images_only": "Alleen afbeeldingen", "videos_only": "Alleen video's", "text_only": "Alleen tekst",
|
||||
"recent_jobs": "Recente Taken", "search_placeholder": "Zoek media...",
|
||||
"media": "Media", "pages_label": "Pagina's", "download": "Downloaden",
|
||||
"no_media": "Geen media gevonden", "no_more": "Geen resultaten meer", "load_error": "Laden mislukt",
|
||||
"copy": "Kopiëren", "copied": "Gekopieerd!", "starting": "Starten...", "started_ok": "Gestart!",
|
||||
"error": "Fout", "watching": "Nieuwe afbeeldingen bekijken...", "content": "Inhoud",
|
||||
"previous": "Vorige", "next": "Volgende", "over_9000": "Meer dan 9.000+!",
|
||||
"loading_stats": "Statistieken laden...", "screenshot": "Schermafbeelding",
|
||||
"valid_uri": "Voer minimaal één geldige URI in", "load_jobs_error": "Taken laden mislukt",
|
||||
"hydrate_subtitle": "Media van het web ophalen", "live_subtitle": "Bekijk afbeeldingen terwijl ze worden gecrawld",
|
||||
"total_images": "Totaal afbeeldingen", "new_session": "Nieuw deze sessie", "per_minute": "Per minuut",
|
||||
"all_domains": "Alle domeinen", "download_phantom": "Fantoomsite downloaden",
|
||||
"phantom_subtitle": "Gearchiveerde pagina's exporteren als statische site met lokale media", "max_pages": "Max. pagina's", "select_domain": "Selecteer domein", "sources": "bronnen",
|
||||
"source_uri": "Bron-URI", "neopig_uri": "Neopig-URI", "source_page": "Bronpagina", "neopig_page": "Neopig-pagina",
|
||||
"type_label": "Type", "mime_label": "MIME", "size_label": "Grootte", "alt_label": "Alt",
|
||||
"keywords_label": "Trefwoorden", "description_label": "Beschrijving", "items": "items", "bytes": "bytes",
|
||||
"used_on": "Gebruikt op", "page": "Pagina", "discovered": "Ontdekt",
|
||||
"console": "Console", "close": "Sluiten", "no_logs": "Nog geen logs",
|
||||
},
|
||||
"pl": {
|
||||
"search": "Szukaj", "crawl": "Indeksuj", "live": "Na żywo", "about": "O nas",
|
||||
"random": "Losowo", "phantom": "Fantom", "loading": "Ładowanie...",
|
||||
"start_crawl": "Rozpocznij", "pause": "Pauza", "resume": "Wznów", "delete": "Usuń",
|
||||
"url": "URL", "mode": "Tryb", "depth": "Głębokość", "keywords": "Słowa kluczowe",
|
||||
"screenshots": "Zrzuty ekranu", "fast_mode": "Tryb szybki", "fresh_start": "Nowy start",
|
||||
"pages": "stron", "found": "znaleziono", "saved": "zapisano", "dupes": "duplikaty",
|
||||
"running": "w toku", "completed": "zakończono", "paused": "wstrzymano",
|
||||
"no_jobs": "Brak zadań", "started": "Rozpoczęto", "filter": "Filtruj",
|
||||
"all_types": "Wszystkie typy", "images": "Obrazy", "videos": "Filmy", "audio": "Audio",
|
||||
"everything": "Wszystko", "all_media": "Wszystkie media",
|
||||
"images_only": "Tylko obrazy", "videos_only": "Tylko filmy", "text_only": "Tylko tekst",
|
||||
"recent_jobs": "Ostatnie Zadania", "search_placeholder": "Szukaj mediów...",
|
||||
"media": "Media", "pages_label": "Strony", "download": "Pobierz",
|
||||
"no_media": "Nie znaleziono mediów", "no_more": "Brak więcej wyników", "load_error": "Błąd ładowania",
|
||||
"copy": "Kopiuj", "copied": "Skopiowano!", "starting": "Uruchamianie...", "started_ok": "Uruchomiono!",
|
||||
"error": "Błąd", "watching": "Obserwowanie nowych obrazów...", "content": "Treść",
|
||||
"previous": "Poprzedni", "next": "Następny", "over_9000": "Ponad 9 000+!",
|
||||
"loading_stats": "Ładowanie statystyk...", "screenshot": "Zrzut ekranu",
|
||||
"valid_uri": "Wprowadź przynajmniej jeden URI", "load_jobs_error": "Nie udało się załadować zadań",
|
||||
"hydrate_subtitle": "Pobierz media z sieci", "live_subtitle": "Oglądaj obrazy podczas indeksowania",
|
||||
"total_images": "Łącznie obrazów", "new_session": "Nowe w tej sesji", "per_minute": "Na minutę",
|
||||
"all_domains": "Wszystkie domeny", "download_phantom": "Pobierz stronę fantomową",
|
||||
"phantom_subtitle": "Eksportuj zarchiwizowane strony jako statyczną witrynę z lokalnymi mediami", "max_pages": "Maks. stron", "select_domain": "Wybierz domenę", "sources": "źródeł",
|
||||
"source_uri": "URI źródła", "neopig_uri": "URI Neopig", "source_page": "Strona źródłowa", "neopig_page": "Strona Neopig",
|
||||
"type_label": "Typ", "mime_label": "MIME", "size_label": "Rozmiar", "alt_label": "Alt",
|
||||
"keywords_label": "Słowa kluczowe", "description_label": "Opis", "items": "elementy", "bytes": "bajty",
|
||||
"used_on": "Używane na", "page": "Strona", "discovered": "Odkryto",
|
||||
"console": "Konsola", "close": "Zamknij", "no_logs": "Brak logów",
|
||||
},
|
||||
"tr": {
|
||||
"search": "Ara", "crawl": "Tara", "live": "Canlı", "about": "Hakkında",
|
||||
"random": "Rastgele", "phantom": "Hayalet", "loading": "Yükleniyor...",
|
||||
"start_crawl": "Taramayı Başlat", "pause": "Duraklat", "resume": "Devam", "delete": "Sil",
|
||||
"url": "URL", "mode": "Mod", "depth": "Derinlik", "keywords": "Anahtar kelimeler",
|
||||
"screenshots": "Ekran görüntüleri", "fast_mode": "Hızlı Mod", "fresh_start": "Yeni Başlangıç",
|
||||
"pages": "sayfa", "found": "bulundu", "saved": "kaydedildi", "dupes": "kopya",
|
||||
"running": "çalışıyor", "completed": "tamamlandı", "paused": "duraklatıldı",
|
||||
"no_jobs": "Görev yok", "started": "Başladı", "filter": "Filtrele",
|
||||
"all_types": "Tüm türler", "images": "Resimler", "videos": "Videolar", "audio": "Ses",
|
||||
"everything": "Her şey", "all_media": "Tüm medya",
|
||||
"images_only": "Sadece resimler", "videos_only": "Sadece videolar", "text_only": "Sadece metin",
|
||||
"recent_jobs": "Son Görevler", "search_placeholder": "Medya ara...",
|
||||
"media": "Medya", "pages_label": "Sayfalar", "download": "İndir",
|
||||
"no_media": "Medya bulunamadı", "no_more": "Daha fazla sonuç yok", "load_error": "Yükleme başarısız",
|
||||
"copy": "Kopyala", "copied": "Kopyalandı!", "starting": "Başlatılıyor...", "started_ok": "Başlatıldı!",
|
||||
"error": "Hata", "watching": "Yeni resimler izleniyor...", "content": "İçerik",
|
||||
"previous": "Önceki", "next": "Sonraki", "over_9000": "9.000'den fazla+!",
|
||||
"loading_stats": "İstatistikler yükleniyor...", "screenshot": "Ekran görüntüsü",
|
||||
"valid_uri": "Lütfen en az bir geçerli URI girin", "load_jobs_error": "Görevler yüklenemedi",
|
||||
"hydrate_subtitle": "Web'den medya al", "live_subtitle": "Taranan resimleri izle",
|
||||
"total_images": "Toplam resim", "new_session": "Bu oturumda yeni", "per_minute": "Dakikada",
|
||||
"all_domains": "Tüm alanlar", "download_phantom": "Hayalet siteyi indir",
|
||||
"phantom_subtitle": "Arşivlenmiş sayfaları yerel medya ile statik site olarak dışa aktar", "max_pages": "Maks. sayfa", "select_domain": "Alan adı seç", "sources": "kaynaklar",
|
||||
"source_uri": "Kaynak URI", "neopig_uri": "Neopig URI", "source_page": "Kaynak Sayfa", "neopig_page": "Neopig Sayfası",
|
||||
"type_label": "Tür", "mime_label": "MIME", "size_label": "Boyut", "alt_label": "Alt",
|
||||
"keywords_label": "Anahtar kelimeler", "description_label": "Açıklama", "items": "öğe", "bytes": "bayt",
|
||||
"used_on": "Kullanıldığı yer", "page": "Sayfa", "discovered": "Keşfedildi",
|
||||
"console": "Konsol", "close": "Kapat", "no_logs": "Henüz log yok",
|
||||
},
|
||||
"vi": {
|
||||
"search": "Tìm kiếm", "crawl": "Thu thập", "live": "Trực tiếp", "about": "Giới thiệu",
|
||||
"random": "Ngẫu nhiên", "phantom": "Bóng ma", "loading": "Đang tải...",
|
||||
"start_crawl": "Bắt đầu", "pause": "Tạm dừng", "resume": "Tiếp tục", "delete": "Xóa",
|
||||
"url": "URL", "mode": "Chế độ", "depth": "Độ sâu", "keywords": "Từ khóa",
|
||||
"screenshots": "Ảnh chụp", "fast_mode": "Chế độ nhanh", "fresh_start": "Bắt đầu mới",
|
||||
"pages": "trang", "found": "tìm thấy", "saved": "đã lưu", "dupes": "trùng lặp",
|
||||
"running": "đang chạy", "completed": "hoàn thành", "paused": "tạm dừng",
|
||||
"no_jobs": "Không có tác vụ", "started": "Đã bắt đầu", "filter": "Lọc",
|
||||
"all_types": "Tất cả loại", "images": "Hình ảnh", "videos": "Video", "audio": "Âm thanh",
|
||||
"everything": "Tất cả", "all_media": "Tất cả media",
|
||||
"images_only": "Chỉ hình ảnh", "videos_only": "Chỉ video", "text_only": "Chỉ văn bản",
|
||||
"recent_jobs": "Tác vụ gần đây", "search_placeholder": "Tìm media...",
|
||||
"media": "Media", "pages_label": "Trang", "download": "Tải xuống",
|
||||
"no_media": "Không tìm thấy media", "no_more": "Không còn kết quả", "load_error": "Tải thất bại",
|
||||
"copy": "Sao chép", "copied": "Đã sao chép!", "starting": "Đang khởi động...", "started_ok": "Đã khởi động!",
|
||||
"error": "Lỗi", "watching": "Đang theo dõi hình ảnh mới...", "content": "Nội dung",
|
||||
"previous": "Trước", "next": "Tiếp", "over_9000": "Hơn 9.000+!",
|
||||
"loading_stats": "Đang tải thống kê...", "screenshot": "Ảnh chụp màn hình",
|
||||
"valid_uri": "Vui lòng nhập ít nhất một URI hợp lệ", "load_jobs_error": "Không thể tải tác vụ",
|
||||
"hydrate_subtitle": "Lấy media từ web", "live_subtitle": "Xem hình ảnh khi đang thu thập",
|
||||
"total_images": "Tổng hình ảnh", "new_session": "Mới trong phiên này", "per_minute": "Mỗi phút",
|
||||
"all_domains": "Tất cả tên miền", "download_phantom": "Tải trang bóng ma",
|
||||
"phantom_subtitle": "Xuất trang lưu trữ dưới dạng trang tĩnh với media cục bộ", "max_pages": "Tối đa trang", "select_domain": "Chọn tên miền", "sources": "nguồn",
|
||||
"source_uri": "URI nguồn", "neopig_uri": "URI Neopig", "source_page": "Trang nguồn", "neopig_page": "Trang Neopig",
|
||||
"type_label": "Loại", "mime_label": "MIME", "size_label": "Kích thước", "alt_label": "Alt",
|
||||
"keywords_label": "Từ khóa", "description_label": "Mô tả", "items": "mục", "bytes": "byte",
|
||||
"used_on": "Sử dụng tại", "page": "Trang", "discovered": "Phát hiện",
|
||||
"console": "Bảng điều khiển", "close": "Đóng", "no_logs": "Chưa có nhật ký",
|
||||
},
|
||||
"th": {
|
||||
"search": "ค้นหา", "crawl": "รวบรวม", "live": "สด", "about": "เกี่ยวกับ",
|
||||
"random": "สุ่ม", "phantom": "แฟนทอม", "loading": "กำลังโหลด...",
|
||||
"start_crawl": "เริ่มรวบรวม", "pause": "หยุดชั่วคราว", "resume": "ดำเนินต่อ", "delete": "ลบ",
|
||||
"url": "URL", "mode": "โหมด", "depth": "ความลึก", "keywords": "คำสำคัญ",
|
||||
"screenshots": "ภาพหน้าจอ", "fast_mode": "โหมดเร็ว", "fresh_start": "เริ่มใหม่",
|
||||
"pages": "หน้า", "found": "พบ", "saved": "บันทึก", "dupes": "ซ้ำ",
|
||||
"running": "กำลังทำงาน", "completed": "เสร็จสิ้น", "paused": "หยุดชั่วคราว",
|
||||
"no_jobs": "ไม่มีงาน", "started": "เริ่มแล้ว", "filter": "กรอง",
|
||||
"all_types": "ทุกประเภท", "images": "รูปภาพ", "videos": "วิดีโอ", "audio": "เสียง",
|
||||
"everything": "ทั้งหมด", "all_media": "สื่อทั้งหมด",
|
||||
"images_only": "เฉพาะรูปภาพ", "videos_only": "เฉพาะวิดีโอ", "text_only": "เฉพาะข้อความ",
|
||||
"recent_jobs": "งานล่าสุด", "search_placeholder": "ค้นหาสื่อ...",
|
||||
"media": "สื่อ", "pages_label": "หน้า", "download": "ดาวน์โหลด",
|
||||
"no_media": "ไม่พบสื่อ", "no_more": "ไม่มีผลลัพธ์เพิ่มเติม", "load_error": "โหลดไม่สำเร็จ",
|
||||
"copy": "คัดลอก", "copied": "คัดลอกแล้ว!", "starting": "กำลังเริ่ม...", "started_ok": "เริ่มแล้ว!",
|
||||
"error": "ข้อผิดพลาด", "watching": "กำลังดูรูปภาพใหม่...", "content": "เนื้อหา",
|
||||
"previous": "ก่อนหน้า", "next": "ถัดไป", "over_9000": "มากกว่า 9,000+!",
|
||||
"loading_stats": "กำลังโหลดสถิติ...", "screenshot": "ภาพหน้าจอ",
|
||||
"valid_uri": "กรุณาใส่ URI ที่ถูกต้องอย่างน้อยหนึ่งรายการ", "load_jobs_error": "ไม่สามารถโหลดงานได้",
|
||||
"hydrate_subtitle": "ดึงสื่อจากเว็บ", "live_subtitle": "ดูรูปภาพขณะรวบรวม",
|
||||
"total_images": "รูปภาพทั้งหมด", "new_session": "ใหม่ในเซสชันนี้", "per_minute": "ต่อนาที",
|
||||
"all_domains": "โดเมนทั้งหมด", "download_phantom": "ดาวน์โหลดเว็บแฟนทอม",
|
||||
"phantom_subtitle": "ส่งออกหน้าที่เก็บถาวรเป็นเว็บไซต์แบบคงที่พร้อมสื่อในเครื่อง", "max_pages": "หน้าสูงสุด", "select_domain": "เลือกโดเมน", "sources": "แหล่งที่มา",
|
||||
"source_uri": "URI แหล่งที่มา", "neopig_uri": "URI Neopig", "source_page": "หน้าแหล่งที่มา", "neopig_page": "หน้า Neopig",
|
||||
"type_label": "ประเภท", "mime_label": "MIME", "size_label": "ขนาด", "alt_label": "Alt",
|
||||
"keywords_label": "คำสำคัญ", "description_label": "คำอธิบาย", "items": "รายการ", "bytes": "ไบต์",
|
||||
"used_on": "ใช้บน", "page": "หน้า", "discovered": "ค้นพบ",
|
||||
"console": "คอนโซล", "close": "ปิด", "no_logs": "ยังไม่มีบันทึก",
|
||||
},
|
||||
"id": {
|
||||
"search": "Cari", "crawl": "Jelajahi", "live": "Langsung", "about": "Tentang",
|
||||
"random": "Acak", "phantom": "Hantu", "loading": "Memuat...",
|
||||
"start_crawl": "Mulai Jelajah", "pause": "Jeda", "resume": "Lanjutkan", "delete": "Hapus",
|
||||
"url": "URL", "mode": "Mode", "depth": "Kedalaman", "keywords": "Kata kunci",
|
||||
"screenshots": "Tangkapan layar", "fast_mode": "Mode Cepat", "fresh_start": "Mulai Baru",
|
||||
"pages": "halaman", "found": "ditemukan", "saved": "disimpan", "dupes": "duplikat",
|
||||
"running": "berjalan", "completed": "selesai", "paused": "dijeda",
|
||||
"no_jobs": "Tidak ada tugas", "started": "Dimulai", "filter": "Filter",
|
||||
"all_types": "Semua jenis", "images": "Gambar", "videos": "Video", "audio": "Audio",
|
||||
"everything": "Semua", "all_media": "Semua media",
|
||||
"images_only": "Hanya gambar", "videos_only": "Hanya video", "text_only": "Hanya teks",
|
||||
"recent_jobs": "Tugas Terbaru", "search_placeholder": "Cari media...",
|
||||
"media": "Media", "pages_label": "Halaman", "download": "Unduh",
|
||||
"no_media": "Media tidak ditemukan", "no_more": "Tidak ada hasil lagi", "load_error": "Gagal memuat",
|
||||
"copy": "Salin", "copied": "Disalin!", "starting": "Memulai...", "started_ok": "Dimulai!",
|
||||
"error": "Kesalahan", "watching": "Memantau gambar baru...", "content": "Konten",
|
||||
"previous": "Sebelumnya", "next": "Berikutnya", "over_9000": "Lebih dari 9.000+!",
|
||||
"loading_stats": "Memuat statistik...", "screenshot": "Tangkapan layar",
|
||||
"valid_uri": "Masukkan setidaknya satu URI yang valid", "load_jobs_error": "Gagal memuat tugas",
|
||||
"hydrate_subtitle": "Ambil media dari web", "live_subtitle": "Lihat gambar saat dijelajahi",
|
||||
"total_images": "Total gambar", "new_session": "Baru sesi ini", "per_minute": "Per menit",
|
||||
"all_domains": "Semua domain", "download_phantom": "Unduh situs hantu",
|
||||
"phantom_subtitle": "Ekspor halaman arsip sebagai situs statis dengan media lokal", "max_pages": "Maks. halaman", "select_domain": "Pilih domain", "sources": "sumber",
|
||||
"source_uri": "URI sumber", "neopig_uri": "URI Neopig", "source_page": "Halaman sumber", "neopig_page": "Halaman Neopig",
|
||||
"type_label": "Jenis", "mime_label": "MIME", "size_label": "Ukuran", "alt_label": "Alt",
|
||||
"keywords_label": "Kata kunci", "description_label": "Deskripsi", "items": "item", "bytes": "byte",
|
||||
"used_on": "Digunakan di", "page": "Halaman", "discovered": "Ditemukan",
|
||||
"console": "Konsol", "close": "Tutup", "no_logs": "Belum ada log",
|
||||
},
|
||||
"uk": {
|
||||
"search": "Пошук", "crawl": "Сканувати", "live": "Наживо", "about": "Про нас",
|
||||
"random": "Випадково", "phantom": "Привид", "loading": "Завантаження...",
|
||||
"start_crawl": "Почати сканування", "pause": "Пауза", "resume": "Продовжити", "delete": "Видалити",
|
||||
"url": "URL", "mode": "Режим", "depth": "Глибина", "keywords": "Ключові слова",
|
||||
"screenshots": "Знімки екрану", "fast_mode": "Швидкий режим", "fresh_start": "Новий старт",
|
||||
"pages": "сторінок", "found": "знайдено", "saved": "збережено", "dupes": "дублікати",
|
||||
"running": "виконується", "completed": "завершено", "paused": "призупинено",
|
||||
"no_jobs": "Немає завдань", "started": "Розпочато", "filter": "Фільтр",
|
||||
"all_types": "Усі типи", "images": "Зображення", "videos": "Відео", "audio": "Аудіо",
|
||||
"everything": "Все", "all_media": "Усі медіа",
|
||||
"images_only": "Лише зображення", "videos_only": "Лише відео", "text_only": "Лише текст",
|
||||
"recent_jobs": "Останні завдання", "search_placeholder": "Пошук медіа...",
|
||||
"media": "Медіа", "pages_label": "Сторінки", "download": "Завантажити",
|
||||
"no_media": "Медіа не знайдено", "no_more": "Більше немає результатів", "load_error": "Помилка завантаження",
|
||||
"copy": "Копіювати", "copied": "Скопійовано!", "starting": "Запуск...", "started_ok": "Запущено!",
|
||||
"error": "Помилка", "watching": "Відстеження нових зображень...", "content": "Вміст",
|
||||
"previous": "Попередній", "next": "Наступний", "over_9000": "Понад 9000+!",
|
||||
"loading_stats": "Завантаження статистики...", "screenshot": "Знімок екрану",
|
||||
"valid_uri": "Введіть хоча б один дійсний URI", "load_jobs_error": "Не вдалося завантажити завдання",
|
||||
"hydrate_subtitle": "Отримати медіа з вебу", "live_subtitle": "Дивіться зображення під час сканування",
|
||||
"total_images": "Всього зображень", "new_session": "Нові в сесії", "per_minute": "За хвилину",
|
||||
"all_domains": "Усі домени", "download_phantom": "Завантажити привид-сайт",
|
||||
"phantom_subtitle": "Експортувати архівні сторінки як статичний сайт з локальними медіа", "max_pages": "Макс. сторінок", "select_domain": "Вибрати домен", "sources": "джерел",
|
||||
"source_uri": "URI джерела", "neopig_uri": "URI Neopig", "source_page": "Сторінка джерела", "neopig_page": "Сторінка Neopig",
|
||||
"type_label": "Тип", "mime_label": "MIME", "size_label": "Розмір", "alt_label": "Alt",
|
||||
"keywords_label": "Ключові слова", "description_label": "Опис", "items": "елементів", "bytes": "байт",
|
||||
"used_on": "Використовується на", "page": "Сторінка", "discovered": "Виявлено",
|
||||
"console": "Консоль", "close": "Закрити", "no_logs": "Ще немає логів",
|
||||
},
|
||||
"sv": {
|
||||
"search": "Sök", "crawl": "Genomsök", "live": "Live", "about": "Om",
|
||||
"random": "Slumpmässig", "phantom": "Fantom", "loading": "Laddar...",
|
||||
"start_crawl": "Starta genomsökning", "pause": "Pausa", "resume": "Återuppta", "delete": "Ta bort",
|
||||
"url": "URL", "mode": "Läge", "depth": "Djup", "keywords": "Nyckelord",
|
||||
"screenshots": "Skärmbilder", "fast_mode": "Snabbläge", "fresh_start": "Ny start",
|
||||
"pages": "sidor", "found": "hittade", "saved": "sparade", "dupes": "dubbletter",
|
||||
"running": "körs", "completed": "slutförd", "paused": "pausad",
|
||||
"no_jobs": "Inga jobb", "started": "Startad", "filter": "Filtrera",
|
||||
"all_types": "Alla typer", "images": "Bilder", "videos": "Videor", "audio": "Ljud",
|
||||
"everything": "Allt", "all_media": "Alla media",
|
||||
"images_only": "Endast bilder", "videos_only": "Endast videor", "text_only": "Endast text",
|
||||
"recent_jobs": "Senaste Jobb", "search_placeholder": "Sök media...",
|
||||
"media": "Media", "pages_label": "Sidor", "download": "Ladda ner",
|
||||
"no_media": "Ingen media hittades", "no_more": "Inga fler resultat", "load_error": "Kunde inte ladda",
|
||||
"copy": "Kopiera", "copied": "Kopierat!", "starting": "Startar...", "started_ok": "Startad!",
|
||||
"error": "Fel", "watching": "Bevakar nya bilder...", "content": "Innehåll",
|
||||
"previous": "Föregående", "next": "Nästa", "over_9000": "Över 9 000+!",
|
||||
"loading_stats": "Laddar statistik...", "screenshot": "Skärmbild",
|
||||
"valid_uri": "Ange minst en giltig URI", "load_jobs_error": "Kunde inte ladda jobb",
|
||||
"hydrate_subtitle": "Hämta media från webben", "live_subtitle": "Se bilder medan de genomsöks",
|
||||
"total_images": "Totalt bilder", "new_session": "Nya denna session", "per_minute": "Per minut",
|
||||
"all_domains": "Alla domäner", "download_phantom": "Ladda ner fantomwebbplats",
|
||||
"phantom_subtitle": "Exportera arkiverade sidor som statisk webbplats med lokal media", "max_pages": "Max sidor", "select_domain": "Välj domän", "sources": "källor",
|
||||
"source_uri": "Käll-URI", "neopig_uri": "Neopig URI", "source_page": "Källsida", "neopig_page": "Neopig-sida",
|
||||
"type_label": "Typ", "mime_label": "MIME", "size_label": "Storlek", "alt_label": "Alt",
|
||||
"keywords_label": "Nyckelord", "description_label": "Beskrivning", "items": "objekt", "bytes": "bytes",
|
||||
"used_on": "Används på", "page": "Sida", "discovered": "Upptäckt",
|
||||
"console": "Konsol", "close": "Stäng", "no_logs": "Inga loggar ännu",
|
||||
},
|
||||
"zh-tw": {
|
||||
"search": "搜尋", "crawl": "爬取", "live": "即時動態", "about": "關於",
|
||||
"random": "隨機", "phantom": "幻影", "loading": "載入中...",
|
||||
"start_crawl": "開始爬取", "pause": "暫停", "resume": "繼續", "delete": "刪除",
|
||||
"url": "網址", "mode": "模式", "depth": "深度", "keywords": "關鍵字",
|
||||
"screenshots": "螢幕截圖", "fast_mode": "快速模式", "fresh_start": "全新開始",
|
||||
"pages": "頁面", "found": "發現", "saved": "儲存", "dupes": "重複",
|
||||
"running": "執行中", "completed": "已完成", "paused": "已暫停",
|
||||
"no_jobs": "暫無爬取任務", "started": "開始於", "filter": "篩選",
|
||||
"all_types": "所有類型", "images": "圖片", "videos": "影片", "audio": "音訊",
|
||||
"everything": "全部（文字+媒體）", "all_media": "所有媒體",
|
||||
"images_only": "僅圖片", "videos_only": "僅影片", "text_only": "僅文字",
|
||||
"recent_jobs": "最近任務", "search_placeholder": "搜尋媒體...",
|
||||
"media": "媒體", "pages_label": "頁面", "download": "下載",
|
||||
"no_media": "未找到媒體", "no_more": "沒有更多結果", "load_error": "載入失敗",
|
||||
"copy": "複製", "copied": "已複製!", "starting": "啟動中...", "started_ok": "已啟動!",
|
||||
"error": "錯誤", "watching": "正在監視新圖片...", "content": "內容",
|
||||
"previous": "上一頁", "next": "下一頁", "over_9000": "超過9000+!",
|
||||
"loading_stats": "載入統計中...", "screenshot": "螢幕截圖",
|
||||
"valid_uri": "請輸入至少一個有效的URI", "load_jobs_error": "載入任務失敗",
|
||||
"hydrate_subtitle": "從網路取得媒體", "live_subtitle": "觀看正在爬取的圖片",
|
||||
"total_images": "總圖片數", "new_session": "本次新增", "per_minute": "每分鐘",
|
||||
"all_domains": "所有網域", "download_phantom": "下載幻影網站",
|
||||
"phantom_subtitle": "將存檔頁面匯出為帶本地媒體的靜態網站", "max_pages": "最大頁數", "select_domain": "選擇網域", "sources": "來源",
|
||||
"source_uri": "來源URI", "neopig_uri": "Neopig URI", "source_page": "來源頁面", "neopig_page": "Neopig頁面",
|
||||
"type_label": "類型", "mime_label": "MIME", "size_label": "大小", "alt_label": "替代文字",
|
||||
"keywords_label": "關鍵字", "description_label": "描述", "items": "項", "bytes": "位元組",
|
||||
"used_on": "使用於", "page": "頁面", "discovered": "發現時間",
|
||||
"console": "主控台", "close": "關閉", "no_logs": "暫無日誌",
|
||||
},
|
||||
"bn": {
|
||||
"search": "অনুসন্ধান", "crawl": "ক্রল", "live": "লাইভ ফিড", "about": "সম্পর্কে",
|
||||
"random": "এলোমেলো", "phantom": "ফ্যান্টম", "loading": "লোড হচ্ছে...",
|
||||
"start_crawl": "ক্রল শুরু করুন", "pause": "বিরতি", "resume": "পুনরায় শুরু", "delete": "মুছুন",
|
||||
"url": "URL", "mode": "মোড", "depth": "গভীরতা", "keywords": "কীওয়ার্ড",
|
||||
"screenshots": "স্ক্রিনশট", "fast_mode": "দ্রুত মোড", "fresh_start": "নতুন শুরু",
|
||||
"pages": "পৃষ্ঠা", "found": "পাওয়া গেছে", "saved": "সংরক্ষিত", "dupes": "ডুপ্লিকেট",
|
||||
"running": "চলছে", "completed": "সম্পন্ন", "paused": "বিরতি",
|
||||
"no_jobs": "কোন ক্রল কাজ নেই", "started": "শুরু", "filter": "ফিল্টার",
|
||||
"all_types": "সব ধরনের", "images": "ছবি", "videos": "ভিডিও", "audio": "অডিও",
|
||||
"everything": "সবকিছু", "all_media": "সব মিডিয়া",
|
||||
"images_only": "শুধু ছবি", "videos_only": "শুধু ভিডিও", "text_only": "শুধু টেক্সট",
|
||||
"recent_jobs": "সাম্প্রতিক কাজ", "search_placeholder": "মিডিয়া অনুসন্ধান...",
|
||||
"media": "মিডিয়া", "pages_label": "পৃষ্ঠা", "download": "ডাউনলোড",
|
||||
"no_media": "কোন মিডিয়া পাওয়া যায়নি", "no_more": "আর কোন ফলাফল নেই", "load_error": "লোড ব্যর্থ",
|
||||
"copy": "কপি", "copied": "কপি হয়েছে!", "starting": "শুরু হচ্ছে...", "started_ok": "শুরু হয়েছে!",
|
||||
"error": "ত্রুটি", "watching": "নতুন ছবি দেখছি...", "content": "বিষয়বস্তু",
|
||||
"previous": "পূর্ববর্তী", "next": "পরবর্তী", "over_9000": "৯,০০০+ এর বেশি!",
|
||||
"loading_stats": "পরিসংখ্যান লোড হচ্ছে...", "screenshot": "স্ক্রিনশট",
|
||||
"valid_uri": "অন্তত একটি বৈধ URI দিন", "load_jobs_error": "কাজ লোড ব্যর্থ",
|
||||
"hydrate_subtitle": "ওয়েব থেকে মিডিয়া আনুন", "live_subtitle": "ক্রল হওয়া ছবি দেখুন",
|
||||
"total_images": "মোট ছবি", "new_session": "এই সেশনে নতুন", "per_minute": "প্রতি মিনিটে",
|
||||
"all_domains": "সব ডোমেইন", "download_phantom": "ফ্যান্টম সাইট ডাউনলোড",
|
||||
"phantom_subtitle": "সংরক্ষিত পৃষ্ঠা স্থানীয় মিডিয়া সহ স্ট্যাটিক সাইট হিসেবে রপ্তানি", "max_pages": "সর্বোচ্চ পৃষ্ঠা", "select_domain": "ডোমেইন নির্বাচন", "sources": "সূত্র",
|
||||
"source_uri": "উৎস URI", "neopig_uri": "Neopig URI", "source_page": "উৎস পৃষ্ঠা", "neopig_page": "Neopig পৃষ্ঠা",
|
||||
"type_label": "ধরন", "mime_label": "MIME", "size_label": "আকার", "alt_label": "Alt",
|
||||
"keywords_label": "কীওয়ার্ড", "description_label": "বিবরণ", "items": "আইটেম", "bytes": "বাইট",
|
||||
"used_on": "ব্যবহৃত", "page": "পৃষ্ঠা", "discovered": "আবিষ্কৃত",
|
||||
"console": "কনসোল", "close": "বন্ধ", "no_logs": "কোন লগ নেই",
|
||||
},
|
||||
"ur": {
|
||||
"search": "تلاش", "crawl": "کرال", "live": "لائیو فیڈ", "about": "کے بارے میں",
|
||||
"random": "بے ترتیب", "phantom": "فینٹم", "loading": "لوڈ ہو رہا ہے...",
|
||||
"start_crawl": "کرال شروع کریں", "pause": "روکیں", "resume": "جاری رکھیں", "delete": "حذف کریں",
|
||||
"url": "یو آر ایل", "mode": "موڈ", "depth": "گہرائی", "keywords": "کلیدی الفاظ",
|
||||
"screenshots": "اسکرین شاٹس", "fast_mode": "تیز موڈ", "fresh_start": "نئی شروعات",
|
||||
"pages": "صفحات", "found": "ملے", "saved": "محفوظ", "dupes": "ڈپلیکیٹ",
|
||||
"running": "چل رہا ہے", "completed": "مکمل", "paused": "روکا ہوا",
|
||||
"no_jobs": "کوئی کرال کام نہیں", "started": "شروع", "filter": "فلٹر",
|
||||
"all_types": "تمام اقسام", "images": "تصاویر", "videos": "ویڈیوز", "audio": "آڈیو",
|
||||
"everything": "سب کچھ", "all_media": "تمام میڈیا",
|
||||
"images_only": "صرف تصاویر", "videos_only": "صرف ویڈیوز", "text_only": "صرف متن",
|
||||
"recent_jobs": "حالیہ کام", "search_placeholder": "میڈیا تلاش کریں...",
|
||||
"media": "میڈیا", "pages_label": "صفحات", "download": "ڈاؤن لوڈ",
|
||||
"no_media": "کوئی میڈیا نہیں ملا", "no_more": "مزید نتائج نہیں", "load_error": "لوڈ ناکام",
|
||||
"copy": "کاپی", "copied": "کاپی ہو گیا!", "starting": "شروع ہو رہا ہے...", "started_ok": "شروع ہو گیا!",
|
||||
"error": "خرابی", "watching": "نئی تصاویر دیکھ رہا ہے...", "content": "مواد",
|
||||
"previous": "پچھلا", "next": "اگلا", "over_9000": "9,000+ سے زیادہ!",
|
||||
"loading_stats": "اعدادوشمار لوڈ ہو رہے ہیں...", "screenshot": "اسکرین شاٹ",
|
||||
"valid_uri": "کم از کم ایک درست URI درج کریں", "load_jobs_error": "کام لوڈ ناکام",
|
||||
"hydrate_subtitle": "ویب سے میڈیا حاصل کریں", "live_subtitle": "کرال ہوتی تصاویر دیکھیں",
|
||||
"total_images": "کل تصاویر", "new_session": "اس سیشن میں نئی", "per_minute": "فی منٹ",
|
||||
"all_domains": "تمام ڈومینز", "download_phantom": "فینٹم سائٹ ڈاؤن لوڈ",
|
||||
"phantom_subtitle": "آرکائیو شدہ صفحات کو مقامی میڈیا کے ساتھ جامد سائٹ کے طور پر برآمد کریں", "max_pages": "زیادہ سے زیادہ صفحات", "select_domain": "ڈومین منتخب کریں", "sources": "ذرائع",
|
||||
"source_uri": "ماخذ URI", "neopig_uri": "Neopig URI", "source_page": "ماخذ صفحہ", "neopig_page": "Neopig صفحہ",
|
||||
"type_label": "قسم", "mime_label": "MIME", "size_label": "سائز", "alt_label": "Alt",
|
||||
"keywords_label": "کلیدی الفاظ", "description_label": "تفصیل", "items": "آئٹمز", "bytes": "بائٹس",
|
||||
"used_on": "استعمال شدہ", "page": "صفحہ", "discovered": "دریافت",
|
||||
"console": "کنسول", "close": "بند کریں", "no_logs": "ابھی کوئی لاگ نہیں",
|
||||
},
|
||||
"sw": {
|
||||
"search": "Tafuta", "crawl": "Tambaa", "live": "Moja kwa Moja", "about": "Kuhusu",
|
||||
"random": "Nasibu", "phantom": "Phantom", "loading": "Inapakia...",
|
||||
"start_crawl": "Anza Kutambaa", "pause": "Simamisha", "resume": "Endelea", "delete": "Futa",
|
||||
"url": "URL", "mode": "Hali", "depth": "Kina", "keywords": "Maneno muhimu",
|
||||
"screenshots": "Picha za skrini", "fast_mode": "Hali ya Haraka", "fresh_start": "Mwanzo Mpya",
|
||||
"pages": "kurasa", "found": "zilizopatikana", "saved": "zilizohifadhiwa", "dupes": "nakala",
|
||||
"running": "inaendelea", "completed": "imekamilika", "paused": "imesimamishwa",
|
||||
"no_jobs": "Hakuna kazi", "started": "Ilianza", "filter": "Chuja",
|
||||
"all_types": "Aina zote", "images": "Picha", "videos": "Video", "audio": "Sauti",
|
||||
"everything": "Kila kitu", "all_media": "Media zote",
|
||||
"images_only": "Picha tu", "videos_only": "Video tu", "text_only": "Maandishi tu",
|
||||
"recent_jobs": "Kazi za Hivi Karibuni", "search_placeholder": "Tafuta media...",
|
||||
"media": "Media", "pages_label": "Kurasa", "download": "Pakua",
|
||||
"no_media": "Hakuna media iliyopatikana", "no_more": "Hakuna matokeo zaidi", "load_error": "Imeshindwa kupakia",
|
||||
"copy": "Nakili", "copied": "Imenakiliwa!", "starting": "Inaanza...", "started_ok": "Imeanza!",
|
||||
"error": "Hitilafu", "watching": "Inatazama picha mpya...", "content": "Maudhui",
|
||||
"previous": "Iliyotangulia", "next": "Inayofuata", "over_9000": "Zaidi ya 9,000+!",
|
||||
"loading_stats": "Inapakia takwimu...", "screenshot": "Picha ya skrini",
|
||||
"valid_uri": "Tafadhali weka URI moja halali", "load_jobs_error": "Imeshindwa kupakia kazi",
|
||||
"hydrate_subtitle": "Pata media kutoka wavuti", "live_subtitle": "Tazama picha zikitambaazwa",
|
||||
"total_images": "Jumla ya Picha", "new_session": "Mpya Kipindi Hiki", "per_minute": "Kwa Dakika",
|
||||
"all_domains": "Vikoa vyote", "download_phantom": "Pakua tovuti ya phantom",
|
||||
"phantom_subtitle": "Hamisha kurasa zilizohifadhiwa kama tovuti tuli na media za ndani", "max_pages": "Kurasa za juu", "select_domain": "Chagua Kikoa", "sources": "vyanzo",
|
||||
"source_uri": "URI Chanzo", "neopig_uri": "Neopig URI", "source_page": "Ukurasa Chanzo", "neopig_page": "Ukurasa Neopig",
|
||||
"type_label": "Aina", "mime_label": "MIME", "size_label": "Ukubwa", "alt_label": "Alt",
|
||||
"keywords_label": "Maneno muhimu", "description_label": "Maelezo", "items": "vitu", "bytes": "baiti",
|
||||
"used_on": "Imetumika", "page": "Ukurasa", "discovered": "Iligunduliwa",
|
||||
"console": "Konsoli", "close": "Funga", "no_logs": "Hakuna kumbukumbu bado",
|
||||
},
|
||||
"mr": {
|
||||
"search": "शोध", "crawl": "क्रॉल", "live": "थेट फीड", "about": "बद्दल",
|
||||
"random": "यादृच्छिक", "phantom": "फँटम", "loading": "लोड होत आहे...",
|
||||
"start_crawl": "क्रॉल सुरू करा", "pause": "थांबवा", "resume": "पुन्हा सुरू करा", "delete": "हटवा",
|
||||
"url": "URL", "mode": "मोड", "depth": "खोली", "keywords": "कीवर्ड",
|
||||
"screenshots": "स्क्रीनशॉट", "fast_mode": "जलद मोड", "fresh_start": "नवीन सुरुवात",
|
||||
"pages": "पृष्ठे", "found": "सापडले", "saved": "जतन केले", "dupes": "डुप्लिकेट",
|
||||
"running": "चालू आहे", "completed": "पूर्ण", "paused": "थांबवले",
|
||||
"no_jobs": "कोणतेही क्रॉल काम नाही", "started": "सुरू", "filter": "फिल्टर",
|
||||
"all_types": "सर्व प्रकार", "images": "प्रतिमा", "videos": "व्हिडिओ", "audio": "ऑडिओ",
|
||||
"everything": "सर्वकाही", "all_media": "सर्व मीडिया",
|
||||
"images_only": "फक्त प्रतिमा", "videos_only": "फक्त व्हिडिओ", "text_only": "फक्त मजकूर",
|
||||
"recent_jobs": "अलीकडील कामे", "search_placeholder": "मीडिया शोधा...",
|
||||
"media": "मीडिया", "pages_label": "पृष्ठे", "download": "डाउनलोड",
|
||||
"no_media": "मीडिया सापडला नाही", "no_more": "आणखी परिणाम नाहीत", "load_error": "लोड अयशस्वी",
|
||||
"copy": "कॉपी", "copied": "कॉपी झाले!", "starting": "सुरू होत आहे...", "started_ok": "सुरू झाले!",
|
||||
"error": "त्रुटी", "watching": "नवीन प्रतिमा पाहत आहे...", "content": "सामग्री",
|
||||
"previous": "मागील", "next": "पुढील", "over_9000": "9,000+ पेक्षा जास्त!",
|
||||
"loading_stats": "आकडेवारी लोड होत आहे...", "screenshot": "स्क्रीनशॉट",
|
||||
"valid_uri": "कृपया किमान एक वैध URI प्रविष्ट करा", "load_jobs_error": "कामे लोड अयशस्वी",
|
||||
"hydrate_subtitle": "वेबवरून मीडिया मिळवा", "live_subtitle": "क्रॉल होताना प्रतिमा पहा",
|
||||
"total_images": "एकूण प्रतिमा", "new_session": "या सत्रात नवीन", "per_minute": "प्रति मिनिट",
|
||||
"all_domains": "सर्व डोमेन", "download_phantom": "फँटम साइट डाउनलोड करा",
|
||||
"phantom_subtitle": "संग्रहित पृष्ठे स्थानिक मीडियासह स्थिर साइट म्हणून निर्यात करा", "max_pages": "जास्तीत जास्त पृष्ठे", "select_domain": "डोमेन निवडा", "sources": "स्रोत",
|
||||
"source_uri": "स्रोत URI", "neopig_uri": "Neopig URI", "source_page": "स्रोत पृष्ठ", "neopig_page": "Neopig पृष्ठ",
|
||||
"type_label": "प्रकार", "mime_label": "MIME", "size_label": "आकार", "alt_label": "Alt",
|
||||
"keywords_label": "कीवर्ड", "description_label": "वर्णन", "items": "आयटम", "bytes": "बाइट्स",
|
||||
"used_on": "वापरलेले", "page": "पृष्ठ", "discovered": "शोधले",
|
||||
"console": "कन्सोल", "close": "बंद करा", "no_logs": "अद्याप कोणतेही लॉग नाहीत",
|
||||
},
|
||||
"te": {
|
||||
"search": "శోధన", "crawl": "క్రాల్", "live": "లైవ్ ఫీడ్", "about": "గురించి",
|
||||
"random": "యాదృచ్ఛిక", "phantom": "ఫాంటమ్", "loading": "లోడ్ అవుతోంది...",
|
||||
"start_crawl": "క్రాల్ ప్రారంభించు", "pause": "పాజ్", "resume": "పునఃప్రారంభించు", "delete": "తొలగించు",
|
||||
"url": "URL", "mode": "మోడ్", "depth": "లోతు", "keywords": "కీవర్డ్లు",
|
||||
"screenshots": "స్క్రీన్షాట్లు", "fast_mode": "వేగవంతమైన మోడ్", "fresh_start": "కొత్త ప్రారంభం",
|
||||
"pages": "పేజీలు", "found": "కనుగొనబడింది", "saved": "సేవ్ చేయబడింది", "dupes": "నకిలీలు",
|
||||
"running": "నడుస్తోంది", "completed": "పూర్తయింది", "paused": "పాజ్ చేయబడింది",
|
||||
"no_jobs": "క్రాల్ జాబ్లు లేవు", "started": "ప్రారంభమైంది", "filter": "ఫిల్టర్",
|
||||
"all_types": "అన్ని రకాలు", "images": "చిత్రాలు", "videos": "వీడియోలు", "audio": "ఆడియో",
|
||||
"everything": "అన్నీ", "all_media": "అన్ని మీడియా",
|
||||
"images_only": "చిత్రాలు మాత్రమే", "videos_only": "వీడియోలు మాత్రమే", "text_only": "వచనం మాత్రమే",
|
||||
"recent_jobs": "ఇటీవలి జాబ్లు", "search_placeholder": "మీడియా శోధించండి...",
|
||||
"media": "మీడియా", "pages_label": "పేజీలు", "download": "డౌన్లోడ్",
|
||||
"no_media": "మీడియా కనుగొనబడలేదు", "no_more": "మరిన్ని ఫలితాలు లేవు", "load_error": "లోడ్ విఫలమైంది",
|
||||
"copy": "కాపీ", "copied": "కాపీ అయింది!", "starting": "ప్రారంభమవుతోంది...", "started_ok": "ప్రారంభమైంది!",
|
||||
"error": "లోపం", "watching": "కొత్త చిత్రాల కోసం చూస్తోంది...", "content": "కంటెంట్",
|
||||
"previous": "మునుపటి", "next": "తదుపరి", "over_9000": "9,000+ కంటే ఎక్కువ!",
|
||||
"loading_stats": "గణాంకాలు లోడ్ అవుతున్నాయి...", "screenshot": "స్క్రీన్షాట్",
|
||||
"valid_uri": "దయచేసి కనీసం ఒక చెల్లుబాటు అయ్యే URI నమోదు చేయండి", "load_jobs_error": "జాబ్లు లోడ్ విఫలమైంది",
|
||||
"hydrate_subtitle": "వెబ్ నుండి మీడియా పొందండి", "live_subtitle": "క్రాల్ అవుతున్న చిత్రాలను చూడండి",
|
||||
"total_images": "మొత్తం చిత్రాలు", "new_session": "ఈ సెషన్లో కొత్తవి", "per_minute": "నిమిషానికి",
|
||||
"all_domains": "అన్ని డొమైన్లు", "download_phantom": "ఫాంటమ్ సైట్ డౌన్లోడ్",
|
||||
"phantom_subtitle": "ఆర్కైవ్ చేసిన పేజీలను స్థానిక మీడియాతో స్టాటిక్ సైట్గా ఎగుమతి చేయండి", "max_pages": "గరిష్ట పేజీలు", "select_domain": "డొమైన్ ఎంచుకోండి", "sources": "మూలాలు",
|
||||
"source_uri": "మూల URI", "neopig_uri": "Neopig URI", "source_page": "మూల పేజీ", "neopig_page": "Neopig పేజీ",
|
||||
"type_label": "రకం", "mime_label": "MIME", "size_label": "పరిమాణం", "alt_label": "Alt",
|
||||
"keywords_label": "కీవర్డ్లు", "description_label": "వివరణ", "items": "అంశాలు", "bytes": "బైట్లు",
|
||||
"used_on": "ఉపయోగించబడింది", "page": "పేజీ", "discovered": "కనుగొనబడింది",
|
||||
"console": "కన్సోల్", "close": "మూసివేయి", "no_logs": "ఇంకా లాగ్లు లేవు",
|
||||
},
|
||||
}
|
||||
|
||||
def get_lang(lang_cookie: str = None, accept_language: str = None) -> str:
|
||||
"""Get language from cookie first, then Accept-Language header."""
|
||||
# Cookie takes priority (user's explicit choice)
|
||||
if lang_cookie and lang_cookie in TRANSLATIONS:
|
||||
return lang_cookie
|
||||
# Fall back to Accept-Language header
|
||||
if not accept_language:
|
||||
return "en"
|
||||
# Parse "en-US,en;q=0.9,zh-CN;q=0.8" format
|
||||
for part in accept_language.split(','):
|
||||
lang = part.split(';')[0].strip().split('-')[0].lower()
|
||||
if lang in TRANSLATIONS:
|
||||
return lang
|
||||
return "en"
|
||||
|
||||
# Language names for the selector dropdown
|
||||
LANG_NAMES = {
|
||||
"en": "English", "zh": "中文", "zh-tw": "繁體中文", "es": "Español", "hi": "हिन्दी", "ar": "العربية",
|
||||
"pt": "Português", "ru": "Русский", "ja": "日本語", "fr": "Français", "de": "Deutsch",
|
||||
"ko": "한국어", "it": "Italiano", "nl": "Nederlands", "pl": "Polski", "tr": "Türkçe",
|
||||
"vi": "Tiếng Việt", "th": "ไทย", "id": "Bahasa", "uk": "Українська", "sv": "Svenska",
|
||||
"bn": "বাংলা", "ur": "اردو", "sw": "Kiswahili", "mr": "मराठी", "te": "తెలుగు",
|
||||
}
|
||||
|
||||
def t(key: str, lang: str = "en") -> str:
|
||||
"""Get translation for key in language."""
|
||||
return TRANSLATIONS.get(lang, TRANSLATIONS["en"]).get(key, TRANSLATIONS["en"].get(key, key))
|
||||
|
||||
# Single source of truth for navigation - uses {{key}} placeholders
|
||||
NAV_HTML = '''<div class="nav">
|
||||
<a href="/" class="brand">🐷 neopig</a>
|
||||
<a href="/">{{search}}</a>
|
||||
<a href="/live">{{live}}</a>
|
||||
<a href="/random">{{random}}</a>
|
||||
<a href="/crawl">{{crawl}}</a>
|
||||
<a href="/about">{{about}}</a>
|
||||
</div>'''
|
||||
|
||||
# Single source of truth for search box
|
||||
SEARCH_BOX_HTML = '''<form class="search-box" action="/" method="get">
|
||||
<input type="text" name="q" placeholder="{{search_placeholder}}">
|
||||
<select name="type">
|
||||
<option value="">{{all_types}}</option>
|
||||
<option value="image">{{images}}</option>
|
||||
<option value="video">{{videos}}</option>
|
||||
<option value="audio">{{audio}}</option>
|
||||
</select>
|
||||
<button type="submit">{{search}}</button>
|
||||
</form>'''
|
||||
|
||||
def inject_i18n(html: str, lang: str) -> str:
|
||||
"""Replace {key} placeholders and inject JS translations + language selector."""
|
||||
trans = TRANSLATIONS.get(lang, TRANSLATIONS["en"])
|
||||
# Add lang attribute and inject JS translations
|
||||
html = html.replace('<html>', f'<html lang="{lang}">')
|
||||
# Replace nav placeholder with actual nav (add import link if enabled)
|
||||
nav = NAV_HTML
|
||||
if IMPORT_MODE:
|
||||
nav = nav.replace('<a href="/about">', '<a href="/import">Import</a>\n <a href="/about">')
|
||||
html = html.replace('<!-- NAV -->', nav)
|
||||
# Replace search placeholder with search box
|
||||
html = html.replace('<!-- SEARCH -->', SEARCH_BOX_HTML)
|
||||
# Build language selector options
|
||||
lang_options = ''.join(f'<option value="{code}"{" selected" if code == lang else ""}>{name}</option>'
|
||||
for code, name in LANG_NAMES.items())
|
||||
lang_selector = f'''<select id="lang-select" onchange="setLang(this.value)" style="background:#1a1a1a;color:#888;border:1px solid #333;border-radius:4px;padding:4px 8px;font-size:12px;cursor:pointer;">{lang_options}</select>'''
|
||||
lang_js = '''
|
||||
function setLang(code) {
|
||||
localStorage.setItem('neopig_lang', code);
|
||||
document.cookie = 'lang=' + code + ';path=/;max-age=31536000';
|
||||
location.reload();
|
||||
}
|
||||
'''
|
||||
html = html.replace('<script>', f'<script>\nconst T={json.dumps(trans)};\n{lang_js}', 1)
|
||||
# Insert language selector after {{about}} link in nav
|
||||
html = html.replace('{{about}}</a>\n</div>', '{{about}}</a>\n ' + lang_selector + '\n</div>')
|
||||
# Replace all {key} and {{key}} placeholders
|
||||
for key, value in trans.items():
|
||||
html = html.replace('{{' + key + '}}', value) # Double braces (static templates)
|
||||
html = html.replace('{' + key + '}', value) # Single braces (after f-string)
|
||||
return html
|
||||
|
||||
app = FastAPI(title="neopig", description="Media crawler + SERP + Screenshot service")
|
||||
|
||||
|
|
|
|||
|
|
@ -401,9 +401,9 @@
|
|||
<section class="hero-section">
|
||||
<img src="/static/images/pig1-color1.png" alt="pig.py" class="hero-pig">
|
||||
<h1 class="hero-title">neopig</h1>
|
||||
<p class="hero-subtitle">Neo Python Image Grabber</p>
|
||||
<p class="hero-subtitle">{{ t.about_subtitle }}</p>
|
||||
<p class="hero-tagline">
|
||||
A phoenix-like chimera grown from the bones of dead repositories & forums. A pig that devours web domains whole. Where others archive pages, neopig <em>consumes all digital files</em>.<br><br>Welcome to An Age of Aquarius, no deceitful secrets.
|
||||
{{ t.about_hero_tagline }}<br><br>{{ t.about_hero_tagline2 }}
|
||||
</p>
|
||||
</section>
|
||||
|
||||
|
|
@ -411,32 +411,22 @@
|
|||
<section class="chapter">
|
||||
<div class="chapter-number">I</div>
|
||||
<div class="chapter-content">
|
||||
<h2 class="chapter-title">Genesis</h2>
|
||||
<h2 class="chapter-title">{{ t.about_ch1_title }}</h2>
|
||||
|
||||
<div class="dramatic-quote">
|
||||
<p>pig.py is a <em>very</em> simple python command line tool to download all the images from a given uri.</p>
|
||||
<cite>— Russell Ballestrini, August 22, 2011</cite>
|
||||
<p>{{ t.about_ch1_quote }}</p>
|
||||
<cite>— {{ t.about_ch1_cite }}</cite>
|
||||
</div>
|
||||
|
||||
<p class="narrative">
|
||||
In the beginning, there was <a href="https://russell.ballestrini.net/python-image-grabber-pig-py/" target="_blank">pig.py</a>. A simple creature. Innocent. <em>Hungry.</em>
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
Russell Ballestrini crafted it in the summer of 2011 — a humble Python script with a singular appetite: <strong>images</strong>. Point it at a webpage, and it would slurp down every pixel it could find. No configuration. No complexity. Just a hungry little pig gobbling up the visual fabric of the web.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch1_p1 }}</p>
|
||||
<p class="narrative">{{ t.about_ch1_p2 }}</p>
|
||||
|
||||
<div class="code-section">
|
||||
<pre><code>python pig.py https://www.foxhop.net</code></pre>
|
||||
</div>
|
||||
|
||||
<p class="narrative">
|
||||
That was it. Entire interface. Pig asked only for a target, and it <em>fed</em>.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
Russell released it into the public domain — a gift to anyone who needed to harvest images from the wild web. Source lived at <code>bitbucket.org/russellballestrini/pig</code>, nestled safely in a Mercurial repository. Pig slept soundly in its pen, unaware of the extinction event approaching.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch1_p3 }}</p>
|
||||
<p class="narrative">{{ t.about_ch1_p4 }}</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
@ -444,37 +434,22 @@
|
|||
<section class="chapter chapter-dark">
|
||||
<div class="chapter-number">II</div>
|
||||
<div class="chapter-content">
|
||||
<h2 class="chapter-title">A Great Bitbucket Extinction</h2>
|
||||
<h2 class="chapter-title">{{ t.about_ch2_title }}</h2>
|
||||
|
||||
<p class="narrative">
|
||||
<strong>Then came the dark times.</strong>
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
In 2020, Atlassian — the corporate leviathan that had swallowed Bitbucket whole — decreed that Mercurial repositories would be <em>purged</em>. All of them. Every hg repo on Bitbucket Cloud, atomized. Scattered like digital ash across the void. Atlassian never disclosed how many.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch2_p1 }}</p>
|
||||
<p class="narrative">{{ t.about_ch2_p2 }}</p>
|
||||
<a href="https://media.unturf.com/c/b86529cc-e673-11ef-b77a-115dc81aa9c4/langolier-time-voidlings-of-null-void" target="_blank"><img src="/static/images/bitbucket.jpg" alt="Bitbucket extinction" style="width:42%"></a>
|
||||
|
||||
<p class="narrative">
|
||||
Original pig.py, that innocent image-gobbling creature, was among the casualties. Its home at <a href="https://bitbucket.org/russellballestrini/pig" class="dead-link">bitbucket.org/russellballestrini/pig</a> became a tombstone. Click it. We dare you. There's nothing there but the echo of corporate indifference.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch2_p3 }}</p>
|
||||
|
||||
<div class="dramatic-quote">
|
||||
<p>They literally killed pig.py & sent him to bitbucket.</p>
|
||||
<cite>— Cruel irony of naming your graveyard after a slang term for death</cite>
|
||||
<p>{{ t.about_ch2_quote }}</p>
|
||||
<cite>— {{ t.about_ch2_cite }}</cite>
|
||||
</div>
|
||||
|
||||
<p class="narrative">
|
||||
<em>"Sent to the bitbucket"</em> — programmer slang from the age of punch cards. When you discarded bad data, you threw it in the bit bucket. Atlassian named their service after a trash can, then proved the prophecy true by throwing away everyone's code.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
Pig was dead. Its bones scattered across cached search results & archived blog posts. A ghost in the machine, referenced but unreachable.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
But here's where it gets <em>weird</em>.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch2_p4 }}</p>
|
||||
<p class="narrative">{{ t.about_ch2_p5 }}</p>
|
||||
<p class="narrative">{{ t.about_ch2_p6 }}</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
@ -482,41 +457,23 @@
|
|||
<section class="chapter chapter-rebirth">
|
||||
<div class="chapter-number">III</div>
|
||||
<div class="chapter-content">
|
||||
<h2 class="chapter-title">An Alchemy</h2>
|
||||
<h2 class="chapter-title">{{ t.about_ch3_title }}</h2>
|
||||
|
||||
<p class="narrative">
|
||||
Russell once wrote that <a href="https://russell.ballestrini.net/programming-is-like-alchemy/" target="_blank">"programming is like alchemy — instead of exchanging matter, we programmers exchange time."</a>
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch3_p1 }}</p>
|
||||
<img src="/static/images/alchemy.jpg" alt="Alchemy" style="width:42%">
|
||||
|
||||
<p class="narrative">
|
||||
Programs are <strong>golems</strong>. Familiar spirits. Magical servants performing repetitive tasks so we don't have to. <em>"It is more accurate to group programs with technology than magic, but less fun."</em>
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
Years passed. Web continued its relentless churn — sites going dark, forums shutting down, communities scattering like startled birds. Somewhere, a very angry ex-Ruby developer (Python too) watched the digital decay & remembered the pig.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch3_p2 }}</p>
|
||||
<p class="narrative">{{ t.about_ch3_p3 }}</p>
|
||||
|
||||
<div class="dramatic-quote">
|
||||
<p>What if we could transmute that dead code into gold?</p>
|
||||
<cite>— Thought that started everything</cite>
|
||||
<p>{{ t.about_ch3_quote }}</p>
|
||||
<cite>— {{ t.about_ch3_cite }}</cite>
|
||||
</div>
|
||||
|
||||
<p class="narrative">
|
||||
And speaking of transmutation: <a href="https://phys.org/news/2025-07-marathon-fusion-mercury-gold-energy.html.j2" target="_blank">Marathon Fusion</a> discovered that tokamak breeding blankets — wrapped in Mercury-Lithium alloy, like <em>pigs in a blanket</em> — can transmute Mercury-198 into Gold-197 through chrysopoeia. Fast neutrons trigger (n, 2n) reactions; unstable mercury decays into stable gold within 64 hours.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
Two metric tons of gold per gigawatt. Alchemists' dream realized, wrapped in radioactive patience (17.7 years of cooling before you can touch your transmuted treasure).
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
A golden goose born from the ashes of deprecated version control. Old pig was slaughtered, while <strong>a new creature stirred in the digital depths...</strong>
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
Machine learning uses an equivalent amount of <a href="https://en.wikipedia.org/wiki/Wu_wei" target="_blank">wu wei</a> energy that elites used to kill pig.py to unfold neopig.py.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch3_p4 }}</p>
|
||||
<p class="narrative">{{ t.about_ch3_p5 }}</p>
|
||||
<p class="narrative">{{ t.about_ch3_p6 }}</p>
|
||||
<p class="narrative">{{ t.about_ch3_p7 }}</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
@ -524,32 +481,20 @@
|
|||
<section class="chapter chapter-beast">
|
||||
<div class="chapter-number">IV</div>
|
||||
<div class="chapter-content">
|
||||
<h2 class="chapter-title">A Pig Awakens</h2>
|
||||
<h2 class="chapter-title">{{ t.about_ch4_title }}</h2>
|
||||
|
||||
<p class="narrative">
|
||||
Like a phoenix rising from dead Bitbucket repos, <strong>neopig</strong> emerged.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch4_p1 }}</p>
|
||||
<a href="https://media.unturf.com/c/fbc48bff-e895-11f0-8d29-02dfe05770ee/gluttony" target="_blank"><img src="/static/images/hungry-pig.jpg" alt="Neopig grows hungrier" style="width:42%"></a>
|
||||
|
||||
<p class="narrative">
|
||||
Not a resurrection. Not a mere fork. Something <em>else</em>. A chimera. A griffin. A creature stitched together from the DNA of the original pig & the fevered dreams of someone who had watched too many sites die.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch4_p2 }}</p>
|
||||
|
||||
<div class="dramatic-quote">
|
||||
<p>A pig with an appetite like Gluttony from Fullmetal Alchemist — a homunculus that devours everything in its path, absorbing knowledge, power & form.</p>
|
||||
<p>{{ t.about_ch4_quote }}</p>
|
||||
</div>
|
||||
|
||||
<p class="narrative">
|
||||
Where pig.py sipped politely from single pages, neopig <strong>devours entire domains</strong>. It doesn't just grab images — it consumes HTML, screenshots pages, converts content to markdown, indexes every scrap of text, and stores it all in content-addressed vaults that will outlast the original servers.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
Code lives now at <a href="https://git.unturf.com/engineering/unturf/pig.py" target="_blank">git.unturf.com/engineering/unturf/pig.py</a>. And it keeps evolving. Every crawl adds new capabilities. Every archived site teaches it new tricks.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
<em>We are adding the ability to backup entire codebases.</em> Neopig grows hungrier.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch4_p3 }}</p>
|
||||
<p class="narrative">{{ t.about_ch4_p4 }}</p>
|
||||
<p class="narrative">{{ t.about_ch4_p5 }}</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
@ -557,32 +502,20 @@
|
|||
<section class="chapter chapter-mirror">
|
||||
<div class="chapter-number">V</div>
|
||||
<div class="chapter-content">
|
||||
<h2 class="chapter-title">A Mirror Dimension</h2>
|
||||
<h2 class="chapter-title">{{ t.about_ch5_title }}</h2>
|
||||
|
||||
<p class="narrative">
|
||||
We are about to <strong>invert the git tree into itself</strong>.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch5_p1 }}</p>
|
||||
<a href="https://media.unturf.com/c/fcddf99d-a2da-11f0-af1d-02dfe05770ee/the-tempest-overtakes" target="_blank"><img src="/static/images/sleeping-pig.gif" alt="A sleeping pig" style="width:42%"></a>
|
||||
|
||||
<p class="narrative">
|
||||
Like Dr. Strange versus Spider-Man in the mirror dimension — that impossible space where geometry folds back on itself, where buildings become M.C. Escher nightmares & reality reflects endlessly into its own depths.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
neopig doesn't just archive websites. It can archive <em>itself</em>. It can archive the repositories that contain the code that does the archiving. It can crawl git forges & preserve the source code of tools that preserve source code.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch5_p2 }}</p>
|
||||
<p class="narrative">{{ t.about_ch5_p3 }}</p>
|
||||
|
||||
<div class="dramatic-quote">
|
||||
<p>Our spider-pig gets out of the danger of the mirror void — where days are years — and exits absorbing everything like a griffin, like a chimera, like a pig that feeds on bones of dead platforms.</p>
|
||||
<p>{{ t.about_ch5_quote }}</p>
|
||||
</div>
|
||||
|
||||
<p class="narrative">
|
||||
When a code hosting platform dies (and they all die eventually — Google Code, Gitorious, BitBucket's hg repos), neopig ensures the knowledge survives. We are the ark. We are the vault. We are the pig that remembers.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
Archives become self-extracting executables. <code>.run</code> files that contain their own server, their own search engine, their own reality. Drop one on a machine with Python, and it <em>wakes up</em>. A sleeping pig, carrying an entire website in its belly, ready to serve it to anyone who asks.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch5_p4 }}</p>
|
||||
<p class="narrative">{{ t.about_ch5_p5 }}</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
@ -590,56 +523,56 @@
|
|||
<section class="feature-section">
|
||||
<div class="chapter-number">VI</div>
|
||||
<div class="chapter-content" style="max-width: 100%;">
|
||||
<h2 class="chapter-title" style="text-align: center; margin-bottom: 60px;">The Arsenal</h2>
|
||||
<h2 class="chapter-title" style="text-align: center; margin-bottom: 60px;">{{ t.about_ch6_title }}</h2>
|
||||
|
||||
<div class="feature-grid">
|
||||
<div class="feature-card">
|
||||
<h3>Async Domain Crawling</h3>
|
||||
<p>Full-domain recursive crawling with configurable depth. Respects robots.txt & crawl delays. Polite but <em>relentless</em>.</p>
|
||||
<h3>{{ t.about_feat_async }}</h3>
|
||||
<p>{{ t.about_feat_async_desc }}</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<h3>Content-Addressed Vault</h3>
|
||||
<p>MD5-based deduplication in a 9-deep hex directory structure. Same image from 100 pages? Stored once. Space is memory.</p>
|
||||
<h3>{{ t.about_feat_vault }}</h3>
|
||||
<p>{{ t.about_feat_vault_desc }}</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<h3>Full-Text Search</h3>
|
||||
<p>Every image indexed with its surrounding context — page title, alt text, captions, nearby headings. Find images by <em>meaning</em>, not filename.</p>
|
||||
<h3>{{ t.about_feat_fts }}</h3>
|
||||
<p>{{ t.about_feat_fts_desc }}</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<h3>Page Screenshots</h3>
|
||||
<p><a href="https://linkpeek.com">LinkPeek</a> reborn! Full-page captures via <a href="https://github.com/russellballestrini/uri2png">uri2png</a>. wkhtmltoimage, cutycapt, or Playwright backends.</p>
|
||||
<h3>{{ t.about_feat_screenshots }}</h3>
|
||||
<p>{{ t.about_feat_screenshots_desc }}</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<h3>Hydra Mode</h3>
|
||||
<p>Auto-discover RSS/Atom/Sitemap feeds. Persists feed URLs, checks for new content on every crawl. Self-healing archive maintenance.</p>
|
||||
<h3>{{ t.about_feat_hydra }}</h3>
|
||||
<p>{{ t.about_feat_hydra_desc }}</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<h3>Media Provenance</h3>
|
||||
<p>Track first_seen, last_seen, and when each page first linked to media. Watch content spread across the web over time.</p>
|
||||
<h3>{{ t.about_feat_provenance }}</h3>
|
||||
<p>{{ t.about_feat_provenance_desc }}</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<h3>Markdown Conversion</h3>
|
||||
<p>Intelligent HTML-to-Markdown with forum post detection, noise removal & content extraction. Human-readable archives.</p>
|
||||
<h3>{{ t.about_feat_markdown }}</h3>
|
||||
<p>{{ t.about_feat_markdown_desc }}</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<h3>Self-Extracting Archives</h3>
|
||||
<p>Create <code>.run</code> executables that contain entire websites. Drop on any machine, execute, browse. Pig travels light.</p>
|
||||
<h3>{{ t.about_feat_selfextract }}</h3>
|
||||
<p>{{ t.about_feat_selfextract_desc }}</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<h3>Code File Support</h3>
|
||||
<p>Archive source code, stylesheets, fonts, and scripts alongside media. Complete preservation for developer communities.</p>
|
||||
<h3>{{ t.about_feat_code }}</h3>
|
||||
<p>{{ t.about_feat_code_desc }}</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<h3>VCS Repository Cloning</h3>
|
||||
<p>Auto-detect GitHub, GitLab, Bitbucket & more. Clone repos, symlink files to vault. Deduplicated code across forks.</p>
|
||||
<h3>{{ t.about_feat_vcs }}</h3>
|
||||
<p>{{ t.about_feat_vcs_desc }}</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<h3>Live Feed</h3>
|
||||
<p>Watch media appear in real-time as crawls progress. SSE-powered live updates. Pig hunts while you watch.</p>
|
||||
<h3>{{ t.about_feat_live }}</h3>
|
||||
<p>{{ t.about_feat_live_desc }}</p>
|
||||
</div>
|
||||
<div class="feature-card">
|
||||
<h3>SERP Web Interface</h3>
|
||||
<p>FastAPI-powered search engine. Browse by type, search by context, view media details. Your personal media search engine.</p>
|
||||
<h3>{{ t.about_feat_serp }}</h3>
|
||||
<p>{{ t.about_feat_serp_desc }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -649,27 +582,14 @@
|
|||
<section class="chapter">
|
||||
<div class="chapter-number">VII</div>
|
||||
<div class="chapter-content">
|
||||
<h2 class="chapter-title">A Skeleton Key 🗝️🔑</h2>
|
||||
<h2 class="chapter-title">{{ t.about_ch7_title }}</h2>
|
||||
|
||||
<p class="narrative">
|
||||
neopig uses what we call the <strong>"skeleton key"</strong> approach to media indexing. For every image, we capture <em>all</em> the text that might help you find it later:
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch7_p1 }}</p>
|
||||
<a href="https://media.unturf.com/c/b45632be-9120-11ec-9431-eb3419618d4a/grow-food-not-lawn" target="_blank"><img src="/static/images/grow-food-not-lawn.jpg" alt="Grow food not lawn" style="width:42%"></a>
|
||||
|
||||
<p class="narrative">
|
||||
<strong>Page context:</strong> Title, description, headings near the image.<br>
|
||||
<strong>Image attributes:</strong> Alt text, title, surrounding captions.<br>
|
||||
<strong>Link context:</strong> Text of links pointing to the image.<br>
|
||||
<strong>Detail pages:</strong> For gallery sites, we follow through to detail pages & harvest their metadata too.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
Result? You can search for <em>"sunset over mountains"</em> and find that image even if it was named <code>IMG_4372.jpg</code> with no alt text — because the page title mentioned it, or someone linked to it with descriptive text, or a caption three divs away contained the words.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
Every door opens. Every search succeeds. Skeleton key fits every lock because it carries <em>every possible key</em> inside it.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch7_p2 }}</p>
|
||||
<p class="narrative">{{ t.about_ch7_p3 }}</p>
|
||||
<p class="narrative">{{ t.about_ch7_p4 }}</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
@ -677,28 +597,12 @@
|
|||
<section class="chapter chapter-to-cyan">
|
||||
<div class="chapter-number">VIII</div>
|
||||
<div class="chapter-content">
|
||||
<h2 class="chapter-title">Why We Archive</h2>
|
||||
<h2 class="chapter-title">{{ t.about_ch8_title }}</h2>
|
||||
|
||||
<p class="narrative">
|
||||
<strong>Web is ephemeral.</strong> Sites go dark. Forums shut down. Communities scatter. Platforms get acquired & gutted. Executives decide that Mercurial isn't profitable & delete every repository with a single corporate memo.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
We have watched beloved communities vanish. Game modding forums. Technical documentation. Art galleries. Fan wikis. Personal blogs with decades of writing. Gone. Not archived by the Wayback Machine. Not cached by Google. Just <em>gone</em>.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
neopig is built for <strong>preservation</strong> — capturing not just the media, but the <em>context</em> that gives it meaning. When you archive a site with neopig, you get:
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
• Original HTML with rewritten media links pointing to your local vault<br>
|
||||
• Full-text searchable markdown versions of every page<br>
|
||||
• Screenshots showing exactly how pages looked<br>
|
||||
• A SQLite database you can query, backup, and migrate forever<br>
|
||||
• Self-extracting archives that work offline forever
|
||||
<br><br><br>
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch8_p1 }}</p>
|
||||
<p class="narrative">{{ t.about_ch8_p2 }}</p>
|
||||
<p class="narrative">{{ t.about_ch8_p3 }}</p>
|
||||
<p class="narrative">{{ t.about_ch8_list }}<br><br><br></p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
@ -706,50 +610,25 @@
|
|||
<section class="chapter chapter-forge">
|
||||
<div class="chapter-number">IX</div>
|
||||
<div class="chapter-content">
|
||||
<h2 class="chapter-title">A Forge</h2>
|
||||
<h2 class="chapter-title">{{ t.about_ch9_title }}</h2>
|
||||
|
||||
<p class="narrative">
|
||||
This tool was not built for profit. It was <strong>forged</strong>.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
Forged in the fires of <a href="https://permacomputer.com" target="_blank">permacomputer.com</a> — the belief that computing should be <em>permanent</em>. That knowledge should not evaporate when a company pivots. That communities should not lose their histories when a platform gets acquired. That the lore doesn't have to die when you get acqui-hired or outright <span style="color: #8B00FF;">terminated</span> 🟣. A digital commons that belongs to <em>everyone</em>, forever.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch9_p1 }}</p>
|
||||
<p class="narrative">{{ t.about_ch9_p2 }}</p>
|
||||
|
||||
<a href="https://templeos.org" target="_blank"><img src="/static/images/templeos.png" alt="TempleOS" class="forge-logo"></a>
|
||||
|
||||
<div class="dramatic-quote">
|
||||
<p>Truth. Harmony. Freedom. Love.</p>
|
||||
<cite>— The quadrivium of a permacomputer</cite>
|
||||
<p>{{ t.about_ch9_quote }}</p>
|
||||
<cite>— {{ t.about_ch9_cite }}</cite>
|
||||
</div>
|
||||
|
||||
<p class="narrative">
|
||||
<strong>Truth</strong> — We preserve what actually existed, not sanitized versions. Original HTML. Exact timestamps. Unedited conversations. History as it happened, not as someone wishes it had.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
<strong>Harmony</strong> — Every piece connects. Media links to pages. Pages link to context. Context links to meaning. Archive is not a graveyard of disconnected files but a living web of relationships, searchable, traversable, <em>whole</em>.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
<strong>Freedom</strong> — Public domain. No licenses to parse. No corporate terms of service. Take it. Fork it. Run it. Share it. Code is yours. Archives you create are yours. No one can revoke your access to your own preserved knowledge.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
<strong>Love</strong> — We archive because we <em>care</em>. About the communities that built these spaces. About the people who poured years into forum posts & wiki pages & blog entries. About the future researchers who will want to understand how we lived & thought & created. This is an act of love for human knowledge.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
Pig is imbued with <em>righteous fire</em>. It does not crawl politely because it fears the powerful — it respects robots.txt because it honors the wishes of creators. It does not hoard in darkness — it preserves in light, making archives searchable, shareable, <em>alive</em>. A pig imbued with <em>righteous fire</em>.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
When the platforms fall (and they will fall), when the corporate servers go dark (and they will go dark), when the executives decide your community isn't profitable enough to maintain (and they will decide this) — a permacomputer endures. Archives persist. Knowledge survives.
|
||||
</p>
|
||||
|
||||
<p class="narrative">
|
||||
<strong>This is what we were forged to do.</strong>
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch9_truth }}</p>
|
||||
<p class="narrative">{{ t.about_ch9_harmony }}</p>
|
||||
<p class="narrative">{{ t.about_ch9_freedom }}</p>
|
||||
<p class="narrative">{{ t.about_ch9_love }}</p>
|
||||
<p class="narrative">{{ t.about_ch9_p3 }}</p>
|
||||
<p class="narrative">{{ t.about_ch9_p4 }}</p>
|
||||
<p class="narrative">{{ t.about_ch9_p5 }}</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
@ -757,15 +636,11 @@
|
|||
<section class="chapter chapter-from-cyan">
|
||||
<div class="chapter-number">X</div>
|
||||
<div class="chapter-content">
|
||||
<h2 class="chapter-title">A Way.</h2>
|
||||
<h2 class="chapter-title">{{ t.about_ch10_title }}</h2>
|
||||
|
||||
<p class="narrative">
|
||||
neopig is built on <a href="https://sqlite.org" target="_blank">SQLite</a> — the most deployed database in existence. SQLite adopted the <a href="https://sqlite.org/codeofethics.html.j2" target="_blank">Rule of St. Benedict</a> as their Code of Ethics. 72 principles from a 6th-century monastery, governing software in the 21st.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch10_p1 }}</p>
|
||||
|
||||
<p class="narrative">
|
||||
We inherit this lineage. All 72 rules, unabridged:
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch10_p2 }}</p>
|
||||
|
||||
<div class="rule-grid">
|
||||
<div class="rule-item"><strong>1.</strong> Love the Lord God with your whole heart, soul, and strength.</div>
|
||||
|
|
@ -842,9 +717,7 @@
|
|||
<div class="rule-item"><strong>72.</strong> Never despair of God's mercy.</div>
|
||||
</div>
|
||||
|
||||
<p class="narrative" style="margin-top: 60px;">
|
||||
We archive not to hoard, but to <em>give</em>. We preserve not to control, but to <em>liberate</em>. Database that holds your memories runs on a foundation of ancient wisdom — and so do we.
|
||||
</p>
|
||||
<p class="narrative" style="margin-top: 60px;">{{ t.about_ch10_p3 }}</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
|
@ -852,36 +725,32 @@
|
|||
<section class="chapter chapter-beast">
|
||||
<div class="chapter-number">XI</div>
|
||||
<div class="chapter-content">
|
||||
<h2 class="chapter-title">Invoke the Pig</h2>
|
||||
<h2 class="chapter-title">{{ t.about_ch11_title }}</h2>
|
||||
|
||||
<div class="code-section">
|
||||
<pre><code># open http://127.0.0.1:31337 in browser
|
||||
python neopig.py --serve</code></pre>
|
||||
</div>
|
||||
|
||||
<p class="narrative">
|
||||
<strong>neopig</strong> = <strong>Neo</strong> (new) + <strong>P</strong>ython <strong>I</strong>mage <strong>G</strong>rabber
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch11_p1 }}</p>
|
||||
|
||||
<p class="narrative">
|
||||
A tip of the hat to original pig.py. A nod to Matrix's Neo — seeing through surface of web to underlying content within. And perhaps a warning: this pig has grown teeth.
|
||||
</p>
|
||||
<p class="narrative">{{ t.about_ch11_p2 }}</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- FOOTER -->
|
||||
<footer class="about-footer">
|
||||
<p>neopig is open source, gifted into our public domain.</p>
|
||||
<p>{{ t.about_footer_p1 }}</p>
|
||||
<img src="/static/images/pixel-art-gift.png" alt="Gift" style="width:200px">
|
||||
<p>Neopig lives:</p>
|
||||
<p>{{ t.about_footer_p2 }}</p>
|
||||
<a href="https://git.unturf.com/engineering/unturf/pig.py" target="_blank">git.unturf.com/engineering/unturf/pig.py</a>
|
||||
<p style="margin-top: 40px; color: #444; font-size: 0.9em;">
|
||||
In memory of all the repositories Atlassian killed.<br>
|
||||
Neopig remembers. Neopig archives. Neopig <em>persists</em>.
|
||||
{{ t.about_footer_p3 }}<br>
|
||||
{{ t.about_footer_remember }}
|
||||
</p>
|
||||
<a href="https://media.unturf.com/c/28697a5a-b2ad-11ec-9431-eb3419618d4a/oven-fresh-internet-4-all" target="_blank"><img src="/static/images/oven-fresh-internet.jpg" alt="Oven fresh internet" style="width:21%"></a>
|
||||
<p style="margin-top: 60px; color: #666;">
|
||||
Sorry for the convenience. ✝️
|
||||
{{ t.about_footer_p5 }}
|
||||
</p>
|
||||
</footer>
|
||||
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ button:hover { background: #ff5252; }
|
|||
<a href="/live">{{ t.live }}</a>
|
||||
<a href="/random">{{ t.random }}</a>
|
||||
<a href="/crawl">{{ t.crawl }}</a>
|
||||
{% if import_mode %}<a href="/import">Import</a>{% endif %}
|
||||
{% if import_mode %}<a href="/import">{{ t.import_nav }}</a>{% endif %}
|
||||
<a href="/about">{{ t.about }}</a>
|
||||
<select id="lang-select" onchange="setLang(this.value)" style="background:#1a1a1a;color:#888;border:1px solid #333;border-radius:4px;padding:4px 8px;font-size:12px;cursor:pointer;">
|
||||
{% for code, name in langs.items() %}<option value="{{ code }}"{% if code == lang %} selected{% endif %}>{{ name }}</option>{% endfor %}
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ h2 { color: #ff6b6b; font-size: 18px; margin-bottom: 15px; }
|
|||
</div>
|
||||
<div>
|
||||
<input type="checkbox" id="hydra">
|
||||
<label for="hydra" style="display:inline; margin:0;" title="Parse RSS/Atom/Sitemap for fast discovery">Hydra</label>
|
||||
<label for="hydra" style="display:inline; margin:0;" title="Parse RSS/Atom/Sitemap for fast discovery">{{ t.hydra }}</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -180,7 +180,7 @@ if (CRAWL_DISABLED) {
|
|||
});
|
||||
const banner = document.createElement('div');
|
||||
banner.style.cssText = 'background:#442;border:1px solid #664;padding:12px 20px;border-radius:8px;margin-bottom:20px;color:#ffa;';
|
||||
banner.innerHTML = 'Crawling is disabled on this server (read-only archive mode)';
|
||||
banner.innerHTML = T.crawl_disabled;
|
||||
form.parentNode.insertBefore(banner, form);
|
||||
});
|
||||
}
|
||||
|
|
@ -290,7 +290,7 @@ const activeReplays = new Set();
|
|||
function replayJob(jobId) {
|
||||
// Prevent duplicate replays of same job
|
||||
if (activeReplays.has(jobId)) {
|
||||
alert('Replay for job #' + jobId + ' is already running');
|
||||
alert(T.replay_already_running + ' #' + jobId);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -299,7 +299,7 @@ function replayJob(jobId) {
|
|||
// Update button to show active state
|
||||
const btn = document.querySelector(`button[onclick="replayJob(${jobId})"]`);
|
||||
if (btn) {
|
||||
btn.textContent = '⏳ Replaying...';
|
||||
btn.textContent = T.replaying;
|
||||
btn.disabled = true;
|
||||
}
|
||||
|
||||
|
|
@ -312,7 +312,7 @@ function replayJob(jobId) {
|
|||
clearInterval(checkClosed);
|
||||
activeReplays.delete(jobId);
|
||||
if (btn) {
|
||||
btn.textContent = '▶ Replay';
|
||||
btn.textContent = T.replay;
|
||||
btn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
|
@ -323,7 +323,7 @@ function replayJob(jobId) {
|
|||
clearInterval(checkClosed);
|
||||
activeReplays.delete(jobId);
|
||||
if (btn) {
|
||||
btn.textContent = '▶ Replay';
|
||||
btn.textContent = T.replay;
|
||||
btn.disabled = false;
|
||||
}
|
||||
}, 300000);
|
||||
|
|
@ -331,13 +331,13 @@ function replayJob(jobId) {
|
|||
|
||||
// Recrawl all sources from an import job (async background crawls)
|
||||
async function recrawlSources(jobId, sourceCount) {
|
||||
if (!confirm('Start ' + sourceCount + ' new crawl jobs for the imported sources?\n\nThis will crawl each source asynchronously in the background.')) {
|
||||
if (!confirm(T.confirm_recrawl + '\n\n' + T.recrawl_async_note)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const btn = document.querySelector(`button[onclick="recrawlSources(${jobId}, ${sourceCount})"]`);
|
||||
if (btn) {
|
||||
btn.textContent = '⏳ Starting...';
|
||||
btn.textContent = T.starting;
|
||||
btn.disabled = true;
|
||||
}
|
||||
|
||||
|
|
@ -346,16 +346,16 @@ async function recrawlSources(jobId, sourceCount) {
|
|||
const data = await res.json();
|
||||
|
||||
if (res.ok) {
|
||||
alert('Started ' + data.job_ids.length + ' crawl jobs!\n\nJob IDs: ' + data.job_ids.join(', '));
|
||||
alert(T.started_jobs + '\n\nJob IDs: ' + data.job_ids.join(', '));
|
||||
loadJobs(); // Refresh to show new jobs
|
||||
} else {
|
||||
alert(T.error + ': ' + (data.detail || 'Failed to start recrawl'));
|
||||
alert(T.error + ': ' + (data.detail || T.load_error));
|
||||
}
|
||||
} catch (err) {
|
||||
alert(T.error + ': ' + err.message);
|
||||
} finally {
|
||||
if (btn) {
|
||||
btn.textContent = '🔄 Recrawl ' + sourceCount + ' sources';
|
||||
btn.textContent = T.recrawl_sources + ' (' + sourceCount + ')';
|
||||
btn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
|
@ -456,8 +456,8 @@ function renderJob(job) {
|
|||
${job.status === 'running' && job.job_kind === 'crawl' ? '<button class="pause-btn" onclick="pauseJob(' + job.id + ')">' + T.pause + '</button>' : ''}
|
||||
${(job.status === 'paused' || job.status === 'cancelled') && job.job_kind === 'crawl' ? '<button class="start-btn" onclick="resumeJob(' + job.id + ')">' + T.resume + '</button>' : ''}
|
||||
${(job.status === 'completed' || job.status === 'failed' || job.status === 'cancelled') && job.job_kind === 'crawl' && job.mode !== 'import' ? "<button class=\"recrawl-btn\" onclick='loadJobSettings(" + JSON.stringify(job) + ")'>↻</button>" : ''}
|
||||
${job.status === 'completed' && job.job_kind === 'crawl' ? '<button class="replay-btn" onclick="replayJob(' + job.id + ')">▶ Replay</button>' : ''}
|
||||
${job.status === 'completed' && job.mode === 'import' && stats.imported_sources?.length ? '<button class="recrawl-btn" onclick="recrawlSources(' + job.id + ', ' + stats.imported_sources.length + ')">🔄 Recrawl ' + stats.imported_sources.length + ' sources</button>' : ''}
|
||||
${job.status === 'completed' && job.job_kind === 'crawl' ? '<button class="replay-btn" onclick="replayJob(' + job.id + ')">' + T.replay + '</button>' : ''}
|
||||
${job.status === 'completed' && job.mode === 'import' && stats.imported_sources?.length ? '<button class="recrawl-btn" onclick="recrawlSources(' + job.id + ', ' + stats.imported_sources.length + ')">' + T.recrawl_sources + ' (' + stats.imported_sources.length + ')</button>' : ''}
|
||||
<button class="console-btn" onclick="openConsole(${job.id})">${T.console}</button>
|
||||
${job.status !== 'running' ? '<button class="delete-btn" onclick="deleteJob(' + job.id + ')">' + T.delete + '</button>' : ''}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -52,24 +52,24 @@ h1 { color: #ff6b6b; margin-bottom: 5px; }
|
|||
|
||||
{% block content %}
|
||||
<div class="container">
|
||||
<h1>Import Mode</h1>
|
||||
<p class="subtitle">Upload a neopig archive (.tar.gz or .run) to explore it</p>
|
||||
<h1>{{ t.import_mode }}</h1>
|
||||
<p class="subtitle">{{ t.import_subtitle }}</p>
|
||||
|
||||
<div class="upload-zone" id="upload-zone" onclick="document.getElementById('file-input').click()">
|
||||
<div class="upload-icon">📦</div>
|
||||
<div class="upload-text">Drop archive here or click to browse</div>
|
||||
<div class="upload-hint">Supports .tar.gz, .tgz, and .run files</div>
|
||||
<div class="upload-text">{{ t.drop_or_browse }}</div>
|
||||
<div class="upload-hint">{{ t.supported_formats }}</div>
|
||||
<input type="file" id="file-input" accept=".tar.gz,.tgz,.run">
|
||||
</div>
|
||||
|
||||
<div class="progress" id="progress">
|
||||
<div class="progress-bar"><div class="progress-fill" id="progress-fill"></div></div>
|
||||
<div class="progress-text" id="progress-text">Uploading...</div>
|
||||
<div class="progress-text" id="progress-text">{{ t.uploading }}</div>
|
||||
</div>
|
||||
|
||||
<div class="status" id="status">
|
||||
<h3 style="margin-top:0;color:#ff6b6b;">Current Archive</h3>
|
||||
<div id="status-content">Loading...</div>
|
||||
<h3 style="margin-top:0;color:#ff6b6b;">{{ t.current_archive }}</h3>
|
||||
<div id="status-content">{{ t.loading }}</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
|
@ -104,7 +104,7 @@ let currentUpload = null;
|
|||
async function uploadFile(file) {
|
||||
progress.style.display = 'block';
|
||||
progressFill.style.width = '0%';
|
||||
progressText.textContent = `Initializing upload for ${file.name}...`;
|
||||
progressText.textContent = `${T.initializing_upload} ${file.name}...`;
|
||||
|
||||
try {
|
||||
const initRes = await fetch(`/api/import/upload/init?filename=${encodeURIComponent(file.name)}&size=${file.size}`, { method: 'POST' });
|
||||
|
|
@ -162,10 +162,10 @@ async function uploadChunks() {
|
|||
} catch (err) {
|
||||
retries++;
|
||||
if (retries > MAX_RETRIES) {
|
||||
progressText.innerHTML = `<span class="error">✗ Upload failed after ${MAX_RETRIES} retries: ${err.message}</span>`;
|
||||
progressText.innerHTML = `<span class="error">✗ ${T.upload_failed_retries}: ${err.message}</span>`;
|
||||
return;
|
||||
}
|
||||
progressText.textContent = `Connection lost, retrying (${retries}/${MAX_RETRIES})...`;
|
||||
progressText.textContent = `${T.connection_lost_retry} (${retries}/${MAX_RETRIES})...`;
|
||||
await new Promise(r => setTimeout(r, RETRY_DELAY));
|
||||
|
||||
try {
|
||||
|
|
@ -182,7 +182,7 @@ async function uploadChunks() {
|
|||
throw new Error(err.detail || 'Failed to complete upload');
|
||||
}
|
||||
const result = await completeRes.json();
|
||||
progressText.innerHTML = `<span class="success">✓ Archive loaded successfully! Job #${result.job_id}</span>`;
|
||||
progressText.innerHTML = `<span class="success">✓ ${T.archive_loaded} #${result.job_id}</span>`;
|
||||
currentUpload = null;
|
||||
loadStatus();
|
||||
} catch (err) {
|
||||
|
|
@ -203,13 +203,13 @@ async function loadStatus() {
|
|||
const content = document.getElementById('status-content');
|
||||
|
||||
content.innerHTML = `
|
||||
<div class="status-row"><span class="status-label">Media in database</span><span class="status-value">${(data.media_count || 0).toLocaleString()}</span></div>
|
||||
<div class="status-row"><span class="status-label">Pages indexed</span><span class="status-value">${(data.pages_count || 0).toLocaleString()}</span></div>
|
||||
<div class="status-row"><span class="status-label">Recent imports</span><span class="status-value">${data.recent_imports || 0}</span></div>
|
||||
<div class="status-row"><a href="/crawl" style="color:#ff6b6b;">View all jobs →</a></div>
|
||||
<div class="status-row"><span class="status-label">${T.media_in_db}</span><span class="status-value">${(data.media_count || 0).toLocaleString()}</span></div>
|
||||
<div class="status-row"><span class="status-label">${T.pages_indexed}</span><span class="status-value">${(data.pages_count || 0).toLocaleString()}</span></div>
|
||||
<div class="status-row"><span class="status-label">${T.recent_imports}</span><span class="status-value">${data.recent_imports || 0}</span></div>
|
||||
<div class="status-row"><a href="/crawl" style="color:#ff6b6b;">${T.view_all_jobs} →</a></div>
|
||||
`;
|
||||
} catch (err) {
|
||||
document.getElementById('status-content').innerHTML = '<div class="error">Failed to load status</div>';
|
||||
document.getElementById('status-content').innerHTML = '<div class="error">' + T.failed_load_status + '</div>';
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -92,10 +92,10 @@ let replayMode = !!REPLAY_JOB_ID;
|
|||
if (replayMode) {
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
const h1 = document.querySelector('h1');
|
||||
h1.textContent = '▶ Replay Job #' + REPLAY_JOB_ID;
|
||||
h1.textContent = T.replay_job + ' #' + REPLAY_JOB_ID;
|
||||
h1.style.color = '#9b59b6';
|
||||
const subtitle = document.querySelector('.subtitle');
|
||||
subtitle.textContent = 'Replaying crawl at original discovery pace';
|
||||
subtitle.textContent = T.replay_subtitle;
|
||||
subtitle.style.color = '#9b59b6';
|
||||
});
|
||||
}
|
||||
|
|
@ -118,7 +118,7 @@ function connectSSE() {
|
|||
// Handle replay end marker
|
||||
if (item.type === 'replay_end') {
|
||||
const status = document.getElementById('status');
|
||||
status.textContent = '✓ Replay complete (' + item.total + ' items)';
|
||||
status.textContent = T.replay_complete + ' (' + item.total + ' items)';
|
||||
status.style.color = '#10b981';
|
||||
eventSource.close();
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -48,13 +48,13 @@ select {
|
|||
</form>
|
||||
|
||||
<div class="info">
|
||||
<h3>What is a Phantom Site?</h3>
|
||||
<h3>{{ t.what_is_phantom }}</h3>
|
||||
<ul>
|
||||
<li>Original HTML preserved exactly as crawled</li>
|
||||
<li>All media URLs rewritten to local paths</li>
|
||||
<li>Ready to host statically (nginx, Caddy, S3, etc.)</li>
|
||||
<li>Works offline - all assets included</li>
|
||||
<li>Perfect for archival & preservation</li>
|
||||
<li>{{ t.phantom_feature_1 }}</li>
|
||||
<li>{{ t.phantom_feature_2 }}</li>
|
||||
<li>{{ t.phantom_feature_3 }}</li>
|
||||
<li>{{ t.phantom_feature_4 }}</li>
|
||||
<li>{{ t.phantom_feature_5 }}</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -181,7 +181,7 @@ h3 { color: #ff6b6b; font-size: 16px; margin: 30px 0 15px 0; border-bottom: 1px
|
|||
</div>
|
||||
{% if screenshot_hashes %}
|
||||
<div class="screenshot-col">
|
||||
<h3 onclick="toggleScreenshots()" style="cursor:pointer;">{{ t.screenshots or 'Screenshots' }} ({{ screenshot_hashes|length }}) <span id="ss-toggle" style="font-size:12px;color:#888;">[hide]</span></h3>
|
||||
<h3 onclick="toggleScreenshots()" style="cursor:pointer;">{{ t.screenshots or 'Screenshots' }} ({{ screenshot_hashes|length }}) <span id="ss-toggle" style="font-size:12px;color:#888;">[{{ t.hide }}]</span></h3>
|
||||
<div class="screenshot-stack" id="screenshot-container">
|
||||
{% for hash in screenshot_hashes %}
|
||||
<a href="/view/{{ hash }}"><img src="/media/{{ hash }}" loading="lazy"></a>
|
||||
|
|
@ -206,7 +206,7 @@ function toggleScreenshots() {
|
|||
const toggle = document.getElementById('ss-toggle');
|
||||
const collapsed = container.classList.toggle('collapsed');
|
||||
grid.classList.toggle('ss-collapsed');
|
||||
toggle.textContent = collapsed ? '[show]' : '[hide]';
|
||||
toggle.textContent = collapsed ? '[' + T.show + ']' : '[' + T.hide + ']';
|
||||
}
|
||||
|
||||
// Forum post restructuring with anchor IDs
|
||||
|
|
|
|||
414
tests/unit/test_archive.py
Normal file
414
tests/unit/test_archive.py
Normal file
|
|
@ -0,0 +1,414 @@
|
|||
"""
|
||||
Tests for archive module.
|
||||
|
||||
Tests site archiver functionality for creating distributable tar.gz packages.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, AsyncMock, patch, MagicMock
|
||||
import tempfile
|
||||
import shutil
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
from archive import (
|
||||
SiteArchiver,
|
||||
sanitize_filename,
|
||||
url_to_path,
|
||||
html_to_markdown,
|
||||
HAS_HTML2TEXT,
|
||||
)
|
||||
|
||||
|
||||
class TestSanitizeFilename:
|
||||
"""Test filename sanitization."""
|
||||
|
||||
def test_simple_name(self):
|
||||
"""Test simple name passes through."""
|
||||
result = sanitize_filename("example")
|
||||
assert result == "example"
|
||||
|
||||
def test_removes_special_chars(self):
|
||||
"""Test special characters are removed."""
|
||||
result = sanitize_filename("test<>:/\\|?*file")
|
||||
# Should not contain any of those characters
|
||||
for char in '<>:"/\\|?*':
|
||||
assert char not in result
|
||||
|
||||
def test_domain_with_dots(self):
|
||||
"""Test domain with dots is sanitized."""
|
||||
result = sanitize_filename("www.example.com")
|
||||
assert "." not in result
|
||||
|
||||
def test_collapses_multiple_dashes(self):
|
||||
"""Test multiple dashes are collapsed."""
|
||||
result = sanitize_filename("test---file")
|
||||
assert "---" not in result
|
||||
|
||||
def test_strips_leading_trailing(self):
|
||||
"""Test leading/trailing dashes are stripped."""
|
||||
result = sanitize_filename("--test--")
|
||||
assert not result.startswith("-")
|
||||
assert not result.endswith("-")
|
||||
|
||||
def test_empty_string(self):
|
||||
"""Test empty string returns 'unnamed'."""
|
||||
result = sanitize_filename("")
|
||||
assert result == "unnamed"
|
||||
|
||||
def test_only_special_chars(self):
|
||||
"""Test string of only special chars returns 'unnamed'."""
|
||||
result = sanitize_filename(":::///")
|
||||
assert result == "unnamed"
|
||||
|
||||
def test_max_length(self):
|
||||
"""Test max length truncation."""
|
||||
long_name = "a" * 300
|
||||
result = sanitize_filename(long_name)
|
||||
assert len(result) <= 200
|
||||
|
||||
|
||||
class TestUrlToPath:
|
||||
"""Test URL to filesystem path conversion."""
|
||||
|
||||
def test_root_url(self):
|
||||
"""Test root URL becomes index.html."""
|
||||
result = url_to_path("https://example.com/")
|
||||
assert result == "index.html"
|
||||
|
||||
def test_root_url_with_path_slash(self):
|
||||
"""Test root URL with explicit path slash."""
|
||||
# Note: url_to_path expects URLs to have at least a "/" path
|
||||
result = url_to_path("https://example.com/about/")
|
||||
assert "about" in result
|
||||
assert result.endswith("index.html")
|
||||
|
||||
def test_html_file(self):
|
||||
"""Test .html file preserves extension."""
|
||||
result = url_to_path("https://example.com/page.html")
|
||||
assert result == "page.html"
|
||||
|
||||
def test_htm_file(self):
|
||||
"""Test .htm file preserves extension."""
|
||||
result = url_to_path("https://example.com/page.htm")
|
||||
assert result == "page.htm"
|
||||
|
||||
def test_directory_path(self):
|
||||
"""Test directory path gets index.html."""
|
||||
result = url_to_path("https://example.com/about")
|
||||
assert result.endswith("index.html")
|
||||
assert "about" in result
|
||||
|
||||
def test_nested_directory(self):
|
||||
"""Test nested directory path."""
|
||||
result = url_to_path("https://example.com/blog/posts")
|
||||
assert "blog" in result
|
||||
assert "posts" in result
|
||||
assert result.endswith("index.html")
|
||||
|
||||
def test_file_with_extension(self):
|
||||
"""Test file with non-html extension."""
|
||||
result = url_to_path("https://example.com/document.pdf")
|
||||
assert result == "document.pdf"
|
||||
|
||||
|
||||
class TestHtmlToMarkdown:
|
||||
"""Test HTML to markdown conversion."""
|
||||
|
||||
@pytest.mark.skipif(not HAS_HTML2TEXT, reason="html2text not installed")
|
||||
def test_simple_html(self):
|
||||
"""Test converting simple HTML."""
|
||||
html = "<h1>Title</h1><p>Paragraph text.</p>"
|
||||
result = html_to_markdown(html)
|
||||
assert "Title" in result
|
||||
assert "Paragraph" in result
|
||||
|
||||
@pytest.mark.skipif(not HAS_HTML2TEXT, reason="html2text not installed")
|
||||
def test_preserves_links(self):
|
||||
"""Test links are preserved."""
|
||||
html = '<a href="https://example.com">Link</a>'
|
||||
result = html_to_markdown(html)
|
||||
assert "https://example.com" in result or "[Link]" in result
|
||||
|
||||
@pytest.mark.skipif(not HAS_HTML2TEXT, reason="html2text not installed")
|
||||
def test_preserves_images(self):
|
||||
"""Test images are preserved."""
|
||||
html = '<img src="image.jpg" alt="Test">'
|
||||
result = html_to_markdown(html)
|
||||
# Should have image reference
|
||||
assert "image.jpg" in result or "Test" in result
|
||||
|
||||
def test_without_html2text(self):
|
||||
"""Test fallback when html2text not available."""
|
||||
# When html2text is not installed, should return original HTML
|
||||
if not HAS_HTML2TEXT:
|
||||
html = "<h1>Test</h1>"
|
||||
result = html_to_markdown(html)
|
||||
assert result == html
|
||||
|
||||
|
||||
class TestSiteArchiverInitialization:
|
||||
"""Test SiteArchiver class initialization."""
|
||||
|
||||
def test_default_initialization(self):
|
||||
"""Test default initialization values."""
|
||||
archiver = SiteArchiver()
|
||||
|
||||
assert archiver.output_dir == Path(".")
|
||||
assert archiver.include_screenshots is True
|
||||
assert archiver.include_markdown == HAS_HTML2TEXT
|
||||
assert archiver.fast_mode is False
|
||||
assert archiver.trim_wrapper is False
|
||||
assert archiver.show_progress is True
|
||||
assert archiver.fresh_start is False
|
||||
|
||||
def test_custom_output_dir(self):
|
||||
"""Test custom output directory."""
|
||||
archiver = SiteArchiver(output_dir="/tmp/archives")
|
||||
assert archiver.output_dir == Path("/tmp/archives")
|
||||
|
||||
def test_disable_screenshots(self):
|
||||
"""Test disabling screenshots."""
|
||||
archiver = SiteArchiver(include_screenshots=False)
|
||||
assert archiver.include_screenshots is False
|
||||
assert archiver.screenshot_config.enabled is False
|
||||
|
||||
def test_disable_markdown(self):
|
||||
"""Test disabling markdown."""
|
||||
archiver = SiteArchiver(include_markdown=False)
|
||||
assert archiver.include_markdown is False
|
||||
|
||||
def test_fast_mode(self):
|
||||
"""Test enabling fast mode."""
|
||||
archiver = SiteArchiver(fast_mode=True)
|
||||
assert archiver.fast_mode is True
|
||||
|
||||
def test_trim_wrapper(self):
|
||||
"""Test trim wrapper option."""
|
||||
archiver = SiteArchiver(trim_wrapper=True)
|
||||
assert archiver.trim_wrapper is True
|
||||
|
||||
def test_fresh_start(self):
|
||||
"""Test fresh start option."""
|
||||
archiver = SiteArchiver(fresh_start=True)
|
||||
assert archiver.fresh_start is True
|
||||
|
||||
|
||||
class TestSiteArchiverExtractTitle:
|
||||
"""Test SiteArchiver._extract_title method."""
|
||||
|
||||
def setup_method(self):
|
||||
self.archiver = SiteArchiver()
|
||||
|
||||
def test_extract_title_from_title_tag(self):
|
||||
"""Test extracting title from <title> tag."""
|
||||
html = "<html><head><title>Page Title</title></head><body></body></html>"
|
||||
result = self.archiver._extract_title(html)
|
||||
assert result == "Page Title"
|
||||
|
||||
def test_extract_title_from_h1(self):
|
||||
"""Test extracting title from <h1> when no title tag."""
|
||||
html = "<html><body><h1>Heading Title</h1></body></html>"
|
||||
result = self.archiver._extract_title(html)
|
||||
assert result == "Heading Title"
|
||||
|
||||
def test_extract_title_empty(self):
|
||||
"""Test extracting title from empty HTML."""
|
||||
html = "<html><body></body></html>"
|
||||
result = self.archiver._extract_title(html)
|
||||
assert result is None
|
||||
|
||||
def test_extract_title_invalid_html(self):
|
||||
"""Test extracting title from invalid HTML."""
|
||||
html = "not html at all"
|
||||
result = self.archiver._extract_title(html)
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestSiteArchiverWriteIndex:
|
||||
"""Test SiteArchiver._write_index_html method."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.archiver = SiteArchiver()
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_write_index_creates_file(self):
|
||||
"""Test that index.html is created."""
|
||||
sitemap = [
|
||||
{"path": "html/page1.html", "title": "Page 1"},
|
||||
{"path": "html/page2.html", "title": "Page 2"},
|
||||
]
|
||||
self.archiver._write_index_html(Path(self.temp_dir), sitemap, "example.com")
|
||||
|
||||
index_path = Path(self.temp_dir) / "index.html"
|
||||
assert index_path.exists()
|
||||
|
||||
def test_write_index_contains_domain(self):
|
||||
"""Test that index contains domain name."""
|
||||
sitemap = []
|
||||
self.archiver._write_index_html(Path(self.temp_dir), sitemap, "example.com")
|
||||
|
||||
index_path = Path(self.temp_dir) / "index.html"
|
||||
content = index_path.read_text()
|
||||
assert "example.com" in content
|
||||
|
||||
def test_write_index_contains_links(self):
|
||||
"""Test that index contains page links."""
|
||||
sitemap = [
|
||||
{"path": "html/page1.html", "title": "Page 1"},
|
||||
]
|
||||
self.archiver._write_index_html(Path(self.temp_dir), sitemap, "example.com")
|
||||
|
||||
index_path = Path(self.temp_dir) / "index.html"
|
||||
content = index_path.read_text()
|
||||
assert "Page 1" in content
|
||||
assert "html/page1.html" in content
|
||||
|
||||
def test_write_index_limits_to_1000(self):
|
||||
"""Test that index limits sitemap to 1000 entries."""
|
||||
sitemap = [{"path": f"html/page{i}.html", "title": f"Page {i}"} for i in range(1500)]
|
||||
self.archiver._write_index_html(Path(self.temp_dir), sitemap, "example.com")
|
||||
|
||||
index_path = Path(self.temp_dir) / "index.html"
|
||||
content = index_path.read_text()
|
||||
# Should mention there are more pages
|
||||
assert "more pages" in content or "500 more" in content or "and" in content
|
||||
|
||||
|
||||
class TestSiteArchiverArchive:
|
||||
"""Test SiteArchiver.archive method."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.archiver = SiteArchiver(output_dir=self.temp_dir)
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_creates_tarball(self):
|
||||
"""Test that archive creates a tar.gz file."""
|
||||
with patch('archive.NeoPig') as MockNeoPig:
|
||||
# Setup mocks
|
||||
mock_pig = AsyncMock()
|
||||
MockNeoPig.return_value = mock_pig
|
||||
mock_pig.init = AsyncMock()
|
||||
mock_pig.crawl = AsyncMock(return_value={})
|
||||
mock_pig.db = AsyncMock()
|
||||
mock_pig.db.get_all_media_uri_mappings = AsyncMock(return_value={})
|
||||
mock_pig.db.get_pages_by_domain = AsyncMock(return_value=[])
|
||||
mock_pig.db.close = AsyncMock()
|
||||
mock_pig.screenshot_config = MagicMock()
|
||||
mock_pig.screenshot_config.enabled = True
|
||||
mock_pig._clear_state = MagicMock()
|
||||
mock_pig.seen_pages = set()
|
||||
mock_pig.seen_media = {}
|
||||
mock_pig.seen_screenshots = set()
|
||||
|
||||
# Run archive
|
||||
try:
|
||||
result = await self.archiver.archive(
|
||||
"https://example.com",
|
||||
depth=1,
|
||||
max_pages=1,
|
||||
)
|
||||
# Check if tarball was created (might not be if mocking isn't complete)
|
||||
# assert result.suffix == ".gz" or str(result).endswith(".tar.gz")
|
||||
except Exception:
|
||||
pass # Full archive requires extensive mocking
|
||||
|
||||
|
||||
class TestHAS_HTML2TEXT:
|
||||
"""Test html2text availability constant."""
|
||||
|
||||
def test_has_html2text_is_bool(self):
|
||||
"""Test HAS_HTML2TEXT is a boolean."""
|
||||
assert isinstance(HAS_HTML2TEXT, bool)
|
||||
|
||||
|
||||
class TestArchiveIntegration:
|
||||
"""Integration tests for archive functionality."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_sanitize_domain_for_filename(self):
|
||||
"""Test domain sanitization produces valid filename."""
|
||||
domains = [
|
||||
"example.com",
|
||||
"sub.example.com",
|
||||
"example.com:8080",
|
||||
"example.co.uk",
|
||||
]
|
||||
for domain in domains:
|
||||
result = sanitize_filename(domain)
|
||||
# Should not contain problematic characters
|
||||
assert "/" not in result
|
||||
assert "\\" not in result
|
||||
assert ":" not in result
|
||||
# Should be valid filename
|
||||
assert len(result) > 0
|
||||
|
||||
def test_url_to_path_for_common_patterns(self):
|
||||
"""Test URL to path for common URL patterns."""
|
||||
test_cases = [
|
||||
("https://example.com/", "index.html"),
|
||||
("https://example.com/about", "about/index.html"),
|
||||
("https://example.com/blog/", "blog/index.html"),
|
||||
("https://example.com/doc.html", "doc.html"),
|
||||
("https://example.com/file.pdf", "file.pdf"),
|
||||
]
|
||||
for url, expected in test_cases:
|
||||
result = url_to_path(url)
|
||||
assert expected in result or result == expected
|
||||
|
||||
|
||||
class TestUpgradeNeopigInArchive:
|
||||
"""Test upgrade_neopig_in_archive function."""
|
||||
|
||||
def test_function_exists(self):
|
||||
"""Test function is importable."""
|
||||
from archive import upgrade_neopig_in_archive
|
||||
assert callable(upgrade_neopig_in_archive)
|
||||
|
||||
|
||||
class TestArchiverOptions:
|
||||
"""Test various archiver configuration options."""
|
||||
|
||||
def test_screenshot_config_created(self):
|
||||
"""Test screenshot config is created."""
|
||||
archiver = SiteArchiver(include_screenshots=True)
|
||||
assert archiver.screenshot_config is not None
|
||||
assert archiver.screenshot_config.enabled is True
|
||||
|
||||
def test_screenshot_config_disabled(self):
|
||||
"""Test screenshot config when disabled."""
|
||||
archiver = SiteArchiver(include_screenshots=False)
|
||||
assert archiver.screenshot_config.enabled is False
|
||||
|
||||
def test_markdown_respects_html2text(self):
|
||||
"""Test markdown option respects html2text availability."""
|
||||
archiver = SiteArchiver(include_markdown=True)
|
||||
# Should be True only if html2text is installed
|
||||
assert archiver.include_markdown == HAS_HTML2TEXT
|
||||
|
||||
def test_output_dir_path_conversion(self):
|
||||
"""Test output_dir is converted to Path."""
|
||||
archiver = SiteArchiver(output_dir="/custom/path")
|
||||
assert isinstance(archiver.output_dir, Path)
|
||||
assert str(archiver.output_dir) == "/custom/path"
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
534
tests/unit/test_database.py
Normal file
534
tests/unit/test_database.py
Normal file
|
|
@ -0,0 +1,534 @@
|
|||
"""
|
||||
Tests for database module.
|
||||
|
||||
Tests SQLAlchemy models, database operations, and queries.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, AsyncMock, patch, MagicMock
|
||||
import tempfile
|
||||
import shutil
|
||||
import os
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from database import (
|
||||
Database,
|
||||
CrawlJob,
|
||||
Media,
|
||||
MediaSource,
|
||||
Page,
|
||||
BackfillJob,
|
||||
SCORE_SCREENSHOT,
|
||||
SCORE_OG_IMAGE,
|
||||
SCORE_THUMBNAIL,
|
||||
SCORE_FULL_RES,
|
||||
OVER_9000,
|
||||
)
|
||||
|
||||
|
||||
class TestScoreConstants:
|
||||
"""Test scoring constants."""
|
||||
|
||||
def test_score_hierarchy(self):
|
||||
"""Test that scores follow expected hierarchy."""
|
||||
assert SCORE_SCREENSHOT < SCORE_OG_IMAGE
|
||||
assert SCORE_OG_IMAGE < SCORE_THUMBNAIL
|
||||
assert SCORE_THUMBNAIL < SCORE_FULL_RES
|
||||
|
||||
def test_score_values(self):
|
||||
"""Test specific score values."""
|
||||
assert SCORE_SCREENSHOT == 1
|
||||
assert SCORE_OG_IMAGE == 3
|
||||
assert SCORE_THUMBNAIL == 5
|
||||
assert SCORE_FULL_RES == 10
|
||||
|
||||
def test_over_9000(self):
|
||||
"""Test the limit constant."""
|
||||
assert OVER_9000 == 9000
|
||||
|
||||
|
||||
class TestDatabaseInitialization:
|
||||
"""Test Database class initialization."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test.db")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_database_init(self):
|
||||
"""Test database initialization creates tables."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
assert db._initialized is True
|
||||
assert db._engine is not None
|
||||
assert db._session_factory is not None
|
||||
assert os.path.exists(self.db_path)
|
||||
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_database_double_init(self):
|
||||
"""Test that double initialization is safe."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
await db.init() # Should not raise
|
||||
|
||||
assert db._initialized is True
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_database_session(self):
|
||||
"""Test getting a session."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
session = db.session()
|
||||
assert session is not None
|
||||
|
||||
await db.close()
|
||||
|
||||
|
||||
class TestCrawlJobOperations:
|
||||
"""Test crawl job database operations."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test.db")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_crawl_job(self):
|
||||
"""Test creating a crawl job."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(
|
||||
target_uri="https://example.com",
|
||||
keywords=["test", "demo"],
|
||||
mode="images",
|
||||
depth=10,
|
||||
max_pages=100,
|
||||
fast=False,
|
||||
screenshots=True,
|
||||
)
|
||||
|
||||
assert job_id > 0
|
||||
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_crawl_job_defaults(self):
|
||||
"""Test creating crawl job with defaults."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(target_uri="https://example.com")
|
||||
|
||||
assert job_id > 0
|
||||
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_crawl_job_stats(self):
|
||||
"""Test updating crawl job stats."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(target_uri="https://example.com")
|
||||
stats = {"pages_crawled": 10, "media_found": 50}
|
||||
|
||||
await db.update_crawl_job_stats(job_id, stats)
|
||||
|
||||
# Verify stats were saved
|
||||
job = await db.get_crawl_job(job_id)
|
||||
assert job is not None
|
||||
saved_stats = json.loads(job['stats'])
|
||||
assert saved_stats['pages_crawled'] == 10
|
||||
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_complete_crawl_job(self):
|
||||
"""Test completing a crawl job."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(target_uri="https://example.com")
|
||||
stats = {"pages_crawled": 100, "media_found": 500}
|
||||
|
||||
await db.complete_crawl_job(job_id, stats)
|
||||
|
||||
job = await db.get_crawl_job(job_id)
|
||||
assert job['status'] == 'completed'
|
||||
assert job['completed_at'] is not None
|
||||
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pause_crawl_job(self):
|
||||
"""Test pausing a crawl job."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(target_uri="https://example.com")
|
||||
stats = {"pages_crawled": 50}
|
||||
|
||||
await db.pause_crawl_job(job_id, stats)
|
||||
|
||||
job = await db.get_crawl_job(job_id)
|
||||
assert job['status'] == 'paused'
|
||||
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fail_crawl_job(self):
|
||||
"""Test failing a crawl job."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(target_uri="https://example.com")
|
||||
|
||||
await db.fail_crawl_job(job_id, "Connection timeout")
|
||||
|
||||
job = await db.get_crawl_job(job_id)
|
||||
assert job['status'] == 'failed'
|
||||
assert job['error'] == 'Connection timeout'
|
||||
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_crawl_job(self):
|
||||
"""Test deleting a crawl job."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(target_uri="https://example.com")
|
||||
|
||||
result = await db.delete_crawl_job(job_id)
|
||||
|
||||
assert result['deleted'] is True
|
||||
assert result['target_uri'] == "https://example.com"
|
||||
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_nonexistent_job(self):
|
||||
"""Test deleting a job that doesn't exist."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
result = await db.delete_crawl_job(99999)
|
||||
|
||||
assert result['deleted'] is False
|
||||
|
||||
await db.close()
|
||||
|
||||
|
||||
class TestMediaOperations:
|
||||
"""Test media database operations."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test.db")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_media_record(self):
|
||||
"""Test creating a media record."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(target_uri="https://example.com")
|
||||
|
||||
await db.create_media_record(
|
||||
md5_hash="d41d8cd98f00b204e9800998ecf8427e",
|
||||
media_uri="https://example.com/image.jpg",
|
||||
page_uri="https://example.com/",
|
||||
crawl_job_id=job_id,
|
||||
media_type="image",
|
||||
mime_type="image/jpeg",
|
||||
file_size=12345,
|
||||
page_title="Example Page",
|
||||
alt_text="A sample image",
|
||||
score=SCORE_FULL_RES,
|
||||
)
|
||||
|
||||
# Verify media was created
|
||||
media = await db.get_media("d41d8cd98f00b204e9800998ecf8427e")
|
||||
assert media is not None
|
||||
assert media['media_type'] == 'image'
|
||||
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_duplicate_media(self):
|
||||
"""Test that duplicate media is deduplicated."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(target_uri="https://example.com")
|
||||
|
||||
# Create first media
|
||||
await db.create_media_record(
|
||||
md5_hash="d41d8cd98f00b204e9800998ecf8427e",
|
||||
media_uri="https://example.com/image.jpg",
|
||||
page_uri="https://example.com/page1",
|
||||
crawl_job_id=job_id,
|
||||
media_type="image",
|
||||
)
|
||||
|
||||
# Create same media from different page
|
||||
await db.create_media_record(
|
||||
md5_hash="d41d8cd98f00b204e9800998ecf8427e",
|
||||
media_uri="https://example.com/image.jpg",
|
||||
page_uri="https://example.com/page2",
|
||||
crawl_job_id=job_id,
|
||||
media_type="image",
|
||||
)
|
||||
|
||||
# Should still have only one media record
|
||||
media = await db.get_media("d41d8cd98f00b204e9800998ecf8427e")
|
||||
assert media is not None
|
||||
|
||||
# But should have two sources
|
||||
sources = await db.get_media_sources("d41d8cd98f00b204e9800998ecf8427e")
|
||||
# Note: depends on unique constraint behavior
|
||||
assert len(sources) >= 1
|
||||
|
||||
await db.close()
|
||||
|
||||
|
||||
class TestPageOperations:
|
||||
"""Test page database operations."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test.db")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_page(self):
|
||||
"""Test creating a page record."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(target_uri="https://example.com")
|
||||
|
||||
await db.upsert_page(
|
||||
uri="https://example.com/about",
|
||||
title="About Page",
|
||||
content="This is the about page content.",
|
||||
crawl_job_id=job_id,
|
||||
)
|
||||
|
||||
# Verify page was created
|
||||
page = await db.get_page_by_uri("https://example.com/about")
|
||||
assert page is not None
|
||||
assert page['title'] == 'About Page'
|
||||
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_page_hashes(self):
|
||||
"""Test backfilling page URI hashes."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(target_uri="https://example.com")
|
||||
|
||||
await db.upsert_page(
|
||||
uri="https://example.com/test",
|
||||
title="Test",
|
||||
content="Test content",
|
||||
crawl_job_id=job_id,
|
||||
)
|
||||
|
||||
count = await db.backfill_page_hashes()
|
||||
# Should have backfilled at least one
|
||||
assert count >= 0
|
||||
|
||||
await db.close()
|
||||
|
||||
|
||||
class TestSearchOperations:
|
||||
"""Test search functionality."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test.db")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_media(self):
|
||||
"""Test searching for media."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(target_uri="https://example.com")
|
||||
|
||||
# Create media with searchable text
|
||||
await db.create_media_record(
|
||||
md5_hash="abc123def456",
|
||||
media_uri="https://example.com/sunset.jpg",
|
||||
page_uri="https://example.com/",
|
||||
crawl_job_id=job_id,
|
||||
media_type="image",
|
||||
alt_text="Beautiful sunset over the ocean",
|
||||
searchable_text="sunset ocean beautiful landscape",
|
||||
)
|
||||
|
||||
results = await db.search_media("sunset", limit=10)
|
||||
# Result depends on search implementation
|
||||
assert isinstance(results, list)
|
||||
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_stats(self):
|
||||
"""Test getting database stats."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
stats = await db.get_stats()
|
||||
|
||||
assert 'media_count' in stats or 'images' in stats or isinstance(stats, dict)
|
||||
|
||||
await db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_recent_media(self):
|
||||
"""Test getting recent media."""
|
||||
db = Database(self.db_path)
|
||||
await db.init()
|
||||
|
||||
job_id = await db.create_crawl_job(target_uri="https://example.com")
|
||||
|
||||
# Create some media
|
||||
for i in range(5):
|
||||
await db.create_media_record(
|
||||
md5_hash=f"hash{i}{'0' * 27}",
|
||||
media_uri=f"https://example.com/image{i}.jpg",
|
||||
page_uri="https://example.com/",
|
||||
crawl_job_id=job_id,
|
||||
media_type="image",
|
||||
)
|
||||
|
||||
recent = await db.get_recent_media(limit=3)
|
||||
assert len(recent) <= 3
|
||||
|
||||
await db.close()
|
||||
|
||||
|
||||
class TestCrawlJobModel:
|
||||
"""Test CrawlJob model."""
|
||||
|
||||
def test_crawl_job_table_name(self):
|
||||
"""Test CrawlJob table name."""
|
||||
assert CrawlJob.__tablename__ == 'crawl_jobs'
|
||||
|
||||
def test_crawl_job_columns(self):
|
||||
"""Test CrawlJob has required columns."""
|
||||
columns = [c.name for c in CrawlJob.__table__.columns]
|
||||
assert 'id' in columns
|
||||
assert 'target_uri' in columns
|
||||
assert 'keywords' in columns
|
||||
assert 'mode' in columns
|
||||
assert 'status' in columns
|
||||
assert 'started_at' in columns
|
||||
assert 'completed_at' in columns
|
||||
assert 'stats' in columns
|
||||
|
||||
|
||||
class TestMediaModel:
|
||||
"""Test Media model."""
|
||||
|
||||
def test_media_table_name(self):
|
||||
"""Test Media table name."""
|
||||
assert Media.__tablename__ == 'media'
|
||||
|
||||
def test_media_primary_key(self):
|
||||
"""Test Media primary key is md5_hash."""
|
||||
pk_columns = [c.name for c in Media.__table__.primary_key.columns]
|
||||
assert 'md5_hash' in pk_columns
|
||||
|
||||
def test_media_columns(self):
|
||||
"""Test Media has required columns."""
|
||||
columns = [c.name for c in Media.__table__.columns]
|
||||
assert 'md5_hash' in columns
|
||||
assert 'media_type' in columns
|
||||
assert 'mime_type' in columns
|
||||
assert 'file_size' in columns
|
||||
assert 'score' in columns
|
||||
assert 'first_seen_at' in columns
|
||||
|
||||
|
||||
class TestMediaSourceModel:
|
||||
"""Test MediaSource model."""
|
||||
|
||||
def test_media_source_table_name(self):
|
||||
"""Test MediaSource table name."""
|
||||
assert MediaSource.__tablename__ == 'media_sources'
|
||||
|
||||
def test_media_source_columns(self):
|
||||
"""Test MediaSource has required columns."""
|
||||
columns = [c.name for c in MediaSource.__table__.columns]
|
||||
assert 'md5_hash' in columns
|
||||
assert 'media_uri' in columns
|
||||
assert 'page_uri' in columns
|
||||
assert 'page_title' in columns
|
||||
assert 'alt_text' in columns
|
||||
assert 'discovered_at' in columns
|
||||
|
||||
|
||||
class TestPageModel:
|
||||
"""Test Page model."""
|
||||
|
||||
def test_page_table_name(self):
|
||||
"""Test Page table name."""
|
||||
assert Page.__tablename__ == 'pages'
|
||||
|
||||
def test_page_columns(self):
|
||||
"""Test Page has required columns."""
|
||||
columns = [c.name for c in Page.__table__.columns]
|
||||
assert 'uri' in columns
|
||||
assert 'uri_hash' in columns
|
||||
assert 'title' in columns
|
||||
assert 'content' in columns
|
||||
|
||||
|
||||
class TestBackfillJobModel:
|
||||
"""Test BackfillJob model."""
|
||||
|
||||
def test_backfill_job_table_name(self):
|
||||
"""Test BackfillJob table name."""
|
||||
assert BackfillJob.__tablename__ == 'backfill_jobs'
|
||||
|
||||
def test_backfill_job_columns(self):
|
||||
"""Test BackfillJob has required columns."""
|
||||
columns = [c.name for c in BackfillJob.__table__.columns]
|
||||
assert 'job_type' in columns
|
||||
assert 'status' in columns
|
||||
assert 'total_records' in columns
|
||||
assert 'processed_records' in columns
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
579
tests/unit/test_neopig.py
Normal file
579
tests/unit/test_neopig.py
Normal file
|
|
@ -0,0 +1,579 @@
|
|||
"""
|
||||
Tests for neopig module.
|
||||
|
||||
Tests the main NeoPig crawler class and related functionality.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, AsyncMock, patch, MagicMock
|
||||
import tempfile
|
||||
import shutil
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from neopig import (
|
||||
NeoPig,
|
||||
AppendOnlyStateLog,
|
||||
get_state_log_path,
|
||||
get_state_file_path,
|
||||
rotate_state_file,
|
||||
get_live_queue,
|
||||
emit_live_media,
|
||||
setup_logging,
|
||||
start_job_logging,
|
||||
stop_job_logging,
|
||||
)
|
||||
|
||||
|
||||
class TestAppendOnlyStateLog:
|
||||
"""Test AppendOnlyStateLog class."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.log_path = Path(self.temp_dir) / "test.log"
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_state_log_open_close(self):
|
||||
"""Test opening and closing state log."""
|
||||
log = AppendOnlyStateLog(self.log_path)
|
||||
log.open()
|
||||
assert log._file is not None
|
||||
log.close()
|
||||
assert log._file is None
|
||||
|
||||
def test_state_log_context_manager(self):
|
||||
"""Test state log as context manager."""
|
||||
with AppendOnlyStateLog(self.log_path) as log:
|
||||
assert log._file is not None
|
||||
assert log._file is None
|
||||
|
||||
def test_state_log_page(self):
|
||||
"""Test recording page as seen."""
|
||||
with AppendOnlyStateLog(self.log_path) as log:
|
||||
log.page("https://example.com/page1")
|
||||
log.page("https://example.com/page2")
|
||||
|
||||
content = self.log_path.read_text()
|
||||
assert "P https://example.com/page1" in content
|
||||
assert "P https://example.com/page2" in content
|
||||
|
||||
def test_state_log_media(self):
|
||||
"""Test recording media as downloaded."""
|
||||
with AppendOnlyStateLog(self.log_path) as log:
|
||||
log.media("abc123", "https://example.com/image.jpg")
|
||||
|
||||
content = self.log_path.read_text()
|
||||
assert "M abc123 https://example.com/image.jpg" in content
|
||||
|
||||
def test_state_log_screenshot(self):
|
||||
"""Test recording screenshot as taken."""
|
||||
with AppendOnlyStateLog(self.log_path) as log:
|
||||
log.screenshot("https://example.com/page")
|
||||
|
||||
content = self.log_path.read_text()
|
||||
assert "S https://example.com/page" in content
|
||||
|
||||
def test_state_log_skip_domain(self):
|
||||
"""Test recording domain to skip."""
|
||||
with AppendOnlyStateLog(self.log_path) as log:
|
||||
log.skip_domain("blocked.com")
|
||||
|
||||
content = self.log_path.read_text()
|
||||
assert "D blocked.com" in content
|
||||
|
||||
def test_state_log_stats(self):
|
||||
"""Test recording stats checkpoint."""
|
||||
stats = {"pages_crawled": 100, "media_found": 500}
|
||||
with AppendOnlyStateLog(self.log_path) as log:
|
||||
log.stats(stats)
|
||||
|
||||
content = self.log_path.read_text()
|
||||
assert "X stats" in content
|
||||
assert "100" in content
|
||||
|
||||
def test_state_log_load_empty(self):
|
||||
"""Test loading empty/nonexistent log."""
|
||||
log = AppendOnlyStateLog(self.log_path)
|
||||
result = log.load()
|
||||
|
||||
assert result['seen_pages'] == set()
|
||||
assert result['seen_media'] == {}
|
||||
assert result['seen_screenshots'] == set()
|
||||
assert result['skip_domains'] == set()
|
||||
assert result['stats'] == {}
|
||||
|
||||
def test_state_log_load_with_data(self):
|
||||
"""Test loading log with data."""
|
||||
with AppendOnlyStateLog(self.log_path) as log:
|
||||
log.page("https://example.com/page1")
|
||||
log.media("abc123", "https://example.com/image.jpg")
|
||||
log.screenshot("https://example.com/")
|
||||
log.skip_domain("blocked.com")
|
||||
log.stats({"pages_crawled": 10})
|
||||
|
||||
log = AppendOnlyStateLog(self.log_path)
|
||||
result = log.load()
|
||||
|
||||
assert "https://example.com/page1" in result['seen_pages']
|
||||
assert result['seen_media'].get("https://example.com/image.jpg") == "abc123"
|
||||
assert "https://example.com/" in result['seen_screenshots']
|
||||
assert "blocked.com" in result['skip_domains']
|
||||
assert result['stats'].get('pages_crawled') == 10
|
||||
|
||||
|
||||
class TestStateFilePaths:
|
||||
"""Test state file path generation."""
|
||||
|
||||
def test_get_state_log_path(self):
|
||||
"""Test state log path generation."""
|
||||
path = get_state_log_path("example.com")
|
||||
assert path == Path("data/example-com.log")
|
||||
|
||||
def test_get_state_log_path_with_subdomain(self):
|
||||
"""Test state log path with subdomain."""
|
||||
path = get_state_log_path("sub.example.com")
|
||||
assert "sub" in str(path)
|
||||
assert path.suffix == ".log"
|
||||
|
||||
def test_get_state_file_path(self):
|
||||
"""Test state file path generation."""
|
||||
path = get_state_file_path("example.com")
|
||||
assert path == Path("data/example-com.state")
|
||||
|
||||
def test_get_state_file_path_special_chars(self):
|
||||
"""Test state file path with special characters."""
|
||||
path = get_state_file_path("example.com:8080")
|
||||
assert "state" in str(path)
|
||||
# The path should be a valid state file path
|
||||
assert path.suffix == ".state"
|
||||
|
||||
|
||||
class TestRotateStateFile:
|
||||
"""Test state file rotation."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_rotate_nonexistent(self):
|
||||
"""Test rotating nonexistent file returns None."""
|
||||
path = Path(self.temp_dir) / "nonexistent.state"
|
||||
result = rotate_state_file(path)
|
||||
assert result is None
|
||||
|
||||
def test_rotate_existing(self):
|
||||
"""Test rotating existing file."""
|
||||
path = Path(self.temp_dir) / "test.state"
|
||||
path.write_text('{"key": "value"}')
|
||||
|
||||
rotated = rotate_state_file(path)
|
||||
|
||||
assert rotated is not None
|
||||
assert rotated.exists()
|
||||
assert not path.exists()
|
||||
assert ".1" in str(rotated)
|
||||
|
||||
def test_rotate_multiple(self):
|
||||
"""Test multiple rotations increment number."""
|
||||
path = Path(self.temp_dir) / "test.state"
|
||||
|
||||
# First rotation
|
||||
path.write_text('{"v": 1}')
|
||||
rotated1 = rotate_state_file(path)
|
||||
assert ".1" in str(rotated1)
|
||||
|
||||
# Second rotation
|
||||
path.write_text('{"v": 2}')
|
||||
rotated2 = rotate_state_file(path)
|
||||
assert ".2" in str(rotated2)
|
||||
|
||||
def test_rotate_with_preserve_keys(self):
|
||||
"""Test rotation with key preservation."""
|
||||
path = Path(self.temp_dir) / "test.state"
|
||||
path.write_text('{"keep": "value", "discard": "other"}')
|
||||
|
||||
rotated = rotate_state_file(path, preserve_keys=["keep"])
|
||||
|
||||
assert rotated.exists()
|
||||
assert path.exists() # New file with preserved keys
|
||||
new_content = json.loads(path.read_text())
|
||||
assert new_content.get("keep") == "value"
|
||||
assert "discard" not in new_content
|
||||
|
||||
|
||||
class TestLiveQueue:
|
||||
"""Test live media queue functions."""
|
||||
|
||||
def test_get_live_queue(self):
|
||||
"""Test getting live queue creates queue."""
|
||||
queue = get_live_queue()
|
||||
assert queue is not None
|
||||
# Should return same queue on second call
|
||||
queue2 = get_live_queue()
|
||||
assert queue is queue2
|
||||
|
||||
def test_emit_live_media(self):
|
||||
"""Test emitting media to queue."""
|
||||
queue = get_live_queue()
|
||||
# Empty queue first
|
||||
while not queue.empty():
|
||||
try:
|
||||
queue.get_nowait()
|
||||
except:
|
||||
break
|
||||
|
||||
media_info = {"md5_hash": "abc123", "media_type": "image"}
|
||||
emit_live_media(media_info)
|
||||
|
||||
# Should be able to get the item
|
||||
assert not queue.empty()
|
||||
|
||||
|
||||
class TestNeoPigInitialization:
|
||||
"""Test NeoPig class initialization."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test.db")
|
||||
self.vault_path = os.path.join(self.temp_dir, "vault")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_neopig_creation(self):
|
||||
"""Test creating NeoPig instance."""
|
||||
pig = NeoPig(
|
||||
db_path=self.db_path,
|
||||
vault_path=self.vault_path,
|
||||
)
|
||||
|
||||
assert pig.db is not None
|
||||
assert pig.vault is not None
|
||||
assert pig.fetcher is not None
|
||||
assert pig.screenshot is not None
|
||||
|
||||
def test_neopig_fast_mode(self):
|
||||
"""Test NeoPig in fast mode."""
|
||||
pig = NeoPig(
|
||||
db_path=self.db_path,
|
||||
vault_path=self.vault_path,
|
||||
fast_mode=True,
|
||||
)
|
||||
|
||||
assert pig.fast_mode is True
|
||||
assert pig.fetcher.default_crawl_delay == 0.0
|
||||
|
||||
def test_neopig_stats_initialized(self):
|
||||
"""Test NeoPig stats are initialized."""
|
||||
pig = NeoPig(
|
||||
db_path=self.db_path,
|
||||
vault_path=self.vault_path,
|
||||
)
|
||||
|
||||
assert pig.stats['pages_crawled'] == 0
|
||||
assert pig.stats['media_found'] == 0
|
||||
assert pig.stats['errors'] == 0
|
||||
|
||||
def test_neopig_state_tracking(self):
|
||||
"""Test NeoPig state tracking structures initialized."""
|
||||
pig = NeoPig(
|
||||
db_path=self.db_path,
|
||||
vault_path=self.vault_path,
|
||||
)
|
||||
|
||||
assert pig.seen_media == {}
|
||||
assert pig.seen_screenshots == set()
|
||||
assert pig.seen_pages == set()
|
||||
|
||||
|
||||
class TestNeoPigStateMethods:
|
||||
"""Test NeoPig state management methods."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test.db")
|
||||
self.vault_path = os.path.join(self.temp_dir, "vault")
|
||||
self.pig = NeoPig(
|
||||
db_path=self.db_path,
|
||||
vault_path=self.vault_path,
|
||||
)
|
||||
# Override state dir for tests
|
||||
self.pig._state_dir = Path(self.temp_dir)
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_get_state_file(self):
|
||||
"""Test getting state file path for URL."""
|
||||
path = self.pig._get_state_file("https://example.com/page")
|
||||
assert "example" in str(path)
|
||||
assert path.suffix == ".state"
|
||||
|
||||
def test_get_state_log(self):
|
||||
"""Test getting state log for URL."""
|
||||
log = self.pig._get_state_log("https://example.com/page")
|
||||
assert isinstance(log, AppendOnlyStateLog)
|
||||
|
||||
def test_get_hydra_state_file(self):
|
||||
"""Test getting hydra state file path."""
|
||||
path = self.pig._get_hydra_state_file("example.com")
|
||||
assert "hydra" in str(path)
|
||||
assert ".json" in str(path)
|
||||
|
||||
def test_load_hydra_state_empty(self):
|
||||
"""Test loading empty hydra state."""
|
||||
state = self.pig._load_hydra_state("example.com")
|
||||
assert state['seen_urls'] == {}
|
||||
assert state['feeds'] == {}
|
||||
|
||||
def test_save_and_load_hydra_state(self):
|
||||
"""Test saving and loading hydra state."""
|
||||
domain = "example.com"
|
||||
state = {
|
||||
'seen_urls': {'https://example.com/article': {}},
|
||||
'feeds': {'https://example.com/rss': {}},
|
||||
}
|
||||
self.pig._save_hydra_state(domain, state)
|
||||
|
||||
loaded = self.pig._load_hydra_state(domain)
|
||||
assert 'https://example.com/article' in loaded['seen_urls']
|
||||
assert 'https://example.com/rss' in loaded['feeds']
|
||||
|
||||
def test_get_known_feeds(self):
|
||||
"""Test getting known feeds for domain."""
|
||||
domain = "example.com"
|
||||
state = {
|
||||
'seen_urls': {},
|
||||
'feeds': {
|
||||
'https://example.com/rss': {},
|
||||
'https://example.com/atom': {},
|
||||
},
|
||||
}
|
||||
self.pig._save_hydra_state(domain, state)
|
||||
|
||||
feeds = self.pig._get_known_feeds(domain)
|
||||
assert len(feeds) == 2
|
||||
assert 'https://example.com/rss' in feeds
|
||||
|
||||
def test_add_hydra_feeds(self):
|
||||
"""Test adding new feeds to hydra state."""
|
||||
domain = "example.com"
|
||||
self.pig._add_hydra_feeds(domain, ['https://example.com/feed1'])
|
||||
|
||||
state = self.pig._load_hydra_state(domain)
|
||||
assert 'https://example.com/feed1' in state['feeds']
|
||||
|
||||
|
||||
class TestNeoPigAsync:
|
||||
"""Test NeoPig async methods."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test.db")
|
||||
self.vault_path = os.path.join(self.temp_dir, "vault")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_neopig_init_async(self):
|
||||
"""Test NeoPig async initialization."""
|
||||
pig = NeoPig(
|
||||
db_path=self.db_path,
|
||||
vault_path=self.vault_path,
|
||||
)
|
||||
await pig.init()
|
||||
|
||||
assert pig.db._initialized is True
|
||||
assert pig.vault._initialized is True
|
||||
|
||||
# Cleanup
|
||||
await pig.db.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_neopig_db_close(self):
|
||||
"""Test NeoPig database cleanup."""
|
||||
pig = NeoPig(
|
||||
db_path=self.db_path,
|
||||
vault_path=self.vault_path,
|
||||
)
|
||||
await pig.init()
|
||||
await pig.db.close()
|
||||
|
||||
# Should not raise
|
||||
|
||||
|
||||
class TestJobLogging:
|
||||
"""Test job logging functions."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
@patch('neopig.LOGS_PATH', None)
|
||||
def test_start_job_logging(self):
|
||||
"""Test starting job logging."""
|
||||
# This modifies global state, so we just verify it doesn't crash
|
||||
# The actual logging is tested by checking log file creation
|
||||
pass # Would need to mock LOGS_PATH properly
|
||||
|
||||
|
||||
class TestSetupLogging:
|
||||
"""Test logging setup."""
|
||||
|
||||
def test_setup_logging(self):
|
||||
"""Test setting up logging doesn't crash."""
|
||||
import logging
|
||||
setup_logging(level=logging.DEBUG)
|
||||
# Verify root logger has handlers
|
||||
assert len(logging.getLogger().handlers) > 0
|
||||
|
||||
|
||||
class TestNeoPigStats:
|
||||
"""Test NeoPig statistics tracking."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test.db")
|
||||
self.vault_path = os.path.join(self.temp_dir, "vault")
|
||||
self.pig = NeoPig(
|
||||
db_path=self.db_path,
|
||||
vault_path=self.vault_path,
|
||||
)
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_stats_structure(self):
|
||||
"""Test stats dictionary structure."""
|
||||
required_keys = [
|
||||
'pages_crawled',
|
||||
'pages_pending',
|
||||
'media_found',
|
||||
'media_downloaded',
|
||||
'duplicates_skipped',
|
||||
'screenshots_taken',
|
||||
'errors',
|
||||
'bytes_downloaded',
|
||||
'bytes_stored',
|
||||
]
|
||||
for key in required_keys:
|
||||
assert key in self.pig.stats
|
||||
|
||||
def test_stats_initial_values(self):
|
||||
"""Test stats start at zero."""
|
||||
for key, value in self.pig.stats.items():
|
||||
if isinstance(value, int):
|
||||
assert value == 0
|
||||
|
||||
|
||||
class TestNeoPigVaultManager:
|
||||
"""Test NeoPig vault manager integration."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test.db")
|
||||
self.vault_path = os.path.join(self.temp_dir, "vault")
|
||||
self.pig = NeoPig(
|
||||
db_path=self.db_path,
|
||||
vault_path=self.vault_path,
|
||||
)
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_domain_vaults_initialized(self):
|
||||
"""Test domain vaults are initialized."""
|
||||
assert self.pig.domain_vaults is not None
|
||||
|
||||
def test_vault_paths(self):
|
||||
"""Test vault paths are set correctly."""
|
||||
assert self.pig.vault_path == self.vault_path
|
||||
|
||||
|
||||
class TestTqdmLoggingHandler:
|
||||
"""Test TqdmLoggingHandler."""
|
||||
|
||||
def test_tqdm_handler_exists(self):
|
||||
"""Test TqdmLoggingHandler class exists."""
|
||||
from neopig import TqdmLoggingHandler
|
||||
handler = TqdmLoggingHandler()
|
||||
assert handler is not None
|
||||
|
||||
def test_tqdm_handler_emit(self):
|
||||
"""Test handler emit doesn't crash."""
|
||||
from neopig import TqdmLoggingHandler
|
||||
import logging
|
||||
|
||||
handler = TqdmLoggingHandler()
|
||||
handler.setFormatter(logging.Formatter('%(message)s'))
|
||||
|
||||
record = logging.LogRecord(
|
||||
name='test',
|
||||
level=logging.INFO,
|
||||
pathname='test.py',
|
||||
lineno=1,
|
||||
msg='Test message',
|
||||
args=(),
|
||||
exc_info=None
|
||||
)
|
||||
|
||||
# Should not raise
|
||||
handler.emit(record)
|
||||
|
||||
|
||||
class TestNeoPigCrawlMethods:
|
||||
"""Test NeoPig crawl-related methods."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test.db")
|
||||
self.vault_path = os.path.join(self.temp_dir, "vault")
|
||||
self.pig = NeoPig(
|
||||
db_path=self.db_path,
|
||||
vault_path=self.vault_path,
|
||||
)
|
||||
self.pig._state_dir = Path(self.temp_dir)
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_save_state_interval(self):
|
||||
"""Test state save interval."""
|
||||
# State is saved every N items
|
||||
assert self.pig._state_save_interval == 10
|
||||
assert self.pig._items_since_save == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_neopig_crawl_returns_stats(self):
|
||||
"""Test that crawl stats structure exists."""
|
||||
await self.pig.init()
|
||||
|
||||
# We can't fully test crawl without extensive mocking,
|
||||
# but we can verify the structure
|
||||
assert 'pages_crawled' in self.pig.stats
|
||||
assert 'media_found' in self.pig.stats
|
||||
assert 'errors' in self.pig.stats
|
||||
|
||||
await self.pig.db.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
562
tests/unit/test_repo.py
Normal file
562
tests/unit/test_repo.py
Normal file
|
|
@ -0,0 +1,562 @@
|
|||
"""
|
||||
Tests for repo module.
|
||||
|
||||
Tests VCS detection, repository cloning, and file walking.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, AsyncMock, patch, MagicMock
|
||||
import tempfile
|
||||
import shutil
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from repo import (
|
||||
detect_vcs,
|
||||
get_repo_path,
|
||||
run_cmd,
|
||||
run_cmd_async,
|
||||
clone_repo,
|
||||
clone_repo_async,
|
||||
pull_repo,
|
||||
pull_repo_async,
|
||||
get_commit_hash,
|
||||
walk_files,
|
||||
is_binary_file,
|
||||
get_file_language,
|
||||
VCS_HOSTS,
|
||||
VCS_URL_PATTERNS,
|
||||
VCS_DIRS,
|
||||
BINARY_EXTENSIONS,
|
||||
)
|
||||
|
||||
|
||||
class TestVCSDetection:
|
||||
"""Test VCS type detection from URLs."""
|
||||
|
||||
def test_detect_github(self):
|
||||
"""Test detecting GitHub as git."""
|
||||
vcs, uri = detect_vcs("https://github.com/user/repo")
|
||||
assert vcs == "git"
|
||||
assert uri.endswith(".git")
|
||||
|
||||
def test_detect_github_with_git_suffix(self):
|
||||
"""Test GitHub URL already ending in .git."""
|
||||
vcs, uri = detect_vcs("https://github.com/user/repo.git")
|
||||
assert vcs == "git"
|
||||
assert uri == "https://github.com/user/repo.git"
|
||||
|
||||
def test_detect_gitlab(self):
|
||||
"""Test detecting GitLab as git."""
|
||||
vcs, uri = detect_vcs("https://gitlab.com/user/repo")
|
||||
assert vcs == "git"
|
||||
assert uri.endswith(".git")
|
||||
|
||||
def test_detect_bitbucket(self):
|
||||
"""Test detecting Bitbucket as git."""
|
||||
vcs, uri = detect_vcs("https://bitbucket.org/user/repo")
|
||||
assert vcs == "git"
|
||||
|
||||
def test_detect_codeberg(self):
|
||||
"""Test detecting Codeberg as git."""
|
||||
vcs, uri = detect_vcs("https://codeberg.org/user/repo")
|
||||
assert vcs == "git"
|
||||
|
||||
def test_detect_hg_mozilla(self):
|
||||
"""Test detecting Mozilla HG as mercurial."""
|
||||
vcs, uri = detect_vcs("https://hg.mozilla.org/mozilla-central")
|
||||
assert vcs == "hg"
|
||||
assert uri == "https://hg.mozilla.org/mozilla-central"
|
||||
|
||||
def test_detect_hg_python(self):
|
||||
"""Test detecting Python HG as mercurial."""
|
||||
vcs, uri = detect_vcs("https://hg.python.org/cpython")
|
||||
assert vcs == "hg"
|
||||
|
||||
def test_detect_svn_apache(self):
|
||||
"""Test detecting Apache SVN."""
|
||||
vcs, uri = detect_vcs("https://svn.apache.org/repos/asf/project")
|
||||
assert vcs == "svn"
|
||||
|
||||
def test_detect_ssh_git_url(self):
|
||||
"""Test detecting SSH git URL."""
|
||||
vcs, uri = detect_vcs("git@github.com:user/repo.git")
|
||||
assert vcs == "git"
|
||||
assert uri == "git@github.com:user/repo.git"
|
||||
|
||||
def test_detect_ssh_hg_url(self):
|
||||
"""Test detecting SSH hg URL."""
|
||||
vcs, uri = detect_vcs("hg@bitbucket.org:user/repo")
|
||||
assert vcs == "hg"
|
||||
|
||||
def test_detect_fossil_extension(self):
|
||||
"""Test detecting fossil from .fossil extension."""
|
||||
vcs, uri = detect_vcs("https://example.com/project.fossil")
|
||||
assert vcs == "fossil"
|
||||
|
||||
def test_detect_svn_trunk_pattern(self):
|
||||
"""Test detecting SVN from /trunk pattern."""
|
||||
vcs, uri = detect_vcs("https://svn.example.com/project/trunk")
|
||||
assert vcs == "svn"
|
||||
|
||||
def test_detect_svn_branches_pattern(self):
|
||||
"""Test detecting SVN from /branches pattern."""
|
||||
vcs, uri = detect_vcs("https://svn.example.com/project/branches/feature")
|
||||
assert vcs == "svn"
|
||||
|
||||
def test_detect_unknown_url(self):
|
||||
"""Test non-VCS URL returns None."""
|
||||
vcs, uri = detect_vcs("https://example.com/page.html")
|
||||
assert vcs is None
|
||||
assert uri is None
|
||||
|
||||
def test_detect_non_repo_url(self):
|
||||
"""Test regular website URL."""
|
||||
vcs, uri = detect_vcs("https://google.com")
|
||||
assert vcs is None
|
||||
|
||||
|
||||
class TestGetRepoPath:
|
||||
"""Test repository path generation."""
|
||||
|
||||
def test_github_repo_path(self):
|
||||
"""Test path for GitHub repo."""
|
||||
path = get_repo_path("https://github.com/user/repo", Path("vault"))
|
||||
assert path == Path("vault/github.com/user/repo")
|
||||
|
||||
def test_github_repo_path_with_git_suffix(self):
|
||||
"""Test path strips .git suffix."""
|
||||
path = get_repo_path("https://github.com/user/repo.git", Path("vault"))
|
||||
assert path == Path("vault/github.com/user/repo")
|
||||
|
||||
def test_nested_repo_path(self):
|
||||
"""Test path for nested repo structure."""
|
||||
path = get_repo_path("https://github.com/org/sub/repo", Path("vault"))
|
||||
assert path == Path("vault/github.com/org/sub/repo")
|
||||
|
||||
def test_ssh_repo_path(self):
|
||||
"""Test path for SSH URL."""
|
||||
path = get_repo_path("git@github.com:user/repo.git", Path("vault"))
|
||||
assert path == Path("vault/github.com/user/repo")
|
||||
|
||||
def test_repo_path_with_port(self):
|
||||
"""Test path strips port from host."""
|
||||
path = get_repo_path("https://git.example.com:8443/user/repo", Path("vault"))
|
||||
assert "8443" not in str(path) or "git.example.com" in str(path)
|
||||
|
||||
|
||||
class TestRunCmd:
|
||||
"""Test command execution."""
|
||||
|
||||
def test_run_cmd_success(self):
|
||||
"""Test successful command execution."""
|
||||
ret, stdout, stderr = run_cmd(["echo", "hello"])
|
||||
assert ret == 0
|
||||
assert "hello" in stdout
|
||||
|
||||
def test_run_cmd_failure(self):
|
||||
"""Test failed command execution."""
|
||||
ret, stdout, stderr = run_cmd(["false"])
|
||||
assert ret != 0
|
||||
|
||||
def test_run_cmd_nonexistent(self):
|
||||
"""Test nonexistent command."""
|
||||
ret, stdout, stderr = run_cmd(["nonexistent_command_xyz"])
|
||||
assert ret != 0
|
||||
|
||||
def test_run_cmd_timeout(self):
|
||||
"""Test command timeout."""
|
||||
# This should timeout quickly
|
||||
ret, stdout, stderr = run_cmd(["sleep", "10"], timeout=1)
|
||||
assert ret == -1
|
||||
assert "timed out" in stderr.lower() or "timeout" in stderr.lower()
|
||||
|
||||
|
||||
class TestRunCmdAsync:
|
||||
"""Test async command execution."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cmd_async_success(self):
|
||||
"""Test async successful command."""
|
||||
ret, stdout, stderr = await run_cmd_async(["echo", "hello"])
|
||||
assert ret == 0
|
||||
assert "hello" in stdout
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cmd_async_failure(self):
|
||||
"""Test async failed command."""
|
||||
ret, stdout, stderr = await run_cmd_async(["false"])
|
||||
assert ret != 0
|
||||
|
||||
|
||||
class TestCloneRepo:
|
||||
"""Test repository cloning."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_clone_repo_unknown_vcs(self):
|
||||
"""Test cloning with unknown VCS type."""
|
||||
dest = Path(self.temp_dir) / "unknown"
|
||||
success, msg = clone_repo("https://example.com/repo", dest, "unknown")
|
||||
assert success is False
|
||||
assert "Unknown VCS type" in msg
|
||||
|
||||
@patch('repo.run_cmd')
|
||||
def test_clone_git_shallow(self, mock_run):
|
||||
"""Test git clone with shallow flag."""
|
||||
mock_run.return_value = (0, "", "")
|
||||
dest = Path(self.temp_dir) / "repo"
|
||||
|
||||
success, msg = clone_repo("https://github.com/user/repo.git", dest, "git", shallow=True)
|
||||
|
||||
mock_run.assert_called_once()
|
||||
call_args = mock_run.call_args[0][0]
|
||||
assert "--depth" in call_args
|
||||
assert "1" in call_args
|
||||
|
||||
@patch('repo.run_cmd')
|
||||
def test_clone_git_full(self, mock_run):
|
||||
"""Test git clone without shallow flag."""
|
||||
mock_run.return_value = (0, "", "")
|
||||
dest = Path(self.temp_dir) / "repo"
|
||||
|
||||
success, msg = clone_repo("https://github.com/user/repo.git", dest, "git", shallow=False)
|
||||
|
||||
mock_run.assert_called_once()
|
||||
call_args = mock_run.call_args[0][0]
|
||||
assert "--depth" not in call_args
|
||||
|
||||
@patch('repo.run_cmd')
|
||||
def test_clone_hg(self, mock_run):
|
||||
"""Test mercurial clone."""
|
||||
mock_run.return_value = (0, "", "")
|
||||
dest = Path(self.temp_dir) / "repo"
|
||||
|
||||
success, msg = clone_repo("https://hg.example.com/repo", dest, "hg")
|
||||
|
||||
mock_run.assert_called_once()
|
||||
call_args = mock_run.call_args[0][0]
|
||||
assert "hg" in call_args
|
||||
assert "clone" in call_args
|
||||
|
||||
@patch('repo.run_cmd')
|
||||
def test_clone_svn(self, mock_run):
|
||||
"""Test SVN checkout."""
|
||||
mock_run.return_value = (0, "", "")
|
||||
dest = Path(self.temp_dir) / "repo"
|
||||
|
||||
success, msg = clone_repo("https://svn.example.com/repo", dest, "svn")
|
||||
|
||||
mock_run.assert_called_once()
|
||||
call_args = mock_run.call_args[0][0]
|
||||
assert "svn" in call_args
|
||||
assert "checkout" in call_args
|
||||
|
||||
|
||||
class TestCloneRepoAsync:
|
||||
"""Test async repository cloning."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch('repo.clone_repo')
|
||||
async def test_clone_repo_async(self, mock_clone):
|
||||
"""Test async clone delegates to sync."""
|
||||
mock_clone.return_value = (True, "Cloned")
|
||||
|
||||
success, msg = await clone_repo_async(
|
||||
"https://github.com/user/repo.git",
|
||||
Path("/tmp/repo"),
|
||||
"git"
|
||||
)
|
||||
|
||||
assert success is True
|
||||
mock_clone.assert_called_once()
|
||||
|
||||
|
||||
class TestPullRepo:
|
||||
"""Test repository pulling/updating."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.repo_path = Path(self.temp_dir) / "repo"
|
||||
self.repo_path.mkdir()
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_pull_no_vcs(self):
|
||||
"""Test pull on directory without VCS."""
|
||||
success, msg = pull_repo(self.repo_path)
|
||||
assert success is False
|
||||
assert "No VCS directory found" in msg
|
||||
|
||||
@patch('repo.run_cmd')
|
||||
def test_pull_git(self, mock_run):
|
||||
"""Test git pull."""
|
||||
mock_run.return_value = (0, "Already up to date", "")
|
||||
(self.repo_path / ".git").mkdir()
|
||||
|
||||
success, msg = pull_repo(self.repo_path)
|
||||
|
||||
assert success is True
|
||||
mock_run.assert_called_once()
|
||||
call_args = mock_run.call_args[0][0]
|
||||
assert "git" in call_args
|
||||
assert "pull" in call_args
|
||||
|
||||
@patch('repo.run_cmd')
|
||||
def test_pull_hg(self, mock_run):
|
||||
"""Test mercurial pull."""
|
||||
mock_run.return_value = (0, "pulling from...", "")
|
||||
(self.repo_path / ".hg").mkdir()
|
||||
|
||||
success, msg = pull_repo(self.repo_path)
|
||||
|
||||
assert success is True
|
||||
call_args = mock_run.call_args[0][0]
|
||||
assert "hg" in call_args
|
||||
assert "pull" in call_args
|
||||
|
||||
@patch('repo.run_cmd')
|
||||
def test_pull_svn(self, mock_run):
|
||||
"""Test SVN update."""
|
||||
mock_run.return_value = (0, "Updating...", "")
|
||||
(self.repo_path / ".svn").mkdir()
|
||||
|
||||
success, msg = pull_repo(self.repo_path)
|
||||
|
||||
assert success is True
|
||||
call_args = mock_run.call_args[0][0]
|
||||
assert "svn" in call_args
|
||||
assert "update" in call_args
|
||||
|
||||
|
||||
class TestGetCommitHash:
|
||||
"""Test getting commit/revision hash."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.repo_path = Path(self.temp_dir) / "repo"
|
||||
self.repo_path.mkdir()
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_get_hash_no_vcs(self):
|
||||
"""Test getting hash without VCS."""
|
||||
result = get_commit_hash(self.repo_path)
|
||||
assert result is None
|
||||
|
||||
@patch('repo.run_cmd')
|
||||
def test_get_git_hash(self, mock_run):
|
||||
"""Test getting git commit hash."""
|
||||
mock_run.return_value = (0, "abc123def456\n", "")
|
||||
(self.repo_path / ".git").mkdir()
|
||||
|
||||
result = get_commit_hash(self.repo_path)
|
||||
|
||||
assert result == "abc123def456"
|
||||
|
||||
@patch('repo.run_cmd')
|
||||
def test_get_hg_hash(self, mock_run):
|
||||
"""Test getting mercurial changeset hash."""
|
||||
mock_run.return_value = (0, "abc123+\n", "")
|
||||
(self.repo_path / ".hg").mkdir()
|
||||
|
||||
result = get_commit_hash(self.repo_path)
|
||||
|
||||
assert result == "abc123+"
|
||||
|
||||
|
||||
class TestWalkFiles:
|
||||
"""Test walking files in a repository."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.repo_path = Path(self.temp_dir) / "repo"
|
||||
self.repo_path.mkdir()
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_walk_empty_repo(self):
|
||||
"""Test walking empty directory."""
|
||||
files = list(walk_files(self.repo_path))
|
||||
assert files == []
|
||||
|
||||
def test_walk_with_files(self):
|
||||
"""Test walking directory with files."""
|
||||
# Create some files
|
||||
(self.repo_path / "main.py").write_text("print('hello')")
|
||||
(self.repo_path / "README.md").write_text("# Test")
|
||||
|
||||
files = list(walk_files(self.repo_path))
|
||||
|
||||
assert len(files) == 2
|
||||
names = [f.name for f in files]
|
||||
assert "main.py" in names
|
||||
assert "README.md" in names
|
||||
|
||||
def test_walk_skips_vcs_dirs(self):
|
||||
"""Test that VCS directories are skipped."""
|
||||
(self.repo_path / ".git").mkdir()
|
||||
(self.repo_path / ".git" / "config").write_text("[core]")
|
||||
(self.repo_path / "main.py").write_text("print('hello')")
|
||||
|
||||
files = list(walk_files(self.repo_path))
|
||||
|
||||
# Should only have main.py, not .git/config
|
||||
assert len(files) == 1
|
||||
assert files[0].name == "main.py"
|
||||
|
||||
def test_walk_skips_binary_by_default(self):
|
||||
"""Test that binary files are skipped by default."""
|
||||
(self.repo_path / "image.png").write_bytes(b"\x89PNG\r\n\x1a\n")
|
||||
(self.repo_path / "main.py").write_text("print('hello')")
|
||||
|
||||
files = list(walk_files(self.repo_path, include_binary=False))
|
||||
|
||||
names = [f.name for f in files]
|
||||
assert "main.py" in names
|
||||
assert "image.png" not in names
|
||||
|
||||
def test_walk_includes_binary_when_requested(self):
|
||||
"""Test that binary files are included when requested."""
|
||||
(self.repo_path / "image.png").write_bytes(b"\x89PNG\r\n\x1a\n")
|
||||
(self.repo_path / "main.py").write_text("print('hello')")
|
||||
|
||||
files = list(walk_files(self.repo_path, include_binary=True))
|
||||
|
||||
names = [f.name for f in files]
|
||||
assert "main.py" in names
|
||||
assert "image.png" in names
|
||||
|
||||
def test_walk_skips_empty_files(self):
|
||||
"""Test that empty files are skipped."""
|
||||
(self.repo_path / "empty.txt").write_text("")
|
||||
(self.repo_path / "main.py").write_text("print('hello')")
|
||||
|
||||
files = list(walk_files(self.repo_path))
|
||||
|
||||
names = [f.name for f in files]
|
||||
assert "main.py" in names
|
||||
assert "empty.txt" not in names
|
||||
|
||||
def test_walk_nested_directories(self):
|
||||
"""Test walking nested directories."""
|
||||
(self.repo_path / "src").mkdir()
|
||||
(self.repo_path / "src" / "app.py").write_text("app code")
|
||||
(self.repo_path / "tests").mkdir()
|
||||
(self.repo_path / "tests" / "test_app.py").write_text("test code")
|
||||
|
||||
files = list(walk_files(self.repo_path))
|
||||
|
||||
names = [f.name for f in files]
|
||||
assert "app.py" in names
|
||||
assert "test_app.py" in names
|
||||
|
||||
|
||||
class TestIsBinaryFile:
|
||||
"""Test binary file detection."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_binary_by_extension(self):
|
||||
"""Test detecting binary by extension."""
|
||||
path = Path(self.temp_dir) / "image.png"
|
||||
path.write_bytes(b"fake png")
|
||||
|
||||
assert is_binary_file(path) is True
|
||||
|
||||
def test_binary_by_content(self):
|
||||
"""Test detecting binary by content (null bytes)."""
|
||||
path = Path(self.temp_dir) / "data.bin"
|
||||
path.write_bytes(b"hello\x00world")
|
||||
|
||||
assert is_binary_file(path) is True
|
||||
|
||||
def test_text_file(self):
|
||||
"""Test detecting text file."""
|
||||
path = Path(self.temp_dir) / "code.py"
|
||||
path.write_text("print('hello')")
|
||||
|
||||
assert is_binary_file(path) is False
|
||||
|
||||
def test_binary_extensions_set(self):
|
||||
"""Test that binary extensions are defined."""
|
||||
assert ".png" in BINARY_EXTENSIONS
|
||||
assert ".jpg" in BINARY_EXTENSIONS
|
||||
assert ".exe" in BINARY_EXTENSIONS
|
||||
assert ".zip" in BINARY_EXTENSIONS
|
||||
|
||||
|
||||
class TestGetFileLanguage:
|
||||
"""Test programming language detection."""
|
||||
|
||||
def test_python_extension(self):
|
||||
"""Test detecting Python."""
|
||||
assert get_file_language(Path("script.py")) == "python"
|
||||
|
||||
def test_javascript_extension(self):
|
||||
"""Test detecting JavaScript."""
|
||||
assert get_file_language(Path("app.js")) == "javascript"
|
||||
|
||||
def test_typescript_extension(self):
|
||||
"""Test detecting TypeScript."""
|
||||
assert get_file_language(Path("app.ts")) == "typescript"
|
||||
|
||||
def test_jsx_extension(self):
|
||||
"""Test detecting JSX."""
|
||||
assert get_file_language(Path("component.jsx")) == "javascript"
|
||||
|
||||
def test_tsx_extension(self):
|
||||
"""Test detecting TSX."""
|
||||
assert get_file_language(Path("component.tsx")) == "typescript"
|
||||
|
||||
def test_ruby_extension(self):
|
||||
"""Test detecting Ruby."""
|
||||
assert get_file_language(Path("app.rb")) == "ruby"
|
||||
|
||||
def test_unknown_extension(self):
|
||||
"""Test unknown extension returns None."""
|
||||
result = get_file_language(Path("file.xyz"))
|
||||
assert result is None or result == ""
|
||||
|
||||
|
||||
class TestVCSConstants:
|
||||
"""Test VCS-related constants."""
|
||||
|
||||
def test_vcs_hosts_contains_major_hosts(self):
|
||||
"""Test VCS hosts map contains major hosts."""
|
||||
assert "github.com" in VCS_HOSTS
|
||||
assert "gitlab.com" in VCS_HOSTS
|
||||
assert "bitbucket.org" in VCS_HOSTS
|
||||
|
||||
def test_vcs_dirs_contains_standard_dirs(self):
|
||||
"""Test VCS dirs contains standard directories."""
|
||||
assert ".git" in VCS_DIRS
|
||||
assert ".hg" in VCS_DIRS
|
||||
assert ".svn" in VCS_DIRS
|
||||
|
||||
def test_vcs_url_patterns_is_list(self):
|
||||
"""Test VCS URL patterns is a list of tuples."""
|
||||
assert isinstance(VCS_URL_PATTERNS, list)
|
||||
for pattern, vcs_type in VCS_URL_PATTERNS:
|
||||
assert isinstance(pattern, str)
|
||||
assert isinstance(vcs_type, str)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
422
tests/unit/test_serp.py
Normal file
422
tests/unit/test_serp.py
Normal file
|
|
@ -0,0 +1,422 @@
|
|||
"""
|
||||
Tests for serp module.
|
||||
|
||||
Tests FastAPI SERP server endpoints and functionality.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, AsyncMock, patch, MagicMock
|
||||
import tempfile
|
||||
import shutil
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Test client for FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from httpx import AsyncClient, ASGITransport
|
||||
|
||||
from serp import (
|
||||
app,
|
||||
get_lang,
|
||||
t,
|
||||
inject_i18n,
|
||||
slugify,
|
||||
TRANSLATIONS,
|
||||
LANG_NAMES,
|
||||
CrawlRequest,
|
||||
)
|
||||
|
||||
|
||||
class TestLanguageDetection:
|
||||
"""Test language detection functions."""
|
||||
|
||||
def test_get_lang_default(self):
|
||||
"""Test default language is English."""
|
||||
lang = get_lang(None, None)
|
||||
assert lang == "en"
|
||||
|
||||
def test_get_lang_from_cookie(self):
|
||||
"""Test language from cookie takes priority."""
|
||||
lang = get_lang("zh", "en-US")
|
||||
assert lang == "zh"
|
||||
|
||||
def test_get_lang_from_accept_header(self):
|
||||
"""Test language from Accept-Language header."""
|
||||
lang = get_lang(None, "zh-CN,zh;q=0.9,en;q=0.8")
|
||||
assert lang == "zh"
|
||||
|
||||
def test_get_lang_accept_header_fallback(self):
|
||||
"""Test Accept-Language fallback to English for unknown."""
|
||||
lang = get_lang(None, "xyz-XY,abc;q=0.8")
|
||||
assert lang == "en"
|
||||
|
||||
def test_get_lang_unsupported_cookie(self):
|
||||
"""Test unsupported cookie falls back to header."""
|
||||
lang = get_lang("xyz", "ja,en;q=0.8")
|
||||
assert lang == "ja"
|
||||
|
||||
|
||||
class TestTranslationFunction:
|
||||
"""Test translation function."""
|
||||
|
||||
def test_translation_english(self):
|
||||
"""Test English translation."""
|
||||
result = t("search", "en")
|
||||
assert result == "Search"
|
||||
|
||||
def test_translation_chinese(self):
|
||||
"""Test Chinese translation."""
|
||||
result = t("search", "zh")
|
||||
assert result == "搜索"
|
||||
|
||||
def test_translation_spanish(self):
|
||||
"""Test Spanish translation."""
|
||||
result = t("search", "es")
|
||||
assert result == "Buscar"
|
||||
|
||||
def test_translation_missing_key(self):
|
||||
"""Test missing key returns key."""
|
||||
result = t("nonexistent_key", "en")
|
||||
assert result == "nonexistent_key"
|
||||
|
||||
def test_translation_missing_language(self):
|
||||
"""Test missing language falls back to English."""
|
||||
result = t("search", "xyz")
|
||||
assert result == "Search"
|
||||
|
||||
|
||||
class TestSlugify:
|
||||
"""Test slugify function."""
|
||||
|
||||
def test_slugify_basic(self):
|
||||
"""Test basic slugification."""
|
||||
result = slugify("Hello World")
|
||||
assert result == "hello-world"
|
||||
|
||||
def test_slugify_special_chars(self):
|
||||
"""Test removing special characters."""
|
||||
result = slugify("Hello! World? Test@123")
|
||||
assert "hello" in result
|
||||
assert "world" in result
|
||||
|
||||
def test_slugify_max_length(self):
|
||||
"""Test max length truncation."""
|
||||
long_text = "a" * 100
|
||||
result = slugify(long_text, max_len=10)
|
||||
assert len(result) <= 10
|
||||
|
||||
def test_slugify_empty(self):
|
||||
"""Test empty string."""
|
||||
result = slugify("")
|
||||
assert result == ""
|
||||
|
||||
def test_slugify_unicode(self):
|
||||
"""Test unicode handling."""
|
||||
result = slugify("Cafe with accents")
|
||||
assert result == "cafe-with-accents" or "cafe" in result
|
||||
|
||||
|
||||
class TestTranslationsStructure:
|
||||
"""Test translations dictionary structure."""
|
||||
|
||||
def test_all_languages_have_required_keys(self):
|
||||
"""Test all languages have core translation keys."""
|
||||
required_keys = ["search", "crawl", "live", "about", "loading"]
|
||||
for lang_code, translations in TRANSLATIONS.items():
|
||||
for key in required_keys:
|
||||
assert key in translations, f"Missing '{key}' in language '{lang_code}'"
|
||||
|
||||
def test_lang_names_match_translations(self):
|
||||
"""Test language names match translation keys."""
|
||||
for lang_code in LANG_NAMES:
|
||||
assert lang_code in TRANSLATIONS, f"Language '{lang_code}' in LANG_NAMES but not TRANSLATIONS"
|
||||
|
||||
|
||||
class TestCrawlRequestModel:
|
||||
"""Test CrawlRequest Pydantic model."""
|
||||
|
||||
def test_crawl_request_defaults(self):
|
||||
"""Test CrawlRequest default values."""
|
||||
request = CrawlRequest()
|
||||
assert request.targets == []
|
||||
assert request.target_uri == ""
|
||||
assert request.mode == "all"
|
||||
assert request.depth == -1
|
||||
assert request.max_pages == -1
|
||||
assert request.fresh is False
|
||||
assert request.fast is False
|
||||
assert request.screenshots is True
|
||||
assert request.hydra is False
|
||||
|
||||
def test_crawl_request_with_values(self):
|
||||
"""Test CrawlRequest with custom values."""
|
||||
request = CrawlRequest(
|
||||
targets=["https://example.com", "https://test.com"],
|
||||
mode="images",
|
||||
depth=5,
|
||||
max_pages=100,
|
||||
fresh=True,
|
||||
)
|
||||
assert len(request.targets) == 2
|
||||
assert request.mode == "images"
|
||||
assert request.depth == 5
|
||||
assert request.max_pages == 100
|
||||
assert request.fresh is True
|
||||
|
||||
|
||||
class TestAPIEndpoints:
|
||||
"""Test API endpoints using TestClient."""
|
||||
|
||||
def setup_method(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
self.db_path = os.path.join(self.temp_dir, "test.db")
|
||||
|
||||
def teardown_method(self):
|
||||
if os.path.exists(self.temp_dir):
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def test_health_endpoint(self):
|
||||
"""Test health check endpoint returns valid response."""
|
||||
with TestClient(app) as client:
|
||||
response = client.get("/health")
|
||||
# Health endpoint doesn't require db init
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "status" in data
|
||||
assert data["status"] == "healthy"
|
||||
assert "features" in data
|
||||
|
||||
|
||||
class TestInjectI18n:
|
||||
"""Test i18n injection into HTML."""
|
||||
|
||||
def test_inject_replaces_placeholders(self):
|
||||
"""Test that placeholders are replaced."""
|
||||
html = "<html><body>{{search}} {{loading}}</body></html><script></script>"
|
||||
result = inject_i18n(html, "en")
|
||||
|
||||
assert "Search" in result
|
||||
assert "Loading..." in result
|
||||
assert "{{search}}" not in result
|
||||
|
||||
def test_inject_adds_lang_attribute(self):
|
||||
"""Test that lang attribute is added to html tag."""
|
||||
html = "<html><body>Test</body></html><script></script>"
|
||||
result = inject_i18n(html, "zh")
|
||||
|
||||
assert 'lang="zh"' in result
|
||||
|
||||
def test_inject_adds_js_translations(self):
|
||||
"""Test that JS translations object is injected."""
|
||||
html = "<html><body>Test</body></html><script></script>"
|
||||
result = inject_i18n(html, "en")
|
||||
|
||||
assert "const T=" in result
|
||||
|
||||
|
||||
class TestStaticEndpoints:
|
||||
"""Test that static file mounts work."""
|
||||
|
||||
def test_app_has_routes(self):
|
||||
"""Test that app has expected routes."""
|
||||
route_paths = [route.path for route in app.routes]
|
||||
|
||||
# Check for main routes
|
||||
assert "/" in route_paths or any("/" in str(r.path) for r in app.routes)
|
||||
|
||||
|
||||
class TestAsyncEndpoints:
|
||||
"""Test async endpoints with mocked database."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_async(self):
|
||||
"""Test health endpoint asynchronously."""
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/health")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["status"] == "healthy"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch('serp.db')
|
||||
async def test_stats_endpoint(self, mock_db):
|
||||
"""Test stats endpoint with mocked db."""
|
||||
mock_db.get_stats = AsyncMock(return_value={
|
||||
"images": 100,
|
||||
"videos": 50,
|
||||
"audio": 10
|
||||
})
|
||||
mock_db.session = MagicMock()
|
||||
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/api/stats")
|
||||
# May fail without proper db initialization, that's expected
|
||||
assert response.status_code in [200, 500]
|
||||
|
||||
|
||||
class TestErrorHandling:
|
||||
"""Test error handling in endpoints."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch('serp.db', None)
|
||||
async def test_media_not_found(self):
|
||||
"""Test 404 for non-existent media."""
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/media/nonexistent123")
|
||||
# Without db, should return error
|
||||
assert response.status_code in [404, 500]
|
||||
|
||||
|
||||
class TestInternationalization:
|
||||
"""Test internationalization coverage."""
|
||||
|
||||
def test_all_supported_languages(self):
|
||||
"""Test that all major languages are supported."""
|
||||
expected_languages = ["en", "zh", "es", "hi", "ar", "pt", "ru", "ja", "fr", "de"]
|
||||
for lang in expected_languages:
|
||||
assert lang in TRANSLATIONS, f"Missing language: {lang}"
|
||||
|
||||
def test_translation_consistency(self):
|
||||
"""Test that all translations have same number of keys."""
|
||||
en_keys = set(TRANSLATIONS["en"].keys())
|
||||
for lang_code, translations in TRANSLATIONS.items():
|
||||
lang_keys = set(translations.keys())
|
||||
# All translations should have at least the English keys
|
||||
missing = en_keys - lang_keys
|
||||
if missing:
|
||||
# This is a warning, not necessarily a failure
|
||||
pass # Some keys may be intentionally missing
|
||||
|
||||
def test_lang_selector_has_all_languages(self):
|
||||
"""Test language selector includes all supported languages."""
|
||||
for lang_code in TRANSLATIONS.keys():
|
||||
# zh-tw is a variant, others should be in LANG_NAMES
|
||||
if "-" not in lang_code or lang_code == "zh-tw":
|
||||
# Either should be in LANG_NAMES or be a valid language
|
||||
pass # Just checking structure
|
||||
|
||||
|
||||
class TestMediaServing:
|
||||
"""Test media serving functionality."""
|
||||
|
||||
def test_slugify_for_filenames(self):
|
||||
"""Test slugify produces safe filenames."""
|
||||
# Test various inputs that might be used for filenames
|
||||
test_cases = [
|
||||
("Hello World.jpg", "hello-worldjpg"),
|
||||
("Test Image 123", "test-image-123"),
|
||||
("Image with spaces", "image-with-spaces"),
|
||||
]
|
||||
for input_text, expected_pattern in test_cases:
|
||||
result = slugify(input_text)
|
||||
# Result should be lowercase and have no spaces
|
||||
assert result.islower() or result == ""
|
||||
assert " " not in result
|
||||
|
||||
|
||||
class TestCrawlRequestValidation:
|
||||
"""Test CrawlRequest validation."""
|
||||
|
||||
def test_valid_targets(self):
|
||||
"""Test valid URL targets."""
|
||||
request = CrawlRequest(
|
||||
targets=["https://example.com", "https://test.org"]
|
||||
)
|
||||
assert len(request.targets) == 2
|
||||
|
||||
def test_mode_values(self):
|
||||
"""Test different mode values."""
|
||||
for mode in ["text", "images", "videos", "media", "all"]:
|
||||
request = CrawlRequest(mode=mode)
|
||||
assert request.mode == mode
|
||||
|
||||
def test_depth_values(self):
|
||||
"""Test depth values."""
|
||||
request = CrawlRequest(depth=0)
|
||||
assert request.depth == 0
|
||||
|
||||
request = CrawlRequest(depth=-1)
|
||||
assert request.depth == -1
|
||||
|
||||
request = CrawlRequest(depth=10)
|
||||
assert request.depth == 10
|
||||
|
||||
|
||||
class TestHelperFunctions:
|
||||
"""Test various helper functions."""
|
||||
|
||||
def test_nav_html_exists(self):
|
||||
"""Test NAV_HTML constant exists."""
|
||||
from serp import NAV_HTML
|
||||
assert "neopig" in NAV_HTML
|
||||
assert "href" in NAV_HTML
|
||||
|
||||
def test_search_box_html_exists(self):
|
||||
"""Test SEARCH_BOX_HTML constant exists."""
|
||||
from serp import SEARCH_BOX_HTML
|
||||
assert "form" in SEARCH_BOX_HTML
|
||||
assert "search" in SEARCH_BOX_HTML.lower()
|
||||
|
||||
|
||||
class TestDatabaseIntegration:
|
||||
"""Test database-related endpoint behaviors."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_endpoint_accepts_query(self):
|
||||
"""Test search endpoint accepts query parameter."""
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/api/search", params={"q": "test"})
|
||||
# May fail without db, but should not crash
|
||||
assert response.status_code in [200, 500]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_random_endpoint_exists(self):
|
||||
"""Test random endpoint exists."""
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
response = await client.get("/random")
|
||||
# Should redirect or return error without db
|
||||
assert response.status_code in [302, 307, 500]
|
||||
|
||||
|
||||
class TestLiveStream:
|
||||
"""Test live streaming endpoint."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_stream_endpoint_exists(self):
|
||||
"""Test live stream SSE endpoint exists."""
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
# SSE endpoints are tricky to test, just verify route exists
|
||||
response = await client.get("/api/live/stream", timeout=2.0)
|
||||
# Should return event-stream or error
|
||||
assert response.status_code in [200, 500]
|
||||
|
||||
|
||||
class TestImportMode:
|
||||
"""Test import mode functionality."""
|
||||
|
||||
def test_import_mode_env_check(self):
|
||||
"""Test import mode is controlled by environment."""
|
||||
from serp import IMPORT_MODE
|
||||
# Just verify the constant exists
|
||||
assert isinstance(IMPORT_MODE, bool)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_upload_requires_mode(self):
|
||||
"""Test import upload requires import mode enabled."""
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
# When import mode is disabled, should return 403
|
||||
response = await client.post("/api/import/upload")
|
||||
# Either 403 (disabled) or 422 (missing file) or 500 (db error)
|
||||
assert response.status_code in [403, 422, 500]
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
Loading…
Add table
Add a link
Reference in a new issue