Russell Ballestrini
feb6eed596
- Fix asyncio error: use create_task() directly instead of BackgroundTasks - Default depth=9, max_pages=-1 for full site crawls - Add update_crawl_job_stats() for live progress updates - Poll jobs every 2s while running, show live stats - Animated progress bar for running jobs - Fix get_media_type_from_extension() None path crash
2451 lines
91 KiB
Python
2451 lines
91 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
neopig SERP - Search Engine Results Page
|
|
|
|
Fast image/video search across hydrated metadata.
|
|
Serves files directly from filevault via Caddy with 1GB memory cache.
|
|
|
|
Search across:
|
|
- keywords (crawl tags)
|
|
- alt_text (image alt attributes)
|
|
- title (media titles)
|
|
- analysis_result (Qwen 3 VL descriptions)
|
|
- source_page / source_url (origin)
|
|
|
|
Usage:
|
|
python serp.py --port 8000 --vault ./vault --db neopig.db
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import mimetypes
|
|
from pathlib import Path
|
|
from typing import List, Dict, Any, Optional
|
|
|
|
from fastapi import FastAPI, Query, HTTPException, BackgroundTasks
|
|
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response
|
|
from fastapi.staticfiles import StaticFiles
|
|
from pydantic import BaseModel
|
|
from sqlalchemy import text
|
|
import uvicorn
|
|
|
|
from database import Database
|
|
from miniuri import Uri
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
app = FastAPI(title="neopig", description="Media crawler + SERP + Screenshot service")
|
|
|
|
# Try to include uri2png screenshot router (optional dependency)
|
|
try:
|
|
from uri2png import get_screenshot_router
|
|
app.include_router(get_screenshot_router())
|
|
logger.info("Screenshot router loaded from uri2png")
|
|
except ImportError:
|
|
logger.warning("uri2png not installed - screenshot endpoints not available")
|
|
|
|
# Config - set via startup
|
|
DB_PATH = "data/neopig.db"
|
|
VAULT_PATH = Path("data/vault")
|
|
|
|
# Global database instance
|
|
db: Database = None
|
|
|
|
# Active crawl jobs (in-memory tracking)
|
|
ACTIVE_CRAWLS: Dict[int, Dict[str, Any]] = {}
|
|
|
|
# Tarball mode - serve directly from tar.gz archive
|
|
import tarfile
|
|
import tempfile
|
|
TAR_FILE: tarfile.TarFile = None
|
|
TAR_MEMBERS: Dict[str, tarfile.TarInfo] = {}
|
|
ARCHIVE_ROOT: str = None # e.g., "example.com-20251230"
|
|
TEMP_DB_PATH: str = None # Extracted database (SQLite needs real file)
|
|
|
|
|
|
# Shared CSS for view pages
|
|
VIEW_CSS = """
|
|
* { box-sizing: border-box; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0; padding: 0;
|
|
background: #0a0a0a; color: #e0e0e0;
|
|
}
|
|
.nav {
|
|
background: #1a1a1a; padding: 10px 20px;
|
|
display: flex; gap: 20px; align-items: center;
|
|
border-bottom: 1px solid #333;
|
|
}
|
|
.nav a { color: #ff6b6b; text-decoration: none; }
|
|
.nav a:hover { text-decoration: underline; }
|
|
.nav .brand { font-weight: bold; font-size: 18px; }
|
|
.container { padding: 20px; }
|
|
h1 { color: #ff6b6b; font-size: 20px; margin: 0 0 10px 0; }
|
|
h3 { color: #ff6b6b; font-size: 16px; margin: 30px 0 15px 0; border-bottom: 1px solid #333; padding-bottom: 8px; }
|
|
a { color: #ff6b6b; }
|
|
.meta { background: #1a1a1a; padding: 15px; border-radius: 8px; margin: 15px 0; }
|
|
.meta-row { display: flex; margin: 8px 0; }
|
|
.meta-label { width: 100px; color: #888; font-size: 13px; }
|
|
.meta-value { flex: 1; word-break: break-all; font-size: 13px; }
|
|
.meta-value a { color: #4ade80; }
|
|
.hero { text-align: center; margin-bottom: 20px; }
|
|
.hero img, .hero video { max-width: 100%; max-height: 60vh; border-radius: 8px; }
|
|
.content-rendered {
|
|
background: #1a1a1a; padding: 20px; border-radius: 8px;
|
|
line-height: 1.7; font-size: 14px; color: #ccc;
|
|
}
|
|
.content-rendered img { max-width: 100%; height: auto; margin: 10px 0; }
|
|
.content-rendered img.avatar {
|
|
display: inline-block; vertical-align: middle;
|
|
width: 40px; height: 40px; border-radius: 50%;
|
|
margin: 0 10px 0 0; object-fit: cover;
|
|
}
|
|
.content-rendered img.emoji,
|
|
.content-rendered img[alt^=":"][alt$=":"],
|
|
.content-rendered img[src*="emoji"] {
|
|
display: inline; width: 20px; height: 20px;
|
|
max-width: 20px; margin: 0 2px; vertical-align: text-bottom;
|
|
}
|
|
.content-rendered a { color: #ff6b6b; }
|
|
.content-rendered pre, .content-rendered code {
|
|
background: #252525; padding: 2px 6px;
|
|
border-radius: 4px; font-family: monospace; font-size: 13px;
|
|
}
|
|
.content-rendered pre {
|
|
padding: 15px; display: block;
|
|
white-space: pre-wrap; overflow-x: auto;
|
|
}
|
|
.content-rendered blockquote {
|
|
background: #151520; border-left: 3px solid #4a9eff;
|
|
padding: 12px 16px; margin: 16px 0;
|
|
border-radius: 0 6px 6px 0; color: #aaa; font-style: italic;
|
|
}
|
|
.content-rendered hr { border: none; border-top: 1px solid #333; margin: 24px 0; }
|
|
.content-rendered p { margin: 0 0 16px 0; line-height: 1.7; }
|
|
.content-rendered h1, .content-rendered h2, .content-rendered h3 {
|
|
color: #ff6b6b; margin-top: 28px; margin-bottom: 12px;
|
|
}
|
|
.media-grid {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fill, minmax(150px, 1fr));
|
|
gap: 10px; margin-top: 15px;
|
|
}
|
|
.media-card {
|
|
background: #1a1a1a; border-radius: 8px;
|
|
overflow: hidden; display: block; transition: transform 0.2s;
|
|
}
|
|
.media-card:hover { transform: scale(1.02); }
|
|
.media-card img, .media-card video {
|
|
width: 100%; height: 120px;
|
|
object-fit: contain; background: #0a0a0a;
|
|
}
|
|
.tag { background: #333; padding: 2px 8px; border-radius: 4px; font-size: 12px; margin-right: 5px; }
|
|
"""
|
|
|
|
|
|
def render_detail_page(
|
|
title: str,
|
|
hero_html: str,
|
|
meta_rows: list,
|
|
media_items: list,
|
|
content_html: str,
|
|
screenshot_hashes: list,
|
|
source_domain: str,
|
|
page_uri: str,
|
|
download_btn_html: str = "",
|
|
noai: bool = False,
|
|
) -> str:
|
|
"""
|
|
Shared template for media view and page view.
|
|
|
|
Args:
|
|
title: Page title
|
|
hero_html: HTML for hero section (media element or empty)
|
|
meta_rows: List of (label, value_html) tuples for metadata
|
|
media_items: List of media dicts with md5_hash, media_type
|
|
content_html: Rendered markdown/content HTML
|
|
screenshot_hashes: List of screenshot md5 hashes
|
|
source_domain: Domain for AI prompt
|
|
page_uri: Page URI for AI prompt
|
|
download_btn_html: Optional download button HTML
|
|
noai: Disable AI assistant
|
|
"""
|
|
import html as html_module
|
|
|
|
# Build metadata section
|
|
meta_html = ''.join(
|
|
f'<div class="meta-row"><span class="meta-label">{label}:</span><span class="meta-value">{value}</span></div>'
|
|
for label, value in meta_rows
|
|
)
|
|
|
|
# Build media gallery
|
|
media_grid = ""
|
|
if media_items:
|
|
media_cards = []
|
|
for m in media_items:
|
|
is_video = m.get("media_type") == "video"
|
|
if is_video:
|
|
el = f'<video src="/media/{m["md5_hash"]}" muted loop preload="metadata" onmouseenter="this.play()" onmouseleave="this.pause()"></video>'
|
|
else:
|
|
el = f'<img src="/media/{m["md5_hash"]}" loading="lazy">'
|
|
media_cards.append(f'''<a href="/view/{m["md5_hash"]}" class="media-card">{el}</a>''')
|
|
media_grid = f'''
|
|
<div class="page-media">
|
|
<h3>Media ({len(media_items)})</h3>
|
|
<div class="media-grid">{''.join(media_cards)}</div>
|
|
</div>'''
|
|
|
|
escaped_title = html_module.escape(title)
|
|
escaped_uri = html_module.escape(page_uri or "")
|
|
|
|
return f"""<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>{escaped_title} - neopig</title>
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css">
|
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
|
|
<style>
|
|
{VIEW_CSS}
|
|
.screenshot-stack {{ border-radius: 8px; }}
|
|
.screenshot-stack img {{ border-radius: 0; }}
|
|
.screenshot-stack img:first-child {{ border-radius: 8px 8px 0 0; }}
|
|
.screenshot-stack img:last-child {{ border-radius: 0 0 8px 8px; }}
|
|
.screenshot-stack img:only-child {{ border-radius: 8px; }}
|
|
#screenshot-container.collapsed {{ display: none; }}
|
|
.hero-grid {{ display: grid; grid-template-columns: 1fr; gap: 20px; margin-bottom: 20px; }}
|
|
.hero-grid.has-hero {{ grid-template-columns: 1fr 1fr; }}
|
|
.content-grid {{ display: grid; grid-template-columns: 1fr; gap: 20px; margin-top: 20px; }}
|
|
.content-grid.has-screenshots {{ grid-template-columns: 1fr 1fr; }}
|
|
.content-grid.ss-collapsed {{ grid-template-columns: 1fr 20px; }}
|
|
.content-grid.ss-collapsed .screenshot-col h3 {{ writing-mode: vertical-rl; text-orientation: mixed; margin: 0; font-size: 12px; }}
|
|
.content-rendered img.emoji {{ display: inline; width: 20px; height: 20px; margin: 0 2px; vertical-align: text-bottom; }}
|
|
.content-rendered .post-block {{ display: grid; grid-template-columns: 48px 1fr; gap: 12px; padding: 16px 0; border-bottom: 1px solid #252530; }}
|
|
.content-rendered .post-block:last-child {{ border-bottom: none; }}
|
|
.content-rendered .post-avatar-col {{ display: flex; flex-direction: column; align-items: center; }}
|
|
.content-rendered .post-avatar {{ width: 40px; height: 40px; border-radius: 50%; object-fit: cover; }}
|
|
.content-rendered .post-avatar-placeholder {{ width: 40px; height: 40px; border-radius: 50%; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: #fff; display: flex; align-items: center; justify-content: center; font-weight: bold; font-size: 18px; }}
|
|
.content-rendered .post-content {{ min-width: 0; }}
|
|
.content-rendered pre {{ position: relative; }}
|
|
.content-rendered pre .code-actions {{ position: absolute; top: 8px; right: 8px; display: flex; gap: 5px; }}
|
|
.content-rendered pre .code-actions button {{ background: #444; border: none; color: #ccc; padding: 4px 8px; border-radius: 4px; cursor: pointer; font-size: 11px; }}
|
|
.content-rendered pre .code-actions button:hover {{ background: #555; }}
|
|
.content-rendered pre .code-actions button.copied {{ background: #4ade80; color: #000; }}
|
|
@media (max-width: 768px) {{ .hero-grid, .hero-grid.has-hero, .content-grid, .content-grid.has-screenshots {{ grid-template-columns: 1fr; }} }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="nav">
|
|
<a href="/" class="brand">🐷 neopig</a>
|
|
<a href="/">Search</a>
|
|
<a href="/live">Live</a>
|
|
<a href="/random">Random</a>
|
|
<a href="/crawl">Crawl</a>
|
|
<a href="/phantom">Phantom</a>
|
|
</div>
|
|
<div class="container">
|
|
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
|
<input type="text" name="q" placeholder="Search..." style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;">
|
|
<button type="submit" style="padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">Search</button>
|
|
</form>
|
|
<div class="hero-grid{' has-hero' if hero_html else ''}">
|
|
{f'<div class="hero-left"><div style="background:#111;border-radius:12px;padding:20px;text-align:center;">{hero_html}</div></div>' if hero_html else ''}
|
|
<div class="hero-right" style="background:#151515;border-radius:12px;padding:20px;">
|
|
<h1 style="margin:0 0 15px 0;font-size:1.3em;">{escaped_title}</h1>
|
|
<div class="meta">{meta_html}</div>
|
|
{download_btn_html}
|
|
</div>
|
|
</div>
|
|
{media_grid}
|
|
<div id="content-grid" class="content-grid{' has-screenshots' if screenshot_hashes else ''}">
|
|
<div class="content-col">
|
|
{f'<h3>Content</h3><div class="content-rendered">{content_html}</div>' if content_html else ''}
|
|
</div>
|
|
{f'''<div class="screenshot-col">
|
|
<h3 style="cursor:pointer;" onclick="toggleScreenshot()">
|
|
<span id="ss-toggle">▼</span> Screenshot
|
|
</h3>
|
|
<div id="screenshot-container" class="screenshot-stack">
|
|
{''.join(f'<img src="/media/{h}" alt="Screenshot" style="width:100%;display:block;">' for h in screenshot_hashes)}
|
|
</div>
|
|
</div>''' if screenshot_hashes else ''}
|
|
</div>
|
|
<script>
|
|
hljs.highlightAll();
|
|
|
|
// Screenshot toggle
|
|
function toggleScreenshot() {{
|
|
const container = document.getElementById('screenshot-container');
|
|
const toggle = document.getElementById('ss-toggle');
|
|
const grid = document.getElementById('content-grid');
|
|
if (container) {{
|
|
const collapsed = !container.classList.contains('collapsed');
|
|
container.classList.toggle('collapsed', collapsed);
|
|
grid.classList.toggle('ss-collapsed', collapsed);
|
|
toggle.textContent = collapsed ? '▶' : '▼';
|
|
localStorage.setItem('neopig_ss_collapsed', collapsed);
|
|
}}
|
|
}}
|
|
if (localStorage.getItem('neopig_ss_collapsed') === 'true') {{
|
|
const container = document.getElementById('screenshot-container');
|
|
const toggle = document.getElementById('ss-toggle');
|
|
const grid = document.getElementById('content-grid');
|
|
if (container) {{
|
|
container.classList.add('collapsed');
|
|
grid.classList.add('ss-collapsed');
|
|
toggle.textContent = '▶';
|
|
}}
|
|
}}
|
|
|
|
// Auto-detect emojis/avatars
|
|
document.querySelectorAll('.content-rendered img').forEach(img => {{
|
|
const check = () => {{
|
|
const w = img.naturalWidth || img.width, h = img.naturalHeight || img.height;
|
|
if (w > 0 && h > 0) {{
|
|
if (w <= 24 && h <= 24) img.classList.add('emoji');
|
|
else if (w <= 60 && h <= 60) img.classList.add('avatar');
|
|
}}
|
|
}};
|
|
if (img.complete) check(); else img.onload = check;
|
|
}});
|
|
|
|
// Code block copy/download
|
|
const extMap = {{'python':'py','javascript':'js','typescript':'ts','cpp':'cpp','c':'c','java':'java','rust':'rs','go':'go','ruby':'rb','php':'php','html':'html','css':'css','json':'json','yaml':'yaml','sql':'sql','bash':'sh','sh':'sh','markdown':'md'}};
|
|
document.querySelectorAll('.content-rendered pre').forEach((pre, idx) => {{
|
|
const code = pre.querySelector('code') || pre;
|
|
const text = code.textContent;
|
|
let ext = 'txt';
|
|
(code.className || '').split(/\\s+/).forEach(cls => {{
|
|
const m = cls.match(/^(?:language-)?(.+)$/);
|
|
if (m && extMap[m[1].toLowerCase()]) ext = extMap[m[1].toLowerCase()];
|
|
}});
|
|
const actions = document.createElement('div');
|
|
actions.className = 'code-actions';
|
|
const copyBtn = document.createElement('button');
|
|
copyBtn.textContent = 'Copy';
|
|
copyBtn.onclick = async () => {{
|
|
await navigator.clipboard.writeText(text);
|
|
copyBtn.textContent = 'Copied!'; copyBtn.classList.add('copied');
|
|
setTimeout(() => {{ copyBtn.textContent = 'Copy'; copyBtn.classList.remove('copied'); }}, 2000);
|
|
}};
|
|
const dlBtn = document.createElement('button');
|
|
dlBtn.textContent = 'Download';
|
|
dlBtn.onclick = () => {{
|
|
const blob = new Blob([text], {{type: 'text/plain'}});
|
|
const a = document.createElement('a');
|
|
a.href = URL.createObjectURL(blob);
|
|
a.download = `code-${{idx + 1}}.${{ext}}`;
|
|
a.click();
|
|
}};
|
|
actions.appendChild(copyBtn); actions.appendChild(dlBtn);
|
|
pre.appendChild(actions);
|
|
}});
|
|
|
|
// Forum post restructuring
|
|
(function() {{
|
|
const container = document.querySelector('.content-rendered');
|
|
if (!container) return;
|
|
const postStarts = [];
|
|
container.querySelectorAll('p').forEach(p => {{
|
|
const img = p.querySelector('img[alt="avatar"]');
|
|
const strong = p.querySelector('strong');
|
|
if (img && strong) postStarts.push({{p, img, username: strong.textContent}});
|
|
}});
|
|
if (!postStarts.length) return;
|
|
postStarts.forEach(({{p, img, username}}, idx) => {{
|
|
if (p.closest('.post-block')) return;
|
|
const block = document.createElement('div');
|
|
block.className = 'post-block';
|
|
const avatarCol = document.createElement('div');
|
|
avatarCol.className = 'post-avatar-col';
|
|
if (img.src && !img.src.includes('undefined')) {{
|
|
const avatar = document.createElement('img');
|
|
avatar.className = 'post-avatar';
|
|
avatar.src = img.src;
|
|
avatarCol.appendChild(avatar);
|
|
}} else {{
|
|
const ph = document.createElement('div');
|
|
ph.className = 'post-avatar-placeholder';
|
|
ph.textContent = username.charAt(0).toUpperCase();
|
|
avatarCol.appendChild(ph);
|
|
}}
|
|
const contentCol = document.createElement('div');
|
|
contentCol.className = 'post-content';
|
|
contentCol.innerHTML = `<div class="post-header"><strong>${{username}}</strong></div>`;
|
|
let sib = p.nextElementSibling;
|
|
const nextP = idx < postStarts.length - 1 ? postStarts[idx + 1].p : null;
|
|
while (sib && sib !== nextP && sib.tagName !== 'HR') {{
|
|
contentCol.appendChild(sib.cloneNode(true));
|
|
const rm = sib; sib = sib.nextElementSibling; rm.remove();
|
|
}}
|
|
if (sib && sib.tagName === 'HR') sib.remove();
|
|
block.appendChild(avatarCol); block.appendChild(contentCol);
|
|
p.parentNode.insertBefore(block, p); p.remove();
|
|
}});
|
|
}})();
|
|
</script>
|
|
{'' if noai else f'''<script>
|
|
window.UNCLOSEAI_SYSTEM_PROMPT = "Archived copy of {source_domain} by neopig. Viewing: {escaped_title} ({escaped_uri})";
|
|
</script>
|
|
<script src="https://uncloseai.com/uncloseai.js" type="module"></script>'''}
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
def read_from_tarball(path: str) -> bytes:
|
|
"""Read a file from the tarball. Path is relative to archive root."""
|
|
if not TAR_FILE or not ARCHIVE_ROOT:
|
|
return None
|
|
full_path = f"{ARCHIVE_ROOT}/{path}"
|
|
if full_path in TAR_MEMBERS:
|
|
member = TAR_MEMBERS[full_path]
|
|
f = TAR_FILE.extractfile(member)
|
|
if f:
|
|
return f.read()
|
|
return None
|
|
|
|
|
|
def find_media_in_tarball(md5_hash: str) -> tuple:
|
|
"""Find media file in tarball by hash. Returns (data, extension) or (None, None)."""
|
|
if not TAR_FILE or not ARCHIVE_ROOT:
|
|
return None, None
|
|
prefix = f"{ARCHIVE_ROOT}/media/{md5_hash}"
|
|
for name, member in TAR_MEMBERS.items():
|
|
if name.startswith(prefix):
|
|
f = TAR_FILE.extractfile(member)
|
|
if f:
|
|
ext = Path(name).suffix
|
|
return f.read(), ext
|
|
return None, None
|
|
|
|
|
|
class ArchiveDB:
|
|
"""Simple sync SQLite wrapper for archive.db (FTS5 search only)."""
|
|
def __init__(self, db_path):
|
|
import sqlite3
|
|
self.conn = sqlite3.connect(db_path)
|
|
self.conn.row_factory = sqlite3.Row
|
|
|
|
async def search_pages(self, query, limit=50):
|
|
cursor = self.conn.execute(
|
|
"SELECT uri, title, snippet(pages_fts, 2, '<b>', '</b>', '...', 32) as snippet "
|
|
"FROM pages_fts WHERE pages_fts MATCH ? LIMIT ?",
|
|
(query, limit)
|
|
)
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
async def get_stats(self):
|
|
cursor = self.conn.execute("SELECT COUNT(*) FROM pages")
|
|
return {"pages": cursor.fetchone()[0], "media": 0, "screenshots": 0}
|
|
|
|
|
|
@app.on_event("startup")
|
|
async def startup_event():
|
|
"""Initialize database on startup."""
|
|
global db
|
|
if TAR_FILE:
|
|
# Tarball mode: use simple archive DB
|
|
db = ArchiveDB(DB_PATH)
|
|
logger.info(f"Using archive database: {DB_PATH}")
|
|
else:
|
|
# Normal mode: use full async database
|
|
db = Database(DB_PATH)
|
|
await db.init() # Handles schema + WAL mode
|
|
# Backfill uri_hash for existing pages
|
|
count = await db.backfill_page_hashes()
|
|
if count:
|
|
logger.info(f"Backfilled {count} page URI hashes")
|
|
VAULT_PATH.mkdir(parents=True, exist_ok=True)
|
|
logger.info(f"Vault directory ready: {VAULT_PATH}")
|
|
|
|
|
|
class CrawlRequest(BaseModel):
|
|
"""Request to start a new crawl."""
|
|
targets: List[str] = [] # Multiple target URIs
|
|
target_uri: str = "" # Deprecated: single target (for backwards compat)
|
|
keywords: List[str] = []
|
|
mode: str = "images" # text, images, videos, media, all
|
|
depth: int = -1 # -1 = unlimited
|
|
max_pages: int = -1 # -1 = unlimited
|
|
download_media: bool = True
|
|
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
async def index():
|
|
"""Simple search UI."""
|
|
return SEARCH_HTML
|
|
|
|
|
|
@app.get("/crawl", response_class=HTMLResponse)
|
|
async def crawl_page():
|
|
"""Crawler command page."""
|
|
return CRAWL_HTML
|
|
|
|
|
|
# HTML Templates
|
|
SEARCH_HTML = """
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>neopig SERP</title>
|
|
<style>
|
|
* { box-sizing: border-box; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0;
|
|
padding: 0;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}
|
|
.nav {
|
|
background: #1a1a1a;
|
|
padding: 10px 20px;
|
|
display: flex;
|
|
gap: 20px;
|
|
align-items: center;
|
|
border-bottom: 1px solid #333;
|
|
}
|
|
.nav a { color: #ff6b6b; text-decoration: none; }
|
|
.nav a:hover { text-decoration: underline; }
|
|
.nav .brand { font-weight: bold; font-size: 18px; }
|
|
.container { padding: 20px; }
|
|
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
|
.subtitle { color: #666; margin-bottom: 20px; }
|
|
.search-box {
|
|
display: flex;
|
|
gap: 10px;
|
|
margin-bottom: 20px;
|
|
}
|
|
input[type="text"] {
|
|
flex: 1;
|
|
padding: 12px 16px;
|
|
font-size: 16px;
|
|
border: 2px solid #333;
|
|
border-radius: 8px;
|
|
background: #1a1a1a;
|
|
color: #fff;
|
|
}
|
|
input[type="text"]:focus {
|
|
outline: none;
|
|
border-color: #ff6b6b;
|
|
}
|
|
select {
|
|
padding: 12px 16px;
|
|
font-size: 16px;
|
|
border: 2px solid #333;
|
|
border-radius: 8px;
|
|
background: #1a1a1a;
|
|
color: #fff;
|
|
}
|
|
button {
|
|
padding: 12px 24px;
|
|
font-size: 16px;
|
|
background: #ff6b6b;
|
|
color: #fff;
|
|
border: none;
|
|
border-radius: 8px;
|
|
cursor: pointer;
|
|
}
|
|
button:hover { background: #ff5252; }
|
|
.stats {
|
|
padding: 10px 15px;
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
margin-bottom: 20px;
|
|
font-size: 14px;
|
|
color: #888;
|
|
}
|
|
.section-title {
|
|
color: #ff6b6b;
|
|
font-size: 18px;
|
|
margin: 25px 0 15px 0;
|
|
border-bottom: 1px solid #333;
|
|
padding-bottom: 8px;
|
|
}
|
|
.results {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
|
|
gap: 15px;
|
|
}
|
|
.result {
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
overflow: hidden;
|
|
transition: transform 0.2s;
|
|
}
|
|
.result:hover {
|
|
transform: scale(1.02);
|
|
}
|
|
.result img, .result video {
|
|
width: 100%;
|
|
height: 200px;
|
|
object-fit: contain;
|
|
background: #1a1a1a;
|
|
}
|
|
.result-info {
|
|
padding: 10px;
|
|
}
|
|
.result-hash {
|
|
font-family: monospace;
|
|
font-size: 11px;
|
|
color: #666;
|
|
word-break: break-all;
|
|
}
|
|
.result-meta {
|
|
font-size: 12px;
|
|
color: #888;
|
|
margin-top: 5px;
|
|
}
|
|
.result-keywords {
|
|
display: flex;
|
|
flex-wrap: wrap;
|
|
gap: 5px;
|
|
margin-top: 8px;
|
|
}
|
|
.tag {
|
|
background: #333;
|
|
padding: 2px 8px;
|
|
border-radius: 4px;
|
|
font-size: 11px;
|
|
color: #aaa;
|
|
}
|
|
.no-results {
|
|
text-align: center;
|
|
padding: 60px;
|
|
color: #666;
|
|
}
|
|
.media-link {
|
|
display: block;
|
|
cursor: pointer;
|
|
}
|
|
.media-link:hover img, .media-link:hover video {
|
|
opacity: 0.8;
|
|
}
|
|
.result-hash {
|
|
color: #ff6b6b;
|
|
text-decoration: none;
|
|
}
|
|
.result-hash:hover {
|
|
text-decoration: underline;
|
|
}
|
|
/* Page results */
|
|
.page-results {
|
|
display: flex;
|
|
flex-direction: column;
|
|
gap: 10px;
|
|
}
|
|
.page-result {
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
padding: 15px;
|
|
transition: background 0.2s;
|
|
}
|
|
.page-result:hover {
|
|
background: #252525;
|
|
}
|
|
.page-result a {
|
|
color: #ff6b6b;
|
|
text-decoration: none;
|
|
font-size: 16px;
|
|
font-weight: 500;
|
|
}
|
|
.page-result a:hover {
|
|
text-decoration: underline;
|
|
}
|
|
.page-path {
|
|
font-size: 12px;
|
|
color: #4ade80;
|
|
margin-top: 4px;
|
|
font-family: monospace;
|
|
}
|
|
.page-snippet {
|
|
font-size: 13px;
|
|
color: #999;
|
|
margin-top: 8px;
|
|
line-height: 1.5;
|
|
}
|
|
.page-snippet mark {
|
|
background: #ff6b6b33;
|
|
color: #ff9999;
|
|
padding: 1px 3px;
|
|
border-radius: 2px;
|
|
}
|
|
/* Two-column layout: pages left, media right */
|
|
.search-columns {
|
|
display: grid;
|
|
grid-template-columns: 1fr;
|
|
gap: 30px;
|
|
align-items: start;
|
|
}
|
|
.search-columns.has-pages {
|
|
grid-template-columns: 1fr 2fr;
|
|
}
|
|
.search-columns .page-column {
|
|
display: none;
|
|
}
|
|
.search-columns.has-pages .page-column {
|
|
display: block;
|
|
position: sticky;
|
|
top: 20px;
|
|
max-height: 85vh;
|
|
overflow-y: auto;
|
|
}
|
|
@media (max-width: 1000px) {
|
|
.search-columns.has-pages {
|
|
grid-template-columns: 1fr;
|
|
}
|
|
.search-columns.has-pages .page-column {
|
|
position: static;
|
|
max-height: none;
|
|
}
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="nav">
|
|
<a href="/" class="brand">🐷 neopig</a>
|
|
<a href="/">Search</a>
|
|
<a href="/live">Live</a>
|
|
<a href="/random">Random</a>
|
|
<a href="/crawl">Crawl</a>
|
|
<a href="/phantom">Phantom</a>
|
|
</div>
|
|
<div class="container">
|
|
|
|
<div class="search-box">
|
|
<input type="text" id="query" placeholder="Search keywords, alt text, page content..." autofocus>
|
|
<select id="type">
|
|
<option value="">All types</option>
|
|
<option value="image">Images</option>
|
|
<option value="video">Videos</option>
|
|
<option value="audio">Audio</option>
|
|
</select>
|
|
<button onclick="search()">Search</button>
|
|
</div>
|
|
|
|
<div class="stats" id="stats">Loading stats...</div>
|
|
|
|
<div class="search-columns">
|
|
<div class="page-column" id="page-section">
|
|
<h2 class="section-title">Pages</h2>
|
|
<div class="page-results" id="page-results"></div>
|
|
</div>
|
|
|
|
<div class="media-column" id="media-section">
|
|
<h2 class="section-title">Media</h2>
|
|
<div class="results" id="results"></div>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
async function loadStats() {
|
|
const res = await fetch('/api/stats');
|
|
const stats = await res.json();
|
|
document.getElementById('stats').innerHTML =
|
|
`<strong>${stats.total_media}</strong> media | ` +
|
|
`<strong>${stats.by_type?.image || 0}</strong> images | ` +
|
|
`<strong>${stats.by_type?.video || 0}</strong> videos | ` +
|
|
`<strong>${stats.total_sources}</strong> sources | ` +
|
|
`<strong>${stats.total_pages || 0}</strong> pages`;
|
|
}
|
|
|
|
async function search() {
|
|
const query = document.getElementById('query').value;
|
|
const type = document.getElementById('type').value;
|
|
|
|
// Search media
|
|
let mediaUrl = `/api/search?q=${encodeURIComponent(query)}&limit=100`;
|
|
if (type) mediaUrl += `&type=${type}`;
|
|
|
|
const mediaRes = await fetch(mediaUrl);
|
|
const mediaResults = await mediaRes.json();
|
|
|
|
const mediaContainer = document.getElementById('results');
|
|
const mediaSection = document.getElementById('media-section');
|
|
|
|
if (mediaResults.length === 0) {
|
|
mediaContainer.innerHTML = '<div class="no-results">No media found</div>';
|
|
} else {
|
|
mediaContainer.innerHTML = mediaResults.map(r => {
|
|
const isVideo = r.media_type === 'video';
|
|
const mediaEl = isVideo
|
|
? `<video src="/media/${r.md5_hash}" muted loop preload="metadata" onmouseenter="this.play()" onmouseleave="this.pause()"></video>`
|
|
: `<img src="/media/${r.md5_hash}" alt="${r.alt_text || ''}" loading="lazy">`;
|
|
|
|
const keywords = JSON.parse(r.keywords || '[]');
|
|
const tagsHtml = keywords.map(k => `<span class="tag">${k}</span>`).join('');
|
|
|
|
return `
|
|
<div class="result">
|
|
<a href="/view/${r.md5_hash}" class="media-link">
|
|
${mediaEl}
|
|
</a>
|
|
<div class="result-info">
|
|
<a href="/view/${r.md5_hash}" class="result-hash">${r.md5_hash}</a>
|
|
<div class="result-meta">
|
|
${r.media_type} · ${formatBytes(r.file_size)}
|
|
${r.alt_text ? ` · ${r.alt_text.substring(0, 50)}` : ''}
|
|
</div>
|
|
<div class="result-keywords">${tagsHtml}</div>
|
|
</div>
|
|
</div>
|
|
`;
|
|
}).join('');
|
|
}
|
|
|
|
// Search pages
|
|
const pageContainer = document.getElementById('page-results');
|
|
const searchColumns = document.querySelector('.search-columns');
|
|
|
|
if (query.trim()) {
|
|
const pageRes = await fetch(`/api/search/pages?q=${encodeURIComponent(query)}&limit=30`);
|
|
const pageResults = await pageRes.json();
|
|
|
|
if (pageResults.length > 0) {
|
|
searchColumns.classList.add('has-pages');
|
|
pageContainer.innerHTML = pageResults.map(p => `
|
|
<div class="page-result">
|
|
<a href="/page/${p.uri_hash}">${p.title || p.uri}</a>
|
|
<div class="page-path">${p.path || p.uri}</div>
|
|
<div class="page-snippet">${p.snippet || ''}</div>
|
|
</div>
|
|
`).join('');
|
|
} else {
|
|
searchColumns.classList.remove('has-pages');
|
|
pageContainer.innerHTML = '';
|
|
}
|
|
} else {
|
|
searchColumns.classList.remove('has-pages');
|
|
pageContainer.innerHTML = '';
|
|
}
|
|
}
|
|
|
|
function formatBytes(bytes) {
|
|
if (!bytes) return '?';
|
|
if (bytes < 1024) return bytes + ' B';
|
|
if (bytes < 1024*1024) return (bytes/1024).toFixed(1) + ' KB';
|
|
return (bytes/1024/1024).toFixed(1) + ' MB';
|
|
}
|
|
|
|
// Enter key to search
|
|
document.getElementById('query').addEventListener('keypress', e => {
|
|
if (e.key === 'Enter') search();
|
|
});
|
|
|
|
// Load stats on page load
|
|
loadStats();
|
|
|
|
// Check for query param from nav search
|
|
const urlParams = new URLSearchParams(window.location.search);
|
|
const q = urlParams.get('q');
|
|
if (q) {
|
|
document.getElementById('query').value = q;
|
|
}
|
|
|
|
// Initial search (show all media or query)
|
|
search();
|
|
</script>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
CRAWL_HTML = """
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>neopig Crawler</title>
|
|
<style>
|
|
* { box-sizing: border-box; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0;
|
|
padding: 0;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}
|
|
.nav {
|
|
background: #1a1a1a;
|
|
padding: 10px 20px;
|
|
display: flex;
|
|
gap: 20px;
|
|
align-items: center;
|
|
border-bottom: 1px solid #333;
|
|
}
|
|
.nav a { color: #ff6b6b; text-decoration: none; }
|
|
.nav a:hover { text-decoration: underline; }
|
|
.nav .brand { font-weight: bold; font-size: 18px; }
|
|
.container { padding: 20px; }
|
|
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
|
.subtitle { color: #666; margin-bottom: 20px; }
|
|
|
|
.form-group {
|
|
margin-bottom: 15px;
|
|
}
|
|
label {
|
|
display: block;
|
|
margin-bottom: 5px;
|
|
color: #aaa;
|
|
font-size: 14px;
|
|
}
|
|
input[type="text"], input[type="number"], select {
|
|
width: 100%;
|
|
padding: 12px 16px;
|
|
font-size: 16px;
|
|
border: 2px solid #333;
|
|
border-radius: 8px;
|
|
background: #1a1a1a;
|
|
color: #fff;
|
|
}
|
|
input:focus, select:focus {
|
|
outline: none;
|
|
border-color: #ff6b6b;
|
|
}
|
|
.row {
|
|
display: flex;
|
|
gap: 15px;
|
|
}
|
|
.row > div { flex: 1; }
|
|
|
|
button {
|
|
padding: 14px 28px;
|
|
font-size: 16px;
|
|
background: #ff6b6b;
|
|
color: #fff;
|
|
border: none;
|
|
border-radius: 8px;
|
|
cursor: pointer;
|
|
margin-top: 10px;
|
|
}
|
|
button:hover { background: #ff5252; }
|
|
button:disabled {
|
|
background: #444;
|
|
cursor: not-allowed;
|
|
}
|
|
button.secondary {
|
|
background: #333;
|
|
}
|
|
button.secondary:hover {
|
|
background: #444;
|
|
}
|
|
|
|
.checkbox-group {
|
|
display: flex;
|
|
align-items: center;
|
|
gap: 8px;
|
|
}
|
|
.checkbox-group input {
|
|
width: auto;
|
|
}
|
|
|
|
.jobs-section {
|
|
margin-top: 30px;
|
|
padding-top: 20px;
|
|
border-top: 1px solid #333;
|
|
}
|
|
h2 {
|
|
color: #ff6b6b;
|
|
font-size: 18px;
|
|
margin-bottom: 15px;
|
|
}
|
|
|
|
.job {
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
padding: 15px;
|
|
margin-bottom: 10px;
|
|
}
|
|
.job-header {
|
|
display: flex;
|
|
justify-content: space-between;
|
|
align-items: center;
|
|
margin-bottom: 10px;
|
|
}
|
|
.job-id {
|
|
font-family: monospace;
|
|
color: #666;
|
|
}
|
|
.job-status {
|
|
padding: 4px 10px;
|
|
border-radius: 4px;
|
|
font-size: 12px;
|
|
font-weight: bold;
|
|
}
|
|
.job-status.running { background: #2d5a27; color: #7bed72; }
|
|
.job-status.completed { background: #1a3a4a; color: #6bc5e8; }
|
|
.job-status.failed { background: #5a2727; color: #ed7272; }
|
|
|
|
.job-target {
|
|
font-size: 14px;
|
|
word-break: break-all;
|
|
margin-bottom: 5px;
|
|
}
|
|
.job-meta {
|
|
font-size: 12px;
|
|
color: #666;
|
|
}
|
|
.job-stats {
|
|
display: flex;
|
|
gap: 15px;
|
|
margin-top: 10px;
|
|
font-size: 13px;
|
|
}
|
|
.job-stats span {
|
|
background: #252525;
|
|
padding: 4px 10px;
|
|
border-radius: 4px;
|
|
}
|
|
|
|
.progress-bar {
|
|
height: 4px;
|
|
background: #333;
|
|
border-radius: 2px;
|
|
margin-top: 10px;
|
|
overflow: hidden;
|
|
}
|
|
.progress-bar-fill {
|
|
height: 100%;
|
|
background: #ff6b6b;
|
|
transition: width 0.3s;
|
|
}
|
|
.progress-bar-fill.running {
|
|
width: 100%;
|
|
background: linear-gradient(90deg, #ff6b6b 0%, #4ecdc4 50%, #ff6b6b 100%);
|
|
background-size: 200% 100%;
|
|
animation: progress-wave 1.5s ease-in-out infinite;
|
|
}
|
|
@keyframes progress-wave {
|
|
0% { background-position: 200% 0; }
|
|
100% { background-position: -200% 0; }
|
|
}
|
|
.job.running {
|
|
border-color: #4ecdc4;
|
|
}
|
|
.job-stats.live {
|
|
color: #4ecdc4;
|
|
}
|
|
|
|
.no-jobs {
|
|
color: #666;
|
|
text-align: center;
|
|
padding: 30px;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="nav">
|
|
<a href="/" class="brand">🐷 neopig</a>
|
|
<a href="/">Search</a>
|
|
<a href="/live">Live</a>
|
|
<a href="/random">Random</a>
|
|
<a href="/crawl">Crawl</a>
|
|
<a href="/phantom">Phantom</a>
|
|
</div>
|
|
<div class="container">
|
|
|
|
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
|
<input type="text" name="q" placeholder="Search keywords, alt text, page content..." style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<option value="">All types</option>
|
|
<option value="image">Images</option>
|
|
<option value="video">Videos</option>
|
|
<option value="audio">Audio</option>
|
|
</select>
|
|
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">Search</button>
|
|
</form>
|
|
|
|
<h1>Crawler</h1>
|
|
<p class="subtitle">Hydrate media from the web</p>
|
|
|
|
<form id="crawl-form" onsubmit="startCrawl(event)">
|
|
<div class="form-group">
|
|
<label>Target URIs (space, comma, or newline separated)</label>
|
|
<textarea id="target" placeholder="https://example.com https://another.com" rows="2" required style="width:100%;padding:10px;border:1px solid #333;border-radius:4px;background:#1a1a1a;color:#e0e0e0;font-size:14px;resize:vertical;"></textarea>
|
|
</div>
|
|
|
|
<div class="form-group">
|
|
<label>Keywords (space or comma separated)</label>
|
|
<input type="text" id="keywords" placeholder="rick and morty, adult swim">
|
|
</div>
|
|
|
|
<div class="row">
|
|
<div class="form-group">
|
|
<label>Mode</label>
|
|
<select id="mode">
|
|
<option value="images">Images only</option>
|
|
<option value="videos">Videos only</option>
|
|
<option value="media">All media (images + videos + audio)</option>
|
|
<option value="all">Everything (text + media)</option>
|
|
<option value="text">Text only</option>
|
|
</select>
|
|
</div>
|
|
<div class="form-group">
|
|
<label>Depth (9 = full site, max 15)</label>
|
|
<input type="number" id="depth" value="9" min="1" max="15">
|
|
</div>
|
|
<div class="form-group">
|
|
<label>Max Pages (-1 = unlimited)</label>
|
|
<input type="number" id="max_pages" value="-1" min="-1">
|
|
</div>
|
|
</div>
|
|
|
|
<div class="form-group checkbox-group">
|
|
<input type="checkbox" id="download" checked>
|
|
<label for="download" style="display:inline; margin:0;">Download media (uncheck to just index URLs)</label>
|
|
</div>
|
|
|
|
<button type="submit" id="start-btn">Start Crawl</button>
|
|
</form>
|
|
|
|
<div class="jobs-section">
|
|
<h2>Crawl Jobs</h2>
|
|
<div id="jobs">Loading...</div>
|
|
</div>
|
|
|
|
<script>
|
|
async function startCrawl(e) {
|
|
e.preventDefault();
|
|
|
|
const btn = document.getElementById('start-btn');
|
|
btn.disabled = true;
|
|
btn.textContent = 'Starting...';
|
|
|
|
const keywordsRaw = document.getElementById('keywords').value;
|
|
const keywords = keywordsRaw
|
|
.split(/[,\\s]+/)
|
|
.map(k => k.trim())
|
|
.filter(k => k.length > 0);
|
|
|
|
// Parse multiple target URIs (space, comma, or newline separated)
|
|
const targetsRaw = document.getElementById('target').value;
|
|
const targets = targetsRaw
|
|
.split(/[,\\s\\n]+/)
|
|
.map(t => t.trim())
|
|
.filter(t => t.length > 0 && t.startsWith('http'));
|
|
|
|
if (targets.length === 0) {
|
|
alert('Please enter at least one valid URI (must start with http)');
|
|
btn.disabled = false;
|
|
btn.textContent = 'Start Crawl';
|
|
return;
|
|
}
|
|
|
|
const payload = {
|
|
targets: targets,
|
|
keywords: keywords,
|
|
mode: document.getElementById('mode').value,
|
|
depth: parseInt(document.getElementById('depth').value),
|
|
max_pages: parseInt(document.getElementById('max_pages').value),
|
|
download_media: document.getElementById('download').checked
|
|
};
|
|
|
|
try {
|
|
const res = await fetch('/api/crawl', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(payload)
|
|
});
|
|
|
|
if (!res.ok) {
|
|
const err = await res.json();
|
|
alert('Error: ' + (err.detail || 'Failed to start crawl'));
|
|
} else {
|
|
const result = await res.json();
|
|
const jobIds = result.job_ids || [result.job_id];
|
|
alert('Crawl started! Job IDs: ' + jobIds.join(', '));
|
|
loadJobs();
|
|
}
|
|
} catch (err) {
|
|
alert('Error: ' + err.message);
|
|
}
|
|
|
|
btn.disabled = false;
|
|
btn.textContent = 'Start Crawl';
|
|
}
|
|
|
|
async function loadJobs() {
|
|
try {
|
|
const res = await fetch('/api/crawl/jobs');
|
|
const jobs = await res.json();
|
|
|
|
const container = document.getElementById('jobs');
|
|
|
|
if (jobs.length === 0) {
|
|
container.innerHTML = '<div class="no-jobs">No crawl jobs yet</div>';
|
|
return;
|
|
}
|
|
|
|
const hasRunning = jobs.some(j => j.status === 'running');
|
|
|
|
container.innerHTML = jobs.map(job => {
|
|
const stats = job.stats ? JSON.parse(job.stats) : {};
|
|
const keywords = job.keywords ? JSON.parse(job.keywords) : [];
|
|
|
|
// Show live stats for running jobs
|
|
const liveStats = job.status === 'running' ? `
|
|
<div class="job-stats live">
|
|
<span>📄 ${stats.pages_crawled || 0} pages</span>
|
|
<span>🖼️ ${stats.media_found || 0} found</span>
|
|
<span>💾 ${stats.media_downloaded || 0} saved</span>
|
|
<span>📸 ${stats.screenshots_taken || 0} screenshots</span>
|
|
</div>
|
|
` : '';
|
|
|
|
return `
|
|
<div class="job ${job.status}">
|
|
<div class="job-header">
|
|
<span class="job-id">Job #${job.id}</span>
|
|
<span class="job-status ${job.status}">${job.status}</span>
|
|
</div>
|
|
<div class="job-target">${job.target_uri}</div>
|
|
<div class="job-meta">
|
|
Mode: ${job.mode || 'images'} |
|
|
Keywords: ${keywords.join(', ') || 'none'} |
|
|
Started: ${new Date(job.started_at).toLocaleString()}
|
|
</div>
|
|
${job.status === 'completed' ? `
|
|
<div class="job-stats">
|
|
<span>📄 ${stats.pages_crawled || 0} pages</span>
|
|
<span>🖼️ ${stats.media_found || 0} found</span>
|
|
<span>💾 ${stats.media_downloaded || 0} saved</span>
|
|
<span>♻️ ${stats.duplicates_skipped || 0} dupes</span>
|
|
</div>
|
|
` : ''}
|
|
${liveStats}
|
|
${job.status === 'running' ? `
|
|
<div class="progress-bar">
|
|
<div class="progress-bar-fill running"></div>
|
|
</div>
|
|
` : ''}
|
|
</div>
|
|
`;
|
|
}).join('');
|
|
|
|
// Poll faster while jobs are running
|
|
if (hasRunning) {
|
|
setTimeout(loadJobs, 2000);
|
|
}
|
|
} catch (err) {
|
|
document.getElementById('jobs').innerHTML = '<div class="no-jobs">Failed to load jobs</div>';
|
|
}
|
|
}
|
|
|
|
// Load jobs on page load
|
|
loadJobs();
|
|
|
|
// Refresh jobs every 5 seconds
|
|
setInterval(loadJobs, 5000);
|
|
</script>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
LIVE_HTML = """
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>neopig LIVE - Watch Images Crawl In</title>
|
|
<style>
|
|
* { box-sizing: border-box; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0;
|
|
padding: 0;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}
|
|
.nav {
|
|
background: #1a1a1a;
|
|
padding: 10px 20px;
|
|
display: flex;
|
|
gap: 20px;
|
|
align-items: center;
|
|
border-bottom: 1px solid #333;
|
|
}
|
|
.nav a { color: #ff6b6b; text-decoration: none; }
|
|
.nav a:hover { text-decoration: underline; }
|
|
.nav .brand { font-weight: bold; font-size: 18px; }
|
|
.container { padding: 20px; }
|
|
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
|
.subtitle { color: #666; margin-bottom: 10px; }
|
|
.stats {
|
|
background: #1a1a1a;
|
|
padding: 15px;
|
|
border-radius: 8px;
|
|
margin-bottom: 20px;
|
|
display: flex;
|
|
gap: 30px;
|
|
flex-wrap: wrap;
|
|
}
|
|
.stat { display: flex; flex-direction: column; }
|
|
.stat-value { font-size: 24px; font-weight: bold; color: #ff6b6b; }
|
|
.stat-label { font-size: 12px; color: #888; }
|
|
.controls {
|
|
margin-bottom: 20px;
|
|
display: flex;
|
|
gap: 10px;
|
|
align-items: center;
|
|
}
|
|
.controls button {
|
|
padding: 8px 16px;
|
|
border: none;
|
|
border-radius: 4px;
|
|
cursor: pointer;
|
|
font-size: 14px;
|
|
}
|
|
.controls button.primary { background: #ff6b6b; color: white; }
|
|
.controls button.secondary { background: #333; color: #ddd; }
|
|
.controls button:hover { opacity: 0.8; }
|
|
.status { color: #4ade80; font-size: 14px; }
|
|
.status.paused { color: #fbbf24; }
|
|
.live-grid {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
|
|
gap: 15px;
|
|
}
|
|
.live-card {
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
overflow: hidden;
|
|
animation: fadeIn 0.5s ease-out;
|
|
}
|
|
@keyframes fadeIn {
|
|
from { opacity: 0; transform: translateY(20px); }
|
|
to { opacity: 1; transform: translateY(0); }
|
|
}
|
|
.live-card.new {
|
|
box-shadow: 0 0 20px rgba(255, 107, 107, 0.5);
|
|
}
|
|
.live-card img, .live-card video {
|
|
width: 100%;
|
|
height: 180px;
|
|
object-fit: contain;
|
|
background: #222;
|
|
}
|
|
.live-card-info {
|
|
padding: 10px;
|
|
}
|
|
.live-card-title {
|
|
font-size: 12px;
|
|
color: #888;
|
|
overflow: hidden;
|
|
text-overflow: ellipsis;
|
|
white-space: nowrap;
|
|
}
|
|
.live-card-source {
|
|
font-size: 10px;
|
|
color: #555;
|
|
margin-top: 4px;
|
|
overflow: hidden;
|
|
text-overflow: ellipsis;
|
|
white-space: nowrap;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="nav">
|
|
<a href="/" class="brand">🐷 neopig</a>
|
|
<a href="/">Search</a>
|
|
<a href="/live">Live</a>
|
|
<a href="/random">Random</a>
|
|
<a href="/crawl">Crawl</a>
|
|
<a href="/phantom">Phantom</a>
|
|
</div>
|
|
<div class="container">
|
|
|
|
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
|
<input type="text" name="q" placeholder="Search keywords, alt text, page content..." style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<option value="">All types</option>
|
|
<option value="image">Images</option>
|
|
<option value="video">Videos</option>
|
|
<option value="audio">Audio</option>
|
|
</select>
|
|
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">Search</button>
|
|
</form>
|
|
|
|
<h1>Live Feed</h1>
|
|
<p class="subtitle">Watch images appear as they're crawled</p>
|
|
|
|
<div class="stats">
|
|
<div class="stat">
|
|
<span class="stat-value" id="total-count">0</span>
|
|
<span class="stat-label">Total Images</span>
|
|
</div>
|
|
<div class="stat">
|
|
<span class="stat-value" id="new-count">0</span>
|
|
<span class="stat-label">New This Session</span>
|
|
</div>
|
|
<div class="stat">
|
|
<span class="stat-value" id="rate">0</span>
|
|
<span class="stat-label">Per Minute</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="controls">
|
|
<button class="primary" id="toggle-btn" onclick="toggleFeed()">Pause</button>
|
|
<button class="secondary" onclick="clearFeed()">Clear</button>
|
|
<span class="status" id="status">Watching for new images...</span>
|
|
</div>
|
|
|
|
<div class="live-grid" id="grid"></div>
|
|
|
|
<script>
|
|
let running = true;
|
|
let lastCheck = new Date().toISOString();
|
|
let seenHashes = new Set();
|
|
let newCount = 0;
|
|
let startTime = Date.now();
|
|
|
|
function toggleFeed() {
|
|
running = !running;
|
|
const btn = document.getElementById('toggle-btn');
|
|
const status = document.getElementById('status');
|
|
if (running) {
|
|
btn.textContent = 'Pause';
|
|
status.textContent = 'Watching for new images...';
|
|
status.className = 'status';
|
|
poll();
|
|
} else {
|
|
btn.textContent = 'Resume';
|
|
status.textContent = 'Paused';
|
|
status.className = 'status paused';
|
|
}
|
|
}
|
|
|
|
function clearFeed() {
|
|
document.getElementById('grid').innerHTML = '';
|
|
seenHashes.clear();
|
|
newCount = 0;
|
|
startTime = Date.now();
|
|
updateStats(0);
|
|
}
|
|
|
|
function updateStats(total) {
|
|
document.getElementById('total-count').textContent = total;
|
|
document.getElementById('new-count').textContent = newCount;
|
|
|
|
const minutes = (Date.now() - startTime) / 60000;
|
|
const rate = minutes > 0 ? Math.round(newCount / minutes) : 0;
|
|
document.getElementById('rate').textContent = rate;
|
|
}
|
|
|
|
async function poll() {
|
|
if (!running) return;
|
|
|
|
try {
|
|
// Get recent media sorted by first_seen_at descending
|
|
const res = await fetch('/api/search?q=&limit=50');
|
|
const media = await res.json();
|
|
|
|
// Get stats
|
|
const statsRes = await fetch('/api/stats');
|
|
const stats = await statsRes.json();
|
|
updateStats(stats.total_media || 0);
|
|
|
|
const grid = document.getElementById('grid');
|
|
|
|
// Find new items (on first poll, show all; after that only new ones)
|
|
const isFirstPoll = seenHashes.size === 0;
|
|
const newItems = media.filter(m => !seenHashes.has(m.md5_hash));
|
|
|
|
// Add new items to the top (or all items on first load)
|
|
newItems.reverse().forEach(item => {
|
|
seenHashes.add(item.md5_hash);
|
|
newCount++;
|
|
|
|
const card = document.createElement('div');
|
|
card.className = 'live-card new';
|
|
|
|
const isVideo = item.media_type === 'video';
|
|
const mediaEl = isVideo
|
|
? `<video src="/media/${item.md5_hash}" muted loop onmouseenter="this.play()" onmouseleave="this.pause()"></video>`
|
|
: `<img src="/media/${item.md5_hash}" loading="lazy" onerror="this.src='data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 width=%22200%22 height=%22180%22><rect fill=%22%23333%22 width=%22200%22 height=%22180%22/><text x=%2250%%22 y=%2250%%22 fill=%22%23666%22 text-anchor=%22middle%22>Error</text></svg>'">`;
|
|
|
|
card.innerHTML = `
|
|
<a href="/view/${item.md5_hash}">
|
|
${mediaEl}
|
|
</a>
|
|
<div class="live-card-info">
|
|
<div class="live-card-title">${item.alt_text || item.title || item.md5_hash.slice(0,12)}</div>
|
|
<div class="live-card-source">${item.media_type} - ${formatSize(item.file_size)}</div>
|
|
</div>
|
|
`;
|
|
|
|
grid.insertBefore(card, grid.firstChild);
|
|
|
|
// Remove 'new' highlight after animation
|
|
setTimeout(() => card.classList.remove('new'), 2000);
|
|
});
|
|
|
|
// Update stats
|
|
updateStats(stats.total_media || 0);
|
|
|
|
} catch (err) {
|
|
console.error('Poll error:', err);
|
|
}
|
|
|
|
// Poll again
|
|
setTimeout(poll, 2000);
|
|
}
|
|
|
|
function formatSize(bytes) {
|
|
if (!bytes) return '?';
|
|
if (bytes < 1024) return bytes + ' B';
|
|
if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB';
|
|
return (bytes / (1024 * 1024)).toFixed(1) + ' MB';
|
|
}
|
|
|
|
// Start polling
|
|
poll();
|
|
</script>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
@app.get("/live", response_class=HTMLResponse)
|
|
async def live_page():
|
|
"""Live feed page - watch images appear as they're crawled."""
|
|
return LIVE_HTML
|
|
|
|
|
|
@app.get("/view/{md5_hash}", response_class=HTMLResponse)
|
|
async def view_media_page(md5_hash: str, noai: bool = Query(False)):
|
|
"""Detail view page for a single media item."""
|
|
import html as html_module
|
|
import re
|
|
from urllib.parse import quote, urljoin
|
|
|
|
async with db.session() as session:
|
|
result = await session.execute(
|
|
text("SELECT * FROM media WHERE md5_hash = :hash"),
|
|
{'hash': md5_hash}
|
|
)
|
|
media = result.fetchone()
|
|
if not media:
|
|
raise HTTPException(status_code=404, detail="Media not found")
|
|
|
|
result = await session.execute(
|
|
text("""SELECT media_uri, page_uri, page_title, page_content,
|
|
detail_page_uri, detail_title, detail_content, discovered_at
|
|
FROM media_sources WHERE md5_hash = :hash"""),
|
|
{'hash': md5_hash}
|
|
)
|
|
sources = result.fetchall()
|
|
|
|
media = dict(media._mapping)
|
|
sources = [dict(s._mapping) for s in sources]
|
|
keywords = json.loads(media.get('keywords') or '[]')
|
|
sorted_sources = sorted(sources, key=lambda s: len(s["page_uri"] or ""), reverse=True)
|
|
|
|
# Get page URI
|
|
page_uri = sorted_sources[0]["page_uri"] if sorted_sources else None
|
|
source_uri = page_uri or (sorted_sources[0]["media_uri"] if sorted_sources else "unknown")
|
|
source_domain = Uri(source_uri).hostname if source_uri else "unknown"
|
|
|
|
# Display title
|
|
display_title = media.get('alt_text') or media.get('title')
|
|
if not display_title and sources:
|
|
display_title = sources[0].get('page_title')
|
|
if not display_title:
|
|
display_title = f"Media {md5_hash[:12]}"
|
|
|
|
# Hero: media element
|
|
is_video = media['media_type'] == 'video'
|
|
is_audio = media['media_type'] == 'audio'
|
|
if is_video:
|
|
hero_html = f'<a href="/media/{md5_hash}" target="_blank"><video src="/media/{md5_hash}" controls muted loop style="max-width:100%;max-height:70vh;"></video></a>'
|
|
elif is_audio:
|
|
hero_html = f'<audio src="/media/{md5_hash}" controls></audio>'
|
|
else:
|
|
hero_html = f'<a href="/media/{md5_hash}" target="_blank"><img src="/media/{md5_hash}" alt="{html_module.escape(media.get("alt_text") or "")}" style="max-width:100%;max-height:70vh;"></a>'
|
|
|
|
# Metadata rows
|
|
import hashlib
|
|
media_uri = sorted_sources[0]["media_uri"] if sorted_sources else None
|
|
neopig_media_uri = f"/view/{md5_hash}"
|
|
page_uri_hash = hashlib.md5(page_uri.encode()).hexdigest() if page_uri else None
|
|
neopig_page_uri = f"/page/{page_uri_hash}" if page_uri_hash else None
|
|
keywords_html = ''.join([f'<span class="tag">{k}</span>' for k in keywords]) or '-'
|
|
meta_rows = [
|
|
("Source URI", f'<a href="{media_uri}" target="_blank" style="font-size:11px;">{media_uri}</a>' if media_uri else '-'),
|
|
("Neopig Media URI", f'<a href="{neopig_media_uri}" style="font-size:11px;">{neopig_media_uri}</a>'),
|
|
("Source Page URI", f'<a href="{page_uri}" target="_blank" style="font-size:11px;">{page_uri}</a>' if page_uri else '-'),
|
|
("Neopig Page URI", f'<a href="{neopig_page_uri}" style="font-size:11px;">{neopig_page_uri}</a>' if neopig_page_uri else '-'),
|
|
("MD5", f'<code style="font-size:11px;">{md5_hash}</code>'),
|
|
("Type", media['media_type']),
|
|
("MIME", media.get('mime_type') or 'unknown'),
|
|
("Size", f"{media.get('file_size') or 0:,} bytes"),
|
|
("Alt", media.get('alt_text') or '-'),
|
|
("Keywords", keywords_html),
|
|
]
|
|
|
|
# Download button
|
|
name_source = media.get('alt_text') or media.get('title')
|
|
if not name_source and sources:
|
|
pt = sources[0].get('page_title', '')
|
|
media_idx = int(md5_hash[:4], 16)
|
|
name_source = f"{pt}-{media_idx}" if pt else f"media-{media_idx}"
|
|
download_name = slugify(name_source or f"media-{md5_hash[:8]}")
|
|
ext_map = {'image/jpeg': '.jpg', 'image/png': '.png', 'image/gif': '.gif', 'image/webp': '.webp',
|
|
'video/mp4': '.mp4', 'video/webm': '.webm', 'audio/mpeg': '.mp3', 'audio/wav': '.wav'}
|
|
ext = ext_map.get(media.get('mime_type', ''), '.bin')
|
|
download_btn = f'<a href="/media/{md5_hash}?download=1" style="display:block;padding:12px 20px;background:#4a9eff;color:#fff;text-decoration:none;border-radius:6px;text-align:center;font-weight:500;margin-top:15px;">Download ({download_name}{ext})</a>'
|
|
|
|
# Get page media items (siblings)
|
|
media_items = await db.get_page_media(page_uri) if page_uri else []
|
|
# Exclude current item from gallery
|
|
media_items = [m for m in media_items if m.get("md5_hash") != md5_hash]
|
|
|
|
# Get rendered content
|
|
content_html = ""
|
|
if page_uri:
|
|
page_row = await db.get_page_by_uri(page_uri)
|
|
if page_row and page_row.get("markdown"):
|
|
try:
|
|
import markdown
|
|
md_converter = markdown.Markdown(extensions=['fenced_code', 'tables', 'nl2br'])
|
|
content_html = md_converter.convert(page_row["markdown"][:100000])
|
|
# Hydrate images and links
|
|
img_urls = re.findall(r'<img[^>]+src=["\']([^"\']+)["\']', content_html, re.I)
|
|
link_urls = re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', content_html, re.I)
|
|
all_urls = list(set(img_urls + link_urls))
|
|
if all_urls:
|
|
resolved = {u: u if u.startswith(('http://', 'https://', '//')) else urljoin(page_uri, u) for u in all_urls}
|
|
url_to_hash = await db.lookup_media_by_uris(list(set(resolved.values())))
|
|
# Fallback for imgur and other CDNs - lookup by filename/ID
|
|
for orig, res in resolved.items():
|
|
if res not in url_to_hash:
|
|
from pathlib import Path as P
|
|
fname = P(res.split('?')[0]).stem # Remove query params, get stem
|
|
if fname and len(fname) >= 5:
|
|
result = await db.lookup_media_by_filename(fname)
|
|
if result:
|
|
url_to_hash[res] = result[1]
|
|
for orig, res in resolved.items():
|
|
if res in url_to_hash:
|
|
md5 = url_to_hash[res]
|
|
content_html = content_html.replace(f'src="{orig}"', f'src="/media/{md5}"')
|
|
content_html = content_html.replace(f"src='{orig}'", f'src="/media/{md5}"')
|
|
content_html = content_html.replace(f'href="{orig}"', f'href="/view/{md5}"')
|
|
content_html = content_html.replace(f"href='{orig}'", f'href="/view/{md5}"')
|
|
except ImportError:
|
|
content_html = f"<pre>{html_module.escape(page_row['markdown'][:50000])}</pre>"
|
|
|
|
# Get screenshots (exclude current if it's a screenshot)
|
|
screenshot_hashes = await db.get_page_screenshots(page_uri) if page_uri else []
|
|
screenshot_hashes = [h for h in screenshot_hashes if h != md5_hash]
|
|
|
|
return render_detail_page(
|
|
title=display_title,
|
|
hero_html=hero_html,
|
|
meta_rows=meta_rows,
|
|
media_items=media_items,
|
|
content_html=content_html,
|
|
screenshot_hashes=screenshot_hashes,
|
|
source_domain=source_domain,
|
|
page_uri=page_uri or "",
|
|
download_btn_html=download_btn,
|
|
noai=noai,
|
|
)
|
|
|
|
|
|
|
|
|
|
@app.get("/page/{uri_hash}", response_class=HTMLResponse)
|
|
async def view_page_by_hash(
|
|
uri_hash: str,
|
|
noai: bool = Query(False, description="Disable AI assistant")
|
|
):
|
|
"""View an archived page by URI hash."""
|
|
page = await db.get_page_by_hash(uri_hash)
|
|
if not page:
|
|
raise HTTPException(status_code=404, detail="Page not found")
|
|
# Redirect to the URI-based view (reuses same logic)
|
|
return await view_page(uri=page['uri'], noai=noai)
|
|
|
|
|
|
@app.get("/page/view", response_class=HTMLResponse)
|
|
async def view_page(
|
|
uri: str = Query(..., description="Page URI to view"),
|
|
noai: bool = Query(False, description="Disable AI assistant")
|
|
):
|
|
"""View an archived page with markdown and screenshot."""
|
|
import html as html_module
|
|
import re
|
|
from urllib.parse import urljoin, quote
|
|
|
|
source_domain = Uri(uri).hostname
|
|
|
|
page = await db.get_page_by_uri(uri)
|
|
if not page:
|
|
raise HTTPException(status_code=404, detail="Page not found")
|
|
|
|
page_title = page.get("title") or uri
|
|
|
|
# Render markdown
|
|
content_html = ""
|
|
if page.get("markdown"):
|
|
try:
|
|
import markdown
|
|
md_converter = markdown.Markdown(extensions=['fenced_code', 'tables', 'nl2br'])
|
|
content_html = md_converter.convert(page["markdown"][:100000])
|
|
|
|
# Hydrate images and links from vault
|
|
img_urls = re.findall(r'<img[^>]+src=["\']([^"\']+)["\']', content_html, re.I)
|
|
link_urls = re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', content_html, re.I)
|
|
all_urls = list(set(img_urls + link_urls))
|
|
if all_urls:
|
|
resolved = {u: u if u.startswith(('http://', 'https://', '//')) else urljoin(uri, u) for u in all_urls}
|
|
url_to_hash = await db.lookup_media_by_uris(list(set(resolved.values())))
|
|
# Fallback for imgur and other CDNs - lookup by filename/ID
|
|
for murl in [u for u in resolved.values() if u not in url_to_hash]:
|
|
from pathlib import Path as P
|
|
fname = P(murl.split('?')[0]).stem # Remove query params, get stem
|
|
if fname and len(fname) >= 5:
|
|
result = await db.lookup_media_by_filename(fname)
|
|
if result:
|
|
url_to_hash[murl] = result[1]
|
|
for orig, res in resolved.items():
|
|
if res in url_to_hash:
|
|
md5 = url_to_hash[res]
|
|
# Rewrite img src to serve media directly
|
|
content_html = content_html.replace(f'src="{orig}"', f'src="/media/{md5}"')
|
|
content_html = content_html.replace(f"src='{orig}'", f'src="/media/{md5}"')
|
|
# Rewrite a href to neopig media view page
|
|
content_html = content_html.replace(f'href="{orig}"', f'href="/view/{md5}"')
|
|
content_html = content_html.replace(f"href='{orig}'", f'href="/view/{md5}"')
|
|
except ImportError:
|
|
content_html = f"<pre>{html_module.escape(page.get('markdown', '')[:50000])}</pre>"
|
|
elif page.get("content"):
|
|
escaped = html_module.escape(page["content"][:50000])
|
|
content_html = f"<pre style='white-space:pre-wrap;'>{escaped}</pre>"
|
|
|
|
# Get screenshots and media
|
|
screenshot_hashes = await db.get_page_screenshots(uri)
|
|
media_items = await db.get_page_media(uri)
|
|
|
|
# Metadata rows for page view
|
|
import hashlib
|
|
uri_hash = hashlib.md5(uri.encode()).hexdigest()
|
|
neopig_page_uri = f"/page/{uri_hash}"
|
|
keywords = json.loads(page.get('keywords') or '[]') if page.get('keywords') else []
|
|
keywords_html = ''.join([f'<span class="tag">{k}</span>' for k in keywords]) or '-'
|
|
meta_rows = [
|
|
("Source URI", f'<a href="{uri}" target="_blank" style="font-size:11px;">{uri}</a>'),
|
|
("Neopig URI", f'<a href="{neopig_page_uri}" style="font-size:11px;">{neopig_page_uri}</a>'),
|
|
("Description", page.get('description') or '-'),
|
|
("Keywords", keywords_html),
|
|
("Media", f"{len(media_items)} items"),
|
|
]
|
|
|
|
return render_detail_page(
|
|
title=page_title,
|
|
hero_html="", # No hero media for page view
|
|
meta_rows=meta_rows,
|
|
media_items=media_items,
|
|
content_html=content_html,
|
|
screenshot_hashes=screenshot_hashes,
|
|
source_domain=source_domain,
|
|
page_uri=uri,
|
|
noai=noai,
|
|
)
|
|
@app.get("/phantom/export")
|
|
async def phantom_export(domain: str = Query(None, description="Filter by domain")):
|
|
"""
|
|
Export a phantom HTML site - original HTML with media URLs rewritten to vault.
|
|
|
|
Creates a downloadable zip of the phantom site ready for static hosting.
|
|
"""
|
|
import io
|
|
import re
|
|
import zipfile
|
|
from urllib.parse import urljoin
|
|
from fastapi.responses import StreamingResponse
|
|
|
|
# Get all pages with raw_html
|
|
pages = await db.get_pages_by_domain(domain)
|
|
|
|
if not pages:
|
|
raise HTTPException(status_code=404, detail="No pages with raw HTML found")
|
|
|
|
# Get all media URL to hash mappings
|
|
url_to_hash = await db.get_all_media_uri_mappings()
|
|
|
|
# Create zip in memory
|
|
zip_buffer = io.BytesIO()
|
|
|
|
with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
|
|
pages_written = 0
|
|
media_hashes = set()
|
|
|
|
for page in pages:
|
|
uri = page['uri']
|
|
raw_html = page.get('raw_html')
|
|
if not raw_html:
|
|
continue
|
|
|
|
# Parse URI to get path
|
|
parsed = Uri(uri)
|
|
site_domain = parsed.hostname
|
|
path = parsed.path.strip('/') or 'index'
|
|
if not path.endswith('.html') and '.' not in path.split('/')[-1]:
|
|
path = f"{path}/index.html" if path else "index.html"
|
|
|
|
# Rewrite media URLs to local paths
|
|
html = raw_html
|
|
|
|
# Find all src and href attributes pointing to media
|
|
patterns = [
|
|
(r'src=["\']([^"\']+)["\']', 'src'),
|
|
(r'href=["\']([^"\']+\.(jpg|jpeg|png|gif|webp|mp4|webm|svg|ico))["\']', 'href'),
|
|
]
|
|
|
|
for pattern, attr in patterns:
|
|
matches = re.findall(pattern, html, re.IGNORECASE)
|
|
for match in matches:
|
|
url = match[0] if isinstance(match, tuple) else match
|
|
|
|
# Resolve relative URLs
|
|
full_url = urljoin(uri, url)
|
|
|
|
# Check if we have this media
|
|
if full_url in url_to_hash:
|
|
md5 = url_to_hash[full_url]
|
|
media_hashes.add(md5)
|
|
# Replace with local path
|
|
ext = Path(url).suffix or '.bin'
|
|
local_path = f"media/{md5}{ext}"
|
|
html = html.replace(f'{attr}="{url}"', f'{attr}="{local_path}"')
|
|
html = html.replace(f"{attr}='{url}'", f'{attr}="{local_path}"')
|
|
elif url in url_to_hash:
|
|
md5 = url_to_hash[url]
|
|
media_hashes.add(md5)
|
|
ext = Path(url).suffix or '.bin'
|
|
local_path = f"media/{md5}{ext}"
|
|
html = html.replace(f'{attr}="{url}"', f'{attr}="{local_path}"')
|
|
html = html.replace(f"{attr}='{url}'", f'{attr}="{local_path}"')
|
|
|
|
# Write HTML file
|
|
zf.writestr(f"site/{path}", html.encode('utf-8'))
|
|
pages_written += 1
|
|
|
|
# Copy media files from vault
|
|
media_copied = 0
|
|
for md5 in media_hashes:
|
|
subdir = VAULT_PATH / md5[:2]
|
|
if subdir.exists():
|
|
for f in subdir.iterdir():
|
|
if f.name.startswith(md5):
|
|
ext = f.suffix or '.bin'
|
|
zf.write(f, f"site/media/{md5}{ext}")
|
|
media_copied += 1
|
|
break
|
|
|
|
# Write index
|
|
index_html = f"""<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>Phantom Site - {site_domain}</title>
|
|
<style>
|
|
body {{ font-family: sans-serif; max-width: 800px; margin: 50px auto; padding: 20px; }}
|
|
h1 {{ color: #333; }}
|
|
ul {{ line-height: 2; }}
|
|
a {{ color: #0066cc; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>Phantom Site Archive</h1>
|
|
<p>Domain: {site_domain}</p>
|
|
<p>Pages: {pages_written}</p>
|
|
<p>Media: {media_copied}</p>
|
|
<h2>Pages</h2>
|
|
<ul>
|
|
"""
|
|
for page in pages[:100]:
|
|
parsed = Uri(page['uri'])
|
|
path = parsed.path.strip('/') or 'index'
|
|
if not path.endswith('.html') and '.' not in path.split('/')[-1]:
|
|
path = f"{path}/index.html" if path else "index.html"
|
|
title = page.get('title') or path
|
|
index_html += f' <li><a href="{path}">{title}</a></li>\n'
|
|
|
|
index_html += """ </ul>
|
|
</body>
|
|
</html>"""
|
|
zf.writestr("site/phantom_index.html", index_html.encode('utf-8'))
|
|
|
|
# Return zip
|
|
zip_buffer.seek(0)
|
|
return StreamingResponse(
|
|
zip_buffer,
|
|
media_type="application/zip",
|
|
headers={"Content-Disposition": f"attachment; filename=phantom_{site_domain or 'site'}.zip"}
|
|
)
|
|
|
|
|
|
@app.get("/phantom", response_class=HTMLResponse)
|
|
async def phantom_page():
|
|
"""Phantom site export UI."""
|
|
domains = await db.get_domains_with_pages()
|
|
|
|
domain_options = ''.join([
|
|
f'<option value="{d[0]}">{d[0]} ({d[1]} pages)</option>'
|
|
for d in domains if d[0]
|
|
])
|
|
|
|
return f"""
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>Phantom Site Export - neopig</title>
|
|
<style>
|
|
* {{ box-sizing: border-box; }}
|
|
body {{
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0;
|
|
padding: 0;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}}
|
|
.nav {{
|
|
background: #1a1a1a;
|
|
padding: 10px 20px;
|
|
display: flex;
|
|
gap: 20px;
|
|
align-items: center;
|
|
border-bottom: 1px solid #333;
|
|
}}
|
|
.nav a {{ color: #ff6b6b; text-decoration: none; }}
|
|
.nav a:hover {{ text-decoration: underline; }}
|
|
.nav .brand {{ font-weight: bold; font-size: 18px; }}
|
|
.container {{ padding: 20px; }}
|
|
h1 {{ color: #ff6b6b; margin-bottom: 10px; }}
|
|
.subtitle {{ color: #888; margin-bottom: 30px; }}
|
|
.form-group {{ margin-bottom: 20px; }}
|
|
label {{ display: block; margin-bottom: 8px; color: #aaa; }}
|
|
select {{
|
|
width: 100%;
|
|
padding: 12px 16px;
|
|
font-size: 16px;
|
|
border: 2px solid #333;
|
|
border-radius: 8px;
|
|
background: #1a1a1a;
|
|
color: #fff;
|
|
}}
|
|
button {{
|
|
padding: 14px 28px;
|
|
font-size: 16px;
|
|
background: #ff6b6b;
|
|
color: #fff;
|
|
border: none;
|
|
border-radius: 8px;
|
|
cursor: pointer;
|
|
}}
|
|
button:hover {{ background: #ff5252; }}
|
|
.info {{
|
|
background: #1a1a1a;
|
|
padding: 20px;
|
|
border-radius: 8px;
|
|
margin-top: 30px;
|
|
}}
|
|
.info h3 {{ color: #ff6b6b; margin-top: 0; }}
|
|
.info ul {{ color: #aaa; line-height: 1.8; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="nav">
|
|
<a href="/" class="brand">🐷 neopig</a>
|
|
<a href="/">Search</a>
|
|
<a href="/live">Live</a>
|
|
<a href="/random">Random</a>
|
|
<a href="/crawl">Crawl</a>
|
|
<a href="/phantom">Phantom</a>
|
|
</div>
|
|
<div class="container">
|
|
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
|
<input type="text" name="q" placeholder="Search keywords, alt text, page content..." style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<option value="">All types</option>
|
|
<option value="image">Images</option>
|
|
<option value="video">Videos</option>
|
|
<option value="audio">Audio</option>
|
|
</select>
|
|
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">Search</button>
|
|
</form>
|
|
<h1>Phantom Site Export</h1>
|
|
<p class="subtitle">Export archived pages as a static site with local media</p>
|
|
|
|
<form action="/phantom/export" method="get">
|
|
<div class="form-group">
|
|
<label>Select Domain</label>
|
|
<select name="domain">
|
|
<option value="">All domains</option>
|
|
{domain_options}
|
|
</select>
|
|
</div>
|
|
<button type="submit">Download Phantom Site (.zip)</button>
|
|
</form>
|
|
|
|
<div class="info">
|
|
<h3>What is a Phantom Site?</h3>
|
|
<ul>
|
|
<li>Original HTML preserved exactly as crawled</li>
|
|
<li>All media URLs rewritten to local paths</li>
|
|
<li>Ready to host statically (nginx, Caddy, S3, etc.)</li>
|
|
<li>Works offline - all assets included</li>
|
|
<li>Perfect for archival and preservation</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
@app.get("/health")
|
|
async def health():
|
|
"""Health check endpoint."""
|
|
has_screenshot = False
|
|
try:
|
|
from uri2png import get_available_engines
|
|
has_screenshot = True
|
|
except ImportError:
|
|
pass
|
|
|
|
return {
|
|
"status": "healthy",
|
|
"features": {
|
|
"search": True,
|
|
"crawl": True,
|
|
"screenshot": has_screenshot
|
|
}
|
|
}
|
|
|
|
|
|
@app.get("/api/stats")
|
|
async def get_stats_endpoint():
|
|
"""Get database statistics."""
|
|
stats = await db.get_stats()
|
|
# Add page count (handled separately since table may not exist)
|
|
try:
|
|
from sqlalchemy import select, func
|
|
from database import Page
|
|
async with db.session() as session:
|
|
result = await session.execute(select(func.count()).select_from(Page))
|
|
stats['total_pages'] = result.scalar() or 0
|
|
except Exception:
|
|
stats['total_pages'] = 0
|
|
return stats
|
|
|
|
|
|
@app.get("/random")
|
|
async def random_item(type: str = Query(None, description="Type: media or page (random if not specified)")):
|
|
"""Redirect to a random media item or page."""
|
|
from sqlalchemy import select, func
|
|
from database import Media, Page
|
|
import random
|
|
|
|
# If no type specified, randomly pick between media and page
|
|
if type is None:
|
|
type = random.choice(["media", "page"])
|
|
|
|
async with db.session() as session:
|
|
if type == "page":
|
|
# Get a random page
|
|
stmt = select(Page.uri_hash).order_by(func.random()).limit(1)
|
|
result = await session.execute(stmt)
|
|
row = result.fetchone()
|
|
if row and row[0]:
|
|
return RedirectResponse(url=f"/page/{row[0]}", status_code=302)
|
|
else:
|
|
# Get a random media item (excluding screenshots)
|
|
stmt = (
|
|
select(Media.md5_hash)
|
|
.where(Media.media_type != 'screenshot')
|
|
.order_by(func.random())
|
|
.limit(1)
|
|
)
|
|
result = await session.execute(stmt)
|
|
row = result.fetchone()
|
|
if row:
|
|
return RedirectResponse(url=f"/view/{row[0]}", status_code=302)
|
|
|
|
return RedirectResponse(url="/", status_code=302)
|
|
|
|
|
|
@app.get("/api/search")
|
|
async def search(
|
|
q: str = Query("", description="Search query"),
|
|
type: Optional[str] = Query(None, description="Filter by media type"),
|
|
status: Optional[str] = Query(None, description="Filter by analysis status"),
|
|
limit: int = Query(100, le=1000),
|
|
offset: int = Query(0)
|
|
):
|
|
"""
|
|
Search media by text query.
|
|
|
|
Searches across: keywords, alt_text, title, source URLs, analysis results.
|
|
"""
|
|
results = await db.search_media_advanced(
|
|
q=q if q else None,
|
|
media_type=type,
|
|
limit=limit,
|
|
offset=offset
|
|
)
|
|
return results
|
|
|
|
|
|
@app.get("/api/search/pages")
|
|
async def search_pages_endpoint(
|
|
q: str = Query("", description="Search query"),
|
|
limit: int = Query(50, le=500),
|
|
):
|
|
"""
|
|
Search pages by text query using FTS5.
|
|
"""
|
|
if not q:
|
|
return []
|
|
return await db.search_pages(q, limit)
|
|
|
|
|
|
@app.get("/api/media/{md5_hash}")
|
|
async def get_media_info(md5_hash: str):
|
|
"""Get full media info including all source URLs."""
|
|
media = await db.get_media_by_hash(md5_hash)
|
|
if not media:
|
|
raise HTTPException(status_code=404, detail="Media not found")
|
|
|
|
media['sources'] = await db.get_media_sources(md5_hash)
|
|
return media
|
|
|
|
|
|
def slugify(text: str, max_len: int = 60) -> str:
|
|
"""Convert text to a safe filename slug."""
|
|
import re
|
|
import unicodedata
|
|
# Normalize unicode
|
|
text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('ascii')
|
|
# Lowercase and replace spaces/special chars with hyphens
|
|
text = re.sub(r'[^\w\s-]', '', text.lower())
|
|
text = re.sub(r'[-\s]+', '-', text).strip('-')
|
|
return text[:max_len] if text else ""
|
|
|
|
|
|
@app.get("/media/{md5_hash}")
|
|
async def serve_media(md5_hash: str, download: bool = False):
|
|
"""
|
|
Serve media file from vault or tarball.
|
|
|
|
Use ?download=1 for attachment mode with smart filename.
|
|
Caddy should be configured to cache these responses.
|
|
"""
|
|
# Tarball mode: serve from tar.gz
|
|
if TAR_FILE:
|
|
data, ext = find_media_in_tarball(md5_hash)
|
|
if data:
|
|
mime_type, _ = mimetypes.guess_type(f"file{ext}")
|
|
if not mime_type:
|
|
mime_type = "application/octet-stream"
|
|
filename = f"{md5_hash[:12]}{ext}"
|
|
headers = {
|
|
"Cache-Control": "public, max-age=31536000, immutable",
|
|
"X-Content-Hash": md5_hash,
|
|
}
|
|
if download:
|
|
headers["Content-Disposition"] = f'attachment; filename="{filename}"'
|
|
return Response(content=data, media_type=mime_type, headers=headers)
|
|
raise HTTPException(status_code=404, detail="Media not found in archive")
|
|
|
|
# Filesystem mode: find file in vault
|
|
subdir = VAULT_PATH / md5_hash[:2]
|
|
if not subdir.exists():
|
|
raise HTTPException(status_code=404, detail="Media not found")
|
|
|
|
# Find file with this hash prefix
|
|
for f in subdir.iterdir():
|
|
if f.name.startswith(md5_hash):
|
|
# Guess content type and get extension
|
|
mime_type, _ = mimetypes.guess_type(f.name)
|
|
ext = f.suffix or ""
|
|
|
|
# Get metadata for filename generation
|
|
filename = None
|
|
media_record = await db.get_media_by_hash(md5_hash)
|
|
if media_record:
|
|
if not mime_type and media_record.get("mime_type"):
|
|
mime_type = media_record["mime_type"]
|
|
# Generate filename from alt_text or title
|
|
name_source = media_record.get("alt_text") or media_record.get("title")
|
|
if name_source:
|
|
slug = slugify(name_source)
|
|
if slug:
|
|
filename = f"{slug}{ext}"
|
|
|
|
# Fallback: try to get page_title from media_sources
|
|
if not filename:
|
|
sources = await db.get_media_sources(md5_hash)
|
|
if sources:
|
|
row2 = sources[0]
|
|
# Try page_title + hash index
|
|
if row2.get("page_title"):
|
|
media_idx = int(md5_hash[:4], 16)
|
|
slug = slugify(f"{row2['page_title']}-{media_idx}")
|
|
if slug:
|
|
filename = f"{slug}{ext}"
|
|
# Fallback: original filename from URL
|
|
if not filename and row2.get("media_uri"):
|
|
from urllib.parse import unquote
|
|
parsed = Uri(row2["media_uri"])
|
|
orig_name = Path(unquote(parsed.path)).name
|
|
if orig_name and '.' in orig_name:
|
|
filename = orig_name
|
|
|
|
if not mime_type:
|
|
mime_type = "application/octet-stream"
|
|
|
|
# Default filename if nothing else
|
|
if not filename:
|
|
filename = f"{md5_hash[:12]}{ext}"
|
|
|
|
# Download mode: attachment with smart filename
|
|
# Inline mode: no filename header, browser shows inline
|
|
if download:
|
|
return FileResponse(
|
|
f,
|
|
media_type=mime_type,
|
|
filename=filename,
|
|
content_disposition_type="attachment",
|
|
headers={
|
|
"Cache-Control": "public, max-age=31536000, immutable",
|
|
"X-Content-Hash": md5_hash,
|
|
}
|
|
)
|
|
else:
|
|
return FileResponse(
|
|
f,
|
|
media_type=mime_type,
|
|
content_disposition_type="inline",
|
|
headers={
|
|
"Cache-Control": "public, max-age=31536000, immutable",
|
|
"X-Content-Hash": md5_hash,
|
|
}
|
|
)
|
|
|
|
raise HTTPException(status_code=404, detail="Media not found")
|
|
|
|
|
|
# ============================================================================
|
|
# Crawler API
|
|
# ============================================================================
|
|
|
|
@app.get("/api/crawl/jobs")
|
|
async def get_crawl_jobs_endpoint(limit: int = Query(50, le=200)):
|
|
"""Get recent crawl jobs."""
|
|
return await db.get_crawl_jobs(limit)
|
|
|
|
|
|
@app.get("/api/crawl/jobs/{job_id}")
|
|
async def get_crawl_job_endpoint(job_id: int):
|
|
"""Get a specific crawl job."""
|
|
job = await db.get_crawl_job(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
return job
|
|
|
|
|
|
@app.post("/api/crawl")
|
|
async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
|
|
"""
|
|
Start new crawl job(s).
|
|
|
|
Supports multiple targets - creates one job per target.
|
|
The crawls run in the background. Poll /api/crawl/jobs/{id} for status.
|
|
"""
|
|
# Import neopig here to avoid circular imports
|
|
from neopig import NeoPig
|
|
from async_web_fetcher import CrawlMode
|
|
|
|
# Map mode string to enum
|
|
mode_map = {
|
|
"text": CrawlMode.TEXT,
|
|
"images": CrawlMode.IMAGES,
|
|
"videos": CrawlMode.VIDEOS,
|
|
"media": CrawlMode.MEDIA,
|
|
"all": CrawlMode.ALL,
|
|
}
|
|
mode = mode_map.get(request.mode, CrawlMode.IMAGES)
|
|
|
|
# Get targets (support both new 'targets' array and old 'target_uri' single value)
|
|
targets = request.targets if request.targets else [request.target_uri] if request.target_uri else []
|
|
if not targets:
|
|
raise HTTPException(status_code=400, detail="No target URIs provided")
|
|
|
|
job_ids = []
|
|
|
|
# Create a job for each target
|
|
for target_uri in targets:
|
|
job_id = await db.create_crawl_job(target_uri, request.keywords, request.mode)
|
|
job_ids.append(job_id)
|
|
|
|
# Run crawl in background (closure captures job_id and target_uri)
|
|
async def run_crawl(jid=job_id, uri=target_uri):
|
|
try:
|
|
pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH))
|
|
await pig.init()
|
|
|
|
# Load existing seen media for resume capability
|
|
crawled_media = await pig.db.get_crawled_media_uris()
|
|
if crawled_media:
|
|
pig.seen_media = crawled_media
|
|
|
|
# Cap depth at 15 (convert -1 or values > 15 to 15)
|
|
depth = request.depth if 1 <= request.depth <= 15 else 15
|
|
|
|
# Progress updater - runs alongside crawl
|
|
stop_progress = asyncio.Event()
|
|
|
|
async def update_progress():
|
|
while not stop_progress.is_set():
|
|
await asyncio.sleep(2) # Update every 2 seconds
|
|
if not stop_progress.is_set():
|
|
await db.update_crawl_job_stats(jid, pig.stats)
|
|
|
|
progress_task = asyncio.create_task(update_progress())
|
|
|
|
try:
|
|
stats = await pig.crawl(
|
|
target_uri=uri,
|
|
keywords=request.keywords,
|
|
mode=mode,
|
|
depth=depth,
|
|
max_pages=request.max_pages,
|
|
download_media=request.download_media,
|
|
)
|
|
finally:
|
|
stop_progress.set()
|
|
progress_task.cancel()
|
|
try:
|
|
await progress_task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
|
|
# Update job as completed
|
|
await db.complete_crawl_job(jid, stats)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Crawl job {jid} failed: {e}")
|
|
await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})
|
|
|
|
# Schedule async task on the event loop (not BackgroundTasks which runs in threadpool)
|
|
asyncio.create_task(run_crawl())
|
|
|
|
return {"job_ids": job_ids, "status": "running", "count": len(job_ids)}
|
|
|
|
|
|
def init_tarball_mode(tarball_path: str):
|
|
"""Initialize serving from a tar.gz archive."""
|
|
global TAR_FILE, TAR_MEMBERS, ARCHIVE_ROOT, DB_PATH, TEMP_DB_PATH
|
|
|
|
logger.info(f"Opening archive: {tarball_path}")
|
|
tarball = Path(tarball_path)
|
|
|
|
# Handle .run files with NEOPIG trailer
|
|
offset = 0
|
|
if tarball.suffix == '.run' or tarball.stat().st_size > 100000:
|
|
try:
|
|
with open(tarball, 'rb') as f:
|
|
f.seek(-22, 2)
|
|
trailer = f.read(22)
|
|
if trailer[:6] == b'NEOPIG':
|
|
offset = int(trailer[6:22].decode(), 16)
|
|
logger.info(f"Detected .run format, tarball offset: {offset}")
|
|
except Exception:
|
|
pass
|
|
|
|
# Open tarball
|
|
if offset > 0:
|
|
# Create a wrapper that presents just the tarball portion
|
|
class OffsetFile:
|
|
"""File wrapper that starts reading from an offset."""
|
|
def __init__(self, path, offset):
|
|
self._f = open(path, 'rb')
|
|
self._offset = offset
|
|
self._f.seek(offset)
|
|
def read(self, size=-1):
|
|
return self._f.read(size)
|
|
def seek(self, pos, whence=0):
|
|
if whence == 0: # SEEK_SET
|
|
return self._f.seek(self._offset + pos)
|
|
elif whence == 1: # SEEK_CUR
|
|
return self._f.seek(pos, 1)
|
|
else: # SEEK_END
|
|
return self._f.seek(pos, 2)
|
|
def tell(self):
|
|
return self._f.tell() - self._offset
|
|
def close(self):
|
|
self._f.close()
|
|
|
|
TAR_FILE = tarfile.open(fileobj=OffsetFile(tarball, offset), mode='r:gz')
|
|
else:
|
|
TAR_FILE = tarfile.open(tarball_path, 'r:gz')
|
|
|
|
# Build member lookup
|
|
for member in TAR_FILE.getmembers():
|
|
TAR_MEMBERS[member.name] = member
|
|
|
|
# Get archive root from first member
|
|
first = list(TAR_MEMBERS.keys())[0]
|
|
ARCHIVE_ROOT = first.split('/')[0]
|
|
logger.info(f"Archive root: {ARCHIVE_ROOT}")
|
|
|
|
# Extract database to temp (SQLite needs real file)
|
|
db_member = f"{ARCHIVE_ROOT}/archive.db"
|
|
if db_member in TAR_MEMBERS:
|
|
temp_dir = tempfile.mkdtemp(prefix="neopig_")
|
|
TEMP_DB_PATH = f"{temp_dir}/archive.db"
|
|
member = TAR_MEMBERS[db_member]
|
|
f = TAR_FILE.extractfile(member)
|
|
if f:
|
|
with open(TEMP_DB_PATH, 'wb') as out:
|
|
out.write(f.read())
|
|
DB_PATH = TEMP_DB_PATH
|
|
logger.info(f"Extracted database to: {TEMP_DB_PATH}")
|
|
else:
|
|
logger.warning("No archive.db found in tarball")
|
|
|
|
|
|
def main():
|
|
global DB_PATH, VAULT_PATH
|
|
|
|
parser = argparse.ArgumentParser(description="neopig SERP")
|
|
parser.add_argument("tarball", nargs='?', help="Path to archive.tar.gz or .run file")
|
|
parser.add_argument("--port", type=int, default=8000)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--db", default="data/neopig.db")
|
|
parser.add_argument("--vault", default="data/vault")
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Tarball mode
|
|
if args.tarball:
|
|
init_tarball_mode(args.tarball)
|
|
logger.info(f"Starting neopig SERP (archive mode) on {args.host}:{args.port}")
|
|
else:
|
|
DB_PATH = args.db
|
|
VAULT_PATH = Path(args.vault)
|
|
logger.info(f"Starting neopig SERP on {args.host}:{args.port}")
|
|
logger.info(f"Database: {DB_PATH}")
|
|
logger.info(f"Vault: {VAULT_PATH}")
|
|
|
|
uvicorn.run(app, host=args.host, port=args.port)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|