Russell Ballestrini
bc11c516b6
New files: - filevault.py: Hash-based file storage with use_pairs option (v1.1.0) - async_filevault.py: Async wrapper using asyncio.to_thread() - domain_vault.py: Triple vault system for web archival (HTML, Media, Linkpeek) - screenshot.py: Async screenshot capture using uri2png - tests/unit/: Comprehensive test suite (85 tests) Sync-to-async conversions: - storage.py: Wrap Path operations in asyncio.to_thread() - domain_vault.py: Wrap exists(), mkdir(), rglob(), os.walk() in asyncio.to_thread() - screenshot.py: Wrap read_bytes(), write_bytes(), unlink() in asyncio.to_thread() All sync filesystem operations now run in thread pool to avoid blocking async loop.
1394 lines
46 KiB
Python
1394 lines
46 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
neopig SERP - Search Engine Results Page
|
|
|
|
Fast image/video search across hydrated metadata.
|
|
Serves files directly from filevault via Caddy with 1GB memory cache.
|
|
|
|
Search across:
|
|
- keywords (crawl tags)
|
|
- alt_text (image alt attributes)
|
|
- title (media titles)
|
|
- analysis_result (Qwen 3 VL descriptions)
|
|
- source_page / source_url (origin)
|
|
|
|
Usage:
|
|
python serp.py --port 8000 --vault ./vault --db neopig.db
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import mimetypes
|
|
from pathlib import Path
|
|
from typing import List, Dict, Any, Optional
|
|
|
|
from fastapi import FastAPI, Query, HTTPException, BackgroundTasks
|
|
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
from pydantic import BaseModel
|
|
import aiosqlite
|
|
import uvicorn
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
app = FastAPI(title="neopig", description="Media crawler + SERP + Screenshot service")
|
|
|
|
# Try to include uri2png screenshot router (optional dependency)
|
|
try:
|
|
from uri2png import get_screenshot_router
|
|
app.include_router(get_screenshot_router())
|
|
logger.info("Screenshot router loaded from uri2png")
|
|
except ImportError:
|
|
logger.warning("uri2png not installed - screenshot endpoints not available")
|
|
|
|
# Config - set via startup
|
|
DB_PATH = "neopig.db"
|
|
VAULT_PATH = Path("vault")
|
|
|
|
# Active crawl jobs (in-memory tracking)
|
|
ACTIVE_CRAWLS: Dict[int, Dict[str, Any]] = {}
|
|
|
|
|
|
async def init_database():
|
|
"""Initialize database schema if needed."""
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
# Crawl jobs table
|
|
await db.execute("""
|
|
CREATE TABLE IF NOT EXISTS crawl_jobs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
target_uri TEXT NOT NULL,
|
|
keywords TEXT,
|
|
mode TEXT DEFAULT 'images',
|
|
status TEXT DEFAULT 'running',
|
|
started_at TEXT NOT NULL,
|
|
completed_at TEXT,
|
|
stats TEXT
|
|
)
|
|
""")
|
|
|
|
# Media records table
|
|
await db.execute("""
|
|
CREATE TABLE IF NOT EXISTS media (
|
|
md5_hash TEXT PRIMARY KEY,
|
|
media_type TEXT,
|
|
mime_type TEXT,
|
|
file_size INTEGER,
|
|
keywords TEXT,
|
|
alt_text TEXT,
|
|
title TEXT,
|
|
first_seen_at TEXT NOT NULL,
|
|
analysis_status TEXT DEFAULT 'pending',
|
|
analysis_result TEXT
|
|
)
|
|
""")
|
|
|
|
# Media sources table
|
|
await db.execute("""
|
|
CREATE TABLE IF NOT EXISTS media_sources (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
md5_hash TEXT NOT NULL,
|
|
media_uri TEXT NOT NULL,
|
|
page_uri TEXT,
|
|
page_title TEXT,
|
|
page_description TEXT,
|
|
page_keywords TEXT,
|
|
alt_text TEXT,
|
|
link_text TEXT,
|
|
crawl_job_id INTEGER,
|
|
discovered_at TEXT NOT NULL,
|
|
FOREIGN KEY (md5_hash) REFERENCES media(md5_hash),
|
|
FOREIGN KEY (crawl_job_id) REFERENCES crawl_jobs(id),
|
|
UNIQUE(md5_hash, media_uri, page_uri)
|
|
)
|
|
""")
|
|
|
|
# Indexes
|
|
await db.execute("CREATE INDEX IF NOT EXISTS idx_media_type ON media(media_type)")
|
|
await db.execute("CREATE INDEX IF NOT EXISTS idx_media_analysis ON media(analysis_status)")
|
|
await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_hash ON media_sources(md5_hash)")
|
|
await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_job ON media_sources(crawl_job_id)")
|
|
await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_media_uri ON media_sources(media_uri)")
|
|
await db.execute("CREATE INDEX IF NOT EXISTS idx_sources_page_uri ON media_sources(page_uri)")
|
|
|
|
await db.commit()
|
|
logger.info(f"Database initialized: {DB_PATH}")
|
|
|
|
|
|
@app.on_event("startup")
|
|
async def startup_event():
|
|
"""Initialize database on startup."""
|
|
await init_database()
|
|
VAULT_PATH.mkdir(parents=True, exist_ok=True)
|
|
logger.info(f"Vault directory ready: {VAULT_PATH}")
|
|
|
|
|
|
class CrawlRequest(BaseModel):
|
|
"""Request to start a new crawl."""
|
|
targets: List[str] = [] # Multiple target URIs
|
|
target_uri: str = "" # Deprecated: single target (for backwards compat)
|
|
keywords: List[str] = []
|
|
mode: str = "images" # text, images, videos, media, all
|
|
depth: int = -1 # -1 = unlimited
|
|
max_pages: int = -1 # -1 = unlimited
|
|
download_media: bool = True
|
|
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
async def index():
|
|
"""Simple search UI."""
|
|
return SEARCH_HTML
|
|
|
|
|
|
@app.get("/crawl", response_class=HTMLResponse)
|
|
async def crawl_page():
|
|
"""Crawler command page."""
|
|
return CRAWL_HTML
|
|
|
|
|
|
# HTML Templates
|
|
SEARCH_HTML = """
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>neopig SERP</title>
|
|
<style>
|
|
* { box-sizing: border-box; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
max-width: 1400px;
|
|
margin: 0 auto;
|
|
padding: 20px;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}
|
|
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
|
.subtitle { color: #666; margin-bottom: 20px; }
|
|
.search-box {
|
|
display: flex;
|
|
gap: 10px;
|
|
margin-bottom: 20px;
|
|
}
|
|
input[type="text"] {
|
|
flex: 1;
|
|
padding: 12px 16px;
|
|
font-size: 16px;
|
|
border: 2px solid #333;
|
|
border-radius: 8px;
|
|
background: #1a1a1a;
|
|
color: #fff;
|
|
}
|
|
input[type="text"]:focus {
|
|
outline: none;
|
|
border-color: #ff6b6b;
|
|
}
|
|
select {
|
|
padding: 12px 16px;
|
|
font-size: 16px;
|
|
border: 2px solid #333;
|
|
border-radius: 8px;
|
|
background: #1a1a1a;
|
|
color: #fff;
|
|
}
|
|
button {
|
|
padding: 12px 24px;
|
|
font-size: 16px;
|
|
background: #ff6b6b;
|
|
color: #fff;
|
|
border: none;
|
|
border-radius: 8px;
|
|
cursor: pointer;
|
|
}
|
|
button:hover { background: #ff5252; }
|
|
.stats {
|
|
padding: 10px 15px;
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
margin-bottom: 20px;
|
|
font-size: 14px;
|
|
color: #888;
|
|
}
|
|
.results {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
|
|
gap: 15px;
|
|
}
|
|
.result {
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
overflow: hidden;
|
|
transition: transform 0.2s;
|
|
}
|
|
.result:hover {
|
|
transform: scale(1.02);
|
|
}
|
|
.result img, .result video {
|
|
width: 100%;
|
|
height: 200px;
|
|
object-fit: contain;
|
|
background: #1a1a1a;
|
|
}
|
|
.result-info {
|
|
padding: 10px;
|
|
}
|
|
.result-hash {
|
|
font-family: monospace;
|
|
font-size: 11px;
|
|
color: #666;
|
|
word-break: break-all;
|
|
}
|
|
.result-meta {
|
|
font-size: 12px;
|
|
color: #888;
|
|
margin-top: 5px;
|
|
}
|
|
.result-keywords {
|
|
display: flex;
|
|
flex-wrap: wrap;
|
|
gap: 5px;
|
|
margin-top: 8px;
|
|
}
|
|
.tag {
|
|
background: #333;
|
|
padding: 2px 8px;
|
|
border-radius: 4px;
|
|
font-size: 11px;
|
|
color: #aaa;
|
|
}
|
|
.no-results {
|
|
text-align: center;
|
|
padding: 60px;
|
|
color: #666;
|
|
}
|
|
.media-link {
|
|
display: block;
|
|
cursor: pointer;
|
|
}
|
|
.media-link:hover img, .media-link:hover video {
|
|
opacity: 0.8;
|
|
}
|
|
.result-hash {
|
|
color: #ff6b6b;
|
|
text-decoration: none;
|
|
}
|
|
.result-hash:hover {
|
|
text-decoration: underline;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>🐷 neopig</h1>
|
|
<p class="subtitle">Search hydrated media</p>
|
|
|
|
<div class="search-box">
|
|
<input type="text" id="query" placeholder="Search keywords, alt text, titles..." autofocus>
|
|
<select id="type">
|
|
<option value="">All types</option>
|
|
<option value="image">Images</option>
|
|
<option value="video">Videos</option>
|
|
<option value="audio">Audio</option>
|
|
</select>
|
|
<button onclick="search()">Search</button>
|
|
</div>
|
|
|
|
<div class="stats" id="stats">Loading stats...</div>
|
|
<div class="results" id="results"></div>
|
|
|
|
<script>
|
|
async function loadStats() {
|
|
const res = await fetch('/api/stats');
|
|
const stats = await res.json();
|
|
document.getElementById('stats').innerHTML =
|
|
`<strong>${stats.total_media}</strong> media indexed | ` +
|
|
`<strong>${stats.by_type?.image || 0}</strong> images | ` +
|
|
`<strong>${stats.by_type?.video || 0}</strong> videos | ` +
|
|
`<strong>${stats.total_sources}</strong> source URLs`;
|
|
}
|
|
|
|
async function search() {
|
|
const query = document.getElementById('query').value;
|
|
const type = document.getElementById('type').value;
|
|
|
|
let url = `/api/search?q=${encodeURIComponent(query)}&limit=100`;
|
|
if (type) url += `&type=${type}`;
|
|
|
|
const res = await fetch(url);
|
|
const results = await res.json();
|
|
|
|
const container = document.getElementById('results');
|
|
|
|
if (results.length === 0) {
|
|
container.innerHTML = '<div class="no-results">No results found</div>';
|
|
return;
|
|
}
|
|
|
|
container.innerHTML = results.map(r => {
|
|
const isVideo = r.media_type === 'video';
|
|
const mediaEl = isVideo
|
|
? `<video src="/media/${r.md5_hash}" controls preload="metadata"></video>`
|
|
: `<img src="/media/${r.md5_hash}" alt="${r.alt_text || ''}" loading="lazy">`;
|
|
|
|
const keywords = JSON.parse(r.keywords || '[]');
|
|
const tagsHtml = keywords.map(k => `<span class="tag">${k}</span>`).join('');
|
|
|
|
return `
|
|
<div class="result">
|
|
<a href="/media/${r.md5_hash}" target="_blank" class="media-link">
|
|
${mediaEl}
|
|
</a>
|
|
<div class="result-info">
|
|
<a href="/view/${r.md5_hash}" class="result-hash">${r.md5_hash}</a>
|
|
<div class="result-meta">
|
|
${r.media_type} · ${formatBytes(r.file_size)}
|
|
${r.alt_text ? ` · ${r.alt_text.substring(0, 50)}` : ''}
|
|
</div>
|
|
<div class="result-keywords">${tagsHtml}</div>
|
|
</div>
|
|
</div>
|
|
`;
|
|
}).join('');
|
|
}
|
|
|
|
function formatBytes(bytes) {
|
|
if (!bytes) return '?';
|
|
if (bytes < 1024) return bytes + ' B';
|
|
if (bytes < 1024*1024) return (bytes/1024).toFixed(1) + ' KB';
|
|
return (bytes/1024/1024).toFixed(1) + ' MB';
|
|
}
|
|
|
|
// Enter key to search
|
|
document.getElementById('query').addEventListener('keypress', e => {
|
|
if (e.key === 'Enter') search();
|
|
});
|
|
|
|
// Load stats on page load
|
|
loadStats();
|
|
|
|
// Initial search (show all)
|
|
search();
|
|
</script>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
CRAWL_HTML = """
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>neopig Crawler</title>
|
|
<style>
|
|
* { box-sizing: border-box; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
max-width: 1000px;
|
|
margin: 0 auto;
|
|
padding: 20px;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}
|
|
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
|
.subtitle { color: #666; margin-bottom: 20px; }
|
|
nav { margin-bottom: 20px; }
|
|
nav a { color: #ff6b6b; margin-right: 15px; }
|
|
|
|
.form-group {
|
|
margin-bottom: 15px;
|
|
}
|
|
label {
|
|
display: block;
|
|
margin-bottom: 5px;
|
|
color: #aaa;
|
|
font-size: 14px;
|
|
}
|
|
input[type="text"], input[type="number"], select {
|
|
width: 100%;
|
|
padding: 12px 16px;
|
|
font-size: 16px;
|
|
border: 2px solid #333;
|
|
border-radius: 8px;
|
|
background: #1a1a1a;
|
|
color: #fff;
|
|
}
|
|
input:focus, select:focus {
|
|
outline: none;
|
|
border-color: #ff6b6b;
|
|
}
|
|
.row {
|
|
display: flex;
|
|
gap: 15px;
|
|
}
|
|
.row > div { flex: 1; }
|
|
|
|
button {
|
|
padding: 14px 28px;
|
|
font-size: 16px;
|
|
background: #ff6b6b;
|
|
color: #fff;
|
|
border: none;
|
|
border-radius: 8px;
|
|
cursor: pointer;
|
|
margin-top: 10px;
|
|
}
|
|
button:hover { background: #ff5252; }
|
|
button:disabled {
|
|
background: #444;
|
|
cursor: not-allowed;
|
|
}
|
|
button.secondary {
|
|
background: #333;
|
|
}
|
|
button.secondary:hover {
|
|
background: #444;
|
|
}
|
|
|
|
.checkbox-group {
|
|
display: flex;
|
|
align-items: center;
|
|
gap: 8px;
|
|
}
|
|
.checkbox-group input {
|
|
width: auto;
|
|
}
|
|
|
|
.jobs-section {
|
|
margin-top: 30px;
|
|
padding-top: 20px;
|
|
border-top: 1px solid #333;
|
|
}
|
|
h2 {
|
|
color: #ff6b6b;
|
|
font-size: 18px;
|
|
margin-bottom: 15px;
|
|
}
|
|
|
|
.job {
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
padding: 15px;
|
|
margin-bottom: 10px;
|
|
}
|
|
.job-header {
|
|
display: flex;
|
|
justify-content: space-between;
|
|
align-items: center;
|
|
margin-bottom: 10px;
|
|
}
|
|
.job-id {
|
|
font-family: monospace;
|
|
color: #666;
|
|
}
|
|
.job-status {
|
|
padding: 4px 10px;
|
|
border-radius: 4px;
|
|
font-size: 12px;
|
|
font-weight: bold;
|
|
}
|
|
.job-status.running { background: #2d5a27; color: #7bed72; }
|
|
.job-status.completed { background: #1a3a4a; color: #6bc5e8; }
|
|
.job-status.failed { background: #5a2727; color: #ed7272; }
|
|
|
|
.job-target {
|
|
font-size: 14px;
|
|
word-break: break-all;
|
|
margin-bottom: 5px;
|
|
}
|
|
.job-meta {
|
|
font-size: 12px;
|
|
color: #666;
|
|
}
|
|
.job-stats {
|
|
display: flex;
|
|
gap: 15px;
|
|
margin-top: 10px;
|
|
font-size: 13px;
|
|
}
|
|
.job-stats span {
|
|
background: #252525;
|
|
padding: 4px 10px;
|
|
border-radius: 4px;
|
|
}
|
|
|
|
.progress-bar {
|
|
height: 4px;
|
|
background: #333;
|
|
border-radius: 2px;
|
|
margin-top: 10px;
|
|
overflow: hidden;
|
|
}
|
|
.progress-bar-fill {
|
|
height: 100%;
|
|
background: #ff6b6b;
|
|
transition: width 0.3s;
|
|
}
|
|
|
|
.no-jobs {
|
|
color: #666;
|
|
text-align: center;
|
|
padding: 30px;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>🐷 neopig crawler</h1>
|
|
<p class="subtitle">Hydrate media from the web</p>
|
|
|
|
<nav>
|
|
<a href="/">← Search</a>
|
|
<a href="/crawl">Crawler</a>
|
|
</nav>
|
|
|
|
<form id="crawl-form" onsubmit="startCrawl(event)">
|
|
<div class="form-group">
|
|
<label>Target URIs (space, comma, or newline separated)</label>
|
|
<textarea id="target" placeholder="https://example.com https://another.com" rows="2" required style="width:100%;padding:10px;border:1px solid #333;border-radius:4px;background:#1a1a1a;color:#e0e0e0;font-size:14px;resize:vertical;"></textarea>
|
|
</div>
|
|
|
|
<div class="form-group">
|
|
<label>Keywords (space or comma separated)</label>
|
|
<input type="text" id="keywords" placeholder="rick and morty, adult swim">
|
|
</div>
|
|
|
|
<div class="row">
|
|
<div class="form-group">
|
|
<label>Mode</label>
|
|
<select id="mode">
|
|
<option value="images">Images only</option>
|
|
<option value="videos">Videos only</option>
|
|
<option value="media">All media (images + videos + audio)</option>
|
|
<option value="all">Everything (text + media)</option>
|
|
<option value="text">Text only</option>
|
|
</select>
|
|
</div>
|
|
<div class="form-group">
|
|
<label>Depth (5 recommended, max 15)</label>
|
|
<input type="number" id="depth" value="5" min="1" max="15">
|
|
</div>
|
|
<div class="form-group">
|
|
<label>Max Pages (-1 = unlimited)</label>
|
|
<input type="number" id="max_pages" value="500" min="-1">
|
|
</div>
|
|
</div>
|
|
|
|
<div class="form-group checkbox-group">
|
|
<input type="checkbox" id="download" checked>
|
|
<label for="download" style="display:inline; margin:0;">Download media (uncheck to just index URLs)</label>
|
|
</div>
|
|
|
|
<button type="submit" id="start-btn">Start Crawl</button>
|
|
</form>
|
|
|
|
<div class="jobs-section">
|
|
<h2>Crawl Jobs</h2>
|
|
<div id="jobs">Loading...</div>
|
|
</div>
|
|
|
|
<script>
|
|
async function startCrawl(e) {
|
|
e.preventDefault();
|
|
|
|
const btn = document.getElementById('start-btn');
|
|
btn.disabled = true;
|
|
btn.textContent = 'Starting...';
|
|
|
|
const keywordsRaw = document.getElementById('keywords').value;
|
|
const keywords = keywordsRaw
|
|
.split(/[,\\s]+/)
|
|
.map(k => k.trim())
|
|
.filter(k => k.length > 0);
|
|
|
|
// Parse multiple target URIs (space, comma, or newline separated)
|
|
const targetsRaw = document.getElementById('target').value;
|
|
const targets = targetsRaw
|
|
.split(/[,\\s\\n]+/)
|
|
.map(t => t.trim())
|
|
.filter(t => t.length > 0 && t.startsWith('http'));
|
|
|
|
if (targets.length === 0) {
|
|
alert('Please enter at least one valid URI (must start with http)');
|
|
btn.disabled = false;
|
|
btn.textContent = 'Start Crawl';
|
|
return;
|
|
}
|
|
|
|
const payload = {
|
|
targets: targets,
|
|
keywords: keywords,
|
|
mode: document.getElementById('mode').value,
|
|
depth: parseInt(document.getElementById('depth').value),
|
|
max_pages: parseInt(document.getElementById('max_pages').value),
|
|
download_media: document.getElementById('download').checked
|
|
};
|
|
|
|
try {
|
|
const res = await fetch('/api/crawl', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(payload)
|
|
});
|
|
|
|
if (!res.ok) {
|
|
const err = await res.json();
|
|
alert('Error: ' + (err.detail || 'Failed to start crawl'));
|
|
} else {
|
|
const result = await res.json();
|
|
const jobIds = result.job_ids || [result.job_id];
|
|
alert('Crawl started! Job IDs: ' + jobIds.join(', '));
|
|
loadJobs();
|
|
}
|
|
} catch (err) {
|
|
alert('Error: ' + err.message);
|
|
}
|
|
|
|
btn.disabled = false;
|
|
btn.textContent = 'Start Crawl';
|
|
}
|
|
|
|
async function loadJobs() {
|
|
try {
|
|
const res = await fetch('/api/crawl/jobs');
|
|
const jobs = await res.json();
|
|
|
|
const container = document.getElementById('jobs');
|
|
|
|
if (jobs.length === 0) {
|
|
container.innerHTML = '<div class="no-jobs">No crawl jobs yet</div>';
|
|
return;
|
|
}
|
|
|
|
container.innerHTML = jobs.map(job => {
|
|
const stats = job.stats ? JSON.parse(job.stats) : {};
|
|
const keywords = job.keywords ? JSON.parse(job.keywords) : [];
|
|
|
|
return `
|
|
<div class="job">
|
|
<div class="job-header">
|
|
<span class="job-id">Job #${job.id}</span>
|
|
<span class="job-status ${job.status}">${job.status}</span>
|
|
</div>
|
|
<div class="job-target">${job.target_uri}</div>
|
|
<div class="job-meta">
|
|
Mode: ${job.mode || 'images'} |
|
|
Keywords: ${keywords.join(', ') || 'none'} |
|
|
Started: ${new Date(job.started_at).toLocaleString()}
|
|
</div>
|
|
${job.status === 'completed' ? `
|
|
<div class="job-stats">
|
|
<span>📄 ${stats.pages_crawled || 0} pages</span>
|
|
<span>🖼️ ${stats.media_found || 0} found</span>
|
|
<span>💾 ${stats.media_downloaded || 0} saved</span>
|
|
<span>♻️ ${stats.duplicates_skipped || 0} dupes</span>
|
|
</div>
|
|
` : ''}
|
|
${job.status === 'running' ? `
|
|
<div class="progress-bar">
|
|
<div class="progress-bar-fill" style="width: 50%"></div>
|
|
</div>
|
|
` : ''}
|
|
</div>
|
|
`;
|
|
}).join('');
|
|
} catch (err) {
|
|
document.getElementById('jobs').innerHTML = '<div class="no-jobs">Failed to load jobs</div>';
|
|
}
|
|
}
|
|
|
|
// Load jobs on page load
|
|
loadJobs();
|
|
|
|
// Refresh jobs every 5 seconds
|
|
setInterval(loadJobs, 5000);
|
|
</script>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
LIVE_HTML = """
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>neopig LIVE - Watch Images Crawl In</title>
|
|
<style>
|
|
* { box-sizing: border-box; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0;
|
|
padding: 20px;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}
|
|
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
|
.subtitle { color: #666; margin-bottom: 10px; }
|
|
.stats {
|
|
background: #1a1a1a;
|
|
padding: 15px;
|
|
border-radius: 8px;
|
|
margin-bottom: 20px;
|
|
display: flex;
|
|
gap: 30px;
|
|
flex-wrap: wrap;
|
|
}
|
|
.stat { display: flex; flex-direction: column; }
|
|
.stat-value { font-size: 24px; font-weight: bold; color: #ff6b6b; }
|
|
.stat-label { font-size: 12px; color: #888; }
|
|
.controls {
|
|
margin-bottom: 20px;
|
|
display: flex;
|
|
gap: 10px;
|
|
align-items: center;
|
|
}
|
|
.controls button {
|
|
padding: 8px 16px;
|
|
border: none;
|
|
border-radius: 4px;
|
|
cursor: pointer;
|
|
font-size: 14px;
|
|
}
|
|
.controls button.primary { background: #ff6b6b; color: white; }
|
|
.controls button.secondary { background: #333; color: #ddd; }
|
|
.controls button:hover { opacity: 0.8; }
|
|
.status { color: #4ade80; font-size: 14px; }
|
|
.status.paused { color: #fbbf24; }
|
|
.grid {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
|
|
gap: 15px;
|
|
}
|
|
.card {
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
overflow: hidden;
|
|
animation: fadeIn 0.5s ease-out;
|
|
}
|
|
@keyframes fadeIn {
|
|
from { opacity: 0; transform: translateY(20px); }
|
|
to { opacity: 1; transform: translateY(0); }
|
|
}
|
|
.card.new {
|
|
box-shadow: 0 0 20px rgba(255, 107, 107, 0.5);
|
|
}
|
|
.card img, .card video {
|
|
width: 100%;
|
|
height: 180px;
|
|
object-fit: contain;
|
|
background: #222;
|
|
}
|
|
.card-info {
|
|
padding: 10px;
|
|
}
|
|
.card-title {
|
|
font-size: 12px;
|
|
color: #888;
|
|
overflow: hidden;
|
|
text-overflow: ellipsis;
|
|
white-space: nowrap;
|
|
}
|
|
.card-source {
|
|
font-size: 10px;
|
|
color: #555;
|
|
margin-top: 4px;
|
|
overflow: hidden;
|
|
text-overflow: ellipsis;
|
|
white-space: nowrap;
|
|
}
|
|
.nav { margin-bottom: 20px; }
|
|
.nav a { color: #ff6b6b; margin-right: 15px; text-decoration: none; }
|
|
.nav a:hover { text-decoration: underline; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="nav">
|
|
<a href="/">Search</a>
|
|
<a href="/crawl">Crawl</a>
|
|
<a href="/live">Live Feed</a>
|
|
</div>
|
|
|
|
<h1>Live Feed</h1>
|
|
<p class="subtitle">Watch images appear as they're crawled</p>
|
|
|
|
<div class="stats">
|
|
<div class="stat">
|
|
<span class="stat-value" id="total-count">0</span>
|
|
<span class="stat-label">Total Images</span>
|
|
</div>
|
|
<div class="stat">
|
|
<span class="stat-value" id="new-count">0</span>
|
|
<span class="stat-label">New This Session</span>
|
|
</div>
|
|
<div class="stat">
|
|
<span class="stat-value" id="rate">0</span>
|
|
<span class="stat-label">Per Minute</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="controls">
|
|
<button class="primary" id="toggle-btn" onclick="toggleFeed()">Pause</button>
|
|
<button class="secondary" onclick="clearFeed()">Clear</button>
|
|
<span class="status" id="status">Watching for new images...</span>
|
|
</div>
|
|
|
|
<div class="grid" id="grid"></div>
|
|
|
|
<script>
|
|
let running = true;
|
|
let lastCheck = new Date().toISOString();
|
|
let seenHashes = new Set();
|
|
let newCount = 0;
|
|
let startTime = Date.now();
|
|
|
|
function toggleFeed() {
|
|
running = !running;
|
|
const btn = document.getElementById('toggle-btn');
|
|
const status = document.getElementById('status');
|
|
if (running) {
|
|
btn.textContent = 'Pause';
|
|
status.textContent = 'Watching for new images...';
|
|
status.className = 'status';
|
|
poll();
|
|
} else {
|
|
btn.textContent = 'Resume';
|
|
status.textContent = 'Paused';
|
|
status.className = 'status paused';
|
|
}
|
|
}
|
|
|
|
function clearFeed() {
|
|
document.getElementById('grid').innerHTML = '';
|
|
seenHashes.clear();
|
|
newCount = 0;
|
|
startTime = Date.now();
|
|
updateStats(0);
|
|
}
|
|
|
|
function updateStats(total) {
|
|
document.getElementById('total-count').textContent = total;
|
|
document.getElementById('new-count').textContent = newCount;
|
|
|
|
const minutes = (Date.now() - startTime) / 60000;
|
|
const rate = minutes > 0 ? Math.round(newCount / minutes) : 0;
|
|
document.getElementById('rate').textContent = rate;
|
|
}
|
|
|
|
async function poll() {
|
|
if (!running) return;
|
|
|
|
try {
|
|
// Get recent media sorted by first_seen_at descending
|
|
const res = await fetch('/api/search?q=&limit=50');
|
|
const media = await res.json();
|
|
|
|
// Get stats
|
|
const statsRes = await fetch('/api/stats');
|
|
const stats = await statsRes.json();
|
|
updateStats(stats.total_media || 0);
|
|
|
|
const grid = document.getElementById('grid');
|
|
|
|
// Find new items
|
|
const newItems = media.filter(m => !seenHashes.has(m.md5_hash));
|
|
|
|
// Add new items to the top
|
|
newItems.reverse().forEach(item => {
|
|
seenHashes.add(item.md5_hash);
|
|
newCount++;
|
|
|
|
const card = document.createElement('div');
|
|
card.className = 'card new';
|
|
|
|
const isVideo = item.media_type === 'video';
|
|
const mediaEl = isVideo
|
|
? `<video src="/media/${item.md5_hash}" muted loop onmouseenter="this.play()" onmouseleave="this.pause()"></video>`
|
|
: `<img src="/media/${item.md5_hash}" loading="lazy" onerror="this.src='data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 width=%22200%22 height=%22180%22><rect fill=%22%23333%22 width=%22200%22 height=%22180%22/><text x=%2250%%22 y=%2250%%22 fill=%22%23666%22 text-anchor=%22middle%22>Error</text></svg>'">`;
|
|
|
|
card.innerHTML = `
|
|
<a href="/view/${item.md5_hash}">
|
|
${mediaEl}
|
|
</a>
|
|
<div class="card-info">
|
|
<div class="card-title">${item.alt_text || item.title || item.md5_hash.slice(0,12)}</div>
|
|
<div class="card-source">${item.media_type} - ${formatSize(item.file_size)}</div>
|
|
</div>
|
|
`;
|
|
|
|
grid.insertBefore(card, grid.firstChild);
|
|
|
|
// Remove 'new' class after animation
|
|
setTimeout(() => card.classList.remove('new'), 2000);
|
|
});
|
|
|
|
// Update stats
|
|
updateStats(stats.total_media || 0);
|
|
|
|
} catch (err) {
|
|
console.error('Poll error:', err);
|
|
}
|
|
|
|
// Poll again
|
|
setTimeout(poll, 2000);
|
|
}
|
|
|
|
function formatSize(bytes) {
|
|
if (!bytes) return '?';
|
|
if (bytes < 1024) return bytes + ' B';
|
|
if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB';
|
|
return (bytes / (1024 * 1024)).toFixed(1) + ' MB';
|
|
}
|
|
|
|
// Start polling
|
|
poll();
|
|
</script>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
@app.get("/live", response_class=HTMLResponse)
|
|
async def live_page():
|
|
"""Live feed page - watch images appear as they're crawled."""
|
|
return LIVE_HTML
|
|
|
|
|
|
@app.get("/view/{md5_hash}", response_class=HTMLResponse)
|
|
async def view_media_page(md5_hash: str):
|
|
"""Detail view page for a single media item."""
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
|
|
cursor = await db.execute("SELECT * FROM media WHERE md5_hash = ?", (md5_hash,))
|
|
media = await cursor.fetchone()
|
|
if not media:
|
|
raise HTTPException(status_code=404, detail="Media not found")
|
|
|
|
cursor = await db.execute(
|
|
"SELECT media_uri, page_uri, discovered_at FROM media_sources WHERE md5_hash = ?",
|
|
(md5_hash,)
|
|
)
|
|
sources = await cursor.fetchall()
|
|
|
|
media = dict(media)
|
|
keywords = json.loads(media.get('keywords') or '[]')
|
|
|
|
is_video = media['media_type'] == 'video'
|
|
is_audio = media['media_type'] == 'audio'
|
|
|
|
if is_video:
|
|
media_html = f'<video src="/media/{md5_hash}" controls style="max-width:100%;max-height:70vh;"></video>'
|
|
elif is_audio:
|
|
media_html = f'<audio src="/media/{md5_hash}" controls></audio>'
|
|
else:
|
|
media_html = f'<img src="/media/{md5_hash}" alt="{media.get("alt_text") or ""}" style="max-width:100%;max-height:70vh;">'
|
|
|
|
# Build sources - show all pages that embed this image
|
|
# Prioritize more specific pages (longer paths) over generic ones
|
|
sorted_sources = sorted(sources, key=lambda s: len(s["page_uri"] or ""), reverse=True)
|
|
|
|
sources_html = ''.join([
|
|
f'<li><a href="{s["page_uri"]}" target="_blank">{s["page_uri"]}</a></li>'
|
|
for s in sorted_sources
|
|
])
|
|
|
|
# Also show the direct media URL(s) separately
|
|
media_urls = list(set(s["media_uri"] for s in sources))
|
|
media_urls_html = ''.join([
|
|
f'<li><a href="{url}" target="_blank">{url}</a></li>'
|
|
for url in media_urls
|
|
])
|
|
|
|
keywords_html = ''.join([f'<span class="tag">{k}</span>' for k in keywords])
|
|
|
|
return f"""
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>{media.get('alt_text') or md5_hash} - neopig</title>
|
|
<style>
|
|
* {{ box-sizing: border-box; }}
|
|
body {{
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
max-width: 1200px;
|
|
margin: 0 auto;
|
|
padding: 20px;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}}
|
|
h1 {{ color: #ff6b6b; font-size: 20px; }}
|
|
a {{ color: #ff6b6b; }}
|
|
.media-container {{ text-align: center; margin: 20px 0; }}
|
|
.meta {{ background: #1a1a1a; padding: 15px; border-radius: 8px; margin: 15px 0; }}
|
|
.meta-row {{ display: flex; margin: 8px 0; }}
|
|
.meta-label {{ width: 120px; color: #888; }}
|
|
.meta-value {{ flex: 1; word-break: break-all; }}
|
|
.tag {{ background: #333; padding: 2px 8px; border-radius: 4px; font-size: 12px; margin-right: 5px; }}
|
|
.sources {{ margin-top: 20px; }}
|
|
.sources ul {{ padding-left: 20px; }}
|
|
.sources li {{ margin: 10px 0; }}
|
|
.sources small {{ color: #666; }}
|
|
.sources .urls {{ display: block; word-break: break-all; font-size: 11px; margin-top: 4px; }}
|
|
nav {{ margin-bottom: 20px; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<nav><a href="/">← Back to Search</a></nav>
|
|
<h1>{media.get('alt_text') or media.get('title') or 'Untitled'}</h1>
|
|
|
|
<div class="media-container">
|
|
<a href="/media/{md5_hash}" target="_blank">{media_html}</a>
|
|
</div>
|
|
|
|
<div class="meta">
|
|
<div class="meta-row"><span class="meta-label">MD5 Hash:</span><span class="meta-value"><code>{md5_hash}</code></span></div>
|
|
<div class="meta-row"><span class="meta-label">Type:</span><span class="meta-value">{media['media_type']}</span></div>
|
|
<div class="meta-row"><span class="meta-label">MIME:</span><span class="meta-value">{media.get('mime_type') or 'unknown'}</span></div>
|
|
<div class="meta-row"><span class="meta-label">Size:</span><span class="meta-value">{media.get('file_size') or 0:,} bytes</span></div>
|
|
<div class="meta-row"><span class="meta-label">First seen:</span><span class="meta-value">{media.get('first_seen_at')}</span></div>
|
|
<div class="meta-row"><span class="meta-label">Alt text:</span><span class="meta-value">{media.get('alt_text') or '-'}</span></div>
|
|
<div class="meta-row"><span class="meta-label">Title:</span><span class="meta-value">{media.get('title') or '-'}</span></div>
|
|
<div class="meta-row"><span class="meta-label">Keywords:</span><span class="meta-value">{keywords_html or '-'}</span></div>
|
|
<div class="meta-row"><span class="meta-label">Analysis:</span><span class="meta-value">{media.get('analysis_status', 'pending')}</span></div>
|
|
</div>
|
|
|
|
<div class="sources">
|
|
<h3>Pages embedding this media ({len(sources)})</h3>
|
|
<ul>{sources_html}</ul>
|
|
</div>
|
|
|
|
<div class="sources">
|
|
<h3>Direct media URLs ({len(media_urls)})</h3>
|
|
<ul>{media_urls_html}</ul>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
@app.get("/health")
|
|
async def health():
|
|
"""Health check endpoint."""
|
|
has_screenshot = False
|
|
try:
|
|
from uri2png import get_available_engines
|
|
has_screenshot = True
|
|
except ImportError:
|
|
pass
|
|
|
|
return {
|
|
"status": "healthy",
|
|
"features": {
|
|
"search": True,
|
|
"crawl": True,
|
|
"screenshot": has_screenshot
|
|
}
|
|
}
|
|
|
|
|
|
@app.get("/api/stats")
|
|
async def get_stats():
|
|
"""Get database statistics."""
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
stats = {}
|
|
|
|
cursor = await db.execute("SELECT COUNT(*) FROM media")
|
|
stats['total_media'] = (await cursor.fetchone())[0]
|
|
|
|
cursor = await db.execute(
|
|
"SELECT media_type, COUNT(*) FROM media GROUP BY media_type"
|
|
)
|
|
stats['by_type'] = {row[0]: row[1] for row in await cursor.fetchall()}
|
|
|
|
cursor = await db.execute(
|
|
"SELECT analysis_status, COUNT(*) FROM media GROUP BY analysis_status"
|
|
)
|
|
stats['by_analysis'] = {row[0]: row[1] for row in await cursor.fetchall()}
|
|
|
|
cursor = await db.execute("SELECT COUNT(*) FROM media_sources")
|
|
stats['total_sources'] = (await cursor.fetchone())[0]
|
|
|
|
cursor = await db.execute("SELECT COUNT(*) FROM crawl_jobs")
|
|
stats['total_jobs'] = (await cursor.fetchone())[0]
|
|
|
|
return stats
|
|
|
|
|
|
@app.get("/api/search")
|
|
async def search(
|
|
q: str = Query("", description="Search query"),
|
|
type: Optional[str] = Query(None, description="Filter by media type"),
|
|
status: Optional[str] = Query(None, description="Filter by analysis status"),
|
|
limit: int = Query(100, le=1000),
|
|
offset: int = Query(0)
|
|
):
|
|
"""
|
|
Search media by text query.
|
|
|
|
Searches across: keywords, alt_text, title, source URLs, analysis results.
|
|
"""
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
|
|
# Build query
|
|
conditions = []
|
|
params = []
|
|
|
|
if q:
|
|
conditions.append("""(
|
|
m.keywords LIKE ? OR
|
|
m.alt_text LIKE ? OR
|
|
m.title LIKE ? OR
|
|
m.analysis_result LIKE ? OR
|
|
EXISTS (SELECT 1 FROM media_sources ms WHERE ms.md5_hash = m.md5_hash AND (ms.media_uri LIKE ? OR ms.page_uri LIKE ? OR ms.page_title LIKE ? OR ms.page_description LIKE ? OR ms.page_keywords LIKE ?))
|
|
)""")
|
|
like_q = f"%{q}%"
|
|
params.extend([like_q, like_q, like_q, like_q, like_q, like_q, like_q, like_q, like_q])
|
|
|
|
if type:
|
|
conditions.append("m.media_type = ?")
|
|
params.append(type)
|
|
|
|
if status:
|
|
conditions.append("m.analysis_status = ?")
|
|
params.append(status)
|
|
|
|
where_clause = " AND ".join(conditions) if conditions else "1=1"
|
|
|
|
query = f"""
|
|
SELECT m.*
|
|
FROM media m
|
|
WHERE {where_clause}
|
|
ORDER BY m.first_seen_at DESC
|
|
LIMIT ? OFFSET ?
|
|
"""
|
|
params.extend([limit, offset])
|
|
|
|
cursor = await db.execute(query, params)
|
|
rows = await cursor.fetchall()
|
|
|
|
return [dict(row) for row in rows]
|
|
|
|
|
|
@app.get("/api/media/{md5_hash}")
|
|
async def get_media_info(md5_hash: str):
|
|
"""Get full media info including all source URLs."""
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
|
|
# Get media record
|
|
cursor = await db.execute(
|
|
"SELECT * FROM media WHERE md5_hash = ?",
|
|
(md5_hash,)
|
|
)
|
|
media = await cursor.fetchone()
|
|
if not media:
|
|
raise HTTPException(status_code=404, detail="Media not found")
|
|
|
|
result = dict(media)
|
|
|
|
# Get all sources
|
|
cursor = await db.execute(
|
|
"SELECT media_uri, page_uri, page_title, page_description, page_keywords, alt_text, link_text, discovered_at FROM media_sources WHERE md5_hash = ?",
|
|
(md5_hash,)
|
|
)
|
|
result['sources'] = [dict(row) for row in await cursor.fetchall()]
|
|
|
|
return result
|
|
|
|
|
|
@app.get("/media/{md5_hash}")
|
|
async def serve_media(md5_hash: str):
|
|
"""
|
|
Serve media file from vault.
|
|
|
|
Caddy should be configured to cache these responses.
|
|
"""
|
|
# Find file in vault
|
|
subdir = VAULT_PATH / md5_hash[:2]
|
|
if not subdir.exists():
|
|
raise HTTPException(status_code=404, detail="Media not found")
|
|
|
|
# Find file with this hash prefix
|
|
for f in subdir.iterdir():
|
|
if f.name.startswith(md5_hash):
|
|
# Guess content type
|
|
mime_type, _ = mimetypes.guess_type(f.name)
|
|
if not mime_type:
|
|
# Try to get from database
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
cursor = await db.execute(
|
|
"SELECT mime_type FROM media WHERE md5_hash = ?",
|
|
(md5_hash,)
|
|
)
|
|
row = await cursor.fetchone()
|
|
if row and row[0]:
|
|
mime_type = row[0]
|
|
else:
|
|
mime_type = "application/octet-stream"
|
|
|
|
return FileResponse(
|
|
f,
|
|
media_type=mime_type,
|
|
headers={
|
|
"Cache-Control": "public, max-age=31536000, immutable",
|
|
"X-Content-Hash": md5_hash,
|
|
}
|
|
)
|
|
|
|
raise HTTPException(status_code=404, detail="Media not found")
|
|
|
|
|
|
# ============================================================================
|
|
# Crawler API
|
|
# ============================================================================
|
|
|
|
@app.get("/api/crawl/jobs")
|
|
async def get_crawl_jobs(limit: int = Query(50, le=200)):
|
|
"""Get recent crawl jobs."""
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
cursor = await db.execute(
|
|
"""
|
|
SELECT * FROM crawl_jobs
|
|
ORDER BY started_at DESC
|
|
LIMIT ?
|
|
""",
|
|
(limit,)
|
|
)
|
|
rows = await cursor.fetchall()
|
|
return [dict(row) for row in rows]
|
|
|
|
|
|
@app.get("/api/crawl/jobs/{job_id}")
|
|
async def get_crawl_job(job_id: int):
|
|
"""Get a specific crawl job."""
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
cursor = await db.execute(
|
|
"SELECT * FROM crawl_jobs WHERE id = ?",
|
|
(job_id,)
|
|
)
|
|
row = await cursor.fetchone()
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
return dict(row)
|
|
|
|
|
|
@app.post("/api/crawl")
|
|
async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
|
|
"""
|
|
Start new crawl job(s).
|
|
|
|
Supports multiple targets - creates one job per target.
|
|
The crawls run in the background. Poll /api/crawl/jobs/{id} for status.
|
|
"""
|
|
# Import neopig here to avoid circular imports
|
|
from neopig import NeoPig
|
|
from async_web_fetcher import CrawlMode
|
|
|
|
# Map mode string to enum
|
|
mode_map = {
|
|
"text": CrawlMode.TEXT,
|
|
"images": CrawlMode.IMAGES,
|
|
"videos": CrawlMode.VIDEOS,
|
|
"media": CrawlMode.MEDIA,
|
|
"all": CrawlMode.ALL,
|
|
}
|
|
mode = mode_map.get(request.mode, CrawlMode.IMAGES)
|
|
|
|
# Get targets (support both new 'targets' array and old 'target_uri' single value)
|
|
targets = request.targets if request.targets else [request.target_uri] if request.target_uri else []
|
|
if not targets:
|
|
raise HTTPException(status_code=400, detail="No target URIs provided")
|
|
|
|
job_ids = []
|
|
|
|
# Create a job for each target
|
|
for target_uri in targets:
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
cursor = await db.execute(
|
|
"""
|
|
INSERT INTO crawl_jobs (target_uri, keywords, mode, status, started_at)
|
|
VALUES (?, ?, ?, 'running', datetime('now'))
|
|
""",
|
|
(target_uri, json.dumps(request.keywords), request.mode)
|
|
)
|
|
await db.commit()
|
|
job_id = cursor.lastrowid
|
|
job_ids.append(job_id)
|
|
|
|
# Run crawl in background (closure captures job_id and target_uri)
|
|
async def run_crawl(jid=job_id, uri=target_uri):
|
|
try:
|
|
pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH))
|
|
await pig.init()
|
|
|
|
# Load existing seen media for resume capability
|
|
crawled_media = await pig.db.get_crawled_media_uris()
|
|
if crawled_media:
|
|
pig.seen_media = crawled_media
|
|
|
|
# Cap depth at 15 (convert -1 or values > 15 to 15)
|
|
depth = request.depth if 1 <= request.depth <= 15 else 15
|
|
|
|
stats = await pig.crawl(
|
|
target_uri=uri,
|
|
keywords=request.keywords,
|
|
mode=mode,
|
|
depth=depth,
|
|
max_pages=request.max_pages,
|
|
download_media=request.download_media,
|
|
)
|
|
|
|
# Update job as completed
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
await db.execute(
|
|
"""
|
|
UPDATE crawl_jobs
|
|
SET status = 'completed', completed_at = datetime('now'), stats = ?
|
|
WHERE id = ?
|
|
""",
|
|
(json.dumps(stats), jid)
|
|
)
|
|
await db.commit()
|
|
|
|
except Exception as e:
|
|
logger.error(f"Crawl job {jid} failed: {e}")
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
await db.execute(
|
|
"""
|
|
UPDATE crawl_jobs
|
|
SET status = 'failed', completed_at = datetime('now'), stats = ?
|
|
WHERE id = ?
|
|
""",
|
|
(json.dumps({"error": str(e)}), jid)
|
|
)
|
|
await db.commit()
|
|
|
|
# Schedule background task
|
|
background_tasks.add_task(asyncio.create_task, run_crawl())
|
|
|
|
return {"job_ids": job_ids, "status": "running", "count": len(job_ids)}
|
|
|
|
|
|
def main():
|
|
global DB_PATH, VAULT_PATH
|
|
|
|
parser = argparse.ArgumentParser(description="neopig SERP")
|
|
parser.add_argument("--port", type=int, default=8000)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--db", default="neopig.db")
|
|
parser.add_argument("--vault", default="vault")
|
|
|
|
args = parser.parse_args()
|
|
|
|
DB_PATH = args.db
|
|
VAULT_PATH = Path(args.vault)
|
|
|
|
logger.info(f"Starting neopig SERP on {args.host}:{args.port}")
|
|
logger.info(f"Database: {DB_PATH}")
|
|
logger.info(f"Vault: {VAULT_PATH}")
|
|
|
|
uvicorn.run(app, host=args.host, port=args.port)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|