Russell Ballestrini
623213f5b6
- Removed embedded serve.py (872 lines) from archive.py - bootstrap.c extracts neopig/*.py to /tmp and runs serp.py - serp.py: tarball mode serves media directly from tar.gz - OffsetFile wrapper for reading .run files at correct offset - ArchiveDB: simple SQLite wrapper for archive search (no async deps) - Archives bundle all neopig source files for self-contained operation - --upgrade-neopig flag with progress logging
2964 lines
106 KiB
Python
2964 lines
106 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
neopig SERP - Search Engine Results Page
|
|
|
|
Fast image/video search across hydrated metadata.
|
|
Serves files directly from filevault via Caddy with 1GB memory cache.
|
|
|
|
Search across:
|
|
- keywords (crawl tags)
|
|
- alt_text (image alt attributes)
|
|
- title (media titles)
|
|
- analysis_result (Qwen 3 VL descriptions)
|
|
- source_page / source_url (origin)
|
|
|
|
Usage:
|
|
python serp.py --port 8000 --vault ./vault --db neopig.db
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import mimetypes
|
|
from pathlib import Path
|
|
from typing import List, Dict, Any, Optional
|
|
|
|
from fastapi import FastAPI, Query, HTTPException, BackgroundTasks
|
|
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response
|
|
from fastapi.staticfiles import StaticFiles
|
|
from pydantic import BaseModel
|
|
from sqlalchemy import text
|
|
import uvicorn
|
|
|
|
from database import Database
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
app = FastAPI(title="neopig", description="Media crawler + SERP + Screenshot service")
|
|
|
|
# Try to include uri2png screenshot router (optional dependency)
|
|
try:
|
|
from uri2png import get_screenshot_router
|
|
app.include_router(get_screenshot_router())
|
|
logger.info("Screenshot router loaded from uri2png")
|
|
except ImportError:
|
|
logger.warning("uri2png not installed - screenshot endpoints not available")
|
|
|
|
# Config - set via startup
|
|
DB_PATH = "data/neopig.db"
|
|
VAULT_PATH = Path("data/vault")
|
|
|
|
# Global database instance
|
|
db: Database = None
|
|
|
|
# Active crawl jobs (in-memory tracking)
|
|
ACTIVE_CRAWLS: Dict[int, Dict[str, Any]] = {}
|
|
|
|
# Tarball mode - serve directly from tar.gz archive
|
|
import tarfile
|
|
import tempfile
|
|
TAR_FILE: tarfile.TarFile = None
|
|
TAR_MEMBERS: Dict[str, tarfile.TarInfo] = {}
|
|
ARCHIVE_ROOT: str = None # e.g., "example.com-20251230"
|
|
TEMP_DB_PATH: str = None # Extracted database (SQLite needs real file)
|
|
|
|
|
|
def read_from_tarball(path: str) -> bytes:
|
|
"""Read a file from the tarball. Path is relative to archive root."""
|
|
if not TAR_FILE or not ARCHIVE_ROOT:
|
|
return None
|
|
full_path = f"{ARCHIVE_ROOT}/{path}"
|
|
if full_path in TAR_MEMBERS:
|
|
member = TAR_MEMBERS[full_path]
|
|
f = TAR_FILE.extractfile(member)
|
|
if f:
|
|
return f.read()
|
|
return None
|
|
|
|
|
|
def find_media_in_tarball(md5_hash: str) -> tuple:
|
|
"""Find media file in tarball by hash. Returns (data, extension) or (None, None)."""
|
|
if not TAR_FILE or not ARCHIVE_ROOT:
|
|
return None, None
|
|
prefix = f"{ARCHIVE_ROOT}/media/{md5_hash}"
|
|
for name, member in TAR_MEMBERS.items():
|
|
if name.startswith(prefix):
|
|
f = TAR_FILE.extractfile(member)
|
|
if f:
|
|
ext = Path(name).suffix
|
|
return f.read(), ext
|
|
return None, None
|
|
|
|
|
|
class ArchiveDB:
|
|
"""Simple sync SQLite wrapper for archive.db (FTS5 search only)."""
|
|
def __init__(self, db_path):
|
|
import sqlite3
|
|
self.conn = sqlite3.connect(db_path)
|
|
self.conn.row_factory = sqlite3.Row
|
|
|
|
async def search_pages(self, query, limit=50):
|
|
cursor = self.conn.execute(
|
|
"SELECT uri, title, snippet(pages_fts, 2, '<b>', '</b>', '...', 32) as snippet "
|
|
"FROM pages_fts WHERE pages_fts MATCH ? LIMIT ?",
|
|
(query, limit)
|
|
)
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
async def get_stats(self):
|
|
cursor = self.conn.execute("SELECT COUNT(*) FROM pages")
|
|
return {"pages": cursor.fetchone()[0], "media": 0, "screenshots": 0}
|
|
|
|
|
|
@app.on_event("startup")
|
|
async def startup_event():
|
|
"""Initialize database on startup."""
|
|
global db
|
|
if TAR_FILE:
|
|
# Tarball mode: use simple archive DB
|
|
db = ArchiveDB(DB_PATH)
|
|
logger.info(f"Using archive database: {DB_PATH}")
|
|
else:
|
|
# Normal mode: use full async database
|
|
db = Database(DB_PATH)
|
|
await db.init() # Handles schema + WAL mode
|
|
VAULT_PATH.mkdir(parents=True, exist_ok=True)
|
|
logger.info(f"Vault directory ready: {VAULT_PATH}")
|
|
|
|
|
|
class CrawlRequest(BaseModel):
|
|
"""Request to start a new crawl."""
|
|
targets: List[str] = [] # Multiple target URIs
|
|
target_uri: str = "" # Deprecated: single target (for backwards compat)
|
|
keywords: List[str] = []
|
|
mode: str = "images" # text, images, videos, media, all
|
|
depth: int = -1 # -1 = unlimited
|
|
max_pages: int = -1 # -1 = unlimited
|
|
download_media: bool = True
|
|
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
async def index():
|
|
"""Simple search UI."""
|
|
return SEARCH_HTML
|
|
|
|
|
|
@app.get("/crawl", response_class=HTMLResponse)
|
|
async def crawl_page():
|
|
"""Crawler command page."""
|
|
return CRAWL_HTML
|
|
|
|
|
|
# HTML Templates
|
|
SEARCH_HTML = """
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>neopig SERP</title>
|
|
<style>
|
|
* { box-sizing: border-box; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0;
|
|
padding: 0;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}
|
|
.nav {
|
|
background: #1a1a1a;
|
|
padding: 10px 20px;
|
|
display: flex;
|
|
gap: 20px;
|
|
align-items: center;
|
|
border-bottom: 1px solid #333;
|
|
}
|
|
.nav a { color: #ff6b6b; text-decoration: none; }
|
|
.nav a:hover { text-decoration: underline; }
|
|
.nav .brand { font-weight: bold; font-size: 18px; }
|
|
.container { padding: 20px; }
|
|
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
|
.subtitle { color: #666; margin-bottom: 20px; }
|
|
.search-box {
|
|
display: flex;
|
|
gap: 10px;
|
|
margin-bottom: 20px;
|
|
}
|
|
input[type="text"] {
|
|
flex: 1;
|
|
padding: 12px 16px;
|
|
font-size: 16px;
|
|
border: 2px solid #333;
|
|
border-radius: 8px;
|
|
background: #1a1a1a;
|
|
color: #fff;
|
|
}
|
|
input[type="text"]:focus {
|
|
outline: none;
|
|
border-color: #ff6b6b;
|
|
}
|
|
select {
|
|
padding: 12px 16px;
|
|
font-size: 16px;
|
|
border: 2px solid #333;
|
|
border-radius: 8px;
|
|
background: #1a1a1a;
|
|
color: #fff;
|
|
}
|
|
button {
|
|
padding: 12px 24px;
|
|
font-size: 16px;
|
|
background: #ff6b6b;
|
|
color: #fff;
|
|
border: none;
|
|
border-radius: 8px;
|
|
cursor: pointer;
|
|
}
|
|
button:hover { background: #ff5252; }
|
|
.stats {
|
|
padding: 10px 15px;
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
margin-bottom: 20px;
|
|
font-size: 14px;
|
|
color: #888;
|
|
}
|
|
.section-title {
|
|
color: #ff6b6b;
|
|
font-size: 18px;
|
|
margin: 25px 0 15px 0;
|
|
border-bottom: 1px solid #333;
|
|
padding-bottom: 8px;
|
|
}
|
|
.results {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
|
|
gap: 15px;
|
|
}
|
|
.result {
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
overflow: hidden;
|
|
transition: transform 0.2s;
|
|
}
|
|
.result:hover {
|
|
transform: scale(1.02);
|
|
}
|
|
.result img, .result video {
|
|
width: 100%;
|
|
height: 200px;
|
|
object-fit: contain;
|
|
background: #1a1a1a;
|
|
}
|
|
.result-info {
|
|
padding: 10px;
|
|
}
|
|
.result-hash {
|
|
font-family: monospace;
|
|
font-size: 11px;
|
|
color: #666;
|
|
word-break: break-all;
|
|
}
|
|
.result-meta {
|
|
font-size: 12px;
|
|
color: #888;
|
|
margin-top: 5px;
|
|
}
|
|
.result-keywords {
|
|
display: flex;
|
|
flex-wrap: wrap;
|
|
gap: 5px;
|
|
margin-top: 8px;
|
|
}
|
|
.tag {
|
|
background: #333;
|
|
padding: 2px 8px;
|
|
border-radius: 4px;
|
|
font-size: 11px;
|
|
color: #aaa;
|
|
}
|
|
.no-results {
|
|
text-align: center;
|
|
padding: 60px;
|
|
color: #666;
|
|
}
|
|
.media-link {
|
|
display: block;
|
|
cursor: pointer;
|
|
}
|
|
.media-link:hover img, .media-link:hover video {
|
|
opacity: 0.8;
|
|
}
|
|
.result-hash {
|
|
color: #ff6b6b;
|
|
text-decoration: none;
|
|
}
|
|
.result-hash:hover {
|
|
text-decoration: underline;
|
|
}
|
|
/* Page results */
|
|
.page-results {
|
|
display: flex;
|
|
flex-direction: column;
|
|
gap: 10px;
|
|
}
|
|
.page-result {
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
padding: 15px;
|
|
transition: background 0.2s;
|
|
}
|
|
.page-result:hover {
|
|
background: #252525;
|
|
}
|
|
.page-result a {
|
|
color: #ff6b6b;
|
|
text-decoration: none;
|
|
font-size: 16px;
|
|
font-weight: 500;
|
|
}
|
|
.page-result a:hover {
|
|
text-decoration: underline;
|
|
}
|
|
.page-path {
|
|
font-size: 12px;
|
|
color: #4ade80;
|
|
margin-top: 4px;
|
|
font-family: monospace;
|
|
}
|
|
.page-snippet {
|
|
font-size: 13px;
|
|
color: #999;
|
|
margin-top: 8px;
|
|
line-height: 1.5;
|
|
}
|
|
.page-snippet mark {
|
|
background: #ff6b6b33;
|
|
color: #ff9999;
|
|
padding: 1px 3px;
|
|
border-radius: 2px;
|
|
}
|
|
/* Two-column layout: pages left, media right */
|
|
.search-columns {
|
|
display: grid;
|
|
grid-template-columns: 1fr;
|
|
gap: 30px;
|
|
align-items: start;
|
|
}
|
|
.search-columns.has-pages {
|
|
grid-template-columns: 1fr 2fr;
|
|
}
|
|
.search-columns .page-column {
|
|
display: none;
|
|
}
|
|
.search-columns.has-pages .page-column {
|
|
display: block;
|
|
position: sticky;
|
|
top: 20px;
|
|
max-height: 85vh;
|
|
overflow-y: auto;
|
|
}
|
|
@media (max-width: 1000px) {
|
|
.search-columns.has-pages {
|
|
grid-template-columns: 1fr;
|
|
}
|
|
.search-columns.has-pages .page-column {
|
|
position: static;
|
|
max-height: none;
|
|
}
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="nav">
|
|
<a href="/" class="brand">🐷 neopig</a>
|
|
<a href="/">Search</a>
|
|
<a href="/live">Live</a>
|
|
<a href="/random">Random</a>
|
|
<a href="/crawl">Crawl</a>
|
|
<a href="/phantom">Phantom</a>
|
|
</div>
|
|
<div class="container">
|
|
|
|
<div class="search-box">
|
|
<input type="text" id="query" placeholder="Search keywords, alt text, page content..." autofocus>
|
|
<select id="type">
|
|
<option value="">All types</option>
|
|
<option value="image">Images</option>
|
|
<option value="video">Videos</option>
|
|
<option value="audio">Audio</option>
|
|
</select>
|
|
<button onclick="search()">Search</button>
|
|
</div>
|
|
|
|
<div class="stats" id="stats">Loading stats...</div>
|
|
|
|
<div class="search-columns">
|
|
<div class="page-column" id="page-section">
|
|
<h2 class="section-title">Pages</h2>
|
|
<div class="page-results" id="page-results"></div>
|
|
</div>
|
|
|
|
<div class="media-column" id="media-section">
|
|
<h2 class="section-title">Media</h2>
|
|
<div class="results" id="results"></div>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
async function loadStats() {
|
|
const res = await fetch('/api/stats');
|
|
const stats = await res.json();
|
|
document.getElementById('stats').innerHTML =
|
|
`<strong>${stats.total_media}</strong> media | ` +
|
|
`<strong>${stats.by_type?.image || 0}</strong> images | ` +
|
|
`<strong>${stats.by_type?.video || 0}</strong> videos | ` +
|
|
`<strong>${stats.total_sources}</strong> sources | ` +
|
|
`<strong>${stats.total_pages || 0}</strong> pages`;
|
|
}
|
|
|
|
async function search() {
|
|
const query = document.getElementById('query').value;
|
|
const type = document.getElementById('type').value;
|
|
|
|
// Search media
|
|
let mediaUrl = `/api/search?q=${encodeURIComponent(query)}&limit=100`;
|
|
if (type) mediaUrl += `&type=${type}`;
|
|
|
|
const mediaRes = await fetch(mediaUrl);
|
|
const mediaResults = await mediaRes.json();
|
|
|
|
const mediaContainer = document.getElementById('results');
|
|
const mediaSection = document.getElementById('media-section');
|
|
|
|
if (mediaResults.length === 0) {
|
|
mediaContainer.innerHTML = '<div class="no-results">No media found</div>';
|
|
} else {
|
|
mediaContainer.innerHTML = mediaResults.map(r => {
|
|
const isVideo = r.media_type === 'video';
|
|
const mediaEl = isVideo
|
|
? `<video src="/media/${r.md5_hash}" muted loop preload="metadata" onmouseenter="this.play()" onmouseleave="this.pause()"></video>`
|
|
: `<img src="/media/${r.md5_hash}" alt="${r.alt_text || ''}" loading="lazy">`;
|
|
|
|
const keywords = JSON.parse(r.keywords || '[]');
|
|
const tagsHtml = keywords.map(k => `<span class="tag">${k}</span>`).join('');
|
|
|
|
return `
|
|
<div class="result">
|
|
<a href="/view/${r.md5_hash}" class="media-link">
|
|
${mediaEl}
|
|
</a>
|
|
<div class="result-info">
|
|
<a href="/view/${r.md5_hash}" class="result-hash">${r.md5_hash}</a>
|
|
<div class="result-meta">
|
|
${r.media_type} · ${formatBytes(r.file_size)}
|
|
${r.alt_text ? ` · ${r.alt_text.substring(0, 50)}` : ''}
|
|
</div>
|
|
<div class="result-keywords">${tagsHtml}</div>
|
|
</div>
|
|
</div>
|
|
`;
|
|
}).join('');
|
|
}
|
|
|
|
// Search pages
|
|
const pageContainer = document.getElementById('page-results');
|
|
const searchColumns = document.querySelector('.search-columns');
|
|
|
|
if (query.trim()) {
|
|
const pageRes = await fetch(`/api/search/pages?q=${encodeURIComponent(query)}&limit=30`);
|
|
const pageResults = await pageRes.json();
|
|
|
|
if (pageResults.length > 0) {
|
|
searchColumns.classList.add('has-pages');
|
|
pageContainer.innerHTML = pageResults.map(p => `
|
|
<div class="page-result">
|
|
<a href="/page/view?uri=${encodeURIComponent(p.uri)}">${p.title || p.uri}</a>
|
|
<div class="page-path">${p.path || p.uri}</div>
|
|
<div class="page-snippet">${p.snippet || ''}</div>
|
|
</div>
|
|
`).join('');
|
|
} else {
|
|
searchColumns.classList.remove('has-pages');
|
|
pageContainer.innerHTML = '';
|
|
}
|
|
} else {
|
|
searchColumns.classList.remove('has-pages');
|
|
pageContainer.innerHTML = '';
|
|
}
|
|
}
|
|
|
|
function formatBytes(bytes) {
|
|
if (!bytes) return '?';
|
|
if (bytes < 1024) return bytes + ' B';
|
|
if (bytes < 1024*1024) return (bytes/1024).toFixed(1) + ' KB';
|
|
return (bytes/1024/1024).toFixed(1) + ' MB';
|
|
}
|
|
|
|
// Enter key to search
|
|
document.getElementById('query').addEventListener('keypress', e => {
|
|
if (e.key === 'Enter') search();
|
|
});
|
|
|
|
// Load stats on page load
|
|
loadStats();
|
|
|
|
// Check for query param from nav search
|
|
const urlParams = new URLSearchParams(window.location.search);
|
|
const q = urlParams.get('q');
|
|
if (q) {
|
|
document.getElementById('query').value = q;
|
|
}
|
|
|
|
// Initial search (show all media or query)
|
|
search();
|
|
</script>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
CRAWL_HTML = """
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>neopig Crawler</title>
|
|
<style>
|
|
* { box-sizing: border-box; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0;
|
|
padding: 0;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}
|
|
.nav {
|
|
background: #1a1a1a;
|
|
padding: 10px 20px;
|
|
display: flex;
|
|
gap: 20px;
|
|
align-items: center;
|
|
border-bottom: 1px solid #333;
|
|
}
|
|
.nav a { color: #ff6b6b; text-decoration: none; }
|
|
.nav a:hover { text-decoration: underline; }
|
|
.nav .brand { font-weight: bold; font-size: 18px; }
|
|
.container { padding: 20px; }
|
|
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
|
.subtitle { color: #666; margin-bottom: 20px; }
|
|
|
|
.form-group {
|
|
margin-bottom: 15px;
|
|
}
|
|
label {
|
|
display: block;
|
|
margin-bottom: 5px;
|
|
color: #aaa;
|
|
font-size: 14px;
|
|
}
|
|
input[type="text"], input[type="number"], select {
|
|
width: 100%;
|
|
padding: 12px 16px;
|
|
font-size: 16px;
|
|
border: 2px solid #333;
|
|
border-radius: 8px;
|
|
background: #1a1a1a;
|
|
color: #fff;
|
|
}
|
|
input:focus, select:focus {
|
|
outline: none;
|
|
border-color: #ff6b6b;
|
|
}
|
|
.row {
|
|
display: flex;
|
|
gap: 15px;
|
|
}
|
|
.row > div { flex: 1; }
|
|
|
|
button {
|
|
padding: 14px 28px;
|
|
font-size: 16px;
|
|
background: #ff6b6b;
|
|
color: #fff;
|
|
border: none;
|
|
border-radius: 8px;
|
|
cursor: pointer;
|
|
margin-top: 10px;
|
|
}
|
|
button:hover { background: #ff5252; }
|
|
button:disabled {
|
|
background: #444;
|
|
cursor: not-allowed;
|
|
}
|
|
button.secondary {
|
|
background: #333;
|
|
}
|
|
button.secondary:hover {
|
|
background: #444;
|
|
}
|
|
|
|
.checkbox-group {
|
|
display: flex;
|
|
align-items: center;
|
|
gap: 8px;
|
|
}
|
|
.checkbox-group input {
|
|
width: auto;
|
|
}
|
|
|
|
.jobs-section {
|
|
margin-top: 30px;
|
|
padding-top: 20px;
|
|
border-top: 1px solid #333;
|
|
}
|
|
h2 {
|
|
color: #ff6b6b;
|
|
font-size: 18px;
|
|
margin-bottom: 15px;
|
|
}
|
|
|
|
.job {
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
padding: 15px;
|
|
margin-bottom: 10px;
|
|
}
|
|
.job-header {
|
|
display: flex;
|
|
justify-content: space-between;
|
|
align-items: center;
|
|
margin-bottom: 10px;
|
|
}
|
|
.job-id {
|
|
font-family: monospace;
|
|
color: #666;
|
|
}
|
|
.job-status {
|
|
padding: 4px 10px;
|
|
border-radius: 4px;
|
|
font-size: 12px;
|
|
font-weight: bold;
|
|
}
|
|
.job-status.running { background: #2d5a27; color: #7bed72; }
|
|
.job-status.completed { background: #1a3a4a; color: #6bc5e8; }
|
|
.job-status.failed { background: #5a2727; color: #ed7272; }
|
|
|
|
.job-target {
|
|
font-size: 14px;
|
|
word-break: break-all;
|
|
margin-bottom: 5px;
|
|
}
|
|
.job-meta {
|
|
font-size: 12px;
|
|
color: #666;
|
|
}
|
|
.job-stats {
|
|
display: flex;
|
|
gap: 15px;
|
|
margin-top: 10px;
|
|
font-size: 13px;
|
|
}
|
|
.job-stats span {
|
|
background: #252525;
|
|
padding: 4px 10px;
|
|
border-radius: 4px;
|
|
}
|
|
|
|
.progress-bar {
|
|
height: 4px;
|
|
background: #333;
|
|
border-radius: 2px;
|
|
margin-top: 10px;
|
|
overflow: hidden;
|
|
}
|
|
.progress-bar-fill {
|
|
height: 100%;
|
|
background: #ff6b6b;
|
|
transition: width 0.3s;
|
|
}
|
|
|
|
.no-jobs {
|
|
color: #666;
|
|
text-align: center;
|
|
padding: 30px;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="nav">
|
|
<a href="/" class="brand">🐷 neopig</a>
|
|
<a href="/">Search</a>
|
|
<a href="/live">Live</a>
|
|
<a href="/random">Random</a>
|
|
<a href="/crawl">Crawl</a>
|
|
<a href="/phantom">Phantom</a>
|
|
</div>
|
|
<div class="container">
|
|
|
|
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
|
<input type="text" name="q" placeholder="Search keywords, alt text, page content..." style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<option value="">All types</option>
|
|
<option value="image">Images</option>
|
|
<option value="video">Videos</option>
|
|
<option value="audio">Audio</option>
|
|
</select>
|
|
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">Search</button>
|
|
</form>
|
|
|
|
<h1>Crawler</h1>
|
|
<p class="subtitle">Hydrate media from the web</p>
|
|
|
|
<form id="crawl-form" onsubmit="startCrawl(event)">
|
|
<div class="form-group">
|
|
<label>Target URIs (space, comma, or newline separated)</label>
|
|
<textarea id="target" placeholder="https://example.com https://another.com" rows="2" required style="width:100%;padding:10px;border:1px solid #333;border-radius:4px;background:#1a1a1a;color:#e0e0e0;font-size:14px;resize:vertical;"></textarea>
|
|
</div>
|
|
|
|
<div class="form-group">
|
|
<label>Keywords (space or comma separated)</label>
|
|
<input type="text" id="keywords" placeholder="rick and morty, adult swim">
|
|
</div>
|
|
|
|
<div class="row">
|
|
<div class="form-group">
|
|
<label>Mode</label>
|
|
<select id="mode">
|
|
<option value="images">Images only</option>
|
|
<option value="videos">Videos only</option>
|
|
<option value="media">All media (images + videos + audio)</option>
|
|
<option value="all">Everything (text + media)</option>
|
|
<option value="text">Text only</option>
|
|
</select>
|
|
</div>
|
|
<div class="form-group">
|
|
<label>Depth (5 recommended, max 15)</label>
|
|
<input type="number" id="depth" value="5" min="1" max="15">
|
|
</div>
|
|
<div class="form-group">
|
|
<label>Max Pages (-1 = unlimited)</label>
|
|
<input type="number" id="max_pages" value="500" min="-1">
|
|
</div>
|
|
</div>
|
|
|
|
<div class="form-group checkbox-group">
|
|
<input type="checkbox" id="download" checked>
|
|
<label for="download" style="display:inline; margin:0;">Download media (uncheck to just index URLs)</label>
|
|
</div>
|
|
|
|
<button type="submit" id="start-btn">Start Crawl</button>
|
|
</form>
|
|
|
|
<div class="jobs-section">
|
|
<h2>Crawl Jobs</h2>
|
|
<div id="jobs">Loading...</div>
|
|
</div>
|
|
|
|
<script>
|
|
async function startCrawl(e) {
|
|
e.preventDefault();
|
|
|
|
const btn = document.getElementById('start-btn');
|
|
btn.disabled = true;
|
|
btn.textContent = 'Starting...';
|
|
|
|
const keywordsRaw = document.getElementById('keywords').value;
|
|
const keywords = keywordsRaw
|
|
.split(/[,\\s]+/)
|
|
.map(k => k.trim())
|
|
.filter(k => k.length > 0);
|
|
|
|
// Parse multiple target URIs (space, comma, or newline separated)
|
|
const targetsRaw = document.getElementById('target').value;
|
|
const targets = targetsRaw
|
|
.split(/[,\\s\\n]+/)
|
|
.map(t => t.trim())
|
|
.filter(t => t.length > 0 && t.startsWith('http'));
|
|
|
|
if (targets.length === 0) {
|
|
alert('Please enter at least one valid URI (must start with http)');
|
|
btn.disabled = false;
|
|
btn.textContent = 'Start Crawl';
|
|
return;
|
|
}
|
|
|
|
const payload = {
|
|
targets: targets,
|
|
keywords: keywords,
|
|
mode: document.getElementById('mode').value,
|
|
depth: parseInt(document.getElementById('depth').value),
|
|
max_pages: parseInt(document.getElementById('max_pages').value),
|
|
download_media: document.getElementById('download').checked
|
|
};
|
|
|
|
try {
|
|
const res = await fetch('/api/crawl', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(payload)
|
|
});
|
|
|
|
if (!res.ok) {
|
|
const err = await res.json();
|
|
alert('Error: ' + (err.detail || 'Failed to start crawl'));
|
|
} else {
|
|
const result = await res.json();
|
|
const jobIds = result.job_ids || [result.job_id];
|
|
alert('Crawl started! Job IDs: ' + jobIds.join(', '));
|
|
loadJobs();
|
|
}
|
|
} catch (err) {
|
|
alert('Error: ' + err.message);
|
|
}
|
|
|
|
btn.disabled = false;
|
|
btn.textContent = 'Start Crawl';
|
|
}
|
|
|
|
async function loadJobs() {
|
|
try {
|
|
const res = await fetch('/api/crawl/jobs');
|
|
const jobs = await res.json();
|
|
|
|
const container = document.getElementById('jobs');
|
|
|
|
if (jobs.length === 0) {
|
|
container.innerHTML = '<div class="no-jobs">No crawl jobs yet</div>';
|
|
return;
|
|
}
|
|
|
|
container.innerHTML = jobs.map(job => {
|
|
const stats = job.stats ? JSON.parse(job.stats) : {};
|
|
const keywords = job.keywords ? JSON.parse(job.keywords) : [];
|
|
|
|
return `
|
|
<div class="job">
|
|
<div class="job-header">
|
|
<span class="job-id">Job #${job.id}</span>
|
|
<span class="job-status ${job.status}">${job.status}</span>
|
|
</div>
|
|
<div class="job-target">${job.target_uri}</div>
|
|
<div class="job-meta">
|
|
Mode: ${job.mode || 'images'} |
|
|
Keywords: ${keywords.join(', ') || 'none'} |
|
|
Started: ${new Date(job.started_at).toLocaleString()}
|
|
</div>
|
|
${job.status === 'completed' ? `
|
|
<div class="job-stats">
|
|
<span>📄 ${stats.pages_crawled || 0} pages</span>
|
|
<span>🖼️ ${stats.media_found || 0} found</span>
|
|
<span>💾 ${stats.media_downloaded || 0} saved</span>
|
|
<span>♻️ ${stats.duplicates_skipped || 0} dupes</span>
|
|
</div>
|
|
` : ''}
|
|
${job.status === 'running' ? `
|
|
<div class="progress-bar">
|
|
<div class="progress-bar-fill" style="width: 50%"></div>
|
|
</div>
|
|
` : ''}
|
|
</div>
|
|
`;
|
|
}).join('');
|
|
} catch (err) {
|
|
document.getElementById('jobs').innerHTML = '<div class="no-jobs">Failed to load jobs</div>';
|
|
}
|
|
}
|
|
|
|
// Load jobs on page load
|
|
loadJobs();
|
|
|
|
// Refresh jobs every 5 seconds
|
|
setInterval(loadJobs, 5000);
|
|
</script>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
LIVE_HTML = """
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>neopig LIVE - Watch Images Crawl In</title>
|
|
<style>
|
|
* { box-sizing: border-box; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0;
|
|
padding: 0;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}
|
|
.nav {
|
|
background: #1a1a1a;
|
|
padding: 10px 20px;
|
|
display: flex;
|
|
gap: 20px;
|
|
align-items: center;
|
|
border-bottom: 1px solid #333;
|
|
}
|
|
.nav a { color: #ff6b6b; text-decoration: none; }
|
|
.nav a:hover { text-decoration: underline; }
|
|
.nav .brand { font-weight: bold; font-size: 18px; }
|
|
.container { padding: 20px; }
|
|
h1 { color: #ff6b6b; margin-bottom: 5px; }
|
|
.subtitle { color: #666; margin-bottom: 10px; }
|
|
.stats {
|
|
background: #1a1a1a;
|
|
padding: 15px;
|
|
border-radius: 8px;
|
|
margin-bottom: 20px;
|
|
display: flex;
|
|
gap: 30px;
|
|
flex-wrap: wrap;
|
|
}
|
|
.stat { display: flex; flex-direction: column; }
|
|
.stat-value { font-size: 24px; font-weight: bold; color: #ff6b6b; }
|
|
.stat-label { font-size: 12px; color: #888; }
|
|
.controls {
|
|
margin-bottom: 20px;
|
|
display: flex;
|
|
gap: 10px;
|
|
align-items: center;
|
|
}
|
|
.controls button {
|
|
padding: 8px 16px;
|
|
border: none;
|
|
border-radius: 4px;
|
|
cursor: pointer;
|
|
font-size: 14px;
|
|
}
|
|
.controls button.primary { background: #ff6b6b; color: white; }
|
|
.controls button.secondary { background: #333; color: #ddd; }
|
|
.controls button:hover { opacity: 0.8; }
|
|
.status { color: #4ade80; font-size: 14px; }
|
|
.status.paused { color: #fbbf24; }
|
|
.live-grid {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
|
|
gap: 15px;
|
|
}
|
|
.live-card {
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
overflow: hidden;
|
|
animation: fadeIn 0.5s ease-out;
|
|
}
|
|
@keyframes fadeIn {
|
|
from { opacity: 0; transform: translateY(20px); }
|
|
to { opacity: 1; transform: translateY(0); }
|
|
}
|
|
.live-card.new {
|
|
box-shadow: 0 0 20px rgba(255, 107, 107, 0.5);
|
|
}
|
|
.live-card img, .live-card video {
|
|
width: 100%;
|
|
height: 180px;
|
|
object-fit: contain;
|
|
background: #222;
|
|
}
|
|
.live-card-info {
|
|
padding: 10px;
|
|
}
|
|
.live-card-title {
|
|
font-size: 12px;
|
|
color: #888;
|
|
overflow: hidden;
|
|
text-overflow: ellipsis;
|
|
white-space: nowrap;
|
|
}
|
|
.live-card-source {
|
|
font-size: 10px;
|
|
color: #555;
|
|
margin-top: 4px;
|
|
overflow: hidden;
|
|
text-overflow: ellipsis;
|
|
white-space: nowrap;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="nav">
|
|
<a href="/" class="brand">🐷 neopig</a>
|
|
<a href="/">Search</a>
|
|
<a href="/live">Live</a>
|
|
<a href="/random">Random</a>
|
|
<a href="/crawl">Crawl</a>
|
|
<a href="/phantom">Phantom</a>
|
|
</div>
|
|
<div class="container">
|
|
|
|
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
|
<input type="text" name="q" placeholder="Search keywords, alt text, page content..." style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<option value="">All types</option>
|
|
<option value="image">Images</option>
|
|
<option value="video">Videos</option>
|
|
<option value="audio">Audio</option>
|
|
</select>
|
|
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">Search</button>
|
|
</form>
|
|
|
|
<h1>Live Feed</h1>
|
|
<p class="subtitle">Watch images appear as they're crawled</p>
|
|
|
|
<div class="stats">
|
|
<div class="stat">
|
|
<span class="stat-value" id="total-count">0</span>
|
|
<span class="stat-label">Total Images</span>
|
|
</div>
|
|
<div class="stat">
|
|
<span class="stat-value" id="new-count">0</span>
|
|
<span class="stat-label">New This Session</span>
|
|
</div>
|
|
<div class="stat">
|
|
<span class="stat-value" id="rate">0</span>
|
|
<span class="stat-label">Per Minute</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="controls">
|
|
<button class="primary" id="toggle-btn" onclick="toggleFeed()">Pause</button>
|
|
<button class="secondary" onclick="clearFeed()">Clear</button>
|
|
<span class="status" id="status">Watching for new images...</span>
|
|
</div>
|
|
|
|
<div class="live-grid" id="grid"></div>
|
|
|
|
<script>
|
|
let running = true;
|
|
let lastCheck = new Date().toISOString();
|
|
let seenHashes = new Set();
|
|
let newCount = 0;
|
|
let startTime = Date.now();
|
|
|
|
function toggleFeed() {
|
|
running = !running;
|
|
const btn = document.getElementById('toggle-btn');
|
|
const status = document.getElementById('status');
|
|
if (running) {
|
|
btn.textContent = 'Pause';
|
|
status.textContent = 'Watching for new images...';
|
|
status.className = 'status';
|
|
poll();
|
|
} else {
|
|
btn.textContent = 'Resume';
|
|
status.textContent = 'Paused';
|
|
status.className = 'status paused';
|
|
}
|
|
}
|
|
|
|
function clearFeed() {
|
|
document.getElementById('grid').innerHTML = '';
|
|
seenHashes.clear();
|
|
newCount = 0;
|
|
startTime = Date.now();
|
|
updateStats(0);
|
|
}
|
|
|
|
function updateStats(total) {
|
|
document.getElementById('total-count').textContent = total;
|
|
document.getElementById('new-count').textContent = newCount;
|
|
|
|
const minutes = (Date.now() - startTime) / 60000;
|
|
const rate = minutes > 0 ? Math.round(newCount / minutes) : 0;
|
|
document.getElementById('rate').textContent = rate;
|
|
}
|
|
|
|
async function poll() {
|
|
if (!running) return;
|
|
|
|
try {
|
|
// Get recent media sorted by first_seen_at descending
|
|
const res = await fetch('/api/search?q=&limit=50');
|
|
const media = await res.json();
|
|
|
|
// Get stats
|
|
const statsRes = await fetch('/api/stats');
|
|
const stats = await statsRes.json();
|
|
updateStats(stats.total_media || 0);
|
|
|
|
const grid = document.getElementById('grid');
|
|
|
|
// Find new items (on first poll, show all; after that only new ones)
|
|
const isFirstPoll = seenHashes.size === 0;
|
|
const newItems = media.filter(m => !seenHashes.has(m.md5_hash));
|
|
|
|
// Add new items to the top (or all items on first load)
|
|
newItems.reverse().forEach(item => {
|
|
seenHashes.add(item.md5_hash);
|
|
newCount++;
|
|
|
|
const card = document.createElement('div');
|
|
card.className = 'live-card new';
|
|
|
|
const isVideo = item.media_type === 'video';
|
|
const mediaEl = isVideo
|
|
? `<video src="/media/${item.md5_hash}" muted loop onmouseenter="this.play()" onmouseleave="this.pause()"></video>`
|
|
: `<img src="/media/${item.md5_hash}" loading="lazy" onerror="this.src='data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 width=%22200%22 height=%22180%22><rect fill=%22%23333%22 width=%22200%22 height=%22180%22/><text x=%2250%%22 y=%2250%%22 fill=%22%23666%22 text-anchor=%22middle%22>Error</text></svg>'">`;
|
|
|
|
card.innerHTML = `
|
|
<a href="/view/${item.md5_hash}">
|
|
${mediaEl}
|
|
</a>
|
|
<div class="live-card-info">
|
|
<div class="live-card-title">${item.alt_text || item.title || item.md5_hash.slice(0,12)}</div>
|
|
<div class="live-card-source">${item.media_type} - ${formatSize(item.file_size)}</div>
|
|
</div>
|
|
`;
|
|
|
|
grid.insertBefore(card, grid.firstChild);
|
|
|
|
// Remove 'new' highlight after animation
|
|
setTimeout(() => card.classList.remove('new'), 2000);
|
|
});
|
|
|
|
// Update stats
|
|
updateStats(stats.total_media || 0);
|
|
|
|
} catch (err) {
|
|
console.error('Poll error:', err);
|
|
}
|
|
|
|
// Poll again
|
|
setTimeout(poll, 2000);
|
|
}
|
|
|
|
function formatSize(bytes) {
|
|
if (!bytes) return '?';
|
|
if (bytes < 1024) return bytes + ' B';
|
|
if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + ' KB';
|
|
return (bytes / (1024 * 1024)).toFixed(1) + ' MB';
|
|
}
|
|
|
|
// Start polling
|
|
poll();
|
|
</script>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
@app.get("/live", response_class=HTMLResponse)
|
|
async def live_page():
|
|
"""Live feed page - watch images appear as they're crawled."""
|
|
return LIVE_HTML
|
|
|
|
|
|
@app.get("/view/{md5_hash}", response_class=HTMLResponse)
|
|
async def view_media_page(md5_hash: str):
|
|
"""Detail view page for a single media item."""
|
|
async with db.session() as session:
|
|
result = await session.execute(
|
|
text("SELECT * FROM media WHERE md5_hash = :hash"),
|
|
{'hash': md5_hash}
|
|
)
|
|
media = result.fetchone()
|
|
if not media:
|
|
raise HTTPException(status_code=404, detail="Media not found")
|
|
|
|
result = await session.execute(
|
|
text("""SELECT media_uri, page_uri, page_title, page_content,
|
|
detail_page_uri, detail_title, detail_content, discovered_at
|
|
FROM media_sources WHERE md5_hash = :hash"""),
|
|
{'hash': md5_hash}
|
|
)
|
|
sources = result.fetchall()
|
|
|
|
media = dict(media._mapping)
|
|
sources = [dict(s._mapping) for s in sources]
|
|
keywords = json.loads(media.get('keywords') or '[]')
|
|
|
|
# Generate download filename
|
|
# Priority: alt_text -> title -> (page_title + consistent index)
|
|
name_source = media.get('alt_text') or media.get('title')
|
|
if not name_source and sources:
|
|
page_title = sources[0].get('page_title', '')
|
|
# Use first 4 hex chars of md5 as consistent index (0-65535)
|
|
media_idx = int(md5_hash[:4], 16)
|
|
name_source = f"{page_title}-{media_idx}" if page_title else f"media-{media_idx}"
|
|
|
|
download_name = slugify(name_source or f"media-{md5_hash[:8]}")
|
|
|
|
# Get extension from mime type
|
|
ext_map = {
|
|
'image/jpeg': '.jpg', 'image/png': '.png', 'image/gif': '.gif',
|
|
'image/webp': '.webp', 'image/svg+xml': '.svg', 'image/bmp': '.bmp',
|
|
'video/mp4': '.mp4', 'video/webm': '.webm', 'video/quicktime': '.mov',
|
|
'audio/mpeg': '.mp3', 'audio/wav': '.wav', 'audio/ogg': '.ogg',
|
|
}
|
|
ext = ext_map.get(media.get('mime_type', ''), '.bin')
|
|
download_filename = f"{download_name}{ext}"
|
|
|
|
# Display title: alt_text -> title -> page_title -> hash
|
|
display_title = media.get('alt_text') or media.get('title')
|
|
if not display_title and sources:
|
|
display_title = sources[0].get('page_title')
|
|
if not display_title:
|
|
display_title = f"Media {md5_hash[:12]}"
|
|
|
|
is_video = media['media_type'] == 'video'
|
|
is_audio = media['media_type'] == 'audio'
|
|
|
|
if is_video:
|
|
media_html = f'<video src="/media/{md5_hash}" controls muted loop style="max-width:100%;max-height:70vh;" onmouseenter="this.play()" onmouseleave="this.pause()"></video>'
|
|
elif is_audio:
|
|
media_html = f'<audio src="/media/{md5_hash}" controls></audio>'
|
|
else:
|
|
media_html = f'<img src="/media/{md5_hash}" alt="{media.get("alt_text") or ""}" style="max-width:100%;max-height:70vh;">'
|
|
|
|
# Build sources - show all pages that embed this image
|
|
# Prioritize more specific pages (longer paths) over generic ones
|
|
sorted_sources = sorted(sources, key=lambda s: len(s["page_uri"] or ""), reverse=True)
|
|
|
|
sources_html = ''.join([
|
|
f'<li><a href="{s["page_uri"]}" target="_blank">{s["page_uri"]}</a></li>'
|
|
for s in sorted_sources
|
|
])
|
|
|
|
# Also show the direct media URL(s) separately
|
|
media_urls = list(set(s["media_uri"] for s in sources))
|
|
media_urls_html = ''.join([
|
|
f'<li><a href="{url}" target="_blank">{url}</a></li>'
|
|
for url in media_urls
|
|
])
|
|
|
|
keywords_html = ''.join([f'<span class="tag">{k}</span>' for k in keywords])
|
|
|
|
# Get page URI for content lookup
|
|
page_uri = sorted_sources[0]["page_uri"] if sorted_sources else None
|
|
|
|
# Get page content - prefer markdown, fallback to raw HTML or text
|
|
page_content_html = ""
|
|
|
|
if page_uri:
|
|
page_row = await db.get_page_by_uri(page_uri)
|
|
|
|
if page_row:
|
|
import html as html_module
|
|
import re
|
|
page_title = page_row.get("title") or ""
|
|
|
|
# Render markdown to HTML with our stylesheet
|
|
if page_row.get("markdown"):
|
|
# Render markdown to HTML
|
|
try:
|
|
import markdown
|
|
md_converter = markdown.Markdown(extensions=['fenced_code', 'tables', 'nl2br'])
|
|
rendered = md_converter.convert(page_row["markdown"][:100000])
|
|
|
|
# Hydrate: rewrite image URLs to use our vault
|
|
img_pattern = re.compile(r'<img[^>]+src=["\']([^"\']+)["\']', re.IGNORECASE)
|
|
img_urls = img_pattern.findall(rendered)
|
|
|
|
if img_urls:
|
|
from urllib.parse import urljoin
|
|
resolved_urls = {}
|
|
for url in img_urls:
|
|
if url.startswith(('http://', 'https://', '//')):
|
|
resolved_urls[url] = url
|
|
else:
|
|
resolved_urls[url] = urljoin(page_uri, url)
|
|
|
|
# Look up md5_hash for resolved URLs
|
|
all_urls = list(set(resolved_urls.values()))
|
|
resolved_to_hash = await db.lookup_media_by_uris(all_urls)
|
|
|
|
# Fallback: for URLs not found, try matching by filename
|
|
missing_urls = [u for u in all_urls if u not in resolved_to_hash]
|
|
if missing_urls:
|
|
from pathlib import Path as P
|
|
for murl in missing_urls:
|
|
fname = P(murl).stem
|
|
if len(fname) >= 20:
|
|
result = await db.lookup_media_by_filename(fname)
|
|
if result:
|
|
resolved_to_hash[murl] = result[1]
|
|
|
|
# Replace original URLs with vault paths
|
|
for orig_url, resolved_url in resolved_urls.items():
|
|
if resolved_url in resolved_to_hash:
|
|
md5 = resolved_to_hash[resolved_url]
|
|
rendered = rendered.replace(f'src="{orig_url}"', f'src="/media/{md5}"')
|
|
rendered = rendered.replace(f"src='{orig_url}'", f'src="/media/{md5}"')
|
|
|
|
# Look for screenshot of this page
|
|
screenshot_html = ""
|
|
screenshot_hash = await db.get_page_screenshot(page_uri, exclude_hash=md5_hash)
|
|
if screenshot_hash:
|
|
screenshot_html = f'''
|
|
<div class="page-screenshot">
|
|
<a href="/media/{screenshot_hash}" target="_blank">
|
|
<img src="/media/{screenshot_hash}" alt="Page screenshot" loading="lazy">
|
|
</a>
|
|
</div>'''
|
|
|
|
if screenshot_html:
|
|
page_content_html = f'''
|
|
<div class="page-content">
|
|
<h3>Page Context{(' - ' + html_module.escape(page_title)) if page_title else ''}</h3>
|
|
<div class="page-context-grid">
|
|
<div class="content-rendered">{rendered}</div>
|
|
{screenshot_html}
|
|
</div>
|
|
</div>
|
|
'''
|
|
else:
|
|
page_content_html = f'''
|
|
<div class="page-content">
|
|
<h3>Page Context{(' - ' + html_module.escape(page_title)) if page_title else ''}</h3>
|
|
<div class="content-rendered">{rendered}</div>
|
|
</div>
|
|
'''
|
|
except ImportError:
|
|
escaped = html_module.escape(page_row["markdown"][:50000])
|
|
page_content_html = f'''
|
|
<div class="page-content">
|
|
<h3>Page Context{(' - ' + html_module.escape(page_title)) if page_title else ''}</h3>
|
|
<div class="content-text"><pre style="white-space:pre-wrap;">{escaped}</pre></div>
|
|
</div>
|
|
'''
|
|
elif page_row.get("content"):
|
|
content = page_row["content"][:50000]
|
|
escaped = html_module.escape(content)
|
|
paragraphs = escaped.split('\n\n')
|
|
formatted = ''.join(f'<p>{p.replace(chr(10), "<br>")}</p>' for p in paragraphs if p.strip())
|
|
page_content_html = f'''
|
|
<div class="page-content">
|
|
<h3>Page Context{(' - ' + html_module.escape(page_title)) if page_title else ''}</h3>
|
|
<div class="content-text">{formatted}</div>
|
|
</div>
|
|
'''
|
|
|
|
return f"""
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>{display_title} - neopig</title>
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css">
|
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
|
|
<style>
|
|
* {{ box-sizing: border-box; }}
|
|
body {{
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0;
|
|
padding: 0;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}}
|
|
.nav {{
|
|
background: #1a1a1a;
|
|
padding: 10px 20px;
|
|
display: flex;
|
|
gap: 20px;
|
|
align-items: center;
|
|
border-bottom: 1px solid #333;
|
|
}}
|
|
.nav a {{ color: #ff6b6b; text-decoration: none; }}
|
|
.nav a:hover {{ text-decoration: underline; }}
|
|
.nav .brand {{ font-weight: bold; font-size: 18px; }}
|
|
.container {{ padding: 20px; }}
|
|
h1 {{ color: #ff6b6b; font-size: 20px; }}
|
|
h3 {{ color: #ff6b6b; font-size: 16px; margin-top: 30px; border-bottom: 1px solid #333; padding-bottom: 8px; }}
|
|
a {{ color: #ff6b6b; }}
|
|
.media-container {{ text-align: center; margin-bottom: 20px; }}
|
|
.media-container img, .media-container video {{ max-width: 100%; max-height: 80vh; }}
|
|
.meta {{ background: #1a1a1a; padding: 15px; border-radius: 8px; margin: 15px 0; }}
|
|
.meta-row {{ display: flex; margin: 8px 0; }}
|
|
.meta-label {{ width: 120px; color: #888; }}
|
|
.meta-value {{ flex: 1; word-break: break-all; }}
|
|
.tag {{ background: #333; padding: 2px 8px; border-radius: 4px; font-size: 12px; margin-right: 5px; }}
|
|
.sources {{ margin-top: 20px; }}
|
|
.sources ul {{ padding-left: 20px; }}
|
|
.sources li {{ margin: 10px 0; }}
|
|
.sources small {{ color: #666; }}
|
|
.sources .urls {{ display: block; word-break: break-all; font-size: 11px; margin-top: 4px; }}
|
|
.page-content {{ margin-top: 30px; }}
|
|
.content-text {{
|
|
background: #1a1a1a;
|
|
padding: 20px;
|
|
border-radius: 8px;
|
|
line-height: 1.7;
|
|
font-size: 14px;
|
|
color: #ccc;
|
|
max-height: 600px;
|
|
overflow-y: auto;
|
|
}}
|
|
.content-text p {{ margin: 0 0 15px 0; }}
|
|
.content-rendered {{
|
|
background: #1a1a1a;
|
|
padding: 20px;
|
|
border-radius: 8px;
|
|
line-height: 1.7;
|
|
font-size: 14px;
|
|
color: #ccc;
|
|
}}
|
|
.content-rendered img {{ max-width: 100%; height: auto; }}
|
|
.content-rendered img.avatar {{
|
|
display: inline-block;
|
|
vertical-align: middle;
|
|
width: 40px;
|
|
height: 40px;
|
|
border-radius: 50%;
|
|
margin: 0 10px 0 0;
|
|
object-fit: cover;
|
|
}}
|
|
/* Forum-style post layout */
|
|
.content-rendered .post-block {{
|
|
display: grid;
|
|
grid-template-columns: 48px 1fr;
|
|
gap: 12px;
|
|
padding: 16px 0;
|
|
border-bottom: 1px solid #252530;
|
|
}}
|
|
.content-rendered .post-block:last-child {{
|
|
border-bottom: none;
|
|
}}
|
|
.content-rendered .post-avatar-col {{
|
|
display: flex;
|
|
flex-direction: column;
|
|
align-items: center;
|
|
}}
|
|
.content-rendered .post-avatar {{
|
|
width: 40px;
|
|
height: 40px;
|
|
border-radius: 50%;
|
|
object-fit: cover;
|
|
}}
|
|
.content-rendered .post-avatar-placeholder {{
|
|
width: 40px;
|
|
height: 40px;
|
|
border-radius: 50%;
|
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
color: #fff;
|
|
display: flex;
|
|
align-items: center;
|
|
justify-content: center;
|
|
font-weight: bold;
|
|
font-size: 18px;
|
|
}}
|
|
.content-rendered .post-content {{
|
|
min-width: 0;
|
|
}}
|
|
.content-rendered .post-content p:first-child {{
|
|
margin-top: 0;
|
|
}}
|
|
.content-rendered a {{ color: #ff6b6b; }}
|
|
.content-rendered pre, .content-rendered code {{
|
|
background: #252525;
|
|
padding: 2px 6px;
|
|
border-radius: 4px;
|
|
font-family: monospace;
|
|
font-size: 13px;
|
|
overflow-x: auto;
|
|
}}
|
|
.content-rendered pre {{
|
|
padding: 15px;
|
|
display: block;
|
|
white-space: pre-wrap;
|
|
position: relative;
|
|
}}
|
|
.content-rendered pre .code-actions {{
|
|
position: absolute;
|
|
top: 8px;
|
|
right: 8px;
|
|
display: flex;
|
|
gap: 5px;
|
|
opacity: 0;
|
|
transition: opacity 0.2s;
|
|
}}
|
|
.content-rendered pre:hover .code-actions {{ opacity: 1; }}
|
|
.content-rendered pre .code-actions button {{
|
|
background: #444;
|
|
border: none;
|
|
color: #ccc;
|
|
padding: 4px 8px;
|
|
border-radius: 4px;
|
|
cursor: pointer;
|
|
font-size: 11px;
|
|
}}
|
|
.content-rendered pre .code-actions button:hover {{ background: #555; }}
|
|
.content-rendered pre .code-actions button.copied {{ background: #4ade80; color: #000; }}
|
|
.content-rendered blockquote {{
|
|
background: #151520;
|
|
border-left: 3px solid #4a9eff;
|
|
padding: 12px 16px;
|
|
margin: 16px 0;
|
|
border-radius: 0 6px 6px 0;
|
|
color: #aaa;
|
|
font-style: italic;
|
|
}}
|
|
.content-rendered blockquote p {{
|
|
margin: 0 0 8px 0;
|
|
}}
|
|
.content-rendered blockquote p:last-child {{
|
|
margin-bottom: 0;
|
|
}}
|
|
/* Nested quotes */
|
|
.content-rendered blockquote blockquote {{
|
|
background: #1a1a25;
|
|
border-left-color: #666;
|
|
margin: 12px 0;
|
|
}}
|
|
/* Post/comment spacing - add visual separation */
|
|
.content-rendered hr {{
|
|
border: none;
|
|
border-top: 1px solid #333;
|
|
margin: 24px 0;
|
|
}}
|
|
.content-rendered p {{
|
|
margin: 0 0 16px 0;
|
|
line-height: 1.7;
|
|
}}
|
|
.content-rendered > p + p {{
|
|
margin-top: 16px;
|
|
}}
|
|
.content-rendered h1, .content-rendered h2, .content-rendered h3 {{
|
|
color: #ff6b6b;
|
|
margin-top: 28px;
|
|
margin-bottom: 12px;
|
|
padding-top: 16px;
|
|
border-top: 1px solid #252530;
|
|
}}
|
|
.content-rendered h1:first-child, .content-rendered h2:first-child, .content-rendered h3:first-child {{
|
|
margin-top: 0;
|
|
padding-top: 0;
|
|
border-top: none;
|
|
}}
|
|
.page-context-grid {{
|
|
display: grid;
|
|
grid-template-columns: 1fr 1fr;
|
|
gap: 20px;
|
|
margin-top: 15px;
|
|
}}
|
|
.page-screenshot {{
|
|
position: sticky;
|
|
top: 20px;
|
|
}}
|
|
.page-screenshot img {{
|
|
width: 100%;
|
|
border-radius: 8px;
|
|
border: 1px solid #333;
|
|
}}
|
|
@media (max-width: 900px) {{
|
|
.page-context-grid {{
|
|
grid-template-columns: 1fr;
|
|
}}
|
|
.page-screenshot {{
|
|
position: static;
|
|
}}
|
|
.hero-grid {{
|
|
grid-template-columns: 1fr !important;
|
|
}}
|
|
}}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="nav">
|
|
<a href="/" class="brand">🐷 neopig</a>
|
|
<a href="/">Search</a>
|
|
<a href="/live">Live</a>
|
|
<a href="/random">Random</a>
|
|
<a href="/crawl">Crawl</a>
|
|
<a href="/phantom">Phantom</a>
|
|
</div>
|
|
<div class="container">
|
|
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
|
<input type="text" name="q" placeholder="Search keywords, alt text, page content..." style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<option value="">All types</option>
|
|
<option value="image">Images</option>
|
|
<option value="video">Videos</option>
|
|
<option value="audio">Audio</option>
|
|
</select>
|
|
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">Search</button>
|
|
</form>
|
|
<div class="hero-grid" style="display:grid;grid-template-columns:1fr 1fr;gap:20px;margin-bottom:30px;">
|
|
<div class="hero-left">
|
|
<div class="media-container" style="background:#111;border-radius:12px;padding:20px;text-align:center;">
|
|
<a href="/media/{md5_hash}" target="_blank">{media_html}</a>
|
|
</div>
|
|
<h1 style="margin-top:15px;font-size:1.3em;">{display_title}</h1>
|
|
</div>
|
|
<div class="hero-right" style="background:#151515;border-radius:12px;padding:20px;">
|
|
<div class="meta" style="margin-bottom:20px;">
|
|
<div class="meta-row"><span class="meta-label">MD5 Hash:</span><span class="meta-value"><code style="font-size:11px;">{md5_hash}</code></span></div>
|
|
<div class="meta-row"><span class="meta-label">Type:</span><span class="meta-value">{media['media_type']}</span></div>
|
|
<div class="meta-row"><span class="meta-label">MIME:</span><span class="meta-value">{media.get('mime_type') or 'unknown'}</span></div>
|
|
<div class="meta-row"><span class="meta-label">Size:</span><span class="meta-value">{media.get('file_size') or 0:,} bytes</span></div>
|
|
<div class="meta-row"><span class="meta-label">First seen:</span><span class="meta-value">{media.get('first_seen_at')}</span></div>
|
|
<div class="meta-row"><span class="meta-label">Alt text:</span><span class="meta-value">{media.get('alt_text') or '-'}</span></div>
|
|
<div class="meta-row"><span class="meta-label">Title:</span><span class="meta-value">{media.get('title') or '-'}</span></div>
|
|
<div class="meta-row"><span class="meta-label">Keywords:</span><span class="meta-value">{keywords_html or '-'}</span></div>
|
|
</div>
|
|
<a href="/media/{md5_hash}?download=1" class="download-btn" style="display:block;padding:12px 20px;background:#4a9eff;color:#fff;text-decoration:none;border-radius:6px;text-align:center;font-weight:500;">
|
|
⬇ Download ({download_filename})
|
|
</a>
|
|
<div class="sources" style="margin-top:20px;">
|
|
<h4 style="color:#888;font-size:12px;margin-bottom:8px;">Source pages ({len(sources)})</h4>
|
|
<ul style="font-size:12px;padding-left:15px;margin:0;">{sources_html}</ul>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
{page_content_html}
|
|
</div>
|
|
<script>
|
|
// Syntax highlighting
|
|
hljs.highlightAll();
|
|
|
|
// Auto-detect small images as emojis/avatars
|
|
document.querySelectorAll('.content-rendered img').forEach(img => {{
|
|
const checkSize = () => {{
|
|
const w = img.naturalWidth || img.width;
|
|
const h = img.naturalHeight || img.height;
|
|
if (w > 0 && h > 0) {{
|
|
if (w <= 24 && h <= 24) {{
|
|
img.classList.add('emoji');
|
|
}} else if (w <= 60 && h <= 60) {{
|
|
img.classList.add('avatar');
|
|
}}
|
|
}}
|
|
}};
|
|
if (img.complete) checkSize();
|
|
else img.onload = checkSize;
|
|
}});
|
|
|
|
// Add copy/download buttons to all code blocks
|
|
const langExtMap = {{
|
|
'python': 'py', 'py': 'py', 'javascript': 'js', 'js': 'js', 'typescript': 'ts', 'ts': 'ts',
|
|
'cpp': 'cpp', 'c++': 'cpp', 'c': 'c', 'java': 'java', 'rust': 'rs', 'go': 'go',
|
|
'ruby': 'rb', 'php': 'php', 'swift': 'swift', 'kotlin': 'kt', 'scala': 'scala',
|
|
'html': 'html', 'css': 'css', 'scss': 'scss', 'json': 'json', 'yaml': 'yaml', 'yml': 'yml',
|
|
'xml': 'xml', 'sql': 'sql', 'bash': 'sh', 'sh': 'sh', 'shell': 'sh', 'powershell': 'ps1',
|
|
'markdown': 'md', 'md': 'md', 'lua': 'lua', 'perl': 'pl', 'r': 'r'
|
|
}};
|
|
document.querySelectorAll('.content-rendered pre').forEach((pre, idx) => {{
|
|
const code = pre.querySelector('code') || pre;
|
|
const text = code.textContent;
|
|
|
|
// Detect language from class
|
|
let ext = 'txt';
|
|
const classes = (code.className || '').split(/\\s+/);
|
|
for (const cls of classes) {{
|
|
const match = cls.match(/^(?:language-|lang-)?(.+)$/);
|
|
if (match && langExtMap[match[1].toLowerCase()]) {{
|
|
ext = langExtMap[match[1].toLowerCase()];
|
|
break;
|
|
}}
|
|
}}
|
|
|
|
const actions = document.createElement('div');
|
|
actions.className = 'code-actions';
|
|
|
|
const copyBtn = document.createElement('button');
|
|
copyBtn.textContent = 'Copy';
|
|
copyBtn.onclick = async () => {{
|
|
await navigator.clipboard.writeText(text);
|
|
copyBtn.textContent = 'Copied!';
|
|
copyBtn.classList.add('copied');
|
|
setTimeout(() => {{ copyBtn.textContent = 'Copy'; copyBtn.classList.remove('copied'); }}, 2000);
|
|
}};
|
|
|
|
const dlBtn = document.createElement('button');
|
|
dlBtn.textContent = 'Download';
|
|
dlBtn.onclick = () => {{
|
|
const blob = new Blob([text], {{ type: 'text/plain' }});
|
|
const url = URL.createObjectURL(blob);
|
|
const a = document.createElement('a');
|
|
a.href = url;
|
|
a.download = `code-${{idx + 1}}.${{ext}}`;
|
|
a.click();
|
|
URL.revokeObjectURL(url);
|
|
}};
|
|
|
|
actions.appendChild(copyBtn);
|
|
actions.appendChild(dlBtn);
|
|
pre.appendChild(actions);
|
|
}});
|
|
// Restructure forum posts into two-column layout
|
|
// Looks for pattern: <p> <strong>username</strong></p> followed by content until <hr>
|
|
function restructureForumPosts() {{
|
|
const container = document.querySelector('.content-rendered');
|
|
if (!container) return;
|
|
|
|
// Find paragraphs that start with an avatar image followed by bold username
|
|
const postStarts = [];
|
|
container.querySelectorAll('p').forEach(p => {{
|
|
const img = p.querySelector('img[alt="avatar"]');
|
|
const strong = p.querySelector('strong');
|
|
if (img && strong) {{
|
|
postStarts.push({{ p, img, username: strong.textContent }});
|
|
}}
|
|
}});
|
|
|
|
if (postStarts.length === 0) return;
|
|
|
|
// For each post header, create a post-block
|
|
postStarts.forEach(({{ p, img, username }}, idx) => {{
|
|
if (p.closest('.post-block')) return;
|
|
|
|
const postBlock = document.createElement('div');
|
|
postBlock.className = 'post-block';
|
|
|
|
// Avatar column - use placeholder if no src
|
|
const avatarCol = document.createElement('div');
|
|
avatarCol.className = 'post-avatar-col';
|
|
|
|
const hasAvatar = img.src && !img.src.endsWith('#') && img.getAttribute('src') !== '#';
|
|
if (hasAvatar) {{
|
|
const avatarImg = img.cloneNode(true);
|
|
avatarImg.classList.add('post-avatar');
|
|
avatarCol.appendChild(avatarImg);
|
|
}} else {{
|
|
// Placeholder for posts without avatar
|
|
const placeholder = document.createElement('div');
|
|
placeholder.className = 'post-avatar-placeholder';
|
|
placeholder.textContent = username.charAt(0).toUpperCase();
|
|
avatarCol.appendChild(placeholder);
|
|
}}
|
|
|
|
// Content column
|
|
const contentCol = document.createElement('div');
|
|
contentCol.className = 'post-content';
|
|
|
|
// Add username header
|
|
const header = document.createElement('div');
|
|
header.className = 'post-header';
|
|
header.innerHTML = `<strong>${{username}}</strong>`;
|
|
contentCol.appendChild(header);
|
|
|
|
// Collect siblings until next post or HR
|
|
let sibling = p.nextElementSibling;
|
|
const nextP = idx < postStarts.length - 1 ? postStarts[idx + 1].p : null;
|
|
|
|
while (sibling && sibling !== nextP && sibling.tagName !== 'HR') {{
|
|
contentCol.appendChild(sibling.cloneNode(true));
|
|
const toRemove = sibling;
|
|
sibling = sibling.nextElementSibling;
|
|
toRemove.remove();
|
|
}}
|
|
|
|
// Remove the HR separator if present
|
|
if (sibling && sibling.tagName === 'HR') {{
|
|
sibling.remove();
|
|
}}
|
|
|
|
postBlock.appendChild(avatarCol);
|
|
postBlock.appendChild(contentCol);
|
|
p.parentNode.insertBefore(postBlock, p);
|
|
p.remove();
|
|
}});
|
|
}}
|
|
|
|
// Run after DOM ready (avatars may be empty placeholders)
|
|
restructureForumPosts();
|
|
</script>
|
|
<script src="https://uncloseai.com/uncloseai.js" type="module"></script>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
@app.get("/page/view", response_class=HTMLResponse)
|
|
async def view_page(
|
|
uri: str = Query(..., description="Page URI to view"),
|
|
noai: bool = Query(False, description="Disable AI assistant")
|
|
):
|
|
"""View an archived page with markdown and screenshot."""
|
|
import html as html_module
|
|
import re
|
|
|
|
page = await db.get_page_by_uri(uri)
|
|
if not page:
|
|
raise HTTPException(status_code=404, detail="Page not found")
|
|
|
|
page_title = page.get("title") or uri
|
|
|
|
# Render markdown
|
|
content_html = ""
|
|
if page.get("markdown"):
|
|
try:
|
|
import markdown
|
|
md_converter = markdown.Markdown(extensions=['fenced_code', 'tables', 'nl2br'])
|
|
content_html = md_converter.convert(page["markdown"][:100000])
|
|
|
|
# Hydrate images from vault
|
|
img_pattern = re.compile(r'<img[^>]+src=["\']([^"\']+)["\']', re.IGNORECASE)
|
|
img_urls = img_pattern.findall(content_html)
|
|
if img_urls:
|
|
from urllib.parse import urljoin
|
|
resolved_urls = {}
|
|
for url in img_urls:
|
|
if url.startswith(('http://', 'https://', '//')):
|
|
resolved_urls[url] = url
|
|
else:
|
|
resolved_urls[url] = urljoin(uri, url)
|
|
|
|
# Look up md5_hash for resolved URLs
|
|
all_urls = list(set(resolved_urls.values()))
|
|
resolved_to_hash = await db.lookup_media_by_uris(all_urls)
|
|
|
|
# Fallback: for URLs not found, try matching by filename
|
|
missing_urls = [u for u in all_urls if u not in resolved_to_hash]
|
|
if missing_urls:
|
|
from pathlib import Path as P
|
|
for murl in missing_urls:
|
|
fname = P(murl).stem
|
|
if len(fname) >= 20:
|
|
result = await db.lookup_media_by_filename(fname)
|
|
if result:
|
|
resolved_to_hash[murl] = result[1]
|
|
|
|
# Replace original URLs with vault paths
|
|
for orig_url, resolved_url in resolved_urls.items():
|
|
if resolved_url in resolved_to_hash:
|
|
md5 = resolved_to_hash[resolved_url]
|
|
content_html = content_html.replace(f'src="{orig_url}"', f'src="/media/{md5}"')
|
|
content_html = content_html.replace(f"src='{orig_url}'", f'src="/media/{md5}"')
|
|
except ImportError:
|
|
content_html = f"<pre>{html_module.escape(page.get('markdown', '')[:50000])}</pre>"
|
|
elif page.get("content"):
|
|
escaped = html_module.escape(page["content"][:50000])
|
|
content_html = f"<pre style='white-space:pre-wrap;'>{escaped}</pre>"
|
|
|
|
# Find screenshot
|
|
screenshot_html = ""
|
|
screenshot_hash = await db.get_page_screenshot(uri)
|
|
if screenshot_hash:
|
|
screenshot_html = f'''<a href="/media/{screenshot_hash}" target="_blank">
|
|
<img src="/media/{screenshot_hash}" alt="Page screenshot" style="max-width:100%;max-height:50vh;border-radius:8px;">
|
|
</a>'''
|
|
|
|
# Find media from this page
|
|
media_items = await db.get_page_media(uri)
|
|
|
|
media_grid = ""
|
|
if media_items:
|
|
media_cards = []
|
|
for m in media_items:
|
|
is_video = m.get("media_type") == "video"
|
|
if is_video:
|
|
el = f'<video src="/media/{m["md5_hash"]}" muted loop preload="metadata" onmouseenter="this.play()" onmouseleave="this.pause()"></video>'
|
|
else:
|
|
el = f'<img src="/media/{m["md5_hash"]}" loading="lazy">'
|
|
media_cards.append(f'''
|
|
<a href="/view/{m["md5_hash"]}" class="media-card">
|
|
{el}
|
|
</a>''')
|
|
media_grid = f'''
|
|
<div class="page-media">
|
|
<h3>Media from this page ({len(media_items)})</h3>
|
|
<div class="media-grid">{''.join(media_cards)}</div>
|
|
</div>'''
|
|
|
|
return f"""
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>{html_module.escape(page_title)} - neopig</title>
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css">
|
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
|
|
<style>
|
|
* {{ box-sizing: border-box; }}
|
|
body {{
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0;
|
|
padding: 0;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}}
|
|
.nav {{
|
|
background: #1a1a1a;
|
|
padding: 10px 20px;
|
|
display: flex;
|
|
gap: 20px;
|
|
align-items: center;
|
|
border-bottom: 1px solid #333;
|
|
}}
|
|
.nav a {{ color: #ff6b6b; text-decoration: none; }}
|
|
.nav a:hover {{ text-decoration: underline; }}
|
|
.nav .brand {{ font-weight: bold; font-size: 18px; }}
|
|
.container {{ padding: 20px; }}
|
|
h1 {{ color: #ff6b6b; font-size: 22px; margin-bottom: 5px; }}
|
|
.page-uri {{ color: #4ade80; font-size: 12px; font-family: monospace; margin-bottom: 20px; }}
|
|
.page-uri a {{ color: #4ade80; }}
|
|
.hero-grid {{
|
|
display: grid;
|
|
grid-template-columns: 1fr 1fr;
|
|
gap: 20px;
|
|
margin-bottom: 30px;
|
|
}}
|
|
.hero-left {{
|
|
background: #111;
|
|
border-radius: 12px;
|
|
padding: 20px;
|
|
}}
|
|
.hero-right {{
|
|
background: #151515;
|
|
border-radius: 12px;
|
|
padding: 20px;
|
|
}}
|
|
.meta {{
|
|
background: #1a1a1a;
|
|
padding: 15px;
|
|
border-radius: 8px;
|
|
margin-bottom: 15px;
|
|
}}
|
|
.meta-row {{
|
|
display: flex;
|
|
margin: 8px 0;
|
|
}}
|
|
.meta-label {{
|
|
width: 100px;
|
|
color: #888;
|
|
font-size: 13px;
|
|
}}
|
|
.meta-value {{
|
|
flex: 1;
|
|
word-break: break-all;
|
|
font-size: 13px;
|
|
}}
|
|
.meta-value a {{
|
|
color: #4ade80;
|
|
}}
|
|
.content-rendered {{
|
|
background: #1a1a1a;
|
|
padding: 20px;
|
|
border-radius: 8px;
|
|
line-height: 1.7;
|
|
font-size: 14px;
|
|
color: #ccc;
|
|
}}
|
|
.content-rendered img {{ max-width: 100%; height: auto; margin: 10px 0; }}
|
|
.content-rendered img.avatar, .content-rendered img.small-img {{
|
|
display: inline-block;
|
|
vertical-align: middle;
|
|
width: 32px;
|
|
height: 32px;
|
|
border-radius: 50%;
|
|
margin: 0 8px 0 0;
|
|
object-fit: cover;
|
|
}}
|
|
.content-rendered img.emoji {{
|
|
display: inline;
|
|
width: 20px;
|
|
height: 20px;
|
|
margin: 0 2px;
|
|
vertical-align: text-bottom;
|
|
}}
|
|
/* Forum-style post layout */
|
|
.content-rendered .post-block {{
|
|
display: grid;
|
|
grid-template-columns: 48px 1fr;
|
|
gap: 12px;
|
|
padding: 16px 0;
|
|
border-bottom: 1px solid #252530;
|
|
}}
|
|
.content-rendered .post-block:last-child {{
|
|
border-bottom: none;
|
|
}}
|
|
.content-rendered .post-avatar-col {{
|
|
display: flex;
|
|
flex-direction: column;
|
|
align-items: center;
|
|
}}
|
|
.content-rendered .post-avatar {{
|
|
width: 40px;
|
|
height: 40px;
|
|
border-radius: 50%;
|
|
object-fit: cover;
|
|
}}
|
|
.content-rendered .post-avatar-placeholder {{
|
|
width: 40px;
|
|
height: 40px;
|
|
border-radius: 50%;
|
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
color: #fff;
|
|
display: flex;
|
|
align-items: center;
|
|
justify-content: center;
|
|
font-weight: bold;
|
|
font-size: 18px;
|
|
}}
|
|
.content-rendered .post-content {{
|
|
min-width: 0;
|
|
}}
|
|
.content-rendered .post-content p:first-child {{
|
|
margin-top: 0;
|
|
}}
|
|
.content-rendered a {{ color: #ff6b6b; }}
|
|
.content-rendered pre, .content-rendered code {{
|
|
background: #252525;
|
|
padding: 2px 6px;
|
|
border-radius: 4px;
|
|
font-family: monospace;
|
|
font-size: 13px;
|
|
}}
|
|
.content-rendered pre {{
|
|
padding: 15px;
|
|
display: block;
|
|
white-space: pre-wrap;
|
|
overflow-x: auto;
|
|
position: relative;
|
|
}}
|
|
.content-rendered pre .code-actions {{
|
|
position: absolute;
|
|
top: 5px;
|
|
right: 5px;
|
|
display: flex;
|
|
gap: 5px;
|
|
opacity: 0;
|
|
transition: opacity 0.2s;
|
|
}}
|
|
.content-rendered pre:hover .code-actions {{ opacity: 1; }}
|
|
.content-rendered pre .code-actions button {{
|
|
background: #444;
|
|
border: none;
|
|
color: #ccc;
|
|
padding: 4px 8px;
|
|
border-radius: 4px;
|
|
cursor: pointer;
|
|
font-size: 11px;
|
|
}}
|
|
.content-rendered pre .code-actions button:hover {{ background: #555; }}
|
|
.content-rendered pre .code-actions button.copied {{ background: #4ade80; color: #000; }}
|
|
.content-rendered blockquote {{
|
|
background: #151520;
|
|
border-left: 3px solid #4a9eff;
|
|
padding: 12px 16px;
|
|
margin: 16px 0;
|
|
border-radius: 0 6px 6px 0;
|
|
color: #aaa;
|
|
font-style: italic;
|
|
}}
|
|
.content-rendered blockquote p {{
|
|
margin: 0 0 8px 0;
|
|
}}
|
|
.content-rendered blockquote p:last-child {{
|
|
margin-bottom: 0;
|
|
}}
|
|
/* Nested quotes */
|
|
.content-rendered blockquote blockquote {{
|
|
background: #1a1a25;
|
|
border-left-color: #666;
|
|
margin: 12px 0;
|
|
}}
|
|
/* Post/comment spacing */
|
|
.content-rendered hr {{
|
|
border: none;
|
|
border-top: 1px solid #333;
|
|
margin: 24px 0;
|
|
}}
|
|
.content-rendered p {{
|
|
margin: 0 0 16px 0;
|
|
line-height: 1.7;
|
|
}}
|
|
.content-rendered h1, .content-rendered h2, .content-rendered h3 {{
|
|
color: #ff6b6b;
|
|
margin-top: 28px;
|
|
margin-bottom: 12px;
|
|
padding-top: 16px;
|
|
border-top: 1px solid #252530;
|
|
}}
|
|
.content-rendered h1:first-child, .content-rendered h2:first-child, .content-rendered h3:first-child {{
|
|
margin-top: 0;
|
|
padding-top: 0;
|
|
border-top: none;
|
|
}}
|
|
.page-screenshot {{
|
|
position: sticky;
|
|
top: 20px;
|
|
}}
|
|
.page-screenshot img {{
|
|
width: 100%;
|
|
border-radius: 8px;
|
|
border: 1px solid #333;
|
|
}}
|
|
.page-media {{ margin-top: 30px; }}
|
|
.page-media h3 {{ color: #ff6b6b; font-size: 16px; border-bottom: 1px solid #333; padding-bottom: 8px; margin: 0 0 15px 0; }}
|
|
.media-grid {{
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fill, minmax(150px, 1fr));
|
|
gap: 10px;
|
|
}}
|
|
.media-card {{
|
|
background: #1a1a1a;
|
|
border-radius: 8px;
|
|
overflow: hidden;
|
|
display: block;
|
|
transition: transform 0.2s;
|
|
}}
|
|
.media-card:hover {{
|
|
transform: scale(1.02);
|
|
}}
|
|
.media-card img, .media-card video {{
|
|
width: 100%;
|
|
height: 120px;
|
|
object-fit: contain;
|
|
background: #0a0a0a;
|
|
}}
|
|
@media (max-width: 900px) {{
|
|
.hero-grid {{
|
|
grid-template-columns: 1fr;
|
|
}}
|
|
}}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="nav">
|
|
<a href="/" class="brand">🐷 neopig</a>
|
|
<a href="/">Search</a>
|
|
<a href="/live">Live</a>
|
|
<a href="/random">Random</a>
|
|
<a href="/crawl">Crawl</a>
|
|
<a href="/phantom">Phantom</a>
|
|
</div>
|
|
<div class="container">
|
|
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
|
<input type="text" name="q" placeholder="Search keywords, alt text, page content..." style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<option value="">All types</option>
|
|
<option value="image">Images</option>
|
|
<option value="video">Videos</option>
|
|
<option value="audio">Audio</option>
|
|
</select>
|
|
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">Search</button>
|
|
</form>
|
|
<div class="hero-grid">
|
|
<div class="hero-left">
|
|
{screenshot_html or '<div style="color:#666;text-align:center;padding:40px;">No screenshot available</div>'}
|
|
<h1 style="margin-top:15px;font-size:1.3em;">{html_module.escape(page_title)}</h1>
|
|
</div>
|
|
<div class="hero-right">
|
|
<div class="meta">
|
|
<div class="meta-row"><span class="meta-label">URL:</span><span class="meta-value"><a href="{uri}" target="_blank">{uri}</a></span></div>
|
|
<div class="meta-row"><span class="meta-label">Media:</span><span class="meta-value">{len(media_items)} items</span></div>
|
|
</div>
|
|
<div class="content-rendered" style="max-height:400px;overflow-y:auto;">{content_html or '<p>No content available</p>'}</div>
|
|
</div>
|
|
</div>
|
|
{media_grid}
|
|
</div>
|
|
<script>
|
|
// Syntax highlighting
|
|
hljs.highlightAll();
|
|
|
|
// Auto-detect small images as emojis/avatars
|
|
document.querySelectorAll('.content-rendered img').forEach(img => {{
|
|
const checkSize = () => {{
|
|
const w = img.naturalWidth || img.width;
|
|
const h = img.naturalHeight || img.height;
|
|
if (w > 0 && h > 0) {{
|
|
if (w <= 24 && h <= 24) {{
|
|
img.classList.add('emoji');
|
|
}} else if (w <= 60 && h <= 60) {{
|
|
img.classList.add('avatar');
|
|
}}
|
|
}}
|
|
}};
|
|
if (img.complete) checkSize();
|
|
else img.onload = checkSize;
|
|
}});
|
|
|
|
// Add copy/download buttons to all code blocks
|
|
const langExtMap = {{
|
|
'python': 'py', 'py': 'py', 'javascript': 'js', 'js': 'js', 'typescript': 'ts', 'ts': 'ts',
|
|
'cpp': 'cpp', 'c++': 'cpp', 'c': 'c', 'java': 'java', 'rust': 'rs', 'go': 'go',
|
|
'ruby': 'rb', 'php': 'php', 'swift': 'swift', 'kotlin': 'kt', 'scala': 'scala',
|
|
'html': 'html', 'css': 'css', 'scss': 'scss', 'json': 'json', 'yaml': 'yaml', 'yml': 'yml',
|
|
'xml': 'xml', 'sql': 'sql', 'bash': 'sh', 'sh': 'sh', 'shell': 'sh', 'powershell': 'ps1',
|
|
'markdown': 'md', 'md': 'md', 'lua': 'lua', 'perl': 'pl', 'r': 'r'
|
|
}};
|
|
document.querySelectorAll('.content-rendered pre').forEach((pre, idx) => {{
|
|
const code = pre.querySelector('code') || pre;
|
|
const text = code.textContent;
|
|
|
|
// Detect language from class
|
|
let ext = 'txt';
|
|
const classes = (code.className || '').split(/\\s+/);
|
|
for (const cls of classes) {{
|
|
const match = cls.match(/^(?:language-|lang-)?(.+)$/);
|
|
if (match && langExtMap[match[1].toLowerCase()]) {{
|
|
ext = langExtMap[match[1].toLowerCase()];
|
|
break;
|
|
}}
|
|
}}
|
|
|
|
const actions = document.createElement('div');
|
|
actions.className = 'code-actions';
|
|
|
|
const copyBtn = document.createElement('button');
|
|
copyBtn.textContent = 'Copy';
|
|
copyBtn.onclick = async () => {{
|
|
await navigator.clipboard.writeText(text);
|
|
copyBtn.textContent = 'Copied!';
|
|
copyBtn.classList.add('copied');
|
|
setTimeout(() => {{ copyBtn.textContent = 'Copy'; copyBtn.classList.remove('copied'); }}, 2000);
|
|
}};
|
|
|
|
const dlBtn = document.createElement('button');
|
|
dlBtn.textContent = 'Download';
|
|
dlBtn.onclick = () => {{
|
|
const blob = new Blob([text], {{ type: 'text/plain' }});
|
|
const url = URL.createObjectURL(blob);
|
|
const a = document.createElement('a');
|
|
a.href = url;
|
|
a.download = `code-${{idx + 1}}.${{ext}}`;
|
|
a.click();
|
|
URL.revokeObjectURL(url);
|
|
}};
|
|
|
|
actions.appendChild(copyBtn);
|
|
actions.appendChild(dlBtn);
|
|
pre.appendChild(actions);
|
|
}});
|
|
// Restructure forum posts into two-column layout
|
|
// Looks for pattern: <p> <strong>username</strong></p> followed by content until <hr>
|
|
function restructureForumPosts() {{
|
|
const container = document.querySelector('.content-rendered');
|
|
if (!container) return;
|
|
|
|
// Find paragraphs that start with an avatar image followed by bold username
|
|
const postStarts = [];
|
|
container.querySelectorAll('p').forEach(p => {{
|
|
const img = p.querySelector('img[alt="avatar"]');
|
|
const strong = p.querySelector('strong');
|
|
if (img && strong) {{
|
|
postStarts.push({{ p, img, username: strong.textContent }});
|
|
}}
|
|
}});
|
|
|
|
if (postStarts.length === 0) return;
|
|
|
|
// For each post header, create a post-block
|
|
postStarts.forEach(({{ p, img, username }}, idx) => {{
|
|
if (p.closest('.post-block')) return;
|
|
|
|
const postBlock = document.createElement('div');
|
|
postBlock.className = 'post-block';
|
|
|
|
// Avatar column - use placeholder if no src
|
|
const avatarCol = document.createElement('div');
|
|
avatarCol.className = 'post-avatar-col';
|
|
|
|
const hasAvatar = img.src && !img.src.endsWith('#') && img.getAttribute('src') !== '#';
|
|
if (hasAvatar) {{
|
|
const avatarImg = img.cloneNode(true);
|
|
avatarImg.classList.add('post-avatar');
|
|
avatarCol.appendChild(avatarImg);
|
|
}} else {{
|
|
// Placeholder for posts without avatar
|
|
const placeholder = document.createElement('div');
|
|
placeholder.className = 'post-avatar-placeholder';
|
|
placeholder.textContent = username.charAt(0).toUpperCase();
|
|
avatarCol.appendChild(placeholder);
|
|
}}
|
|
|
|
// Content column
|
|
const contentCol = document.createElement('div');
|
|
contentCol.className = 'post-content';
|
|
|
|
// Add username header
|
|
const header = document.createElement('div');
|
|
header.className = 'post-header';
|
|
header.innerHTML = `<strong>${{username}}</strong>`;
|
|
contentCol.appendChild(header);
|
|
|
|
// Collect siblings until next post or HR
|
|
let sibling = p.nextElementSibling;
|
|
const nextP = idx < postStarts.length - 1 ? postStarts[idx + 1].p : null;
|
|
|
|
while (sibling && sibling !== nextP && sibling.tagName !== 'HR') {{
|
|
contentCol.appendChild(sibling.cloneNode(true));
|
|
const toRemove = sibling;
|
|
sibling = sibling.nextElementSibling;
|
|
toRemove.remove();
|
|
}}
|
|
|
|
// Remove the HR separator if present
|
|
if (sibling && sibling.tagName === 'HR') {{
|
|
sibling.remove();
|
|
}}
|
|
|
|
postBlock.appendChild(avatarCol);
|
|
postBlock.appendChild(contentCol);
|
|
p.parentNode.insertBefore(postBlock, p);
|
|
p.remove();
|
|
}});
|
|
}}
|
|
|
|
// Run after DOM ready (avatars may be empty placeholders)
|
|
restructureForumPosts();
|
|
</script>
|
|
{'' if noai else '<script src="https://uncloseai.com/uncloseai.js" type="module"></script>'}
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
@app.get("/phantom/export")
|
|
async def phantom_export(domain: str = Query(None, description="Filter by domain")):
|
|
"""
|
|
Export a phantom HTML site - original HTML with media URLs rewritten to vault.
|
|
|
|
Creates a downloadable zip of the phantom site ready for static hosting.
|
|
"""
|
|
import io
|
|
import re
|
|
import zipfile
|
|
from urllib.parse import urlparse, urljoin
|
|
from fastapi.responses import StreamingResponse
|
|
|
|
# Get all pages with raw_html
|
|
pages = await db.get_pages_by_domain(domain)
|
|
|
|
if not pages:
|
|
raise HTTPException(status_code=404, detail="No pages with raw HTML found")
|
|
|
|
# Get all media URL to hash mappings
|
|
url_to_hash = await db.get_all_media_uri_mappings()
|
|
|
|
# Create zip in memory
|
|
zip_buffer = io.BytesIO()
|
|
|
|
with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:
|
|
pages_written = 0
|
|
media_hashes = set()
|
|
|
|
for page in pages:
|
|
uri = page['uri']
|
|
raw_html = page.get('raw_html')
|
|
if not raw_html:
|
|
continue
|
|
|
|
# Parse URI to get path
|
|
parsed = urlparse(uri)
|
|
site_domain = parsed.netloc
|
|
path = parsed.path.strip('/') or 'index'
|
|
if not path.endswith('.html') and '.' not in path.split('/')[-1]:
|
|
path = f"{path}/index.html" if path else "index.html"
|
|
|
|
# Rewrite media URLs to local paths
|
|
html = raw_html
|
|
|
|
# Find all src and href attributes pointing to media
|
|
patterns = [
|
|
(r'src=["\']([^"\']+)["\']', 'src'),
|
|
(r'href=["\']([^"\']+\.(jpg|jpeg|png|gif|webp|mp4|webm|svg|ico))["\']', 'href'),
|
|
]
|
|
|
|
for pattern, attr in patterns:
|
|
matches = re.findall(pattern, html, re.IGNORECASE)
|
|
for match in matches:
|
|
url = match[0] if isinstance(match, tuple) else match
|
|
|
|
# Resolve relative URLs
|
|
full_url = urljoin(uri, url)
|
|
|
|
# Check if we have this media
|
|
if full_url in url_to_hash:
|
|
md5 = url_to_hash[full_url]
|
|
media_hashes.add(md5)
|
|
# Replace with local path
|
|
ext = Path(url).suffix or '.bin'
|
|
local_path = f"media/{md5}{ext}"
|
|
html = html.replace(f'{attr}="{url}"', f'{attr}="{local_path}"')
|
|
html = html.replace(f"{attr}='{url}'", f'{attr}="{local_path}"')
|
|
elif url in url_to_hash:
|
|
md5 = url_to_hash[url]
|
|
media_hashes.add(md5)
|
|
ext = Path(url).suffix or '.bin'
|
|
local_path = f"media/{md5}{ext}"
|
|
html = html.replace(f'{attr}="{url}"', f'{attr}="{local_path}"')
|
|
html = html.replace(f"{attr}='{url}'", f'{attr}="{local_path}"')
|
|
|
|
# Write HTML file
|
|
zf.writestr(f"site/{path}", html.encode('utf-8'))
|
|
pages_written += 1
|
|
|
|
# Copy media files from vault
|
|
media_copied = 0
|
|
for md5 in media_hashes:
|
|
subdir = VAULT_PATH / md5[:2]
|
|
if subdir.exists():
|
|
for f in subdir.iterdir():
|
|
if f.name.startswith(md5):
|
|
ext = f.suffix or '.bin'
|
|
zf.write(f, f"site/media/{md5}{ext}")
|
|
media_copied += 1
|
|
break
|
|
|
|
# Write index
|
|
index_html = f"""<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>Phantom Site - {site_domain}</title>
|
|
<style>
|
|
body {{ font-family: sans-serif; max-width: 800px; margin: 50px auto; padding: 20px; }}
|
|
h1 {{ color: #333; }}
|
|
ul {{ line-height: 2; }}
|
|
a {{ color: #0066cc; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>Phantom Site Archive</h1>
|
|
<p>Domain: {site_domain}</p>
|
|
<p>Pages: {pages_written}</p>
|
|
<p>Media: {media_copied}</p>
|
|
<h2>Pages</h2>
|
|
<ul>
|
|
"""
|
|
for page in pages[:100]:
|
|
parsed = urlparse(page['uri'])
|
|
path = parsed.path.strip('/') or 'index'
|
|
if not path.endswith('.html') and '.' not in path.split('/')[-1]:
|
|
path = f"{path}/index.html" if path else "index.html"
|
|
title = page.get('title') or path
|
|
index_html += f' <li><a href="{path}">{title}</a></li>\n'
|
|
|
|
index_html += """ </ul>
|
|
</body>
|
|
</html>"""
|
|
zf.writestr("site/phantom_index.html", index_html.encode('utf-8'))
|
|
|
|
# Return zip
|
|
zip_buffer.seek(0)
|
|
return StreamingResponse(
|
|
zip_buffer,
|
|
media_type="application/zip",
|
|
headers={"Content-Disposition": f"attachment; filename=phantom_{site_domain or 'site'}.zip"}
|
|
)
|
|
|
|
|
|
@app.get("/phantom", response_class=HTMLResponse)
|
|
async def phantom_page():
|
|
"""Phantom site export UI."""
|
|
domains = await db.get_domains_with_pages()
|
|
|
|
domain_options = ''.join([
|
|
f'<option value="{d[0]}">{d[0]} ({d[1]} pages)</option>'
|
|
for d in domains if d[0]
|
|
])
|
|
|
|
return f"""
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>Phantom Site Export - neopig</title>
|
|
<style>
|
|
* {{ box-sizing: border-box; }}
|
|
body {{
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0;
|
|
padding: 0;
|
|
background: #0a0a0a;
|
|
color: #e0e0e0;
|
|
}}
|
|
.nav {{
|
|
background: #1a1a1a;
|
|
padding: 10px 20px;
|
|
display: flex;
|
|
gap: 20px;
|
|
align-items: center;
|
|
border-bottom: 1px solid #333;
|
|
}}
|
|
.nav a {{ color: #ff6b6b; text-decoration: none; }}
|
|
.nav a:hover {{ text-decoration: underline; }}
|
|
.nav .brand {{ font-weight: bold; font-size: 18px; }}
|
|
.container {{ padding: 20px; }}
|
|
h1 {{ color: #ff6b6b; margin-bottom: 10px; }}
|
|
.subtitle {{ color: #888; margin-bottom: 30px; }}
|
|
.form-group {{ margin-bottom: 20px; }}
|
|
label {{ display: block; margin-bottom: 8px; color: #aaa; }}
|
|
select {{
|
|
width: 100%;
|
|
padding: 12px 16px;
|
|
font-size: 16px;
|
|
border: 2px solid #333;
|
|
border-radius: 8px;
|
|
background: #1a1a1a;
|
|
color: #fff;
|
|
}}
|
|
button {{
|
|
padding: 14px 28px;
|
|
font-size: 16px;
|
|
background: #ff6b6b;
|
|
color: #fff;
|
|
border: none;
|
|
border-radius: 8px;
|
|
cursor: pointer;
|
|
}}
|
|
button:hover {{ background: #ff5252; }}
|
|
.info {{
|
|
background: #1a1a1a;
|
|
padding: 20px;
|
|
border-radius: 8px;
|
|
margin-top: 30px;
|
|
}}
|
|
.info h3 {{ color: #ff6b6b; margin-top: 0; }}
|
|
.info ul {{ color: #aaa; line-height: 1.8; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="nav">
|
|
<a href="/" class="brand">🐷 neopig</a>
|
|
<a href="/">Search</a>
|
|
<a href="/live">Live</a>
|
|
<a href="/random">Random</a>
|
|
<a href="/crawl">Crawl</a>
|
|
<a href="/phantom">Phantom</a>
|
|
</div>
|
|
<div class="container">
|
|
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
|
<input type="text" name="q" placeholder="Search keywords, alt text, page content..." style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
|
<option value="">All types</option>
|
|
<option value="image">Images</option>
|
|
<option value="video">Videos</option>
|
|
<option value="audio">Audio</option>
|
|
</select>
|
|
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">Search</button>
|
|
</form>
|
|
<h1>Phantom Site Export</h1>
|
|
<p class="subtitle">Export archived pages as a static site with local media</p>
|
|
|
|
<form action="/phantom/export" method="get">
|
|
<div class="form-group">
|
|
<label>Select Domain</label>
|
|
<select name="domain">
|
|
<option value="">All domains</option>
|
|
{domain_options}
|
|
</select>
|
|
</div>
|
|
<button type="submit">Download Phantom Site (.zip)</button>
|
|
</form>
|
|
|
|
<div class="info">
|
|
<h3>What is a Phantom Site?</h3>
|
|
<ul>
|
|
<li>Original HTML preserved exactly as crawled</li>
|
|
<li>All media URLs rewritten to local paths</li>
|
|
<li>Ready to host statically (nginx, Caddy, S3, etc.)</li>
|
|
<li>Works offline - all assets included</li>
|
|
<li>Perfect for archival and preservation</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
|
|
@app.get("/health")
|
|
async def health():
|
|
"""Health check endpoint."""
|
|
has_screenshot = False
|
|
try:
|
|
from uri2png import get_available_engines
|
|
has_screenshot = True
|
|
except ImportError:
|
|
pass
|
|
|
|
return {
|
|
"status": "healthy",
|
|
"features": {
|
|
"search": True,
|
|
"crawl": True,
|
|
"screenshot": has_screenshot
|
|
}
|
|
}
|
|
|
|
|
|
@app.get("/api/stats")
|
|
async def get_stats_endpoint():
|
|
"""Get database statistics."""
|
|
stats = await db.get_stats()
|
|
# Add page count (handled separately since table may not exist)
|
|
try:
|
|
from sqlalchemy import select, func
|
|
from database import Page
|
|
async with db.session() as session:
|
|
result = await session.execute(select(func.count()).select_from(Page))
|
|
stats['total_pages'] = result.scalar() or 0
|
|
except Exception:
|
|
stats['total_pages'] = 0
|
|
return stats
|
|
|
|
|
|
@app.get("/random")
|
|
async def random_media():
|
|
"""Redirect to a random media item."""
|
|
from sqlalchemy import select, func
|
|
from database import Media
|
|
import random
|
|
|
|
async with db.session() as session:
|
|
# Get a random media item (excluding screenshots)
|
|
stmt = (
|
|
select(Media.md5_hash)
|
|
.where(Media.media_type != 'screenshot')
|
|
.order_by(func.random())
|
|
.limit(1)
|
|
)
|
|
result = await session.execute(stmt)
|
|
row = result.fetchone()
|
|
|
|
if row:
|
|
return RedirectResponse(url=f"/view/{row[0]}", status_code=302)
|
|
else:
|
|
return RedirectResponse(url="/", status_code=302)
|
|
|
|
|
|
@app.get("/api/search")
|
|
async def search(
|
|
q: str = Query("", description="Search query"),
|
|
type: Optional[str] = Query(None, description="Filter by media type"),
|
|
status: Optional[str] = Query(None, description="Filter by analysis status"),
|
|
limit: int = Query(100, le=1000),
|
|
offset: int = Query(0)
|
|
):
|
|
"""
|
|
Search media by text query.
|
|
|
|
Searches across: keywords, alt_text, title, source URLs, analysis results.
|
|
"""
|
|
results = await db.search_media_advanced(
|
|
q=q if q else None,
|
|
media_type=type,
|
|
limit=limit,
|
|
offset=offset
|
|
)
|
|
return results
|
|
|
|
|
|
@app.get("/api/search/pages")
|
|
async def search_pages_endpoint(
|
|
q: str = Query("", description="Search query"),
|
|
limit: int = Query(50, le=500),
|
|
):
|
|
"""
|
|
Search pages by text query using FTS5.
|
|
"""
|
|
if not q:
|
|
return []
|
|
return await db.search_pages(q, limit)
|
|
|
|
|
|
@app.get("/api/media/{md5_hash}")
|
|
async def get_media_info(md5_hash: str):
|
|
"""Get full media info including all source URLs."""
|
|
media = await db.get_media_by_hash(md5_hash)
|
|
if not media:
|
|
raise HTTPException(status_code=404, detail="Media not found")
|
|
|
|
media['sources'] = await db.get_media_sources(md5_hash)
|
|
return media
|
|
|
|
|
|
def slugify(text: str, max_len: int = 60) -> str:
|
|
"""Convert text to a safe filename slug."""
|
|
import re
|
|
import unicodedata
|
|
# Normalize unicode
|
|
text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('ascii')
|
|
# Lowercase and replace spaces/special chars with hyphens
|
|
text = re.sub(r'[^\w\s-]', '', text.lower())
|
|
text = re.sub(r'[-\s]+', '-', text).strip('-')
|
|
return text[:max_len] if text else ""
|
|
|
|
|
|
@app.get("/media/{md5_hash}")
|
|
async def serve_media(md5_hash: str, download: bool = False):
|
|
"""
|
|
Serve media file from vault or tarball.
|
|
|
|
Use ?download=1 for attachment mode with smart filename.
|
|
Caddy should be configured to cache these responses.
|
|
"""
|
|
# Tarball mode: serve from tar.gz
|
|
if TAR_FILE:
|
|
data, ext = find_media_in_tarball(md5_hash)
|
|
if data:
|
|
mime_type, _ = mimetypes.guess_type(f"file{ext}")
|
|
if not mime_type:
|
|
mime_type = "application/octet-stream"
|
|
filename = f"{md5_hash[:12]}{ext}"
|
|
headers = {
|
|
"Cache-Control": "public, max-age=31536000, immutable",
|
|
"X-Content-Hash": md5_hash,
|
|
}
|
|
if download:
|
|
headers["Content-Disposition"] = f'attachment; filename="{filename}"'
|
|
return Response(content=data, media_type=mime_type, headers=headers)
|
|
raise HTTPException(status_code=404, detail="Media not found in archive")
|
|
|
|
# Filesystem mode: find file in vault
|
|
subdir = VAULT_PATH / md5_hash[:2]
|
|
if not subdir.exists():
|
|
raise HTTPException(status_code=404, detail="Media not found")
|
|
|
|
# Find file with this hash prefix
|
|
for f in subdir.iterdir():
|
|
if f.name.startswith(md5_hash):
|
|
# Guess content type and get extension
|
|
mime_type, _ = mimetypes.guess_type(f.name)
|
|
ext = f.suffix or ""
|
|
|
|
# Get metadata for filename generation
|
|
filename = None
|
|
media_record = await db.get_media_by_hash(md5_hash)
|
|
if media_record:
|
|
if not mime_type and media_record.get("mime_type"):
|
|
mime_type = media_record["mime_type"]
|
|
# Generate filename from alt_text or title
|
|
name_source = media_record.get("alt_text") or media_record.get("title")
|
|
if name_source:
|
|
slug = slugify(name_source)
|
|
if slug:
|
|
filename = f"{slug}{ext}"
|
|
|
|
# Fallback: try to get page_title from media_sources
|
|
if not filename:
|
|
sources = await db.get_media_sources(md5_hash)
|
|
if sources:
|
|
row2 = sources[0]
|
|
# Try page_title + hash index
|
|
if row2.get("page_title"):
|
|
media_idx = int(md5_hash[:4], 16)
|
|
slug = slugify(f"{row2['page_title']}-{media_idx}")
|
|
if slug:
|
|
filename = f"{slug}{ext}"
|
|
# Fallback: original filename from URL
|
|
if not filename and row2.get("media_uri"):
|
|
from urllib.parse import urlparse, unquote
|
|
parsed = urlparse(row2["media_uri"])
|
|
orig_name = Path(unquote(parsed.path)).name
|
|
if orig_name and '.' in orig_name:
|
|
filename = orig_name
|
|
|
|
if not mime_type:
|
|
mime_type = "application/octet-stream"
|
|
|
|
# Default filename if nothing else
|
|
if not filename:
|
|
filename = f"{md5_hash[:12]}{ext}"
|
|
|
|
# Download mode: attachment with smart filename
|
|
# Inline mode: no filename header, browser shows inline
|
|
if download:
|
|
return FileResponse(
|
|
f,
|
|
media_type=mime_type,
|
|
filename=filename,
|
|
content_disposition_type="attachment",
|
|
headers={
|
|
"Cache-Control": "public, max-age=31536000, immutable",
|
|
"X-Content-Hash": md5_hash,
|
|
}
|
|
)
|
|
else:
|
|
return FileResponse(
|
|
f,
|
|
media_type=mime_type,
|
|
content_disposition_type="inline",
|
|
headers={
|
|
"Cache-Control": "public, max-age=31536000, immutable",
|
|
"X-Content-Hash": md5_hash,
|
|
}
|
|
)
|
|
|
|
raise HTTPException(status_code=404, detail="Media not found")
|
|
|
|
|
|
# ============================================================================
|
|
# Crawler API
|
|
# ============================================================================
|
|
|
|
@app.get("/api/crawl/jobs")
|
|
async def get_crawl_jobs_endpoint(limit: int = Query(50, le=200)):
|
|
"""Get recent crawl jobs."""
|
|
return await db.get_crawl_jobs(limit)
|
|
|
|
|
|
@app.get("/api/crawl/jobs/{job_id}")
|
|
async def get_crawl_job_endpoint(job_id: int):
|
|
"""Get a specific crawl job."""
|
|
job = await db.get_crawl_job(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="Job not found")
|
|
return job
|
|
|
|
|
|
@app.post("/api/crawl")
|
|
async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
|
|
"""
|
|
Start new crawl job(s).
|
|
|
|
Supports multiple targets - creates one job per target.
|
|
The crawls run in the background. Poll /api/crawl/jobs/{id} for status.
|
|
"""
|
|
# Import neopig here to avoid circular imports
|
|
from neopig import NeoPig
|
|
from async_web_fetcher import CrawlMode
|
|
|
|
# Map mode string to enum
|
|
mode_map = {
|
|
"text": CrawlMode.TEXT,
|
|
"images": CrawlMode.IMAGES,
|
|
"videos": CrawlMode.VIDEOS,
|
|
"media": CrawlMode.MEDIA,
|
|
"all": CrawlMode.ALL,
|
|
}
|
|
mode = mode_map.get(request.mode, CrawlMode.IMAGES)
|
|
|
|
# Get targets (support both new 'targets' array and old 'target_uri' single value)
|
|
targets = request.targets if request.targets else [request.target_uri] if request.target_uri else []
|
|
if not targets:
|
|
raise HTTPException(status_code=400, detail="No target URIs provided")
|
|
|
|
job_ids = []
|
|
|
|
# Create a job for each target
|
|
for target_uri in targets:
|
|
job_id = await db.create_crawl_job(target_uri, request.keywords, request.mode)
|
|
job_ids.append(job_id)
|
|
|
|
# Run crawl in background (closure captures job_id and target_uri)
|
|
async def run_crawl(jid=job_id, uri=target_uri):
|
|
try:
|
|
pig = NeoPig(db_path=DB_PATH, vault_path=str(VAULT_PATH))
|
|
await pig.init()
|
|
|
|
# Load existing seen media for resume capability
|
|
crawled_media = await pig.db.get_crawled_media_uris()
|
|
if crawled_media:
|
|
pig.seen_media = crawled_media
|
|
|
|
# Cap depth at 15 (convert -1 or values > 15 to 15)
|
|
depth = request.depth if 1 <= request.depth <= 15 else 15
|
|
|
|
stats = await pig.crawl(
|
|
target_uri=uri,
|
|
keywords=request.keywords,
|
|
mode=mode,
|
|
depth=depth,
|
|
max_pages=request.max_pages,
|
|
download_media=request.download_media,
|
|
)
|
|
|
|
# Update job as completed
|
|
await db.complete_crawl_job(jid, stats)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Crawl job {jid} failed: {e}")
|
|
await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})
|
|
|
|
# Schedule background task
|
|
background_tasks.add_task(asyncio.create_task, run_crawl())
|
|
|
|
return {"job_ids": job_ids, "status": "running", "count": len(job_ids)}
|
|
|
|
|
|
def init_tarball_mode(tarball_path: str):
|
|
"""Initialize serving from a tar.gz archive."""
|
|
global TAR_FILE, TAR_MEMBERS, ARCHIVE_ROOT, DB_PATH, TEMP_DB_PATH
|
|
|
|
logger.info(f"Opening archive: {tarball_path}")
|
|
tarball = Path(tarball_path)
|
|
|
|
# Handle .run files with NEOPIG trailer
|
|
offset = 0
|
|
if tarball.suffix == '.run' or tarball.stat().st_size > 100000:
|
|
try:
|
|
with open(tarball, 'rb') as f:
|
|
f.seek(-22, 2)
|
|
trailer = f.read(22)
|
|
if trailer[:6] == b'NEOPIG':
|
|
offset = int(trailer[6:22].decode(), 16)
|
|
logger.info(f"Detected .run format, tarball offset: {offset}")
|
|
except Exception:
|
|
pass
|
|
|
|
# Open tarball
|
|
if offset > 0:
|
|
# Create a wrapper that presents just the tarball portion
|
|
class OffsetFile:
|
|
"""File wrapper that starts reading from an offset."""
|
|
def __init__(self, path, offset):
|
|
self._f = open(path, 'rb')
|
|
self._offset = offset
|
|
self._f.seek(offset)
|
|
def read(self, size=-1):
|
|
return self._f.read(size)
|
|
def seek(self, pos, whence=0):
|
|
if whence == 0: # SEEK_SET
|
|
return self._f.seek(self._offset + pos)
|
|
elif whence == 1: # SEEK_CUR
|
|
return self._f.seek(pos, 1)
|
|
else: # SEEK_END
|
|
return self._f.seek(pos, 2)
|
|
def tell(self):
|
|
return self._f.tell() - self._offset
|
|
def close(self):
|
|
self._f.close()
|
|
|
|
TAR_FILE = tarfile.open(fileobj=OffsetFile(tarball, offset), mode='r:gz')
|
|
else:
|
|
TAR_FILE = tarfile.open(tarball_path, 'r:gz')
|
|
|
|
# Build member lookup
|
|
for member in TAR_FILE.getmembers():
|
|
TAR_MEMBERS[member.name] = member
|
|
|
|
# Get archive root from first member
|
|
first = list(TAR_MEMBERS.keys())[0]
|
|
ARCHIVE_ROOT = first.split('/')[0]
|
|
logger.info(f"Archive root: {ARCHIVE_ROOT}")
|
|
|
|
# Extract database to temp (SQLite needs real file)
|
|
db_member = f"{ARCHIVE_ROOT}/archive.db"
|
|
if db_member in TAR_MEMBERS:
|
|
temp_dir = tempfile.mkdtemp(prefix="neopig_")
|
|
TEMP_DB_PATH = f"{temp_dir}/archive.db"
|
|
member = TAR_MEMBERS[db_member]
|
|
f = TAR_FILE.extractfile(member)
|
|
if f:
|
|
with open(TEMP_DB_PATH, 'wb') as out:
|
|
out.write(f.read())
|
|
DB_PATH = TEMP_DB_PATH
|
|
logger.info(f"Extracted database to: {TEMP_DB_PATH}")
|
|
else:
|
|
logger.warning("No archive.db found in tarball")
|
|
|
|
|
|
def main():
|
|
global DB_PATH, VAULT_PATH
|
|
|
|
parser = argparse.ArgumentParser(description="neopig SERP")
|
|
parser.add_argument("tarball", nargs='?', help="Path to archive.tar.gz or .run file")
|
|
parser.add_argument("--port", type=int, default=8000)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--db", default="data/neopig.db")
|
|
parser.add_argument("--vault", default="data/vault")
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Tarball mode
|
|
if args.tarball:
|
|
init_tarball_mode(args.tarball)
|
|
logger.info(f"Starting neopig SERP (archive mode) on {args.host}:{args.port}")
|
|
else:
|
|
DB_PATH = args.db
|
|
VAULT_PATH = Path(args.vault)
|
|
logger.info(f"Starting neopig SERP on {args.host}:{args.port}")
|
|
logger.info(f"Database: {DB_PATH}")
|
|
logger.info(f"Vault: {VAULT_PATH}")
|
|
|
|
uvicorn.run(app, host=args.host, port=args.port)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|