Russell Ballestrini
416b3cb760
- Bootstrap C program extracts serve.py and runs from tarball - Archives now include neopig source files for self-contained crawling - --upgrade-neopig flag to update neopig in existing archives - html2md.py: smart HTML-to-markdown converter for forums/blogs/Q&A - Fix vault path defaults (data/vault instead of vault) - Streaming tar.gz creation without temp copies - URL rewriting for local media references in archives
1654 lines
61 KiB
Python
1654 lines
61 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
archive.py - Site Archiver for Sunset Sites
|
|
|
|
Thin wrapper around neopig that packages crawl results into a distributable
|
|
tar.gz archive. Uses neopig for the actual crawling.
|
|
|
|
Output structure:
|
|
{domain}-{date}/
|
|
html/ # Original HTML pages
|
|
markdown/ # Converted markdown (optional)
|
|
media/ # Images, videos, audio
|
|
screenshots/ # Page screenshots (optional)
|
|
archive.db # SQLite search database
|
|
serve.py # Embedded search server
|
|
metadata.json # Crawl metadata
|
|
|
|
Usage:
|
|
python archive.py https://discourse-urho3d.github.io/
|
|
python archive.py https://example.com --no-screenshots --no-markdown
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, Any, List, Optional
|
|
from urllib.parse import urlparse
|
|
|
|
import aiofiles
|
|
import aiofiles.os
|
|
from bs4 import BeautifulSoup
|
|
from tqdm import tqdm
|
|
|
|
from neopig import NeoPig, setup_logging
|
|
from async_web_fetcher import CrawlMode
|
|
from screenshot import ScreenshotConfig
|
|
|
|
# Optional markdown conversion
|
|
try:
|
|
import html2text
|
|
HAS_HTML2TEXT = True
|
|
except ImportError:
|
|
HAS_HTML2TEXT = False
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def sanitize_filename(name: str) -> str:
|
|
"""Sanitize a string for use as filename. Uses - as separator."""
|
|
name = re.sub(r'[<>:"/\\|?*.]', '-', name)
|
|
name = re.sub(r'-+', '-', name) # collapse multiple dashes
|
|
name = name.strip('- ')
|
|
return name[:200] if name else 'unnamed'
|
|
|
|
|
|
def url_to_path(url: str) -> str:
|
|
"""Convert URL to filesystem path."""
|
|
parsed = urlparse(url)
|
|
path = parsed.path.strip('/')
|
|
if not path:
|
|
return 'index.html'
|
|
if path.endswith('.html') or path.endswith('.htm'):
|
|
return path
|
|
if '.' in path.split('/')[-1]:
|
|
return path
|
|
return f"{path}/index.html"
|
|
|
|
|
|
def html_to_markdown(html: str, base_url: str = '', trim_wrapper: bool = False) -> str:
|
|
"""Convert HTML to markdown."""
|
|
if not HAS_HTML2TEXT:
|
|
return html
|
|
# Optionally strip nav/header/footer/logo before conversion
|
|
if trim_wrapper:
|
|
from neopig import trim_html_wrapper
|
|
html = trim_html_wrapper(html)
|
|
h = html2text.HTML2Text()
|
|
h.ignore_links = False
|
|
h.ignore_images = False
|
|
h.body_width = 0
|
|
h.unicode_snob = True
|
|
if base_url:
|
|
h.baseurl = base_url
|
|
return h.handle(html)
|
|
|
|
|
|
class SiteArchiver:
|
|
"""
|
|
Packages neopig crawl results into a distributable tar.gz archive.
|
|
|
|
Uses neopig for crawling, then reads from its vaults to build the archive.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
output_dir: str = '.',
|
|
include_screenshots: bool = True,
|
|
include_markdown: bool = True,
|
|
screenshot_config: ScreenshotConfig = None,
|
|
fast_mode: bool = False,
|
|
trim_wrapper: bool = False,
|
|
):
|
|
self.output_dir = Path(output_dir)
|
|
self.include_screenshots = include_screenshots
|
|
self.include_markdown = include_markdown and HAS_HTML2TEXT
|
|
self.screenshot_config = screenshot_config or ScreenshotConfig(enabled=include_screenshots)
|
|
self.fast_mode = fast_mode
|
|
self.trim_wrapper = trim_wrapper
|
|
|
|
async def archive(
|
|
self,
|
|
target_url: str,
|
|
depth: int = -1,
|
|
max_pages: int = -1,
|
|
db_path: str = None,
|
|
vault_path: str = None,
|
|
) -> Path:
|
|
"""
|
|
Archive a site using neopig and package into tar.gz.
|
|
"""
|
|
parsed = urlparse(target_url)
|
|
domain = parsed.netloc.lower()
|
|
date_str = datetime.now().strftime('%Y%m%d')
|
|
archive_name = f"{sanitize_filename(domain)}-{date_str}"
|
|
|
|
# Use standard neopig data paths - one vault, multiple domains
|
|
if not db_path:
|
|
db_path = "data/neopig.db"
|
|
if not vault_path:
|
|
vault_path = "data/vault"
|
|
|
|
# Ensure data directory exists
|
|
Path("data").mkdir(exist_ok=True)
|
|
|
|
logger.info(f"Starting archive of {target_url}")
|
|
logger.info(f"Archive name: {archive_name}")
|
|
|
|
# Create neopig instance and crawl
|
|
pig = NeoPig(
|
|
db_path=db_path,
|
|
vault_path=vault_path,
|
|
screenshot_config=self.screenshot_config,
|
|
fast_mode=self.fast_mode,
|
|
trim_wrapper=self.trim_wrapper,
|
|
)
|
|
await pig.init()
|
|
|
|
# Load previously crawled media/screenshots from DB (source of truth for resume)
|
|
crawled_media = await pig.db.get_crawled_media_uris()
|
|
crawled_screenshots = await pig.db.get_crawled_screenshot_uris()
|
|
logger.info(f"Resume state: {len(crawled_media)} media URIs, {len(crawled_screenshots)} screenshotted pages in DB")
|
|
if crawled_media:
|
|
pig.seen_media = crawled_media
|
|
if crawled_screenshots:
|
|
pig.seen_screenshots = crawled_screenshots
|
|
logger.info(f"Screenshots enabled: {pig.screenshot_config.enabled}")
|
|
|
|
# Run the crawl
|
|
stats = await pig.crawl(
|
|
target_uri=target_url,
|
|
mode=CrawlMode.ALL,
|
|
depth=depth,
|
|
max_pages=max_pages,
|
|
download_media=True,
|
|
)
|
|
|
|
# Now package the results
|
|
logger.info("Packaging archive (streaming mode)...")
|
|
|
|
tar_path = self.output_dir / f"{archive_name}.tar.gz"
|
|
html_vault_path = Path(vault_path) / 'html_vault' / domain
|
|
media_vault_path = Path(vault_path) / 'media_vault' / domain
|
|
linkpeek_vault_path = Path(vault_path) / 'linkpeek_vault' / domain
|
|
|
|
# Get URL-to-hash mapping for rewriting external URLs to local copies
|
|
logger.info("Loading media URL mappings...")
|
|
url_to_hash = await pig.db.get_all_media_uri_mappings()
|
|
logger.info(f"Loaded {len(url_to_hash)} URL mappings for rewriting")
|
|
|
|
def rewrite_urls(html_content: str) -> str:
|
|
"""Rewrite external image/media URLs to local archive paths."""
|
|
import re
|
|
def replace_url(match):
|
|
url = match.group(1)
|
|
if url in url_to_hash:
|
|
md5 = url_to_hash[url]
|
|
# Get extension from original URL
|
|
ext = Path(url.split('?')[0]).suffix or '.bin'
|
|
return match.group(0).replace(url, f'../media/{md5}{ext}')
|
|
return match.group(0)
|
|
|
|
# Replace src="url" and href="url" patterns
|
|
html_content = re.sub(r'src=["\']([^"\']+)["\']', replace_url, html_content)
|
|
html_content = re.sub(r'href=["\']([^"\']+\.(jpg|jpeg|png|gif|webp|mp4|webm|svg))["\']',
|
|
replace_url, html_content, flags=re.IGNORECASE)
|
|
return html_content
|
|
|
|
# Step 1: Scan HTML files to build sitemap and search index (no copying)
|
|
logger.info("Building search index...")
|
|
sitemap = []
|
|
html_contents = {}
|
|
|
|
if html_vault_path.exists():
|
|
for html_file in html_vault_path.rglob('*.html'):
|
|
rel_path = html_file.relative_to(html_vault_path)
|
|
try:
|
|
content = html_file.read_text(encoding='utf-8', errors='replace')
|
|
# Rewrite external URLs to local copies
|
|
content = rewrite_urls(content)
|
|
title = self._extract_title(content) or str(rel_path)
|
|
sitemap.append({'path': f'html/{rel_path}', 'title': title})
|
|
html_contents[str(rel_path)] = content
|
|
except Exception:
|
|
pass
|
|
|
|
# Step 2: Create generated files in small temp dir
|
|
local_tmpdir = self.output_dir / '.tmp'
|
|
local_tmpdir.mkdir(parents=True, exist_ok=True)
|
|
|
|
with tempfile.TemporaryDirectory(dir=local_tmpdir) as tmpdir:
|
|
tmpdir_path = Path(tmpdir)
|
|
|
|
# Create search database
|
|
await self._create_search_database_streaming(tmpdir_path, sitemap, domain, html_contents)
|
|
|
|
# Write serve.py
|
|
self._write_serve_py(tmpdir_path)
|
|
|
|
# Write metadata
|
|
metadata = {
|
|
'domain': domain,
|
|
'target_url': target_url,
|
|
'created': datetime.now(timezone.utc).isoformat(),
|
|
'stats': stats,
|
|
'include_screenshots': self.include_screenshots,
|
|
'include_markdown': self.include_markdown,
|
|
}
|
|
(tmpdir_path / 'metadata.json').write_text(json.dumps(metadata, indent=2))
|
|
|
|
# Copy state file
|
|
state_domain = domain.replace('.', '-').replace(':', '-')
|
|
state_file = Path("data") / f"crawl-state-{state_domain}.json"
|
|
if state_file.exists():
|
|
shutil.copy(state_file, tmpdir_path / 'crawl_state.json')
|
|
logger.info(f"Included crawl state for future delta crawls")
|
|
|
|
# Write index.html
|
|
self._write_index_html(tmpdir_path, sitemap, domain)
|
|
|
|
# Step 3: Stream everything to tar.gz in one pass
|
|
logger.info("Streaming to archive...")
|
|
|
|
def stream_to_tar():
|
|
with tarfile.open(tar_path, 'w:gz') as tar:
|
|
# Add generated files first (from temp)
|
|
for f in tmpdir_path.iterdir():
|
|
tar.add(f, arcname=f"{archive_name}/{f.name}")
|
|
|
|
# Stream HTML files (with rewritten URLs)
|
|
for rel_path_str, content in html_contents.items():
|
|
try:
|
|
html_bytes = content.encode('utf-8')
|
|
arcname = f"{archive_name}/html/{rel_path_str}"
|
|
info = tarfile.TarInfo(name=arcname)
|
|
info.size = len(html_bytes)
|
|
tar.addfile(info, io.BytesIO(html_bytes))
|
|
|
|
# Generate markdown on the fly
|
|
if self.include_markdown:
|
|
md_content = html_to_markdown(content, trim_wrapper=self.trim_wrapper)
|
|
md_bytes = md_content.encode('utf-8')
|
|
md_rel = rel_path_str.replace('.html', '.md')
|
|
md_arcname = f"{archive_name}/markdown/{md_rel}"
|
|
md_info = tarfile.TarInfo(name=md_arcname)
|
|
md_info.size = len(md_bytes)
|
|
tar.addfile(md_info, io.BytesIO(md_bytes))
|
|
except Exception as e:
|
|
logger.debug(f"Error adding HTML {rel_path_str}: {e}")
|
|
|
|
# Stream media files by hash (matching rewritten URLs)
|
|
# Build hash -> file path mapping from hash vault
|
|
hash_vault = Path(vault_path)
|
|
added_hashes = set()
|
|
for url, md5 in url_to_hash.items():
|
|
if md5 in added_hashes:
|
|
continue
|
|
# Find file in hash vault: vault/xx/hash.ext
|
|
bucket = md5[:2]
|
|
bucket_dir = hash_vault / bucket
|
|
if bucket_dir.exists():
|
|
for f in bucket_dir.iterdir():
|
|
if f.stem == md5:
|
|
try:
|
|
arcname = f"{archive_name}/media/{f.name}"
|
|
tar.add(f, arcname=arcname)
|
|
added_hashes.add(md5)
|
|
except Exception as e:
|
|
logger.debug(f"Error adding media {f}: {e}")
|
|
break
|
|
|
|
# Stream screenshots (follow symlinks)
|
|
if self.include_screenshots and linkpeek_vault_path.exists():
|
|
for screenshot_file in linkpeek_vault_path.rglob('*.png'):
|
|
try:
|
|
rel_path = screenshot_file.relative_to(linkpeek_vault_path)
|
|
arcname = f"{archive_name}/screenshots/{rel_path}"
|
|
if screenshot_file.is_symlink():
|
|
target = screenshot_file.resolve()
|
|
if target.exists():
|
|
tar.add(target, arcname=arcname)
|
|
else:
|
|
tar.add(screenshot_file, arcname=arcname)
|
|
except Exception as e:
|
|
logger.debug(f"Error adding screenshot {screenshot_file}: {e}")
|
|
|
|
# Bundle neopig source files for self-contained crawling
|
|
neopig_src = Path(__file__).parent
|
|
neopig_files = [
|
|
'neopig.py', 'database.py', 'async_web_fetcher.py',
|
|
'storage.py', 'domain_vault.py', 'screenshot.py',
|
|
'html2md.py', 'serp.py', 'filevault.py', 'async_filevault.py',
|
|
]
|
|
for pyfile in neopig_files:
|
|
src_path = neopig_src / pyfile
|
|
if src_path.exists():
|
|
tar.add(src_path, arcname=f"{archive_name}/neopig/{pyfile}")
|
|
|
|
# Add requirements.txt for neopig dependencies
|
|
req_path = neopig_src / 'requirements.txt'
|
|
if req_path.exists():
|
|
tar.add(req_path, arcname=f"{archive_name}/requirements.txt")
|
|
|
|
await asyncio.to_thread(stream_to_tar)
|
|
|
|
# Clear html_contents to free memory
|
|
html_contents.clear()
|
|
|
|
final_size = tar_path.stat().st_size
|
|
logger.info(f"Archive complete: {tar_path} ({final_size / 1024 / 1024:.1f} MB)")
|
|
|
|
return tar_path
|
|
|
|
async def _create_search_database_streaming(self, tmpdir: Path, sitemap: list, domain: str, html_contents: dict):
|
|
"""Create FTS5 search database from collected html contents."""
|
|
import sqlite3
|
|
|
|
db_path = tmpdir / 'archive.db'
|
|
|
|
def create_db():
|
|
conn = sqlite3.connect(db_path)
|
|
c = conn.cursor()
|
|
|
|
c.execute('''
|
|
CREATE TABLE pages (
|
|
id INTEGER PRIMARY KEY,
|
|
path TEXT UNIQUE,
|
|
title TEXT,
|
|
content TEXT
|
|
)
|
|
''')
|
|
c.execute('''
|
|
CREATE VIRTUAL TABLE pages_fts USING fts5(
|
|
title, content, path,
|
|
content='pages',
|
|
content_rowid='id'
|
|
)
|
|
''')
|
|
|
|
for item in sitemap:
|
|
html_path = item['path'].replace('html/', '', 1)
|
|
content = html_contents.get(html_path, '')
|
|
if content:
|
|
try:
|
|
soup = BeautifulSoup(content, 'html.parser')
|
|
for tag in soup(['script', 'style', 'nav', 'header', 'footer']):
|
|
tag.decompose()
|
|
text = soup.get_text(separator=' ', strip=True)[:50000]
|
|
except Exception:
|
|
text = ''
|
|
else:
|
|
text = ''
|
|
|
|
try:
|
|
c.execute('INSERT INTO pages (path, title, content) VALUES (?, ?, ?)',
|
|
(item['path'], item['title'], text))
|
|
except Exception:
|
|
pass
|
|
|
|
c.execute('''
|
|
INSERT INTO pages_fts(rowid, title, content, path)
|
|
SELECT id, title, content, path FROM pages
|
|
''')
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
await asyncio.to_thread(create_db)
|
|
|
|
def _write_index_html(self, tmpdir: Path, sitemap: list, domain: str):
|
|
"""Write index.html with sitemap."""
|
|
html = f'''<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>{domain} Archive</title>
|
|
<style>
|
|
body {{ font-family: sans-serif; max-width: 800px; margin: 40px auto; padding: 0 20px; }}
|
|
h1 {{ color: #333; }}
|
|
ul {{ list-style: none; padding: 0; }}
|
|
li {{ padding: 8px 0; border-bottom: 1px solid #eee; }}
|
|
a {{ color: #0066cc; text-decoration: none; }}
|
|
a:hover {{ text-decoration: underline; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>{domain} Archive</h1>
|
|
<p>{len(sitemap)} pages archived</p>
|
|
<ul>
|
|
'''
|
|
for item in sitemap[:1000]: # Limit to 1000 in index
|
|
html += f' <li><a href="{item["path"]}">{item["title"]}</a></li>\n'
|
|
if len(sitemap) > 1000:
|
|
html += f' <li>... and {len(sitemap) - 1000} more pages</li>\n'
|
|
html += ''' </ul>
|
|
</body>
|
|
</html>'''
|
|
(tmpdir / 'index.html').write_text(html)
|
|
|
|
def _extract_title(self, html: str) -> Optional[str]:
|
|
"""Extract title from HTML."""
|
|
try:
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
if soup.title and soup.title.string:
|
|
return soup.title.string.strip()
|
|
h1 = soup.find('h1')
|
|
if h1:
|
|
return h1.get_text().strip()
|
|
except Exception:
|
|
pass
|
|
return None
|
|
|
|
def _extract_text(self, html: str) -> str:
|
|
"""Extract readable text from HTML for search indexing."""
|
|
try:
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
for tag in soup(['script', 'style', 'nav', 'footer', 'header']):
|
|
tag.decompose()
|
|
text = soup.get_text(separator=' ', strip=True)
|
|
text = re.sub(r'\s+', ' ', text)
|
|
return text[:50000]
|
|
except Exception:
|
|
return ''
|
|
|
|
async def _create_search_database(
|
|
self,
|
|
archive_root: Path,
|
|
sitemap: List[Dict[str, str]],
|
|
domain: str,
|
|
html_dir: Path,
|
|
):
|
|
"""Create SQLite database with searchable page content."""
|
|
db_path = archive_root / 'archive.db'
|
|
|
|
def create_db():
|
|
conn = sqlite3.connect(db_path)
|
|
c = conn.cursor()
|
|
|
|
c.execute('''
|
|
CREATE TABLE pages (
|
|
id INTEGER PRIMARY KEY,
|
|
url TEXT NOT NULL,
|
|
path TEXT NOT NULL,
|
|
title TEXT,
|
|
content TEXT
|
|
)
|
|
''')
|
|
|
|
c.execute('''
|
|
CREATE VIRTUAL TABLE pages_fts USING fts5(
|
|
title, content, url, path,
|
|
content='pages',
|
|
content_rowid='id'
|
|
)
|
|
''')
|
|
|
|
c.execute('''
|
|
CREATE TRIGGER pages_ai AFTER INSERT ON pages BEGIN
|
|
INSERT INTO pages_fts(rowid, title, content, url, path)
|
|
VALUES (new.id, new.title, new.content, new.url, new.path);
|
|
END
|
|
''')
|
|
|
|
for item in sitemap:
|
|
path = item['path']
|
|
title = item['title']
|
|
|
|
html_path = archive_root / path
|
|
if html_path.exists():
|
|
html_content = html_path.read_text(encoding='utf-8', errors='replace')
|
|
text_content = self._extract_text(html_content)
|
|
else:
|
|
text_content = ''
|
|
|
|
c.execute(
|
|
'INSERT INTO pages (url, path, title, content) VALUES (?, ?, ?, ?)',
|
|
(path, path, title, text_content)
|
|
)
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
await asyncio.to_thread(create_db)
|
|
logger.info(f"Created search database: {db_path}")
|
|
|
|
def _write_serve_py(self, archive_root: Path):
|
|
"""Write embedded search server (stdlib only, no dependencies)."""
|
|
serve_py = '''#!/usr/bin/env python3
|
|
"""
|
|
neopig Archive Server - Browse and search archived sites.
|
|
|
|
Zero dependencies - uses only Python stdlib.
|
|
|
|
Usage:
|
|
python serve.py # Serve from extracted archive
|
|
python serve.py archive.tar.gz # Serve directly from tarball
|
|
python serve.py -p 8080 # Custom port
|
|
./archive.run # Self-extracting archive
|
|
"""
|
|
|
|
import argparse
|
|
import html
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
from http.server import HTTPServer, BaseHTTPRequestHandler
|
|
from pathlib import Path
|
|
from urllib.parse import parse_qs, urlparse, unquote
|
|
|
|
# Globals set at startup
|
|
ARCHIVE_ROOT = None
|
|
TAR_FILE = None
|
|
TAR_MEMBERS = {}
|
|
DB_PATH = None
|
|
METADATA = {}
|
|
|
|
|
|
def get_archive_source():
|
|
"""Determine if we are in a tarball, .run, or extracted directory."""
|
|
exe_path = Path(sys.argv[0]).resolve()
|
|
|
|
# Check for .run format (has NEOPIG trailer)
|
|
if exe_path.suffix == '.run' or (len(sys.argv) == 1 and exe_path.stat().st_size > 1000000):
|
|
try:
|
|
with open(exe_path, 'rb') as f:
|
|
f.seek(-22, 2)
|
|
trailer = f.read(22)
|
|
if trailer[:6] == b'NEOPIG':
|
|
offset = int(trailer[6:22].decode(), 16)
|
|
return ('run', exe_path, offset)
|
|
except Exception:
|
|
pass
|
|
|
|
# Check command line for tarball or .run argument
|
|
for arg in sys.argv[1:]:
|
|
if not arg.startswith('-'):
|
|
p = Path(arg)
|
|
if p.exists():
|
|
if p.suffix in ('.gz', '.tar', '.tgz'):
|
|
return ('tarball', p, 0)
|
|
# Check if it's a .run file with NEOPIG trailer
|
|
if p.suffix == '.run' or p.stat().st_size > 100000:
|
|
try:
|
|
with open(p, 'rb') as f:
|
|
f.seek(-22, 2)
|
|
trailer = f.read(22)
|
|
if trailer[:6] == b'NEOPIG':
|
|
offset = int(trailer[6:22].decode(), 16)
|
|
return ('run', p, offset)
|
|
except Exception:
|
|
pass
|
|
|
|
# Must be extracted directory
|
|
return ('directory', Path(__file__).parent, 0)
|
|
|
|
|
|
def init_archive():
|
|
"""Initialize archive access."""
|
|
global ARCHIVE_ROOT, TAR_FILE, TAR_MEMBERS, DB_PATH, METADATA
|
|
|
|
source_type, source_path, offset = get_archive_source()
|
|
|
|
if source_type == 'run':
|
|
print(f"Serving from self-extracting archive: {source_path}")
|
|
f = open(source_path, 'rb')
|
|
f.seek(offset)
|
|
TAR_FILE = tarfile.open(fileobj=f, mode='r:gz')
|
|
elif source_type == 'tarball':
|
|
print(f"Serving from tarball: {source_path}")
|
|
TAR_FILE = tarfile.open(source_path, 'r:gz')
|
|
else:
|
|
print(f"Serving from directory: {source_path}")
|
|
ARCHIVE_ROOT = source_path
|
|
|
|
if TAR_FILE:
|
|
# Build member lookup and find archive root
|
|
for member in TAR_FILE.getmembers():
|
|
TAR_MEMBERS[member.name] = member
|
|
first = list(TAR_MEMBERS.keys())[0]
|
|
archive_name = first.split('/')[0]
|
|
ARCHIVE_ROOT = Path(archive_name)
|
|
|
|
# Extract database to temp for searching
|
|
db_member = f"{archive_name}/archive.db"
|
|
if db_member in TAR_MEMBERS:
|
|
temp_dir = tempfile.mkdtemp()
|
|
TAR_FILE.extract(TAR_MEMBERS[db_member], temp_dir)
|
|
DB_PATH = Path(temp_dir) / db_member
|
|
else:
|
|
DB_PATH = ARCHIVE_ROOT / 'archive.db'
|
|
|
|
# Load metadata
|
|
meta_path = ARCHIVE_ROOT / 'metadata.json' if not TAR_FILE else None
|
|
if meta_path and meta_path.exists():
|
|
METADATA = json.loads(meta_path.read_text())
|
|
elif TAR_FILE:
|
|
meta_member = f"{ARCHIVE_ROOT}/metadata.json"
|
|
if meta_member in TAR_MEMBERS:
|
|
f = TAR_FILE.extractfile(TAR_MEMBERS[meta_member])
|
|
if f:
|
|
METADATA = json.loads(f.read().decode())
|
|
|
|
|
|
def read_file(path: str) -> tuple:
|
|
"""Read file from archive. Returns (content_bytes, mime_type) or (None, None)."""
|
|
if TAR_FILE:
|
|
# Normalize path for tarball
|
|
tar_path = f"{ARCHIVE_ROOT}/{path}".lstrip('/')
|
|
if tar_path in TAR_MEMBERS:
|
|
f = TAR_FILE.extractfile(TAR_MEMBERS[tar_path])
|
|
if f:
|
|
mime, _ = mimetypes.guess_type(path)
|
|
return f.read(), mime or 'application/octet-stream'
|
|
return None, None
|
|
else:
|
|
file_path = ARCHIVE_ROOT / path
|
|
if file_path.exists() and file_path.is_file():
|
|
# Security: prevent path traversal
|
|
try:
|
|
file_path.resolve().relative_to(ARCHIVE_ROOT.resolve())
|
|
except ValueError:
|
|
return None, None
|
|
mime, _ = mimetypes.guess_type(str(file_path))
|
|
return file_path.read_bytes(), mime or 'application/octet-stream'
|
|
return None, None
|
|
|
|
|
|
def list_files(subdir: str, pattern: str = '*') -> list:
|
|
"""List files in a subdirectory."""
|
|
files = []
|
|
if TAR_FILE:
|
|
prefix = f"{ARCHIVE_ROOT}/{subdir}/"
|
|
for name in TAR_MEMBERS:
|
|
if name.startswith(prefix) and not name.endswith('/'):
|
|
rel = name[len(prefix):]
|
|
if pattern == '*' or rel.endswith(pattern.replace('*', '')):
|
|
files.append(rel)
|
|
else:
|
|
dir_path = ARCHIVE_ROOT / subdir
|
|
if dir_path.exists():
|
|
for f in dir_path.rglob(pattern):
|
|
if f.is_file():
|
|
files.append(str(f.relative_to(dir_path)))
|
|
return sorted(files)
|
|
|
|
|
|
def search_pages(query: str, limit: int = 50) -> list:
|
|
"""Search pages using FTS5 with LIKE fallback."""
|
|
if not DB_PATH or not DB_PATH.exists():
|
|
return []
|
|
|
|
conn = sqlite3.connect(DB_PATH)
|
|
c = conn.cursor()
|
|
results = []
|
|
try:
|
|
# Try FTS5 with prefix matching (add * for partial matches)
|
|
fts_query = ' '.join(f'"{word}"*' for word in query.split())
|
|
c.execute("""
|
|
SELECT p.path, p.title, snippet(pages_fts, 1, '<mark>', '</mark>', '...', 40)
|
|
FROM pages_fts
|
|
JOIN pages p ON pages_fts.rowid = p.id
|
|
WHERE pages_fts MATCH ?
|
|
ORDER BY rank
|
|
LIMIT ?
|
|
""", (fts_query, limit))
|
|
results = [{'path': r[0], 'title': r[1], 'snippet': r[2]} for r in c.fetchall()]
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
|
|
# Fallback to LIKE for substring matching if FTS5 found nothing
|
|
if not results:
|
|
try:
|
|
like_q = f'%{query}%'
|
|
c.execute("""
|
|
SELECT path, title, substr(content, 1, 200) as snippet
|
|
FROM pages
|
|
WHERE title LIKE ? COLLATE NOCASE
|
|
OR content LIKE ? COLLATE NOCASE
|
|
OR path LIKE ? COLLATE NOCASE
|
|
LIMIT ?
|
|
""", (like_q, like_q, like_q, limit))
|
|
results = [{'path': r[0], 'title': r[1], 'snippet': r[2] + '...'} for r in c.fetchall()]
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
|
|
conn.close()
|
|
return results
|
|
|
|
|
|
def get_stats() -> dict:
|
|
"""Get archive statistics."""
|
|
stats = {
|
|
'pages': len(list_files('html', '*.html')),
|
|
'media': len(list_files('media')),
|
|
'screenshots': len(list_files('screenshots', '*.png')),
|
|
'domain': METADATA.get('domain', 'unknown'),
|
|
'created': METADATA.get('created', 'unknown'),
|
|
}
|
|
if METADATA.get('stats'):
|
|
stats.update(METADATA['stats'])
|
|
return stats
|
|
|
|
|
|
# HTML Templates
|
|
INDEX_HTML = """<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>{domain} - neopig Archive</title>
|
|
<style>
|
|
* {{ box-sizing: border-box; }}
|
|
body {{
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0; padding: 20px;
|
|
background: #0a0a0a; color: #e0e0e0;
|
|
}}
|
|
.container {{ max-width: 1400px; margin: 0 auto; }}
|
|
h1 {{ color: #ff6b6b; margin-bottom: 5px; }}
|
|
.subtitle {{ color: #666; margin-bottom: 20px; }}
|
|
nav {{ margin-bottom: 20px; }}
|
|
nav a {{ color: #ff6b6b; margin-right: 15px; text-decoration: none; }}
|
|
nav a:hover {{ text-decoration: underline; }}
|
|
nav a.active {{ font-weight: bold; border-bottom: 2px solid #ff6b6b; }}
|
|
.search-box {{
|
|
display: flex; gap: 10px; margin-bottom: 20px;
|
|
}}
|
|
input[type="text"] {{
|
|
flex: 1; padding: 12px 16px; font-size: 16px;
|
|
border: 2px solid #333; border-radius: 8px;
|
|
background: #1a1a1a; color: #fff;
|
|
}}
|
|
input[type="text"]:focus {{ outline: none; border-color: #ff6b6b; }}
|
|
button {{
|
|
padding: 12px 24px; font-size: 16px;
|
|
background: #ff6b6b; color: #fff;
|
|
border: none; border-radius: 8px; cursor: pointer;
|
|
}}
|
|
button:hover {{ background: #ff5252; }}
|
|
.stats {{
|
|
display: flex; gap: 20px; flex-wrap: wrap;
|
|
padding: 15px; background: #1a1a1a;
|
|
border-radius: 8px; margin-bottom: 20px;
|
|
}}
|
|
.stat {{ text-align: center; }}
|
|
.stat-value {{ font-size: 24px; font-weight: bold; color: #ff6b6b; }}
|
|
.stat-label {{ font-size: 12px; color: #888; }}
|
|
.results {{ margin-top: 20px; }}
|
|
.result {{
|
|
background: #1a1a1a; border-radius: 8px;
|
|
padding: 15px; margin-bottom: 10px;
|
|
}}
|
|
.result h3 {{ margin: 0 0 8px 0; }}
|
|
.result a {{ color: #ff6b6b; text-decoration: none; }}
|
|
.result a:hover {{ text-decoration: underline; }}
|
|
.result .snippet {{ color: #888; font-size: 14px; }}
|
|
.result .snippet mark {{ background: #ff6b6b; color: #fff; padding: 1px 3px; border-radius: 2px; }}
|
|
.result .path {{ font-size: 12px; color: #666; margin-top: 5px; }}
|
|
.grid {{
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
|
|
gap: 15px;
|
|
}}
|
|
.card {{
|
|
background: #1a1a1a; border-radius: 8px;
|
|
overflow: hidden; transition: transform 0.2s;
|
|
}}
|
|
.card:hover {{ transform: scale(1.02); }}
|
|
.card img {{
|
|
width: 100%; height: 150px;
|
|
object-fit: cover; background: #222;
|
|
}}
|
|
.card-info {{ padding: 10px; }}
|
|
.card-title {{
|
|
font-size: 12px; color: #ccc;
|
|
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
|
}}
|
|
.empty {{ text-align: center; padding: 60px; color: #666; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<h1>neopig Archive</h1>
|
|
<p class="subtitle">{domain} - archived {created}</p>
|
|
|
|
<nav>
|
|
<a href="/" class="active">Search</a>
|
|
<a href="/browse">Browse Pages</a>
|
|
<a href="/media">Media</a>
|
|
<a href="/screenshots">Screenshots</a>
|
|
</nav>
|
|
|
|
<div class="stats">
|
|
<div class="stat"><div class="stat-value">{pages}</div><div class="stat-label">Pages</div></div>
|
|
<div class="stat"><div class="stat-value">{media}</div><div class="stat-label">Media</div></div>
|
|
<div class="stat"><div class="stat-value">{screenshots}</div><div class="stat-label">Screenshots</div></div>
|
|
</div>
|
|
|
|
<div class="search-box">
|
|
<input type="text" id="query" placeholder="Search archived pages..." autofocus>
|
|
<button onclick="search()">Search</button>
|
|
</div>
|
|
|
|
<div class="results" id="results"></div>
|
|
</div>
|
|
|
|
<script>
|
|
async function search() {{
|
|
const q = document.getElementById('query').value;
|
|
if (!q) return;
|
|
const res = await fetch('/api/search?q=' + encodeURIComponent(q));
|
|
const data = await res.json();
|
|
const container = document.getElementById('results');
|
|
if (data.length === 0) {{
|
|
container.innerHTML = '<div class="empty">No results found</div>';
|
|
return;
|
|
}}
|
|
container.innerHTML = data.map(r => `
|
|
<div class="result">
|
|
<h3><a href="/view/${{r.path.replace('html/', '')}}">${{r.title}}</a></h3>
|
|
<p class="snippet">${{r.snippet}}</p>
|
|
<p class="path">${{r.path}}</p>
|
|
</div>
|
|
`).join('');
|
|
}}
|
|
document.getElementById('query').addEventListener('keypress', e => {{
|
|
if (e.key === 'Enter') search();
|
|
}});
|
|
</script>
|
|
</body>
|
|
</html>"""
|
|
|
|
BROWSE_HTML = """<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Browse - {domain}</title>
|
|
<style>
|
|
* {{ box-sizing: border-box; }}
|
|
body {{
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0; padding: 20px;
|
|
background: #0a0a0a; color: #e0e0e0;
|
|
}}
|
|
.container {{ max-width: 1400px; margin: 0 auto; }}
|
|
h1 {{ color: #ff6b6b; }}
|
|
nav {{ margin-bottom: 20px; }}
|
|
nav a {{ color: #ff6b6b; margin-right: 15px; text-decoration: none; }}
|
|
nav a:hover {{ text-decoration: underline; }}
|
|
.list {{ background: #1a1a1a; border-radius: 8px; }}
|
|
.list-item {{
|
|
padding: 12px 15px; border-bottom: 1px solid #333;
|
|
display: flex; align-items: center; gap: 10px;
|
|
}}
|
|
.list-item:last-child {{ border-bottom: none; }}
|
|
.list-item a {{ color: #ff6b6b; text-decoration: none; flex: 1; }}
|
|
.list-item a:hover {{ text-decoration: underline; }}
|
|
.list-item .path {{ color: #666; font-size: 12px; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<h1>Browse Pages</h1>
|
|
<nav>
|
|
<a href="/">Search</a>
|
|
<a href="/browse">Browse Pages</a>
|
|
<a href="/media">Media</a>
|
|
<a href="/screenshots">Screenshots</a>
|
|
</nav>
|
|
<div class="list">
|
|
{items}
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>"""
|
|
|
|
MEDIA_HTML = """<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Media - {domain}</title>
|
|
<style>
|
|
* {{ box-sizing: border-box; }}
|
|
body {{
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0; padding: 20px;
|
|
background: #0a0a0a; color: #e0e0e0;
|
|
}}
|
|
.container {{ max-width: 1600px; margin: 0 auto; }}
|
|
h1 {{ color: #ff6b6b; }}
|
|
nav {{ margin-bottom: 20px; }}
|
|
nav a {{ color: #ff6b6b; margin-right: 15px; text-decoration: none; }}
|
|
nav a:hover {{ text-decoration: underline; }}
|
|
.grid {{
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fill, minmax(220px, 1fr));
|
|
grid-auto-rows: min-content;
|
|
gap: 12px;
|
|
}}
|
|
.card {{
|
|
background: #1a1a1a; border-radius: 8px;
|
|
overflow: hidden; transition: transform 0.2s;
|
|
break-inside: avoid;
|
|
}}
|
|
.card:hover {{ transform: scale(1.02); }}
|
|
.card a {{ display: block; }}
|
|
.card img, .card video {{
|
|
width: 100%;
|
|
height: auto;
|
|
display: block;
|
|
background: #222;
|
|
}}
|
|
.card-info {{ padding: 8px 10px; }}
|
|
.card-title {{
|
|
font-size: 11px; color: #888;
|
|
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
|
}}
|
|
.empty {{ text-align: center; padding: 60px; color: #666; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<h1>{title}</h1>
|
|
<nav>
|
|
<a href="/">Search</a>
|
|
<a href="/browse">Browse Pages</a>
|
|
<a href="/media">Media</a>
|
|
<a href="/screenshots">Screenshots</a>
|
|
</nav>
|
|
<div class="grid">
|
|
{items}
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>"""
|
|
|
|
MEDIA_DETAIL_HTML = """<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>{caption} - neopig Archive</title>
|
|
<style>
|
|
* {{ box-sizing: border-box; }}
|
|
body {{
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0; padding: 20px;
|
|
background: #0a0a0a; color: #e0e0e0;
|
|
line-height: 1.6;
|
|
}}
|
|
.container {{ max-width: 1000px; margin: 0 auto; }}
|
|
nav {{ margin-bottom: 20px; padding-bottom: 15px; border-bottom: 1px solid #333; }}
|
|
nav a {{ color: #ff6b6b; margin-right: 15px; text-decoration: none; }}
|
|
nav a:hover {{ text-decoration: underline; }}
|
|
.hero {{
|
|
background: #111; border-radius: 12px; padding: 20px;
|
|
margin-bottom: 25px; text-align: center;
|
|
}}
|
|
.hero img, .hero video {{
|
|
max-width: 100%; max-height: 70vh;
|
|
border-radius: 8px; margin-bottom: 15px;
|
|
}}
|
|
.caption {{
|
|
font-size: 1.2em; color: #fff; margin: 15px 0 10px;
|
|
}}
|
|
.meta {{
|
|
font-size: 12px; color: #666; margin: 10px 0;
|
|
}}
|
|
.meta a {{ color: #6bb3ff; word-break: break-all; }}
|
|
.meta-row {{ margin: 5px 0; }}
|
|
.meta-label {{ color: #888; }}
|
|
.divider {{
|
|
border: none; border-top: 1px solid #333;
|
|
margin: 25px 0;
|
|
}}
|
|
.source-heading {{
|
|
color: #ff6b6b; font-size: 1.1em; margin-bottom: 15px;
|
|
}}
|
|
.content {{
|
|
background: #111; border-radius: 8px; padding: 25px;
|
|
font-size: 15px;
|
|
}}
|
|
.content h1, .content h2, .content h3 {{ color: #ff6b6b; margin-top: 1.5em; }}
|
|
.content h1:first-child, .content h2:first-child {{ margin-top: 0; }}
|
|
.content a {{ color: #6bb3ff; }}
|
|
.content code {{
|
|
background: #1a1a1a; padding: 2px 6px; border-radius: 3px;
|
|
font-family: monospace; font-size: 0.9em;
|
|
}}
|
|
.content pre {{
|
|
background: #1a1a1a; padding: 15px; border-radius: 6px;
|
|
overflow-x: auto; font-size: 0.85em;
|
|
}}
|
|
.content pre code {{ background: none; padding: 0; }}
|
|
.content blockquote {{
|
|
border-left: 3px solid #ff6b6b; margin: 1em 0;
|
|
padding-left: 15px; color: #aaa;
|
|
}}
|
|
.content img {{ max-width: 100%; height: auto; border-radius: 4px; }}
|
|
.content ul, .content ol {{ padding-left: 25px; }}
|
|
.content li {{ margin: 0.3em 0; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<nav>
|
|
<a href="/">Search</a>
|
|
<a href="/browse">Browse Pages</a>
|
|
<a href="/media">Media</a>
|
|
<a href="/screenshots">Screenshots</a>
|
|
</nav>
|
|
|
|
<div class="hero">
|
|
{media_element}
|
|
<div class="caption">{caption}</div>
|
|
<div class="meta">
|
|
<div class="meta-row"><span class="meta-label">Image:</span> <a href="{media_uri}" target="_blank">{media_uri}</a></div>
|
|
<div class="meta-row"><span class="meta-label">Source:</span> <a href="/view/{page_path}">{page_uri}</a></div>
|
|
</div>
|
|
</div>
|
|
|
|
<hr class="divider">
|
|
|
|
<div class="source-heading">Source Page Content</div>
|
|
<div class="content">
|
|
{page_content}
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
PAGE_VIEW_HTML = """<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>{title} - neopig Archive</title>
|
|
<style>
|
|
* {{ box-sizing: border-box; }}
|
|
body {{
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
margin: 0; padding: 20px;
|
|
background: #0a0a0a; color: #e0e0e0;
|
|
line-height: 1.6;
|
|
}}
|
|
.container {{ max-width: 900px; margin: 0 auto; }}
|
|
h1 {{ color: #ff6b6b; margin-bottom: 5px; font-size: 1.5em; }}
|
|
nav {{ margin-bottom: 20px; padding-bottom: 15px; border-bottom: 1px solid #333; }}
|
|
nav a {{ color: #ff6b6b; margin-right: 15px; text-decoration: none; }}
|
|
nav a:hover {{ text-decoration: underline; }}
|
|
.meta {{ color: #666; font-size: 12px; margin-bottom: 20px; }}
|
|
.meta a {{ color: #888; }}
|
|
.content {{
|
|
background: #111; border-radius: 8px; padding: 25px;
|
|
font-size: 15px;
|
|
}}
|
|
.content h1, .content h2, .content h3 {{ color: #ff6b6b; margin-top: 1.5em; }}
|
|
.content h1:first-child, .content h2:first-child {{ margin-top: 0; }}
|
|
.content a {{ color: #6bb3ff; }}
|
|
.content code {{
|
|
background: #1a1a1a; padding: 2px 6px; border-radius: 3px;
|
|
font-family: monospace; font-size: 0.9em;
|
|
}}
|
|
.content pre {{
|
|
background: #1a1a1a; padding: 15px; border-radius: 6px;
|
|
overflow-x: auto; font-size: 0.85em;
|
|
}}
|
|
.content pre code {{ background: none; padding: 0; }}
|
|
.content blockquote {{
|
|
border-left: 3px solid #ff6b6b; margin: 1em 0;
|
|
padding-left: 15px; color: #aaa;
|
|
}}
|
|
.content img {{ max-width: 100%; height: auto; border-radius: 4px; }}
|
|
.content ul, .content ol {{ padding-left: 25px; }}
|
|
.content li {{ margin: 0.3em 0; }}
|
|
.content hr {{ border: none; border-top: 1px solid #333; margin: 2em 0; }}
|
|
.screenshot {{ margin-top: 20px; }}
|
|
.screenshot img {{ max-width: 100%; border: 1px solid #333; border-radius: 4px; }}
|
|
.screenshot-label {{ color: #666; font-size: 12px; margin-bottom: 5px; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<nav>
|
|
<a href="/">Search</a>
|
|
<a href="/browse">Browse Pages</a>
|
|
<a href="/media">Media</a>
|
|
<a href="/screenshots">Screenshots</a>
|
|
</nav>
|
|
<h1>{title}</h1>
|
|
<div class="meta">
|
|
<a href="/html/{path}">[View Original HTML]</a>
|
|
{screenshot_link}
|
|
</div>
|
|
<div class="content">
|
|
{content}
|
|
</div>
|
|
{screenshot_embed}
|
|
</div>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
def simple_markdown_to_html(md: str) -> str:
|
|
"""Convert markdown to HTML (simple stdlib-only implementation)."""
|
|
import re
|
|
lines = md.split('\n')
|
|
html_lines = []
|
|
in_code_block = False
|
|
in_list = False
|
|
|
|
for line in lines:
|
|
# Code blocks
|
|
if line.startswith('```'):
|
|
if in_code_block:
|
|
html_lines.append('</code></pre>')
|
|
in_code_block = False
|
|
else:
|
|
html_lines.append('<pre><code>')
|
|
in_code_block = True
|
|
continue
|
|
|
|
if in_code_block:
|
|
html_lines.append(html.escape(line))
|
|
continue
|
|
|
|
# Close list if needed
|
|
if in_list and not line.strip().startswith(('- ', '* ', '1. ')):
|
|
html_lines.append('</ul>')
|
|
in_list = False
|
|
|
|
# Headers
|
|
if line.startswith('### '):
|
|
html_lines.append(f'<h3>{html.escape(line[4:])}</h3>')
|
|
elif line.startswith('## '):
|
|
html_lines.append(f'<h2>{html.escape(line[3:])}</h2>')
|
|
elif line.startswith('# '):
|
|
html_lines.append(f'<h1>{html.escape(line[2:])}</h1>')
|
|
# Blockquotes
|
|
elif line.startswith('> '):
|
|
html_lines.append(f'<blockquote>{html.escape(line[2:])}</blockquote>')
|
|
# Horizontal rule
|
|
elif line.strip() in ('---', '***', '___'):
|
|
html_lines.append('<hr>')
|
|
# Lists
|
|
elif line.strip().startswith(('- ', '* ')):
|
|
if not in_list:
|
|
html_lines.append('<ul>')
|
|
in_list = True
|
|
content = line.strip()[2:]
|
|
html_lines.append(f'<li>{html.escape(content)}</li>')
|
|
# Empty line
|
|
elif not line.strip():
|
|
html_lines.append('<br>')
|
|
# Regular paragraph
|
|
else:
|
|
escaped = html.escape(line)
|
|
# Inline code
|
|
escaped = re.sub(r'`([^`]+)`', r'<code>\1</code>', escaped)
|
|
# Bold
|
|
escaped = re.sub(r'[*][*]([^*]+)[*][*]', r'<strong>\1</strong>', escaped)
|
|
# Italic
|
|
escaped = re.sub(r'[*]([^*]+)[*]', r'<em>\1</em>', escaped)
|
|
# Links [text](url)
|
|
link_re = re.compile(r'\[([^]]+)\]\(([^)]+)\)')
|
|
escaped = link_re.sub(r'<a href="\2">\1</a>', escaped)
|
|
# Images 
|
|
img_re = re.compile(r'!\[([^]]*)\]\(([^)]+)\)')
|
|
escaped = img_re.sub(r'<img src="\2" alt="\1">', escaped)
|
|
html_lines.append(f'<p>{escaped}</p>')
|
|
|
|
if in_list:
|
|
html_lines.append('</ul>')
|
|
if in_code_block:
|
|
html_lines.append('</code></pre>')
|
|
|
|
return '\n'.join(html_lines)
|
|
|
|
|
|
class ArchiveHandler(BaseHTTPRequestHandler):
|
|
"""HTTP request handler for the archive."""
|
|
|
|
def log_message(self, format, *args):
|
|
print(f"[{self.log_date_time_string()}] {args[0]}")
|
|
|
|
def send_html(self, content: str, status: int = 200):
|
|
self.send_response(status)
|
|
self.send_header('Content-Type', 'text/html; charset=utf-8')
|
|
self.send_header('Content-Length', len(content.encode()))
|
|
self.end_headers()
|
|
self.wfile.write(content.encode())
|
|
|
|
def send_json(self, data, status: int = 200):
|
|
content = json.dumps(data)
|
|
self.send_response(status)
|
|
self.send_header('Content-Type', 'application/json')
|
|
self.send_header('Content-Length', len(content.encode()))
|
|
self.end_headers()
|
|
self.wfile.write(content.encode())
|
|
|
|
def send_file(self, content: bytes, mime: str):
|
|
self.send_response(200)
|
|
self.send_header('Content-Type', mime)
|
|
self.send_header('Content-Length', len(content))
|
|
self.send_header('Cache-Control', 'public, max-age=86400')
|
|
self.end_headers()
|
|
self.wfile.write(content)
|
|
|
|
def send_404(self):
|
|
self.send_html('<h1>404 Not Found</h1>', 404)
|
|
|
|
def do_GET(self):
|
|
parsed = urlparse(self.path)
|
|
path = unquote(parsed.path)
|
|
query = parse_qs(parsed.query)
|
|
|
|
# API endpoints
|
|
if path == '/api/search':
|
|
q = query.get('q', [''])[0]
|
|
results = search_pages(q) if q else []
|
|
self.send_json(results)
|
|
return
|
|
|
|
if path == '/api/stats':
|
|
self.send_json(get_stats())
|
|
return
|
|
|
|
# Pages
|
|
if path == '/':
|
|
stats = get_stats()
|
|
content = INDEX_HTML.format(**stats)
|
|
self.send_html(content)
|
|
return
|
|
|
|
if path == '/browse':
|
|
pages = list_files('html', '*.html')
|
|
items = ''.join([
|
|
f'<div class="list-item"><a href="/view/{html.escape(p)}">{html.escape(p)}</a></div>'
|
|
for p in pages[:500]
|
|
])
|
|
if not items:
|
|
items = '<div class="list-item">No pages found</div>'
|
|
content = BROWSE_HTML.format(domain=METADATA.get('domain', ''), items=items)
|
|
self.send_html(content)
|
|
return
|
|
|
|
# Page view - render markdown with neopig styling
|
|
if path.startswith('/view/'):
|
|
page_path = path[6:] # Remove '/view/'
|
|
# Try markdown first, fall back to HTML
|
|
md_path = 'markdown/' + page_path.replace('.html', '.md')
|
|
html_path = 'html/' + page_path
|
|
|
|
md_content, _ = read_file(md_path)
|
|
html_content_raw, _ = read_file(html_path)
|
|
|
|
if md_content:
|
|
# Render markdown
|
|
rendered = simple_markdown_to_html(md_content.decode('utf-8', errors='replace'))
|
|
elif html_content_raw:
|
|
# Extract body from HTML and show as-is
|
|
html_str = html_content_raw.decode('utf-8', errors='replace')
|
|
# Simple body extraction
|
|
import re as re_mod
|
|
body_match = re_mod.search(r'<body[^>]*>(.*?)</body>', html_str, re_mod.DOTALL | re_mod.IGNORECASE)
|
|
rendered = body_match.group(1) if body_match else html_str
|
|
else:
|
|
self.send_404()
|
|
return
|
|
|
|
# Extract title
|
|
title = page_path.replace('.html', '').replace('/', ' > ')
|
|
|
|
# Check for screenshot
|
|
ss_path = page_path.replace('.html', '.png').replace('/', '_')
|
|
screenshot_exists = f'screenshots/{ss_path}' in [f'screenshots/{f}' for f in list_files('screenshots', '*.png')]
|
|
|
|
screenshot_link = f' | <a href="/screenshots/{html.escape(ss_path)}">[View Screenshot]</a>' if screenshot_exists else ''
|
|
screenshot_embed = (
|
|
'<div class="screenshot">'
|
|
'<div class="screenshot-label">Page Screenshot:</div>'
|
|
f'<img src="/screenshots/{html.escape(ss_path)}" alt="Screenshot">'
|
|
'</div>'
|
|
) if screenshot_exists else ''
|
|
|
|
content = PAGE_VIEW_HTML.format(
|
|
title=html.escape(title),
|
|
path=html.escape(page_path),
|
|
content=rendered,
|
|
screenshot_link=screenshot_link,
|
|
screenshot_embed=screenshot_embed,
|
|
)
|
|
self.send_html(content)
|
|
return
|
|
|
|
if path == '/media':
|
|
files = list_files('media')
|
|
items = []
|
|
for f in files[:200]:
|
|
ext = Path(f).suffix.lower()
|
|
if ext in ('.mp4', '.webm', '.mov'):
|
|
media_el = f'<video src="/media/{html.escape(f)}" preload="metadata"></video>'
|
|
else:
|
|
media_el = f'<img src="/media/{html.escape(f)}" loading="lazy">'
|
|
# Link to detail view instead of raw file
|
|
items.append(
|
|
'<div class="card">'
|
|
f'<a href="/media/detail/{html.escape(f)}">{media_el}</a>'
|
|
f'<div class="card-info"><div class="card-title">{html.escape(f)}</div></div>'
|
|
'</div>'
|
|
)
|
|
content = MEDIA_HTML.format(
|
|
domain=METADATA.get('domain', ''),
|
|
title='Media',
|
|
items=''.join(items) if items else '<div class="empty">No media found</div>'
|
|
)
|
|
self.send_html(content)
|
|
return
|
|
|
|
# Media detail view - image at top with source page content below
|
|
if path.startswith('/media/detail/'):
|
|
media_path = path[14:] # Remove '/media/detail/'
|
|
media_file = 'media/' + media_path
|
|
|
|
# Check media exists
|
|
media_content, mime = read_file(media_file)
|
|
if not media_content:
|
|
self.send_404()
|
|
return
|
|
|
|
# Determine media element type
|
|
ext = Path(media_path).suffix.lower()
|
|
if ext in ('.mp4', '.webm', '.mov'):
|
|
media_el = f'<video src="/media/{html.escape(media_path)}" controls autoplay muted></video>'
|
|
else:
|
|
media_el = f'<img src="/media/{html.escape(media_path)}">'
|
|
|
|
# Try to find the source page - media path mirrors URL structure
|
|
# e.g., media/images/foo.jpg might come from t/topic-name/123.html
|
|
# For now, use filename as caption
|
|
caption = Path(media_path).stem.replace('-', ' ').replace('_', ' ')
|
|
media_uri = f'/media/{media_path}'
|
|
page_uri = METADATA.get('target_url', METADATA.get('domain', 'unknown'))
|
|
page_path = 'index.html'
|
|
|
|
# Try to find associated page content from the database
|
|
page_content = '<p class="empty">Source page content not available in archive database.</p>'
|
|
if DB_PATH and DB_PATH.exists():
|
|
conn = sqlite3.connect(DB_PATH)
|
|
c = conn.cursor()
|
|
try:
|
|
# Search for pages that might contain this media
|
|
media_name = Path(media_path).name
|
|
c.execute("""
|
|
SELECT path, title, content FROM pages
|
|
WHERE content LIKE ? OR path LIKE ?
|
|
LIMIT 1
|
|
""", (f'%{media_name}%', f'%{media_name}%'))
|
|
row = c.fetchone()
|
|
if row:
|
|
page_path = row[0].replace('html/', '')
|
|
caption = row[1] or caption
|
|
# Render the content
|
|
md_file = 'markdown/' + page_path.replace('.html', '.md')
|
|
md_content, _ = read_file(md_file)
|
|
if md_content:
|
|
page_content = simple_markdown_to_html(md_content.decode('utf-8', errors='replace'))
|
|
else:
|
|
page_content = f'<p>{html.escape(row[2][:2000] if row[2] else "")}...</p>'
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
conn.close()
|
|
|
|
content = MEDIA_DETAIL_HTML.format(
|
|
media_element=media_el,
|
|
caption=html.escape(caption),
|
|
media_uri=html.escape(media_uri),
|
|
page_uri=html.escape(page_uri),
|
|
page_path=html.escape(page_path),
|
|
page_content=page_content,
|
|
)
|
|
self.send_html(content)
|
|
return
|
|
|
|
if path == '/screenshots':
|
|
files = list_files('screenshots', '*.png')
|
|
items = []
|
|
for f in files[:200]:
|
|
items.append(
|
|
'<div class="card">'
|
|
f'<a href="/screenshots/{html.escape(f)}" target="_blank">'
|
|
f'<img src="/screenshots/{html.escape(f)}" loading="lazy">'
|
|
'</a>'
|
|
f'<div class="card-info"><div class="card-title">{html.escape(f)}</div></div>'
|
|
'</div>'
|
|
)
|
|
content = MEDIA_HTML.format(
|
|
domain=METADATA.get('domain', ''),
|
|
title='Screenshots',
|
|
items=''.join(items) if items else '<div class="empty">No screenshots found</div>'
|
|
)
|
|
self.send_html(content)
|
|
return
|
|
|
|
# Serve static files
|
|
file_path = path.lstrip('/')
|
|
content, mime = read_file(file_path)
|
|
if content is not None:
|
|
self.send_file(content, mime)
|
|
else:
|
|
self.send_404()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='neopig Archive Server')
|
|
parser.add_argument('archive', nargs='?', help='Path to archive.tar.gz (optional)')
|
|
parser.add_argument('-p', '--port', type=int, default=8000, help='Port to listen on')
|
|
parser.add_argument('--host', default='0.0.0.0', help='Host to bind to')
|
|
args = parser.parse_args()
|
|
|
|
init_archive()
|
|
|
|
server = HTTPServer((args.host, args.port), ArchiveHandler)
|
|
print(f"Starting neopig archive server at http://{args.host}:{args.port}")
|
|
print(f"Archive: {METADATA.get('domain', 'unknown')} ({get_stats()['pages']} pages)")
|
|
print("Press Ctrl+C to stop")
|
|
|
|
try:
|
|
server.serve_forever()
|
|
except KeyboardInterrupt:
|
|
print("\\nShutting down...")
|
|
server.shutdown()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|
|
'''
|
|
(archive_root / 'serve.py').write_text(serve_py)
|
|
|
|
|
|
def upgrade_neopig_in_archive(archive_path: Path) -> Path:
|
|
"""Replace neopig/ directory in existing archive with current source."""
|
|
import shutil
|
|
|
|
if not archive_path.exists():
|
|
raise FileNotFoundError(f"Archive not found: {archive_path}")
|
|
|
|
neopig_src = Path(__file__).parent
|
|
neopig_files = [
|
|
'neopig.py', 'database.py', 'async_web_fetcher.py',
|
|
'storage.py', 'domain_vault.py', 'screenshot.py',
|
|
'html2md.py', 'serp.py', 'filevault.py', 'async_filevault.py',
|
|
]
|
|
|
|
# Create new archive with updated neopig
|
|
output_path = archive_path.with_suffix('.upgraded.tar.gz')
|
|
|
|
with tarfile.open(archive_path, 'r:gz') as old_tar:
|
|
with tarfile.open(output_path, 'w:gz') as new_tar:
|
|
# Get archive name from first member
|
|
members = old_tar.getmembers()
|
|
archive_name = members[0].name.split('/')[0]
|
|
|
|
# Copy all members except neopig/ and requirements.txt
|
|
for member in members:
|
|
parts = member.name.split('/')
|
|
if len(parts) > 1 and parts[1] == 'neopig':
|
|
continue # Skip old neopig files
|
|
if len(parts) > 1 and parts[1] == 'requirements.txt':
|
|
continue # Skip old requirements
|
|
if member.isfile():
|
|
f = old_tar.extractfile(member)
|
|
if f:
|
|
new_tar.addfile(member, f)
|
|
else:
|
|
new_tar.addfile(member)
|
|
|
|
# Add new neopig files
|
|
for pyfile in neopig_files:
|
|
src_path = neopig_src / pyfile
|
|
if src_path.exists():
|
|
new_tar.add(src_path, arcname=f"{archive_name}/neopig/{pyfile}")
|
|
|
|
# Add requirements.txt
|
|
req_path = neopig_src / 'requirements.txt'
|
|
if req_path.exists():
|
|
new_tar.add(req_path, arcname=f"{archive_name}/requirements.txt")
|
|
|
|
# Replace original with upgraded
|
|
shutil.move(output_path, archive_path)
|
|
return archive_path
|
|
|
|
|
|
async def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Archive a website for preservation (uses neopig for crawling)",
|
|
epilog="Example: python archive.py https://discourse-urho3d.github.io/"
|
|
)
|
|
|
|
parser.add_argument("url", nargs='?', help="URL of the site to archive")
|
|
parser.add_argument("-o", "--output", default=".", help="Output directory for tar.gz")
|
|
parser.add_argument("-d", "--depth", type=int, default=-1, help="Crawl depth (-1 = unlimited)")
|
|
parser.add_argument("-p", "--max-pages", type=int, default=-1, help="Max pages (-1 = unlimited)")
|
|
parser.add_argument("--no-screenshots", action="store_true", help="Disable screenshots")
|
|
parser.add_argument("--no-markdown", action="store_true", help="Disable markdown conversion")
|
|
parser.add_argument("--screenshot-width", type=int, default=1280, help="Screenshot width")
|
|
parser.add_argument("--screenshot-height", type=int, default=1024, help="Screenshot height")
|
|
parser.add_argument("--screenshot-engine", type=str, default=None, help="Screenshot engine")
|
|
parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
|
|
parser.add_argument("--fresh", action="store_true", help="Start fresh, ignore resume state")
|
|
parser.add_argument("--fast", action="store_true", help="Fast mode: no crawl delay (for sites without robots.txt)")
|
|
parser.add_argument("--serve", action="store_true", help="Start SERP server to watch crawl live")
|
|
parser.add_argument("--port", type=int, default=8000, help="Port for SERP server (default: 8000)")
|
|
parser.add_argument("--backfill-markdown", metavar="DOMAIN", help="Re-process stored HTML for DOMAIN to regenerate markdown with absolute URLs")
|
|
parser.add_argument("--trim-wrapper", action="store_true", help="With --backfill-markdown: strip nav/header/footer/logo before conversion")
|
|
parser.add_argument("--db", default="data/neopig.db", help="Database path (for --backfill-markdown)")
|
|
parser.add_argument("--upgrade-neopig", metavar="TARBALL", help="Upgrade neopig inside an existing archive")
|
|
|
|
args = parser.parse_args()
|
|
|
|
setup_logging(level=logging.DEBUG if args.verbose else logging.INFO)
|
|
|
|
# Handle --upgrade-neopig
|
|
if args.upgrade_neopig:
|
|
archive_path = Path(args.upgrade_neopig)
|
|
logger.info(f"Upgrading neopig in {archive_path}...")
|
|
upgrade_neopig_in_archive(archive_path)
|
|
logger.info(f"Done! Archive updated: {archive_path}")
|
|
return
|
|
|
|
# Handle --backfill-markdown
|
|
if args.backfill_markdown:
|
|
from neopig import backfill_markdown
|
|
await backfill_markdown(args.db, domain_filter=args.backfill_markdown, trim_wrapper=args.trim_wrapper)
|
|
return
|
|
|
|
# URL required for archiving
|
|
if not args.url:
|
|
parser.error("URL required (use --upgrade-neopig or --backfill-markdown for other operations)")
|
|
|
|
if args.no_markdown and not HAS_HTML2TEXT:
|
|
pass
|
|
elif not HAS_HTML2TEXT:
|
|
logger.warning("html2text not installed - markdown disabled")
|
|
|
|
screenshot_config = ScreenshotConfig(
|
|
enabled=not args.no_screenshots,
|
|
width=args.screenshot_width,
|
|
height=args.screenshot_height,
|
|
engine=args.screenshot_engine,
|
|
full_page=True, # Archive captures full page by default
|
|
)
|
|
|
|
archiver = SiteArchiver(
|
|
output_dir=args.output,
|
|
include_screenshots=not args.no_screenshots,
|
|
include_markdown=not args.no_markdown,
|
|
screenshot_config=screenshot_config,
|
|
fast_mode=args.fast,
|
|
trim_wrapper=args.trim_wrapper,
|
|
)
|
|
|
|
# Start SERP server if requested
|
|
serp_process = None
|
|
if args.serve:
|
|
serp_script = Path(__file__).parent / 'serp.py'
|
|
if serp_script.exists():
|
|
serp_cmd = [
|
|
sys.executable, str(serp_script),
|
|
'--port', str(args.port),
|
|
'--db', 'data/neopig.db',
|
|
'--vault', 'data/vault',
|
|
]
|
|
serp_process = subprocess.Popen(
|
|
serp_cmd,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
logger.info(f"SERP server started at http://localhost:{args.port}")
|
|
logger.info("Watch the crawl live - pages appear as they're indexed!")
|
|
else:
|
|
logger.warning("serp.py not found - --serve disabled")
|
|
|
|
try:
|
|
archive_path = await archiver.archive(
|
|
target_url=args.url,
|
|
depth=args.depth,
|
|
max_pages=args.max_pages,
|
|
)
|
|
|
|
print(f"\nArchive created: {archive_path}")
|
|
print(f"Extract with: tar -xzf {archive_path.name}")
|
|
finally:
|
|
# Cleanup SERP server
|
|
if serp_process:
|
|
logger.info("Stopping SERP server...")
|
|
serp_process.terminate()
|
|
try:
|
|
serp_process.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
serp_process.kill()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|