pig.py/archive.py

#!/usr/bin/env python3
"""
archive.py - Site Archiver for Sunset Sites

Thin wrapper around neopig that packages crawl results into a distributable
tar.gz archive. Uses neopig for the actual crawling.

Output structure:
    {domain}-{date}/
        html/                   # Original HTML pages
        markdown/               # Converted markdown (optional)
        media/                  # Images, videos, audio
        screenshots/            # Page screenshots (optional)
        archive.db              # SQLite search database
        serve.py                # Embedded search server
        metadata.json           # Crawl metadata

Usage:
    python archive.py https://discourse-urho3d.github.io/
    python archive.py https://example.com --no-screenshots --no-markdown
"""

import argparse
import asyncio
import json
import logging
import os
import re
import shutil
import signal
import sqlite3
import subprocess
import sys
import tarfile
import tempfile
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, Any, List, Optional
from urllib.parse import urlparse

import aiofiles
import aiofiles.os
from bs4 import BeautifulSoup
from tqdm import tqdm

from neopig import NeoPig, setup_logging
from async_web_fetcher import CrawlMode
from screenshot import ScreenshotConfig

# Optional markdown conversion
try:
    import html2text
    HAS_HTML2TEXT = True
except ImportError:
    HAS_HTML2TEXT = False

logger = logging.getLogger(__name__)


def sanitize_filename(name: str) -> str:
    """Sanitize a string for use as filename. Uses - as separator."""
    name = re.sub(r'[<>:"/\\|?*.]', '-', name)
    name = re.sub(r'-+', '-', name)  # collapse multiple dashes
    name = name.strip('- ')
    return name[:200] if name else 'unnamed'


def url_to_path(url: str) -> str:
    """Convert URL to filesystem path."""
    parsed = urlparse(url)
    path = parsed.path.strip('/')
    if not path:
        return 'index.html'
    if path.endswith('.html') or path.endswith('.htm'):
        return path
    if '.' in path.split('/')[-1]:
        return path
    return f"{path}/index.html"


def html_to_markdown(html: str, base_url: str = '') -> str:
    """Convert HTML to markdown."""
    if not HAS_HTML2TEXT:
        return html
    h = html2text.HTML2Text()
    h.ignore_links = False
    h.ignore_images = False
    h.body_width = 0
    h.unicode_snob = True
    if base_url:
        h.baseurl = base_url
    return h.handle(html)


class SiteArchiver:
    """
    Packages neopig crawl results into a distributable tar.gz archive.

    Uses neopig for crawling, then reads from its vaults to build the archive.
    """

    def __init__(
        self,
        output_dir: str = '.',
        include_screenshots: bool = True,
        include_markdown: bool = True,
        screenshot_config: ScreenshotConfig = None,
        fast_mode: bool = False,
    ):
        self.output_dir = Path(output_dir)
        self.include_screenshots = include_screenshots
        self.include_markdown = include_markdown and HAS_HTML2TEXT
        self.screenshot_config = screenshot_config or ScreenshotConfig(enabled=include_screenshots)
        self.fast_mode = fast_mode

    async def archive(
        self,
        target_url: str,
        depth: int = -1,
        max_pages: int = -1,
        db_path: str = None,
        vault_path: str = None,
    ) -> Path:
        """
        Archive a site using neopig and package into tar.gz.
        """
        parsed = urlparse(target_url)
        domain = parsed.netloc.lower()
        date_str = datetime.now().strftime('%Y%m%d')
        archive_name = f"{sanitize_filename(domain)}-{date_str}"

        # Use standard neopig data paths - one vault, multiple domains
        if not db_path:
            db_path = "data/neopig.db"
        if not vault_path:
            vault_path = "data/vault"

        # Ensure data directory exists
        Path("data").mkdir(exist_ok=True)

        logger.info(f"Starting archive of {target_url}")
        logger.info(f"Archive name: {archive_name}")

        # Create neopig instance and crawl
        pig = NeoPig(
            db_path=db_path,
            vault_path=vault_path,
            screenshot_config=self.screenshot_config,
            fast_mode=self.fast_mode,
        )
        await pig.init()

        # Run the crawl
        stats = await pig.crawl(
            target_uri=target_url,
            mode=CrawlMode.ALL,
            depth=depth,
            max_pages=max_pages,
            download_media=True,
        )

        # Now package the results
        logger.info("Packaging archive...")

        with tempfile.TemporaryDirectory() as tmpdir:
            archive_root = Path(tmpdir) / archive_name
            archive_root.mkdir(parents=True)

            # Create subdirectories
            html_dir = archive_root / 'html'
            media_dir = archive_root / 'media'
            html_dir.mkdir()
            media_dir.mkdir()

            if self.include_markdown:
                md_dir = archive_root / 'markdown'
                md_dir.mkdir()

            if self.include_screenshots:
                screenshots_dir = archive_root / 'screenshots'
                screenshots_dir.mkdir()

            # Read pages from neopig's html vault
            html_vault_path = Path(vault_path) / 'html_vault' / domain
            sitemap = []

            if html_vault_path.exists():
                for html_file in html_vault_path.rglob('*.html'):
                    rel_path = html_file.relative_to(html_vault_path)
                    html_content = html_file.read_text(encoding='utf-8', errors='replace')

                    # Write HTML
                    dest_path = html_dir / rel_path
                    await aiofiles.os.makedirs(dest_path.parent, exist_ok=True)
                    async with aiofiles.open(dest_path, 'w', encoding='utf-8') as f:
                        await f.write(html_content)

                    # Write markdown
                    if self.include_markdown:
                        md_path = md_dir / str(rel_path).replace('.html', '.md')
                        await aiofiles.os.makedirs(md_path.parent, exist_ok=True)
                        md_content = html_to_markdown(html_content)
                        async with aiofiles.open(md_path, 'w', encoding='utf-8') as f:
                            await f.write(md_content)

                    # Extract title for sitemap
                    title = self._extract_title(html_content) or str(rel_path)
                    sitemap.append({
                        'path': f'html/{rel_path}',
                        'title': title,
                    })

            # Copy media from neopig's media vault (follows symlinks to hash vault)
            media_vault_path = Path(vault_path) / 'media_vault' / domain
            if media_vault_path.exists():
                for media_file in media_vault_path.rglob('*'):
                    if media_file.is_file() or media_file.is_symlink():
                        try:
                            # Follow symlinks to get actual content
                            if media_file.is_symlink():
                                target = media_file.resolve()
                                if not target.exists():
                                    logger.debug(f"Skipping broken symlink: {media_file}")
                                    continue
                                content = target.read_bytes()
                            else:
                                content = media_file.read_bytes()
                            rel_path = media_file.relative_to(media_vault_path)
                            dest_path = media_dir / rel_path
                            await aiofiles.os.makedirs(dest_path.parent, exist_ok=True)
                            async with aiofiles.open(dest_path, 'wb') as f:
                                await f.write(content)
                        except Exception as e:
                            logger.debug(f"Error copying media {media_file}: {e}")

            # Copy screenshots from neopig's linkpeek vault (follows symlinks to hash vault)
            if self.include_screenshots:
                linkpeek_vault_path = Path(vault_path) / 'linkpeek_vault' / domain
                if linkpeek_vault_path.exists():
                    for screenshot_file in linkpeek_vault_path.rglob('*.png'):
                        try:
                            # Follow symlinks to get actual content
                            if screenshot_file.is_symlink():
                                target = screenshot_file.resolve()
                                if not target.exists():
                                    logger.debug(f"Skipping broken symlink: {screenshot_file}")
                                    continue
                                content = target.read_bytes()
                            else:
                                content = screenshot_file.read_bytes()
                            rel_path = screenshot_file.relative_to(linkpeek_vault_path)
                            dest_path = screenshots_dir / rel_path
                            await aiofiles.os.makedirs(dest_path.parent, exist_ok=True)
                            async with aiofiles.open(dest_path, 'wb') as f:
                                await f.write(content)
                        except Exception as e:
                            logger.debug(f"Error copying screenshot {screenshot_file}: {e}")

            # Create search database
            await self._create_search_database(archive_root, sitemap, domain, html_dir)

            # Write embedded serve.py
            self._write_serve_py(archive_root)

            # Write metadata
            metadata = {
                'domain': domain,
                'target_url': target_url,
                'created': datetime.now(timezone.utc).isoformat(),
                'stats': stats,
                'include_screenshots': self.include_screenshots,
                'include_markdown': self.include_markdown,
            }
            async with aiofiles.open(archive_root / 'metadata.json', 'w') as f:
                await f.write(json.dumps(metadata, indent=2))

            # Copy state file into archive for future delta crawls
            state_domain = domain.replace('.', '-').replace(':', '-')
            state_file = Path("data") / f"crawl-state-{state_domain}.json"
            if state_file.exists():
                shutil.copy(state_file, archive_root / 'crawl_state.json')
                logger.info(f"Included crawl state for future delta crawls")

            # Create tar.gz
            tar_path = self.output_dir / f"{archive_name}.tar.gz"

            def create_tarball():
                with tarfile.open(tar_path, 'w:gz') as tar:
                    tar.add(archive_root, arcname=archive_name)

            await asyncio.to_thread(create_tarball)

            final_size = tar_path.stat().st_size
            logger.info(f"Archive complete: {tar_path} ({final_size / 1024 / 1024:.1f} MB)")

            return tar_path

    def _extract_title(self, html: str) -> Optional[str]:
        """Extract title from HTML."""
        try:
            soup = BeautifulSoup(html, 'html.parser')
            if soup.title and soup.title.string:
                return soup.title.string.strip()
            h1 = soup.find('h1')
            if h1:
                return h1.get_text().strip()
        except Exception:
            pass
        return None

    def _extract_text(self, html: str) -> str:
        """Extract readable text from HTML for search indexing."""
        try:
            soup = BeautifulSoup(html, 'html.parser')
            for tag in soup(['script', 'style', 'nav', 'footer', 'header']):
                tag.decompose()
            text = soup.get_text(separator=' ', strip=True)
            text = re.sub(r'\s+', ' ', text)
            return text[:50000]
        except Exception:
            return ''

    async def _create_search_database(
        self,
        archive_root: Path,
        sitemap: List[Dict[str, str]],
        domain: str,
        html_dir: Path,
    ):
        """Create SQLite database with searchable page content."""
        db_path = archive_root / 'archive.db'

        def create_db():
            conn = sqlite3.connect(db_path)
            c = conn.cursor()

            c.execute('''
                CREATE TABLE pages (
                    id INTEGER PRIMARY KEY,
                    url TEXT NOT NULL,
                    path TEXT NOT NULL,
                    title TEXT,
                    content TEXT
                )
            ''')

            c.execute('''
                CREATE VIRTUAL TABLE pages_fts USING fts5(
                    title, content, url, path,
                    content='pages',
                    content_rowid='id'
                )
            ''')

            c.execute('''
                CREATE TRIGGER pages_ai AFTER INSERT ON pages BEGIN
                    INSERT INTO pages_fts(rowid, title, content, url, path)
                    VALUES (new.id, new.title, new.content, new.url, new.path);
                END
            ''')

            for item in sitemap:
                path = item['path']
                title = item['title']

                html_path = archive_root / path
                if html_path.exists():
                    html_content = html_path.read_text(encoding='utf-8', errors='replace')
                    text_content = self._extract_text(html_content)
                else:
                    text_content = ''

                c.execute(
                    'INSERT INTO pages (url, path, title, content) VALUES (?, ?, ?, ?)',
                    (path, path, title, text_content)
                )

            conn.commit()
            conn.close()

        await asyncio.to_thread(create_db)
        logger.info(f"Created search database: {db_path}")

    def _write_serve_py(self, archive_root: Path):
        """Write embedded search server (stdlib only, no dependencies)."""
        serve_py = '''#!/usr/bin/env python3
"""
neopig Archive Server - Browse and search archived sites.

Zero dependencies - uses only Python stdlib.

Usage:
    python serve.py                    # Serve from extracted archive
    python serve.py archive.tar.gz     # Serve directly from tarball
    python serve.py -p 8080            # Custom port
    ./archive.run                      # Self-extracting archive
"""

import argparse
import html
import json
import mimetypes
import os
import re
import sqlite3
import sys
import tarfile
import tempfile
from http.server import HTTPServer, BaseHTTPRequestHandler
from pathlib import Path
from urllib.parse import parse_qs, urlparse, unquote

# Globals set at startup
ARCHIVE_ROOT = None
TAR_FILE = None
TAR_MEMBERS = {}
DB_PATH = None
METADATA = {}


def get_archive_source():
    """Determine if we are in a tarball, .run, or extracted directory."""
    exe_path = Path(sys.argv[0]).resolve()

    # Check for .run format (has NEOPIG trailer)
    if exe_path.suffix == '.run' or (len(sys.argv) == 1 and exe_path.stat().st_size > 1000000):
        try:
            with open(exe_path, 'rb') as f:
                f.seek(-22, 2)
                trailer = f.read(22)
                if trailer[:6] == b'NEOPIG':
                    offset = int(trailer[6:22].decode(), 16)
                    return ('run', exe_path, offset)
        except Exception:
            pass

    # Check command line for tarball argument
    for arg in sys.argv[1:]:
        if not arg.startswith('-'):
            p = Path(arg)
            if p.exists() and p.suffix in ('.gz', '.tar', '.tgz'):
                return ('tarball', p, 0)

    # Must be extracted directory
    return ('directory', Path(__file__).parent, 0)


def init_archive():
    """Initialize archive access."""
    global ARCHIVE_ROOT, TAR_FILE, TAR_MEMBERS, DB_PATH, METADATA

    source_type, source_path, offset = get_archive_source()

    if source_type == 'run':
        print(f"Serving from self-extracting archive: {source_path}")
        f = open(source_path, 'rb')
        f.seek(offset)
        TAR_FILE = tarfile.open(fileobj=f, mode='r:gz')
    elif source_type == 'tarball':
        print(f"Serving from tarball: {source_path}")
        TAR_FILE = tarfile.open(source_path, 'r:gz')
    else:
        print(f"Serving from directory: {source_path}")
        ARCHIVE_ROOT = source_path

    if TAR_FILE:
        # Build member lookup and find archive root
        for member in TAR_FILE.getmembers():
            TAR_MEMBERS[member.name] = member
        first = list(TAR_MEMBERS.keys())[0]
        archive_name = first.split('/')[0]
        ARCHIVE_ROOT = Path(archive_name)

        # Extract database to temp for searching
        db_member = f"{archive_name}/archive.db"
        if db_member in TAR_MEMBERS:
            temp_dir = tempfile.mkdtemp()
            TAR_FILE.extract(TAR_MEMBERS[db_member], temp_dir)
            DB_PATH = Path(temp_dir) / db_member
    else:
        DB_PATH = ARCHIVE_ROOT / 'archive.db'

    # Load metadata
    meta_path = ARCHIVE_ROOT / 'metadata.json' if not TAR_FILE else None
    if meta_path and meta_path.exists():
        METADATA = json.loads(meta_path.read_text())
    elif TAR_FILE:
        meta_member = f"{ARCHIVE_ROOT}/metadata.json"
        if meta_member in TAR_MEMBERS:
            f = TAR_FILE.extractfile(TAR_MEMBERS[meta_member])
            if f:
                METADATA = json.loads(f.read().decode())


def read_file(path: str) -> tuple:
    """Read file from archive. Returns (content_bytes, mime_type) or (None, None)."""
    if TAR_FILE:
        # Normalize path for tarball
        tar_path = f"{ARCHIVE_ROOT}/{path}".lstrip('/')
        if tar_path in TAR_MEMBERS:
            f = TAR_FILE.extractfile(TAR_MEMBERS[tar_path])
            if f:
                mime, _ = mimetypes.guess_type(path)
                return f.read(), mime or 'application/octet-stream'
        return None, None
    else:
        file_path = ARCHIVE_ROOT / path
        if file_path.exists() and file_path.is_file():
            # Security: prevent path traversal
            try:
                file_path.resolve().relative_to(ARCHIVE_ROOT.resolve())
            except ValueError:
                return None, None
            mime, _ = mimetypes.guess_type(str(file_path))
            return file_path.read_bytes(), mime or 'application/octet-stream'
        return None, None


def list_files(subdir: str, pattern: str = '*') -> list:
    """List files in a subdirectory."""
    files = []
    if TAR_FILE:
        prefix = f"{ARCHIVE_ROOT}/{subdir}/"
        for name in TAR_MEMBERS:
            if name.startswith(prefix) and not name.endswith('/'):
                rel = name[len(prefix):]
                if pattern == '*' or rel.endswith(pattern.replace('*', '')):
                    files.append(rel)
    else:
        dir_path = ARCHIVE_ROOT / subdir
        if dir_path.exists():
            for f in dir_path.rglob(pattern):
                if f.is_file():
                    files.append(str(f.relative_to(dir_path)))
    return sorted(files)


def search_pages(query: str, limit: int = 50) -> list:
    """Search pages using FTS5."""
    if not DB_PATH or not DB_PATH.exists():
        return []

    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    try:
        c.execute("""
            SELECT p.path, p.title, snippet(pages_fts, 1, '<mark>', '</mark>', '...', 40)
            FROM pages_fts
            JOIN pages p ON pages_fts.rowid = p.id
            WHERE pages_fts MATCH ?
            ORDER BY rank
            LIMIT ?
        """, (query, limit))
        results = [{'path': r[0], 'title': r[1], 'snippet': r[2]} for r in c.fetchall()]
    except sqlite3.OperationalError:
        results = []
    finally:
        conn.close()
    return results


def get_stats() -> dict:
    """Get archive statistics."""
    stats = {
        'pages': len(list_files('html', '*.html')),
        'media': len(list_files('media')),
        'screenshots': len(list_files('screenshots', '*.png')),
        'domain': METADATA.get('domain', 'unknown'),
        'created': METADATA.get('created', 'unknown'),
    }
    if METADATA.get('stats'):
        stats.update(METADATA['stats'])
    return stats


# HTML Templates
INDEX_HTML = """<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>{domain} - neopig Archive</title>
    <style>
        * {{ box-sizing: border-box; }}
        body {{
            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
            margin: 0; padding: 20px;
            background: #0a0a0a; color: #e0e0e0;
        }}
        .container {{ max-width: 1400px; margin: 0 auto; }}
        h1 {{ color: #ff6b6b; margin-bottom: 5px; }}
        .subtitle {{ color: #666; margin-bottom: 20px; }}
        nav {{ margin-bottom: 20px; }}
        nav a {{ color: #ff6b6b; margin-right: 15px; text-decoration: none; }}
        nav a:hover {{ text-decoration: underline; }}
        nav a.active {{ font-weight: bold; border-bottom: 2px solid #ff6b6b; }}
        .search-box {{
            display: flex; gap: 10px; margin-bottom: 20px;
        }}
        input[type="text"] {{
            flex: 1; padding: 12px 16px; font-size: 16px;
            border: 2px solid #333; border-radius: 8px;
            background: #1a1a1a; color: #fff;
        }}
        input[type="text"]:focus {{ outline: none; border-color: #ff6b6b; }}
        button {{
            padding: 12px 24px; font-size: 16px;
            background: #ff6b6b; color: #fff;
            border: none; border-radius: 8px; cursor: pointer;
        }}
        button:hover {{ background: #ff5252; }}
        .stats {{
            display: flex; gap: 20px; flex-wrap: wrap;
            padding: 15px; background: #1a1a1a;
            border-radius: 8px; margin-bottom: 20px;
        }}
        .stat {{ text-align: center; }}
        .stat-value {{ font-size: 24px; font-weight: bold; color: #ff6b6b; }}
        .stat-label {{ font-size: 12px; color: #888; }}
        .results {{ margin-top: 20px; }}
        .result {{
            background: #1a1a1a; border-radius: 8px;
            padding: 15px; margin-bottom: 10px;
        }}
        .result h3 {{ margin: 0 0 8px 0; }}
        .result a {{ color: #ff6b6b; text-decoration: none; }}
        .result a:hover {{ text-decoration: underline; }}
        .result .snippet {{ color: #888; font-size: 14px; }}
        .result .snippet mark {{ background: #ff6b6b; color: #fff; padding: 1px 3px; border-radius: 2px; }}
        .result .path {{ font-size: 12px; color: #666; margin-top: 5px; }}
        .grid {{
            display: grid;
            grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
            gap: 15px;
        }}
        .card {{
            background: #1a1a1a; border-radius: 8px;
            overflow: hidden; transition: transform 0.2s;
        }}
        .card:hover {{ transform: scale(1.02); }}
        .card img {{
            width: 100%; height: 150px;
            object-fit: cover; background: #222;
        }}
        .card-info {{ padding: 10px; }}
        .card-title {{
            font-size: 12px; color: #ccc;
            overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
        }}
        .empty {{ text-align: center; padding: 60px; color: #666; }}
    </style>
</head>
<body>
<div class="container">
    <h1>neopig Archive</h1>
    <p class="subtitle">{domain} - archived {created}</p>

    <nav>
        <a href="/" class="active">Search</a>
        <a href="/browse">Browse Pages</a>
        <a href="/media">Media</a>
        <a href="/screenshots">Screenshots</a>
    </nav>

    <div class="stats">
        <div class="stat"><div class="stat-value">{pages}</div><div class="stat-label">Pages</div></div>
        <div class="stat"><div class="stat-value">{media}</div><div class="stat-label">Media</div></div>
        <div class="stat"><div class="stat-value">{screenshots}</div><div class="stat-label">Screenshots</div></div>
    </div>

    <div class="search-box">
        <input type="text" id="query" placeholder="Search archived pages..." autofocus>
        <button onclick="search()">Search</button>
    </div>

    <div class="results" id="results"></div>
</div>

<script>
async function search() {{
    const q = document.getElementById('query').value;
    if (!q) return;
    const res = await fetch('/api/search?q=' + encodeURIComponent(q));
    const data = await res.json();
    const container = document.getElementById('results');
    if (data.length === 0) {{
        container.innerHTML = '<div class="empty">No results found</div>';
        return;
    }}
    container.innerHTML = data.map(r => `
        <div class="result">
            <h3><a href="/${{r.path}}">${{r.title}}</a></h3>
            <p class="snippet">${{r.snippet}}</p>
            <p class="path">${{r.path}}</p>
        </div>
    `).join('');
}}
document.getElementById('query').addEventListener('keypress', e => {{
    if (e.key === 'Enter') search();
}});
</script>
</body>
</html>"""

BROWSE_HTML = """<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Browse - {domain}</title>
    <style>
        * {{ box-sizing: border-box; }}
        body {{
            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
            margin: 0; padding: 20px;
            background: #0a0a0a; color: #e0e0e0;
        }}
        .container {{ max-width: 1400px; margin: 0 auto; }}
        h1 {{ color: #ff6b6b; }}
        nav {{ margin-bottom: 20px; }}
        nav a {{ color: #ff6b6b; margin-right: 15px; text-decoration: none; }}
        nav a:hover {{ text-decoration: underline; }}
        .list {{ background: #1a1a1a; border-radius: 8px; }}
        .list-item {{
            padding: 12px 15px; border-bottom: 1px solid #333;
            display: flex; align-items: center; gap: 10px;
        }}
        .list-item:last-child {{ border-bottom: none; }}
        .list-item a {{ color: #ff6b6b; text-decoration: none; flex: 1; }}
        .list-item a:hover {{ text-decoration: underline; }}
        .list-item .path {{ color: #666; font-size: 12px; }}
    </style>
</head>
<body>
<div class="container">
    <h1>Browse Pages</h1>
    <nav>
        <a href="/">Search</a>
        <a href="/browse">Browse Pages</a>
        <a href="/media">Media</a>
        <a href="/screenshots">Screenshots</a>
    </nav>
    <div class="list">
        {items}
    </div>
</div>
</body>
</html>"""

MEDIA_HTML = """<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Media - {domain}</title>
    <style>
        * {{ box-sizing: border-box; }}
        body {{
            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
            margin: 0; padding: 20px;
            background: #0a0a0a; color: #e0e0e0;
        }}
        .container {{ max-width: 1400px; margin: 0 auto; }}
        h1 {{ color: #ff6b6b; }}
        nav {{ margin-bottom: 20px; }}
        nav a {{ color: #ff6b6b; margin-right: 15px; text-decoration: none; }}
        nav a:hover {{ text-decoration: underline; }}
        .grid {{
            display: grid;
            grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
            gap: 15px;
        }}
        .card {{
            background: #1a1a1a; border-radius: 8px;
            overflow: hidden; transition: transform 0.2s;
        }}
        .card:hover {{ transform: scale(1.02); }}
        .card a {{ display: block; }}
        .card img, .card video {{
            width: 100%; height: 150px;
            object-fit: contain; background: #222;
        }}
        .card-info {{ padding: 10px; }}
        .card-title {{
            font-size: 11px; color: #888;
            overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
        }}
        .empty {{ text-align: center; padding: 60px; color: #666; }}
    </style>
</head>
<body>
<div class="container">
    <h1>{title}</h1>
    <nav>
        <a href="/">Search</a>
        <a href="/browse">Browse Pages</a>
        <a href="/media">Media</a>
        <a href="/screenshots">Screenshots</a>
    </nav>
    <div class="grid">
        {items}
    </div>
</div>
</body>
</html>"""


class ArchiveHandler(BaseHTTPRequestHandler):
    """HTTP request handler for the archive."""

    def log_message(self, format, *args):
        print(f"[{self.log_date_time_string()}] {args[0]}")

    def send_html(self, content: str, status: int = 200):
        self.send_response(status)
        self.send_header('Content-Type', 'text/html; charset=utf-8')
        self.send_header('Content-Length', len(content.encode()))
        self.end_headers()
        self.wfile.write(content.encode())

    def send_json(self, data, status: int = 200):
        content = json.dumps(data)
        self.send_response(status)
        self.send_header('Content-Type', 'application/json')
        self.send_header('Content-Length', len(content.encode()))
        self.end_headers()
        self.wfile.write(content.encode())

    def send_file(self, content: bytes, mime: str):
        self.send_response(200)
        self.send_header('Content-Type', mime)
        self.send_header('Content-Length', len(content))
        self.send_header('Cache-Control', 'public, max-age=86400')
        self.end_headers()
        self.wfile.write(content)

    def send_404(self):
        self.send_html('<h1>404 Not Found</h1>', 404)

    def do_GET(self):
        parsed = urlparse(self.path)
        path = unquote(parsed.path)
        query = parse_qs(parsed.query)

        # API endpoints
        if path == '/api/search':
            q = query.get('q', [''])[0]
            results = search_pages(q) if q else []
            self.send_json(results)
            return

        if path == '/api/stats':
            self.send_json(get_stats())
            return

        # Pages
        if path == '/':
            stats = get_stats()
            content = INDEX_HTML.format(**stats)
            self.send_html(content)
            return

        if path == '/browse':
            pages = list_files('html', '*.html')
            items = ''.join([
                f'<div class="list-item"><a href="/html/{html.escape(p)}">{html.escape(p)}</a></div>'
                for p in pages[:500]
            ])
            if not items:
                items = '<div class="list-item">No pages found</div>'
            content = BROWSE_HTML.format(domain=METADATA.get('domain', ''), items=items)
            self.send_html(content)
            return

        if path == '/media':
            files = list_files('media')
            items = []
            for f in files[:200]:
                ext = Path(f).suffix.lower()
                if ext in ('.mp4', '.webm', '.mov'):
                    media_el = f'<video src="/media/{html.escape(f)}" preload="metadata"></video>'
                else:
                    media_el = f'<img src="/media/{html.escape(f)}" loading="lazy">'
                items.append(
                    '<div class="card">'
                    f'<a href="/media/{html.escape(f)}" target="_blank">{media_el}</a>'
                    f'<div class="card-info"><div class="card-title">{html.escape(f)}</div></div>'
                    '</div>'
                )
            content = MEDIA_HTML.format(
                domain=METADATA.get('domain', ''),
                title='Media',
                items=''.join(items) if items else '<div class="empty">No media found</div>'
            )
            self.send_html(content)
            return

        if path == '/screenshots':
            files = list_files('screenshots', '*.png')
            items = []
            for f in files[:200]:
                items.append(
                    '<div class="card">'
                    f'<a href="/screenshots/{html.escape(f)}" target="_blank">'
                    f'<img src="/screenshots/{html.escape(f)}" loading="lazy">'
                    '</a>'
                    f'<div class="card-info"><div class="card-title">{html.escape(f)}</div></div>'
                    '</div>'
                )
            content = MEDIA_HTML.format(
                domain=METADATA.get('domain', ''),
                title='Screenshots',
                items=''.join(items) if items else '<div class="empty">No screenshots found</div>'
            )
            self.send_html(content)
            return

        # Serve static files
        file_path = path.lstrip('/')
        content, mime = read_file(file_path)
        if content is not None:
            self.send_file(content, mime)
        else:
            self.send_404()


def main():
    parser = argparse.ArgumentParser(description='neopig Archive Server')
    parser.add_argument('archive', nargs='?', help='Path to archive.tar.gz (optional)')
    parser.add_argument('-p', '--port', type=int, default=8000, help='Port to listen on')
    parser.add_argument('--host', default='0.0.0.0', help='Host to bind to')
    args = parser.parse_args()

    init_archive()

    server = HTTPServer((args.host, args.port), ArchiveHandler)
    print(f"Starting neopig archive server at http://{args.host}:{args.port}")
    print(f"Archive: {METADATA.get('domain', 'unknown')} ({get_stats()['pages']} pages)")
    print("Press Ctrl+C to stop")

    try:
        server.serve_forever()
    except KeyboardInterrupt:
        print("\\nShutting down...")
        server.shutdown()


if __name__ == '__main__':
    main()
'''
        (archive_root / 'serve.py').write_text(serve_py)


async def main():
    parser = argparse.ArgumentParser(
        description="Archive a website for preservation (uses neopig for crawling)",
        epilog="Example: python archive.py https://discourse-urho3d.github.io/"
    )

    parser.add_argument("url", help="URL of the site to archive")
    parser.add_argument("-o", "--output", default=".", help="Output directory for tar.gz")
    parser.add_argument("-d", "--depth", type=int, default=-1, help="Crawl depth (-1 = unlimited)")
    parser.add_argument("-p", "--max-pages", type=int, default=-1, help="Max pages (-1 = unlimited)")
    parser.add_argument("--no-screenshots", action="store_true", help="Disable screenshots")
    parser.add_argument("--no-markdown", action="store_true", help="Disable markdown conversion")
    parser.add_argument("--screenshot-width", type=int, default=1280, help="Screenshot width")
    parser.add_argument("--screenshot-height", type=int, default=1024, help="Screenshot height")
    parser.add_argument("--screenshot-engine", type=str, default=None, help="Screenshot engine")
    parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
    parser.add_argument("--fresh", action="store_true", help="Start fresh, ignore resume state")
    parser.add_argument("--fast", action="store_true", help="Fast mode: no crawl delay (for sites without robots.txt)")
    parser.add_argument("--serve", action="store_true", help="Start SERP server to watch crawl live")
    parser.add_argument("--port", type=int, default=8000, help="Port for SERP server (default: 8000)")

    args = parser.parse_args()

    setup_logging(level=logging.DEBUG if args.verbose else logging.INFO)

    if args.no_markdown and not HAS_HTML2TEXT:
        pass
    elif not HAS_HTML2TEXT:
        logger.warning("html2text not installed - markdown disabled")

    screenshot_config = ScreenshotConfig(
        enabled=not args.no_screenshots,
        width=args.screenshot_width,
        height=args.screenshot_height,
        engine=args.screenshot_engine,
    )

    archiver = SiteArchiver(
        output_dir=args.output,
        include_screenshots=not args.no_screenshots,
        include_markdown=not args.no_markdown,
        screenshot_config=screenshot_config,
        fast_mode=args.fast,
    )

    # Start SERP server if requested
    serp_process = None
    if args.serve:
        serp_script = Path(__file__).parent / 'serp.py'
        if serp_script.exists():
            serp_cmd = [
                sys.executable, str(serp_script),
                '--port', str(args.port),
                '--db', 'data/neopig.db',
                '--vault', 'data/vault',
            ]
            serp_process = subprocess.Popen(
                serp_cmd,
                stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL,
            )
            logger.info(f"SERP server started at http://localhost:{args.port}")
            logger.info("Watch the crawl live - pages appear as they're indexed!")
        else:
            logger.warning("serp.py not found - --serve disabled")

    try:
        archive_path = await archiver.archive(
            target_url=args.url,
            depth=args.depth,
            max_pages=args.max_pages,
        )

        print(f"\nArchive created: {archive_path}")
        print(f"Extract with: tar -xzf {archive_path.name}")
    finally:
        # Cleanup SERP server
        if serp_process:
            logger.info("Stopping SERP server...")
            serp_process.terminate()
            try:
                serp_process.wait(timeout=5)
            except subprocess.TimeoutExpired:
                serp_process.kill()


if __name__ == "__main__":
    asyncio.run(main())