From 623213f5b6dd144eeb8a0667cfb4e7c925a37348 Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Tue, 30 Dec 2025 07:46:59 -0500 Subject: [PATCH] Self-extracting archives now use bundled neopig/serp.py - Removed embedded serve.py (872 lines) from archive.py - bootstrap.c extracts neopig/*.py to /tmp and runs serp.py - serp.py: tarball mode serves media directly from tar.gz - OffsetFile wrapper for reading .run files at correct offset - ArchiveDB: simple SQLite wrapper for archive search (no async deps) - Archives bundle all neopig source files for self-contained operation - --upgrade-neopig flag with progress logging --- archive.py | 995 ++-------------------------------------------------- bootstrap.c | 160 +++++---- serp.py | 181 +++++++++- 3 files changed, 293 insertions(+), 1043 deletions(-) diff --git a/archive.py b/archive.py index eb30eec..2bfa68f 100644 --- a/archive.py +++ b/archive.py @@ -235,9 +235,6 @@ class SiteArchiver: # Create search database await self._create_search_database_streaming(tmpdir_path, sitemap, domain, html_contents) - # Write serve.py - self._write_serve_py(tmpdir_path) - # Write metadata metadata = { 'domain': domain, @@ -523,968 +520,10 @@ class SiteArchiver: await asyncio.to_thread(create_db) logger.info(f"Created search database: {db_path}") - def _write_serve_py(self, archive_root: Path): - """Write embedded search server (stdlib only, no dependencies).""" - serve_py = '''#!/usr/bin/env python3 -""" -neopig Archive Server - Browse and search archived sites. -Zero dependencies - uses only Python stdlib. - -Usage: - python serve.py # Serve from extracted archive - python serve.py archive.tar.gz # Serve directly from tarball - python serve.py -p 8080 # Custom port - ./archive.run # Self-extracting archive -""" - -import argparse -import html -import json -import mimetypes -import os -import re -import sqlite3 -import sys -import tarfile -import tempfile -from http.server import HTTPServer, BaseHTTPRequestHandler -from pathlib import Path -from urllib.parse import parse_qs, urlparse, unquote - -# Globals set at startup -ARCHIVE_ROOT = None -TAR_FILE = None -TAR_MEMBERS = {} -DB_PATH = None -METADATA = {} - - -def get_archive_source(): - """Determine if we are in a tarball, .run, or extracted directory.""" - exe_path = Path(sys.argv[0]).resolve() - - # Check for .run format (has NEOPIG trailer) - if exe_path.suffix == '.run' or (len(sys.argv) == 1 and exe_path.stat().st_size > 1000000): - try: - with open(exe_path, 'rb') as f: - f.seek(-22, 2) - trailer = f.read(22) - if trailer[:6] == b'NEOPIG': - offset = int(trailer[6:22].decode(), 16) - return ('run', exe_path, offset) - except Exception: - pass - - # Check command line for tarball or .run argument - for arg in sys.argv[1:]: - if not arg.startswith('-'): - p = Path(arg) - if p.exists(): - if p.suffix in ('.gz', '.tar', '.tgz'): - return ('tarball', p, 0) - # Check if it's a .run file with NEOPIG trailer - if p.suffix == '.run' or p.stat().st_size > 100000: - try: - with open(p, 'rb') as f: - f.seek(-22, 2) - trailer = f.read(22) - if trailer[:6] == b'NEOPIG': - offset = int(trailer[6:22].decode(), 16) - return ('run', p, offset) - except Exception: - pass - - # Must be extracted directory - return ('directory', Path(__file__).parent, 0) - - -def init_archive(): - """Initialize archive access.""" - global ARCHIVE_ROOT, TAR_FILE, TAR_MEMBERS, DB_PATH, METADATA - - source_type, source_path, offset = get_archive_source() - - if source_type == 'run': - print(f"Serving from self-extracting archive: {source_path}") - f = open(source_path, 'rb') - f.seek(offset) - TAR_FILE = tarfile.open(fileobj=f, mode='r:gz') - elif source_type == 'tarball': - print(f"Serving from tarball: {source_path}") - TAR_FILE = tarfile.open(source_path, 'r:gz') - else: - print(f"Serving from directory: {source_path}") - ARCHIVE_ROOT = source_path - - if TAR_FILE: - # Build member lookup and find archive root - for member in TAR_FILE.getmembers(): - TAR_MEMBERS[member.name] = member - first = list(TAR_MEMBERS.keys())[0] - archive_name = first.split('/')[0] - ARCHIVE_ROOT = Path(archive_name) - - # Extract database to temp for searching - db_member = f"{archive_name}/archive.db" - if db_member in TAR_MEMBERS: - temp_dir = tempfile.mkdtemp() - TAR_FILE.extract(TAR_MEMBERS[db_member], temp_dir) - DB_PATH = Path(temp_dir) / db_member - else: - DB_PATH = ARCHIVE_ROOT / 'archive.db' - - # Load metadata - meta_path = ARCHIVE_ROOT / 'metadata.json' if not TAR_FILE else None - if meta_path and meta_path.exists(): - METADATA = json.loads(meta_path.read_text()) - elif TAR_FILE: - meta_member = f"{ARCHIVE_ROOT}/metadata.json" - if meta_member in TAR_MEMBERS: - f = TAR_FILE.extractfile(TAR_MEMBERS[meta_member]) - if f: - METADATA = json.loads(f.read().decode()) - - -def read_file(path: str) -> tuple: - """Read file from archive. Returns (content_bytes, mime_type) or (None, None).""" - if TAR_FILE: - # Normalize path for tarball - tar_path = f"{ARCHIVE_ROOT}/{path}".lstrip('/') - if tar_path in TAR_MEMBERS: - f = TAR_FILE.extractfile(TAR_MEMBERS[tar_path]) - if f: - mime, _ = mimetypes.guess_type(path) - return f.read(), mime or 'application/octet-stream' - return None, None - else: - file_path = ARCHIVE_ROOT / path - if file_path.exists() and file_path.is_file(): - # Security: prevent path traversal - try: - file_path.resolve().relative_to(ARCHIVE_ROOT.resolve()) - except ValueError: - return None, None - mime, _ = mimetypes.guess_type(str(file_path)) - return file_path.read_bytes(), mime or 'application/octet-stream' - return None, None - - -def list_files(subdir: str, pattern: str = '*') -> list: - """List files in a subdirectory.""" - files = [] - if TAR_FILE: - prefix = f"{ARCHIVE_ROOT}/{subdir}/" - for name in TAR_MEMBERS: - if name.startswith(prefix) and not name.endswith('/'): - rel = name[len(prefix):] - if pattern == '*' or rel.endswith(pattern.replace('*', '')): - files.append(rel) - else: - dir_path = ARCHIVE_ROOT / subdir - if dir_path.exists(): - for f in dir_path.rglob(pattern): - if f.is_file(): - files.append(str(f.relative_to(dir_path))) - return sorted(files) - - -def search_pages(query: str, limit: int = 50) -> list: - """Search pages using FTS5 with LIKE fallback.""" - if not DB_PATH or not DB_PATH.exists(): - return [] - - conn = sqlite3.connect(DB_PATH) - c = conn.cursor() - results = [] - try: - # Try FTS5 with prefix matching (add * for partial matches) - fts_query = ' '.join(f'"{word}"*' for word in query.split()) - c.execute(""" - SELECT p.path, p.title, snippet(pages_fts, 1, '', '', '...', 40) - FROM pages_fts - JOIN pages p ON pages_fts.rowid = p.id - WHERE pages_fts MATCH ? - ORDER BY rank - LIMIT ? - """, (fts_query, limit)) - results = [{'path': r[0], 'title': r[1], 'snippet': r[2]} for r in c.fetchall()] - except sqlite3.OperationalError: - pass - - # Fallback to LIKE for substring matching if FTS5 found nothing - if not results: - try: - like_q = f'%{query}%' - c.execute(""" - SELECT path, title, substr(content, 1, 200) as snippet - FROM pages - WHERE title LIKE ? COLLATE NOCASE - OR content LIKE ? COLLATE NOCASE - OR path LIKE ? COLLATE NOCASE - LIMIT ? - """, (like_q, like_q, like_q, limit)) - results = [{'path': r[0], 'title': r[1], 'snippet': r[2] + '...'} for r in c.fetchall()] - except sqlite3.OperationalError: - pass - - conn.close() - return results - - -def get_stats() -> dict: - """Get archive statistics.""" - stats = { - 'pages': len(list_files('html', '*.html')), - 'media': len(list_files('media')), - 'screenshots': len(list_files('screenshots', '*.png')), - 'domain': METADATA.get('domain', 'unknown'), - 'created': METADATA.get('created', 'unknown'), - } - if METADATA.get('stats'): - stats.update(METADATA['stats']) - return stats - - -# HTML Templates -INDEX_HTML = """ - - - - - {domain} - neopig Archive - - - -
-

neopig Archive

-

{domain} - archived {created}

- - - -
-
{pages}
Pages
-
{media}
Media
-
{screenshots}
Screenshots
-
- -
- - -
- -
-
- - - -""" - -BROWSE_HTML = """ - - - - - Browse - {domain} - - - -
-

Browse Pages

- -
- {items} -
-
- -""" - -MEDIA_HTML = """ - - - - - Media - {domain} - - - -
-

{title}

- -
- {items} -
-
- -""" - -MEDIA_DETAIL_HTML = """ - - - - - {caption} - neopig Archive - - - -
- - -
- {media_element} -
{caption}
-
-
Image: {media_uri}
-
Source: {page_uri}
-
-
- -
- -
Source Page Content
-
- {page_content} -
-
- -""" - - -PAGE_VIEW_HTML = """ - - - - - {title} - neopig Archive - - - -
- -

{title}

-
- [View Original HTML] - {screenshot_link} -
-
- {content} -
- {screenshot_embed} -
- -""" - - -def simple_markdown_to_html(md: str) -> str: - """Convert markdown to HTML (simple stdlib-only implementation).""" - import re - lines = md.split('\n') - html_lines = [] - in_code_block = False - in_list = False - - for line in lines: - # Code blocks - if line.startswith('```'): - if in_code_block: - html_lines.append('') - in_code_block = False - else: - html_lines.append('
')
-                in_code_block = True
-            continue
-
-        if in_code_block:
-            html_lines.append(html.escape(line))
-            continue
-
-        # Close list if needed
-        if in_list and not line.strip().startswith(('- ', '* ', '1. ')):
-            html_lines.append('')
-            in_list = False
-
-        # Headers
-        if line.startswith('### '):
-            html_lines.append(f'
{html.escape(line[4:])}
')
-        elif line.startswith('## '):
-            html_lines.append(f'
{html.escape(line[3:])}
')
-        elif line.startswith('# '):
-            html_lines.append(f'
{html.escape(line[2:])}
')
-        # Blockquotes
-        elif line.startswith('> '):
-            html_lines.append(f'
{html.escape(line[2:])}
')
-        # Horizontal rule
-        elif line.strip() in ('---', '***', '___'):
-            html_lines.append('
')
-        # Lists
-        elif line.strip().startswith(('- ', '* ')):
-            if not in_list:
-                html_lines.append('
    ')
-                in_list = True
-            content = line.strip()[2:]
-            html_lines.append(f'
  • {html.escape(content)}
    • ')
-        # Empty line
-        elif not line.strip():
-            html_lines.append('
    ')
-        # Regular paragraph
-        else:
-            escaped = html.escape(line)
-            # Inline code
-            escaped = re.sub(r'`([^`]+)`', r'\1', escaped)
-            # Bold
-            escaped = re.sub(r'[*][*]([^*]+)[*][*]', r'\1', escaped)
-            # Italic
-            escaped = re.sub(r'[*]([^*]+)[*]', r'\1', escaped)
-            # Links [text](url)
-            link_re = re.compile(r'\[([^]]+)\]\(([^)]+)\)')
-            escaped = link_re.sub(r'\1', escaped)
-            # Images ![alt](url)
-            img_re = re.compile(r'!\[([^]]*)\]\(([^)]+)\)')
-            escaped = img_re.sub(r'\1', escaped)
-            html_lines.append(f'
    {escaped}
    ')
-
-    if in_list:
-        html_lines.append('
')
-    if in_code_block:
-        html_lines.append('
') - - return '\n'.join(html_lines) - - -class ArchiveHandler(BaseHTTPRequestHandler): - """HTTP request handler for the archive.""" - - def log_message(self, format, *args): - print(f"[{self.log_date_time_string()}] {args[0]}") - - def send_html(self, content: str, status: int = 200): - self.send_response(status) - self.send_header('Content-Type', 'text/html; charset=utf-8') - self.send_header('Content-Length', len(content.encode())) - self.end_headers() - self.wfile.write(content.encode()) - - def send_json(self, data, status: int = 200): - content = json.dumps(data) - self.send_response(status) - self.send_header('Content-Type', 'application/json') - self.send_header('Content-Length', len(content.encode())) - self.end_headers() - self.wfile.write(content.encode()) - - def send_file(self, content: bytes, mime: str): - self.send_response(200) - self.send_header('Content-Type', mime) - self.send_header('Content-Length', len(content)) - self.send_header('Cache-Control', 'public, max-age=86400') - self.end_headers() - self.wfile.write(content) - - def send_404(self): - self.send_html('

404 Not Found

', 404) - - def do_GET(self): - parsed = urlparse(self.path) - path = unquote(parsed.path) - query = parse_qs(parsed.query) - - # API endpoints - if path == '/api/search': - q = query.get('q', [''])[0] - results = search_pages(q) if q else [] - self.send_json(results) - return - - if path == '/api/stats': - self.send_json(get_stats()) - return - - # Pages - if path == '/': - stats = get_stats() - content = INDEX_HTML.format(**stats) - self.send_html(content) - return - - if path == '/browse': - pages = list_files('html', '*.html') - items = ''.join([ - f'
{html.escape(p)}
' - for p in pages[:500] - ]) - if not items: - items = '
No pages found
' - content = BROWSE_HTML.format(domain=METADATA.get('domain', ''), items=items) - self.send_html(content) - return - - # Page view - render markdown with neopig styling - if path.startswith('/view/'): - page_path = path[6:] # Remove '/view/' - # Try markdown first, fall back to HTML - md_path = 'markdown/' + page_path.replace('.html', '.md') - html_path = 'html/' + page_path - - md_content, _ = read_file(md_path) - html_content_raw, _ = read_file(html_path) - - if md_content: - # Render markdown - rendered = simple_markdown_to_html(md_content.decode('utf-8', errors='replace')) - elif html_content_raw: - # Extract body from HTML and show as-is - html_str = html_content_raw.decode('utf-8', errors='replace') - # Simple body extraction - import re as re_mod - body_match = re_mod.search(r']*>(.*?)', html_str, re_mod.DOTALL | re_mod.IGNORECASE) - rendered = body_match.group(1) if body_match else html_str - else: - self.send_404() - return - - # Extract title - title = page_path.replace('.html', '').replace('/', ' > ') - - # Check for screenshot - ss_path = page_path.replace('.html', '.png').replace('/', '_') - screenshot_exists = f'screenshots/{ss_path}' in [f'screenshots/{f}' for f in list_files('screenshots', '*.png')] - - screenshot_link = f' | [View Screenshot]' if screenshot_exists else '' - screenshot_embed = ( - '
' - '
Page Screenshot:
' - f'Screenshot' - '
' - ) if screenshot_exists else '' - - content = PAGE_VIEW_HTML.format( - title=html.escape(title), - path=html.escape(page_path), - content=rendered, - screenshot_link=screenshot_link, - screenshot_embed=screenshot_embed, - ) - self.send_html(content) - return - - if path == '/media': - files = list_files('media') - items = [] - for f in files[:200]: - ext = Path(f).suffix.lower() - if ext in ('.mp4', '.webm', '.mov'): - media_el = f'' - else: - media_el = f'' - # Link to detail view instead of raw file - items.append( - '
' - f'{media_el}' - f'
{html.escape(f)}
' - '
' - ) - content = MEDIA_HTML.format( - domain=METADATA.get('domain', ''), - title='Media', - items=''.join(items) if items else '
No media found
' - ) - self.send_html(content) - return - - # Media detail view - image at top with source page content below - if path.startswith('/media/detail/'): - media_path = path[14:] # Remove '/media/detail/' - media_file = 'media/' + media_path - - # Check media exists - media_content, mime = read_file(media_file) - if not media_content: - self.send_404() - return - - # Determine media element type - ext = Path(media_path).suffix.lower() - if ext in ('.mp4', '.webm', '.mov'): - media_el = f'' - else: - media_el = f'' - - # Try to find the source page - media path mirrors URL structure - # e.g., media/images/foo.jpg might come from t/topic-name/123.html - # For now, use filename as caption - caption = Path(media_path).stem.replace('-', ' ').replace('_', ' ') - media_uri = f'/media/{media_path}' - page_uri = METADATA.get('target_url', METADATA.get('domain', 'unknown')) - page_path = 'index.html' - - # Try to find associated page content from the database - page_content = '

Source page content not available in archive database.

' - if DB_PATH and DB_PATH.exists(): - conn = sqlite3.connect(DB_PATH) - c = conn.cursor() - try: - # Search for pages that might contain this media - media_name = Path(media_path).name - c.execute(""" - SELECT path, title, content FROM pages - WHERE content LIKE ? OR path LIKE ? - LIMIT 1 - """, (f'%{media_name}%', f'%{media_name}%')) - row = c.fetchone() - if row: - page_path = row[0].replace('html/', '') - caption = row[1] or caption - # Render the content - md_file = 'markdown/' + page_path.replace('.html', '.md') - md_content, _ = read_file(md_file) - if md_content: - page_content = simple_markdown_to_html(md_content.decode('utf-8', errors='replace')) - else: - page_content = f'

{html.escape(row[2][:2000] if row[2] else "")}...

' - except Exception: - pass - finally: - conn.close() - - content = MEDIA_DETAIL_HTML.format( - media_element=media_el, - caption=html.escape(caption), - media_uri=html.escape(media_uri), - page_uri=html.escape(page_uri), - page_path=html.escape(page_path), - page_content=page_content, - ) - self.send_html(content) - return - - if path == '/screenshots': - files = list_files('screenshots', '*.png') - items = [] - for f in files[:200]: - items.append( - '
' - f'' - f'' - '' - f'
{html.escape(f)}
' - '
' - ) - content = MEDIA_HTML.format( - domain=METADATA.get('domain', ''), - title='Screenshots', - items=''.join(items) if items else '
No screenshots found
' - ) - self.send_html(content) - return - - # Serve static files - file_path = path.lstrip('/') - content, mime = read_file(file_path) - if content is not None: - self.send_file(content, mime) - else: - self.send_404() - - -def main(): - parser = argparse.ArgumentParser(description='neopig Archive Server') - parser.add_argument('archive', nargs='?', help='Path to archive.tar.gz (optional)') - parser.add_argument('-p', '--port', type=int, default=8000, help='Port to listen on') - parser.add_argument('--host', default='0.0.0.0', help='Host to bind to') - args = parser.parse_args() - - init_archive() - - server = HTTPServer((args.host, args.port), ArchiveHandler) - print(f"Starting neopig archive server at http://{args.host}:{args.port}") - print(f"Archive: {METADATA.get('domain', 'unknown')} ({get_stats()['pages']} pages)") - print("Press Ctrl+C to stop") - - try: - server.serve_forever() - except KeyboardInterrupt: - print("\\nShutting down...") - server.shutdown() - - -if __name__ == '__main__': - main() -''' - (archive_root / 'serve.py').write_text(serve_py) +# NOTE: Embedded serve.py removed - archives now use serp.py from bundled neopig/ +# The bootstrap.c extracts neopig/*.py and runs serp.py with the tarball as argument. +# See: make run TARBALL=archive.tar.gz def upgrade_neopig_in_archive(archive_path: Path) -> Path: @@ -1504,38 +543,56 @@ def upgrade_neopig_in_archive(archive_path: Path) -> Path: # Create new archive with updated neopig output_path = archive_path.with_suffix('.upgraded.tar.gz') + logger.info(f"Reading archive: {archive_path}") with tarfile.open(archive_path, 'r:gz') as old_tar: - with tarfile.open(output_path, 'w:gz') as new_tar: - # Get archive name from first member - members = old_tar.getmembers() - archive_name = members[0].name.split('/')[0] + # Get members list (this reads the whole index) + members = old_tar.getmembers() + total = len(members) + archive_name = members[0].name.split('/')[0] + logger.info(f"Found {total} members in {archive_name}") + with tarfile.open(output_path, 'w:gz') as new_tar: # Copy all members except neopig/ and requirements.txt - for member in members: + copied = 0 + skipped = 0 + for i, member in enumerate(members): parts = member.name.split('/') if len(parts) > 1 and parts[1] == 'neopig': + skipped += 1 continue # Skip old neopig files if len(parts) > 1 and parts[1] == 'requirements.txt': + skipped += 1 continue # Skip old requirements if member.isfile(): f = old_tar.extractfile(member) if f: new_tar.addfile(member, f) + copied += 1 else: new_tar.addfile(member) + copied += 1 + + if (i + 1) % 1000 == 0: + logger.info(f" Progress: {i+1}/{total} ({copied} copied, {skipped} skipped)") + + logger.info(f"Copied {copied} members, skipped {skipped}") # Add new neopig files + logger.info("Adding neopig source files...") for pyfile in neopig_files: src_path = neopig_src / pyfile if src_path.exists(): new_tar.add(src_path, arcname=f"{archive_name}/neopig/{pyfile}") + logger.info(f" Added: neopig/{pyfile}") # Add requirements.txt req_path = neopig_src / 'requirements.txt' if req_path.exists(): new_tar.add(req_path, arcname=f"{archive_name}/requirements.txt") + logger.info(" Added: requirements.txt") # Replace original with upgraded + logger.info(f"Replacing original archive...") shutil.move(output_path, archive_path) return archive_path diff --git a/bootstrap.c b/bootstrap.c index 405787f..6b1f377 100644 --- a/bootstrap.c +++ b/bootstrap.c @@ -1,10 +1,10 @@ /* * bootstrap.c - Self-extracting archive bootstrap * - * A small C program that boots an embedded Python archive server. + * A small C program that boots neopig from an embedded tar.gz archive. * When compiled and concatenated with a tar.gz archive, it: - * 1. Extracts serve.py from the embedded tarball - * 2. Runs: python3 serve.py + * 1. Extracts neopig/*.py from the embedded tarball to /tmp + * 2. Runs: python3 /tmp/neopig/serp.py * 3. The Python script serves directly from the tarball portion * * Build: @@ -16,7 +16,7 @@ * echo -n "NEOPIG" >> archive.run # Magic marker * printf '%016x' $(stat -c%s bootstrap) >> archive.run # Offset (hex) * - * Or use the provided make-executable.sh script. + * Or use: make run TARBALL=archive.tar.gz */ #define _GNU_SOURCE @@ -35,7 +35,6 @@ #define OFFSET_LEN 16 #define TRAILER_LEN (MAGIC_LEN + OFFSET_LEN) -#define SERVE_PY_NAME "serve.py" #define CHUNK_SIZE 16384 /* Find python3 interpreter */ @@ -94,17 +93,35 @@ static long read_trailer(const char *self_path) { return offset; } -/* Extract serve.py from gzipped tarball */ -static char *extract_serve_py(const char *self_path, long tar_offset) { - /* Create temp file for serve.py */ - char *temp_path = strdup("/tmp/neopig_serve_XXXXXX.py"); - if (!temp_path) return NULL; +/* Create directory recursively */ +static int mkdir_p(const char *path) { + char tmp[4096]; + char *p = NULL; + size_t len; - /* mkstemps for .py suffix */ - int fd = mkstemps(temp_path, 3); - if (fd < 0) { - perror("mkstemps"); - free(temp_path); + snprintf(tmp, sizeof(tmp), "%s", path); + len = strlen(tmp); + if (tmp[len - 1] == '/') tmp[len - 1] = 0; + + for (p = tmp + 1; *p; p++) { + if (*p == '/') { + *p = 0; + mkdir(tmp, 0755); + *p = '/'; + } + } + return mkdir(tmp, 0755); +} + +/* Extract neopig/*.py from gzipped tarball */ +static char *extract_neopig(const char *self_path, long tar_offset) { + /* Create temp directory */ + char *temp_dir = strdup("/tmp/neopig_XXXXXX"); + if (!temp_dir) return NULL; + + if (!mkdtemp(temp_dir)) { + perror("mkdtemp"); + free(temp_dir); return NULL; } @@ -112,35 +129,30 @@ static char *extract_serve_py(const char *self_path, long tar_offset) { FILE *self = fopen(self_path, "rb"); if (!self) { perror("fopen self for tar"); - close(fd); - unlink(temp_path); - free(temp_path); + free(temp_dir); return NULL; } if (fseek(self, tar_offset, SEEK_SET) != 0) { perror("fseek to tar"); fclose(self); - close(fd); - unlink(temp_path); - free(temp_path); + free(temp_dir); return NULL; } /* Open gzip stream */ - gzFile gz = gzdopen(fileno(self), "rb"); + gzFile gz = gzdopen(dup(fileno(self)), "rb"); if (!gz) { fprintf(stderr, "gzdopen failed\n"); fclose(self); - close(fd); - unlink(temp_path); - free(temp_path); + free(temp_dir); return NULL; } + fclose(self); - /* Read tar headers looking for serve.py */ + /* Read tar headers looking for neopig/*.py */ unsigned char header[512]; - int found = 0; + int found_any = 0; while (gzread(gz, header, 512) == 512) { /* Check for end of archive (all zeros) */ @@ -161,25 +173,44 @@ static char *extract_serve_py(const char *self_path, long tar_offset) { size_str[12] = '\0'; long filesize = strtol(size_str, NULL, 8); - /* Check if this is serve.py */ - char *basename = strrchr(filename, '/'); - basename = basename ? basename + 1 : filename; + /* Check if this is a neopig/*.py file */ + char *neopig_pos = strstr(filename, "/neopig/"); + char *py_ext = strstr(filename, ".py"); - if (strcmp(basename, SERVE_PY_NAME) == 0) { + if (neopig_pos && py_ext && py_ext > neopig_pos) { /* Extract this file */ - unsigned char buf[CHUNK_SIZE]; - long remaining = filesize; + char *basename = neopig_pos + 8; /* Skip "/neopig/" */ + char out_path[4096]; + snprintf(out_path, sizeof(out_path), "%s/%s", temp_dir, basename); - while (remaining > 0) { - int to_read = remaining > CHUNK_SIZE ? CHUNK_SIZE : remaining; - int got = gzread(gz, buf, to_read); - if (got <= 0) break; - write(fd, buf, got); - remaining -= got; + int fd = open(out_path, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd >= 0) { + unsigned char buf[CHUNK_SIZE]; + long remaining = filesize; + + while (remaining > 0) { + int to_read = remaining > CHUNK_SIZE ? CHUNK_SIZE : remaining; + int got = gzread(gz, buf, to_read); + if (got <= 0) break; + write(fd, buf, got); + remaining -= got; + } + close(fd); + found_any = 1; + printf(" Extracted: %s\n", basename); + + /* Skip padding to 512 boundary */ + int pad = (512 - (filesize % 512)) % 512; + if (pad > 0) { + gzread(gz, buf, pad); + } + } else { + /* Skip this file's content */ + long blocks = (filesize + 511) / 512; + for (long i = 0; i < blocks; i++) { + gzread(gz, header, 512); + } } - - found = 1; - break; } else { /* Skip this file's content (padded to 512 bytes) */ long blocks = (filesize + 511) / 512; @@ -190,16 +221,14 @@ static char *extract_serve_py(const char *self_path, long tar_offset) { } gzclose(gz); - close(fd); - if (!found) { - fprintf(stderr, "Error: serve.py not found in archive\n"); - unlink(temp_path); - free(temp_path); + if (!found_any) { + fprintf(stderr, "Error: No neopig/*.py files found in archive\n"); + free(temp_dir); return NULL; } - return temp_path; + return temp_dir; } int main(int argc, char *argv[]) { @@ -219,6 +248,9 @@ int main(int argc, char *argv[]) { self_path[len] = '\0'; } + printf("neopig self-extracting archive\n"); + printf("==============================\n"); + /* Read trailer to get tarball offset */ long tar_offset = read_trailer(self_path); if (tar_offset < 0) { @@ -227,43 +259,49 @@ int main(int argc, char *argv[]) { printf("Archive offset: %ld bytes\n", tar_offset); - /* Extract serve.py */ - char *serve_py_path = extract_serve_py(self_path, tar_offset); - if (!serve_py_path) { + /* Extract neopig directory */ + printf("Extracting neopig...\n"); + char *neopig_dir = extract_neopig(self_path, tar_offset); + if (!neopig_dir) { return 1; } - printf("Extracted: %s\n", serve_py_path); + /* Build path to serp.py */ + char serp_path[4096]; + snprintf(serp_path, sizeof(serp_path), "%s/serp.py", neopig_dir); /* Find python */ const char *python = find_python(); - /* Build command: python3 serve.py */ - printf("Starting: %s %s %s\n", python, serve_py_path, self_path); + /* Build command: python3 serp.py */ + printf("\nStarting server...\n"); + printf("Command: %s %s %s\n\n", python, serp_path, self_path); /* Fork and exec */ pid_t pid = fork(); if (pid < 0) { perror("fork"); - unlink(serve_py_path); - free(serve_py_path); + free(neopig_dir); return 1; } if (pid == 0) { /* Child - exec python */ - execl(python, "python3", serve_py_path, self_path, NULL); + execl(python, "python3", serp_path, self_path, NULL); perror("execl python3"); _exit(1); } - /* Parent - wait for child and cleanup */ + /* Parent - wait for child */ int status; waitpid(pid, &status, 0); - /* Cleanup temp file */ - unlink(serve_py_path); - free(serve_py_path); + /* Cleanup temp directory */ + printf("\nCleaning up %s...\n", neopig_dir); + char rm_cmd[4096]; + snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", neopig_dir); + system(rm_cmd); + free(neopig_dir); return WIFEXITED(status) ? WEXITSTATUS(status) : 1; } diff --git a/serp.py b/serp.py index 3a24318..d4c5129 100644 --- a/serp.py +++ b/serp.py @@ -25,7 +25,7 @@ from pathlib import Path from typing import List, Dict, Any, Optional from fastapi import FastAPI, Query, HTTPException, BackgroundTasks -from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response from fastapi.staticfiles import StaticFiles from pydantic import BaseModel from sqlalchemy import text @@ -56,15 +56,76 @@ db: Database = None # Active crawl jobs (in-memory tracking) ACTIVE_CRAWLS: Dict[int, Dict[str, Any]] = {} +# Tarball mode - serve directly from tar.gz archive +import tarfile +import tempfile +TAR_FILE: tarfile.TarFile = None +TAR_MEMBERS: Dict[str, tarfile.TarInfo] = {} +ARCHIVE_ROOT: str = None # e.g., "example.com-20251230" +TEMP_DB_PATH: str = None # Extracted database (SQLite needs real file) + + +def read_from_tarball(path: str) -> bytes: + """Read a file from the tarball. Path is relative to archive root.""" + if not TAR_FILE or not ARCHIVE_ROOT: + return None + full_path = f"{ARCHIVE_ROOT}/{path}" + if full_path in TAR_MEMBERS: + member = TAR_MEMBERS[full_path] + f = TAR_FILE.extractfile(member) + if f: + return f.read() + return None + + +def find_media_in_tarball(md5_hash: str) -> tuple: + """Find media file in tarball by hash. Returns (data, extension) or (None, None).""" + if not TAR_FILE or not ARCHIVE_ROOT: + return None, None + prefix = f"{ARCHIVE_ROOT}/media/{md5_hash}" + for name, member in TAR_MEMBERS.items(): + if name.startswith(prefix): + f = TAR_FILE.extractfile(member) + if f: + ext = Path(name).suffix + return f.read(), ext + return None, None + + +class ArchiveDB: + """Simple sync SQLite wrapper for archive.db (FTS5 search only).""" + def __init__(self, db_path): + import sqlite3 + self.conn = sqlite3.connect(db_path) + self.conn.row_factory = sqlite3.Row + + async def search_pages(self, query, limit=50): + cursor = self.conn.execute( + "SELECT uri, title, snippet(pages_fts, 2, '', '', '...', 32) as snippet " + "FROM pages_fts WHERE pages_fts MATCH ? LIMIT ?", + (query, limit) + ) + return [dict(row) for row in cursor.fetchall()] + + async def get_stats(self): + cursor = self.conn.execute("SELECT COUNT(*) FROM pages") + return {"pages": cursor.fetchone()[0], "media": 0, "screenshots": 0} + @app.on_event("startup") async def startup_event(): """Initialize database on startup.""" global db - db = Database(DB_PATH) - await db.init() # Handles schema + WAL mode - VAULT_PATH.mkdir(parents=True, exist_ok=True) - logger.info(f"Vault directory ready: {VAULT_PATH}") + if TAR_FILE: + # Tarball mode: use simple archive DB + db = ArchiveDB(DB_PATH) + logger.info(f"Using archive database: {DB_PATH}") + else: + # Normal mode: use full async database + db = Database(DB_PATH) + await db.init() # Handles schema + WAL mode + VAULT_PATH.mkdir(parents=True, exist_ok=True) + logger.info(f"Vault directory ready: {VAULT_PATH}") class CrawlRequest(BaseModel): @@ -2612,12 +2673,29 @@ def slugify(text: str, max_len: int = 60) -> str: @app.get("/media/{md5_hash}") async def serve_media(md5_hash: str, download: bool = False): """ - Serve media file from vault. + Serve media file from vault or tarball. Use ?download=1 for attachment mode with smart filename. Caddy should be configured to cache these responses. """ - # Find file in vault + # Tarball mode: serve from tar.gz + if TAR_FILE: + data, ext = find_media_in_tarball(md5_hash) + if data: + mime_type, _ = mimetypes.guess_type(f"file{ext}") + if not mime_type: + mime_type = "application/octet-stream" + filename = f"{md5_hash[:12]}{ext}" + headers = { + "Cache-Control": "public, max-age=31536000, immutable", + "X-Content-Hash": md5_hash, + } + if download: + headers["Content-Disposition"] = f'attachment; filename="{filename}"' + return Response(content=data, media_type=mime_type, headers=headers) + raise HTTPException(status_code=404, detail="Media not found in archive") + + # Filesystem mode: find file in vault subdir = VAULT_PATH / md5_hash[:2] if not subdir.exists(): raise HTTPException(status_code=404, detail="Media not found") @@ -2784,10 +2862,83 @@ async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks): return {"job_ids": job_ids, "status": "running", "count": len(job_ids)} +def init_tarball_mode(tarball_path: str): + """Initialize serving from a tar.gz archive.""" + global TAR_FILE, TAR_MEMBERS, ARCHIVE_ROOT, DB_PATH, TEMP_DB_PATH + + logger.info(f"Opening archive: {tarball_path}") + tarball = Path(tarball_path) + + # Handle .run files with NEOPIG trailer + offset = 0 + if tarball.suffix == '.run' or tarball.stat().st_size > 100000: + try: + with open(tarball, 'rb') as f: + f.seek(-22, 2) + trailer = f.read(22) + if trailer[:6] == b'NEOPIG': + offset = int(trailer[6:22].decode(), 16) + logger.info(f"Detected .run format, tarball offset: {offset}") + except Exception: + pass + + # Open tarball + if offset > 0: + # Create a wrapper that presents just the tarball portion + class OffsetFile: + """File wrapper that starts reading from an offset.""" + def __init__(self, path, offset): + self._f = open(path, 'rb') + self._offset = offset + self._f.seek(offset) + def read(self, size=-1): + return self._f.read(size) + def seek(self, pos, whence=0): + if whence == 0: # SEEK_SET + return self._f.seek(self._offset + pos) + elif whence == 1: # SEEK_CUR + return self._f.seek(pos, 1) + else: # SEEK_END + return self._f.seek(pos, 2) + def tell(self): + return self._f.tell() - self._offset + def close(self): + self._f.close() + + TAR_FILE = tarfile.open(fileobj=OffsetFile(tarball, offset), mode='r:gz') + else: + TAR_FILE = tarfile.open(tarball_path, 'r:gz') + + # Build member lookup + for member in TAR_FILE.getmembers(): + TAR_MEMBERS[member.name] = member + + # Get archive root from first member + first = list(TAR_MEMBERS.keys())[0] + ARCHIVE_ROOT = first.split('/')[0] + logger.info(f"Archive root: {ARCHIVE_ROOT}") + + # Extract database to temp (SQLite needs real file) + db_member = f"{ARCHIVE_ROOT}/archive.db" + if db_member in TAR_MEMBERS: + temp_dir = tempfile.mkdtemp(prefix="neopig_") + TEMP_DB_PATH = f"{temp_dir}/archive.db" + member = TAR_MEMBERS[db_member] + f = TAR_FILE.extractfile(member) + if f: + with open(TEMP_DB_PATH, 'wb') as out: + out.write(f.read()) + DB_PATH = TEMP_DB_PATH + logger.info(f"Extracted database to: {TEMP_DB_PATH}") + else: + logger.warning("No archive.db found in tarball") + + def main(): global DB_PATH, VAULT_PATH parser = argparse.ArgumentParser(description="neopig SERP") + parser.add_argument("tarball", nargs='?', help="Path to archive.tar.gz or .run file") parser.add_argument("--port", type=int, default=8000) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--db", default="data/neopig.db") @@ -2795,12 +2946,16 @@ def main(): args = parser.parse_args() - DB_PATH = args.db - VAULT_PATH = Path(args.vault) - - logger.info(f"Starting neopig SERP on {args.host}:{args.port}") - logger.info(f"Database: {DB_PATH}") - logger.info(f"Vault: {VAULT_PATH}") + # Tarball mode + if args.tarball: + init_tarball_mode(args.tarball) + logger.info(f"Starting neopig SERP (archive mode) on {args.host}:{args.port}") + else: + DB_PATH = args.db + VAULT_PATH = Path(args.vault) + logger.info(f"Starting neopig SERP on {args.host}:{args.port}") + logger.info(f"Database: {DB_PATH}") + logger.info(f"Vault: {VAULT_PATH}") uvicorn.run(app, host=args.host, port=args.port)