engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Self-extracting archives now use bundled neopig/serp.py

Browse source 
- Removed embedded serve.py (872 lines) from archive.py
- bootstrap.c extracts neopig/*.py to /tmp and runs serp.py
- serp.py: tarball mode serves media directly from tar.gz
- OffsetFile wrapper for reading .run files at correct offset
- ArchiveDB: simple SQLite wrapper for archive search (no async deps)
- Archives bundle all neopig source files for self-contained operation
- --upgrade-neopig flag with progress logging
This commit is contained in:
Russell Ballestrini 2025-12-30 07:46:59 -05:00
parent 416b3cb760
commit 623213f5b6
3 changed files with 293 additions and 1043 deletions
Download patch file Download diff file Expand all files Collapse all files

995
archive.py
View file

File diff suppressed because it is too large Load diff

160
bootstrap.c
View file

@ -1,10 +1,10 @@
/*
* bootstrap.c - Self-extracting archive bootstrap
*
* A small C program that boots an embedded Python archive server.
* A small C program that boots neopig from an embedded tar.gz archive.
* When compiled and concatenated with a tar.gz archive, it:
* 1. Extracts serve.py from the embedded tarball
* 2. Runs: python3 serve.py <self>
* 1. Extracts neopig/*.py from the embedded tarball to /tmp
* 2. Runs: python3 /tmp/neopig/serp.py <self>
* 3. The Python script serves directly from the tarball portion
*
* Build:
@ -16,7 +16,7 @@
* echo -n "NEOPIG" >> archive.run # Magic marker
* printf '%016x' $(stat -c%s bootstrap) >> archive.run # Offset (hex)
*
* Or use the provided make-executable.sh script.
* Or use: make run TARBALL=archive.tar.gz
*/
#define _GNU_SOURCE
@ -35,7 +35,6 @@
#define OFFSET_LEN 16
#define TRAILER_LEN (MAGIC_LEN + OFFSET_LEN)
#define SERVE_PY_NAME "serve.py"
#define CHUNK_SIZE 16384
/* Find python3 interpreter */
@ -94,17 +93,35 @@ static long read_trailer(const char *self_path) {
return offset;
}
/* Extract serve.py from gzipped tarball */
static char *extract_serve_py(const char *self_path, long tar_offset) {
/* Create temp file for serve.py */
char *temp_path = strdup("/tmp/neopig_serve_XXXXXX.py");
if (!temp_path) return NULL;
/* Create directory recursively */
static int mkdir_p(const char *path) {
char tmp[4096];
char *p = NULL;
size_t len;
/* mkstemps for .py suffix */
int fd = mkstemps(temp_path, 3);
if (fd < 0) {
perror("mkstemps");
free(temp_path);
snprintf(tmp, sizeof(tmp), "%s", path);
len = strlen(tmp);
if (tmp[len - 1] == '/') tmp[len - 1] = 0;
for (p = tmp + 1; *p; p++) {
if (*p == '/') {
*p = 0;
mkdir(tmp, 0755);
*p = '/';
}
}
return mkdir(tmp, 0755);
}
/* Extract neopig/*.py from gzipped tarball */
static char *extract_neopig(const char *self_path, long tar_offset) {
/* Create temp directory */
char *temp_dir = strdup("/tmp/neopig_XXXXXX");
if (!temp_dir) return NULL;
if (!mkdtemp(temp_dir)) {
perror("mkdtemp");
free(temp_dir);
return NULL;
}
@ -112,35 +129,30 @@ static char *extract_serve_py(const char *self_path, long tar_offset) {
FILE *self = fopen(self_path, "rb");
if (!self) {
perror("fopen self for tar");
close(fd);
unlink(temp_path);
free(temp_path);
free(temp_dir);
return NULL;
}
if (fseek(self, tar_offset, SEEK_SET) != 0) {
perror("fseek to tar");
fclose(self);
close(fd);
unlink(temp_path);
free(temp_path);
free(temp_dir);
return NULL;
}
/* Open gzip stream */
gzFile gz = gzdopen(fileno(self), "rb");
gzFile gz = gzdopen(dup(fileno(self)), "rb");
if (!gz) {
fprintf(stderr, "gzdopen failed\n");
fclose(self);
close(fd);
unlink(temp_path);
free(temp_path);
free(temp_dir);
return NULL;
}
fclose(self);
/* Read tar headers looking for serve.py */
/* Read tar headers looking for neopig/*.py */
unsigned char header[512];
int found = 0;
int found_any = 0;
while (gzread(gz, header, 512) == 512) {
/* Check for end of archive (all zeros) */
@ -161,25 +173,44 @@ static char *extract_serve_py(const char *self_path, long tar_offset) {
size_str[12] = '\0';
long filesize = strtol(size_str, NULL, 8);
/* Check if this is serve.py */
char *basename = strrchr(filename, '/');
basename = basename ? basename + 1 : filename;
/* Check if this is a neopig/*.py file */
char *neopig_pos = strstr(filename, "/neopig/");
char *py_ext = strstr(filename, ".py");
if (strcmp(basename, SERVE_PY_NAME) == 0) {
if (neopig_pos && py_ext && py_ext > neopig_pos) {
/* Extract this file */
unsigned char buf[CHUNK_SIZE];
long remaining = filesize;
char *basename = neopig_pos + 8; /* Skip "/neopig/" */
char out_path[4096];
snprintf(out_path, sizeof(out_path), "%s/%s", temp_dir, basename);
while (remaining > 0) {
int to_read = remaining > CHUNK_SIZE ? CHUNK_SIZE : remaining;
int got = gzread(gz, buf, to_read);
if (got <= 0) break;
write(fd, buf, got);
remaining -= got;
int fd = open(out_path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
if (fd >= 0) {
unsigned char buf[CHUNK_SIZE];
long remaining = filesize;
while (remaining > 0) {
int to_read = remaining > CHUNK_SIZE ? CHUNK_SIZE : remaining;
int got = gzread(gz, buf, to_read);
if (got <= 0) break;
write(fd, buf, got);
remaining -= got;
}
close(fd);
found_any = 1;
printf(" Extracted: %s\n", basename);
/* Skip padding to 512 boundary */
int pad = (512 - (filesize % 512)) % 512;
if (pad > 0) {
gzread(gz, buf, pad);
}
} else {
/* Skip this file's content */
long blocks = (filesize + 511) / 512;
for (long i = 0; i < blocks; i++) {
gzread(gz, header, 512);
}
}
found = 1;
break;
} else {
/* Skip this file's content (padded to 512 bytes) */
long blocks = (filesize + 511) / 512;
@ -190,16 +221,14 @@ static char *extract_serve_py(const char *self_path, long tar_offset) {
}
gzclose(gz);
close(fd);
if (!found) {
fprintf(stderr, "Error: serve.py not found in archive\n");
unlink(temp_path);
free(temp_path);
if (!found_any) {
fprintf(stderr, "Error: No neopig/*.py files found in archive\n");
free(temp_dir);
return NULL;
}
return temp_path;
return temp_dir;
}
int main(int argc, char *argv[]) {
@ -219,6 +248,9 @@ int main(int argc, char *argv[]) {
self_path[len] = '\0';
}
printf("neopig self-extracting archive\n");
printf("==============================\n");
/* Read trailer to get tarball offset */
long tar_offset = read_trailer(self_path);
if (tar_offset < 0) {
@ -227,43 +259,49 @@ int main(int argc, char *argv[]) {
printf("Archive offset: %ld bytes\n", tar_offset);
/* Extract serve.py */
char *serve_py_path = extract_serve_py(self_path, tar_offset);
if (!serve_py_path) {
/* Extract neopig directory */
printf("Extracting neopig...\n");
char *neopig_dir = extract_neopig(self_path, tar_offset);
if (!neopig_dir) {
return 1;
}
printf("Extracted: %s\n", serve_py_path);
/* Build path to serp.py */
char serp_path[4096];
snprintf(serp_path, sizeof(serp_path), "%s/serp.py", neopig_dir);
/* Find python */
const char *python = find_python();
/* Build command: python3 serve.py <self_path> */
printf("Starting: %s %s %s\n", python, serve_py_path, self_path);
/* Build command: python3 serp.py <self_path> */
printf("\nStarting server...\n");
printf("Command: %s %s %s\n\n", python, serp_path, self_path);
/* Fork and exec */
pid_t pid = fork();
if (pid < 0) {
perror("fork");
unlink(serve_py_path);
free(serve_py_path);
free(neopig_dir);
return 1;
}
if (pid == 0) {
/* Child - exec python */
execl(python, "python3", serve_py_path, self_path, NULL);
execl(python, "python3", serp_path, self_path, NULL);
perror("execl python3");
_exit(1);
}
/* Parent - wait for child and cleanup */
/* Parent - wait for child */
int status;
waitpid(pid, &status, 0);
/* Cleanup temp file */
unlink(serve_py_path);
free(serve_py_path);
/* Cleanup temp directory */
printf("\nCleaning up %s...\n", neopig_dir);
char rm_cmd[4096];
snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", neopig_dir);
system(rm_cmd);
free(neopig_dir);
return WIFEXITED(status) ? WEXITSTATUS(status) : 1;
}

181
serp.py
View file

@ -25,7 +25,7 @@ from pathlib import Path
from typing import List, Dict, Any, Optional
from fastapi import FastAPI, Query, HTTPException, BackgroundTasks
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse, Response
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
from sqlalchemy import text
@ -56,15 +56,76 @@ db: Database = None
# Active crawl jobs (in-memory tracking)
ACTIVE_CRAWLS: Dict[int, Dict[str, Any]] = {}
# Tarball mode - serve directly from tar.gz archive
import tarfile
import tempfile
TAR_FILE: tarfile.TarFile = None
TAR_MEMBERS: Dict[str, tarfile.TarInfo] = {}
ARCHIVE_ROOT: str = None # e.g., "example.com-20251230"
TEMP_DB_PATH: str = None # Extracted database (SQLite needs real file)
def read_from_tarball(path: str) -> bytes:
"""Read a file from the tarball. Path is relative to archive root."""
if not TAR_FILE or not ARCHIVE_ROOT:
return None
full_path = f"{ARCHIVE_ROOT}/{path}"
if full_path in TAR_MEMBERS:
member = TAR_MEMBERS[full_path]
f = TAR_FILE.extractfile(member)
if f:
return f.read()
return None
def find_media_in_tarball(md5_hash: str) -> tuple:
"""Find media file in tarball by hash. Returns (data, extension) or (None, None)."""
if not TAR_FILE or not ARCHIVE_ROOT:
return None, None
prefix = f"{ARCHIVE_ROOT}/media/{md5_hash}"
for name, member in TAR_MEMBERS.items():
if name.startswith(prefix):
f = TAR_FILE.extractfile(member)
if f:
ext = Path(name).suffix
return f.read(), ext
return None, None
class ArchiveDB:
"""Simple sync SQLite wrapper for archive.db (FTS5 search only)."""
def __init__(self, db_path):
import sqlite3
self.conn = sqlite3.connect(db_path)
self.conn.row_factory = sqlite3.Row
async def search_pages(self, query, limit=50):
cursor = self.conn.execute(
"SELECT uri, title, snippet(pages_fts, 2, '<b>', '</b>', '...', 32) as snippet "
"FROM pages_fts WHERE pages_fts MATCH ? LIMIT ?",
(query, limit)
)
return [dict(row) for row in cursor.fetchall()]
async def get_stats(self):
cursor = self.conn.execute("SELECT COUNT(*) FROM pages")
return {"pages": cursor.fetchone()[0], "media": 0, "screenshots": 0}
@app.on_event("startup")
async def startup_event():
"""Initialize database on startup."""
global db
db = Database(DB_PATH)
await db.init() # Handles schema + WAL mode
VAULT_PATH.mkdir(parents=True, exist_ok=True)
logger.info(f"Vault directory ready: {VAULT_PATH}")
if TAR_FILE:
# Tarball mode: use simple archive DB
db = ArchiveDB(DB_PATH)
logger.info(f"Using archive database: {DB_PATH}")
else:
# Normal mode: use full async database
db = Database(DB_PATH)
await db.init() # Handles schema + WAL mode
VAULT_PATH.mkdir(parents=True, exist_ok=True)
logger.info(f"Vault directory ready: {VAULT_PATH}")
class CrawlRequest(BaseModel):
@ -2612,12 +2673,29 @@ def slugify(text: str, max_len: int = 60) -> str:
@app.get("/media/{md5_hash}")
async def serve_media(md5_hash: str, download: bool = False):
"""
Serve media file from vault.
Serve media file from vault or tarball.
Use ?download=1 for attachment mode with smart filename.
Caddy should be configured to cache these responses.
"""
# Find file in vault
# Tarball mode: serve from tar.gz
if TAR_FILE:
data, ext = find_media_in_tarball(md5_hash)
if data:
mime_type, _ = mimetypes.guess_type(f"file{ext}")
if not mime_type:
mime_type = "application/octet-stream"
filename = f"{md5_hash[:12]}{ext}"
headers = {
"Cache-Control": "public, max-age=31536000, immutable",
"X-Content-Hash": md5_hash,
}
if download:
headers["Content-Disposition"] = f'attachment; filename="{filename}"'
return Response(content=data, media_type=mime_type, headers=headers)
raise HTTPException(status_code=404, detail="Media not found in archive")
# Filesystem mode: find file in vault
subdir = VAULT_PATH / md5_hash[:2]
if not subdir.exists():
raise HTTPException(status_code=404, detail="Media not found")
@ -2784,10 +2862,83 @@ async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
return {"job_ids": job_ids, "status": "running", "count": len(job_ids)}
def init_tarball_mode(tarball_path: str):
"""Initialize serving from a tar.gz archive."""
global TAR_FILE, TAR_MEMBERS, ARCHIVE_ROOT, DB_PATH, TEMP_DB_PATH
logger.info(f"Opening archive: {tarball_path}")
tarball = Path(tarball_path)
# Handle .run files with NEOPIG trailer
offset = 0
if tarball.suffix == '.run' or tarball.stat().st_size > 100000:
try:
with open(tarball, 'rb') as f:
f.seek(-22, 2)
trailer = f.read(22)
if trailer[:6] == b'NEOPIG':
offset = int(trailer[6:22].decode(), 16)
logger.info(f"Detected .run format, tarball offset: {offset}")
except Exception:
pass
# Open tarball
if offset > 0:
# Create a wrapper that presents just the tarball portion
class OffsetFile:
"""File wrapper that starts reading from an offset."""
def __init__(self, path, offset):
self._f = open(path, 'rb')
self._offset = offset
self._f.seek(offset)
def read(self, size=-1):
return self._f.read(size)
def seek(self, pos, whence=0):
if whence == 0: # SEEK_SET
return self._f.seek(self._offset + pos)
elif whence == 1: # SEEK_CUR
return self._f.seek(pos, 1)
else: # SEEK_END
return self._f.seek(pos, 2)
def tell(self):
return self._f.tell() - self._offset
def close(self):
self._f.close()
TAR_FILE = tarfile.open(fileobj=OffsetFile(tarball, offset), mode='r:gz')
else:
TAR_FILE = tarfile.open(tarball_path, 'r:gz')
# Build member lookup
for member in TAR_FILE.getmembers():
TAR_MEMBERS[member.name] = member
# Get archive root from first member
first = list(TAR_MEMBERS.keys())[0]
ARCHIVE_ROOT = first.split('/')[0]
logger.info(f"Archive root: {ARCHIVE_ROOT}")
# Extract database to temp (SQLite needs real file)
db_member = f"{ARCHIVE_ROOT}/archive.db"
if db_member in TAR_MEMBERS:
temp_dir = tempfile.mkdtemp(prefix="neopig_")
TEMP_DB_PATH = f"{temp_dir}/archive.db"
member = TAR_MEMBERS[db_member]
f = TAR_FILE.extractfile(member)
if f:
with open(TEMP_DB_PATH, 'wb') as out:
out.write(f.read())
DB_PATH = TEMP_DB_PATH
logger.info(f"Extracted database to: {TEMP_DB_PATH}")
else:
logger.warning("No archive.db found in tarball")
def main():
global DB_PATH, VAULT_PATH
parser = argparse.ArgumentParser(description="neopig SERP")
parser.add_argument("tarball", nargs='?', help="Path to archive.tar.gz or .run file")
parser.add_argument("--port", type=int, default=8000)
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--db", default="data/neopig.db")
@ -2795,12 +2946,16 @@ def main():
args = parser.parse_args()
DB_PATH = args.db
VAULT_PATH = Path(args.vault)
logger.info(f"Starting neopig SERP on {args.host}:{args.port}")
logger.info(f"Database: {DB_PATH}")
logger.info(f"Vault: {VAULT_PATH}")
# Tarball mode
if args.tarball:
init_tarball_mode(args.tarball)
logger.info(f"Starting neopig SERP (archive mode) on {args.host}:{args.port}")
else:
DB_PATH = args.db
VAULT_PATH = Path(args.vault)
logger.info(f"Starting neopig SERP on {args.host}:{args.port}")
logger.info(f"Database: {DB_PATH}")
logger.info(f"Vault: {VAULT_PATH}")
uvicorn.run(app, host=args.host, port=args.port)