engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rename sandbox to import mode, hide phantom from nav

Browse source
This commit is contained in:
Russell Ballestrini 2026-01-02 19:09:47 -05:00
parent 2ebaa78f16
commit 71ad0c9ba2
1 changed files with 61 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

123
serp.py
View file

@ -583,7 +583,6 @@ NAV_HTML = '''<div class="nav">
<a href="/live">{{live}}</a>
<a href="/random">{{random}}</a>
<a href="/crawl">{{crawl}}</a>
<a href="/phantom">{{phantom}}</a>
<a href="/about">{{about}}</a>
</div>'''
@ -592,10 +591,10 @@ def inject_i18n(html: str, lang: str) -> str:
trans = TRANSLATIONS.get(lang, TRANSLATIONS["en"])
# Add lang attribute and inject JS translations
html = html.replace('<html>', f'<html lang="{lang}">')
# Replace nav placeholder with actual nav (add sandbox link if enabled)
# Replace nav placeholder with actual nav (add import link if enabled)
nav = NAV_HTML
if SANDBOX_MODE:
nav = nav.replace('</div>', ' <a href="/sandbox">🧪 Sandbox</a>\n</div>')
if IMPORT_MODE:
nav = nav.replace('</div>', ' <a href="/import">🧪 Import</a>\n</div>')
html = html.replace('<!-- NAV -->', nav)
# Build language selector options
lang_options = ''.join(f'<option value="{code}"{" selected" if code == lang else ""}>{name}</option>'
@ -638,7 +637,7 @@ DB_PATH = "data/neopig.db"
VAULT_PATH = Path("data/vault")
LOGS_PATH = Path("data/logs")
CRAWL_DISABLED = os.environ.get("NEOPIG_DISABLE_CRAWL", "").lower() in ("1", "true", "yes")
SANDBOX_MODE = os.environ.get("NEOPIG_SANDBOX", "").lower() in ("1", "true", "yes")
IMPORT_MODE = os.environ.get("NEOPIG_IMPORT", "").lower() in ("1", "true", "yes")
# Global database instance
db: Database = None
@ -1210,8 +1209,8 @@ async def startup_event():
global db
if CRAWL_DISABLED:
logger.info("Crawl disabled via NEOPIG_DISABLE_CRAWL")
if SANDBOX_MODE:
logger.info("Sandbox mode enabled via NEOPIG_SANDBOX - archive uploads allowed")
if IMPORT_MODE:
logger.info("Import mode enabled via NEOPIG_IMPORT - archive uploads allowed")
if TAR_PATH:
# Tarball mode: use full Database for neopig.db, legacy ArchiveDB for archive.db
if DB_PATH.endswith('neopig.db'):
@ -4483,10 +4482,10 @@ async def serve_media(md5_hash: str, download: bool = False):
# ============================================================================
# Sandbox Mode - Upload and serve archives
# Import Mode - Upload and serve archives
# ============================================================================
SANDBOX_UPLOAD_DIR = Path(tempfile.gettempdir()) / "neopig_sandbox"
IMPORT_UPLOAD_DIR = Path(tempfile.gettempdir()) / "neopig_import"
# Resumable upload tracking: upload_id -> {filename, total_size, created_at}
PENDING_UPLOADS: Dict[str, dict] = {}
@ -4497,24 +4496,24 @@ def generate_upload_id() -> str:
return f"upload-{secrets.token_hex(8)}"
@app.post("/api/sandbox/upload/init")
@app.post("/api/import/upload/init")
async def init_resumable_upload(filename: str = Query(...), size: int = Query(...)):
"""Initialize a resumable upload session.
Returns upload_id that client uses for subsequent chunk uploads.
Client can resume from any disconnect by checking /api/sandbox/upload/{upload_id}/status
Client can resume from any disconnect by checking /api/import/upload/{upload_id}/status
"""
if not SANDBOX_MODE:
raise HTTPException(status_code=403, detail="Sandbox mode not enabled")
if not IMPORT_MODE:
raise HTTPException(status_code=403, detail="Import mode not enabled")
if not filename.endswith(('.tar.gz', '.tgz', '.run')):
raise HTTPException(status_code=400, detail="File must be .tar.gz, .tgz, or .run")
upload_id = generate_upload_id()
SANDBOX_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
IMPORT_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
# Create empty file for this upload
upload_path = SANDBOX_UPLOAD_DIR / f"{upload_id}.part"
upload_path = IMPORT_UPLOAD_DIR / f"{upload_id}.part"
upload_path.touch()
PENDING_UPLOADS[upload_id] = {
@ -4524,7 +4523,7 @@ async def init_resumable_upload(filename: str = Query(...), size: int = Query(..
"path": str(upload_path)
}
logger.info(f"Sandbox: initialized resumable upload {upload_id} for {filename} ({size / 1024 / 1024:.1f} MB)")
logger.info(f"Import: initialized resumable upload {upload_id} for {filename} ({size / 1024 / 1024:.1f} MB)")
return {
"upload_id": upload_id,
@ -4534,11 +4533,11 @@ async def init_resumable_upload(filename: str = Query(...), size: int = Query(..
}
@app.get("/api/sandbox/upload/{upload_id}/status")
@app.get("/api/import/upload/{upload_id}/status")
async def get_upload_status(upload_id: str):
"""Get status of a resumable upload. Use this to resume after disconnect."""
if not SANDBOX_MODE:
raise HTTPException(status_code=403, detail="Sandbox mode not enabled")
if not IMPORT_MODE:
raise HTTPException(status_code=403, detail="Import mode not enabled")
if upload_id not in PENDING_UPLOADS:
raise HTTPException(status_code=404, detail="Upload not found or expired")
@ -4560,7 +4559,7 @@ async def get_upload_status(upload_id: str):
}
@app.patch("/api/sandbox/upload/{upload_id}")
@app.patch("/api/import/upload/{upload_id}")
async def upload_chunk(
upload_id: str,
request: Request,
@ -4573,8 +4572,8 @@ async def upload_chunk(
Or use X-Upload-Offset header for simpler resumption.
"""
if not SANDBOX_MODE:
raise HTTPException(status_code=403, detail="Sandbox mode not enabled")
if not IMPORT_MODE:
raise HTTPException(status_code=403, detail="Import mode not enabled")
if upload_id not in PENDING_UPLOADS:
raise HTTPException(status_code=404, detail="Upload not found or expired")
@ -4614,7 +4613,7 @@ async def upload_chunk(
f.write(chunk_data)
new_size = upload_path.stat().st_size
logger.info(f"Sandbox: {upload_id} received chunk {chunk_size} bytes, total {new_size}/{info['total_size']}")
logger.info(f"Import: {upload_id} received chunk {chunk_size} bytes, total {new_size}/{info['total_size']}")
return {
"upload_id": upload_id,
@ -4624,11 +4623,11 @@ async def upload_chunk(
}
@app.post("/api/sandbox/upload/{upload_id}/complete")
@app.post("/api/import/upload/{upload_id}/complete")
async def complete_resumable_upload(upload_id: str):
"""Finalize upload and start import job."""
if not SANDBOX_MODE:
raise HTTPException(status_code=403, detail="Sandbox mode not enabled")
if not IMPORT_MODE:
raise HTTPException(status_code=403, detail="Import mode not enabled")
if upload_id not in PENDING_UPLOADS:
raise HTTPException(status_code=404, detail="Upload not found or expired")
@ -4647,18 +4646,18 @@ async def complete_resumable_upload(upload_id: str):
)
# Rename to final filename
final_path = SANDBOX_UPLOAD_DIR / info["filename"]
final_path = IMPORT_UPLOAD_DIR / info["filename"]
upload_path.rename(final_path)
# Remove from pending
del PENDING_UPLOADS[upload_id]
logger.info(f"Sandbox: {upload_id} completed, saved as {info['filename']}")
logger.info(f"Import: {upload_id} completed, saved as {info['filename']}")
# Create import job
job_id = await db.create_crawl_job(
target_uri=f"import://{info['filename']}",
keywords=["sandbox", "import"],
keywords=["import"],
mode="import"
)
@ -4669,9 +4668,9 @@ async def complete_resumable_upload(upload_id: str):
stats = await import_archive_to_db(final_path, job_id)
await db.complete_crawl_job(job_id, stats)
final_path.unlink(missing_ok=True)
logger.info(f"Sandbox import complete: {stats}")
logger.info(f"Import complete: {stats}")
except Exception as e:
logger.error(f"Sandbox import failed: {e}")
logger.error(f"Import failed: {e}")
await db.fail_crawl_job(job_id, str(e))
task = asyncio.create_task(run_import())
@ -4686,19 +4685,19 @@ async def complete_resumable_upload(upload_id: str):
}
@app.post("/api/sandbox/upload")
@app.post("/api/import/upload")
async def upload_archive(file: UploadFile = File(...)):
"""Upload a tar.gz archive and import into local database as a job."""
if not SANDBOX_MODE:
raise HTTPException(status_code=403, detail="Sandbox mode not enabled (NEOPIG_SANDBOX=1)")
if not IMPORT_MODE:
raise HTTPException(status_code=403, detail="Import mode not enabled (NEOPIG_IMPORT=1)")
if not file.filename.endswith(('.tar.gz', '.tgz', '.run')):
raise HTTPException(status_code=400, detail="File must be .tar.gz, .tgz, or .run")
# Save uploaded file
SANDBOX_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
upload_path = SANDBOX_UPLOAD_DIR / file.filename
logger.info(f"Sandbox: receiving upload {file.filename}")
IMPORT_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
upload_path = IMPORT_UPLOAD_DIR / file.filename
logger.info(f"Import: receiving upload {file.filename}")
try:
with open(upload_path, 'wb') as f:
@ -4708,12 +4707,12 @@ async def upload_archive(file: UploadFile = File(...)):
raise HTTPException(status_code=400, detail=f"Upload failed: {e}")
file_size = upload_path.stat().st_size
logger.info(f"Sandbox: saved {file.filename} ({file_size / 1024 / 1024:.1f} MB)")
logger.info(f"Import: saved {file.filename} ({file_size / 1024 / 1024:.1f} MB)")
# Create import job
job_id = await db.create_crawl_job(
target_uri=f"import://{file.filename}",
keywords=["sandbox", "import"],
keywords=["import"],
mode="import"
)
@ -4724,9 +4723,9 @@ async def upload_archive(file: UploadFile = File(...)):
stats = await import_archive_to_db(upload_path, job_id)
await db.complete_crawl_job(job_id, stats)
upload_path.unlink(missing_ok=True)
logger.info(f"Sandbox import complete: {stats}")
logger.info(f"Import complete: {stats}")
except Exception as e:
logger.error(f"Sandbox import failed: {e}")
logger.error(f"Import failed: {e}")
await db.fail_crawl_job(job_id, str(e))
task = asyncio.create_task(run_import())
@ -4769,7 +4768,7 @@ async def import_archive_to_db(archive_path: Path, job_id: int) -> dict:
# Extract neopig.db to temp
db_member = f"{archive_root}/neopig.db"
temp_db = SANDBOX_UPLOAD_DIR / f"import_{job_id}.db"
temp_db = IMPORT_UPLOAD_DIR / f"import_{job_id}.db"
for m in members:
if m.name == db_member:
@ -4872,11 +4871,11 @@ async def import_archive_to_db(archive_path: Path, job_id: int) -> dict:
return stats
@app.get("/api/sandbox/status")
async def sandbox_status():
"""Get current sandbox status."""
if not SANDBOX_MODE:
raise HTTPException(status_code=403, detail="Sandbox mode not enabled")
@app.get("/api/import/status")
async def import_status():
"""Get current import status."""
if not IMPORT_MODE:
raise HTTPException(status_code=403, detail="Import mode not enabled")
# Get import job stats
stats = await db.get_stats()
@ -4884,14 +4883,14 @@ async def sandbox_status():
imports = [j for j in import_jobs if j.get("mode") == "import"]
return {
"sandbox_mode": True,
"import_mode": True,
"media_count": stats.get("media", 0),
"pages_count": stats.get("pages", 0),
"recent_imports": len(imports)
}
SANDBOX_CSS = """
IMPORT_CSS = """
.container { max-width: 800px; margin: 0 auto; }
h1 { color: #ff6b6b; margin-bottom: 5px; }
.subtitle { color: #666; margin-bottom: 30px; }
@ -4939,9 +4938,9 @@ SANDBOX_CSS = """
.error { color: #f66; }
"""
SANDBOX_CONTENT = """
IMPORT_CONTENT = """
<div class="container">
<h1>🧪 Sandbox Mode</h1>
<h1>🧪 Import Mode</h1>
<p class="subtitle">Upload a neopig archive (.tar.gz or .run) to explore it</p>
<div class="upload-zone" id="upload-zone" onclick="document.getElementById('file-input').click()">
@ -4996,7 +4995,7 @@ SANDBOX_CONTENT = """
try {
// Step 1: Initialize resumable upload
const initRes = await fetch(`/api/sandbox/upload/init?filename=${encodeURIComponent(file.name)}&size=${file.size}`, {
const initRes = await fetch(`/api/import/upload/init?filename=${encodeURIComponent(file.name)}&size=${file.size}`, {
method: 'POST'
});
if (!initRes.ok) {
@ -5024,7 +5023,7 @@ SANDBOX_CONTENT = """
const chunk = file.slice(start, end);
try {
const res = await fetch(`/api/sandbox/upload/${uploadId}`, {
const res = await fetch(`/api/import/upload/${uploadId}`, {
method: 'PATCH',
headers: {
'Content-Type': 'application/octet-stream',
@ -5076,7 +5075,7 @@ SANDBOX_CONTENT = """
// Step 3: Complete upload
try {
const completeRes = await fetch(`/api/sandbox/upload/${uploadId}/complete`, { method: 'POST' });
const completeRes = await fetch(`/api/import/upload/${uploadId}/complete`, { method: 'POST' });
if (!completeRes.ok) {
const err = await completeRes.json();
throw new Error(err.detail || 'Failed to complete upload');
@ -5091,7 +5090,7 @@ SANDBOX_CONTENT = """
}
async function getUploadStatus(uploadId) {
const res = await fetch(`/api/sandbox/upload/${uploadId}/status`);
const res = await fetch(`/api/import/upload/${uploadId}/status`);
if (!res.ok) throw new Error('Failed to get upload status');
return await res.json();
}
@ -5106,7 +5105,7 @@ SANDBOX_CONTENT = """
async function loadStatus() {
try {
const res = await fetch('/api/sandbox/status');
const res = await fetch('/api/import/status');
const data = await res.json();
const content = document.getElementById('status-content');
@ -5126,13 +5125,13 @@ SANDBOX_CONTENT = """
"""
@app.get("/sandbox", response_class=HTMLResponse)
async def sandbox_page(lang: str = Cookie(None), accept_language: str = Header(None)):
"""Sandbox page for uploading archives."""
if not SANDBOX_MODE:
raise HTTPException(status_code=403, detail="Sandbox mode not enabled (NEOPIG_SANDBOX=1)")
@app.get("/import", response_class=HTMLResponse)
async def import_page(lang: str = Cookie(None), accept_language: str = Header(None)):
"""Import page for uploading archives."""
if not IMPORT_MODE:
raise HTTPException(status_code=403, detail="Import mode not enabled (NEOPIG_IMPORT=1)")
language = get_lang(lang, accept_language)
html = layout("Sandbox", SANDBOX_CONTENT, extra_css=SANDBOX_CSS)
html = layout("Import", IMPORT_CONTENT, extra_css=IMPORT_CSS)
return inject_i18n(html, language)