diff --git a/serp.py b/serp.py index ed0d124..fc4927a 100644 --- a/serp.py +++ b/serp.py @@ -583,7 +583,6 @@ NAV_HTML = '''
{{live}} {{random}} {{crawl}} - {{phantom}} {{about}}
''' @@ -592,10 +591,10 @@ def inject_i18n(html: str, lang: str) -> str: trans = TRANSLATIONS.get(lang, TRANSLATIONS["en"]) # Add lang attribute and inject JS translations html = html.replace('', f'') - # Replace nav placeholder with actual nav (add sandbox link if enabled) + # Replace nav placeholder with actual nav (add import link if enabled) nav = NAV_HTML - if SANDBOX_MODE: - nav = nav.replace('', ' 🧪 Sandbox\n') + if IMPORT_MODE: + nav = nav.replace('', ' 🧪 Import\n') html = html.replace('', nav) # Build language selector options lang_options = ''.join(f'' @@ -638,7 +637,7 @@ DB_PATH = "data/neopig.db" VAULT_PATH = Path("data/vault") LOGS_PATH = Path("data/logs") CRAWL_DISABLED = os.environ.get("NEOPIG_DISABLE_CRAWL", "").lower() in ("1", "true", "yes") -SANDBOX_MODE = os.environ.get("NEOPIG_SANDBOX", "").lower() in ("1", "true", "yes") +IMPORT_MODE = os.environ.get("NEOPIG_IMPORT", "").lower() in ("1", "true", "yes") # Global database instance db: Database = None @@ -1210,8 +1209,8 @@ async def startup_event(): global db if CRAWL_DISABLED: logger.info("Crawl disabled via NEOPIG_DISABLE_CRAWL") - if SANDBOX_MODE: - logger.info("Sandbox mode enabled via NEOPIG_SANDBOX - archive uploads allowed") + if IMPORT_MODE: + logger.info("Import mode enabled via NEOPIG_IMPORT - archive uploads allowed") if TAR_PATH: # Tarball mode: use full Database for neopig.db, legacy ArchiveDB for archive.db if DB_PATH.endswith('neopig.db'): @@ -4483,10 +4482,10 @@ async def serve_media(md5_hash: str, download: bool = False): # ============================================================================ -# Sandbox Mode - Upload and serve archives +# Import Mode - Upload and serve archives # ============================================================================ -SANDBOX_UPLOAD_DIR = Path(tempfile.gettempdir()) / "neopig_sandbox" +IMPORT_UPLOAD_DIR = Path(tempfile.gettempdir()) / "neopig_import" # Resumable upload tracking: upload_id -> {filename, total_size, created_at} PENDING_UPLOADS: Dict[str, dict] = {} @@ -4497,24 +4496,24 @@ def generate_upload_id() -> str: return f"upload-{secrets.token_hex(8)}" -@app.post("/api/sandbox/upload/init") +@app.post("/api/import/upload/init") async def init_resumable_upload(filename: str = Query(...), size: int = Query(...)): """Initialize a resumable upload session. Returns upload_id that client uses for subsequent chunk uploads. - Client can resume from any disconnect by checking /api/sandbox/upload/{upload_id}/status + Client can resume from any disconnect by checking /api/import/upload/{upload_id}/status """ - if not SANDBOX_MODE: - raise HTTPException(status_code=403, detail="Sandbox mode not enabled") + if not IMPORT_MODE: + raise HTTPException(status_code=403, detail="Import mode not enabled") if not filename.endswith(('.tar.gz', '.tgz', '.run')): raise HTTPException(status_code=400, detail="File must be .tar.gz, .tgz, or .run") upload_id = generate_upload_id() - SANDBOX_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) + IMPORT_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) # Create empty file for this upload - upload_path = SANDBOX_UPLOAD_DIR / f"{upload_id}.part" + upload_path = IMPORT_UPLOAD_DIR / f"{upload_id}.part" upload_path.touch() PENDING_UPLOADS[upload_id] = { @@ -4524,7 +4523,7 @@ async def init_resumable_upload(filename: str = Query(...), size: int = Query(.. "path": str(upload_path) } - logger.info(f"Sandbox: initialized resumable upload {upload_id} for {filename} ({size / 1024 / 1024:.1f} MB)") + logger.info(f"Import: initialized resumable upload {upload_id} for {filename} ({size / 1024 / 1024:.1f} MB)") return { "upload_id": upload_id, @@ -4534,11 +4533,11 @@ async def init_resumable_upload(filename: str = Query(...), size: int = Query(.. } -@app.get("/api/sandbox/upload/{upload_id}/status") +@app.get("/api/import/upload/{upload_id}/status") async def get_upload_status(upload_id: str): """Get status of a resumable upload. Use this to resume after disconnect.""" - if not SANDBOX_MODE: - raise HTTPException(status_code=403, detail="Sandbox mode not enabled") + if not IMPORT_MODE: + raise HTTPException(status_code=403, detail="Import mode not enabled") if upload_id not in PENDING_UPLOADS: raise HTTPException(status_code=404, detail="Upload not found or expired") @@ -4560,7 +4559,7 @@ async def get_upload_status(upload_id: str): } -@app.patch("/api/sandbox/upload/{upload_id}") +@app.patch("/api/import/upload/{upload_id}") async def upload_chunk( upload_id: str, request: Request, @@ -4573,8 +4572,8 @@ async def upload_chunk( Or use X-Upload-Offset header for simpler resumption. """ - if not SANDBOX_MODE: - raise HTTPException(status_code=403, detail="Sandbox mode not enabled") + if not IMPORT_MODE: + raise HTTPException(status_code=403, detail="Import mode not enabled") if upload_id not in PENDING_UPLOADS: raise HTTPException(status_code=404, detail="Upload not found or expired") @@ -4614,7 +4613,7 @@ async def upload_chunk( f.write(chunk_data) new_size = upload_path.stat().st_size - logger.info(f"Sandbox: {upload_id} received chunk {chunk_size} bytes, total {new_size}/{info['total_size']}") + logger.info(f"Import: {upload_id} received chunk {chunk_size} bytes, total {new_size}/{info['total_size']}") return { "upload_id": upload_id, @@ -4624,11 +4623,11 @@ async def upload_chunk( } -@app.post("/api/sandbox/upload/{upload_id}/complete") +@app.post("/api/import/upload/{upload_id}/complete") async def complete_resumable_upload(upload_id: str): """Finalize upload and start import job.""" - if not SANDBOX_MODE: - raise HTTPException(status_code=403, detail="Sandbox mode not enabled") + if not IMPORT_MODE: + raise HTTPException(status_code=403, detail="Import mode not enabled") if upload_id not in PENDING_UPLOADS: raise HTTPException(status_code=404, detail="Upload not found or expired") @@ -4647,18 +4646,18 @@ async def complete_resumable_upload(upload_id: str): ) # Rename to final filename - final_path = SANDBOX_UPLOAD_DIR / info["filename"] + final_path = IMPORT_UPLOAD_DIR / info["filename"] upload_path.rename(final_path) # Remove from pending del PENDING_UPLOADS[upload_id] - logger.info(f"Sandbox: {upload_id} completed, saved as {info['filename']}") + logger.info(f"Import: {upload_id} completed, saved as {info['filename']}") # Create import job job_id = await db.create_crawl_job( target_uri=f"import://{info['filename']}", - keywords=["sandbox", "import"], + keywords=["import"], mode="import" ) @@ -4669,9 +4668,9 @@ async def complete_resumable_upload(upload_id: str): stats = await import_archive_to_db(final_path, job_id) await db.complete_crawl_job(job_id, stats) final_path.unlink(missing_ok=True) - logger.info(f"Sandbox import complete: {stats}") + logger.info(f"Import complete: {stats}") except Exception as e: - logger.error(f"Sandbox import failed: {e}") + logger.error(f"Import failed: {e}") await db.fail_crawl_job(job_id, str(e)) task = asyncio.create_task(run_import()) @@ -4686,19 +4685,19 @@ async def complete_resumable_upload(upload_id: str): } -@app.post("/api/sandbox/upload") +@app.post("/api/import/upload") async def upload_archive(file: UploadFile = File(...)): """Upload a tar.gz archive and import into local database as a job.""" - if not SANDBOX_MODE: - raise HTTPException(status_code=403, detail="Sandbox mode not enabled (NEOPIG_SANDBOX=1)") + if not IMPORT_MODE: + raise HTTPException(status_code=403, detail="Import mode not enabled (NEOPIG_IMPORT=1)") if not file.filename.endswith(('.tar.gz', '.tgz', '.run')): raise HTTPException(status_code=400, detail="File must be .tar.gz, .tgz, or .run") # Save uploaded file - SANDBOX_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) - upload_path = SANDBOX_UPLOAD_DIR / file.filename - logger.info(f"Sandbox: receiving upload {file.filename}") + IMPORT_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) + upload_path = IMPORT_UPLOAD_DIR / file.filename + logger.info(f"Import: receiving upload {file.filename}") try: with open(upload_path, 'wb') as f: @@ -4708,12 +4707,12 @@ async def upload_archive(file: UploadFile = File(...)): raise HTTPException(status_code=400, detail=f"Upload failed: {e}") file_size = upload_path.stat().st_size - logger.info(f"Sandbox: saved {file.filename} ({file_size / 1024 / 1024:.1f} MB)") + logger.info(f"Import: saved {file.filename} ({file_size / 1024 / 1024:.1f} MB)") # Create import job job_id = await db.create_crawl_job( target_uri=f"import://{file.filename}", - keywords=["sandbox", "import"], + keywords=["import"], mode="import" ) @@ -4724,9 +4723,9 @@ async def upload_archive(file: UploadFile = File(...)): stats = await import_archive_to_db(upload_path, job_id) await db.complete_crawl_job(job_id, stats) upload_path.unlink(missing_ok=True) - logger.info(f"Sandbox import complete: {stats}") + logger.info(f"Import complete: {stats}") except Exception as e: - logger.error(f"Sandbox import failed: {e}") + logger.error(f"Import failed: {e}") await db.fail_crawl_job(job_id, str(e)) task = asyncio.create_task(run_import()) @@ -4769,7 +4768,7 @@ async def import_archive_to_db(archive_path: Path, job_id: int) -> dict: # Extract neopig.db to temp db_member = f"{archive_root}/neopig.db" - temp_db = SANDBOX_UPLOAD_DIR / f"import_{job_id}.db" + temp_db = IMPORT_UPLOAD_DIR / f"import_{job_id}.db" for m in members: if m.name == db_member: @@ -4872,11 +4871,11 @@ async def import_archive_to_db(archive_path: Path, job_id: int) -> dict: return stats -@app.get("/api/sandbox/status") -async def sandbox_status(): - """Get current sandbox status.""" - if not SANDBOX_MODE: - raise HTTPException(status_code=403, detail="Sandbox mode not enabled") +@app.get("/api/import/status") +async def import_status(): + """Get current import status.""" + if not IMPORT_MODE: + raise HTTPException(status_code=403, detail="Import mode not enabled") # Get import job stats stats = await db.get_stats() @@ -4884,14 +4883,14 @@ async def sandbox_status(): imports = [j for j in import_jobs if j.get("mode") == "import"] return { - "sandbox_mode": True, + "import_mode": True, "media_count": stats.get("media", 0), "pages_count": stats.get("pages", 0), "recent_imports": len(imports) } -SANDBOX_CSS = """ +IMPORT_CSS = """ .container { max-width: 800px; margin: 0 auto; } h1 { color: #ff6b6b; margin-bottom: 5px; } .subtitle { color: #666; margin-bottom: 30px; } @@ -4939,9 +4938,9 @@ SANDBOX_CSS = """ .error { color: #f66; } """ -SANDBOX_CONTENT = """ +IMPORT_CONTENT = """
-

🧪 Sandbox Mode

+

🧪 Import Mode

Upload a neopig archive (.tar.gz or .run) to explore it

@@ -4996,7 +4995,7 @@ SANDBOX_CONTENT = """ try { // Step 1: Initialize resumable upload - const initRes = await fetch(`/api/sandbox/upload/init?filename=${encodeURIComponent(file.name)}&size=${file.size}`, { + const initRes = await fetch(`/api/import/upload/init?filename=${encodeURIComponent(file.name)}&size=${file.size}`, { method: 'POST' }); if (!initRes.ok) { @@ -5024,7 +5023,7 @@ SANDBOX_CONTENT = """ const chunk = file.slice(start, end); try { - const res = await fetch(`/api/sandbox/upload/${uploadId}`, { + const res = await fetch(`/api/import/upload/${uploadId}`, { method: 'PATCH', headers: { 'Content-Type': 'application/octet-stream', @@ -5076,7 +5075,7 @@ SANDBOX_CONTENT = """ // Step 3: Complete upload try { - const completeRes = await fetch(`/api/sandbox/upload/${uploadId}/complete`, { method: 'POST' }); + const completeRes = await fetch(`/api/import/upload/${uploadId}/complete`, { method: 'POST' }); if (!completeRes.ok) { const err = await completeRes.json(); throw new Error(err.detail || 'Failed to complete upload'); @@ -5091,7 +5090,7 @@ SANDBOX_CONTENT = """ } async function getUploadStatus(uploadId) { - const res = await fetch(`/api/sandbox/upload/${uploadId}/status`); + const res = await fetch(`/api/import/upload/${uploadId}/status`); if (!res.ok) throw new Error('Failed to get upload status'); return await res.json(); } @@ -5106,7 +5105,7 @@ SANDBOX_CONTENT = """ async function loadStatus() { try { - const res = await fetch('/api/sandbox/status'); + const res = await fetch('/api/import/status'); const data = await res.json(); const content = document.getElementById('status-content'); @@ -5126,13 +5125,13 @@ SANDBOX_CONTENT = """ """ -@app.get("/sandbox", response_class=HTMLResponse) -async def sandbox_page(lang: str = Cookie(None), accept_language: str = Header(None)): - """Sandbox page for uploading archives.""" - if not SANDBOX_MODE: - raise HTTPException(status_code=403, detail="Sandbox mode not enabled (NEOPIG_SANDBOX=1)") +@app.get("/import", response_class=HTMLResponse) +async def import_page(lang: str = Cookie(None), accept_language: str = Header(None)): + """Import page for uploading archives.""" + if not IMPORT_MODE: + raise HTTPException(status_code=403, detail="Import mode not enabled (NEOPIG_IMPORT=1)") language = get_lang(lang, accept_language) - html = layout("Sandbox", SANDBOX_CONTENT, extra_css=SANDBOX_CSS) + html = layout("Import", IMPORT_CONTENT, extra_css=IMPORT_CSS) return inject_i18n(html, language)