Move job logging to neopig.py - always log for both CLI and web UI

neopig.py

@@ -47,6 +47,27 @@ from tqdm import tqdm
 
 logger = logging.getLogger(__name__)
 
+# Per-job log handlers
+LOGS_PATH = Path("data/logs")
+JOB_LOG_HANDLERS: Dict[int, logging.FileHandler] = {}
+
+def start_job_logging(job_id: int) -> None:
+    """Start capturing logs for a crawl job to file."""
+    LOGS_PATH.mkdir(parents=True, exist_ok=True)
+    log_file = LOGS_PATH / f"{job_id}.log"
+    handler = logging.FileHandler(log_file, mode='w', encoding='utf-8')
+    handler.setLevel(logging.INFO)
+    handler.setFormatter(logging.Formatter('%(asctime)s %(name)s %(levelname)s: %(message)s', datefmt='%H:%M:%S'))
+    logging.getLogger().addHandler(handler)
+    JOB_LOG_HANDLERS[job_id] = handler
+
+def stop_job_logging(job_id: int) -> None:
+    """Stop capturing logs for a crawl job."""
+    handler = JOB_LOG_HANDLERS.pop(job_id, None)
+    if handler:
+        handler.close()
+        logging.getLogger().removeHandler(handler)
+
 
 def get_state_file_path(domain: str) -> Path:
     """Get unified state file path for a domain.
@@ -485,6 +506,9 @@ class NeoPig:
                 mode=mode.value,
             )
 
+        # Always log to file for this job
+        start_job_logging(job_id)
+
         logger.info(f"Starting neopig crawl job {job_id}")
         logger.info(f"Target: {target_uri}")
         logger.info(f"Mode: {mode.value}")
@@ -662,6 +686,9 @@ class NeoPig:
         self._items_since_save = self._state_save_interval  # Force save
         self._save_state(target_uri)
 
+        # Stop logging to file
+        stop_job_logging(job_id)
+
         return self.stats
 
     async def _process_media_item(

serp.py

@@ -3630,7 +3630,6 @@ async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
 
         # Run crawl in background (closure captures job_id and target_uri)
         async def run_crawl(jid=job_id, uri=target_uri):
-            start_job_logging(jid)
             try:
                 from screenshot import ScreenshotConfig
                 screenshot_config = ScreenshotConfig(enabled=request.screenshots)
@@ -3687,8 +3686,6 @@ async def start_crawl(request: CrawlRequest, background_tasks: BackgroundTasks):
             except Exception as e:
                 logger.error(f"Crawl job {jid} failed: {e}")
                 await db.complete_crawl_job(jid, {"error": str(e), "status": "failed"})
-            finally:
-                stop_job_logging(jid)
 
         # Schedule async task on the event loop (not BackgroundTasks which runs in threadpool)
         task = asyncio.create_task(run_crawl())