Russell Ballestrini
7883ae0eba
Phase 1 of module restructuring as outlined in docs/REFACTOR.md: New neopig/ package modules: - live.py: Live media queue (get_live_queue, emit_live_media) - state.py: AppendOnlyStateLog, state file helpers - html_utils.py: trim_html_wrapper, extract_meta_from_html - logging.py: TqdmLoggingHandler, job logging functions - backfill/: markdown and screenshot backfill operations Package features: - Lazy import of NeoPig/main from neopig.py via __getattr__ - Full backwards compatibility with existing imports - 42 new unit tests for extracted modules Total: 458 tests passing
75 lines
2.1 KiB
Python
75 lines
2.1 KiB
Python
"""
|
|
Logging utilities for neopig.
|
|
|
|
Provides tqdm-safe logging handlers and job-specific log capture.
|
|
"""
|
|
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import Dict
|
|
|
|
from tqdm import tqdm
|
|
|
|
# Per-job log handlers (job_id -> handler)
|
|
LOGS_PATH = Path("data/logs")
|
|
JOB_LOG_HANDLERS: Dict[int, logging.FileHandler] = {}
|
|
|
|
|
|
class TqdmLoggingHandler(logging.Handler):
|
|
"""Logging handler that writes through tqdm to avoid progress bar corruption."""
|
|
|
|
def emit(self, record):
|
|
try:
|
|
msg = self.format(record)
|
|
tqdm.write(msg)
|
|
except Exception:
|
|
self.handleError(record)
|
|
|
|
|
|
def setup_logging(level=logging.INFO):
|
|
"""Setup logging to work with tqdm progress bars."""
|
|
handler = TqdmLoggingHandler()
|
|
handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
|
|
root = logging.getLogger()
|
|
root.handlers = [handler]
|
|
root.setLevel(level)
|
|
|
|
|
|
def start_job_logging(job_id: int) -> None:
|
|
"""Start capturing logs for a crawl job to file."""
|
|
LOGS_PATH.mkdir(parents=True, exist_ok=True)
|
|
log_file = LOGS_PATH / f"{job_id}.log"
|
|
handler = logging.FileHandler(log_file, mode='w', encoding='utf-8')
|
|
handler.setLevel(logging.INFO)
|
|
handler.setFormatter(logging.Formatter('%(asctime)s %(name)s %(levelname)s: %(message)s', datefmt='%H:%M:%S'))
|
|
logging.getLogger().addHandler(handler)
|
|
JOB_LOG_HANDLERS[job_id] = handler
|
|
|
|
|
|
def stop_job_logging(job_id: int) -> None:
|
|
"""Stop capturing logs for a crawl job."""
|
|
handler = JOB_LOG_HANDLERS.pop(job_id, None)
|
|
if handler:
|
|
handler.close()
|
|
logging.getLogger().removeHandler(handler)
|
|
|
|
|
|
def get_job_logs(job_id: int, tail: int = 0) -> str:
|
|
"""Read logs for a crawl job.
|
|
|
|
Args:
|
|
job_id: Job ID to get logs for
|
|
tail: If > 0, return only last N lines
|
|
|
|
Returns:
|
|
Log contents as string
|
|
"""
|
|
log_file = LOGS_PATH / f"{job_id}.log"
|
|
if not log_file.exists():
|
|
return ""
|
|
|
|
content = log_file.read_text(encoding='utf-8')
|
|
if tail > 0:
|
|
lines = content.splitlines()
|
|
return '\n'.join(lines[-tail:])
|
|
return content
|