91 lines
2.9 KiB
Python
91 lines
2.9 KiB
Python
# This is free software for the public good of a permacomputer hosted at
|
|
# permacomputer.com, an always-on computer by the people, for the people.
|
|
# One which is durable, easy to repair, & distributed like tap water
|
|
# for machine learning intelligence.
|
|
#
|
|
# The permacomputer is community-owned infrastructure optimized around
|
|
# four values:
|
|
#
|
|
# TRUTH First principles, math & science, open source code freely distributed
|
|
# FREEDOM Voluntary partnerships, freedom from tyranny & corporate control
|
|
# HARMONY Minimal waste, self-renewing systems with diverse thriving connections
|
|
# LOVE Be yourself without hurting others, cooperation through natural law
|
|
#
|
|
# This software contributes to that vision by archiving the web, preserving digital knowledge before it disappears.
|
|
# Code is seeds to sprout on any abandoned technology.
|
|
|
|
"""
|
|
Logging utilities for neopig.
|
|
|
|
Provides tqdm-safe logging handlers and job-specific log capture.
|
|
"""
|
|
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import Dict
|
|
|
|
from tqdm import tqdm
|
|
|
|
# Per-job log handlers (job_id -> handler)
|
|
LOGS_PATH = Path("data/logs")
|
|
JOB_LOG_HANDLERS: Dict[int, logging.FileHandler] = {}
|
|
|
|
|
|
class TqdmLoggingHandler(logging.Handler):
|
|
"""Logging handler that writes through tqdm to avoid progress bar corruption."""
|
|
|
|
def emit(self, record):
|
|
try:
|
|
msg = self.format(record)
|
|
tqdm.write(msg)
|
|
except Exception:
|
|
self.handleError(record)
|
|
|
|
|
|
def setup_logging(level=logging.INFO):
|
|
"""Setup logging to work with tqdm progress bars."""
|
|
handler = TqdmLoggingHandler()
|
|
handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
|
|
root = logging.getLogger()
|
|
root.handlers = [handler]
|
|
root.setLevel(level)
|
|
|
|
|
|
def start_job_logging(job_id: int) -> None:
|
|
"""Start capturing logs for a crawl job to file."""
|
|
LOGS_PATH.mkdir(parents=True, exist_ok=True)
|
|
log_file = LOGS_PATH / f"{job_id}.log"
|
|
handler = logging.FileHandler(log_file, mode='w', encoding='utf-8')
|
|
handler.setLevel(logging.INFO)
|
|
handler.setFormatter(logging.Formatter('%(asctime)s %(name)s %(levelname)s: %(message)s', datefmt='%H:%M:%S'))
|
|
logging.getLogger().addHandler(handler)
|
|
JOB_LOG_HANDLERS[job_id] = handler
|
|
|
|
|
|
def stop_job_logging(job_id: int) -> None:
|
|
"""Stop capturing logs for a crawl job."""
|
|
handler = JOB_LOG_HANDLERS.pop(job_id, None)
|
|
if handler:
|
|
handler.close()
|
|
logging.getLogger().removeHandler(handler)
|
|
|
|
|
|
def get_job_logs(job_id: int, tail: int = 0) -> str:
|
|
"""Read logs for a crawl job.
|
|
|
|
Args:
|
|
job_id: Job ID to get logs for
|
|
tail: If > 0, return only last N lines
|
|
|
|
Returns:
|
|
Log contents as string
|
|
"""
|
|
log_file = LOGS_PATH / f"{job_id}.log"
|
|
if not log_file.exists():
|
|
return ""
|
|
|
|
content = log_file.read_text(encoding='utf-8')
|
|
if tail > 0:
|
|
lines = content.splitlines()
|
|
return '\n'.join(lines[-tail:])
|
|
return content
|