Russell Ballestrini
7883ae0eba
Phase 1 of module restructuring as outlined in docs/REFACTOR.md: New neopig/ package modules: - live.py: Live media queue (get_live_queue, emit_live_media) - state.py: AppendOnlyStateLog, state file helpers - html_utils.py: trim_html_wrapper, extract_meta_from_html - logging.py: TqdmLoggingHandler, job logging functions - backfill/: markdown and screenshot backfill operations Package features: - Lazy import of NeoPig/main from neopig.py via __getattr__ - Full backwards compatibility with existing imports - 42 new unit tests for extracted modules Total: 458 tests passing
65 lines
1.7 KiB
Python
65 lines
1.7 KiB
Python
"""
|
|
neopig - Neo Python Image Grabber package.
|
|
|
|
Full-domain async media crawler with content-addressed deduplication.
|
|
"""
|
|
|
|
from .live import get_live_queue, emit_live_media
|
|
from .state import (
|
|
AppendOnlyStateLog,
|
|
get_state_log_path,
|
|
get_state_file_path,
|
|
rotate_state_file,
|
|
)
|
|
from .html_utils import trim_html_wrapper, extract_meta_from_html
|
|
from .logging import (
|
|
TqdmLoggingHandler,
|
|
setup_logging,
|
|
start_job_logging,
|
|
stop_job_logging,
|
|
get_job_logs,
|
|
)
|
|
from .backfill import backfill_markdown, backfill_screenshots
|
|
|
|
# Lazy imports for items still in neopig.py (to avoid circular imports)
|
|
_lazy_imports = {'NeoPig', 'main'}
|
|
|
|
def __getattr__(name):
|
|
"""Lazy import NeoPig and main from neopig.py to avoid circular imports."""
|
|
if name in _lazy_imports:
|
|
import importlib.util
|
|
import os
|
|
spec = importlib.util.spec_from_file_location(
|
|
"neopig_main",
|
|
os.path.join(os.path.dirname(os.path.dirname(__file__)), "neopig.py")
|
|
)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return getattr(module, name)
|
|
raise AttributeError(f"module 'neopig' has no attribute {name!r}")
|
|
|
|
__all__ = [
|
|
# Live queue
|
|
'get_live_queue',
|
|
'emit_live_media',
|
|
# State management
|
|
'AppendOnlyStateLog',
|
|
'get_state_log_path',
|
|
'get_state_file_path',
|
|
'rotate_state_file',
|
|
# HTML utilities
|
|
'trim_html_wrapper',
|
|
'extract_meta_from_html',
|
|
# Logging
|
|
'TqdmLoggingHandler',
|
|
'setup_logging',
|
|
'start_job_logging',
|
|
'stop_job_logging',
|
|
'get_job_logs',
|
|
# Backfill
|
|
'backfill_markdown',
|
|
'backfill_screenshots',
|
|
# Main (lazy imports)
|
|
'NeoPig',
|
|
'main',
|
|
]
|