81 lines
2.5 KiB
Python
81 lines
2.5 KiB
Python
# This is free software for the public good of a permacomputer hosted at
|
|
# permacomputer.com, an always-on computer by the people, for the people.
|
|
# One which is durable, easy to repair, & distributed like tap water
|
|
# for machine learning intelligence.
|
|
#
|
|
# The permacomputer is community-owned infrastructure optimized around
|
|
# four values:
|
|
#
|
|
# TRUTH First principles, math & science, open source code freely distributed
|
|
# FREEDOM Voluntary partnerships, freedom from tyranny & corporate control
|
|
# HARMONY Minimal waste, self-renewing systems with diverse thriving connections
|
|
# LOVE Be yourself without hurting others, cooperation through natural law
|
|
#
|
|
# This software contributes to that vision by archiving the web, preserving digital knowledge before it disappears.
|
|
# Code is seeds to sprout on any abandoned technology.
|
|
|
|
"""
|
|
neopig - Neo Python Image Grabber package.
|
|
|
|
Full-domain async media crawler with content-addressed deduplication.
|
|
"""
|
|
|
|
from .live import get_live_queue, emit_live_media
|
|
from .state import (
|
|
AppendOnlyStateLog,
|
|
get_state_log_path,
|
|
get_state_file_path,
|
|
rotate_state_file,
|
|
)
|
|
from .html_utils import trim_html_wrapper, extract_meta_from_html
|
|
from .logging import (
|
|
TqdmLoggingHandler,
|
|
setup_logging,
|
|
start_job_logging,
|
|
stop_job_logging,
|
|
get_job_logs,
|
|
)
|
|
from .backfill import backfill_markdown, backfill_screenshots
|
|
|
|
# Lazy imports for items still in neopig.py (to avoid circular imports)
|
|
_lazy_imports = {'NeoPig', 'main'}
|
|
|
|
def __getattr__(name):
|
|
"""Lazy import NeoPig and main from neopig.py to avoid circular imports."""
|
|
if name in _lazy_imports:
|
|
import importlib.util
|
|
import os
|
|
spec = importlib.util.spec_from_file_location(
|
|
"neopig_main",
|
|
os.path.join(os.path.dirname(os.path.dirname(__file__)), "neopig.py")
|
|
)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return getattr(module, name)
|
|
raise AttributeError(f"module 'neopig' has no attribute {name!r}")
|
|
|
|
__all__ = [
|
|
# Live queue
|
|
'get_live_queue',
|
|
'emit_live_media',
|
|
# State management
|
|
'AppendOnlyStateLog',
|
|
'get_state_log_path',
|
|
'get_state_file_path',
|
|
'rotate_state_file',
|
|
# HTML utilities
|
|
'trim_html_wrapper',
|
|
'extract_meta_from_html',
|
|
# Logging
|
|
'TqdmLoggingHandler',
|
|
'setup_logging',
|
|
'start_job_logging',
|
|
'stop_job_logging',
|
|
'get_job_logs',
|
|
# Backfill
|
|
'backfill_markdown',
|
|
'backfill_screenshots',
|
|
# Main (lazy imports)
|
|
'NeoPig',
|
|
'main',
|
|
]
|