"""Crawler — ethical async web fetching with robots.txt compliance.

Verbatim lift from ``~/git/agents.ai.unturf.com/core/`` (rev as of 2026-04-28):

    core/async_web_fetcher.py  ->  aborist/sources/crawler/async_web_fetcher.py
    core/web_fetch.py          ->  aborist/sources/crawler/web_fetch.py

Two source-side adaptations during the lift:

1. Chat-bot fetch triggers (``has_fresh_fetch_trigger`` /
   ``has_web_fetch_trigger`` from ``core.keywords``) were dropped. Aborist
   has no chat surface — fetch intent is detected at the application
   layer, not from message content.

2. ``web_cache_manager.py`` (SQLAlchemy-backed page cache) was NOT
   lifted. Aborist has its own content-addressed store; cache through
   that path instead of carrying SQLAlchemy as a dep.

**Off by default.** Heavy dependencies (aiohttp, beautifulsoup4, lxml,
html5lib, html2text, miniuri, feedparser, Pillow, cairosvg, pypdf) ship
as the ``[crawler]`` extras. Importing this module without the extras
raises ``ImportError`` with the install hint. The default test suite
(``make test``) does not exercise the crawler; ``make test-crawler``
runs the lifted tests with the extras installed.

Public surface — same as the agents repo:

    from aborist.sources.crawler import AsyncWebFetcher, CrawlMode
    from aborist.sources.crawler import (
        URIContentType,
        extract_all_uris_from_content,
        extract_url_from_content,
        build_sources_footer,
        fetch_and_cache,
    )

Adopters should pass ``user_agent`` reflecting their deployment and
honor ``robots.txt`` — that's what the underlying parser already does
when given a URL.
"""

# Suppress bs4's "you're parsing XML with the HTML parser" warning when the
# crawler hits a sitemap.xml / RSS feed via the generic page handler. Benign
# (parsing still works) but spams stderr during a deep BFS. Filtering here —
# at the aborist subpackage boundary — keeps the verbatim lift untouched.
import warnings as _warnings
try:
    from bs4 import XMLParsedAsHTMLWarning as _XMLParsedAsHTMLWarning
    _warnings.filterwarnings("ignore", category=_XMLParsedAsHTMLWarning)
except ImportError:  # pragma: no cover
    pass

from aborist.sources.crawler.async_web_fetcher import (
    AsyncWebFetcher,
    CrawlMode,
    FeedItem,
    MediaItem,
    MediaMetadata,
    discover_feeds,
    extract_media_from_html,
    fetch_robots_txt,
    parse_atom,
    parse_feed,
    parse_rss,
    parse_sitemap,
)
from aborist.sources.crawler.web_fetch import (
    URIContentType,
    build_sources_footer,
    detect_uri_content_type,
    extract_all_uris_from_content,
    extract_domain,
    extract_url_from_content,
    fetch_and_cache,
    format_progress_message,
)

__all__ = [
    "AsyncWebFetcher",
    "CrawlMode",
    "FeedItem",
    "MediaItem",
    "MediaMetadata",
    "URIContentType",
    "build_sources_footer",
    "detect_uri_content_type",
    "discover_feeds",
    "extract_all_uris_from_content",
    "extract_domain",
    "extract_media_from_html",
    "extract_url_from_content",
    "fetch_and_cache",
    "fetch_robots_txt",
    "format_progress_message",
    "parse_atom",
    "parse_feed",
    "parse_rss",
    "parse_sitemap",
]
