diff --git a/aborist/sources/crawler/__init__.py b/aborist/sources/crawler/__init__.py index 8928083..fbcd665 100644 --- a/aborist/sources/crawler/__init__.py +++ b/aborist/sources/crawler/__init__.py @@ -39,6 +39,17 @@ honor ``robots.txt`` — that's what the underlying parser already does when given a URL. """ +# Suppress bs4's "you're parsing XML with the HTML parser" warning when the +# crawler hits a sitemap.xml / RSS feed via the generic page handler. Benign +# (parsing still works) but spams stderr during a deep BFS. Filtering here — +# at the aborist subpackage boundary — keeps the verbatim lift untouched. +import warnings as _warnings +try: + from bs4 import XMLParsedAsHTMLWarning as _XMLParsedAsHTMLWarning + _warnings.filterwarnings("ignore", category=_XMLParsedAsHTMLWarning) +except ImportError: # pragma: no cover + pass + from aborist.sources.crawler.async_web_fetcher import ( AsyncWebFetcher, CrawlMode,