From f04417161c6f8010be3898e5da5593ae589f16fb Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 28 Apr 2026 21:07:27 -0400 Subject: [PATCH] crawler: filter bs4 XMLParsedAsHTMLWarning at subpackage boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A real BFS hits sitemap.xml / RSS feeds via the verbatim crawler's generic page handler. bs4 warns "you're using an HTML parser on XML" on each — benign (parsing still works) but spams stderr during a deep crawl, drowning the progress heartbeat we just wired in. Filter at aborist/sources/crawler/__init__.py so the lifted source stays untouched. The filter applies whenever any caller imports from this subpackage, including the CLI's `from ... bridge import ...` (which runs __init__.py first per Python import semantics). --- aborist/sources/crawler/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/aborist/sources/crawler/__init__.py b/aborist/sources/crawler/__init__.py index 8928083..fbcd665 100644 --- a/aborist/sources/crawler/__init__.py +++ b/aborist/sources/crawler/__init__.py @@ -39,6 +39,17 @@ honor ``robots.txt`` — that's what the underlying parser already does when given a URL. """ +# Suppress bs4's "you're parsing XML with the HTML parser" warning when the +# crawler hits a sitemap.xml / RSS feed via the generic page handler. Benign +# (parsing still works) but spams stderr during a deep BFS. Filtering here — +# at the aborist subpackage boundary — keeps the verbatim lift untouched. +import warnings as _warnings +try: + from bs4 import XMLParsedAsHTMLWarning as _XMLParsedAsHTMLWarning + _warnings.filterwarnings("ignore", category=_XMLParsedAsHTMLWarning) +except ImportError: # pragma: no cover + pass + from aborist.sources.crawler.async_web_fetcher import ( AsyncWebFetcher, CrawlMode,