diff --git a/aborist/sources/crawler/bridge.py b/aborist/sources/crawler/bridge.py
index 8299b47..68ffe73 100644
--- a/aborist/sources/crawler/bridge.py
+++ b/aborist/sources/crawler/bridge.py
@@ -48,6 +48,63 @@ def _normalize(url: str) -> str:
return urllib.parse.urlunparse(cleaned)
+# Feeds, sitemaps, and similar XML discovery artifacts are crawl
+# infrastructure — they tell you which URLs exist on a site. They are
+# NOT knowledge content, so they must not enter the Merkle tree as
+# documents. We exclude them in two passes: (1) URL pattern match on
+# the path so we skip even before the response arrives, (2) content-
+# type check on the response so feeds served at unconventional paths
+# still get rejected. xhtml is preserved because it IS HTML.
+_FEED_URL_SUFFIXES = (
+ ".atom",
+ ".atom.xml",
+ ".rss",
+ ".rss.xml",
+ ".rdf",
+ # Common feed filename patterns regardless of where they live.
+ "/feed.xml",
+ "/atom.xml",
+ "/rss.xml",
+ "/rss2.xml",
+ "wp-rss2.xml", # WordPress legacy
+ "wp-rss.xml",
+ "wp-atom.xml",
+ "wp-rdf.xml",
+)
+_FEED_URL_SUBSTRINGS = (
+ "/feed/",
+ "/feeds/",
+ "/atom",
+ "/rss",
+ "/sitemap", # sitemap.xml, sitemap_index.xml, sitemaps/foo.xml
+)
+_FEED_CONTENT_TYPES = (
+ "application/atom+xml",
+ "application/rss+xml",
+ "application/rdf+xml",
+ "application/xml",
+ "text/xml",
+)
+
+
+def _looks_like_feed_url(url: str) -> bool:
+ """True if the URL path matches a known feed/sitemap pattern."""
+ path = urllib.parse.urlparse(url).path.lower()
+ if any(path.endswith(s) for s in _FEED_URL_SUFFIXES):
+ return True
+ if any(s in path for s in _FEED_URL_SUBSTRINGS):
+ return True
+ if path.endswith("/sitemap.xml") or path.endswith("sitemapindex.xml"):
+ return True
+ return False
+
+
+def _looks_like_feed_response(content_type: str) -> bool:
+ """True if a response Content-Type is a feed/sitemap rather than HTML."""
+ ct = content_type.lower().split(";", 1)[0].strip()
+ return ct in _FEED_CONTENT_TYPES
+
+
async def _crawl_seed_async(
seed_url: str,
*,
@@ -174,13 +231,20 @@ class _CrawledHtmlSource:
follow_redirects=True,
) as client:
for url in self.urls:
+ # Cheap pre-flight: skip feed/sitemap URLs without a fetch.
+ if _looks_like_feed_url(url):
+ continue
try:
resp = client.get(url)
resp.raise_for_status()
except httpx.HTTPError:
continue
ctype = resp.headers.get("content-type", "").lower()
- if "html" not in ctype and "xml" not in ctype:
+ # Defense in depth: a feed served at a non-feed path still
+ # gets rejected by Content-Type.
+ if _looks_like_feed_response(ctype):
+ continue
+ if "html" not in ctype:
continue
doc = parse_html(str(resp.url), resp.text, self.source_type)
if doc is None:
diff --git a/tests/crawler/test_bridge.py b/tests/crawler/test_bridge.py
index 2483fb7..5f1a26d 100644
--- a/tests/crawler/test_bridge.py
+++ b/tests/crawler/test_bridge.py
@@ -334,6 +334,88 @@ def test_ingest_crawled_skips_non_html(http_meta_db):
assert result["inserted"] == 0
+@pytest.mark.parametrize(
+ "feed_url",
+ [
+ "https://x.com/feeds/all.atom.xml",
+ "https://x.com/feed/",
+ "https://x.com/atom",
+ "https://x.com/rss",
+ "https://x.com/sitemap.xml",
+ "https://x.com/sitemaps/posts.xml",
+ "https://x.com/wp-rss2.xml",
+ "https://x.com/index.atom",
+ ],
+)
+def test_ingest_crawled_skips_feed_urls_by_path(http_meta_db, feed_url):
+ """Feed/sitemap URLs are crawl infrastructure, not knowledge. Skip
+ even before the fetch — pin so a future "but XML can be useful"
+ rewrite can't silently re-introduce feed pollution."""
+ # Response stub never gets used because the path filter is pre-fetch,
+ # but we provide one in case the filter regresses.
+ responses = {feed_url: {"status": 200, "headers": {"content-type": "text/html"}}}
+ with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
+ conn = connect(http_meta_db)
+ try:
+ result = ingest_crawled(conn, [feed_url])
+ finally:
+ conn.close()
+ assert result["http_meta_written"] == 0
+ assert result["inserted"] == 0
+
+
+@pytest.mark.parametrize(
+ "ct",
+ [
+ "application/atom+xml",
+ "application/rss+xml",
+ "application/rdf+xml",
+ "application/xml",
+ "text/xml",
+ "application/atom+xml; charset=utf-8",
+ ],
+)
+def test_ingest_crawled_skips_feed_content_types(http_meta_db, ct):
+ """A feed served at a non-feed path still gets rejected by
+ Content-Type. xhtml is allowed (treated as HTML elsewhere)."""
+ url = "https://x.com/innocent-looking-path" # no feed-pattern in URL
+ responses = {
+ url: {
+ "status": 200,
+ "headers": {"content-type": ct},
+ "body": "",
+ }
+ }
+ with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
+ conn = connect(http_meta_db)
+ try:
+ result = ingest_crawled(conn, [url])
+ finally:
+ conn.close()
+ assert result["http_meta_written"] == 0
+ assert result["inserted"] == 0
+
+
+def test_ingest_crawled_keeps_xhtml(http_meta_db):
+ """xhtml IS HTML — keep it, even though the content type ends in +xml."""
+ url = "https://x.com/page"
+ responses = {
+ url: {
+ "status": 200,
+ "headers": {"content-type": "application/xhtml+xml"},
+ "body": "T
Real content here.",
+ }
+ }
+ with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
+ conn = connect(http_meta_db)
+ try:
+ result = ingest_crawled(conn, [url])
+ finally:
+ conn.close()
+ assert result["inserted"] == 1
+ assert result["http_meta_written"] == 1
+
+
# ---------------------------------------------------------------------------
# recrawl_check: conditional HEAD classification
# ---------------------------------------------------------------------------