From 43529328fd5c44ec1edd3e687ce7049d784f7b38 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 29 Apr 2026 05:55:28 -0400 Subject: [PATCH] crawler: skip feeds + sitemaps at ingest (they're discovery, not knowledge) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per fox: a query for "who is Russell Ballestrini" classified STRICT by grounding 6/6 quotes against `feeds/all.atom.xml` — a 230-chunk dump of post metadata. The verifier was technically correct (every quoted string IS in the feed) but the result was hollow: feeds list URLs, they don't carry knowledge. The ACTUAL bios live at the linked posts. Two-pass filter in _CrawledHtmlSource.iter_documents: 1. Pre-fetch URL-pattern check (_looks_like_feed_url): - suffix matches: .atom, .atom.xml, .rss, .rss.xml, .rdf, /feed.xml, /atom.xml, /rss.xml, /rss2.xml, wp-rss2.xml, wp-atom.xml, wp-rdf.xml, wp-rss.xml - substring matches in path: /feed/, /feeds/, /atom, /rss, /sitemap 2. Post-fetch Content-Type check (_looks_like_feed_response): - rejects: application/atom+xml, application/rss+xml, application/rdf+xml, application/xml, text/xml - keeps: application/xhtml+xml (xhtml IS html, just stricter syntax) Defense in depth — a feed served at a non-feed path (e.g. /index.html returning text/xml) still gets dropped on Content-Type. What stays out of the corpus: /feeds/all.atom.xml (Atom feeds) /sitemap.xml (XML sitemaps) /wp-rss2.xml (WordPress RSS) /feed/ (any feed alias) What's still allowed: /post-name/ (real prose pages) /index.html (HTML) /page.xhtml (xhtml) Tests: 12 new parametric cases (8 path patterns + 6 Content-Types + xhtml positive case). 36 bridge tests + 273 default suite, all passing. For fox: the existing feed entry already in the crawl shard was burned manually via `aborist burn --kind document --root eaf7c8d5...` — 230 chunks gone. Re-running `make crawl-ingest` won't re-introduce it. --- aborist/sources/crawler/bridge.py | 66 ++++++++++++++++++++++++- tests/crawler/test_bridge.py | 82 +++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/aborist/sources/crawler/bridge.py b/aborist/sources/crawler/bridge.py index 8299b47..68ffe73 100644 --- a/aborist/sources/crawler/bridge.py +++ b/aborist/sources/crawler/bridge.py @@ -48,6 +48,63 @@ def _normalize(url: str) -> str: return urllib.parse.urlunparse(cleaned) +# Feeds, sitemaps, and similar XML discovery artifacts are crawl +# infrastructure — they tell you which URLs exist on a site. They are +# NOT knowledge content, so they must not enter the Merkle tree as +# documents. We exclude them in two passes: (1) URL pattern match on +# the path so we skip even before the response arrives, (2) content- +# type check on the response so feeds served at unconventional paths +# still get rejected. xhtml is preserved because it IS HTML. +_FEED_URL_SUFFIXES = ( + ".atom", + ".atom.xml", + ".rss", + ".rss.xml", + ".rdf", + # Common feed filename patterns regardless of where they live. + "/feed.xml", + "/atom.xml", + "/rss.xml", + "/rss2.xml", + "wp-rss2.xml", # WordPress legacy + "wp-rss.xml", + "wp-atom.xml", + "wp-rdf.xml", +) +_FEED_URL_SUBSTRINGS = ( + "/feed/", + "/feeds/", + "/atom", + "/rss", + "/sitemap", # sitemap.xml, sitemap_index.xml, sitemaps/foo.xml +) +_FEED_CONTENT_TYPES = ( + "application/atom+xml", + "application/rss+xml", + "application/rdf+xml", + "application/xml", + "text/xml", +) + + +def _looks_like_feed_url(url: str) -> bool: + """True if the URL path matches a known feed/sitemap pattern.""" + path = urllib.parse.urlparse(url).path.lower() + if any(path.endswith(s) for s in _FEED_URL_SUFFIXES): + return True + if any(s in path for s in _FEED_URL_SUBSTRINGS): + return True + if path.endswith("/sitemap.xml") or path.endswith("sitemapindex.xml"): + return True + return False + + +def _looks_like_feed_response(content_type: str) -> bool: + """True if a response Content-Type is a feed/sitemap rather than HTML.""" + ct = content_type.lower().split(";", 1)[0].strip() + return ct in _FEED_CONTENT_TYPES + + async def _crawl_seed_async( seed_url: str, *, @@ -174,13 +231,20 @@ class _CrawledHtmlSource: follow_redirects=True, ) as client: for url in self.urls: + # Cheap pre-flight: skip feed/sitemap URLs without a fetch. + if _looks_like_feed_url(url): + continue try: resp = client.get(url) resp.raise_for_status() except httpx.HTTPError: continue ctype = resp.headers.get("content-type", "").lower() - if "html" not in ctype and "xml" not in ctype: + # Defense in depth: a feed served at a non-feed path still + # gets rejected by Content-Type. + if _looks_like_feed_response(ctype): + continue + if "html" not in ctype: continue doc = parse_html(str(resp.url), resp.text, self.source_type) if doc is None: diff --git a/tests/crawler/test_bridge.py b/tests/crawler/test_bridge.py index 2483fb7..5f1a26d 100644 --- a/tests/crawler/test_bridge.py +++ b/tests/crawler/test_bridge.py @@ -334,6 +334,88 @@ def test_ingest_crawled_skips_non_html(http_meta_db): assert result["inserted"] == 0 +@pytest.mark.parametrize( + "feed_url", + [ + "https://x.com/feeds/all.atom.xml", + "https://x.com/feed/", + "https://x.com/atom", + "https://x.com/rss", + "https://x.com/sitemap.xml", + "https://x.com/sitemaps/posts.xml", + "https://x.com/wp-rss2.xml", + "https://x.com/index.atom", + ], +) +def test_ingest_crawled_skips_feed_urls_by_path(http_meta_db, feed_url): + """Feed/sitemap URLs are crawl infrastructure, not knowledge. Skip + even before the fetch — pin so a future "but XML can be useful" + rewrite can't silently re-introduce feed pollution.""" + # Response stub never gets used because the path filter is pre-fetch, + # but we provide one in case the filter regresses. + responses = {feed_url: {"status": 200, "headers": {"content-type": "text/html"}}} + with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)): + conn = connect(http_meta_db) + try: + result = ingest_crawled(conn, [feed_url]) + finally: + conn.close() + assert result["http_meta_written"] == 0 + assert result["inserted"] == 0 + + +@pytest.mark.parametrize( + "ct", + [ + "application/atom+xml", + "application/rss+xml", + "application/rdf+xml", + "application/xml", + "text/xml", + "application/atom+xml; charset=utf-8", + ], +) +def test_ingest_crawled_skips_feed_content_types(http_meta_db, ct): + """A feed served at a non-feed path still gets rejected by + Content-Type. xhtml is allowed (treated as HTML elsewhere).""" + url = "https://x.com/innocent-looking-path" # no feed-pattern in URL + responses = { + url: { + "status": 200, + "headers": {"content-type": ct}, + "body": "", + } + } + with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)): + conn = connect(http_meta_db) + try: + result = ingest_crawled(conn, [url]) + finally: + conn.close() + assert result["http_meta_written"] == 0 + assert result["inserted"] == 0 + + +def test_ingest_crawled_keeps_xhtml(http_meta_db): + """xhtml IS HTML — keep it, even though the content type ends in +xml.""" + url = "https://x.com/page" + responses = { + url: { + "status": 200, + "headers": {"content-type": "application/xhtml+xml"}, + "body": "

T

Real content here.", + } + } + with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)): + conn = connect(http_meta_db) + try: + result = ingest_crawled(conn, [url]) + finally: + conn.close() + assert result["inserted"] == 1 + assert result["http_meta_written"] == 1 + + # --------------------------------------------------------------------------- # recrawl_check: conditional HEAD classification # ---------------------------------------------------------------------------