crawler: skip feeds + sitemaps at ingest (they're discovery, not knowledge)
Per fox: a query for "who is Russell Ballestrini" classified STRICT by
grounding 6/6 quotes against `feeds/all.atom.xml` — a 230-chunk dump
of post metadata. The verifier was technically correct (every quoted
string IS in the feed) but the result was hollow: feeds list URLs,
they don't carry knowledge. The ACTUAL bios live at the linked posts.
Two-pass filter in _CrawledHtmlSource.iter_documents:
1. Pre-fetch URL-pattern check (_looks_like_feed_url):
- suffix matches: .atom, .atom.xml, .rss, .rss.xml, .rdf,
/feed.xml, /atom.xml, /rss.xml, /rss2.xml, wp-rss2.xml,
wp-atom.xml, wp-rdf.xml, wp-rss.xml
- substring matches in path: /feed/, /feeds/, /atom, /rss, /sitemap
2. Post-fetch Content-Type check (_looks_like_feed_response):
- rejects: application/atom+xml, application/rss+xml,
application/rdf+xml, application/xml, text/xml
- keeps: application/xhtml+xml (xhtml IS html, just stricter syntax)
Defense in depth — a feed served at a non-feed path (e.g.
/index.html returning text/xml) still gets dropped on Content-Type.
What stays out of the corpus:
/feeds/all.atom.xml (Atom feeds)
/sitemap.xml (XML sitemaps)
/wp-rss2.xml (WordPress RSS)
/feed/ (any feed alias)
What's still allowed:
/post-name/ (real prose pages)
/index.html (HTML)
/page.xhtml (xhtml)
Tests: 12 new parametric cases (8 path patterns + 6 Content-Types +
xhtml positive case). 36 bridge tests + 273 default suite, all
passing.
For fox: the existing feed entry already in the crawl shard was burned
manually via `aborist burn --kind document --root eaf7c8d5...` — 230
chunks gone. Re-running `make crawl-ingest` won't re-introduce it.
This commit is contained in:
parent
cf103fad78
commit
43529328fd
2 changed files with 147 additions and 1 deletions
|
|
@ -48,6 +48,63 @@ def _normalize(url: str) -> str:
|
|||
return urllib.parse.urlunparse(cleaned)
|
||||
|
||||
|
||||
# Feeds, sitemaps, and similar XML discovery artifacts are crawl
|
||||
# infrastructure — they tell you which URLs exist on a site. They are
|
||||
# NOT knowledge content, so they must not enter the Merkle tree as
|
||||
# documents. We exclude them in two passes: (1) URL pattern match on
|
||||
# the path so we skip even before the response arrives, (2) content-
|
||||
# type check on the response so feeds served at unconventional paths
|
||||
# still get rejected. xhtml is preserved because it IS HTML.
|
||||
_FEED_URL_SUFFIXES = (
|
||||
".atom",
|
||||
".atom.xml",
|
||||
".rss",
|
||||
".rss.xml",
|
||||
".rdf",
|
||||
# Common feed filename patterns regardless of where they live.
|
||||
"/feed.xml",
|
||||
"/atom.xml",
|
||||
"/rss.xml",
|
||||
"/rss2.xml",
|
||||
"wp-rss2.xml", # WordPress legacy
|
||||
"wp-rss.xml",
|
||||
"wp-atom.xml",
|
||||
"wp-rdf.xml",
|
||||
)
|
||||
_FEED_URL_SUBSTRINGS = (
|
||||
"/feed/",
|
||||
"/feeds/",
|
||||
"/atom",
|
||||
"/rss",
|
||||
"/sitemap", # sitemap.xml, sitemap_index.xml, sitemaps/foo.xml
|
||||
)
|
||||
_FEED_CONTENT_TYPES = (
|
||||
"application/atom+xml",
|
||||
"application/rss+xml",
|
||||
"application/rdf+xml",
|
||||
"application/xml",
|
||||
"text/xml",
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_feed_url(url: str) -> bool:
|
||||
"""True if the URL path matches a known feed/sitemap pattern."""
|
||||
path = urllib.parse.urlparse(url).path.lower()
|
||||
if any(path.endswith(s) for s in _FEED_URL_SUFFIXES):
|
||||
return True
|
||||
if any(s in path for s in _FEED_URL_SUBSTRINGS):
|
||||
return True
|
||||
if path.endswith("/sitemap.xml") or path.endswith("sitemapindex.xml"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _looks_like_feed_response(content_type: str) -> bool:
|
||||
"""True if a response Content-Type is a feed/sitemap rather than HTML."""
|
||||
ct = content_type.lower().split(";", 1)[0].strip()
|
||||
return ct in _FEED_CONTENT_TYPES
|
||||
|
||||
|
||||
async def _crawl_seed_async(
|
||||
seed_url: str,
|
||||
*,
|
||||
|
|
@ -174,13 +231,20 @@ class _CrawledHtmlSource:
|
|||
follow_redirects=True,
|
||||
) as client:
|
||||
for url in self.urls:
|
||||
# Cheap pre-flight: skip feed/sitemap URLs without a fetch.
|
||||
if _looks_like_feed_url(url):
|
||||
continue
|
||||
try:
|
||||
resp = client.get(url)
|
||||
resp.raise_for_status()
|
||||
except httpx.HTTPError:
|
||||
continue
|
||||
ctype = resp.headers.get("content-type", "").lower()
|
||||
if "html" not in ctype and "xml" not in ctype:
|
||||
# Defense in depth: a feed served at a non-feed path still
|
||||
# gets rejected by Content-Type.
|
||||
if _looks_like_feed_response(ctype):
|
||||
continue
|
||||
if "html" not in ctype:
|
||||
continue
|
||||
doc = parse_html(str(resp.url), resp.text, self.source_type)
|
||||
if doc is None:
|
||||
|
|
|
|||
|
|
@ -334,6 +334,88 @@ def test_ingest_crawled_skips_non_html(http_meta_db):
|
|||
assert result["inserted"] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"feed_url",
|
||||
[
|
||||
"https://x.com/feeds/all.atom.xml",
|
||||
"https://x.com/feed/",
|
||||
"https://x.com/atom",
|
||||
"https://x.com/rss",
|
||||
"https://x.com/sitemap.xml",
|
||||
"https://x.com/sitemaps/posts.xml",
|
||||
"https://x.com/wp-rss2.xml",
|
||||
"https://x.com/index.atom",
|
||||
],
|
||||
)
|
||||
def test_ingest_crawled_skips_feed_urls_by_path(http_meta_db, feed_url):
|
||||
"""Feed/sitemap URLs are crawl infrastructure, not knowledge. Skip
|
||||
even before the fetch — pin so a future "but XML can be useful"
|
||||
rewrite can't silently re-introduce feed pollution."""
|
||||
# Response stub never gets used because the path filter is pre-fetch,
|
||||
# but we provide one in case the filter regresses.
|
||||
responses = {feed_url: {"status": 200, "headers": {"content-type": "text/html"}}}
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(http_meta_db)
|
||||
try:
|
||||
result = ingest_crawled(conn, [feed_url])
|
||||
finally:
|
||||
conn.close()
|
||||
assert result["http_meta_written"] == 0
|
||||
assert result["inserted"] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"ct",
|
||||
[
|
||||
"application/atom+xml",
|
||||
"application/rss+xml",
|
||||
"application/rdf+xml",
|
||||
"application/xml",
|
||||
"text/xml",
|
||||
"application/atom+xml; charset=utf-8",
|
||||
],
|
||||
)
|
||||
def test_ingest_crawled_skips_feed_content_types(http_meta_db, ct):
|
||||
"""A feed served at a non-feed path still gets rejected by
|
||||
Content-Type. xhtml is allowed (treated as HTML elsewhere)."""
|
||||
url = "https://x.com/innocent-looking-path" # no feed-pattern in URL
|
||||
responses = {
|
||||
url: {
|
||||
"status": 200,
|
||||
"headers": {"content-type": ct},
|
||||
"body": "<?xml version='1.0'?><feed></feed>",
|
||||
}
|
||||
}
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(http_meta_db)
|
||||
try:
|
||||
result = ingest_crawled(conn, [url])
|
||||
finally:
|
||||
conn.close()
|
||||
assert result["http_meta_written"] == 0
|
||||
assert result["inserted"] == 0
|
||||
|
||||
|
||||
def test_ingest_crawled_keeps_xhtml(http_meta_db):
|
||||
"""xhtml IS HTML — keep it, even though the content type ends in +xml."""
|
||||
url = "https://x.com/page"
|
||||
responses = {
|
||||
url: {
|
||||
"status": 200,
|
||||
"headers": {"content-type": "application/xhtml+xml"},
|
||||
"body": "<html><body><h1>T</h1>Real content here.</body></html>",
|
||||
}
|
||||
}
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(http_meta_db)
|
||||
try:
|
||||
result = ingest_crawled(conn, [url])
|
||||
finally:
|
||||
conn.close()
|
||||
assert result["inserted"] == 1
|
||||
assert result["http_meta_written"] == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# recrawl_check: conditional HEAD classification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue