engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

perf+cleanup: skip feeds in crawl discovery; lxml link extraction

Browse source 
Two crawler-discovery changes surfaced while chasing fast-crawl wall
time on russell.ballestrini.net:

1. Feed-skip in BFS discovery: the bridge fetched feed/sitemap URLs
   (a multi-MB atom.xml among them) only for ingest_crawled to discard
   them. Gate enqueue on the existing _looks_like_feed_url so we never
   fetch crawl-infrastructure URLs — less wasted work and one fewer
   slow wave straggler.

2. lxml link extraction, DRY'd: the three duplicated BeautifulSoup
   html.parser closures (fresh fetch + 2 cache paths) collapse into one
   module-level extract_page_links() backed by lxml.html (C parser,
   releases the GIL so to_thread actually parallelises) with a BS4
   fallback for markup lxml rejects. Parse on a 24-page wave 3.5s->2.5s.

Honest scope: neither moves full-crawl wall time much — measurement
showed the dominant cost is the per-page HEAD+GET double round-trip on
a per-call ClientSession, not parsing. These are correct-and-cleaner
on their own; the wall-time lever (shared session + drop redundant
HEAD) is a separate change. lxml extraction is regression-pinned
against the BS4 fallback for parity.
This commit is contained in:
russell@unturf.com 2026-05-22 06:46:55 -04:00
parent 8298bf8618
commit 9e196bcd82
No known key found for this signature in database
4 changed files with 155 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

51
tests/crawler/test_async_web_fetcher.py
View file

@ -28,6 +28,57 @@ from arborist.sources.crawler.async_web_fetcher import (
)
class TestExtractPageLinks:
"""Link extraction for BFS (lxml fast path + BS4 fallback parity)."""
HTML = (
'<html><body>'
'<a href="/a">A</a>'
'<a href="https://other.com/x">ext</a>'
'<a href="javascript:void(0)">js</a>'
'<a href="#frag">frag</a>'
'<a href="/b" rel="nofollow">nf</a>'
'<a href="/c#sec">C</a>'
'</body></html>'
)
def test_lxml_extraction_rules(self):
from arborist.sources.crawler.async_web_fetcher import extract_page_links
out = extract_page_links(self.HTML, "https://x.com/")
assert "https://x.com/a" in out # relative resolved
assert "https://other.com/x" in out # off-domain kept (bridge filters)
assert "https://x.com/c" in out # fragment stripped
assert not any("javascript" in u for u in out)
assert "https://x.com/b" not in out # rel=nofollow dropped
assert not any("#" in u for u in out)
def test_lxml_matches_bs4_fallback(self):
from arborist.sources.crawler.async_web_fetcher import (
extract_page_links,
_extract_page_links_bs4,
)
assert (
extract_page_links(self.HTML, "https://x.com/")
== _extract_page_links_bs4(self.HTML, "https://x.com/")
)
def test_anchor_text_mode(self):
from arborist.sources.crawler.async_web_fetcher import extract_page_links
out = extract_page_links(self.HTML, "https://x.com/", extract_anchor_text=True)
a = next(d for d in out if d["url"] == "https://x.com/a")
assert a["anchor_text"] == "A"
def test_malformed_html_falls_back_gracefully(self):
from arborist.sources.crawler.async_web_fetcher import extract_page_links
# Empty / whitespace markup makes lxml.fromstring raise — the
# helper must fall back, not crash.
assert extract_page_links(" ", "https://x.com/") == []
class TestCrawlMode:
"""Test CrawlMode enum."""

20
tests/crawler/test_bridge.py
View file

@ -138,6 +138,26 @@ def test_crawl_seed_drops_off_domain_links():
assert "https://x.com/local" in out
def test_crawl_seed_skips_feed_urls_during_discovery():
"""Feed/sitemap links must not be crawled — they're discarded at
ingest and a big feed is a slow wave straggler. Pin that the BFS
drops them at enqueue (not just at ingest)."""
page_links = {
"https://x.com/": [
"/a",
"/feeds/all.atom.xml", # big feed — must be skipped
"/sitemap.xml",
"/b.rss",
],
"https://x.com/a": [],
}
out = _run_crawl(page_links, "https://x.com/", depth=2, max_pages=10)
assert "https://x.com/a" in out
assert "https://x.com/feeds/all.atom.xml" not in out
assert "https://x.com/sitemap.xml" not in out
assert "https://x.com/b.rss" not in out
def test_crawl_seed_dedups():
page_links = {
"https://x.com/": ["/a", "/a", "/b"],