diff --git a/aborist/sources/crawler/bridge.py b/aborist/sources/crawler/bridge.py index 969a079..fbb37f2 100644 --- a/aborist/sources/crawler/bridge.py +++ b/aborist/sources/crawler/bridge.py @@ -72,7 +72,10 @@ async def _crawl_seed_async( seen.add(url) try: - html, links = await fetcher.fetch_webpage(url) + # extract_links=True is REQUIRED for BFS — the verbatim + # AsyncWebFetcher defaults it to False (caller opts in for the + # link-graph use case). Without it BFS terminates at the seed. + html, links = await fetcher.fetch_webpage(url, extract_links=True) except Exception: continue if html is None: diff --git a/tests/crawler/test_bridge.py b/tests/crawler/test_bridge.py index 831cb25..d81b077 100644 --- a/tests/crawler/test_bridge.py +++ b/tests/crawler/test_bridge.py @@ -64,7 +64,7 @@ class _StubFetcher: def __init__(self, page_links: dict[str, list[str]]): self.page_links = page_links - async def fetch_webpage(self, url: str): + async def fetch_webpage(self, url: str, extract_links: bool = False, **kw): return "", self.page_links.get(url, []) @@ -137,6 +137,36 @@ def test_crawl_seed_dedups(): assert len(out) == len(set(out)) +def test_crawl_seed_passes_extract_links_true(): + """Regression: AsyncWebFetcher.fetch_webpage defaults extract_links=False, + so without the explicit kwarg BFS terminates after the seed (only one + URL ever lands in `discovered`). Pin the kwarg by recording the calls + on a stub fetcher.""" + + class _RecordingFetcher: + def __init__(self): + self.calls: list[dict] = [] + + async def fetch_webpage(self, url, extract_links=False, **kw): + self.calls.append({"url": url, "extract_links": extract_links}) + return "", [] + + import asyncio + from aborist.sources.crawler import bridge + + fetcher = _RecordingFetcher() + asyncio.run( + bridge._crawl_seed_async( + "https://x.com/", max_depth=2, max_pages=5, fetcher=fetcher + ) + ) + assert fetcher.calls, "fetcher was never called" + for call in fetcher.calls: + assert call["extract_links"] is True, ( + f"BFS must request link extraction; got {call!r}" + ) + + # --------------------------------------------------------------------------- # ingest_crawled: stores ETag + Last-Modified per document_root # ---------------------------------------------------------------------------