From 552dc0def2968dba869c9f095083e6de021289ea Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 28 Apr 2026 20:59:22 -0400 Subject: [PATCH] crawler: --max-pages defaults to 0 (no cap); depth is the only bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per fox: 20 was a wrong default — typical sites have many more pages within depth 2 than that, and the cap was silently truncating real crawls. New behavior: aborist crawl --depth 2 # no page cap aborist crawl --depth 2 --max-pages 50 # opt-in cap when needed make crawl-ingest URL=... DEPTH=2 # no page cap make crawl-ingest URL=... DEPTH=2 MAX=50 # explicit cap Implementation: bridge's BFS loop treats max_pages=0 as "unbounded" (while-condition becomes `queue and (max_pages == 0 or len(...) < max_pages)`). Default flows from CLI argparse default=0 down to the bridge. Tests: pinned max_pages=0 → no cap with a 30-page chain regression test (19 passed in tests/crawler/test_bridge.py). Default suite still 273 passed, 1 skipped. --- Makefile | 6 +++--- aborist/cli.py | 4 ++-- aborist/sources/crawler/bridge.py | 12 +++++++++--- tests/crawler/test_bridge.py | 12 ++++++++++++ 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 08860b5..034cf35 100644 --- a/Makefile +++ b/Makefile @@ -326,9 +326,9 @@ test-crawler: bootstrap-crawler ## run only the lifted crawler tests # capture ETag/Last-Modified for each so recrawl-check can do conditional # HEAD requests later. URL is required; DEPTH and MAX have safe defaults. CRAWL_DEPTH ?= 2 -CRAWL_MAX ?= 20 -crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=20] and ingest into $(DB) - @if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=20]" >&2; exit 2; fi +CRAWL_MAX ?= 0 +crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0] and ingest into $(DB) (MAX=0 = no cap, depth is only bound) + @if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0]" >&2; exit 2; fi $(ABORIST) --db $(DB) crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) --ingest # Fast freshness probe: conditional HEAD per doc, classify fresh/stale/gone. diff --git a/aborist/cli.py b/aborist/cli.py index c9f612f..28c7e77 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -2737,8 +2737,8 @@ def build_parser() -> argparse.ArgumentParser: "--max-pages", dest="max_pages", type=int, - default=20, - help="cap discovery at N URLs (default: 20)", + default=0, + help="cap discovery at N URLs (0 = no cap, depth is the only bound; default: 0)", ) crawl_cmd.add_argument( "--ingest", diff --git a/aborist/sources/crawler/bridge.py b/aborist/sources/crawler/bridge.py index fbb37f2..7101554 100644 --- a/aborist/sources/crawler/bridge.py +++ b/aborist/sources/crawler/bridge.py @@ -65,7 +65,10 @@ async def _crawl_seed_async( discovered: list[str] = [] queue: deque[tuple[str, int]] = deque([(_normalize(seed_url), 0)]) - while queue and len(discovered) < max_pages: + # max_pages == 0 means "no cap" — depth is the only bound. Useful for + # exhaustive same-domain crawls where the operator trusts depth to + # bound the crawl naturally. + while queue and (max_pages == 0 or len(discovered) < max_pages): url, depth = queue.popleft() if url in seen: continue @@ -106,9 +109,12 @@ def crawl_seed( seed_url: str, *, max_depth: int = 2, - max_pages: int = 20, + max_pages: int = 0, ) -> list[str]: - """Sync wrapper around the async crawler. Returns same-domain URLs.""" + """Sync wrapper around the async crawler. Returns same-domain URLs. + + ``max_pages=0`` means "no cap" — depth is the only bound. + """ return asyncio.run( _crawl_seed_async(seed_url, max_depth=max_depth, max_pages=max_pages) ) diff --git a/tests/crawler/test_bridge.py b/tests/crawler/test_bridge.py index d81b077..70eb6bb 100644 --- a/tests/crawler/test_bridge.py +++ b/tests/crawler/test_bridge.py @@ -102,6 +102,18 @@ def test_crawl_seed_respects_max_pages(): assert len(out) == 5 +def test_crawl_seed_max_pages_zero_means_no_cap(): + """Default `--max-pages 0` lets depth be the only bound. Pin so a + future "let's set a sensible default" PR can't silently re-introduce + a cap.""" + # Linear chain of 30 pages within depth 30; max_pages=0 should + # discover all of them. + page_links = {f"https://x.com/p{i}": [f"/p{i+1}"] for i in range(30)} + page_links["https://x.com/"] = ["/p0"] + out = _run_crawl(page_links, "https://x.com/", depth=30, max_pages=0) + assert len(out) >= 30 # seed + at least 30 chained pages + + def test_crawl_seed_respects_max_depth(): page_links = { "https://x.com/": ["/a"],