diff --git a/Makefile b/Makefile index 08860b5..034cf35 100644 --- a/Makefile +++ b/Makefile @@ -326,9 +326,9 @@ test-crawler: bootstrap-crawler ## run only the lifted crawler tests # capture ETag/Last-Modified for each so recrawl-check can do conditional # HEAD requests later. URL is required; DEPTH and MAX have safe defaults. CRAWL_DEPTH ?= 2 -CRAWL_MAX ?= 20 -crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=20] and ingest into $(DB) - @if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=20]" >&2; exit 2; fi +CRAWL_MAX ?= 0 +crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0] and ingest into $(DB) (MAX=0 = no cap, depth is only bound) + @if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0]" >&2; exit 2; fi $(ABORIST) --db $(DB) crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) --ingest # Fast freshness probe: conditional HEAD per doc, classify fresh/stale/gone. diff --git a/aborist/cli.py b/aborist/cli.py index c9f612f..28c7e77 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -2737,8 +2737,8 @@ def build_parser() -> argparse.ArgumentParser: "--max-pages", dest="max_pages", type=int, - default=20, - help="cap discovery at N URLs (default: 20)", + default=0, + help="cap discovery at N URLs (0 = no cap, depth is the only bound; default: 0)", ) crawl_cmd.add_argument( "--ingest", diff --git a/aborist/sources/crawler/bridge.py b/aborist/sources/crawler/bridge.py index fbb37f2..7101554 100644 --- a/aborist/sources/crawler/bridge.py +++ b/aborist/sources/crawler/bridge.py @@ -65,7 +65,10 @@ async def _crawl_seed_async( discovered: list[str] = [] queue: deque[tuple[str, int]] = deque([(_normalize(seed_url), 0)]) - while queue and len(discovered) < max_pages: + # max_pages == 0 means "no cap" — depth is the only bound. Useful for + # exhaustive same-domain crawls where the operator trusts depth to + # bound the crawl naturally. + while queue and (max_pages == 0 or len(discovered) < max_pages): url, depth = queue.popleft() if url in seen: continue @@ -106,9 +109,12 @@ def crawl_seed( seed_url: str, *, max_depth: int = 2, - max_pages: int = 20, + max_pages: int = 0, ) -> list[str]: - """Sync wrapper around the async crawler. Returns same-domain URLs.""" + """Sync wrapper around the async crawler. Returns same-domain URLs. + + ``max_pages=0`` means "no cap" — depth is the only bound. + """ return asyncio.run( _crawl_seed_async(seed_url, max_depth=max_depth, max_pages=max_pages) ) diff --git a/tests/crawler/test_bridge.py b/tests/crawler/test_bridge.py index d81b077..70eb6bb 100644 --- a/tests/crawler/test_bridge.py +++ b/tests/crawler/test_bridge.py @@ -102,6 +102,18 @@ def test_crawl_seed_respects_max_pages(): assert len(out) == 5 +def test_crawl_seed_max_pages_zero_means_no_cap(): + """Default `--max-pages 0` lets depth be the only bound. Pin so a + future "let's set a sensible default" PR can't silently re-introduce + a cap.""" + # Linear chain of 30 pages within depth 30; max_pages=0 should + # discover all of them. + page_links = {f"https://x.com/p{i}": [f"/p{i+1}"] for i in range(30)} + page_links["https://x.com/"] = ["/p0"] + out = _run_crawl(page_links, "https://x.com/", depth=30, max_pages=0) + assert len(out) >= 30 # seed + at least 30 chained pages + + def test_crawl_seed_respects_max_depth(): page_links = { "https://x.com/": ["/a"],