crawler: --max-pages defaults to 0 (no cap); depth is the only bound

Per fox: 20 was a wrong default — typical sites have many more pages
within depth 2 than that, and the cap was silently truncating real
crawls. New behavior:

  aborist crawl --depth 2                  # no page cap
  aborist crawl --depth 2 --max-pages 50   # opt-in cap when needed
  make crawl-ingest URL=... DEPTH=2        # no page cap
  make crawl-ingest URL=... DEPTH=2 MAX=50 # explicit cap

Implementation: bridge's BFS loop treats max_pages=0 as "unbounded"
(while-condition becomes `queue and (max_pages == 0 or len(...) < max_pages)`).
Default flows from CLI argparse default=0 down to the bridge.

Tests: pinned max_pages=0 → no cap with a 30-page chain regression
test (19 passed in tests/crawler/test_bridge.py). Default suite still
273 passed, 1 skipped.

Makefile

@@ -326,9 +326,9 @@ test-crawler: bootstrap-crawler ## run only the lifted crawler tests
 # capture ETag/Last-Modified for each so recrawl-check can do conditional
 # HEAD requests later. URL is required; DEPTH and MAX have safe defaults.
 CRAWL_DEPTH ?= 2
-CRAWL_MAX   ?= 20
-crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=20] and ingest into $(DB)
-	@if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=20]" >&2; exit 2; fi
+CRAWL_MAX   ?= 0
+crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0] and ingest into $(DB) (MAX=0 = no cap, depth is only bound)
+	@if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0]" >&2; exit 2; fi
 	$(ABORIST) --db $(DB) crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) --ingest
 
 # Fast freshness probe: conditional HEAD per doc, classify fresh/stale/gone.

aborist/cli.py

@@ -2737,8 +2737,8 @@ def build_parser() -> argparse.ArgumentParser:
         "--max-pages",
         dest="max_pages",
         type=int,
-        default=20,
-        help="cap discovery at N URLs (default: 20)",
+        default=0,
+        help="cap discovery at N URLs (0 = no cap, depth is the only bound; default: 0)",
     )
     crawl_cmd.add_argument(
         "--ingest",

aborist/sources/crawler/bridge.py

@@ -65,7 +65,10 @@ async def _crawl_seed_async(
     discovered: list[str] = []
     queue: deque[tuple[str, int]] = deque([(_normalize(seed_url), 0)])
 
-    while queue and len(discovered) < max_pages:
+    # max_pages == 0 means "no cap" — depth is the only bound. Useful for
+    # exhaustive same-domain crawls where the operator trusts depth to
+    # bound the crawl naturally.
+    while queue and (max_pages == 0 or len(discovered) < max_pages):
         url, depth = queue.popleft()
         if url in seen:
             continue
@@ -106,9 +109,12 @@ def crawl_seed(
     seed_url: str,
     *,
     max_depth: int = 2,
-    max_pages: int = 20,
+    max_pages: int = 0,
 ) -> list[str]:
-    """Sync wrapper around the async crawler. Returns same-domain URLs."""
+    """Sync wrapper around the async crawler. Returns same-domain URLs.
+
+    ``max_pages=0`` means "no cap" — depth is the only bound.
+    """
     return asyncio.run(
         _crawl_seed_async(seed_url, max_depth=max_depth, max_pages=max_pages)
     )

tests/crawler/test_bridge.py

@@ -102,6 +102,18 @@ def test_crawl_seed_respects_max_pages():
     assert len(out) == 5
 
 
+def test_crawl_seed_max_pages_zero_means_no_cap():
+    """Default `--max-pages 0` lets depth be the only bound. Pin so a
+    future "let's set a sensible default" PR can't silently re-introduce
+    a cap."""
+    # Linear chain of 30 pages within depth 30; max_pages=0 should
+    # discover all of them.
+    page_links = {f"https://x.com/p{i}": [f"/p{i+1}"] for i in range(30)}
+    page_links["https://x.com/"] = ["/p0"]
+    out = _run_crawl(page_links, "https://x.com/", depth=30, max_pages=0)
+    assert len(out) >= 30  # seed + at least 30 chained pages
+
+
 def test_crawl_seed_respects_max_depth():
     page_links = {
         "https://x.com/": ["/a"],