crawler: --fast flag exposes the verbatim AsyncWebFetcher fast_mode

The lifted AsyncWebFetcher already had a fast_mode constructor knob;
just unbroken plumbing was missing. fast_mode trades politeness for
throughput:

  - timeouts: 5s vs 15-60s
  - parallel page workers: cpu_count() * 3 vs 1
  - ignores robots.txt 'crawl-delay' directive
  - robots.txt 'Disallow' is STILL honored (only the delay drops)

Surface:

  aborist crawl --seed-url ... --fast
  make crawl-ingest URL=... FAST=1

Tests: two regressions pin that fast=True / fast=False each translate
to the right AsyncWebFetcher(fast_mode=...) construction. Capturing
fetcher fixture to avoid network. 21 bridge tests + 273 default
suite, all passing.

Makefile

@@ -327,9 +327,9 @@ test-crawler: bootstrap-crawler ## run only the lifted crawler tests
 # HEAD requests later. URL is required; DEPTH and MAX have safe defaults.
 CRAWL_DEPTH ?= 2
 CRAWL_MAX   ?= 0
-crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0] and ingest into $(DB) (MAX=0 = no cap, depth is only bound)
-	@if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0]" >&2; exit 2; fi
-	$(ABORIST) --db $(DB) crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) --ingest
+crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] and ingest into $(DB)
+	@if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1]" >&2; exit 2; fi
+	$(ABORIST) --db $(DB) crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest
 
 # Fast freshness probe: conditional HEAD per doc, classify fresh/stale/gone.
 # Send only If-None-Match + If-Modified-Since headers — server returns 304

aborist/cli.py

@@ -2123,8 +2123,9 @@ def _cmd_crawl(args: argparse.Namespace) -> int:
     from aborist.progress import Progress
 
     cap = "no cap" if args.max_pages == 0 else f"max {args.max_pages}"
+    speed = "fast" if args.fast else "polite"
     print(
-        f"  crawl: seed={args.seed_url} depth={args.depth} {cap}",
+        f"  crawl: seed={args.seed_url} depth={args.depth} {cap} ({speed})",
         file=sys.stderr,
         flush=True,
     )
@@ -2134,6 +2135,7 @@ def _cmd_crawl(args: argparse.Namespace) -> int:
         max_depth=args.depth,
         max_pages=args.max_pages,
         progress=crawl_progress,
+        fast=args.fast,
     )
     print(
         f"  crawl: discovery done — {len(urls)} URLs",
@@ -2769,6 +2771,15 @@ def build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="ingest the discovered pages into --db (default: print URL list only)",
     )
+    crawl_cmd.add_argument(
+        "--fast",
+        action="store_true",
+        help=(
+            "fast_mode: 5s timeouts, CPU*3 parallel page workers, ignore "
+            "robots.txt crawl-delay (Disallow is still honored). Use only "
+            "against domains where aggressive fetching is acceptable."
+        ),
+    )
     crawl_cmd.set_defaults(func=_cmd_crawl)
 
     crawler_cmd = sub.add_parser(

aborist/sources/crawler/bridge.py

@@ -55,6 +55,7 @@ async def _crawl_seed_async(
     max_pages: int,
     fetcher: AsyncWebFetcher | None = None,
     progress: Progress | None = None,
+    fast: bool = False,
 ) -> list[str]:
     """BFS from `seed_url`, staying on the same hostname.
 
@@ -66,8 +67,15 @@ async def _crawl_seed_async(
     heartbeats while the BFS runs. Lines are rate-limited by the
     Progress instance (default 2s) so a fast crawl prints once or
     twice; a slow one prints every interval.
+
+    ``fast=True`` flips the verbatim AsyncWebFetcher into fast_mode:
+    5s timeouts (vs 15-60s), CPU*3 parallel page workers, and the
+    target's robots.txt ``crawl-delay`` directive is ignored. Robots
+    Disallow is still honored. Use only against domains where
+    aggressive fetching is acceptable (your own sites, dumps you've
+    been authorized to mirror, etc.).
     """
-    fetcher = fetcher or AsyncWebFetcher(user_agent=USER_AGENT)
+    fetcher = fetcher or AsyncWebFetcher(user_agent=USER_AGENT, fast_mode=fast)
     seen: set[str] = set()
     discovered: list[str] = []
     queue: deque[tuple[str, int]] = deque([(_normalize(seed_url), 0)])
@@ -122,11 +130,14 @@ def crawl_seed(
     max_depth: int = 2,
     max_pages: int = 0,
     progress: Progress | None = None,
+    fast: bool = False,
 ) -> list[str]:
     """Sync wrapper around the async crawler. Returns same-domain URLs.
 
     ``max_pages=0`` means "no cap" — depth is the only bound. Pass
-    ``progress`` for stderr heartbeats while BFS runs.
+    ``progress`` for stderr heartbeats while BFS runs. ``fast=True``
+    enables AsyncWebFetcher's fast_mode (5s timeouts, CPU*3 workers,
+    ignores robots crawl-delay).
     """
     return asyncio.run(
         _crawl_seed_async(
@@ -134,6 +145,7 @@ def crawl_seed(
             max_depth=max_depth,
             max_pages=max_pages,
             progress=progress,
+            fast=fast,
         )
     )
 

tests/crawler/test_bridge.py

@@ -149,6 +149,44 @@ def test_crawl_seed_dedups():
     assert len(out) == len(set(out))
 
 
+def test_crawl_seed_fast_flag_creates_fast_mode_fetcher(monkeypatch):
+    """When fast=True and no fetcher is supplied, the bridge constructs an
+    AsyncWebFetcher with fast_mode=True. Pin so the kwarg can't silently
+    drop on the floor."""
+    captured: dict = {}
+
+    class _CapturingFetcher:
+        def __init__(self, **kwargs):
+            captured.update(kwargs)
+
+        async def fetch_webpage(self, url, extract_links=False, **kw):
+            return None, []
+
+    from aborist.sources.crawler import bridge
+
+    monkeypatch.setattr(bridge, "AsyncWebFetcher", _CapturingFetcher)
+    bridge.crawl_seed("https://x.com/", max_depth=1, max_pages=1, fast=True)
+    assert captured.get("fast_mode") is True
+
+
+def test_crawl_seed_default_creates_polite_fetcher(monkeypatch):
+    """fast=False (default) → fast_mode=False on the underlying fetcher."""
+    captured: dict = {}
+
+    class _CapturingFetcher:
+        def __init__(self, **kwargs):
+            captured.update(kwargs)
+
+        async def fetch_webpage(self, url, extract_links=False, **kw):
+            return None, []
+
+    from aborist.sources.crawler import bridge
+
+    monkeypatch.setattr(bridge, "AsyncWebFetcher", _CapturingFetcher)
+    bridge.crawl_seed("https://x.com/", max_depth=1, max_pages=1)
+    assert captured.get("fast_mode") is False
+
+
 def test_crawl_seed_passes_extract_links_true():
     """Regression: AsyncWebFetcher.fetch_webpage defaults extract_links=False,
     so without the explicit kwarg BFS terminates after the seed (only one