diff --git a/Makefile b/Makefile index 034cf35..caa5f7a 100644 --- a/Makefile +++ b/Makefile @@ -327,9 +327,9 @@ test-crawler: bootstrap-crawler ## run only the lifted crawler tests # HEAD requests later. URL is required; DEPTH and MAX have safe defaults. CRAWL_DEPTH ?= 2 CRAWL_MAX ?= 0 -crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0] and ingest into $(DB) (MAX=0 = no cap, depth is only bound) - @if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0]" >&2; exit 2; fi - $(ABORIST) --db $(DB) crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) --ingest +crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] and ingest into $(DB) + @if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1]" >&2; exit 2; fi + $(ABORIST) --db $(DB) crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest # Fast freshness probe: conditional HEAD per doc, classify fresh/stale/gone. # Send only If-None-Match + If-Modified-Since headers — server returns 304 diff --git a/aborist/cli.py b/aborist/cli.py index 04f9c31..77b6a1d 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -2123,8 +2123,9 @@ def _cmd_crawl(args: argparse.Namespace) -> int: from aborist.progress import Progress cap = "no cap" if args.max_pages == 0 else f"max {args.max_pages}" + speed = "fast" if args.fast else "polite" print( - f" crawl: seed={args.seed_url} depth={args.depth} {cap}", + f" crawl: seed={args.seed_url} depth={args.depth} {cap} ({speed})", file=sys.stderr, flush=True, ) @@ -2134,6 +2135,7 @@ def _cmd_crawl(args: argparse.Namespace) -> int: max_depth=args.depth, max_pages=args.max_pages, progress=crawl_progress, + fast=args.fast, ) print( f" crawl: discovery done — {len(urls)} URLs", @@ -2769,6 +2771,15 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="ingest the discovered pages into --db (default: print URL list only)", ) + crawl_cmd.add_argument( + "--fast", + action="store_true", + help=( + "fast_mode: 5s timeouts, CPU*3 parallel page workers, ignore " + "robots.txt crawl-delay (Disallow is still honored). Use only " + "against domains where aggressive fetching is acceptable." + ), + ) crawl_cmd.set_defaults(func=_cmd_crawl) crawler_cmd = sub.add_parser( diff --git a/aborist/sources/crawler/bridge.py b/aborist/sources/crawler/bridge.py index 922e75f..8299b47 100644 --- a/aborist/sources/crawler/bridge.py +++ b/aborist/sources/crawler/bridge.py @@ -55,6 +55,7 @@ async def _crawl_seed_async( max_pages: int, fetcher: AsyncWebFetcher | None = None, progress: Progress | None = None, + fast: bool = False, ) -> list[str]: """BFS from `seed_url`, staying on the same hostname. @@ -66,8 +67,15 @@ async def _crawl_seed_async( heartbeats while the BFS runs. Lines are rate-limited by the Progress instance (default 2s) so a fast crawl prints once or twice; a slow one prints every interval. + + ``fast=True`` flips the verbatim AsyncWebFetcher into fast_mode: + 5s timeouts (vs 15-60s), CPU*3 parallel page workers, and the + target's robots.txt ``crawl-delay`` directive is ignored. Robots + Disallow is still honored. Use only against domains where + aggressive fetching is acceptable (your own sites, dumps you've + been authorized to mirror, etc.). """ - fetcher = fetcher or AsyncWebFetcher(user_agent=USER_AGENT) + fetcher = fetcher or AsyncWebFetcher(user_agent=USER_AGENT, fast_mode=fast) seen: set[str] = set() discovered: list[str] = [] queue: deque[tuple[str, int]] = deque([(_normalize(seed_url), 0)]) @@ -122,11 +130,14 @@ def crawl_seed( max_depth: int = 2, max_pages: int = 0, progress: Progress | None = None, + fast: bool = False, ) -> list[str]: """Sync wrapper around the async crawler. Returns same-domain URLs. ``max_pages=0`` means "no cap" — depth is the only bound. Pass - ``progress`` for stderr heartbeats while BFS runs. + ``progress`` for stderr heartbeats while BFS runs. ``fast=True`` + enables AsyncWebFetcher's fast_mode (5s timeouts, CPU*3 workers, + ignores robots crawl-delay). """ return asyncio.run( _crawl_seed_async( @@ -134,6 +145,7 @@ def crawl_seed( max_depth=max_depth, max_pages=max_pages, progress=progress, + fast=fast, ) ) diff --git a/tests/crawler/test_bridge.py b/tests/crawler/test_bridge.py index 70eb6bb..2483fb7 100644 --- a/tests/crawler/test_bridge.py +++ b/tests/crawler/test_bridge.py @@ -149,6 +149,44 @@ def test_crawl_seed_dedups(): assert len(out) == len(set(out)) +def test_crawl_seed_fast_flag_creates_fast_mode_fetcher(monkeypatch): + """When fast=True and no fetcher is supplied, the bridge constructs an + AsyncWebFetcher with fast_mode=True. Pin so the kwarg can't silently + drop on the floor.""" + captured: dict = {} + + class _CapturingFetcher: + def __init__(self, **kwargs): + captured.update(kwargs) + + async def fetch_webpage(self, url, extract_links=False, **kw): + return None, [] + + from aborist.sources.crawler import bridge + + monkeypatch.setattr(bridge, "AsyncWebFetcher", _CapturingFetcher) + bridge.crawl_seed("https://x.com/", max_depth=1, max_pages=1, fast=True) + assert captured.get("fast_mode") is True + + +def test_crawl_seed_default_creates_polite_fetcher(monkeypatch): + """fast=False (default) → fast_mode=False on the underlying fetcher.""" + captured: dict = {} + + class _CapturingFetcher: + def __init__(self, **kwargs): + captured.update(kwargs) + + async def fetch_webpage(self, url, extract_links=False, **kw): + return None, [] + + from aborist.sources.crawler import bridge + + monkeypatch.setattr(bridge, "AsyncWebFetcher", _CapturingFetcher) + bridge.crawl_seed("https://x.com/", max_depth=1, max_pages=1) + assert captured.get("fast_mode") is False + + def test_crawl_seed_passes_extract_links_true(): """Regression: AsyncWebFetcher.fetch_webpage defaults extract_links=False, so without the explicit kwarg BFS terminates after the seed (only one