crawler: --fast flag exposes the verbatim AsyncWebFetcher fast_mode
The lifted AsyncWebFetcher already had a fast_mode constructor knob; just unbroken plumbing was missing. fast_mode trades politeness for throughput: - timeouts: 5s vs 15-60s - parallel page workers: cpu_count() * 3 vs 1 - ignores robots.txt 'crawl-delay' directive - robots.txt 'Disallow' is STILL honored (only the delay drops) Surface: aborist crawl --seed-url ... --fast make crawl-ingest URL=... FAST=1 Tests: two regressions pin that fast=True / fast=False each translate to the right AsyncWebFetcher(fast_mode=...) construction. Capturing fetcher fixture to avoid network. 21 bridge tests + 273 default suite, all passing.
This commit is contained in:
parent
f04417161c
commit
dfe39a0bf4
4 changed files with 67 additions and 6 deletions
|
|
@ -149,6 +149,44 @@ def test_crawl_seed_dedups():
|
|||
assert len(out) == len(set(out))
|
||||
|
||||
|
||||
def test_crawl_seed_fast_flag_creates_fast_mode_fetcher(monkeypatch):
|
||||
"""When fast=True and no fetcher is supplied, the bridge constructs an
|
||||
AsyncWebFetcher with fast_mode=True. Pin so the kwarg can't silently
|
||||
drop on the floor."""
|
||||
captured: dict = {}
|
||||
|
||||
class _CapturingFetcher:
|
||||
def __init__(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
|
||||
async def fetch_webpage(self, url, extract_links=False, **kw):
|
||||
return None, []
|
||||
|
||||
from aborist.sources.crawler import bridge
|
||||
|
||||
monkeypatch.setattr(bridge, "AsyncWebFetcher", _CapturingFetcher)
|
||||
bridge.crawl_seed("https://x.com/", max_depth=1, max_pages=1, fast=True)
|
||||
assert captured.get("fast_mode") is True
|
||||
|
||||
|
||||
def test_crawl_seed_default_creates_polite_fetcher(monkeypatch):
|
||||
"""fast=False (default) → fast_mode=False on the underlying fetcher."""
|
||||
captured: dict = {}
|
||||
|
||||
class _CapturingFetcher:
|
||||
def __init__(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
|
||||
async def fetch_webpage(self, url, extract_links=False, **kw):
|
||||
return None, []
|
||||
|
||||
from aborist.sources.crawler import bridge
|
||||
|
||||
monkeypatch.setattr(bridge, "AsyncWebFetcher", _CapturingFetcher)
|
||||
bridge.crawl_seed("https://x.com/", max_depth=1, max_pages=1)
|
||||
assert captured.get("fast_mode") is False
|
||||
|
||||
|
||||
def test_crawl_seed_passes_extract_links_true():
|
||||
"""Regression: AsyncWebFetcher.fetch_webpage defaults extract_links=False,
|
||||
so without the explicit kwarg BFS terminates after the seed (only one
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue