diff --git a/neopig/async_web_fetcher.py b/neopig/async_web_fetcher.py index 377a41c..a69aa00 100644 --- a/neopig/async_web_fetcher.py +++ b/neopig/async_web_fetcher.py @@ -1204,7 +1204,17 @@ class AsyncWebFetcher: return can_fetch async def _enforce_crawl_delay(self, domain: str): - """Enforce crawl delay for a domain""" + """Enforce crawl delay for a domain. + + fast_mode's contract is "ignore crawl-delay" — so short-circuit + here, NOT just on the robots-200 path. The per-domain delay is + only zeroed in _fetch_robots_txt when robots.txt returns 200; on + a 404/error it falls back to default_crawl_delay (2s), which + would otherwise make every concurrent fetch wave sleep ~2s even + under fast_mode (the dominant cost on a robots-less site). + """ + if self.fast_mode: + return delay = self.domain_crawl_delays.get(domain, self.default_crawl_delay) last_fetched = self.domain_last_fetched.get(domain, 0) elapsed = time.time() - last_fetched diff --git a/tests/unit/test_async_web_fetcher.py b/tests/unit/test_async_web_fetcher.py index 318387e..b008bab 100644 --- a/tests/unit/test_async_web_fetcher.py +++ b/tests/unit/test_async_web_fetcher.py @@ -426,6 +426,25 @@ class TestAsyncWebFetcherAsync: # Second call should have some delay (at least part of 0.1s) assert second_time >= 0.05 # Allow some tolerance + @pytest.mark.asyncio + async def test_fast_mode_ignores_crawl_delay(self): + """fast_mode must ignore crawl-delay regardless of robots status. + + Regression: the delay was only zeroed on the robots-200 path, so + a site with no robots.txt (404) fell back to default_crawl_delay + and every concurrent fetch wave slept ~2s even under fast_mode. + """ + fetcher = AsyncWebFetcher(fast_mode=True, default_crawl_delay=2.0) + domain = "example.com" + # Simulate the robots-404 fallback: a non-zero per-domain delay. + fetcher.domain_crawl_delays[domain] = 2.0 + fetcher.domain_last_fetched[domain] = asyncio.get_event_loop().time() + + start = asyncio.get_event_loop().time() + await fetcher._enforce_crawl_delay(domain) + elapsed = asyncio.get_event_loop().time() - start + assert elapsed < 0.5, "fast_mode must not sleep on a crawl delay" + @pytest.mark.asyncio async def test_check_media_url_signature(self): """Test check_media_url function signature."""