From 10382bd104c53365e355b154679b39ad26d69317 Mon Sep 17 00:00:00 2001
From: "russell@unturf.com" <russell@unturf.com>
Date: Fri, 22 May 2026 09:15:22 -0400
Subject: [PATCH] fix: fast_mode must ignore crawl-delay on robots-less sites
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

fast_mode is meant to ignore crawl-delay, but the per-domain delay was
only zeroed on the robots.txt-200 path. A site whose robots.txt 404s
(or errors) fell back to default_crawl_delay (2s), so under fast_mode
every concurrent fetch wave still slept ~2s — the dominant cost of a
wide crawl on a robots-less host. _enforce_crawl_delay now short-circuits
when fast_mode. Disallow unaffected; polite mode still enforces the delay.

Fixed upstream in agents.ai.unturf.com/core; this is the same one-line
fix in this copy of the verbatim-lift fetcher.
---
 neopig/async_web_fetcher.py          | 12 +++++++++++-
 tests/unit/test_async_web_fetcher.py | 19 +++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/neopig/async_web_fetcher.py b/neopig/async_web_fetcher.py
index 377a41c..a69aa00 100644
--- a/neopig/async_web_fetcher.py
+++ b/neopig/async_web_fetcher.py
@@ -1204,7 +1204,17 @@ class AsyncWebFetcher:
         return can_fetch
 
     async def _enforce_crawl_delay(self, domain: str):
-        """Enforce crawl delay for a domain"""
+        """Enforce crawl delay for a domain.
+
+        fast_mode's contract is "ignore crawl-delay" — so short-circuit
+        here, NOT just on the robots-200 path. The per-domain delay is
+        only zeroed in _fetch_robots_txt when robots.txt returns 200; on
+        a 404/error it falls back to default_crawl_delay (2s), which
+        would otherwise make every concurrent fetch wave sleep ~2s even
+        under fast_mode (the dominant cost on a robots-less site).
+        """
+        if self.fast_mode:
+            return
         delay = self.domain_crawl_delays.get(domain, self.default_crawl_delay)
         last_fetched = self.domain_last_fetched.get(domain, 0)
         elapsed = time.time() - last_fetched
diff --git a/tests/unit/test_async_web_fetcher.py b/tests/unit/test_async_web_fetcher.py
index 318387e..b008bab 100644
--- a/tests/unit/test_async_web_fetcher.py
+++ b/tests/unit/test_async_web_fetcher.py
@@ -426,6 +426,25 @@ class TestAsyncWebFetcherAsync:
         # Second call should have some delay (at least part of 0.1s)
         assert second_time >= 0.05  # Allow some tolerance
 
+    @pytest.mark.asyncio
+    async def test_fast_mode_ignores_crawl_delay(self):
+        """fast_mode must ignore crawl-delay regardless of robots status.
+
+        Regression: the delay was only zeroed on the robots-200 path, so
+        a site with no robots.txt (404) fell back to default_crawl_delay
+        and every concurrent fetch wave slept ~2s even under fast_mode.
+        """
+        fetcher = AsyncWebFetcher(fast_mode=True, default_crawl_delay=2.0)
+        domain = "example.com"
+        # Simulate the robots-404 fallback: a non-zero per-domain delay.
+        fetcher.domain_crawl_delays[domain] = 2.0
+        fetcher.domain_last_fetched[domain] = asyncio.get_event_loop().time()
+
+        start = asyncio.get_event_loop().time()
+        await fetcher._enforce_crawl_delay(domain)
+        elapsed = asyncio.get_event_loop().time() - start
+        assert elapsed < 0.5, "fast_mode must not sleep on a crawl delay"
+
     @pytest.mark.asyncio
     async def test_check_media_url_signature(self):
         """Test check_media_url function signature."""