fix+perf: fast-mode ignores crawl-delay; shared session; drop HEAD

Full --fast crawl of russell.ballestrini.net (242 URIs): 26s -> ~5s.

Three changes, biggest first:

1. fast_mode now actually ignores crawl-delay (the ~5x). The delay was
   only zeroed on the robots-200 path; a site with no robots.txt (404)
   or a robots fetch error fell back to default_crawl_delay (2s). Under
   --fast that made every concurrent fetch wave sleep ~2s — ~10 waves
   x 2s dominated the wall time. _enforce_crawl_delay now short-circuits
   when fast_mode, matching the documented "ignore crawl-delay"
   contract regardless of robots status. Disallow is still honored
   (separate path).

2. One shared ClientSession for the fetcher's lifetime (keepalive TCP
   connector sized to page-worker width) instead of a fresh session per
   fetch — ~3x on a 24-page wave. Lazily built in-loop via _get_session;
   the bridge closes it in a finally (guarded on owning the fetcher).

3. Drop the per-page preflight HEAD. aiohttp exposes response headers
   before the body is read, so the existing content-type binary guard
   skips images/video/audio without downloading them — the HEAD was a
   redundant round trip that doubled per-page latency.

Diverges arborist's AsyncWebFetcher from the agents.ai.unturf.com/core
verbatim lift (fox-approved); candidate to upstream. Regression tests
pin fast=no-delay / polite=delay, shared-session lifecycle, and bridge
session teardown (owned vs injected).

tests/crawler/test_async_web_fetcher.py

@@ -28,6 +28,76 @@ from arborist.sources.crawler.async_web_fetcher import (
 )
 
 
+class TestSharedSession:
+    """Shared keepalive session lifecycle (fast-crawl connection reuse)."""
+
+    def test_get_session_is_shared(self):
+        async def go():
+            f = AsyncWebFetcher()
+            s1 = await f._get_session()
+            s2 = await f._get_session()
+            assert s1 is s2
+            await f.aclose()
+
+        asyncio.run(go())
+
+    def test_aclose_closes_and_resets(self):
+        async def go():
+            f = AsyncWebFetcher()
+            s = await f._get_session()
+            await f.aclose()
+            assert s.closed
+            assert f._session is None
+            await f.aclose()  # idempotent — must not raise
+
+        asyncio.run(go())
+
+    def test_get_session_rebuilds_after_close(self):
+        async def go():
+            f = AsyncWebFetcher()
+            s1 = await f._get_session()
+            await f.aclose()
+            s2 = await f._get_session()
+            assert s2 is not s1
+            assert not s2.closed
+            await f.aclose()
+
+        asyncio.run(go())
+
+
+class TestFastModeIgnoresCrawlDelay:
+    """fast_mode must ignore crawl-delay regardless of robots.txt status.
+
+    Regression: the delay was only zeroed on the robots-200 path, so a
+    site with no robots.txt (404) fell back to default_crawl_delay (2s)
+    and every concurrent fetch wave slept ~2s even under --fast."""
+
+    def test_fast_mode_skips_delay_even_with_default_delay_set(self):
+        async def go():
+            f = AsyncWebFetcher(fast_mode=True)
+            # Simulate the robots-404 fallback: a non-zero per-domain delay.
+            f.domain_crawl_delays["x.com"] = 2.0
+            f.domain_last_fetched["x.com"] = __import__("time").time()
+            t = __import__("time").time()
+            await f._enforce_crawl_delay("x.com")
+            assert (__import__("time").time() - t) < 0.5, "fast_mode must not sleep"
+
+        asyncio.run(go())
+
+    def test_polite_mode_still_enforces_delay(self):
+        async def go():
+            import time as _t
+
+            f = AsyncWebFetcher(fast_mode=False, default_crawl_delay=0.3)
+            f.domain_crawl_delays["x.com"] = 0.3
+            f.domain_last_fetched["x.com"] = _t.time()
+            t = _t.time()
+            await f._enforce_crawl_delay("x.com")
+            assert (_t.time() - t) >= 0.25, "polite mode must still wait"
+
+        asyncio.run(go())
+
+
 class TestExtractPageLinks:
     """Link extraction for BFS (lxml fast path + BS4 fallback parity)."""
 

tests/crawler/test_bridge.py

@@ -297,6 +297,57 @@ def test_crawl_seed_polite_fetches_one_at_a_time():
     assert fetcher.peak_in_flight == 1, "polite crawl must stay one-at-a-time"
 
 
+def test_crawl_seed_closes_owned_fetcher(monkeypatch):
+    """When the bridge builds the fetcher, it tears down the shared
+    session in a finally — no leaked sockets after a fast crawl."""
+    closed = {"count": 0}
+
+    class _ClosableFetcher:
+        max_page_workers = 4
+
+        def __init__(self, **kwargs):
+            pass
+
+        async def fetch_webpage(self, url, extract_links=False, **kw):
+            return "<html></html>", []
+
+        async def aclose(self):
+            closed["count"] += 1
+
+    from arborist.sources.crawler import bridge
+
+    monkeypatch.setattr(bridge, "AsyncWebFetcher", _ClosableFetcher)
+    bridge.crawl_seed("https://x.com/", max_depth=1, max_pages=1)
+    assert closed["count"] == 1, "bridge must close a fetcher it created"
+
+
+def test_crawl_seed_leaves_injected_fetcher_open():
+    """An injected fetcher belongs to the caller — the bridge must NOT
+    close its session (it may be reused across crawls)."""
+    import asyncio
+
+    closed = {"count": 0}
+
+    class _ClosableFetcher:
+        max_page_workers = 4
+
+        async def fetch_webpage(self, url, extract_links=False, **kw):
+            return "<html></html>", []
+
+        async def aclose(self):
+            closed["count"] += 1
+
+    from arborist.sources.crawler import bridge
+
+    fetcher = _ClosableFetcher()
+    asyncio.run(
+        bridge._crawl_seed_async(
+            "https://x.com/", max_depth=1, max_pages=1, fetcher=fetcher
+        )
+    )
+    assert closed["count"] == 0, "bridge must not close a caller-supplied fetcher"
+
+
 # ---------------------------------------------------------------------------
 # ingest_crawled: stores ETag + Last-Modified per document_root
 # ---------------------------------------------------------------------------