russell@unturf.com
dee689cd91
Full --fast crawl of russell.ballestrini.net (242 URIs): 26s -> ~5s. Three changes, biggest first: 1. fast_mode now actually ignores crawl-delay (the ~5x). The delay was only zeroed on the robots-200 path; a site with no robots.txt (404) or a robots fetch error fell back to default_crawl_delay (2s). Under --fast that made every concurrent fetch wave sleep ~2s — ~10 waves x 2s dominated the wall time. _enforce_crawl_delay now short-circuits when fast_mode, matching the documented "ignore crawl-delay" contract regardless of robots status. Disallow is still honored (separate path). 2. One shared ClientSession for the fetcher's lifetime (keepalive TCP connector sized to page-worker width) instead of a fresh session per fetch — ~3x on a 24-page wave. Lazily built in-loop via _get_session; the bridge closes it in a finally (guarded on owning the fetcher). 3. Drop the per-page preflight HEAD. aiohttp exposes response headers before the body is read, so the existing content-type binary guard skips images/video/audio without downloading them — the HEAD was a redundant round trip that doubled per-page latency. Diverges arborist's AsyncWebFetcher from the agents.ai.unturf.com/core verbatim lift (fox-approved); candidate to upstream. Regression tests pin fast=no-delay / polite=delay, shared-session lifecycle, and bridge session teardown (owned vs injected).
680 lines
23 KiB
Python
680 lines
23 KiB
Python
"""Bridge tests: crawler discovery + ingest + recrawl-check.
|
|
|
|
These tests stub the network. The verbatim crawler tests in this same
|
|
directory exercise the underlying AsyncWebFetcher; here we exercise
|
|
the arborist bridge logic on top of it (BFS bounds, same-domain filter,
|
|
ETag capture, conditional HEAD classification).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from arborist.sources.crawler.bridge import (
|
|
_normalize,
|
|
_same_domain,
|
|
crawl_seed,
|
|
ingest_crawled,
|
|
recrawl_check,
|
|
)
|
|
from arborist.store import connect
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# URL helpers (pure)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_same_domain_exact_hostname():
|
|
assert _same_domain("https://x.com/a", "https://x.com/b") is True
|
|
|
|
|
|
def test_same_domain_rejects_subdomain():
|
|
assert _same_domain("https://x.com/a", "https://docs.x.com/b") is False
|
|
|
|
|
|
def test_same_domain_rejects_other_host():
|
|
assert _same_domain("https://x.com/a", "https://y.com/a") is False
|
|
|
|
|
|
def test_normalize_strips_fragment():
|
|
assert _normalize("https://x.com/a#section") == "https://x.com/a"
|
|
|
|
|
|
def test_normalize_preserves_query():
|
|
assert _normalize("https://x.com/a?q=1") == "https://x.com/a?q=1"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# crawl_seed: BFS discovery (no real network — mock the fetcher)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class _StubFetcher:
|
|
"""Stand-in for AsyncWebFetcher that returns pre-canned (html, links)
|
|
based on a URL→links map. Enough to drive the bridge's BFS.
|
|
"""
|
|
|
|
def __init__(self, page_links: dict[str, list[str]]):
|
|
self.page_links = page_links
|
|
|
|
async def fetch_webpage(self, url: str, extract_links: bool = False, **kw):
|
|
return "<html></html>", self.page_links.get(url, [])
|
|
|
|
|
|
def _run_crawl(page_links, seed, depth, max_pages):
|
|
"""Helper: drive _crawl_seed_async with a stub fetcher."""
|
|
import asyncio
|
|
from arborist.sources.crawler import bridge
|
|
|
|
return asyncio.run(
|
|
bridge._crawl_seed_async(
|
|
seed, max_depth=depth, max_pages=max_pages, fetcher=_StubFetcher(page_links)
|
|
)
|
|
)
|
|
|
|
|
|
def test_crawl_seed_breadth_first_within_limit():
|
|
page_links = {
|
|
"https://x.com/": ["/a", "/b"],
|
|
"https://x.com/a": ["/c"],
|
|
"https://x.com/b": [],
|
|
"https://x.com/c": [],
|
|
}
|
|
out = _run_crawl(page_links, "https://x.com/", depth=2, max_pages=10)
|
|
# Seed first, then both depth-1 pages, then the depth-2 page.
|
|
assert out[0] == "https://x.com/"
|
|
assert "https://x.com/a" in out
|
|
assert "https://x.com/b" in out
|
|
assert "https://x.com/c" in out
|
|
|
|
|
|
def test_crawl_seed_respects_max_pages():
|
|
page_links = {f"https://x.com/p{i}": [f"/p{i+1}"] for i in range(50)}
|
|
page_links["https://x.com/"] = ["/p0"]
|
|
out = _run_crawl(page_links, "https://x.com/", depth=10, max_pages=5)
|
|
assert len(out) == 5
|
|
|
|
|
|
def test_crawl_seed_max_pages_zero_means_no_cap():
|
|
"""Default `--max-pages 0` lets depth be the only bound. Pin so a
|
|
future "let's set a sensible default" PR can't silently re-introduce
|
|
a cap."""
|
|
# Linear chain of 30 pages within depth 30; max_pages=0 should
|
|
# discover all of them.
|
|
page_links = {f"https://x.com/p{i}": [f"/p{i+1}"] for i in range(30)}
|
|
page_links["https://x.com/"] = ["/p0"]
|
|
out = _run_crawl(page_links, "https://x.com/", depth=30, max_pages=0)
|
|
assert len(out) >= 30 # seed + at least 30 chained pages
|
|
|
|
|
|
def test_crawl_seed_respects_max_depth():
|
|
page_links = {
|
|
"https://x.com/": ["/a"],
|
|
"https://x.com/a": ["/b"],
|
|
"https://x.com/b": ["/c"],
|
|
"https://x.com/c": [],
|
|
}
|
|
out = _run_crawl(page_links, "https://x.com/", depth=1, max_pages=10)
|
|
# Seed (depth 0) + /a (depth 1) only. /b is depth 2, skipped.
|
|
assert "https://x.com/" in out
|
|
assert "https://x.com/a" in out
|
|
assert "https://x.com/b" not in out
|
|
|
|
|
|
def test_crawl_seed_drops_off_domain_links():
|
|
page_links = {
|
|
"https://x.com/": ["https://other.com/foo", "/local"],
|
|
"https://x.com/local": [],
|
|
}
|
|
out = _run_crawl(page_links, "https://x.com/", depth=2, max_pages=10)
|
|
assert "https://other.com/foo" not in out
|
|
assert "https://x.com/local" in out
|
|
|
|
|
|
def test_crawl_seed_skips_feed_urls_during_discovery():
|
|
"""Feed/sitemap links must not be crawled — they're discarded at
|
|
ingest and a big feed is a slow wave straggler. Pin that the BFS
|
|
drops them at enqueue (not just at ingest)."""
|
|
page_links = {
|
|
"https://x.com/": [
|
|
"/a",
|
|
"/feeds/all.atom.xml", # big feed — must be skipped
|
|
"/sitemap.xml",
|
|
"/b.rss",
|
|
],
|
|
"https://x.com/a": [],
|
|
}
|
|
out = _run_crawl(page_links, "https://x.com/", depth=2, max_pages=10)
|
|
assert "https://x.com/a" in out
|
|
assert "https://x.com/feeds/all.atom.xml" not in out
|
|
assert "https://x.com/sitemap.xml" not in out
|
|
assert "https://x.com/b.rss" not in out
|
|
|
|
|
|
def test_crawl_seed_dedups():
|
|
page_links = {
|
|
"https://x.com/": ["/a", "/a", "/b"],
|
|
"https://x.com/a": ["/b"],
|
|
"https://x.com/b": ["/a"],
|
|
}
|
|
out = _run_crawl(page_links, "https://x.com/", depth=3, max_pages=10)
|
|
# Each URL appears at most once even with cycles.
|
|
assert len(out) == len(set(out))
|
|
|
|
|
|
def test_crawl_seed_fast_flag_creates_fast_mode_fetcher(monkeypatch):
|
|
"""When fast=True and no fetcher is supplied, the bridge constructs an
|
|
AsyncWebFetcher with fast_mode=True. Pin so the kwarg can't silently
|
|
drop on the floor."""
|
|
captured: dict = {}
|
|
|
|
class _CapturingFetcher:
|
|
def __init__(self, **kwargs):
|
|
captured.update(kwargs)
|
|
|
|
async def fetch_webpage(self, url, extract_links=False, **kw):
|
|
return None, []
|
|
|
|
from arborist.sources.crawler import bridge
|
|
|
|
monkeypatch.setattr(bridge, "AsyncWebFetcher", _CapturingFetcher)
|
|
bridge.crawl_seed("https://x.com/", max_depth=1, max_pages=1, fast=True)
|
|
assert captured.get("fast_mode") is True
|
|
|
|
|
|
def test_crawl_seed_default_creates_polite_fetcher(monkeypatch):
|
|
"""fast=False (default) → fast_mode=False on the underlying fetcher."""
|
|
captured: dict = {}
|
|
|
|
class _CapturingFetcher:
|
|
def __init__(self, **kwargs):
|
|
captured.update(kwargs)
|
|
|
|
async def fetch_webpage(self, url, extract_links=False, **kw):
|
|
return None, []
|
|
|
|
from arborist.sources.crawler import bridge
|
|
|
|
monkeypatch.setattr(bridge, "AsyncWebFetcher", _CapturingFetcher)
|
|
bridge.crawl_seed("https://x.com/", max_depth=1, max_pages=1)
|
|
assert captured.get("fast_mode") is False
|
|
|
|
|
|
def test_crawl_seed_passes_extract_links_true():
|
|
"""Regression: AsyncWebFetcher.fetch_webpage defaults extract_links=False,
|
|
so without the explicit kwarg BFS terminates after the seed (only one
|
|
URL ever lands in `discovered`). Pin the kwarg by recording the calls
|
|
on a stub fetcher."""
|
|
|
|
class _RecordingFetcher:
|
|
def __init__(self):
|
|
self.calls: list[dict] = []
|
|
|
|
async def fetch_webpage(self, url, extract_links=False, **kw):
|
|
self.calls.append({"url": url, "extract_links": extract_links})
|
|
return "<html></html>", []
|
|
|
|
import asyncio
|
|
from arborist.sources.crawler import bridge
|
|
|
|
fetcher = _RecordingFetcher()
|
|
asyncio.run(
|
|
bridge._crawl_seed_async(
|
|
"https://x.com/", max_depth=2, max_pages=5, fetcher=fetcher
|
|
)
|
|
)
|
|
assert fetcher.calls, "fetcher was never called"
|
|
for call in fetcher.calls:
|
|
assert call["extract_links"] is True, (
|
|
f"BFS must request link extraction; got {call!r}"
|
|
)
|
|
|
|
|
|
class _ConcurrencyFetcher:
|
|
"""Stub that records peak in-flight fetches so a test can prove the
|
|
BFS fans out (or doesn't). `max_page_workers` mimics what fast_mode
|
|
sets on the real AsyncWebFetcher; the bridge keys wave width off it.
|
|
"""
|
|
|
|
def __init__(self, page_links: dict[str, list[str]], max_page_workers: int):
|
|
self.page_links = page_links
|
|
self.max_page_workers = max_page_workers
|
|
self._in_flight = 0
|
|
self.peak_in_flight = 0
|
|
|
|
async def fetch_webpage(self, url: str, extract_links: bool = False, **kw):
|
|
import asyncio
|
|
|
|
self._in_flight += 1
|
|
self.peak_in_flight = max(self.peak_in_flight, self._in_flight)
|
|
try:
|
|
await asyncio.sleep(0.02) # hold the slot so overlap is observable
|
|
return "<html></html>", self.page_links.get(url, [])
|
|
finally:
|
|
self._in_flight -= 1
|
|
|
|
|
|
def test_crawl_seed_fast_fetches_a_wave_concurrently():
|
|
"""fast_mode sets max_page_workers>1 → a level of sibling pages is
|
|
fetched with multiple requests in flight, not one-at-a-time."""
|
|
import asyncio
|
|
|
|
from arborist.sources.crawler import bridge
|
|
|
|
# Seed fans out to 6 leaf pages, all at depth 1 — one wave.
|
|
page_links = {"https://x.com/": [f"/p{i}" for i in range(6)]}
|
|
fetcher = _ConcurrencyFetcher(page_links, max_page_workers=6)
|
|
out = asyncio.run(
|
|
bridge._crawl_seed_async(
|
|
"https://x.com/", max_depth=1, max_pages=0, fetcher=fetcher
|
|
)
|
|
)
|
|
assert len(out) == 7 # seed + 6 leaves
|
|
assert fetcher.peak_in_flight > 1, "fast crawl must overlap fetches in a wave"
|
|
|
|
|
|
def test_crawl_seed_polite_fetches_one_at_a_time():
|
|
"""max_page_workers==1 (polite default) → strictly sequential, so the
|
|
per-page crawl-delay still serialises same-domain fetches."""
|
|
import asyncio
|
|
|
|
from arborist.sources.crawler import bridge
|
|
|
|
page_links = {"https://x.com/": [f"/p{i}" for i in range(6)]}
|
|
fetcher = _ConcurrencyFetcher(page_links, max_page_workers=1)
|
|
asyncio.run(
|
|
bridge._crawl_seed_async(
|
|
"https://x.com/", max_depth=1, max_pages=0, fetcher=fetcher
|
|
)
|
|
)
|
|
assert fetcher.peak_in_flight == 1, "polite crawl must stay one-at-a-time"
|
|
|
|
|
|
def test_crawl_seed_closes_owned_fetcher(monkeypatch):
|
|
"""When the bridge builds the fetcher, it tears down the shared
|
|
session in a finally — no leaked sockets after a fast crawl."""
|
|
closed = {"count": 0}
|
|
|
|
class _ClosableFetcher:
|
|
max_page_workers = 4
|
|
|
|
def __init__(self, **kwargs):
|
|
pass
|
|
|
|
async def fetch_webpage(self, url, extract_links=False, **kw):
|
|
return "<html></html>", []
|
|
|
|
async def aclose(self):
|
|
closed["count"] += 1
|
|
|
|
from arborist.sources.crawler import bridge
|
|
|
|
monkeypatch.setattr(bridge, "AsyncWebFetcher", _ClosableFetcher)
|
|
bridge.crawl_seed("https://x.com/", max_depth=1, max_pages=1)
|
|
assert closed["count"] == 1, "bridge must close a fetcher it created"
|
|
|
|
|
|
def test_crawl_seed_leaves_injected_fetcher_open():
|
|
"""An injected fetcher belongs to the caller — the bridge must NOT
|
|
close its session (it may be reused across crawls)."""
|
|
import asyncio
|
|
|
|
closed = {"count": 0}
|
|
|
|
class _ClosableFetcher:
|
|
max_page_workers = 4
|
|
|
|
async def fetch_webpage(self, url, extract_links=False, **kw):
|
|
return "<html></html>", []
|
|
|
|
async def aclose(self):
|
|
closed["count"] += 1
|
|
|
|
from arborist.sources.crawler import bridge
|
|
|
|
fetcher = _ClosableFetcher()
|
|
asyncio.run(
|
|
bridge._crawl_seed_async(
|
|
"https://x.com/", max_depth=1, max_pages=1, fetcher=fetcher
|
|
)
|
|
)
|
|
assert closed["count"] == 0, "bridge must not close a caller-supplied fetcher"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ingest_crawled: stores ETag + Last-Modified per document_root
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.fixture
|
|
def http_meta_db(tmp_path):
|
|
db = tmp_path / "meta.db"
|
|
# Touch the DB so the schema is initialised, then close — we'll
|
|
# reopen as the test needs.
|
|
conn = connect(db)
|
|
try:
|
|
pass
|
|
finally:
|
|
conn.close()
|
|
return db
|
|
|
|
|
|
def _patched_httpx_get(url_to_response: dict[str, dict]):
|
|
"""Return a context-manager class that fakes httpx.Client.get."""
|
|
|
|
class _FakeClient:
|
|
def __init__(self, *a, **kw):
|
|
self._responses = url_to_response
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *a):
|
|
pass
|
|
|
|
def get(self, url):
|
|
spec = self._responses.get(url)
|
|
if spec is None:
|
|
raise httpx.HTTPError(f"no stub for {url}")
|
|
req = httpx.Request("GET", url)
|
|
return httpx.Response(
|
|
spec.get("status", 200),
|
|
headers=spec.get("headers", {"content-type": "text/html"}),
|
|
content=spec.get("body", "<html><body>hi</body></html>").encode(),
|
|
request=req,
|
|
)
|
|
|
|
def head(self, url, headers=None):
|
|
spec = self._responses.get(url)
|
|
if spec is None:
|
|
raise httpx.HTTPError(f"no stub for {url}")
|
|
head_status = spec.get("head_status", 200)
|
|
req = httpx.Request("HEAD", url, headers=headers or {})
|
|
return httpx.Response(
|
|
head_status,
|
|
headers=spec.get("head_headers", {}),
|
|
request=req,
|
|
)
|
|
|
|
return _FakeClient
|
|
|
|
|
|
def test_ingest_crawled_writes_http_meta(http_meta_db):
|
|
url = "https://x.com/page"
|
|
responses = {
|
|
url: {
|
|
"status": 200,
|
|
"headers": {
|
|
"content-type": "text/html",
|
|
"etag": '"abc123"',
|
|
"last-modified": "Tue, 28 Apr 2026 00:00:00 GMT",
|
|
},
|
|
"body": "<html><body><h1>Title</h1>Hello world.</body></html>",
|
|
}
|
|
}
|
|
|
|
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
|
conn = connect(http_meta_db)
|
|
try:
|
|
result = ingest_crawled(conn, [url])
|
|
finally:
|
|
conn.close()
|
|
|
|
assert result["http_meta_written"] == 1
|
|
assert len(result["documents"]) == 1
|
|
document_root = result["documents"][0]["document_root"]
|
|
|
|
conn = connect(http_meta_db)
|
|
try:
|
|
row = conn.execute(
|
|
"SELECT etag, last_modified, last_status, last_fetched_at, last_checked_at "
|
|
"FROM document_http_meta WHERE document_root = ?",
|
|
(document_root,),
|
|
).fetchone()
|
|
finally:
|
|
conn.close()
|
|
assert row["etag"] == '"abc123"'
|
|
assert row["last_modified"] == "Tue, 28 Apr 2026 00:00:00 GMT"
|
|
assert row["last_status"] == 200
|
|
assert row["last_fetched_at"] > 0
|
|
|
|
|
|
def test_ingest_crawled_skips_non_html(http_meta_db):
|
|
url = "https://x.com/file.pdf"
|
|
responses = {
|
|
url: {
|
|
"status": 200,
|
|
"headers": {"content-type": "application/pdf"},
|
|
"body": "%PDF-1.4 binary",
|
|
}
|
|
}
|
|
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
|
conn = connect(http_meta_db)
|
|
try:
|
|
result = ingest_crawled(conn, [url])
|
|
finally:
|
|
conn.close()
|
|
assert result["http_meta_written"] == 0
|
|
assert result["inserted"] == 0
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"feed_url",
|
|
[
|
|
"https://x.com/feeds/all.atom.xml",
|
|
"https://x.com/feed/",
|
|
"https://x.com/atom",
|
|
"https://x.com/rss",
|
|
"https://x.com/sitemap.xml",
|
|
"https://x.com/sitemaps/posts.xml",
|
|
"https://x.com/wp-rss2.xml",
|
|
"https://x.com/index.atom",
|
|
],
|
|
)
|
|
def test_ingest_crawled_skips_feed_urls_by_path(http_meta_db, feed_url):
|
|
"""Feed/sitemap URLs are crawl infrastructure, not knowledge. Skip
|
|
even before the fetch — pin so a future "but XML can be useful"
|
|
rewrite can't silently re-introduce feed pollution."""
|
|
# Response stub never gets used because the path filter is pre-fetch,
|
|
# but we provide one in case the filter regresses.
|
|
responses = {feed_url: {"status": 200, "headers": {"content-type": "text/html"}}}
|
|
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
|
conn = connect(http_meta_db)
|
|
try:
|
|
result = ingest_crawled(conn, [feed_url])
|
|
finally:
|
|
conn.close()
|
|
assert result["http_meta_written"] == 0
|
|
assert result["inserted"] == 0
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"ct",
|
|
[
|
|
"application/atom+xml",
|
|
"application/rss+xml",
|
|
"application/rdf+xml",
|
|
"application/xml",
|
|
"text/xml",
|
|
"application/atom+xml; charset=utf-8",
|
|
],
|
|
)
|
|
def test_ingest_crawled_skips_feed_content_types(http_meta_db, ct):
|
|
"""A feed served at a non-feed path still gets rejected by
|
|
Content-Type. xhtml is allowed (treated as HTML elsewhere)."""
|
|
url = "https://x.com/innocent-looking-path" # no feed-pattern in URL
|
|
responses = {
|
|
url: {
|
|
"status": 200,
|
|
"headers": {"content-type": ct},
|
|
"body": "<?xml version='1.0'?><feed></feed>",
|
|
}
|
|
}
|
|
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
|
conn = connect(http_meta_db)
|
|
try:
|
|
result = ingest_crawled(conn, [url])
|
|
finally:
|
|
conn.close()
|
|
assert result["http_meta_written"] == 0
|
|
assert result["inserted"] == 0
|
|
|
|
|
|
def test_ingest_crawled_keeps_xhtml(http_meta_db):
|
|
"""xhtml IS HTML — keep it, even though the content type ends in +xml."""
|
|
url = "https://x.com/page"
|
|
responses = {
|
|
url: {
|
|
"status": 200,
|
|
"headers": {"content-type": "application/xhtml+xml"},
|
|
"body": "<html><body><h1>T</h1>Real content here.</body></html>",
|
|
}
|
|
}
|
|
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
|
conn = connect(http_meta_db)
|
|
try:
|
|
result = ingest_crawled(conn, [url])
|
|
finally:
|
|
conn.close()
|
|
assert result["inserted"] == 1
|
|
assert result["http_meta_written"] == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# recrawl_check: conditional HEAD classification
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _seed_meta(conn, *, document_root: str, document_uri: str, etag: str | None = None):
|
|
"""Insert a documents row + http_meta row for recheck tests."""
|
|
from arborist.store import transaction
|
|
|
|
with transaction(conn):
|
|
conn.execute(
|
|
"INSERT INTO documents "
|
|
"(document_root, document_uri, source_type, kind, compression_depth, "
|
|
" title, chunking_version, canonicalization_version, schema_version, "
|
|
" ingest_ts, hit_count) "
|
|
"VALUES (?, ?, 'html', 'surface', 0, NULL, 'tok-512-v1', 'norm-v1', 'v9.8.0', ?, 0)",
|
|
(document_root, document_uri, int(time.time())),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO document_http_meta "
|
|
"(document_root, etag, last_modified, last_fetched_at, last_status, last_checked_at) "
|
|
"VALUES (?, ?, NULL, ?, 200, NULL)",
|
|
(document_root, etag, int(time.time())),
|
|
)
|
|
|
|
|
|
def test_recrawl_check_classifies_304_as_fresh(tmp_path):
|
|
db = tmp_path / "rc.db"
|
|
conn = connect(db)
|
|
try:
|
|
_seed_meta(conn, document_root="aa" * 32, document_uri="https://x.com/p", etag='"v1"')
|
|
finally:
|
|
conn.close()
|
|
|
|
responses = {"https://x.com/p": {"head_status": 304}}
|
|
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
|
conn = connect(db)
|
|
try:
|
|
result = recrawl_check(conn, limit=10)
|
|
finally:
|
|
conn.close()
|
|
assert result["fresh"] == 1
|
|
assert result["stale"] == 0
|
|
assert result["items"][0]["verdict"] == "fresh"
|
|
|
|
|
|
def test_recrawl_check_classifies_200_as_stale(tmp_path):
|
|
db = tmp_path / "rc.db"
|
|
conn = connect(db)
|
|
try:
|
|
_seed_meta(conn, document_root="bb" * 32, document_uri="https://x.com/q", etag='"v1"')
|
|
finally:
|
|
conn.close()
|
|
|
|
responses = {"https://x.com/q": {"head_status": 200}}
|
|
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
|
conn = connect(db)
|
|
try:
|
|
result = recrawl_check(conn, limit=10)
|
|
finally:
|
|
conn.close()
|
|
assert result["stale"] == 1
|
|
assert result["items"][0]["verdict"] == "stale"
|
|
|
|
|
|
def test_recrawl_check_classifies_404_as_gone(tmp_path):
|
|
db = tmp_path / "rc.db"
|
|
conn = connect(db)
|
|
try:
|
|
_seed_meta(conn, document_root="cc" * 32, document_uri="https://x.com/r", etag='"v1"')
|
|
finally:
|
|
conn.close()
|
|
|
|
responses = {"https://x.com/r": {"head_status": 404}}
|
|
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
|
conn = connect(db)
|
|
try:
|
|
result = recrawl_check(conn, limit=10)
|
|
finally:
|
|
conn.close()
|
|
assert result["gone"] == 1
|
|
|
|
|
|
def test_recrawl_check_domain_filter(tmp_path):
|
|
db = tmp_path / "rc.db"
|
|
conn = connect(db)
|
|
try:
|
|
_seed_meta(conn, document_root="11" * 32, document_uri="https://keep.com/a", etag='"v"')
|
|
_seed_meta(conn, document_root="22" * 32, document_uri="https://drop.com/a", etag='"v"')
|
|
finally:
|
|
conn.close()
|
|
|
|
responses = {
|
|
"https://keep.com/a": {"head_status": 304},
|
|
# drop.com would error if the filter is broken; we expect it to be skipped.
|
|
}
|
|
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
|
conn = connect(db)
|
|
try:
|
|
result = recrawl_check(conn, domain="keep.com", limit=10)
|
|
finally:
|
|
conn.close()
|
|
assert result["examined"] == 1
|
|
assert result["items"][0]["document_uri"] == "https://keep.com/a"
|
|
|
|
|
|
def test_recrawl_check_updates_last_checked_at(tmp_path):
|
|
db = tmp_path / "rc.db"
|
|
conn = connect(db)
|
|
try:
|
|
_seed_meta(conn, document_root="33" * 32, document_uri="https://x.com/s", etag='"v1"')
|
|
finally:
|
|
conn.close()
|
|
|
|
responses = {"https://x.com/s": {"head_status": 304}}
|
|
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
|
conn = connect(db)
|
|
try:
|
|
recrawl_check(conn, limit=10)
|
|
finally:
|
|
conn.close()
|
|
|
|
conn = connect(db)
|
|
try:
|
|
row = conn.execute(
|
|
"SELECT last_status, last_checked_at FROM document_http_meta WHERE document_root='33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33' || '33'"
|
|
).fetchone()
|
|
finally:
|
|
conn.close()
|
|
assert row["last_status"] == 304
|
|
assert row["last_checked_at"] is not None
|