aborist/arborist
modified: .gitlab-ci.yml modified: bench/qa_questions.txt modified: bench/qa_sweep.py modified: bench/run.sh modified: docs/TICKETS.md modified: docs/_source/README.md modified: docs/_source/_ext/makefile_targets.py modified: docs/_source/api/cli.rst modified: docs/_source/api/distill.rst modified: docs/_source/api/mesh.rst modified: docs/_source/api/qa.rst modified: docs/_source/api/retrieval.rst modified: docs/_source/api/storage.rst modified: docs/_source/api/substrate.rst modified: docs/_source/concepts.rst modified: docs/_source/conf.py modified: docs/_source/cookbook.rst modified: docs/_source/index.rst modified: docs/_source/license.rst modified: docs/_source/quickstart.rst modified: docs/bench-maxing.md modified: docs/benchmarks.md modified: docs/cti-architecture.md modified: docs/diagrams/aborist-modules.dot modified: docs/diagrams/aborist-modules.svg modified: docs/diagrams/mesh-data-flow.dot modified: docs/diagrams/mesh-epoch-lifecycle.dot modified: docs/diagrams/mesh-epoch-lifecycle.svg modified: docs/diagrams/mesh-group-decisions.dot modified: docs/diagrams/mesh-group-decisions.svg modified: docs/diagrams/mesh-identity-stack.dot modified: docs/diagrams/mesh-secret-envelope.dot modified: docs/mesh.md modified: docs/qa-modes-bench.md modified: docs/seven-point-program.md modified: docs/tickets/ticket-000001-retrieval-keywords-audit-gap.md modified: docs/tickets/ticket-000002-reference-frame-polarity-contract.md modified: docs/tickets/ticket-000003-anchor-class-warrant.md modified: docs/tickets/ticket-000005-label-ladder-migration.md modified: docs/tickets/ticket-000006-bench-emergent-findings.md modified: docs/tickets/ticket-000007-query-layer-hyphen-fold.md modified: docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md modified: docs/tickets/ticket-000009-quantifier-preflight-dag-binding.md modified: docs/tickets/ticket-000010-metacognition-preflight-guard.md modified: docs/tickets/ticket-000011-soft-preflight-hint-sidecar.md modified: scripts/backfill_concepts.py modified: scripts/bench_emergent.py modified: tests/crawler/test_async_web_fetcher.py modified: tests/crawler/test_bridge.py modified: tests/crawler/test_web_fetch.py modified: tests/test_bench_qa_sweep.py modified: tests/test_burn.py modified: tests/test_burn_doc.py modified: tests/test_claim_lattice.py modified: tests/test_cli_render.py modified: tests/test_compress.py modified: tests/test_concepts.py modified: tests/test_dag.py modified: tests/test_directives.py modified: tests/test_distill.py modified: tests/test_distill_recursive.py modified: tests/test_evict.py modified: tests/test_frame.py modified: tests/test_grok_source.py modified: tests/test_html_source.py modified: tests/test_ingest.py modified: tests/test_inspect.py modified: tests/test_journal.py modified: tests/test_keys.py modified: tests/test_llm_context_base.py modified: tests/test_merkle.py modified: tests/test_mesh.py modified: tests/test_mesh_aead.py modified: tests/test_mesh_chain.py modified: tests/test_mesh_cli.py modified: tests/test_mesh_cli_pull.py modified: tests/test_mesh_wire.py modified: tests/test_mesh_wire_e2e.py modified: tests/test_metacognition.py modified: tests/test_migration_audit_mode.py modified: tests/test_providence_source.py modified: tests/test_qa.py modified: tests/test_qa_quality_live.py modified: tests/test_quantifier_caps.py modified: tests/test_quantifier_classifier.py modified: tests/test_quantifier_phase4.py modified: tests/test_quantifier_reminder.py modified: tests/test_query.py modified: tests/test_reclassify.py modified: tests/test_repair.py modified: tests/test_resume.py modified: tests/test_snapshot.py modified: tests/test_soft_preflight.py modified: tests/test_tfidf.py modified: tests/test_vcs_source.py modified: tests/test_verify.py modified: tests/test_verify_json.py modified: tests/test_versioned_ingest.py modified: tests/test_warrant.py modified: tests/test_wikipedia_old.py modified: tests/test_wikipedia_xml.py modified: tests/test_wikitext.py
This commit is contained in:
parent
2c9d00ac95
commit
8d6961fcc1
102 changed files with 716 additions and 716 deletions
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
These tests stub the network. The verbatim crawler tests in this same
|
||||
directory exercise the underlying AsyncWebFetcher; here we exercise
|
||||
the aborist bridge logic on top of it (BFS bounds, same-domain filter,
|
||||
the arborist bridge logic on top of it (BFS bounds, same-domain filter,
|
||||
ETag capture, conditional HEAD classification).
|
||||
"""
|
||||
|
||||
|
|
@ -16,14 +16,14 @@ from unittest.mock import patch
|
|||
import httpx
|
||||
import pytest
|
||||
|
||||
from aborist.sources.crawler.bridge import (
|
||||
from arborist.sources.crawler.bridge import (
|
||||
_normalize,
|
||||
_same_domain,
|
||||
crawl_seed,
|
||||
ingest_crawled,
|
||||
recrawl_check,
|
||||
)
|
||||
from aborist.store import connect
|
||||
from arborist.store import connect
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -71,7 +71,7 @@ class _StubFetcher:
|
|||
def _run_crawl(page_links, seed, depth, max_pages):
|
||||
"""Helper: drive _crawl_seed_async with a stub fetcher."""
|
||||
import asyncio
|
||||
from aborist.sources.crawler import bridge
|
||||
from arborist.sources.crawler import bridge
|
||||
|
||||
return asyncio.run(
|
||||
bridge._crawl_seed_async(
|
||||
|
|
@ -162,7 +162,7 @@ def test_crawl_seed_fast_flag_creates_fast_mode_fetcher(monkeypatch):
|
|||
async def fetch_webpage(self, url, extract_links=False, **kw):
|
||||
return None, []
|
||||
|
||||
from aborist.sources.crawler import bridge
|
||||
from arborist.sources.crawler import bridge
|
||||
|
||||
monkeypatch.setattr(bridge, "AsyncWebFetcher", _CapturingFetcher)
|
||||
bridge.crawl_seed("https://x.com/", max_depth=1, max_pages=1, fast=True)
|
||||
|
|
@ -180,7 +180,7 @@ def test_crawl_seed_default_creates_polite_fetcher(monkeypatch):
|
|||
async def fetch_webpage(self, url, extract_links=False, **kw):
|
||||
return None, []
|
||||
|
||||
from aborist.sources.crawler import bridge
|
||||
from arborist.sources.crawler import bridge
|
||||
|
||||
monkeypatch.setattr(bridge, "AsyncWebFetcher", _CapturingFetcher)
|
||||
bridge.crawl_seed("https://x.com/", max_depth=1, max_pages=1)
|
||||
|
|
@ -202,7 +202,7 @@ def test_crawl_seed_passes_extract_links_true():
|
|||
return "<html></html>", []
|
||||
|
||||
import asyncio
|
||||
from aborist.sources.crawler import bridge
|
||||
from arborist.sources.crawler import bridge
|
||||
|
||||
fetcher = _RecordingFetcher()
|
||||
asyncio.run(
|
||||
|
|
@ -289,7 +289,7 @@ def test_ingest_crawled_writes_http_meta(http_meta_db):
|
|||
}
|
||||
}
|
||||
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(http_meta_db)
|
||||
try:
|
||||
result = ingest_crawled(conn, [url])
|
||||
|
|
@ -324,7 +324,7 @@ def test_ingest_crawled_skips_non_html(http_meta_db):
|
|||
"body": "%PDF-1.4 binary",
|
||||
}
|
||||
}
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(http_meta_db)
|
||||
try:
|
||||
result = ingest_crawled(conn, [url])
|
||||
|
|
@ -354,7 +354,7 @@ def test_ingest_crawled_skips_feed_urls_by_path(http_meta_db, feed_url):
|
|||
# Response stub never gets used because the path filter is pre-fetch,
|
||||
# but we provide one in case the filter regresses.
|
||||
responses = {feed_url: {"status": 200, "headers": {"content-type": "text/html"}}}
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(http_meta_db)
|
||||
try:
|
||||
result = ingest_crawled(conn, [feed_url])
|
||||
|
|
@ -386,7 +386,7 @@ def test_ingest_crawled_skips_feed_content_types(http_meta_db, ct):
|
|||
"body": "<?xml version='1.0'?><feed></feed>",
|
||||
}
|
||||
}
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(http_meta_db)
|
||||
try:
|
||||
result = ingest_crawled(conn, [url])
|
||||
|
|
@ -406,7 +406,7 @@ def test_ingest_crawled_keeps_xhtml(http_meta_db):
|
|||
"body": "<html><body><h1>T</h1>Real content here.</body></html>",
|
||||
}
|
||||
}
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(http_meta_db)
|
||||
try:
|
||||
result = ingest_crawled(conn, [url])
|
||||
|
|
@ -423,7 +423,7 @@ def test_ingest_crawled_keeps_xhtml(http_meta_db):
|
|||
|
||||
def _seed_meta(conn, *, document_root: str, document_uri: str, etag: str | None = None):
|
||||
"""Insert a documents row + http_meta row for recheck tests."""
|
||||
from aborist.store import transaction
|
||||
from arborist.store import transaction
|
||||
|
||||
with transaction(conn):
|
||||
conn.execute(
|
||||
|
|
@ -451,7 +451,7 @@ def test_recrawl_check_classifies_304_as_fresh(tmp_path):
|
|||
conn.close()
|
||||
|
||||
responses = {"https://x.com/p": {"head_status": 304}}
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(db)
|
||||
try:
|
||||
result = recrawl_check(conn, limit=10)
|
||||
|
|
@ -471,7 +471,7 @@ def test_recrawl_check_classifies_200_as_stale(tmp_path):
|
|||
conn.close()
|
||||
|
||||
responses = {"https://x.com/q": {"head_status": 200}}
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(db)
|
||||
try:
|
||||
result = recrawl_check(conn, limit=10)
|
||||
|
|
@ -490,7 +490,7 @@ def test_recrawl_check_classifies_404_as_gone(tmp_path):
|
|||
conn.close()
|
||||
|
||||
responses = {"https://x.com/r": {"head_status": 404}}
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(db)
|
||||
try:
|
||||
result = recrawl_check(conn, limit=10)
|
||||
|
|
@ -512,7 +512,7 @@ def test_recrawl_check_domain_filter(tmp_path):
|
|||
"https://keep.com/a": {"head_status": 304},
|
||||
# drop.com would error if the filter is broken; we expect it to be skipped.
|
||||
}
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(db)
|
||||
try:
|
||||
result = recrawl_check(conn, domain="keep.com", limit=10)
|
||||
|
|
@ -531,7 +531,7 @@ def test_recrawl_check_updates_last_checked_at(tmp_path):
|
|||
conn.close()
|
||||
|
||||
responses = {"https://x.com/s": {"head_status": 304}}
|
||||
with patch("aborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
with patch("arborist.sources.crawler.bridge.httpx.Client", _patched_httpx_get(responses)):
|
||||
conn = connect(db)
|
||||
try:
|
||||
recrawl_check(conn, limit=10)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue