crawl central-db + query auto-include + read-seam provenance
- make crawl-ingest writes to one central crawl db (CRAWL_DB, default ~/.arborist/crawl/web.db) instead of per-domain shards in the peer-shared main dir: keeps locally-crawled content out of peer sharing by default and a growing domain set under SQLite's 10-attach cap (Makefile, docs/crawler.md). - arborist query auto-includes the local crawl db (query() gains extra_shards; CLI --include-shard / --no-crawl-db, default-on when web.db exists). Fix latent --db single-file query AttributeError (cli.py). Persist used / used_pointer_ids + retrieval_purity into merkle_proof so read-only consumers can see which chunks fed the answer (qa/query.py). - arborist.read: read-only seam for dashboards / verifiers; on a multi-source context root surface the real primary source instead of the opaque corpus://multi-source sentinel (read.py). Backs the arborist-viz Merkle Command Center (#000069). - tests for extra_shards, the CLI crawl-db resolver, and the read seam.
This commit is contained in:
parent
6eada6dc89
commit
7f7eeefeb9
10 changed files with 3460 additions and 20 deletions
|
|
@ -383,3 +383,40 @@ def test_render_label_format_collapsed_surfaces_tail():
|
|||
out = _render_query_human(r, "winners of all major sports?")
|
||||
assert "format collapsed" in out
|
||||
assert "UNGROUNDED" in out
|
||||
|
||||
|
||||
# --- auto-include of the local crawl db -----------------------------
|
||||
|
||||
|
||||
def test_resolve_extra_shards_auto_includes_crawl_db(tmp_path, monkeypatch):
|
||||
"""`arborist query` auto-includes the local crawl db (when present),
|
||||
honors --no-crawl-db, and appends --include-shard extras de-duped."""
|
||||
import argparse
|
||||
|
||||
from arborist.cli import _default_crawl_db, _resolve_extra_shards
|
||||
|
||||
crawl_db = tmp_path / "web.db"
|
||||
crawl_db.write_bytes(b"") # resolver only checks presence
|
||||
monkeypatch.setenv("ARBORIST_CRAWL_DB", str(crawl_db))
|
||||
assert _default_crawl_db() == crawl_db
|
||||
|
||||
# present → auto-included
|
||||
ns = argparse.Namespace(no_crawl_db=False, include_shard=None)
|
||||
assert _resolve_extra_shards(ns) == [crawl_db]
|
||||
|
||||
# --no-crawl-db opts out
|
||||
ns_off = argparse.Namespace(no_crawl_db=True, include_shard=None)
|
||||
assert _resolve_extra_shards(ns_off) == []
|
||||
|
||||
# explicit --include-shard appends extras; crawl db deduped
|
||||
extra = tmp_path / "extra.db"
|
||||
extra.write_bytes(b"")
|
||||
ns_inc = argparse.Namespace(
|
||||
no_crawl_db=False, include_shard=[str(extra), str(crawl_db)]
|
||||
)
|
||||
assert _resolve_extra_shards(ns_inc) == [crawl_db, extra]
|
||||
|
||||
# missing crawl db → silently dropped
|
||||
monkeypatch.setenv("ARBORIST_CRAWL_DB", str(tmp_path / "nope.db"))
|
||||
ns_missing = argparse.Namespace(no_crawl_db=False, include_shard=None)
|
||||
assert _resolve_extra_shards(ns_missing) == []
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue