engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

crawl central-db + query auto-include + read-seam provenance

Browse source 
- make crawl-ingest writes to one central crawl db (CRAWL_DB, default
  ~/.arborist/crawl/web.db) instead of per-domain shards in the
  peer-shared main dir: keeps locally-crawled content out of peer
  sharing by default and a growing domain set under SQLite's 10-attach
  cap (Makefile, docs/crawler.md).

- arborist query auto-includes the local crawl db (query() gains
  extra_shards; CLI --include-shard / --no-crawl-db, default-on when
  web.db exists). Fix latent --db single-file query AttributeError
  (cli.py). Persist used / used_pointer_ids + retrieval_purity into
  merkle_proof so read-only consumers can see which chunks fed the
  answer (qa/query.py).

- arborist.read: read-only seam for dashboards / verifiers; on a
  multi-source context root surface the real primary source instead of
  the opaque corpus://multi-source sentinel (read.py). Backs the
  arborist-viz Merkle Command Center (#000069).

- tests for extra_shards, the CLI crawl-db resolver, and the read seam.
This commit is contained in:
russell@unturf.com 2026-05-29 13:45:47 -04:00
parent 6eada6dc89
commit 7f7eeefeb9
No known key found for this signature in database
10 changed files with 3460 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

24
Makefile
View file

@ -1331,18 +1331,20 @@ test-crawler: bootstrap-crawler ## run only the lifted crawler tests
# HEAD requests later. URL is required; DEPTH and MAX have safe defaults.
CRAWL_DEPTH ?= 2
CRAWL_MAX ?= 0
# Per-domain shard so `make query` (which reads $(SHARDS_DIR)) sees the
# crawled content. Shard filename derived from the seed URL's hostname:
# https://russell.ballestrini.net -> $(SHARDS_DIR)/crawl_russell_ballestrini_net.db
# Override with CRAWL_SHARD=... when you want a custom path.
crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] into $(SHARDS_DIR)/crawl_<domain>.db
@if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1 CRAWL_SHARD=path]" >&2; exit 2; fi
@mkdir -p $(SHARDS_DIR)
# General web crawls land in ONE central crawl db ($(CRAWL_DB)), inside
# $(CRAWL_SHARDS_DIR) so they stay OUT of the peer-shared main $(SHARDS_DIR)
# by default and a growing set of crawled domains never trips SQLite's
# 10-attached-database limit. Content-addressing lets many domains share
# one file safely (idempotent re-ingest; `supersedes` edges on change).
# To query it: `--db $(CRAWL_DB)` standalone, or attach it alongside the
# main corpus (still just one extra file, under the 10-attach cap).
# Override the path with CRAWL_DB=... (or CRAWL_SHARD=... for a one-off).
CRAWL_DB ?= $(CRAWL_SHARDS_DIR)/web.db
crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1 CRAWL_DB=path] into the central crawl db ($(CRAWL_DB))
@if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1 CRAWL_DB=path]" >&2; exit 2; fi
@mkdir -p $(CRAWL_SHARDS_DIR)
@shard="$(CRAWL_SHARD)"; \
if [ -z "$$shard" ]; then \
domain=$$(echo "$(URL)" | sed -E 's,^https?://([^/]+).*$$,\1,' | tr '.' '_'); \
shard="$(SHARDS_DIR)/crawl_$${domain}.db"; \
fi; \
if [ -z "$$shard" ]; then shard="$(CRAWL_DB)"; fi; \
echo " shard: $$shard" >&2; \
$(ARBORIST) --db "$$shard" crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest

52
arborist/cli.py
View file

@ -488,6 +488,37 @@ def _cmd_ask(args: argparse.Namespace) -> int:
return 0 if result.get("status") in ("cache_hit", "cache_miss_then_written") else 1
def _default_crawl_db() -> Path:
"""Path of the local crawl db auto-included on the read path.
A single central file for locally web-crawled content (``make
crawl-ingest``). Kept SEPARATE from the peer-shared main shards so
it is never bundled into cold-packs / mesh by default; only the read
path attaches it. Override with ``ARBORIST_CRAWL_DB``."""
import os
env = os.environ.get("ARBORIST_CRAWL_DB")
if env:
return Path(env).expanduser()
return Path.home() / ".arborist" / "crawl" / "web.db"
def _resolve_extra_shards(args: argparse.Namespace) -> list[Path]:
"""Extra shard files to search alongside the main corpus: the local
crawl db (auto-included unless ``--no-crawl-db``) plus any explicit
``--include-shard`` paths. Missing files are dropped silently — the
crawl db only exists once something has been crawled locally."""
extras: list[Path] = []
if not getattr(args, "no_crawl_db", False):
crawl_db = _default_crawl_db()
if crawl_db.is_file():
extras.append(crawl_db)
for p in (getattr(args, "include_shard", None) or []):
pp = Path(p).expanduser()
if pp.is_file() and pp not in extras:
extras.append(pp)
return extras
def _cmd_query(args: argparse.Namespace) -> int:
"""Multi-source RAG: question -> top-K corpus docs -> Hermes -> cache."""
import os
@ -527,6 +558,7 @@ def _cmd_query(args: argparse.Namespace) -> int:
Path(args.global_shards_dir) if args.global_shards_dir else None
)
single_db = None if shards_dir else args.db
extra_shards = _resolve_extra_shards(args)
# Apply per-call policy overrides (question_dedup, repair, answer_mode)
# on top of the default. fidelity is a function-level kwarg, not in
@ -621,6 +653,7 @@ def _cmd_query(args: argparse.Namespace) -> int:
quantization=quantization,
shards_dir=shards_dir,
single_db=single_db,
extra_shards=extra_shards,
top_k=args.top_k,
over_fetch=args.over_fetch,
max_context_chars=args.max_context_chars,
@ -645,7 +678,7 @@ def _cmd_query(args: argparse.Namespace) -> int:
# so the render-layer warrant-chain tail can look up
# warrant-resolver derivations without needing args. Stripped
# before JSON output to keep the json shape stable.
_shards_dir = args.global_shards_dir or args.shards_dir
_shards_dir = args.global_shards_dir or getattr(args, "shards_dir", None)
if _shards_dir:
result["_shards_dir"] = str(_shards_dir)
@ -5340,6 +5373,23 @@ def build_parser() -> argparse.ArgumentParser:
"--over-fetch", dest="over_fetch", type=int, default=32,
help="FTS5 hits to fetch per shard before dedup (default 32)",
)
query_cmd.add_argument(
"--include-shard", dest="include_shard", action="append", default=None,
metavar="PATH",
help=(
"extra shard db file(s) to search alongside the main corpus "
"(repeatable). Per-shard fan-out, so no 10-attach limit here. "
"The local crawl db (ARBORIST_CRAWL_DB, default "
"~/.arborist/crawl/web.db) is auto-included when present."
),
)
query_cmd.add_argument(
"--no-crawl-db", dest="no_crawl_db", action="store_true",
help=(
"do not auto-include the local crawl db; query only the main "
"corpus (--shards-dir / --db)."
),
)
query_cmd.add_argument(
"--max-context-chars", dest="max_context_chars", type=int, default=None,
help=(

27
arborist/qa/query.py
View file

@ -1375,6 +1375,7 @@ def _search_corpus(
question: str,
over_fetch: int,
*,
extra_shards: list[Path] | None = None,
progress: Progress | None = None,
) -> list[_Hit]:
"""Two parallel searches across shards, merged:
@ -1425,6 +1426,14 @@ def _search_corpus(
paths = [Path(single_db)]
else:
paths = []
# Opt-in extra shards (e.g. the local crawl db auto-included by the
# CLI) searched alongside the main corpus. Per-shard fan-out below
# means there is no SQLite attach-limit ceiling here. Dedup so a path
# already discovered in shards_dir isn't searched twice.
for x in (extra_shards or []):
xp = Path(x)
if xp not in paths:
paths.append(xp)
progress = progress or _progress_disabled()
@ -1936,6 +1945,7 @@ def query(
quantization: str = "",
shards_dir: Path | None = None,
single_db: Path | None = None,
extra_shards: list[Path] | None = None,
top_k: int = 8,
over_fetch: int = 32,
max_context_chars: int | None = None,
@ -2753,7 +2763,8 @@ def query(
)
t_phase = time.monotonic()
hits, core_match_roots, phrase_match_roots, root_to_shard = _search_corpus(
shards_dir, single_db, retrieval_query, over_fetch, progress=progress,
shards_dir, single_db, retrieval_query, over_fetch,
extra_shards=extra_shards, progress=progress,
)
if not hits:
return {
@ -3604,7 +3615,10 @@ def query(
for h in chosen
],
}
proof_blob = json.dumps(proof_obj, separators=(",", ":"))
# proof_blob is serialized AFTER the lattice block below, so the
# persisted merkle_proof carries the `used` / `used_pointer_ids`
# annotations and `retrieval_purity` that the JSON result already
# exposes — see the note at the json.dumps call.
# Per-run Merkle-DAG. Quote mode base shape: 7 stages
# (question / retrieval / context / prompt / answer / verify
@ -3692,6 +3706,15 @@ def query(
),
}
proof_obj["retrieval_purity"] = retrieval_purity
# Serialize merkle_proof here (not at proof_obj construction) so the
# stored blob includes the render-layer `used` / `used_pointer_ids`
# per source and `retrieval_purity`. merkle_proof is stored
# provenance, NOT a commitment — the providence_query audit-event
# body excludes it — so adding these derived signals does not touch
# the proof path. Lets read-only consumers (the VIZ dashboard)
# highlight the chunks that actually fed the answer.
proof_blob = json.dumps(proof_obj, separators=(",", ":"))
# Retrieval-plan hash (Ticket #000001 / Directive D4):
# capture the operator-influenceable retrieval inputs so the
# run-DAG's retrieval stage binds BOTH plan (what guided the

965
arborist/read.py Normal file
View file

@ -0,0 +1,965 @@
"""arborist.read — read-only seam for dashboards and verifiers.
Counterpart to :mod:`arborist.embed`: that module is the seam embedders
use to *write* into arborist (Document → ingest → root). This module is
the seam read-only consumers (Merkle Command Center / arborist-viz,
third-party verifiers, archival mirrors) use to *read* from arborist.
Why a seam? Same reason as ``arborist.embed`` — so internal refactors
(schema bumps, compression scheme changes, sharding plans, wikitext
projection version, qa-vs-document shard separation) don't break the
downstream consumers. Consumers import from here; they never touch
arborist's tables, columns, or helper modules directly.
Surface summary
---------------
::
open_shards(paths) → Shards
shards.roots(limit=50) → list[Root]
shards.root(document_root) → Root | None
shards.leaves(document_root, *, reveal_private=False, project=True)
→ list[Leaf]
shards.proof(document_root, leaf_index)
→ Proof
shards.tree_layers(document_root) → list[list[str]]
shards.audit_event(event_hash) → AuditEvent | None
shards.audit_chain(head, limit) → list[AuditEvent]
shards.audit_recent(limit) → list[AuditEvent]
shards.audit_by_root(root, limit) → list[AuditEvent]
shards.audit_since(cursor) → (list[AuditEvent], dict)
shards.qa(cache_key) → QaRecord | None
shards.qa_recent(limit) → list[QaRecord]
shards.qa_by_root(root, limit) → list[QaRecord]
shards.qa_search(text, limit) → list[QaRecord]
shards.resolve(hex_hash) → ResolveResult
shards.counts() → ShardCounts
Privacy
-------
By default, ``leaves()`` does not decompress chunk content (the bytes
the leaf hash commits to). Pass ``reveal_private=True`` to opt in. The
chunk content stays compressed-on-disk regardless; reveal only affects
whether the read seam decompresses and returns it.
The ``project=True`` (default) flag adds a deterministic prose
projection (:mod:`arborist.wikitext.to_base`, versioned by
``BASE_VERSION``) alongside the raw bytes. Pass ``project=False`` to
skip the projection when only raw bytes are needed.
Privacy does **not** apply to: hashes, leaf indices, audit-event
metadata, providence_cache rows excluding ``answer_text`` /
``question_text`` (which are part of the Q&A record by design).
"""
from __future__ import annotations
import json
import logging
import sqlite3
from collections.abc import Iterable
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
from arborist.compress import unpack_chunk
from arborist.merkle import MerkleTree, proof_to_dict, verify_proof
from arborist.store import connect
log = logging.getLogger(__name__)
__all__ = [
"open_shards",
"Shards",
"Root",
"Leaf",
"Proof",
"AuditEvent",
"QaRecord",
"ContextRoot",
"ResolveResult",
"ShardCounts",
]
# ---------- dataclasses ------------------------------------------------
@dataclass(frozen=True)
class Root:
"""One document root + its surface metadata. ``shard_path`` lets the
consumer cite where the answer came from."""
document_root: str
document_uri: str
source_type: str
kind: str
title: Optional[str]
chunking_version: str
canonicalization_version: str
schema_version: str
ingest_ts: int
hit_count: int
leaf_count: int
shard_path: str
@dataclass(frozen=True)
class Leaf:
"""One chunk of a document. ``content`` is the raw bytes-under-the-hash
(decompressed only when ``reveal_private=True`` was passed). ``prose``
is the deterministic projection — derivable from ``content``, versioned
by ``base_version``."""
document_root: str
idx: int
leaf_hash: str
tier: str
content: Optional[str] = None
prose: Optional[str] = None
base_version: Optional[str] = None
@dataclass(frozen=True)
class Proof:
"""A Merkle inclusion proof. Hash check is performed at construction;
``passed`` is final."""
document_root: str
leaf_index: int
leaf_hash: str
sibling_hashes: list[str]
left_right_flags: list[bool]
computed_root: str
expected_root: str
passed: bool
shard_path: str
raw: dict # the arborist.merkle.proof_to_dict form, for verifiers
@dataclass(frozen=True)
class AuditEvent:
seq: int
event_hash: str
prev_event_hash: Optional[str]
event_type: str
subject_root: Optional[str]
body: dict
ts: int
shard_path: str
@dataclass(frozen=True)
class QaRecord:
"""One providence_cache row — the v9.8 8-dim invariant + the cached
answer. Only consumers that respect the §14 privacy contract should
surface ``question_text`` / ``answer_text``."""
cache_key: str
source_root: str
document_uri: str
question_hash: str
question_text: str
answer_text: str
model_profile_hash: str
conversation_hash: str
governance_policy_hash: str
schema_version: str
canonicalization_version: str
chunking_version: str
falsification_state: str
audit_mode: Optional[str]
audit_event_hash: Optional[str]
run_dag_root: Optional[str]
created_at: int
last_hit_at: Optional[int]
hit_count: int
n_quotes: Optional[int]
n_verified: Optional[int]
verifier_method: Optional[str]
merkle_proof: dict
run_dag_blob: dict
shard_path: str
@dataclass(frozen=True)
class ContextRoot:
"""A synthetic root for a multi-source QA context.
Arborist's providence_cache rows reference a ``source_root`` that
isn't always a single document — when a Q&A is answered against
chunks assembled from several documents, the ``source_root`` is a
merkle root *over those sources*. That root has no row in the
``documents`` table; its evidence manifest lives in the QA record's
``merkle_proof`` blob.
A ``ContextRoot`` exposes that synthesized identity so consumers can
look up any hash uniformly via the read seam, without knowing whether
the row landed in ``documents`` or only in ``providence_cache``.
QA may eventually merge back into the document shards; that's the
seam's problem, not the consumer's.
"""
context_root: str
cache_key: str
question_text: str
document_uri: str
sources: list[dict]
chunking_version: str
canonicalization_version: str
schema_version: str
created_at: int
hit_count: int
shard_path: str
# Real provenance, derived from `sources`. ``document_uri`` above is
# headlined with ``primary_source_uri`` (instead of the opaque
# ``corpus://multi-source`` sentinel) so consumers show where the
# knowledge actually came from. ``source_domains`` lists the distinct
# contributing hostnames, primary answer source first.
primary_source_uri: Optional[str] = None
source_domains: list[str] = field(default_factory=list)
@dataclass(frozen=True)
class ResolveResult:
"""Outcome of ``shards.resolve(hex_hash)``. ``kind`` is one of:
document_root | leaf_hash | audit_event | merkle_interior |
qa_cache_key | context_root | qa_run_dag_root | unknown.
``context_root`` fires when the hash matches a ``providence_cache``
row's ``source_root`` but isn't itself a ``documents`` row — i.e. a
synthesized multi-source context (see :class:`ContextRoot`)."""
kind: str
hash: str
shard_path: Optional[str] = None
extra: dict = field(default_factory=dict)
@dataclass(frozen=True)
class ShardCounts:
documents: int
audit_events: int
qa_records: int
shard_count: int
# ---------- the Shards handle ------------------------------------------
class Shards:
"""Read-only handle over one or more arborist shards.
Open with :func:`open_shards`. The handle holds a per-shard SQLite
connection (opened ``mode=ro``) and fans out every query across the
set. ``mode=ro`` plus ``check_same_thread=False`` makes a single
connection safe for read-only sharing across a thread pool (waitress,
gunicorn) — see SQLite docs on shared-read connections.
The schema versions arborist stores are *not* exposed by Shards as
a strict version check; that is the caller's job if they want it.
Schema migrations are arborist's contract, not the seam's.
"""
def __init__(self, paths: list[str]):
self._paths: list[str] = list(paths)
self._conns: dict[str, sqlite3.Connection] = {}
for p in self._paths:
try:
# Run arborist's migration probe via the canonical connect()
# to bring the schema current, then drop it.
connect(p).close()
conn = sqlite3.connect(
"file:{0}?mode=ro".format(p), uri=True, check_same_thread=False
)
conn.row_factory = sqlite3.Row
self._conns[p] = conn
log.debug("arborist.read: opened %s (read-only)", p)
except Exception as exc:
log.warning("arborist.read: failed to open shard %s: %s", p, exc)
# ---------- meta ---------------------------------------------------
@property
def paths(self) -> list[str]:
return list(self._conns.keys())
def close(self) -> None:
for c in self._conns.values():
try:
c.close()
except Exception:
pass
self._conns.clear()
def counts(self) -> ShardCounts:
docs = events = qa = 0
for c in self._conns.values():
try:
docs += int(c.execute("SELECT COUNT(*) FROM documents").fetchone()[0])
except sqlite3.Error:
pass
try:
events += int(
c.execute("SELECT COALESCE(MAX(seq), 0) FROM audit_events").fetchone()[0]
)
except sqlite3.Error:
pass
try:
qa += int(c.execute("SELECT COUNT(*) FROM providence_cache").fetchone()[0])
except sqlite3.Error:
pass
return ShardCounts(documents=docs, audit_events=events, qa_records=qa, shard_count=len(self._conns))
# ---------- documents / roots --------------------------------------
def root(self, document_root: str) -> Optional[Root]:
for path, c in self._conns.items():
r = c.execute(
"SELECT document_root, document_uri, source_type, kind, title, "
" chunking_version, canonicalization_version, schema_version, "
" ingest_ts, hit_count "
"FROM documents WHERE document_root = ?",
(document_root,),
).fetchone()
if r is None:
continue
leaf_count = c.execute(
"SELECT COUNT(*) FROM chunks WHERE document_root = ?",
(document_root,),
).fetchone()[0]
return Root(leaf_count=int(leaf_count), shard_path=path, **dict(r))
return None
def roots(self, limit: int = 50) -> list[Root]:
merged: list[Root] = []
for path, c in self._conns.items():
rows = c.execute(
"SELECT document_root, document_uri, source_type, kind, title, "
" chunking_version, canonicalization_version, schema_version, "
" ingest_ts, hit_count "
"FROM documents ORDER BY ingest_ts DESC LIMIT ?",
(limit,),
).fetchall()
for r in rows:
d = dict(r)
lc = c.execute(
"SELECT COUNT(*) FROM chunks WHERE document_root = ?",
(d["document_root"],),
).fetchone()[0]
merged.append(Root(leaf_count=int(lc), shard_path=path, **d))
merged.sort(key=lambda r: r.ingest_ts, reverse=True)
return merged[:limit]
# ---------- leaves / chunks ----------------------------------------
def leaves(
self,
document_root: str,
*,
reveal_private: bool = False,
project: bool = True,
) -> list[Leaf]:
"""Read every chunk under ``document_root`` from its shard.
Decompresses content only when ``reveal_private=True`` (§14
default-deny). When ``project=True`` (default) and the
``arborist[wikitext]`` extra is installed, also computes the
deterministic prose projection alongside the raw bytes.
Returns an empty list when the root is unknown.
"""
conn = self._owning_conn_for_root(document_root)
if conn is None:
return []
rows = conn.execute(
"SELECT document_root, idx, leaf_hash, content, tier "
"FROM chunks WHERE document_root = ? ORDER BY idx",
(document_root,),
).fetchall()
to_base, base_version = _get_wikitext_projector() if project else (None, None)
out: list[Leaf] = []
for r in rows:
content: Optional[str] = None
prose: Optional[str] = None
stamp: Optional[str] = None
if reveal_private and r["content"] is not None:
try:
content = unpack_chunk(r["content"])
except Exception as exc:
log.warning("unpack_chunk failed for leaf %s: %s", r["leaf_hash"], exc)
if content is not None and to_base is not None:
try:
prose = to_base(content)
stamp = base_version
except Exception as exc:
log.warning("wikitext.to_base failed for leaf %s: %s", r["leaf_hash"], exc)
out.append(
Leaf(
document_root=r["document_root"],
idx=int(r["idx"]),
leaf_hash=r["leaf_hash"],
tier=r["tier"],
content=content,
prose=prose,
base_version=stamp,
)
)
return out
# ---------- proofs / tree ------------------------------------------
def proof(self, document_root: str, leaf_index: int) -> Optional[Proof]:
"""Compute a Merkle inclusion proof against the shard that owns
the root. Returns ``None`` if the root is unknown; raises
``IndexError`` if ``leaf_index`` is out of range."""
conn = self._owning_conn_for_root(document_root)
path = self._path_for_root(document_root)
if conn is None or path is None:
return None
rows = conn.execute(
"SELECT idx, leaf_hash FROM chunks WHERE document_root = ? ORDER BY idx",
(document_root,),
).fetchall()
if not rows:
return None
if leaf_index < 0 or leaf_index >= len(rows):
raise IndexError(
"leaf_index {0} out of range (root has {1} leaves)".format(
leaf_index, len(rows)
)
)
leaves = [bytes.fromhex(r["leaf_hash"]) for r in rows]
tree = MerkleTree.build(leaves)
p = tree.proof(leaf_index)
passed = verify_proof(p) and tree.root.hex() == document_root
raw = proof_to_dict(p)
return Proof(
document_root=document_root,
leaf_index=leaf_index,
leaf_hash=p.leaf.hex(),
sibling_hashes=[s["hash"] for s in raw["siblings"]],
left_right_flags=[bool(s["is_left"]) for s in raw["siblings"]],
computed_root=tree.root.hex(),
expected_root=document_root,
passed=passed,
shard_path=path,
raw=raw,
)
def tree_layers(self, document_root: str) -> Optional[list[list[str]]]:
"""All Merkle tree layers as hex (layer 0 = leaves, last = root).
Drives the 3D lattice visualization."""
conn = self._owning_conn_for_root(document_root)
if conn is None:
return None
rows = conn.execute(
"SELECT leaf_hash FROM chunks WHERE document_root = ? ORDER BY idx",
(document_root,),
).fetchall()
if not rows:
return None
leaves = [bytes.fromhex(r["leaf_hash"]) for r in rows]
tree = MerkleTree.build(leaves)
return [[node.hex() for node in layer] for layer in tree.layers]
# ---------- audit --------------------------------------------------
def audit_event(self, event_hash: str) -> Optional[AuditEvent]:
for path, c in self._conns.items():
r = c.execute(
"SELECT seq, event_hash, prev_event_hash, event_type, subject_root, body, ts "
"FROM audit_events WHERE event_hash = ?",
(event_hash,),
).fetchone()
if r is not None:
return _row_to_audit(r, path)
return None
def audit_chain(self, head_event_hash: str, *, limit: int = 100) -> list[AuditEvent]:
out: list[AuditEvent] = []
cursor = head_event_hash
for _ in range(limit):
if cursor is None:
break
ev = self.audit_event(cursor)
if ev is None:
break
out.append(ev)
cursor = ev.prev_event_hash
return out
def audit_recent(self, *, limit: int = 100) -> list[AuditEvent]:
merged: list[AuditEvent] = []
for path, c in self._conns.items():
for r in c.execute(
"SELECT seq, event_hash, prev_event_hash, event_type, subject_root, body, ts "
"FROM audit_events ORDER BY seq DESC LIMIT ?",
(limit,),
).fetchall():
merged.append(_row_to_audit(r, path))
merged.sort(key=lambda e: e.ts, reverse=True)
return merged[:limit]
def audit_by_root(self, root: str, *, limit: int = 50) -> list[AuditEvent]:
merged: list[AuditEvent] = []
for path, c in self._conns.items():
for r in c.execute(
"SELECT seq, event_hash, prev_event_hash, event_type, subject_root, body, ts "
"FROM audit_events WHERE subject_root = ? ORDER BY seq DESC LIMIT ?",
(root, limit),
).fetchall():
merged.append(_row_to_audit(r, path))
merged.sort(key=lambda e: e.seq, reverse=True)
return merged[:limit]
def audit_since(
self, cursor: Optional[dict] = None, *, limit: int = 200
) -> tuple[list[AuditEvent], dict]:
"""Stream-friendly tail. ``cursor`` is opaque to the caller — pass
``None`` first time, then pass the dict returned from the previous
call. Returns ``(new_events, next_cursor)``.
Each shard tracks its own ``last_seq`` so the seam can drive an
SSE stream over all shards without the consumer knowing the
sharding layout.
"""
cursor = dict(cursor or {})
new_events: list[AuditEvent] = []
for path, c in self._conns.items():
last_seq = int(cursor.get(path, 0))
for r in c.execute(
"SELECT seq, event_hash, prev_event_hash, event_type, subject_root, body, ts "
"FROM audit_events WHERE seq > ? ORDER BY seq ASC LIMIT ?",
(last_seq, limit),
).fetchall():
ev = _row_to_audit(r, path)
new_events.append(ev)
cursor[path] = max(int(cursor.get(path, 0)), ev.seq)
return new_events, cursor
def audit_cursor(self) -> dict:
"""Cursor at the current head of every shard (use as starting
point for a tail that only wants future events)."""
return {
path: int(c.execute("SELECT COALESCE(MAX(seq), 0) FROM audit_events").fetchone()[0])
for path, c in self._conns.items()
}
# ---------- providence_cache (Q&A) ---------------------------------
def context(self, context_root: str) -> Optional[ContextRoot]:
"""Synthesize a :class:`ContextRoot` for a multi-source QA root.
Looks up the first ``providence_cache`` row whose ``source_root``
equals ``context_root`` and surfaces the evidence manifest parsed
out of its ``merkle_proof`` JSON. Returns ``None`` when no QA
record cites this root.
Callers should fall through from :meth:`root` to this method when
a hash is "a root but not a document" — that case happens when
the QA pipeline answered against assembled context rather than a
single ingested document. Consumers don't need to know which
case applies; they just chain the calls.
"""
for path, c in self._conns.items():
try:
r = c.execute(
_QA_SELECT + " WHERE source_root = ? LIMIT 1",
(context_root,),
).fetchone()
except sqlite3.Error:
continue
if r is None:
continue
qa = _row_to_qa(r, path)
sources = qa.merkle_proof.get("sources", []) if isinstance(qa.merkle_proof, dict) else []
primary_uri, domains = _summarize_sources(sources)
# Headline the real source. The stored document_uri for a
# multi-source answer is the opaque ``corpus://multi-source``
# sentinel — replace it with the primary answer source so the
# node says where the knowledge came from (e.g.
# russell.ballestrini.net), not a placeholder.
headline_uri = qa.document_uri or ""
if (not headline_uri or headline_uri == _MULTI_SOURCE_URI) and primary_uri:
headline_uri = primary_uri
return ContextRoot(
context_root=context_root,
cache_key=qa.cache_key,
question_text=qa.question_text or "",
document_uri=headline_uri,
sources=sources,
chunking_version=qa.chunking_version,
canonicalization_version=qa.canonicalization_version,
schema_version=qa.schema_version,
created_at=qa.created_at,
hit_count=qa.hit_count,
shard_path=path,
primary_source_uri=primary_uri,
source_domains=domains,
)
return None
def context_layers(self, context_root: str) -> Optional[list[list[str]]]:
"""Rebuild the context tree's layers from the sorted source roots.
Mirrors :meth:`tree_layers` for synthetic context roots. Layer 0
is leaves (the sorted source document_roots); the last layer is
``[context_root]``. Uses the same canonicalization as
:func:`arborist.qa.query._context_root` — sort source roots, build
a :class:`MerkleTree`, return its layers as hex.
Returns ``None`` when the context root isn't in providence_cache.
"""
ctx = self.context(context_root)
if ctx is None:
return None
leaves_hex = _context_leaf_hashes(ctx.sources)
if not leaves_hex:
return [[context_root]]
leaves = [bytes.fromhex(h) for h in leaves_hex]
tree = MerkleTree.build(leaves)
return [[node.hex() for node in layer] for layer in tree.layers]
def context_proof(self, context_root: str, leaf_index: int) -> Optional[Proof]:
"""Inclusion proof for one source under a context root.
Mirrors :meth:`proof`. The leaf at index ``leaf_index`` is the
sorted-source document_root. Raises ``IndexError`` if out of
range. Returns ``None`` when the context root isn't known.
"""
ctx = self.context(context_root)
if ctx is None:
return None
leaves_hex = _context_leaf_hashes(ctx.sources)
if not leaves_hex:
return None
if leaf_index < 0 or leaf_index >= len(leaves_hex):
raise IndexError(
"leaf_index {0} out of range (context has {1} sources)".format(
leaf_index, len(leaves_hex)
)
)
leaves = [bytes.fromhex(h) for h in leaves_hex]
tree = MerkleTree.build(leaves)
p = tree.proof(leaf_index)
passed = verify_proof(p) and tree.root.hex() == context_root
raw = proof_to_dict(p)
return Proof(
document_root=context_root,
leaf_index=leaf_index,
leaf_hash=p.leaf.hex(),
sibling_hashes=[s["hash"] for s in raw["siblings"]],
left_right_flags=[bool(s["is_left"]) for s in raw["siblings"]],
computed_root=tree.root.hex(),
expected_root=context_root,
passed=passed,
shard_path=ctx.shard_path,
raw=raw,
)
def qa(self, cache_key: str) -> Optional[QaRecord]:
for path, c in self._conns.items():
r = c.execute(
_QA_SELECT + " WHERE cache_key = ?", (cache_key,)
).fetchone()
if r is not None:
return _row_to_qa(r, path)
return None
def qa_recent(self, *, limit: int = 50) -> list[QaRecord]:
merged: list[QaRecord] = []
for path, c in self._conns.items():
for r in c.execute(
_QA_SELECT + " ORDER BY created_at DESC LIMIT ?", (limit,)
).fetchall():
merged.append(_row_to_qa(r, path))
merged.sort(key=lambda q: q.created_at, reverse=True)
return merged[:limit]
def qa_by_root(self, source_root: str, *, limit: int = 50) -> list[QaRecord]:
merged: list[QaRecord] = []
for path, c in self._conns.items():
for r in c.execute(
_QA_SELECT + " WHERE source_root = ? ORDER BY created_at DESC LIMIT ?",
(source_root, limit),
).fetchall():
merged.append(_row_to_qa(r, path))
merged.sort(key=lambda q: q.created_at, reverse=True)
return merged[:limit]
def qa_search(self, text: str, *, limit: int = 50) -> list[QaRecord]:
"""Substring search over ``question_text`` (case-insensitive).
FTS5 isn't indexed on providence_cache rows in the current
arborist schema; a LIKE scan over 4-5k QA records is cheap
enough for v1. Swap to FTS5 if the corpus grows past ~100k
records.
"""
pat = "%{0}%".format(text.replace("%", r"\%").replace("_", r"\_"))
merged: list[QaRecord] = []
for path, c in self._conns.items():
for r in c.execute(
_QA_SELECT + " WHERE question_text LIKE ? ESCAPE '\\' "
"ORDER BY created_at DESC LIMIT ?",
(pat, limit),
).fetchall():
merged.append(_row_to_qa(r, path))
merged.sort(key=lambda q: q.created_at, reverse=True)
return merged[:limit]
# ---------- hash resolver ------------------------------------------
def resolve(self, hex_hash: str) -> ResolveResult:
"""Classify any hex hash. Order of search:
document_root → leaf_hash → audit event_hash → providence_cache
cache_key → providence_cache source_root → providence_cache
run_dag_root → merkle_nodes interior hash → unknown.
"""
h = hex_hash.lower().strip()
for path, c in self._conns.items():
if c.execute("SELECT 1 FROM documents WHERE document_root = ? LIMIT 1", (h,)).fetchone():
return ResolveResult(kind="document_root", hash=h, shard_path=path)
for path, c in self._conns.items():
r = c.execute("SELECT document_root, idx FROM chunks WHERE leaf_hash = ? LIMIT 1", (h,)).fetchone()
if r:
return ResolveResult(
kind="leaf_hash", hash=h, shard_path=path,
extra={"document_root": r["document_root"], "leaf_index": int(r["idx"])},
)
for path, c in self._conns.items():
r = c.execute("SELECT subject_root FROM audit_events WHERE event_hash = ? LIMIT 1", (h,)).fetchone()
if r:
return ResolveResult(
kind="audit_event", hash=h, shard_path=path,
extra={"subject_root": r["subject_root"]},
)
for path, c in self._conns.items():
try:
r = c.execute(
"SELECT source_root FROM providence_cache WHERE cache_key = ? LIMIT 1", (h,)
).fetchone()
if r:
return ResolveResult(
kind="qa_cache_key", hash=h, shard_path=path,
extra={"source_root": r["source_root"]},
)
except sqlite3.Error:
continue
# providence_cache.source_root: the QA's evidence root. Reaches
# this branch only when the hash isn't in ``documents`` (the
# documents probe runs first and returns "document_root"), so
# any match here is a synthesized multi-source context.
for path, c in self._conns.items():
try:
r = c.execute(
"SELECT cache_key FROM providence_cache WHERE source_root = ? LIMIT 1", (h,)
).fetchone()
if r:
return ResolveResult(
kind="context_root", hash=h, shard_path=path,
extra={"cache_key": r["cache_key"]},
)
except sqlite3.Error:
continue
for path, c in self._conns.items():
try:
r = c.execute(
"SELECT cache_key FROM providence_cache WHERE run_dag_root = ? LIMIT 1", (h,)
).fetchone()
if r:
return ResolveResult(
kind="qa_run_dag_root", hash=h, shard_path=path,
extra={"cache_key": r["cache_key"]},
)
except sqlite3.Error:
continue
for path, c in self._conns.items():
r = c.execute(
"SELECT document_root, layer, idx FROM merkle_nodes WHERE hash = ? LIMIT 1", (h,)
).fetchone()
if r:
return ResolveResult(
kind="merkle_interior", hash=h, shard_path=path,
extra={
"document_root": r["document_root"],
"layer": int(r["layer"]),
"idx": int(r["idx"]),
},
)
return ResolveResult(kind="unknown", hash=h)
# ---------- internals ----------------------------------------------
def _owning_conn_for_root(self, document_root: str) -> Optional[sqlite3.Connection]:
for _path, c in self._conns.items():
if c.execute(
"SELECT 1 FROM documents WHERE document_root = ? LIMIT 1",
(document_root,),
).fetchone():
return c
return None
def _path_for_root(self, document_root: str) -> Optional[str]:
for path, c in self._conns.items():
if c.execute(
"SELECT 1 FROM documents WHERE document_root = ? LIMIT 1",
(document_root,),
).fetchone():
return path
return None
def open_shards(paths: Iterable[str | Path]) -> Shards:
"""Open every shard at ``paths`` (read-only) and return a ``Shards``
handle. Shards that fail to open are skipped with a warning."""
return Shards([str(p) for p in paths])
# ---------- row decoders -----------------------------------------------
_QA_SELECT = (
"SELECT cache_key, source_root, document_uri, question_hash, question_text, "
" answer_text, model_profile_hash, conversation_hash, "
" governance_policy_hash, schema_version, canonicalization_version, "
" chunking_version, falsification_state, audit_mode, audit_event_hash, "
" run_dag_root, created_at, last_hit_at, hit_count, n_quotes, "
" n_verified, verifier_method, merkle_proof, run_dag_blob "
"FROM providence_cache"
)
# Sentinel ``document_uri`` stored on multi-source providence_cache rows
# (see ``arborist.qa.query``). Opaque on purpose at write time; the read
# seam projects it back to real provenance via ``_summarize_sources``.
_MULTI_SOURCE_URI = "corpus://multi-source"
def _domain_of(uri: str) -> str:
"""Hostname of a source URI, or '' for opaque/relative URIs."""
try:
from urllib.parse import urlparse
return (urlparse(uri).netloc or "").lower()
except Exception:
return ""
def _summarize_sources(sources: list[dict]) -> tuple[Optional[str], list[str]]:
"""Derive (primary_source_uri, source_domains) from a context root's
sources. The primary answer source ranks first; remaining sources keep
their order. Domains are de-duplicated, primary first."""
if not sources:
return None, []
primaries = [
s for s in sources
if isinstance(s, dict) and s.get("source_role") == "primary_answer_source"
]
others = [s for s in sources if isinstance(s, dict) and s not in primaries]
ordered = primaries + others
primary_uri = None
for s in ordered:
if s.get("document_uri"):
primary_uri = s["document_uri"]
break
domains: list[str] = []
for s in ordered:
dom = _domain_of(s.get("document_uri") or "")
if dom and dom not in domains:
domains.append(dom)
return primary_uri, domains
def _context_leaf_hashes(sources: list[dict]) -> list[str]:
"""Canonical leaf hashes for a context tree.
Mirrors :func:`arborist.qa.query._context_root` — the leaves are the
sources' ``document_root`` values, sorted lexicographically. Keeping
this helper next to :meth:`Shards.context_layers` /
:meth:`Shards.context_proof` lets the read seam reconstruct the tree
and prove inclusion without re-importing the QA pipeline.
"""
roots = [s.get("document_root") for s in sources if isinstance(s, dict) and s.get("document_root")]
return sorted(r.lower() for r in roots)
def _row_to_audit(r: sqlite3.Row, shard_path: str) -> AuditEvent:
try:
body = json.loads(r["body"]) if r["body"] else {}
except Exception:
body = {"_raw": r["body"]}
return AuditEvent(
seq=int(r["seq"]),
event_hash=r["event_hash"],
prev_event_hash=r["prev_event_hash"],
event_type=r["event_type"],
subject_root=r["subject_root"],
body=body,
ts=int(r["ts"]),
shard_path=shard_path,
)
def _row_to_qa(r: sqlite3.Row, shard_path: str) -> QaRecord:
def _json_or_empty(s):
if not s:
return {}
try:
return json.loads(s)
except Exception:
return {"_raw": s}
return QaRecord(
cache_key=r["cache_key"],
source_root=r["source_root"],
document_uri=r["document_uri"],
question_hash=r["question_hash"],
question_text=r["question_text"],
answer_text=r["answer_text"],
model_profile_hash=r["model_profile_hash"],
conversation_hash=r["conversation_hash"],
governance_policy_hash=r["governance_policy_hash"],
schema_version=r["schema_version"],
canonicalization_version=r["canonicalization_version"],
chunking_version=r["chunking_version"],
falsification_state=r["falsification_state"],
audit_mode=r["audit_mode"] if "audit_mode" in r.keys() else None,
audit_event_hash=r["audit_event_hash"] if "audit_event_hash" in r.keys() else None,
run_dag_root=r["run_dag_root"] if "run_dag_root" in r.keys() else None,
created_at=int(r["created_at"]),
last_hit_at=int(r["last_hit_at"]) if r["last_hit_at"] is not None else None,
hit_count=int(r["hit_count"]),
n_quotes=int(r["n_quotes"]) if "n_quotes" in r.keys() and r["n_quotes"] is not None else None,
n_verified=int(r["n_verified"]) if "n_verified" in r.keys() and r["n_verified"] is not None else None,
verifier_method=r["verifier_method"] if "verifier_method" in r.keys() else None,
merkle_proof=_json_or_empty(r["merkle_proof"]),
run_dag_blob=_json_or_empty(r["run_dag_blob"]) if "run_dag_blob" in r.keys() else {},
shard_path=shard_path,
)
def _get_wikitext_projector():
"""Resolve arborist.wikitext.to_base lazily (the wikitext extra is
optional). Returns ``(callable, base_version)`` or ``(None, None)``."""
try:
from arborist.wikitext import to_base, BASE_VERSION
return to_base, BASE_VERSION
except Exception:
return None, None

3
docs/TICKETS.md
View file

@ -111,6 +111,7 @@ Newest first. Update on every open/close.
| ID | Title | Status | Opened | Directive |
|----------|------------------------------------------------|-----------------------|------------|-----------|
| #000069 | Arborist VIZ / Merkle Command Center (Pyramid + six.js + SSE browser dashboard) | **open · awaiting go/no-go · doc-only scaffold** (2026-05-27; filed from `/home/fox/Downloads/TICKET_0000VIZ_*`, stack corrected same day per fox). Configurable browser dashboard for inspecting arborist's content-addressed state: Merkle root explorer, proof verifier, claim warrant + graveyard, audit timeline, run-DAG replay, cache-key explainer, root diff, 3D Merkle lattice, optional circuit/activation traces. Read-only consumer; arborist proper stays source-of-truth, dashboard projects state. **Stack pinned to unturf-native** (fox 2026-05-27, supersedes proposal §3): **Pyramid + Jinja2 + SQLAlchemy** (matches `remarkbox` / `make_post_sell` / `unhomeschool.com` idiom), **SSE** (`text/event-stream` via Pyramid streaming response) for live audit/claim/falsifier patches, **vanilla JS + six.js** (fox's patched three.js fork at `git.unturf.com/gumyum/six.js` — three.js r175 + CWE-407 patches incl. ObjectBVH O(N)→O(log N); bundles vendored from `~/git/cupPCB/cdn/six/`; third-instance MOAD-0001 dogfood alongside `java-topology` + gumyum-engine) for 3D widgets and large-graph rendering, SQLite for dashboard metadata (no PostgreSQL/ClickHouse/Redis/NATS by default — promote on measured need), no React / no Next.js / no Node build step. Server-rendered SVG (or Graphviz `.dot` per existing `docs/diagrams/*.dot` pattern) replaces React Flow for run-DAG widgets. Browser-side proof verification dropped from v1 (server-side Pyramid view returns PASS/FAIL + receipt; reinstate phase-N only if third-party-verification use case surfaces). **Three filing-note gates before phase 0** (in ticket body): **F-1** sibling-repo home — implementation lives in a new `~/git/arborist-viz` (Pyramid Python, matches existing unturf apps), not in-tree; arborist's contribution is the read-API spec + view package + arborist library import via `arborist.embed`. **F-2** scope split — proposal carries 8 phases (§17 phases 0–8); recommended cut keeps phases 0–3 (schema + shell + proof/root widgets + claim/audit/run widgets) inside #000069, and spawns sibling tickets for SSE streaming (4), 3D six.js (5), massive-graph (6, only if measured need surfaces), circuit-tracing (7, gated on #000062), embeddable widgets (8) — Dav1d-audience rule. **F-3** upstream prereqs — phase 7 (circuit/activation) consumes **#000062 Mechanistic Witness**'s `MechanisticWitnessRoot`; phase 3's claim-graveyard widget projects **#000059**'s bounded-ingestion graveyard. Hard constraints: arborist soft-vs-hard discipline applies verbatim (attribution weights renderable but never `audit_mode`, never causal without intervention/ablation evidence); private-leaf default-deny (commitments + hashes + redacted maps only without explicit auth); every widget exposes its data query + source roots. Reserved scope: NOT a replacement for `arborist controller-events` / `arborist analyze` / `arborist inspect` CLI — those stay canonical inspector surfaces; VIZ is the projection layer. | 2026-05-27 | — |
| #000068 | Verifier-blind missed-answer falsification guard | **in progress · Phase 1+2+3 landed 2026-05-27 · Phase 4 default flip NO-GO** (Phase 2 bench 2026-05-27 76q × n=3 claim_lattice Hermes-3-8B: 2/228 sidecar fires, both STRONG confidence, both the Ballestrini regression fixture, 100% precision, 0/226 false positives across non-Ballestrini runs. Phase 3 demote flag opt-in via `--demote-on-missed-answer` on `query`/`ask` — wires `EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL` for strong/medium confidence on lattice modes; lower rungs + non-lattice modes get `· missed-answer` tail tag. `answerability_demote_enabled` added to `_VERIFIER_POLICY_FIELDS` so flipping the flag partitions cache via verifier_policy_hash. Default OFF per Dav1d Phase 4 NO-GO — 100% precision at n=2 fires is too few samples to claim precision floor empirically; default flip blocks on wider bench + human spot-check. 47 tests (36 Phase 1 + 11 Phase 3) all passing. End-to-end verified live: 4/4 Hermes runs on Ballestrini with --demote-on-missed-answer rendered EVIDENCE-MISSED-PARTIAL.) Original opening 2026-05-27 (Dav1d de-novo review GO for Phase 1 with seven hardenings folded into spec — subject-token cue-stripping, answer-type alignment, confidence_class, candidate cap=10, precise offset_start/end/basis, cache-hit recompute-on-read, Phase 1 out of verifier_policy_hash). Original opening 2026-05-27; sibling to the user-payload-layout work shipped 2026-05-26, split out per the Dav1d-audience rule — `feedback_ticket_proliferation`). Surfaced by the Ballestrini case: evidence E2 literally contained the song names, Hermes-3-8B under `user_payload_layout=tail` said *"specific songs by her are not mentioned in the provided evidence blocks"*, verifier marked the run `EVIDENCE-WARRANTED` 2/2 because nothing positive was unsupported. **Verifier-blind false-negative class** — existing layered verifier (quote/span/entity/paraphrase + Rule 8 + Rule 9 + claim ceiling) guards unsupported *presence*, has no hook for unsupported *absence*. Layout fixes attention placement on the specific instance (n=3 bench 2026-05-27 confirms bookend/per_chunk recover Ballestrini); layout alone can't close the class — adversarial phrasing or bigger prompt resurfaces it under any layout. Proposed deterministic sidecar in `arborist/qa/inspect.py:diagnose_missed_answer`: three-clause conjunction — **(A)** answer matches denial pattern ("not mentioned", "not provided", "the evidence does not say", …, closed list versioned via `denial_patterns_version`); **(B)** question is extraction shape (reuse `arborist.qa.quantifier` classifier — `ALL`/`COMPREHENSIVE`/`OPEN_REQUEST` intensities, OR surface cues "songs by"/"works by"/"who wrote"/"list"/"name all"); **(C)** evidence contains candidate spans near subject tokens (reuse `entity_proximity_n`/`entity_proximity_window` from verify.py — quoted strings, title-case spans, comma-separated title lists within W chars of stemmed subject content tokens). All three must fire. Output: `result["answerability"]` with `missed_answer_candidate_spans` list (evidence_id + offset + text). **Hash discipline:** sidecar fields (`denial_patterns_version`, `extraction_cues_version`, `answerability_threshold`) fold into `governance_policy_hash` only; an optional `answerability_demote_enabled` flag (default OFF) wires `EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL` in `_render_audit_label`, and IF on folds into BOTH `governance_policy_hash` AND `verifier_policy_hash` (changes rendered audit_mode, so verifier hash must move — the deliberate opt-in moves the verifier hash, sidecar-only stays out). No LLM-as-judge. Never writes `providence_cache`/`audit_events`. Never promotes claims. Pattern verbatim from `arborist.qa.inspect.diagnose_*` (deflection, coherence, title-relevance). Phases: 1 sidecar read-only, 2 bench + threshold tuning, 3 demote opt-in, 4 default decision (bench-gated). 5F-Falsification fixture: Ballestrini case already in `bench/qa_questions.txt` under "entity list". Full spec in `docs/tickets/ticket-000068-verifier-blind-missed-answer-guard.md`. | 2026-05-27 | D2 |
| #000067 | M-aware cold-pack hydration (route incoming docs by content hash into M target shards) | **open · scaffold · prereq for #46 genesis test** (2026-05-26; surfaced while preparing the 3090 SPV-wallet validation). Today's `hydrate_from_metadata_pack` takes a single `conn` and writes every incoming row into one shard. With the corpus now in M=4 hash-routed topology (#000065), a fresh peer needs to land each document on `shard_for_document(document_root, M)` — same routing function as the producer. Without this, a fresh peer's `~/.arborist/shards/` is just one big single-shard DB and the M=4 ATTACH-and-route assumption #000065 was sized for doesn't hold consumer-side. Two coherent shapes: **(α) two-step kludge** — hydrate into single shard, then `arborist corpus reshard --to M` on the consumer. Works today (proven by the 2026-05-26 reshard executor) but doubles the wall time and treats packed shards as if they came from an arbitrary topology. **(β) direct M-aware hydrate** — extend `hydrate_from_metadata_pack` to accept `targets: list[sqlite3.Connection]` + `M: int` and route per-row at restore time (reusing `arborist.document.shard_for_document` + the table-routing rules in `arborist/migrate.py`). Manifest carries `corpus_shard_count` so the unpacker knows M from the pack itself. β is the right answer — α exists only as a fallback if 20-min-window pressure forces it. Sequence: (1) add `corpus_shard_count` to pack manifest (read from source meta during `dump_shard_metadata`); (2) `restore_shard_metadata_routed(targets, M, table_dir)` in `cold_pack_metadata.py` mirroring `_route_per_doc_table` from migrate.py; (3) `hydrate_from_metadata_pack` gains a `targets`/`shards_dir` param; (4) `arborist cold unpack --shards-dir DIR` initialises M target shards from the manifest's `corpus_shard_count` and routes; (5) regression test: pack 2 shards → hydrate into fresh 4 shards → assert every doc on its hash-routed target. Refactor opportunity: the routing rules (ROUTED_BY_DOCUMENT_ROOT, CONSOLIDATED_TABLES) currently live in migrate.py; this ticket can either duplicate them in cold_pack_metadata.py (fast) or factor into a shared `arborist/multi_shard.py` module (cleaner). The shared-module path is more honest given graft mode (#000066) wants the same primitives. Out of scope: graft / overlay mode (that's #000066 — overlays onto populated, this is hydrate-into-empty). | 2026-05-26 | — |
| #000066 | Cold-pack overlay / graft mode (pack-as-package, witness-pattern audit chain) | **scaffold-only · awaiting go/no-go** (2026-05-26; surfaced while running #000065 reshard, fox extension: "we could envision a pack for wikipedia 2010, wikipedia current, etc"). Extend #000061 cold-pack hydration with a second mode: overlay an existing pack onto a populated shard set instead of hydrating into empty. Doc/chunk/edge/concept overlay is trivial (`INSERT OR IGNORE` on content-addressed PKs collapses dupes); FTS5 overlay is trivial (new chunks → new rowids → new FTS rows). The interesting part is the audit chain — can't naively append the pack's events because `prev_event_hash` linkage breaks across the join. Chosen approach: **graft receipt**. Append one new `event_type='graft'` event to the host chain carrying `(pack_hash, snapshot_root, corpus_name, event_count, first_event_hash, last_event_hash, manifest_root)`; the pack file itself becomes the durable witness for the absorbed events (anyone can re-fetch the pack, walk its internal chain, and verify it matches the receipt). Host chain stays linear; pack chain is a "witnessed subgraph." This is the same witness pattern Merkle-AGI v8/v9 is heading toward, but bought at near-zero schema cost. Rejected alternatives: re-chain everything (breaks external refs to old event_hashes — cache_keys anchoring to old `audit_event_hash`, snapshots, etc. — silently invalid); chain forest with new `chain_id` column (right answer when graft dominates the lifecycle, but premature now). **Pack-as-package extension** (fox 2026-05-26): each pack carries a `corpus_name` field in its manifest (`wikipedia-2010`, `wikipedia-current`, `arxiv-cs`, `textbooks-undergrad`, …) so operators pick which corpora to graft — `arborist cold graft wikipedia-current` becomes as natural as `apt install firefox`. Multiple packs of the same corpus name: most-recent `snapshot_root` wins; older packs stay in the bucket until GC. URI conflicts across corpora (e.g., `wikipedia.org/wiki/Foo` in both 2010 and current): different content → different `document_root` → both stored, `supersedes` edges per CLAUDE.md invariant. Providence-cache conflicts: same `cache_key` with different answer → existing v9.8 falsification framework handles it (`state='stale'` or `quarantined`). Mesh-peer-corpus-merge: each peer's pack is a graftable package; partition reconciliation becomes "exchange the packs you each carry, graft what you lack". The mesh-of-arborists semantic. Sequence: (1) `corpus_name` field in #000061 manifest format + alias index in bucket (`corpora/<name>/latest.json` pointer to active pack_hash); (2) `arborist cold graft <pack_hash>` / `arborist cold graft --corpus <name>` mode in evict.py — read pack, INSERT OR IGNORE per-table, emit graft receipt; (3) conflict-policy flag (`--on-uri-conflict {supersedes,skip,fail}`, default `supersedes`); (4) `arborist cold list-corpora` shows available packages in a bucket. Scaffold first, code only when (a) #000065 reshard lands and stabilises (b) a second corpus exists (the wikipedia-current snapshot, or first textbook bundle ready to graft onto wikipedia-2010 base) (c) at least two peers want to exchange. | 2026-05-26 | — |
@ -182,4 +183,4 @@ Newest first. Update on every open/close.
## Next ID
`000069`
`000070`

23
docs/crawler.md
View file

@ -226,16 +226,29 @@ arborist crawler recrawl-check [--domain D] [--limit N]
- `--author` — default author surname appended to titles for warrant
resolution (only with `--ingest`).
Make targets drive the textbook-crawl workflow; crawl shards land in
Make targets drive both crawl workflows; crawl shards land in
`~/.arborist/crawl/` (separate from the main `~/.arborist/shards` so
SQLite's attached-DB limit isn't tripped):
SQLite's 10-attached-DB limit isn't tripped, and so locally crawled
content isn't shared as a peer by default):
```
make crawl-textbooks # BFS-crawl every manifest entry with a crawl_url
make textbook ID=<id> # ingest one textbook by id
make crawl-textbooks-stats # docs-per-shard summary
make crawl-ingest URL=https://x.com # general web crawl → ONE central
# db (CRAWL_DB, default web.db)
make crawl-textbooks # BFS-crawl every manifest entry
# with a crawl_url (warrant substrate)
make textbook ID=<id> # ingest one textbook by id
make crawl-textbooks-stats # docs-per-shard summary
```
General web crawls (`make crawl-ingest`) all flow into a **single**
central db rather than one-per-domain: content-addressing lets many
domains coexist in one file (idempotent re-ingest, `supersedes` edges
on change), and a single file always attaches under the 10-DB cap.
Query it standalone with `arborist --db ~/.arborist/crawl/web.db query
"…"`, or attach it alongside the main corpus when you want unified
results. The per-host textbook crawls stay separate — they are warrant
substrate, resolved through a different path.
## Source map
| File | Role |

1900
docs/tickets/ticket-000069-arborist-viz-merkle-command-center.md Normal file
View file

File diff suppressed because it is too large Load diff

37
tests/test_cli_render.py
View file

@ -383,3 +383,40 @@ def test_render_label_format_collapsed_surfaces_tail():
out = _render_query_human(r, "winners of all major sports?")
assert "format collapsed" in out
assert "UNGROUNDED" in out
# --- auto-include of the local crawl db -----------------------------
def test_resolve_extra_shards_auto_includes_crawl_db(tmp_path, monkeypatch):
"""`arborist query` auto-includes the local crawl db (when present),
honors --no-crawl-db, and appends --include-shard extras de-duped."""
import argparse
from arborist.cli import _default_crawl_db, _resolve_extra_shards
crawl_db = tmp_path / "web.db"
crawl_db.write_bytes(b"") # resolver only checks presence
monkeypatch.setenv("ARBORIST_CRAWL_DB", str(crawl_db))
assert _default_crawl_db() == crawl_db
# present → auto-included
ns = argparse.Namespace(no_crawl_db=False, include_shard=None)
assert _resolve_extra_shards(ns) == [crawl_db]
# --no-crawl-db opts out
ns_off = argparse.Namespace(no_crawl_db=True, include_shard=None)
assert _resolve_extra_shards(ns_off) == []
# explicit --include-shard appends extras; crawl db deduped
extra = tmp_path / "extra.db"
extra.write_bytes(b"")
ns_inc = argparse.Namespace(
no_crawl_db=False, include_shard=[str(extra), str(crawl_db)]
)
assert _resolve_extra_shards(ns_inc) == [crawl_db, extra]
# missing crawl db → silently dropped
monkeypatch.setenv("ARBORIST_CRAWL_DB", str(tmp_path / "nope.db"))
ns_missing = argparse.Namespace(no_crawl_db=False, include_shard=None)
assert _resolve_extra_shards(ns_missing) == []

54
tests/test_query.py
View file

@ -1424,3 +1424,57 @@ def test_integration_hyphenated_query_retrieves_joined_title(tmp_path):
assert any("bipolar-disorder" in u for u in src_uris), (
f"Ticket #000007 regression: bipolar-disorder not in {src_uris}"
)
def test_query_extra_shards_pulls_in_separate_db(tmp_path):
"""`extra_shards` searches a separate db alongside the main corpus —
the seam the CLI uses to auto-include the local crawl db without
folding it into the peer-shared main shards."""
main_db = tmp_path / "corpus.db"
crawl_db = tmp_path / "web.db"
conn = connect(main_db)
try:
ingest_source(conn, FakeSource(DOCS))
finally:
conn.close()
crawl_doc = _doc(
"test://virtback",
"Virtback is a python libvirt backup utility for kvm xen virtualbox. " * 12
+ "Virtback restores domains from backups. " * 8,
)
conn = connect(crawl_db)
try:
ingest_source(conn, FakeSource([crawl_doc]))
finally:
conn.close()
client = StubClient(
answer='Per the source: "Virtback is a python libvirt backup utility for kvm xen virtualbox"'
)
# Without extra_shards: the crawl-only topic is absent from the main corpus.
r_without = query(
question="What is virtback?",
qa_db=tmp_path / "qa1.db",
chat_client=client,
model_id="m",
single_db=main_db,
top_k=3,
)
assert all(
"virtback" not in s["document_uri"] for s in r_without.get("sources") or []
)
# With extra_shards: the separate crawl db is searched alongside it.
r_with = query(
question="What is virtback?",
qa_db=tmp_path / "qa2.db",
chat_client=client,
model_id="m",
single_db=main_db,
extra_shards=[crawl_db],
top_k=3,
)
assert any(
"virtback" in s["document_uri"] for s in r_with.get("sources") or []
), f"extra_shards crawl db not searched: {[s['document_uri'] for s in r_with.get('sources') or []]}"

395
tests/test_read.py Normal file
View file

@ -0,0 +1,395 @@
"""Tests for arborist.read — the supported read-only seam.
These pin the contract that downstream read-only consumers
(arborist-viz / Merkle Command Center, third-party verifiers, archival
mirrors) depend on: open shards, query roots/leaves/proofs/audit/qa,
resolve any hex hash.
"""
from __future__ import annotations
import sqlite3
import pytest
from arborist.embed import Document, ingest_documents, open_store
from arborist.read import (
open_shards,
Root,
Leaf,
Proof,
AuditEvent,
QaRecord,
ResolveResult,
ShardCounts,
)
# ---------- fixtures ----------------------------------------------------
@pytest.fixture
def shard_path(tmp_path):
p = str(tmp_path / "shard.db")
conn = open_store(p)
ingest_documents(
conn,
[
Document(
uri="https://test.local/doc1",
title="Doc 1",
source_type="test",
content="One sentence. Two sentence. Three sentence. Four sentence. Five.",
),
Document(
uri="https://test.local/doc2",
title="Doc 2",
source_type="test",
content="Alpha beta gamma delta epsilon.",
),
],
source_type="test",
chunker_name="sent-v1",
)
conn.close()
return p
@pytest.fixture
def shards(shard_path):
s = open_shards([shard_path])
yield s
s.close()
def _root_of(shards) -> str:
return shards.roots(limit=10)[0].document_root
# ---------- handle ------------------------------------------------------
def test_open_skips_broken_shards(tmp_path):
s = open_shards([str(tmp_path / "missing.db")])
# Broken / missing shard is skipped, handle is still usable.
assert s.paths == [str(tmp_path / "missing.db")] or s.paths == []
def test_counts(shards):
c = shards.counts()
assert isinstance(c, ShardCounts)
assert c.documents == 2
assert c.shard_count == 1
assert c.audit_events >= 2 # at least one ingest event per doc
# ---------- roots / leaves ---------------------------------------------
def test_roots_listing(shards):
rs = shards.roots(limit=10)
assert len(rs) == 2
titles = {r.title for r in rs}
assert {"Doc 1", "Doc 2"} == titles
for r in rs:
assert isinstance(r, Root)
assert r.shard_path # source-of-record citation
def test_root_lookup(shards):
root_hash = _root_of(shards)
r = shards.root(root_hash)
assert r is not None
assert r.document_root == root_hash
assert r.leaf_count >= 1
def test_root_not_found(shards):
assert shards.root("0" * 64) is None
def test_leaves_privacy_default_deny(shards):
"""Default: bytes-under-the-hash NOT decompressed (§14)."""
root_hash = _root_of(shards)
leaves = shards.leaves(root_hash)
assert leaves, "shard must have at least one chunk"
for L in leaves:
assert isinstance(L, Leaf)
assert L.content is None
assert L.prose is None
def test_leaves_reveal_returns_content(shards):
root_hash = _root_of(shards)
leaves = shards.leaves(root_hash, reveal_private=True)
assert any(L.content for L in leaves), "reveal must surface chunk content"
def test_leaves_project_off(shards):
root_hash = _root_of(shards)
leaves = shards.leaves(root_hash, reveal_private=True, project=False)
for L in leaves:
assert L.prose is None
assert L.base_version is None
# ---------- proof / tree ------------------------------------------------
def test_proof_passes(shards):
root_hash = _root_of(shards)
p = shards.proof(root_hash, 0)
assert isinstance(p, Proof)
assert p.passed is True
assert p.computed_root == p.expected_root == root_hash
def test_proof_out_of_range(shards):
root_hash = _root_of(shards)
with pytest.raises(IndexError):
shards.proof(root_hash, 9999)
def test_proof_unknown_root(shards):
assert shards.proof("0" * 64, 0) is None
def test_tree_layers(shards):
root_hash = _root_of(shards)
layers = shards.tree_layers(root_hash)
assert layers is not None
assert len(layers[-1]) == 1
assert layers[-1][0] == root_hash
# ---------- audit -------------------------------------------------------
def test_audit_recent(shards):
evs = shards.audit_recent(limit=10)
assert evs, "ingest writes at least one audit event per document"
for ev in evs:
assert isinstance(ev, AuditEvent)
assert ev.shard_path
def test_audit_event_lookup(shards):
ev = shards.audit_recent(limit=1)[0]
assert shards.audit_event(ev.event_hash) == ev
def test_audit_by_root(shards):
root_hash = _root_of(shards)
evs = shards.audit_by_root(root_hash)
assert all(ev.subject_root == root_hash for ev in evs)
def test_audit_chain_walks_prev(shards):
head = shards.audit_recent(limit=1)[0]
chain = shards.audit_chain(head.event_hash, limit=10)
assert chain[0].event_hash == head.event_hash
def test_audit_since_yields_then_empties(shards):
# First call (empty cursor) returns every event.
events, cur = shards.audit_since(None)
assert events
# Second call with the cursor returns nothing new.
again, _ = shards.audit_since(cur)
assert again == []
def test_audit_cursor_is_head(shards):
cur = shards.audit_cursor()
assert all(seq >= 1 for seq in cur.values())
# ---------- hash resolver ----------------------------------------------
def test_resolve_document_root(shards):
root_hash = _root_of(shards)
r = shards.resolve(root_hash)
assert r.kind == "document_root"
assert r.hash == root_hash
assert r.shard_path
def test_resolve_leaf_hash(shards):
root_hash = _root_of(shards)
leaf = shards.leaves(root_hash)[0]
r = shards.resolve(leaf.leaf_hash)
assert r.kind == "leaf_hash"
assert r.extra["document_root"] == root_hash
assert r.extra["leaf_index"] == leaf.idx
def test_resolve_audit_event(shards):
ev = shards.audit_recent(limit=1)[0]
r = shards.resolve(ev.event_hash)
assert r.kind == "audit_event"
def test_resolve_unknown(shards):
assert shards.resolve("0" * 64).kind == "unknown"
# ---------- multi-shard fan-out ----------------------------------------
def test_multi_shard_resolve_and_count(tmp_path):
"""Fan-out: roots / counts / resolve all aggregate across shards."""
paths = []
for i, content in enumerate(["First doc one.", "Second doc two.", "Third doc three."]):
p = str(tmp_path / f"s{i}.db")
c = open_store(p)
ingest_documents(
c,
[Document(uri=f"test://{i}", content=content, source_type="test")],
source_type="test",
)
c.close()
paths.append(p)
s = open_shards(paths)
try:
assert s.counts().documents == 3
assert s.counts().shard_count == 3
roots = s.roots(limit=10)
assert len(roots) == 3
# Each root's shard_path matches one of the three configured paths.
assert {r.shard_path for r in roots} == set(paths)
# Resolving each root identifies its owning shard.
for r in roots:
res = s.resolve(r.document_root)
assert res.kind == "document_root"
assert res.shard_path == r.shard_path
finally:
s.close()
# ---------- providence_cache (Q&A) -------------------------------------
def _seed_qa(path: str, *, cache_key: str, source_root: str, question: str, answer: str):
"""Manually insert a providence_cache row — arborist's qa.runner
would normally do this; we synthesize a row so the test doesn't need
a live LLM."""
c = sqlite3.connect(path)
c.execute(
"INSERT OR REPLACE INTO providence_cache ("
" cache_key, source_root, document_uri, question_hash, question_text, "
" answer_text, merkle_proof, model_profile_hash, conversation_hash, "
" governance_policy_hash, schema_version, canonicalization_version, "
" chunking_version, falsification_state, created_at, hit_count"
") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
(
cache_key, source_root, "test://doc", "q" * 64, question, answer,
"{}", "m" * 64, "c" * 64, "g" * 64,
"v9.8.0", "norm-v1", "tok-512-v1", "live", 1, 1,
),
)
c.commit()
c.close()
def test_qa_lookup_and_search(shard_path):
_seed_qa(
shard_path,
cache_key="a" * 64,
source_root="b" * 64,
question="who wrote the song?",
answer="Joey Tempest.",
)
s = open_shards([shard_path])
try:
rec = s.qa("a" * 64)
assert rec is not None
assert rec.question_text == "who wrote the song?"
assert rec.answer_text == "Joey Tempest."
hits = s.qa_search("wrote")
assert any(r.cache_key == "a" * 64 for r in hits)
by_root = s.qa_by_root("b" * 64)
assert by_root[0].cache_key == "a" * 64
# cache_key resolves
r = s.resolve("a" * 64)
assert r.kind == "qa_cache_key"
# source_root resolves as a synthetic context_root (no
# corresponding row in the documents table).
r = s.resolve("b" * 64)
assert r.kind == "context_root"
assert r.extra.get("cache_key") == "a" * 64
# And the dedicated context() lookup returns the synthesized root.
ctx = s.context("b" * 64)
assert ctx is not None
assert ctx.context_root == "b" * 64
assert ctx.cache_key == "a" * 64
assert ctx.question_text == "who wrote the song?"
finally:
s.close()
def test_summarize_sources_picks_primary_and_domains():
"""The provenance helper ranks the primary answer source first and
de-dupes contributing domains (primary first)."""
from arborist.read import _summarize_sources
sources = [
{"document_uri": "https://en.wikipedia.org/wiki/Virt",
"source_role": "background_source"},
{"document_uri": "https://russell.ballestrini.net/virt-back-restoring-from-backups/",
"source_role": "primary_answer_source"},
{"document_uri": "https://russell.ballestrini.net/virt-backs-domfetcher/",
"source_role": "primary_answer_source"},
]
primary, domains = _summarize_sources(sources)
assert primary == "https://russell.ballestrini.net/virt-back-restoring-from-backups/"
assert domains == ["russell.ballestrini.net", "en.wikipedia.org"]
assert _summarize_sources([]) == (None, [])
def test_context_headlines_real_source_not_sentinel(shard_path):
"""A multi-source context root stores the opaque
``corpus://multi-source`` sentinel as document_uri; the read seam
must headline the real primary source so a consumer (dashboard /
verifier) shows where the knowledge came from."""
import json
c = sqlite3.connect(shard_path)
c.execute(
"INSERT OR REPLACE INTO providence_cache ("
" cache_key, source_root, document_uri, question_hash, question_text, "
" answer_text, merkle_proof, model_profile_hash, conversation_hash, "
" governance_policy_hash, schema_version, canonicalization_version, "
" chunking_version, falsification_state, created_at, hit_count"
") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
(
"d" * 64, "e" * 64, "corpus://multi-source", "q" * 64,
"what is virt-back?", "answer",
json.dumps({"sources": [
{"document_uri": "https://en.wikipedia.org/wiki/Virt",
"source_role": "background_source", "document_root": "1" * 64},
{"document_uri": "https://russell.ballestrini.net/virt-back/",
"source_role": "primary_answer_source", "document_root": "2" * 64},
]}),
"m" * 64, "c" * 64, "g" * 64,
"v9.8.0", "norm-v1", "tok-512-v1", "live", 1, 1,
),
)
c.commit()
c.close()
s = open_shards([shard_path])
try:
ctx = s.context("e" * 64)
assert ctx is not None
# Headline is the real primary source, not the opaque sentinel.
assert ctx.document_uri == "https://russell.ballestrini.net/virt-back/"
assert ctx.primary_source_uri == "https://russell.ballestrini.net/virt-back/"
assert ctx.source_domains == ["russell.ballestrini.net", "en.wikipedia.org"]
finally:
s.close()