From 7f7eeefeb9df923d40936eccbb3cdb8c7e79148f Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 29 May 2026 13:45:47 -0400 Subject: [PATCH] crawl central-db + query auto-include + read-seam provenance - make crawl-ingest writes to one central crawl db (CRAWL_DB, default ~/.arborist/crawl/web.db) instead of per-domain shards in the peer-shared main dir: keeps locally-crawled content out of peer sharing by default and a growing domain set under SQLite's 10-attach cap (Makefile, docs/crawler.md). - arborist query auto-includes the local crawl db (query() gains extra_shards; CLI --include-shard / --no-crawl-db, default-on when web.db exists). Fix latent --db single-file query AttributeError (cli.py). Persist used / used_pointer_ids + retrieval_purity into merkle_proof so read-only consumers can see which chunks fed the answer (qa/query.py). - arborist.read: read-only seam for dashboards / verifiers; on a multi-source context root surface the real primary source instead of the opaque corpus://multi-source sentinel (read.py). Backs the arborist-viz Merkle Command Center (#000069). - tests for extra_shards, the CLI crawl-db resolver, and the read seam. --- Makefile | 24 +- arborist/cli.py | 52 +- arborist/qa/query.py | 27 +- arborist/read.py | 965 +++++++++ docs/TICKETS.md | 3 +- docs/crawler.md | 23 +- ...0069-arborist-viz-merkle-command-center.md | 1900 +++++++++++++++++ tests/test_cli_render.py | 37 + tests/test_query.py | 54 + tests/test_read.py | 395 ++++ 10 files changed, 3460 insertions(+), 20 deletions(-) create mode 100644 arborist/read.py create mode 100644 docs/tickets/ticket-000069-arborist-viz-merkle-command-center.md create mode 100644 tests/test_read.py diff --git a/Makefile b/Makefile index 8106ef5..b943df0 100644 --- a/Makefile +++ b/Makefile @@ -1331,18 +1331,20 @@ test-crawler: bootstrap-crawler ## run only the lifted crawler tests # HEAD requests later. URL is required; DEPTH and MAX have safe defaults. CRAWL_DEPTH ?= 2 CRAWL_MAX ?= 0 -# Per-domain shard so `make query` (which reads $(SHARDS_DIR)) sees the -# crawled content. Shard filename derived from the seed URL's hostname: -# https://russell.ballestrini.net -> $(SHARDS_DIR)/crawl_russell_ballestrini_net.db -# Override with CRAWL_SHARD=... when you want a custom path. -crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] into $(SHARDS_DIR)/crawl_.db - @if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1 CRAWL_SHARD=path]" >&2; exit 2; fi - @mkdir -p $(SHARDS_DIR) +# General web crawls land in ONE central crawl db ($(CRAWL_DB)), inside +# $(CRAWL_SHARDS_DIR) so they stay OUT of the peer-shared main $(SHARDS_DIR) +# by default and a growing set of crawled domains never trips SQLite's +# 10-attached-database limit. Content-addressing lets many domains share +# one file safely (idempotent re-ingest; `supersedes` edges on change). +# To query it: `--db $(CRAWL_DB)` standalone, or attach it alongside the +# main corpus (still just one extra file, under the 10-attach cap). +# Override the path with CRAWL_DB=... (or CRAWL_SHARD=... for a one-off). +CRAWL_DB ?= $(CRAWL_SHARDS_DIR)/web.db +crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1 CRAWL_DB=path] into the central crawl db ($(CRAWL_DB)) + @if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1 CRAWL_DB=path]" >&2; exit 2; fi + @mkdir -p $(CRAWL_SHARDS_DIR) @shard="$(CRAWL_SHARD)"; \ - if [ -z "$$shard" ]; then \ - domain=$$(echo "$(URL)" | sed -E 's,^https?://([^/]+).*$$,\1,' | tr '.' '_'); \ - shard="$(SHARDS_DIR)/crawl_$${domain}.db"; \ - fi; \ + if [ -z "$$shard" ]; then shard="$(CRAWL_DB)"; fi; \ echo " shard: $$shard" >&2; \ $(ARBORIST) --db "$$shard" crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest diff --git a/arborist/cli.py b/arborist/cli.py index 9714997..3e4cd27 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -488,6 +488,37 @@ def _cmd_ask(args: argparse.Namespace) -> int: return 0 if result.get("status") in ("cache_hit", "cache_miss_then_written") else 1 +def _default_crawl_db() -> Path: + """Path of the local crawl db auto-included on the read path. + + A single central file for locally web-crawled content (``make + crawl-ingest``). Kept SEPARATE from the peer-shared main shards so + it is never bundled into cold-packs / mesh by default; only the read + path attaches it. Override with ``ARBORIST_CRAWL_DB``.""" + import os + env = os.environ.get("ARBORIST_CRAWL_DB") + if env: + return Path(env).expanduser() + return Path.home() / ".arborist" / "crawl" / "web.db" + + +def _resolve_extra_shards(args: argparse.Namespace) -> list[Path]: + """Extra shard files to search alongside the main corpus: the local + crawl db (auto-included unless ``--no-crawl-db``) plus any explicit + ``--include-shard`` paths. Missing files are dropped silently — the + crawl db only exists once something has been crawled locally.""" + extras: list[Path] = [] + if not getattr(args, "no_crawl_db", False): + crawl_db = _default_crawl_db() + if crawl_db.is_file(): + extras.append(crawl_db) + for p in (getattr(args, "include_shard", None) or []): + pp = Path(p).expanduser() + if pp.is_file() and pp not in extras: + extras.append(pp) + return extras + + def _cmd_query(args: argparse.Namespace) -> int: """Multi-source RAG: question -> top-K corpus docs -> Hermes -> cache.""" import os @@ -527,6 +558,7 @@ def _cmd_query(args: argparse.Namespace) -> int: Path(args.global_shards_dir) if args.global_shards_dir else None ) single_db = None if shards_dir else args.db + extra_shards = _resolve_extra_shards(args) # Apply per-call policy overrides (question_dedup, repair, answer_mode) # on top of the default. fidelity is a function-level kwarg, not in @@ -621,6 +653,7 @@ def _cmd_query(args: argparse.Namespace) -> int: quantization=quantization, shards_dir=shards_dir, single_db=single_db, + extra_shards=extra_shards, top_k=args.top_k, over_fetch=args.over_fetch, max_context_chars=args.max_context_chars, @@ -645,7 +678,7 @@ def _cmd_query(args: argparse.Namespace) -> int: # so the render-layer warrant-chain tail can look up # warrant-resolver derivations without needing args. Stripped # before JSON output to keep the json shape stable. - _shards_dir = args.global_shards_dir or args.shards_dir + _shards_dir = args.global_shards_dir or getattr(args, "shards_dir", None) if _shards_dir: result["_shards_dir"] = str(_shards_dir) @@ -5340,6 +5373,23 @@ def build_parser() -> argparse.ArgumentParser: "--over-fetch", dest="over_fetch", type=int, default=32, help="FTS5 hits to fetch per shard before dedup (default 32)", ) + query_cmd.add_argument( + "--include-shard", dest="include_shard", action="append", default=None, + metavar="PATH", + help=( + "extra shard db file(s) to search alongside the main corpus " + "(repeatable). Per-shard fan-out, so no 10-attach limit here. " + "The local crawl db (ARBORIST_CRAWL_DB, default " + "~/.arborist/crawl/web.db) is auto-included when present." + ), + ) + query_cmd.add_argument( + "--no-crawl-db", dest="no_crawl_db", action="store_true", + help=( + "do not auto-include the local crawl db; query only the main " + "corpus (--shards-dir / --db)." + ), + ) query_cmd.add_argument( "--max-context-chars", dest="max_context_chars", type=int, default=None, help=( diff --git a/arborist/qa/query.py b/arborist/qa/query.py index 557441e..3a53b3b 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -1375,6 +1375,7 @@ def _search_corpus( question: str, over_fetch: int, *, + extra_shards: list[Path] | None = None, progress: Progress | None = None, ) -> list[_Hit]: """Two parallel searches across shards, merged: @@ -1425,6 +1426,14 @@ def _search_corpus( paths = [Path(single_db)] else: paths = [] + # Opt-in extra shards (e.g. the local crawl db auto-included by the + # CLI) searched alongside the main corpus. Per-shard fan-out below + # means there is no SQLite attach-limit ceiling here. Dedup so a path + # already discovered in shards_dir isn't searched twice. + for x in (extra_shards or []): + xp = Path(x) + if xp not in paths: + paths.append(xp) progress = progress or _progress_disabled() @@ -1936,6 +1945,7 @@ def query( quantization: str = "", shards_dir: Path | None = None, single_db: Path | None = None, + extra_shards: list[Path] | None = None, top_k: int = 8, over_fetch: int = 32, max_context_chars: int | None = None, @@ -2753,7 +2763,8 @@ def query( ) t_phase = time.monotonic() hits, core_match_roots, phrase_match_roots, root_to_shard = _search_corpus( - shards_dir, single_db, retrieval_query, over_fetch, progress=progress, + shards_dir, single_db, retrieval_query, over_fetch, + extra_shards=extra_shards, progress=progress, ) if not hits: return { @@ -3604,7 +3615,10 @@ def query( for h in chosen ], } - proof_blob = json.dumps(proof_obj, separators=(",", ":")) + # proof_blob is serialized AFTER the lattice block below, so the + # persisted merkle_proof carries the `used` / `used_pointer_ids` + # annotations and `retrieval_purity` that the JSON result already + # exposes — see the note at the json.dumps call. # Per-run Merkle-DAG. Quote mode base shape: 7 stages # (question / retrieval / context / prompt / answer / verify @@ -3692,6 +3706,15 @@ def query( ), } proof_obj["retrieval_purity"] = retrieval_purity + + # Serialize merkle_proof here (not at proof_obj construction) so the + # stored blob includes the render-layer `used` / `used_pointer_ids` + # per source and `retrieval_purity`. merkle_proof is stored + # provenance, NOT a commitment — the providence_query audit-event + # body excludes it — so adding these derived signals does not touch + # the proof path. Lets read-only consumers (the VIZ dashboard) + # highlight the chunks that actually fed the answer. + proof_blob = json.dumps(proof_obj, separators=(",", ":")) # Retrieval-plan hash (Ticket #000001 / Directive D4): # capture the operator-influenceable retrieval inputs so the # run-DAG's retrieval stage binds BOTH plan (what guided the diff --git a/arborist/read.py b/arborist/read.py new file mode 100644 index 0000000..fbf69e6 --- /dev/null +++ b/arborist/read.py @@ -0,0 +1,965 @@ +"""arborist.read — read-only seam for dashboards and verifiers. + +Counterpart to :mod:`arborist.embed`: that module is the seam embedders +use to *write* into arborist (Document → ingest → root). This module is +the seam read-only consumers (Merkle Command Center / arborist-viz, +third-party verifiers, archival mirrors) use to *read* from arborist. + +Why a seam? Same reason as ``arborist.embed`` — so internal refactors +(schema bumps, compression scheme changes, sharding plans, wikitext +projection version, qa-vs-document shard separation) don't break the +downstream consumers. Consumers import from here; they never touch +arborist's tables, columns, or helper modules directly. + +Surface summary +--------------- + +:: + + open_shards(paths) → Shards + shards.roots(limit=50) → list[Root] + shards.root(document_root) → Root | None + shards.leaves(document_root, *, reveal_private=False, project=True) + → list[Leaf] + shards.proof(document_root, leaf_index) + → Proof + shards.tree_layers(document_root) → list[list[str]] + shards.audit_event(event_hash) → AuditEvent | None + shards.audit_chain(head, limit) → list[AuditEvent] + shards.audit_recent(limit) → list[AuditEvent] + shards.audit_by_root(root, limit) → list[AuditEvent] + shards.audit_since(cursor) → (list[AuditEvent], dict) + shards.qa(cache_key) → QaRecord | None + shards.qa_recent(limit) → list[QaRecord] + shards.qa_by_root(root, limit) → list[QaRecord] + shards.qa_search(text, limit) → list[QaRecord] + shards.resolve(hex_hash) → ResolveResult + shards.counts() → ShardCounts + +Privacy +------- + +By default, ``leaves()`` does not decompress chunk content (the bytes +the leaf hash commits to). Pass ``reveal_private=True`` to opt in. The +chunk content stays compressed-on-disk regardless; reveal only affects +whether the read seam decompresses and returns it. + +The ``project=True`` (default) flag adds a deterministic prose +projection (:mod:`arborist.wikitext.to_base`, versioned by +``BASE_VERSION``) alongside the raw bytes. Pass ``project=False`` to +skip the projection when only raw bytes are needed. + +Privacy does **not** apply to: hashes, leaf indices, audit-event +metadata, providence_cache rows excluding ``answer_text`` / +``question_text`` (which are part of the Q&A record by design). +""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +from collections.abc import Iterable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +from arborist.compress import unpack_chunk +from arborist.merkle import MerkleTree, proof_to_dict, verify_proof +from arborist.store import connect + + +log = logging.getLogger(__name__) + + +__all__ = [ + "open_shards", + "Shards", + "Root", + "Leaf", + "Proof", + "AuditEvent", + "QaRecord", + "ContextRoot", + "ResolveResult", + "ShardCounts", +] + + +# ---------- dataclasses ------------------------------------------------ + + +@dataclass(frozen=True) +class Root: + """One document root + its surface metadata. ``shard_path`` lets the + consumer cite where the answer came from.""" + + document_root: str + document_uri: str + source_type: str + kind: str + title: Optional[str] + chunking_version: str + canonicalization_version: str + schema_version: str + ingest_ts: int + hit_count: int + leaf_count: int + shard_path: str + + +@dataclass(frozen=True) +class Leaf: + """One chunk of a document. ``content`` is the raw bytes-under-the-hash + (decompressed only when ``reveal_private=True`` was passed). ``prose`` + is the deterministic projection — derivable from ``content``, versioned + by ``base_version``.""" + + document_root: str + idx: int + leaf_hash: str + tier: str + content: Optional[str] = None + prose: Optional[str] = None + base_version: Optional[str] = None + + +@dataclass(frozen=True) +class Proof: + """A Merkle inclusion proof. Hash check is performed at construction; + ``passed`` is final.""" + + document_root: str + leaf_index: int + leaf_hash: str + sibling_hashes: list[str] + left_right_flags: list[bool] + computed_root: str + expected_root: str + passed: bool + shard_path: str + raw: dict # the arborist.merkle.proof_to_dict form, for verifiers + + +@dataclass(frozen=True) +class AuditEvent: + seq: int + event_hash: str + prev_event_hash: Optional[str] + event_type: str + subject_root: Optional[str] + body: dict + ts: int + shard_path: str + + +@dataclass(frozen=True) +class QaRecord: + """One providence_cache row — the v9.8 8-dim invariant + the cached + answer. Only consumers that respect the §14 privacy contract should + surface ``question_text`` / ``answer_text``.""" + + cache_key: str + source_root: str + document_uri: str + question_hash: str + question_text: str + answer_text: str + model_profile_hash: str + conversation_hash: str + governance_policy_hash: str + schema_version: str + canonicalization_version: str + chunking_version: str + falsification_state: str + audit_mode: Optional[str] + audit_event_hash: Optional[str] + run_dag_root: Optional[str] + created_at: int + last_hit_at: Optional[int] + hit_count: int + n_quotes: Optional[int] + n_verified: Optional[int] + verifier_method: Optional[str] + merkle_proof: dict + run_dag_blob: dict + shard_path: str + + +@dataclass(frozen=True) +class ContextRoot: + """A synthetic root for a multi-source QA context. + + Arborist's providence_cache rows reference a ``source_root`` that + isn't always a single document — when a Q&A is answered against + chunks assembled from several documents, the ``source_root`` is a + merkle root *over those sources*. That root has no row in the + ``documents`` table; its evidence manifest lives in the QA record's + ``merkle_proof`` blob. + + A ``ContextRoot`` exposes that synthesized identity so consumers can + look up any hash uniformly via the read seam, without knowing whether + the row landed in ``documents`` or only in ``providence_cache``. + QA may eventually merge back into the document shards; that's the + seam's problem, not the consumer's. + """ + + context_root: str + cache_key: str + question_text: str + document_uri: str + sources: list[dict] + chunking_version: str + canonicalization_version: str + schema_version: str + created_at: int + hit_count: int + shard_path: str + # Real provenance, derived from `sources`. ``document_uri`` above is + # headlined with ``primary_source_uri`` (instead of the opaque + # ``corpus://multi-source`` sentinel) so consumers show where the + # knowledge actually came from. ``source_domains`` lists the distinct + # contributing hostnames, primary answer source first. + primary_source_uri: Optional[str] = None + source_domains: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class ResolveResult: + """Outcome of ``shards.resolve(hex_hash)``. ``kind`` is one of: + document_root | leaf_hash | audit_event | merkle_interior | + qa_cache_key | context_root | qa_run_dag_root | unknown. + + ``context_root`` fires when the hash matches a ``providence_cache`` + row's ``source_root`` but isn't itself a ``documents`` row — i.e. a + synthesized multi-source context (see :class:`ContextRoot`).""" + + kind: str + hash: str + shard_path: Optional[str] = None + extra: dict = field(default_factory=dict) + + +@dataclass(frozen=True) +class ShardCounts: + documents: int + audit_events: int + qa_records: int + shard_count: int + + +# ---------- the Shards handle ------------------------------------------ + + +class Shards: + """Read-only handle over one or more arborist shards. + + Open with :func:`open_shards`. The handle holds a per-shard SQLite + connection (opened ``mode=ro``) and fans out every query across the + set. ``mode=ro`` plus ``check_same_thread=False`` makes a single + connection safe for read-only sharing across a thread pool (waitress, + gunicorn) — see SQLite docs on shared-read connections. + + The schema versions arborist stores are *not* exposed by Shards as + a strict version check; that is the caller's job if they want it. + Schema migrations are arborist's contract, not the seam's. + """ + + def __init__(self, paths: list[str]): + self._paths: list[str] = list(paths) + self._conns: dict[str, sqlite3.Connection] = {} + for p in self._paths: + try: + # Run arborist's migration probe via the canonical connect() + # to bring the schema current, then drop it. + connect(p).close() + conn = sqlite3.connect( + "file:{0}?mode=ro".format(p), uri=True, check_same_thread=False + ) + conn.row_factory = sqlite3.Row + self._conns[p] = conn + log.debug("arborist.read: opened %s (read-only)", p) + except Exception as exc: + log.warning("arborist.read: failed to open shard %s: %s", p, exc) + + # ---------- meta --------------------------------------------------- + + @property + def paths(self) -> list[str]: + return list(self._conns.keys()) + + def close(self) -> None: + for c in self._conns.values(): + try: + c.close() + except Exception: + pass + self._conns.clear() + + def counts(self) -> ShardCounts: + docs = events = qa = 0 + for c in self._conns.values(): + try: + docs += int(c.execute("SELECT COUNT(*) FROM documents").fetchone()[0]) + except sqlite3.Error: + pass + try: + events += int( + c.execute("SELECT COALESCE(MAX(seq), 0) FROM audit_events").fetchone()[0] + ) + except sqlite3.Error: + pass + try: + qa += int(c.execute("SELECT COUNT(*) FROM providence_cache").fetchone()[0]) + except sqlite3.Error: + pass + return ShardCounts(documents=docs, audit_events=events, qa_records=qa, shard_count=len(self._conns)) + + # ---------- documents / roots -------------------------------------- + + def root(self, document_root: str) -> Optional[Root]: + for path, c in self._conns.items(): + r = c.execute( + "SELECT document_root, document_uri, source_type, kind, title, " + " chunking_version, canonicalization_version, schema_version, " + " ingest_ts, hit_count " + "FROM documents WHERE document_root = ?", + (document_root,), + ).fetchone() + if r is None: + continue + leaf_count = c.execute( + "SELECT COUNT(*) FROM chunks WHERE document_root = ?", + (document_root,), + ).fetchone()[0] + return Root(leaf_count=int(leaf_count), shard_path=path, **dict(r)) + return None + + def roots(self, limit: int = 50) -> list[Root]: + merged: list[Root] = [] + for path, c in self._conns.items(): + rows = c.execute( + "SELECT document_root, document_uri, source_type, kind, title, " + " chunking_version, canonicalization_version, schema_version, " + " ingest_ts, hit_count " + "FROM documents ORDER BY ingest_ts DESC LIMIT ?", + (limit,), + ).fetchall() + for r in rows: + d = dict(r) + lc = c.execute( + "SELECT COUNT(*) FROM chunks WHERE document_root = ?", + (d["document_root"],), + ).fetchone()[0] + merged.append(Root(leaf_count=int(lc), shard_path=path, **d)) + merged.sort(key=lambda r: r.ingest_ts, reverse=True) + return merged[:limit] + + # ---------- leaves / chunks ---------------------------------------- + + def leaves( + self, + document_root: str, + *, + reveal_private: bool = False, + project: bool = True, + ) -> list[Leaf]: + """Read every chunk under ``document_root`` from its shard. + + Decompresses content only when ``reveal_private=True`` (§14 + default-deny). When ``project=True`` (default) and the + ``arborist[wikitext]`` extra is installed, also computes the + deterministic prose projection alongside the raw bytes. + + Returns an empty list when the root is unknown. + """ + conn = self._owning_conn_for_root(document_root) + if conn is None: + return [] + + rows = conn.execute( + "SELECT document_root, idx, leaf_hash, content, tier " + "FROM chunks WHERE document_root = ? ORDER BY idx", + (document_root,), + ).fetchall() + + to_base, base_version = _get_wikitext_projector() if project else (None, None) + + out: list[Leaf] = [] + for r in rows: + content: Optional[str] = None + prose: Optional[str] = None + stamp: Optional[str] = None + if reveal_private and r["content"] is not None: + try: + content = unpack_chunk(r["content"]) + except Exception as exc: + log.warning("unpack_chunk failed for leaf %s: %s", r["leaf_hash"], exc) + if content is not None and to_base is not None: + try: + prose = to_base(content) + stamp = base_version + except Exception as exc: + log.warning("wikitext.to_base failed for leaf %s: %s", r["leaf_hash"], exc) + out.append( + Leaf( + document_root=r["document_root"], + idx=int(r["idx"]), + leaf_hash=r["leaf_hash"], + tier=r["tier"], + content=content, + prose=prose, + base_version=stamp, + ) + ) + return out + + # ---------- proofs / tree ------------------------------------------ + + def proof(self, document_root: str, leaf_index: int) -> Optional[Proof]: + """Compute a Merkle inclusion proof against the shard that owns + the root. Returns ``None`` if the root is unknown; raises + ``IndexError`` if ``leaf_index`` is out of range.""" + conn = self._owning_conn_for_root(document_root) + path = self._path_for_root(document_root) + if conn is None or path is None: + return None + rows = conn.execute( + "SELECT idx, leaf_hash FROM chunks WHERE document_root = ? ORDER BY idx", + (document_root,), + ).fetchall() + if not rows: + return None + if leaf_index < 0 or leaf_index >= len(rows): + raise IndexError( + "leaf_index {0} out of range (root has {1} leaves)".format( + leaf_index, len(rows) + ) + ) + leaves = [bytes.fromhex(r["leaf_hash"]) for r in rows] + tree = MerkleTree.build(leaves) + p = tree.proof(leaf_index) + passed = verify_proof(p) and tree.root.hex() == document_root + raw = proof_to_dict(p) + return Proof( + document_root=document_root, + leaf_index=leaf_index, + leaf_hash=p.leaf.hex(), + sibling_hashes=[s["hash"] for s in raw["siblings"]], + left_right_flags=[bool(s["is_left"]) for s in raw["siblings"]], + computed_root=tree.root.hex(), + expected_root=document_root, + passed=passed, + shard_path=path, + raw=raw, + ) + + def tree_layers(self, document_root: str) -> Optional[list[list[str]]]: + """All Merkle tree layers as hex (layer 0 = leaves, last = root). + Drives the 3D lattice visualization.""" + conn = self._owning_conn_for_root(document_root) + if conn is None: + return None + rows = conn.execute( + "SELECT leaf_hash FROM chunks WHERE document_root = ? ORDER BY idx", + (document_root,), + ).fetchall() + if not rows: + return None + leaves = [bytes.fromhex(r["leaf_hash"]) for r in rows] + tree = MerkleTree.build(leaves) + return [[node.hex() for node in layer] for layer in tree.layers] + + # ---------- audit -------------------------------------------------- + + def audit_event(self, event_hash: str) -> Optional[AuditEvent]: + for path, c in self._conns.items(): + r = c.execute( + "SELECT seq, event_hash, prev_event_hash, event_type, subject_root, body, ts " + "FROM audit_events WHERE event_hash = ?", + (event_hash,), + ).fetchone() + if r is not None: + return _row_to_audit(r, path) + return None + + def audit_chain(self, head_event_hash: str, *, limit: int = 100) -> list[AuditEvent]: + out: list[AuditEvent] = [] + cursor = head_event_hash + for _ in range(limit): + if cursor is None: + break + ev = self.audit_event(cursor) + if ev is None: + break + out.append(ev) + cursor = ev.prev_event_hash + return out + + def audit_recent(self, *, limit: int = 100) -> list[AuditEvent]: + merged: list[AuditEvent] = [] + for path, c in self._conns.items(): + for r in c.execute( + "SELECT seq, event_hash, prev_event_hash, event_type, subject_root, body, ts " + "FROM audit_events ORDER BY seq DESC LIMIT ?", + (limit,), + ).fetchall(): + merged.append(_row_to_audit(r, path)) + merged.sort(key=lambda e: e.ts, reverse=True) + return merged[:limit] + + def audit_by_root(self, root: str, *, limit: int = 50) -> list[AuditEvent]: + merged: list[AuditEvent] = [] + for path, c in self._conns.items(): + for r in c.execute( + "SELECT seq, event_hash, prev_event_hash, event_type, subject_root, body, ts " + "FROM audit_events WHERE subject_root = ? ORDER BY seq DESC LIMIT ?", + (root, limit), + ).fetchall(): + merged.append(_row_to_audit(r, path)) + merged.sort(key=lambda e: e.seq, reverse=True) + return merged[:limit] + + def audit_since( + self, cursor: Optional[dict] = None, *, limit: int = 200 + ) -> tuple[list[AuditEvent], dict]: + """Stream-friendly tail. ``cursor`` is opaque to the caller — pass + ``None`` first time, then pass the dict returned from the previous + call. Returns ``(new_events, next_cursor)``. + + Each shard tracks its own ``last_seq`` so the seam can drive an + SSE stream over all shards without the consumer knowing the + sharding layout. + """ + cursor = dict(cursor or {}) + new_events: list[AuditEvent] = [] + for path, c in self._conns.items(): + last_seq = int(cursor.get(path, 0)) + for r in c.execute( + "SELECT seq, event_hash, prev_event_hash, event_type, subject_root, body, ts " + "FROM audit_events WHERE seq > ? ORDER BY seq ASC LIMIT ?", + (last_seq, limit), + ).fetchall(): + ev = _row_to_audit(r, path) + new_events.append(ev) + cursor[path] = max(int(cursor.get(path, 0)), ev.seq) + return new_events, cursor + + def audit_cursor(self) -> dict: + """Cursor at the current head of every shard (use as starting + point for a tail that only wants future events).""" + return { + path: int(c.execute("SELECT COALESCE(MAX(seq), 0) FROM audit_events").fetchone()[0]) + for path, c in self._conns.items() + } + + # ---------- providence_cache (Q&A) --------------------------------- + + def context(self, context_root: str) -> Optional[ContextRoot]: + """Synthesize a :class:`ContextRoot` for a multi-source QA root. + + Looks up the first ``providence_cache`` row whose ``source_root`` + equals ``context_root`` and surfaces the evidence manifest parsed + out of its ``merkle_proof`` JSON. Returns ``None`` when no QA + record cites this root. + + Callers should fall through from :meth:`root` to this method when + a hash is "a root but not a document" — that case happens when + the QA pipeline answered against assembled context rather than a + single ingested document. Consumers don't need to know which + case applies; they just chain the calls. + """ + for path, c in self._conns.items(): + try: + r = c.execute( + _QA_SELECT + " WHERE source_root = ? LIMIT 1", + (context_root,), + ).fetchone() + except sqlite3.Error: + continue + if r is None: + continue + qa = _row_to_qa(r, path) + sources = qa.merkle_proof.get("sources", []) if isinstance(qa.merkle_proof, dict) else [] + primary_uri, domains = _summarize_sources(sources) + # Headline the real source. The stored document_uri for a + # multi-source answer is the opaque ``corpus://multi-source`` + # sentinel — replace it with the primary answer source so the + # node says where the knowledge came from (e.g. + # russell.ballestrini.net), not a placeholder. + headline_uri = qa.document_uri or "" + if (not headline_uri or headline_uri == _MULTI_SOURCE_URI) and primary_uri: + headline_uri = primary_uri + return ContextRoot( + context_root=context_root, + cache_key=qa.cache_key, + question_text=qa.question_text or "", + document_uri=headline_uri, + sources=sources, + chunking_version=qa.chunking_version, + canonicalization_version=qa.canonicalization_version, + schema_version=qa.schema_version, + created_at=qa.created_at, + hit_count=qa.hit_count, + shard_path=path, + primary_source_uri=primary_uri, + source_domains=domains, + ) + return None + + def context_layers(self, context_root: str) -> Optional[list[list[str]]]: + """Rebuild the context tree's layers from the sorted source roots. + + Mirrors :meth:`tree_layers` for synthetic context roots. Layer 0 + is leaves (the sorted source document_roots); the last layer is + ``[context_root]``. Uses the same canonicalization as + :func:`arborist.qa.query._context_root` — sort source roots, build + a :class:`MerkleTree`, return its layers as hex. + + Returns ``None`` when the context root isn't in providence_cache. + """ + ctx = self.context(context_root) + if ctx is None: + return None + leaves_hex = _context_leaf_hashes(ctx.sources) + if not leaves_hex: + return [[context_root]] + leaves = [bytes.fromhex(h) for h in leaves_hex] + tree = MerkleTree.build(leaves) + return [[node.hex() for node in layer] for layer in tree.layers] + + def context_proof(self, context_root: str, leaf_index: int) -> Optional[Proof]: + """Inclusion proof for one source under a context root. + + Mirrors :meth:`proof`. The leaf at index ``leaf_index`` is the + sorted-source document_root. Raises ``IndexError`` if out of + range. Returns ``None`` when the context root isn't known. + """ + ctx = self.context(context_root) + if ctx is None: + return None + leaves_hex = _context_leaf_hashes(ctx.sources) + if not leaves_hex: + return None + if leaf_index < 0 or leaf_index >= len(leaves_hex): + raise IndexError( + "leaf_index {0} out of range (context has {1} sources)".format( + leaf_index, len(leaves_hex) + ) + ) + leaves = [bytes.fromhex(h) for h in leaves_hex] + tree = MerkleTree.build(leaves) + p = tree.proof(leaf_index) + passed = verify_proof(p) and tree.root.hex() == context_root + raw = proof_to_dict(p) + return Proof( + document_root=context_root, + leaf_index=leaf_index, + leaf_hash=p.leaf.hex(), + sibling_hashes=[s["hash"] for s in raw["siblings"]], + left_right_flags=[bool(s["is_left"]) for s in raw["siblings"]], + computed_root=tree.root.hex(), + expected_root=context_root, + passed=passed, + shard_path=ctx.shard_path, + raw=raw, + ) + + def qa(self, cache_key: str) -> Optional[QaRecord]: + for path, c in self._conns.items(): + r = c.execute( + _QA_SELECT + " WHERE cache_key = ?", (cache_key,) + ).fetchone() + if r is not None: + return _row_to_qa(r, path) + return None + + def qa_recent(self, *, limit: int = 50) -> list[QaRecord]: + merged: list[QaRecord] = [] + for path, c in self._conns.items(): + for r in c.execute( + _QA_SELECT + " ORDER BY created_at DESC LIMIT ?", (limit,) + ).fetchall(): + merged.append(_row_to_qa(r, path)) + merged.sort(key=lambda q: q.created_at, reverse=True) + return merged[:limit] + + def qa_by_root(self, source_root: str, *, limit: int = 50) -> list[QaRecord]: + merged: list[QaRecord] = [] + for path, c in self._conns.items(): + for r in c.execute( + _QA_SELECT + " WHERE source_root = ? ORDER BY created_at DESC LIMIT ?", + (source_root, limit), + ).fetchall(): + merged.append(_row_to_qa(r, path)) + merged.sort(key=lambda q: q.created_at, reverse=True) + return merged[:limit] + + def qa_search(self, text: str, *, limit: int = 50) -> list[QaRecord]: + """Substring search over ``question_text`` (case-insensitive). + + FTS5 isn't indexed on providence_cache rows in the current + arborist schema; a LIKE scan over 4-5k QA records is cheap + enough for v1. Swap to FTS5 if the corpus grows past ~100k + records. + """ + pat = "%{0}%".format(text.replace("%", r"\%").replace("_", r"\_")) + merged: list[QaRecord] = [] + for path, c in self._conns.items(): + for r in c.execute( + _QA_SELECT + " WHERE question_text LIKE ? ESCAPE '\\' " + "ORDER BY created_at DESC LIMIT ?", + (pat, limit), + ).fetchall(): + merged.append(_row_to_qa(r, path)) + merged.sort(key=lambda q: q.created_at, reverse=True) + return merged[:limit] + + # ---------- hash resolver ------------------------------------------ + + def resolve(self, hex_hash: str) -> ResolveResult: + """Classify any hex hash. Order of search: + + document_root → leaf_hash → audit event_hash → providence_cache + cache_key → providence_cache source_root → providence_cache + run_dag_root → merkle_nodes interior hash → unknown. + """ + h = hex_hash.lower().strip() + + for path, c in self._conns.items(): + if c.execute("SELECT 1 FROM documents WHERE document_root = ? LIMIT 1", (h,)).fetchone(): + return ResolveResult(kind="document_root", hash=h, shard_path=path) + + for path, c in self._conns.items(): + r = c.execute("SELECT document_root, idx FROM chunks WHERE leaf_hash = ? LIMIT 1", (h,)).fetchone() + if r: + return ResolveResult( + kind="leaf_hash", hash=h, shard_path=path, + extra={"document_root": r["document_root"], "leaf_index": int(r["idx"])}, + ) + + for path, c in self._conns.items(): + r = c.execute("SELECT subject_root FROM audit_events WHERE event_hash = ? LIMIT 1", (h,)).fetchone() + if r: + return ResolveResult( + kind="audit_event", hash=h, shard_path=path, + extra={"subject_root": r["subject_root"]}, + ) + + for path, c in self._conns.items(): + try: + r = c.execute( + "SELECT source_root FROM providence_cache WHERE cache_key = ? LIMIT 1", (h,) + ).fetchone() + if r: + return ResolveResult( + kind="qa_cache_key", hash=h, shard_path=path, + extra={"source_root": r["source_root"]}, + ) + except sqlite3.Error: + continue + + # providence_cache.source_root: the QA's evidence root. Reaches + # this branch only when the hash isn't in ``documents`` (the + # documents probe runs first and returns "document_root"), so + # any match here is a synthesized multi-source context. + for path, c in self._conns.items(): + try: + r = c.execute( + "SELECT cache_key FROM providence_cache WHERE source_root = ? LIMIT 1", (h,) + ).fetchone() + if r: + return ResolveResult( + kind="context_root", hash=h, shard_path=path, + extra={"cache_key": r["cache_key"]}, + ) + except sqlite3.Error: + continue + + for path, c in self._conns.items(): + try: + r = c.execute( + "SELECT cache_key FROM providence_cache WHERE run_dag_root = ? LIMIT 1", (h,) + ).fetchone() + if r: + return ResolveResult( + kind="qa_run_dag_root", hash=h, shard_path=path, + extra={"cache_key": r["cache_key"]}, + ) + except sqlite3.Error: + continue + + for path, c in self._conns.items(): + r = c.execute( + "SELECT document_root, layer, idx FROM merkle_nodes WHERE hash = ? LIMIT 1", (h,) + ).fetchone() + if r: + return ResolveResult( + kind="merkle_interior", hash=h, shard_path=path, + extra={ + "document_root": r["document_root"], + "layer": int(r["layer"]), + "idx": int(r["idx"]), + }, + ) + + return ResolveResult(kind="unknown", hash=h) + + # ---------- internals ---------------------------------------------- + + def _owning_conn_for_root(self, document_root: str) -> Optional[sqlite3.Connection]: + for _path, c in self._conns.items(): + if c.execute( + "SELECT 1 FROM documents WHERE document_root = ? LIMIT 1", + (document_root,), + ).fetchone(): + return c + return None + + def _path_for_root(self, document_root: str) -> Optional[str]: + for path, c in self._conns.items(): + if c.execute( + "SELECT 1 FROM documents WHERE document_root = ? LIMIT 1", + (document_root,), + ).fetchone(): + return path + return None + + +def open_shards(paths: Iterable[str | Path]) -> Shards: + """Open every shard at ``paths`` (read-only) and return a ``Shards`` + handle. Shards that fail to open are skipped with a warning.""" + return Shards([str(p) for p in paths]) + + +# ---------- row decoders ----------------------------------------------- + + +_QA_SELECT = ( + "SELECT cache_key, source_root, document_uri, question_hash, question_text, " + " answer_text, model_profile_hash, conversation_hash, " + " governance_policy_hash, schema_version, canonicalization_version, " + " chunking_version, falsification_state, audit_mode, audit_event_hash, " + " run_dag_root, created_at, last_hit_at, hit_count, n_quotes, " + " n_verified, verifier_method, merkle_proof, run_dag_blob " + "FROM providence_cache" +) + + +# Sentinel ``document_uri`` stored on multi-source providence_cache rows +# (see ``arborist.qa.query``). Opaque on purpose at write time; the read +# seam projects it back to real provenance via ``_summarize_sources``. +_MULTI_SOURCE_URI = "corpus://multi-source" + + +def _domain_of(uri: str) -> str: + """Hostname of a source URI, or '' for opaque/relative URIs.""" + try: + from urllib.parse import urlparse + + return (urlparse(uri).netloc or "").lower() + except Exception: + return "" + + +def _summarize_sources(sources: list[dict]) -> tuple[Optional[str], list[str]]: + """Derive (primary_source_uri, source_domains) from a context root's + sources. The primary answer source ranks first; remaining sources keep + their order. Domains are de-duplicated, primary first.""" + if not sources: + return None, [] + primaries = [ + s for s in sources + if isinstance(s, dict) and s.get("source_role") == "primary_answer_source" + ] + others = [s for s in sources if isinstance(s, dict) and s not in primaries] + ordered = primaries + others + primary_uri = None + for s in ordered: + if s.get("document_uri"): + primary_uri = s["document_uri"] + break + domains: list[str] = [] + for s in ordered: + dom = _domain_of(s.get("document_uri") or "") + if dom and dom not in domains: + domains.append(dom) + return primary_uri, domains + + +def _context_leaf_hashes(sources: list[dict]) -> list[str]: + """Canonical leaf hashes for a context tree. + + Mirrors :func:`arborist.qa.query._context_root` — the leaves are the + sources' ``document_root`` values, sorted lexicographically. Keeping + this helper next to :meth:`Shards.context_layers` / + :meth:`Shards.context_proof` lets the read seam reconstruct the tree + and prove inclusion without re-importing the QA pipeline. + """ + roots = [s.get("document_root") for s in sources if isinstance(s, dict) and s.get("document_root")] + return sorted(r.lower() for r in roots) + + +def _row_to_audit(r: sqlite3.Row, shard_path: str) -> AuditEvent: + try: + body = json.loads(r["body"]) if r["body"] else {} + except Exception: + body = {"_raw": r["body"]} + return AuditEvent( + seq=int(r["seq"]), + event_hash=r["event_hash"], + prev_event_hash=r["prev_event_hash"], + event_type=r["event_type"], + subject_root=r["subject_root"], + body=body, + ts=int(r["ts"]), + shard_path=shard_path, + ) + + +def _row_to_qa(r: sqlite3.Row, shard_path: str) -> QaRecord: + def _json_or_empty(s): + if not s: + return {} + try: + return json.loads(s) + except Exception: + return {"_raw": s} + + return QaRecord( + cache_key=r["cache_key"], + source_root=r["source_root"], + document_uri=r["document_uri"], + question_hash=r["question_hash"], + question_text=r["question_text"], + answer_text=r["answer_text"], + model_profile_hash=r["model_profile_hash"], + conversation_hash=r["conversation_hash"], + governance_policy_hash=r["governance_policy_hash"], + schema_version=r["schema_version"], + canonicalization_version=r["canonicalization_version"], + chunking_version=r["chunking_version"], + falsification_state=r["falsification_state"], + audit_mode=r["audit_mode"] if "audit_mode" in r.keys() else None, + audit_event_hash=r["audit_event_hash"] if "audit_event_hash" in r.keys() else None, + run_dag_root=r["run_dag_root"] if "run_dag_root" in r.keys() else None, + created_at=int(r["created_at"]), + last_hit_at=int(r["last_hit_at"]) if r["last_hit_at"] is not None else None, + hit_count=int(r["hit_count"]), + n_quotes=int(r["n_quotes"]) if "n_quotes" in r.keys() and r["n_quotes"] is not None else None, + n_verified=int(r["n_verified"]) if "n_verified" in r.keys() and r["n_verified"] is not None else None, + verifier_method=r["verifier_method"] if "verifier_method" in r.keys() else None, + merkle_proof=_json_or_empty(r["merkle_proof"]), + run_dag_blob=_json_or_empty(r["run_dag_blob"]) if "run_dag_blob" in r.keys() else {}, + shard_path=shard_path, + ) + + +def _get_wikitext_projector(): + """Resolve arborist.wikitext.to_base lazily (the wikitext extra is + optional). Returns ``(callable, base_version)`` or ``(None, None)``.""" + try: + from arborist.wikitext import to_base, BASE_VERSION + return to_base, BASE_VERSION + except Exception: + return None, None diff --git a/docs/TICKETS.md b/docs/TICKETS.md index bd75ef7..e83b90b 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -111,6 +111,7 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| +| #000069 | Arborist VIZ / Merkle Command Center (Pyramid + six.js + SSE browser dashboard) | **open · awaiting go/no-go · doc-only scaffold** (2026-05-27; filed from `/home/fox/Downloads/TICKET_0000VIZ_*`, stack corrected same day per fox). Configurable browser dashboard for inspecting arborist's content-addressed state: Merkle root explorer, proof verifier, claim warrant + graveyard, audit timeline, run-DAG replay, cache-key explainer, root diff, 3D Merkle lattice, optional circuit/activation traces. Read-only consumer; arborist proper stays source-of-truth, dashboard projects state. **Stack pinned to unturf-native** (fox 2026-05-27, supersedes proposal §3): **Pyramid + Jinja2 + SQLAlchemy** (matches `remarkbox` / `make_post_sell` / `unhomeschool.com` idiom), **SSE** (`text/event-stream` via Pyramid streaming response) for live audit/claim/falsifier patches, **vanilla JS + six.js** (fox's patched three.js fork at `git.unturf.com/gumyum/six.js` — three.js r175 + CWE-407 patches incl. ObjectBVH O(N)→O(log N); bundles vendored from `~/git/cupPCB/cdn/six/`; third-instance MOAD-0001 dogfood alongside `java-topology` + gumyum-engine) for 3D widgets and large-graph rendering, SQLite for dashboard metadata (no PostgreSQL/ClickHouse/Redis/NATS by default — promote on measured need), no React / no Next.js / no Node build step. Server-rendered SVG (or Graphviz `.dot` per existing `docs/diagrams/*.dot` pattern) replaces React Flow for run-DAG widgets. Browser-side proof verification dropped from v1 (server-side Pyramid view returns PASS/FAIL + receipt; reinstate phase-N only if third-party-verification use case surfaces). **Three filing-note gates before phase 0** (in ticket body): **F-1** sibling-repo home — implementation lives in a new `~/git/arborist-viz` (Pyramid Python, matches existing unturf apps), not in-tree; arborist's contribution is the read-API spec + view package + arborist library import via `arborist.embed`. **F-2** scope split — proposal carries 8 phases (§17 phases 0–8); recommended cut keeps phases 0–3 (schema + shell + proof/root widgets + claim/audit/run widgets) inside #000069, and spawns sibling tickets for SSE streaming (4), 3D six.js (5), massive-graph (6, only if measured need surfaces), circuit-tracing (7, gated on #000062), embeddable widgets (8) — Dav1d-audience rule. **F-3** upstream prereqs — phase 7 (circuit/activation) consumes **#000062 Mechanistic Witness**'s `MechanisticWitnessRoot`; phase 3's claim-graveyard widget projects **#000059**'s bounded-ingestion graveyard. Hard constraints: arborist soft-vs-hard discipline applies verbatim (attribution weights renderable but never `audit_mode`, never causal without intervention/ablation evidence); private-leaf default-deny (commitments + hashes + redacted maps only without explicit auth); every widget exposes its data query + source roots. Reserved scope: NOT a replacement for `arborist controller-events` / `arborist analyze` / `arborist inspect` CLI — those stay canonical inspector surfaces; VIZ is the projection layer. | 2026-05-27 | — | | #000068 | Verifier-blind missed-answer falsification guard | **in progress · Phase 1+2+3 landed 2026-05-27 · Phase 4 default flip NO-GO** (Phase 2 bench 2026-05-27 76q × n=3 claim_lattice Hermes-3-8B: 2/228 sidecar fires, both STRONG confidence, both the Ballestrini regression fixture, 100% precision, 0/226 false positives across non-Ballestrini runs. Phase 3 demote flag opt-in via `--demote-on-missed-answer` on `query`/`ask` — wires `EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL` for strong/medium confidence on lattice modes; lower rungs + non-lattice modes get `· missed-answer` tail tag. `answerability_demote_enabled` added to `_VERIFIER_POLICY_FIELDS` so flipping the flag partitions cache via verifier_policy_hash. Default OFF per Dav1d Phase 4 NO-GO — 100% precision at n=2 fires is too few samples to claim precision floor empirically; default flip blocks on wider bench + human spot-check. 47 tests (36 Phase 1 + 11 Phase 3) all passing. End-to-end verified live: 4/4 Hermes runs on Ballestrini with --demote-on-missed-answer rendered EVIDENCE-MISSED-PARTIAL.) Original opening 2026-05-27 (Dav1d de-novo review GO for Phase 1 with seven hardenings folded into spec — subject-token cue-stripping, answer-type alignment, confidence_class, candidate cap=10, precise offset_start/end/basis, cache-hit recompute-on-read, Phase 1 out of verifier_policy_hash). Original opening 2026-05-27; sibling to the user-payload-layout work shipped 2026-05-26, split out per the Dav1d-audience rule — `feedback_ticket_proliferation`). Surfaced by the Ballestrini case: evidence E2 literally contained the song names, Hermes-3-8B under `user_payload_layout=tail` said *"specific songs by her are not mentioned in the provided evidence blocks"*, verifier marked the run `EVIDENCE-WARRANTED` 2/2 because nothing positive was unsupported. **Verifier-blind false-negative class** — existing layered verifier (quote/span/entity/paraphrase + Rule 8 + Rule 9 + claim ceiling) guards unsupported *presence*, has no hook for unsupported *absence*. Layout fixes attention placement on the specific instance (n=3 bench 2026-05-27 confirms bookend/per_chunk recover Ballestrini); layout alone can't close the class — adversarial phrasing or bigger prompt resurfaces it under any layout. Proposed deterministic sidecar in `arborist/qa/inspect.py:diagnose_missed_answer`: three-clause conjunction — **(A)** answer matches denial pattern ("not mentioned", "not provided", "the evidence does not say", …, closed list versioned via `denial_patterns_version`); **(B)** question is extraction shape (reuse `arborist.qa.quantifier` classifier — `ALL`/`COMPREHENSIVE`/`OPEN_REQUEST` intensities, OR surface cues "songs by"/"works by"/"who wrote"/"list"/"name all"); **(C)** evidence contains candidate spans near subject tokens (reuse `entity_proximity_n`/`entity_proximity_window` from verify.py — quoted strings, title-case spans, comma-separated title lists within W chars of stemmed subject content tokens). All three must fire. Output: `result["answerability"]` with `missed_answer_candidate_spans` list (evidence_id + offset + text). **Hash discipline:** sidecar fields (`denial_patterns_version`, `extraction_cues_version`, `answerability_threshold`) fold into `governance_policy_hash` only; an optional `answerability_demote_enabled` flag (default OFF) wires `EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL` in `_render_audit_label`, and IF on folds into BOTH `governance_policy_hash` AND `verifier_policy_hash` (changes rendered audit_mode, so verifier hash must move — the deliberate opt-in moves the verifier hash, sidecar-only stays out). No LLM-as-judge. Never writes `providence_cache`/`audit_events`. Never promotes claims. Pattern verbatim from `arborist.qa.inspect.diagnose_*` (deflection, coherence, title-relevance). Phases: 1 sidecar read-only, 2 bench + threshold tuning, 3 demote opt-in, 4 default decision (bench-gated). 5F-Falsification fixture: Ballestrini case already in `bench/qa_questions.txt` under "entity list". Full spec in `docs/tickets/ticket-000068-verifier-blind-missed-answer-guard.md`. | 2026-05-27 | D2 | | #000067 | M-aware cold-pack hydration (route incoming docs by content hash into M target shards) | **open · scaffold · prereq for #46 genesis test** (2026-05-26; surfaced while preparing the 3090 SPV-wallet validation). Today's `hydrate_from_metadata_pack` takes a single `conn` and writes every incoming row into one shard. With the corpus now in M=4 hash-routed topology (#000065), a fresh peer needs to land each document on `shard_for_document(document_root, M)` — same routing function as the producer. Without this, a fresh peer's `~/.arborist/shards/` is just one big single-shard DB and the M=4 ATTACH-and-route assumption #000065 was sized for doesn't hold consumer-side. Two coherent shapes: **(α) two-step kludge** — hydrate into single shard, then `arborist corpus reshard --to M` on the consumer. Works today (proven by the 2026-05-26 reshard executor) but doubles the wall time and treats packed shards as if they came from an arbitrary topology. **(β) direct M-aware hydrate** — extend `hydrate_from_metadata_pack` to accept `targets: list[sqlite3.Connection]` + `M: int` and route per-row at restore time (reusing `arborist.document.shard_for_document` + the table-routing rules in `arborist/migrate.py`). Manifest carries `corpus_shard_count` so the unpacker knows M from the pack itself. β is the right answer — α exists only as a fallback if 20-min-window pressure forces it. Sequence: (1) add `corpus_shard_count` to pack manifest (read from source meta during `dump_shard_metadata`); (2) `restore_shard_metadata_routed(targets, M, table_dir)` in `cold_pack_metadata.py` mirroring `_route_per_doc_table` from migrate.py; (3) `hydrate_from_metadata_pack` gains a `targets`/`shards_dir` param; (4) `arborist cold unpack --shards-dir DIR` initialises M target shards from the manifest's `corpus_shard_count` and routes; (5) regression test: pack 2 shards → hydrate into fresh 4 shards → assert every doc on its hash-routed target. Refactor opportunity: the routing rules (ROUTED_BY_DOCUMENT_ROOT, CONSOLIDATED_TABLES) currently live in migrate.py; this ticket can either duplicate them in cold_pack_metadata.py (fast) or factor into a shared `arborist/multi_shard.py` module (cleaner). The shared-module path is more honest given graft mode (#000066) wants the same primitives. Out of scope: graft / overlay mode (that's #000066 — overlays onto populated, this is hydrate-into-empty). | 2026-05-26 | — | | #000066 | Cold-pack overlay / graft mode (pack-as-package, witness-pattern audit chain) | **scaffold-only · awaiting go/no-go** (2026-05-26; surfaced while running #000065 reshard, fox extension: "we could envision a pack for wikipedia 2010, wikipedia current, etc"). Extend #000061 cold-pack hydration with a second mode: overlay an existing pack onto a populated shard set instead of hydrating into empty. Doc/chunk/edge/concept overlay is trivial (`INSERT OR IGNORE` on content-addressed PKs collapses dupes); FTS5 overlay is trivial (new chunks → new rowids → new FTS rows). The interesting part is the audit chain — can't naively append the pack's events because `prev_event_hash` linkage breaks across the join. Chosen approach: **graft receipt**. Append one new `event_type='graft'` event to the host chain carrying `(pack_hash, snapshot_root, corpus_name, event_count, first_event_hash, last_event_hash, manifest_root)`; the pack file itself becomes the durable witness for the absorbed events (anyone can re-fetch the pack, walk its internal chain, and verify it matches the receipt). Host chain stays linear; pack chain is a "witnessed subgraph." This is the same witness pattern Merkle-AGI v8/v9 is heading toward, but bought at near-zero schema cost. Rejected alternatives: re-chain everything (breaks external refs to old event_hashes — cache_keys anchoring to old `audit_event_hash`, snapshots, etc. — silently invalid); chain forest with new `chain_id` column (right answer when graft dominates the lifecycle, but premature now). **Pack-as-package extension** (fox 2026-05-26): each pack carries a `corpus_name` field in its manifest (`wikipedia-2010`, `wikipedia-current`, `arxiv-cs`, `textbooks-undergrad`, …) so operators pick which corpora to graft — `arborist cold graft wikipedia-current` becomes as natural as `apt install firefox`. Multiple packs of the same corpus name: most-recent `snapshot_root` wins; older packs stay in the bucket until GC. URI conflicts across corpora (e.g., `wikipedia.org/wiki/Foo` in both 2010 and current): different content → different `document_root` → both stored, `supersedes` edges per CLAUDE.md invariant. Providence-cache conflicts: same `cache_key` with different answer → existing v9.8 falsification framework handles it (`state='stale'` or `quarantined`). Mesh-peer-corpus-merge: each peer's pack is a graftable package; partition reconciliation becomes "exchange the packs you each carry, graft what you lack". The mesh-of-arborists semantic. Sequence: (1) `corpus_name` field in #000061 manifest format + alias index in bucket (`corpora//latest.json` pointer to active pack_hash); (2) `arborist cold graft ` / `arborist cold graft --corpus ` mode in evict.py — read pack, INSERT OR IGNORE per-table, emit graft receipt; (3) conflict-policy flag (`--on-uri-conflict {supersedes,skip,fail}`, default `supersedes`); (4) `arborist cold list-corpora` shows available packages in a bucket. Scaffold first, code only when (a) #000065 reshard lands and stabilises (b) a second corpus exists (the wikipedia-current snapshot, or first textbook bundle ready to graft onto wikipedia-2010 base) (c) at least two peers want to exchange. | 2026-05-26 | — | @@ -182,4 +183,4 @@ Newest first. Update on every open/close. ## Next ID -`000069` +`000070` diff --git a/docs/crawler.md b/docs/crawler.md index 7389edb..5d85232 100644 --- a/docs/crawler.md +++ b/docs/crawler.md @@ -226,16 +226,29 @@ arborist crawler recrawl-check [--domain D] [--limit N] - `--author` — default author surname appended to titles for warrant resolution (only with `--ingest`). -Make targets drive the textbook-crawl workflow; crawl shards land in +Make targets drive both crawl workflows; crawl shards land in `~/.arborist/crawl/` (separate from the main `~/.arborist/shards` so -SQLite's attached-DB limit isn't tripped): +SQLite's 10-attached-DB limit isn't tripped, and so locally crawled +content isn't shared as a peer by default): ``` -make crawl-textbooks # BFS-crawl every manifest entry with a crawl_url -make textbook ID= # ingest one textbook by id -make crawl-textbooks-stats # docs-per-shard summary +make crawl-ingest URL=https://x.com # general web crawl → ONE central + # db (CRAWL_DB, default web.db) +make crawl-textbooks # BFS-crawl every manifest entry + # with a crawl_url (warrant substrate) +make textbook ID= # ingest one textbook by id +make crawl-textbooks-stats # docs-per-shard summary ``` +General web crawls (`make crawl-ingest`) all flow into a **single** +central db rather than one-per-domain: content-addressing lets many +domains coexist in one file (idempotent re-ingest, `supersedes` edges +on change), and a single file always attaches under the 10-DB cap. +Query it standalone with `arborist --db ~/.arborist/crawl/web.db query +"…"`, or attach it alongside the main corpus when you want unified +results. The per-host textbook crawls stay separate — they are warrant +substrate, resolved through a different path. + ## Source map | File | Role | diff --git a/docs/tickets/ticket-000069-arborist-viz-merkle-command-center.md b/docs/tickets/ticket-000069-arborist-viz-merkle-command-center.md new file mode 100644 index 0000000..3fe08a9 --- /dev/null +++ b/docs/tickets/ticket-000069-arborist-viz-merkle-command-center.md @@ -0,0 +1,1900 @@ +# Ticket #000069 — Arborist VIZ / Merkle Command Center + +**Status:** open · awaiting go/no-go +**Opened:** 2026-05-27 +**Scope:** Tableau-inspired, Palantir-style browser dashboard for inspecting +Merkle roots, proofs, claims, audit trails, run-DAGs, claim graveyard, +3D state geometry, and optional circuit/activation traces produced by +arborist. Read-only consumer of arborist's content-addressed state. +**Audience:** product/UX, Dav1d de-novo review, downstream integrators +(neopig, undefect.com). +**Hard constraint:** + +1. **Use unturf's existing stack — not invent a new one** (fox + 2026-05-27). Backend: **Pyramid** (Python) + Jinja2 + SQLAlchemy, + matching `~/git/remarkbox` / `~/git/make_post_sell` / + `~/git/unhomeschool.com` idioms. Streaming: **SSE** + (`text/event-stream`) via Pyramid streaming response. Frontend: + **vanilla JS** + **six.js** (= `~/git/six.js`, fox's vendored + three.js) for 3D widgets. **NO React, NO Next.js, NO Node** + toolchain, NO NATS JetStream, NO PostgreSQL/ClickHouse/Redis + bring-up unless a measured need surfaces. VIZ lives in a + **sibling repo** `~/git/arborist-viz` (Pyramid Python, optional + `[viz]` extra), matching the `arborist-zk-bench` / + `arborist-world` pattern; arborist proper stays + `python3.12 + venv + sqlite3`. See **§3-corrected** below for the + pinned stack; original §3 is preserved as superseded design log. +2. The dashboard projects state, never asserts it. Merkle roots and + audit events remain source-of-truth in arborist shards; every + widget exposes the query + source roots it derived from + (§19.3 of the proposal body). +3. **Soft signals never enter the proof path** (CLAUDE.md "soft hash + vs hard hash" convention). The activation/circuit-trace layer + (§11, §6.10) MAY render attribution weights, but it inherits + #000049's cage verbatim: hash-pinned, governance-hashed, + demotion-only-if-it-ever-feeds-`audit_mode`, never causal without + intervention/ablation evidence. `audit_mode` is unaffected by + anything VIZ displays. +4. **Private-leaf default-deny** (proposal §14). Commitments, hashes, + redacted evidence maps, selective-disclosure objects only — raw + chunks/prompts/traces require explicit authorization. No + widget bypasses this gate. + +--- + +## Filing notes (2026-05-27) + +This ticket files the proposal text verbatim from +`/home/fox/Downloads/TICKET_0000VIZ_Arborist_VIZ__Merkle_Command_Center.txt`, +wrapped in the arborist ticket header above. Three structural decisions +were flagged at file time and stand as open items for the next review: + +**(F-1) Repo home + stack — CORRECTED 2026-05-27.** Proposal §3 +specifies Next.js + React + TypeScript + Node + Fastify + NATS +JetStream + PostgreSQL + ClickHouse + Redis. fox rejected that stack +2026-05-27: *"we will want to use the frameworks that unturf already +uses (not react) three.js (six.js) pyramid python prefered with SSE +see https://www.unturf.com/software/"*. Pinned stack lives in +**§3-corrected** below. The repo home is still a sibling — `~/git/arborist-viz` +(Pyramid Python app, not in-tree) — but now matches `remarkbox` / +`make_post_sell` / `unhomeschool.com` idioms instead of standing up a +parallel TypeScript world. Arborist's in-tree contribution is still +just the **read-API spec + Python read-API impl** (proposal §13: HTTP +routes + SSE endpoints, both Pyramid-flavored); the only code that +might land in `arborist/` itself is `arborist/viz_api/` (an optional +Pyramid view package) or, more likely, nothing in-tree at all (the +sibling repo imports `arborist` as a library via `arborist.embed`, +per `docs/embedding.md`). + +**(F-2) Scope split.** Proposal §17 lists eight implementation phases. +Don't-proliferate (CLAUDE.md) says default to extending one ticket; the +Dav1d-audience rule says split when a sub-piece needs an independent +audience. Recommended cut: + +| Sub-ticket | Phases | Audience | +|---|---|---| +| #000069 (this) | §17 phases 0–3: schema, dashboard shell, proof/root widgets, claim/audit/run widgets | core product + Dav1d (the foundational layer) | +| sibling A | §17 phase 4: NATS → WebSocket streaming | infra/ops review | +| sibling B | §17 phase 5: 3D Merkle lattice (R3F + Three.js) | UX + GPU/perf review | +| sibling C | §17 phase 6: massive-graph widgets (Cosmograph) | scale review | +| sibling D | §17 phase 7: circuit/activation trace widgets | gated on #000062 producing a `MechanisticWitnessRoot`; mechanistic-interp review | +| sibling E | §17 phase 8: embeddable web-components NPM package | API/integration review | + +If fox prefers a single ticket carrying all phases, leave as-is and +spawn sub-tickets only when a phase's review surface diverges from the +others. **Default this ticket scope to phases 0–3 unless told otherwise.** + +**(F-3) Upstream prereqs.** Two arborist tickets are upstream of VIZ +phases and need explicit cross-references for the de-novo review: + +- **#000062 Mechanistic Witness (scaffold-only, awaiting go/no-go)** — + produces `MechanisticWitnessRoot` over (model, capture policy, + neurons/features, intervention deltas). VIZ §6.10 (Circuit Trace + Widget) + §11 (full circuit/activation tracing layer) are + **consumers** of this root. Phase D of this ticket cannot start + until #000062 lands a witness producer. VIZ §11's hard rule + ("never causal unless intervention or ablation evidence exists") + must remain word-identical to #000062's four guardrails so the cage + is consistent across producer + display layer. +- **#000059 Admission discipline / claim-graveyard** — VIZ §6.9 (Claim + Graveyard Widget) is one UI projection of #000059's bounded-ingestion + graveyard. Bounded-ingestion hard constraint (fox 2026-05-20) + applies: the graveyard widget MUST display the recurring-error + surface, never queries-ever, and must not encourage operators to + treat the graveyard as a transcript store. + +The proposal body below stands as the design draft. Phases, widget +catalog, schemas, and APIs are the design log; the four hard +constraints above and the three filing notes are the arborist-specific +gates the design must pass before any phase 0 work lands. + +--- + +## §3-corrected — Pinned unturf-native stack (2026-05-27) + +fox 2026-05-27: *"we will want to use the frameworks that unturf +already uses (not react) three.js (six.js) pyramid python prefered +with SSE — see https://www.unturf.com/software/"*. This section +supersedes the original §3 (preserved unchanged below as the +superseded draft / design log). + +### Backend (sibling repo `~/git/arborist-viz`) + +```text +Python 3.10+ +Pyramid — web framework (matches remarkbox / make_post_sell) +pyramid_jinja2 — Jinja2 server-rendered templates +pyramid_tm — transaction manager +pyramid_retry — retry on conflict +SQLAlchemy — dashboard metadata (saved layouts / filter presets) + (read-only access to arborist shards goes through + `arborist.embed` / direct sqlite3, NOT SQLAlchemy) +alembic — schema migrations for the dashboard metadata DB +waitress — production WSGI server (Pyramid default) +arborist — imported as a Python library for read-only + provenance access; arborist shards stay the + source of truth +``` + +### Storage + +```text +arborist SQLite shards — provenance source of truth (read-only) +SQLite for dashboard + metadata — saved layouts, filter presets, user prefs + (NOT PostgreSQL — keeps unturf's + "python3.12 + venv + sqlite3" floor) +``` + +Pyramid's standard SQLAlchemy-on-PostgreSQL idiom (the remarkbox +pattern) is the obvious *escape hatch* when dashboard metadata +outgrows SQLite — but VIZ v1 ships SQLite-only by default. Promoting +to PostgreSQL requires a measured need (multi-user concurrent edits, +team sharing, hosted SaaS deployment), not a v1 default. + +### Streaming + +```text +SSE (Server-Sent Events) over HTTP — text/event-stream +Pyramid streaming response (`generator → Response.app_iter`) +arborist audit chain → SSE event stream + (audit-event hash, root commitments, claim status changes, + falsifier acceptance, cache invalidation — same subjects + as the original §9 NATS subject list, but flat HTTP) +``` + +**Why SSE over WebSockets+NATS JetStream:** + +- SSE is one-directional (server → browser), which is exactly what a + read-only dashboard needs. No client→server channel = smaller + attack surface, no auth complexity for the streaming side. +- Pyramid serves SSE natively via a streaming view (no extra + gateway, no `uWebSockets.js`, no Node runtime). +- Auto-reconnect is built into the browser `EventSource` API; no + client library needed. +- NATS JetStream is a queue-broker for many-producer many-consumer + fan-out. Arborist is a single producer; the audit chain already + carries sequence + content-addressed event hashes. A queue + broker adds an extra durability layer we don't need. +- The CLAUDE.md "stdout buffering" rule applies: SSE streams must + flush per-event, never line-buffer. Pyramid's `app_iter` pattern + flushes naturally; `gunicorn` workers (if ever used in place of + waitress) need `--worker-class gevent` or equivalent. + +### Frontend + +```text +vanilla JavaScript — NO React, NO Next.js, NO TypeScript build step + (Jinja2 server-rendered HTML is the default; + JS sprinkles only where genuinely needed) +six.js — fox's patched three.js fork for 3D Merkle + lattice, 3D proof-pulse, root-diff explosion, + temporal playback (proposal §6.11). See + "six.js sourcing" subsection below — this is + a CWE-407-patched fork (MOAD-0001 dogfood), + not vanilla three.js. +EventSource (browser — SSE client (no JS dep) + built-in) + + six.js — graph rendering for large claim/falsifier graphs + (replaces the Cosmograph / cosmos.gl + recommendation; if a 100K+ node graph ever + needs WebGL force-directed layout, evaluate a + vanilla-JS lib then — sigma.js / d3-force — + NOT a React-coupled one) +HTML
/ — virtualized tables happen at the server + (Pyramid view paginates), not via TanStack + Table. Default to Jinja2 pagination; only + add client-side virtualization when a real + 100K-row need surfaces. +SVG + vanilla layout — run-DAG / pipeline widgets (proposal §6.5); + replaces React Flow. Either server-render + SVG from Jinja2 + a dagre-python port, or + emit Graphviz `.dot` and render server-side + (matches the existing `docs/diagrams/*.dot` + pattern under `make docs`). +``` + +**No build step by default.** A Pyramid+Jinja2+vanilla-JS dashboard +needs no webpack / no Vite / no `npm install`. Six.js loads as a +single `