diff --git a/docs/_source/concepts.rst b/docs/_source/concepts.rst
index 36462c3..dbb66b4 100644
--- a/docs/_source/concepts.rst
+++ b/docs/_source/concepts.rst
@@ -26,6 +26,19 @@ Three layers stacked on one SQLite file (per shard):
- **Providence cache** — Q&A records keyed on the v9.8 8-dimension
invariant; every record carries an audit_mode and a Merkle proof.
+.. figure:: diagrams/three-layer-stack.svg
+ :alt: Three-layer stack (surface, core, providence cache)
+ :width: 100%
+
+ How the three layers compose. Surface document_roots flow into both
+ core derivations (with per-chunk inclusion proofs in
+ ``derivations.proof_blob``) and providence-cache records (as the
+ ``source_root`` dimension of the 8-dim ``cache_key``). Cores reuse
+ the surface proof structure, so the Merkle binding survives
+ compression. Providence rows then carry their own ``merkle_proof``
+ that ties an answer back to the chunks the verifier ran against —
+ verifiable offline by any peer that holds the same surface roots.
+
.. figure:: diagrams/arborist-modules.svg
:alt: Arborist module graph
:width: 100%
@@ -84,6 +97,20 @@ lookup. This is how the system stays honest across model changes,
schema migrations, prompt edits, etc. — old answers don't silently
serve under new conditions.
+.. figure:: diagrams/cache-key-8dim.svg
+ :alt: 8-dim cache_key composition
+ :width: 100%
+
+ The eight mandatory dimensions split across two groups: the top row
+ (corpus, question, model, conversation) is the dynamic input
+ surface; the bottom row (policy + three pinned versions) is the
+ admissibility scaffold. The optional 9th dimension
+ (``verifier_policy_hash``) exists for audit *legibility* under
+ ticket #000058 — it does not add correctness coverage, because the
+ verifier-policy fields are already a subset of
+ ``governance_policy_hash``. 8-dim is the default write form; 9-dim
+ is opt-in.
+
The audit chain
---------------
@@ -172,6 +199,18 @@ Two ways to remove a wrong answer:
record has children unless ``FORCE=1``. Use during scratch corpus
building.
+.. figure:: diagrams/falsification-states.svg
+ :alt: Falsification state machine
+ :width: 100%
+
+ State machine for ``falsification_state``. Every transition writes
+ one ``audit_event`` so the chain stays intact — the schema column
+ is the lookup gate, the audit chain is the history. ``stale →
+ live`` is the rarely-trodden path where a corpus was rolled back to
+ a prior state; cleanest in single-shard test setups. ``burn``
+ actually deletes the row and is refused if the record has
+ children unless ``FORCE=1``.
+
Sidecar diagnostics
-------------------
diff --git a/docs/_source/cookbook.rst b/docs/_source/cookbook.rst
index 72b4077..50be60b 100644
--- a/docs/_source/cookbook.rst
+++ b/docs/_source/cookbook.rst
@@ -141,3 +141,236 @@ keywords without changing what the LLM sees as the question:
Provenance gap on this is tracked in
:doc:`api/qa` (``arborist.qa.query``).
+
+Use arborist as a Python library
+================================
+
+Everything below uses the supported embedding surface,
+:mod:`arborist.embed`. Import from there, not from internal modules —
+internal refactors are free to move things around behind that seam.
+See :doc:`api/storage` for the full ``arborist.store`` reference and
+:doc:`api/substrate` for the Merkle primitives the recipes call into.
+
+Open a store and ingest documents you already hold
+---------------------------------------------------
+
+The minimum useful contact surface: pass in your own
+:class:`~arborist.document.Document` objects, get back content-addressed
+storage with an audit chain. Idempotent — re-running with the same
+``content`` yields the same ``document_root`` and skips the insert.
+
+.. code-block:: python
+
+ from pathlib import Path
+ from arborist.embed import (
+ open_store, ingest_documents, search, Document, Edge,
+ )
+
+ conn = open_store(Path("data/arborist.db")) # creates + migrates
+
+ stats = ingest_documents(conn, [
+ Document(
+ uri="https://example.com/post-a",
+ content="anarcho-capitalism describes a stateless society "
+ "where private property and free markets coordinate "
+ "without coercion.",
+ source_type="my_app",
+ title="Anarcho-Capitalism Primer",
+ edges=[Edge(edge_type="references",
+ dst_uri="https://example.com/post-b")],
+ extra={"md5": "deadbeef"}, # your provenance, carried along
+ ),
+ ])
+ print(stats) # IngestStats(seen=1, inserted=1, ...)
+
+ for hit in search(conn, "free markets", limit=5):
+ print(hit.document_uri, round(hit.score, 3))
+
+ conn.close()
+
+Define a custom :class:`~arborist.source.Source` for stateful corpora
+---------------------------------------------------------------------
+
+When you have a corpus (a directory tree, a paginated API, a database
+table) it's cleaner to express it as a :class:`Source`. The ABC has one
+required method, :meth:`iter_documents`, which must be deterministic
+and idempotent. That's exactly the contract every built-in source under
+``arborist/sources/`` already implements.
+
+.. code-block:: python
+
+ from arborist.embed import Source, Document
+ from arborist.ingest import ingest_source
+ from arborist.embed import open_store
+
+ class TaggedDocs(Source):
+ """Ingest a list of (uri, body) pairs under a shared tag."""
+
+ source_type = "tagged_example"
+
+ def __init__(self, tag, items):
+ self.tag = tag
+ self._items = items
+
+ def iter_documents(self):
+ for uri, body in self._items:
+ yield Document(
+ uri=uri,
+ content=body,
+ source_type=self.source_type,
+ extra={"tag": self.tag},
+ )
+
+ conn = open_store("data/arborist.db")
+ stats = ingest_source(conn, TaggedDocs("research", [
+ ("https://example.com/c", "third doc body about content-addressing."),
+ ]))
+ print(stats)
+ conn.close()
+
+Walk and verify the audit chain
+-------------------------------
+
+Every state-changing op writes one row in ``audit_events`` with
+``event_hash = sha256(prev_event_hash || canonical(body))``. Verifying
+the chain is just re-running that hash for every row and checking the
+linkage. (``make chain-check-shards`` does this at scale; the recipe
+below is the same logic, inlined.)
+
+.. code-block:: python
+
+ import hashlib
+ from arborist.embed import open_store
+ from arborist.store import latest_event_hash
+
+ conn = open_store("data/arborist.db")
+ print("head:", latest_event_hash(conn))
+
+ prev = None
+ bad = 0
+ for seq, eh, ph, body in conn.execute(
+ "SELECT seq, event_hash, prev_event_hash, body "
+ "FROM audit_events ORDER BY seq"
+ ):
+ h = hashlib.sha256()
+ if ph is not None:
+ h.update(bytes.fromhex(ph))
+ h.update(body.encode("utf-8", errors="surrogatepass"))
+ if h.hexdigest() != eh or (prev is not None and ph != prev):
+ bad += 1
+ prev = eh
+
+ print(f"chain breaks: {bad}") # 0 = intact
+ conn.close()
+
+Round-trip a Merkle inclusion proof
+-----------------------------------
+
+The proof primitives from :mod:`arborist.merkle` are the Python port of
+``proxy.unturf.com``'s Go conventions. Use them to re-derive a
+``document_root`` from its leaves, build a proof for any chunk, and
+serialize the proof for over-the-wire delivery to another peer.
+
+.. code-block:: python
+
+ import json
+ from arborist.embed import open_store
+ from arborist.merkle import (
+ MerkleTree, verify_proof, proof_to_dict, proof_from_dict,
+ )
+
+ conn = open_store("data/arborist.db")
+
+ doc_root_hex, = conn.execute(
+ "SELECT document_root FROM documents LIMIT 1"
+ ).fetchone()
+ leaves = [
+ bytes.fromhex(r[0]) for r in conn.execute(
+ "SELECT leaf_hash FROM chunks WHERE document_root=? ORDER BY idx",
+ (doc_root_hex,),
+ )
+ ]
+
+ tree = MerkleTree.build(leaves)
+ assert tree.root.hex() == doc_root_hex # bit-identical re-derivation
+
+ proof = tree.proof(0)
+ assert verify_proof(proof) # local round-trip
+
+ blob = json.dumps(proof_to_dict(proof)) # serialize for wire
+ proof_received = proof_from_dict(json.loads(blob))
+ assert verify_proof(proof_received) # any peer can verify
+
+ conn.close()
+
+Two peers that ingested the same source with the same chunker and
+canonicalization will compute byte-identical ``document_root`` hashes
+and accept each other's proofs — see :doc:`api/mesh` for the
+federation primitives built on top of this property.
+
+Run a Q&A from Python and read the result programmatically
+-----------------------------------------------------------
+
+The CLI (``arborist query`` / ``make query``) is one entry point to
+the multi-route retrieval + LLM + verifier pipeline. The same surface
+is callable directly from Python via :func:`arborist.qa.query.query` —
+useful when you want to drive a batch sweep, integrate into a notebook,
+or wrap the result in your own application logic. Cache lookups, the
+8-dim cache_key, the verifier, the run-DAG, and the falsification gate
+all behave identically to the CLI path.
+
+.. code-block:: python
+
+ from pathlib import Path
+ from arborist.qa.query import query
+ from arborist.qa.client import OpenAICompatibleClient
+
+ client = OpenAICompatibleClient(
+ base_url="https://hermes.ai.unturf.com/v1", # any OpenAI-compat
+ )
+
+ result = query(
+ question="What is anarcho-capitalism?",
+ qa_db=Path.home() / ".arborist" / "qa.db", # cache lives here
+ chat_client=client,
+ model_id="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic",
+ shards_dir=Path.home() / ".arborist" / "shards", # OR single_db=...
+ )
+
+ print(result["status"]) # cache_hit | cache_miss_then_written
+ print(result["audit_mode"]) # STRICT | HYBRID | UNGROUNDED
+ print(result["cache_key"]) # 64-char hex
+ print(result["answer_text"])
+ for src in result["sources"]:
+ print(" ->", src["document_uri"], src["document_root"][:12],
+ "role:", src["source_role"])
+
+The first call writes one providence-cache row + one audit event;
+the second call with the same question replays from cache in ~100 ms
+(``status == "cache_hit"``). To exercise the pipeline deterministically
+in tests, swap the live client for :class:`~arborist.qa.client.StubClient`:
+
+.. code-block:: python
+
+ from arborist.qa.client import StubClient
+
+ stub = StubClient(answer='Anarcho-capitalism is "a political '
+ 'philosophy that advocates the elimination '
+ 'of centralized state dictums".')
+ result = query(
+ question="What is anarcho-capitalism?",
+ qa_db=Path("/tmp/qa.db"),
+ chat_client=stub,
+ model_id="stub/test",
+ single_db=Path("/tmp/arborist.db"),
+ )
+ assert result["audit_mode"] in {"STRICT", "HYBRID", "UNGROUNDED"}
+
+That stubbed shape is exactly how the test suite drives the runner —
+the verifier still runs lexically against the assembled context, so
+the audit_mode classification is real even though the model output
+is canned.
+
+For the single-document path (``arborist ask`` / one ``document_root``
+in hand), use :func:`arborist.qa.runner.ask` instead — same return
+shape, same cache_key invariants, scoped to one document.
diff --git a/docs/diagrams/cache-key-8dim.dot b/docs/diagrams/cache-key-8dim.dot
new file mode 100644
index 0000000..401f8ad
--- /dev/null
+++ b/docs/diagrams/cache-key-8dim.dot
@@ -0,0 +1,71 @@
+// 8-dim cache_key composition for v9.8 admissibility.
+//
+// Every providence_cache record is keyed on a SHA-256 over eight
+// concatenated bytes — the 8-dim invariant. Bumping any one of the
+// dimensions invalidates prior records on lookup (this is the honesty
+// property: old answers do not silently serve under new conditions).
+//
+// An optional 9th dimension — verifier_policy_hash — exists for *audit
+// legibility* (#000058). It does NOT add correctness coverage; the
+// verifier-policy fields are already a subset of governance_policy_hash.
+// 8-dim is the default write form; 9-dim is opt-in for one-hash-diff
+// answerability of "did the *verifier* rules change?"
+//
+// Render: dot -Tsvg cache-key-8dim.dot -o cache-key-8dim.svg
+
+digraph cache_key_8dim {
+ rankdir=TB
+ node [shape=box, style="rounded,filled", fontname="Helvetica", fontsize=10]
+ edge [fontname="Helvetica", fontsize=9]
+ bgcolor="white"
+ ranksep=0.6
+ nodesep=0.3
+
+ subgraph cluster_dims {
+ label="EIGHT MANDATORY DIMENSIONS (concat in fixed order)"
+ labeljust="l"
+ fontname="Helvetica-Bold"
+ style="rounded,filled"
+ fillcolor="#f5faff"
+ margin=14
+
+ // Top row: corpus / question / model state
+ d1 [label="1. source_root\nMerkle root over\ningested source(s)", fillcolor="#e6f0ff"]
+ d2 [label="2. question_hash\ndedup-mode aware\n(strict | equiv. class)", fillcolor="#e6f0ff"]
+ d3 [label="3. model_profile_hash\nmodel id + sampling +\nsystem prompt + tools", fillcolor="#e6f0ff"]
+ d4 [label="4. conversation_hash\nprior turns committed\ninto the cache_key", fillcolor="#e6f0ff"]
+
+ // Bottom row: policy / pinned versions
+ d5 [label="5. governance_policy_hash\nverifier + answer_mode +\nquantifier guard + …\n(superset of #9)", fillcolor="#e6f0ff"]
+ d6 [label="6. schema_version\nv9.8.0\nschema-level compat", fillcolor="#e6f0ff"]
+ d7 [label="7. canonicalization_version\nnorm-v1\npinned preprocessing", fillcolor="#e6f0ff"]
+ d8 [label="8. chunking_version\ntok-512-v1\npinned leaf boundaries", fillcolor="#e6f0ff"]
+
+ {rank=same; d1; d2; d3; d4}
+ {rank=same; d5; d6; d7; d8}
+
+ // Invisible spine to enforce the 2-row layout
+ d1 -> d5 [style=invis]
+ d2 -> d6 [style=invis]
+ d3 -> d7 [style=invis]
+ d4 -> d8 [style=invis]
+ }
+
+ // Optional 9th — render as sibling-of-cluster
+ d9 [label="9. verifier_policy_hash\n(opt-in, #000058)\nsubset of #5; exists for\none-hash-diff legibility,\nNOT new coverage", fillcolor="#fff0e0", style="rounded,filled,dashed"]
+
+ sha [label="sha256(d1 ‖ d2 ‖ d3 ‖ d4 ‖ d5 ‖ d6 ‖ d7 ‖ d8 [ ‖ d9 ])", fillcolor="#999999", fontcolor="white", shape=cds, fontsize=11]
+
+ key [label="cache_key (32 bytes, hex)\nkeys one providence_cache row", fillcolor="#28a745", fontcolor="white", shape=note]
+
+ invariant [label="HONESTY PROPERTY\nbumping ANY dimension invalidates prior\nrecords on lookup — no silent reuse\nunder new conditions", fillcolor="#fff0e0", shape=note]
+
+ // Cluster -> sha (one collapsed edge by anchoring on d8)
+ d5 -> sha
+ d6 -> sha
+ d7 -> sha
+ d8 -> sha
+ d9 -> sha [style=dashed, label="opt-in"]
+
+ sha -> key -> invariant [style=dotted, color="#666666"]
+}
diff --git a/docs/diagrams/cache-key-8dim.png b/docs/diagrams/cache-key-8dim.png
new file mode 100644
index 0000000..81bd11b
Binary files /dev/null and b/docs/diagrams/cache-key-8dim.png differ
diff --git a/docs/diagrams/cache-key-8dim.svg b/docs/diagrams/cache-key-8dim.svg
new file mode 100644
index 0000000..c459e2c
--- /dev/null
+++ b/docs/diagrams/cache-key-8dim.svg
@@ -0,0 +1,166 @@
+
+
+
+
+
diff --git a/docs/diagrams/falsification-states.dot b/docs/diagrams/falsification-states.dot
new file mode 100644
index 0000000..a54a339
--- /dev/null
+++ b/docs/diagrams/falsification-states.dot
@@ -0,0 +1,50 @@
+// falsification_state state machine — the lifecycle of a providence
+// record after it lands in cache.
+//
+// Cache lookups filter on state='live'. Drift (re-ingest yielding a
+// different document_root than the one bound into the record) flips
+// the record to 'stale'. Explicit human/operator override flips to
+// 'failed' (audit-preserving) or removes the row entirely via burn
+// (which is refused if the record has children unless FORCE=1).
+//
+// Render: dot -Tsvg falsification-states.dot -o falsification-states.svg
+
+digraph falsification_states {
+ rankdir=LR
+ node [shape=ellipse, style="filled", fontname="Helvetica", fontsize=10]
+ edge [fontname="Helvetica", fontsize=9]
+ bgcolor="white"
+
+ start [label="cache write\n(verifier sets audit_mode)", shape=note, fillcolor="#fff7e6"]
+
+ live [label="LIVE\ncache lookups\nreturn this row", fillcolor="#28a745", fontcolor="white"]
+ stale [label="STALE\nsource drifted\n(document_root changed\non re-ingest)", fillcolor="#ffc107"]
+ failed [label="FAILED\nhuman / operator\nfalsified", fillcolor="#dc3545", fontcolor="white"]
+ quarantined [label="QUARANTINED\nverifier downgrade\npending review", fillcolor="#6c757d", fontcolor="white"]
+
+ gone [label="(burn — row deleted)\nrefused if record has children\nunless FORCE=1", shape=octagon, fillcolor="#888888", fontcolor="white"]
+
+ audit [label="every transition\nwrites one audit_event\nchain remains intact", shape=note, fillcolor="#fff0e0"]
+
+ start -> live [label="initial state\nif verifier produced\nany audit_mode"]
+
+ live -> stale [label="re-ingest produces\ndifferent document_root\n(drift detection)"]
+ live -> failed [label="make falsify KEY=…\nREASON='…'\n(audit-preserving)"]
+ live -> quarantined [label="verifier rule change\n+ promotion gate\n(rare)"]
+
+ stale -> live [label="re-ingest matches\nbound root again\n(corpus returned to\nprior state)", style=dotted]
+ quarantined -> live [label="review passed", style=dotted]
+ quarantined -> failed [label="review rejected", style=dotted]
+
+ failed -> gone [label="make burn KEY=…\nREASON='…'\n(kindergarten only)", color="#aa0000"]
+ stale -> gone [label="make burn KEY=…", color="#aa0000", style=dashed]
+
+ // Audit-chain annotation
+ live -> audit [style=dashed, color="#888888"]
+ stale -> audit [style=dashed, color="#888888"]
+ failed -> audit [style=dashed, color="#888888"]
+ quarantined -> audit [style=dashed, color="#888888"]
+
+ {rank=same; live; stale}
+ {rank=same; failed; quarantined}
+}
diff --git a/docs/diagrams/falsification-states.png b/docs/diagrams/falsification-states.png
new file mode 100644
index 0000000..762fb62
Binary files /dev/null and b/docs/diagrams/falsification-states.png differ
diff --git a/docs/diagrams/falsification-states.svg b/docs/diagrams/falsification-states.svg
new file mode 100644
index 0000000..89f9e51
--- /dev/null
+++ b/docs/diagrams/falsification-states.svg
@@ -0,0 +1,173 @@
+
+
+
+
+
diff --git a/docs/diagrams/three-layer-stack.dot b/docs/diagrams/three-layer-stack.dot
new file mode 100644
index 0000000..47565fa
--- /dev/null
+++ b/docs/diagrams/three-layer-stack.dot
@@ -0,0 +1,87 @@
+// Three-layer stack: how surface, core, and providence-cache compose
+// over one SQLite file per shard.
+//
+// Surface = ingested docs (Merkle-rooted, FTS5-indexed).
+// Core = distillates, bound back to surfaces via per-chunk inclusion
+// proofs in derivations.proof_blob. Recursive (depth-N → N+1).
+// Providence cache = Q&A records keyed on the v9.8 8-dim invariant,
+// each carrying audit_mode (STRICT / HYBRID / UNGROUNDED) +
+// a Merkle proof binding the answer to its source chunks.
+//
+// Render: dot -Tsvg three-layer-stack.dot -o three-layer-stack.svg
+
+digraph three_layer_stack {
+ rankdir=BT
+ node [shape=box, style="rounded,filled", fontname="Helvetica", fontsize=10]
+ edge [fontname="Helvetica", fontsize=9]
+ bgcolor="white"
+
+ subgraph cluster_surface {
+ label="SURFACE — ingested documents"
+ labeljust="l"
+ fontname="Helvetica-Bold"
+ style="rounded,filled"
+ fillcolor="#fff7e6"
+ margin=12
+
+ s_wiki [label="Wikipedia\nXML / SQL dump", fillcolor="#fff0d0"]
+ s_html [label="HTML page\n(crawler)", fillcolor="#fff0d0"]
+ s_grok [label="Grok export\nconversation", fillcolor="#fff0d0"]
+ s_vcs [label="git / hg\nrepo HEAD walk", fillcolor="#fff0d0"]
+ s_tex [label="textbook_tex\nProject Gutenberg LaTeX", fillcolor="#fff0d0"]
+ s_root [label="document_root (32 bytes)\nMerkle root over canonical chunks\nidentity = content, not URI", fillcolor="#ffe2a8", shape=note]
+ }
+
+ subgraph cluster_core {
+ label="CORE — distillates"
+ labeljust="l"
+ fontname="Helvetica-Bold"
+ style="rounded,filled"
+ fillcolor="#e7ffe7"
+ margin=12
+
+ c_first [label="first_sentence\n(lead extraction)", fillcolor="#d0f0d0"]
+ c_tfidf [label="tfidf\n(keyword core)", fillcolor="#d0f0d0"]
+ c_recur [label="depth-N → depth-N+1\nrecursive distillation", fillcolor="#d0f0d0"]
+ c_proof [label="derivations.proof_blob\nper contributing chunk:\n • leaf_hash\n • sibling path\n • verify_proof() ≡ True\nbinding survives compression", fillcolor="#b8e6b8", shape=note]
+ }
+
+ subgraph cluster_prov {
+ label="PROVIDENCE CACHE — Q&A records"
+ labeljust="l"
+ fontname="Helvetica-Bold"
+ style="rounded,filled"
+ fillcolor="#e6f0ff"
+ margin=12
+
+ p_key [label="cache_key (8-dim)\nsource_root | question_hash |\nmodel_profile | conversation |\ngovernance_policy | schema_v |\ncanonicalization_v | chunking_v", fillcolor="#cfdfff", shape=note]
+ p_audit [label="audit_mode ∈\n{STRICT, HYBRID, UNGROUNDED}\nset by verifier, never asserted", fillcolor="#cfdfff"]
+ p_state [label="falsification_state ∈\n{live, failed, stale, quarantined}\ncache reads filter on state='live'", fillcolor="#cfdfff"]
+ p_proof [label="merkle_proof\nbinds answer ↔ source chunks\nverifies offline, across peers", fillcolor="#a8c8ff", shape=note]
+ }
+
+ // Surface composes into one document_root
+ s_wiki -> s_root
+ s_html -> s_root
+ s_grok -> s_root
+ s_vcs -> s_root
+ s_tex -> s_root
+
+ // Core binds back to surface via per-chunk proofs
+ s_root -> c_proof [label="ingested chunks\nbind cores back", color="#2d6a2d"]
+ c_first -> c_proof
+ c_tfidf -> c_proof
+ c_recur -> c_proof
+ c_recur -> c_recur [label="self-recursion", color="#2d6a2d"]
+
+ // Providence carries source_root from surface + Merkle proof
+ s_root -> p_key [label="source_root", color="#1a3a8a"]
+ c_proof -> p_proof [label="proof structure\nreused", color="#1a3a8a", style="dashed"]
+
+ p_key -> p_audit [style=invis]
+ p_audit -> p_state [style=invis]
+ p_state -> p_proof [style=invis]
+
+ // Layer separation hint at right edge
+ {rank=same; s_root; c_proof; p_proof}
+}
diff --git a/docs/diagrams/three-layer-stack.png b/docs/diagrams/three-layer-stack.png
new file mode 100644
index 0000000..ea0e4b5
Binary files /dev/null and b/docs/diagrams/three-layer-stack.png differ
diff --git a/docs/diagrams/three-layer-stack.svg b/docs/diagrams/three-layer-stack.svg
new file mode 100644
index 0000000..0b4874f
--- /dev/null
+++ b/docs/diagrams/three-layer-stack.svg
@@ -0,0 +1,226 @@
+
+
+
+
+