diff --git a/Makefile b/Makefile index 6379082..69dbf8f 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ SEARCH_Q ?= computer ingest-xml-attached ingest-abstract \ ingest-self ingest-self-providence ingest-git ingest-hg \ verify search stats test test-ci test-live docs docs-api docs-api-clean \ + docs-one-pager docs-two-pager docs-pagers docs-pagers-clean \ chain-check chain-check-shards \ falsify burn burn-kindergarten inspect bootstrap-crawler test-crawler crawl-ingest \ recrawl-check bench-qa bench-qa-smoke bench-qa-progressive-and \ @@ -1120,6 +1121,36 @@ docs-api: ## generate Sphinx API reference from docstrings (output: docs/_source docs-api-clean: ## remove Sphinx build artifacts rm -rf docs/_source/_build/ +# 1-pager / 2-pager standalone PDFs. Source RST lives at +# docs/_source/arborist-*-pager.rst (so the same files render on the +# Sphinx site AND as standalone PDFs). Style sheet at docs/pager.style. +# rst2pdf is installed lazily into $(VENV) on first build. +RST2PDF := $(VENV)/bin/rst2pdf +DOCS_BUILD := docs/_build +PAGER_STYLE := docs/pager.style +PAGER_FIGURES := docs/diagrams/pager-arch-stack.png docs/diagrams/pager-verifier-flow.png + +$(RST2PDF): $(VENV)/bin/activate + $(PIP) install rst2pdf + +$(DOCS_BUILD): + mkdir -p $(DOCS_BUILD) + +$(DOCS_BUILD)/arborist-one-pager.pdf: docs/_source/arborist-one-pager.rst $(PAGER_STYLE) | $(DOCS_BUILD) $(RST2PDF) + $(RST2PDF) docs/_source/arborist-one-pager.rst -s $(PAGER_STYLE) -o $@ + +$(DOCS_BUILD)/arborist-two-pager.pdf: docs/_source/arborist-two-pager.rst $(PAGER_STYLE) $(PAGER_FIGURES) | $(DOCS_BUILD) $(RST2PDF) + $(RST2PDF) docs/_source/arborist-two-pager.rst -s $(PAGER_STYLE) -o $@ + +docs-one-pager: $(DOCS_BUILD)/arborist-one-pager.pdf ## render docs/_build/arborist-one-pager.pdf + +docs-two-pager: $(DOCS_BUILD)/arborist-two-pager.pdf ## render docs/_build/arborist-two-pager.pdf + +docs-pagers: docs-one-pager docs-two-pager ## render both 1-pager and 2-pager PDFs + +docs-pagers-clean: ## remove generated pager PDFs + rm -f $(DOCS_BUILD)/arborist-one-pager.pdf $(DOCS_BUILD)/arborist-two-pager.pdf + # Reproducible micro-benchmark over a fixed slice of cur. Lets you compare # ETL throughput across configs and catches regressions on optimization # work. Override BENCH_DOCS=N (default 5000). diff --git a/docs/_source/arborist-one-pager.rst b/docs/_source/arborist-one-pager.rst new file mode 100644 index 0000000..d26cc34 --- /dev/null +++ b/docs/_source/arborist-one-pager.rst @@ -0,0 +1,64 @@ +arborist — a Q&A store with cryptographically verified answers +================================================================ + +.. class:: center + +*one-page summary · 2026-05-14 · AGPL-3.0-only · permacomputer.com* + +---- + +Most retrieval-augmented systems hand a language model some context, ask +for an answer, and ship whatever comes back. There is no +machine-checkable claim about what the answer actually quotes from its +source. **arborist inverts that.** Every answer is verified against its +source *after* generation by a deterministic, lexical check that produces +three labels — **STRICT** (every claim verifies), **HYBRID** (mixed), +**UNGROUNDED** (none). The verifier is binary. It does not negotiate. +Answers and labels are written to a content-addressed SQLite cache, every +state change appends a SHA-256 audit-chain row, and a single tampered +record breaks the chain. + +**Three Merkle-bound layers.** Surface (ingested documents — Wikipedia, +HTML, TeX, git, claim-packs), Core (distilled documents bound to their +surfaces by per-chunk inclusion proofs), Providence cache (verified Q&A +keyed on an 8-dimensional invariant that folds in the source-corpus root, +the question hash, the model identity, the governance policy, the schema +version, and three normalization-version dimensions). Bumping any one of +the eight dimensions invalidates prior records on lookup. Cores never +evict. + +**Synthetic-elision-by-construction-impossible.** In claim-lattice-pointer +mode, the model never types the quote string. The runtime hands it +tokens — ``[E1]``, ``[E2]`` — and interpolates the literal source span at +render time. The model proposes pointer-tagged clauses, the runtime +composes the answer. A class of hallucinations (the frankenquote, the +recombined-token claim) becomes impossible because the model cannot type +a string it never sees. + +**Soft signals never enter the proof path.** Deflection, coherence, +title-relevance, NLI vetoes, cross-encoder rerankers — all live as +advisory sidecars or as demote-only governance signals folded into the +cache key. They influence what the substrate caches; they do not +influence what it certifies. The hard channel (SHA-256 commitments, +audit chain, proof-of-inclusion) and the soft channel (embeddings, +similarity scores, lexical heuristics) evolve on independent timelines. + +**What it measures on real traffic.** + +- **100% mis-cite catch at 0% real-STRICT false positive** on the + 808-cell pooled bench-qa run (relevance-reranker veto, + BAAI/bge-reranker-large, n=20 mis-cite fixtures; ticket #000052 §3.2). +- **92 / 92 (100%) warrant-chain coverage** on the claim-pack corpus — + 18 public-domain or open-licensed textbook substrates plus curated + alias chains (ticket #000031). +- **0.54 STRICT-rate** on Hermes-3-Llama-3.1-8B quote mode (n=3 × 71 + questions, real Wikipedia haystack); lattice modes carry 99% + directive-coverage on the seven-point program. + +**What it runs on.** Python 3.12, SQLite (FTS5 + JSON), optional +cross-encoder rerankers for the soft channel. No GPU required for the +proof path. AGPL-3.0-only. Reference inference endpoint: +`hermes.ai.unturf.com `_. Source: +`git.unturf.com/engineering/unturf/arborist `_. +Full whitepaper: `unfirehose.com/merkle-providence-reverse-rag.html +`_. diff --git a/docs/_source/arborist-two-pager.rst b/docs/_source/arborist-two-pager.rst new file mode 100644 index 0000000..0107585 --- /dev/null +++ b/docs/_source/arborist-two-pager.rst @@ -0,0 +1,211 @@ +.. This is free software for the public good of a permacomputer hosted at +.. permacomputer.com, an always-on computer by the people, for the people. +.. One which is durable, easy to repair, & distributed like tap water +.. for machine learning intelligence. + +.. class:: center + +**Unturf · permacomputer.com · uncloseai.com · unfirehose.com** + +arborist — Merkle-Providence Reverse RAG, reference implementation +==================================================================== + +.. class:: center + +**License: AGPL-3.0-only** · schema v9.8 · 2026-05-14 +· *Russell Ballestrini * + +---- + +Abstract +-------- + +**arborist** is a content-addressed Q&A store with cryptographically +verified answers, the reference runtime for the *Merkle Providence +Reverse RAG* whitepaper (`unfirehose.com/merkle-providence-reverse-rag.html +`_, +Ballestrini · Wong · Morgan, April 2026). Every cached answer carries a Merkle +proof of source origin and a *binary, lexical, no-LLM-in-the-loop* +faithfulness label. The verifier never asks a model to grade itself. +Soft signals — embeddings, NLI vetoes, similarity rerankers — exist +as sidecars and demote-only governance signals; they never enter the +proof path. The reference inference endpoint is Hermes-3-Llama-3.1-8B +served freely at `hermes.ai.unturf.com `_; +arborist runs against any OpenAI-compatible alternative with a matching +``model_profile_hash``. + +1. The system in one paragraph +------------------------------- + +Three SQLite-backed layers, all Merkle-bound. **Surface** holds ingested +documents (Wikipedia dumps, HTML pages, TeX textbooks, git histories, +claim-packs), chunked, hash-rooted, and FTS5-indexed. **Core** holds +distillations bound to their surfaces by per-chunk inclusion proofs in +``derivations.proof_blob``. **Providence cache** holds verified Q&A +records keyed on an 8-dimensional invariant. Every state-changing +operation appends one row to ``audit_events`` carrying +``event_hash = sha256(prev || canonical(body))``; a single tampered row +breaks the chain. Re-ingest is idempotent (same content → same +``document_root`` → no-op); same URI with different content yields a new +doc plus a ``supersedes`` edge (lossless history). + +2. What makes arborist different +--------------------------------- + +Eleven design rules govern the substrate. The four that matter most for +a reviewer: + +**The verifier stays binary.** Each evidence unit verifies or does not. +No per-unit confidence scores, no fuzzy-match indicators, no soft +labels. Once a soft signal contaminates the chain, every prior +cryptographic claim becomes negotiable. + +**The runtime owns quote text.** In ``claim_lattice_pointer`` mode the +model emits prose with bracketed pointer tags (``[E1]``, ``[E2]``); the +runtime interpolates the literal source span at render time. The model +never types the quote string. *Synthetic-elision-by-construction-impossible* +— a model cannot produce a frankenquote it cannot type. + +**Soft hash never enters the proof path.** Embeddings, TF-IDF scores, +cross-encoder rerankers, NLI judgments shape ranking and retrieval and +inform demote-only governance. They never feed ``cache_key`` , +``document_root``, or ``audit_event_hash``. The hard channel and the +soft channel evolve on independent timelines. + +**Labels name properties, not vibes.** ``POINTER-LINKED``, +``ANCHOR-WARRANTED``, ``EVIDENCE-WARRANTED``, ``UNGROUNDED`` each name a +property the verifier could lexically confirm or could not. There are +no "high confidence" labels. A reviewer reads the label and immediately +knows what was checked. + +3. The 8-dimensional cache key +------------------------------- + +A cache hit returns a previously computed answer with a Merkle proof of +origin; a cache miss triggers fresh inference. The key folds together: + +================================ =================================================== +``source_root`` Merkle root of the corpus the answer is about +``question_hash`` dedup-mode-aware hash of the question (strict | equivalence_class) +``model_profile_hash`` model identity (name + sampling profile) +``conversation_hash`` prior-turn dependency, if any +``governance_policy_hash`` every policy field that alters generation +``schema_version`` ``v9.8.0`` +``canonicalization_version`` ``norm-v1`` +``chunking_version`` ``tok-512-v1`` +================================ =================================================== + +Bumping any one dimension invalidates prior records on lookup — +governance evolves, the corpus does not get rewritten when governance +evolves. Every preflight policy decision is *bound twice*: once into +``governance_policy_hash`` (cache identity) and once into ``run_dag_root`` +via the preflight stage (audit replay). + +4. The layered verifier +------------------------ + +Four strategies are tried in order; first to find evidence classifies. +**quote** (sequential pair-matching, never regex — prevents phantom +inter-pair captures), **span** (verbatim line match), **entity** +(proximity-clustered proper nouns), **paraphrase** (token-coverage, +prose-shaped only). Trichotomy on the schema column: STRICT = every unit +verifies, HYBRID = mixed, UNGROUNDED = none. The renderer maps that +column to a four-rung claim-lattice ladder +(POINTER-LINKED → ANCHOR-WARRANTED → EVIDENCE-WARRANTED; +ENTAILMENT-VERIFIED reserved); programmatic callers see the trichotomy, +human-facing surfaces see the ladder. The ENTAILMENT-VERIFIED rung is +deliberately empty — committing NLI to the proof path would re-introduce +the soft/hard boundary leak the architecture is built to prevent. +Honesty about the ceiling is a feature. + +---- + +5. What we measure on real traffic +----------------------------------- + +The bench is the scoreboard; the live fixtures are the gates. Numbers +that survive the 5pp signal floor on real-shard, real-LLM runs: + +- **Mis-cite catch (claim about X cited to source about Y).** The + ``BAAI/bge-reranker-large`` veto at threshold ``θ ≤ -2.42`` catches + 20/20 (100%) of mis-cite fixtures at strictly 0/808 (0%) real-STRICT + false positive on the pooled bench-qa, with a 1.4-point margin + between the worst mis-cite and the best STRICT (#000052 §3.2). +- **Deflection catch (off-topic answer to an on-topic question).** Same + reranker catches 11/20 (55%) of real-haystack deflection fixtures at + the same 0/808 FP threshold. The cost-pick alternate + (``ms-marco-MiniLM-L-6-v2``, 5× smaller) catches 13/20 (65%) at 0.4% FP. +- **Recombination catch (right tokens, wrong subject).** The NLI veto + (``facebook/bart-large-mnli``, top-k=12, θc=0.999) catches ~48% of + hand-crafted real-haystack recombination fixtures at 0/808 FP; the + ceiling is selector-bound, not entailment-bound (#000049 §7 #27). +- **Warrant-chain coverage.** 92/92 (100%) of the claim-pack corpus + resolves to a warrant chain (18 public-domain or open-licensed + textbook substrates + curated citation aliases + term aliases; + ticket #000031). +- **STRICT-rate, quote mode, real Wikipedia haystack.** 0.54 on + Hermes-3-Llama-3.1-8B (n=3 × 71 questions, sample-shuffled @ c=4; + see ``docs/qa-modes-bench.md``). + +Sample sizes are stated. Synthetic shortcuts are not. The five-point +sweep-wide/sweep-deep discipline ("real-traffic-sized samples or it +didn't happen") lives in ``docs/bench-maxing.md``. + +6. Closed substrate work +------------------------- + +Recent substrate work that is *landed and pinned*: ticket #000005 +(four-rung ladder display layer), #000008 (broad-quantifier preflight +guard with six-level disable hierarchy), #000010 (metacognition +preflight — false-premise, contradiction, stale-risk, out-of-corpus), +#000015 (π* canonical projections — arithmetic, logic-kernel, +algebra-symbolic), #000027 (canonical-projection persistence in +providence cache), #000031 (claim-pack warrant resolver, 100% +coverage), #000049 §3 (NLI veto, partial closure), #000052 §3.1 +(coherence sidecar — circular, vacuous, phrase-component-reuse) and +§3.2 (relevance reranker, viable runtime soft-veto). The synthesis +brief for the independent review thread is +``docs/relevance-and-veto-synthesis-for-dav1d.md``. + +---- + +License — Permacomputer Preamble +--------------------------------- + +:: + + AGPL-3.0-only · NO WARRANTY + + Free software for the public good of a permacomputer at + permacomputer.com — an always-on computer by the people, for + the people. Durable, easy to repair, distributed like tap water + for machine learning intelligence. + + Four values: TRUTH · FREEDOM · HARMONY · LOVE. + + Copyright (C) 2025-2026 TimeHexOn & foxhop & russell@unturf. + Full text: `LICENSE` at the repository root. + +---- + +Appendix A — Three-layer architecture +-------------------------------------- + +.. figure:: diagrams/pager-arch-stack.png + :width: 5.5in + :align: center + + Surface, Core, Providence cache — each layer hash-binds to the next; + every state change appends a row to the audit chain. + +Appendix B — Verifier flow +--------------------------- + +.. figure:: diagrams/pager-verifier-flow.png + :width: 6.5in + :align: center + + Question → retrieval → pointer-tagged context → LLM → binary + verifier → trichotomy label → audit chain & providence cache. The + LLM never types the quote string. + diff --git a/docs/_source/index.rst b/docs/_source/index.rst index 0269956..5d5ddac 100644 --- a/docs/_source/index.rst +++ b/docs/_source/index.rst @@ -5,6 +5,13 @@ Generated from docstrings. Replaces the static modules.md. Contents: +.. toctree:: + :maxdepth: 1 + :caption: Summary pages + + arborist-one-pager + arborist-two-pager + .. toctree:: :maxdepth: 2 :caption: Getting started diff --git a/docs/diagrams/pager-arch-stack.dot b/docs/diagrams/pager-arch-stack.dot new file mode 100644 index 0000000..b28b561 --- /dev/null +++ b/docs/diagrams/pager-arch-stack.dot @@ -0,0 +1,23 @@ +// Compact 3-layer architecture for the 2-pager appendix. +// Render: dot -Tsvg pager-arch-stack.dot -o pager-arch-stack.svg + +digraph arch_stack { + rankdir=BT + ranksep=0.25 + nodesep=0.20 + node [shape=box, style="rounded,filled", fontname="Helvetica", fontsize=11] + edge [fontname="Helvetica", fontsize=9] + bgcolor="white" + + surface [label="SURFACE — ingested documents\nWikipedia · HTML · TeX · git · claim-packs\nchunked → Merkle-rooted → FTS5-indexed", fillcolor="#e9f3ff"] + core [label="CORE — distilled documents\nMerkle-bound to surfaces by per-chunk\ninclusion proofs (derivations.proof_blob)", fillcolor="#dff5e7"] + cache [label="PROVIDENCE CACHE — verified Q&A\n8-dim cache_key · audit_mode label ∈\n{STRICT, HYBRID, UNGROUNDED}", fillcolor="#fff3d6"] + + audit [label="AUDIT CHAIN — every state change\nappends sha256(prev || canonical(body))\nto audit_events (single SQLite file)", fillcolor="#fde7e9", shape=note] + + surface -> core [label="distill", fontsize=9] + core -> cache [label="verify → cache", fontsize=9] + cache -> audit [style=dashed, color="#888888", label="every write", fontsize=9] + core -> audit [style=dashed, color="#888888"] + surface -> audit [style=dashed, color="#888888"] +} diff --git a/docs/diagrams/pager-arch-stack.png b/docs/diagrams/pager-arch-stack.png new file mode 100644 index 0000000..1281414 Binary files /dev/null and b/docs/diagrams/pager-arch-stack.png differ diff --git a/docs/diagrams/pager-arch-stack.svg b/docs/diagrams/pager-arch-stack.svg new file mode 100644 index 0000000..4b39742 --- /dev/null +++ b/docs/diagrams/pager-arch-stack.svg @@ -0,0 +1,80 @@ + + + + + + +arch_stack + + + +surface + +SURFACE — ingested documents +Wikipedia · HTML · TeX · git · claim-packs +chunked → Merkle-rooted → FTS5-indexed + + + +core + +CORE — distilled documents +Merkle-bound to surfaces by per-chunk +inclusion proofs (derivations.proof_blob) + + + +surface->core + + +distill + + + +audit + + + +AUDIT CHAIN — every state change +appends sha256(prev || canonical(body)) +to audit_events (single SQLite file) + + + +surface->audit + + + + + +cache + +PROVIDENCE CACHE — verified Q&A +8-dim cache_key · audit_mode label ∈ +{STRICT, HYBRID, UNGROUNDED} + + + +core->cache + + +verify → cache + + + +core->audit + + + + + +cache->audit + + +every write + + + diff --git a/docs/diagrams/pager-verifier-flow.dot b/docs/diagrams/pager-verifier-flow.dot new file mode 100644 index 0000000..63d8699 --- /dev/null +++ b/docs/diagrams/pager-verifier-flow.dot @@ -0,0 +1,36 @@ +// Compact verifier flow for the 2-pager appendix. +// (Question, source-chunks) → model → (text, pointers) → verifier → label. +// Render: dot -Tsvg pager-verifier-flow.dot -o pager-verifier-flow.svg + +digraph verifier_flow { + rankdir=LR + ranksep=0.35 + nodesep=0.25 + node [shape=box, style="rounded,filled", fontname="Helvetica", fontsize=12] + edge [fontname="Helvetica", fontsize=11] + bgcolor="white" + + question [label="QUESTION", fillcolor="#fff7e6", shape=note] + retrieval [label="retrieval\n(FTS5 + phrase route\n+ rivalry exclusion)", fillcolor="#e9f3ff"] + pointers [label="POINTER MAP\nE1 → span\nE2 → span\n…", fillcolor="#dff5e7", shape=note] + model [label="LLM\nsees: question +\npointer-tagged chunks\nemits: prose with [E1] [E2] tags\n(never types the quote string)", fillcolor="#f0e6ff"] + verifier [label="verifier\n(quote · span · entity\n· paraphrase — binary,\nno LLM call)", fillcolor="#fff3d6"] + + label_strict [label="STRICT\nevery claim verifies", fillcolor="#28a745", fontcolor="white"] + label_hybrid [label="HYBRID\nsome verify, some don't", fillcolor="#ffc107"] + label_ungrounded [label="UNGROUNDED\nnone verify", fillcolor="#dc3545", fontcolor="white"] + + audit [label="audit chain\n+ providence cache\n(8-dim key, governance-aware)", fillcolor="#fde7e9", shape=note] + + question -> retrieval -> pointers + pointers -> model + model -> verifier [label="answer\n+ pointer IDs"] + verifier -> label_strict + verifier -> label_hybrid + verifier -> label_ungrounded + label_strict -> audit [style=dashed, color="#888888"] + label_hybrid -> audit [style=dashed, color="#888888"] + label_ungrounded -> audit [style=dashed, color="#888888"] + + {rank=same; label_strict; label_hybrid; label_ungrounded} +} diff --git a/docs/diagrams/pager-verifier-flow.png b/docs/diagrams/pager-verifier-flow.png new file mode 100644 index 0000000..c3f435d Binary files /dev/null and b/docs/diagrams/pager-verifier-flow.png differ diff --git a/docs/diagrams/pager-verifier-flow.svg b/docs/diagrams/pager-verifier-flow.svg new file mode 100644 index 0000000..e5fa750 --- /dev/null +++ b/docs/diagrams/pager-verifier-flow.svg @@ -0,0 +1,152 @@ + + + + + + +verifier_flow + + + +question + + + +QUESTION + + + +retrieval + +retrieval +(FTS5 + phrase route ++ rivalry exclusion) + + + +question->retrieval + + + + + +pointers + + + +POINTER MAP +E1 → span +E2 → span +… + + + +retrieval->pointers + + + + + +model + +LLM +sees: question + +pointer-tagged chunks +emits: prose with [E1] [E2] tags +(never types the quote string) + + + +pointers->model + + + + + +verifier + +verifier +(quote · span · entity +· paraphrase — binary, +no LLM call) + + + +model->verifier + + +answer ++ pointer IDs + + + +label_strict + +STRICT +every claim verifies + + + +verifier->label_strict + + + + + +label_hybrid + +HYBRID +some verify, some don't + + + +verifier->label_hybrid + + + + + +label_ungrounded + +UNGROUNDED +none verify + + + +verifier->label_ungrounded + + + + + +audit + + + +audit chain ++ providence cache +(8-dim key, governance-aware) + + + +label_strict->audit + + + + + +label_hybrid->audit + + + + + +label_ungrounded->audit + + + + + diff --git a/docs/pager.style b/docs/pager.style new file mode 100644 index 0000000..2ac7ddc --- /dev/null +++ b/docs/pager.style @@ -0,0 +1,60 @@ +# rst2pdf style sheet for arborist 1-pager and 2-pager. +# Mirrors ~/git/unfirehose-nextjs-logger/whitepaper/whitepaper.style with +# tighter margins, smaller body text, and compact spacing so a dense +# technical summary lands inside a single Letter page (1-pager) or two +# Letter pages + appendix figures (2-pager). + +pageSetup: + size: LETTER + margin-top: 0.55in + margin-bottom: 0.55in + margin-left: 0.65in + margin-right: 0.65in + spacing-header: 4 + spacing-footer: 4 + +styles: + base: + fontSize: 9.5 + leading: 12 + spaceAfter: 4 + bodytext: + parent: base + spaceAfter: 5 + title: + spaceBefore: 0 + spaceAfter: 14 + alignment: TA_CENTER + fontSize: 16 + leading: 19 + heading1: + parent: heading + fontSize: 13 + spaceBefore: 12 + spaceAfter: 6 + leading: 16 + heading2: + parent: heading + fontSize: 11 + spaceBefore: 9 + spaceAfter: 5 + leading: 14 + heading3: + parent: heading + fontSize: 10 + spaceBefore: 8 + spaceAfter: 4 + leading: 13 + code: + fontSize: 8 + leading: 10 + backColor: "#f4f4f4" + field-list: + colWidths: [20%,80%] + bullet-list: + spaceAfter: 6 + figure: + spaceAfter: 6 + spaceBefore: 6 + center: + alignment: TA_CENTER