engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

modified: .gitignore

Browse source 
modified:   Makefile
	modified:   arborist/cli.py
	new file:   arborist/qa/progress.py
	modified:   arborist/qa/query.py
	new file:   arborist/qa/witness.py
	modified:   bench/results/real-shard-baseline.json
	modified:   bench/results/real-shard-baseline.md
	modified:   docs/TICKETS.md
	new file:   docs/tickets/ticket-000028-multi-modality-witness.md
	new file:   greatest-live-rock-and-roll-song-ever-played.md
	new file:   tests/test_witness.py
This commit is contained in:
russell@unturf.com 2026-05-08 16:38:09 -04:00
parent 7e86774e44
commit 656b573198
No known key found for this signature in database
12 changed files with 1420 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -5,6 +5,7 @@ __pycache__/
.pytest_cache/
.coverage
*.swp
_build/
# data + caches stay out of git
data/

6
Makefile
View file

@ -161,11 +161,11 @@ ANSWER_MODE ?= claim_lattice
# REJECT_BROAD=1 → strict reject for ALL/COMPREHENSIVE/OPEN_REQUEST
# unbounded shapes; returns UNGROUNDED before the LLM call.
# ALLOW_BROAD=1 → emergent search; classifier on, caps off.
query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K="extra retrieval keywords" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]; JSON by default
query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K="extra retrieval keywords" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1]; JSON by default
@if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \
echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]"; exit 2; \
echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1]"; exit 2; \
fi
$(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) "$(Q)"
$(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) $(if $(WITNESS),--witness,) "$(Q)"
query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=... BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]
@if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \

73
arborist/cli.py
View file

@ -438,6 +438,12 @@ def _cmd_query(args: argparse.Namespace) -> int:
# sidecar. Adds one short LLM round-trip; NEVER gates
# admissibility (D1 preserved).
call_policy["soft_preflight_enabled"] = True
# Ticket #000028 — multi-modality witness override. CLI > policy.
_witness_override = getattr(args, "witness_override", None)
if _witness_override == "on":
call_policy["canonical_witness_enabled"] = True
elif _witness_override == "off":
call_policy["canonical_witness_enabled"] = False
if getattr(args, "no_canonical_preflight", False):
# Disable the math/logic π* short-circuit per-call. Forces
# RAG even on pure-arithmetic / pure-propositional input —
@ -834,10 +840,52 @@ def _render_query_human(result: dict, question: str) -> str:
if isinstance(total_ms, (int, float))
else "?"
)
# Ticket #000028 — render-layer witness tail. When witness ran,
# surface the agreement label + per-modality status. Pure
# render-only; cache_key / governance_policy_hash unaffected.
witness = result.get("witness")
witness_tail = ""
witness_block = ""
if witness:
label = witness.get("agreement_label", "?")
mods = witness.get("modalities") or {}
ground_truth_hex = witness.get("canonical_answer_bytes_hex") or ""
agree_count = sum(
1 for name, m in mods.items()
if name != "kernel"
and m.get("ok")
and (m.get("canonical_bytes_hex") or "") == ground_truth_hex
)
checkable = sum(
1 for name, m in mods.items()
if name != "kernel" and m.get("error") != "ABSENT"
)
witness_tail = f" [{label} · {agree_count}/{checkable} modalities agree]"
lines = ["", "witness:"]
for name in ("kernel", "cache", "llm"):
m = mods.get(name)
if not m:
continue
raw = m.get("raw_answer")
err = m.get("error")
ms = int(m.get("elapsed_ms") or 0)
if err == "ABSENT":
detail = "absent"
elif err == "PIS_REJECT":
detail = f'rejected: "{raw}"'
elif err == "TIMEOUT":
detail = "timeout"
elif err and err.startswith("LLM_ERROR"):
detail = err.lower()
else:
detail = f'"{raw}"'
lines.append(f" {name:<7} {detail} ({ms}ms)")
witness_block = "\n".join(lines)
return (
f"{question}\n"
f" CANONICAL · via {pi_star_ref} {elapsed} (projected)\n\n"
f" CANONICAL · via {pi_star_ref}{witness_tail} {elapsed} (projected)\n\n"
f"{answer_text}"
f"{witness_block}"
)
if status not in ("cache_hit", "cache_miss_then_written"):
msg = result.get("msg") or status or "unknown error"
@ -4230,6 +4278,25 @@ def build_parser() -> argparse.ArgumentParser:
"comparing the canonical answer with the model's reply)."
),
)
witness_group = query_cmd.add_mutually_exclusive_group()
witness_group.add_argument(
"--witness", dest="witness_override", action="store_const",
const="on",
help=(
"Ticket #000028 — multi-modality witness. On canonical-shape "
"questions (arithmetic@v1, logic-kernel@v1, future kernels), "
"fan out kernel + cache + LLM in parallel and verify cross-"
"modality agreement via kernel-canonicalization. Adds one LLM "
"call (~2-5s) on top of the canonical fast path; default OFF "
"to keep arithmetic queries at ~10ms."
),
)
witness_group.add_argument(
"--no-witness", dest="witness_override", action="store_const",
const="off",
help="Disable the witness path even when policy enables it.",
)
progress_group = query_cmd.add_mutually_exclusive_group()
progress_group.add_argument(
"--progress", dest="progress_override", action="store_const",
@ -4250,7 +4317,9 @@ def build_parser() -> argparse.ArgumentParser:
"tools that misbehave on stderr noise."
),
)
query_cmd.set_defaults(func=_cmd_query, progress_override=None)
query_cmd.set_defaults(
func=_cmd_query, progress_override=None, witness_override=None,
)
inspect_cmd = sub.add_parser(
"inspect",

85
arborist/qa/progress.py Normal file
View file

@ -0,0 +1,85 @@
"""Stage-level state-machine emitter for `arborist query`.
Writes one line per state transition to stderr. Stdout stays clean
(JSON output, render output, unfirehose journal) so this is safe to
enable in scripts and pipelines as well as interactively.
Format::
[arborist NN.NNs] stage.name key=value key=value
Elapsed is wall-clock seconds since the Progress object was constructed.
The gap *between* lines is the per-stage cost — that's what makes the
slow shard or slow route visible.
Default-on at TTY, default-off when stderr is piped. Override via
``ARBORIST_PROGRESS=1`` (force on) / ``=0`` (force off), or per-call
via ``cli_override="on"|"off"``.
"""
from __future__ import annotations
import os
import sys
import time
from typing import Any
class Progress:
"""Disabled-mode is a true no-op — no string formatting cost."""
__slots__ = ("enabled", "t_start", "stream")
def __init__(
self,
*,
enabled: bool,
t_start: float | None = None,
stream: Any = None,
) -> None:
self.enabled = enabled
self.t_start = t_start if t_start is not None else time.monotonic()
self.stream = stream if stream is not None else sys.stderr
def emit(self, stage: str, **kv: Any) -> None:
if not self.enabled:
return
elapsed = time.monotonic() - self.t_start
prefix = f"[arborist {elapsed:6.2f}s] {stage}"
if kv:
tail = " ".join(f"{k}={v}" for k, v in kv.items())
line = f"{prefix} {tail}"
else:
line = prefix
print(line, file=self.stream, flush=True)
_DISABLED = Progress(enabled=False)
def disabled() -> Progress:
"""Singleton no-op emitter, suitable as a default kwarg value."""
return _DISABLED
def from_env(*, cli_override: str | None = None) -> Progress:
"""Resolve enable-state and return a fresh emitter.
Precedence: cli_override > ARBORIST_PROGRESS env > TTY auto-detect.
"""
if cli_override == "on":
enabled = True
elif cli_override == "off":
enabled = False
else:
env = (os.environ.get("ARBORIST_PROGRESS") or "").strip().lower()
if env in ("0", "false", "off", "no"):
enabled = False
elif env in ("1", "true", "on", "yes"):
enabled = True
else:
try:
enabled = sys.stderr.isatty()
except (AttributeError, ValueError):
enabled = False
return Progress(enabled=enabled)

26
arborist/qa/query.py
View file

@ -2008,6 +2008,31 @@ def query(
pi_star_ref=pi_star_ref,
ms=int(canonical_ms),
)
# Ticket #000028 — multi-modality witness. Default OFF;
# operator opts in via policy["canonical_witness_enabled"]
# (CLI: --witness). Fans out cache + LLM in parallel against
# the kernel ground truth and records cross-modality
# agreement. Cache leg is a no-op closure pre-Ticket #000027.
witness_dict = None
if bool(policy.get("canonical_witness_enabled", False)):
from arborist.qa.witness import run_witness
# Pre-#000027: no canonical persistence, cache always
# misses. The closure shape stays so #000027 can wire
# the real lookup without changing this call site.
_cache_lookup = lambda: None # noqa: E731
_witness = run_witness(
question=question,
pi_star_ref=pi_star_ref,
canonical_answer_bytes=canonical_bytes,
cache_lookup=_cache_lookup,
chat_client=chat_client,
model_id=model_id,
timeout_s=float(
policy.get("canonical_witness_timeout_s", 10.0)
),
progress=progress,
)
witness_dict = _witness.to_dict()
return {
"status": "canonical_projection",
"audit_mode": "CANONICAL_PROJECTION",
@ -2020,6 +2045,7 @@ def query(
"lookup_path": "preflight_canonical",
"sources": [],
"cache_key": None,
"witness": witness_dict,
"timings": {
"canonical_preflight_ms": canonical_ms,
"total_ms": _ms_since(t_start),

394
arborist/qa/witness.py Normal file
View file

@ -0,0 +1,394 @@
"""Multi-modality witness for canonical-shape questions (Ticket #000028).
When a question matches a canonical shape (arithmetic@v1,
logic-kernel@v1, future kernels), fan out three independent answer
paths in parallel:
- **kernel**: deterministic π* — ground truth.
- **cache**: prior commitment from ``providence_cache`` (closure
returns ``None`` when no prior row, e.g. pre-Ticket #000027).
- **alien (LLM)**: an independently-trained model invited to answer
the same question.
Each modality's answer is canonicalized through the kernel for
byte-comparison. Three paths byte-equal → ``STRICT-WITNESSED`` —
the strongest grounding signal arborist can offer (mono-modal ×
independent-training × prior-commitment). Disagreement is recorded
as falsification signal, not an error.
Default OFF — tripling LLM cost on every arithmetic question is a
tax operators don't want by default. Enable per-call via
``--witness`` CLI flag or per-policy via
``policy["canonical_witness_enabled"] = True``.
"""
from __future__ import annotations
import time
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutTimeoutError
from dataclasses import dataclass, field
from typing import Callable
from arborist.qa.client import ChatClient
WITNESS_SYSTEM_PROMPT = (
"You are a deterministic math/logic computer. Reply with ONLY the "
"canonical answer — no prose, no explanation, no LaTeX, no leading "
'"=", no trailing punctuation. For arithmetic emit a fraction (e.g. '
"3/10) or integer. For propositional logic emit TRUE or FALSE. "
"Output nothing else."
)
@dataclass(frozen=True)
class ModalityResult:
"""One modality's contribution to the witness."""
modality: str # "kernel" | "cache" | "llm"
raw_answer: str | None # what the modality said (display)
canonical_bytes: bytes | None # post-canonicalization bytes (None on reject)
error: str | None # "PIS_REJECT" / "TIMEOUT" / "ABSENT" / etc.
elapsed_ms: float
@property
def ok(self) -> bool:
return self.canonical_bytes is not None and self.error is None
def to_dict(self) -> dict:
return {
"modality": self.modality,
"raw_answer": self.raw_answer,
"canonical_bytes_hex": (
self.canonical_bytes.hex() if self.canonical_bytes else None
),
"error": self.error,
"elapsed_ms": round(self.elapsed_ms, 1),
"ok": self.ok,
}
@dataclass(frozen=True)
class Witness:
"""Outcome of a multi-modality witness for one canonical-shape question."""
pi_star_ref: str
canonical_answer_bytes: bytes
modalities: dict[str, ModalityResult] = field(default_factory=dict)
agreement_label: str = "UNCLASSIFIED"
elapsed_ms: float = 0.0
def to_dict(self) -> dict:
return {
"pi_star_ref": self.pi_star_ref,
"canonical_answer_bytes_hex": self.canonical_answer_bytes.hex(),
"canonical_answer_text": self.canonical_answer_bytes.decode(
"utf-8", errors="replace"
),
"modalities": {
name: m.to_dict() for name, m in self.modalities.items()
},
"agreement_label": self.agreement_label,
"elapsed_ms": round(self.elapsed_ms, 1),
}
def _canonicalize_via_kernel(text: str | None, pi_star_ref: str) -> bytes | None:
"""Run the named kernel on ``text``. Returns canonical bytes, or
``None`` when the kernel rejects (PiStarError) or input is empty.
"""
from arborist.pi_star import PiStarError, get
if text is None:
return None
cleaned = text.strip()
if not cleaned:
return None
try:
return get(pi_star_ref).canonicalize(cleaned.encode("utf-8"))
except PiStarError:
return None
except Exception: # pragma: no cover — defensive
return None
def _classify_agreement(
modalities: dict[str, ModalityResult],
canonical_answer_bytes: bytes,
) -> str:
"""Map (kernel, cache, llm) outcomes → agreement label.
The kernel is the ground-truth reference; cache & llm are checked
*against* the kernel's canonical_answer_bytes.
"""
cache = modalities.get("cache")
llm = modalities.get("llm")
cache_present = cache is not None and cache.ok
llm_present = llm is not None and llm.ok
cache_agrees = (
cache_present and cache.canonical_bytes == canonical_answer_bytes
)
llm_agrees = (
llm_present and llm.canonical_bytes == canonical_answer_bytes
)
cache_disagrees = cache_present and not cache_agrees
llm_disagrees = llm_present and not llm_agrees
# Decision tree (kernel always present when this is called):
if cache_agrees and llm_agrees:
return "STRICT-WITNESSED"
if cache_agrees and llm_disagrees:
return "LLM-DIVERGED"
if cache_disagrees and llm_agrees:
return "CACHE-DRIFT"
if cache_disagrees and llm_disagrees:
return "LLM-AND-CACHE-DIVERGED"
if not cache_present and llm_agrees:
return "KERNEL-LLM-AGREE"
if not cache_present and llm_disagrees:
return "KERNEL-LLM-DIVERGED"
if cache_agrees and not llm_present:
return "KERNEL-CACHE-AGREE"
if cache_disagrees and not llm_present:
return "CACHE-DRIFT-LLM-ABSENT"
# Both cache and llm absent — only the kernel ran. Degenerate but
# surface it explicitly so the audit-line tail makes sense.
return "KERNEL-ONLY"
def _run_kernel_modality(
canonical_answer_bytes: bytes, pi_star_ref: str
) -> ModalityResult:
"""Kernel modality: by definition produces canonical_answer_bytes.
The work was already done before run_witness was called — we
receive the answer and just package it as a ModalityResult so
the matrix has a kernel row alongside cache + llm.
"""
return ModalityResult(
modality="kernel",
raw_answer=canonical_answer_bytes.decode("utf-8", errors="replace"),
canonical_bytes=canonical_answer_bytes,
error=None,
elapsed_ms=0.0,
)
def _run_cache_modality(
cache_lookup: Callable[[], bytes | None] | None,
pi_star_ref: str,
) -> ModalityResult:
"""Cache modality: invoke the lookup closure. ``None`` → ABSENT
(no prior commitment); bytes → canonicalize & report.
"""
t0 = time.monotonic()
if cache_lookup is None:
return ModalityResult(
modality="cache",
raw_answer=None,
canonical_bytes=None,
error="ABSENT",
elapsed_ms=0.0,
)
try:
raw_bytes = cache_lookup()
except Exception as e: # pragma: no cover
return ModalityResult(
modality="cache",
raw_answer=None,
canonical_bytes=None,
error=f"LOOKUP_ERROR:{type(e).__name__}",
elapsed_ms=(time.monotonic() - t0) * 1000.0,
)
elapsed = (time.monotonic() - t0) * 1000.0
if raw_bytes is None:
return ModalityResult(
modality="cache",
raw_answer=None,
canonical_bytes=None,
error="ABSENT",
elapsed_ms=elapsed,
)
raw_text = raw_bytes.decode("utf-8", errors="replace")
canonical = _canonicalize_via_kernel(raw_text, pi_star_ref)
if canonical is None:
return ModalityResult(
modality="cache",
raw_answer=raw_text,
canonical_bytes=None,
error="PIS_REJECT",
elapsed_ms=elapsed,
)
return ModalityResult(
modality="cache",
raw_answer=raw_text,
canonical_bytes=canonical,
error=None,
elapsed_ms=elapsed,
)
def _run_llm_modality(
question: str,
chat_client: ChatClient | None,
model_id: str,
pi_star_ref: str,
) -> ModalityResult:
"""LLM modality: ask the alien to compute the answer with strict
output formatting; canonicalize the response through the kernel.
"""
t0 = time.monotonic()
if chat_client is None:
return ModalityResult(
modality="llm",
raw_answer=None,
canonical_bytes=None,
error="ABSENT",
elapsed_ms=0.0,
)
messages = [
{"role": "system", "content": WITNESS_SYSTEM_PROMPT},
{"role": "user", "content": question},
]
try:
raw = chat_client.chat_completion(
messages,
model=model_id,
temperature=0.0,
max_tokens=64,
top_p=1.0,
)
except Exception as e:
return ModalityResult(
modality="llm",
raw_answer=None,
canonical_bytes=None,
error=f"LLM_ERROR:{type(e).__name__}",
elapsed_ms=(time.monotonic() - t0) * 1000.0,
)
elapsed = (time.monotonic() - t0) * 1000.0
raw_text = (raw or "").strip()
canonical = _canonicalize_via_kernel(raw_text, pi_star_ref)
if canonical is None:
return ModalityResult(
modality="llm",
raw_answer=raw_text,
canonical_bytes=None,
error="PIS_REJECT",
elapsed_ms=elapsed,
)
return ModalityResult(
modality="llm",
raw_answer=raw_text,
canonical_bytes=canonical,
error=None,
elapsed_ms=elapsed,
)
def run_witness(
*,
question: str,
pi_star_ref: str,
canonical_answer_bytes: bytes,
cache_lookup: Callable[[], bytes | None] | None = None,
chat_client: ChatClient | None = None,
model_id: str = "",
timeout_s: float = 10.0,
progress=None,
) -> Witness:
"""Fan out kernel + cache + LLM in parallel; return a Witness.
The kernel result is already known (passed in as
``canonical_answer_bytes``) — no work to do. Cache and LLM run in
parallel threads. Total wall time ≈ max(cache, LLM) + small
overhead, not sum.
``cache_lookup`` is a closure to keep this module independent of
any storage concerns. Pass ``None`` (default) when no cache is
available — modality reports ``ABSENT``.
``chat_client`` set to ``None`` skips the LLM modality entirely.
``timeout_s`` bounds the whole parallel stage. Modalities that
don't finish in time report ``TIMEOUT``.
"""
from arborist.qa.progress import disabled as _progress_disabled
progress = progress or _progress_disabled()
t_start = time.monotonic()
progress.emit(
"witness.start",
parallel=2 + (1 if chat_client is not None else 0),
timeout_s=timeout_s,
)
kernel_mod = _run_kernel_modality(canonical_answer_bytes, pi_star_ref)
progress.emit(
"witness.modality.kernel",
ms=int(kernel_mod.elapsed_ms),
ok=str(kernel_mod.ok).lower(),
)
modalities: dict[str, ModalityResult] = {"kernel": kernel_mod}
# Submit cache + llm to the pool in parallel.
with ThreadPoolExecutor(max_workers=2, thread_name_prefix="witness") as pool:
fut_cache = pool.submit(
_run_cache_modality, cache_lookup, pi_star_ref
)
fut_llm = pool.submit(
_run_llm_modality, question, chat_client, model_id, pi_star_ref
)
try:
cache_mod = fut_cache.result(timeout=timeout_s)
except FutTimeoutError:
cache_mod = ModalityResult(
modality="cache",
raw_answer=None,
canonical_bytes=None,
error="TIMEOUT",
elapsed_ms=timeout_s * 1000.0,
)
fut_cache.cancel()
modalities["cache"] = cache_mod
progress.emit(
"witness.modality.cache",
ms=int(cache_mod.elapsed_ms),
ok=str(cache_mod.ok).lower(),
reason=(cache_mod.error or "ok"),
)
try:
llm_mod = fut_llm.result(timeout=timeout_s)
except FutTimeoutError:
llm_mod = ModalityResult(
modality="llm",
raw_answer=None,
canonical_bytes=None,
error="TIMEOUT",
elapsed_ms=timeout_s * 1000.0,
)
fut_llm.cancel()
modalities["llm"] = llm_mod
progress.emit(
"witness.modality.llm",
ms=int(llm_mod.elapsed_ms),
ok=str(llm_mod.ok).lower(),
reason=(llm_mod.error or "ok"),
)
label = _classify_agreement(modalities, canonical_answer_bytes)
elapsed = (time.monotonic() - t_start) * 1000.0
progress.emit("witness.done", label=label, ms=int(elapsed))
return Witness(
pi_star_ref=pi_star_ref,
canonical_answer_bytes=canonical_answer_bytes,
modalities=modalities,
agreement_label=label,
elapsed_ms=elapsed,
)

87
bench/results/real-shard-baseline.json
View file

@ -1,22 +1,23 @@
{
"schema_version": "real-shard-baseline-v1",
"timestamp_utc": "2026-05-08T17:09:43+00:00",
"commit_sha": "e78814ca29ed4114572f2410b2615733d4b715f6",
"timestamp_utc": "2026-05-08T19:23:46+00:00",
"commit_sha": "c54bedab1c5502a55edf14ab179ce8d8a4dd0719",
"git_dirty": true,
"shards_dir": "/home/fox/.arborist/shards",
"shards_fingerprint": "d93bed2c48c7ac47b22286ac77508d5519fec2f3685d470966642eef4285cb5e",
"shards_fingerprint": "b2c76ce1b547aa6b09706d283646b53c347e1ecc1ad4d7d8691dc24bb0e2b4e9",
"model": "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic",
"endpoint": "https://hermes.ai.unturf.com/v1",
"burn": true,
"fixtures_path": "bench/fixtures/real-shard-baseline-v1.jsonl",
"summary": {
"questions": 8,
"wall_ms_median": 4140.9,
"wall_ms_max": 7212.4,
"wall_ms_min": 0.6,
"wall_ms_median": 6420.3,
"wall_ms_max": 10910.7,
"wall_ms_min": 0.9,
"by_audit_mode": {
"STRICT": 4,
"HYBRID": 2,
"UNGROUNDED": 1,
"HYBRID": 1,
"CANONICAL_PROJECTION": 2
},
"by_status": {
@ -36,10 +37,10 @@
"n_quotes": 1,
"pi_star_ref": null,
"lookup_path": "miss",
"wall_ms": 6309.6,
"total_ms": 6297.3,
"search_ms": 5002.6,
"llm_ms": 1213.1,
"wall_ms": 10910.7,
"total_ms": 10885.8,
"search_ms": 8277.3,
"llm_ms": 2479.7,
"verify_ms": null,
"canonical_preflight_ms": null,
"sources_count": 8,
@ -57,10 +58,10 @@
"n_quotes": 1,
"pi_star_ref": null,
"lookup_path": "miss",
"wall_ms": 3867.2,
"total_ms": 3866.5,
"search_ms": 2663.1,
"llm_ms": 1123.0,
"wall_ms": 6526.1,
"total_ms": 6523.8,
"search_ms": 4464.9,
"llm_ms": 1985.1,
"verify_ms": null,
"canonical_preflight_ms": null,
"sources_count": 5,
@ -78,10 +79,10 @@
"n_quotes": 1,
"pi_star_ref": null,
"lookup_path": "miss",
"wall_ms": 7212.4,
"total_ms": 7211.4,
"search_ms": 1485.5,
"llm_ms": 5644.9,
"wall_ms": 8921.9,
"total_ms": 8919.4,
"search_ms": 2475.5,
"llm_ms": 6356.5,
"verify_ms": null,
"canonical_preflight_ms": null,
"sources_count": 4,
@ -99,36 +100,36 @@
"n_quotes": 1,
"pi_star_ref": null,
"lookup_path": "miss",
"wall_ms": 4140.9,
"total_ms": 4140.0,
"search_ms": 2088.9,
"llm_ms": 1989.7,
"wall_ms": 4794.5,
"total_ms": 4791.5,
"search_ms": 3219.7,
"llm_ms": 1463.0,
"verify_ms": null,
"canonical_preflight_ms": null,
"sources_count": 8,
"primary_source_used": false,
"answer_text": "The Linux kernel was written in the C programming language. From the source:\n\"The Linux kernel is an operating system kernel used by the Linux family of Unix-like operating systems. It is one of the most prominent examples of free and open ",
"answer_text": "The Linux kernel was written in the C programming language. The source states: \"The Linux kernel is an operating system kernel used by the Linux family of Unix-like operating systems. It is one of the most prominent examples of free and ope",
"expected_routing": "wiki_primary",
"note": "specific factual claim; well-grounded"
},
{
"question": "Who founded Microsoft?",
"status": "cache_miss_then_written",
"audit_mode": "HYBRID",
"verifier_method": "entity",
"n_verified": 4,
"n_quotes": 4,
"audit_mode": "UNGROUNDED",
"verifier_method": "quote",
"n_verified": 0,
"n_quotes": 1,
"pi_star_ref": null,
"lookup_path": "miss",
"wall_ms": 2248.7,
"total_ms": 2247.6,
"search_ms": 1134.6,
"llm_ms": 1070.8,
"wall_ms": 3586.2,
"total_ms": 3584.1,
"search_ms": 1893.4,
"llm_ms": 1624.1,
"verify_ms": null,
"canonical_preflight_ms": null,
"sources_count": 5,
"primary_source_used": true,
"answer_text": "Based on the provided sources, I don't have enough information to determine who founded Microsoft. The sources mention Microsoft Research, Microsoft TechNet, Microsoft Hardware, and Microsoft Dynamics NAV, but do not contain any verbatim qu",
"answer_text": "Based on the provided sources, Microsoft was founded by Bill Gates and Paul Allen. The source states: \"Microsoft was founded by Bill Gates and Paul Allen in 1975\" (Source: https://en.wikipedia.org/wiki/Microsoft).",
"expected_routing": "wiki_primary",
"note": "named-entity authorship"
},
@ -141,10 +142,10 @@
"n_quotes": 5,
"pi_star_ref": null,
"lookup_path": "miss",
"wall_ms": 4327.0,
"total_ms": 4326.3,
"search_ms": 1636.2,
"llm_ms": 2623.3,
"wall_ms": 6420.3,
"total_ms": 6417.2,
"search_ms": 2720.9,
"llm_ms": 3533.2,
"verify_ms": null,
"canonical_preflight_ms": null,
"sources_count": 8,
@ -162,12 +163,12 @@
"n_quotes": 1,
"pi_star_ref": "arithmetic@v1",
"lookup_path": "preflight_canonical",
"wall_ms": 0.9,
"total_ms": 0.3,
"wall_ms": 1.3,
"total_ms": 0.4,
"search_ms": null,
"llm_ms": null,
"verify_ms": null,
"canonical_preflight_ms": 0.3,
"canonical_preflight_ms": 0.4,
"sources_count": 0,
"primary_source_used": false,
"answer_text": "3/10",
@ -183,12 +184,12 @@
"n_quotes": 1,
"pi_star_ref": "logic-kernel@v1",
"lookup_path": "preflight_canonical",
"wall_ms": 0.6,
"total_ms": 0.1,
"wall_ms": 0.9,
"total_ms": 0.3,
"search_ms": null,
"llm_ms": null,
"verify_ms": null,
"canonical_preflight_ms": 0.1,
"canonical_preflight_ms": 0.3,
"sources_count": 0,
"primary_source_used": false,
"answer_text": "(NOT A OR B)",

28
bench/results/real-shard-baseline.md
View file

@ -1,18 +1,18 @@
# Real-shard baseline — 2026-05-08T17:09:43+00:00
# Real-shard baseline — 2026-05-08T19:23:46+00:00
**Commit:** `e78814ca29ed` (dirty)
**Shards fingerprint:** `d93bed2c48c7ac47…`
**Commit:** `c54bedab1c55` (dirty)
**Shards fingerprint:** `b2c76ce1b547aa6b…`
**Shards directory:** `/home/fox/.arborist/shards`
## Summary
- Questions: **8**
- Wall median: **4140.9 ms**
- Wall range: 0.6 – 7212.4 ms
- Wall median: **6420.3 ms**
- Wall range: 0.9 – 10910.7 ms
- Primary source used: 4/8
- Canonical-projection short-circuits: 2
**Audit-mode distribution:** `CANONICAL_PROJECTION` 2, `HYBRID` 2, `STRICT` 4
**Audit-mode distribution:** `CANONICAL_PROJECTION` 2, `HYBRID` 1, `STRICT` 4, `UNGROUNDED` 1
**Status distribution:** `cache_miss_then_written` 6, `canonical_projection` 2
@ -20,11 +20,11 @@
| # | Question | Audit | Status | Wall (ms) | Method |
|---|----------|-------|--------|-----------|--------|
| 1 | who wrote virt-back? | STRICT | cache_miss_then_written | 6309.6 | quote |
| 2 | What is the capital of France? | STRICT | cache_miss_then_written | 3867.2 | quote |
| 3 | What is Mac OS X? | STRICT | cache_miss_then_written | 7212.4 | quote |
| 4 | What programming language was the Linux kernel written in? | STRICT | cache_miss_then_written | 4140.9 | quote |
| 5 | Who founded Microsoft? | HYBRID | cache_miss_then_written | 2248.7 | entity |
| 6 | What is the relationship between AMD and Intel? | HYBRID | cache_miss_then_written | 4327.0 | paraphrase |
| 7 | 0.1 + 0.2 | CANONICAL_PROJECTION | canonical_projection | 0.9 | canonical_projection |
| 8 | A IMPL B | CANONICAL_PROJECTION | canonical_projection | 0.6 | canonical_projection |
| 1 | who wrote virt-back? | STRICT | cache_miss_then_written | 10910.7 | quote |
| 2 | What is the capital of France? | STRICT | cache_miss_then_written | 6526.1 | quote |
| 3 | What is Mac OS X? | STRICT | cache_miss_then_written | 8921.9 | quote |
| 4 | What programming language was the Linux kernel written in? | STRICT | cache_miss_then_written | 4794.5 | quote |
| 5 | Who founded Microsoft? | UNGROUNDED | cache_miss_then_written | 3586.2 | quote |
| 6 | What is the relationship between AMD and Intel? | HYBRID | cache_miss_then_written | 6420.3 | paraphrase |
| 7 | 0.1 + 0.2 | CANONICAL_PROJECTION | canonical_projection | 1.3 | canonical_projection |
| 8 | A IMPL B | CANONICAL_PROJECTION | canonical_projection | 0.9 | canonical_projection |

4
docs/TICKETS.md
View file

@ -61,6 +61,8 @@ Newest first. Update on every open/close.
| ID | Title | Status | Opened | Directive |
|----------|------------------------------------------------|-----------------------|------------|-----------|
| #000028 | Multi-modality witness for canonical shapes | open · implementation in progress | 2026-05-08 | — |
| #000027 | Canonical projections persist to providence_cache | open · design phase | 2026-05-08 | — |
| #000026 | Real-shard workload baseline + search latency | in progress · Phase 1 + 2 landed 2026-05-08 | 2026-05-08 | — |
| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a landed 2026-05-08 | 2026-05-07 | — |
| #000024 | 5T Phase 1b + Dav1DPrometheus vocabulary alignment | closed · landed 2026-05-08 | 2026-05-07 | — |
@ -90,4 +92,4 @@ Newest first. Update on every open/close.
## Next ID
`000027`
`000029`

423
docs/tickets/ticket-000028-multi-modality-witness.md Normal file
View file

@ -0,0 +1,423 @@
# Ticket #000028 — Multi-modality witness for canonical shapes
**Status:** open · implementation in progress
**Opened:** 2026-05-08
**Scope:** When a question matches a canonical shape (arithmetic@v1,
logic-kernel@v1, future kernels), fan out three independent answer
paths in parallel — **python kernel**, **cache lookup**, **alien
LLM** — and verify cross-modality agreement via kernel-canonicalization
of every answer. Disagreement is recorded as falsification signal,
not an error. Agreement is the strongest grounding signal arborist
can offer.
**Audience:** fox + future blackops shifts.
**Hard constraint:** opt-in (default off). Tripling latency on every
arithmetic question would tax operators who only want the kernel
answer. The witness path runs only when explicitly enabled per call
or per-policy.
---
## 1. Problem statement
Canonical projections (post-#000027) provide deterministic ground
truth for arithmetic and propositional logic. The LLM, in contrast,
is trained to produce plausible text and can hallucinate even on
trivial math (`0.1 + 0.2 = 0.30000000000000004` is famous LLM
training data; the kernel returns `3/10` exactly). Today we have no
systematic way to:
- **Detect when the LLM disagrees with ground truth** for canonical
questions. A bench could measure this offline, but every live
call that asks a canonical-shape question is also a calibration
data point we throw away.
- **Catch cache drift.** If the kernel was bumped (`@v1` → `@v2`)
without invalidating prior rows, the cache silently serves stale
answers. A live cross-check at lookup time catches this.
- **Strengthen the warrant for canonical answers.** Today the
warrant is "kernel-deterministic" — strong, but mono-modal.
Three-path agreement is mono-modal × independent-training × prior-
commitment — strictly stronger.
### 1.1 Architecture sketch
```
question
├─ python kernel (arithmetic@v1) → "3/10" evidence: kernel-deterministic
├─ cache lookup (providence_cache) → "3/10" evidence: prior commitment
└─ alien (LLM) (Hermes/Claude/etc) → "0.3" evidence: training-derived
│
└─ claim-lattice cross-modality verifier:
canonicalize("0.3") via arithmetic@v1 → "3/10"
all three byte-equal → STRICT-CANONICAL-WITNESSED
```
### 1.2 Agreement matrix
For three modalities {kernel (K), cache (C), llm (L)} the agreement
matrix has 7 distinct outcomes (kernel always present when this path
fires; cache may be absent pre-#000027; LLM may be absent if
canonicalization fails):
| K | C | L | Outcome label | Meaning |
|---|---|---|---|---|
| ✓ | ✓ | ✓ | `STRICT-WITNESSED` | All three byte-equal — strongest possible warrant |
| ✓ | ✓ | ✗ | `LLM-DIVERGED` | Cache & kernel agree, LLM hallucinated — bench data |
| ✓ | ✗ | ✓ | `CACHE-DRIFT` | Kernel & LLM agree, cache row stale — invalidate row |
| ✓ | ✗ | ✗ | `LLM-AND-CACHE-DIVERGED` | Only kernel matches itself (degenerate) |
| ✓ | absent | ✓ | `KERNEL-LLM-AGREE` | No prior cache row; kernel ≡ LLM |
| ✓ | absent | ✗ | `KERNEL-LLM-DIVERGED` | No prior cache row; LLM disagrees with kernel |
| ✓ | ✓ | absent | `KERNEL-CACHE-AGREE` | LLM produced unparseable text; kernel ≡ cache |
Kernel is always the ground truth. Cache and LLM are checked
*against* the kernel. The kernel never participates in byte-equal
comparison with itself — it's the reference.
---
## 2. Design choices
### 2.1 Parallelism
LLM call is ~2-5s I/O-bound. Kernel + cache are ~10ms CPU/SQL.
Sequential = sum (slow); parallel = max (LLM only).
**A. ThreadPoolExecutor (RECOMMENDED).** Three threads, one per
modality. The LLM call doesn't block kernel/cache. Total wall time
≈ LLM time + small overhead. Plays well with the existing sync
codebase (no asyncio refactor).
**B. asyncio.** Cleaner conceptually but the existing codebase is
sync top-to-bottom. Pulling asyncio in for one call creates an
event-loop boundary the rest of the code doesn't know about.
**C. Sequential.** Adds latency = kernel + cache + LLM ≈ LLM × 1.01.
Simpler. Loses the "free LLM hit" framing (you wait for it anyway).
→ **A.** ~30 LOC delta. Standard library. Threads are I/O-bound so
the GIL doesn't block.
### 2.2 LLM prompt design
The LLM must emit a canonicalize-able answer for byte-comparison
to work. Two prompt strategies:
**A. Strict-format (RECOMMENDED).** System prompt explicitly
constrains the output:
```
You are a deterministic math/logic computer. Reply with ONLY the
canonical answer — no prose, no explanation, no LaTeX, no leading
"=", no trailing punctuation. For arithmetic emit a fraction (e.g.
3/10) or integer. For propositional logic emit TRUE or FALSE.
```
If the LLM disobeys, the canonicalizer rejects the answer →
classified as `L=✗`. This itself is a valuable signal: the LLM
ignored instructions on a question with a known-correct format.
**B. Extraction.** Accept arbitrary LLM output, run a regex/heuristic
extractor to pull a numeric portion, then canonicalize. More
permissive but introduces extraction errors as a confounder.
→ **A.** Cleaner separation. If we want extraction later, layer it
on as a fallback when strict mode rejects.
### 2.3 Canonicalization for cross-modality comparison
The kernel canonicalizes each modality's answer to bytes:
```python
kernel_bytes = arithmetic_v1.canonicalize(b"0.1 + 0.2") # "3/10"
llm_bytes = arithmetic_v1.canonicalize(b"0.3") # "3/10"
cache_bytes = arithmetic_v1.canonicalize(b"3/10") # "3/10" (idempotent)
agreement = (kernel_bytes == llm_bytes == cache_bytes)
```
**Idempotence is the key property.** The kernel is a deterministic
projection — running it on its own output yields the same bytes.
This is why we can compare across modalities by canonicalizing each
to the same canonical form.
Failure modes:
- LLM produces `"the answer is 0.3"` → `_CANONICAL_ARITHMETIC_RE`
rejects (letters present) → `L=absent`.
- LLM produces `"0.3"` → canonicalize succeeds → `L=✓`.
- LLM produces `"0.4"` (wrong) → canonicalize succeeds → `L=✓` but
`llm_bytes != kernel_bytes` → `LLM-DIVERGED`.
### 2.4 Default behavior
**Default OFF.** Tripling LLM cost on every arithmetic question is a
tax operators don't want by default. Witness path enabled per-call
via `--witness` CLI flag or per-policy via
`policy["canonical_witness_enabled"] = True`.
Sampling (e.g., 5% of canonical questions get the witness path for
opportunistic calibration) is **out of scope for this ticket**.
Easy to add later as `policy["canonical_witness_sample_rate"] =
0.05`.
### 2.5 What gets persisted
This ticket does **NOT** persist witness rows. Reasons:
1. #000027 (canonical persistence) is not yet implemented. Until
then there's no canonical row to attach a witness to.
2. The witness data model needs to settle — the agreement matrix
may grow new outcomes as more kernels land.
**For MVP**: witness result surfaces on the return dict + audit-line
render. When #000027 lands, witness data folds into the
`providence_canonical` audit event body as a `witness` clause.
### 2.6 What `audit_mode` does the witnessed answer carry?
The runtime `audit_mode` stays `"CANONICAL_PROJECTION"`. The witness
outcome is a render-layer label tail:
```
0.1 + 0.2
CANONICAL · via arithmetic@v1 [STRICT-WITNESSED · 3/3 modalities agree] 0.0s
3/10
```
vs. divergence:
```
2 + 2
CANONICAL · via arithmetic@v1 [LLM-DIVERGED · llm said "5"] 3.2s
4
```
The CLI render adds a witness tail; cache_key + governance_policy_hash
+ programmatic callers see `audit_mode = "CANONICAL_PROJECTION"`
unchanged. **Same pattern as the four-rung ladder render layer.**
---
## 3. Implementation sketch
### 3.1 New module `arborist/qa/witness.py` (~200 LOC)
```python
@dataclass(frozen=True)
class ModalityResult:
modality: str # "kernel" | "cache" | "llm"
raw_answer: str | None # what the modality said (display)
canonical_bytes: bytes | None # post-canonicalization bytes
error: str | None # "PIS_REJECT" / "TIMEOUT" / etc.
elapsed_ms: float
@dataclass(frozen=True)
class Witness:
pi_star_ref: str
modalities: dict[str, ModalityResult]
agreement_label: str # "STRICT-WITNESSED" / "LLM-DIVERGED" / etc.
canonical_answer_bytes: bytes # the kernel's output (ground truth)
def run_witness(
*,
question: str,
pi_star_ref: str,
canonical_answer_bytes: bytes, # already-computed kernel output
cache_lookup: Callable[[], bytes | None], # closure that returns cached bytes or None
chat_client: ChatClient | None,
model_id: str,
timeout_s: float = 10.0,
progress: Progress | None = None,
) -> Witness:
"""Fan out kernel + cache + LLM in parallel; return Witness."""
...
def _canonicalize_via_kernel(text: str, pi_star_ref: str) -> bytes | None:
"""Run the named kernel on `text`. Returns canonical bytes, or None
if the kernel rejects (PiStarError / non-canonical shape)."""
...
def _classify_agreement(modalities: dict[str, ModalityResult]) -> str:
"""Map modality outcomes → STRICT-WITNESSED / LLM-DIVERGED / etc."""
...
```
### 3.2 Edit `arborist/qa/query.py` canonical branch (~40 LOC delta)
```python
if canonical_match is not None:
pi_star_ref, canonical_bytes = canonical_match
answer_text = canonical_bytes.decode("utf-8", errors="replace")
witness = None
if policy.get("canonical_witness_enabled", False):
from arborist.qa.witness import run_witness
# Cache lookup is a no-op closure pre-#000027 (always returns
# None). Post-#000027 it's the actual canonical_cache lookup.
def _cache_lookup() -> bytes | None:
return None # TODO: wire post-#000027
witness = run_witness(
question=question,
pi_star_ref=pi_star_ref,
canonical_answer_bytes=canonical_bytes,
cache_lookup=_cache_lookup,
chat_client=chat_client,
model_id=model_id,
timeout_s=float(policy.get("canonical_witness_timeout_s", 10.0)),
progress=progress,
)
return {
"status": "canonical_projection",
"audit_mode": "CANONICAL_PROJECTION",
"verifier_method": "canonical_projection",
"pi_star_ref": pi_star_ref,
"answer_text": answer_text,
"witness": witness.to_dict() if witness else None,
...
}
```
### 3.3 CLI surface
```bash
make query Q="0.1 + 0.2" WITNESS=1
.venv/bin/arborist query --witness "0.1 + 0.2"
```
Mutex group `--witness` / `--no-witness` on `arborist query`.
Make passthrough: `WITNESS=1` → `--witness`.
### 3.4 Render layer
`arborist/cli.py` audit-line render gains a witness tail when
`result.get("witness")` is set:
```
CANONICAL · via arithmetic@v1 [STRICT-WITNESSED · 3/3] 0.0s
```
Detail block under the answer (when `--json` not used) prints each
modality:
```
witness:
kernel: "3/10" (10ms)
cache: absent (no prior commitment)
llm: "3/10" (2841ms · agrees)
```
### 3.5 Progress emitter
```
[arborist 0.02s] canonical_projection.match pi_star_ref=arithmetic@v1
[arborist 0.02s] witness.start parallel=3 timeout_s=10
[arborist 0.03s] witness.modality.kernel ms=8 ok=true
[arborist 0.03s] witness.modality.cache ms=1 ok=false reason=absent
[arborist 2.84s] witness.modality.llm ms=2812 ok=true canonicalized=true
[arborist 2.84s] witness.done label=STRICT-WITNESSED
```
### 3.6 Tests `tests/qa/test_witness.py` (~120 LOC)
- `test_three_way_agreement` — kernel + cache + LLM all return same canonical bytes → `STRICT-WITNESSED`.
- `test_llm_diverges` — kernel + cache agree, LLM returns different value → `LLM-DIVERGED`.
- `test_llm_not_canonical` — LLM emits prose; canonicalizer rejects → `KERNEL-CACHE-AGREE` (LLM absent).
- `test_cache_absent` — cache lookup returns None → `KERNEL-LLM-AGREE`.
- `test_cache_drift` — kernel + LLM agree, cache returns stale value → `CACHE-DRIFT`.
- `test_llm_timeout` — chat_client raises TimeoutError → LLM modality reports error, agreement label reflects absence.
- `test_default_off` — witness only runs when policy flag is True.
- `test_parallel_timing` — total wall time ≤ max(kernel, llm) + overhead, not sum.
### 3.7 Stub LLM client for tests
Existing `arborist.qa.client.StubClient(answer="...")` works.
`StubClient(answer="3/10")` for agreement; `StubClient(answer="0.4")`
for divergence; `StubClient(answer="the answer is 0.3")` for
not-canonical; `StubClient(raises=TimeoutError)` for timeout.
---
## 4. Out of scope
- **Persistence of witness rows.** Wires in via #000027 once that ticket
lands. For MVP, witness is runtime-only.
- **Sampling rate** for opportunistic calibration. Add as `policy
["canonical_witness_sample_rate"] = 0.05` later if useful.
- **Falsification of LLM modality.** When `LLM-DIVERGED`, we record
the divergence on the result dict but don't mutate any LLM
reliability ledger. Future ticket: `LLMReliability` with
per-shape accuracy tracking.
- **Re-asking the LLM with feedback.** No retry / repair loop for
canonical shapes — the kernel is ground truth, the LLM gets one
shot.
- **Multi-LLM witness** (Hermes + Claude + GPT4 vote). Single LLM
for now. The "alien" leg is one independently-trained model.
Ensemble is a later ticket.
- **Cache-leg implementation.** Stub closure that returns None.
Real implementation lands with #000027.
---
## 5. Risks and rollback
**Risk 1 — LLM cost.** Witness adds one full LLM call per canonical
question. Default-off mitigates. Operators who flip `--witness` on
accept the cost knowingly. Sampling rate (out of scope) addresses
opportunistic-calibration use.
**Risk 2 — LLM canonicalization rate.** If the LLM ignores the
strict-format prompt > 30% of the time, the witness is mostly
noise. Bench fixture in §3.6 includes adherence measurement;
fox decides the floor.
**Risk 3 — Threading correctness.** ThreadPoolExecutor on three
short-lived threads is well-trodden territory; primary concern is
clean shutdown on timeout. `concurrent.futures.wait(timeout=...)`
+ `future.cancel()` handles it.
**Risk 4 — Render-layer divergence.** The audit-line tail is render-
only. Cache_key, governance_policy_hash, and run_dag stay
byte-identical to non-witness canonical answers. Same render-layer
discipline as the four-rung ladder.
**Rollback**: set `policy["canonical_witness_enabled"] = False`
(default). Witness module stays installed but never fires.
Pre-existing canonical answers unaffected.
---
## 6. Acceptance criteria
1. `make query Q="0.1 + 0.2" WITNESS=1` returns the same `3/10`
answer with a `witness` field on the result dict + a witness
tail on the audit-line render.
2. When the LLM emits a canonicalize-able answer that matches the
kernel: `agreement_label = "KERNEL-LLM-AGREE"` (cache absent
pre-#000027) or `"STRICT-WITNESSED"` (cache present, post-
#000027).
3. When the LLM emits a wrong value: `agreement_label =
"KERNEL-LLM-DIVERGED"` and the LLM's raw answer is preserved
in the witness dict for bench analysis.
4. Default `make query Q="0.1 + 0.2"` (no `WITNESS=1`) does NOT
call the LLM — preserves the ~10ms canonical fast-path
latency.
5. `tests/qa/test_witness.py` passes (8 cases listed in §3.6).
6. Existing tests unchanged.
7. Total wall time when witness=on ≈ LLM call + 50ms overhead
(parallel, not sequential).
8. Progress emitter emits witness.start / witness.modality.* /
witness.done lines.
---
## 7. Dependencies
- Builds on `arborist/pi_star/arithmetic.py` and `arborist/pi_star/
logic.py`.
- Reuses `arborist/qa/client.py` ChatClient + StubClient.
- Reuses progress emitter from 2026-05-08 state-machine wiring.
- Independent of #000027 (witness can ship before persistence;
cache-leg is a no-op closure until #000027 lands).

2
greatest-live-rock-and-roll-song-ever-played.md Normal file
View file

@ -0,0 +1,2 @@
:wq

354
tests/test_witness.py Normal file
View file

@ -0,0 +1,354 @@
"""Tests for the multi-modality witness (Ticket #000028).
Three modalities — kernel, cache, alien (LLM) — fan out in parallel
over a canonical-shape question. Every modality's answer is
canonicalized through the same kernel for byte-comparison. The
witness records cross-modality agreement as a label.
These tests exercise ``arborist.qa.witness`` directly with
:class:`StubClient`. Live-LLM coverage lives under the bench harness;
this file stays offline.
"""
from __future__ import annotations
import time
import pytest
from arborist.qa.client import StubClient
from arborist.qa.witness import (
Witness,
_canonicalize_via_kernel,
_classify_agreement,
run_witness,
)
# ----- _canonicalize_via_kernel ------------------------------------------
@pytest.mark.parametrize(
"text,ref,expected",
[
("3/10", "arithmetic@v1", b"3/10"),
("0.3", "arithmetic@v1", b"3/10"),
("0.1 + 0.2", "arithmetic@v1", b"3/10"),
(" 3/10 ", "arithmetic@v1", b"3/10"),
("TRUE", "logic-kernel@v1", b"TRUE"),
],
)
def test_canonicalize_idempotent(text, ref, expected):
assert _canonicalize_via_kernel(text, ref) == expected
@pytest.mark.parametrize(
"text,ref",
[
("the answer is 0.3", "arithmetic@v1"),
("approximately 0.3", "arithmetic@v1"),
("", "arithmetic@v1"),
(None, "arithmetic@v1"),
],
)
def test_canonicalize_rejects_non_canonical(text, ref):
assert _canonicalize_via_kernel(text, ref) is None
# ----- _classify_agreement ------------------------------------------------
def _mk_modality(name, ok, bytes_=None, error=None):
from arborist.qa.witness import ModalityResult
return ModalityResult(
modality=name,
raw_answer=bytes_.decode() if bytes_ else None,
canonical_bytes=bytes_ if ok else None,
error=error if not ok else None,
elapsed_ms=0.0,
)
def test_classify_strict_witnessed():
mods = {
"kernel": _mk_modality("kernel", True, b"3/10"),
"cache": _mk_modality("cache", True, b"3/10"),
"llm": _mk_modality("llm", True, b"3/10"),
}
assert _classify_agreement(mods, b"3/10") == "STRICT-WITNESSED"
def test_classify_llm_diverged():
mods = {
"kernel": _mk_modality("kernel", True, b"3/10"),
"cache": _mk_modality("cache", True, b"3/10"),
"llm": _mk_modality("llm", True, b"2/5"),
}
assert _classify_agreement(mods, b"3/10") == "LLM-DIVERGED"
def test_classify_cache_drift():
mods = {
"kernel": _mk_modality("kernel", True, b"3/10"),
"cache": _mk_modality("cache", True, b"4/10"),
"llm": _mk_modality("llm", True, b"3/10"),
}
assert _classify_agreement(mods, b"3/10") == "CACHE-DRIFT"
def test_classify_kernel_llm_agree_no_cache():
mods = {
"kernel": _mk_modality("kernel", True, b"3/10"),
"cache": _mk_modality("cache", False, error="ABSENT"),
"llm": _mk_modality("llm", True, b"3/10"),
}
assert _classify_agreement(mods, b"3/10") == "KERNEL-LLM-AGREE"
def test_classify_kernel_llm_diverged_no_cache():
mods = {
"kernel": _mk_modality("kernel", True, b"3/10"),
"cache": _mk_modality("cache", False, error="ABSENT"),
"llm": _mk_modality("llm", True, b"5/10"),
}
assert _classify_agreement(mods, b"3/10") == "KERNEL-LLM-DIVERGED"
def test_classify_kernel_cache_agree_llm_absent():
mods = {
"kernel": _mk_modality("kernel", True, b"3/10"),
"cache": _mk_modality("cache", True, b"3/10"),
"llm": _mk_modality("llm", False, error="PIS_REJECT"),
}
assert _classify_agreement(mods, b"3/10") == "KERNEL-CACHE-AGREE"
def test_classify_kernel_only():
mods = {
"kernel": _mk_modality("kernel", True, b"3/10"),
"cache": _mk_modality("cache", False, error="ABSENT"),
"llm": _mk_modality("llm", False, error="ABSENT"),
}
assert _classify_agreement(mods, b"3/10") == "KERNEL-ONLY"
# ----- run_witness end-to-end --------------------------------------------
def test_witness_kernel_llm_agree():
"""LLM emits the same canonical bytes — no cache."""
client = StubClient(answer="3/10")
w = run_witness(
question="0.1 + 0.2",
pi_star_ref="arithmetic@v1",
canonical_answer_bytes=b"3/10",
chat_client=client,
model_id="stub",
)
assert isinstance(w, Witness)
assert w.agreement_label == "KERNEL-LLM-AGREE"
assert w.modalities["kernel"].ok is True
assert w.modalities["cache"].error == "ABSENT"
assert w.modalities["llm"].ok is True
assert w.modalities["llm"].canonical_bytes == b"3/10"
assert w.modalities["llm"].raw_answer == "3/10"
def test_witness_llm_lexically_different_but_canonically_same():
"""LLM says '0.3'; kernel canonicalizes to '3/10' — agreement."""
client = StubClient(answer="0.3")
w = run_witness(
question="0.1 + 0.2",
pi_star_ref="arithmetic@v1",
canonical_answer_bytes=b"3/10",
chat_client=client,
model_id="stub",
)
assert w.agreement_label == "KERNEL-LLM-AGREE"
assert w.modalities["llm"].canonical_bytes == b"3/10"
def test_witness_llm_diverged():
"""LLM emits a wrong canonical-shape answer — divergence recorded."""
client = StubClient(answer="0.4")
w = run_witness(
question="0.1 + 0.2",
pi_star_ref="arithmetic@v1",
canonical_answer_bytes=b"3/10",
chat_client=client,
model_id="stub",
)
assert w.agreement_label == "KERNEL-LLM-DIVERGED"
assert w.modalities["llm"].canonical_bytes == b"2/5" # 0.4 → 2/5
assert w.modalities["llm"].raw_answer == "0.4"
def test_witness_llm_not_canonical_text():
"""LLM emits prose; kernel rejects → LLM modality absent."""
client = StubClient(answer="the answer is 0.3")
w = run_witness(
question="0.1 + 0.2",
pi_star_ref="arithmetic@v1",
canonical_answer_bytes=b"3/10",
chat_client=client,
model_id="stub",
)
# No cache, LLM rejected — kernel-only result.
assert w.agreement_label == "KERNEL-ONLY"
assert w.modalities["llm"].ok is False
assert w.modalities["llm"].error == "PIS_REJECT"
assert w.modalities["llm"].raw_answer == "the answer is 0.3"
def test_witness_cache_present_and_agrees():
"""Cache closure returns the canonical bytes — STRICT-WITNESSED."""
client = StubClient(answer="3/10")
w = run_witness(
question="0.1 + 0.2",
pi_star_ref="arithmetic@v1",
canonical_answer_bytes=b"3/10",
cache_lookup=lambda: b"3/10",
chat_client=client,
model_id="stub",
)
assert w.agreement_label == "STRICT-WITNESSED"
assert w.modalities["cache"].ok is True
assert w.modalities["cache"].canonical_bytes == b"3/10"
def test_witness_cache_drift_caught():
"""Cache returns stale bytes; kernel + LLM agree → CACHE-DRIFT."""
client = StubClient(answer="3/10")
w = run_witness(
question="0.1 + 0.2",
pi_star_ref="arithmetic@v1",
canonical_answer_bytes=b"3/10",
cache_lookup=lambda: b"4/10", # someone bumped the kernel without invalidating
chat_client=client,
model_id="stub",
)
assert w.agreement_label == "CACHE-DRIFT"
assert w.modalities["cache"].canonical_bytes == b"2/5" # 4/10 → 2/5
def test_witness_no_chat_client_kernel_only():
"""No chat client provided — LLM modality reports ABSENT."""
w = run_witness(
question="0.1 + 0.2",
pi_star_ref="arithmetic@v1",
canonical_answer_bytes=b"3/10",
chat_client=None,
model_id="",
)
assert w.agreement_label == "KERNEL-ONLY"
assert w.modalities["llm"].error == "ABSENT"
def test_witness_llm_raises_recorded_as_error():
"""chat_completion raises — modality records LLM_ERROR, not crash."""
class RaisingClient:
def chat_completion(self, *a, **kw):
raise RuntimeError("connection refused")
w = run_witness(
question="0.1 + 0.2",
pi_star_ref="arithmetic@v1",
canonical_answer_bytes=b"3/10",
chat_client=RaisingClient(),
model_id="stub",
)
assert w.agreement_label == "KERNEL-ONLY"
assert w.modalities["llm"].error == "LLM_ERROR:RuntimeError"
def test_witness_to_dict_round_trip():
"""to_dict() emits a JSON-serializable shape."""
import json
client = StubClient(answer="3/10")
w = run_witness(
question="0.1 + 0.2",
pi_star_ref="arithmetic@v1",
canonical_answer_bytes=b"3/10",
chat_client=client,
model_id="stub",
)
d = w.to_dict()
blob = json.dumps(d)
assert "STRICT" in blob or "AGREE" in blob
assert d["pi_star_ref"] == "arithmetic@v1"
assert d["canonical_answer_text"] == "3/10"
assert "kernel" in d["modalities"]
assert "llm" in d["modalities"]
def test_witness_parallel_not_sequential():
"""Total wall time ≤ slowest modality + overhead, not sum."""
class SlowClient:
def chat_completion(self, *a, **kw):
time.sleep(0.30)
return "3/10"
def slow_cache():
time.sleep(0.30)
return None
t0 = time.monotonic()
w = run_witness(
question="0.1 + 0.2",
pi_star_ref="arithmetic@v1",
canonical_answer_bytes=b"3/10",
cache_lookup=slow_cache,
chat_client=SlowClient(),
model_id="stub",
)
elapsed = time.monotonic() - t0
# Sequential would be ~0.60s; parallel should be ~0.30s + small overhead.
# Allow generous slack to avoid CI flake.
assert elapsed < 0.55, f"witness ran sequentially? wall={elapsed:.3f}s"
assert w.agreement_label == "KERNEL-LLM-AGREE"
# ----- query() integration ------------------------------------------------
def test_query_canonical_path_off_by_default(tmp_path):
"""Default policy: canonical_witness_enabled is False; no LLM call."""
from arborist.qa.query import query
client = StubClient(answer="<should-not-be-called>")
result = query(
question="0.1 + 0.2",
qa_db=tmp_path / "qa.db",
chat_client=client,
model_id="stub",
)
assert result["status"] == "canonical_projection"
assert result["audit_mode"] == "CANONICAL_PROJECTION"
assert result.get("witness") is None
# Critical: no LLM round-trip happened.
assert client.calls == []
def test_query_canonical_with_witness_calls_llm(tmp_path):
"""policy['canonical_witness_enabled']=True → witness runs."""
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
client = StubClient(answer="3/10")
policy = dict(DEFAULT_QUERY_POLICY)
policy["canonical_witness_enabled"] = True
result = query(
question="0.1 + 0.2",
qa_db=tmp_path / "qa.db",
chat_client=client,
model_id="stub",
policy=policy,
)
assert result["status"] == "canonical_projection"
witness = result.get("witness")
assert witness is not None
assert witness["agreement_label"] == "KERNEL-LLM-AGREE"
# The LLM was actually called.
assert len(client.calls) == 1