diff --git a/.gitignore b/.gitignore index 1b38079..be30f16 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ __pycache__/ .pytest_cache/ .coverage *.swp +_build/ # data + caches stay out of git data/ diff --git a/Makefile b/Makefile index 76b170d..315ac95 100644 --- a/Makefile +++ b/Makefile @@ -161,11 +161,11 @@ ANSWER_MODE ?= claim_lattice # REJECT_BROAD=1 → strict reject for ALL/COMPREHENSIVE/OPEN_REQUEST # unbounded shapes; returns UNGROUNDED before the LLM call. # ALLOW_BROAD=1 → emergent search; classifier on, caps off. -query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K="extra retrieval keywords" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]; JSON by default +query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K="extra retrieval keywords" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1]; JSON by default @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ - echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]"; exit 2; \ + echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1]"; exit 2; \ fi - $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) "$(Q)" + $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) $(if $(WITNESS),--witness,) "$(Q)" query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=... BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1] @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ diff --git a/arborist/cli.py b/arborist/cli.py index fff98a4..6763784 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -438,6 +438,12 @@ def _cmd_query(args: argparse.Namespace) -> int: # sidecar. Adds one short LLM round-trip; NEVER gates # admissibility (D1 preserved). call_policy["soft_preflight_enabled"] = True + # Ticket #000028 — multi-modality witness override. CLI > policy. + _witness_override = getattr(args, "witness_override", None) + if _witness_override == "on": + call_policy["canonical_witness_enabled"] = True + elif _witness_override == "off": + call_policy["canonical_witness_enabled"] = False if getattr(args, "no_canonical_preflight", False): # Disable the math/logic π* short-circuit per-call. Forces # RAG even on pure-arithmetic / pure-propositional input — @@ -834,10 +840,52 @@ def _render_query_human(result: dict, question: str) -> str: if isinstance(total_ms, (int, float)) else "?" ) + # Ticket #000028 — render-layer witness tail. When witness ran, + # surface the agreement label + per-modality status. Pure + # render-only; cache_key / governance_policy_hash unaffected. + witness = result.get("witness") + witness_tail = "" + witness_block = "" + if witness: + label = witness.get("agreement_label", "?") + mods = witness.get("modalities") or {} + ground_truth_hex = witness.get("canonical_answer_bytes_hex") or "" + agree_count = sum( + 1 for name, m in mods.items() + if name != "kernel" + and m.get("ok") + and (m.get("canonical_bytes_hex") or "") == ground_truth_hex + ) + checkable = sum( + 1 for name, m in mods.items() + if name != "kernel" and m.get("error") != "ABSENT" + ) + witness_tail = f" [{label} · {agree_count}/{checkable} modalities agree]" + lines = ["", "witness:"] + for name in ("kernel", "cache", "llm"): + m = mods.get(name) + if not m: + continue + raw = m.get("raw_answer") + err = m.get("error") + ms = int(m.get("elapsed_ms") or 0) + if err == "ABSENT": + detail = "absent" + elif err == "PIS_REJECT": + detail = f'rejected: "{raw}"' + elif err == "TIMEOUT": + detail = "timeout" + elif err and err.startswith("LLM_ERROR"): + detail = err.lower() + else: + detail = f'"{raw}"' + lines.append(f" {name:<7} {detail} ({ms}ms)") + witness_block = "\n".join(lines) return ( f"{question}\n" - f" CANONICAL · via {pi_star_ref} {elapsed} (projected)\n\n" + f" CANONICAL · via {pi_star_ref}{witness_tail} {elapsed} (projected)\n\n" f"{answer_text}" + f"{witness_block}" ) if status not in ("cache_hit", "cache_miss_then_written"): msg = result.get("msg") or status or "unknown error" @@ -4230,6 +4278,25 @@ def build_parser() -> argparse.ArgumentParser: "comparing the canonical answer with the model's reply)." ), ) + witness_group = query_cmd.add_mutually_exclusive_group() + witness_group.add_argument( + "--witness", dest="witness_override", action="store_const", + const="on", + help=( + "Ticket #000028 — multi-modality witness. On canonical-shape " + "questions (arithmetic@v1, logic-kernel@v1, future kernels), " + "fan out kernel + cache + LLM in parallel and verify cross-" + "modality agreement via kernel-canonicalization. Adds one LLM " + "call (~2-5s) on top of the canonical fast path; default OFF " + "to keep arithmetic queries at ~10ms." + ), + ) + witness_group.add_argument( + "--no-witness", dest="witness_override", action="store_const", + const="off", + help="Disable the witness path even when policy enables it.", + ) + progress_group = query_cmd.add_mutually_exclusive_group() progress_group.add_argument( "--progress", dest="progress_override", action="store_const", @@ -4250,7 +4317,9 @@ def build_parser() -> argparse.ArgumentParser: "tools that misbehave on stderr noise." ), ) - query_cmd.set_defaults(func=_cmd_query, progress_override=None) + query_cmd.set_defaults( + func=_cmd_query, progress_override=None, witness_override=None, + ) inspect_cmd = sub.add_parser( "inspect", diff --git a/arborist/qa/progress.py b/arborist/qa/progress.py new file mode 100644 index 0000000..d87c954 --- /dev/null +++ b/arborist/qa/progress.py @@ -0,0 +1,85 @@ +"""Stage-level state-machine emitter for `arborist query`. + +Writes one line per state transition to stderr. Stdout stays clean +(JSON output, render output, unfirehose journal) so this is safe to +enable in scripts and pipelines as well as interactively. + +Format:: + + [arborist NN.NNs] stage.name key=value key=value + +Elapsed is wall-clock seconds since the Progress object was constructed. +The gap *between* lines is the per-stage cost — that's what makes the +slow shard or slow route visible. + +Default-on at TTY, default-off when stderr is piped. Override via +``ARBORIST_PROGRESS=1`` (force on) / ``=0`` (force off), or per-call +via ``cli_override="on"|"off"``. +""" + +from __future__ import annotations + +import os +import sys +import time +from typing import Any + + +class Progress: + """Disabled-mode is a true no-op — no string formatting cost.""" + + __slots__ = ("enabled", "t_start", "stream") + + def __init__( + self, + *, + enabled: bool, + t_start: float | None = None, + stream: Any = None, + ) -> None: + self.enabled = enabled + self.t_start = t_start if t_start is not None else time.monotonic() + self.stream = stream if stream is not None else sys.stderr + + def emit(self, stage: str, **kv: Any) -> None: + if not self.enabled: + return + elapsed = time.monotonic() - self.t_start + prefix = f"[arborist {elapsed:6.2f}s] {stage}" + if kv: + tail = " ".join(f"{k}={v}" for k, v in kv.items()) + line = f"{prefix} {tail}" + else: + line = prefix + print(line, file=self.stream, flush=True) + + +_DISABLED = Progress(enabled=False) + + +def disabled() -> Progress: + """Singleton no-op emitter, suitable as a default kwarg value.""" + return _DISABLED + + +def from_env(*, cli_override: str | None = None) -> Progress: + """Resolve enable-state and return a fresh emitter. + + Precedence: cli_override > ARBORIST_PROGRESS env > TTY auto-detect. + """ + if cli_override == "on": + enabled = True + elif cli_override == "off": + enabled = False + else: + env = (os.environ.get("ARBORIST_PROGRESS") or "").strip().lower() + if env in ("0", "false", "off", "no"): + enabled = False + elif env in ("1", "true", "on", "yes"): + enabled = True + else: + try: + enabled = sys.stderr.isatty() + except (AttributeError, ValueError): + enabled = False + return Progress(enabled=enabled) diff --git a/arborist/qa/query.py b/arborist/qa/query.py index f791520..24e8c20 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -2008,6 +2008,31 @@ def query( pi_star_ref=pi_star_ref, ms=int(canonical_ms), ) + # Ticket #000028 — multi-modality witness. Default OFF; + # operator opts in via policy["canonical_witness_enabled"] + # (CLI: --witness). Fans out cache + LLM in parallel against + # the kernel ground truth and records cross-modality + # agreement. Cache leg is a no-op closure pre-Ticket #000027. + witness_dict = None + if bool(policy.get("canonical_witness_enabled", False)): + from arborist.qa.witness import run_witness + # Pre-#000027: no canonical persistence, cache always + # misses. The closure shape stays so #000027 can wire + # the real lookup without changing this call site. + _cache_lookup = lambda: None # noqa: E731 + _witness = run_witness( + question=question, + pi_star_ref=pi_star_ref, + canonical_answer_bytes=canonical_bytes, + cache_lookup=_cache_lookup, + chat_client=chat_client, + model_id=model_id, + timeout_s=float( + policy.get("canonical_witness_timeout_s", 10.0) + ), + progress=progress, + ) + witness_dict = _witness.to_dict() return { "status": "canonical_projection", "audit_mode": "CANONICAL_PROJECTION", @@ -2020,6 +2045,7 @@ def query( "lookup_path": "preflight_canonical", "sources": [], "cache_key": None, + "witness": witness_dict, "timings": { "canonical_preflight_ms": canonical_ms, "total_ms": _ms_since(t_start), diff --git a/arborist/qa/witness.py b/arborist/qa/witness.py new file mode 100644 index 0000000..3b14f1d --- /dev/null +++ b/arborist/qa/witness.py @@ -0,0 +1,394 @@ +"""Multi-modality witness for canonical-shape questions (Ticket #000028). + +When a question matches a canonical shape (arithmetic@v1, +logic-kernel@v1, future kernels), fan out three independent answer +paths in parallel: + +- **kernel**: deterministic π* — ground truth. +- **cache**: prior commitment from ``providence_cache`` (closure + returns ``None`` when no prior row, e.g. pre-Ticket #000027). +- **alien (LLM)**: an independently-trained model invited to answer + the same question. + +Each modality's answer is canonicalized through the kernel for +byte-comparison. Three paths byte-equal → ``STRICT-WITNESSED`` — +the strongest grounding signal arborist can offer (mono-modal × +independent-training × prior-commitment). Disagreement is recorded +as falsification signal, not an error. + +Default OFF — tripling LLM cost on every arithmetic question is a +tax operators don't want by default. Enable per-call via +``--witness`` CLI flag or per-policy via +``policy["canonical_witness_enabled"] = True``. +""" + +from __future__ import annotations + +import time +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutTimeoutError +from dataclasses import dataclass, field +from typing import Callable + +from arborist.qa.client import ChatClient + + +WITNESS_SYSTEM_PROMPT = ( + "You are a deterministic math/logic computer. Reply with ONLY the " + "canonical answer — no prose, no explanation, no LaTeX, no leading " + '"=", no trailing punctuation. For arithmetic emit a fraction (e.g. ' + "3/10) or integer. For propositional logic emit TRUE or FALSE. " + "Output nothing else." +) + + +@dataclass(frozen=True) +class ModalityResult: + """One modality's contribution to the witness.""" + + modality: str # "kernel" | "cache" | "llm" + raw_answer: str | None # what the modality said (display) + canonical_bytes: bytes | None # post-canonicalization bytes (None on reject) + error: str | None # "PIS_REJECT" / "TIMEOUT" / "ABSENT" / etc. + elapsed_ms: float + + @property + def ok(self) -> bool: + return self.canonical_bytes is not None and self.error is None + + def to_dict(self) -> dict: + return { + "modality": self.modality, + "raw_answer": self.raw_answer, + "canonical_bytes_hex": ( + self.canonical_bytes.hex() if self.canonical_bytes else None + ), + "error": self.error, + "elapsed_ms": round(self.elapsed_ms, 1), + "ok": self.ok, + } + + +@dataclass(frozen=True) +class Witness: + """Outcome of a multi-modality witness for one canonical-shape question.""" + + pi_star_ref: str + canonical_answer_bytes: bytes + modalities: dict[str, ModalityResult] = field(default_factory=dict) + agreement_label: str = "UNCLASSIFIED" + elapsed_ms: float = 0.0 + + def to_dict(self) -> dict: + return { + "pi_star_ref": self.pi_star_ref, + "canonical_answer_bytes_hex": self.canonical_answer_bytes.hex(), + "canonical_answer_text": self.canonical_answer_bytes.decode( + "utf-8", errors="replace" + ), + "modalities": { + name: m.to_dict() for name, m in self.modalities.items() + }, + "agreement_label": self.agreement_label, + "elapsed_ms": round(self.elapsed_ms, 1), + } + + +def _canonicalize_via_kernel(text: str | None, pi_star_ref: str) -> bytes | None: + """Run the named kernel on ``text``. Returns canonical bytes, or + ``None`` when the kernel rejects (PiStarError) or input is empty. + """ + from arborist.pi_star import PiStarError, get + if text is None: + return None + cleaned = text.strip() + if not cleaned: + return None + try: + return get(pi_star_ref).canonicalize(cleaned.encode("utf-8")) + except PiStarError: + return None + except Exception: # pragma: no cover — defensive + return None + + +def _classify_agreement( + modalities: dict[str, ModalityResult], + canonical_answer_bytes: bytes, +) -> str: + """Map (kernel, cache, llm) outcomes → agreement label. + + The kernel is the ground-truth reference; cache & llm are checked + *against* the kernel's canonical_answer_bytes. + """ + cache = modalities.get("cache") + llm = modalities.get("llm") + + cache_present = cache is not None and cache.ok + llm_present = llm is not None and llm.ok + + cache_agrees = ( + cache_present and cache.canonical_bytes == canonical_answer_bytes + ) + llm_agrees = ( + llm_present and llm.canonical_bytes == canonical_answer_bytes + ) + + cache_disagrees = cache_present and not cache_agrees + llm_disagrees = llm_present and not llm_agrees + + # Decision tree (kernel always present when this is called): + if cache_agrees and llm_agrees: + return "STRICT-WITNESSED" + if cache_agrees and llm_disagrees: + return "LLM-DIVERGED" + if cache_disagrees and llm_agrees: + return "CACHE-DRIFT" + if cache_disagrees and llm_disagrees: + return "LLM-AND-CACHE-DIVERGED" + if not cache_present and llm_agrees: + return "KERNEL-LLM-AGREE" + if not cache_present and llm_disagrees: + return "KERNEL-LLM-DIVERGED" + if cache_agrees and not llm_present: + return "KERNEL-CACHE-AGREE" + if cache_disagrees and not llm_present: + return "CACHE-DRIFT-LLM-ABSENT" + # Both cache and llm absent — only the kernel ran. Degenerate but + # surface it explicitly so the audit-line tail makes sense. + return "KERNEL-ONLY" + + +def _run_kernel_modality( + canonical_answer_bytes: bytes, pi_star_ref: str +) -> ModalityResult: + """Kernel modality: by definition produces canonical_answer_bytes. + + The work was already done before run_witness was called — we + receive the answer and just package it as a ModalityResult so + the matrix has a kernel row alongside cache + llm. + """ + return ModalityResult( + modality="kernel", + raw_answer=canonical_answer_bytes.decode("utf-8", errors="replace"), + canonical_bytes=canonical_answer_bytes, + error=None, + elapsed_ms=0.0, + ) + + +def _run_cache_modality( + cache_lookup: Callable[[], bytes | None] | None, + pi_star_ref: str, +) -> ModalityResult: + """Cache modality: invoke the lookup closure. ``None`` → ABSENT + (no prior commitment); bytes → canonicalize & report. + """ + t0 = time.monotonic() + if cache_lookup is None: + return ModalityResult( + modality="cache", + raw_answer=None, + canonical_bytes=None, + error="ABSENT", + elapsed_ms=0.0, + ) + try: + raw_bytes = cache_lookup() + except Exception as e: # pragma: no cover + return ModalityResult( + modality="cache", + raw_answer=None, + canonical_bytes=None, + error=f"LOOKUP_ERROR:{type(e).__name__}", + elapsed_ms=(time.monotonic() - t0) * 1000.0, + ) + elapsed = (time.monotonic() - t0) * 1000.0 + if raw_bytes is None: + return ModalityResult( + modality="cache", + raw_answer=None, + canonical_bytes=None, + error="ABSENT", + elapsed_ms=elapsed, + ) + raw_text = raw_bytes.decode("utf-8", errors="replace") + canonical = _canonicalize_via_kernel(raw_text, pi_star_ref) + if canonical is None: + return ModalityResult( + modality="cache", + raw_answer=raw_text, + canonical_bytes=None, + error="PIS_REJECT", + elapsed_ms=elapsed, + ) + return ModalityResult( + modality="cache", + raw_answer=raw_text, + canonical_bytes=canonical, + error=None, + elapsed_ms=elapsed, + ) + + +def _run_llm_modality( + question: str, + chat_client: ChatClient | None, + model_id: str, + pi_star_ref: str, +) -> ModalityResult: + """LLM modality: ask the alien to compute the answer with strict + output formatting; canonicalize the response through the kernel. + """ + t0 = time.monotonic() + if chat_client is None: + return ModalityResult( + modality="llm", + raw_answer=None, + canonical_bytes=None, + error="ABSENT", + elapsed_ms=0.0, + ) + messages = [ + {"role": "system", "content": WITNESS_SYSTEM_PROMPT}, + {"role": "user", "content": question}, + ] + try: + raw = chat_client.chat_completion( + messages, + model=model_id, + temperature=0.0, + max_tokens=64, + top_p=1.0, + ) + except Exception as e: + return ModalityResult( + modality="llm", + raw_answer=None, + canonical_bytes=None, + error=f"LLM_ERROR:{type(e).__name__}", + elapsed_ms=(time.monotonic() - t0) * 1000.0, + ) + elapsed = (time.monotonic() - t0) * 1000.0 + raw_text = (raw or "").strip() + canonical = _canonicalize_via_kernel(raw_text, pi_star_ref) + if canonical is None: + return ModalityResult( + modality="llm", + raw_answer=raw_text, + canonical_bytes=None, + error="PIS_REJECT", + elapsed_ms=elapsed, + ) + return ModalityResult( + modality="llm", + raw_answer=raw_text, + canonical_bytes=canonical, + error=None, + elapsed_ms=elapsed, + ) + + +def run_witness( + *, + question: str, + pi_star_ref: str, + canonical_answer_bytes: bytes, + cache_lookup: Callable[[], bytes | None] | None = None, + chat_client: ChatClient | None = None, + model_id: str = "", + timeout_s: float = 10.0, + progress=None, +) -> Witness: + """Fan out kernel + cache + LLM in parallel; return a Witness. + + The kernel result is already known (passed in as + ``canonical_answer_bytes``) — no work to do. Cache and LLM run in + parallel threads. Total wall time ≈ max(cache, LLM) + small + overhead, not sum. + + ``cache_lookup`` is a closure to keep this module independent of + any storage concerns. Pass ``None`` (default) when no cache is + available — modality reports ``ABSENT``. + + ``chat_client`` set to ``None`` skips the LLM modality entirely. + + ``timeout_s`` bounds the whole parallel stage. Modalities that + don't finish in time report ``TIMEOUT``. + """ + from arborist.qa.progress import disabled as _progress_disabled + progress = progress or _progress_disabled() + + t_start = time.monotonic() + progress.emit( + "witness.start", + parallel=2 + (1 if chat_client is not None else 0), + timeout_s=timeout_s, + ) + + kernel_mod = _run_kernel_modality(canonical_answer_bytes, pi_star_ref) + progress.emit( + "witness.modality.kernel", + ms=int(kernel_mod.elapsed_ms), + ok=str(kernel_mod.ok).lower(), + ) + + modalities: dict[str, ModalityResult] = {"kernel": kernel_mod} + + # Submit cache + llm to the pool in parallel. + with ThreadPoolExecutor(max_workers=2, thread_name_prefix="witness") as pool: + fut_cache = pool.submit( + _run_cache_modality, cache_lookup, pi_star_ref + ) + fut_llm = pool.submit( + _run_llm_modality, question, chat_client, model_id, pi_star_ref + ) + + try: + cache_mod = fut_cache.result(timeout=timeout_s) + except FutTimeoutError: + cache_mod = ModalityResult( + modality="cache", + raw_answer=None, + canonical_bytes=None, + error="TIMEOUT", + elapsed_ms=timeout_s * 1000.0, + ) + fut_cache.cancel() + modalities["cache"] = cache_mod + progress.emit( + "witness.modality.cache", + ms=int(cache_mod.elapsed_ms), + ok=str(cache_mod.ok).lower(), + reason=(cache_mod.error or "ok"), + ) + + try: + llm_mod = fut_llm.result(timeout=timeout_s) + except FutTimeoutError: + llm_mod = ModalityResult( + modality="llm", + raw_answer=None, + canonical_bytes=None, + error="TIMEOUT", + elapsed_ms=timeout_s * 1000.0, + ) + fut_llm.cancel() + modalities["llm"] = llm_mod + progress.emit( + "witness.modality.llm", + ms=int(llm_mod.elapsed_ms), + ok=str(llm_mod.ok).lower(), + reason=(llm_mod.error or "ok"), + ) + + label = _classify_agreement(modalities, canonical_answer_bytes) + elapsed = (time.monotonic() - t_start) * 1000.0 + progress.emit("witness.done", label=label, ms=int(elapsed)) + + return Witness( + pi_star_ref=pi_star_ref, + canonical_answer_bytes=canonical_answer_bytes, + modalities=modalities, + agreement_label=label, + elapsed_ms=elapsed, + ) diff --git a/bench/results/real-shard-baseline.json b/bench/results/real-shard-baseline.json index aa38dc3..13f7aea 100644 --- a/bench/results/real-shard-baseline.json +++ b/bench/results/real-shard-baseline.json @@ -1,22 +1,23 @@ { "schema_version": "real-shard-baseline-v1", - "timestamp_utc": "2026-05-08T17:09:43+00:00", - "commit_sha": "e78814ca29ed4114572f2410b2615733d4b715f6", + "timestamp_utc": "2026-05-08T19:23:46+00:00", + "commit_sha": "c54bedab1c5502a55edf14ab179ce8d8a4dd0719", "git_dirty": true, "shards_dir": "/home/fox/.arborist/shards", - "shards_fingerprint": "d93bed2c48c7ac47b22286ac77508d5519fec2f3685d470966642eef4285cb5e", + "shards_fingerprint": "b2c76ce1b547aa6b09706d283646b53c347e1ecc1ad4d7d8691dc24bb0e2b4e9", "model": "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", "endpoint": "https://hermes.ai.unturf.com/v1", "burn": true, "fixtures_path": "bench/fixtures/real-shard-baseline-v1.jsonl", "summary": { "questions": 8, - "wall_ms_median": 4140.9, - "wall_ms_max": 7212.4, - "wall_ms_min": 0.6, + "wall_ms_median": 6420.3, + "wall_ms_max": 10910.7, + "wall_ms_min": 0.9, "by_audit_mode": { "STRICT": 4, - "HYBRID": 2, + "UNGROUNDED": 1, + "HYBRID": 1, "CANONICAL_PROJECTION": 2 }, "by_status": { @@ -36,10 +37,10 @@ "n_quotes": 1, "pi_star_ref": null, "lookup_path": "miss", - "wall_ms": 6309.6, - "total_ms": 6297.3, - "search_ms": 5002.6, - "llm_ms": 1213.1, + "wall_ms": 10910.7, + "total_ms": 10885.8, + "search_ms": 8277.3, + "llm_ms": 2479.7, "verify_ms": null, "canonical_preflight_ms": null, "sources_count": 8, @@ -57,10 +58,10 @@ "n_quotes": 1, "pi_star_ref": null, "lookup_path": "miss", - "wall_ms": 3867.2, - "total_ms": 3866.5, - "search_ms": 2663.1, - "llm_ms": 1123.0, + "wall_ms": 6526.1, + "total_ms": 6523.8, + "search_ms": 4464.9, + "llm_ms": 1985.1, "verify_ms": null, "canonical_preflight_ms": null, "sources_count": 5, @@ -78,10 +79,10 @@ "n_quotes": 1, "pi_star_ref": null, "lookup_path": "miss", - "wall_ms": 7212.4, - "total_ms": 7211.4, - "search_ms": 1485.5, - "llm_ms": 5644.9, + "wall_ms": 8921.9, + "total_ms": 8919.4, + "search_ms": 2475.5, + "llm_ms": 6356.5, "verify_ms": null, "canonical_preflight_ms": null, "sources_count": 4, @@ -99,36 +100,36 @@ "n_quotes": 1, "pi_star_ref": null, "lookup_path": "miss", - "wall_ms": 4140.9, - "total_ms": 4140.0, - "search_ms": 2088.9, - "llm_ms": 1989.7, + "wall_ms": 4794.5, + "total_ms": 4791.5, + "search_ms": 3219.7, + "llm_ms": 1463.0, "verify_ms": null, "canonical_preflight_ms": null, "sources_count": 8, "primary_source_used": false, - "answer_text": "The Linux kernel was written in the C programming language. From the source:\n\"The Linux kernel is an operating system kernel used by the Linux family of Unix-like operating systems. It is one of the most prominent examples of free and open ", + "answer_text": "The Linux kernel was written in the C programming language. The source states: \"The Linux kernel is an operating system kernel used by the Linux family of Unix-like operating systems. It is one of the most prominent examples of free and ope", "expected_routing": "wiki_primary", "note": "specific factual claim; well-grounded" }, { "question": "Who founded Microsoft?", "status": "cache_miss_then_written", - "audit_mode": "HYBRID", - "verifier_method": "entity", - "n_verified": 4, - "n_quotes": 4, + "audit_mode": "UNGROUNDED", + "verifier_method": "quote", + "n_verified": 0, + "n_quotes": 1, "pi_star_ref": null, "lookup_path": "miss", - "wall_ms": 2248.7, - "total_ms": 2247.6, - "search_ms": 1134.6, - "llm_ms": 1070.8, + "wall_ms": 3586.2, + "total_ms": 3584.1, + "search_ms": 1893.4, + "llm_ms": 1624.1, "verify_ms": null, "canonical_preflight_ms": null, "sources_count": 5, "primary_source_used": true, - "answer_text": "Based on the provided sources, I don't have enough information to determine who founded Microsoft. The sources mention Microsoft Research, Microsoft TechNet, Microsoft Hardware, and Microsoft Dynamics NAV, but do not contain any verbatim qu", + "answer_text": "Based on the provided sources, Microsoft was founded by Bill Gates and Paul Allen. The source states: \"Microsoft was founded by Bill Gates and Paul Allen in 1975\" (Source: https://en.wikipedia.org/wiki/Microsoft).", "expected_routing": "wiki_primary", "note": "named-entity authorship" }, @@ -141,10 +142,10 @@ "n_quotes": 5, "pi_star_ref": null, "lookup_path": "miss", - "wall_ms": 4327.0, - "total_ms": 4326.3, - "search_ms": 1636.2, - "llm_ms": 2623.3, + "wall_ms": 6420.3, + "total_ms": 6417.2, + "search_ms": 2720.9, + "llm_ms": 3533.2, "verify_ms": null, "canonical_preflight_ms": null, "sources_count": 8, @@ -162,12 +163,12 @@ "n_quotes": 1, "pi_star_ref": "arithmetic@v1", "lookup_path": "preflight_canonical", - "wall_ms": 0.9, - "total_ms": 0.3, + "wall_ms": 1.3, + "total_ms": 0.4, "search_ms": null, "llm_ms": null, "verify_ms": null, - "canonical_preflight_ms": 0.3, + "canonical_preflight_ms": 0.4, "sources_count": 0, "primary_source_used": false, "answer_text": "3/10", @@ -183,12 +184,12 @@ "n_quotes": 1, "pi_star_ref": "logic-kernel@v1", "lookup_path": "preflight_canonical", - "wall_ms": 0.6, - "total_ms": 0.1, + "wall_ms": 0.9, + "total_ms": 0.3, "search_ms": null, "llm_ms": null, "verify_ms": null, - "canonical_preflight_ms": 0.1, + "canonical_preflight_ms": 0.3, "sources_count": 0, "primary_source_used": false, "answer_text": "(NOT A OR B)", diff --git a/bench/results/real-shard-baseline.md b/bench/results/real-shard-baseline.md index c55139c..5b29178 100644 --- a/bench/results/real-shard-baseline.md +++ b/bench/results/real-shard-baseline.md @@ -1,18 +1,18 @@ -# Real-shard baseline — 2026-05-08T17:09:43+00:00 +# Real-shard baseline — 2026-05-08T19:23:46+00:00 -**Commit:** `e78814ca29ed` (dirty) -**Shards fingerprint:** `d93bed2c48c7ac47…` +**Commit:** `c54bedab1c55` (dirty) +**Shards fingerprint:** `b2c76ce1b547aa6b…` **Shards directory:** `/home/fox/.arborist/shards` ## Summary - Questions: **8** -- Wall median: **4140.9 ms** -- Wall range: 0.6 – 7212.4 ms +- Wall median: **6420.3 ms** +- Wall range: 0.9 – 10910.7 ms - Primary source used: 4/8 - Canonical-projection short-circuits: 2 -**Audit-mode distribution:** `CANONICAL_PROJECTION` 2, `HYBRID` 2, `STRICT` 4 +**Audit-mode distribution:** `CANONICAL_PROJECTION` 2, `HYBRID` 1, `STRICT` 4, `UNGROUNDED` 1 **Status distribution:** `cache_miss_then_written` 6, `canonical_projection` 2 @@ -20,11 +20,11 @@ | # | Question | Audit | Status | Wall (ms) | Method | |---|----------|-------|--------|-----------|--------| -| 1 | who wrote virt-back? | STRICT | cache_miss_then_written | 6309.6 | quote | -| 2 | What is the capital of France? | STRICT | cache_miss_then_written | 3867.2 | quote | -| 3 | What is Mac OS X? | STRICT | cache_miss_then_written | 7212.4 | quote | -| 4 | What programming language was the Linux kernel written in? | STRICT | cache_miss_then_written | 4140.9 | quote | -| 5 | Who founded Microsoft? | HYBRID | cache_miss_then_written | 2248.7 | entity | -| 6 | What is the relationship between AMD and Intel? | HYBRID | cache_miss_then_written | 4327.0 | paraphrase | -| 7 | 0.1 + 0.2 | CANONICAL_PROJECTION | canonical_projection | 0.9 | canonical_projection | -| 8 | A IMPL B | CANONICAL_PROJECTION | canonical_projection | 0.6 | canonical_projection | +| 1 | who wrote virt-back? | STRICT | cache_miss_then_written | 10910.7 | quote | +| 2 | What is the capital of France? | STRICT | cache_miss_then_written | 6526.1 | quote | +| 3 | What is Mac OS X? | STRICT | cache_miss_then_written | 8921.9 | quote | +| 4 | What programming language was the Linux kernel written in? | STRICT | cache_miss_then_written | 4794.5 | quote | +| 5 | Who founded Microsoft? | UNGROUNDED | cache_miss_then_written | 3586.2 | quote | +| 6 | What is the relationship between AMD and Intel? | HYBRID | cache_miss_then_written | 6420.3 | paraphrase | +| 7 | 0.1 + 0.2 | CANONICAL_PROJECTION | canonical_projection | 1.3 | canonical_projection | +| 8 | A IMPL B | CANONICAL_PROJECTION | canonical_projection | 0.9 | canonical_projection | diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 677fd52..5e711de 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -61,6 +61,8 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| +| #000028 | Multi-modality witness for canonical shapes | open · implementation in progress | 2026-05-08 | — | +| #000027 | Canonical projections persist to providence_cache | open · design phase | 2026-05-08 | — | | #000026 | Real-shard workload baseline + search latency | in progress · Phase 1 + 2 landed 2026-05-08 | 2026-05-08 | — | | #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a landed 2026-05-08 | 2026-05-07 | — | | #000024 | 5T Phase 1b + Dav1DPrometheus vocabulary alignment | closed · landed 2026-05-08 | 2026-05-07 | — | @@ -90,4 +92,4 @@ Newest first. Update on every open/close. ## Next ID -`000027` +`000029` diff --git a/docs/tickets/ticket-000028-multi-modality-witness.md b/docs/tickets/ticket-000028-multi-modality-witness.md new file mode 100644 index 0000000..317b1b3 --- /dev/null +++ b/docs/tickets/ticket-000028-multi-modality-witness.md @@ -0,0 +1,423 @@ +# Ticket #000028 — Multi-modality witness for canonical shapes + +**Status:** open · implementation in progress +**Opened:** 2026-05-08 +**Scope:** When a question matches a canonical shape (arithmetic@v1, +logic-kernel@v1, future kernels), fan out three independent answer +paths in parallel — **python kernel**, **cache lookup**, **alien +LLM** — and verify cross-modality agreement via kernel-canonicalization +of every answer. Disagreement is recorded as falsification signal, +not an error. Agreement is the strongest grounding signal arborist +can offer. +**Audience:** fox + future blackops shifts. +**Hard constraint:** opt-in (default off). Tripling latency on every +arithmetic question would tax operators who only want the kernel +answer. The witness path runs only when explicitly enabled per call +or per-policy. + +--- + +## 1. Problem statement + +Canonical projections (post-#000027) provide deterministic ground +truth for arithmetic and propositional logic. The LLM, in contrast, +is trained to produce plausible text and can hallucinate even on +trivial math (`0.1 + 0.2 = 0.30000000000000004` is famous LLM +training data; the kernel returns `3/10` exactly). Today we have no +systematic way to: + +- **Detect when the LLM disagrees with ground truth** for canonical + questions. A bench could measure this offline, but every live + call that asks a canonical-shape question is also a calibration + data point we throw away. +- **Catch cache drift.** If the kernel was bumped (`@v1` → `@v2`) + without invalidating prior rows, the cache silently serves stale + answers. A live cross-check at lookup time catches this. +- **Strengthen the warrant for canonical answers.** Today the + warrant is "kernel-deterministic" — strong, but mono-modal. + Three-path agreement is mono-modal × independent-training × prior- + commitment — strictly stronger. + +### 1.1 Architecture sketch + +``` +question + ├─ python kernel (arithmetic@v1) → "3/10" evidence: kernel-deterministic + ├─ cache lookup (providence_cache) → "3/10" evidence: prior commitment + └─ alien (LLM) (Hermes/Claude/etc) → "0.3" evidence: training-derived + │ + └─ claim-lattice cross-modality verifier: + canonicalize("0.3") via arithmetic@v1 → "3/10" + all three byte-equal → STRICT-CANONICAL-WITNESSED +``` + +### 1.2 Agreement matrix + +For three modalities {kernel (K), cache (C), llm (L)} the agreement +matrix has 7 distinct outcomes (kernel always present when this path +fires; cache may be absent pre-#000027; LLM may be absent if +canonicalization fails): + +| K | C | L | Outcome label | Meaning | +|---|---|---|---|---| +| ✓ | ✓ | ✓ | `STRICT-WITNESSED` | All three byte-equal — strongest possible warrant | +| ✓ | ✓ | ✗ | `LLM-DIVERGED` | Cache & kernel agree, LLM hallucinated — bench data | +| ✓ | ✗ | ✓ | `CACHE-DRIFT` | Kernel & LLM agree, cache row stale — invalidate row | +| ✓ | ✗ | ✗ | `LLM-AND-CACHE-DIVERGED` | Only kernel matches itself (degenerate) | +| ✓ | absent | ✓ | `KERNEL-LLM-AGREE` | No prior cache row; kernel ≡ LLM | +| ✓ | absent | ✗ | `KERNEL-LLM-DIVERGED` | No prior cache row; LLM disagrees with kernel | +| ✓ | ✓ | absent | `KERNEL-CACHE-AGREE` | LLM produced unparseable text; kernel ≡ cache | + +Kernel is always the ground truth. Cache and LLM are checked +*against* the kernel. The kernel never participates in byte-equal +comparison with itself — it's the reference. + +--- + +## 2. Design choices + +### 2.1 Parallelism + +LLM call is ~2-5s I/O-bound. Kernel + cache are ~10ms CPU/SQL. +Sequential = sum (slow); parallel = max (LLM only). + +**A. ThreadPoolExecutor (RECOMMENDED).** Three threads, one per +modality. The LLM call doesn't block kernel/cache. Total wall time +≈ LLM time + small overhead. Plays well with the existing sync +codebase (no asyncio refactor). + +**B. asyncio.** Cleaner conceptually but the existing codebase is +sync top-to-bottom. Pulling asyncio in for one call creates an +event-loop boundary the rest of the code doesn't know about. + +**C. Sequential.** Adds latency = kernel + cache + LLM ≈ LLM × 1.01. +Simpler. Loses the "free LLM hit" framing (you wait for it anyway). + +→ **A.** ~30 LOC delta. Standard library. Threads are I/O-bound so +the GIL doesn't block. + +### 2.2 LLM prompt design + +The LLM must emit a canonicalize-able answer for byte-comparison +to work. Two prompt strategies: + +**A. Strict-format (RECOMMENDED).** System prompt explicitly +constrains the output: + +``` +You are a deterministic math/logic computer. Reply with ONLY the +canonical answer — no prose, no explanation, no LaTeX, no leading +"=", no trailing punctuation. For arithmetic emit a fraction (e.g. +3/10) or integer. For propositional logic emit TRUE or FALSE. +``` + +If the LLM disobeys, the canonicalizer rejects the answer → +classified as `L=✗`. This itself is a valuable signal: the LLM +ignored instructions on a question with a known-correct format. + +**B. Extraction.** Accept arbitrary LLM output, run a regex/heuristic +extractor to pull a numeric portion, then canonicalize. More +permissive but introduces extraction errors as a confounder. + +→ **A.** Cleaner separation. If we want extraction later, layer it +on as a fallback when strict mode rejects. + +### 2.3 Canonicalization for cross-modality comparison + +The kernel canonicalizes each modality's answer to bytes: + +```python +kernel_bytes = arithmetic_v1.canonicalize(b"0.1 + 0.2") # "3/10" +llm_bytes = arithmetic_v1.canonicalize(b"0.3") # "3/10" +cache_bytes = arithmetic_v1.canonicalize(b"3/10") # "3/10" (idempotent) +agreement = (kernel_bytes == llm_bytes == cache_bytes) +``` + +**Idempotence is the key property.** The kernel is a deterministic +projection — running it on its own output yields the same bytes. +This is why we can compare across modalities by canonicalizing each +to the same canonical form. + +Failure modes: +- LLM produces `"the answer is 0.3"` → `_CANONICAL_ARITHMETIC_RE` + rejects (letters present) → `L=absent`. +- LLM produces `"0.3"` → canonicalize succeeds → `L=✓`. +- LLM produces `"0.4"` (wrong) → canonicalize succeeds → `L=✓` but + `llm_bytes != kernel_bytes` → `LLM-DIVERGED`. + +### 2.4 Default behavior + +**Default OFF.** Tripling LLM cost on every arithmetic question is a +tax operators don't want by default. Witness path enabled per-call +via `--witness` CLI flag or per-policy via +`policy["canonical_witness_enabled"] = True`. + +Sampling (e.g., 5% of canonical questions get the witness path for +opportunistic calibration) is **out of scope for this ticket**. +Easy to add later as `policy["canonical_witness_sample_rate"] = +0.05`. + +### 2.5 What gets persisted + +This ticket does **NOT** persist witness rows. Reasons: + +1. #000027 (canonical persistence) is not yet implemented. Until + then there's no canonical row to attach a witness to. +2. The witness data model needs to settle — the agreement matrix + may grow new outcomes as more kernels land. + +**For MVP**: witness result surfaces on the return dict + audit-line +render. When #000027 lands, witness data folds into the +`providence_canonical` audit event body as a `witness` clause. + +### 2.6 What `audit_mode` does the witnessed answer carry? + +The runtime `audit_mode` stays `"CANONICAL_PROJECTION"`. The witness +outcome is a render-layer label tail: + +``` +0.1 + 0.2 + CANONICAL · via arithmetic@v1 [STRICT-WITNESSED · 3/3 modalities agree] 0.0s + +3/10 +``` + +vs. divergence: + +``` +2 + 2 + CANONICAL · via arithmetic@v1 [LLM-DIVERGED · llm said "5"] 3.2s + +4 +``` + +The CLI render adds a witness tail; cache_key + governance_policy_hash ++ programmatic callers see `audit_mode = "CANONICAL_PROJECTION"` +unchanged. **Same pattern as the four-rung ladder render layer.** + +--- + +## 3. Implementation sketch + +### 3.1 New module `arborist/qa/witness.py` (~200 LOC) + +```python +@dataclass(frozen=True) +class ModalityResult: + modality: str # "kernel" | "cache" | "llm" + raw_answer: str | None # what the modality said (display) + canonical_bytes: bytes | None # post-canonicalization bytes + error: str | None # "PIS_REJECT" / "TIMEOUT" / etc. + elapsed_ms: float + +@dataclass(frozen=True) +class Witness: + pi_star_ref: str + modalities: dict[str, ModalityResult] + agreement_label: str # "STRICT-WITNESSED" / "LLM-DIVERGED" / etc. + canonical_answer_bytes: bytes # the kernel's output (ground truth) + + +def run_witness( + *, + question: str, + pi_star_ref: str, + canonical_answer_bytes: bytes, # already-computed kernel output + cache_lookup: Callable[[], bytes | None], # closure that returns cached bytes or None + chat_client: ChatClient | None, + model_id: str, + timeout_s: float = 10.0, + progress: Progress | None = None, +) -> Witness: + """Fan out kernel + cache + LLM in parallel; return Witness.""" + ... + + +def _canonicalize_via_kernel(text: str, pi_star_ref: str) -> bytes | None: + """Run the named kernel on `text`. Returns canonical bytes, or None + if the kernel rejects (PiStarError / non-canonical shape).""" + ... + + +def _classify_agreement(modalities: dict[str, ModalityResult]) -> str: + """Map modality outcomes → STRICT-WITNESSED / LLM-DIVERGED / etc.""" + ... +``` + +### 3.2 Edit `arborist/qa/query.py` canonical branch (~40 LOC delta) + +```python +if canonical_match is not None: + pi_star_ref, canonical_bytes = canonical_match + answer_text = canonical_bytes.decode("utf-8", errors="replace") + + witness = None + if policy.get("canonical_witness_enabled", False): + from arborist.qa.witness import run_witness + # Cache lookup is a no-op closure pre-#000027 (always returns + # None). Post-#000027 it's the actual canonical_cache lookup. + def _cache_lookup() -> bytes | None: + return None # TODO: wire post-#000027 + witness = run_witness( + question=question, + pi_star_ref=pi_star_ref, + canonical_answer_bytes=canonical_bytes, + cache_lookup=_cache_lookup, + chat_client=chat_client, + model_id=model_id, + timeout_s=float(policy.get("canonical_witness_timeout_s", 10.0)), + progress=progress, + ) + + return { + "status": "canonical_projection", + "audit_mode": "CANONICAL_PROJECTION", + "verifier_method": "canonical_projection", + "pi_star_ref": pi_star_ref, + "answer_text": answer_text, + "witness": witness.to_dict() if witness else None, + ... + } +``` + +### 3.3 CLI surface + +```bash +make query Q="0.1 + 0.2" WITNESS=1 +.venv/bin/arborist query --witness "0.1 + 0.2" +``` + +Mutex group `--witness` / `--no-witness` on `arborist query`. +Make passthrough: `WITNESS=1` → `--witness`. + +### 3.4 Render layer + +`arborist/cli.py` audit-line render gains a witness tail when +`result.get("witness")` is set: + +``` +CANONICAL · via arithmetic@v1 [STRICT-WITNESSED · 3/3] 0.0s +``` + +Detail block under the answer (when `--json` not used) prints each +modality: + +``` +witness: + kernel: "3/10" (10ms) + cache: absent (no prior commitment) + llm: "3/10" (2841ms · agrees) +``` + +### 3.5 Progress emitter + +``` +[arborist 0.02s] canonical_projection.match pi_star_ref=arithmetic@v1 +[arborist 0.02s] witness.start parallel=3 timeout_s=10 +[arborist 0.03s] witness.modality.kernel ms=8 ok=true +[arborist 0.03s] witness.modality.cache ms=1 ok=false reason=absent +[arborist 2.84s] witness.modality.llm ms=2812 ok=true canonicalized=true +[arborist 2.84s] witness.done label=STRICT-WITNESSED +``` + +### 3.6 Tests `tests/qa/test_witness.py` (~120 LOC) + +- `test_three_way_agreement` — kernel + cache + LLM all return same canonical bytes → `STRICT-WITNESSED`. +- `test_llm_diverges` — kernel + cache agree, LLM returns different value → `LLM-DIVERGED`. +- `test_llm_not_canonical` — LLM emits prose; canonicalizer rejects → `KERNEL-CACHE-AGREE` (LLM absent). +- `test_cache_absent` — cache lookup returns None → `KERNEL-LLM-AGREE`. +- `test_cache_drift` — kernel + LLM agree, cache returns stale value → `CACHE-DRIFT`. +- `test_llm_timeout` — chat_client raises TimeoutError → LLM modality reports error, agreement label reflects absence. +- `test_default_off` — witness only runs when policy flag is True. +- `test_parallel_timing` — total wall time ≤ max(kernel, llm) + overhead, not sum. + +### 3.7 Stub LLM client for tests + +Existing `arborist.qa.client.StubClient(answer="...")` works. +`StubClient(answer="3/10")` for agreement; `StubClient(answer="0.4")` +for divergence; `StubClient(answer="the answer is 0.3")` for +not-canonical; `StubClient(raises=TimeoutError)` for timeout. + +--- + +## 4. Out of scope + +- **Persistence of witness rows.** Wires in via #000027 once that ticket + lands. For MVP, witness is runtime-only. +- **Sampling rate** for opportunistic calibration. Add as `policy + ["canonical_witness_sample_rate"] = 0.05` later if useful. +- **Falsification of LLM modality.** When `LLM-DIVERGED`, we record + the divergence on the result dict but don't mutate any LLM + reliability ledger. Future ticket: `LLMReliability` with + per-shape accuracy tracking. +- **Re-asking the LLM with feedback.** No retry / repair loop for + canonical shapes — the kernel is ground truth, the LLM gets one + shot. +- **Multi-LLM witness** (Hermes + Claude + GPT4 vote). Single LLM + for now. The "alien" leg is one independently-trained model. + Ensemble is a later ticket. +- **Cache-leg implementation.** Stub closure that returns None. + Real implementation lands with #000027. + +--- + +## 5. Risks and rollback + +**Risk 1 — LLM cost.** Witness adds one full LLM call per canonical +question. Default-off mitigates. Operators who flip `--witness` on +accept the cost knowingly. Sampling rate (out of scope) addresses +opportunistic-calibration use. + +**Risk 2 — LLM canonicalization rate.** If the LLM ignores the +strict-format prompt > 30% of the time, the witness is mostly +noise. Bench fixture in §3.6 includes adherence measurement; +fox decides the floor. + +**Risk 3 — Threading correctness.** ThreadPoolExecutor on three +short-lived threads is well-trodden territory; primary concern is +clean shutdown on timeout. `concurrent.futures.wait(timeout=...)` ++ `future.cancel()` handles it. + +**Risk 4 — Render-layer divergence.** The audit-line tail is render- +only. Cache_key, governance_policy_hash, and run_dag stay +byte-identical to non-witness canonical answers. Same render-layer +discipline as the four-rung ladder. + +**Rollback**: set `policy["canonical_witness_enabled"] = False` +(default). Witness module stays installed but never fires. +Pre-existing canonical answers unaffected. + +--- + +## 6. Acceptance criteria + +1. `make query Q="0.1 + 0.2" WITNESS=1` returns the same `3/10` + answer with a `witness` field on the result dict + a witness + tail on the audit-line render. +2. When the LLM emits a canonicalize-able answer that matches the + kernel: `agreement_label = "KERNEL-LLM-AGREE"` (cache absent + pre-#000027) or `"STRICT-WITNESSED"` (cache present, post- + #000027). +3. When the LLM emits a wrong value: `agreement_label = + "KERNEL-LLM-DIVERGED"` and the LLM's raw answer is preserved + in the witness dict for bench analysis. +4. Default `make query Q="0.1 + 0.2"` (no `WITNESS=1`) does NOT + call the LLM — preserves the ~10ms canonical fast-path + latency. +5. `tests/qa/test_witness.py` passes (8 cases listed in §3.6). +6. Existing tests unchanged. +7. Total wall time when witness=on ≈ LLM call + 50ms overhead + (parallel, not sequential). +8. Progress emitter emits witness.start / witness.modality.* / + witness.done lines. + +--- + +## 7. Dependencies + +- Builds on `arborist/pi_star/arithmetic.py` and `arborist/pi_star/ + logic.py`. +- Reuses `arborist/qa/client.py` ChatClient + StubClient. +- Reuses progress emitter from 2026-05-08 state-machine wiring. +- Independent of #000027 (witness can ship before persistence; + cache-leg is a no-op closure until #000027 lands). diff --git a/greatest-live-rock-and-roll-song-ever-played.md b/greatest-live-rock-and-roll-song-ever-played.md new file mode 100644 index 0000000..41e5b13 --- /dev/null +++ b/greatest-live-rock-and-roll-song-ever-played.md @@ -0,0 +1,2 @@ +:wq + diff --git a/tests/test_witness.py b/tests/test_witness.py new file mode 100644 index 0000000..10d0a30 --- /dev/null +++ b/tests/test_witness.py @@ -0,0 +1,354 @@ +"""Tests for the multi-modality witness (Ticket #000028). + +Three modalities — kernel, cache, alien (LLM) — fan out in parallel +over a canonical-shape question. Every modality's answer is +canonicalized through the same kernel for byte-comparison. The +witness records cross-modality agreement as a label. + +These tests exercise ``arborist.qa.witness`` directly with +:class:`StubClient`. Live-LLM coverage lives under the bench harness; +this file stays offline. +""" + +from __future__ import annotations + +import time + +import pytest + +from arborist.qa.client import StubClient +from arborist.qa.witness import ( + Witness, + _canonicalize_via_kernel, + _classify_agreement, + run_witness, +) + + +# ----- _canonicalize_via_kernel ------------------------------------------ + + +@pytest.mark.parametrize( + "text,ref,expected", + [ + ("3/10", "arithmetic@v1", b"3/10"), + ("0.3", "arithmetic@v1", b"3/10"), + ("0.1 + 0.2", "arithmetic@v1", b"3/10"), + (" 3/10 ", "arithmetic@v1", b"3/10"), + ("TRUE", "logic-kernel@v1", b"TRUE"), + ], +) +def test_canonicalize_idempotent(text, ref, expected): + assert _canonicalize_via_kernel(text, ref) == expected + + +@pytest.mark.parametrize( + "text,ref", + [ + ("the answer is 0.3", "arithmetic@v1"), + ("approximately 0.3", "arithmetic@v1"), + ("", "arithmetic@v1"), + (None, "arithmetic@v1"), + ], +) +def test_canonicalize_rejects_non_canonical(text, ref): + assert _canonicalize_via_kernel(text, ref) is None + + +# ----- _classify_agreement ------------------------------------------------ + + +def _mk_modality(name, ok, bytes_=None, error=None): + from arborist.qa.witness import ModalityResult + return ModalityResult( + modality=name, + raw_answer=bytes_.decode() if bytes_ else None, + canonical_bytes=bytes_ if ok else None, + error=error if not ok else None, + elapsed_ms=0.0, + ) + + +def test_classify_strict_witnessed(): + mods = { + "kernel": _mk_modality("kernel", True, b"3/10"), + "cache": _mk_modality("cache", True, b"3/10"), + "llm": _mk_modality("llm", True, b"3/10"), + } + assert _classify_agreement(mods, b"3/10") == "STRICT-WITNESSED" + + +def test_classify_llm_diverged(): + mods = { + "kernel": _mk_modality("kernel", True, b"3/10"), + "cache": _mk_modality("cache", True, b"3/10"), + "llm": _mk_modality("llm", True, b"2/5"), + } + assert _classify_agreement(mods, b"3/10") == "LLM-DIVERGED" + + +def test_classify_cache_drift(): + mods = { + "kernel": _mk_modality("kernel", True, b"3/10"), + "cache": _mk_modality("cache", True, b"4/10"), + "llm": _mk_modality("llm", True, b"3/10"), + } + assert _classify_agreement(mods, b"3/10") == "CACHE-DRIFT" + + +def test_classify_kernel_llm_agree_no_cache(): + mods = { + "kernel": _mk_modality("kernel", True, b"3/10"), + "cache": _mk_modality("cache", False, error="ABSENT"), + "llm": _mk_modality("llm", True, b"3/10"), + } + assert _classify_agreement(mods, b"3/10") == "KERNEL-LLM-AGREE" + + +def test_classify_kernel_llm_diverged_no_cache(): + mods = { + "kernel": _mk_modality("kernel", True, b"3/10"), + "cache": _mk_modality("cache", False, error="ABSENT"), + "llm": _mk_modality("llm", True, b"5/10"), + } + assert _classify_agreement(mods, b"3/10") == "KERNEL-LLM-DIVERGED" + + +def test_classify_kernel_cache_agree_llm_absent(): + mods = { + "kernel": _mk_modality("kernel", True, b"3/10"), + "cache": _mk_modality("cache", True, b"3/10"), + "llm": _mk_modality("llm", False, error="PIS_REJECT"), + } + assert _classify_agreement(mods, b"3/10") == "KERNEL-CACHE-AGREE" + + +def test_classify_kernel_only(): + mods = { + "kernel": _mk_modality("kernel", True, b"3/10"), + "cache": _mk_modality("cache", False, error="ABSENT"), + "llm": _mk_modality("llm", False, error="ABSENT"), + } + assert _classify_agreement(mods, b"3/10") == "KERNEL-ONLY" + + +# ----- run_witness end-to-end -------------------------------------------- + + +def test_witness_kernel_llm_agree(): + """LLM emits the same canonical bytes — no cache.""" + client = StubClient(answer="3/10") + w = run_witness( + question="0.1 + 0.2", + pi_star_ref="arithmetic@v1", + canonical_answer_bytes=b"3/10", + chat_client=client, + model_id="stub", + ) + assert isinstance(w, Witness) + assert w.agreement_label == "KERNEL-LLM-AGREE" + assert w.modalities["kernel"].ok is True + assert w.modalities["cache"].error == "ABSENT" + assert w.modalities["llm"].ok is True + assert w.modalities["llm"].canonical_bytes == b"3/10" + assert w.modalities["llm"].raw_answer == "3/10" + + +def test_witness_llm_lexically_different_but_canonically_same(): + """LLM says '0.3'; kernel canonicalizes to '3/10' — agreement.""" + client = StubClient(answer="0.3") + w = run_witness( + question="0.1 + 0.2", + pi_star_ref="arithmetic@v1", + canonical_answer_bytes=b"3/10", + chat_client=client, + model_id="stub", + ) + assert w.agreement_label == "KERNEL-LLM-AGREE" + assert w.modalities["llm"].canonical_bytes == b"3/10" + + +def test_witness_llm_diverged(): + """LLM emits a wrong canonical-shape answer — divergence recorded.""" + client = StubClient(answer="0.4") + w = run_witness( + question="0.1 + 0.2", + pi_star_ref="arithmetic@v1", + canonical_answer_bytes=b"3/10", + chat_client=client, + model_id="stub", + ) + assert w.agreement_label == "KERNEL-LLM-DIVERGED" + assert w.modalities["llm"].canonical_bytes == b"2/5" # 0.4 → 2/5 + assert w.modalities["llm"].raw_answer == "0.4" + + +def test_witness_llm_not_canonical_text(): + """LLM emits prose; kernel rejects → LLM modality absent.""" + client = StubClient(answer="the answer is 0.3") + w = run_witness( + question="0.1 + 0.2", + pi_star_ref="arithmetic@v1", + canonical_answer_bytes=b"3/10", + chat_client=client, + model_id="stub", + ) + # No cache, LLM rejected — kernel-only result. + assert w.agreement_label == "KERNEL-ONLY" + assert w.modalities["llm"].ok is False + assert w.modalities["llm"].error == "PIS_REJECT" + assert w.modalities["llm"].raw_answer == "the answer is 0.3" + + +def test_witness_cache_present_and_agrees(): + """Cache closure returns the canonical bytes — STRICT-WITNESSED.""" + client = StubClient(answer="3/10") + w = run_witness( + question="0.1 + 0.2", + pi_star_ref="arithmetic@v1", + canonical_answer_bytes=b"3/10", + cache_lookup=lambda: b"3/10", + chat_client=client, + model_id="stub", + ) + assert w.agreement_label == "STRICT-WITNESSED" + assert w.modalities["cache"].ok is True + assert w.modalities["cache"].canonical_bytes == b"3/10" + + +def test_witness_cache_drift_caught(): + """Cache returns stale bytes; kernel + LLM agree → CACHE-DRIFT.""" + client = StubClient(answer="3/10") + w = run_witness( + question="0.1 + 0.2", + pi_star_ref="arithmetic@v1", + canonical_answer_bytes=b"3/10", + cache_lookup=lambda: b"4/10", # someone bumped the kernel without invalidating + chat_client=client, + model_id="stub", + ) + assert w.agreement_label == "CACHE-DRIFT" + assert w.modalities["cache"].canonical_bytes == b"2/5" # 4/10 → 2/5 + + +def test_witness_no_chat_client_kernel_only(): + """No chat client provided — LLM modality reports ABSENT.""" + w = run_witness( + question="0.1 + 0.2", + pi_star_ref="arithmetic@v1", + canonical_answer_bytes=b"3/10", + chat_client=None, + model_id="", + ) + assert w.agreement_label == "KERNEL-ONLY" + assert w.modalities["llm"].error == "ABSENT" + + +def test_witness_llm_raises_recorded_as_error(): + """chat_completion raises — modality records LLM_ERROR, not crash.""" + + class RaisingClient: + def chat_completion(self, *a, **kw): + raise RuntimeError("connection refused") + + w = run_witness( + question="0.1 + 0.2", + pi_star_ref="arithmetic@v1", + canonical_answer_bytes=b"3/10", + chat_client=RaisingClient(), + model_id="stub", + ) + assert w.agreement_label == "KERNEL-ONLY" + assert w.modalities["llm"].error == "LLM_ERROR:RuntimeError" + + +def test_witness_to_dict_round_trip(): + """to_dict() emits a JSON-serializable shape.""" + import json + + client = StubClient(answer="3/10") + w = run_witness( + question="0.1 + 0.2", + pi_star_ref="arithmetic@v1", + canonical_answer_bytes=b"3/10", + chat_client=client, + model_id="stub", + ) + d = w.to_dict() + blob = json.dumps(d) + assert "STRICT" in blob or "AGREE" in blob + assert d["pi_star_ref"] == "arithmetic@v1" + assert d["canonical_answer_text"] == "3/10" + assert "kernel" in d["modalities"] + assert "llm" in d["modalities"] + + +def test_witness_parallel_not_sequential(): + """Total wall time ≤ slowest modality + overhead, not sum.""" + + class SlowClient: + def chat_completion(self, *a, **kw): + time.sleep(0.30) + return "3/10" + + def slow_cache(): + time.sleep(0.30) + return None + + t0 = time.monotonic() + w = run_witness( + question="0.1 + 0.2", + pi_star_ref="arithmetic@v1", + canonical_answer_bytes=b"3/10", + cache_lookup=slow_cache, + chat_client=SlowClient(), + model_id="stub", + ) + elapsed = time.monotonic() - t0 + # Sequential would be ~0.60s; parallel should be ~0.30s + small overhead. + # Allow generous slack to avoid CI flake. + assert elapsed < 0.55, f"witness ran sequentially? wall={elapsed:.3f}s" + assert w.agreement_label == "KERNEL-LLM-AGREE" + + +# ----- query() integration ------------------------------------------------ + + +def test_query_canonical_path_off_by_default(tmp_path): + """Default policy: canonical_witness_enabled is False; no LLM call.""" + from arborist.qa.query import query + + client = StubClient(answer="") + result = query( + question="0.1 + 0.2", + qa_db=tmp_path / "qa.db", + chat_client=client, + model_id="stub", + ) + assert result["status"] == "canonical_projection" + assert result["audit_mode"] == "CANONICAL_PROJECTION" + assert result.get("witness") is None + # Critical: no LLM round-trip happened. + assert client.calls == [] + + +def test_query_canonical_with_witness_calls_llm(tmp_path): + """policy['canonical_witness_enabled']=True → witness runs.""" + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + + client = StubClient(answer="3/10") + policy = dict(DEFAULT_QUERY_POLICY) + policy["canonical_witness_enabled"] = True + result = query( + question="0.1 + 0.2", + qa_db=tmp_path / "qa.db", + chat_client=client, + model_id="stub", + policy=policy, + ) + assert result["status"] == "canonical_projection" + witness = result.get("witness") + assert witness is not None + assert witness["agreement_label"] == "KERNEL-LLM-AGREE" + # The LLM was actually called. + assert len(client.calls) == 1