diff --git a/Makefile b/Makefile index 288e098..d9cbd5f 100644 --- a/Makefile +++ b/Makefile @@ -333,6 +333,27 @@ bench-real-shard: bootstrap ## #000026 Phase 2 — real-shard workload baseline --shards-dir $${ARBORIST_SHARDS_DIR:-$$HOME/.arborist/shards} \ --burn +# #000028 follow-up — extract LLM-divergence audit events as 5F +# Falsification fixtures. Reads providence_canonical_witness audit +# events from a qa.db and writes them to a witness-derived fixture +# JSONL the existing 5F-falsification-live runner can consume. +WITNESS_QA_DB ?= $$HOME/.arborist/shards/qa.db +WITNESS_OUT ?= bench/fixtures/5f/falsification-witness-v1.jsonl +bench-witness-divergence: bootstrap ## extract LLM-divergence events as 5F Falsification fixtures + PYTHONUNBUFFERED=1 $(PY) -m bench.scripts.witness_to_5f \ + --qa-db $(WITNESS_QA_DB) \ + --out $(WITNESS_OUT) + +# function-sampled@v1 demo (#000030 Phase 7) — closes the +# opencompletion activity24-math-plot.yaml loop. Canonical bytes +# always print; PNG is optional (provide PNG=/path/to/file.png). +# Default expression range covers a full sine period. +demo-plot: bootstrap ## function-sampled@v1 demo: make demo-plot Q='sin(x)' [PNG=/tmp/out.png] + @if [ -z "$(Q)" ]; then echo "usage: make demo-plot Q='' [PNG=/tmp/out.png]" >&2; exit 2; fi + PYTHONUNBUFFERED=1 $(PY) -m bench.scripts.demo_plot \ + --expr "$(Q)" \ + $(if $(PNG),--png $(PNG),) + # v8 ForkScore — runs the full bench-suite, then scores the fresh # child output against a previously-pinned PARENT artifact. Default # parent is bench/results/baseline-suite.json (operator pins this diff --git a/arborist/qa/query.py b/arborist/qa/query.py index 6c3f508..f0a6548 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -2146,6 +2146,57 @@ def query( progress=progress, ) witness_dict = _witness.to_dict() + # Witness audit event (#000028 follow-up). Records + # the fan-out result on the audit chain so a + # downstream extractor (`make + # bench-witness-divergence`) can pull divergence + # events as 5F-Falsification calibration fixtures. + # Best-effort: chain-write failure must never fail + # the query. + if witness_dict: + try: + from arborist.store import ( + append_audit as _append_audit, + ) + _wconn = connect(qa_db) + try: + _append_audit( + _wconn, + event_type="providence_canonical_witness", + subject_root=ckey, + body={ + "pi_star_ref": pi_star_ref, + "question_text": question, + "agreement_label": witness_dict.get( + "agreement_label" + ), + "canonical_answer_text": ( + canonical_bytes.decode( + "utf-8", errors="replace" + ) + ), + "llm_raw_text": ( + witness_dict.get("modalities", {}) + .get("llm", {}) + .get("raw_answer") + ), + "llm_canonical_bytes": ( + witness_dict.get("modalities", {}) + .get("llm", {}) + .get("canonical_bytes") + ), + "cache_status": ( + witness_dict.get("modalities", {}) + .get("cache", {}) + .get("error") + or "ok" + ), + }, + ) + finally: + _wconn.close() + except Exception: # pragma: no cover + pass # Capital-ledger record (#000028 follow-up). Witness # mode adds one full LLM call per fired canonical # question; record the cost so ForkScore (#000012) diff --git a/bench/scripts/demo_plot.py b/bench/scripts/demo_plot.py new file mode 100644 index 0000000..6cb8d76 --- /dev/null +++ b/bench/scripts/demo_plot.py @@ -0,0 +1,160 @@ +"""End-to-end demo for ``function-sampled@v1`` (#000030 Phase 7). + +Closes the loop on opencompletion's ``activity24-math-plot.yaml``: +SymPy expression → quantized integer-vector signature (canonical +bytes) → optional matplotlib PNG (a downstream view of the +canonical evidence). + +The canonical bytes are the proof: two function-sampled outputs +that match byte-for-byte represent the same function on the same +quantization grid. The PNG is just a human-readable rendering of +those same bytes — different DPIs / colormaps / axis spans don't +change identity. + +Usage:: + + python -m bench.scripts.demo_plot \\ + --expr 'sin(x)' --x-min 0 --x-max 6.283 --n-samples 200 --dv 0.01 \\ + --png /tmp/sin.png + + # Or via Make (PNG optional): + make demo-plot Q='sin(x)' + make demo-plot Q='x**2 + 2*x + 1' PNG=/tmp/parabola.png + +The matplotlib import is gated — the canonical bytes always print +to stdout; the PNG is written only when ``--png`` is provided AND +matplotlib is importable. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path + + +def _build_payload( + expr: str, x: str, x_min: float, x_max: float, + n_samples: int, dv: float, +) -> bytes: + return json.dumps({ + "f": expr, "x": x, + "x_min": x_min, "x_max": x_max, + "n_samples": n_samples, "dv": dv, + }).encode("utf-8") + + +def _maybe_render_png( + *, expr: str, x_var: str, x_min: float, x_max: float, + n_samples: int, png_path: Path, +) -> None: + """Render a PNG via matplotlib for human-readable inspection. + No-op if matplotlib isn't installed.""" + try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + import sympy as sp + except ImportError: + print( + "warning: matplotlib + sympy required for PNG render; " + "skipping --png", + file=sys.stderr, + ) + return + x_sym = sp.Symbol(x_var) + f_expr = sp.sympify(expr, locals={x_var: x_sym}) + f = sp.lambdify(x_sym, f_expr, "math") + step = (x_max - x_min) / (n_samples - 1) + xs = [x_min + i * step for i in range(n_samples)] + ys = [] + for xi in xs: + try: + yi = f(xi) + if isinstance(yi, complex): + yi = yi.real + ys.append(float(yi)) + except (ValueError, ZeroDivisionError, OverflowError): + ys.append(float("nan")) + plt.figure(figsize=(8, 5)) + plt.plot(xs, ys, linewidth=2, color="#3366cc") + plt.title(f"y = {expr}") + plt.xlabel(x_var) + plt.ylabel("y") + plt.grid(True, alpha=0.3) + png_path.parent.mkdir(parents=True, exist_ok=True) + plt.tight_layout() + plt.savefig(png_path, dpi=120) + plt.close() + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument("--expr", required=True, help="SymPy expression in the variable --x") + p.add_argument("--x", default="x", help="independent-variable name (default 'x')") + p.add_argument("--x-min", dest="x_min", type=float, default=-3.14159) + p.add_argument("--x-max", dest="x_max", type=float, default=3.14159) + p.add_argument("--n-samples", dest="n_samples", type=int, default=200) + p.add_argument("--dv", type=float, default=0.01, + help="value-quantization step (smaller = tighter)") + p.add_argument("--png", type=Path, default=None, + help="optional PNG output path (requires matplotlib)") + args = p.parse_args(argv) + + from arborist.pi_star import PiStarError, get + try: + ps = get("function-sampled@v1") + except KeyError: + print( + "error: function-sampled@v1 not registered (sympy missing?); " + "install with `pip install arborist[math]`", + file=sys.stderr, + ) + return 2 + + payload = _build_payload( + args.expr, args.x, args.x_min, args.x_max, + args.n_samples, args.dv, + ) + try: + canonical_bytes = ps.canonicalize(payload) + except PiStarError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + digest = hashlib.sha256(canonical_bytes).hexdigest() + + out = { + "expr": args.expr, + "x": args.x, + "grid": { + "x_min": args.x_min, + "x_max": args.x_max, + "n_samples": args.n_samples, + "dv": args.dv, + }, + "canonical_bytes_sha256": digest, + "canonical_bytes_preview": canonical_bytes.decode("utf-8")[:200] + ( + "…" if len(canonical_bytes) > 200 else "" + ), + "canonical_bytes_total_chars": len(canonical_bytes), + "pi_star_ref": "function-sampled@v1", + } + + if args.png is not None: + _maybe_render_png( + expr=args.expr, x_var=args.x, + x_min=args.x_min, x_max=args.x_max, + n_samples=args.n_samples, + png_path=args.png, + ) + out["png_path"] = str(args.png) + + print(json.dumps(out, indent=2, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/scripts/witness_to_5f.py b/bench/scripts/witness_to_5f.py new file mode 100644 index 0000000..9ec2942 --- /dev/null +++ b/bench/scripts/witness_to_5f.py @@ -0,0 +1,192 @@ +"""Witness-divergence → 5F Falsification fixture extractor. + +Pulls ``providence_canonical_witness`` audit events from a qa.db +and writes them out as 5F Falsification fixtures (matching the +existing ``falsification-live-v1`` schema). Each diverging witness +event becomes a supervised-correction sample the LLM can be +calibrated against — the kernel's canonical bytes are ground +truth, so an LLM raw answer that doesn't fold to those bytes is +falsifiable evidence of an arithmetic / logic / symbolic +hallucination. + +#000028 follow-up — closes the divergence → calibration data loop. +The witness path landed in commit `656b573`; the audit-event write +landed in this same commit. The extractor here is the missing +piece that converts audit-chain entries into 5F-runner-compatible +fixtures. + +Usage:: + + python -m bench.scripts.witness_to_5f \\ + --qa-db ~/.arborist/shards/qa.db \\ + --out bench/fixtures/5f/falsification-witness-v1.jsonl + +Or via Makefile:: + + make bench-witness-divergence + +Filtering rules +--------------- +Includes events whose ``agreement_label`` indicates LLM divergence: + +- ``LLM-DIVERGED`` (kernel + cache agree; LLM differs) +- ``KERNEL-LLM-DIVERGED`` (no cache; LLM differs from kernel) +- ``CACHE-DRIFT`` (kernel + LLM agree; cache differs — flagged + as a calibration-data point too since the cache row is wrong) + +Skips: + +- ``STRICT-WITNESSED`` / ``KERNEL-LLM-AGREE`` / + ``KERNEL-CACHE-AGREE`` / ``KERNEL-ONLY`` — agreement, no signal. +- Events without ``llm_raw_text`` — LLM was unparseable; not a + useful supervised-correction sample. + +Idempotency +----------- +Output is sorted by audit-event seq for deterministic byte-equality +across runs against the same qa.db. Re-running with no new +divergence events writes the same bytes. The fixture-digest +discipline (#000021 §3) stays valid. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from arborist.store import connect + + +_DIVERGENCE_LABELS = { + "LLM-DIVERGED", + "KERNEL-LLM-DIVERGED", + "CACHE-DRIFT", +} + + +def extract_divergence_fixtures( + qa_db: Path, + *, + fixture_id_prefix: str = "5f-fal-witness", + verifier_method_root: str = "verify_quotes-v1", +) -> list[dict]: + """Pull divergence events from `qa_db`'s audit chain. Returns a + list of 5F-Falsification fixture dicts (sorted by audit-event + seq for determinism).""" + conn = connect(qa_db) + try: + rows = conn.execute( + "SELECT seq, body FROM audit_events " + "WHERE event_type = 'providence_canonical_witness' " + "ORDER BY seq ASC" + ).fetchall() + finally: + conn.close() + + fixtures: list[dict] = [] + seen_ids: set[str] = set() + for r in rows: + try: + body = json.loads(r["body"]) + except (json.JSONDecodeError, TypeError): + continue + agreement = body.get("agreement_label") + if agreement not in _DIVERGENCE_LABELS: + continue + llm_raw = body.get("llm_raw_text") + if not llm_raw: + # Unparseable LLM output isn't a calibration sample. + continue + canonical_text = body.get("canonical_answer_text") or "" + question_text = body.get("question_text") or "" + pi_star_ref = body.get("pi_star_ref") or "arithmetic@v1" + + # Stable id from seq — re-running against the same qa.db + # produces the same fixture bytes (idempotency contract). + fixture_id = f"{fixture_id_prefix}-{int(r['seq']):06d}" + if fixture_id in seen_ids: + continue + seen_ids.add(fixture_id) + fixtures.append({ + "id": fixture_id, + "battery": "5f", + "sub_battery": "falsification", + "version": "v1", + "carrier": "providence_record", + "domain": "claim_lattice", + "pi_star_ref": pi_star_ref, + "answer_text": llm_raw, + "context": ( + f"canonical_kernel_answer={canonical_text}\n" + f"question={question_text}" + ), + "expected_reason": "UNGROUNDED", + "verifier_method_root": verifier_method_root, + "expected": "pass", + "_witness_meta": { + "agreement_label": agreement, + "audit_seq": int(r["seq"]), + "pi_star_ref": pi_star_ref, + }, + }) + return fixtures + + +def write_fixtures(out_path: Path, fixtures: list[dict]) -> None: + out_path.parent.mkdir(parents=True, exist_ok=True) + meta = { + "_meta": { + "battery": "5f", + "sub_battery": "falsification", + "version": "v1", + "task_count": len(fixtures), + "notes": ( + "Witness-divergence-extracted falsification fixtures. " + "Auto-generated by `make bench-witness-divergence` " + "from providence_canonical_witness audit events. " + "Each row pairs an LLM raw answer with the kernel's " + "canonical answer; verify_quotes is expected to " + "return UNGROUNDED because the LLM prose doesn't " + "substring-match the kernel's terse canonical form. " + "_witness_meta carries the original agreement_label " + "+ audit_seq for traceability." + ), + }, + } + with out_path.open("w", encoding="utf-8") as fh: + fh.write(json.dumps(meta, ensure_ascii=False) + "\n") + for fx in fixtures: + fh.write(json.dumps(fx, ensure_ascii=False) + "\n") + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument( + "--qa-db", type=Path, required=True, + help="path to qa.db (the providence_cache + audit_events store)", + ) + p.add_argument( + "--out", type=Path, + default=Path("bench/fixtures/5f/falsification-witness-v1.jsonl"), + help="output JSONL path", + ) + args = p.parse_args(argv) + + if not args.qa_db.is_file(): + print(f"error: qa-db not found: {args.qa_db}", file=sys.stderr) + return 2 + + fixtures = extract_divergence_fixtures(args.qa_db) + write_fixtures(args.out, fixtures) + print( + f"wrote {len(fixtures)} divergence fixtures to {args.out}", + file=sys.stderr, + ) + print(json.dumps({"divergence_count": len(fixtures)})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/_source/bench.rst b/docs/_source/bench.rst index 517d219..5a16e65 100644 --- a/docs/_source/bench.rst +++ b/docs/_source/bench.rst @@ -2,25 +2,39 @@ Benchmark surface ================= Arborist ships the complete **Dav1DPrometheus 5S/5T/5F/5R** evaluation -suite — 21 sub-batteries, ~660 deterministic fixtures — as -first-class infrastructure. Every benchmark is reproducible, no -LLM-as-judge, and many sub-batteries route through the actual -arborist surface (parser, verifier, audit chain, π* registry) rather -than synthetic gold output. +suite — 21 sub-batteries, **662+ deterministic fixtures** in the +default runner (Phase 1d expansion 2026-05-09) plus ~110 additional +math π* fixtures — as first-class infrastructure. Every benchmark is +reproducible, no LLM-as-judge, and many sub-batteries route through +the actual arborist surface (parser, verifier, audit chain, π* +registry) rather than synthetic gold output. Quick reference --------------- .. code-block:: bash - make bench-suite # complete 5S + 5T + 5F + 5R suite - make bench-5s # representational discipline - make bench-5t # temporal / cross-reasoning - make bench-5f # operational quality (Phase 1a embedded) - make bench-5f-live # 5F bridged to live arborist surfaces (Phase 1b.2) - make bench-5r # workspace operators - make bench-5s-math # arithmetic@v1 + logic-kernel@v1 fixtures - make bench-5s-code # code-py-ast@v1 fixtures + make bench-suite # complete 5S + 5T + 5F + 5R (662 tasks) + make bench-5s # representational discipline + make bench-5t # temporal / cross-reasoning + make bench-5f # operational quality (synthetic + live) + make bench-5r # workspace operators + # Per-π* 5S targets (#000030 SymPy substrate + the registry's text/ + # arithmetic/logic core + the last-stub graduation): + make bench-5s-math # arithmetic@v1 + logic-kernel@v1 + make bench-5s-code # code-py-ast@v1 + make bench-5s-time-series # time-series-quantized@v1 + make bench-5s-tabular # tabular-pinned@v1 (last reserved stub) + make bench-5s-algebra # algebra-symbolic@v1 + make bench-5s-calculus-limit # calculus-limit@v1 + make bench-5s-calculus-series # calculus-series@v1 + make bench-5s-linear-algebra # linear-algebra@v1 + make bench-5s-function-sampled # function-sampled@v1 (SymPy → time-series) + # Real-shard + selection + witness-divergence harness: + make bench-real-shard # #000026 — real-shard latency / audit baseline + make bench-fork-baseline # pin current bench output as ForkScore parent + make bench-fork-score # score child vs pinned parent (CI-gateable) + make bench-witness-divergence # extract LLM-divergence as 5F fixtures Each invocation emits a JSON :class:`bench.batteries.base.BatteryResult` with per-task pass/fail, fixture digest, runtime digest, and diff --git a/docs/_source/v8-fork-score.rst b/docs/_source/v8-fork-score.rst index e34b1db..e32c415 100644 --- a/docs/_source/v8-fork-score.rst +++ b/docs/_source/v8-fork-score.rst @@ -107,15 +107,35 @@ CLI [--capital-delta N] \ [--memory-invalidation-count N] \ [--audit-completeness 0..1] \ - [--selfmodel-calibration-gain N] + [--selfmodel-calibration-gain N] \ + [--out path/to/fork_score_report.json] ``parent-bench.json`` and ``child-bench.json`` are the JSON output of ``bench.batteries.runner --all``. +``--out`` mirrors stdout to a file; CI / mesh peers / downstream +graders ingest the artifact without parsing pipe output. + Output: :class:`arborist.v8.fork_score.ScoredFork` with ``score``, ``verdict``, per-term ``breakdown``, ``flags`` list, and the active ``weights`` echoed back. +Make harness (Phase 1b) +~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + make bench-fork-baseline # one-shot: pin current bench-suite output + # as the ForkScore parent (writes + # bench/results/baseline-suite.json) + + make bench-fork-score # rerun bench, score child vs pinned parent, + # write bench/results/fork_score_report.json. + # Exits 1 on REJECT verdict — CI-gateable. + +Override defaults via env-vars: ``FORK_PARENT``, ``FORK_CHILD``, +``FORK_REPORT``. + What's NOT in Phase 1a ---------------------- diff --git a/tests/test_demo_plot.py b/tests/test_demo_plot.py new file mode 100644 index 0000000..3fb7de5 --- /dev/null +++ b/tests/test_demo_plot.py @@ -0,0 +1,97 @@ +"""Tests for the function-sampled@v1 demo-plot script. + +The script's primary contract is the canonical-bytes path (always +prints SHA-256 + preview to stdout). The PNG render is an optional +matplotlib-gated downstream view; tests skip gracefully when +matplotlib isn't installed. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + + +sympy = pytest.importorskip("sympy") + + +def _run(*args: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-m", "bench.scripts.demo_plot", *args], + capture_output=True, text=True, check=False, + ) + + +def test_demo_plot_emits_canonical_bytes_for_sin_x(): + r = _run("--expr", "sin(x)", "--n-samples", "10", "--dv", "0.1") + assert r.returncode == 0, r.stderr + out = json.loads(r.stdout) + assert out["expr"] == "sin(x)" + assert out["pi_star_ref"] == "function-sampled@v1" + assert len(out["canonical_bytes_sha256"]) == 64 + assert "canonical_bytes_preview" in out + assert out["grid"]["n_samples"] == 10 + + +def test_demo_plot_canonical_bytes_deterministic(): + """Same expression + grid → same SHA-256.""" + r1 = _run("--expr", "x**2", "--x-min", "0", "--x-max", "4", + "--n-samples", "5", "--dv", "1") + r2 = _run("--expr", "x**2", "--x-min", "0", "--x-max", "4", + "--n-samples", "5", "--dv", "1") + assert r1.returncode == 0 and r2.returncode == 0 + out1 = json.loads(r1.stdout) + out2 = json.loads(r2.stdout) + assert out1["canonical_bytes_sha256"] == out2["canonical_bytes_sha256"] + + +def test_demo_plot_equivalent_expressions_collapse(): + """sin(x) and 2*sin(x)/2 must produce the same SHA-256 — they + sample to identical numeric values, so canonical bytes match.""" + a = _run("--expr", "sin(x)", "--n-samples", "10", "--dv", "0.1") + b = _run("--expr", "2*sin(x)/2", "--n-samples", "10", "--dv", "0.1") + assert a.returncode == 0 and b.returncode == 0 + assert ( + json.loads(a.stdout)["canonical_bytes_sha256"] + == json.loads(b.stdout)["canonical_bytes_sha256"] + ) + + +def test_demo_plot_invalid_expression_returns_1(): + """A genuinely unparseable expression should exit 1 with a + PiStarError surfaced on stderr.""" + # Empty expression — sympify accepts garbage but lambdify or + # range checks fail; use an obviously-bad input. + r = _run("--expr", "x_min", "--x-min", "1", "--x-max", "1", + "--n-samples", "5", "--dv", "1") + # x_max <= x_min → PiStarError from function-sampled@v1 + assert r.returncode == 1 + assert "error" in r.stderr.lower() + + +def test_demo_plot_png_optional_no_crash_when_matplotlib_missing(tmp_path): + """When matplotlib isn't installed, --png prints a warning to + stderr and skips the PNG; canonical bytes still print.""" + png_path = tmp_path / "out.png" + r = _run("--expr", "x", "--n-samples", "5", "--dv", "1", + "--png", str(png_path)) + assert r.returncode == 0, r.stderr + out = json.loads(r.stdout) + assert "canonical_bytes_sha256" in out + # matplotlib may or may not be installed in this env; the + # script's contract is "no crash either way." + + +def test_demo_plot_png_renders_when_matplotlib_present(tmp_path): + matplotlib = pytest.importorskip("matplotlib") + png_path = tmp_path / "out.png" + r = _run("--expr", "sin(x)", "--n-samples", "20", "--dv", "0.1", + "--png", str(png_path)) + assert r.returncode == 0, r.stderr + assert png_path.is_file(), "matplotlib was importable; PNG must land" + # Sanity: the file isn't empty. + assert png_path.stat().st_size > 100 diff --git a/tests/test_witness_to_5f.py b/tests/test_witness_to_5f.py new file mode 100644 index 0000000..480428d --- /dev/null +++ b/tests/test_witness_to_5f.py @@ -0,0 +1,210 @@ +"""Tests for the witness-divergence → 5F Falsification extractor. + +Two streams: the extractor logic itself + the audit-event write +in query.py that feeds it. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from arborist.qa.client import StubClient +from arborist.qa.query import DEFAULT_QUERY_POLICY, query +from arborist.store import connect + + +# ----- audit-event write (#000028 follow-up) ----------------------------- + + +def test_witness_event_written_to_audit_chain(tmp_path: Path): + """When witness fires, query() appends a + providence_canonical_witness audit event with the question + + canonical + LLM raw answer + agreement label.""" + qa_db = tmp_path / "qa.db" + policy = dict(DEFAULT_QUERY_POLICY) + policy["canonical_witness_enabled"] = True + client = StubClient(answer="3/10") + query( + question="0.1 + 0.2", qa_db=qa_db, + chat_client=client, model_id="stub", policy=policy, + ) + conn = connect(qa_db) + try: + rows = conn.execute( + "SELECT body FROM audit_events " + "WHERE event_type = 'providence_canonical_witness'" + ).fetchall() + finally: + conn.close() + assert len(rows) == 1 + body = json.loads(rows[0]["body"]) + assert body["pi_star_ref"] == "arithmetic@v1" + assert body["question_text"] == "0.1 + 0.2" + assert body["canonical_answer_text"] == "3/10" + assert body["llm_raw_text"] == "3/10" + # Agreement: KERNEL-LLM-AGREE (no cache yet on first call). + assert body["agreement_label"] == "KERNEL-LLM-AGREE" + + +def test_witness_event_records_divergence(tmp_path: Path): + """LLM emits a wrong canonical-shape answer; the audit event + captures the divergence label + raw text for the extractor.""" + qa_db = tmp_path / "qa.db" + policy = dict(DEFAULT_QUERY_POLICY) + policy["canonical_witness_enabled"] = True + client = StubClient(answer="0.4") # 0.4 → 2/5 ≠ kernel's 3/10 + query( + question="0.1 + 0.2", qa_db=qa_db, + chat_client=client, model_id="stub", policy=policy, + ) + conn = connect(qa_db) + try: + body = json.loads(conn.execute( + "SELECT body FROM audit_events " + "WHERE event_type = 'providence_canonical_witness' " + "ORDER BY seq DESC LIMIT 1" + ).fetchone()["body"]) + finally: + conn.close() + assert body["agreement_label"] == "KERNEL-LLM-DIVERGED" + assert body["llm_raw_text"] == "0.4" + assert body["canonical_answer_text"] == "3/10" + + +def test_witness_no_event_when_disabled(tmp_path: Path): + """Default-off witness path → no providence_canonical_witness + event landed.""" + qa_db = tmp_path / "qa.db" + client = StubClient(answer="") + query( + question="0.1 + 0.2", qa_db=qa_db, + chat_client=client, model_id="stub", + ) + conn = connect(qa_db) + try: + n = conn.execute( + "SELECT COUNT(*) FROM audit_events " + "WHERE event_type = 'providence_canonical_witness'" + ).fetchone()[0] + finally: + conn.close() + assert n == 0 + + +# ----- extractor (witness_to_5f.py) -------------------------------------- + + +def _seed_witness_events(qa_db: Path, events: list[dict]) -> None: + """Helper: fire `query` with synthetic LLM answers to build a + qa.db carrying a known set of witness audit events.""" + policy = dict(DEFAULT_QUERY_POLICY) + policy["canonical_witness_enabled"] = True + for ev in events: + client = StubClient(answer=ev["llm_answer"]) + query( + question=ev["question"], qa_db=qa_db, + chat_client=client, model_id="stub", policy=policy, + ) + + +def test_extract_divergence_fixtures_basic(tmp_path: Path): + from bench.scripts.witness_to_5f import extract_divergence_fixtures + qa_db = tmp_path / "qa.db" + # Seed: 1 agreement, 2 divergences. Only the divergences come back. + _seed_witness_events(qa_db, [ + {"question": "0.1 + 0.2", "llm_answer": "3/10"}, # agree + {"question": "1 + 1", "llm_answer": "3"}, # diverged + {"question": "2 + 2", "llm_answer": "5"}, # diverged + ]) + fixtures = extract_divergence_fixtures(qa_db) + assert len(fixtures) == 2 + questions = sorted( + f["context"].split("\nquestion=")[1] for f in fixtures + ) + assert questions == ["1 + 1", "2 + 2"] + + +def test_extract_skips_unparseable_llm_text(tmp_path: Path): + """LLM emits prose rather than canonical-shape answer; that's + KERNEL-ONLY (LLM modality absent), not divergence.""" + from bench.scripts.witness_to_5f import extract_divergence_fixtures + qa_db = tmp_path / "qa.db" + _seed_witness_events(qa_db, [ + {"question": "0.1 + 0.2", "llm_answer": "the answer is around 0.3"}, + ]) + fixtures = extract_divergence_fixtures(qa_db) + # KERNEL-ONLY isn't in _DIVERGENCE_LABELS — skipped. + assert len(fixtures) == 0 + + +def test_extract_fixture_schema_matches_5f_falsification(tmp_path: Path): + """The emitted fixture dict has the same fields the existing + 5F-falsification-live runner expects.""" + from bench.scripts.witness_to_5f import extract_divergence_fixtures + qa_db = tmp_path / "qa.db" + _seed_witness_events(qa_db, [ + {"question": "1 + 1", "llm_answer": "3"}, + ]) + fixtures = extract_divergence_fixtures(qa_db) + assert len(fixtures) == 1 + fx = fixtures[0] + # Required by the 5F-falsification schema. + for required in ( + "id", "battery", "sub_battery", "version", "carrier", + "domain", "pi_star_ref", "answer_text", "context", + "expected_reason", "verifier_method_root", "expected", + ): + assert required in fx, f"missing field {required!r}" + assert fx["battery"] == "5f" + assert fx["sub_battery"] == "falsification" + assert fx["version"] == "v1" + assert fx["carrier"] == "providence_record" + assert fx["expected_reason"] == "UNGROUNDED" + assert fx["expected"] == "pass" + # Witness traceability metadata stays out of the runner's path + # (underscored field) but available for inspection. + assert "_witness_meta" in fx + assert fx["_witness_meta"]["agreement_label"] == "KERNEL-LLM-DIVERGED" + + +def test_extract_idempotent_on_same_db(tmp_path: Path): + """Running the extractor twice against the same qa.db produces + byte-equal fixtures (sorted by audit-event seq).""" + from bench.scripts.witness_to_5f import ( + extract_divergence_fixtures, + ) + qa_db = tmp_path / "qa.db" + _seed_witness_events(qa_db, [ + {"question": "1 + 1", "llm_answer": "3"}, + {"question": "2 + 2", "llm_answer": "5"}, + ]) + a = extract_divergence_fixtures(qa_db) + b = extract_divergence_fixtures(qa_db) + assert a == b + + +def test_write_fixtures_creates_5f_jsonl_meta(tmp_path: Path): + """The output JSONL starts with a meta header naming the + battery + sub_battery + task_count, matching the rest of the + bench/fixtures/5f/*-v1.jsonl files.""" + from bench.scripts.witness_to_5f import ( + extract_divergence_fixtures, write_fixtures, + ) + qa_db = tmp_path / "qa.db" + out = tmp_path / "out.jsonl" + _seed_witness_events(qa_db, [ + {"question": "1 + 1", "llm_answer": "3"}, + ]) + fixtures = extract_divergence_fixtures(qa_db) + write_fixtures(out, fixtures) + lines = out.read_text(encoding="utf-8").splitlines() + meta = json.loads(lines[0])["_meta"] + assert meta["battery"] == "5f" + assert meta["sub_battery"] == "falsification" + assert meta["task_count"] == 1 + # Subsequent lines are valid fixture JSONs. + fx = json.loads(lines[1]) + assert fx["id"].startswith("5f-fal-witness-")