arborist/bench/scripts/witness_to_5f.py

"""Witness-divergence → 5F Falsification fixture extractor.

Pulls ``providence_canonical_witness`` audit events from a qa.db
and writes them out as 5F Falsification fixtures (matching the
existing ``falsification-live-v1`` schema). Each diverging witness
event becomes a supervised-correction sample the LLM can be
calibrated against — the kernel's canonical bytes are ground
truth, so an LLM raw answer that doesn't fold to those bytes is
falsifiable evidence of an arithmetic / logic / symbolic
hallucination.

#000028 follow-up — closes the divergence → calibration data loop.
The witness path landed in commit `656b573`; the audit-event write
landed in this same commit. The extractor here is the missing
piece that converts audit-chain entries into 5F-runner-compatible
fixtures.

Usage::

    python -m bench.scripts.witness_to_5f \\
        --qa-db ~/.arborist/shards/qa.db \\
        --out bench/fixtures/5f/falsification-witness-v1.jsonl

Or via Makefile::

    make bench-witness-divergence

Filtering rules
---------------
Includes events whose ``agreement_label`` indicates LLM divergence:

- ``LLM-DIVERGED`` (kernel + cache agree; LLM differs)
- ``KERNEL-LLM-DIVERGED`` (no cache; LLM differs from kernel)
- ``CACHE-DRIFT`` (kernel + LLM agree; cache differs — flagged
  as a calibration-data point too since the cache row is wrong)

Skips:

- ``STRICT-WITNESSED`` / ``KERNEL-LLM-AGREE`` /
  ``KERNEL-CACHE-AGREE`` / ``KERNEL-ONLY`` — agreement, no signal.
- Events without ``llm_raw_text`` — LLM was unparseable; not a
  useful supervised-correction sample.

Idempotency
-----------
Output is sorted by audit-event seq for deterministic byte-equality
across runs against the same qa.db. Re-running with no new
divergence events writes the same bytes. The fixture-digest
discipline (#000021 §3) stays valid.
"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

from arborist.store import connect


_DIVERGENCE_LABELS = {
    "LLM-DIVERGED",
    "KERNEL-LLM-DIVERGED",
    "CACHE-DRIFT",
}


def extract_divergence_fixtures(
    qa_db: Path,
    *,
    fixture_id_prefix: str = "5f-fal-witness",
    verifier_method_root: str = "verify_quotes-v1",
) -> list[dict]:
    """Pull divergence events from `qa_db`'s audit chain. Returns a
    list of 5F-Falsification fixture dicts (sorted by audit-event
    seq for determinism)."""
    conn = connect(qa_db)
    try:
        rows = conn.execute(
            "SELECT seq, body FROM audit_events "
            "WHERE event_type = 'providence_canonical_witness' "
            "ORDER BY seq ASC"
        ).fetchall()
    finally:
        conn.close()

    fixtures: list[dict] = []
    seen_ids: set[str] = set()
    for r in rows:
        try:
            body = json.loads(r["body"])
        except (json.JSONDecodeError, TypeError):
            continue
        agreement = body.get("agreement_label")
        if agreement not in _DIVERGENCE_LABELS:
            continue
        llm_raw = body.get("llm_raw_text")
        if not llm_raw:
            # Unparseable LLM output isn't a calibration sample.
            continue
        canonical_text = body.get("canonical_answer_text") or ""
        question_text = body.get("question_text") or ""
        pi_star_ref = body.get("pi_star_ref") or "arithmetic@v1"

        # Stable id from seq — re-running against the same qa.db
        # produces the same fixture bytes (idempotency contract).
        fixture_id = f"{fixture_id_prefix}-{int(r['seq']):06d}"
        if fixture_id in seen_ids:
            continue
        seen_ids.add(fixture_id)
        fixtures.append({
            "id": fixture_id,
            "battery": "5f",
            "sub_battery": "falsification",
            "version": "v1",
            "carrier": "providence_record",
            "domain": "claim_lattice",
            "pi_star_ref": pi_star_ref,
            "answer_text": llm_raw,
            "context": (
                f"canonical_kernel_answer={canonical_text}\n"
                f"question={question_text}"
            ),
            "expected_reason": "UNGROUNDED",
            "verifier_method_root": verifier_method_root,
            "expected": "pass",
            "_witness_meta": {
                "agreement_label": agreement,
                "audit_seq": int(r["seq"]),
                "pi_star_ref": pi_star_ref,
            },
        })
    return fixtures


def write_fixtures(out_path: Path, fixtures: list[dict]) -> None:
    out_path.parent.mkdir(parents=True, exist_ok=True)
    meta = {
        "_meta": {
            "battery": "5f",
            "sub_battery": "falsification",
            "version": "v1",
            "task_count": len(fixtures),
            "notes": (
                "Witness-divergence-extracted falsification fixtures. "
                "Auto-generated by `make bench-witness-divergence` "
                "from providence_canonical_witness audit events. "
                "Each row pairs an LLM raw answer with the kernel's "
                "canonical answer; verify_quotes is expected to "
                "return UNGROUNDED because the LLM prose doesn't "
                "substring-match the kernel's terse canonical form. "
                "_witness_meta carries the original agreement_label "
                "+ audit_seq for traceability."
            ),
        },
    }
    with out_path.open("w", encoding="utf-8") as fh:
        fh.write(json.dumps(meta, ensure_ascii=False) + "\n")
        for fx in fixtures:
            fh.write(json.dumps(fx, ensure_ascii=False) + "\n")


def main(argv: list[str] | None = None) -> int:
    p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
    p.add_argument(
        "--qa-db", type=Path, required=True,
        help="path to qa.db (the providence_cache + audit_events store)",
    )
    p.add_argument(
        "--out", type=Path,
        default=Path("bench/fixtures/5f/falsification-witness-v1.jsonl"),
        help="output JSONL path",
    )
    args = p.parse_args(argv)

    if not args.qa_db.is_file():
        print(f"error: qa-db not found: {args.qa_db}", file=sys.stderr)
        return 2

    fixtures = extract_divergence_fixtures(args.qa_db)
    write_fixtures(args.out, fixtures)
    print(
        f"wrote {len(fixtures)} divergence fixtures to {args.out}",
        file=sys.stderr,
    )
    print(json.dumps({"divergence_count": len(fixtures)}))
    return 0


if __name__ == "__main__":
    sys.exit(main())