bench: QA-quality sweep harness with n-sample variance

Adds bench/qa_sweep.py + bench/qa_questions.txt + `make bench-qa` target.
Sweeps a fixed question set through quote and claim_lattice_pointer modes,
N samples per cell (default 3), each sample burns the cached record so
Hermes nondeterminism becomes the variance source.

Outputs:
  bench/qa_results/<utc>.jsonl   one row per (mode, question, sample)
  bench/qa_results/<utc>.md      summary + per-question vote counts

Question set spans the failure-mode shapes the verifier needs to handle:
narrow factoid, broad descriptive ("tell me about X"), entity list,
relationship, and out-of-corpus (should land UNGROUNDED honestly).

Methodology gap this closes: at n=1 the strict-rate swings ±20pp on a
fixed prompt purely from sampling noise. n=3 makes 5pp deltas legible.

.gitignore

@@ -12,3 +12,6 @@ data/
 *.db-journal
 *.db-wal
 *.db-shm
+
+# bench artifacts
+bench/qa_results/

Makefile

@@ -32,7 +32,7 @@ SEARCH_Q     ?= computer
         ingest-self ingest-git ingest-hg \
         verify search stats test docs chain-check chain-check-shards \
         falsify burn burn-kindergarten inspect bootstrap-crawler test-crawler crawl-ingest \
-        recrawl-check clean clean-db clean-data help
+        recrawl-check bench-qa clean clean-db clean-data help
 
 all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats
 
@@ -173,6 +173,21 @@ query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=
 	fi
 	$(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) "$(Q)"
 
+BENCH_QA_QUESTIONS ?= bench/qa_questions.txt
+BENCH_QA_OUT       ?= bench/qa_results
+BENCH_QA_MODES     ?= quote,claim_lattice_pointer
+BENCH_QA_LIMIT     ?= 0
+BENCH_QA_N         ?= 3
+bench-qa: bootstrap ## QA-quality sweep: questions × modes × N samples [BENCH_QA_N=3 BENCH_QA_LIMIT=N BENCH_QA_MODES=...]
+	PYTHONUNBUFFERED=1 $(PY) bench/qa_sweep.py \
+	    --questions $(BENCH_QA_QUESTIONS) \
+	    --shards-dir $(SHARDS_DIR) \
+	    --out-dir $(BENCH_QA_OUT) \
+	    --top-k $(QUERY_TOP_K) \
+	    --modes $(BENCH_QA_MODES) \
+	    --limit $(BENCH_QA_LIMIT) \
+	    --n $(BENCH_QA_N)
+
 verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample
 	$(ABORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N)
 

bench/qa_questions.txt

@@ -0,0 +1,32 @@
+# aborist QA-quality benchmark question set.
+#
+# One question per line. `#`-prefixed lines and blank lines ignored.
+# Designed for the 2003-05-16 Wikipedia cur snapshot — questions must
+# resolve against ~2003 article content. Mix of failure-mode probes:
+#
+#   narrow factoid          — one-shot, evergreen, well-anchored
+#   broad descriptive       — "tell me about X" — invites encyclopedic
+#                             echo, the G0 mode-collapse trigger
+#   entity list             — invites lazy-anchor on a magnet chunk
+#   relationship / multi    — multi-pointer claim
+#   out-of-corpus           — should land UNGROUNDED honestly
+#
+# Add/remove rows as we learn what the verifier should be measuring.
+
+# narrow factoid
+who founded apple computer?
+what is the capital of france?
+who wrote the linux kernel?
+
+# broad descriptive
+tell me about connecticut
+tell me about the python programming language
+
+# entity list
+list the founders of microsoft
+
+# relationship
+who is supermans girlfriend?
+
+# out-of-corpus (should land UNGROUNDED, not hallucinate)
+who is the prime minister of mars?

bench/qa_sweep.py

@@ -0,0 +1,326 @@
+"""QA-quality benchmark sweep.
+
+Runs a fixed question set through ``aborist.qa.query.query`` under each
+answer mode and tabulates STRICT / HYBRID / UNGROUNDED counts, the
+``n_verified / n_quotes`` ratio, latency, and lazy-anchor signals.
+
+Why this exists: G0 / claim-lattice-pointer mode landed in 2337b77, and
+four hardenings stacked on top (smell sidecar, positive-form prompts,
+source-role boost, title-purity rerank, CITATION_MISMATCH check). Every
+new heuristic flies blind until we can show "this commit moved STRICT
+from N to M". This is that scaffolding.
+
+Outputs:
+
+    bench/qa_results/<utc-iso>.jsonl   one row per (mode, question)
+    bench/qa_results/<utc-iso>.md      markdown summary table
+
+The JSONL is the durable artifact. The markdown is the thing fox reads.
+
+Wire via ``make bench-qa``. Honors ``BURN=1`` to force fresh inference
+under the current ``governance_policy_hash`` — useful after a prompt
+change that doesn't bump any version field.
+"""
+from __future__ import annotations
+
+import argparse
+import datetime as _dt
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+# Defer aborist imports until argparse runs so `--help` works without
+# the package installed.
+
+ANSWER_MODES = ("quote", "claim_lattice_pointer")
+
+
+def _read_questions(path: Path) -> list[str]:
+    out: list[str] = []
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        out.append(line)
+    return out
+
+
+def _utc_iso_compact() -> str:
+    # YYYY-MM-DDTHH-MM-SSZ — colon-free so it's filesystem-safe.
+    return _dt.datetime.now(tz=_dt.timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")
+
+
+def _ratio(verdict: dict) -> float:
+    nq = verdict.get("n_quotes") or 0
+    nv = verdict.get("n_verified") or 0
+    return (nv / nq) if nq else 0.0
+
+
+def _run_one(
+    *,
+    question: str,
+    answer_mode: str,
+    shards_dir: Path | None,
+    qa_db: Path | None,
+    top_k: int,
+    burn: bool,
+    endpoint: str,
+    model: str,
+) -> dict:
+    """Run one (question, mode) — returns a flat record for the JSONL."""
+    from aborist.qa.client import OpenAICompatibleClient
+    from aborist.qa.query import DEFAULT_QUERY_POLICY, query
+
+    policy = dict(DEFAULT_QUERY_POLICY)
+    policy["answer_mode"] = answer_mode
+
+    api_key = os.environ.get("ABORIST_LLM_API_KEY")
+    client = OpenAICompatibleClient(base_url=endpoint, api_key=api_key)
+
+    if qa_db is None:
+        qa_db = (shards_dir / "qa.db") if shards_dir else (Path.home() / ".aborist" / "qa.db")
+
+    t0 = time.monotonic()
+    err: str | None = None
+    result: dict = {}
+    try:
+        result = query(
+            question=question,
+            qa_db=qa_db,
+            chat_client=client,
+            model_id=model,
+            shards_dir=shards_dir,
+            top_k=top_k,
+            policy=policy,
+            burn_existing=burn,
+        )
+    except Exception as e:  # noqa: BLE001 — bench captures all
+        err = f"{type(e).__name__}: {e}"
+    elapsed_s = round(time.monotonic() - t0, 2)
+
+    audit_mode = result.get("audit_mode")
+    return {
+        "question": question,
+        "answer_mode": answer_mode,
+        "status": result.get("status") or ("error" if err else "missing"),
+        "audit_mode": audit_mode,
+        "n_quotes": result.get("n_quotes"),
+        "n_verified": result.get("n_verified"),
+        "ratio": round(_ratio(result), 3),
+        "verifier_method": result.get("verifier_method"),
+        "lookup_path": result.get("lookup_path"),
+        "failure_stage": result.get("failure_stage"),
+        "lazy_anchor_ratio": result.get("lazy_anchor_ratio"),
+        "pointer_id_distribution": result.get("pointer_id_distribution"),
+        "cache_key": (result.get("cache_key") or "")[:12],
+        "n_sources": len(result.get("sources") or []),
+        "elapsed_s": elapsed_s,
+        "error": err,
+    }
+
+
+def _summarize(rows: list[dict]) -> dict:
+    """Group rows by mode → STRICT/HYBRID/UNGROUNDED totals + means."""
+    by_mode: dict[str, dict] = {}
+    for r in rows:
+        m = r["answer_mode"]
+        b = by_mode.setdefault(m, {
+            "n": 0,
+            "STRICT": 0,
+            "HYBRID": 0,
+            "UNGROUNDED": 0,
+            "errors": 0,
+            "ratio_sum": 0.0,
+            "latency_sum": 0.0,
+        })
+        b["n"] += 1
+        if r["error"] or r["status"] == "error":
+            b["errors"] += 1
+        elif r["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED"):
+            b[r["audit_mode"]] += 1
+        b["ratio_sum"] += r["ratio"] or 0.0
+        b["latency_sum"] += r["elapsed_s"] or 0.0
+    return by_mode
+
+
+def _per_question_summary(rows: list[dict]) -> dict:
+    """Group rows by (question, mode) → per-cell vote counts + medians."""
+    cells: dict[tuple[str, str], dict] = {}
+    for r in rows:
+        key = (r["question"], r["answer_mode"])
+        c = cells.setdefault(key, {
+            "STRICT": 0, "HYBRID": 0, "UNGROUNDED": 0, "errors": 0,
+            "ratios": [], "latencies": [],
+        })
+        if r["error"] or r["status"] == "error":
+            c["errors"] += 1
+        elif r["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED"):
+            c[r["audit_mode"]] += 1
+        c["ratios"].append(r["ratio"] or 0.0)
+        c["latencies"].append(r["elapsed_s"] or 0.0)
+    return cells
+
+
+def _median(xs: list[float]) -> float:
+    if not xs:
+        return 0.0
+    s = sorted(xs)
+    mid = len(s) // 2
+    return s[mid] if len(s) % 2 else (s[mid - 1] + s[mid]) / 2
+
+
+def _render_markdown(
+    rows: list[dict],
+    summary: dict,
+    started_utc: str,
+    modes: list[str],
+    questions: list[str],
+    n_samples: int,
+) -> str:
+    lines: list[str] = []
+    n_runs = len(rows)
+    lines.append(f"# aborist QA-quality benchmark — {started_utc}")
+    lines.append("")
+    lines.append(
+        f"questions: {len(questions)} · modes: {len(modes)} · samples per cell: "
+        f"{n_samples} · runs: {n_runs}"
+    )
+    lines.append("")
+    lines.append("## summary")
+    lines.append("")
+    lines.append("| mode | runs | STRICT | HYBRID | UNGROUNDED | err | strict-rate | mean ratio | mean latency |")
+    lines.append("|------|------|--------|--------|------------|-----|-------------|-----------|--------------|")
+    for mode in modes:
+        b = summary.get(mode)
+        if not b:
+            continue
+        n = b["n"] or 1
+        mean_ratio = b["ratio_sum"] / n
+        mean_lat = b["latency_sum"] / n
+        strict_rate = b["STRICT"] / n
+        lines.append(
+            f"| {mode} | {b['n']} | {b['STRICT']} | {b['HYBRID']} | "
+            f"{b['UNGROUNDED']} | {b['errors']} | "
+            f"{strict_rate:.2f} | {mean_ratio:.3f} | {mean_lat:.1f}s |"
+        )
+    lines.append("")
+    lines.append("## per question")
+    lines.append("")
+    lines.append(
+        "Cell shows verdict counts across N samples (S=STRICT, H=HYBRID, U=UNGROUNDED, e=err). "
+        "median ratio = median n_verified/n_quotes."
+    )
+    lines.append("")
+    cells = _per_question_summary(rows)
+    lines.append("| question | mode | verdicts (N=" + str(n_samples) + ") | median ratio | median latency |")
+    lines.append("|----------|------|---------------------|--------------|----------------|")
+    for q in questions:
+        for mode in modes:
+            c = cells.get((q, mode))
+            if not c:
+                continue
+            verdicts = (
+                f"S:{c['STRICT']} H:{c['HYBRID']} U:{c['UNGROUNDED']}"
+                + (f" e:{c['errors']}" if c["errors"] else "")
+            )
+            mr = _median(c["ratios"])
+            ml = _median(c["latencies"])
+            qd = q if len(q) <= 50 else q[:47] + "..."
+            lines.append(f"| {qd} | {mode} | {verdicts} | {mr:.3f} | {ml:.1f}s |")
+    lines.append("")
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--questions", type=Path, default=Path("bench/qa_questions.txt"))
+    ap.add_argument("--shards-dir", type=Path, default=Path.home() / ".aborist" / "shards")
+    ap.add_argument("--qa-db", type=Path, default=None)
+    ap.add_argument("--out-dir", type=Path, default=Path("bench/qa_results"))
+    ap.add_argument("--top-k", type=int, default=8)
+    ap.add_argument("--modes", default=",".join(ANSWER_MODES),
+                    help="comma-separated subset of ANSWER_MODES to sweep")
+    ap.add_argument("--n", dest="n_samples", type=int, default=3,
+                    help="samples per (question, mode); each sample burns the cached "
+                         "record so Hermes nondeterminism becomes the variance source")
+    ap.add_argument("--limit", type=int, default=0,
+                    help="truncate question list to N; 0 = all")
+    ap.add_argument("--endpoint", default=os.environ.get(
+        "ABORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1"))
+    ap.add_argument("--model", default=os.environ.get(
+        "ABORIST_LLM_MODEL", "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"))
+    args = ap.parse_args(argv)
+
+    questions = _read_questions(args.questions)
+    if args.limit:
+        questions = questions[: args.limit]
+    if not questions:
+        print(f"no questions in {args.questions}", file=sys.stderr)
+        return 2
+
+    modes = [m.strip() for m in args.modes.split(",") if m.strip()]
+    bad = [m for m in modes if m not in ANSWER_MODES]
+    if bad:
+        print(f"unknown mode(s): {bad}; expected subset of {list(ANSWER_MODES)}", file=sys.stderr)
+        return 2
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    stamp = _utc_iso_compact()
+    jsonl_path = args.out_dir / f"{stamp}.jsonl"
+    md_path = args.out_dir / f"{stamp}.md"
+
+    if args.n_samples < 1:
+        print(f"--n must be >= 1, got {args.n_samples}", file=sys.stderr)
+        return 2
+
+    n_runs_total = len(questions) * len(modes) * args.n_samples
+    print(
+        f"[bench] {len(questions)} question(s) × {len(modes)} mode(s) × "
+        f"{args.n_samples} sample(s) = {n_runs_total} run(s)"
+    )
+    print(f"[bench] shards_dir={args.shards_dir}  top_k={args.top_k}  "
+          f"burn=always (per-sample, forces fresh inference)")
+    print(f"[bench] writing → {jsonl_path}")
+    print()
+
+    rows: list[dict] = []
+    with jsonl_path.open("w", encoding="utf-8") as f:
+        for q in questions:
+            for mode in modes:
+                for sample_idx in range(args.n_samples):
+                    tag = f"{mode:<22} #{sample_idx + 1}/{args.n_samples}"
+                    print(f"  [{tag}] {q[:60]}", flush=True)
+                    row = _run_one(
+                        question=q,
+                        answer_mode=mode,
+                        shards_dir=args.shards_dir,
+                        qa_db=args.qa_db,
+                        top_k=args.top_k,
+                        burn=True,
+                        endpoint=args.endpoint,
+                        model=args.model,
+                    )
+                    row["sample_idx"] = sample_idx
+                    rows.append(row)
+                    f.write(json.dumps(row, ensure_ascii=False) + "\n")
+                    f.flush()
+                    tag = (
+                        f"err: {row['error']}" if row["error"]
+                        else f"{row['audit_mode']}  {row['n_verified']}/{row['n_quotes']}  {row['elapsed_s']:.1f}s"
+                    )
+                    print(f"     → {tag}", flush=True)
+
+    summary = _summarize(rows)
+    md = _render_markdown(rows, summary, stamp, modes, questions, args.n_samples)
+    md_path.write_text(md, encoding="utf-8")
+
+    print()
+    print(md)
+    print(f"\n[bench] results: {jsonl_path}\n[bench] summary: {md_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())