russell@unturf.com
f927298353
Per-call accounting of prompt capacity so we can answer 'does
context size affect strict-rate?' at aggregate scale.
Per-call result dict gains:
"prompt_chars": {
"system_prompt": <int>, # system message chars
"grounding_reminder": <int>, # restated rule message (if any)
"user_question": <int>, # the original question text
"evidence_or_context":<int>, # rendered evidence map / wikitext context
"messages_total": <int>, # sum of all messages[].content
}
"answer_chars": <int> # rendered answer length
Char-level for now: model-agnostic, fast, and a workable proxy
for prompt tokens (~4 chars/token English prose, ~2.5 for JSON-
evidence-block heavy contexts). Token-exact accounting can layer
on later if a tokenizer is cheap enough to wire in per-call.
Surfaces in three places:
(1) result["prompt_chars"] — for programmatic callers. Cache hits
return prompt_chars too (computed before lookup).
(2) `aborist query` human render — one-liner showing the breakdown
and the answer length, so an operator can tell at a glance
whether STRICT came from a 5KB tight prompt or a 60KB stuffed
context.
(3) bench/qa_sweep.py — per-row prompt_chars_{total,evidence,system,
question} + answer_chars columns in the JSONL, plus a new
"strict-rate by prompt size" section in the markdown summary
bucketing runs by capacity (<8KB, 8-16KB, 16-32KB, 32-64KB,
>=64KB) so the bench reveals whether input size correlates
with verdict quality across modes.
Never enters cache_key — runtime measurements, not policy.
393 lines
15 KiB
Python
393 lines
15 KiB
Python
"""QA-quality benchmark sweep.
|
||
|
||
Runs a fixed question set through ``aborist.qa.query.query`` under each
|
||
answer mode and tabulates STRICT / HYBRID / UNGROUNDED counts, the
|
||
``n_verified / n_quotes`` ratio, latency, and lazy-anchor signals.
|
||
|
||
Why this exists: G0 / claim-lattice-pointer mode landed in 2337b77, and
|
||
four hardenings stacked on top (smell sidecar, positive-form prompts,
|
||
source-role boost, title-purity rerank, CITATION_MISMATCH check). Every
|
||
new heuristic flies blind until we can show "this commit moved STRICT
|
||
from N to M". This is that scaffolding.
|
||
|
||
Outputs:
|
||
|
||
bench/qa_results/<utc-iso>.jsonl one row per (mode, question)
|
||
bench/qa_results/<utc-iso>.md markdown summary table
|
||
|
||
The JSONL is the durable artifact. The markdown is the thing fox reads.
|
||
|
||
Wire via ``make bench-qa``. Honors ``BURN=1`` to force fresh inference
|
||
under the current ``governance_policy_hash`` — useful after a prompt
|
||
change that doesn't bump any version field.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import datetime as _dt
|
||
import json
|
||
import os
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
# Defer aborist imports until argparse runs so `--help` works without
|
||
# the package installed.
|
||
|
||
ANSWER_MODES = ("quote", "claim_lattice_pointer", "claim_lattice")
|
||
|
||
|
||
def _read_questions(path: Path) -> list[str]:
|
||
out: list[str] = []
|
||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||
line = raw.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
out.append(line)
|
||
return out
|
||
|
||
|
||
def _utc_iso_compact() -> str:
|
||
# YYYY-MM-DDTHH-MM-SSZ — colon-free so it's filesystem-safe.
|
||
return _dt.datetime.now(tz=_dt.timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")
|
||
|
||
|
||
def _ratio(verdict: dict) -> float:
|
||
nq = verdict.get("n_quotes") or 0
|
||
nv = verdict.get("n_verified") or 0
|
||
return (nv / nq) if nq else 0.0
|
||
|
||
|
||
def _run_one(
|
||
*,
|
||
question: str,
|
||
answer_mode: str,
|
||
shards_dir: Path | None,
|
||
qa_db: Path | None,
|
||
top_k: int,
|
||
burn: bool,
|
||
endpoint: str,
|
||
model: str,
|
||
) -> dict:
|
||
"""Run one (question, mode) — returns a flat record for the JSONL."""
|
||
from aborist.qa.client import OpenAICompatibleClient
|
||
from aborist.qa.query import DEFAULT_QUERY_POLICY, query
|
||
|
||
policy = dict(DEFAULT_QUERY_POLICY)
|
||
policy["answer_mode"] = answer_mode
|
||
|
||
api_key = os.environ.get("ABORIST_LLM_API_KEY")
|
||
client = OpenAICompatibleClient(base_url=endpoint, api_key=api_key)
|
||
|
||
if qa_db is None:
|
||
qa_db = (shards_dir / "qa.db") if shards_dir else (Path.home() / ".aborist" / "qa.db")
|
||
|
||
t0 = time.monotonic()
|
||
err: str | None = None
|
||
result: dict = {}
|
||
try:
|
||
result = query(
|
||
question=question,
|
||
qa_db=qa_db,
|
||
chat_client=client,
|
||
model_id=model,
|
||
shards_dir=shards_dir,
|
||
top_k=top_k,
|
||
policy=policy,
|
||
burn_existing=burn,
|
||
)
|
||
except Exception as e: # noqa: BLE001 — bench captures all
|
||
err = f"{type(e).__name__}: {e}"
|
||
elapsed_s = round(time.monotonic() - t0, 2)
|
||
|
||
# Deflection sidecar — sidecar-only signal, never feeds the
|
||
# verifier. Surfaces the Mars-BDFL pattern (STRICT but answer
|
||
# never mentions the question's subject) at bench-aggregate
|
||
# scale so a creeping "model deflects rather than refuses"
|
||
# regression is legible across runs.
|
||
from aborist.qa.inspect import diagnose_deflection
|
||
deflection = diagnose_deflection(question, result.get("answer_text") or "")
|
||
|
||
audit_mode = result.get("audit_mode")
|
||
return {
|
||
"question": question,
|
||
"answer_mode": answer_mode,
|
||
"status": result.get("status") or ("error" if err else "missing"),
|
||
"audit_mode": audit_mode,
|
||
"n_quotes": result.get("n_quotes"),
|
||
"n_verified": result.get("n_verified"),
|
||
"ratio": round(_ratio(result), 3),
|
||
"verifier_method": result.get("verifier_method"),
|
||
"lookup_path": result.get("lookup_path"),
|
||
"failure_stage": result.get("failure_stage"),
|
||
"lazy_anchor_ratio": result.get("lazy_anchor_ratio"),
|
||
"pointer_id_distribution": result.get("pointer_id_distribution"),
|
||
"cache_key": (result.get("cache_key") or "")[:12],
|
||
"n_sources": len(result.get("sources") or []),
|
||
"elapsed_s": elapsed_s,
|
||
"deflection_kind": deflection["kind"],
|
||
"subject_anchor": deflection["subject_anchor"],
|
||
"subject_in_answer": deflection["subject_in_answer"],
|
||
# Capacity metrics — char-level proxy for prompt-token budget.
|
||
# Surfaces "did STRICT come from a tight 5KB prompt or a 50KB
|
||
# context-stuffed one?" at aggregate scale. Lets the bench
|
||
# bucket strict-rate by input-size band.
|
||
"prompt_chars_total": (result.get("prompt_chars") or {}).get("messages_total", 0),
|
||
"prompt_chars_evidence": (result.get("prompt_chars") or {}).get("evidence_or_context", 0),
|
||
"prompt_chars_system": (result.get("prompt_chars") or {}).get("system_prompt", 0),
|
||
"prompt_chars_question": (result.get("prompt_chars") or {}).get("user_question", 0),
|
||
"answer_chars": result.get("answer_chars", 0),
|
||
"error": err,
|
||
}
|
||
|
||
|
||
def _summarize(rows: list[dict]) -> dict:
|
||
"""Group rows by mode → STRICT/HYBRID/UNGROUNDED totals + means."""
|
||
by_mode: dict[str, dict] = {}
|
||
for r in rows:
|
||
m = r["answer_mode"]
|
||
b = by_mode.setdefault(m, {
|
||
"n": 0,
|
||
"STRICT": 0,
|
||
"HYBRID": 0,
|
||
"UNGROUNDED": 0,
|
||
"errors": 0,
|
||
"ratio_sum": 0.0,
|
||
"latency_sum": 0.0,
|
||
"deflections": 0,
|
||
})
|
||
b["n"] += 1
|
||
if r["error"] or r["status"] == "error":
|
||
b["errors"] += 1
|
||
elif r["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED"):
|
||
b[r["audit_mode"]] += 1
|
||
b["ratio_sum"] += r["ratio"] or 0.0
|
||
b["latency_sum"] += r["elapsed_s"] or 0.0
|
||
# Deflection rate: STRICT/HYBRID with subject-anchor missing
|
||
# from answer. UNGROUNDED + deflection isn't interesting (no
|
||
# answer to deflect with), and no_question_tokens is vacuous.
|
||
if r.get("deflection_kind") == "deflection" and r["audit_mode"] in (
|
||
"STRICT", "HYBRID"
|
||
):
|
||
b["deflections"] += 1
|
||
return by_mode
|
||
|
||
|
||
def _per_question_summary(rows: list[dict]) -> dict:
|
||
"""Group rows by (question, mode) → per-cell vote counts + medians."""
|
||
cells: dict[tuple[str, str], dict] = {}
|
||
for r in rows:
|
||
key = (r["question"], r["answer_mode"])
|
||
c = cells.setdefault(key, {
|
||
"STRICT": 0, "HYBRID": 0, "UNGROUNDED": 0, "errors": 0,
|
||
"ratios": [], "latencies": [],
|
||
})
|
||
if r["error"] or r["status"] == "error":
|
||
c["errors"] += 1
|
||
elif r["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED"):
|
||
c[r["audit_mode"]] += 1
|
||
c["ratios"].append(r["ratio"] or 0.0)
|
||
c["latencies"].append(r["elapsed_s"] or 0.0)
|
||
return cells
|
||
|
||
|
||
def _median(xs: list[float]) -> float:
|
||
if not xs:
|
||
return 0.0
|
||
s = sorted(xs)
|
||
mid = len(s) // 2
|
||
return s[mid] if len(s) % 2 else (s[mid - 1] + s[mid]) / 2
|
||
|
||
|
||
def _render_markdown(
|
||
rows: list[dict],
|
||
summary: dict,
|
||
started_utc: str,
|
||
modes: list[str],
|
||
questions: list[str],
|
||
n_samples: int,
|
||
) -> str:
|
||
lines: list[str] = []
|
||
n_runs = len(rows)
|
||
lines.append(f"# aborist QA-quality benchmark — {started_utc}")
|
||
lines.append("")
|
||
lines.append(
|
||
f"questions: {len(questions)} · modes: {len(modes)} · samples per cell: "
|
||
f"{n_samples} · runs: {n_runs}"
|
||
)
|
||
lines.append("")
|
||
lines.append("## summary")
|
||
lines.append("")
|
||
lines.append("| mode | runs | STRICT | HYBRID | UNGROUNDED | err | strict-rate | mean ratio | mean latency | deflections |")
|
||
lines.append("|------|------|--------|--------|------------|-----|-------------|-----------|--------------|-------------|")
|
||
for mode in modes:
|
||
b = summary.get(mode)
|
||
if not b:
|
||
continue
|
||
n = b["n"] or 1
|
||
mean_ratio = b["ratio_sum"] / n
|
||
mean_lat = b["latency_sum"] / n
|
||
strict_rate = b["STRICT"] / n
|
||
deflections = b.get("deflections", 0)
|
||
lines.append(
|
||
f"| {mode} | {b['n']} | {b['STRICT']} | {b['HYBRID']} | "
|
||
f"{b['UNGROUNDED']} | {b['errors']} | "
|
||
f"{strict_rate:.2f} | {mean_ratio:.3f} | {mean_lat:.1f}s | "
|
||
f"{deflections}/{b['n']} |"
|
||
)
|
||
lines.append("")
|
||
lines.append("## strict-rate by prompt size")
|
||
lines.append("")
|
||
lines.append(
|
||
"Buckets capacity in `messages_total` chars (system + reminder + "
|
||
"evidence/context + question). ~4 chars/token English prose; "
|
||
"~2.5 chars/token JSON-evidence-heavy. Tells you whether the "
|
||
"model strict-rate degrades with input size."
|
||
)
|
||
lines.append("")
|
||
buckets = [
|
||
("<8KB", 0, 8000),
|
||
("8-16KB", 8000, 16000),
|
||
("16-32KB", 16000, 32000),
|
||
("32-64KB", 32000, 64000),
|
||
(">=64KB", 64000, 10**9),
|
||
]
|
||
lines.append("| mode | bucket | runs | STRICT | strict-rate | mean evidence chars | mean answer chars |")
|
||
lines.append("|------|--------|------|--------|-------------|---------------------|-------------------|")
|
||
for mode in modes:
|
||
for label, lo, hi in buckets:
|
||
sub = [
|
||
r for r in rows
|
||
if r["answer_mode"] == mode
|
||
and lo <= (r.get("prompt_chars_total") or 0) < hi
|
||
and not r["error"] and r["status"] != "error"
|
||
]
|
||
if not sub:
|
||
continue
|
||
n_b = len(sub)
|
||
n_strict = sum(1 for r in sub if r["audit_mode"] == "STRICT")
|
||
mean_ev = sum((r.get("prompt_chars_evidence") or 0) for r in sub) / n_b
|
||
mean_ans = sum((r.get("answer_chars") or 0) for r in sub) / n_b
|
||
lines.append(
|
||
f"| {mode} | {label} | {n_b} | {n_strict} | "
|
||
f"{n_strict / n_b:.2f} | {int(mean_ev)} | {int(mean_ans)} |"
|
||
)
|
||
lines.append("")
|
||
lines.append("## per question")
|
||
lines.append("")
|
||
lines.append(
|
||
"Cell shows verdict counts across N samples (S=STRICT, H=HYBRID, U=UNGROUNDED, e=err). "
|
||
"median ratio = median n_verified/n_quotes."
|
||
)
|
||
lines.append("")
|
||
cells = _per_question_summary(rows)
|
||
lines.append("| question | mode | verdicts (N=" + str(n_samples) + ") | median ratio | median latency |")
|
||
lines.append("|----------|------|---------------------|--------------|----------------|")
|
||
for q in questions:
|
||
for mode in modes:
|
||
c = cells.get((q, mode))
|
||
if not c:
|
||
continue
|
||
verdicts = (
|
||
f"S:{c['STRICT']} H:{c['HYBRID']} U:{c['UNGROUNDED']}"
|
||
+ (f" e:{c['errors']}" if c["errors"] else "")
|
||
)
|
||
mr = _median(c["ratios"])
|
||
ml = _median(c["latencies"])
|
||
qd = q if len(q) <= 50 else q[:47] + "..."
|
||
lines.append(f"| {qd} | {mode} | {verdicts} | {mr:.3f} | {ml:.1f}s |")
|
||
lines.append("")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
ap = argparse.ArgumentParser(description=__doc__)
|
||
ap.add_argument("--questions", type=Path, default=Path("bench/qa_questions.txt"))
|
||
ap.add_argument("--shards-dir", type=Path, default=Path.home() / ".aborist" / "shards")
|
||
ap.add_argument("--qa-db", type=Path, default=None)
|
||
ap.add_argument("--out-dir", type=Path, default=Path("bench/qa_results"))
|
||
ap.add_argument("--top-k", type=int, default=8)
|
||
ap.add_argument("--modes", default=",".join(ANSWER_MODES),
|
||
help="comma-separated subset of ANSWER_MODES to sweep")
|
||
ap.add_argument("--n", dest="n_samples", type=int, default=3,
|
||
help="samples per (question, mode); each sample burns the cached "
|
||
"record so Hermes nondeterminism becomes the variance source")
|
||
ap.add_argument("--limit", type=int, default=0,
|
||
help="truncate question list to N; 0 = all")
|
||
ap.add_argument("--endpoint", default=os.environ.get(
|
||
"ABORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1"))
|
||
ap.add_argument("--model", default=os.environ.get(
|
||
"ABORIST_LLM_MODEL", "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"))
|
||
args = ap.parse_args(argv)
|
||
|
||
questions = _read_questions(args.questions)
|
||
if args.limit:
|
||
questions = questions[: args.limit]
|
||
if not questions:
|
||
print(f"no questions in {args.questions}", file=sys.stderr)
|
||
return 2
|
||
|
||
modes = [m.strip() for m in args.modes.split(",") if m.strip()]
|
||
bad = [m for m in modes if m not in ANSWER_MODES]
|
||
if bad:
|
||
print(f"unknown mode(s): {bad}; expected subset of {list(ANSWER_MODES)}", file=sys.stderr)
|
||
return 2
|
||
|
||
args.out_dir.mkdir(parents=True, exist_ok=True)
|
||
stamp = _utc_iso_compact()
|
||
jsonl_path = args.out_dir / f"{stamp}.jsonl"
|
||
md_path = args.out_dir / f"{stamp}.md"
|
||
|
||
if args.n_samples < 1:
|
||
print(f"--n must be >= 1, got {args.n_samples}", file=sys.stderr)
|
||
return 2
|
||
|
||
n_runs_total = len(questions) * len(modes) * args.n_samples
|
||
print(
|
||
f"[bench] {len(questions)} question(s) × {len(modes)} mode(s) × "
|
||
f"{args.n_samples} sample(s) = {n_runs_total} run(s)"
|
||
)
|
||
print(f"[bench] shards_dir={args.shards_dir} top_k={args.top_k} "
|
||
f"burn=always (per-sample, forces fresh inference)")
|
||
print(f"[bench] writing → {jsonl_path}")
|
||
print()
|
||
|
||
rows: list[dict] = []
|
||
with jsonl_path.open("w", encoding="utf-8") as f:
|
||
for q in questions:
|
||
for mode in modes:
|
||
for sample_idx in range(args.n_samples):
|
||
tag = f"{mode:<22} #{sample_idx + 1}/{args.n_samples}"
|
||
print(f" [{tag}] {q[:60]}", flush=True)
|
||
row = _run_one(
|
||
question=q,
|
||
answer_mode=mode,
|
||
shards_dir=args.shards_dir,
|
||
qa_db=args.qa_db,
|
||
top_k=args.top_k,
|
||
burn=True,
|
||
endpoint=args.endpoint,
|
||
model=args.model,
|
||
)
|
||
row["sample_idx"] = sample_idx
|
||
rows.append(row)
|
||
f.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||
f.flush()
|
||
tag = (
|
||
f"err: {row['error']}" if row["error"]
|
||
else f"{row['audit_mode']} {row['n_verified']}/{row['n_quotes']} {row['elapsed_s']:.1f}s"
|
||
)
|
||
print(f" → {tag}", flush=True)
|
||
|
||
summary = _summarize(rows)
|
||
md = _render_markdown(rows, summary, stamp, modes, questions, args.n_samples)
|
||
md_path.write_text(md, encoding="utf-8")
|
||
|
||
print()
|
||
print(md)
|
||
print(f"\n[bench] results: {jsonl_path}\n[bench] summary: {md_path}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|