diff --git a/.gitignore b/.gitignore index 37d21cb..e3461d6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ data/ *.db-journal *.db-wal *.db-shm + +# bench artifacts +bench/qa_results/ diff --git a/Makefile b/Makefile index 4c64ff2..26538df 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ SEARCH_Q ?= computer ingest-self ingest-git ingest-hg \ verify search stats test docs chain-check chain-check-shards \ falsify burn burn-kindergarten inspect bootstrap-crawler test-crawler crawl-ingest \ - recrawl-check clean clean-db clean-data help + recrawl-check bench-qa clean clean-db clean-data help all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats @@ -173,6 +173,21 @@ query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON= fi $(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) "$(Q)" +BENCH_QA_QUESTIONS ?= bench/qa_questions.txt +BENCH_QA_OUT ?= bench/qa_results +BENCH_QA_MODES ?= quote,claim_lattice_pointer +BENCH_QA_LIMIT ?= 0 +BENCH_QA_N ?= 3 +bench-qa: bootstrap ## QA-quality sweep: questions × modes × N samples [BENCH_QA_N=3 BENCH_QA_LIMIT=N BENCH_QA_MODES=...] + PYTHONUNBUFFERED=1 $(PY) bench/qa_sweep.py \ + --questions $(BENCH_QA_QUESTIONS) \ + --shards-dir $(SHARDS_DIR) \ + --out-dir $(BENCH_QA_OUT) \ + --top-k $(QUERY_TOP_K) \ + --modes $(BENCH_QA_MODES) \ + --limit $(BENCH_QA_LIMIT) \ + --n $(BENCH_QA_N) + verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample $(ABORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N) diff --git a/bench/qa_questions.txt b/bench/qa_questions.txt new file mode 100644 index 0000000..3fcaaf6 --- /dev/null +++ b/bench/qa_questions.txt @@ -0,0 +1,32 @@ +# aborist QA-quality benchmark question set. +# +# One question per line. `#`-prefixed lines and blank lines ignored. +# Designed for the 2003-05-16 Wikipedia cur snapshot — questions must +# resolve against ~2003 article content. Mix of failure-mode probes: +# +# narrow factoid — one-shot, evergreen, well-anchored +# broad descriptive — "tell me about X" — invites encyclopedic +# echo, the G0 mode-collapse trigger +# entity list — invites lazy-anchor on a magnet chunk +# relationship / multi — multi-pointer claim +# out-of-corpus — should land UNGROUNDED honestly +# +# Add/remove rows as we learn what the verifier should be measuring. + +# narrow factoid +who founded apple computer? +what is the capital of france? +who wrote the linux kernel? + +# broad descriptive +tell me about connecticut +tell me about the python programming language + +# entity list +list the founders of microsoft + +# relationship +who is supermans girlfriend? + +# out-of-corpus (should land UNGROUNDED, not hallucinate) +who is the prime minister of mars? diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py new file mode 100644 index 0000000..a69c639 --- /dev/null +++ b/bench/qa_sweep.py @@ -0,0 +1,326 @@ +"""QA-quality benchmark sweep. + +Runs a fixed question set through ``aborist.qa.query.query`` under each +answer mode and tabulates STRICT / HYBRID / UNGROUNDED counts, the +``n_verified / n_quotes`` ratio, latency, and lazy-anchor signals. + +Why this exists: G0 / claim-lattice-pointer mode landed in 2337b77, and +four hardenings stacked on top (smell sidecar, positive-form prompts, +source-role boost, title-purity rerank, CITATION_MISMATCH check). Every +new heuristic flies blind until we can show "this commit moved STRICT +from N to M". This is that scaffolding. + +Outputs: + + bench/qa_results/.jsonl one row per (mode, question) + bench/qa_results/.md markdown summary table + +The JSONL is the durable artifact. The markdown is the thing fox reads. + +Wire via ``make bench-qa``. Honors ``BURN=1`` to force fresh inference +under the current ``governance_policy_hash`` — useful after a prompt +change that doesn't bump any version field. +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import json +import os +import sys +import time +from pathlib import Path + +# Defer aborist imports until argparse runs so `--help` works without +# the package installed. + +ANSWER_MODES = ("quote", "claim_lattice_pointer") + + +def _read_questions(path: Path) -> list[str]: + out: list[str] = [] + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + out.append(line) + return out + + +def _utc_iso_compact() -> str: + # YYYY-MM-DDTHH-MM-SSZ — colon-free so it's filesystem-safe. + return _dt.datetime.now(tz=_dt.timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ") + + +def _ratio(verdict: dict) -> float: + nq = verdict.get("n_quotes") or 0 + nv = verdict.get("n_verified") or 0 + return (nv / nq) if nq else 0.0 + + +def _run_one( + *, + question: str, + answer_mode: str, + shards_dir: Path | None, + qa_db: Path | None, + top_k: int, + burn: bool, + endpoint: str, + model: str, +) -> dict: + """Run one (question, mode) — returns a flat record for the JSONL.""" + from aborist.qa.client import OpenAICompatibleClient + from aborist.qa.query import DEFAULT_QUERY_POLICY, query + + policy = dict(DEFAULT_QUERY_POLICY) + policy["answer_mode"] = answer_mode + + api_key = os.environ.get("ABORIST_LLM_API_KEY") + client = OpenAICompatibleClient(base_url=endpoint, api_key=api_key) + + if qa_db is None: + qa_db = (shards_dir / "qa.db") if shards_dir else (Path.home() / ".aborist" / "qa.db") + + t0 = time.monotonic() + err: str | None = None + result: dict = {} + try: + result = query( + question=question, + qa_db=qa_db, + chat_client=client, + model_id=model, + shards_dir=shards_dir, + top_k=top_k, + policy=policy, + burn_existing=burn, + ) + except Exception as e: # noqa: BLE001 — bench captures all + err = f"{type(e).__name__}: {e}" + elapsed_s = round(time.monotonic() - t0, 2) + + audit_mode = result.get("audit_mode") + return { + "question": question, + "answer_mode": answer_mode, + "status": result.get("status") or ("error" if err else "missing"), + "audit_mode": audit_mode, + "n_quotes": result.get("n_quotes"), + "n_verified": result.get("n_verified"), + "ratio": round(_ratio(result), 3), + "verifier_method": result.get("verifier_method"), + "lookup_path": result.get("lookup_path"), + "failure_stage": result.get("failure_stage"), + "lazy_anchor_ratio": result.get("lazy_anchor_ratio"), + "pointer_id_distribution": result.get("pointer_id_distribution"), + "cache_key": (result.get("cache_key") or "")[:12], + "n_sources": len(result.get("sources") or []), + "elapsed_s": elapsed_s, + "error": err, + } + + +def _summarize(rows: list[dict]) -> dict: + """Group rows by mode → STRICT/HYBRID/UNGROUNDED totals + means.""" + by_mode: dict[str, dict] = {} + for r in rows: + m = r["answer_mode"] + b = by_mode.setdefault(m, { + "n": 0, + "STRICT": 0, + "HYBRID": 0, + "UNGROUNDED": 0, + "errors": 0, + "ratio_sum": 0.0, + "latency_sum": 0.0, + }) + b["n"] += 1 + if r["error"] or r["status"] == "error": + b["errors"] += 1 + elif r["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED"): + b[r["audit_mode"]] += 1 + b["ratio_sum"] += r["ratio"] or 0.0 + b["latency_sum"] += r["elapsed_s"] or 0.0 + return by_mode + + +def _per_question_summary(rows: list[dict]) -> dict: + """Group rows by (question, mode) → per-cell vote counts + medians.""" + cells: dict[tuple[str, str], dict] = {} + for r in rows: + key = (r["question"], r["answer_mode"]) + c = cells.setdefault(key, { + "STRICT": 0, "HYBRID": 0, "UNGROUNDED": 0, "errors": 0, + "ratios": [], "latencies": [], + }) + if r["error"] or r["status"] == "error": + c["errors"] += 1 + elif r["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED"): + c[r["audit_mode"]] += 1 + c["ratios"].append(r["ratio"] or 0.0) + c["latencies"].append(r["elapsed_s"] or 0.0) + return cells + + +def _median(xs: list[float]) -> float: + if not xs: + return 0.0 + s = sorted(xs) + mid = len(s) // 2 + return s[mid] if len(s) % 2 else (s[mid - 1] + s[mid]) / 2 + + +def _render_markdown( + rows: list[dict], + summary: dict, + started_utc: str, + modes: list[str], + questions: list[str], + n_samples: int, +) -> str: + lines: list[str] = [] + n_runs = len(rows) + lines.append(f"# aborist QA-quality benchmark — {started_utc}") + lines.append("") + lines.append( + f"questions: {len(questions)} · modes: {len(modes)} · samples per cell: " + f"{n_samples} · runs: {n_runs}" + ) + lines.append("") + lines.append("## summary") + lines.append("") + lines.append("| mode | runs | STRICT | HYBRID | UNGROUNDED | err | strict-rate | mean ratio | mean latency |") + lines.append("|------|------|--------|--------|------------|-----|-------------|-----------|--------------|") + for mode in modes: + b = summary.get(mode) + if not b: + continue + n = b["n"] or 1 + mean_ratio = b["ratio_sum"] / n + mean_lat = b["latency_sum"] / n + strict_rate = b["STRICT"] / n + lines.append( + f"| {mode} | {b['n']} | {b['STRICT']} | {b['HYBRID']} | " + f"{b['UNGROUNDED']} | {b['errors']} | " + f"{strict_rate:.2f} | {mean_ratio:.3f} | {mean_lat:.1f}s |" + ) + lines.append("") + lines.append("## per question") + lines.append("") + lines.append( + "Cell shows verdict counts across N samples (S=STRICT, H=HYBRID, U=UNGROUNDED, e=err). " + "median ratio = median n_verified/n_quotes." + ) + lines.append("") + cells = _per_question_summary(rows) + lines.append("| question | mode | verdicts (N=" + str(n_samples) + ") | median ratio | median latency |") + lines.append("|----------|------|---------------------|--------------|----------------|") + for q in questions: + for mode in modes: + c = cells.get((q, mode)) + if not c: + continue + verdicts = ( + f"S:{c['STRICT']} H:{c['HYBRID']} U:{c['UNGROUNDED']}" + + (f" e:{c['errors']}" if c["errors"] else "") + ) + mr = _median(c["ratios"]) + ml = _median(c["latencies"]) + qd = q if len(q) <= 50 else q[:47] + "..." + lines.append(f"| {qd} | {mode} | {verdicts} | {mr:.3f} | {ml:.1f}s |") + lines.append("") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--questions", type=Path, default=Path("bench/qa_questions.txt")) + ap.add_argument("--shards-dir", type=Path, default=Path.home() / ".aborist" / "shards") + ap.add_argument("--qa-db", type=Path, default=None) + ap.add_argument("--out-dir", type=Path, default=Path("bench/qa_results")) + ap.add_argument("--top-k", type=int, default=8) + ap.add_argument("--modes", default=",".join(ANSWER_MODES), + help="comma-separated subset of ANSWER_MODES to sweep") + ap.add_argument("--n", dest="n_samples", type=int, default=3, + help="samples per (question, mode); each sample burns the cached " + "record so Hermes nondeterminism becomes the variance source") + ap.add_argument("--limit", type=int, default=0, + help="truncate question list to N; 0 = all") + ap.add_argument("--endpoint", default=os.environ.get( + "ABORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1")) + ap.add_argument("--model", default=os.environ.get( + "ABORIST_LLM_MODEL", "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic")) + args = ap.parse_args(argv) + + questions = _read_questions(args.questions) + if args.limit: + questions = questions[: args.limit] + if not questions: + print(f"no questions in {args.questions}", file=sys.stderr) + return 2 + + modes = [m.strip() for m in args.modes.split(",") if m.strip()] + bad = [m for m in modes if m not in ANSWER_MODES] + if bad: + print(f"unknown mode(s): {bad}; expected subset of {list(ANSWER_MODES)}", file=sys.stderr) + return 2 + + args.out_dir.mkdir(parents=True, exist_ok=True) + stamp = _utc_iso_compact() + jsonl_path = args.out_dir / f"{stamp}.jsonl" + md_path = args.out_dir / f"{stamp}.md" + + if args.n_samples < 1: + print(f"--n must be >= 1, got {args.n_samples}", file=sys.stderr) + return 2 + + n_runs_total = len(questions) * len(modes) * args.n_samples + print( + f"[bench] {len(questions)} question(s) × {len(modes)} mode(s) × " + f"{args.n_samples} sample(s) = {n_runs_total} run(s)" + ) + print(f"[bench] shards_dir={args.shards_dir} top_k={args.top_k} " + f"burn=always (per-sample, forces fresh inference)") + print(f"[bench] writing → {jsonl_path}") + print() + + rows: list[dict] = [] + with jsonl_path.open("w", encoding="utf-8") as f: + for q in questions: + for mode in modes: + for sample_idx in range(args.n_samples): + tag = f"{mode:<22} #{sample_idx + 1}/{args.n_samples}" + print(f" [{tag}] {q[:60]}", flush=True) + row = _run_one( + question=q, + answer_mode=mode, + shards_dir=args.shards_dir, + qa_db=args.qa_db, + top_k=args.top_k, + burn=True, + endpoint=args.endpoint, + model=args.model, + ) + row["sample_idx"] = sample_idx + rows.append(row) + f.write(json.dumps(row, ensure_ascii=False) + "\n") + f.flush() + tag = ( + f"err: {row['error']}" if row["error"] + else f"{row['audit_mode']} {row['n_verified']}/{row['n_quotes']} {row['elapsed_s']:.1f}s" + ) + print(f" → {tag}", flush=True) + + summary = _summarize(rows) + md = _render_markdown(rows, summary, stamp, modes, questions, args.n_samples) + md_path.write_text(md, encoding="utf-8") + + print() + print(md) + print(f"\n[bench] results: {jsonl_path}\n[bench] summary: {md_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())