diff --git a/.gitignore b/.gitignore index be30f16..0add393 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ data/ # bench artifacts bench/qa_results/ +bench/cross_model_results/ # Claude Code session-local artifacts (worktrees, transient state) .claude/ diff --git a/Makefile b/Makefile index e1c5411..5474568 100644 --- a/Makefile +++ b/Makefile @@ -427,6 +427,24 @@ bench-emergent-pending: bootstrap ## print log entries awaiting teacher review $(PY) scripts/bench_emergent.py --print-pending +# Cross-model self-play — same question through multiple models, tabulate +# audit_mode, agreement, $/grounded. Drives the "ask twice for two +# options" pattern that uncloseai-cli's tool_arborist supports. +BENCH_CM_QUESTIONS ?= bench/qa_questions_smoke.txt +BENCH_CM_OUT ?= bench/cross_model_results +BENCH_CM_TIMEOUT ?= 180 +BENCH_CM_LIMIT ?= 0 +bench-cross-model: bootstrap ## cross-model self-play (Hermes + Qwen on smoke fixture) [BENCH_CM_QUESTIONS=... LIMIT=N BURN=1] + PYTHONUNBUFFERED=1 $(PY) bench/cross_model_selfplay.py \ + --questions $(BENCH_CM_QUESTIONS) \ + --shards-dir $(SHARDS_DIR) \ + --out-dir $(BENCH_CM_OUT) \ + --top-k $(QUERY_TOP_K) \ + --timeout $(BENCH_CM_TIMEOUT) \ + --limit $(BENCH_CM_LIMIT) \ + $(if $(BURN),--burn,) + + bench-5s: bootstrap ## 5S battery (Syntax+Semantics+Syllogism+Synthesis+Semiotics) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax --fixtures bench/fixtures/5s/syntax-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics --fixtures bench/fixtures/5s/semantics-v1.jsonl diff --git a/bench/cross_model_selfplay.py b/bench/cross_model_selfplay.py new file mode 100644 index 0000000..c1b5623 --- /dev/null +++ b/bench/cross_model_selfplay.py @@ -0,0 +1,390 @@ +"""Cross-model self-play bench — same question, multiple models, compare. + +The pattern: the uncloseai-cli harness can call arborist twice per +question with different models to get two opinions. This script bakes +that pattern in as a benchmark. For each question in a fixture, run +arborist once per configured model, then tabulate: + + * audit_mode per model (EVIDENCE-WARRANTED → ANCHOR-WARRANTED → + POINTER-LINKED → UNGROUNDED; HYBRID variants flatten to base) + * agreement on primary source URI + * grounding rate per model + * estimated $/grounded-answer (Hermes $0.09, Qwen $0.16 per call — + override via --price model=cents,model=cents) + * latency per call + +Outputs: + + bench/cross_model_results/.jsonl one row per (question, model) + bench/cross_model_results/.md markdown summary + +The cost-per-grounded-answer column is the score that matters: cheap +calls that ground are wins; expensive calls that ground are wins worth +defending; ungrounded calls are wasted spend. + +Usage: + + make bench-cross-model # smoke fixture, Hermes + Qwen + make bench-cross-model BENCH_CM_QUESTIONS=bench/qa_questions.txt + + # custom model list (name=endpoint:model_id, comma-separated): + python3 bench/cross_model_selfplay.py \\ + --models 'hermes=https://hermes.ai.unturf.com/v1:adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic,qwen=https://qwen.ai.unturf.com/v1:Qwen3.6-27B-UD-Q4_K_XL.gguf' +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import json +import os +import subprocess +import sys +import time +from collections import defaultdict +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +ARBORIST = REPO / ".venv" / "bin" / "arborist" + +DEFAULT_MODELS = [ + ("hermes", "https://hermes.ai.unturf.com/v1", + "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"), + ("qwen", "https://qwen.ai.unturf.com/v1", + "Qwen3.6-27B-UD-Q4_K_XL.gguf"), +] +# Per fox 2026-05-31 — average cost per grounded answer at our infra +# scale. Hermes (8B) on 3090, Qwen (30B) on 4090. Override via --price. +DEFAULT_PRICE_CENTS = {"hermes": 9.0, "qwen": 16.0} + +# audit_mode → numeric rank for "stronger" comparison. Strips +# "-PARTIAL" suffix that HYBRID modes carry. +_AUDIT_RANK = { + "EVIDENCE-WARRANTED": 3, + "ANCHOR-WARRANTED": 2, + "POINTER-LINKED": 1, + "UNGROUNDED": 0, + "STRICT": 2, # legacy alias for ANCHOR-WARRANTED + "HYBRID": 1, # legacy alias for POINTER-LINKED +} + + +def _audit_rank(mode: str | None) -> int: + if not mode: + return 0 + base = mode.replace("-PARTIAL", "") + return _AUDIT_RANK.get(base, 0) + + +def _is_grounded(mode: str | None) -> bool: + """Anything not UNGROUNDED counts as grounded for cost accounting.""" + if not mode: + return False + return not mode.startswith("UNGROUNDED") + + +def _read_questions(path: Path) -> list[str]: + out: list[str] = [] + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + out.append(line) + return out + + +def _parse_models(spec: str) -> list[tuple[str, str, str]]: + """Parse name=endpoint:model_id,... into a list of tuples.""" + out = [] + for chunk in spec.split(","): + chunk = chunk.strip() + if not chunk: + continue + if "=" not in chunk or ":" not in chunk.split("=", 1)[1]: + raise SystemExit( + f"--models entry must be name=endpoint:model_id, got: {chunk!r}" + ) + name, rest = chunk.split("=", 1) + # split on the LAST colon — model_id may contain colons (rare) + # but endpoint always ends at the path boundary, so split on + # the colon BEFORE a non-slash. Simpler: assume model_id + # follows the first colon AFTER the endpoint scheme://host[:port]/path. + # We split on " :" sentinel — but easiest: rsplit once on `:` + # because vLLM model_ids don't contain colons in practice. + endpoint, model_id = rest.rsplit(":", 1) + out.append((name.strip(), endpoint.strip(), model_id.strip())) + return out + + +def _parse_price(spec: str | None) -> dict[str, float]: + if not spec: + return dict(DEFAULT_PRICE_CENTS) + out = dict(DEFAULT_PRICE_CENTS) + for chunk in spec.split(","): + if "=" not in chunk: + continue + name, cents = chunk.split("=", 1) + try: + out[name.strip()] = float(cents.strip()) + except ValueError: + pass + return out + + +def _run_one( + question: str, *, shards_dir: Path | None, endpoint: str, model: str, + top_k: int, timeout_s: int, burn: bool, +) -> dict: + cmd = [str(ARBORIST)] + if shards_dir: + cmd += ["--shards-dir", str(shards_dir)] + cmd += [ + "query", "--json", + "--top-k", str(top_k), + "--endpoint", endpoint, + "--model", model, + ] + if burn: + cmd.append("--burn") + cmd.append(question) + t0 = time.time() + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout_s, + env={**os.environ, "ARBORIST_PROGRESS": "0"}, + ) + except subprocess.TimeoutExpired: + return {"_error": f"timeout after {timeout_s}s", + "_elapsed_s": float(timeout_s)} + elapsed = round(time.time() - t0, 2) + if proc.returncode not in (0, 1): + # Exit 1 is "ungrounded" — still has JSON. Other codes are real errors. + return {"_error": f"exit {proc.returncode}", + "_stderr": (proc.stderr or "")[:500], + "_elapsed_s": elapsed} + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError as e: + return {"_error": f"parse: {e}", + "_stdout_head": proc.stdout[:200], + "_elapsed_s": elapsed} + data["_elapsed_s"] = elapsed + data["_exit_code"] = proc.returncode + return data + + +def _primary_uri(result: dict) -> str: + srcs = result.get("sources") or [] + primary = next( + (s for s in srcs if s.get("source_role") == "primary_answer_source"), + srcs[0] if srcs else {}, + ) + return primary.get("document_uri") or "" + + +def _stronger_model(per_model: dict[str, dict]) -> str: + """Return the name of the model with the higher audit_mode rank + (ties broken by alpha for stability). Empty string if all ungrounded.""" + best = "" + best_rank = -1 + for name, res in sorted(per_model.items()): + r = _audit_rank(res.get("audit_mode")) + if r > best_rank: + best, best_rank = name, r + return best if best_rank > 0 else "" + + +def _summarize( + rows: list[dict], models: list[tuple[str, str, str]], + price_cents: dict[str, float], +) -> str: + """Build the markdown summary.""" + by_q: dict[str, dict[str, dict]] = defaultdict(dict) + for r in rows: + by_q[r["question"]][r["model_name"]] = r["result"] + + n_q = len(by_q) + model_names = [m[0] for m in models] + out = [] + out.append(f"# Cross-model self-play bench — {_dt.datetime.now(_dt.timezone.utc).isoformat()}") + out.append("") + out.append(f"Questions: {n_q}. Models: {', '.join(model_names)}.") + out.append("") + + # Per-model summary + out.append("## Per-model summary") + out.append("") + out.append("| model | grounded | rate | avg latency | est $/grounded |") + out.append("|---|---:|---:|---:|---:|") + for name in model_names: + grounded = sum( + 1 for q in by_q if _is_grounded(by_q[q].get(name, {}).get("audit_mode")) + ) + total = sum(1 for q in by_q if name in by_q[q] and "_error" not in by_q[q][name]) + rate = (grounded / total) if total else 0.0 + avg_lat = ( + sum(by_q[q].get(name, {}).get("_elapsed_s", 0) for q in by_q) / max(total, 1) + ) + per_call_cents = price_cents.get(name, 0.0) + cost_per_grounded = (per_call_cents * total / grounded) if grounded else float("nan") + cost_str = "n/a" if cost_per_grounded != cost_per_grounded else f"${cost_per_grounded / 100:.2f}" + out.append( + f"| {name} | {grounded}/{total} | {rate:.0%} " + f"| {avg_lat:.1f}s | {cost_str} |" + ) + out.append("") + + # Agreement + out.append("## Per-question results") + out.append("") + header = ["question"] + model_names + ["stronger", "primary agree"] + out.append("| " + " | ".join(header) + " |") + out.append("|" + "|".join(["---"] * len(header)) + "|") + for q in by_q: + cells = [q[:64] + ("…" if len(q) > 64 else "")] + uris = set() + for name in model_names: + res = by_q[q].get(name, {}) + audit = res.get("audit_mode") or "—" + if "_error" in res: + audit = f"ERR ({res['_error'][:30]})" + cells.append(audit) + uri = _primary_uri(res) + if uri: + uris.add(uri) + cells.append(_stronger_model(by_q[q]) or "—") + cells.append("✓" if len(uris) == 1 else ("✗" if len(uris) > 1 else "—")) + out.append("| " + " | ".join(cells) + " |") + out.append("") + + # Cost analysis + out.append("## Cost analysis (north star: $/grounded)") + out.append("") + total_cost = 0.0 + total_grounded = 0 + for name in model_names: + per_call_cents = price_cents.get(name, 0.0) + n_calls = sum(1 for q in by_q if name in by_q[q] and "_error" not in by_q[q][name]) + n_g = sum(1 for q in by_q if _is_grounded(by_q[q].get(name, {}).get("audit_mode"))) + spent = per_call_cents * n_calls / 100 + total_cost += spent + total_grounded += n_g + out.append(f"* **{name}** — {n_calls} calls × ${per_call_cents/100:.2f} = ${spent:.2f}, {n_g} grounded") + out.append("") + out.append(f"**Total spend**: ${total_cost:.2f}. **Total grounded answers**: {total_grounded}.") + if total_grounded: + out.append(f"**Effective $/grounded** (sum of all model spend / total grounded answers): ${total_cost / total_grounded:.3f}") + out.append("") + + # Routing hint — what cheap-first cascade would have cost + out.append("## Cheap-first cascade (try cheapest, escalate on UNGROUNDED)") + out.append("") + ordered = sorted(model_names, key=lambda n: price_cents.get(n, 0)) + cascade_cost = 0.0 + cascade_grounded = 0 + for q in by_q: + for i, name in enumerate(ordered): + res = by_q[q].get(name, {}) + if "_error" in res: + continue + cascade_cost += price_cents.get(name, 0) / 100 + if _is_grounded(res.get("audit_mode")): + cascade_grounded += 1 + break + # else fall through to next model + if cascade_grounded: + out.append( + f"Cascade ({' → '.join(ordered)}): ${cascade_cost:.2f} for " + f"{cascade_grounded}/{n_q} grounded — " + f"${cascade_cost / cascade_grounded:.3f}/grounded." + ) + out.append("") + out.append("Cascade beats always-Qwen when the cheaper model grounds often enough that escalation is rare. Compare to per-model rows above.") + out.append("") + return "\n".join(out) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument( + "--questions", type=Path, + default=Path("bench/qa_questions_smoke.txt"), + ) + ap.add_argument("--shards-dir", type=Path, default=None) + ap.add_argument( + "--models", default=None, + help="comma-separated name=endpoint:model_id; defaults to hermes + qwen", + ) + ap.add_argument( + "--price", default=None, + help="comma-separated name=cents per call; defaults to hermes=9,qwen=16", + ) + ap.add_argument("--top-k", type=int, default=8) + ap.add_argument("--timeout", type=int, default=180) + ap.add_argument( + "--burn", action="store_true", + help="force fresh inference (skip cache) on every call", + ) + ap.add_argument("--limit", type=int, default=0, help="cap questions (0 = no cap)") + ap.add_argument( + "--out-dir", type=Path, default=Path("bench/cross_model_results"), + ) + args = ap.parse_args() + + questions = _read_questions(args.questions) + if args.limit: + questions = questions[: args.limit] + if not questions: + print(f"no questions in {args.questions}", file=sys.stderr) + return 1 + + if args.models: + models = _parse_models(args.models) + else: + models = list(DEFAULT_MODELS) + price = _parse_price(args.price) + + if not ARBORIST.exists(): + print(f"arborist binary not found: {ARBORIST}", file=sys.stderr) + return 2 + + args.out_dir.mkdir(parents=True, exist_ok=True) + ts = _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + jsonl_path = args.out_dir / f"{ts}.jsonl" + md_path = args.out_dir / f"{ts}.md" + + rows: list[dict] = [] + print(f"# {len(questions)} questions × {len(models)} models = " + f"{len(questions) * len(models)} calls", file=sys.stderr) + with jsonl_path.open("w") as f: + for q in questions: + for name, endpoint, model_id in models: + print(f" [{name}] {q[:80]}", file=sys.stderr) + res = _run_one( + q, shards_dir=args.shards_dir, + endpoint=endpoint, model=model_id, + top_k=args.top_k, timeout_s=args.timeout, burn=args.burn, + ) + row = { + "question": q, + "model_name": name, + "model_id": model_id, + "endpoint": endpoint, + "result": res, + } + rows.append(row) + f.write(json.dumps(row, ensure_ascii=False) + "\n") + f.flush() + audit = res.get("audit_mode") or res.get("_error") or "?" + elapsed = res.get("_elapsed_s", "?") + print(f" → {audit} ({elapsed}s)", file=sys.stderr) + + md_path.write_text(_summarize(rows, models, price)) + print(f"\nJSONL: {jsonl_path}", file=sys.stderr) + print(f"Summary: {md_path}", file=sys.stderr) + print() + print(md_path.read_text()) + return 0 + + +if __name__ == "__main__": + sys.exit(main())