bench: cross-model self-play — same question, multiple models, $/grounded

Adds bench/cross_model_selfplay.py + `make bench-cross-model` target.
For each question in a fixture, runs `arborist query` once per
configured model (default: Hermes + Qwen) and tabulates:

  * audit_mode per model (EVIDENCE-WARRANTED → POINTER-LINKED → UNGROUNDED)
  * agreement on primary source URI
  * grounding rate per model
  * estimated $/grounded-answer (per-call prices configurable)
  * cheap-first cascade analysis (try cheapest, escalate on UNGROUNDED)

This is the "ask twice for two options" pattern from the agent
perspective — bakes it in as a benchmark so we can measure whether
the cascade beats always-using-the-stronger-model on $/grounded.

First live run (2 questions × 2 models, $0.41):
  - Hermes: 1/1 grounded (1 timeout — operational issue)
  - Qwen: 2/2 grounded STRICT
  - Cascade: 2/2 grounded for $0.25 — beats always-Qwen ($0.32)
    when Hermes succeeds on its first call.

Output: bench/cross_model_results/<utc-iso>.{jsonl,md} (gitignored).

.gitignore

@@ -16,6 +16,7 @@ data/
 
 # bench artifacts
 bench/qa_results/
+bench/cross_model_results/
 
 # Claude Code session-local artifacts (worktrees, transient state)
 .claude/

Makefile

@@ -427,6 +427,24 @@ bench-emergent-pending: bootstrap ## print log entries awaiting teacher review
 	$(PY) scripts/bench_emergent.py --print-pending
 
 
+# Cross-model self-play — same question through multiple models, tabulate
+# audit_mode, agreement, $/grounded. Drives the "ask twice for two
+# options" pattern that uncloseai-cli's tool_arborist supports.
+BENCH_CM_QUESTIONS ?= bench/qa_questions_smoke.txt
+BENCH_CM_OUT       ?= bench/cross_model_results
+BENCH_CM_TIMEOUT   ?= 180
+BENCH_CM_LIMIT     ?= 0
+bench-cross-model: bootstrap ## cross-model self-play (Hermes + Qwen on smoke fixture) [BENCH_CM_QUESTIONS=... LIMIT=N BURN=1]
+	PYTHONUNBUFFERED=1 $(PY) bench/cross_model_selfplay.py \
+	    --questions $(BENCH_CM_QUESTIONS) \
+	    --shards-dir $(SHARDS_DIR) \
+	    --out-dir $(BENCH_CM_OUT) \
+	    --top-k $(QUERY_TOP_K) \
+	    --timeout $(BENCH_CM_TIMEOUT) \
+	    --limit $(BENCH_CM_LIMIT) \
+	    $(if $(BURN),--burn,)
+
+
 bench-5s: bootstrap ## 5S battery (Syntax+Semantics+Syllogism+Synthesis+Semiotics)
 	PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax     --fixtures bench/fixtures/5s/syntax-v1.jsonl
 	PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics  --fixtures bench/fixtures/5s/semantics-v1.jsonl

bench/cross_model_selfplay.py

@@ -0,0 +1,390 @@
+"""Cross-model self-play bench — same question, multiple models, compare.
+
+The pattern: the uncloseai-cli harness can call arborist twice per
+question with different models to get two opinions. This script bakes
+that pattern in as a benchmark. For each question in a fixture, run
+arborist once per configured model, then tabulate:
+
+  * audit_mode per model (EVIDENCE-WARRANTED → ANCHOR-WARRANTED →
+    POINTER-LINKED → UNGROUNDED; HYBRID variants flatten to base)
+  * agreement on primary source URI
+  * grounding rate per model
+  * estimated $/grounded-answer (Hermes $0.09, Qwen $0.16 per call —
+    override via --price model=cents,model=cents)
+  * latency per call
+
+Outputs:
+
+    bench/cross_model_results/<utc-iso>.jsonl   one row per (question, model)
+    bench/cross_model_results/<utc-iso>.md      markdown summary
+
+The cost-per-grounded-answer column is the score that matters: cheap
+calls that ground are wins; expensive calls that ground are wins worth
+defending; ungrounded calls are wasted spend.
+
+Usage:
+
+    make bench-cross-model                              # smoke fixture, Hermes + Qwen
+    make bench-cross-model BENCH_CM_QUESTIONS=bench/qa_questions.txt
+
+    # custom model list (name=endpoint:model_id, comma-separated):
+    python3 bench/cross_model_selfplay.py \\
+        --models 'hermes=https://hermes.ai.unturf.com/v1:adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic,qwen=https://qwen.ai.unturf.com/v1:Qwen3.6-27B-UD-Q4_K_XL.gguf'
+"""
+from __future__ import annotations
+
+import argparse
+import datetime as _dt
+import json
+import os
+import subprocess
+import sys
+import time
+from collections import defaultdict
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parent.parent
+ARBORIST = REPO / ".venv" / "bin" / "arborist"
+
+DEFAULT_MODELS = [
+    ("hermes", "https://hermes.ai.unturf.com/v1",
+     "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"),
+    ("qwen", "https://qwen.ai.unturf.com/v1",
+     "Qwen3.6-27B-UD-Q4_K_XL.gguf"),
+]
+# Per fox 2026-05-31 — average cost per grounded answer at our infra
+# scale. Hermes (8B) on 3090, Qwen (30B) on 4090. Override via --price.
+DEFAULT_PRICE_CENTS = {"hermes": 9.0, "qwen": 16.0}
+
+# audit_mode → numeric rank for "stronger" comparison. Strips
+# "-PARTIAL" suffix that HYBRID modes carry.
+_AUDIT_RANK = {
+    "EVIDENCE-WARRANTED": 3,
+    "ANCHOR-WARRANTED": 2,
+    "POINTER-LINKED": 1,
+    "UNGROUNDED": 0,
+    "STRICT": 2,    # legacy alias for ANCHOR-WARRANTED
+    "HYBRID": 1,    # legacy alias for POINTER-LINKED
+}
+
+
+def _audit_rank(mode: str | None) -> int:
+    if not mode:
+        return 0
+    base = mode.replace("-PARTIAL", "")
+    return _AUDIT_RANK.get(base, 0)
+
+
+def _is_grounded(mode: str | None) -> bool:
+    """Anything not UNGROUNDED counts as grounded for cost accounting."""
+    if not mode:
+        return False
+    return not mode.startswith("UNGROUNDED")
+
+
+def _read_questions(path: Path) -> list[str]:
+    out: list[str] = []
+    for line in path.read_text().splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        out.append(line)
+    return out
+
+
+def _parse_models(spec: str) -> list[tuple[str, str, str]]:
+    """Parse name=endpoint:model_id,... into a list of tuples."""
+    out = []
+    for chunk in spec.split(","):
+        chunk = chunk.strip()
+        if not chunk:
+            continue
+        if "=" not in chunk or ":" not in chunk.split("=", 1)[1]:
+            raise SystemExit(
+                f"--models entry must be name=endpoint:model_id, got: {chunk!r}"
+            )
+        name, rest = chunk.split("=", 1)
+        # split on the LAST colon — model_id may contain colons (rare)
+        # but endpoint always ends at the path boundary, so split on
+        # the colon BEFORE a non-slash. Simpler: assume model_id
+        # follows the first colon AFTER the endpoint scheme://host[:port]/path.
+        # We split on " :" sentinel — but easiest: rsplit once on `:`
+        # because vLLM model_ids don't contain colons in practice.
+        endpoint, model_id = rest.rsplit(":", 1)
+        out.append((name.strip(), endpoint.strip(), model_id.strip()))
+    return out
+
+
+def _parse_price(spec: str | None) -> dict[str, float]:
+    if not spec:
+        return dict(DEFAULT_PRICE_CENTS)
+    out = dict(DEFAULT_PRICE_CENTS)
+    for chunk in spec.split(","):
+        if "=" not in chunk:
+            continue
+        name, cents = chunk.split("=", 1)
+        try:
+            out[name.strip()] = float(cents.strip())
+        except ValueError:
+            pass
+    return out
+
+
+def _run_one(
+    question: str, *, shards_dir: Path | None, endpoint: str, model: str,
+    top_k: int, timeout_s: int, burn: bool,
+) -> dict:
+    cmd = [str(ARBORIST)]
+    if shards_dir:
+        cmd += ["--shards-dir", str(shards_dir)]
+    cmd += [
+        "query", "--json",
+        "--top-k", str(top_k),
+        "--endpoint", endpoint,
+        "--model", model,
+    ]
+    if burn:
+        cmd.append("--burn")
+    cmd.append(question)
+    t0 = time.time()
+    try:
+        proc = subprocess.run(
+            cmd, capture_output=True, text=True, timeout=timeout_s,
+            env={**os.environ, "ARBORIST_PROGRESS": "0"},
+        )
+    except subprocess.TimeoutExpired:
+        return {"_error": f"timeout after {timeout_s}s",
+                "_elapsed_s": float(timeout_s)}
+    elapsed = round(time.time() - t0, 2)
+    if proc.returncode not in (0, 1):
+        # Exit 1 is "ungrounded" — still has JSON. Other codes are real errors.
+        return {"_error": f"exit {proc.returncode}",
+                "_stderr": (proc.stderr or "")[:500],
+                "_elapsed_s": elapsed}
+    try:
+        data = json.loads(proc.stdout)
+    except json.JSONDecodeError as e:
+        return {"_error": f"parse: {e}",
+                "_stdout_head": proc.stdout[:200],
+                "_elapsed_s": elapsed}
+    data["_elapsed_s"] = elapsed
+    data["_exit_code"] = proc.returncode
+    return data
+
+
+def _primary_uri(result: dict) -> str:
+    srcs = result.get("sources") or []
+    primary = next(
+        (s for s in srcs if s.get("source_role") == "primary_answer_source"),
+        srcs[0] if srcs else {},
+    )
+    return primary.get("document_uri") or ""
+
+
+def _stronger_model(per_model: dict[str, dict]) -> str:
+    """Return the name of the model with the higher audit_mode rank
+    (ties broken by alpha for stability). Empty string if all ungrounded."""
+    best = ""
+    best_rank = -1
+    for name, res in sorted(per_model.items()):
+        r = _audit_rank(res.get("audit_mode"))
+        if r > best_rank:
+            best, best_rank = name, r
+    return best if best_rank > 0 else ""
+
+
+def _summarize(
+    rows: list[dict], models: list[tuple[str, str, str]],
+    price_cents: dict[str, float],
+) -> str:
+    """Build the markdown summary."""
+    by_q: dict[str, dict[str, dict]] = defaultdict(dict)
+    for r in rows:
+        by_q[r["question"]][r["model_name"]] = r["result"]
+
+    n_q = len(by_q)
+    model_names = [m[0] for m in models]
+    out = []
+    out.append(f"# Cross-model self-play bench — {_dt.datetime.now(_dt.timezone.utc).isoformat()}")
+    out.append("")
+    out.append(f"Questions: {n_q}. Models: {', '.join(model_names)}.")
+    out.append("")
+
+    # Per-model summary
+    out.append("## Per-model summary")
+    out.append("")
+    out.append("| model | grounded | rate | avg latency | est $/grounded |")
+    out.append("|---|---:|---:|---:|---:|")
+    for name in model_names:
+        grounded = sum(
+            1 for q in by_q if _is_grounded(by_q[q].get(name, {}).get("audit_mode"))
+        )
+        total = sum(1 for q in by_q if name in by_q[q] and "_error" not in by_q[q][name])
+        rate = (grounded / total) if total else 0.0
+        avg_lat = (
+            sum(by_q[q].get(name, {}).get("_elapsed_s", 0) for q in by_q) / max(total, 1)
+        )
+        per_call_cents = price_cents.get(name, 0.0)
+        cost_per_grounded = (per_call_cents * total / grounded) if grounded else float("nan")
+        cost_str = "n/a" if cost_per_grounded != cost_per_grounded else f"${cost_per_grounded / 100:.2f}"
+        out.append(
+            f"| {name} | {grounded}/{total} | {rate:.0%} "
+            f"| {avg_lat:.1f}s | {cost_str} |"
+        )
+    out.append("")
+
+    # Agreement
+    out.append("## Per-question results")
+    out.append("")
+    header = ["question"] + model_names + ["stronger", "primary agree"]
+    out.append("| " + " | ".join(header) + " |")
+    out.append("|" + "|".join(["---"] * len(header)) + "|")
+    for q in by_q:
+        cells = [q[:64] + ("…" if len(q) > 64 else "")]
+        uris = set()
+        for name in model_names:
+            res = by_q[q].get(name, {})
+            audit = res.get("audit_mode") or "—"
+            if "_error" in res:
+                audit = f"ERR ({res['_error'][:30]})"
+            cells.append(audit)
+            uri = _primary_uri(res)
+            if uri:
+                uris.add(uri)
+        cells.append(_stronger_model(by_q[q]) or "—")
+        cells.append("✓" if len(uris) == 1 else ("✗" if len(uris) > 1 else "—"))
+        out.append("| " + " | ".join(cells) + " |")
+    out.append("")
+
+    # Cost analysis
+    out.append("## Cost analysis (north star: $/grounded)")
+    out.append("")
+    total_cost = 0.0
+    total_grounded = 0
+    for name in model_names:
+        per_call_cents = price_cents.get(name, 0.0)
+        n_calls = sum(1 for q in by_q if name in by_q[q] and "_error" not in by_q[q][name])
+        n_g = sum(1 for q in by_q if _is_grounded(by_q[q].get(name, {}).get("audit_mode")))
+        spent = per_call_cents * n_calls / 100
+        total_cost += spent
+        total_grounded += n_g
+        out.append(f"* **{name}** — {n_calls} calls × ${per_call_cents/100:.2f} = ${spent:.2f}, {n_g} grounded")
+    out.append("")
+    out.append(f"**Total spend**: ${total_cost:.2f}. **Total grounded answers**: {total_grounded}.")
+    if total_grounded:
+        out.append(f"**Effective $/grounded** (sum of all model spend / total grounded answers): ${total_cost / total_grounded:.3f}")
+    out.append("")
+
+    # Routing hint — what cheap-first cascade would have cost
+    out.append("## Cheap-first cascade (try cheapest, escalate on UNGROUNDED)")
+    out.append("")
+    ordered = sorted(model_names, key=lambda n: price_cents.get(n, 0))
+    cascade_cost = 0.0
+    cascade_grounded = 0
+    for q in by_q:
+        for i, name in enumerate(ordered):
+            res = by_q[q].get(name, {})
+            if "_error" in res:
+                continue
+            cascade_cost += price_cents.get(name, 0) / 100
+            if _is_grounded(res.get("audit_mode")):
+                cascade_grounded += 1
+                break
+            # else fall through to next model
+    if cascade_grounded:
+        out.append(
+            f"Cascade ({' → '.join(ordered)}): ${cascade_cost:.2f} for "
+            f"{cascade_grounded}/{n_q} grounded — "
+            f"${cascade_cost / cascade_grounded:.3f}/grounded."
+        )
+    out.append("")
+    out.append("Cascade beats always-Qwen when the cheaper model grounds often enough that escalation is rare. Compare to per-model rows above.")
+    out.append("")
+    return "\n".join(out)
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument(
+        "--questions", type=Path,
+        default=Path("bench/qa_questions_smoke.txt"),
+    )
+    ap.add_argument("--shards-dir", type=Path, default=None)
+    ap.add_argument(
+        "--models", default=None,
+        help="comma-separated name=endpoint:model_id; defaults to hermes + qwen",
+    )
+    ap.add_argument(
+        "--price", default=None,
+        help="comma-separated name=cents per call; defaults to hermes=9,qwen=16",
+    )
+    ap.add_argument("--top-k", type=int, default=8)
+    ap.add_argument("--timeout", type=int, default=180)
+    ap.add_argument(
+        "--burn", action="store_true",
+        help="force fresh inference (skip cache) on every call",
+    )
+    ap.add_argument("--limit", type=int, default=0, help="cap questions (0 = no cap)")
+    ap.add_argument(
+        "--out-dir", type=Path, default=Path("bench/cross_model_results"),
+    )
+    args = ap.parse_args()
+
+    questions = _read_questions(args.questions)
+    if args.limit:
+        questions = questions[: args.limit]
+    if not questions:
+        print(f"no questions in {args.questions}", file=sys.stderr)
+        return 1
+
+    if args.models:
+        models = _parse_models(args.models)
+    else:
+        models = list(DEFAULT_MODELS)
+    price = _parse_price(args.price)
+
+    if not ARBORIST.exists():
+        print(f"arborist binary not found: {ARBORIST}", file=sys.stderr)
+        return 2
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    ts = _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    jsonl_path = args.out_dir / f"{ts}.jsonl"
+    md_path = args.out_dir / f"{ts}.md"
+
+    rows: list[dict] = []
+    print(f"# {len(questions)} questions × {len(models)} models = "
+          f"{len(questions) * len(models)} calls", file=sys.stderr)
+    with jsonl_path.open("w") as f:
+        for q in questions:
+            for name, endpoint, model_id in models:
+                print(f"  [{name}] {q[:80]}", file=sys.stderr)
+                res = _run_one(
+                    q, shards_dir=args.shards_dir,
+                    endpoint=endpoint, model=model_id,
+                    top_k=args.top_k, timeout_s=args.timeout, burn=args.burn,
+                )
+                row = {
+                    "question": q,
+                    "model_name": name,
+                    "model_id": model_id,
+                    "endpoint": endpoint,
+                    "result": res,
+                }
+                rows.append(row)
+                f.write(json.dumps(row, ensure_ascii=False) + "\n")
+                f.flush()
+                audit = res.get("audit_mode") or res.get("_error") or "?"
+                elapsed = res.get("_elapsed_s", "?")
+                print(f"    → {audit} ({elapsed}s)", file=sys.stderr)
+
+    md_path.write_text(_summarize(rows, models, price))
+    print(f"\nJSONL: {jsonl_path}", file=sys.stderr)
+    print(f"Summary: {md_path}", file=sys.stderr)
+    print()
+    print(md_path.read_text())
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())