bench: add --teacher-model-judge flag — emit prompts for downstream correctness grading

STRICT audit ≠ factually correct. The verifier passes any answer whose
quotes match source text; a model can quote correctly and still draw a
wrong conclusion. Without a judge, the bench can't detect false-STRICTs.

This flag emits <ts>.judge_input.jsonl alongside the regular results —
one row per (question, model) with a model-agnostic teacher prompt.
Feed to any teacher (claude -p per row, remote API, Hermes self-judge)
to get CORRECT|WRONG|PARTIAL|UNCERTAIN verdicts. Skips rows that
errored or returned empty answers.

Grounded in observed reality from the 76-question 2010-wiki bench:
Hermes produced 2 confidently-wrong STRICTs (Q12 conflated Roman Empire
with Holy Roman Empire; Q48 answered the Niger River when asked about
the Nile). Qwen had 0 false STRICTs over the same fixture. Naming the
flag --teacher-model-judge (not --opus-judge) keeps the harness
provider-agnostic.

bench/cross_model_selfplay.py

@@ -34,6 +34,12 @@ Usage:
     # custom model list (name=endpoint:model_id, comma-separated):
     python3 bench/cross_model_selfplay.py \\
         --models 'hermes=https://hermes.ai.unturf.com/v1:adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic,qwen=https://qwen.ai.unturf.com/v1:Qwen3.6-27B-UD-Q4_K_XL.gguf'
+
+    # emit teacher-judge prompts for downstream correctness grading:
+    python3 bench/cross_model_selfplay.py --teacher-model-judge
+    # → produces <ts>.judge_input.jsonl with one row per (question, model);
+    # feed to a teacher (claude -p, remote API, etc.) to detect false STRICTs
+    # — answers that audit STRICT but are factually wrong.
 """
 from __future__ import annotations
 
@@ -200,6 +206,62 @@ def _stronger_model(per_model: dict[str, dict]) -> str:
     return best if best_rank > 0 else ""
 
 
+def _judge_prompt(question: str, model_name: str, audit_mode: str, answer: str) -> str:
+    """Build the prompt a teacher model receives to grade one (question, answer)
+    pair. Asks for a single verdict token + a one-sentence rationale.
+
+    Designed to be model-agnostic — any teacher (Opus via `claude -p`, a remote
+    API, a Hermes self-judge run) can consume it. The teacher should reply with
+    the first non-empty line as: `CORRECT|WRONG|PARTIAL|UNCERTAIN` followed by
+    a colon and a one-sentence rationale.
+    """
+    return (
+        "You are a hermetic judge grading the factual correctness of one answer "
+        "to one question. Use your own parametric knowledge as ground truth. "
+        "Ignore the answer's stylistic quality and whether it cites sources — "
+        "judge only whether the factual claim is true.\n\n"
+        f"Question: {question}\n\n"
+        f"Answer: {answer}\n\n"
+        "Reply on a single line in this exact format:\n"
+        "  VERDICT: <one sentence rationale>\n\n"
+        "Where VERDICT is exactly one of:\n"
+        "  CORRECT   — the answer's main factual claim is true\n"
+        "  WRONG     — the answer's main factual claim is false\n"
+        "  PARTIAL   — main claim correct but missing required detail "
+        "(or correct on part, wrong on part)\n"
+        "  UNCERTAIN — you don't know the domain well enough to judge\n"
+    )
+
+
+def _emit_judge_inputs(
+    rows: list[dict], out_path: Path,
+) -> int:
+    """Write one JSONL row per (question, model) ready for a teacher to grade.
+    Skips rows that errored or returned no answer text. Returns count written."""
+    written = 0
+    with out_path.open("w") as f:
+        for r in rows:
+            res = r.get("result") or {}
+            if "_error" in res:
+                continue
+            answer = (res.get("answer_text") or "").strip()
+            if not answer:
+                continue
+            audit = res.get("audit_mode") or "UNKNOWN"
+            payload = {
+                "question": r["question"],
+                "model_name": r["model_name"],
+                "audit_mode": audit,
+                "answer_text": answer,
+                "judge_prompt": _judge_prompt(
+                    r["question"], r["model_name"], audit, answer,
+                ),
+            }
+            f.write(json.dumps(payload, ensure_ascii=False) + "\n")
+            written += 1
+    return written
+
+
 def _summarize(
     rows: list[dict], models: list[tuple[str, str, str]],
     baseline_per_1k: dict[str, float],
@@ -372,6 +434,14 @@ def main() -> int:
     ap.add_argument(
         "--out-dir", type=Path, default=Path("bench/cross_model_results"),
     )
+    ap.add_argument(
+        "--teacher-model-judge", action="store_true",
+        help="emit <ts>.judge_input.jsonl alongside results — one row per "
+             "(question, model) with a teacher-ready prompt for grading. "
+             "Feed to any teacher (claude -p, remote API, Hermes self-judge) "
+             "to get correctness verdicts. STRICT audit ≠ correct — this is "
+             "how you measure false-STRICTs.",
+    )
     args = ap.parse_args()
 
     questions = _read_questions(args.questions)
@@ -425,6 +495,15 @@ def main() -> int:
     md_path.write_text(_summarize(rows, models, baseline))
     print(f"\nJSONL: {jsonl_path}", file=sys.stderr)
     print(f"Summary: {md_path}", file=sys.stderr)
+
+    if args.teacher_model_judge:
+        judge_input_path = args.out_dir / f"{ts}.judge_input.jsonl"
+        n = _emit_judge_inputs(rows, judge_input_path)
+        print(f"Teacher-judge inputs: {judge_input_path} ({n} rows)", file=sys.stderr)
+        print(f"  Feed to your teacher of choice (e.g. `claude -p` per row) "
+              f"to grade. Verdicts are CORRECT|WRONG|PARTIAL|UNCERTAIN.",
+              file=sys.stderr)
+
     print()
     print(md_path.read_text())
     return 0