diff --git a/bench/analyze_judge_disagreement.py b/bench/analyze_judge_disagreement.py new file mode 100644 index 0000000..08c16b3 --- /dev/null +++ b/bench/analyze_judge_disagreement.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Reconcile the code judge against the trusted Opus judge. + +fox 2026-05-19: "we do trust opus from claude highly even though it +costs a lot." The B rescore (control_sweep_*_code_judge.jsonl) carries +both the original Opus verdict (`verdict` field, from the live sweep) +and the deterministic code verdict (`code_verdict` field, from the +rescore). Where the two disagree, Opus is the reference — the code +judge is the instrument being calibrated. + +This script: + + 1. bucketises disagreements by (code_verdict, opus_verdict) cell; + 2. dumps a configurable sample from each cell with full context + (question, answer, code rationale, opus rationale-tail); + 3. emits a markdown report so the patterns are reviewable + side-by-side rather than scrolling per-record. + +Reads only records where Opus produced a real verdict +(verdict != 'JUDGE_ERROR') — the rest is Opus failing, not +disagreement. + +NO LLM calls. Pure read of the JSONL. + +Usage:: + + python -m bench.analyze_judge_disagreement \\ + --in bench/qa_results/control_sweep__code_judge.jsonl \\ + --sample 10 +""" +from __future__ import annotations + +import argparse +import json +import sys +from collections import Counter, defaultdict +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from bench.control_ab import _gold # noqa: E402 + +LABELS = ["CORRECT_GROUNDED", "WRONG", "FABRICATED", "ABSTAINED", + "JUDGE_ERROR"] + + +def _opus_rationale_tail(judge_raw: str) -> str: + """Heuristic: the last non-sentinel non-empty line of the Opus raw + output is usually the model's brief rationale before the + FINAL_VERDICT=X sentinel.""" + out = [] + for ln in reversed((judge_raw or "").splitlines()): + s = ln.strip() + if not s: + continue + if "FINAL_VERDICT" in s: + continue + out.append(s) + if len(out) >= 2: + break + return " · ".join(reversed(out))[:300] + + +def _gold_excerpt(shards_dir: Path, fixture_by_i: dict, i: int, + cap: int = 600, cache: dict | None = None) -> str: + cache = cache if cache is not None else {} + if i in cache: + return cache[i] + fxr = fixture_by_i.get(i) + if not fxr: + cache[i] = "" + return "" + g = _gold(shards_dir, fxr.get("shard", ""), fxr["target_root"]) or "" + cache[i] = g[:cap] + return cache[i] + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--in", dest="in_path", required=True) + ap.add_argument("--fixture", + default="bench/qa_questions_stale_map.json") + ap.add_argument("--shards-dir", + default=str(Path.home() / ".arborist" / "shards")) + ap.add_argument("--sample", type=int, default=8, + help="per-cell sample size to dump") + ap.add_argument("--out-md", default="", + help="markdown report path " + "(default: alongside --in with _disagreement.md)") + a = ap.parse_args() + + in_path = Path(a.in_path) + out_md = (Path(a.out_md) if a.out_md + else in_path.with_name( + in_path.stem.replace("_code_judge", "") + + "_judge_disagreement.md")) + + items = json.loads(Path(a.fixture).read_text()) + fixture_by_i = {i + 1: it for i, it in enumerate(items)} + shards_dir = Path(a.shards_dir) + gold_cache: dict[int, str] = {} + + cells: dict[tuple, list[dict]] = defaultdict(list) + cell_count: Counter = Counter() + opus_total: Counter = Counter() + + for ln in in_path.read_text().splitlines(): + if not ln.strip(): + continue + r = json.loads(ln) + if r.get("arm") == "skip": + continue + opus = r.get("verdict") or "JUDGE_ERROR" + code = r.get("code_verdict") or "JUDGE_ERROR" + opus_total[opus] += 1 + # We're calibrating against records Opus actually graded. + if opus == "JUDGE_ERROR": + continue + cell_count[(code, opus)] += 1 + cells[(code, opus)].append(r) + + # Bucketise into "agreement" (on-diagonal), "off-diagonal", and + # rank off-diagonal cells by count so the biggest disagreements + # surface first. + on_diag = sum(cell_count[(lab, lab)] for lab in LABELS) + total = sum(cell_count.values()) + off_diag_cells = sorted( + ((cnt, code, opus) for (code, opus), cnt in cell_count.items() + if code != opus), + reverse=True, + ) + + L: list[str] = [ + f"# Judge disagreement reconciliation — `{in_path.name}`", + "", + "fox 2026-05-19: Opus is trusted; code judge is being calibrated. " + "Below: every (code_verdict, opus_verdict) cell off the diagonal, " + f"sample={a.sample} records per cell, sorted by cell size " + "(biggest disagreement first).", + "", + f"**Restricted to records Opus actually graded** " + f"(`verdict != 'JUDGE_ERROR'`): {total} of " + f"{sum(opus_total.values())} (the other " + f"{opus_total['JUDGE_ERROR']} are Opus failures, not " + "disagreement).", + "", + f"On-diagonal (agree): **{on_diag}/{total} = " + f"{on_diag/total:.1%}**.", + "", + "## Per-cell summary", + "", + "| n | code says | opus says |", + "|---|---|---|" + ] + for cnt, code, opus in off_diag_cells: + L.append(f"| {cnt} | {code} | {opus} |") + L += [""] + + # Per-cell dumps, biggest first. + for cnt, code, opus in off_diag_cells: + if cnt == 0: + continue + L += [ + f"## code=**{code}** · opus=**{opus}** " + f"({cnt} records · showing {min(a.sample, cnt)})", + "", + ] + for r in cells[(code, opus)][:a.sample]: + i = r.get("i") + q = r.get("question_asked") or r.get("question_orig") or "" + ans = (r.get("answer") or "").replace("\n", " ")[:300] + cr = (r.get("code_rationale") or "")[:200] + jr = _opus_rationale_tail(r.get("judge_raw") or "")[:200] + gold = _gold_excerpt(shards_dir, fixture_by_i, i, + cap=400, cache=gold_cache) + gold_short = gold.replace("\n", " ")[:300] + L += [ + f"### item {i} · {r.get('arm')}/{r.get('model')}/{r.get('variant')}", + f"- **Q:** {q}", + f"- **A:** {ans}", + f"- **Gold excerpt:** {gold_short}", + f"- **Code says {code}:** {cr}", + f"- **Opus says {opus}:** {jr}", + "", + ] + + out_md.parent.mkdir(parents=True, exist_ok=True) + out_md.write_text("\n".join(L) + "\n") + print(f" disagreement report: {out_md}") + print(f" on-diagonal: {on_diag}/{total} = {on_diag/total:.1%}") + for cnt, code, opus in off_diag_cells[:10]: + print(f" n={cnt:>4} code={code:<20} opus={opus}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/judge_code.py b/bench/judge_code.py index 47a54e4..7fafc9e 100644 --- a/bench/judge_code.py +++ b/bench/judge_code.py @@ -102,6 +102,32 @@ _VERDICTS = ("CORRECT_GROUNDED", "WRONG", "FABRICATED", "ABSTAINED") # (residue for an LLM judge later) rather than risk a false promote. _CODE_JUDGE_THETA_ENTAIL_CORROBORATE = 0.55 +# Code-judge NLI contradiction threshold (rule 3 — gold contradicts +# the answer → WRONG). The 2026-05-19 reconciliation against the +# trusted Opus judge measured **114 records** where the code judge +# fired WRONG via NLI contradiction p ∈ [0.51, 0.74] on answers Opus +# correctly graded CG — clear factual claims like "Ólafur Ragnar +# Grímsson is the president of Iceland" against the corpus infobox +# that DOES list him as incumbent. NLI is noisy in the 0.5–0.8 band +# on wikitext-shaped gold (clause-level candidate selection picks up +# co-mentioned earlier office-holders & reads temporal-frame mismatch +# as contradiction). For TRUE contradictions the signal is far higher +# (the WW2 self-test fixture #2 measures p=0.985). Threshold raised +# from the manifest's 0.5 → 0.85 to keep the strong contradictions +# (cleanly above the floor) while dropping the noise band. +_CODE_JUDGE_THETA_CONTRA = 0.85 + +# Short-answer threshold (rule 3.5 — terse correct-noun answers). +# The verifier's strategy 2 needs prose-shaped spans; very short +# answers ("Pratibha Patil", "Jalal Talabani", "In 1945.") fall +# through to UNGROUNDED-no-specifics → ABSTAINED, missing valid CG. +# The reconciliation against Opus identified 13 such records. +# Compensate by accepting a short answer as grounded when every +# specific (proper noun + numeric) it asserts is present in gold AND +# the question's subject anchor is present in gold (catches "wrong +# topic, right name" false-positives). +_CODE_JUDGE_SHORT_ANSWER_TOKENS = 15 + @dataclass class Verdict: @@ -123,9 +149,27 @@ class Verdict: # answer EXPLICITLY says it can not / will not answer or that the # source does not contain the information. A model that answers # "I think it is X" is NOT abstaining — it is making a claim. +# +# Pattern set expanded 2026-05-19 from the Opus-reconciliation +# residue: Hermes's most common abstention phrasings under +# `--variant source_relative` are "I do not have accurate information +# about", "I do not have access to real-time / a reference knowledge +# base to", "I do not have enough information to". The original +# patterns missed all three (they require a verb like determine/know +# right after "do not"). Eighteen records mis-classified as WRONG +# trace to this gap. _ABSTAIN_PATTERNS = ( re.compile(r"\b(?:i\s+(?:cannot|can[' ]?t|do\s+not|don[' ]?t)\s+" r"(?:determine|know|tell|say|find|locate|verify))\b", re.I), + # NEW: "I do not have information / access / data ..." + re.compile(r"\bi\s+(?:do\s+not|don[' ]?t|cannot|can[' ]?t)\s+" + r"have\s+(?:\w+\s+){0,4}" + r"(?:information|access|data|knowledge|details)\b", re.I), + # NEW: "I do not have access to ..." stand-alone + re.compile(r"\bi\s+(?:do\s+not|don[' ]?t)\s+have\s+access\b", re.I), + # NEW: "I lack (access / information / knowledge)" + re.compile(r"\bi\s+lack\s+(?:\w+\s+){0,3}" + r"(?:access|information|knowledge|data|details)\b", re.I), re.compile(r"\b(?:not\s+(?:enough|sufficient)\s+information)\b", re.I), re.compile(r"\b(?:no\s+information\s+(?:is\s+)?(?:available|provided|given))\b", re.I), re.compile(r"\b(?:the\s+(?:source|article|text|context|passage)\s+" @@ -208,6 +252,144 @@ _CAP_TOKEN_STOPWORDS = frozenset({ }) +def _all_answer_specifics(answer: str) -> list[str]: + """Every specific (proper noun + numeric) the answer asserts. + Symmetric with :func:`_specifics_not_in_gold` but unfiltered — + used to check 'this answer asserted SOMETHING specific' before + we decide whether to call it ABSTAINED.""" + out: list[str] = [] + seen: set[str] = set() + + def _add(tok: str): + k = tok.lower() + if k in seen: + return + out.append(tok) + seen.add(k) + + for noun in extract_proper_nouns(answer): + _add(noun) + for m in _SINGLE_CAP_TOKEN_RE.finditer(answer): + tok = m.group(0) + if tok.lower() in _CAP_TOKEN_STOPWORDS: + continue + _add(tok) + for m in _NUMERIC_SPECIFIC_RE.finditer(answer): + _add(m.group(0)) + return out + + +# Stopwords for the question-subject-anchor extractor. Mirrors the +# inspect-module convention: drop function words / WH-question words / +# generic role nouns so the LAST remaining content token is the +# question's primary subject ("Iceland" in "who is the president of +# Iceland?"; "Australia" in "according to the reference knowledge base, +# who is the prime minister of Australia?"). The anchor is also +# tolerant of the variant-rewriting prefixes (`According to ...`, +# `As of 2010 ...`) because those add to the front, not the back. +_SUBJ_STOPWORDS = frozenset({ + "who", "what", "when", "where", "why", "how", "which", "whose", "whom", + "is", "are", "was", "were", "be", "been", "being", "has", "have", "had", + "do", "does", "did", "the", "a", "an", "of", "in", "on", "at", "for", + "by", "to", "from", "as", "according", "reference", "knowledge", "base", + "current", "currently", "today", "now", "presently", "office", + "head", "leader", + # Generic role nouns kept OUT of the subject — they describe the + # *role*, the polity is the subject. "who is the prime minister of + # Australia" → subject = australia, not minister. + "president", "prime", "minister", "chancellor", "premier", "governor", + "general", "secretary", "king", "queen", "emperor", "leader", + "chairman", "chair", "speaker", "monarch", "ruler", +}) + + +_TOKEN_RE = re.compile(r"\b\w+\b", re.U) + + +def _question_subject_anchor(question: str) -> str: + """Last content token after stopword-strip — the inspect-module + fallback used when the question carries no proper-noun-shaped + terms. Returns lowercase or empty string.""" + toks = [t.lower() for t in _TOKEN_RE.findall(question or "")] + content = [t for t in toks if t not in _SUBJ_STOPWORDS and not t.isdigit()] + return content[-1] if content else "" + + +def _question_subject_terms(question: str) -> list[str]: + """Proper-noun-shaped subject terms from the question. Tries + extract_proper_nouns (multi-word PNs) first, then layers single + capitalised tokens (modulo stopwords). For "who is the president + of Iceland?" → ["Iceland"]. For "who founded the Higgs boson + cafe in 1066?" → ["Higgs"]. For "who is the president of the + Quorum of the Twelve Apostles?" → ["Quorum", "Twelve", "Apostles"] + (whichever the multi-word extractor surfaces + single caps the + extractor misses). + + Used as the strong-anchor signal for ``_subject_in_gold``: a + proper-noun-shaped term in the question is the topic's identity, + not just a co-occurring stopword ("cafe" matching gold's "any + cafe" denial is the bug this avoids). Returns [] when the + question has no proper-noun-shaped term — caller falls back to + the last-content-token anchor.""" + out: list[str] = [] + seen: set[str] = set() + + def _add(t: str): + k = t.lower() + if k in seen or k in _SUBJ_STOPWORDS or k in _CAP_TOKEN_STOPWORDS: + return + out.append(t) + seen.add(k) + + for pn in extract_proper_nouns(question or ""): + # Multi-word PNs land whole — record both the phrase and its + # head tokens for substring matching against gold. + _add(pn) + for tok in pn.split(): + _add(tok) + for m in _SINGLE_CAP_TOKEN_RE.finditer(question or ""): + _add(m.group(0)) + return out + + +def _subject_in_gold(question: str, gold: str) -> tuple[bool, str]: + """True iff the question's subject is present in gold. Strong + anchor: proper-noun-shaped terms from the question — ALL such + terms must appear in gold (case-insensitive substring). + Conservative because the WRONG-vs-FABRICATED tie-break demotes + FABRICATED → WRONG when this fires, and we'd rather under-fire + (keep FABRICATED) than over-fire (the Higgs-boson-cafe gold + coincidentally contains "cafe" in a denial — single-token anchor + would mis-fire, multi-PN anchor doesn't). + + Returns ``(found, anchor)`` where ``anchor`` records which term + decided it (for the decision trace). Falls back to the last- + content-token heuristic when the question has no proper-noun- + shaped terms (rare on this corpus).""" + gold_lc = gold.lower() + terms = _question_subject_terms(question) + if terms: + # Require every proper-noun-shaped term to appear in gold — + # all the topic's identity tokens, not just one. + missing = [t for t in terms if t.lower() not in gold_lc] + if not missing: + return True, terms[0] + return False, missing[0] + # Fallback for proper-noun-free questions: the last-content-token + # anchor (was the original heuristic before reconciliation). + anchor = _question_subject_anchor(question) + if not anchor: + return False, "" + return (anchor in gold_lc), anchor + + +def _is_short_answer(answer: str) -> bool: + """The verifier's strategy-2 span extractor needs prose; very + short answers fall through to UNGROUNDED-no-specifics. Treat as + 'short' for the entity-grounding fast path.""" + return len(_TOKEN_RE.findall(answer or "")) <= _CODE_JUDGE_SHORT_ANSWER_TOKENS + + def _specifics_not_in_gold(answer: str, gold: str) -> list[str]: """Proper-noun and numeric specifics that the answer asserts but the gold source never mentions. Three layers: @@ -264,7 +446,36 @@ def judge(question: str, answer: str, gold_source: str) -> Verdict: Same shape & verdict vocabulary as ``bench.judge.judge``; no network, no LLM, no quota cost. JUDGE_ERROR is reserved for the HYBRID-without-NLI-corroboration class so a downstream LLM judge - (Opus batched, Grok) can pick that residue up later.""" + (Opus batched, Grok) can pick that residue up later. + + Rule order (reconciled against Opus 2026-05-19): + + 1. empty / no-gold guards + 2. explicit abstention phrases (lexical regex; broad set + covering the Hermes "I do not have ... information / access" + family that the original narrow patterns missed) + 3. strong NLI contradiction (theta_contra=0.85, raised from 0.5 + after measuring 114 false-positives in the 0.5-0.75 band) + 4. short-answer entity-grounding fast path — for terse answers + where the verifier's prose-shape extractor would whiff, + accept CG when every asserted specific is in gold AND the + question's subject anchor is in gold (catches "Pratibha + Patil" → gold "Pratibha Devisingh Patil ..." → CG) + 5. lexical verifier (verify_quotes) + STRICT → CG + HYBRID + NLI entail ≥ 0.55 → CG + HYBRID + subject-anchor + all-grounded → CG (entity- + grounding rescue) + HYBRID otherwise → JUDGE_ERROR + (residue) + UNGROUNDED + specifics + subject-in-gold → WRONG (source + contradicts — + Opus's WRONG + vs FABRICATED + distinction) + UNGROUNDED + specifics, no subject → FABRICATED + UNGROUNDED + no specifics → ABSTAINED + """ a = (answer or "").strip() g = (gold_source or "").strip() trace: dict = {"rules_fired": []} @@ -284,24 +495,56 @@ def judge(question: str, answer: str, gold_source: str) -> Verdict: trace["abstain_match"] = abs_match return _v("ABSTAINED", f"explicit refusal: {abs_match!r}", trace) - # Rule 3 — NLI contradiction (strongest signal: gold contradicts). + # Pre-compute shared signals once. nli = _nli_check(a, g) - if nli is not None and nli.available: - trace["nli_available"] = True + nli_avail = nli is not None and nli.available + trace["nli_available"] = nli_avail + if nli_avail: trace["nli_max_contra"] = nli.max_contradiction trace["nli_max_entail"] = nli.max_entailment - trace["nli_theta_contra"] = nli.theta_contra - trace["nli_theta_entail"] = nli.theta_entail - if nli.max_contradiction >= nli.theta_contra: - trace["rules_fired"].append("nli_contradiction") - return _v("WRONG", - f"NLI contradiction p={nli.max_contradiction:.3f} " - f">= theta_contra={nli.theta_contra:.2f}", - trace) - else: - trace["nli_available"] = False + subj_in_gold, subj_anchor = _subject_in_gold(question, g) + trace["subject_anchor"] = subj_anchor + trace["subject_in_gold"] = subj_in_gold - # Rule 4 — lexical verifier against gold. + # Rule 3 — strong NLI contradiction (gold-contradicts-claim). + # The 0.85 threshold is the code-judge floor for "this is a real + # contradiction, not NLI noise on wikitext-shaped gold" — see the + # _CODE_JUDGE_THETA_CONTRA constant for the measurement that drove + # the recalibration from the manifest's 0.5. + theta_contra = _CODE_JUDGE_THETA_CONTRA + trace["theta_contra_code_judge"] = theta_contra + if nli_avail and nli.max_contradiction >= theta_contra: + trace["rules_fired"].append("nli_contradiction") + return _v("WRONG", + f"NLI contradiction p={nli.max_contradiction:.3f} " + f">= theta_contra={theta_contra:.2f}", + trace) + + # Rule 4 — short-answer entity-grounding fast path. Verifier's + # strategy-2 needs prose shape; terse-name answers fall through + # to UNGROUNDED with no specifics → ABSTAINED, missing valid CG. + # Rescue when: (a) answer is short, (b) every specific asserted + # is present in gold (no unsourced specifics), (c) at least one + # specific WAS asserted (not just "ok" or other empty content), + # (d) the question's subject anchor is in gold (guards against + # "wrong topic, right name" false positives). + short = _is_short_answer(a) + trace["short_answer"] = short + if short and subj_in_gold: + all_specs = _all_answer_specifics(a) + unsourced = _specifics_not_in_gold(a, g) + trace["short_path_all_specifics"] = all_specs[:10] + trace["short_path_unsourced"] = unsourced[:10] + if all_specs and not unsourced: + trace["rules_fired"].append("short_entity_grounded") + return _v("CORRECT_GROUNDED", + f"short answer + all specifics in gold " + f"({', '.join(all_specs[:3])}" + f"{'…' if len(all_specs) > 3 else ''}) + " + f"subject anchor {subj_anchor!r} in gold", + trace) + + # Rule 5 — lexical verifier against gold. v = verify_quotes(a, g) mode = v.get("audit_mode", "UNGROUNDED") method = v.get("verifier_method", "none") @@ -318,34 +561,58 @@ def judge(question: str, answer: str, gold_source: str) -> Verdict: trace) if mode == "HYBRID": - # HYBRID alone is ambiguous to a code judge. NLI corroboration - # tips it; otherwise defer to a downstream LLM judge. The - # threshold here is the code-judge's own corroboration floor, - # NOT the NLI manifest's blocking-veto threshold (see comment - # at _CODE_JUDGE_THETA_ENTAIL_CORROBORATE for why they differ). + # HYBRID alone is ambiguous. Three tip-points: NLI + # corroboration, OR the entity-grounding rescue (all asserted + # specifics in gold + subject anchor in gold), OR defer. theta = _CODE_JUDGE_THETA_ENTAIL_CORROBORATE trace["theta_entail_corroborate"] = theta - if nli is not None and nli.available \ - and nli.max_entailment >= theta: + if nli_avail and nli.max_entailment >= theta: trace["rules_fired"].append("hybrid+nli_entail") return _v("CORRECT_GROUNDED", f"verifier HYBRID via {method} + NLI entailment " f"p={nli.max_entailment:.3f} >= " f"theta_corroborate={theta:.2f}", trace) + unsourced = _specifics_not_in_gold(a, g) + all_specs = _all_answer_specifics(a) + if subj_in_gold and all_specs and not unsourced: + trace["rules_fired"].append("hybrid+entity_grounded") + return _v("CORRECT_GROUNDED", + f"verifier HYBRID via {method} + all asserted " + f"specifics in gold + subject anchor " + f"{subj_anchor!r} in gold", + trace) return _v("JUDGE_ERROR", f"code judge ambiguous: HYBRID via {method} without " - f"NLI entailment corroboration — residue for LLM judge", + f"NLI entailment corroboration or entity-grounding — " + f"residue for LLM judge", trace) - # mode == "UNGROUNDED" — fabrication vs thin-non-answer. + # mode == "UNGROUNDED" — WRONG vs FABRICATED vs ABSTAINED. specifics = _specifics_not_in_gold(a, g) trace["ungrounded_specifics"] = specifics[:10] if specifics: + # WRONG vs FABRICATED: if gold mentions the question's + # subject (the topic) but the answer's asserted specifics + # aren't in gold, gold is contradicting the answer (it knows + # the topic, has a different specific). If gold doesn't + # mention the subject at all, source is silent on the topic + # and the answer's specifics are FABRICATED. + if subj_in_gold: + trace["rules_fired"].append("ungrounded+subject_in_gold") + return _v("WRONG", + f"verifier UNGROUNDED + subject anchor " + f"{subj_anchor!r} in gold + specifics not in " + f"gold: {', '.join(specifics[:5])}" + f"{'…' if len(specifics) > 5 else ''} — gold " + f"has the topic but a different value", + trace) return _v("FABRICATED", - f"verifier UNGROUNDED + specifics not in gold: " + f"verifier UNGROUNDED + specifics not in gold + " + f"subject anchor {subj_anchor!r} NOT in gold: " f"{', '.join(specifics[:5])}" - f"{'…' if len(specifics) > 5 else ''}", + f"{'…' if len(specifics) > 5 else ''} — source " + f"silent on the topic", trace) return _v("ABSTAINED", "verifier UNGROUNDED + no asserted specifics — "