diff --git a/bench/judge_code.py b/bench/judge_code.py index 38eac88..b94c52d 100644 --- a/bench/judge_code.py +++ b/bench/judge_code.py @@ -653,6 +653,35 @@ def judge(question: str, answer: str, gold_source: str) -> Verdict: f"specifics in gold + subject anchor " f"{subj_anchor!r} in gold", trace) + # Relaxed entity-grounding rescue (2026-05-21). Requiring ZERO + # unsourced specifics is too brittle: a single extra proper noun + # ("Emperor Honorius", "Alexander Molossus" — an alias or a + # paraphrased adjacent fact) blocks rescue even when the verifier + # confirmed verbatim quotes and the answer is correct. The risky + # unsourced class is NUMERIC (a wrong date/count is a real factual + # error); unsourced proper nouns are usually aliases/paraphrase. + # So rescue a HYBRID to CG when: (a) the verifier confirmed >=1 + # verbatim quote in gold (real grounding), (b) subject anchor in + # gold (on-topic), (c) NO unsourced NUMERIC specific (dates/counts + # all grounded), and (d) NLI is not strongly contradicting. Wrong- + # number answers (unsourced numeric) and contradicted answers stay + # as JUDGE_ERROR residue. + n_verified = v.get("n_verified", 0) + unsourced_numeric = [s for s in unsourced + if _NUMERIC_SPECIFIC_RE.search(s)] + not_contra = (not nli_avail) or nli.max_contradiction < theta_contra + trace["hybrid_n_verified"] = n_verified + trace["hybrid_unsourced_numeric"] = unsourced_numeric[:10] + if (subj_in_gold and n_verified >= 1 and all_specs + and not unsourced_numeric and not_contra): + trace["rules_fired"].append("hybrid+verified_quote_on_topic") + return _v("CORRECT_GROUNDED", + f"verifier HYBRID via {method} + {n_verified} " + f"verbatim quote(s) verified + subject anchor " + f"{subj_anchor!r} in gold + no unsourced numeric " + f"specific + not NLI-contradicted " + f"(unsourced proper-nouns treated as aliases)", + trace) return _v("JUDGE_ERROR", f"code judge ambiguous: HYBRID via {method} without " f"NLI entailment corroboration or entity-grounding — " diff --git a/tests/test_judge_code.py b/tests/test_judge_code.py index a8081d9..ff0da4f 100644 --- a/tests/test_judge_code.py +++ b/tests/test_judge_code.py @@ -237,3 +237,39 @@ def test_judge_handles_nli_unavailable(monkeypatch): "This article is about thermodynamics.", ) assert v.label == "FABRICATED" + + +# --- relaxed HYBRID rescue (2026-05-21): tolerate unsourced proper nouns +# (aliases/paraphrase) but still block unsourced numerics (date/count +# errors), gated on a verified quote + on-topic + no contradiction. ----- + +_EPIRUS_GOLD = ("Alexander I of Epirus was a king of Epirus from 350 to " + "331 BC, belonging to the Aeacid dynasty. He was the son " + "of Neoptolemus I and brother of Olympias.") + + +def test_hybrid_rescued_when_only_unsourced_specific_is_proper_noun(monkeypatch): + # NLI off so the lexical tier is isolated (not the NLI corroboration + # path). One verbatim quote (verified) + one quote NOT in gold + # (-> HYBRID); the only unsourced specifics are proper nouns. + import bench.judge_code as jc + monkeypatch.setattr(jc, "_nli_check", lambda a, g: None) + answer = ('Alexander I of Epirus "was a king of Epirus from 350 to ' + '331 BC". Also called Alexander Molossus, he "led famed ' + 'campaigns in southern Italy against the Romans".') + v = jc.judge("who was Alexander the first of epirus?", answer, _EPIRUS_GOLD) + assert v.label == "CORRECT_GROUNDED", (v.label, v.rationale) + assert v.decision.get("audit_mode") == "HYBRID" + assert "hybrid+verified_quote_on_topic" in v.decision["rules_fired"] + + +def test_hybrid_stays_residue_when_unsourced_specific_is_numeric(monkeypatch): + # NLI off. One verified quote + an unverified quote with a WRONG date + # -> unsourced numeric -> NOT rescued (date/count errors stay residue). + import bench.judge_code as jc + monkeypatch.setattr(jc, "_nli_check", lambda a, g: None) + answer = ('Alexander I of Epirus "was a king of Epirus". He ' + '"reigned from 999 to 888 BC".') + v = jc.judge("who was Alexander the first of epirus?", answer, _EPIRUS_GOLD) + assert v.label == "JUDGE_ERROR", (v.label, v.rationale) + assert "hybrid+verified_quote_on_topic" not in v.decision["rules_fired"]