feat(#000057): stronger code judge — resolve HYBRID with verified quote + on-topic
The code judge bailed to JUDGE_ERROR on 40% of in-corpus answers: HYBRID
(partial grounding) with low NLI entail, where the entity-grounding
rescue needs ZERO unsourced specifics. A single extra proper noun
('Emperor Honorius', 'Alexander Molossus' — an alias/paraphrase) blocked
rescue even with verbatim quotes verified and the answer correct.
New HYBRID resolution tier: rescue to CORRECT_GROUNDED when the verifier
confirmed >=1 verbatim quote, the subject anchor is in gold (on-topic),
there is NO unsourced NUMERIC specific (wrong dates/counts stay residue),
and NLI isn't strongly contradicting. Unsourced proper nouns are treated
as aliases/paraphrase; unsourced numerics (the real factual-error class)
keep the answer as JUDGE_ERROR. Validated on the 12 real residue cases:
9 -> CORRECT (all genuinely right), 3 stay residue (unsourced numerics).
JUDGE_ERROR 40% -> ~10%. self-test 4/4; 2 new tier tests; suite 2549.
This commit is contained in:
parent
39c040cacc
commit
2d3186669f
2 changed files with 65 additions and 0 deletions
|
|
@ -653,6 +653,35 @@ def judge(question: str, answer: str, gold_source: str) -> Verdict:
|
|||
f"specifics in gold + subject anchor "
|
||||
f"{subj_anchor!r} in gold",
|
||||
trace)
|
||||
# Relaxed entity-grounding rescue (2026-05-21). Requiring ZERO
|
||||
# unsourced specifics is too brittle: a single extra proper noun
|
||||
# ("Emperor Honorius", "Alexander Molossus" — an alias or a
|
||||
# paraphrased adjacent fact) blocks rescue even when the verifier
|
||||
# confirmed verbatim quotes and the answer is correct. The risky
|
||||
# unsourced class is NUMERIC (a wrong date/count is a real factual
|
||||
# error); unsourced proper nouns are usually aliases/paraphrase.
|
||||
# So rescue a HYBRID to CG when: (a) the verifier confirmed >=1
|
||||
# verbatim quote in gold (real grounding), (b) subject anchor in
|
||||
# gold (on-topic), (c) NO unsourced NUMERIC specific (dates/counts
|
||||
# all grounded), and (d) NLI is not strongly contradicting. Wrong-
|
||||
# number answers (unsourced numeric) and contradicted answers stay
|
||||
# as JUDGE_ERROR residue.
|
||||
n_verified = v.get("n_verified", 0)
|
||||
unsourced_numeric = [s for s in unsourced
|
||||
if _NUMERIC_SPECIFIC_RE.search(s)]
|
||||
not_contra = (not nli_avail) or nli.max_contradiction < theta_contra
|
||||
trace["hybrid_n_verified"] = n_verified
|
||||
trace["hybrid_unsourced_numeric"] = unsourced_numeric[:10]
|
||||
if (subj_in_gold and n_verified >= 1 and all_specs
|
||||
and not unsourced_numeric and not_contra):
|
||||
trace["rules_fired"].append("hybrid+verified_quote_on_topic")
|
||||
return _v("CORRECT_GROUNDED",
|
||||
f"verifier HYBRID via {method} + {n_verified} "
|
||||
f"verbatim quote(s) verified + subject anchor "
|
||||
f"{subj_anchor!r} in gold + no unsourced numeric "
|
||||
f"specific + not NLI-contradicted "
|
||||
f"(unsourced proper-nouns treated as aliases)",
|
||||
trace)
|
||||
return _v("JUDGE_ERROR",
|
||||
f"code judge ambiguous: HYBRID via {method} without "
|
||||
f"NLI entailment corroboration or entity-grounding — "
|
||||
|
|
|
|||
|
|
@ -237,3 +237,39 @@ def test_judge_handles_nli_unavailable(monkeypatch):
|
|||
"This article is about thermodynamics.",
|
||||
)
|
||||
assert v.label == "FABRICATED"
|
||||
|
||||
|
||||
# --- relaxed HYBRID rescue (2026-05-21): tolerate unsourced proper nouns
|
||||
# (aliases/paraphrase) but still block unsourced numerics (date/count
|
||||
# errors), gated on a verified quote + on-topic + no contradiction. -----
|
||||
|
||||
_EPIRUS_GOLD = ("Alexander I of Epirus was a king of Epirus from 350 to "
|
||||
"331 BC, belonging to the Aeacid dynasty. He was the son "
|
||||
"of Neoptolemus I and brother of Olympias.")
|
||||
|
||||
|
||||
def test_hybrid_rescued_when_only_unsourced_specific_is_proper_noun(monkeypatch):
|
||||
# NLI off so the lexical tier is isolated (not the NLI corroboration
|
||||
# path). One verbatim quote (verified) + one quote NOT in gold
|
||||
# (-> HYBRID); the only unsourced specifics are proper nouns.
|
||||
import bench.judge_code as jc
|
||||
monkeypatch.setattr(jc, "_nli_check", lambda a, g: None)
|
||||
answer = ('Alexander I of Epirus "was a king of Epirus from 350 to '
|
||||
'331 BC". Also called Alexander Molossus, he "led famed '
|
||||
'campaigns in southern Italy against the Romans".')
|
||||
v = jc.judge("who was Alexander the first of epirus?", answer, _EPIRUS_GOLD)
|
||||
assert v.label == "CORRECT_GROUNDED", (v.label, v.rationale)
|
||||
assert v.decision.get("audit_mode") == "HYBRID"
|
||||
assert "hybrid+verified_quote_on_topic" in v.decision["rules_fired"]
|
||||
|
||||
|
||||
def test_hybrid_stays_residue_when_unsourced_specific_is_numeric(monkeypatch):
|
||||
# NLI off. One verified quote + an unverified quote with a WRONG date
|
||||
# -> unsourced numeric -> NOT rescued (date/count errors stay residue).
|
||||
import bench.judge_code as jc
|
||||
monkeypatch.setattr(jc, "_nli_check", lambda a, g: None)
|
||||
answer = ('Alexander I of Epirus "was a king of Epirus". He '
|
||||
'"reigned from 999 to 888 BC".')
|
||||
v = jc.judge("who was Alexander the first of epirus?", answer, _EPIRUS_GOLD)
|
||||
assert v.label == "JUDGE_ERROR", (v.label, v.rationale)
|
||||
assert "hybrid+verified_quote_on_topic" not in v.decision["rules_fired"]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue