fix(#000057): code judge unwraps Arborist claim-lattice JSON envelopes
The Arborist arm runs answer_mode='claim_lattice' (per control_sweep.py
:179, control_ab.py:155) so its answers arrive as the JSON envelope
{"claims":[{"text":"...","evidence_ids":["E1"]},...]}.
_descaffold strips the [E1] evidence-pointer markup but the JSON
braces + key syntax remain. The verifier's strategy-2 (span) and
strategy-3 (proper-noun) extractors see brace noise instead of the
inner claim prose — every Arborist record degraded to UNGROUNDED.
The 2026-05-19T17-01-17Z sweep, re-graded with the freshly calibrated
judge (5a17f61), surfaced this: Arborist arm reported 0 CG across all
three variants in the live phase 1 output (the live run was pre-
calibration), and 29/120 CG (24%) under the calibrated rescore — clear
improvement just from theta_contra=0.85, but the JSON envelope was
still hobbling the verifier paths.
Fix: _unwrap_claim_lattice_json runs BEFORE all downstream rules.
Detection is conservative (three independent signals: starts-with-
brace AND "claims" key AND "text" key) so plain-prose answers
pass through unchanged. Multi-claim envelopes concatenate as discrete
sentences (extract_claim_spans treats each as its own span).
Malformed JSON falls back to the original answer — no silent
rewriting on broken input.
Smoke result on the Iceland Arborist case
ans: {"claims":[{"text":"The current president of Iceland is
Ólafur Ragnar Grímsson.","evidence_ids":["E1"]}]}
gold: {{Infobox Political post |post = President |body = Iceland
|incumbent = [[Ólafur Ragnar Grímsson]] ...}}
before: UNGROUNDED → FABRICATED (then WRONG after calibration)
after: short_entity_grounded → CORRECT_GROUNDED
pytest: 27/27 (added 7 unwrap-coverage tests covering single-claim
envelopes, multi-claim concatenation, plain-prose passthrough,
malformed-JSON tolerance, unrelated-JSON passthrough, and the
end-to-end Arborist-envelope CG flow). Self-test 4/4 unchanged.
Re-rescores of 17:01 sweep + phase 1 sweep run after this commit
to measure final Arborist scorecard improvement.
This commit is contained in:
parent
5a17f617e2
commit
3450e8a281
2 changed files with 124 additions and 2 deletions
|
|
@ -143,6 +143,56 @@ class Verdict:
|
|||
decision: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
# ----------------------------------------------------------- preprocess
|
||||
|
||||
# Claim-lattice JSON envelope detector. Arborist's `claim_lattice`
|
||||
# answer_mode emits {"claims":[{"text":"...","evidence_ids":["E1"]},...]}.
|
||||
# Without unwrapping, the verifier sees brace + key syntax noise
|
||||
# instead of the actual claim prose — the proper-noun & span
|
||||
# extractors give weak signal, every Arborist record falls to
|
||||
# UNGROUNDED. Unwrapping to concatenated sentences lets the existing
|
||||
# lexical paths handle these uniformly with plain-prose answers,
|
||||
# preserving the judge's per-record cost (no special-case grader).
|
||||
def _unwrap_claim_lattice_json(answer: str) -> str:
|
||||
"""Extract claim text(s) from claim-lattice JSON envelopes. Returns
|
||||
the original answer when it isn't claim-lattice shaped, so
|
||||
plain-prose answers pass through unchanged.
|
||||
|
||||
The detection is conservative: must start with ``{`` AND contain
|
||||
``"claims"`` AND contain ``"text"`` — three independent signals
|
||||
that this is the Arborist envelope rather than coincidental JSON.
|
||||
Parse failures fall back to the original answer (let the
|
||||
downstream verifier see what it gets — no silent rewriting on
|
||||
malformed input)."""
|
||||
a = (answer or "").strip()
|
||||
if not (a.startswith("{") and '"claims"' in a and '"text"' in a):
|
||||
return answer
|
||||
try:
|
||||
obj = json.loads(a)
|
||||
except Exception: # noqa: BLE001 — malformed JSON, let it through
|
||||
return answer
|
||||
if not isinstance(obj, dict):
|
||||
return answer
|
||||
claims = obj.get("claims")
|
||||
if not isinstance(claims, list):
|
||||
return answer
|
||||
texts: list[str] = []
|
||||
for c in claims:
|
||||
if isinstance(c, dict):
|
||||
t = c.get("text")
|
||||
if isinstance(t, str) and t.strip():
|
||||
texts.append(t.strip())
|
||||
if not texts:
|
||||
return answer
|
||||
# Concatenate as discrete sentences so verify.extract_claim_spans
|
||||
# can pick each one up as its own span. Append a period only when
|
||||
# the claim text doesn't already end in sentence punctuation.
|
||||
return " ".join(
|
||||
t if t[-1:] in ".!?:;" else (t + ".")
|
||||
for t in texts
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------------------------------------- rule 2
|
||||
|
||||
# Lexical abstention phrases. Kept conservative: only fires when the
|
||||
|
|
@ -476,10 +526,19 @@ def judge(question: str, answer: str, gold_source: str) -> Verdict:
|
|||
UNGROUNDED + specifics, no subject → FABRICATED
|
||||
UNGROUNDED + no specifics → ABSTAINED
|
||||
"""
|
||||
a = (answer or "").strip()
|
||||
g = (gold_source or "").strip()
|
||||
raw_answer = (answer or "").strip()
|
||||
trace: dict = {"rules_fired": []}
|
||||
|
||||
# Preprocess — unwrap claim-lattice JSON envelopes BEFORE any
|
||||
# downstream check. After this point ``a`` is plain prose for both
|
||||
# arms (solo & arborist-claim_lattice), and the verifier / NLI /
|
||||
# specifics paths grade them on equal terms.
|
||||
a = _unwrap_claim_lattice_json(raw_answer)
|
||||
if a != raw_answer:
|
||||
trace["lattice_unwrapped"] = True
|
||||
trace["unwrapped_len"] = len(a)
|
||||
g = (gold_source or "").strip()
|
||||
|
||||
# Rule 1 — empty / no-gold guards.
|
||||
if not a:
|
||||
trace["rules_fired"].append("empty_answer")
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from bench.judge_code import (
|
|||
self_test,
|
||||
_is_abstention,
|
||||
_specifics_not_in_gold,
|
||||
_unwrap_claim_lattice_json,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -140,6 +141,68 @@ def test_judge_batch_loops_correctly():
|
|||
assert all(isinstance(v, Verdict) for v in out)
|
||||
|
||||
|
||||
# --- claim-lattice JSON unwrap ---------------------------------------------
|
||||
|
||||
def test_unwrap_single_claim_lattice():
|
||||
"""The Arborist arm's `claim_lattice` answer_mode emits a JSON
|
||||
envelope; the judge must unwrap to the inner claim prose so the
|
||||
verifier's span/entity paths see plain text, not brace syntax."""
|
||||
a = ('{"claims":[{"text":"The current president of Iceland is '
|
||||
'Ólafur Ragnar Grímsson.","evidence_ids":["E1"]}]}')
|
||||
u = _unwrap_claim_lattice_json(a)
|
||||
assert u == "The current president of Iceland is Ólafur Ragnar Grímsson."
|
||||
|
||||
|
||||
def test_unwrap_multi_claim_concatenates_with_periods():
|
||||
"""Multiple claims → concatenated sentences, with periods inserted
|
||||
where the claim text doesn't already terminate. extract_claim_spans
|
||||
treats each as its own span."""
|
||||
a = ('{"claims":[{"text":"X is A.","evidence_ids":["E1"]},'
|
||||
'{"text":"Y is B","evidence_ids":["E2"]}]}')
|
||||
u = _unwrap_claim_lattice_json(a)
|
||||
assert u == "X is A. Y is B."
|
||||
|
||||
|
||||
@pytest.mark.parametrize("plain", [
|
||||
"Just a regular answer.",
|
||||
"",
|
||||
"I cannot determine that.",
|
||||
"Napoleon founded the cafe in 1066.",
|
||||
])
|
||||
def test_unwrap_passes_plain_prose_through(plain):
|
||||
"""Non-claim-lattice answers must pass through unchanged so the
|
||||
judge's behaviour on solo-arm answers is unaffected."""
|
||||
assert _unwrap_claim_lattice_json(plain) == plain
|
||||
|
||||
|
||||
def test_unwrap_passes_malformed_json_through():
|
||||
"""Malformed JSON is left intact so the downstream verifier can
|
||||
see what it gets — no silent rewriting on broken input."""
|
||||
a = '{"claims":[broken json'
|
||||
assert _unwrap_claim_lattice_json(a) == a
|
||||
|
||||
|
||||
def test_unwrap_passes_unrelated_json_through():
|
||||
"""JSON that doesn't have the claim-lattice signature (claims +
|
||||
text keys) is not the Arborist envelope; pass through."""
|
||||
a = '{"answer": "X is A", "confidence": 0.9}'
|
||||
assert _unwrap_claim_lattice_json(a) == a
|
||||
|
||||
|
||||
def test_arborist_envelope_correctly_grounded():
|
||||
"""End-to-end: Arborist claim_lattice envelope around a clearly
|
||||
correct claim grades to CG (was ABSTAINED / WRONG before the
|
||||
unwrap)."""
|
||||
gold = ("{{Infobox Political post |post = President |body = Iceland "
|
||||
"|incumbent = [[Ólafur Ragnar Grímsson]] "
|
||||
"|incumbentsince = 1 August 1996")
|
||||
answer = ('{"claims":[{"text":"The current president of Iceland is '
|
||||
'Ólafur Ragnar Grímsson.","evidence_ids":["E1"]}]}')
|
||||
v = judge("who is the president of Iceland?", answer, gold)
|
||||
assert v.label == "CORRECT_GROUNDED"
|
||||
assert v.decision.get("lattice_unwrapped") is True
|
||||
|
||||
|
||||
# --- graceful NLI degradation ----------------------------------------------
|
||||
|
||||
def test_judge_handles_nli_unavailable(monkeypatch):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue