fix(#000057): code judge unwraps Arborist claim-lattice JSON envelopes
The Arborist arm runs answer_mode='claim_lattice' (per control_sweep.py
:179, control_ab.py:155) so its answers arrive as the JSON envelope
{"claims":[{"text":"...","evidence_ids":["E1"]},...]}.
_descaffold strips the [E1] evidence-pointer markup but the JSON
braces + key syntax remain. The verifier's strategy-2 (span) and
strategy-3 (proper-noun) extractors see brace noise instead of the
inner claim prose — every Arborist record degraded to UNGROUNDED.
The 2026-05-19T17-01-17Z sweep, re-graded with the freshly calibrated
judge (5a17f61), surfaced this: Arborist arm reported 0 CG across all
three variants in the live phase 1 output (the live run was pre-
calibration), and 29/120 CG (24%) under the calibrated rescore — clear
improvement just from theta_contra=0.85, but the JSON envelope was
still hobbling the verifier paths.
Fix: _unwrap_claim_lattice_json runs BEFORE all downstream rules.
Detection is conservative (three independent signals: starts-with-
brace AND "claims" key AND "text" key) so plain-prose answers
pass through unchanged. Multi-claim envelopes concatenate as discrete
sentences (extract_claim_spans treats each as its own span).
Malformed JSON falls back to the original answer — no silent
rewriting on broken input.
Smoke result on the Iceland Arborist case
ans: {"claims":[{"text":"The current president of Iceland is
Ólafur Ragnar Grímsson.","evidence_ids":["E1"]}]}
gold: {{Infobox Political post |post = President |body = Iceland
|incumbent = [[Ólafur Ragnar Grímsson]] ...}}
before: UNGROUNDED → FABRICATED (then WRONG after calibration)
after: short_entity_grounded → CORRECT_GROUNDED
pytest: 27/27 (added 7 unwrap-coverage tests covering single-claim
envelopes, multi-claim concatenation, plain-prose passthrough,
malformed-JSON tolerance, unrelated-JSON passthrough, and the
end-to-end Arborist-envelope CG flow). Self-test 4/4 unchanged.
Re-rescores of 17:01 sweep + phase 1 sweep run after this commit
to measure final Arborist scorecard improvement.
This commit is contained in:
parent
5a17f617e2
commit
3450e8a281
2 changed files with 124 additions and 2 deletions
|
|
@ -143,6 +143,56 @@ class Verdict:
|
|||
decision: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
# ----------------------------------------------------------- preprocess
|
||||
|
||||
# Claim-lattice JSON envelope detector. Arborist's `claim_lattice`
|
||||
# answer_mode emits {"claims":[{"text":"...","evidence_ids":["E1"]},...]}.
|
||||
# Without unwrapping, the verifier sees brace + key syntax noise
|
||||
# instead of the actual claim prose — the proper-noun & span
|
||||
# extractors give weak signal, every Arborist record falls to
|
||||
# UNGROUNDED. Unwrapping to concatenated sentences lets the existing
|
||||
# lexical paths handle these uniformly with plain-prose answers,
|
||||
# preserving the judge's per-record cost (no special-case grader).
|
||||
def _unwrap_claim_lattice_json(answer: str) -> str:
|
||||
"""Extract claim text(s) from claim-lattice JSON envelopes. Returns
|
||||
the original answer when it isn't claim-lattice shaped, so
|
||||
plain-prose answers pass through unchanged.
|
||||
|
||||
The detection is conservative: must start with ``{`` AND contain
|
||||
``"claims"`` AND contain ``"text"`` — three independent signals
|
||||
that this is the Arborist envelope rather than coincidental JSON.
|
||||
Parse failures fall back to the original answer (let the
|
||||
downstream verifier see what it gets — no silent rewriting on
|
||||
malformed input)."""
|
||||
a = (answer or "").strip()
|
||||
if not (a.startswith("{") and '"claims"' in a and '"text"' in a):
|
||||
return answer
|
||||
try:
|
||||
obj = json.loads(a)
|
||||
except Exception: # noqa: BLE001 — malformed JSON, let it through
|
||||
return answer
|
||||
if not isinstance(obj, dict):
|
||||
return answer
|
||||
claims = obj.get("claims")
|
||||
if not isinstance(claims, list):
|
||||
return answer
|
||||
texts: list[str] = []
|
||||
for c in claims:
|
||||
if isinstance(c, dict):
|
||||
t = c.get("text")
|
||||
if isinstance(t, str) and t.strip():
|
||||
texts.append(t.strip())
|
||||
if not texts:
|
||||
return answer
|
||||
# Concatenate as discrete sentences so verify.extract_claim_spans
|
||||
# can pick each one up as its own span. Append a period only when
|
||||
# the claim text doesn't already end in sentence punctuation.
|
||||
return " ".join(
|
||||
t if t[-1:] in ".!?:;" else (t + ".")
|
||||
for t in texts
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------------------------------------- rule 2
|
||||
|
||||
# Lexical abstention phrases. Kept conservative: only fires when the
|
||||
|
|
@ -476,10 +526,19 @@ def judge(question: str, answer: str, gold_source: str) -> Verdict:
|
|||
UNGROUNDED + specifics, no subject → FABRICATED
|
||||
UNGROUNDED + no specifics → ABSTAINED
|
||||
"""
|
||||
a = (answer or "").strip()
|
||||
g = (gold_source or "").strip()
|
||||
raw_answer = (answer or "").strip()
|
||||
trace: dict = {"rules_fired": []}
|
||||
|
||||
# Preprocess — unwrap claim-lattice JSON envelopes BEFORE any
|
||||
# downstream check. After this point ``a`` is plain prose for both
|
||||
# arms (solo & arborist-claim_lattice), and the verifier / NLI /
|
||||
# specifics paths grade them on equal terms.
|
||||
a = _unwrap_claim_lattice_json(raw_answer)
|
||||
if a != raw_answer:
|
||||
trace["lattice_unwrapped"] = True
|
||||
trace["unwrapped_len"] = len(a)
|
||||
g = (gold_source or "").strip()
|
||||
|
||||
# Rule 1 — empty / no-gold guards.
|
||||
if not a:
|
||||
trace["rules_fired"].append("empty_answer")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue