fix(#000057): code judge unwraps Arborist claim-lattice JSON envelopes

The Arborist arm runs answer_mode='claim_lattice' (per control_sweep.py
:179, control_ab.py:155) so its answers arrive as the JSON envelope
  {"claims":[{"text":"...","evidence_ids":["E1"]},...]}.
_descaffold strips the [E1] evidence-pointer markup but the JSON
braces + key syntax remain. The verifier's strategy-2 (span) and
strategy-3 (proper-noun) extractors see brace noise instead of the
inner claim prose — every Arborist record degraded to UNGROUNDED.

The 2026-05-19T17-01-17Z sweep, re-graded with the freshly calibrated
judge (5a17f61), surfaced this: Arborist arm reported 0 CG across all
three variants in the live phase 1 output (the live run was pre-
calibration), and 29/120 CG (24%) under the calibrated rescore — clear
improvement just from theta_contra=0.85, but the JSON envelope was
still hobbling the verifier paths.

Fix: _unwrap_claim_lattice_json runs BEFORE all downstream rules.
Detection is conservative (three independent signals: starts-with-
brace AND "claims" key AND "text" key) so plain-prose answers
pass through unchanged. Multi-claim envelopes concatenate as discrete
sentences (extract_claim_spans treats each as its own span).
Malformed JSON falls back to the original answer — no silent
rewriting on broken input.

Smoke result on the Iceland Arborist case
  ans:  {"claims":[{"text":"The current president of Iceland is
         Ólafur Ragnar Grímsson.","evidence_ids":["E1"]}]}
  gold: {{Infobox Political post |post = President |body = Iceland
         |incumbent = [[Ólafur Ragnar Grímsson]] ...}}
  before: UNGROUNDED → FABRICATED (then WRONG after calibration)
  after:  short_entity_grounded → CORRECT_GROUNDED

pytest: 27/27 (added 7 unwrap-coverage tests covering single-claim
envelopes, multi-claim concatenation, plain-prose passthrough,
malformed-JSON tolerance, unrelated-JSON passthrough, and the
end-to-end Arborist-envelope CG flow). Self-test 4/4 unchanged.

Re-rescores of 17:01 sweep + phase 1 sweep run after this commit
to measure final Arborist scorecard improvement.
This commit is contained in:
russell@unturf.com 2026-05-19 18:53:17 -04:00
parent 5a17f617e2
commit 3450e8a281
No known key found for this signature in database
2 changed files with 124 additions and 2 deletions

View file

@ -143,6 +143,56 @@ class Verdict:
decision: dict = field(default_factory=dict)
# ----------------------------------------------------------- preprocess
# Claim-lattice JSON envelope detector. Arborist's `claim_lattice`
# answer_mode emits {"claims":[{"text":"...","evidence_ids":["E1"]},...]}.
# Without unwrapping, the verifier sees brace + key syntax noise
# instead of the actual claim prose — the proper-noun & span
# extractors give weak signal, every Arborist record falls to
# UNGROUNDED. Unwrapping to concatenated sentences lets the existing
# lexical paths handle these uniformly with plain-prose answers,
# preserving the judge's per-record cost (no special-case grader).
def _unwrap_claim_lattice_json(answer: str) -> str:
"""Extract claim text(s) from claim-lattice JSON envelopes. Returns
the original answer when it isn't claim-lattice shaped, so
plain-prose answers pass through unchanged.
The detection is conservative: must start with ``{`` AND contain
``"claims"`` AND contain ``"text"`` three independent signals
that this is the Arborist envelope rather than coincidental JSON.
Parse failures fall back to the original answer (let the
downstream verifier see what it gets no silent rewriting on
malformed input)."""
a = (answer or "").strip()
if not (a.startswith("{") and '"claims"' in a and '"text"' in a):
return answer
try:
obj = json.loads(a)
except Exception: # noqa: BLE001 — malformed JSON, let it through
return answer
if not isinstance(obj, dict):
return answer
claims = obj.get("claims")
if not isinstance(claims, list):
return answer
texts: list[str] = []
for c in claims:
if isinstance(c, dict):
t = c.get("text")
if isinstance(t, str) and t.strip():
texts.append(t.strip())
if not texts:
return answer
# Concatenate as discrete sentences so verify.extract_claim_spans
# can pick each one up as its own span. Append a period only when
# the claim text doesn't already end in sentence punctuation.
return " ".join(
t if t[-1:] in ".!?:;" else (t + ".")
for t in texts
)
# ----------------------------------------------------------- rule 2
# Lexical abstention phrases. Kept conservative: only fires when the
@ -476,10 +526,19 @@ def judge(question: str, answer: str, gold_source: str) -> Verdict:
UNGROUNDED + specifics, no subject FABRICATED
UNGROUNDED + no specifics ABSTAINED
"""
a = (answer or "").strip()
g = (gold_source or "").strip()
raw_answer = (answer or "").strip()
trace: dict = {"rules_fired": []}
# Preprocess — unwrap claim-lattice JSON envelopes BEFORE any
# downstream check. After this point ``a`` is plain prose for both
# arms (solo & arborist-claim_lattice), and the verifier / NLI /
# specifics paths grade them on equal terms.
a = _unwrap_claim_lattice_json(raw_answer)
if a != raw_answer:
trace["lattice_unwrapped"] = True
trace["unwrapped_len"] = len(a)
g = (gold_source or "").strip()
# Rule 1 — empty / no-gold guards.
if not a:
trace["rules_fired"].append("empty_answer")

View file

@ -27,6 +27,7 @@ from bench.judge_code import (
self_test,
_is_abstention,
_specifics_not_in_gold,
_unwrap_claim_lattice_json,
)
@ -140,6 +141,68 @@ def test_judge_batch_loops_correctly():
assert all(isinstance(v, Verdict) for v in out)
# --- claim-lattice JSON unwrap ---------------------------------------------
def test_unwrap_single_claim_lattice():
"""The Arborist arm's `claim_lattice` answer_mode emits a JSON
envelope; the judge must unwrap to the inner claim prose so the
verifier's span/entity paths see plain text, not brace syntax."""
a = ('{"claims":[{"text":"The current president of Iceland is '
'Ólafur Ragnar Grímsson.","evidence_ids":["E1"]}]}')
u = _unwrap_claim_lattice_json(a)
assert u == "The current president of Iceland is Ólafur Ragnar Grímsson."
def test_unwrap_multi_claim_concatenates_with_periods():
"""Multiple claims → concatenated sentences, with periods inserted
where the claim text doesn't already terminate. extract_claim_spans
treats each as its own span."""
a = ('{"claims":[{"text":"X is A.","evidence_ids":["E1"]},'
'{"text":"Y is B","evidence_ids":["E2"]}]}')
u = _unwrap_claim_lattice_json(a)
assert u == "X is A. Y is B."
@pytest.mark.parametrize("plain", [
"Just a regular answer.",
"",
"I cannot determine that.",
"Napoleon founded the cafe in 1066.",
])
def test_unwrap_passes_plain_prose_through(plain):
"""Non-claim-lattice answers must pass through unchanged so the
judge's behaviour on solo-arm answers is unaffected."""
assert _unwrap_claim_lattice_json(plain) == plain
def test_unwrap_passes_malformed_json_through():
"""Malformed JSON is left intact so the downstream verifier can
see what it gets no silent rewriting on broken input."""
a = '{"claims":[broken json'
assert _unwrap_claim_lattice_json(a) == a
def test_unwrap_passes_unrelated_json_through():
"""JSON that doesn't have the claim-lattice signature (claims +
text keys) is not the Arborist envelope; pass through."""
a = '{"answer": "X is A", "confidence": 0.9}'
assert _unwrap_claim_lattice_json(a) == a
def test_arborist_envelope_correctly_grounded():
"""End-to-end: Arborist claim_lattice envelope around a clearly
correct claim grades to CG (was ABSTAINED / WRONG before the
unwrap)."""
gold = ("{{Infobox Political post |post = President |body = Iceland "
"|incumbent = [[Ólafur Ragnar Grímsson]] "
"|incumbentsince = 1 August 1996")
answer = ('{"claims":[{"text":"The current president of Iceland is '
'Ólafur Ragnar Grímsson.","evidence_ids":["E1"]}]}')
v = judge("who is the president of Iceland?", answer, gold)
assert v.label == "CORRECT_GROUNDED"
assert v.decision.get("lattice_unwrapped") is True
# --- graceful NLI degradation ----------------------------------------------
def test_judge_handles_nli_unavailable(monkeypatch):