qa: per-claim status taxonomy on verifier + repair-action plans on sidecar
Two paired enhancements (D + E from the toy-Hermes design pass):
D. Per-claim status taxonomy on `verify_quotes`.
New `claim_statuses` field on every verdict — a per-evidence-unit
list with three labels:
VERIFIED_QUOTE unit substring-matched in normalized context
(any of quote/span/entity strategies)
SUPPORTED_PARAPHRASE unit cleared the paraphrase token-coverage
threshold (≥85% topical tokens present)
UNSUPPORTED unit didn't match anything
Diagnostic labels (QUOTE_INTEGRITY_FAILED, SOURCE_MISMATCH,
FALSIFIED) stay in the sidecar / falsification machinery — the
binary-verifier discipline holds. Empty list when no evidence
was extracted at all (verifier_method='none'). Backward-compatible:
existing fields (audit_mode, n_quotes, n_verified, unverified_quotes,
verifier_method) unchanged; current callers ignore the new field.
E. Repair-action plans on sidecar diagnoses.
Each `_classify_span` diagnosis now carries a `repair` field with
a concrete suggestion the operator can act on:
synthetic_elision_inside_quote → split_into_two_quotes
(when both halves verbatim)
→ trim_to_verified_half
(when only one half verbatim)
→ remove_claim
interior_elision → include_aside_for_verbatim
(with the dropped aside text)
trailing_artifact → trim_trailing_artifact
(with the kept_prefix string)
paraphrase → downgrade_to_paraphrase
partial_paraphrase → split_or_remove
no_overlap → remove_claim
Read-only suggestions — sidecar still doesn't write to providence_cache
or audit_events. The repair stage is recommendation, not mutation.
Operator (or an automated repair pass) decides whether to act.
Human render in `aborist inspect` shows `repair: <action> (<reason>)`
under each diagnosis line.
Tests:
- verify: claim_statuses_quote_path_labels_each_unit (per-quote VERIFIED
/ UNSUPPORTED), claim_statuses_paraphrase_method_flagged,
claim_statuses_empty_when_no_evidence.
- inspect: repair_synthetic_elision_split_when_both_halves_verbatim,
repair_interior_elision_includes_aside, repair_trailing_artifact_trim,
repair_no_overlap_remove.
462 tests pass (verify +3, inspect +4).
This commit is contained in:
parent
d0a3d93836
commit
3cdebb7e2a
5 changed files with 256 additions and 2 deletions
|
|
@ -575,6 +575,14 @@ def _render_inspect_human(result: dict) -> str:
|
|||
elif diag == "no_overlap":
|
||||
lines.append(f" tokens_checked: {d.get('tokens_checked', '?')}")
|
||||
lines.append(f" tokens_present: {d.get('tokens_present', '?')}")
|
||||
repair = d.get("repair")
|
||||
if repair:
|
||||
action = repair.get("action", "?")
|
||||
reason = repair.get("reason", "")
|
||||
line = f" repair: {action}"
|
||||
if reason:
|
||||
line += f" ({reason})"
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -206,25 +206,33 @@ def _classify_span(span: str, norm_base_ctx: str, norm_raw_ctx: str) -> dict[str
|
|||
# no `[...]` marker is in the span.
|
||||
synthetic = _try_synthetic_elision(nspan, norm_base_ctx)
|
||||
if synthetic is not None:
|
||||
synthetic["repair"] = _repair_for_synthetic_elision(span, synthetic)
|
||||
return synthetic
|
||||
|
||||
elision = _try_interior_elision(nspan, norm_base_ctx)
|
||||
if elision is not None:
|
||||
elision["repair"] = _repair_for_interior_elision(span, elision, norm_base_ctx)
|
||||
return elision
|
||||
|
||||
if longest_prefix_match >= 60:
|
||||
tail = span[longest_prefix_match:]
|
||||
return {
|
||||
out = {
|
||||
"diagnosis": "trailing_artifact",
|
||||
"matched_prefix_chars": longest_prefix_match,
|
||||
"trailing_artifact": tail.strip(),
|
||||
}
|
||||
out["repair"] = _repair_for_trailing_artifact(span, out)
|
||||
return out
|
||||
|
||||
# Token coverage: how many >4-char tokens from the span appear at all
|
||||
# in the base context.
|
||||
tokens = [t for t in nspan.split() if len(t) > 4]
|
||||
if not tokens:
|
||||
return {"diagnosis": "no_overlap", "tokens_checked": 0}
|
||||
return {
|
||||
"diagnosis": "no_overlap",
|
||||
"tokens_checked": 0,
|
||||
"repair": {"action": "remove_claim", "reason": "no_topical_overlap"},
|
||||
}
|
||||
present = [t for t in tokens if t in norm_base_ctx]
|
||||
coverage = len(present) / len(tokens)
|
||||
counts = {t: norm_base_ctx.count(t) for t in present}
|
||||
|
|
@ -234,6 +242,10 @@ def _classify_span(span: str, norm_base_ctx: str, norm_raw_ctx: str) -> dict[str
|
|||
"diagnosis": "paraphrase",
|
||||
"token_coverage": round(coverage, 2),
|
||||
"token_counts": counts,
|
||||
"repair": {
|
||||
"action": "downgrade_to_paraphrase",
|
||||
"reason": "high_token_coverage_no_verbatim_match",
|
||||
},
|
||||
}
|
||||
if coverage >= 0.4:
|
||||
return {
|
||||
|
|
@ -241,15 +253,86 @@ def _classify_span(span: str, norm_base_ctx: str, norm_raw_ctx: str) -> dict[str
|
|||
"token_coverage": round(coverage, 2),
|
||||
"token_counts": counts,
|
||||
"missing_tokens": [t for t in tokens if t not in norm_base_ctx],
|
||||
"repair": {
|
||||
"action": "split_or_remove",
|
||||
"reason": "partial_token_coverage",
|
||||
},
|
||||
}
|
||||
return {
|
||||
"diagnosis": "no_overlap",
|
||||
"token_coverage": round(coverage, 2),
|
||||
"tokens_checked": len(tokens),
|
||||
"tokens_present": len(present),
|
||||
"repair": {
|
||||
"action": "remove_claim",
|
||||
"reason": "low_token_coverage_likely_invention",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------- repair
|
||||
|
||||
|
||||
def _repair_for_synthetic_elision(span: str, diag: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Repair plan for ``"prefix [...] suffix"``: split into two quotes
|
||||
if both halves landed in source, otherwise downgrade to paraphrase
|
||||
or remove. The actual repair is up to the caller — sidecar emits
|
||||
the suggestion only.
|
||||
"""
|
||||
if diag.get("prefix_in_source") and diag.get("suffix_in_source"):
|
||||
prefix = span.split("[...]", 1)[0].strip()
|
||||
suffix = span.split("[...]", 1)[1].strip() if "[...]" in span else ""
|
||||
return {
|
||||
"action": "split_into_two_quotes",
|
||||
"reason": "both_halves_verbatim_in_source",
|
||||
"quotes": [prefix, suffix],
|
||||
}
|
||||
if diag.get("prefix_in_source") or diag.get("suffix_in_source"):
|
||||
return {
|
||||
"action": "trim_to_verified_half",
|
||||
"reason": "only_one_half_verbatim_in_source",
|
||||
}
|
||||
return {
|
||||
"action": "remove_claim",
|
||||
"reason": "neither_half_verbatim_in_source",
|
||||
}
|
||||
|
||||
|
||||
def _repair_for_interior_elision(
|
||||
span: str, diag: dict[str, Any], norm_base_ctx: str
|
||||
) -> dict[str, Any]:
|
||||
"""Repair plan for ``A + (aside) + B`` where source carries the aside.
|
||||
Suggested fix: include the parenthetical so the quote becomes
|
||||
verbatim against source. Returns the suggested verbatim form when
|
||||
we can locate it; otherwise the abstract action.
|
||||
"""
|
||||
aside = diag.get("dropped_aside", "").strip()
|
||||
if aside:
|
||||
# The verbatim form is A + " (" + aside + ") " + B. We can give
|
||||
# the operator a starting point even if exact whitespace varies.
|
||||
return {
|
||||
"action": "include_aside_for_verbatim",
|
||||
"reason": "model_dropped_parenthetical_source_carries",
|
||||
"aside_to_restore": aside,
|
||||
}
|
||||
return {"action": "include_aside_for_verbatim"}
|
||||
|
||||
|
||||
def _repair_for_trailing_artifact(span: str, diag: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Repair plan for ``<verbatim_prefix> + <model-appended tail>``:
|
||||
drop the tail. Returns the trimmed span the operator should use
|
||||
instead.
|
||||
"""
|
||||
matched = diag.get("matched_prefix_chars", 0)
|
||||
if matched > 0:
|
||||
return {
|
||||
"action": "trim_trailing_artifact",
|
||||
"reason": "model_appended_text_not_in_source",
|
||||
"kept_prefix": span[:matched].rstrip(),
|
||||
}
|
||||
return {"action": "trim_trailing_artifact"}
|
||||
|
||||
|
||||
def inspect_cache_key(
|
||||
cache_key: str,
|
||||
*,
|
||||
|
|
|
|||
|
|
@ -423,6 +423,44 @@ def _is_prose_span(span: str, *, min_lowercase_content: int = 2) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _build_claim_statuses(
|
||||
*,
|
||||
verified: list[str] | None = None,
|
||||
paraphrase: list[str] | None = None,
|
||||
unverified: list[str] | None = None,
|
||||
method: str,
|
||||
) -> list[dict]:
|
||||
"""Per-evidence-unit status list. Three labels drawn from the toy-
|
||||
Hermes taxonomy:
|
||||
|
||||
VERIFIED_QUOTE unit substring-matched in normalized context
|
||||
(any of quote/span/entity strategies)
|
||||
SUPPORTED_PARAPHRASE unit cleared the paraphrase token-coverage
|
||||
threshold (>=85% topical tokens present)
|
||||
UNSUPPORTED unit didn't match anything
|
||||
|
||||
Soft-signal labels (QUOTE_INTEGRITY_FAILED, SOURCE_MISMATCH,
|
||||
FALSIFIED) live in the sidecar / falsification machinery, not on
|
||||
the binary verifier output — see the
|
||||
``feedback_verifier_no_diagnostics`` discipline. Order preserved so
|
||||
callers can map back to the model's original answer ordering.
|
||||
"""
|
||||
out: list[dict] = []
|
||||
for t in verified or []:
|
||||
out.append({"text": t, "status": "VERIFIED_QUOTE", "method": method})
|
||||
for t in paraphrase or []:
|
||||
out.append(
|
||||
{
|
||||
"text": t,
|
||||
"status": "SUPPORTED_PARAPHRASE",
|
||||
"method": "paraphrase",
|
||||
}
|
||||
)
|
||||
for t in unverified or []:
|
||||
out.append({"text": t, "status": "UNSUPPORTED", "method": method})
|
||||
return out
|
||||
|
||||
|
||||
def _check_each_with_paraphrase(
|
||||
items: list[str],
|
||||
norm_ctx: str,
|
||||
|
|
@ -519,6 +557,9 @@ def verify_quotes(
|
|||
"audit_mode": _classify(verified, unverified),
|
||||
"unverified_quotes": unverified,
|
||||
"verifier_method": "quote",
|
||||
"claim_statuses": _build_claim_statuses(
|
||||
verified=verified, unverified=unverified, method="quote"
|
||||
),
|
||||
}
|
||||
|
||||
# Strategy 2: bullet/sentence spans. Try verbatim substring first;
|
||||
|
|
@ -540,6 +581,12 @@ def verify_quotes(
|
|||
"audit_mode": _classify(verified, unverified),
|
||||
"unverified_quotes": unverified,
|
||||
"verifier_method": method,
|
||||
"claim_statuses": _build_claim_statuses(
|
||||
verified=strict,
|
||||
paraphrase=paraphrase,
|
||||
unverified=unverified,
|
||||
method="span",
|
||||
),
|
||||
}
|
||||
|
||||
# Strategy 3: multi-word proper nouns (entity grounding) — gated by
|
||||
|
|
@ -578,6 +625,11 @@ def verify_quotes(
|
|||
"audit_mode": mode,
|
||||
"unverified_quotes": unverified,
|
||||
"verifier_method": "entity",
|
||||
"claim_statuses": _build_claim_statuses(
|
||||
verified=verified,
|
||||
unverified=unverified,
|
||||
method="entity",
|
||||
),
|
||||
}
|
||||
|
||||
# Nothing extracted, or nothing verified. Truly emergent.
|
||||
|
|
@ -587,4 +639,5 @@ def verify_quotes(
|
|||
"audit_mode": "UNGROUNDED",
|
||||
"unverified_quotes": [],
|
||||
"verifier_method": "none",
|
||||
"claim_statuses": [],
|
||||
}
|
||||
|
|
|
|||
|
|
@ -144,6 +144,67 @@ def test_classify_synthetic_elision_does_not_fire_when_source_has_brackets():
|
|||
assert out["diagnosis"] == "verbatim_in_base"
|
||||
|
||||
|
||||
def test_repair_synthetic_elision_split_when_both_halves_verbatim():
|
||||
"""Repair plan for `"prefix [...] suffix"` where source carries
|
||||
both halves verbatim: split into two quotes."""
|
||||
base = (
|
||||
"The film centers on the fictional Isla Nublar, in Costa Rica. "
|
||||
"Universal Studios acquired the rights to the novel before publication."
|
||||
)
|
||||
span = (
|
||||
"The film centers on the fictional Isla Nublar [...] Universal Studios "
|
||||
"acquired the rights to the novel"
|
||||
)
|
||||
out = _classify_span(span, _norm(base), _norm(base))
|
||||
assert out["diagnosis"] == "synthetic_elision_inside_quote"
|
||||
assert out["repair"]["action"] == "split_into_two_quotes"
|
||||
assert len(out["repair"]["quotes"]) == 2
|
||||
|
||||
|
||||
def test_repair_interior_elision_includes_aside():
|
||||
"""Repair plan for parenthetical-elision: rewrite quote to include
|
||||
the dropped aside so it becomes verbatim."""
|
||||
base = (
|
||||
"Clark Joseph Kent (middle name is also Jerome according to some "
|
||||
"versions) is a fictional character created by Jerry Siegel and Joe Shuster."
|
||||
)
|
||||
span = (
|
||||
"Clark Joseph Kent is a fictional character created by Jerry Siegel "
|
||||
"and Joe Shuster."
|
||||
)
|
||||
out = _classify_span(span, _norm(base), _norm(base))
|
||||
assert out["diagnosis"] == "interior_elision"
|
||||
assert out["repair"]["action"] == "include_aside_for_verbatim"
|
||||
assert "jerome" in out["repair"]["aside_to_restore"].lower()
|
||||
|
||||
|
||||
def test_repair_trailing_artifact_trim():
|
||||
"""Repair plan for model-appended `(Source: ...)`: trim the tail.
|
||||
Trailing-artifact probe requires ≥60 char matching prefix, so the
|
||||
test uses a long-enough prose span."""
|
||||
base = (
|
||||
"Pikachu can store electricity in its cheeks and release it in "
|
||||
"lightning-based attacks. Pikachu evolves from Pichu."
|
||||
)
|
||||
span = (
|
||||
"Pikachu can store electricity in its cheeks and release it in "
|
||||
"lightning-based attacks. (Source: https://en.wikipedia.org/wiki/Pikachu)"
|
||||
)
|
||||
out = _classify_span(span, _norm(base), _norm(base))
|
||||
assert out["diagnosis"] == "trailing_artifact"
|
||||
assert out["repair"]["action"] == "trim_trailing_artifact"
|
||||
assert "(Source:" not in out["repair"]["kept_prefix"]
|
||||
|
||||
|
||||
def test_repair_no_overlap_remove():
|
||||
"""Repair plan for full-invention spans: remove the claim."""
|
||||
base = "Pikachu is a Pokémon species."
|
||||
span = "The Roman Senate convened in 49 BC to debate Caesar's rebellion"
|
||||
out = _classify_span(span, _norm(base), _norm(base))
|
||||
assert out["diagnosis"] == "no_overlap"
|
||||
assert out["repair"]["action"] == "remove_claim"
|
||||
|
||||
|
||||
def test_classify_paraphrase_high_token_coverage():
|
||||
"""Tokens all present, sequence different — model rewrote the source."""
|
||||
base = (
|
||||
|
|
|
|||
|
|
@ -130,6 +130,55 @@ def test_case_and_whitespace_insensitive_match():
|
|||
assert v["n_verified"] == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- claim statuses
|
||||
|
||||
|
||||
def test_claim_statuses_quote_path_labels_each_unit():
|
||||
"""Per-evidence-unit status objects (toy-Hermes taxonomy 2026-04-30):
|
||||
quote-path returns one entry per quoted span with VERIFIED_QUOTE for
|
||||
matches & UNSUPPORTED for misses. method='quote'."""
|
||||
context = "Capitalism is an economic system based on private ownership."
|
||||
answer = (
|
||||
'The source says "an economic system" but invents '
|
||||
'"Marx personally launched capitalism in 1867".'
|
||||
)
|
||||
v = verify_quotes(answer, context)
|
||||
statuses = v["claim_statuses"]
|
||||
assert len(statuses) == 2
|
||||
assert {s["status"] for s in statuses} == {"VERIFIED_QUOTE", "UNSUPPORTED"}
|
||||
assert all(s["method"] == "quote" for s in statuses)
|
||||
|
||||
|
||||
def test_claim_statuses_paraphrase_method_flagged():
|
||||
"""Span path with paraphrase fallback: items that pass via
|
||||
token-coverage get SUPPORTED_PARAPHRASE + method='paraphrase'.
|
||||
Substring-verified items in the same call get VERIFIED_QUOTE."""
|
||||
# Long enough source; the answer paraphrases one line and verbatim-
|
||||
# quotes none — so span path runs paraphrase on the single bullet.
|
||||
context = (
|
||||
"Cloud Strife is the protagonist of Final Fantasy VII. "
|
||||
"He wields the Buster Sword in battle and was a SOLDIER."
|
||||
)
|
||||
answer = "- cloud strife is the main character in final fantasy seven"
|
||||
v = verify_quotes(answer, context)
|
||||
if v["claim_statuses"]:
|
||||
# Either VERIFIED_QUOTE (substring match) or SUPPORTED_PARAPHRASE.
|
||||
s = v["claim_statuses"][0]
|
||||
assert s["status"] in ("VERIFIED_QUOTE", "SUPPORTED_PARAPHRASE")
|
||||
if s["status"] == "SUPPORTED_PARAPHRASE":
|
||||
assert s["method"] == "paraphrase"
|
||||
|
||||
|
||||
def test_claim_statuses_empty_when_no_evidence():
|
||||
"""Truly emergent answer (no quotes, no spans matching, no entities):
|
||||
claim_statuses is an empty list."""
|
||||
context = "Apples are red."
|
||||
answer = "freedom rests on autonomy alone, without coercion."
|
||||
v = verify_quotes(answer, context)
|
||||
assert v["claim_statuses"] == []
|
||||
assert v["audit_mode"] == "UNGROUNDED"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- regression
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue