qa: pointer-format includes source title + chunk prefix; lock EVIDENCE-LINKED label
Two UX fixes addressing the 2026-05-01 Orwell run feedback: (1) Evidence pointer format. Old form `[E5: "<excerpt>"]` looked visually like a 1-indexed source rank — paired with a "sources (8)" list whose `[5]` slot was a different document, operators easily mis-attributed citations. New form: [E5 | Nineteen Eighty-Four | 682f0a11: "<excerpt>"] Title comes from EvidenceObject.title (URI-tail fallback when None); chunk_root prefix is the first 8 hex chars — enough to disambiguate chunks from the same source while staying compact. Renderer change in render_claim_lattice; pre-existing claim-lattice tests updated; 3 new unit tests pin the format (title-inline, URI-tail fallback, no `[E1:` collision-shape). (2) Audit-label discipline. _render_audit_label already maps STRICT → EVIDENCE-LINKED for claim_lattice* modes (verifier output stays binary, schema CHECK constraint stays); 4 new tests in test_cli_render.py pin the relabel for STRICT / HYBRID / UNGROUNDED in claim_lattice modes and the no-relabel for quote-mode where the verifier verifies pinned spans not synthesis. Net: 8 new tests, full suite 649 passed.
This commit is contained in:
parent
bf8d93caba
commit
3586eeeb0a
2 changed files with 141 additions and 5 deletions
|
|
@ -192,7 +192,11 @@ def test_verify_renders_with_literal_spans_from_runtime():
|
|||
v = verify_claim_lattice(answer, em)
|
||||
# Renderer pulls literal span from runtime, not from model output.
|
||||
assert em[0].span in v["rendered_text"]
|
||||
assert "[E1:" in v["rendered_text"]
|
||||
# New pointer format: [E1 | <title> | <chunk_prefix>: "<excerpt>"]
|
||||
# Closes the visual provenance gap where E# could be confused
|
||||
# with the source-list 1-indexed rank.
|
||||
assert "[E1 |" in v["rendered_text"]
|
||||
assert "Jurassic Park (film)" in v["rendered_text"]
|
||||
|
||||
|
||||
def test_verify_passes_multi_pointer_claims():
|
||||
|
|
@ -295,8 +299,9 @@ def test_partial_pointer_within_claim_yields_partial():
|
|||
v = verify_claim_lattice(answer, em, min_claim_content_tokens=0)
|
||||
assert v["audit_mode"] == "HYBRID"
|
||||
assert v["claim_statuses"][0]["status"] == "EVIDENCE_LINKED_PARTIAL"
|
||||
# Renderer keeps only resolved ids.
|
||||
assert "E1:" in v["rendered_text"]
|
||||
# Renderer keeps only resolved ids — new pointer format
|
||||
# `[E1 | <title> | <chunk_prefix>: ...]` since 2026-05-01.
|
||||
assert "[E1 |" in v["rendered_text"]
|
||||
assert "E99" not in v["rendered_text"]
|
||||
|
||||
|
||||
|
|
@ -588,8 +593,11 @@ def test_render_claim_lattice_with_pointer_ids():
|
|||
)
|
||||
assert "Tyrannosaurus rex appears" in text
|
||||
assert "Velociraptors are shown" in text
|
||||
assert "[E1:" in text
|
||||
assert "[E2:" in text
|
||||
# New format includes source title + chunk_root prefix to avoid
|
||||
# E# / source-rank visual confusion (2026-05-01).
|
||||
assert "[E1 |" in text
|
||||
assert "[E2 |" in text
|
||||
assert "Jurassic Park (film)" in text
|
||||
|
||||
|
||||
def test_render_claim_lattice_marks_unknown():
|
||||
|
|
@ -602,6 +610,70 @@ def test_render_claim_lattice_marks_unknown():
|
|||
assert "E99: ?" in text
|
||||
|
||||
|
||||
def test_render_pointer_format_includes_source_title_and_chunk_prefix():
|
||||
"""Pointer format: `[E# | <title> | <chunk_prefix>: "<excerpt>"]`.
|
||||
Closes the 2026-05-01 visual provenance gap on the Orwell run
|
||||
where `[E5: ...]` paired with a source list whose `[5]` was an
|
||||
unrelated document — operator could mis-attribute the citation.
|
||||
Now title + chunk_root prefix are inline so each pointer is
|
||||
self-identifying."""
|
||||
em = build_evidence_map(_sample_chunks())
|
||||
by_pid = evidence_map_by_pointer_id(em)
|
||||
text = render_claim_lattice(
|
||||
[{"text": "trex appears", "pointer_ids": ["E1"]}],
|
||||
by_pid,
|
||||
)
|
||||
# Title appears inline with the pointer.
|
||||
assert "[E1 | Jurassic Park (film) | " in text
|
||||
# chunk_root prefix is the first 8 hex chars of em[0].chunk_root.
|
||||
chunk_prefix = em[0].chunk_root[:8]
|
||||
assert chunk_prefix in text
|
||||
# Excerpt still rendered after the prefix.
|
||||
assert "Tyrannosaurus rex appears" in text
|
||||
|
||||
|
||||
def test_render_pointer_format_falls_back_to_uri_tail_when_title_none():
|
||||
"""If EvidenceObject.title is None (rare — source had no title
|
||||
metadata), the renderer falls back to the URI tail. Defensive,
|
||||
keeps the pointer self-identifying even when title indexing
|
||||
fails for a chunk."""
|
||||
chunks = [{
|
||||
"source_root": "f" * 64,
|
||||
"document_uri": "https://example.org/some-article-slug",
|
||||
"title": None, # explicit absence
|
||||
"chunk_idx": 0,
|
||||
"chunk_root": "ab" * 32,
|
||||
"span": "Some content here.",
|
||||
"source_role": "primary_answer_source",
|
||||
}]
|
||||
em = build_evidence_map(chunks)
|
||||
by_pid = evidence_map_by_pointer_id(em)
|
||||
text = render_claim_lattice(
|
||||
[{"text": "Some content", "pointer_ids": ["E1"]}],
|
||||
by_pid,
|
||||
)
|
||||
# URI tail used as label.
|
||||
assert "[E1 | some-article-slug | " in text
|
||||
|
||||
|
||||
def test_render_pointer_format_no_visual_collision_with_source_rank():
|
||||
"""Regression for the Orwell case: the substring `[E5:` (which
|
||||
would visually collide with a source-rank `[5]` in older
|
||||
renders) does not appear in the new output. The format change
|
||||
isn't optional — it's enforced."""
|
||||
em = build_evidence_map(_sample_chunks())
|
||||
by_pid = evidence_map_by_pointer_id(em)
|
||||
text = render_claim_lattice(
|
||||
[{"text": "trex appears", "pointer_ids": ["E1"]}],
|
||||
by_pid,
|
||||
)
|
||||
# Old format was `[E1: "..."]`; new format is `[E1 | ...: "..."]`.
|
||||
# The colon-after-id must be gone.
|
||||
assert "[E1:" not in text
|
||||
# New format always has the pipe separator.
|
||||
assert "[E1 |" in text
|
||||
|
||||
|
||||
# ---------- spotlight excerpt (G0.2) --------------------------------------
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -213,3 +213,67 @@ def test_render_capacity_thousand_separators():
|
|||
assert "61,550 chars" in out
|
||||
assert "60,000" in out
|
||||
assert "1,234 chars" in out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# label discipline — claim_lattice* modes get EVIDENCE-LINKED relabel
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_render_label_strict_in_claim_lattice_becomes_evidence_linked():
|
||||
"""claim_lattice / claim_lattice_pointer 'STRICT' verdicts mean
|
||||
'every pointer resolved + source_role allowed + token-coverage
|
||||
passed' — NOT semantic entailment. The display label spells out
|
||||
that distinction so users don't read STRICT as 'the answer is
|
||||
correct' on synthesis-heavy claims."""
|
||||
r = _result(
|
||||
audit_mode="STRICT",
|
||||
verifier_method="claim_lattice",
|
||||
n_quotes=1, n_verified=1,
|
||||
)
|
||||
out = _render_query_human(r, "q")
|
||||
assert "EVIDENCE-LINKED" in out
|
||||
assert "via claim_lattice" in out
|
||||
# The bare STRICT token should NOT appear on the summary line —
|
||||
# we relabeled deliberately.
|
||||
summary_line = out.splitlines()[1]
|
||||
assert "STRICT" not in summary_line
|
||||
|
||||
|
||||
def test_render_label_hybrid_in_claim_lattice_becomes_evidence_linked_partial():
|
||||
r = _result(
|
||||
audit_mode="HYBRID",
|
||||
verifier_method="claim_lattice_pointer",
|
||||
n_quotes=3, n_verified=1,
|
||||
)
|
||||
out = _render_query_human(r, "q")
|
||||
assert "EVIDENCE-LINKED-PARTIAL" in out
|
||||
assert "via claim_lattice_pointer" in out
|
||||
|
||||
|
||||
def test_render_label_ungrounded_in_claim_lattice_stays_ungrounded():
|
||||
"""UNGROUNDED already names what it means — no relabel needed."""
|
||||
r = _result(
|
||||
audit_mode="UNGROUNDED",
|
||||
verifier_method="claim_lattice",
|
||||
n_quotes=2, n_verified=0,
|
||||
)
|
||||
out = _render_query_human(r, "q")
|
||||
assert "UNGROUNDED" in out
|
||||
assert "via claim_lattice" in out
|
||||
|
||||
|
||||
def test_render_label_quote_mode_keeps_audit_mode_token():
|
||||
"""Quote / span / entity / paraphrase modes verify against
|
||||
pinned spans, not synthesis. STRICT in quote mode IS a strong
|
||||
claim about evidence units — keep the label as-is."""
|
||||
r = _result(
|
||||
audit_mode="STRICT",
|
||||
verifier_method="quote",
|
||||
n_quotes=2, n_verified=2,
|
||||
)
|
||||
out = _render_query_human(r, "q")
|
||||
assert "STRICT" in out
|
||||
assert "via quote" in out
|
||||
# Don't relabel quote-mode STRICT.
|
||||
assert "EVIDENCE-LINKED" not in out
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue