russell@unturf.com
0d8eed0df8
Promotes the title-relevance sidecar (diagnose_title_relevance,
qa/inspect.py) to a hard check inside both verify_claim_lattice
and verify_claim_lattice_json. New violation kind TITLE_MISMATCH;
demote-to-HYBRID semantics matching the existing WARRANT_MISSING
pattern.
Catches the 2026-05-02 fox-surfaced retrieval-driven hallucination
class:
Q: "explain spin glass modeling & tensors?"
Pre-Rule-8: EVIDENCE-WARRANTED 1/1, claim cited to Quantum
chromodynamics chunk (single-line "See Also: spin
glass" reference). Token-coverage check passed
accidentally on shared physics vocabulary.
Post-Rule-8: POINTER-LINKED-PARTIAL · title mismatch 1/1.
Cited source title 'Quantum chromodynamics' shares
zero stemmed tokens with claim's {spin, glass,
modeling, tensor, ...} → demote.
Implementation:
_claim_title_overlap(claim_text, source_title) returns True iff
the source title shares ≥1 stemmed content token with the claim.
Uses qa.evidence._content_tokens (≥4-char, post-stopword) and
inline minimal stem (s-strip on tokens >4 chars, skip ss-enders)
to avoid an import cycle.
Per-claim loop in both verifiers checks every cited source's
title; ANY-match suffices (only TITLE_MISMATCH when ALL cited
titles miss). Vacuous-pass when claim or title has no
extractable tokens.
Renderer (_ladder_rung_for_lattice) treats TITLE_MISMATCH
alongside WARRANT_MISSING as the POINTER-LINKED-triggering
signal — both indicate citation/claim structural misalignment.
_render_warrant_tail surfaces "· title mismatch" alongside
"· warrant missing" so an operator sees the specific failure
mode at the audit-line.
Test fixtures updated where the synthetic claims were too minimal
(e.g. "Velociraptors are shown attacking workers" cited to
"Jurassic Park (film)" — claim had no topic anchor). Real model
output naturally references the topic (the model sees the title
in the evidence map and uses it); the fixture revisions reflect
that. 4 new tests in test_verify_json.py covering the helper +
end-to-end TITLE_MISMATCH demote.
Live verification:
Spin-glass query: POINTER-LINKED-PARTIAL · title mismatch ✓
Homer/Mr. Burns: EVIDENCE-WARRANTED ✓ (no regression)
CLAUDE.md updated with the Rule 8 convention; existing
diagnose_title_relevance sidecar marked legacy / dict-form for
per-cache-key inspect use.
Full suite: 738 passed (was 734, +4).
1167 lines
43 KiB
Python
1167 lines
43 KiB
Python
"""G0 / CTI — claim-lattice-pointer mode (quote-by-pointer).
|
|
|
|
Covers the pointer-line protocol Hermes emits, plus the runtime
|
|
two-layer id mapping (prompt-facing E1/E2/…, content-addressed
|
|
E########). Tests the deterministic verifier + 9-stage run-DAG +
|
|
schema migration round-trip.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from aborist.qa.dag import build_run_dag, verify_run_dag
|
|
from aborist.qa.evidence import (
|
|
build_evidence_map,
|
|
evidence_map_by_evidence_id,
|
|
evidence_map_by_pointer_id,
|
|
evidence_map_root,
|
|
render_claim_lattice,
|
|
render_evidence_map,
|
|
)
|
|
from aborist.qa.keys import governance_policy_hash
|
|
from aborist.qa.parse_claims import parse_pointer_claims
|
|
from aborist.qa.runner import DEFAULT_POLICY
|
|
from aborist.qa.query import DEFAULT_QUERY_POLICY
|
|
from aborist.qa.verify import (
|
|
ANSWER_MODES,
|
|
DEFAULT_ANSWER_MODE,
|
|
verify_claim_lattice,
|
|
)
|
|
|
|
|
|
# ---------- evidence map ----------------------------------------------------
|
|
|
|
|
|
def _sample_chunks() -> list[dict]:
|
|
"""Three chunks across two sources, two source roles."""
|
|
return [
|
|
{
|
|
"source_root": "a" * 64,
|
|
"document_uri": "https://example.org/jp",
|
|
"title": "Jurassic Park (film)",
|
|
"chunk_idx": 0,
|
|
"chunk_root": "11" * 32,
|
|
"span": "Tyrannosaurus rex appears in the climactic scene.",
|
|
"source_role": "primary_answer_source",
|
|
},
|
|
{
|
|
"source_root": "a" * 64,
|
|
"document_uri": "https://example.org/jp",
|
|
"title": "Jurassic Park (film)",
|
|
"chunk_idx": 1,
|
|
"chunk_root": "22" * 32,
|
|
"span": "Velociraptors are shown attacking workers.",
|
|
"source_role": "primary_answer_source",
|
|
},
|
|
{
|
|
"source_root": "b" * 64,
|
|
"document_uri": "https://example.org/jp-game",
|
|
"title": "Jurassic Park (video game)",
|
|
"chunk_idx": 0,
|
|
"chunk_root": "33" * 32,
|
|
"span": "The Sega Genesis adaptation was released in 1993.",
|
|
"source_role": "noisy_background_source",
|
|
},
|
|
]
|
|
|
|
|
|
def test_pointer_id_is_sequential():
|
|
em = build_evidence_map(_sample_chunks())
|
|
assert [e.pointer_id for e in em] == ["E1", "E2", "E3"]
|
|
|
|
|
|
def test_evidence_id_is_content_addressed_and_stable():
|
|
chunks = _sample_chunks()
|
|
map1 = build_evidence_map(chunks)
|
|
map2 = build_evidence_map(chunks)
|
|
# Content-addressed evidence_ids stay stable across runs.
|
|
assert [e.evidence_id for e in map1] == [e.evidence_id for e in map2]
|
|
assert all(e.evidence_id.startswith("E") and len(e.evidence_id) == 9
|
|
for e in map1)
|
|
# Pointer ids are deterministic too (position-based).
|
|
assert [e.pointer_id for e in map1] == [e.pointer_id for e in map2]
|
|
|
|
|
|
def test_pointer_id_is_run_dependent_when_chunks_reorder():
|
|
chunks = _sample_chunks()
|
|
map1 = build_evidence_map(chunks)
|
|
map2 = build_evidence_map(list(reversed(chunks)))
|
|
# Chunk B's content-addressed evidence_id stays stable...
|
|
assert {e.chunk_root: e.evidence_id for e in map1} == \
|
|
{e.chunk_root: e.evidence_id for e in map2}
|
|
# ...but its pointer_id moves with position.
|
|
assert {e.chunk_root: e.pointer_id for e in map1} != \
|
|
{e.chunk_root: e.pointer_id for e in map2}
|
|
|
|
|
|
def test_evidence_map_root_order_independent():
|
|
chunks = _sample_chunks()
|
|
r1 = evidence_map_root(build_evidence_map(chunks))
|
|
r2 = evidence_map_root(build_evidence_map(list(reversed(chunks))))
|
|
assert r1 == r2
|
|
assert r1 != "00" * 32
|
|
|
|
|
|
def test_render_evidence_map_uses_pointer_ids():
|
|
em = build_evidence_map(_sample_chunks())
|
|
text = render_evidence_map(em)
|
|
for e in em:
|
|
assert f"=== {e.pointer_id} " in text
|
|
# Hex evidence_id must NOT leak into the prompt.
|
|
assert e.evidence_id not in text
|
|
assert e.span in text
|
|
|
|
|
|
# ---------- parser ---------------------------------------------------------
|
|
|
|
|
|
def test_parse_pointer_claims_basic():
|
|
out = parse_pointer_claims(
|
|
"Trex appears in the film. [E1]\n"
|
|
"Velociraptors stalk workers. [E2,E3]\n"
|
|
)
|
|
assert len(out) == 2
|
|
assert out[0].claim_text == "Trex appears in the film."
|
|
assert out[0].pointer_ids == ["E1"]
|
|
assert out[0].parse_status == "PARSED"
|
|
assert out[1].pointer_ids == ["E2", "E3"]
|
|
|
|
|
|
def test_parse_handles_bullet_markers():
|
|
out = parse_pointer_claims("- Trex appears. [E1]\n")
|
|
assert out[0].claim_text == "Trex appears."
|
|
|
|
|
|
def test_parse_handles_whitespace_inside_brackets():
|
|
out = parse_pointer_claims("Trex appears. [E1, E2 ,E3]\n")
|
|
assert out[0].pointer_ids == ["E1", "E2", "E3"]
|
|
|
|
|
|
def test_parse_skips_blank_lines():
|
|
out = parse_pointer_claims("\n \nTrex appears. [E1]\n\n")
|
|
assert len(out) == 1
|
|
|
|
|
|
def test_parse_no_evidence_pointer_status():
|
|
out = parse_pointer_claims("Trex appears in the film.\n")
|
|
assert out[0].parse_status == "NO_EVIDENCE_POINTER"
|
|
assert out[0].pointer_ids == []
|
|
assert out[0].claim_text == "Trex appears in the film."
|
|
|
|
|
|
def test_parse_inline_tag_preserves_prose():
|
|
out = parse_pointer_claims("Trex [E1] appears in the film.\n")
|
|
# Tag stripped; prose around it stays.
|
|
assert "Trex" in out[0].claim_text
|
|
assert "appears in the film" in out[0].claim_text
|
|
assert out[0].pointer_ids == ["E1"]
|
|
|
|
|
|
# ---------- verifier: happy path ------------------------------------------
|
|
|
|
|
|
def test_verify_strict_when_all_resolve():
|
|
em = build_evidence_map(_sample_chunks())
|
|
# Claim text must textually overlap the cited span (G+: 6th hard
|
|
# check — claim content tokens must appear in the cited evidence).
|
|
# AND at least one cited source's title must share a content
|
|
# token with the claim (Rule 8, post-2026-05-02): so we keep
|
|
# "film" in each claim to anchor against the
|
|
# `Jurassic Park (film)` title.
|
|
# E1 span = "Tyrannosaurus rex appears in the climactic scene."
|
|
# E2 span = "Velociraptors are shown attacking workers."
|
|
answer = (
|
|
"Tyrannosaurus rex appears in the film. [E1]\n"
|
|
"Velociraptors are shown attacking workers in the film. [E2]\n"
|
|
)
|
|
v = verify_claim_lattice(answer, em)
|
|
assert v["audit_mode"] == "STRICT"
|
|
assert v["verifier_method"] == "claim_lattice"
|
|
assert v["n_quotes"] == 2
|
|
assert v["n_verified"] == 2
|
|
assert v["violations"] == []
|
|
# Evidence_id pairs use the content-addressed form for run-DAG.
|
|
pairs = v["evidence_id_pairs"]
|
|
assert pairs[0] == [em[0].evidence_id]
|
|
assert pairs[1] == [em[1].evidence_id]
|
|
|
|
|
|
def test_verify_renders_with_literal_spans_from_runtime():
|
|
em = build_evidence_map(_sample_chunks())
|
|
answer = "Tyrannosaurus rex is in the film. [E1]\n"
|
|
v = verify_claim_lattice(answer, em)
|
|
# Renderer pulls literal span from runtime, not from model output.
|
|
assert em[0].span in v["rendered_text"]
|
|
# New pointer format: [E1 | <title> | <chunk_prefix>: "<excerpt>"]
|
|
# Closes the visual provenance gap where E# could be confused
|
|
# with the source-list 1-indexed rank.
|
|
assert "[E1 |" in v["rendered_text"]
|
|
assert "Jurassic Park (film)" in v["rendered_text"]
|
|
|
|
|
|
def test_verify_passes_multi_pointer_claims():
|
|
em = build_evidence_map(_sample_chunks())
|
|
# Title is "Jurassic Park (film)" — claim must share a content
|
|
# token (Rule 8). Add "film" so the claim anchors against the
|
|
# source title.
|
|
answer = "Tyrannosaurus and velociraptors appear in the film. [E1,E2]\n"
|
|
v = verify_claim_lattice(answer, em)
|
|
assert v["audit_mode"] == "STRICT"
|
|
assert v["n_quotes"] == 2
|
|
assert v["n_verified"] == 2
|
|
|
|
|
|
# ---------- verifier: failure modes ---------------------------------------
|
|
|
|
|
|
def test_unknown_pointer_id_is_violation():
|
|
em = build_evidence_map(_sample_chunks())
|
|
answer = "Imaginary claim. [E99]\n"
|
|
v = verify_claim_lattice(answer, em)
|
|
assert v["audit_mode"] == "UNGROUNDED"
|
|
assert v["n_verified"] == 0
|
|
assert any(viol["kind"] == "UNKNOWN_EVIDENCE_ID" for viol in v["violations"])
|
|
assert v["claim_statuses"][0]["status"] == "UNKNOWN_EVIDENCE_ID"
|
|
|
|
|
|
def test_source_role_blocked_violation():
|
|
em = build_evidence_map(_sample_chunks())
|
|
# E3 → noisy_background_source, disallowed by default.
|
|
answer = "Sega adaptation exists. [E3]\n"
|
|
v = verify_claim_lattice(answer, em)
|
|
assert v["audit_mode"] == "UNGROUNDED"
|
|
assert any(viol["kind"] == "SOURCE_ROLE_BLOCKED" for viol in v["violations"])
|
|
assert v["claim_statuses"][0]["status"] == "SOURCE_ROLE_BLOCKED"
|
|
|
|
|
|
def test_double_quote_in_claim_text_no_longer_blocks_verification():
|
|
"""Pre-2026-04-30: any `"` in claim text was a hard MANUAL_QUOTE_VIOLATION
|
|
that blocked every pointer on the claim — even when the claim was
|
|
factually correct and source-grounded. Hermes-3-8B paraphrases prose
|
|
but copies named-quoted phrases verbatim from source (e.g.
|
|
`"Constitution State"` from a Connecticut chunk), so the rule was
|
|
rejecting good claims for cosmetic punctuation. Removed in favor
|
|
of the coverage-threshold check (Rule 5) and pointer cap (Rule 6).
|
|
|
|
The claim below would previously have failed with MANUAL_QUOTE_VIOLATION;
|
|
now it stands or falls on whether the cited evidence actually supports
|
|
it — same as any quote-free claim.
|
|
"""
|
|
em = build_evidence_map(_sample_chunks())
|
|
answer = 'The "T-rex" appears. [E1]\n'
|
|
v = verify_claim_lattice(answer, em)
|
|
# No MANUAL_QUOTE_VIOLATION emitted anywhere.
|
|
assert not any(viol["kind"] == "MANUAL_QUOTE_VIOLATION"
|
|
for viol in v["violations"])
|
|
assert v["claim_statuses"][0]["status"] != "MANUAL_QUOTE_VIOLATION"
|
|
|
|
|
|
def test_curly_quotes_also_no_longer_block():
|
|
"""Mirrors the ASCII-quote case for curly typographic quotes."""
|
|
em = build_evidence_map(_sample_chunks())
|
|
answer = "The “T-rex” appears. [E1]\n"
|
|
v = verify_claim_lattice(answer, em)
|
|
assert not any(viol["kind"] == "MANUAL_QUOTE_VIOLATION"
|
|
for viol in v["violations"])
|
|
|
|
|
|
def test_no_evidence_pointer_downgrades():
|
|
em = build_evidence_map(_sample_chunks())
|
|
answer = (
|
|
"Trex appears in the film.\n" # no pointer
|
|
"Velociraptors are shown. [E2]\n"
|
|
)
|
|
# Opt out of bare-name check (Velociraptors stems to 1 content token
|
|
# after spotlight stopword strip); test exercises NO_EVIDENCE_POINTER.
|
|
v = verify_claim_lattice(answer, em, min_claim_content_tokens=0)
|
|
# Mixed: one verified, one NO_EVIDENCE_POINTER → HYBRID.
|
|
assert v["audit_mode"] == "HYBRID"
|
|
assert any(viol["kind"] == "NO_EVIDENCE_POINTER" for viol in v["violations"])
|
|
statuses = {c["status"] for c in v["claim_statuses"]}
|
|
assert "NO_EVIDENCE_POINTER" in statuses
|
|
assert "EVIDENCE_LINKED" in statuses
|
|
|
|
|
|
def test_partial_resolution_yields_hybrid():
|
|
em = build_evidence_map(_sample_chunks())
|
|
answer = (
|
|
"Tyrannosaurus rex appears. [E1]\n"
|
|
"Made-up. [E99]\n"
|
|
)
|
|
# Test exercises mixed-pointer-resolution behavior; bare-name check
|
|
# is orthogonal so opt out.
|
|
v = verify_claim_lattice(answer, em, min_claim_content_tokens=0)
|
|
assert v["audit_mode"] == "HYBRID"
|
|
assert v["n_verified"] == 1
|
|
assert v["n_quotes"] == 2
|
|
|
|
|
|
def test_partial_pointer_within_claim_yields_partial():
|
|
em = build_evidence_map(_sample_chunks())
|
|
answer = "Tyrannosaurus rex appears. [E1,E99]\n"
|
|
v = verify_claim_lattice(answer, em, min_claim_content_tokens=0)
|
|
assert v["audit_mode"] == "HYBRID"
|
|
assert v["claim_statuses"][0]["status"] == "EVIDENCE_LINKED_PARTIAL"
|
|
# Renderer keeps only resolved ids — new pointer format
|
|
# `[E1 | <title> | <chunk_prefix>: ...]` since 2026-05-01.
|
|
assert "[E1 |" in v["rendered_text"]
|
|
assert "E99" not in v["rendered_text"]
|
|
|
|
|
|
def test_empty_response_yields_ungrounded():
|
|
em = build_evidence_map(_sample_chunks())
|
|
v = verify_claim_lattice("", em)
|
|
assert v["audit_mode"] == "UNGROUNDED"
|
|
assert v["n_verified"] == 0
|
|
assert v["claim_statuses"] == []
|
|
assert v["evidence_id_pairs"] == []
|
|
|
|
|
|
def test_non_pointer_response_yields_ungrounded():
|
|
em = build_evidence_map(_sample_chunks())
|
|
# Model wrote prose without any pointer tags.
|
|
v = verify_claim_lattice(
|
|
"I don't know based on the provided sources.", em
|
|
)
|
|
assert v["audit_mode"] == "UNGROUNDED"
|
|
assert v["n_verified"] == 0
|
|
# Each non-empty line without a tag becomes a NO_EVIDENCE_POINTER claim.
|
|
assert v["claim_statuses"][0]["status"] == "NO_EVIDENCE_POINTER"
|
|
|
|
|
|
def test_orphan_tag_with_no_text_yields_schema_invalid():
|
|
em = build_evidence_map(_sample_chunks())
|
|
answer = "[E1]\n"
|
|
v = verify_claim_lattice(answer, em)
|
|
assert v["audit_mode"] == "UNGROUNDED"
|
|
assert v["claim_statuses"][0]["status"] == "SCHEMA_INVALID"
|
|
|
|
|
|
# ---------- policy & cache aliasing ---------------------------------------
|
|
|
|
|
|
def test_default_answer_mode_is_quote():
|
|
assert DEFAULT_POLICY["answer_mode"] == "quote"
|
|
assert DEFAULT_QUERY_POLICY["answer_mode"] == "quote"
|
|
assert DEFAULT_ANSWER_MODE == "quote"
|
|
assert "claim_lattice_pointer" in ANSWER_MODES
|
|
assert "quote" in ANSWER_MODES
|
|
|
|
|
|
def test_governance_hash_differs_between_modes():
|
|
p_quote = dict(DEFAULT_POLICY, answer_mode="quote")
|
|
p_lattice = dict(DEFAULT_POLICY, answer_mode="claim_lattice_pointer")
|
|
assert governance_policy_hash(p_quote) != governance_policy_hash(p_lattice)
|
|
|
|
|
|
# ---------- run-DAG -------------------------------------------------------
|
|
|
|
|
|
def test_run_dag_quote_mode_is_seven_stages():
|
|
common = dict(
|
|
question_hash="11" * 32,
|
|
sources=[{"document_root": "aa" * 32, "source_role": "primary_answer_source"}],
|
|
context_root="bb" * 32,
|
|
conversation_hash="cc" * 32,
|
|
answer_text="hello",
|
|
audit_mode="STRICT",
|
|
verifier_method="quote",
|
|
n_quotes=1,
|
|
n_verified=1,
|
|
claim_statuses=[],
|
|
lookup_path="miss",
|
|
)
|
|
dag = build_run_dag(**common)
|
|
stages = [n["stage"] for n in dag["nodes"]]
|
|
assert stages == [
|
|
"question", "retrieval", "context", "prompt",
|
|
"answer", "verify", "final_label",
|
|
]
|
|
assert verify_run_dag(dag) is True
|
|
|
|
|
|
def test_run_dag_pointer_mode_is_nine_stages():
|
|
common = dict(
|
|
question_hash="11" * 32,
|
|
sources=[{"document_root": "aa" * 32, "source_role": "primary_answer_source"}],
|
|
context_root="bb" * 32,
|
|
conversation_hash="cc" * 32,
|
|
answer_text="rendered prose",
|
|
audit_mode="STRICT",
|
|
verifier_method="claim_lattice",
|
|
n_quotes=1,
|
|
n_verified=1,
|
|
claim_statuses=[],
|
|
lookup_path="miss",
|
|
evidence_map_root="dd" * 32,
|
|
answer_mode="claim_lattice_pointer",
|
|
violations=[],
|
|
raw_answer_text="Claim. [E1]",
|
|
parsed_lattice=[{"claim_text": "Claim.", "evidence_ids": ["E12345678"]}],
|
|
rendered_text="rendered prose",
|
|
)
|
|
dag = build_run_dag(**common)
|
|
stages = [n["stage"] for n in dag["nodes"]]
|
|
assert stages == [
|
|
"question", "retrieval", "evidence_map", "prompt",
|
|
"raw_answer", "parsed_claim_lattice", "verify", "render",
|
|
"final_label",
|
|
]
|
|
# No 'context' or single 'answer' nodes in pointer mode.
|
|
assert "context" not in stages
|
|
assert "answer" not in stages
|
|
assert verify_run_dag(dag) is True
|
|
|
|
|
|
def test_run_dag_root_changes_when_evidence_map_changes():
|
|
base = dict(
|
|
question_hash="11" * 32,
|
|
sources=[{"document_root": "aa" * 32, "source_role": "primary_answer_source"}],
|
|
context_root="bb" * 32,
|
|
conversation_hash="cc" * 32,
|
|
answer_text="hello",
|
|
audit_mode="STRICT",
|
|
verifier_method="claim_lattice",
|
|
n_quotes=1,
|
|
n_verified=1,
|
|
claim_statuses=[],
|
|
lookup_path="miss",
|
|
answer_mode="claim_lattice_pointer",
|
|
violations=[],
|
|
raw_answer_text="raw",
|
|
parsed_lattice=[],
|
|
rendered_text="hello",
|
|
)
|
|
a = build_run_dag(**base, evidence_map_root="dd" * 32)
|
|
b = build_run_dag(**base, evidence_map_root="ee" * 32)
|
|
assert a["root"] != b["root"]
|
|
|
|
|
|
def test_run_dag_root_changes_when_parsed_lattice_changes():
|
|
base = dict(
|
|
question_hash="11" * 32,
|
|
sources=[{"document_root": "aa" * 32, "source_role": "primary_answer_source"}],
|
|
context_root="bb" * 32,
|
|
conversation_hash="cc" * 32,
|
|
answer_text="hello",
|
|
audit_mode="STRICT",
|
|
verifier_method="claim_lattice",
|
|
n_quotes=1,
|
|
n_verified=1,
|
|
claim_statuses=[],
|
|
lookup_path="miss",
|
|
evidence_map_root="dd" * 32,
|
|
answer_mode="claim_lattice_pointer",
|
|
violations=[],
|
|
raw_answer_text="raw",
|
|
rendered_text="hello",
|
|
)
|
|
a = build_run_dag(**base, parsed_lattice=[{"claim_text": "x", "evidence_ids": ["E12345678"]}])
|
|
b = build_run_dag(**base, parsed_lattice=[{"claim_text": "y", "evidence_ids": ["E12345678"]}])
|
|
assert a["root"] != b["root"]
|
|
|
|
|
|
# ---------- schema / migration --------------------------------------------
|
|
|
|
|
|
def test_verifier_method_check_accepts_claim_lattice(tmp_path: Path):
|
|
"""Fresh shards include 'claim_lattice' in the verifier_method CHECK."""
|
|
from aborist.store import connect
|
|
|
|
db = tmp_path / "fresh.db"
|
|
conn = connect(db)
|
|
try:
|
|
ddl = conn.execute(
|
|
"SELECT sql FROM sqlite_master WHERE type='table' AND name='providence_cache'"
|
|
).fetchone()[0]
|
|
assert "'claim_lattice'" in ddl
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def test_legacy_shard_migrates_in_place(tmp_path: Path):
|
|
"""An old shard with the pre-claim_lattice CHECK gets rebuilt on open
|
|
AND keeps run_dag_root / run_dag_blob populated."""
|
|
from aborist.store import connect
|
|
|
|
db = tmp_path / "legacy.db"
|
|
c = sqlite3.connect(db)
|
|
c.executescript(
|
|
"""
|
|
PRAGMA journal_mode=WAL;
|
|
CREATE TABLE providence_cache (
|
|
cache_key TEXT PRIMARY KEY,
|
|
source_root TEXT NOT NULL,
|
|
document_uri TEXT NOT NULL,
|
|
question_hash TEXT NOT NULL,
|
|
question_text TEXT NOT NULL,
|
|
answer_text TEXT NOT NULL,
|
|
merkle_proof TEXT NOT NULL,
|
|
model_profile_hash TEXT NOT NULL,
|
|
conversation_hash TEXT NOT NULL,
|
|
governance_policy_hash TEXT NOT NULL,
|
|
schema_version TEXT NOT NULL,
|
|
canonicalization_version TEXT NOT NULL,
|
|
chunking_version TEXT NOT NULL,
|
|
falsification_state TEXT NOT NULL DEFAULT 'live'
|
|
CHECK (falsification_state IN ('live','failed','stale','quarantined')),
|
|
chain TEXT NOT NULL DEFAULT 'private'
|
|
CHECK (chain IN ('private','public')),
|
|
audit_event_hash TEXT,
|
|
created_at INTEGER NOT NULL,
|
|
last_hit_at INTEGER,
|
|
hit_count INTEGER NOT NULL DEFAULT 0,
|
|
audit_mode TEXT NOT NULL DEFAULT 'UNGROUNDED'
|
|
CHECK (audit_mode IN ('STRICT','HYBRID','UNGROUNDED')),
|
|
n_quotes INTEGER NOT NULL DEFAULT 0,
|
|
n_verified INTEGER NOT NULL DEFAULT 0,
|
|
unverified_quotes TEXT,
|
|
verifier_method TEXT NOT NULL DEFAULT 'none'
|
|
CHECK (verifier_method IN ('quote','span','entity','paraphrase','none')),
|
|
run_dag_root TEXT,
|
|
run_dag_blob TEXT
|
|
);
|
|
INSERT INTO providence_cache
|
|
(cache_key, source_root, document_uri, question_hash, question_text,
|
|
answer_text, merkle_proof, model_profile_hash, conversation_hash,
|
|
governance_policy_hash, schema_version, canonicalization_version,
|
|
chunking_version, audit_event_hash, created_at,
|
|
run_dag_root, run_dag_blob)
|
|
VALUES
|
|
('ck1','sr1','u','qh','q','a','{}','mh','ch','gh','v','c','c1','eh',
|
|
1, 'rdr', '{"root":"rdr","nodes":[]}');
|
|
"""
|
|
)
|
|
c.commit()
|
|
c.close()
|
|
|
|
conn = connect(db)
|
|
try:
|
|
ddl = conn.execute(
|
|
"SELECT sql FROM sqlite_master WHERE type='table' AND name='providence_cache'"
|
|
).fetchone()[0]
|
|
assert "'claim_lattice'" in ddl
|
|
row = conn.execute(
|
|
"SELECT run_dag_root, run_dag_blob FROM providence_cache WHERE cache_key=?",
|
|
("ck1",),
|
|
).fetchone()
|
|
assert row[0] == "rdr"
|
|
assert row[1] == '{"root":"rdr","nodes":[]}'
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
# ---------- soft-signal isolation -----------------------------------------
|
|
|
|
|
|
def test_no_entailment_field_in_verify_output():
|
|
"""Hard verifier must not assert semantic claims."""
|
|
em = build_evidence_map(_sample_chunks())
|
|
answer = "Trex appears. [E1]\n"
|
|
v = verify_claim_lattice(answer, em)
|
|
forbidden = {"entailment", "entailed", "completeness",
|
|
"predicate_compatibility", "scope_match"}
|
|
assert not (set(v) & forbidden)
|
|
for c in v["claim_statuses"]:
|
|
assert not (set(c) & forbidden)
|
|
|
|
|
|
# ---------- evidence_map index helpers ------------------------------------
|
|
|
|
|
|
def test_evidence_map_by_pointer_id_index():
|
|
em = build_evidence_map(_sample_chunks())
|
|
by_pid = evidence_map_by_pointer_id(em)
|
|
assert set(by_pid) == {"E1", "E2", "E3"}
|
|
assert by_pid["E1"] is em[0]
|
|
|
|
|
|
def test_evidence_map_by_evidence_id_index():
|
|
em = build_evidence_map(_sample_chunks())
|
|
by_eid = evidence_map_by_evidence_id(em)
|
|
assert set(by_eid) == {e.evidence_id for e in em}
|
|
# Ids are 9 chars: "E" + 8 hex.
|
|
assert all(len(eid) == 9 for eid in by_eid)
|
|
|
|
|
|
def test_render_claim_lattice_with_pointer_ids():
|
|
em = build_evidence_map(_sample_chunks())
|
|
by_pid = evidence_map_by_pointer_id(em)
|
|
text = render_claim_lattice(
|
|
[
|
|
{"text": "trex appears", "pointer_ids": ["E1"]},
|
|
{"text": "raptors attack", "pointer_ids": ["E2"]},
|
|
],
|
|
by_pid,
|
|
)
|
|
assert "Tyrannosaurus rex appears" in text
|
|
assert "Velociraptors are shown" in text
|
|
# New format includes source title + chunk_root prefix to avoid
|
|
# E# / source-rank visual confusion (2026-05-01).
|
|
assert "[E1 |" in text
|
|
assert "[E2 |" in text
|
|
assert "Jurassic Park (film)" in text
|
|
|
|
|
|
def test_render_claim_lattice_marks_unknown():
|
|
em = build_evidence_map(_sample_chunks())
|
|
by_pid = evidence_map_by_pointer_id(em)
|
|
text = render_claim_lattice(
|
|
[{"text": "fake", "pointer_ids": ["E99"]}],
|
|
by_pid,
|
|
)
|
|
assert "E99: ?" in text
|
|
|
|
|
|
def test_render_pointer_format_includes_source_title_and_chunk_prefix():
|
|
"""Pointer format: `[E# | <title> | <chunk_prefix>: "<excerpt>"]`.
|
|
Closes the 2026-05-01 visual provenance gap on the Orwell run
|
|
where `[E5: ...]` paired with a source list whose `[5]` was an
|
|
unrelated document — operator could mis-attribute the citation.
|
|
Now title + chunk_root prefix are inline so each pointer is
|
|
self-identifying."""
|
|
em = build_evidence_map(_sample_chunks())
|
|
by_pid = evidence_map_by_pointer_id(em)
|
|
text = render_claim_lattice(
|
|
[{"text": "trex appears", "pointer_ids": ["E1"]}],
|
|
by_pid,
|
|
)
|
|
# Title appears inline with the pointer.
|
|
assert "[E1 | Jurassic Park (film) | " in text
|
|
# chunk_root prefix is the first 8 hex chars of em[0].chunk_root.
|
|
chunk_prefix = em[0].chunk_root[:8]
|
|
assert chunk_prefix in text
|
|
# Excerpt still rendered after the prefix.
|
|
assert "Tyrannosaurus rex appears" in text
|
|
|
|
|
|
def test_render_pointer_format_falls_back_to_uri_tail_when_title_none():
|
|
"""If EvidenceObject.title is None (rare — source had no title
|
|
metadata), the renderer falls back to the URI tail. Defensive,
|
|
keeps the pointer self-identifying even when title indexing
|
|
fails for a chunk."""
|
|
chunks = [{
|
|
"source_root": "f" * 64,
|
|
"document_uri": "https://example.org/some-article-slug",
|
|
"title": None, # explicit absence
|
|
"chunk_idx": 0,
|
|
"chunk_root": "ab" * 32,
|
|
"span": "Some content here.",
|
|
"source_role": "primary_answer_source",
|
|
}]
|
|
em = build_evidence_map(chunks)
|
|
by_pid = evidence_map_by_pointer_id(em)
|
|
text = render_claim_lattice(
|
|
[{"text": "Some content", "pointer_ids": ["E1"]}],
|
|
by_pid,
|
|
)
|
|
# URI tail used as label.
|
|
assert "[E1 | some-article-slug | " in text
|
|
|
|
|
|
def test_render_pointer_format_no_visual_collision_with_source_rank():
|
|
"""Regression for the Orwell case: the substring `[E5:` (which
|
|
would visually collide with a source-rank `[5]` in older
|
|
renders) does not appear in the new output. The format change
|
|
isn't optional — it's enforced."""
|
|
em = build_evidence_map(_sample_chunks())
|
|
by_pid = evidence_map_by_pointer_id(em)
|
|
text = render_claim_lattice(
|
|
[{"text": "trex appears", "pointer_ids": ["E1"]}],
|
|
by_pid,
|
|
)
|
|
# Old format was `[E1: "..."]`; new format is `[E1 | ...: "..."]`.
|
|
# The colon-after-id must be gone.
|
|
assert "[E1:" not in text
|
|
# New format always has the pipe separator.
|
|
assert "[E1 |" in text
|
|
|
|
|
|
# ---------- spotlight excerpt (G0.2) --------------------------------------
|
|
|
|
|
|
def _long_chunk_with_buried_term() -> list[dict]:
|
|
"""Single chunk much longer than the spotlight window with the
|
|
target term ('Brachiosaurus') buried mid-span."""
|
|
intro = "Jurassic Park is a 1993 American science fiction thriller film. " * 30
|
|
middle = (
|
|
"Among the dinosaurs depicted on screen, Brachiosaurus is shown "
|
|
"lifting its long neck to eat from tall trees in an early scene."
|
|
)
|
|
tail = "Production took place across multiple studios. " * 30
|
|
long_span = intro + middle + tail
|
|
return [{
|
|
"source_root": "a" * 64,
|
|
"document_uri": "https://example.org/jp",
|
|
"title": "Jurassic Park (film)",
|
|
"chunk_idx": 0,
|
|
"chunk_root": "11" * 32,
|
|
"span": long_span,
|
|
"source_role": "primary_answer_source",
|
|
}]
|
|
|
|
|
|
def test_spotlight_finds_buried_topic_token():
|
|
em = build_evidence_map(_long_chunk_with_buried_term())
|
|
by_pid = evidence_map_by_pointer_id(em)
|
|
text = render_claim_lattice(
|
|
[{"text": "Brachiosaurus appears in the film.",
|
|
"pointer_ids": ["E1"]}],
|
|
by_pid,
|
|
)
|
|
# The displayed excerpt must contain the actual brachiosaurus
|
|
# mention from mid-span, not the leading article-intro sentence.
|
|
assert "Brachiosaurus is shown lifting" in text
|
|
|
|
|
|
def test_spotlight_falls_back_when_no_token_matches():
|
|
em = build_evidence_map(_long_chunk_with_buried_term())
|
|
by_pid = evidence_map_by_pointer_id(em)
|
|
# Claim contains no content tokens that appear in the span.
|
|
text = render_claim_lattice(
|
|
[{"text": "Quokka spotted yesterday morning.",
|
|
"pointer_ids": ["E1"]}],
|
|
by_pid,
|
|
)
|
|
# Falls back to leading window with trailing ellipsis.
|
|
assert text.rstrip().endswith('..."]')
|
|
# And the leading content of the span is shown.
|
|
assert "Jurassic Park is a 1993" in text
|
|
|
|
|
|
def test_spotlight_no_truncation_for_short_spans():
|
|
em = build_evidence_map(_sample_chunks())
|
|
by_pid = evidence_map_by_pointer_id(em)
|
|
text = render_claim_lattice(
|
|
[{"text": "trex", "pointer_ids": ["E1"]}],
|
|
by_pid,
|
|
)
|
|
# Sample span is <100 chars, fits in window; rendered intact.
|
|
assert em[0].span in text
|
|
|
|
|
|
def test_spotlight_window_size_overridable():
|
|
em = build_evidence_map(_long_chunk_with_buried_term())
|
|
by_pid = evidence_map_by_pointer_id(em)
|
|
narrow = render_claim_lattice(
|
|
[{"text": "Brachiosaurus appears in the film.",
|
|
"pointer_ids": ["E1"]}],
|
|
by_pid,
|
|
window=80,
|
|
)
|
|
wide = render_claim_lattice(
|
|
[{"text": "Brachiosaurus appears in the film.",
|
|
"pointer_ids": ["E1"]}],
|
|
by_pid,
|
|
window=400,
|
|
)
|
|
# Both windows hit the buried mention.
|
|
assert "Brachiosaurus" in narrow
|
|
assert "Brachiosaurus" in wide
|
|
# Wider window carries more surrounding prose.
|
|
assert len(wide) > len(narrow)
|
|
|
|
|
|
def test_spotlight_picks_longest_token_for_specificity():
|
|
"""Claim has both a generic ('film') and a specific ('Brachiosaurus')
|
|
content token. Spotlight should pick the specific one — sorted by
|
|
length desc — so the displayed excerpt anchors on what the claim
|
|
is actually *about*."""
|
|
em = build_evidence_map(_long_chunk_with_buried_term())
|
|
by_pid = evidence_map_by_pointer_id(em)
|
|
text = render_claim_lattice(
|
|
[{"text": "Brachiosaurus appears in the film.",
|
|
"pointer_ids": ["E1"]}],
|
|
by_pid,
|
|
)
|
|
# Window is centered on Brachiosaurus, not on the first occurrence
|
|
# of "film" (which is in the intro at offset 0).
|
|
assert "Brachiosaurus is shown" in text
|
|
|
|
|
|
def test_spotlight_deterministic():
|
|
em = build_evidence_map(_long_chunk_with_buried_term())
|
|
by_pid = evidence_map_by_pointer_id(em)
|
|
args = (
|
|
[{"text": "Brachiosaurus appears in the film.",
|
|
"pointer_ids": ["E1"]}],
|
|
by_pid,
|
|
)
|
|
a = render_claim_lattice(*args)
|
|
b = render_claim_lattice(*args)
|
|
assert a == b
|
|
|
|
|
|
# ---------- G0.1: per-chunk evidence granularity --------------------------
|
|
|
|
|
|
def test_per_chunk_evidence_map_query_path(tmp_path):
|
|
"""A single source with three chunks should produce three evidence
|
|
objects in the prompt-facing map — not one whole-doc entry."""
|
|
from typing import Iterator
|
|
from aborist.document import Document
|
|
from aborist.ingest import ingest_source
|
|
from aborist.qa.client import StubClient
|
|
from aborist.qa.query import DEFAULT_QUERY_POLICY, query
|
|
from aborist.source import Source
|
|
from aborist.store import connect
|
|
|
|
class FakeSource(Source):
|
|
source_type = "test"
|
|
def __init__(self, docs): self.docs = docs
|
|
def iter_documents(self) -> Iterator[Document]:
|
|
yield from self.docs
|
|
|
|
# Long enough to chunk into multiple pieces under tok-512-v1.
|
|
txt = (
|
|
"Paragraph one introduces Brachiosaurus as a long-necked sauropod. "
|
|
* 80
|
|
+ "Paragraph two introduces Velociraptor as a small carnivorous theropod. "
|
|
* 80
|
|
+ "Paragraph three introduces Tyrannosaurus rex as the apex predator. "
|
|
* 80
|
|
)
|
|
shard = tmp_path / "shard.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
src_conn = connect(shard)
|
|
try:
|
|
ingest_source(src_conn, FakeSource([
|
|
Document(uri="t://chunked", content=txt, source_type="test", title="Dinosaurs")
|
|
]))
|
|
finally:
|
|
src_conn.close()
|
|
|
|
# Stub answer that hits all three pointer ids — proves the
|
|
# evidence map exposed >1 chunk to the model. Each claim names
|
|
# "dinosaur" so Rule 8 (title-relevance) anchors against the
|
|
# source title "Dinosaurs".
|
|
stub = StubClient(answer=(
|
|
"Brachiosaurus is a sauropod dinosaur. [E1]\n"
|
|
"Velociraptor is a theropod dinosaur. [E2]\n"
|
|
"T-rex is the apex dinosaur. [E3]\n"
|
|
))
|
|
# Override the per-source chunk cap so all three chunks of this
|
|
# single test source surface as E1/E2/E3. Production default caps
|
|
# at 2 chunks per source to keep the evidence catalog small for
|
|
# broad-descriptive questions; this test is asserting the per-chunk
|
|
# mapping itself, so it opts into the unbounded path.
|
|
policy = dict(
|
|
DEFAULT_QUERY_POLICY,
|
|
answer_mode="claim_lattice_pointer",
|
|
claim_lattice_max_chunks_per_source=8,
|
|
)
|
|
result = query(
|
|
question="dinosaur paragraphs",
|
|
qa_db=qa_db,
|
|
chat_client=stub,
|
|
model_id="stub",
|
|
single_db=shard,
|
|
top_k=4,
|
|
over_fetch=8,
|
|
max_context_chars=60000,
|
|
policy=policy,
|
|
)
|
|
# All three pointers resolved → STRICT.
|
|
assert result["audit_mode"] == "STRICT"
|
|
assert result["n_quotes"] == 3
|
|
assert result["n_verified"] == 3
|
|
|
|
|
|
def test_chunk_query_relevance_scores_by_overlap():
|
|
from aborist.qa.query import _chunk_query_relevance
|
|
|
|
qstem = {"brachiosaurus", "film"}
|
|
# Many distinct + many mentions wins.
|
|
a = _chunk_query_relevance(
|
|
"Brachiosaurus appears in the film. The film features Brachiosaurus.",
|
|
qstem,
|
|
)
|
|
# Only one of the two query tokens present.
|
|
b = _chunk_query_relevance(
|
|
"The film opens with credits and a logo.", qstem,
|
|
)
|
|
# Neither token present.
|
|
c = _chunk_query_relevance(
|
|
"Production took place across multiple studios.", qstem,
|
|
)
|
|
assert a > b > c
|
|
assert a[0] == 2 and a[1] >= 3
|
|
assert b[0] == 1
|
|
assert c == (0, 0)
|
|
|
|
|
|
def test_chunk_query_relevance_stem_aware():
|
|
from aborist.qa.query import _chunk_query_relevance
|
|
|
|
# Query has the plural "dinosaurs"; chunk has the singular "dinosaur".
|
|
# The stem strip in _body_count_with_stem should still match.
|
|
qstem = {"dinosaur"} # already stemmed
|
|
score = _chunk_query_relevance(
|
|
"Each dinosaur was modeled with animatronics.", qstem
|
|
)
|
|
assert score[0] == 1
|
|
|
|
|
|
def test_query_orders_chunks_by_relevance_g03(tmp_path):
|
|
"""End-to-end: a source whose query-relevant chunk is in the MIDDLE
|
|
of the document — G0.3 must promote it to E1 so a model that
|
|
lazy-anchors on early pointers still cites the relevant chunk."""
|
|
from typing import Iterator
|
|
from aborist.document import Document
|
|
from aborist.ingest import ingest_source
|
|
from aborist.qa.client import StubClient
|
|
from aborist.qa.evidence import build_evidence_map
|
|
from aborist.qa.query import (
|
|
DEFAULT_QUERY_POLICY, _load_doc_chunks, _chunk_query_relevance,
|
|
)
|
|
from aborist.qa.query import _stem_token_for_match, _title_query_tokens
|
|
from aborist.source import Source
|
|
from aborist.store import connect
|
|
|
|
class FakeSource(Source):
|
|
source_type = "test"
|
|
def __init__(self, docs): self.docs = docs
|
|
def iter_documents(self) -> Iterator[Document]:
|
|
yield from self.docs
|
|
|
|
# Three chunks worth of content; the middle one mentions the
|
|
# rare term. tok-512-v1 chunks ~500 tokens (~2000 chars) each.
|
|
irrelevant = "Production notes and scheduling details. " * 80
|
|
relevant = "The Brachiosaurus appears in a key scene. " * 80
|
|
more_irrelevant = "Cast and crew biographies follow. " * 80
|
|
txt = irrelevant + relevant + more_irrelevant
|
|
|
|
shard = tmp_path / "shard.db"
|
|
qa_db = tmp_path / "qa.db"
|
|
src_conn = connect(shard)
|
|
try:
|
|
ingest_source(src_conn, FakeSource([
|
|
Document(uri="t://reorder", content=txt, source_type="test", title="r")
|
|
]))
|
|
doc_root = src_conn.execute(
|
|
"SELECT document_root FROM documents WHERE document_uri='t://reorder'"
|
|
).fetchone()["document_root"]
|
|
finally:
|
|
src_conn.close()
|
|
|
|
chunks = _load_doc_chunks(str(shard), doc_root)
|
|
assert chunks is not None
|
|
assert len(chunks) >= 3
|
|
|
|
# Mirror what query() does: rank chunks by query-relevance.
|
|
qstem = {
|
|
_stem_token_for_match(t.lower())
|
|
for t in _title_query_tokens("Brachiosaurus appearance")
|
|
}
|
|
scored = []
|
|
for idx, leaf, span in chunks:
|
|
d, t = _chunk_query_relevance(span, qstem)
|
|
scored.append((idx, leaf, span, d, t))
|
|
scored.sort(key=lambda r: (-r[3], -r[4], r[0]))
|
|
|
|
# The first chunk after sorting MUST be the one containing
|
|
# Brachiosaurus — even though it's not chunk_idx=0 in doc order.
|
|
assert "Brachiosaurus" in scored[0][2]
|
|
# Document-order first chunk should NOT be the top-ranked one
|
|
# (otherwise the test fixture isn't exercising the reorder).
|
|
assert scored[0][0] != 0
|
|
|
|
|
|
def test_lazy_anchor_ratio_one_when_all_share_pointer():
|
|
em = build_evidence_map(_sample_chunks())
|
|
# All three claims must textually overlap E1's span
|
|
# ("Tyrannosaurus rex appears in the climactic scene.").
|
|
answer = (
|
|
"Tyrannosaurus rex appears. [E1]\n"
|
|
"Tyrannosaurus rex roars. [E1]\n"
|
|
"Tyrannosaurus rex broke out. [E1]\n"
|
|
)
|
|
# Test exercises the smell ratio counter; bare-name check is
|
|
# orthogonal so opt out so all three claims reach EVIDENCE_LINKED
|
|
# and populate pointer_distribution.
|
|
v = verify_claim_lattice(answer, em, min_claim_content_tokens=0)
|
|
assert v["lazy_anchor_ratio"] == 1.0
|
|
assert v["pointer_id_distribution"] == {"E1": 3}
|
|
|
|
|
|
def test_lazy_anchor_ratio_one_third_when_diversified():
|
|
em = build_evidence_map(_sample_chunks())
|
|
# Each claim's content tokens must overlap its cited span.
|
|
answer = (
|
|
"Tyrannosaurus rex appears. [E1]\n"
|
|
"Velociraptors stalk workers. [E2]\n"
|
|
"The climactic Tyrannosaurus moment. [E1]\n"
|
|
)
|
|
v = verify_claim_lattice(answer, em, min_claim_content_tokens=0)
|
|
# 2 cite E1, 1 cites E2 → max share 2/3.
|
|
assert v["pointer_id_distribution"] == {"E1": 2, "E2": 1}
|
|
assert abs(v["lazy_anchor_ratio"] - 2 / 3) < 1e-9
|
|
|
|
|
|
def test_lazy_anchor_ratio_zero_when_no_verified_pairs():
|
|
em = build_evidence_map(_sample_chunks())
|
|
# Unknown pointer → 0 verified pairs.
|
|
answer = "Imaginary claim. [E99]\n"
|
|
v = verify_claim_lattice(answer, em)
|
|
assert v["lazy_anchor_ratio"] == 0.0
|
|
assert v["pointer_id_distribution"] == {}
|
|
|
|
|
|
def test_citation_mismatch_when_claim_token_absent_from_evidence():
|
|
"""6th hard check: cited evidence span must textually contain at
|
|
least one content token from the claim text."""
|
|
em = build_evidence_map(_sample_chunks())
|
|
# E1's span is "Tyrannosaurus rex appears in the climactic scene."
|
|
# — the claim says "Brachiosaurus" (not present in the span).
|
|
answer = "Brachiosaurus appears in the film. [E1]\n"
|
|
v = verify_claim_lattice(answer, em)
|
|
assert v["audit_mode"] == "UNGROUNDED"
|
|
assert any(viol["kind"] == "CITATION_MISMATCH" for viol in v["violations"])
|
|
assert v["claim_statuses"][0]["status"] == "CITATION_MISMATCH"
|
|
assert v["n_verified"] == 0
|
|
|
|
|
|
def test_citation_mismatch_partial_when_one_pointer_overlaps():
|
|
em = build_evidence_map(_sample_chunks())
|
|
# E1 contains "Tyrannosaurus rex"; E2 contains "Velociraptors".
|
|
# Claim mentions Tyrannosaurus → E1 overlaps, E2 does not.
|
|
answer = "Tyrannosaurus rex appears. [E1,E2]\n"
|
|
# Test exercises CITATION_MISMATCH on a single pointer; bare-name
|
|
# check is orthogonal so opt out.
|
|
v = verify_claim_lattice(answer, em, min_claim_content_tokens=0)
|
|
assert v["audit_mode"] == "HYBRID"
|
|
assert v["n_verified"] == 1
|
|
assert v["claim_statuses"][0]["status"] == "EVIDENCE_LINKED_PARTIAL"
|
|
# E1 resolved, E2 mismatched.
|
|
assert any(
|
|
viol["kind"] == "CITATION_MISMATCH" and viol["pointer_id"] == "E2"
|
|
for viol in v["violations"]
|
|
)
|
|
|
|
|
|
def test_pure_stopword_claim_passes_overlap_check_vacuously():
|
|
"""A claim composed entirely of stopwords has no content tokens to
|
|
check; the overlap rule returns True vacuously and other hard
|
|
checks own that case."""
|
|
em = build_evidence_map(_sample_chunks())
|
|
# Tyrannosaurus rex appears... is in E1's span; the claim text
|
|
# below has effectively no content tokens after stopword filter
|
|
# (just function words), so the overlap check shouldn't fail it.
|
|
answer = "It was so. [E1]\n"
|
|
# Bare-name check would reject this for 0 content tokens; opt out
|
|
# since this test is specifically about the overlap rule's
|
|
# vacuous-True branch for claims with no content tokens.
|
|
v = verify_claim_lattice(answer, em, min_claim_content_tokens=0)
|
|
# Overlap check passed; claim is short but valid.
|
|
assert v["claim_statuses"][0]["status"] == "EVIDENCE_LINKED"
|
|
|
|
|
|
def test_lazy_anchor_signals_not_in_run_dag_payload():
|
|
"""Sidecar invariant: pointer_id_distribution and lazy_anchor_ratio
|
|
must NOT enter build_run_dag's verify_payload — they're soft
|
|
signals recoverable from claim_statuses, and threading them in
|
|
would make run_dag_root depend on the model's anchoring habits."""
|
|
import inspect as _inspect
|
|
from aborist.qa.dag import build_run_dag
|
|
sig = _inspect.signature(build_run_dag)
|
|
params = set(sig.parameters)
|
|
assert "pointer_id_distribution" not in params
|
|
assert "lazy_anchor_ratio" not in params
|
|
|
|
|
|
def test_per_chunk_evidence_map_uses_distinct_chunk_roots(tmp_path):
|
|
"""Chunk-level evidence_ids must be content-addressed per chunk —
|
|
different chunks of the same source produce different evidence_ids
|
|
even though they share source_root."""
|
|
from typing import Iterator
|
|
from aborist.document import Document
|
|
from aborist.ingest import ingest_source
|
|
from aborist.qa.query import _load_doc_chunks
|
|
from aborist.source import Source
|
|
from aborist.store import connect
|
|
|
|
class FakeSource(Source):
|
|
source_type = "test"
|
|
def __init__(self, docs): self.docs = docs
|
|
def iter_documents(self) -> Iterator[Document]:
|
|
yield from self.docs
|
|
|
|
txt = "para alpha. " * 200 + "para beta. " * 200 + "para gamma. " * 200
|
|
shard = tmp_path / "shard.db"
|
|
conn = connect(shard)
|
|
try:
|
|
ingest_source(conn, FakeSource([
|
|
Document(uri="t://multi", content=txt, source_type="test", title="m")
|
|
]))
|
|
doc_root = conn.execute(
|
|
"SELECT document_root FROM documents WHERE document_uri='t://multi'"
|
|
).fetchone()["document_root"]
|
|
finally:
|
|
conn.close()
|
|
|
|
chunks = _load_doc_chunks(str(shard), doc_root)
|
|
assert chunks is not None
|
|
assert len(chunks) > 1, "test fixture must produce multiple chunks"
|
|
|
|
# Build a chunks_for_map dict like query() does, then verify each
|
|
# chunk gets a distinct content-addressed evidence_id.
|
|
cfm = [
|
|
{
|
|
"source_root": doc_root,
|
|
"document_uri": "t://multi",
|
|
"title": "m",
|
|
"chunk_idx": idx,
|
|
"chunk_root": leaf_hash,
|
|
"span": span,
|
|
"source_role": "primary_answer_source",
|
|
}
|
|
for idx, leaf_hash, span in chunks
|
|
]
|
|
em = build_evidence_map(cfm)
|
|
eids = [e.evidence_id for e in em]
|
|
pids = [e.pointer_id for e in em]
|
|
assert len(set(eids)) == len(eids), "evidence_ids must be unique per chunk"
|
|
# Pointer ids are sequential.
|
|
assert pids == [f"E{i + 1}" for i in range(len(em))]
|
|
# All share the same source_root but different chunk_roots.
|
|
assert len({e.source_root for e in em}) == 1
|
|
assert len({e.chunk_root for e in em}) == len(em)
|
|
|
|
|
|
# ---------- claim-count ceiling --------------------------------------------
|
|
|
|
|
|
def test_too_many_claims_demotes_pointer_mode_to_hybrid():
|
|
"""Pointer-mode mirror of test_verify_json_too_many_claims_demotes_to_hybrid.
|
|
13 well-formed pointer claims (default cap 12) trips TOO_MANY_CLAIMS
|
|
even when each individually verifies. Demotes STRICT to HYBRID so
|
|
'tell me all there is to know about X' runaway is operator-visible.
|
|
"""
|
|
chunks = [
|
|
{
|
|
"source_root": "f" * 64,
|
|
"document_uri": "https://example.org/runaway",
|
|
"title": "Runaway Source",
|
|
"chunk_idx": i,
|
|
"chunk_root": f"{i:02x}" * 32,
|
|
"span": f"Fact {i} appears in the runaway source span.",
|
|
"source_role": "primary_answer_source",
|
|
}
|
|
for i in range(13)
|
|
]
|
|
em = build_evidence_map(chunks)
|
|
answer = "\n".join(
|
|
f"Fact {i} appears in the runaway source span. [E{i + 1}]"
|
|
for i in range(13)
|
|
) + "\n"
|
|
v = verify_claim_lattice(answer, em)
|
|
assert v["audit_mode"] == "HYBRID", \
|
|
f"expected HYBRID (TOO_MANY_CLAIMS demote), got {v['audit_mode']}"
|
|
assert any(vio["kind"] == "TOO_MANY_CLAIMS" for vio in v["violations"])
|
|
# The cap doesn't truncate — every claim still verifies.
|
|
assert v["n_verified"] == 13, \
|
|
f"all 13 should still verify; got {v['n_verified']}"
|