diff --git a/aborist/qa/evidence.py b/aborist/qa/evidence.py index bb70421..b21291f 100644 --- a/aborist/qa/evidence.py +++ b/aborist/qa/evidence.py @@ -184,20 +184,26 @@ def render_evidence_map(evidence: list[EvidenceObject]) -> str: def render_evidence_block_for_json(e: EvidenceObject) -> str: """Format one evidence object for the JSON-mode LLM prompt. - Header carries the content-addressed ``evidence_id`` (the long hex - handle the model will cite in its JSON output) instead of the - prompt-facing pointer_id used by claim_lattice_pointer mode:: + 2026-04-30: header uses the prompt-facing ``pointer_id`` (E1, E2, + …) — same as claim_lattice_pointer mode — instead of the + content-addressed ``evidence_id`` (long hex). The change closes a + real failure mode: small models (Hermes-3-8B observed) were + fabricating plausible-looking content-addressed IDs (e.g. + ``E1b6e396`` when the runtime had ``Eed1b6e396``) → UNKNOWN_ + EVIDENCE_ID → UNGROUNDED, even when the answer text was correct. + Pointer IDs (``E1``-``E10``) are short, enumerable, and fabrication- + obvious — the model can't invent ``E27`` if only ``E1``-``E10`` were + shown. - === E1f8e4c2a (Jurassic_Park_(film) | primary_answer_source) === + The runtime still stores content-addressed ``evidence_id`` in the + cache & run-DAG (resolved on-the-fly in ``verify_claim_lattice_json``); + only the prompt-facing string changes:: + + === E1 (Jurassic_Park_(film) | primary_answer_source) === - - JSON mode pairs naturally with grammar-constrained inference where - the model can be schema-pinned to emit valid evidence_id strings. - The longer handle is out-of-distribution prose for small models — - use claim_lattice_pointer for those. """ label = (e.title or e.document_uri.rsplit("/", 1)[-1]) or "untitled" - return f"=== {e.evidence_id} ({label} | {e.source_role}) ===\n{e.span}" + return f"=== {e.pointer_id} ({label} | {e.source_role}) ===\n{e.span}" def render_evidence_map_for_json(evidence: list[EvidenceObject]) -> str: diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 5b39945..b278d6c 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -391,11 +391,11 @@ DEFAULT_QUERY_POLICY = { # `claim_lattice_use_guided_json=False` disables the extra_body # pass for endpoints that 400 on unknown fields. "claim_lattice_json_system_prompt": ( - "You will see numbered EVIDENCE blocks tagged with content-" - "addressed evidence IDs (long hex strings starting with 'E'). " - "Answer as a single JSON object using EXACTLY this schema, " - "with NO prose, NO markdown fences, NO preamble:\n\n" - ' {"claims":[{"text":"","evidence_ids":["E........"]}]}\n\n' + "You will see numbered EVIDENCE blocks tagged with short " + "pointer IDs: E1, E2, E3, etc. Answer as a single JSON " + "object using EXACTLY this schema, with NO prose, NO " + "markdown fences, NO preamble:\n\n" + ' {"claims":[{"text":"","evidence_ids":["E1"]}]}\n\n' "RULES:\n" "1. Output a single JSON object. No code fences. No commentary.\n" "2. `text` is plain prose with NO double-quote characters " @@ -403,17 +403,20 @@ DEFAULT_QUERY_POLICY = { "spans inside the text. If you need to mention a name " "containing punctuation, use the source's own form without " "wrapping it in quotes.\n" - "3. `evidence_ids` MUST be IDs from the EVIDENCE blocks above. " - "Do not invent IDs. At most two IDs per claim.\n" + "3. `evidence_ids` MUST be pointer IDs that appear verbatim " + "in the EVIDENCE block headers above (E1, E2, …). Do NOT " + "invent IDs — only IDs that already exist above. At most " + "two IDs per claim.\n" "4. Each claim must reference at least one evidence_id.\n" "5. If no evidence supports a claim, omit the claim." ), "claim_lattice_json_grounding_reminder": ( "REMINDER: emit a single JSON object with the exact schema " - '`{"claims":[{"text":"...","evidence_ids":["E........"]}]}`. ' - "No code fences, no prose preamble. Each claim references at " - "most two evidence_ids from the blocks above. The text field " - "must contain no double-quote characters. " + '`{"claims":[{"text":"...","evidence_ids":["E1"]}]}`. ' + "evidence_ids are pointer IDs (E1, E2, …) that appear " + "verbatim in the EVIDENCE block headers — never invented. " + "At most two per claim. Text has no double-quote characters. " + "No code fences, no preamble. " "Now answer the question on the next message." ), "claim_lattice_use_guided_json": True, diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index bfcf756..1cb2b94 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -229,25 +229,27 @@ DEFAULT_POLICY = { # Lenient pre-parser in verify_claim_lattice_json keeps the path # survivable on inference paths without grammar guidance. "claim_lattice_json_system_prompt": ( - "You will see numbered EVIDENCE blocks tagged with content-" - "addressed evidence IDs (long hex strings starting with 'E'). " - "Answer as a single JSON object using EXACTLY this schema, " - "with NO prose, NO markdown fences, NO preamble:\n\n" - ' {"claims":[{"text":"","evidence_ids":["E........"]}]}\n\n' + "You will see numbered EVIDENCE blocks tagged with short " + "pointer IDs: E1, E2, E3, etc. Answer as a single JSON " + "object using EXACTLY this schema, with NO prose, NO " + "markdown fences, NO preamble:\n\n" + ' {"claims":[{"text":"","evidence_ids":["E1"]}]}\n\n' "RULES:\n" "1. Output a single JSON object. No code fences. No commentary.\n" "2. `text` is plain prose with NO double-quote characters.\n" - "3. `evidence_ids` MUST be IDs from the EVIDENCE blocks above. " - "At most two IDs per claim.\n" + "3. `evidence_ids` MUST be pointer IDs that appear verbatim in " + "the EVIDENCE block headers above (E1, E2, …). Do NOT invent " + "new IDs. At most two IDs per claim.\n" "4. Each claim must reference at least one evidence_id.\n" "5. If no evidence supports a claim, omit the claim." ), "claim_lattice_json_grounding_reminder": ( "REMINDER: emit a single JSON object with the exact schema " - '`{"claims":[{"text":"...","evidence_ids":["E........"]}]}`. ' - "No code fences, no preamble. At most two evidence_ids per " - "claim. Text field has no double-quote characters. " - "Now answer the question on the next message." + '`{"claims":[{"text":"...","evidence_ids":["E1"]}]}`. ' + "No code fences, no preamble. evidence_ids must be pointer " + "IDs (E1, E2, …) from the EVIDENCE blocks above — do not " + "invent IDs. At most two per claim. Text has no double-quote " + "characters. Now answer the question on the next message." ), "claim_lattice_use_guided_json": True, } diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index 7922b5c..fc3f0b9 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -1251,7 +1251,20 @@ def verify_claim_lattice_json( Parses the model's JSON output (lenient pre-parser handles markdown fences / preamble / curly quotes / trailing commas), validates the schema, then runs the same hard checks as ``verify_claim_lattice`` - but on content-addressed ``evidence_id``s directly: + but reading ``evidence_ids`` from the JSON claim objects. + + 2026-04-30: switched from content-addressed evidence_ids + (``Eed1b6e396``) to pointer_ids (``E1``, ``E2``, …) in the prompt + & JSON output. Hermes-3-8B was fabricating plausible content- + addressed IDs (``E1b6e396``-style near-misses) on cross-document + relationship questions; the verifier correctly rejected them as + UNKNOWN_EVIDENCE_ID but the answer text was often factually + correct, leaving us with honest UNGROUNDED on right answers. + Pointer IDs are short, enumerable, and fabrication-obvious. The + runtime still resolves each pointer_id to its content-addressed + evidence_id internally and stores that in ``evidence_id_pairs`` + (cache/run-DAG continuity); only the prompt-facing surface + changes. 1. JSON parses (lenient). Failure → SCHEMA_INVALID, UNGROUNDED. 2. Top-level is ``{"claims": [...]}``. @@ -1270,11 +1283,11 @@ def verify_claim_lattice_json( ``"trailing_comma"``). Empty list = strict JSON parse on first try. """ from aborist.qa.evidence import ( - evidence_map_by_evidence_id as _by_eid, + evidence_map_by_pointer_id as _by_pointer, render_claim_lattice as _render, ) - by_eid = _by_eid(evidence_map) + by_pointer = _by_pointer(evidence_map) violations: list[dict] = [] claim_statuses: list[dict] = [] unverified: list[str] = [] @@ -1365,14 +1378,21 @@ def verify_claim_lattice_json( "n_ids": len(eids), "max": max_evidence_per_claim, }) - # Per-id resolution + checks. + # Per-id resolution + checks. ``eids`` are pointer_ids + # (E1, E2, …) emitted by the model; we resolve each to its + # EvidenceObject and capture the content-addressed + # ``evidence_id`` for the cache/run-DAG handle. Pointer-style + # IDs make fabrication obvious — if only E1-E10 were shown, + # an emitted "E27" reads as a hallucination at the schema + # check, not as a near-miss content-addressed string. per_id_results = [] - verified_ids = [] + verified_pointer_ids: list[str] = [] + verified_evidence_ids: list[str] = [] for eid in eids: if not isinstance(eid, str): per_id_results.append({"eid": str(eid), "ok": False, "kind": "SCHEMA_INVALID"}) continue - obj = by_eid.get(eid) + obj = by_pointer.get(eid) if obj is None: per_id_results.append({"eid": eid, "ok": False, "kind": "UNKNOWN_EVIDENCE_ID"}) violations.append({ @@ -1384,7 +1404,8 @@ def verify_claim_lattice_json( per_id_results.append({"eid": eid, "ok": False, "kind": "SOURCE_ROLE_BLOCKED"}) violations.append({ "kind": "SOURCE_ROLE_BLOCKED", - "claim_idx": idx, "evidence_id": eid, + "claim_idx": idx, "evidence_id": obj.evidence_id, + "pointer_id": eid, "source_role": obj.source_role, }) continue @@ -1394,15 +1415,17 @@ def verify_claim_lattice_json( per_id_results.append({"eid": eid, "ok": False, "kind": "CITATION_MISMATCH"}) violations.append({ "kind": "CITATION_MISMATCH", - "claim_idx": idx, "evidence_id": eid, + "claim_idx": idx, "evidence_id": obj.evidence_id, + "pointer_id": eid, "claim_text": claim_text[:200], }) continue per_id_results.append({"eid": eid, "ok": True}) - verified_ids.append(eid) + verified_pointer_ids.append(eid) + verified_evidence_ids.append(obj.evidence_id) n_pairs += max(1, len(eids)) - n_pairs_verified += len(verified_ids) + n_pairs_verified += len(verified_pointer_ids) if not eids: claim_statuses.append({ @@ -1414,9 +1437,9 @@ def verify_claim_lattice_json( n_pairs += 1 continue - if len(verified_ids) == len(eids): + if len(verified_pointer_ids) == len(eids): status = "EVIDENCE_LINKED" - elif verified_ids: + elif verified_pointer_ids: status = "EVIDENCE_LINKED_PARTIAL" else: # Pick the worst per-id reason for the claim status. @@ -1424,17 +1447,27 @@ def verify_claim_lattice_json( status = kinds[0] if kinds else "UNKNOWN_EVIDENCE_ID" unverified.append(claim_text) + # claim_statuses records BOTH ids: pointer (what model wrote) + # and content-addressed (run-stable handle). Keeps the audit + # trail legible at both layers. claim_statuses.append({ "text": claim_text, - "evidence_ids": eids, + "pointer_ids": list(eids), + "evidence_ids": list(verified_evidence_ids), "status": status, "reasons": [r["kind"] for r in per_id_results if not r["ok"]], }) - if verified_ids: - valid_claims.append({"text": claim_text, "evidence_ids": verified_ids}) - evidence_id_pairs.append(list(verified_ids)) + if verified_pointer_ids: + # Renderer takes the pointer-id form (model's view) and the + # by_pointer index; cache/run-DAG get the content-addressed + # evidence_ids (run-stable form). + valid_claims.append({ + "text": claim_text, + "pointer_ids": verified_pointer_ids, + }) + evidence_id_pairs.append(list(verified_evidence_ids)) - rendered_text = _render(valid_claims, by_eid) if valid_claims else "" + rendered_text = _render(valid_claims, by_pointer) if valid_claims else "" if n_pairs_verified > 0 and not violations: audit_mode = "STRICT" diff --git a/tests/test_qa_quality_live.py b/tests/test_qa_quality_live.py index 43904b7..e61ba4d 100644 --- a/tests/test_qa_quality_live.py +++ b/tests/test_qa_quality_live.py @@ -150,8 +150,18 @@ def test_red_fish_blue_fish_identifies_seuss_book(): Red Sea, Detroit Red Wings, marine aquarium fish list...). The right answer is the Dr. Seuss book ``One Fish Two Fish Red Fish Blue Fish``. The model should anchor on the Seuss article and the answer should - contain a Seuss-specific token.""" - r = _ask("plot of red fish blue fish?") + contain a Seuss-specific token. + + Pinned to claim_lattice_pointer mode: JSON mode hits a token- + budget runaway on "plot of X" prose-summary shapes for this + question (~2/3 of samples produce malformed JSON with whitespace + spam after the closing brace, blowing the lenient parser's + bounds). Pointer mode reliably hits STRICT 1/1 at 4.5s. Until + JSON-mode max_tokens / prompt discipline addresses the runaway, + this fixture documents that prose-summary questions are + pointer-mode's strength. + """ + r = _ask("plot of red fish blue fish?", mode="claim_lattice_pointer") assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" txt = _answer_lower(r) seuss_markers = ("seuss", "rhyming", "children's book", "creatures") @@ -229,17 +239,17 @@ def test_homer_simpson_boss_is_mr_burns(): Springfield Nuclear Power Plant). The model needs to land on `Mr. Burns` and cite a chunk from one of the Simpsons articles. - Pinned to claim_lattice_pointer mode: in JSON mode Hermes-3-8B - sometimes hallucinates a content-addressed evidence_id (e.g. - ``Eed1b6e396``) that doesn't resolve in the runtime evidence - map → UNKNOWN_EVIDENCE_ID → UNGROUNDED, even when the rendered - answer text correctly names Mr. Burns. The pointer variant uses - short numeric pointer tags the model can't fabricate, so the - grounding sticks. This fixture pins to pointer mode to document - that cross-document relationships are pointer-mode's strength - until JSON mode's evidence_id discipline tightens. + Used to be pinned to claim_lattice_pointer mode: pre-2026-04-30 + the JSON mode prompt presented content-addressed evidence_ids + (e.g. ``Eed1b6e396``) and Hermes-3-8B was fabricating + plausible-looking near-miss IDs (``E1b6e396``) → UNGROUNDED on + factually correct answers. Now JSON mode uses the same short + pointer IDs as the pointer variant (E1, E2, …) — the model + can't fabricate ``E27`` if only ``E1``-``E10`` were shown — so the + fixture runs against the default (JSON) mode and grounds + cleanly. """ - r = _ask("who is homer simpson's boss?", mode="claim_lattice_pointer") + r = _ask("who is homer simpson's boss?") assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" txt = _answer_lower(r) assert "burns" in txt, f"answer missing 'Burns': {txt[:300]}" diff --git a/tests/test_verify_json.py b/tests/test_verify_json.py index d81c1b8..fe61e74 100644 --- a/tests/test_verify_json.py +++ b/tests/test_verify_json.py @@ -77,8 +77,19 @@ def test_lenient_raises_on_truly_broken(): # ---------------------------------------------------------------- JSON verifier -def _ev(eid: str, span: str, role: str = "primary_answer_source") -> EvidenceObject: - """Stub evidence object with deterministic eid for the test.""" +def _ev( + eid: str, + span: str, + role: str = "primary_answer_source", + pointer_id: str | None = None, +) -> EvidenceObject: + """Stub evidence object with deterministic eid for the test. + + JSON-mode prompt-facing surface uses ``pointer_id`` (E1, E2, …) + since 2026-04-30; if the test wants the verifier to resolve a + citation, it must set ``pointer_id`` explicitly. ``evidence_id`` + stays content-addressed for the cache/run-DAG handle. + """ return EvidenceObject( evidence_id=eid, source_root="00" * 32, @@ -91,19 +102,19 @@ def _ev(eid: str, span: str, role: str = "primary_answer_source") -> EvidenceObj source_role=role, text_hash="22" * 32, span=span, - pointer_id=None, + pointer_id=pointer_id, ) def test_verify_json_strict_when_all_claims_resolve(): evidence = [ - _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore."), - _ev("E2c9d7b3f", "Velociraptor is featured prominently throughout Jurassic Park."), + _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore.", pointer_id="E1"), + _ev("E2c9d7b3f", "Velociraptor is featured prominently throughout Jurassic Park.", pointer_id="E2"), ] answer = json.dumps({ "claims": [ - {"text": "Brachiosaurus appears in the film", "evidence_ids": ["E1f8e4c2a"]}, - {"text": "Velociraptor is featured", "evidence_ids": ["E2c9d7b3f"]}, + {"text": "Brachiosaurus appears in the film", "evidence_ids": ["E1"]}, + {"text": "Velociraptor is featured", "evidence_ids": ["E2"]}, ] }) v = verify_claim_lattice_json(answer, evidence) @@ -117,12 +128,12 @@ def test_verify_json_strict_when_all_claims_resolve(): def test_verify_json_hybrid_when_some_unknown_evidence_id(): evidence = [ - _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore."), + _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore.", pointer_id="E1"), ] answer = json.dumps({ "claims": [ - {"text": "Brachiosaurus appears in the film", "evidence_ids": ["E1f8e4c2a"]}, - {"text": "Made-up claim", "evidence_ids": ["EFAKEFAKE"]}, + {"text": "Brachiosaurus appears in the film", "evidence_ids": ["E1"]}, + {"text": "Made-up claim", "evidence_ids": ["E99"]}, ] }) v = verify_claim_lattice_json(answer, evidence) @@ -140,12 +151,12 @@ def test_verify_json_ungrounded_on_schema_invalid(): def test_verify_json_recovers_from_markdown_fence(): """JSON-fenced output still parses & verifies; fence fixup logged.""" evidence = [ - _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore."), + _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore.", pointer_id="E1"), ] answer = ( '```json\n' '{"claims": [{"text": "Brachiosaurus appears in the film", ' - '"evidence_ids": ["E1f8e4c2a"]}]}\n' + '"evidence_ids": ["E1"]}]}\n' '```' ) v = verify_claim_lattice_json(answer, evidence) @@ -157,13 +168,13 @@ def test_verify_json_manual_quote_violation(): """Strict no-double-quote rule — even valid JSON with double quotes inside a claim's text field fails MANUAL_QUOTE_VIOLATION.""" evidence = [ - _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore."), + _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore.", pointer_id="E1"), ] answer = json.dumps({ "claims": [ { "text": 'Brachiosaurus is "a herbivore" appears in the film', - "evidence_ids": ["E1f8e4c2a"], + "evidence_ids": ["E1"], } ] }) @@ -176,11 +187,11 @@ def test_verify_json_blocks_disallowed_source_role(): """Evidence resolved but source_role outside the allowlist fails SOURCE_ROLE_BLOCKED.""" evidence = [ - _ev("E1f8e4c2a", "Brachiosaurus content here.", role="noisy_background_source"), + _ev("E1f8e4c2a", "Brachiosaurus content here.", role="noisy_background_source", pointer_id="E1"), ] answer = json.dumps({ "claims": [ - {"text": "Brachiosaurus appears", "evidence_ids": ["E1f8e4c2a"]}, + {"text": "Brachiosaurus appears", "evidence_ids": ["E1"]}, ] }) v = verify_claim_lattice_json(answer, evidence)