From c3da725a621e40499a5b5811149d17d8a2558dbc Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 30 Apr 2026 12:06:22 -0400 Subject: [PATCH] qa: wire claim_lattice JSON mode into runner; partial-grounding split JSON mode (answer_mode="claim_lattice") is now reachable end-to-end via runner.ask: message build, optional vLLM guided_json extra_body, verifier dispatch, and providence_cache persistence. Verifier returns verifier_method="claim_lattice" (same as pointer variant) so both modes share the existing CHECK constraint; answer_mode on the run-DAG and json_fixups on the verdict disambiguate. Pointer verifier now splits EVIDENCE_LINKED_PARTIAL into its own partially_verified_quotes bucket so a claim never renders as both a verified bullet and an unverified footer. CLI shows a "partially grounded" section between the verified body and the unverified list. --- aborist/cli.py | 8 +++ aborist/qa/query.py | 7 +++ aborist/qa/runner.py | 105 ++++++++++++++++++++++++++++++++++++-- aborist/qa/verify.py | 21 +++++++- tests/test_verify_json.py | 61 +++++++++++++++++++++- 5 files changed, 195 insertions(+), 7 deletions(-) diff --git a/aborist/cli.py b/aborist/cli.py index cdc3e53..42d612e 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -459,6 +459,14 @@ def _render_query_human(result: dict, question: str) -> str: lines.append(f" [{i}] {title} — {_strip_scheme(uri)}{shard_part}") lines.append("") + partially = result.get("partially_verified_quotes") or [] + if partially: + lines.append(f"partially grounded ({len(partially)}):") + for q in partially: + qtxt = q if len(q) <= 100 else q[:97] + "..." + lines.append(f' - "{qtxt}"') + lines.append("") + unverified = result.get("unverified_quotes") or [] if unverified: lines.append(f"unverified ({len(unverified)}):") diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 7b092a7..2510c18 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -1459,6 +1459,12 @@ def query( if cached["unverified_quotes"] else [] ), + # Cache schema doesn't carry the partially-verified split — + # those claims were folded into unverified_quotes pre-2026- + # 04-30. Cache hits surface an empty partial list; new + # writes populate it correctly. Acceptable degradation + # since governance_policy_hash invalidated prior records. + "partially_verified_quotes": [], "timings": { "search_ms": search_ms, "context_ms": context_ms, @@ -1792,6 +1798,7 @@ def query( "n_verified": verdict["n_verified"], "verifier_method": verdict["verifier_method"], "unverified_quotes": verdict["unverified_quotes"], + "partially_verified_quotes": verdict.get("partially_verified_quotes") or [], # Sidecar smell signals (claim_lattice mode only) — surfaced # for the renderer; never persisted in providence_cache and # never threaded into run_dag_root. diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 5f2c896..3ccf93b 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -39,12 +39,15 @@ from aborist.qa.evidence import ( build_evidence_map, evidence_map_root, render_evidence_map, + render_evidence_map_for_json, ) from aborist.qa.repair import mechanical_repair, reprompt_repair from aborist.qa.verify import ( ANSWER_MODES, + CLAIM_LATTICE_JSON_SCHEMA, DEFAULT_ANSWER_MODE, verify_claim_lattice, + verify_claim_lattice_json, verify_quotes, ) from aborist.store import append_audit, transaction @@ -201,6 +204,33 @@ DEFAULT_POLICY = { # like "Steve Jobs co-founded Apple" still pass. Folds into # governance_policy_hash on change. "claim_lattice_min_citation_coverage": 0.30, + # JSON variant — `answer_mode="claim_lattice"`. Mirrors the + # multi-source query path. Pairs with grammar-constrained inference + # (vLLM guided_json, Claude/GPT-4 native JSON, Qwen 3.6 reasoner). + # Lenient pre-parser in verify_claim_lattice_json keeps the path + # survivable on inference paths without grammar guidance. + "claim_lattice_json_system_prompt": ( + "You will see numbered EVIDENCE blocks tagged with content-" + "addressed evidence IDs (long hex strings starting with 'E'). " + "Answer as a single JSON object using EXACTLY this schema, " + "with NO prose, NO markdown fences, NO preamble:\n\n" + ' {"claims":[{"text":"","evidence_ids":["E........"]}]}\n\n' + "RULES:\n" + "1. Output a single JSON object. No code fences. No commentary.\n" + "2. `text` is plain prose with NO double-quote characters.\n" + "3. `evidence_ids` MUST be IDs from the EVIDENCE blocks above. " + "At most two IDs per claim.\n" + "4. Each claim must reference at least one evidence_id.\n" + "5. If no evidence supports a claim, omit the claim." + ), + "claim_lattice_json_grounding_reminder": ( + "REMINDER: emit a single JSON object with the exact schema " + '`{"claims":[{"text":"...","evidence_ids":["E........"]}]}`. ' + "No code fences, no preamble. At most two evidence_ids per " + "claim. Text field has no double-quote characters. " + "Now answer the question on the next message." + ), + "claim_lattice_use_guided_json": True, } @@ -298,6 +328,36 @@ def ask( grounding_reminder = policy.get("claim_lattice_grounding_reminder") rendered_evidence = render_evidence_map(evidence_map) + def _user_payload(q: str) -> str: + return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}" + elif answer_mode == "claim_lattice": + # JSON variant — same per-chunk evidence map as pointer mode, + # blocks labeled with content-addressed evidence_id (long hex) + # since the model emits IDs in JSON. Pairs with grammar- + # constrained inference; lenient pre-parser handles drift. + chunks_for_map = [ + { + "source_root": document_root, + "document_uri": doc["document_uri"], + "title": None, + "chunk_idx": r["idx"], + "chunk_root": r["leaf_hash"], + "span": chunk_texts[i], + "source_role": "primary_answer_source", + } + for i, r in enumerate(chunk_rows) + ] + evidence_map = build_evidence_map(chunks_for_map) + sys_prompt = policy.get( + "claim_lattice_json_system_prompt", + policy["claim_lattice_system_prompt"], + ) + grounding_reminder = policy.get( + "claim_lattice_json_grounding_reminder", + policy.get("claim_lattice_grounding_reminder"), + ) + rendered_evidence = render_evidence_map_for_json(evidence_map) + def _user_payload(q: str) -> str: return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}" else: @@ -406,6 +466,7 @@ def ask( if cached["unverified_quotes"] else [] ), + "partially_verified_quotes": [], "timings": { "cache_lookup_ms": cache_lookup_ms, "llm_ms": None, @@ -414,12 +475,21 @@ def ask( } t_llm = time.monotonic() + # JSON mode: pass guided_json schema so vLLM constrains output at + # sampling time. Endpoints without guided-decoding silently drop the + # field; the lenient pre-parser handles whatever drift remains. + extra_body: dict | None = None + if answer_mode == "claim_lattice" and policy.get( + "claim_lattice_use_guided_json", True + ): + extra_body = {"guided_json": CLAIM_LATTICE_JSON_SCHEMA} raw_answer = client.chat_completion( messages, model=model_id, temperature=policy["temperature"], max_tokens=policy["max_tokens"], top_p=policy.get("top_p", 1.0), + extra_body=extra_body, ) llm_ms = _ms_since(t_llm) @@ -454,6 +524,30 @@ def ask( # output so an operator can see what the model actually said. rendered = verdict["rendered_text"] answer_text = rendered if rendered else raw_answer + elif answer_mode == "claim_lattice": + verdict = verify_claim_lattice_json( + raw_answer, + evidence_map, + allowed_source_roles=tuple( + policy.get( + "claim_lattice_allowed_source_roles", + [ + "primary_answer_source", + "secondary_context_source", + "background_source", + "unclassified", + ], + ) + ), + max_evidence_per_claim=int(policy.get( + "claim_lattice_max_pointers_per_claim", 2 + )), + min_citation_coverage=float(policy.get( + "claim_lattice_min_citation_coverage", 0.30 + )), + ) + rendered = verdict["rendered_text"] + answer_text = rendered if rendered else raw_answer else: answer_text = raw_answer verdict = verify_quotes( @@ -539,11 +633,13 @@ def ask( # answer splits into raw_answer / parsed_claim_lattice / render. ev_root = evidence_map_root(evidence_map) if evidence_map else None parsed_lattice = None - if answer_mode == "claim_lattice_pointer": + is_lattice_mode = answer_mode in ("claim_lattice_pointer", "claim_lattice") + if is_lattice_mode: # Per-claim list of {claim_text, content-addressed evidence_ids} # for the parsed_claim_lattice node hash. Pointer ids are # run-dependent; evidence_ids are content-addressed → the run- - # DAG hashes the run-stable form. + # DAG hashes the run-stable form. Same shape for JSON and + # pointer; verifier already returns evidence_id_pairs. evidence_id_pairs = verdict.get("evidence_id_pairs") or [] parsed_lattice = [ { @@ -572,9 +668,9 @@ def ask( evidence_map_root=ev_root, answer_mode=answer_mode if answer_mode != "quote" else None, violations=verdict.get("violations"), - raw_answer_text=raw_answer if answer_mode == "claim_lattice_pointer" else None, + raw_answer_text=raw_answer if is_lattice_mode else None, parsed_lattice=parsed_lattice, - rendered_text=answer_text if answer_mode == "claim_lattice_pointer" else None, + rendered_text=answer_text if is_lattice_mode else None, ) run_dag_blob = json.dumps(run_dag, separators=(",", ":")) @@ -676,6 +772,7 @@ def ask( "n_verified": verdict["n_verified"], "verifier_method": verdict["verifier_method"], "unverified_quotes": verdict["unverified_quotes"], + "partially_verified_quotes": verdict.get("partially_verified_quotes") or [], # Sidecar smell signals (claim_lattice mode only) — render- # layer; never persisted, never in run_dag_root. "pointer_id_distribution": verdict.get("pointer_id_distribution"), diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index 522fde9..92ecb89 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -906,7 +906,17 @@ def verify_claim_lattice( by_pointer = _by_pointer(evidence_map) violations: list[dict] = [] claim_statuses: list[dict] = [] + # Three-bucket rendering. ``unverified`` holds claims whose + # status reached neither EVIDENCE_LINKED nor EVIDENCE_LINKED_PARTIAL + # — i.e. fully failed (no pointer verified). ``partially_verified`` + # holds EVIDENCE_LINKED_PARTIAL claims (some pointers ok, some + # failed). The renderer shows them as their own section so a + # claim never appears in BOTH a verified bullet and the unverified + # footer; that previously happened for partial-status claims and + # read as "is it grounded or not?". Per-pointer detail lives in + # ``claim_statuses`` for audit. unverified: list[str] = [] + partially_verified: list[str] = [] raw_claims = parse_pointer_claims(answer_text or "") @@ -1062,7 +1072,9 @@ def verify_claim_lattice( evidence_id_pairs.append(resolved_evidence_ids) if status in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"): valid_claims.append({"text": claim_text, "pointer_ids": ok_pids}) - if status != "EVIDENCE_LINKED": + if status == "EVIDENCE_LINKED_PARTIAL": + partially_verified.append(claim_text) + elif status != "EVIDENCE_LINKED": unverified.append(claim_text) rendered_text = _render(valid_claims, by_pointer) if valid_claims else "" @@ -1108,6 +1120,7 @@ def verify_claim_lattice( "n_verified": n_pairs_verified, "audit_mode": audit_mode, "unverified_quotes": unverified, + "partially_verified_quotes": partially_verified, "verifier_method": "claim_lattice", "claim_statuses": claim_statuses, "violations": violations, @@ -1359,12 +1372,16 @@ def verify_claim_lattice_json( else: audit_mode = "UNGROUNDED" + # Same `verifier_method` as the pointer variant ("claim_lattice") + # so the providence_cache CHECK constraint accepts both. The mode + # is disambiguated downstream via `answer_mode` on the run-DAG & + # via the JSON-only `json_fixups` field on this verdict. return { "n_quotes": n_pairs, "n_verified": n_pairs_verified, "audit_mode": audit_mode, "unverified_quotes": unverified, - "verifier_method": "claim_lattice_json", + "verifier_method": "claim_lattice", "claim_statuses": claim_statuses, "violations": violations, "rendered_text": rendered_text, diff --git a/tests/test_verify_json.py b/tests/test_verify_json.py index 66264e4..d81c1b8 100644 --- a/tests/test_verify_json.py +++ b/tests/test_verify_json.py @@ -108,7 +108,9 @@ def test_verify_json_strict_when_all_claims_resolve(): }) v = verify_claim_lattice_json(answer, evidence) assert v["audit_mode"] == "STRICT" - assert v["verifier_method"] == "claim_lattice_json" + # Same verifier_method as the pointer variant — disambiguated downstream + # via `answer_mode` on the run-DAG and `json_fixups` on the verdict. + assert v["verifier_method"] == "claim_lattice" assert v["n_verified"] == 2 assert v["violations"] == [] @@ -184,3 +186,60 @@ def test_verify_json_blocks_disallowed_source_role(): v = verify_claim_lattice_json(answer, evidence) assert any(vio["kind"] == "SOURCE_ROLE_BLOCKED" for vio in v["violations"]) assert v["audit_mode"] == "UNGROUNDED" + + +# ---------------------------------------------------------------- runner integration + + +def test_runner_ask_json_mode_passes_guided_json_extra_body(tmp_path): + """`ask()` in JSON mode forwards `extra_body={"guided_json": SCHEMA}` + through the chat client. Stub captures the kwargs so we can assert.""" + from typing import Iterator + + from aborist.document import Document + from aborist.ingest import ingest_source + from aborist.qa import ask + from aborist.qa.client import StubClient + from aborist.qa.runner import DEFAULT_POLICY + from aborist.source import Source + from aborist.store import connect + + class _FakeSource(Source): + source_type = "test" + def __init__(self, docs): self.docs = docs + def iter_documents(self) -> Iterator[Document]: yield from self.docs + + db = tmp_path / "qa.db" + conn = connect(db) + try: + ingest_source(conn, _FakeSource([ + Document( + uri="test://doc", + content="Cloud Strife is the protagonist. " * 30, + source_type="test", + title="Cloud", + ) + ])) + root = conn.execute( + "SELECT document_root FROM documents WHERE document_uri='test://doc'" + ).fetchone()["document_root"] + + client = StubClient(answer='{"claims":[]}') + policy = dict(DEFAULT_POLICY) + policy["answer_mode"] = "claim_lattice" + + ask( + conn, + document_root=root, + question="Who is Cloud?", + client=client, + model_id="m", + policy=policy, + ) + assert len(client.calls) == 1 + kwargs = client.calls[0]["kwargs"] + assert "extra_body" in kwargs + assert kwargs["extra_body"] is not None + assert "guided_json" in kwargs["extra_body"] + finally: + conn.close()