engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

qa: wire claim_lattice JSON mode into runner; partial-grounding split

Browse source 
JSON mode (answer_mode="claim_lattice") is now reachable end-to-end via
runner.ask: message build, optional vLLM guided_json extra_body, verifier
dispatch, and providence_cache persistence. Verifier returns
verifier_method="claim_lattice" (same as pointer variant) so both modes
share the existing CHECK constraint; answer_mode on the run-DAG and
json_fixups on the verdict disambiguate.

Pointer verifier now splits EVIDENCE_LINKED_PARTIAL into its own
partially_verified_quotes bucket so a claim never renders as both a
verified bullet and an unverified footer. CLI shows a "partially
grounded" section between the verified body and the unverified list.
This commit is contained in:
russell@unturf.com 2026-04-30 12:06:22 -04:00
parent 224bfd6a2b
commit c3da725a62
No known key found for this signature in database
5 changed files with 195 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

8
aborist/cli.py
View file

@ -459,6 +459,14 @@ def _render_query_human(result: dict, question: str) -> str:
lines.append(f" [{i}] {title} — {_strip_scheme(uri)}{shard_part}")
lines.append("")
partially = result.get("partially_verified_quotes") or []
if partially:
lines.append(f"partially grounded ({len(partially)}):")
for q in partially:
qtxt = q if len(q) <= 100 else q[:97] + "..."
lines.append(f' - "{qtxt}"')
lines.append("")
unverified = result.get("unverified_quotes") or []
if unverified:
lines.append(f"unverified ({len(unverified)}):")

7
aborist/qa/query.py
View file

@ -1459,6 +1459,12 @@ def query(
if cached["unverified_quotes"]
else []
),
# Cache schema doesn't carry the partially-verified split —
# those claims were folded into unverified_quotes pre-2026-
# 04-30. Cache hits surface an empty partial list; new
# writes populate it correctly. Acceptable degradation
# since governance_policy_hash invalidated prior records.
"partially_verified_quotes": [],
"timings": {
"search_ms": search_ms,
"context_ms": context_ms,
@ -1792,6 +1798,7 @@ def query(
"n_verified": verdict["n_verified"],
"verifier_method": verdict["verifier_method"],
"unverified_quotes": verdict["unverified_quotes"],
"partially_verified_quotes": verdict.get("partially_verified_quotes") or [],
# Sidecar smell signals (claim_lattice mode only) — surfaced
# for the renderer; never persisted in providence_cache and
# never threaded into run_dag_root.

105
aborist/qa/runner.py
View file

@ -39,12 +39,15 @@ from aborist.qa.evidence import (
build_evidence_map,
evidence_map_root,
render_evidence_map,
render_evidence_map_for_json,
)
from aborist.qa.repair import mechanical_repair, reprompt_repair
from aborist.qa.verify import (
ANSWER_MODES,
CLAIM_LATTICE_JSON_SCHEMA,
DEFAULT_ANSWER_MODE,
verify_claim_lattice,
verify_claim_lattice_json,
verify_quotes,
)
from aborist.store import append_audit, transaction
@ -201,6 +204,33 @@ DEFAULT_POLICY = {
# like "Steve Jobs co-founded Apple" still pass. Folds into
# governance_policy_hash on change.
"claim_lattice_min_citation_coverage": 0.30,
# JSON variant — `answer_mode="claim_lattice"`. Mirrors the
# multi-source query path. Pairs with grammar-constrained inference
# (vLLM guided_json, Claude/GPT-4 native JSON, Qwen 3.6 reasoner).
# Lenient pre-parser in verify_claim_lattice_json keeps the path
# survivable on inference paths without grammar guidance.
"claim_lattice_json_system_prompt": (
"You will see numbered EVIDENCE blocks tagged with content-"
"addressed evidence IDs (long hex strings starting with 'E'). "
"Answer as a single JSON object using EXACTLY this schema, "
"with NO prose, NO markdown fences, NO preamble:\n\n"
' {"claims":[{"text":"<claim text>","evidence_ids":["E........"]}]}\n\n'
"RULES:\n"
"1. Output a single JSON object. No code fences. No commentary.\n"
"2. `text` is plain prose with NO double-quote characters.\n"
"3. `evidence_ids` MUST be IDs from the EVIDENCE blocks above. "
"At most two IDs per claim.\n"
"4. Each claim must reference at least one evidence_id.\n"
"5. If no evidence supports a claim, omit the claim."
),
"claim_lattice_json_grounding_reminder": (
"REMINDER: emit a single JSON object with the exact schema "
'`{"claims":[{"text":"...","evidence_ids":["E........"]}]}`. '
"No code fences, no preamble. At most two evidence_ids per "
"claim. Text field has no double-quote characters. "
"Now answer the question on the next message."
),
"claim_lattice_use_guided_json": True,
}
@ -298,6 +328,36 @@ def ask(
grounding_reminder = policy.get("claim_lattice_grounding_reminder")
rendered_evidence = render_evidence_map(evidence_map)
def _user_payload(q: str) -> str:
return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}"
elif answer_mode == "claim_lattice":
# JSON variant — same per-chunk evidence map as pointer mode,
# blocks labeled with content-addressed evidence_id (long hex)
# since the model emits IDs in JSON. Pairs with grammar-
# constrained inference; lenient pre-parser handles drift.
chunks_for_map = [
{
"source_root": document_root,
"document_uri": doc["document_uri"],
"title": None,
"chunk_idx": r["idx"],
"chunk_root": r["leaf_hash"],
"span": chunk_texts[i],
"source_role": "primary_answer_source",
}
for i, r in enumerate(chunk_rows)
]
evidence_map = build_evidence_map(chunks_for_map)
sys_prompt = policy.get(
"claim_lattice_json_system_prompt",
policy["claim_lattice_system_prompt"],
)
grounding_reminder = policy.get(
"claim_lattice_json_grounding_reminder",
policy.get("claim_lattice_grounding_reminder"),
)
rendered_evidence = render_evidence_map_for_json(evidence_map)
def _user_payload(q: str) -> str:
return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}"
else:
@ -406,6 +466,7 @@ def ask(
if cached["unverified_quotes"]
else []
),
"partially_verified_quotes": [],
"timings": {
"cache_lookup_ms": cache_lookup_ms,
"llm_ms": None,
@ -414,12 +475,21 @@ def ask(
}
t_llm = time.monotonic()
# JSON mode: pass guided_json schema so vLLM constrains output at
# sampling time. Endpoints without guided-decoding silently drop the
# field; the lenient pre-parser handles whatever drift remains.
extra_body: dict | None = None
if answer_mode == "claim_lattice" and policy.get(
"claim_lattice_use_guided_json", True
):
extra_body = {"guided_json": CLAIM_LATTICE_JSON_SCHEMA}
raw_answer = client.chat_completion(
messages,
model=model_id,
temperature=policy["temperature"],
max_tokens=policy["max_tokens"],
top_p=policy.get("top_p", 1.0),
extra_body=extra_body,
)
llm_ms = _ms_since(t_llm)
@ -454,6 +524,30 @@ def ask(
# output so an operator can see what the model actually said.
rendered = verdict["rendered_text"]
answer_text = rendered if rendered else raw_answer
elif answer_mode == "claim_lattice":
verdict = verify_claim_lattice_json(
raw_answer,
evidence_map,
allowed_source_roles=tuple(
policy.get(
"claim_lattice_allowed_source_roles",
[
"primary_answer_source",
"secondary_context_source",
"background_source",
"unclassified",
],
)
),
max_evidence_per_claim=int(policy.get(
"claim_lattice_max_pointers_per_claim", 2
)),
min_citation_coverage=float(policy.get(
"claim_lattice_min_citation_coverage", 0.30
)),
)
rendered = verdict["rendered_text"]
answer_text = rendered if rendered else raw_answer
else:
answer_text = raw_answer
verdict = verify_quotes(
@ -539,11 +633,13 @@ def ask(
# answer splits into raw_answer / parsed_claim_lattice / render.
ev_root = evidence_map_root(evidence_map) if evidence_map else None
parsed_lattice = None
if answer_mode == "claim_lattice_pointer":
is_lattice_mode = answer_mode in ("claim_lattice_pointer", "claim_lattice")
if is_lattice_mode:
# Per-claim list of {claim_text, content-addressed evidence_ids}
# for the parsed_claim_lattice node hash. Pointer ids are
# run-dependent; evidence_ids are content-addressed → the run-
# DAG hashes the run-stable form.
# DAG hashes the run-stable form. Same shape for JSON and
# pointer; verifier already returns evidence_id_pairs.
evidence_id_pairs = verdict.get("evidence_id_pairs") or []
parsed_lattice = [
{
@ -572,9 +668,9 @@ def ask(
evidence_map_root=ev_root,
answer_mode=answer_mode if answer_mode != "quote" else None,
violations=verdict.get("violations"),
raw_answer_text=raw_answer if answer_mode == "claim_lattice_pointer" else None,
raw_answer_text=raw_answer if is_lattice_mode else None,
parsed_lattice=parsed_lattice,
rendered_text=answer_text if answer_mode == "claim_lattice_pointer" else None,
rendered_text=answer_text if is_lattice_mode else None,
)
run_dag_blob = json.dumps(run_dag, separators=(",", ":"))
@ -676,6 +772,7 @@ def ask(
"n_verified": verdict["n_verified"],
"verifier_method": verdict["verifier_method"],
"unverified_quotes": verdict["unverified_quotes"],
"partially_verified_quotes": verdict.get("partially_verified_quotes") or [],
# Sidecar smell signals (claim_lattice mode only) — render-
# layer; never persisted, never in run_dag_root.
"pointer_id_distribution": verdict.get("pointer_id_distribution"),

21
aborist/qa/verify.py
View file

@ -906,7 +906,17 @@ def verify_claim_lattice(
by_pointer = _by_pointer(evidence_map)
violations: list[dict] = []
claim_statuses: list[dict] = []
# Three-bucket rendering. ``unverified`` holds claims whose
# status reached neither EVIDENCE_LINKED nor EVIDENCE_LINKED_PARTIAL
# — i.e. fully failed (no pointer verified). ``partially_verified``
# holds EVIDENCE_LINKED_PARTIAL claims (some pointers ok, some
# failed). The renderer shows them as their own section so a
# claim never appears in BOTH a verified bullet and the unverified
# footer; that previously happened for partial-status claims and
# read as "is it grounded or not?". Per-pointer detail lives in
# ``claim_statuses`` for audit.
unverified: list[str] = []
partially_verified: list[str] = []
raw_claims = parse_pointer_claims(answer_text or "")
@ -1062,7 +1072,9 @@ def verify_claim_lattice(
evidence_id_pairs.append(resolved_evidence_ids)
if status in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"):
valid_claims.append({"text": claim_text, "pointer_ids": ok_pids})
if status != "EVIDENCE_LINKED":
if status == "EVIDENCE_LINKED_PARTIAL":
partially_verified.append(claim_text)
elif status != "EVIDENCE_LINKED":
unverified.append(claim_text)
rendered_text = _render(valid_claims, by_pointer) if valid_claims else ""
@ -1108,6 +1120,7 @@ def verify_claim_lattice(
"n_verified": n_pairs_verified,
"audit_mode": audit_mode,
"unverified_quotes": unverified,
"partially_verified_quotes": partially_verified,
"verifier_method": "claim_lattice",
"claim_statuses": claim_statuses,
"violations": violations,
@ -1359,12 +1372,16 @@ def verify_claim_lattice_json(
else:
audit_mode = "UNGROUNDED"
# Same `verifier_method` as the pointer variant ("claim_lattice")
# so the providence_cache CHECK constraint accepts both. The mode
# is disambiguated downstream via `answer_mode` on the run-DAG &
# via the JSON-only `json_fixups` field on this verdict.
return {
"n_quotes": n_pairs,
"n_verified": n_pairs_verified,
"audit_mode": audit_mode,
"unverified_quotes": unverified,
"verifier_method": "claim_lattice_json",
"verifier_method": "claim_lattice",
"claim_statuses": claim_statuses,
"violations": violations,
"rendered_text": rendered_text,

61
tests/test_verify_json.py
View file

@ -108,7 +108,9 @@ def test_verify_json_strict_when_all_claims_resolve():
})
v = verify_claim_lattice_json(answer, evidence)
assert v["audit_mode"] == "STRICT"
assert v["verifier_method"] == "claim_lattice_json"
# Same verifier_method as the pointer variant — disambiguated downstream
# via `answer_mode` on the run-DAG and `json_fixups` on the verdict.
assert v["verifier_method"] == "claim_lattice"
assert v["n_verified"] == 2
assert v["violations"] == []
@ -184,3 +186,60 @@ def test_verify_json_blocks_disallowed_source_role():
v = verify_claim_lattice_json(answer, evidence)
assert any(vio["kind"] == "SOURCE_ROLE_BLOCKED" for vio in v["violations"])
assert v["audit_mode"] == "UNGROUNDED"
# ---------------------------------------------------------------- runner integration
def test_runner_ask_json_mode_passes_guided_json_extra_body(tmp_path):
"""`ask()` in JSON mode forwards `extra_body={"guided_json": SCHEMA}`
through the chat client. Stub captures the kwargs so we can assert."""
from typing import Iterator
from aborist.document import Document
from aborist.ingest import ingest_source
from aborist.qa import ask
from aborist.qa.client import StubClient
from aborist.qa.runner import DEFAULT_POLICY
from aborist.source import Source
from aborist.store import connect
class _FakeSource(Source):
source_type = "test"
def __init__(self, docs): self.docs = docs
def iter_documents(self) -> Iterator[Document]: yield from self.docs
db = tmp_path / "qa.db"
conn = connect(db)
try:
ingest_source(conn, _FakeSource([
Document(
uri="test://doc",
content="Cloud Strife is the protagonist. " * 30,
source_type="test",
title="Cloud",
)
]))
root = conn.execute(
"SELECT document_root FROM documents WHERE document_uri='test://doc'"
).fetchone()["document_root"]
client = StubClient(answer='{"claims":[]}')
policy = dict(DEFAULT_POLICY)
policy["answer_mode"] = "claim_lattice"
ask(
conn,
document_root=root,
question="Who is Cloud?",
client=client,
model_id="m",
policy=policy,
)
assert len(client.calls) == 1
kwargs = client.calls[0]["kwargs"]
assert "extra_body" in kwargs
assert kwargs["extra_body"] is not None
assert "guided_json" in kwargs["extra_body"]
finally:
conn.close()