russell@unturf.com
8fb5148e8e
Three fixes that meaningfully improve claim_lattice_pointer behavior
on real queries.
1. Prompts in positive form. Hermes-3-8B (and instruction-tuned 8Bs
in general) follow positive directives ("do X") much more
reliably than negations ("don't do Y"). Rewrote the
claim_lattice_system_prompt and claim_lattice_grounding_reminder
in DEFAULT_POLICY and DEFAULT_QUERY_POLICY so every rule says
what TO do — "Reference evidence by pointer ID", "Use only
pointer IDs that appear in the EVIDENCE blocks", "Cite 1 or 2
pointers per claim", "Stop when the evidence runs out — a short
answer is the right answer when only short evidence exists",
etc. The cite-count constraint (1-2 per claim) directly
addresses the spray-anchor failure where Hermes attached every
available pointer to one claim line ("where is florida"
produced 1 claim with 33 cites pre-fix).
2. Source-role rank boost. _rerank() now ends with a new
_rerank_by_source_role stage that classifies each hit's role
(mutating h.source_role for downstream reuse) and rescales the
score by SOURCE_ROLE_RANK_WEIGHTS — primary 2.0, secondary 0.7,
noisy/sequel 0.3. The Florida defect: list-pages
(List_of_State_Roads_in_Florida, List_of_places_in_Florida:S/C/B,
Florida_locations_by_per_capita_income) all match
_SECONDARY_TITLE_MARKERS for "list of" and now sort below the
actual Florida article instead of dominating the top-8 by body
density. Affects ranking for both quote and pointer modes; the
earlier per-source budget weights stay (still primary gets 2x
the cap), they just don't have to fight a sort order that put
list-pages first.
3. Lazy-anchor smell line gates on n_verified >= 3. With one
verified pair the ratio is trivially 1.00 ("1 of 1 verified
pairs cite [E11]") which is vacuous. Below 3 pairs the metric
has no comparison surface; suppress the warning in those cases.
Live results on the 'where is florida' query went 33/33 with
cite-spray to 1/1 with a single targeted citation; spotlight
now hits "...largest metropolitan area in the state as well as
the entire southeastern United States is the South Florida..."
instead of state-road-number tables. The JP-dinosaurs benchmark
went from 11/11 with all cites at [E1] (ratio 1.00) to 13/14
with cites distributed across [E8], [E9], [E13] (ratio 0.77),
spotlight finding film-specific spans like "the film's
Dilophosaurus stands about 1.2 meters (4 ft) tall."
442 tests still passing. All 7 production shards report 0 chain
breaks.
640 lines
25 KiB
Python
640 lines
25 KiB
Python
"""Q&A runner: cache-first lookup -> inference fallback -> provable record.
|
|
|
|
Implements the v9.8 admissibility invariant:
|
|
No record reused unless all 8 cache_key dimensions match AND state
|
|
is 'live' (not failed/stale/quarantined).
|
|
|
|
Cache hit -> persisted audit_mode (STRICT/HYBRID/UNGROUNDED).
|
|
Cache miss -> call ChatClient, run faithfulness check, classify, store
|
|
record, audit event.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
import time
|
|
|
|
from aborist import (
|
|
CANONICALIZATION_VERSION,
|
|
SCHEMA_VERSION,
|
|
)
|
|
from aborist.compress import unpack_chunk
|
|
from aborist.merkle import MerkleTree, proof_to_dict
|
|
from aborist.qa.client import ChatClient
|
|
from aborist.qa.keys import (
|
|
DEFAULT_FIDELITY,
|
|
DEFAULT_QUESTION_DEDUP,
|
|
FIDELITY_MODES,
|
|
QUESTION_DEDUP_MODES,
|
|
cache_key,
|
|
canonical_question,
|
|
conversation_hash,
|
|
governance_policy_hash,
|
|
model_profile_hash,
|
|
question_hash,
|
|
)
|
|
from aborist.qa.dag import build_run_dag
|
|
from aborist.qa.evidence import (
|
|
build_evidence_map,
|
|
evidence_map_root,
|
|
render_evidence_map,
|
|
)
|
|
from aborist.qa.repair import mechanical_repair, reprompt_repair
|
|
from aborist.qa.verify import (
|
|
ANSWER_MODES,
|
|
DEFAULT_ANSWER_MODE,
|
|
verify_claim_lattice,
|
|
verify_quotes,
|
|
)
|
|
from aborist.store import append_audit, transaction
|
|
|
|
try:
|
|
from aborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION
|
|
from aborist.wikitext import to_base as _wikitext_to_base
|
|
except ImportError: # pragma: no cover
|
|
_WIKITEXT_BASE_VERSION = None
|
|
_wikitext_to_base = None
|
|
|
|
|
|
DEFAULT_POLICY = {
|
|
"system_prompt": (
|
|
"Answer the user's question based ONLY on the document below. "
|
|
"For EVERY factual claim, include a verbatim quote from the "
|
|
"document enclosed in double quotes (\"...\"). The quoted span "
|
|
"must appear word-for-word. If you cannot find a verbatim quote "
|
|
"supporting a claim, do not make the claim. "
|
|
"If the answer is not in the document, say 'I don't know based "
|
|
"on the provided document.' Do not speculate."
|
|
),
|
|
# Restated rule fired as a user message right before the document +
|
|
# question arrive. See aborist/qa/query.py for the rationale (recent
|
|
# user-turn instructions outweigh decayed system-turn rules in 8B
|
|
# instruction-tuned models).
|
|
"grounding_reminder": (
|
|
"REMINDER: every factual claim in your reply must be wrapped in "
|
|
"double quotes (\"...\") and the quoted span must appear "
|
|
"word-for-word in the document. No quote, no claim. "
|
|
"Now answer the question on the next message."
|
|
),
|
|
"temperature": 0.1,
|
|
"top_p": 1.0,
|
|
"max_tokens": 512,
|
|
"entity_policy": "proximity",
|
|
"entity_proximity_n": 3,
|
|
"entity_proximity_window": 300,
|
|
# Mechanical answer repair after first verify. Off by default; see
|
|
# aborist/qa/query.py for semantics.
|
|
"repair_enabled": False,
|
|
"repair_max_reprompts": 0,
|
|
# Strip wikitext markup before the LLM ever sees the context. Lets
|
|
# Hermes quote prose verbatim and shrinks token bills (~43% on
|
|
# Wikipedia chunks). Bumps governance_policy_hash so prior cached
|
|
# answers under raw-wikitext policy stay distinct on lookup. Set
|
|
# via the wikitext extras; no-op if mwparserfromhell isn't installed.
|
|
"base_version": _WIKITEXT_BASE_VERSION,
|
|
# G0 / CTI — claim-lattice-pointer answer mode. "quote" (default):
|
|
# existing behavior, model writes prose with verbatim quotes inline.
|
|
# "claim_lattice_pointer": runtime builds an evidence map and shows
|
|
# the model short pointer ids (E1, E2, …); model writes natural
|
|
# prose with bracket pointer tags ("Claim. [E12]") instead of
|
|
# quoting source text. Renderer interpolates literal spans at
|
|
# display time. Synthetic-elision-by-construction-impossible: the
|
|
# model never types the quote string. Two-layer id discipline keeps
|
|
# the cache & run-DAG keyed on content-addressed evidence_ids.
|
|
# Folds into governance_policy_hash so two modes write under
|
|
# different cache_keys and never alias. No iterative repair in
|
|
# pointer mode (one-shot benchmark discipline).
|
|
"answer_mode": DEFAULT_ANSWER_MODE,
|
|
"claim_lattice_system_prompt": (
|
|
"You will see numbered EVIDENCE blocks tagged E1, E2, E3, etc. "
|
|
"Answer using natural-language pointer-lines: one claim per "
|
|
"line, followed by a bracket tag with the pointer IDs that "
|
|
"directly support that claim.\n\n"
|
|
"WORKED EXAMPLE\n"
|
|
"--------------\n"
|
|
"EVIDENCE:\n\n"
|
|
"=== E1 (Apple_Inc | primary_answer_source) ===\n"
|
|
"Apple Inc. was founded by Steve Jobs, Steve Wozniak, and "
|
|
"Ronald Wayne in April 1976.\n\n"
|
|
"=== E2 (Steve_Wozniak | secondary_context_source) ===\n"
|
|
"Steve Wozniak co-founded Apple Computer Company alongside "
|
|
"Steve Jobs in 1976 and designed the Apple I.\n\n"
|
|
"QUESTION: who founded Apple?\n\n"
|
|
"ANSWER:\n"
|
|
"Steve Jobs co-founded Apple. [E1]\n"
|
|
"Steve Wozniak co-founded Apple. [E1,E2]\n"
|
|
"Ronald Wayne co-founded Apple. [E1]\n\n"
|
|
"END OF EXAMPLE\n\n"
|
|
"RULES (each rule says what TO do):\n"
|
|
"1. Reference evidence by pointer ID. The runtime displays "
|
|
"the literal source span beside each claim — referencing is "
|
|
"your job; quoting is the runtime's job.\n"
|
|
"2. Use only pointer IDs that appear in the EVIDENCE blocks "
|
|
"above.\n"
|
|
"3. Cite 1 or 2 pointers per claim — the blocks whose text "
|
|
"directly contains the claim's key terms.\n"
|
|
"4. End every claim line with [E#] or [E#,E#].\n"
|
|
"5. Make a claim only when an EVIDENCE block textually "
|
|
"supports it. Stop when the evidence runs out — a short "
|
|
"answer is the right answer when only short evidence "
|
|
"exists.\n"
|
|
"6. Write each claim as one plain-prose sentence on its own "
|
|
"line."
|
|
),
|
|
"claim_lattice_grounding_reminder": (
|
|
"REMINDER: format = pointer-line — `Claim text. [E1]` per "
|
|
"line, plain prose with bracket tags. Pointer IDs come from "
|
|
"the EVIDENCE blocks above. Cite 1 or 2 pointers per claim. "
|
|
"Now answer the question on the next message."
|
|
),
|
|
# Allowed source roles for claim-lattice verification. Roles outside
|
|
# this set get classified SOURCE_ROLE_BLOCKED and downgrade the
|
|
# verdict. Mirrors aborist.qa.verify.DEFAULT_ALLOWED_SOURCE_ROLES;
|
|
# noisy_background_source / sequel_background_source are excluded by
|
|
# default. Folds into governance_policy_hash on change.
|
|
"claim_lattice_allowed_source_roles": [
|
|
"primary_answer_source",
|
|
"secondary_context_source",
|
|
"background_source",
|
|
"unclassified",
|
|
],
|
|
}
|
|
|
|
|
|
def _ms_since(t: float) -> float:
|
|
return round((time.monotonic() - t) * 1000, 1)
|
|
|
|
|
|
def ask(
|
|
conn: sqlite3.Connection,
|
|
*,
|
|
document_root: str,
|
|
question: str,
|
|
client: ChatClient,
|
|
model_id: str,
|
|
revision: str = "",
|
|
quantization: str = "",
|
|
policy: dict | None = None,
|
|
chain: str = "private",
|
|
fidelity: str | None = None,
|
|
) -> dict:
|
|
"""Look up cached answer or run inference. Returns a result dict.
|
|
|
|
See ``aborist.qa.query.query`` for `fidelity` semantics — it
|
|
controls lookup tolerance: ``"strict"`` only checks the cache_key
|
|
matching the call's ``policy["question_dedup"]``; the default
|
|
``"equivalence_class"`` falls back to the alternate dedup mode's
|
|
cache_key on miss so a fast-cache agent can reuse records written
|
|
under either mode. Result includes ``lookup_path``.
|
|
"""
|
|
policy = policy or DEFAULT_POLICY
|
|
if fidelity is None:
|
|
fidelity = policy.get("fidelity", DEFAULT_FIDELITY)
|
|
if fidelity not in FIDELITY_MODES:
|
|
raise ValueError(
|
|
f"fidelity must be one of {FIDELITY_MODES}, got {fidelity!r}"
|
|
)
|
|
t_start = time.monotonic()
|
|
|
|
doc = conn.execute(
|
|
"SELECT document_uri, chunking_version FROM documents "
|
|
"WHERE document_root = ?",
|
|
(document_root,),
|
|
).fetchone()
|
|
if doc is None:
|
|
return {"status": "unknown_document"}
|
|
|
|
chunk_rows = conn.execute(
|
|
"SELECT idx, leaf_hash, content FROM chunks "
|
|
"WHERE document_root = ? ORDER BY idx ASC",
|
|
(document_root,),
|
|
).fetchall()
|
|
if not chunk_rows:
|
|
return {"status": "unknown_document"}
|
|
if any(r["content"] is None for r in chunk_rows):
|
|
return {"status": "source_cold", "msg": "rehydrate before asking"}
|
|
|
|
answer_mode = policy.get("answer_mode", DEFAULT_ANSWER_MODE)
|
|
if answer_mode not in ANSWER_MODES:
|
|
raise ValueError(
|
|
f"policy['answer_mode'] must be one of {ANSWER_MODES}, got {answer_mode!r}"
|
|
)
|
|
|
|
chunk_texts = [unpack_chunk(r["content"]) for r in chunk_rows]
|
|
document_text = "\n\n".join(chunk_texts)
|
|
|
|
# Wikitext → prose before the LLM sees it. The model can then quote
|
|
# verbatim against the prose form; the verifier compares like-against-
|
|
# like. Idempotent if context is already plain prose. Gated on
|
|
# policy["base_version"] so this is part of governance_policy_hash.
|
|
if policy.get("base_version") and _wikitext_to_base is not None:
|
|
document_text = _wikitext_to_base(document_text)
|
|
chunk_texts = [_wikitext_to_base(t) for t in chunk_texts]
|
|
|
|
evidence_map = []
|
|
if answer_mode == "claim_lattice_pointer":
|
|
# Quote-by-pointer: one evidence object per chunk. The model sees
|
|
# the literal spans labeled with content-addressed IDs and is
|
|
# instructed to reference IDs, not type quote text. Synthetic
|
|
# elision is impossible by construction — the model never produces
|
|
# the quote string.
|
|
chunks_for_map = [
|
|
{
|
|
"source_root": document_root,
|
|
"document_uri": doc["document_uri"],
|
|
"title": None,
|
|
"chunk_idx": r["idx"],
|
|
"chunk_root": r["leaf_hash"],
|
|
"span": chunk_texts[i],
|
|
"source_role": "primary_answer_source",
|
|
}
|
|
for i, r in enumerate(chunk_rows)
|
|
]
|
|
evidence_map = build_evidence_map(chunks_for_map)
|
|
sys_prompt = policy["claim_lattice_system_prompt"]
|
|
grounding_reminder = policy.get("claim_lattice_grounding_reminder")
|
|
rendered_evidence = render_evidence_map(evidence_map)
|
|
|
|
def _user_payload(q: str) -> str:
|
|
return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}"
|
|
else:
|
|
sys_prompt = policy["system_prompt"]
|
|
grounding_reminder = policy.get("grounding_reminder")
|
|
|
|
def _user_payload(q: str) -> str:
|
|
return f"Document:\n\n{document_text}\n\n---\n\nQuestion: {q}"
|
|
|
|
# System sets the policy; a user-turn reminder restates the rule one
|
|
# message before the payload arrives. Payload (document or evidence
|
|
# map + question) lands last as the most-recent tokens before
|
|
# generation.
|
|
messages = [{"role": "system", "content": sys_prompt}]
|
|
if grounding_reminder:
|
|
messages.append({"role": "user", "content": grounding_reminder})
|
|
messages.append({"role": "user", "content": _user_payload(question)})
|
|
|
|
mhash = model_profile_hash(model_id, revision, quantization)
|
|
|
|
# Dedup-mode-aware cache_key. See aborist/qa/query.py for rationale —
|
|
# policy_variant matches the alternate mode so governance_policy_hash
|
|
# agrees with what an agent under that mode would have written,
|
|
# enabling cross-silo fallback.
|
|
def _ckey_for_mode(mode: str) -> str:
|
|
canon_q = canonical_question(question, mode=mode)
|
|
canon_msgs = list(messages[:-1]) + [
|
|
{"role": "user", "content": _user_payload(canon_q)},
|
|
]
|
|
policy_variant = dict(policy, question_dedup=mode)
|
|
return cache_key(
|
|
document_root,
|
|
question_hash(question, mode=mode),
|
|
mhash,
|
|
conversation_hash(canon_msgs),
|
|
governance_policy_hash(policy_variant),
|
|
SCHEMA_VERSION,
|
|
CANONICALIZATION_VERSION,
|
|
doc["chunking_version"],
|
|
)
|
|
|
|
ghash = governance_policy_hash(policy) # for the legacy INSERT below
|
|
|
|
primary_dedup = policy.get("question_dedup", DEFAULT_QUESTION_DEDUP)
|
|
if primary_dedup not in QUESTION_DEDUP_MODES:
|
|
raise ValueError(
|
|
f"policy['question_dedup'] must be one of {QUESTION_DEDUP_MODES}, "
|
|
f"got {primary_dedup!r}"
|
|
)
|
|
# Re-derive the per-mode hashes for use in the INSERT below. _ckey_for_mode
|
|
# already builds them, but the legacy INSERT references qhash/chash by name.
|
|
qhash = question_hash(question, mode=primary_dedup)
|
|
canonical_q_primary = canonical_question(question, mode=primary_dedup)
|
|
canonical_messages_primary = list(messages[:-1]) + [
|
|
{"role": "user", "content": _user_payload(canonical_q_primary)},
|
|
]
|
|
chash = conversation_hash(canonical_messages_primary)
|
|
primary_ckey = _ckey_for_mode(primary_dedup)
|
|
ckey = primary_ckey # legacy name for the rest of the function
|
|
|
|
t_lookup = time.monotonic()
|
|
cached = conn.execute(
|
|
"SELECT * FROM providence_cache "
|
|
"WHERE cache_key = ? AND falsification_state = 'live'",
|
|
(primary_ckey,),
|
|
).fetchone()
|
|
hit_ckey = primary_ckey
|
|
lookup_path = primary_dedup if cached is not None else None
|
|
if cached is None and fidelity == "equivalence_class":
|
|
other_mode = (
|
|
"equivalence_class" if primary_dedup == "strict" else "strict"
|
|
)
|
|
other_ckey = _ckey_for_mode(other_mode)
|
|
if other_ckey != primary_ckey:
|
|
cached = conn.execute(
|
|
"SELECT * FROM providence_cache "
|
|
"WHERE cache_key = ? AND falsification_state = 'live'",
|
|
(other_ckey,),
|
|
).fetchone()
|
|
if cached is not None:
|
|
hit_ckey = other_ckey
|
|
lookup_path = f"{other_mode}_fallback"
|
|
cache_lookup_ms = _ms_since(t_lookup)
|
|
if cached is not None:
|
|
with transaction(conn):
|
|
now = int(time.time())
|
|
conn.execute(
|
|
"UPDATE providence_cache "
|
|
"SET hit_count = hit_count + 1, last_hit_at = ? "
|
|
"WHERE cache_key = ?",
|
|
(now, hit_ckey),
|
|
)
|
|
return {
|
|
"status": "cache_hit",
|
|
"audit_mode": cached["audit_mode"],
|
|
"cache_key": hit_ckey,
|
|
"lookup_path": lookup_path,
|
|
"source_root": document_root,
|
|
"answer_text": cached["answer_text"],
|
|
"merkle_proof": json.loads(cached["merkle_proof"]),
|
|
"n_quotes": cached["n_quotes"],
|
|
"n_verified": cached["n_verified"],
|
|
"verifier_method": cached["verifier_method"],
|
|
"unverified_quotes": (
|
|
json.loads(cached["unverified_quotes"])
|
|
if cached["unverified_quotes"]
|
|
else []
|
|
),
|
|
"timings": {
|
|
"cache_lookup_ms": cache_lookup_ms,
|
|
"llm_ms": None,
|
|
"total_ms": _ms_since(t_start),
|
|
},
|
|
}
|
|
|
|
t_llm = time.monotonic()
|
|
raw_answer = client.chat_completion(
|
|
messages,
|
|
model=model_id,
|
|
temperature=policy["temperature"],
|
|
max_tokens=policy["max_tokens"],
|
|
top_p=policy.get("top_p", 1.0),
|
|
)
|
|
llm_ms = _ms_since(t_llm)
|
|
|
|
repair_changes: list[dict] = []
|
|
pre_repair_verdict: dict | None = None
|
|
|
|
if answer_mode == "claim_lattice_pointer":
|
|
verdict = verify_claim_lattice(
|
|
raw_answer,
|
|
evidence_map,
|
|
allowed_source_roles=tuple(
|
|
policy.get(
|
|
"claim_lattice_allowed_source_roles",
|
|
[
|
|
"primary_answer_source",
|
|
"secondary_context_source",
|
|
"background_source",
|
|
"unclassified",
|
|
],
|
|
)
|
|
),
|
|
)
|
|
# Rendered prose (literal spans interpolated) is the user-facing
|
|
# answer text — never the model's raw pointer-line output. If
|
|
# rendering produced nothing (no valid claims), persist the raw
|
|
# output so an operator can see what the model actually said.
|
|
rendered = verdict["rendered_text"]
|
|
answer_text = rendered if rendered else raw_answer
|
|
else:
|
|
answer_text = raw_answer
|
|
verdict = verify_quotes(
|
|
answer_text,
|
|
document_text,
|
|
entity_policy=policy.get("entity_policy", "hybrid"),
|
|
proximity_n=policy.get("entity_proximity_n", 3),
|
|
proximity_window=policy.get("entity_proximity_window", 300),
|
|
)
|
|
|
|
def _verify(text: str) -> dict:
|
|
return verify_quotes(
|
|
text,
|
|
document_text,
|
|
entity_policy=policy.get("entity_policy", "hybrid"),
|
|
proximity_n=policy.get("entity_proximity_n", 3),
|
|
proximity_window=policy.get("entity_proximity_window", 300),
|
|
)
|
|
|
|
if (
|
|
policy.get("repair_enabled")
|
|
and verdict["audit_mode"] != "STRICT"
|
|
and verdict.get("unverified_quotes")
|
|
):
|
|
repair_result = mechanical_repair(
|
|
answer_text, verdict["unverified_quotes"], document_text
|
|
)
|
|
if repair_result["changes"]:
|
|
new_verdict = _verify(repair_result["repaired_text"])
|
|
if new_verdict["n_verified"] >= verdict["n_verified"]:
|
|
pre_repair_verdict = verdict
|
|
answer_text = repair_result["repaired_text"]
|
|
verdict = new_verdict
|
|
repair_changes = list(repair_result["changes"])
|
|
|
|
max_reprompts = int(policy.get("repair_max_reprompts", 0))
|
|
for _ in range(max_reprompts):
|
|
if (
|
|
verdict["audit_mode"] == "STRICT"
|
|
or not verdict.get("unverified_quotes")
|
|
):
|
|
break
|
|
new_text = reprompt_repair(
|
|
chat_client=client,
|
|
model_id=model_id,
|
|
original_messages=messages,
|
|
original_answer=answer_text,
|
|
failed_quotes=verdict["unverified_quotes"],
|
|
policy=policy,
|
|
)
|
|
if not new_text:
|
|
break
|
|
new_verdict = _verify(new_text)
|
|
if new_verdict["n_verified"] > verdict["n_verified"]:
|
|
if pre_repair_verdict is None:
|
|
pre_repair_verdict = verdict
|
|
answer_text = new_text
|
|
verdict = new_verdict
|
|
repair_changes.append({
|
|
"action": "reprompt_rewrite",
|
|
"diagnosis": "model_feedback_loop",
|
|
})
|
|
else:
|
|
break
|
|
|
|
unverified_blob = (
|
|
json.dumps(verdict["unverified_quotes"], separators=(",", ":"))
|
|
if verdict["unverified_quotes"]
|
|
else None
|
|
)
|
|
|
|
leaves = [bytes.fromhex(r["leaf_hash"]) for r in chunk_rows]
|
|
tree = MerkleTree.build(leaves)
|
|
proof_obj = {
|
|
"document_root": document_root,
|
|
"chunk_0_proof": proof_to_dict(tree.proof(0)),
|
|
}
|
|
proof_blob = json.dumps(proof_obj, separators=(",", ":"))
|
|
|
|
# Per-run Merkle-DAG (see aborist/qa/dag.py). Single-doc shape: the
|
|
# only "source" is document_root. Pointer mode swaps the 7-stage
|
|
# quote shape for the 9-stage CTI shape — context drops out and
|
|
# answer splits into raw_answer / parsed_claim_lattice / render.
|
|
ev_root = evidence_map_root(evidence_map) if evidence_map else None
|
|
parsed_lattice = None
|
|
if answer_mode == "claim_lattice_pointer":
|
|
# Per-claim list of {claim_text, content-addressed evidence_ids}
|
|
# for the parsed_claim_lattice node hash. Pointer ids are
|
|
# run-dependent; evidence_ids are content-addressed → the run-
|
|
# DAG hashes the run-stable form.
|
|
evidence_id_pairs = verdict.get("evidence_id_pairs") or []
|
|
parsed_lattice = [
|
|
{
|
|
"claim_text": cs.get("text", ""),
|
|
"evidence_ids": evidence_id_pairs[i] if i < len(evidence_id_pairs) else [],
|
|
}
|
|
for i, cs in enumerate(verdict.get("claim_statuses") or [])
|
|
]
|
|
run_dag = build_run_dag(
|
|
question_hash=qhash,
|
|
sources=[{
|
|
"document_root": document_root,
|
|
"source_role": "primary_answer_source",
|
|
"score": None,
|
|
"chunk_idx": None,
|
|
}],
|
|
context_root=document_root,
|
|
conversation_hash=chash,
|
|
answer_text=answer_text,
|
|
audit_mode=verdict["audit_mode"],
|
|
verifier_method=verdict["verifier_method"],
|
|
n_quotes=verdict["n_quotes"],
|
|
n_verified=verdict["n_verified"],
|
|
claim_statuses=verdict.get("claim_statuses", []),
|
|
lookup_path="miss",
|
|
evidence_map_root=ev_root,
|
|
answer_mode=answer_mode if answer_mode != "quote" else None,
|
|
violations=verdict.get("violations"),
|
|
raw_answer_text=raw_answer if answer_mode == "claim_lattice_pointer" else None,
|
|
parsed_lattice=parsed_lattice,
|
|
rendered_text=answer_text if answer_mode == "claim_lattice_pointer" else None,
|
|
)
|
|
run_dag_blob = json.dumps(run_dag, separators=(",", ":"))
|
|
|
|
now = int(time.time())
|
|
with transaction(conn):
|
|
if repair_changes and pre_repair_verdict is not None:
|
|
append_audit(
|
|
conn,
|
|
event_type="providence_repair",
|
|
subject_root=ckey,
|
|
body={
|
|
"kind": "mechanical",
|
|
"n_changes": len(repair_changes),
|
|
"changes": repair_changes,
|
|
"pre_audit_mode": pre_repair_verdict["audit_mode"],
|
|
"post_audit_mode": verdict["audit_mode"],
|
|
"pre_n_verified": pre_repair_verdict["n_verified"],
|
|
"post_n_verified": verdict["n_verified"],
|
|
},
|
|
ts=now,
|
|
)
|
|
event_hash = append_audit(
|
|
conn,
|
|
event_type="providence_write",
|
|
subject_root=ckey,
|
|
body={
|
|
"source_root": document_root,
|
|
"model_id": model_id,
|
|
"revision": revision,
|
|
"quantization": quantization,
|
|
"chunks_in_context": len(chunk_rows),
|
|
"answer_chars": len(answer_text),
|
|
"audit_mode": verdict["audit_mode"],
|
|
"n_quotes": verdict["n_quotes"],
|
|
"n_verified": verdict["n_verified"],
|
|
"verifier_method": verdict["verifier_method"],
|
|
},
|
|
ts=now,
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO providence_cache "
|
|
"(cache_key, source_root, document_uri, question_hash, question_text, "
|
|
" answer_text, merkle_proof, model_profile_hash, conversation_hash, "
|
|
" governance_policy_hash, schema_version, canonicalization_version, "
|
|
" chunking_version, falsification_state, chain, audit_event_hash, "
|
|
" created_at, hit_count, audit_mode, n_quotes, n_verified, "
|
|
" unverified_quotes, verifier_method, run_dag_root, run_dag_blob) "
|
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'live', ?, ?, ?, 0, "
|
|
" ?, ?, ?, ?, ?, ?, ?)",
|
|
(
|
|
ckey,
|
|
document_root,
|
|
doc["document_uri"],
|
|
qhash,
|
|
question,
|
|
answer_text,
|
|
proof_blob,
|
|
mhash,
|
|
chash,
|
|
ghash,
|
|
SCHEMA_VERSION,
|
|
CANONICALIZATION_VERSION,
|
|
doc["chunking_version"],
|
|
chain,
|
|
event_hash,
|
|
now,
|
|
verdict["audit_mode"],
|
|
verdict["n_quotes"],
|
|
verdict["n_verified"],
|
|
unverified_blob,
|
|
verdict["verifier_method"],
|
|
run_dag["root"],
|
|
run_dag_blob,
|
|
),
|
|
)
|
|
|
|
from aborist.qa.dag import localize_failure as _localize
|
|
failure_stage = _localize(
|
|
audit_mode=verdict["audit_mode"],
|
|
n_sources=1, # ask() runs against one document
|
|
n_quotes=verdict["n_quotes"],
|
|
n_verified=verdict["n_verified"],
|
|
)
|
|
return {
|
|
"status": "cache_miss_then_written",
|
|
"audit_mode": verdict["audit_mode"],
|
|
"cache_key": ckey,
|
|
"run_dag_root": run_dag["root"],
|
|
"lookup_path": "miss",
|
|
"failure_stage": failure_stage,
|
|
"repair_changes": repair_changes,
|
|
"pre_repair_audit_mode": (
|
|
pre_repair_verdict["audit_mode"] if pre_repair_verdict else None
|
|
),
|
|
"source_root": document_root,
|
|
"answer_text": answer_text,
|
|
"merkle_proof": proof_obj,
|
|
"n_quotes": verdict["n_quotes"],
|
|
"n_verified": verdict["n_verified"],
|
|
"verifier_method": verdict["verifier_method"],
|
|
"unverified_quotes": verdict["unverified_quotes"],
|
|
# Sidecar smell signals (claim_lattice mode only) — render-
|
|
# layer; never persisted, never in run_dag_root.
|
|
"pointer_id_distribution": verdict.get("pointer_id_distribution"),
|
|
"lazy_anchor_ratio": verdict.get("lazy_anchor_ratio"),
|
|
"timings": {
|
|
"cache_lookup_ms": cache_lookup_ms,
|
|
"llm_ms": llm_ms,
|
|
"total_ms": _ms_since(t_start),
|
|
},
|
|
}
|