russell@unturf.com
71507f9327
Both ask() (single-doc) and query() (multi-source) now run the assembled context through aborist.wikitext.to_base() before message construction. Hermes sees clean prose instead of raw wikitext, which: - lets the model quote verbatim (raw [[Cloud Strife]] was unquotable in fluent prose; the model paraphrased to bare "Cloud Strife" and the verifier flagged that as UNGROUNDED) - shrinks token bills ~43% on Wikipedia chunks (measured: 125,152 → 71,771 chars on the FF7 main-character query) - aligns LLM-input with what the verifier already strips, so the prose-vs-prose substring check is end-to-end consistent Gated on policy["base_version"] (= wikitext-base-v1) so: - governance_policy_hash flows the choice — prior answers cached under raw-wikitext policy retain distinct cache_keys, no migration - operators can opt out (set base_version=None) for debugging - environments without mwparserfromhell soft-fall to raw wikitext with no other behavior change Tests (4): default policy carries base_version (tripwire), ask() user messages have wikitext markers stripped, opt-out reverts to raw wikitext, governance_hash differs with vs without base_version.
300 lines
10 KiB
Python
300 lines
10 KiB
Python
"""Q&A runner: cache-first lookup -> inference fallback -> provable record.
|
|
|
|
Implements the v9.8 admissibility invariant:
|
|
No record reused unless all 8 cache_key dimensions match AND state
|
|
is 'live' (not failed/stale/quarantined).
|
|
|
|
Cache hit -> persisted audit_mode (STRICT/HYBRID/UNGROUNDED).
|
|
Cache miss -> call ChatClient, run faithfulness check, classify, store
|
|
record, audit event.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
import time
|
|
|
|
from aborist import (
|
|
CANONICALIZATION_VERSION,
|
|
SCHEMA_VERSION,
|
|
)
|
|
from aborist.compress import unpack_chunk
|
|
from aborist.merkle import MerkleTree, proof_to_dict
|
|
from aborist.qa.client import ChatClient
|
|
from aborist.qa.keys import (
|
|
cache_key,
|
|
conversation_hash,
|
|
governance_policy_hash,
|
|
model_profile_hash,
|
|
question_hash,
|
|
)
|
|
from aborist.qa.verify import verify_quotes
|
|
from aborist.store import append_audit, transaction
|
|
|
|
try:
|
|
from aborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION
|
|
from aborist.wikitext import to_base as _wikitext_to_base
|
|
except ImportError: # pragma: no cover
|
|
_WIKITEXT_BASE_VERSION = None
|
|
_wikitext_to_base = None
|
|
|
|
|
|
DEFAULT_POLICY = {
|
|
"system_prompt": (
|
|
"Answer the user's question based ONLY on the document below. "
|
|
"For EVERY factual claim, include a verbatim quote from the "
|
|
"document enclosed in double quotes (\"...\"). The quoted span "
|
|
"must appear word-for-word. If you cannot find a verbatim quote "
|
|
"supporting a claim, do not make the claim. "
|
|
"If the answer is not in the document, say 'I don't know based "
|
|
"on the provided document.' Do not speculate."
|
|
),
|
|
# Restated rule fired as a user message right before the document +
|
|
# question arrive. See aborist/qa/query.py for the rationale (recent
|
|
# user-turn instructions outweigh decayed system-turn rules in 8B
|
|
# instruction-tuned models).
|
|
"grounding_reminder": (
|
|
"REMINDER: every factual claim in your reply must be wrapped in "
|
|
"double quotes (\"...\") and the quoted span must appear "
|
|
"word-for-word in the document. No quote, no claim. "
|
|
"Now answer the question on the next message."
|
|
),
|
|
"temperature": 0.1,
|
|
"top_p": 1.0,
|
|
"max_tokens": 512,
|
|
"entity_policy": "proximity",
|
|
"entity_proximity_n": 3,
|
|
"entity_proximity_window": 300,
|
|
# Strip wikitext markup before the LLM ever sees the context. Lets
|
|
# Hermes quote prose verbatim and shrinks token bills (~43% on
|
|
# Wikipedia chunks). Bumps governance_policy_hash so prior cached
|
|
# answers under raw-wikitext policy stay distinct on lookup. Set
|
|
# via the wikitext extras; no-op if mwparserfromhell isn't installed.
|
|
"base_version": _WIKITEXT_BASE_VERSION,
|
|
}
|
|
|
|
|
|
def _ms_since(t: float) -> float:
|
|
return round((time.monotonic() - t) * 1000, 1)
|
|
|
|
|
|
def ask(
|
|
conn: sqlite3.Connection,
|
|
*,
|
|
document_root: str,
|
|
question: str,
|
|
client: ChatClient,
|
|
model_id: str,
|
|
revision: str = "",
|
|
quantization: str = "",
|
|
policy: dict | None = None,
|
|
chain: str = "private",
|
|
) -> dict:
|
|
"""Look up cached answer or run inference. Returns a result dict."""
|
|
policy = policy or DEFAULT_POLICY
|
|
t_start = time.monotonic()
|
|
|
|
doc = conn.execute(
|
|
"SELECT document_uri, chunking_version FROM documents "
|
|
"WHERE document_root = ?",
|
|
(document_root,),
|
|
).fetchone()
|
|
if doc is None:
|
|
return {"status": "unknown_document"}
|
|
|
|
chunk_rows = conn.execute(
|
|
"SELECT idx, leaf_hash, content FROM chunks "
|
|
"WHERE document_root = ? ORDER BY idx ASC",
|
|
(document_root,),
|
|
).fetchall()
|
|
if not chunk_rows:
|
|
return {"status": "unknown_document"}
|
|
if any(r["content"] is None for r in chunk_rows):
|
|
return {"status": "source_cold", "msg": "rehydrate before asking"}
|
|
|
|
document_text = "\n\n".join(unpack_chunk(r["content"]) for r in chunk_rows)
|
|
|
|
# Wikitext → prose before the LLM sees it. The model can then quote
|
|
# verbatim against the prose form; the verifier compares like-against-
|
|
# like. Idempotent if context is already plain prose. Gated on
|
|
# policy["base_version"] so this is part of governance_policy_hash.
|
|
if policy.get("base_version") and _wikitext_to_base is not None:
|
|
document_text = _wikitext_to_base(document_text)
|
|
|
|
# System sets the policy; a user-turn reminder restates the verbatim-quote
|
|
# rule one message before the document arrives. Document + question land
|
|
# last as the most-recent tokens before generation.
|
|
messages = [
|
|
{"role": "system", "content": policy["system_prompt"]},
|
|
]
|
|
if policy.get("grounding_reminder"):
|
|
messages.append(
|
|
{"role": "user", "content": policy["grounding_reminder"]}
|
|
)
|
|
messages.append(
|
|
{
|
|
"role": "user",
|
|
"content": (
|
|
f"Document:\n\n{document_text}\n\n---\n\n"
|
|
f"Question: {question}"
|
|
),
|
|
}
|
|
)
|
|
|
|
qhash = question_hash(question)
|
|
mhash = model_profile_hash(model_id, revision, quantization)
|
|
chash = conversation_hash(messages)
|
|
ghash = governance_policy_hash(policy)
|
|
ckey = cache_key(
|
|
document_root,
|
|
qhash,
|
|
mhash,
|
|
chash,
|
|
ghash,
|
|
SCHEMA_VERSION,
|
|
CANONICALIZATION_VERSION,
|
|
doc["chunking_version"],
|
|
)
|
|
|
|
t_lookup = time.monotonic()
|
|
cached = conn.execute(
|
|
"SELECT * FROM providence_cache "
|
|
"WHERE cache_key = ? AND falsification_state = 'live'",
|
|
(ckey,),
|
|
).fetchone()
|
|
cache_lookup_ms = _ms_since(t_lookup)
|
|
if cached is not None:
|
|
with transaction(conn):
|
|
now = int(time.time())
|
|
conn.execute(
|
|
"UPDATE providence_cache "
|
|
"SET hit_count = hit_count + 1, last_hit_at = ? "
|
|
"WHERE cache_key = ?",
|
|
(now, ckey),
|
|
)
|
|
return {
|
|
"status": "cache_hit",
|
|
"audit_mode": cached["audit_mode"],
|
|
"cache_key": ckey,
|
|
"source_root": document_root,
|
|
"answer_text": cached["answer_text"],
|
|
"merkle_proof": json.loads(cached["merkle_proof"]),
|
|
"n_quotes": cached["n_quotes"],
|
|
"n_verified": cached["n_verified"],
|
|
"verifier_method": cached["verifier_method"],
|
|
"unverified_quotes": (
|
|
json.loads(cached["unverified_quotes"])
|
|
if cached["unverified_quotes"]
|
|
else []
|
|
),
|
|
"timings": {
|
|
"cache_lookup_ms": cache_lookup_ms,
|
|
"llm_ms": None,
|
|
"total_ms": _ms_since(t_start),
|
|
},
|
|
}
|
|
|
|
t_llm = time.monotonic()
|
|
answer_text = client.chat_completion(
|
|
messages,
|
|
model=model_id,
|
|
temperature=policy["temperature"],
|
|
max_tokens=policy["max_tokens"],
|
|
top_p=policy.get("top_p", 1.0),
|
|
)
|
|
llm_ms = _ms_since(t_llm)
|
|
|
|
verdict = verify_quotes(
|
|
answer_text,
|
|
document_text,
|
|
entity_policy=policy.get("entity_policy", "hybrid"),
|
|
proximity_n=policy.get("entity_proximity_n", 3),
|
|
proximity_window=policy.get("entity_proximity_window", 300),
|
|
)
|
|
unverified_blob = (
|
|
json.dumps(verdict["unverified_quotes"], separators=(",", ":"))
|
|
if verdict["unverified_quotes"]
|
|
else None
|
|
)
|
|
|
|
leaves = [bytes.fromhex(r["leaf_hash"]) for r in chunk_rows]
|
|
tree = MerkleTree.build(leaves)
|
|
proof_obj = {
|
|
"document_root": document_root,
|
|
"chunk_0_proof": proof_to_dict(tree.proof(0)),
|
|
}
|
|
proof_blob = json.dumps(proof_obj, separators=(",", ":"))
|
|
|
|
now = int(time.time())
|
|
with transaction(conn):
|
|
event_hash = append_audit(
|
|
conn,
|
|
event_type="providence_write",
|
|
subject_root=ckey,
|
|
body={
|
|
"source_root": document_root,
|
|
"model_id": model_id,
|
|
"revision": revision,
|
|
"quantization": quantization,
|
|
"chunks_in_context": len(chunk_rows),
|
|
"answer_chars": len(answer_text),
|
|
"audit_mode": verdict["audit_mode"],
|
|
"n_quotes": verdict["n_quotes"],
|
|
"n_verified": verdict["n_verified"],
|
|
"verifier_method": verdict["verifier_method"],
|
|
},
|
|
ts=now,
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO providence_cache "
|
|
"(cache_key, source_root, document_uri, question_hash, question_text, "
|
|
" answer_text, merkle_proof, model_profile_hash, conversation_hash, "
|
|
" governance_policy_hash, schema_version, canonicalization_version, "
|
|
" chunking_version, falsification_state, chain, audit_event_hash, "
|
|
" created_at, hit_count, audit_mode, n_quotes, n_verified, "
|
|
" unverified_quotes, verifier_method) "
|
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'live', ?, ?, ?, 0, "
|
|
" ?, ?, ?, ?, ?)",
|
|
(
|
|
ckey,
|
|
document_root,
|
|
doc["document_uri"],
|
|
qhash,
|
|
question,
|
|
answer_text,
|
|
proof_blob,
|
|
mhash,
|
|
chash,
|
|
ghash,
|
|
SCHEMA_VERSION,
|
|
CANONICALIZATION_VERSION,
|
|
doc["chunking_version"],
|
|
chain,
|
|
event_hash,
|
|
now,
|
|
verdict["audit_mode"],
|
|
verdict["n_quotes"],
|
|
verdict["n_verified"],
|
|
unverified_blob,
|
|
verdict["verifier_method"],
|
|
),
|
|
)
|
|
|
|
return {
|
|
"status": "cache_miss_then_written",
|
|
"audit_mode": verdict["audit_mode"],
|
|
"cache_key": ckey,
|
|
"source_root": document_root,
|
|
"answer_text": answer_text,
|
|
"merkle_proof": proof_obj,
|
|
"n_quotes": verdict["n_quotes"],
|
|
"n_verified": verdict["n_verified"],
|
|
"verifier_method": verdict["verifier_method"],
|
|
"unverified_quotes": verdict["unverified_quotes"],
|
|
"timings": {
|
|
"cache_lookup_ms": cache_lookup_ms,
|
|
"llm_ms": llm_ms,
|
|
"total_ms": _ms_since(t_start),
|
|
},
|
|
}
|