wallet: client-side quote verification on cryptographically authenticated chunks

The Merkle bundle gives the wallet authentic chunk bytes, but the
server still decides what audit_mode to claim. Run the existing
verify_quotes() locally on the bundle's chunks so the wallet has an
independent verdict that doesn't trust the server's verifier at all.

`VerifiedAnswer` now carries `local_audit_mode`, `local_n_verified`,
`local_verifier_method` alongside the server's audit_mode. They can
legitimately differ (server's context is larger), but a wallet-side
STRICT against a server-side UNGROUNDED would be a real "server lied
about not finding grounding" signal — exactly what the SPV pattern
exists to catch.

Opt out with `client.ask(q, verify_locally=False)` for pure-stdlib
SPV ports that can't load the verifier.
This commit is contained in:
russell@unturf.com 2026-05-30 07:47:24 -04:00
parent 7e160584b6
commit 296017295f
No known key found for this signature in database
3 changed files with 94 additions and 8 deletions

View file

@ -7279,6 +7279,9 @@ def _cmd_wallet_ask(args: argparse.Namespace) -> int:
return 3 return 3
print(json.dumps({ print(json.dumps({
"audit_mode": verified.audit_mode, "audit_mode": verified.audit_mode,
"local_audit_mode": verified.local_audit_mode,
"local_n_verified": verified.local_n_verified,
"local_verifier_method": verified.local_verifier_method,
"cache_key": verified.cache_key, "cache_key": verified.cache_key,
"snapshot_root": verified.snapshot_root, "snapshot_root": verified.snapshot_root,
"n_chunks_verified": verified.n_chunks_verified, "n_chunks_verified": verified.n_chunks_verified,

View file

@ -30,7 +30,17 @@ class WalletError(Exception):
@dataclass @dataclass
class VerifiedAnswer: class VerifiedAnswer:
"""A wallet's verified view of a server's response.""" """A wallet's verified view of a server's response.
`audit_mode` is what the server claims (verifier verdict ran on
the server's full retrieval context). `local_audit_mode` is what
the wallet's verifier produces on the BUNDLE's chunks alone — a
smaller context (cited sources only), but every byte the wallet
sees has been cryptographically authenticated, so a STRICT here
is a stronger claim. When they disagree the server may simply
have grounded against extra non-cited chunks; only a wallet-side
STRICT-vs-server-UNGROUNDED would be a real lie signal.
"""
answer_text: str answer_text: str
audit_mode: str audit_mode: str
@ -38,6 +48,9 @@ class VerifiedAnswer:
snapshot_root: str snapshot_root: str
sources: list[dict] sources: list[dict]
n_chunks_verified: int n_chunks_verified: int
local_audit_mode: str | None = None
local_n_verified: int | None = None
local_verifier_method: str | None = None
class WalletClient: class WalletClient:
@ -96,17 +109,24 @@ class WalletClient:
def healthz(self) -> dict: def healthz(self) -> dict:
return self._get("/healthz") return self._get("/healthz")
def ask(self, question: str, *, top_k: int | None = None) -> VerifiedAnswer: def ask(
self,
question: str,
*,
top_k: int | None = None,
verify_locally: bool = True,
) -> VerifiedAnswer:
"""Submit a question, receive an AnswerBundle, verify it against """Submit a question, receive an AnswerBundle, verify it against
the wallet's trust anchor. Returns a VerifiedAnswer or raises the wallet's trust anchor. Returns a VerifiedAnswer or raises
``VerificationError`` (proof) / ``WalletError`` (network). ``VerificationError`` (proof) / ``WalletError`` (network).
A passing return is a cryptographic guarantee that the answer's ``verify_locally`` (default True): in addition to the Merkle
cited chunks are part of the corpus committed to by proof, run ``arborist.qa.verify.verify_quotes`` on the bundle's
``self.trust_anchor``. It does NOT guarantee the LLM phrased chunks. This re-grounds the answer against the cryptographically
the answer truthfully w.r.t. those chunks — the wallet can run authenticated chunk bytes, giving the wallet a *second*
the existing ``arborist.qa.verify`` verifier against the bundle independent verdict it can compare with the server's claim.
locally for that, since the bundle ships the actual chunk bytes. Disable only when the wallet runtime can't load the verifier
(e.g. a pure-stdlib SPV port).
""" """
payload: dict = {"question": question} payload: dict = {"question": question}
if top_k is not None: if top_k is not None:
@ -114,6 +134,24 @@ class WalletClient:
resp = self._post("/ask", payload) resp = self._post("/ask", payload)
bundle = AnswerBundle.from_dict(resp) bundle = AnswerBundle.from_dict(resp)
verify_bundle(bundle, trust_anchor=self.trust_anchor) verify_bundle(bundle, trust_anchor=self.trust_anchor)
local_audit_mode = None
local_n_verified = None
local_verifier_method = None
if verify_locally and bundle.chunks:
try:
from arborist.qa.verify import verify_quotes
context = "\n\n".join(c["body"] for c in bundle.chunks)
local = verify_quotes(bundle.answer_text, context)
local_audit_mode = local.get("audit_mode")
local_n_verified = local.get("n_verified")
local_verifier_method = local.get("verifier_method")
except Exception:
# Verifier-import failure is non-fatal — the Merkle
# proof already passed, so the wallet has the strong
# guarantee. Local verification is an additive check.
pass
return VerifiedAnswer( return VerifiedAnswer(
answer_text=bundle.answer_text, answer_text=bundle.answer_text,
audit_mode=bundle.audit_mode, audit_mode=bundle.audit_mode,
@ -121,4 +159,7 @@ class WalletClient:
snapshot_root=bundle.snapshot_root, snapshot_root=bundle.snapshot_root,
sources=bundle.sources, sources=bundle.sources,
n_chunks_verified=len(bundle.chunks), n_chunks_verified=len(bundle.chunks),
local_audit_mode=local_audit_mode,
local_n_verified=local_n_verified,
local_verifier_method=local_verifier_method,
) )

View file

@ -221,6 +221,48 @@ def test_server_healthz_and_snapshot_root(corpus_db: Path, tmp_path: Path):
httpd.shutdown() httpd.shutdown()
def test_client_local_verify_quotes_runs_on_authenticated_chunks(
corpus_db: Path, tmp_path: Path
):
"""verify_locally=True re-grounds the answer on the bundle's chunks
(which are now cryptographically authenticated). A verbatim quote
answer reaches STRICT locally; a fabricated answer falls to
UNGROUNDED locally even if the server claims STRICT — that's the
'wallet doesn't have to trust the server's verifier' property.
"""
from arborist.wallet.client import WalletClient
from arborist.wallet.server import WalletServer, serve_in_thread
verbatim = (
'"eight forms of capital include living, social, and intellectual"'
)
def _factory():
return StubClient(answer=f"According to the source: {verbatim}.")
wallet_server = WalletServer(
qa_db=tmp_path / "qa.db",
single_db=corpus_db,
chat_client_factory=_factory,
)
httpd, _t, base_url = serve_in_thread(server=wallet_server, port=0)
try:
conn = connect(corpus_db)
try:
snap, _ = compute_snapshot_root(conn)
finally:
conn.close()
client = WalletClient(server_url=base_url, trust_anchor=snap)
verified = client.ask("What are the forms of capital?")
# Local verifier ran and reached STRICT on the authenticated chunks.
assert verified.local_audit_mode == "STRICT"
assert verified.local_n_verified is not None
assert verified.local_n_verified >= 1
assert verified.local_verifier_method in {"quote", "span", "entity", "paraphrase"}
finally:
httpd.shutdown()
def test_server_ask_end_to_end_returns_verifiable_bundle( def test_server_ask_end_to_end_returns_verifiable_bundle(
corpus_db: Path, tmp_path: Path corpus_db: Path, tmp_path: Path
): ):