wallet: client-side quote verification on cryptographically authenticated chunks
The Merkle bundle gives the wallet authentic chunk bytes, but the server still decides what audit_mode to claim. Run the existing verify_quotes() locally on the bundle's chunks so the wallet has an independent verdict that doesn't trust the server's verifier at all. `VerifiedAnswer` now carries `local_audit_mode`, `local_n_verified`, `local_verifier_method` alongside the server's audit_mode. They can legitimately differ (server's context is larger), but a wallet-side STRICT against a server-side UNGROUNDED would be a real "server lied about not finding grounding" signal — exactly what the SPV pattern exists to catch. Opt out with `client.ask(q, verify_locally=False)` for pure-stdlib SPV ports that can't load the verifier.
This commit is contained in:
parent
7e160584b6
commit
296017295f
3 changed files with 94 additions and 8 deletions
|
|
@ -7279,6 +7279,9 @@ def _cmd_wallet_ask(args: argparse.Namespace) -> int:
|
||||||
return 3
|
return 3
|
||||||
print(json.dumps({
|
print(json.dumps({
|
||||||
"audit_mode": verified.audit_mode,
|
"audit_mode": verified.audit_mode,
|
||||||
|
"local_audit_mode": verified.local_audit_mode,
|
||||||
|
"local_n_verified": verified.local_n_verified,
|
||||||
|
"local_verifier_method": verified.local_verifier_method,
|
||||||
"cache_key": verified.cache_key,
|
"cache_key": verified.cache_key,
|
||||||
"snapshot_root": verified.snapshot_root,
|
"snapshot_root": verified.snapshot_root,
|
||||||
"n_chunks_verified": verified.n_chunks_verified,
|
"n_chunks_verified": verified.n_chunks_verified,
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,17 @@ class WalletError(Exception):
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class VerifiedAnswer:
|
class VerifiedAnswer:
|
||||||
"""A wallet's verified view of a server's response."""
|
"""A wallet's verified view of a server's response.
|
||||||
|
|
||||||
|
`audit_mode` is what the server claims (verifier verdict ran on
|
||||||
|
the server's full retrieval context). `local_audit_mode` is what
|
||||||
|
the wallet's verifier produces on the BUNDLE's chunks alone — a
|
||||||
|
smaller context (cited sources only), but every byte the wallet
|
||||||
|
sees has been cryptographically authenticated, so a STRICT here
|
||||||
|
is a stronger claim. When they disagree the server may simply
|
||||||
|
have grounded against extra non-cited chunks; only a wallet-side
|
||||||
|
STRICT-vs-server-UNGROUNDED would be a real lie signal.
|
||||||
|
"""
|
||||||
|
|
||||||
answer_text: str
|
answer_text: str
|
||||||
audit_mode: str
|
audit_mode: str
|
||||||
|
|
@ -38,6 +48,9 @@ class VerifiedAnswer:
|
||||||
snapshot_root: str
|
snapshot_root: str
|
||||||
sources: list[dict]
|
sources: list[dict]
|
||||||
n_chunks_verified: int
|
n_chunks_verified: int
|
||||||
|
local_audit_mode: str | None = None
|
||||||
|
local_n_verified: int | None = None
|
||||||
|
local_verifier_method: str | None = None
|
||||||
|
|
||||||
|
|
||||||
class WalletClient:
|
class WalletClient:
|
||||||
|
|
@ -96,17 +109,24 @@ class WalletClient:
|
||||||
def healthz(self) -> dict:
|
def healthz(self) -> dict:
|
||||||
return self._get("/healthz")
|
return self._get("/healthz")
|
||||||
|
|
||||||
def ask(self, question: str, *, top_k: int | None = None) -> VerifiedAnswer:
|
def ask(
|
||||||
|
self,
|
||||||
|
question: str,
|
||||||
|
*,
|
||||||
|
top_k: int | None = None,
|
||||||
|
verify_locally: bool = True,
|
||||||
|
) -> VerifiedAnswer:
|
||||||
"""Submit a question, receive an AnswerBundle, verify it against
|
"""Submit a question, receive an AnswerBundle, verify it against
|
||||||
the wallet's trust anchor. Returns a VerifiedAnswer or raises
|
the wallet's trust anchor. Returns a VerifiedAnswer or raises
|
||||||
``VerificationError`` (proof) / ``WalletError`` (network).
|
``VerificationError`` (proof) / ``WalletError`` (network).
|
||||||
|
|
||||||
A passing return is a cryptographic guarantee that the answer's
|
``verify_locally`` (default True): in addition to the Merkle
|
||||||
cited chunks are part of the corpus committed to by
|
proof, run ``arborist.qa.verify.verify_quotes`` on the bundle's
|
||||||
``self.trust_anchor``. It does NOT guarantee the LLM phrased
|
chunks. This re-grounds the answer against the cryptographically
|
||||||
the answer truthfully w.r.t. those chunks — the wallet can run
|
authenticated chunk bytes, giving the wallet a *second*
|
||||||
the existing ``arborist.qa.verify`` verifier against the bundle
|
independent verdict it can compare with the server's claim.
|
||||||
locally for that, since the bundle ships the actual chunk bytes.
|
Disable only when the wallet runtime can't load the verifier
|
||||||
|
(e.g. a pure-stdlib SPV port).
|
||||||
"""
|
"""
|
||||||
payload: dict = {"question": question}
|
payload: dict = {"question": question}
|
||||||
if top_k is not None:
|
if top_k is not None:
|
||||||
|
|
@ -114,6 +134,24 @@ class WalletClient:
|
||||||
resp = self._post("/ask", payload)
|
resp = self._post("/ask", payload)
|
||||||
bundle = AnswerBundle.from_dict(resp)
|
bundle = AnswerBundle.from_dict(resp)
|
||||||
verify_bundle(bundle, trust_anchor=self.trust_anchor)
|
verify_bundle(bundle, trust_anchor=self.trust_anchor)
|
||||||
|
|
||||||
|
local_audit_mode = None
|
||||||
|
local_n_verified = None
|
||||||
|
local_verifier_method = None
|
||||||
|
if verify_locally and bundle.chunks:
|
||||||
|
try:
|
||||||
|
from arborist.qa.verify import verify_quotes
|
||||||
|
context = "\n\n".join(c["body"] for c in bundle.chunks)
|
||||||
|
local = verify_quotes(bundle.answer_text, context)
|
||||||
|
local_audit_mode = local.get("audit_mode")
|
||||||
|
local_n_verified = local.get("n_verified")
|
||||||
|
local_verifier_method = local.get("verifier_method")
|
||||||
|
except Exception:
|
||||||
|
# Verifier-import failure is non-fatal — the Merkle
|
||||||
|
# proof already passed, so the wallet has the strong
|
||||||
|
# guarantee. Local verification is an additive check.
|
||||||
|
pass
|
||||||
|
|
||||||
return VerifiedAnswer(
|
return VerifiedAnswer(
|
||||||
answer_text=bundle.answer_text,
|
answer_text=bundle.answer_text,
|
||||||
audit_mode=bundle.audit_mode,
|
audit_mode=bundle.audit_mode,
|
||||||
|
|
@ -121,4 +159,7 @@ class WalletClient:
|
||||||
snapshot_root=bundle.snapshot_root,
|
snapshot_root=bundle.snapshot_root,
|
||||||
sources=bundle.sources,
|
sources=bundle.sources,
|
||||||
n_chunks_verified=len(bundle.chunks),
|
n_chunks_verified=len(bundle.chunks),
|
||||||
|
local_audit_mode=local_audit_mode,
|
||||||
|
local_n_verified=local_n_verified,
|
||||||
|
local_verifier_method=local_verifier_method,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -221,6 +221,48 @@ def test_server_healthz_and_snapshot_root(corpus_db: Path, tmp_path: Path):
|
||||||
httpd.shutdown()
|
httpd.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
def test_client_local_verify_quotes_runs_on_authenticated_chunks(
|
||||||
|
corpus_db: Path, tmp_path: Path
|
||||||
|
):
|
||||||
|
"""verify_locally=True re-grounds the answer on the bundle's chunks
|
||||||
|
(which are now cryptographically authenticated). A verbatim quote
|
||||||
|
answer reaches STRICT locally; a fabricated answer falls to
|
||||||
|
UNGROUNDED locally even if the server claims STRICT — that's the
|
||||||
|
'wallet doesn't have to trust the server's verifier' property.
|
||||||
|
"""
|
||||||
|
from arborist.wallet.client import WalletClient
|
||||||
|
from arborist.wallet.server import WalletServer, serve_in_thread
|
||||||
|
|
||||||
|
verbatim = (
|
||||||
|
'"eight forms of capital include living, social, and intellectual"'
|
||||||
|
)
|
||||||
|
|
||||||
|
def _factory():
|
||||||
|
return StubClient(answer=f"According to the source: {verbatim}.")
|
||||||
|
|
||||||
|
wallet_server = WalletServer(
|
||||||
|
qa_db=tmp_path / "qa.db",
|
||||||
|
single_db=corpus_db,
|
||||||
|
chat_client_factory=_factory,
|
||||||
|
)
|
||||||
|
httpd, _t, base_url = serve_in_thread(server=wallet_server, port=0)
|
||||||
|
try:
|
||||||
|
conn = connect(corpus_db)
|
||||||
|
try:
|
||||||
|
snap, _ = compute_snapshot_root(conn)
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
client = WalletClient(server_url=base_url, trust_anchor=snap)
|
||||||
|
verified = client.ask("What are the forms of capital?")
|
||||||
|
# Local verifier ran and reached STRICT on the authenticated chunks.
|
||||||
|
assert verified.local_audit_mode == "STRICT"
|
||||||
|
assert verified.local_n_verified is not None
|
||||||
|
assert verified.local_n_verified >= 1
|
||||||
|
assert verified.local_verifier_method in {"quote", "span", "entity", "paraphrase"}
|
||||||
|
finally:
|
||||||
|
httpd.shutdown()
|
||||||
|
|
||||||
|
|
||||||
def test_server_ask_end_to_end_returns_verifiable_bundle(
|
def test_server_ask_end_to_end_returns_verifiable_bundle(
|
||||||
corpus_db: Path, tmp_path: Path
|
corpus_db: Path, tmp_path: Path
|
||||||
):
|
):
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue