wallet: client-side quote verification on cryptographically authenticated chunks
The Merkle bundle gives the wallet authentic chunk bytes, but the server still decides what audit_mode to claim. Run the existing verify_quotes() locally on the bundle's chunks so the wallet has an independent verdict that doesn't trust the server's verifier at all. `VerifiedAnswer` now carries `local_audit_mode`, `local_n_verified`, `local_verifier_method` alongside the server's audit_mode. They can legitimately differ (server's context is larger), but a wallet-side STRICT against a server-side UNGROUNDED would be a real "server lied about not finding grounding" signal — exactly what the SPV pattern exists to catch. Opt out with `client.ask(q, verify_locally=False)` for pure-stdlib SPV ports that can't load the verifier.
This commit is contained in:
parent
7e160584b6
commit
296017295f
3 changed files with 94 additions and 8 deletions
|
|
@ -7279,6 +7279,9 @@ def _cmd_wallet_ask(args: argparse.Namespace) -> int:
|
|||
return 3
|
||||
print(json.dumps({
|
||||
"audit_mode": verified.audit_mode,
|
||||
"local_audit_mode": verified.local_audit_mode,
|
||||
"local_n_verified": verified.local_n_verified,
|
||||
"local_verifier_method": verified.local_verifier_method,
|
||||
"cache_key": verified.cache_key,
|
||||
"snapshot_root": verified.snapshot_root,
|
||||
"n_chunks_verified": verified.n_chunks_verified,
|
||||
|
|
|
|||
|
|
@ -30,7 +30,17 @@ class WalletError(Exception):
|
|||
|
||||
@dataclass
|
||||
class VerifiedAnswer:
|
||||
"""A wallet's verified view of a server's response."""
|
||||
"""A wallet's verified view of a server's response.
|
||||
|
||||
`audit_mode` is what the server claims (verifier verdict ran on
|
||||
the server's full retrieval context). `local_audit_mode` is what
|
||||
the wallet's verifier produces on the BUNDLE's chunks alone — a
|
||||
smaller context (cited sources only), but every byte the wallet
|
||||
sees has been cryptographically authenticated, so a STRICT here
|
||||
is a stronger claim. When they disagree the server may simply
|
||||
have grounded against extra non-cited chunks; only a wallet-side
|
||||
STRICT-vs-server-UNGROUNDED would be a real lie signal.
|
||||
"""
|
||||
|
||||
answer_text: str
|
||||
audit_mode: str
|
||||
|
|
@ -38,6 +48,9 @@ class VerifiedAnswer:
|
|||
snapshot_root: str
|
||||
sources: list[dict]
|
||||
n_chunks_verified: int
|
||||
local_audit_mode: str | None = None
|
||||
local_n_verified: int | None = None
|
||||
local_verifier_method: str | None = None
|
||||
|
||||
|
||||
class WalletClient:
|
||||
|
|
@ -96,17 +109,24 @@ class WalletClient:
|
|||
def healthz(self) -> dict:
|
||||
return self._get("/healthz")
|
||||
|
||||
def ask(self, question: str, *, top_k: int | None = None) -> VerifiedAnswer:
|
||||
def ask(
|
||||
self,
|
||||
question: str,
|
||||
*,
|
||||
top_k: int | None = None,
|
||||
verify_locally: bool = True,
|
||||
) -> VerifiedAnswer:
|
||||
"""Submit a question, receive an AnswerBundle, verify it against
|
||||
the wallet's trust anchor. Returns a VerifiedAnswer or raises
|
||||
``VerificationError`` (proof) / ``WalletError`` (network).
|
||||
|
||||
A passing return is a cryptographic guarantee that the answer's
|
||||
cited chunks are part of the corpus committed to by
|
||||
``self.trust_anchor``. It does NOT guarantee the LLM phrased
|
||||
the answer truthfully w.r.t. those chunks — the wallet can run
|
||||
the existing ``arborist.qa.verify`` verifier against the bundle
|
||||
locally for that, since the bundle ships the actual chunk bytes.
|
||||
``verify_locally`` (default True): in addition to the Merkle
|
||||
proof, run ``arborist.qa.verify.verify_quotes`` on the bundle's
|
||||
chunks. This re-grounds the answer against the cryptographically
|
||||
authenticated chunk bytes, giving the wallet a *second*
|
||||
independent verdict it can compare with the server's claim.
|
||||
Disable only when the wallet runtime can't load the verifier
|
||||
(e.g. a pure-stdlib SPV port).
|
||||
"""
|
||||
payload: dict = {"question": question}
|
||||
if top_k is not None:
|
||||
|
|
@ -114,6 +134,24 @@ class WalletClient:
|
|||
resp = self._post("/ask", payload)
|
||||
bundle = AnswerBundle.from_dict(resp)
|
||||
verify_bundle(bundle, trust_anchor=self.trust_anchor)
|
||||
|
||||
local_audit_mode = None
|
||||
local_n_verified = None
|
||||
local_verifier_method = None
|
||||
if verify_locally and bundle.chunks:
|
||||
try:
|
||||
from arborist.qa.verify import verify_quotes
|
||||
context = "\n\n".join(c["body"] for c in bundle.chunks)
|
||||
local = verify_quotes(bundle.answer_text, context)
|
||||
local_audit_mode = local.get("audit_mode")
|
||||
local_n_verified = local.get("n_verified")
|
||||
local_verifier_method = local.get("verifier_method")
|
||||
except Exception:
|
||||
# Verifier-import failure is non-fatal — the Merkle
|
||||
# proof already passed, so the wallet has the strong
|
||||
# guarantee. Local verification is an additive check.
|
||||
pass
|
||||
|
||||
return VerifiedAnswer(
|
||||
answer_text=bundle.answer_text,
|
||||
audit_mode=bundle.audit_mode,
|
||||
|
|
@ -121,4 +159,7 @@ class WalletClient:
|
|||
snapshot_root=bundle.snapshot_root,
|
||||
sources=bundle.sources,
|
||||
n_chunks_verified=len(bundle.chunks),
|
||||
local_audit_mode=local_audit_mode,
|
||||
local_n_verified=local_n_verified,
|
||||
local_verifier_method=local_verifier_method,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -221,6 +221,48 @@ def test_server_healthz_and_snapshot_root(corpus_db: Path, tmp_path: Path):
|
|||
httpd.shutdown()
|
||||
|
||||
|
||||
def test_client_local_verify_quotes_runs_on_authenticated_chunks(
|
||||
corpus_db: Path, tmp_path: Path
|
||||
):
|
||||
"""verify_locally=True re-grounds the answer on the bundle's chunks
|
||||
(which are now cryptographically authenticated). A verbatim quote
|
||||
answer reaches STRICT locally; a fabricated answer falls to
|
||||
UNGROUNDED locally even if the server claims STRICT — that's the
|
||||
'wallet doesn't have to trust the server's verifier' property.
|
||||
"""
|
||||
from arborist.wallet.client import WalletClient
|
||||
from arborist.wallet.server import WalletServer, serve_in_thread
|
||||
|
||||
verbatim = (
|
||||
'"eight forms of capital include living, social, and intellectual"'
|
||||
)
|
||||
|
||||
def _factory():
|
||||
return StubClient(answer=f"According to the source: {verbatim}.")
|
||||
|
||||
wallet_server = WalletServer(
|
||||
qa_db=tmp_path / "qa.db",
|
||||
single_db=corpus_db,
|
||||
chat_client_factory=_factory,
|
||||
)
|
||||
httpd, _t, base_url = serve_in_thread(server=wallet_server, port=0)
|
||||
try:
|
||||
conn = connect(corpus_db)
|
||||
try:
|
||||
snap, _ = compute_snapshot_root(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
client = WalletClient(server_url=base_url, trust_anchor=snap)
|
||||
verified = client.ask("What are the forms of capital?")
|
||||
# Local verifier ran and reached STRICT on the authenticated chunks.
|
||||
assert verified.local_audit_mode == "STRICT"
|
||||
assert verified.local_n_verified is not None
|
||||
assert verified.local_n_verified >= 1
|
||||
assert verified.local_verifier_method in {"quote", "span", "entity", "paraphrase"}
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
|
||||
|
||||
def test_server_ask_end_to_end_returns_verifiable_bundle(
|
||||
corpus_db: Path, tmp_path: Path
|
||||
):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue