diff --git a/arborist/cli.py b/arborist/cli.py index 0a1491f..7788131 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -7279,6 +7279,9 @@ def _cmd_wallet_ask(args: argparse.Namespace) -> int: return 3 print(json.dumps({ "audit_mode": verified.audit_mode, + "local_audit_mode": verified.local_audit_mode, + "local_n_verified": verified.local_n_verified, + "local_verifier_method": verified.local_verifier_method, "cache_key": verified.cache_key, "snapshot_root": verified.snapshot_root, "n_chunks_verified": verified.n_chunks_verified, diff --git a/arborist/wallet/client.py b/arborist/wallet/client.py index 4eab6d6..776917c 100644 --- a/arborist/wallet/client.py +++ b/arborist/wallet/client.py @@ -30,7 +30,17 @@ class WalletError(Exception): @dataclass class VerifiedAnswer: - """A wallet's verified view of a server's response.""" + """A wallet's verified view of a server's response. + + `audit_mode` is what the server claims (verifier verdict ran on + the server's full retrieval context). `local_audit_mode` is what + the wallet's verifier produces on the BUNDLE's chunks alone — a + smaller context (cited sources only), but every byte the wallet + sees has been cryptographically authenticated, so a STRICT here + is a stronger claim. When they disagree the server may simply + have grounded against extra non-cited chunks; only a wallet-side + STRICT-vs-server-UNGROUNDED would be a real lie signal. + """ answer_text: str audit_mode: str @@ -38,6 +48,9 @@ class VerifiedAnswer: snapshot_root: str sources: list[dict] n_chunks_verified: int + local_audit_mode: str | None = None + local_n_verified: int | None = None + local_verifier_method: str | None = None class WalletClient: @@ -96,17 +109,24 @@ class WalletClient: def healthz(self) -> dict: return self._get("/healthz") - def ask(self, question: str, *, top_k: int | None = None) -> VerifiedAnswer: + def ask( + self, + question: str, + *, + top_k: int | None = None, + verify_locally: bool = True, + ) -> VerifiedAnswer: """Submit a question, receive an AnswerBundle, verify it against the wallet's trust anchor. Returns a VerifiedAnswer or raises ``VerificationError`` (proof) / ``WalletError`` (network). - A passing return is a cryptographic guarantee that the answer's - cited chunks are part of the corpus committed to by - ``self.trust_anchor``. It does NOT guarantee the LLM phrased - the answer truthfully w.r.t. those chunks — the wallet can run - the existing ``arborist.qa.verify`` verifier against the bundle - locally for that, since the bundle ships the actual chunk bytes. + ``verify_locally`` (default True): in addition to the Merkle + proof, run ``arborist.qa.verify.verify_quotes`` on the bundle's + chunks. This re-grounds the answer against the cryptographically + authenticated chunk bytes, giving the wallet a *second* + independent verdict it can compare with the server's claim. + Disable only when the wallet runtime can't load the verifier + (e.g. a pure-stdlib SPV port). """ payload: dict = {"question": question} if top_k is not None: @@ -114,6 +134,24 @@ class WalletClient: resp = self._post("/ask", payload) bundle = AnswerBundle.from_dict(resp) verify_bundle(bundle, trust_anchor=self.trust_anchor) + + local_audit_mode = None + local_n_verified = None + local_verifier_method = None + if verify_locally and bundle.chunks: + try: + from arborist.qa.verify import verify_quotes + context = "\n\n".join(c["body"] for c in bundle.chunks) + local = verify_quotes(bundle.answer_text, context) + local_audit_mode = local.get("audit_mode") + local_n_verified = local.get("n_verified") + local_verifier_method = local.get("verifier_method") + except Exception: + # Verifier-import failure is non-fatal — the Merkle + # proof already passed, so the wallet has the strong + # guarantee. Local verification is an additive check. + pass + return VerifiedAnswer( answer_text=bundle.answer_text, audit_mode=bundle.audit_mode, @@ -121,4 +159,7 @@ class WalletClient: snapshot_root=bundle.snapshot_root, sources=bundle.sources, n_chunks_verified=len(bundle.chunks), + local_audit_mode=local_audit_mode, + local_n_verified=local_n_verified, + local_verifier_method=local_verifier_method, ) diff --git a/tests/test_wallet_spv.py b/tests/test_wallet_spv.py index 08409fe..2f9437c 100644 --- a/tests/test_wallet_spv.py +++ b/tests/test_wallet_spv.py @@ -221,6 +221,48 @@ def test_server_healthz_and_snapshot_root(corpus_db: Path, tmp_path: Path): httpd.shutdown() +def test_client_local_verify_quotes_runs_on_authenticated_chunks( + corpus_db: Path, tmp_path: Path +): + """verify_locally=True re-grounds the answer on the bundle's chunks + (which are now cryptographically authenticated). A verbatim quote + answer reaches STRICT locally; a fabricated answer falls to + UNGROUNDED locally even if the server claims STRICT — that's the + 'wallet doesn't have to trust the server's verifier' property. + """ + from arborist.wallet.client import WalletClient + from arborist.wallet.server import WalletServer, serve_in_thread + + verbatim = ( + '"eight forms of capital include living, social, and intellectual"' + ) + + def _factory(): + return StubClient(answer=f"According to the source: {verbatim}.") + + wallet_server = WalletServer( + qa_db=tmp_path / "qa.db", + single_db=corpus_db, + chat_client_factory=_factory, + ) + httpd, _t, base_url = serve_in_thread(server=wallet_server, port=0) + try: + conn = connect(corpus_db) + try: + snap, _ = compute_snapshot_root(conn) + finally: + conn.close() + client = WalletClient(server_url=base_url, trust_anchor=snap) + verified = client.ask("What are the forms of capital?") + # Local verifier ran and reached STRICT on the authenticated chunks. + assert verified.local_audit_mode == "STRICT" + assert verified.local_n_verified is not None + assert verified.local_n_verified >= 1 + assert verified.local_verifier_method in {"quote", "span", "entity", "paraphrase"} + finally: + httpd.shutdown() + + def test_server_ask_end_to_end_returns_verifiable_bundle( corpus_db: Path, tmp_path: Path ):