From d3168fa236b625fb5ec289df4a1e67f847dff741 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 29 Apr 2026 10:29:06 -0400 Subject: [PATCH] =?UTF-8?q?qa:=20'aborist=20inspect'=20=E2=80=94=20sidecar?= =?UTF-8?q?=20diagnose=20for=20unverified=20spans?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per fox: the verifier reports n/N verified but doesn't tell you WHY a span didn't ground. Some are paraphrase, some are model-added trailing artifacts (e.g. `(Source: https://...)` citations the model appended to verbatim prose), some are full inventions. Operator needs to see which is which to triage. Read-only sidecar — no audit events, no providence_cache mutations, no v9.8 field changes. Per the verifier-no-diagnostics memory rule: hard chain stays binary; soft signals live in sidecar verbs that never feed back. Five diagnosis labels: verbatim_in_base span IS in base context — verifier or canonicalization bug (worth flagging loudly). verbatim_in_raw_only raw wikitext match but base form differs — wikitext-strip edge case. trailing_artifact a >=60-char prefix matches; the tail (often a model-added citation) doesn't. Surfaces the tail explicitly. paraphrase >=85% query tokens (>4 chars) present in base context but not in this sequence. Model rewrote source content. partial_paraphrase 40-85% coverage — mixed sourced/emergent. no_overlap <40% coverage — likely full invention. Verified live against the real Pikachu cache record (5 sources, 71KB raw -> 34KB base): two spans flagged paraphrase (token_coverage 1.0 for both, just rewritten sequence), one flagged trailing_artifact (100-char prefix matches; tail is `(Source: https://...)` citation). That's the actual reason HYBRID — model paraphrased + appended citations that aren't in the corpus. Surface: aborist inspect --cache-key [--qa-db ...] [--json] make inspect KEY= [JSON=1] Tests: 9 (6 classifier on synthetic contexts pinning each label, 3 end-to-end with seeded fixture record + verification that inspect writes nothing). 318 passed, 1 skipped overall. --- Makefile | 8 +- aborist/cli.py | 121 +++++++++++++++++++ aborist/qa/inspect.py | 211 ++++++++++++++++++++++++++++++++ tests/test_inspect.py | 275 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 613 insertions(+), 2 deletions(-) create mode 100644 aborist/qa/inspect.py create mode 100644 tests/test_inspect.py diff --git a/Makefile b/Makefile index aa9ff63..be4c39a 100644 --- a/Makefile +++ b/Makefile @@ -31,8 +31,8 @@ SEARCH_Q ?= computer ingest-grok ingest-grok-media \ ingest-self ingest-git ingest-hg \ verify search stats test docs chain-check chain-check-shards \ - falsify burn bootstrap-crawler test-crawler crawl-ingest recrawl-check \ - clean clean-db clean-data help + falsify burn inspect bootstrap-crawler test-crawler crawl-ingest \ + recrawl-check clean clean-db clean-data help all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats @@ -120,6 +120,10 @@ ACTIVITY_LIMIT ?= 10 activity: bootstrap ## recent Q&A + freshly cached docs (agent timeline) $(ABORIST) --shards-dir $(SHARDS_DIR) activity --limit $(ACTIVITY_LIMIT) +inspect: bootstrap ## sidecar diagnose unverified spans for a cache_key: make inspect KEY=hex [JSON=1] + @if [ -z "$(KEY)" ]; then echo "usage: make inspect KEY= [JSON=1]" >&2; exit 2; fi + $(ABORIST) --shards-dir $(SHARDS_DIR) inspect --cache-key $(KEY) $(if $(JSON),--json,) + falsify: bootstrap ## mark a cached answer wrong: make falsify KEY=hex REASON='why' @if [ -z "$(KEY)" ]; then echo "usage: make falsify KEY= REASON='why'" >&2; exit 2; fi $(ABORIST) --shards-dir $(SHARDS_DIR) providence --falsify $(KEY) --reason "$(REASON)" diff --git a/aborist/cli.py b/aborist/cli.py index e03214a..fad4d44 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -454,6 +454,104 @@ def _short_path(uri: str) -> str: return s +def _cmd_inspect(args: argparse.Namespace) -> int: + """Sidecar diagnostic — pulls source chunks for a cache_key and + classifies each unverified span. Read-only; no audit events, no + providence_cache mutations. + """ + from aborist.qa.inspect import inspect_cache_key + + qa_db = args.qa_db + if qa_db is None: + qa_db = ( + Path(args.global_shards_dir) / "qa.db" + if args.global_shards_dir + else Path.home() / ".aborist" / "qa.db" + ) + shards_dir = ( + Path(args.global_shards_dir) if args.global_shards_dir else None + ) + single_db = None if shards_dir else Path(args.db) if args.db else None + + result = inspect_cache_key( + args.cache_key, + qa_db=Path(qa_db), + shards_dir=shards_dir, + single_db=single_db, + ) + + if args.json: + print(json.dumps(result, indent=2, ensure_ascii=False)) + else: + print(_render_inspect_human(result)) + return 0 if result.get("status") == "ok" else 1 + + +def _render_inspect_human(result: dict) -> str: + """Pretty-print an inspect result so an operator can scan + paraphrase vs invention vs trailing-artifact at a glance.""" + if result.get("status") != "ok": + return f" {result.get('status', 'error')}: cache_key={result.get('cache_key', '?')}" + + rec = result["record"] + ctx = result["context"] + sources = result["sources"] + diagnoses = result["unverified"] + + lines: list[str] = [] + lines.append(rec["question_text"]) + lines.append( + f" {rec['audit_mode']} {rec['n_verified']}/{rec['n_quotes']} verified " + f"via {rec['verifier_method']} state={rec['falsification_state']}" + ) + lines.append("") + lines.append( + f"context: {ctx['raw_chars']:,} raw -> {ctx['base_chars']:,} base " + f"(wikitext-strip {'on' if ctx['wikitext_strip_active'] else 'off'})" + ) + if sources: + lines.append(f"sources ({len(sources)}):") + for i, s in enumerate(sources, start=1): + lines.append( + f" [{i}] {s.get('title') or '(untitled)'} — " + f"{s.get('chunk_count', '?')} chunks, " + f"{s.get('raw_chars', 0):,} chars" + ) + lines.append("") + + if not diagnoses: + lines.append("(no unverified spans)") + return "\n".join(lines) + + lines.append(f"unverified diagnoses ({len(diagnoses)}):") + for i, d in enumerate(diagnoses, start=1): + span = d.get("span", "") + diag = d.get("diagnosis", "?") + lines.append("") + lines.append(f" [{i}] {diag}") + lines.append(f" span: {_short(span, 140)}") + if diag == "trailing_artifact": + lines.append(f" matched_prefix_chars: {d.get('matched_prefix_chars')}") + lines.append(f" trailing_artifact: {_short(d.get('trailing_artifact', ''), 100)}") + elif diag in ("paraphrase", "partial_paraphrase"): + lines.append(f" token_coverage: {d.get('token_coverage')}") + counts = d.get("token_counts", {}) + if counts: + top = ", ".join(f"{k}×{v}" for k, v in list(counts.items())[:6]) + lines.append(f" tokens_in_base: {top}") + missing = d.get("missing_tokens", []) + if missing: + lines.append(f" missing_tokens: {missing[:8]}") + elif diag == "no_overlap": + lines.append(f" tokens_checked: {d.get('tokens_checked', '?')}") + lines.append(f" tokens_present: {d.get('tokens_present', '?')}") + return "\n".join(lines) + + +def _short(s: str, n: int) -> str: + return s if len(s) <= n else s[: n - 3] + "..." + + def _falsify_cache_key( cache_key_value: str, *, @@ -2535,6 +2633,29 @@ def build_parser() -> argparse.ArgumentParser: ) query_cmd.set_defaults(func=_cmd_query) + inspect_cmd = sub.add_parser( + "inspect", + help=( + "sidecar diagnostic for a providence_cache record — pulls " + "source chunks and classifies each unverified span " + "(paraphrase / trailing_artifact / no_overlap). Read-only, " + "no audit events, no v9.8 field changes." + ), + ) + inspect_cmd.add_argument( + "--cache-key", dest="cache_key", required=True, + help="64-char hex cache_key of the providence record to inspect", + ) + inspect_cmd.add_argument( + "--qa-db", dest="qa_db", default=None, + help="path to qa.db (default: /qa.db or ~/.aborist/qa.db)", + ) + inspect_cmd.add_argument( + "--json", action="store_true", + help="emit raw diagnosis as JSON (default: human render)", + ) + inspect_cmd.set_defaults(func=_cmd_inspect) + prov_cmd = sub.add_parser( "providence", help="list or falsify providence_cache records", diff --git a/aborist/qa/inspect.py b/aborist/qa/inspect.py new file mode 100644 index 0000000..669a5a3 --- /dev/null +++ b/aborist/qa/inspect.py @@ -0,0 +1,211 @@ +"""Sidecar diagnostic for `providence_cache` records. + +Read-only inspection tool. Pulls the same context the verifier saw, +runs the same canonicalization (norm-v1 + lowercase + wikitext strip), +and classifies each ``unverified_quote`` against the corpus so an +operator can tell paraphrase from invention from formatting wart. + +By design this writes nothing — no audit events, no providence_cache +mutations, no v9.8 field changes. The verifier's binary output stays +authoritative; this tool is a debugging lens. See the memory rule +``feedback_verifier_no_diagnostics``: soft signals stay in sidecar +verbs, never feed back into the hard chain. +""" + +from __future__ import annotations + +import json +import sqlite3 +import unicodedata +from pathlib import Path +from typing import Any + +from aborist.compress import unpack_chunk +from aborist.store import connect + +try: + from aborist.wikitext import to_base as _wikitext_to_base +except ImportError: # pragma: no cover + _wikitext_to_base = None + + +def _normalize(s: str) -> str: + """Same as aborist.qa.verify._normalize — kept local to avoid an + import cycle and to make the diagnostic self-contained.""" + s = unicodedata.normalize("NFC", s) + s = " ".join(s.split()) + return s.lower() + + +def _classify_span(span: str, norm_base_ctx: str, norm_raw_ctx: str) -> dict[str, Any]: + """Classify one unverified span against the canonical context. + + Returns one of: + + - ``verbatim_in_base``: substring match (verifier should have caught + this — likely a verifier or canonicalization bug). + - ``verbatim_in_raw_only``: matches raw wikitext but not the base + form (wikitext-strip edge case). + - ``trailing_artifact``: a prefix of length >= 60 chars matches; the + tail (often a model-added citation like ``(Source: ...)``) doesn't. + - ``paraphrase``: query tokens (>4 chars) all present in base context + but not in this sequence — model rewrote source content. + - ``partial_paraphrase``: some query tokens present, others missing. + - ``no_overlap``: most query tokens missing — likely full invention. + """ + nspan = _normalize(span) + if nspan in norm_base_ctx: + return {"diagnosis": "verbatim_in_base"} + if nspan in norm_raw_ctx: + return {"diagnosis": "verbatim_in_raw_only"} + + # Trailing-artifact probe: sweep prefix lengths from longest down, + # find the longest prefix that DOES match. If >= 60 chars matched, + # the tail is the artifact. + longest_prefix_match = 0 + for plen in (len(nspan) - 5, 200, 150, 120, 100, 80, 60): + if plen <= 0 or plen > len(nspan): + continue + if nspan[:plen] in norm_base_ctx: + longest_prefix_match = plen + break + if longest_prefix_match >= 60: + tail = span[longest_prefix_match:] + return { + "diagnosis": "trailing_artifact", + "matched_prefix_chars": longest_prefix_match, + "trailing_artifact": tail.strip(), + } + + # Token coverage: how many >4-char tokens from the span appear at all + # in the base context. + tokens = [t for t in nspan.split() if len(t) > 4] + if not tokens: + return {"diagnosis": "no_overlap", "tokens_checked": 0} + present = [t for t in tokens if t in norm_base_ctx] + coverage = len(present) / len(tokens) + counts = {t: norm_base_ctx.count(t) for t in present} + + if coverage >= 0.85: + return { + "diagnosis": "paraphrase", + "token_coverage": round(coverage, 2), + "token_counts": counts, + } + if coverage >= 0.4: + return { + "diagnosis": "partial_paraphrase", + "token_coverage": round(coverage, 2), + "token_counts": counts, + "missing_tokens": [t for t in tokens if t not in norm_base_ctx], + } + return { + "diagnosis": "no_overlap", + "token_coverage": round(coverage, 2), + "tokens_checked": len(tokens), + "tokens_present": len(present), + } + + +def inspect_cache_key( + cache_key: str, + *, + qa_db: Path, + shards_dir: Path | None, + single_db: Path | None = None, +) -> dict[str, Any]: + """Pull the cache record + sources + classify each unverified span. + + Returns a dict containing: + - record: the providence_cache row (subset) + - sources: per-source size + chunk count + - context: char counts for raw and base forms + - unverified: per-span diagnosis (list) + """ + conn = connect(qa_db) + try: + row = conn.execute( + "SELECT cache_key, source_root, document_uri, question_text, " + " answer_text, audit_mode, n_quotes, n_verified, " + " unverified_quotes, verifier_method, merkle_proof, " + " falsification_state, created_at " + "FROM providence_cache WHERE cache_key = ?", + (cache_key,), + ).fetchone() + finally: + conn.close() + if row is None: + return {"status": "not_found", "cache_key": cache_key} + + proof = json.loads(row["merkle_proof"]) if row["merkle_proof"] else {} + sources = proof.get("sources") or [] + unverified_list = ( + json.loads(row["unverified_quotes"]) if row["unverified_quotes"] else [] + ) + + parts_raw: list[str] = [] + parts_base: list[str] = [] + src_summary: list[dict[str, Any]] = [] + for s in sources: + shard_path = ( + shards_dir / s["shard"] if (shards_dir and s.get("shard")) + else single_db + ) + if shard_path is None: + continue + sc = sqlite3.connect(str(shard_path)) + sc.row_factory = sqlite3.Row + try: + rows = sc.execute( + "SELECT idx, content FROM chunks " + "WHERE document_root = ? AND tier = 'hot' ORDER BY idx", + (s["document_root"],), + ).fetchall() + finally: + sc.close() + text = "\n\n".join(unpack_chunk(r["content"]) or "" for r in rows) + parts_raw.append(text) + if _wikitext_to_base is not None: + parts_base.append(_wikitext_to_base(text)) + else: + parts_base.append(text) + src_summary.append( + { + "document_uri": s.get("document_uri"), + "title": s.get("title"), + "shard": s.get("shard"), + "chunk_count": len(rows), + "raw_chars": len(text), + } + ) + + full_raw = "\n\n".join(parts_raw) + full_base = "\n\n".join(parts_base) + n_raw = _normalize(full_raw) + n_base = _normalize(full_base) + + diagnoses: list[dict[str, Any]] = [] + for q in unverified_list: + d = _classify_span(q, norm_base_ctx=n_base, norm_raw_ctx=n_raw) + diagnoses.append({"span": q, **d}) + + return { + "status": "ok", + "record": { + "cache_key": row["cache_key"], + "question_text": row["question_text"], + "audit_mode": row["audit_mode"], + "n_quotes": row["n_quotes"], + "n_verified": row["n_verified"], + "verifier_method": row["verifier_method"], + "falsification_state": row["falsification_state"], + "created_at": row["created_at"], + }, + "sources": src_summary, + "context": { + "raw_chars": len(full_raw), + "base_chars": len(full_base), + "wikitext_strip_active": _wikitext_to_base is not None, + }, + "unverified": diagnoses, + } diff --git a/tests/test_inspect.py b/tests/test_inspect.py new file mode 100644 index 0000000..1fa9d3d --- /dev/null +++ b/tests/test_inspect.py @@ -0,0 +1,275 @@ +"""Sidecar diagnostic for unverified spans. + +`aborist inspect --cache-key X` reads a providence_cache record, pulls +the same source chunks the verifier saw, and classifies each +unverified_quote into one of: verbatim_in_base, verbatim_in_raw_only, +trailing_artifact, paraphrase, partial_paraphrase, no_overlap. + +These tests pin the classifier on synthetic contexts so the diagnoses +are deterministic. End-to-end coverage of `inspect_cache_key` (DB +plumbing) goes through a small fixture record below. +""" + +from __future__ import annotations + +import json +import time +from pathlib import Path + +import pytest + +from aborist.qa.inspect import _classify_span, _normalize, inspect_cache_key +from aborist.store import append_audit, connect, transaction + + +# --------------------------------------------------------------------------- +# _classify_span — pure function, no DB +# --------------------------------------------------------------------------- + + +def _norm(s: str) -> str: + return _normalize(s) + + +def test_classify_verbatim_in_base(): + """If the verifier flagged a span but it IS in the base context, the + diagnosis surfaces that as a likely verifier or canonicalization + bug. Deliberately distinct label so an operator notices.""" + span = "the quick brown fox" + base = "lorem ipsum the quick brown fox jumps over" + raw = base + out = _classify_span(span, _norm(base), _norm(raw)) + assert out["diagnosis"] == "verbatim_in_base" + + +def test_classify_verbatim_in_raw_only(): + """Span matches raw wikitext but base form differs — wikitext-strip + edge case worth flagging separately.""" + span = "[[Cloud Strife]] is the protagonist" + base = "Cloud Strife is the protagonist" # wikitext stripped + raw = "[[Cloud Strife]] is the protagonist" + out = _classify_span(span, _norm(base), _norm(raw)) + assert out["diagnosis"] == "verbatim_in_raw_only" + + +def test_classify_trailing_artifact_citation_appended(): + """The Pikachu case — model appended `(Source: ...)` to a verbatim + sentence; verifier flagged the whole thing. inspect should isolate + the artifact.""" + base = ( + "Pikachu can store electricity in its cheeks and release it in " + "lightning-based attacks. Pikachu evolves from Pichu." + ) + span = ( + "Pikachu can store electricity in its cheeks and release it in " + "lightning-based attacks. (Source: https://en.wikipedia.org/wiki/Pikachu)" + ) + out = _classify_span(span, _norm(base), _norm(base)) + assert out["diagnosis"] == "trailing_artifact" + assert out["matched_prefix_chars"] >= 60 + assert "Source:" in out["trailing_artifact"] + + +def test_classify_paraphrase_high_token_coverage(): + """Tokens all present, sequence different — model rewrote the source.""" + base = ( + "Pikachu is a Pokémon species in the Pokémon franchise developed by " + "Game Freak and Nintendo. Pokémon games feature numerous creatures." + ) + span = "Pikachu is a species of Pokémon creatures from the Pokémon franchise" + out = _classify_span(span, _norm(base), _norm(base)) + assert out["diagnosis"] == "paraphrase" + assert out["token_coverage"] >= 0.85 + + +def test_classify_partial_paraphrase_some_tokens_missing(): + """Some content overlap but a meaningful token isn't there at all — + likely model invention mixed with corpus tokens.""" + base = "Pikachu is a Pokémon. Pikachu battles other creatures." + span = "Pikachu invented the lightning attack in Tokyo on Tuesday" + out = _classify_span(span, _norm(base), _norm(base)) + assert out["diagnosis"] in ("partial_paraphrase", "no_overlap") + if out["diagnosis"] == "partial_paraphrase": + assert "missing_tokens" in out + + +def test_classify_no_overlap_full_invention(): + """Almost no content tokens shared with the corpus — pure model output.""" + base = "Pikachu is a Pokémon species." + span = "The Roman Senate convened in 49 BC to debate Caesar's rebellion" + out = _classify_span(span, _norm(base), _norm(base)) + assert out["diagnosis"] == "no_overlap" + + +# --------------------------------------------------------------------------- +# inspect_cache_key — end-to-end with a tiny fixture +# --------------------------------------------------------------------------- + + +def _seed_record(qa_db: Path, *, cache_key: str, sources: list[dict], + unverified: list[str]) -> None: + """Insert a minimal providence_cache row with a merkle_proof that + points at the given sources. Caller has already populated each + shard with the actual document + chunks.""" + conn = connect(qa_db) + try: + with transaction(conn): + event_hash = append_audit( + conn, + event_type="providence_query", + subject_root=cache_key, + body={"cache_key": cache_key}, + ) + conn.execute( + "INSERT INTO providence_cache " + "(cache_key, source_root, document_uri, question_hash, " + " question_text, answer_text, merkle_proof, " + " model_profile_hash, conversation_hash, " + " governance_policy_hash, schema_version, " + " canonicalization_version, chunking_version, " + " falsification_state, chain, audit_event_hash, " + " created_at, hit_count, audit_mode, n_quotes, " + " n_verified, unverified_quotes, verifier_method) " + "VALUES (?, ?, 'corpus://multi', ?, ?, ?, ?, ?, ?, ?, " + " ?, ?, ?, 'live', 'private', ?, ?, 0, 'HYBRID', ?, ?, ?, 'span')", + ( + cache_key, + "00" * 32, + "qh", + "what is foo?", + "answer here", + json.dumps({"sources": sources}, ensure_ascii=False), + "mh", + "ch", + "gh", + "v9.8.0", + "norm-v1", + "tok-512-v1", + event_hash, + int(time.time()), + len(unverified) + 1, # n_quotes (one verified plus N unverified) + 1, # n_verified + json.dumps(unverified, ensure_ascii=False), + ), + ) + finally: + conn.close() + + +def _seed_doc(shard: Path, *, document_root: str, document_uri: str, + chunk_text: str) -> None: + """Insert a documents row + one hot chunk with the given text.""" + conn = connect(shard) + try: + from aborist.compress import pack_chunk + + with transaction(conn): + conn.execute( + "INSERT INTO documents " + "(document_root, document_uri, source_type, kind, " + " compression_depth, chunking_version, " + " canonicalization_version, schema_version, ingest_ts, hit_count) " + "VALUES (?, ?, 'html', 'surface', 0, 'tok-512-v1', " + " 'norm-v1', 'v9.8.0', ?, 0)", + (document_root, document_uri, int(time.time())), + ) + conn.execute( + "INSERT INTO chunks (document_root, idx, leaf_hash, content, tier) " + "VALUES (?, 0, ?, ?, 'hot')", + (document_root, "ff" * 32, pack_chunk(chunk_text)), + ) + finally: + conn.close() + + +def test_inspect_cache_key_classifies_each_unverified(tmp_path): + qa_db = tmp_path / "qa.db" + shard = tmp_path / "001.db" + DOC = "ab" * 32 + _seed_doc( + shard, + document_root=DOC, + document_uri="https://example.com/x", + chunk_text=( + "Pikachu can store electricity in its cheeks. " + "Pikachu evolves from Pichu." + ), + ) + _seed_record( + qa_db, + cache_key="01" * 32, + sources=[{ + "document_root": DOC, + "document_uri": "https://example.com/x", + "title": "X", + "shard": shard.name, + "chunk_idx": 0, + }], + unverified=[ + "Pikachu can store electricity in its cheeks.", # verbatim + "Pikachu fought in the 1988 Olympics in Seoul", # invention + ], + ) + + result = inspect_cache_key( + "01" * 32, qa_db=qa_db, shards_dir=tmp_path + ) + assert result["status"] == "ok" + assert result["record"]["cache_key"] == "01" * 32 + assert len(result["sources"]) == 1 + assert result["sources"][0]["chunk_count"] == 1 + assert len(result["unverified"]) == 2 + diags = [d["diagnosis"] for d in result["unverified"]] + assert "verbatim_in_base" in diags + # The Olympics span should be flagged as no_overlap or partial_paraphrase + # depending on token coincidences — both honest classifications. + assert any(d in ("no_overlap", "partial_paraphrase") for d in diags) + + +def test_inspect_cache_key_unknown_returns_not_found(tmp_path): + qa_db = tmp_path / "qa.db" + # touch DB so connect() initialises schema + connect(qa_db).close() + result = inspect_cache_key( + "00" * 32, qa_db=qa_db, shards_dir=tmp_path + ) + assert result["status"] == "not_found" + + +def test_inspect_does_not_mutate_state(tmp_path): + """Sidecar invariant: inspect must not write to providence_cache, + audit_events, or anything else. Pin so a future refactor can't + silently add side effects.""" + qa_db = tmp_path / "qa.db" + shard = tmp_path / "001.db" + DOC = "cd" * 32 + _seed_doc( + shard, document_root=DOC, document_uri="https://example.com/y", + chunk_text="hello world", + ) + _seed_record( + qa_db, cache_key="02" * 32, + sources=[{ + "document_root": DOC, "document_uri": "https://example.com/y", + "title": "Y", "shard": shard.name, "chunk_idx": 0, + }], + unverified=["something the model said"], + ) + + conn = connect(qa_db) + try: + before_audit = conn.execute("SELECT COUNT(*) FROM audit_events").fetchone()[0] + before_prov = conn.execute("SELECT COUNT(*) FROM providence_cache").fetchone()[0] + finally: + conn.close() + + inspect_cache_key("02" * 32, qa_db=qa_db, shards_dir=tmp_path) + + conn = connect(qa_db) + try: + after_audit = conn.execute("SELECT COUNT(*) FROM audit_events").fetchone()[0] + after_prov = conn.execute("SELECT COUNT(*) FROM providence_cache").fetchone()[0] + finally: + conn.close() + assert after_audit == before_audit + assert after_prov == before_prov