From 01a439069330cb799c70ad3c6031dc0b3f7ed7f2 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 28 Apr 2026 16:58:31 -0400 Subject: [PATCH] =?UTF-8?q?verify:=20layered=20strategies=20+=20entity=20p?= =?UTF-8?q?olicies,=20rename=20VISUAL=20=E2=86=92=20UNGROUNDED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - aborist/qa/verify.py: three strategies tried in sequence. quote uses sequential pairing (1st & 2nd `"`, 3rd & 4th, ...) which eliminates the phantom inter-pair captures regex pairing produced on adjacent quote pairs like `"title" prose "quote"`. span checks bullet/sentence lines verbatim. entity matches multi-word proper nouns; gated by policy ∈ {strict, hybrid, drop, proximity}. Default proximity promotes to STRICT only when N=3 verified entities cluster within W=300 chars in source — separates structural grounding (cast list, infobox) from incidental mention (scattered plot summary). - aborist/store.py: schema CHECK now `('STRICT','HYBRID','UNGROUNDED')`. Migration helper rebuilds legacy `('STRICT','HYBRID','VISUAL')` tables via temp-table copy, translating VISUAL → UNGROUNDED in the SELECT. Idempotent — DDL inspection skips the rebuild on already- migrated DBs. - aborist/cli.py: new `aborist reclassify` re-runs the verifier against existing live records under the current entity policy; no LLM calls. --compare runs all four policies side-by-side, --dry-run reports transitions without writing. Cold-source records skipped. One providence_reclassify audit event per changed row. - AuditMode.UNGROUNDED replaces VISUAL across search backend, FTS5, test fixtures, CLAUDE.md. The substrate (Merkle-AGI) name was about FOR-style visualization; in the RAG layer the semantic is "no recoverable grounding," so the label now says that. DEFAULT_QUERY_POLICY gains entity_policy + entity_proximity_n + entity_proximity_window so any tuning folds into governance_policy_hash and invalidates cache cleanly. --- CLAUDE.md | 17 +++-- aborist/cli.py | 12 ++-- aborist/qa/query.py | 2 +- aborist/qa/runner.py | 2 +- aborist/qa/verify.py | 25 +++---- aborist/search/base.py | 19 +++-- aborist/search/fts5.py | 4 +- aborist/store.py | 109 +++++++++++++++++++++++++---- tests/test_ingest.py | 4 +- tests/test_migration_audit_mode.py | 4 +- tests/test_reclassify.py | 12 ++-- tests/test_verify.py | 12 ++-- tests/test_wikitext.py | 2 +- 13 files changed, 160 insertions(+), 64 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6910e02..6192038 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,7 +12,7 @@ Three layers stacked on one SQLite file: 1. **Surface** — ingested documents (Wikipedia dumps, HTML pages, anything with a URI). Chunked, Merkle-rooted, FTS5-indexed. 2. **Core** — distilled documents (haiku/keyword/equation-scale) Merkle-bound back to source surface(s) via per-chunk inclusion proofs in `derivations.proof_blob`. Recursive: cores derive from cores. The "planet toward center" compression. -3. **Providence cache** — Q&A records keyed on the v9.8 8-dim invariant. Each record carries an `audit_mode` set by the post-LLM faithfulness verifier (`aborist/qa/verify.py`): STRICT (every quoted claim verbatim-grounded), HYBRID (mixed source/emergent), VISUAL (no verbatim grounding — purely emergent from training). +3. **Providence cache** — Q&A records keyed on the v9.8 8-dim invariant. Each record carries an `audit_mode` set by the post-LLM faithfulness verifier (`aborist/qa/verify.py`): STRICT (every quoted claim verbatim-grounded), HYBRID (mixed source/emergent), UNGROUNDED (no verbatim grounding — purely emergent from training). ## Source papers (read first if confused) @@ -44,11 +44,12 @@ aborist/ ├── qa/ # Q&A: 8-dim cache_key + Merkle-bound answers │ ├── client.py # ChatClient + StubClient + OpenAICompat │ ├── keys.py # cache_key, question_hash, ... (pure functions) -│ ├── verify.py # verify_quotes(): post-LLM faithfulness check +│ ├── verify.py # layered verifier: quote → span → entity │ └── runner.py # ask(): cache -> infer -> verify -> classify -> write +├── wikitext.py # to_base(): wikitext → plain prose (BASE_VERSION-pinned) └── cli.py # ingest / search / verify / stats / distill / # evict / rehydrate / ask / providence / emergent / - # analyze + # reclassify / analyze ``` ## Build, test, run @@ -66,7 +67,7 @@ make verify # round-trip Merkle proofs on a random sample make analyze # compression spectrum + audit chain integrity ``` -`aborist/cli.py` adds: `analyze`, `distill --kind {surface,core}`, `evict`, `rehydrate`, `ask`, `providence`, `emergent` (list VISUAL/HYBRID records or `--aggregate` to rank unverified quotes — corpus-growth signal). `--batch-size` defaults to 200 docs/transaction; lower it only to bound memory peaks. +`aborist/cli.py` adds: `analyze`, `distill --kind {surface,core}`, `evict`, `rehydrate`, `ask`, `providence`, `emergent` (list UNGROUNDED/HYBRID records or `--aggregate` to rank unverified quotes — corpus-growth signal), `reclassify` (re-run the verifier against existing live providence records under the current entity policy; no LLM calls; `--compare` runs all four policies side-by-side, `--dry-run` reports without writing). `--batch-size` defaults to 200 docs/transaction; lower it only to bound memory peaks. ## Schema invariants (do not break) @@ -82,7 +83,13 @@ make analyze # compression spectrum + audit chain integrity - **Chunker default = `tok-512-v1`.** Changing the default bumps `chunking_version` and **stales every prior cache record**. Add a new chunker as a new `name` instead. - **Canonicalization = `norm-v1`** (NFC, collapsed whitespace). Same rule. - **Schema = `v9.8.0`.** Same rule. -- **`audit_mode` is decided by the verifier, never asserted unconditionally.** Trichotomy: STRICT = every double-quoted span in the answer matches verbatim against `context` (post-LLM, lexical, NFC + collapsed-ws + lowercase). HYBRID = some quotes verify, some don't (the model mixed source with training). VISUAL = no quotes, or none verify (raw FTS5 hits, or an answer that emerged from training alone). Persisted on `providence_cache.audit_mode`; cache-hits return the stored mode. Never overclaim — STRICT is a verifiable claim, not a default. +- **`audit_mode` is decided by the verifier, never asserted unconditionally.** Three layered strategies in `aborist/qa/verify.py`, tried in order; first to find evidence classifies the answer: + 1. **quote** — model wrapped claims in double quotes per system prompt. Sequential pairing: 1st & 2nd `"`, 3rd & 4th, etc. (NOT regex pairing — that captures inter-pair prose as a phantom span when the model writes `"title" prose "quote"`). + 2. **span** — bullet/sentence lines from the answer appear verbatim in context. Catches models that quote inline without `"..."` marks. + 3. **entity** — multi-word proper-noun phrases appear verbatim in context. Gated by `entity_policy ∈ {strict, hybrid, drop, proximity}`. Default `proximity`: STRICT only when N=3 verified entities cluster within W=300 chars in source (cast list / infobox / roster). Otherwise HYBRID/UNGROUNDED. Distinguishes structural grounding from incidental mention. Lives in `DEFAULT_QUERY_POLICY["entity_policy"]` so any change bumps `governance_policy_hash`. + Trichotomy across all paths: STRICT = every evidence unit (≥1) verifies. HYBRID = some verify, some don't. UNGROUNDED = no evidence or none verifies. Persisted on `providence_cache.audit_mode` + `verifier_method`; cache-hits return the stored mode. Never overclaim — STRICT is a verifiable claim, not a default. +- **Verifier stays binary; falsifications carry soft signal.** No per-quote diagnosis fields on hard verifier output. `verify_quotes` returns evidence units + classification; the falsify+reclassify loop owns "why didn't this ground" for the operator. Don't bolt confidence scores or partial-match indicators onto `verify.py`. +- **Wikitext context strip.** `aborist/wikitext.py:to_base(raw)` converts MediaWiki wikitext → plain prose deterministically (mwparserfromhell-backed; pinned by `BASE_VERSION`). Applied to context inside `verify_quotes` before substring tests so a model's clean prose ("Stephen King") matches against `[[Stephen King]]`-style source markup. Optional dep — falls back to identity if mwparserfromhell isn't installed. Bumping `BASE_VERSION` should fold into `governance_policy_hash` via `policy["base_version"]` to invalidate cache. - **Soft hash vs hard hash.** Hard = SHA-256 (commitments, proofs, cache_key). Soft = embeddings/TF-IDF/similarity (training, ranking, distillation candidate selection). Never mix — soft never enters proof path. ## Live endpoints diff --git a/aborist/cli.py b/aborist/cli.py index e66d2b9..4307adb 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -720,7 +720,7 @@ def _cmd_reclassify(args: argparse.Namespace) -> int: def _cmd_emergent(args: argparse.Namespace) -> int: - """Surface emergent claims from VISUAL/HYBRID providence records. + """Surface emergent claims from UNGROUNDED/HYBRID providence records. These are spans the model produced that don't appear verbatim in the corpus — candidate ingest targets. Frequent unverified quotes signal @@ -735,7 +735,7 @@ def _cmd_emergent(args: argparse.Namespace) -> int: if args.aggregate: rows = conn.execute( "SELECT unverified_quotes FROM providence_cache " - "WHERE audit_mode IN ('VISUAL','HYBRID') " + "WHERE audit_mode IN ('UNGROUNDED','HYBRID') " " AND falsification_state = 'live' " " AND unverified_quotes IS NOT NULL" ).fetchall() @@ -753,7 +753,7 @@ def _cmd_emergent(args: argparse.Namespace) -> int: "SELECT cache_key, audit_mode, verifier_method, question_text, " " n_quotes, n_verified, unverified_quotes, created_at " "FROM providence_cache " - "WHERE audit_mode IN ('VISUAL','HYBRID') " + "WHERE audit_mode IN ('UNGROUNDED','HYBRID') " " AND falsification_state = 'live' " "ORDER BY created_at DESC LIMIT ?", (args.limit,), @@ -1617,7 +1617,7 @@ def build_parser() -> argparse.ArgumentParser: ) ingest.set_defaults(func=_cmd_ingest) - search = sub.add_parser("search", help="keyword search (VISUAL audit mode)") + search = sub.add_parser("search", help="keyword search (UNGROUNDED audit mode)") search.add_argument("query", help="query string") search.add_argument("--limit", type=int, default=20) search.add_argument("--json", action="store_true", help="output JSON") @@ -1790,7 +1790,7 @@ def build_parser() -> argparse.ArgumentParser: help=( "how the entity path classifies: 'strict' (legacy, overclaims), " "'hybrid' (default — caps at HYBRID), 'drop' (skip entity path → " - "VISUAL), 'proximity' (STRICT only if N entities cluster within " + "UNGROUNDED), 'proximity' (STRICT only if N entities cluster within " "W chars in source)" ), ) @@ -1802,7 +1802,7 @@ def build_parser() -> argparse.ArgumentParser: emergent_cmd = sub.add_parser( "emergent", - help="surface VISUAL/HYBRID claims — corpus-growth signal", + help="surface UNGROUNDED/HYBRID claims — corpus-growth signal", ) emergent_cmd.add_argument( "--aggregate", diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 5d1c427..7f76d50 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -21,7 +21,7 @@ The flow: Result classifies the answer: STRICT every quote (>=1) verified against context HYBRID some claims sourced, some emergent (training-derived) - VISUAL no quotes verify — purely emergent + UNGROUNDED no quotes verify — purely emergent 7. Persist record with merkle_proof = {context_root, sources: [...]}, audit_mode, and unverified_quotes (the spans the model produced that didn't appear in any source — corpus-growth signal). diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 551a9cb..c06f915 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -4,7 +4,7 @@ Implements the v9.8 admissibility invariant: No record reused unless all 8 cache_key dimensions match AND state is 'live' (not failed/stale/quarantined). -Cache hit -> persisted audit_mode (STRICT/HYBRID/VISUAL). +Cache hit -> persisted audit_mode (STRICT/HYBRID/UNGROUNDED). Cache miss -> call ChatClient, run faithfulness check, classify, store record, audit event. """ diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index 9f4347f..ae8e070 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -15,11 +15,12 @@ fired so the audit chain stays diagnostic. paraphrases structure so spans diverge, but every named entity is intact and grounded. -Each strategy classifies into v9.8's audit-mode trichotomy: +Each strategy classifies into v9.8's audit-mode trichotomy (RAG-adapted +vocabulary; substrate calls UNGROUNDED "VISUAL"): - STRICT every evidence unit (>=1) verifies verbatim against context - HYBRID some verify, others do not (mixed source / emergent) - VISUAL no evidence, or none verify (purely emergent) + STRICT every evidence unit (>=1) verifies verbatim against context + HYBRID some verify, others do not (mixed source / emergent) + UNGROUNDED no evidence, or none verify (purely emergent) `unverified_quotes` (kept under that name for schema continuity) collects spans the model produced that don't appear in any source — the @@ -34,7 +35,7 @@ Wikitext context is run through ``aborist.wikitext.to_base`` before the substring test. The corpus stores raw wikitext (so the link graph is recoverable from any page), but the LLM produces clean prose. Without the strip, every wikilink-carrying source paragraph compares as -"different surface form" and the verifier wrongly reports VISUAL on +"different surface form" and the verifier wrongly reports UNGROUNDED on genuine source-grounded quotes. With the strip, paraphrases of *markup* (``[[Cloud]]`` vs ``Cloud``) verify, while paraphrases of *prose* still flag honestly. mwparserfromhell is an optional dep; if absent, the @@ -88,10 +89,10 @@ MIN_SPAN_CHARS = 12 # # strict all entities verify → STRICT (legacy behavior; overclaims) # hybrid any entity verifies → HYBRID (honest cap; safe default) -# drop skip entity path entirely → VISUAL (most conservative) +# drop skip entity path entirely → UNGROUNDED (most conservative) # proximity STRICT only if N verified entities cluster within W chars # of each other in context (e.g. an infobox cast list). -# Otherwise demotes to HYBRID/VISUAL based on partial match. +# Otherwise demotes to HYBRID/UNGROUNDED based on partial match. ENTITY_POLICIES = ("strict", "hybrid", "drop", "proximity") DEFAULT_ENTITY_POLICY = "proximity" @@ -196,7 +197,7 @@ def _classify(verified: list[str], unverified: list[str]) -> str: return "STRICT" if verified: return "HYBRID" - return "VISUAL" + return "UNGROUNDED" def _has_entity_cluster( @@ -259,7 +260,7 @@ def verify_quotes( { "n_quotes": int, # evidence units extracted (any path) "n_verified": int, # of those, how many appear verbatim - "audit_mode": str, # STRICT | HYBRID | VISUAL + "audit_mode": str, # STRICT | HYBRID | UNGROUNDED "unverified_quotes": [str], # spans we couldn't ground in context "verifier_method": str, # 'quote' | 'span' | 'entity' | 'none' } @@ -305,7 +306,7 @@ def verify_quotes( # `entity_policy`. Entity-existence is weaker proof than quote or span; # the operator chooses how much weight to give it. if entity_policy == "drop": - # Skip entity path entirely. Falls through to VISUAL/none. + # Skip entity path entirely. Falls through to UNGROUNDED/none. pass else: entities = extract_proper_nouns(answer_text) @@ -330,7 +331,7 @@ def verify_quotes( elif verified: mode = "HYBRID" else: - mode = "VISUAL" + mode = "UNGROUNDED" return { "n_quotes": len(entities), "n_verified": len(verified), @@ -343,7 +344,7 @@ def verify_quotes( return { "n_quotes": 0, "n_verified": 0, - "audit_mode": "VISUAL", + "audit_mode": "UNGROUNDED", "unverified_quotes": [], "verifier_method": "none", } diff --git a/aborist/search/base.py b/aborist/search/base.py index a9238bd..4382c2d 100644 --- a/aborist/search/base.py +++ b/aborist/search/base.py @@ -1,11 +1,16 @@ """Search backend ABC + Hit dataclass with explicit audit mode. -Every search hit carries an `audit_mode` so callers never overclaim. Per -Merkle-AGI v7 §: -- STRICT — Merkle-verified, deterministic, full local reveal. Supports formal - claims (e.g., providence_cache hit with verified proof). -- HYBRID — random-challenge spot check (RCA(K)). Non-normative. -- VISUAL — exploration / debug only. No proof claim. Keyword search lives here. +Every search hit carries an `audit_mode` so callers never overclaim. Aborist +adapts the Merkle-AGI v7 audit-mode trichotomy to the RAG layer: +- STRICT — Merkle-verified evidence: every claim cited verbatim against + the source-content tree. +- HYBRID — partial / mixed evidence: some claims source-grounded, others + emerged from training. Cache or search hit is partially trusted. +- UNGROUNDED — no recoverable proof of grounding. Keyword (FTS5) hits land + here by default; LLM answers fall here when no double-quoted + span, sentence, or proper-noun phrase verifies against context. + Substrate name was VISUAL (no formal guarantees attached); we + renamed to UNGROUNDED so the RAG semantic is explicit. """ from __future__ import annotations @@ -19,7 +24,7 @@ from dataclasses import dataclass class AuditMode(str, enum.Enum): STRICT = "STRICT" HYBRID = "HYBRID" - VISUAL = "VISUAL" + UNGROUNDED = "UNGROUNDED" @dataclass(frozen=True) diff --git a/aborist/search/fts5.py b/aborist/search/fts5.py index acf1945..9fb7b6d 100644 --- a/aborist/search/fts5.py +++ b/aborist/search/fts5.py @@ -1,4 +1,4 @@ -"""SQLite FTS5 keyword search. Returns VISUAL-mode hits (no proof claim). +"""SQLite FTS5 keyword search. Returns UNGROUNDED-mode hits (no proof claim). Snippet generation runs in Python because chunks_fts is contentless (`content=''`) — SQLite's snippet()/highlight() functions return empty @@ -126,7 +126,7 @@ def _build_snippet(text: str, query: str) -> str: class FTS5Backend(SearchBackend): name = "fts5" - audit_mode = AuditMode.VISUAL + audit_mode = AuditMode.UNGROUNDED def search(self, query: str, limit: int = 20) -> list[Hit]: if not query.strip(): diff --git a/aborist/store.py b/aborist/store.py index 3f47b29..8972d1a 100644 --- a/aborist/store.py +++ b/aborist/store.py @@ -152,12 +152,13 @@ CREATE TABLE IF NOT EXISTS providence_cache ( created_at INTEGER NOT NULL, last_hit_at INTEGER, hit_count INTEGER NOT NULL DEFAULT 0, - -- v9.8 audit_mode trichotomy: STRICT (every quote in answer verified - -- against context), HYBRID (some claims verified, some emergent), - -- VISUAL (no verbatim grounding — purely emergent from training). - -- Default VISUAL: an unclassified record is the weakest claim. - audit_mode TEXT NOT NULL DEFAULT 'VISUAL' - CHECK (audit_mode IN ('STRICT','HYBRID','VISUAL')), + -- v9.8 audit_mode trichotomy (RAG-adapted vocabulary; substrate calls + -- UNGROUNDED "VISUAL"): STRICT (every quote in answer verified against + -- context), HYBRID (some claims verified, some emergent), UNGROUNDED + -- (no verbatim grounding — purely emergent from training). + -- Default UNGROUNDED: an unclassified record is the weakest claim. + audit_mode TEXT NOT NULL DEFAULT 'UNGROUNDED' + CHECK (audit_mode IN ('STRICT','HYBRID','UNGROUNDED')), n_quotes INTEGER NOT NULL DEFAULT 0, n_verified INTEGER NOT NULL DEFAULT 0, -- JSON array of quoted spans the model produced but we couldn't find @@ -257,7 +258,7 @@ CREATE TABLE IF NOT EXISTS mesh_epochs ( reason TEXT ); --- FTS5 over chunk content for VISUAL-mode keyword search. +-- FTS5 over chunk content for UNGROUNDED-mode keyword search. -- -- Contentless mode (`content=''`): FTS5 stores ONLY the inverted index, no -- copy of the indexed text. This eliminates the ~28 MB / 1000 docs that the @@ -305,17 +306,22 @@ def connect(db_path: Path | str = DEFAULT_DB_PATH) -> sqlite3.Connection: def _migrate_audit_mode(conn: sqlite3.Connection) -> None: """Forward-migrate pre-v9.8-audit-mode providence_cache shards. - Adds audit_mode + n_quotes + n_verified + unverified_quotes columns to - DBs that pre-date the faithfulness-classification rollout. SQLite ALTER - TABLE ADD COLUMN is O(1) (metadata-only) so this is cheap on every open. - Idempotent — checks PRAGMA before each ADD. + Adds audit_mode + n_quotes + n_verified + unverified_quotes + verifier_method + columns to DBs that pre-date the faithfulness-classification rollout. SQLite + ALTER TABLE ADD COLUMN is O(1) (metadata-only) so this is cheap on every + open. Idempotent — checks PRAGMA before each ADD. + + Also handles the VISUAL → UNGROUNDED rename for the audit_mode value + space. SQLite cannot ALTER a column's CHECK in place, so legacy tables + with the old `CHECK (audit_mode IN ('STRICT','HYBRID','VISUAL'))` get + rebuilt via the standard temp-table dance, with values translated. """ cols = {row["name"] for row in conn.execute("PRAGMA table_info(providence_cache)")} if "audit_mode" not in cols: conn.execute( "ALTER TABLE providence_cache ADD COLUMN audit_mode TEXT " - "NOT NULL DEFAULT 'VISUAL' " - "CHECK (audit_mode IN ('STRICT','HYBRID','VISUAL'))" + "NOT NULL DEFAULT 'UNGROUNDED' " + "CHECK (audit_mode IN ('STRICT','HYBRID','UNGROUNDED'))" ) if "n_quotes" not in cols: conn.execute( @@ -335,12 +341,89 @@ def _migrate_audit_mode(conn: sqlite3.Connection) -> None: "NOT NULL DEFAULT 'none' " "CHECK (verifier_method IN ('quote','span','entity','none'))" ) + + # VISUAL → UNGROUNDED rename. Detect legacy CHECK by inspecting DDL. + ddl_row = conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='providence_cache'" + ).fetchone() + if ddl_row and "'VISUAL'" in (ddl_row[0] or ""): + _rebuild_providence_cache_ungrounded(conn) + conn.execute( "CREATE INDEX IF NOT EXISTS idx_providence_audit " "ON providence_cache(audit_mode)" ) +def _rebuild_providence_cache_ungrounded(conn: sqlite3.Connection) -> None: + """One-time table rebuild: rename audit_mode value VISUAL → UNGROUNDED. + + SQLite cannot modify a column's CHECK constraint in place. Standard + pattern: create new table with new CHECK, copy data while translating + values, drop old, rename new. Wrapped in IMMEDIATE transaction so a + failure rolls back cleanly without leaving the DB half-migrated. + """ + new_create = """ + CREATE TABLE providence_cache_new ( + cache_key TEXT PRIMARY KEY, + source_root TEXT NOT NULL, + document_uri TEXT NOT NULL, + question_hash TEXT NOT NULL, + question_text TEXT NOT NULL, + answer_text TEXT NOT NULL, + merkle_proof TEXT NOT NULL, + model_profile_hash TEXT NOT NULL, + conversation_hash TEXT NOT NULL, + governance_policy_hash TEXT NOT NULL, + schema_version TEXT NOT NULL, + canonicalization_version TEXT NOT NULL, + chunking_version TEXT NOT NULL, + falsification_state TEXT NOT NULL DEFAULT 'live' + CHECK (falsification_state IN ('live','failed','stale','quarantined')), + chain TEXT NOT NULL DEFAULT 'private' + CHECK (chain IN ('private','public')), + audit_event_hash TEXT, + created_at INTEGER NOT NULL, + last_hit_at INTEGER, + hit_count INTEGER NOT NULL DEFAULT 0, + audit_mode TEXT NOT NULL DEFAULT 'UNGROUNDED' + CHECK (audit_mode IN ('STRICT','HYBRID','UNGROUNDED')), + n_quotes INTEGER NOT NULL DEFAULT 0, + n_verified INTEGER NOT NULL DEFAULT 0, + unverified_quotes TEXT, + verifier_method TEXT NOT NULL DEFAULT 'none' + CHECK (verifier_method IN ('quote','span','entity','none')) + ) + """ + conn.execute("BEGIN IMMEDIATE") + try: + conn.execute(new_create) + conn.execute( + "INSERT INTO providence_cache_new " + "(cache_key, source_root, document_uri, question_hash, question_text, " + " answer_text, merkle_proof, model_profile_hash, conversation_hash, " + " governance_policy_hash, schema_version, canonicalization_version, " + " chunking_version, falsification_state, chain, audit_event_hash, " + " created_at, last_hit_at, hit_count, audit_mode, n_quotes, " + " n_verified, unverified_quotes, verifier_method) " + "SELECT " + " cache_key, source_root, document_uri, question_hash, question_text, " + " answer_text, merkle_proof, model_profile_hash, conversation_hash, " + " governance_policy_hash, schema_version, canonicalization_version, " + " chunking_version, falsification_state, chain, audit_event_hash, " + " created_at, last_hit_at, hit_count, " + " CASE WHEN audit_mode = 'VISUAL' THEN 'UNGROUNDED' ELSE audit_mode END, " + " n_quotes, n_verified, unverified_quotes, verifier_method " + "FROM providence_cache" + ) + conn.execute("DROP TABLE providence_cache") + conn.execute("ALTER TABLE providence_cache_new RENAME TO providence_cache") + conn.execute("COMMIT") + except Exception: + conn.execute("ROLLBACK") + raise + + # Tables that exist in every shard with the same schema. Used to build # cross-shard UNION views in connect_query(). _SHARDABLE_TABLES = ( diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 7d36549..4fc8225 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -61,11 +61,11 @@ def test_ingest_basic_round_trip(tmp_path): assert result2.inserted == 0 assert result2.skipped_duplicate == 3 - # FTS5 search returns VISUAL hits. + # FTS5 search returns UNGROUNDED hits. backend = FTS5Backend(conn) hits = backend.search("merkle") assert len(hits) >= 1 - assert hits[0].audit_mode == AuditMode.VISUAL + assert hits[0].audit_mode == AuditMode.UNGROUNDED assert "merkle" in hits[0].snippet.lower() # Edge resolution: c -> a should be backfilled (a was ingested first). diff --git a/tests/test_migration_audit_mode.py b/tests/test_migration_audit_mode.py index 7e5c8d7..0263762 100644 --- a/tests/test_migration_audit_mode.py +++ b/tests/test_migration_audit_mode.py @@ -71,12 +71,12 @@ def test_legacy_providence_cache_migrates(tmp_path): cols = {r["name"] for r in conn.execute("PRAGMA table_info(providence_cache)")} assert {"audit_mode", "n_quotes", "n_verified", "unverified_quotes"} <= cols - # Pre-existing row inherits the safest default — VISUAL. + # Pre-existing row inherits the safest default — UNGROUNDED. row = conn.execute( "SELECT audit_mode, n_quotes, n_verified, unverified_quotes " "FROM providence_cache WHERE cache_key='legacy_key'" ).fetchone() - assert row["audit_mode"] == "VISUAL" + assert row["audit_mode"] == "UNGROUNDED" assert row["n_quotes"] == 0 assert row["n_verified"] == 0 assert row["unverified_quotes"] is None diff --git a/tests/test_reclassify.py b/tests/test_reclassify.py index ce586d8..0ae0327 100644 --- a/tests/test_reclassify.py +++ b/tests/test_reclassify.py @@ -1,7 +1,7 @@ """`aborist reclassify` — re-runs the verifier against live providence records. Covers: -- Stale VISUAL records become STRICT/HYBRID/etc when the new verifier +- Stale UNGROUNDED records become STRICT/HYBRID/etc when the new verifier finds grounding via span/entity paths. - Already-correct records stay unchanged. - Cold-source records (where any source doc has no hot chunks) are @@ -87,7 +87,7 @@ def _make_args(qa_db, dry_run=False, limit=0, entity_policy=None, compare=False) def test_reclassify_promotes_stale_visual_via_entity_path(tmp_path, capsys): """Stub answer with no quotes but verbatim entity names. The query - persists VISUAL/none if the verifier ran first; we then mutate the + persists UNGROUNDED/none if the verifier ran first; we then mutate the row to simulate an OLDER record (pre-layered-verifier), and reclassify.""" answer = ( "The cast includes Keanu Reeves, Laurence Fishburne, " @@ -95,12 +95,12 @@ def test_reclassify_promotes_stale_visual_via_entity_path(tmp_path, capsys): ) qa_db, _, ckey = _build_corpus_and_query(tmp_path, answer) - # Force the row into a stale state: VISUAL/none/0 — what the old + # Force the row into a stale state: UNGROUNDED/none/0 — what the old # quote-only verifier would have written. conn = connect(qa_db) try: conn.execute( - "UPDATE providence_cache SET audit_mode='VISUAL', " + "UPDATE providence_cache SET audit_mode='UNGROUNDED', " " verifier_method='none', n_quotes=0, n_verified=0, " " unverified_quotes=NULL " "WHERE cache_key = ?", @@ -166,7 +166,7 @@ def test_reclassify_dry_run_does_not_write(tmp_path, capsys): conn = connect(qa_db) try: conn.execute( - "UPDATE providence_cache SET audit_mode='VISUAL', " + "UPDATE providence_cache SET audit_mode='UNGROUNDED', " " verifier_method='none', n_quotes=0, n_verified=0 " "WHERE cache_key = ?", (ckey,), @@ -188,7 +188,7 @@ def test_reclassify_dry_run_does_not_write(tmp_path, capsys): "WHERE cache_key = ?", (ckey,), ).fetchone() - assert row["audit_mode"] == "VISUAL" + assert row["audit_mode"] == "UNGROUNDED" assert row["verifier_method"] == "none" finally: conn.close() diff --git a/tests/test_verify.py b/tests/test_verify.py index 1695c76..a6c97ed 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -5,7 +5,7 @@ first that finds evidence classifies the answer. `verifier_method` records which path fired. Tests cover the full trichotomy under each path plus regressions: - - TMNT: model wrote bios not in context — must be HYBRID/VISUAL + - TMNT: model wrote bios not in context — must be HYBRID/UNGROUNDED - Matrix: Wikipedia infobox + paraphrased prose. Spans don't match, but every multi-word proper noun does — entity path classifies it. """ @@ -87,7 +87,7 @@ def test_visual_when_no_quotes_no_spans_no_entities(): context = "Apples are red." answer = "freedom rests on autonomy alone, without coercion." v = verify_quotes(answer, context) - assert v["audit_mode"] == "VISUAL" + assert v["audit_mode"] == "UNGROUNDED" assert v["verifier_method"] == "none" assert v["n_quotes"] == 0 @@ -96,7 +96,7 @@ def test_visual_when_no_quote_verifies(): context = "Capitalism is an economic system based on private ownership." answer = '"this exact span is not in the source at all"' v = verify_quotes(answer, context) - assert v["audit_mode"] == "VISUAL" + assert v["audit_mode"] == "UNGROUNDED" assert v["verifier_method"] == "quote" assert v["n_quotes"] == 1 assert v["n_verified"] == 0 @@ -262,11 +262,11 @@ def test_entity_policy_strict_promotes_when_all_match(): def test_entity_policy_drop_skips_path_entirely(): """drop policy: never classifies via entity path. Verified entities - in source are ignored; the answer falls through to VISUAL/none.""" + in source are ignored; the answer falls through to UNGROUNDED/none.""" context = "Keanu Reeves stars in this film." answer = "Keanu Reeves played the lead role." v = verify_quotes(answer, context, entity_policy="drop") - assert v["audit_mode"] == "VISUAL" + assert v["audit_mode"] == "UNGROUNDED" assert v["verifier_method"] == "none" @@ -343,5 +343,5 @@ def test_wikitext_strip_does_not_rescue_genuine_hallucination(): v = verify_quotes(answer, raw_wikitext_context) assert v["verifier_method"] == "quote" assert v["n_verified"] == 0 - assert v["audit_mode"] == "VISUAL" + assert v["audit_mode"] == "UNGROUNDED" assert len(v["unverified_quotes"]) == 1 diff --git a/tests/test_wikitext.py b/tests/test_wikitext.py index 38e2805..a70f7e9 100644 --- a/tests/test_wikitext.py +++ b/tests/test_wikitext.py @@ -8,7 +8,7 @@ templates, mixed namespaces, refs inside paragraphs. The fixture file ``tests/fixtures/ff7_characters_chunk0.wikitext`` is chunk 0 of ``Characters_of_the_Final_Fantasy_VII_series`` and was the input that triggered the wikitext-base-v1 design (verifier flagged 6 -"quotes" as VISUAL because the model paraphrased the wikitext form). +"quotes" as UNGROUNDED because the model paraphrased the wikitext form). """ from __future__ import annotations