russell@unturf.com
1e8d2d2dba
Fox 2026-04-29 (Clark Kent / Superman query): the verifier flagged a
quoted span as unverified and the sidecar diagnosed it as 'paraphrase'
with token_coverage=1.0. Investigation showed the source actually said
`Clark Joseph Kent (middle name is also Jerome according to some
versions) is a fictional character...` and the model elided the
parenthetical aside, quoting `Clark Joseph Kent is a fictional
character...`. Every word in source, but the sequence has a 60-char
gap where the aside was. Distinct failure mode from real paraphrase
(token reordering) and trailing_artifact (model APPENDS).
Sidecar refinement, no verifier change. The binary verifier discipline
(memory rule `feedback_verifier_no_diagnostics`) holds — quote strategy
is still verbatim-only; soft signal stays in the inspect verb.
Algorithm walks every `(` in base. For each open paren at position P:
the longest k where base[:P] ends with span[:k] is the model's prefix;
the span tail (≥20 chars) must then match the source after the close
paren. First paren that satisfies both checks wins.
CLI human-render shows `matched: N prefix + M suffix chars
(parenthetical aside dropped)` plus the dropped_aside text so an
operator can decide at a glance whether the elision is benign.
Live record now reads:
[1] interior_elision
matched: 17 prefix + 138 suffix chars (parenthetical aside dropped)
dropped_aside: middle name is also jerome according to some versions
439 tests pass (sidecar +2, full suite still green).
317 lines
12 KiB
Python
317 lines
12 KiB
Python
"""Sidecar diagnostic for unverified spans.
|
|
|
|
`aborist inspect --cache-key X` reads a providence_cache record, pulls
|
|
the same source chunks the verifier saw, and classifies each
|
|
unverified_quote into one of: verbatim_in_base, verbatim_in_raw_only,
|
|
trailing_artifact, paraphrase, partial_paraphrase, no_overlap.
|
|
|
|
These tests pin the classifier on synthetic contexts so the diagnoses
|
|
are deterministic. End-to-end coverage of `inspect_cache_key` (DB
|
|
plumbing) goes through a small fixture record below.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from aborist.qa.inspect import _classify_span, _normalize, inspect_cache_key
|
|
from aborist.store import append_audit, connect, transaction
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _classify_span — pure function, no DB
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _norm(s: str) -> str:
|
|
return _normalize(s)
|
|
|
|
|
|
def test_classify_verbatim_in_base():
|
|
"""If the verifier flagged a span but it IS in the base context, the
|
|
diagnosis surfaces that as a likely verifier or canonicalization
|
|
bug. Deliberately distinct label so an operator notices."""
|
|
span = "the quick brown fox"
|
|
base = "lorem ipsum the quick brown fox jumps over"
|
|
raw = base
|
|
out = _classify_span(span, _norm(base), _norm(raw))
|
|
assert out["diagnosis"] == "verbatim_in_base"
|
|
|
|
|
|
def test_classify_verbatim_in_raw_only():
|
|
"""Span matches raw wikitext but base form differs — wikitext-strip
|
|
edge case worth flagging separately."""
|
|
span = "[[Cloud Strife]] is the protagonist"
|
|
base = "Cloud Strife is the protagonist" # wikitext stripped
|
|
raw = "[[Cloud Strife]] is the protagonist"
|
|
out = _classify_span(span, _norm(base), _norm(raw))
|
|
assert out["diagnosis"] == "verbatim_in_raw_only"
|
|
|
|
|
|
def test_classify_trailing_artifact_citation_appended():
|
|
"""The Pikachu case — model appended `(Source: ...)` to a verbatim
|
|
sentence; verifier flagged the whole thing. inspect should isolate
|
|
the artifact."""
|
|
base = (
|
|
"Pikachu can store electricity in its cheeks and release it in "
|
|
"lightning-based attacks. Pikachu evolves from Pichu."
|
|
)
|
|
span = (
|
|
"Pikachu can store electricity in its cheeks and release it in "
|
|
"lightning-based attacks. (Source: https://en.wikipedia.org/wiki/Pikachu)"
|
|
)
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "trailing_artifact"
|
|
assert out["matched_prefix_chars"] >= 60
|
|
assert "Source:" in out["trailing_artifact"]
|
|
|
|
|
|
def test_classify_interior_elision_drops_parenthetical_aside():
|
|
"""Fox 2026-04-29 catch (Clark Kent): source has
|
|
`Clark Joseph Kent (middle name is also Jerome ...) is a fictional
|
|
character...`; model quoted `Clark Joseph Kent is a fictional
|
|
character...` — every token in source, but a `(...)` aside got
|
|
elided for prose flow. Distinct from trailing_artifact (model
|
|
APPENDS) and paraphrase (different sequence). Sidecar must
|
|
surface this as `interior_elision` with the dropped aside reported."""
|
|
base = (
|
|
"Clark Joseph Kent (middle name is also Jerome according to some "
|
|
"versions) is a fictional character created by Jerry Siegel and "
|
|
"Joe Shuster. He serves as the civilian and secret identity of "
|
|
"the superhero Superman."
|
|
)
|
|
span = (
|
|
"Clark Joseph Kent is a fictional character created by Jerry Siegel "
|
|
"and Joe Shuster. He serves as the civilian and secret identity of "
|
|
"the superhero Superman."
|
|
)
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "interior_elision"
|
|
assert out["matched_prefix_chars"] >= 17 # "clark joseph kent"
|
|
assert out["matched_suffix_chars"] >= 60
|
|
assert "jerome" in out["dropped_aside"].lower()
|
|
|
|
|
|
def test_classify_interior_elision_falls_through_when_suffix_doesnt_match():
|
|
"""Probe is conservative: suffix must land verbatim after the close
|
|
paren. A model that drops the aside AND rewords the rest should NOT
|
|
classify as interior_elision."""
|
|
base = (
|
|
"Clark Joseph Kent (middle name is Jerome) is a fictional character "
|
|
"created by Jerry Siegel and Joe Shuster."
|
|
)
|
|
span = (
|
|
"Clark Joseph Kent is the most famous superhero in comics history "
|
|
"and was invented by Jerry Siegel."
|
|
)
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] != "interior_elision"
|
|
|
|
|
|
def test_classify_paraphrase_high_token_coverage():
|
|
"""Tokens all present, sequence different — model rewrote the source."""
|
|
base = (
|
|
"Pikachu is a Pokémon species in the Pokémon franchise developed by "
|
|
"Game Freak and Nintendo. Pokémon games feature numerous creatures."
|
|
)
|
|
span = "Pikachu is a species of Pokémon creatures from the Pokémon franchise"
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "paraphrase"
|
|
assert out["token_coverage"] >= 0.85
|
|
|
|
|
|
def test_classify_partial_paraphrase_some_tokens_missing():
|
|
"""Some content overlap but a meaningful token isn't there at all —
|
|
likely model invention mixed with corpus tokens."""
|
|
base = "Pikachu is a Pokémon. Pikachu battles other creatures."
|
|
span = "Pikachu invented the lightning attack in Tokyo on Tuesday"
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] in ("partial_paraphrase", "no_overlap")
|
|
if out["diagnosis"] == "partial_paraphrase":
|
|
assert "missing_tokens" in out
|
|
|
|
|
|
def test_classify_no_overlap_full_invention():
|
|
"""Almost no content tokens shared with the corpus — pure model output."""
|
|
base = "Pikachu is a Pokémon species."
|
|
span = "The Roman Senate convened in 49 BC to debate Caesar's rebellion"
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "no_overlap"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# inspect_cache_key — end-to-end with a tiny fixture
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _seed_record(qa_db: Path, *, cache_key: str, sources: list[dict],
|
|
unverified: list[str]) -> None:
|
|
"""Insert a minimal providence_cache row with a merkle_proof that
|
|
points at the given sources. Caller has already populated each
|
|
shard with the actual document + chunks."""
|
|
conn = connect(qa_db)
|
|
try:
|
|
with transaction(conn):
|
|
event_hash = append_audit(
|
|
conn,
|
|
event_type="providence_query",
|
|
subject_root=cache_key,
|
|
body={"cache_key": cache_key},
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO providence_cache "
|
|
"(cache_key, source_root, document_uri, question_hash, "
|
|
" question_text, answer_text, merkle_proof, "
|
|
" model_profile_hash, conversation_hash, "
|
|
" governance_policy_hash, schema_version, "
|
|
" canonicalization_version, chunking_version, "
|
|
" falsification_state, chain, audit_event_hash, "
|
|
" created_at, hit_count, audit_mode, n_quotes, "
|
|
" n_verified, unverified_quotes, verifier_method) "
|
|
"VALUES (?, ?, 'corpus://multi', ?, ?, ?, ?, ?, ?, ?, "
|
|
" ?, ?, ?, 'live', 'private', ?, ?, 0, 'HYBRID', ?, ?, ?, 'span')",
|
|
(
|
|
cache_key,
|
|
"00" * 32,
|
|
"qh",
|
|
"what is foo?",
|
|
"answer here",
|
|
json.dumps({"sources": sources}, ensure_ascii=False),
|
|
"mh",
|
|
"ch",
|
|
"gh",
|
|
"v9.8.0",
|
|
"norm-v1",
|
|
"tok-512-v1",
|
|
event_hash,
|
|
int(time.time()),
|
|
len(unverified) + 1, # n_quotes (one verified plus N unverified)
|
|
1, # n_verified
|
|
json.dumps(unverified, ensure_ascii=False),
|
|
),
|
|
)
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _seed_doc(shard: Path, *, document_root: str, document_uri: str,
|
|
chunk_text: str) -> None:
|
|
"""Insert a documents row + one hot chunk with the given text."""
|
|
conn = connect(shard)
|
|
try:
|
|
from aborist.compress import pack_chunk
|
|
|
|
with transaction(conn):
|
|
conn.execute(
|
|
"INSERT INTO documents "
|
|
"(document_root, document_uri, source_type, kind, "
|
|
" compression_depth, chunking_version, "
|
|
" canonicalization_version, schema_version, ingest_ts, hit_count) "
|
|
"VALUES (?, ?, 'html', 'surface', 0, 'tok-512-v1', "
|
|
" 'norm-v1', 'v9.8.0', ?, 0)",
|
|
(document_root, document_uri, int(time.time())),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO chunks (document_root, idx, leaf_hash, content, tier) "
|
|
"VALUES (?, 0, ?, ?, 'hot')",
|
|
(document_root, "ff" * 32, pack_chunk(chunk_text)),
|
|
)
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def test_inspect_cache_key_classifies_each_unverified(tmp_path):
|
|
qa_db = tmp_path / "qa.db"
|
|
shard = tmp_path / "001.db"
|
|
DOC = "ab" * 32
|
|
_seed_doc(
|
|
shard,
|
|
document_root=DOC,
|
|
document_uri="https://example.com/x",
|
|
chunk_text=(
|
|
"Pikachu can store electricity in its cheeks. "
|
|
"Pikachu evolves from Pichu."
|
|
),
|
|
)
|
|
_seed_record(
|
|
qa_db,
|
|
cache_key="01" * 32,
|
|
sources=[{
|
|
"document_root": DOC,
|
|
"document_uri": "https://example.com/x",
|
|
"title": "X",
|
|
"shard": shard.name,
|
|
"chunk_idx": 0,
|
|
}],
|
|
unverified=[
|
|
"Pikachu can store electricity in its cheeks.", # verbatim
|
|
"Pikachu fought in the 1988 Olympics in Seoul", # invention
|
|
],
|
|
)
|
|
|
|
result = inspect_cache_key(
|
|
"01" * 32, qa_db=qa_db, shards_dir=tmp_path
|
|
)
|
|
assert result["status"] == "ok"
|
|
assert result["record"]["cache_key"] == "01" * 32
|
|
assert len(result["sources"]) == 1
|
|
assert result["sources"][0]["chunk_count"] == 1
|
|
assert len(result["unverified"]) == 2
|
|
diags = [d["diagnosis"] for d in result["unverified"]]
|
|
assert "verbatim_in_base" in diags
|
|
# The Olympics span should be flagged as no_overlap or partial_paraphrase
|
|
# depending on token coincidences — both honest classifications.
|
|
assert any(d in ("no_overlap", "partial_paraphrase") for d in diags)
|
|
|
|
|
|
def test_inspect_cache_key_unknown_returns_not_found(tmp_path):
|
|
qa_db = tmp_path / "qa.db"
|
|
# touch DB so connect() initialises schema
|
|
connect(qa_db).close()
|
|
result = inspect_cache_key(
|
|
"00" * 32, qa_db=qa_db, shards_dir=tmp_path
|
|
)
|
|
assert result["status"] == "not_found"
|
|
|
|
|
|
def test_inspect_does_not_mutate_state(tmp_path):
|
|
"""Sidecar invariant: inspect must not write to providence_cache,
|
|
audit_events, or anything else. Pin so a future refactor can't
|
|
silently add side effects."""
|
|
qa_db = tmp_path / "qa.db"
|
|
shard = tmp_path / "001.db"
|
|
DOC = "cd" * 32
|
|
_seed_doc(
|
|
shard, document_root=DOC, document_uri="https://example.com/y",
|
|
chunk_text="hello world",
|
|
)
|
|
_seed_record(
|
|
qa_db, cache_key="02" * 32,
|
|
sources=[{
|
|
"document_root": DOC, "document_uri": "https://example.com/y",
|
|
"title": "Y", "shard": shard.name, "chunk_idx": 0,
|
|
}],
|
|
unverified=["something the model said"],
|
|
)
|
|
|
|
conn = connect(qa_db)
|
|
try:
|
|
before_audit = conn.execute("SELECT COUNT(*) FROM audit_events").fetchone()[0]
|
|
before_prov = conn.execute("SELECT COUNT(*) FROM providence_cache").fetchone()[0]
|
|
finally:
|
|
conn.close()
|
|
|
|
inspect_cache_key("02" * 32, qa_db=qa_db, shards_dir=tmp_path)
|
|
|
|
conn = connect(qa_db)
|
|
try:
|
|
after_audit = conn.execute("SELECT COUNT(*) FROM audit_events").fetchone()[0]
|
|
after_prov = conn.execute("SELECT COUNT(*) FROM providence_cache").fetchone()[0]
|
|
finally:
|
|
conn.close()
|
|
assert after_audit == before_audit
|
|
assert after_prov == before_prov
|