1090 lines
44 KiB
Python
1090 lines
44 KiB
Python
"""Sidecar diagnostic for unverified spans.
|
|
|
|
`arborist inspect --cache-key X` reads a providence_cache record, pulls
|
|
the same source chunks the verifier saw, and classifies each
|
|
unverified_quote into one of: verbatim_in_base, verbatim_in_raw_only,
|
|
trailing_artifact, paraphrase, partial_paraphrase, no_overlap.
|
|
|
|
These tests pin the classifier on synthetic contexts so the diagnoses
|
|
are deterministic. End-to-end coverage of `inspect_cache_key` (DB
|
|
plumbing) goes through a small fixture record below.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from arborist.qa.inspect import (
|
|
_classify_span,
|
|
_normalize,
|
|
diagnose_coherence,
|
|
diagnose_deflection,
|
|
diagnose_metaphor_deflection,
|
|
diagnose_title_relevance,
|
|
inspect_cache_key,
|
|
)
|
|
from arborist.store import append_audit, connect, transaction
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _classify_span — pure function, no DB
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _norm(s: str) -> str:
|
|
return _normalize(s)
|
|
|
|
|
|
def test_classify_verbatim_in_base():
|
|
"""If the verifier flagged a span but it IS in the base context, the
|
|
diagnosis surfaces that as a likely verifier or canonicalization
|
|
bug. Deliberately distinct label so an operator notices."""
|
|
span = "the quick brown fox"
|
|
base = "lorem ipsum the quick brown fox jumps over"
|
|
raw = base
|
|
out = _classify_span(span, _norm(base), _norm(raw))
|
|
assert out["diagnosis"] == "verbatim_in_base"
|
|
|
|
|
|
def test_classify_verbatim_in_raw_only():
|
|
"""Span matches raw wikitext but base form differs — wikitext-strip
|
|
edge case worth flagging separately."""
|
|
span = "[[Cloud Strife]] is the protagonist"
|
|
base = "Cloud Strife is the protagonist" # wikitext stripped
|
|
raw = "[[Cloud Strife]] is the protagonist"
|
|
out = _classify_span(span, _norm(base), _norm(raw))
|
|
assert out["diagnosis"] == "verbatim_in_raw_only"
|
|
|
|
|
|
def test_classify_trailing_artifact_citation_appended():
|
|
"""The Pikachu case — model appended `(Source: ...)` to a verbatim
|
|
sentence; verifier flagged the whole thing. inspect should isolate
|
|
the artifact."""
|
|
base = (
|
|
"Pikachu can store electricity in its cheeks and release it in "
|
|
"lightning-based attacks. Pikachu evolves from Pichu."
|
|
)
|
|
span = (
|
|
"Pikachu can store electricity in its cheeks and release it in "
|
|
"lightning-based attacks. (Source: https://en.wikipedia.org/wiki/Pikachu)"
|
|
)
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "trailing_artifact"
|
|
assert out["matched_prefix_chars"] >= 60
|
|
assert "Source:" in out["trailing_artifact"]
|
|
|
|
|
|
def test_classify_interior_elision_drops_parenthetical_aside():
|
|
"""Fox 2026-04-29 catch (Clark Kent): source has
|
|
`Clark Joseph Kent (middle name is also Jerome ...) is a fictional
|
|
character...`; model quoted `Clark Joseph Kent is a fictional
|
|
character...` — every token in source, but a `(...)` aside got
|
|
elided for prose flow. Distinct from trailing_artifact (model
|
|
APPENDS) and paraphrase (different sequence). Sidecar must
|
|
surface this as `interior_elision` with the dropped aside reported."""
|
|
base = (
|
|
"Clark Joseph Kent (middle name is also Jerome according to some "
|
|
"versions) is a fictional character created by Jerry Siegel and "
|
|
"Joe Shuster. He serves as the civilian and secret identity of "
|
|
"the superhero Superman."
|
|
)
|
|
span = (
|
|
"Clark Joseph Kent is a fictional character created by Jerry Siegel "
|
|
"and Joe Shuster. He serves as the civilian and secret identity of "
|
|
"the superhero Superman."
|
|
)
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "interior_elision"
|
|
assert out["matched_prefix_chars"] >= 17 # "clark joseph kent"
|
|
assert out["matched_suffix_chars"] >= 60
|
|
assert "jerome" in out["dropped_aside"].lower()
|
|
|
|
|
|
def test_classify_interior_elision_falls_through_when_suffix_doesnt_match():
|
|
"""Probe is conservative: suffix must land verbatim after the close
|
|
paren. A model that drops the aside AND rewords the rest should NOT
|
|
classify as interior_elision."""
|
|
base = (
|
|
"Clark Joseph Kent (middle name is Jerome) is a fictional character "
|
|
"created by Jerry Siegel and Joe Shuster."
|
|
)
|
|
span = (
|
|
"Clark Joseph Kent is the most famous superhero in comics history "
|
|
"and was invented by Jerry Siegel."
|
|
)
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] != "interior_elision"
|
|
|
|
|
|
def test_classify_synthetic_elision_caught():
|
|
"""Fox 2026-04-30 (Brachiosaurus / Jurassic Park): the model wrote
|
|
a `"..."` quote with literal `[...]` between fragments, signaling
|
|
self-elision while claiming verbatim citation. Distinct from
|
|
`interior_elision` (model dropped a `(...)` aside source carries).
|
|
Sidecar reports prefix/suffix presence in source."""
|
|
base = (
|
|
"The film centers on the fictional Isla Nublar, in Costa Rica, where "
|
|
"billionaire philanthropist John Hammond has created an amusement park "
|
|
"of cloned dinosaurs. Universal Studios acquired the rights."
|
|
)
|
|
span = (
|
|
"The film centers on the fictional Isla Nublar [...] Universal Studios "
|
|
"acquired the rights."
|
|
)
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "synthetic_elision_inside_quote"
|
|
assert out["elision_marker"] == "[...]"
|
|
assert out["prefix_in_source"] is True
|
|
assert out["suffix_in_source"] is True
|
|
|
|
|
|
def test_classify_synthetic_elision_does_not_fire_when_source_has_brackets():
|
|
"""If `[...]` literally appears in source (e.g. a citation
|
|
formatting), the substring check would have passed earlier — sidecar
|
|
falls through to its other diagnoses."""
|
|
base = "Some prose with [...] literal brackets in source."
|
|
span = "Some prose with [...] literal brackets in source."
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "verbatim_in_base"
|
|
|
|
|
|
def test_repair_synthetic_elision_split_when_both_halves_verbatim():
|
|
"""Repair plan for `"prefix [...] suffix"` where source carries
|
|
both halves verbatim: split into two quotes."""
|
|
base = (
|
|
"The film centers on the fictional Isla Nublar, in Costa Rica. "
|
|
"Universal Studios acquired the rights to the novel before publication."
|
|
)
|
|
span = (
|
|
"The film centers on the fictional Isla Nublar [...] Universal Studios "
|
|
"acquired the rights to the novel"
|
|
)
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "synthetic_elision_inside_quote"
|
|
assert out["repair"]["action"] == "split_into_two_quotes"
|
|
assert len(out["repair"]["quotes"]) == 2
|
|
|
|
|
|
def test_repair_interior_elision_includes_aside():
|
|
"""Repair plan for parenthetical-elision: rewrite quote to include
|
|
the dropped aside so it becomes verbatim."""
|
|
base = (
|
|
"Clark Joseph Kent (middle name is also Jerome according to some "
|
|
"versions) is a fictional character created by Jerry Siegel and Joe Shuster."
|
|
)
|
|
span = (
|
|
"Clark Joseph Kent is a fictional character created by Jerry Siegel "
|
|
"and Joe Shuster."
|
|
)
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "interior_elision"
|
|
assert out["repair"]["action"] == "include_aside_for_verbatim"
|
|
assert "jerome" in out["repair"]["aside_to_restore"].lower()
|
|
|
|
|
|
def test_repair_trailing_artifact_trim():
|
|
"""Repair plan for model-appended `(Source: ...)`: trim the tail.
|
|
Trailing-artifact probe requires ≥60 char matching prefix, so the
|
|
test uses a long-enough prose span."""
|
|
base = (
|
|
"Pikachu can store electricity in its cheeks and release it in "
|
|
"lightning-based attacks. Pikachu evolves from Pichu."
|
|
)
|
|
span = (
|
|
"Pikachu can store electricity in its cheeks and release it in "
|
|
"lightning-based attacks. (Source: https://en.wikipedia.org/wiki/Pikachu)"
|
|
)
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "trailing_artifact"
|
|
assert out["repair"]["action"] == "trim_trailing_artifact"
|
|
assert "(Source:" not in out["repair"]["kept_prefix"]
|
|
|
|
|
|
def test_repair_no_overlap_remove():
|
|
"""Repair plan for full-invention spans: remove the claim."""
|
|
base = "Pikachu is a Pokémon species."
|
|
span = "The Roman Senate convened in 49 BC to debate Caesar's rebellion"
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "no_overlap"
|
|
assert out["repair"]["action"] == "remove_claim"
|
|
|
|
|
|
def test_classify_paraphrase_high_token_coverage():
|
|
"""Tokens all present, sequence different — model rewrote the source."""
|
|
base = (
|
|
"Pikachu is a Pokémon species in the Pokémon franchise developed by "
|
|
"Game Freak and Nintendo. Pokémon games feature numerous creatures."
|
|
)
|
|
span = "Pikachu is a species of Pokémon creatures from the Pokémon franchise"
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "paraphrase"
|
|
assert out["token_coverage"] >= 0.85
|
|
|
|
|
|
def test_classify_partial_paraphrase_some_tokens_missing():
|
|
"""Some content overlap but a meaningful token isn't there at all —
|
|
likely model invention mixed with corpus tokens."""
|
|
base = "Pikachu is a Pokémon. Pikachu battles other creatures."
|
|
span = "Pikachu invented the lightning attack in Tokyo on Tuesday"
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] in ("partial_paraphrase", "no_overlap")
|
|
if out["diagnosis"] == "partial_paraphrase":
|
|
assert "missing_tokens" in out
|
|
|
|
|
|
def test_classify_no_overlap_full_invention():
|
|
"""Almost no content tokens shared with the corpus — pure model output."""
|
|
base = "Pikachu is a Pokémon species."
|
|
span = "The Roman Senate convened in 49 BC to debate Caesar's rebellion"
|
|
out = _classify_span(span, _norm(base), _norm(base))
|
|
assert out["diagnosis"] == "no_overlap"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# inspect_cache_key — end-to-end with a tiny fixture
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _seed_record(qa_db: Path, *, cache_key: str, sources: list[dict],
|
|
unverified: list[str],
|
|
question_text: str = "what is foo?",
|
|
answer_text: str = "answer here") -> None:
|
|
"""Insert a minimal providence_cache row with a merkle_proof that
|
|
points at the given sources. Caller has already populated each
|
|
shard with the actual document + chunks. ``question_text`` /
|
|
``answer_text`` default to the canonical pre-warrant fixture
|
|
pair; tests that exercise warrant-shape sidecars override them."""
|
|
conn = connect(qa_db)
|
|
try:
|
|
with transaction(conn):
|
|
event_hash = append_audit(
|
|
conn,
|
|
event_type="providence_query",
|
|
subject_root=cache_key,
|
|
body={"cache_key": cache_key},
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO providence_cache "
|
|
"(cache_key, source_root, document_uri, question_hash, "
|
|
" question_text, answer_text, merkle_proof, "
|
|
" model_profile_hash, conversation_hash, "
|
|
" governance_policy_hash, schema_version, "
|
|
" canonicalization_version, chunking_version, "
|
|
" falsification_state, chain, audit_event_hash, "
|
|
" created_at, hit_count, audit_mode, n_quotes, "
|
|
" n_verified, unverified_quotes, verifier_method) "
|
|
"VALUES (?, ?, 'corpus://multi', ?, ?, ?, ?, ?, ?, ?, "
|
|
" ?, ?, ?, 'live', 'private', ?, ?, 0, 'HYBRID', ?, ?, ?, 'span')",
|
|
(
|
|
cache_key,
|
|
"00" * 32,
|
|
"qh",
|
|
question_text,
|
|
answer_text,
|
|
json.dumps({"sources": sources}, ensure_ascii=False),
|
|
"mh",
|
|
"ch",
|
|
"gh",
|
|
"v9.8.0",
|
|
"norm-v1",
|
|
"tok-512-v1",
|
|
event_hash,
|
|
int(time.time()),
|
|
len(unverified) + 1, # n_quotes (one verified plus N unverified)
|
|
1, # n_verified
|
|
json.dumps(unverified, ensure_ascii=False),
|
|
),
|
|
)
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _seed_doc(shard: Path, *, document_root: str, document_uri: str,
|
|
chunk_text: str) -> None:
|
|
"""Insert a documents row + one hot chunk with the given text."""
|
|
conn = connect(shard)
|
|
try:
|
|
from arborist.compress import pack_chunk
|
|
|
|
with transaction(conn):
|
|
conn.execute(
|
|
"INSERT INTO documents "
|
|
"(document_root, document_uri, source_type, kind, "
|
|
" compression_depth, chunking_version, "
|
|
" canonicalization_version, schema_version, ingest_ts, hit_count) "
|
|
"VALUES (?, ?, 'html', 'surface', 0, 'tok-512-v1', "
|
|
" 'norm-v1', 'v9.8.0', ?, 0)",
|
|
(document_root, document_uri, int(time.time())),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO chunks (document_root, idx, leaf_hash, content, tier) "
|
|
"VALUES (?, 0, ?, ?, 'hot')",
|
|
(document_root, "ff" * 32, pack_chunk(chunk_text)),
|
|
)
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def test_inspect_cache_key_classifies_each_unverified(tmp_path):
|
|
qa_db = tmp_path / "qa.db"
|
|
shard = tmp_path / "001.db"
|
|
DOC = "ab" * 32
|
|
_seed_doc(
|
|
shard,
|
|
document_root=DOC,
|
|
document_uri="https://example.com/x",
|
|
chunk_text=(
|
|
"Pikachu can store electricity in its cheeks. "
|
|
"Pikachu evolves from Pichu."
|
|
),
|
|
)
|
|
_seed_record(
|
|
qa_db,
|
|
cache_key="01" * 32,
|
|
sources=[{
|
|
"document_root": DOC,
|
|
"document_uri": "https://example.com/x",
|
|
"title": "X",
|
|
"shard": shard.name,
|
|
"chunk_idx": 0,
|
|
}],
|
|
unverified=[
|
|
"Pikachu can store electricity in its cheeks.", # verbatim
|
|
"Pikachu fought in the 1988 Olympics in Seoul", # invention
|
|
],
|
|
)
|
|
|
|
result = inspect_cache_key(
|
|
"01" * 32, qa_db=qa_db, shards_dir=tmp_path
|
|
)
|
|
assert result["status"] == "ok"
|
|
assert result["record"]["cache_key"] == "01" * 32
|
|
assert len(result["sources"]) == 1
|
|
assert result["sources"][0]["chunk_count"] == 1
|
|
assert len(result["unverified"]) == 2
|
|
diags = [d["diagnosis"] for d in result["unverified"]]
|
|
assert "verbatim_in_base" in diags
|
|
# The Olympics span should be flagged as no_overlap or partial_paraphrase
|
|
# depending on token coincidences — both honest classifications.
|
|
assert any(d in ("no_overlap", "partial_paraphrase") for d in diags)
|
|
|
|
|
|
def test_inspect_cache_key_unknown_returns_not_found(tmp_path):
|
|
qa_db = tmp_path / "qa.db"
|
|
# touch DB so connect() initialises schema
|
|
connect(qa_db).close()
|
|
result = inspect_cache_key(
|
|
"00" * 32, qa_db=qa_db, shards_dir=tmp_path
|
|
)
|
|
assert result["status"] == "not_found"
|
|
|
|
|
|
def test_inspect_does_not_mutate_state(tmp_path):
|
|
"""Sidecar invariant: inspect must not write to providence_cache,
|
|
audit_events, or anything else. Pin so a future refactor can't
|
|
silently add side effects."""
|
|
qa_db = tmp_path / "qa.db"
|
|
shard = tmp_path / "001.db"
|
|
DOC = "cd" * 32
|
|
_seed_doc(
|
|
shard, document_root=DOC, document_uri="https://example.com/y",
|
|
chunk_text="hello world",
|
|
)
|
|
_seed_record(
|
|
qa_db, cache_key="02" * 32,
|
|
sources=[{
|
|
"document_root": DOC, "document_uri": "https://example.com/y",
|
|
"title": "Y", "shard": shard.name, "chunk_idx": 0,
|
|
}],
|
|
unverified=["something the model said"],
|
|
)
|
|
|
|
conn = connect(qa_db)
|
|
try:
|
|
before_audit = conn.execute("SELECT COUNT(*) FROM audit_events").fetchone()[0]
|
|
before_prov = conn.execute("SELECT COUNT(*) FROM providence_cache").fetchone()[0]
|
|
finally:
|
|
conn.close()
|
|
|
|
inspect_cache_key("02" * 32, qa_db=qa_db, shards_dir=tmp_path)
|
|
|
|
conn = connect(qa_db)
|
|
try:
|
|
after_audit = conn.execute("SELECT COUNT(*) FROM audit_events").fetchone()[0]
|
|
after_prov = conn.execute("SELECT COUNT(*) FROM providence_cache").fetchone()[0]
|
|
finally:
|
|
conn.close()
|
|
assert after_audit == before_audit
|
|
assert after_prov == before_prov
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# deflection diagnostic
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_deflection_mars_bdfl_to_python_guido():
|
|
"""Empirical 2026-04-30: 'who is a benevolent dictator for life
|
|
for mars?' returned STRICT with answer 'Guido van Rossum is a BDFL
|
|
for the Python programming language.' Verifier did its job (claims
|
|
grounded) but answer never mentions mars — pure topic-shift.
|
|
|
|
Subject-anchor heuristic catches this: 3/4 of question tokens
|
|
(benevolent, dictator, life) match the answer, but the SUBJECT
|
|
anchor (last content token, 'mars') is missing. That's the signal
|
|
that a generic-vocabulary overlap can't paper over."""
|
|
d = diagnose_deflection(
|
|
"who is a benevolent dictator for life for mars?",
|
|
"Guido van Rossum is a Benevolent Dictator For Life (BDFL) "
|
|
"for the Python programming language.",
|
|
)
|
|
assert d["kind"] == "deflection", d
|
|
assert "mars" in d["question_tokens"]
|
|
assert d["subject_anchor"] == "mars"
|
|
assert d["subject_in_answer"] is False
|
|
assert "mars" not in d["overlap"]
|
|
# Generic-vocabulary overlap is high (3/4) but subject is missing
|
|
# — that's the deflection signal subject-anchor catches.
|
|
assert d["overlap_ratio"] > 0.5
|
|
|
|
|
|
def test_deflection_partial_overlap_when_some_subjects_match():
|
|
"""Some content tokens overlap, others don't — soft signal."""
|
|
d = diagnose_deflection(
|
|
"what is the relationship between linux and unix?",
|
|
"Linux is a Unix-like operating system inspired by Unix design.",
|
|
)
|
|
assert d["kind"] == "partial_overlap" or d["kind"] == "on_topic", d
|
|
assert "linux" in d["overlap"]
|
|
assert "unix" in d["overlap"]
|
|
# 'relationship' is a question-shape word that may or may not appear
|
|
# in answer; key is that core subjects overlap.
|
|
assert d["overlap_ratio"] > 0.0
|
|
|
|
|
|
def test_deflection_on_topic_when_all_subjects_present():
|
|
"""All question content tokens appear in answer — clean on-topic."""
|
|
d = diagnose_deflection(
|
|
"who painted the mona lisa?",
|
|
"Leonardo da Vinci painted the Mona Lisa.",
|
|
)
|
|
assert d["kind"] == "on_topic"
|
|
assert d["overlap_ratio"] == 1.0
|
|
|
|
|
|
def test_deflection_handles_empty_question_or_answer():
|
|
"""Vacuous case: question with no content tokens — no signal to give."""
|
|
d = diagnose_deflection("?", "Some answer text here.")
|
|
assert d["kind"] == "no_question_tokens"
|
|
assert d["overlap_ratio"] == 0.0
|
|
|
|
|
|
def test_deflection_strips_possessive_s_for_overlap():
|
|
"""Possessive 'mars's' should match 'mars' in the answer."""
|
|
d = diagnose_deflection(
|
|
"what is mars's atmosphere?",
|
|
"Mars has a thin atmosphere of carbon dioxide.",
|
|
)
|
|
assert "mars" in d["overlap"]
|
|
assert "atmosphere" in d["overlap"]
|
|
|
|
|
|
def test_deflection_suppressed_for_when_year_questions():
|
|
"""Bench finding 2026-05-01: 'what year did the berlin wall fall?'
|
|
answer '1989' has zero subject overlap (subject anchor 'fall' is
|
|
a verb the answer doesn't echo). Pre-suppression, classified as
|
|
deflection across all three modes (false positive)."""
|
|
d = diagnose_deflection(
|
|
"what year did the berlin wall fall?",
|
|
"1989.",
|
|
)
|
|
assert d["shape_suppressed"] is True
|
|
# Even with shape_suppressed, zero overlap on a date answer falls
|
|
# back to overlap-ratio = 0 → deflection. The shape_suppressed
|
|
# field tells the operator the signal was a known false-positive
|
|
# shape, not an actual topic shift.
|
|
assert d["kind"] == "deflection" # overlap-ratio path still fires
|
|
# But the operator can filter on shape_suppressed when scoring.
|
|
|
|
|
|
def test_deflection_suppressed_when_numeric_answer_overlaps():
|
|
"""When the date answer DOES include topic words ('Berlin Wall
|
|
fell in 1989'), shape suppression doesn't matter — overlap is
|
|
high enough for partial_overlap or on_topic."""
|
|
d = diagnose_deflection(
|
|
"what year did the berlin wall fall?",
|
|
"The Berlin Wall fell in 1989, ending the Cold War.",
|
|
)
|
|
assert d["shape_suppressed"] is True
|
|
assert d["kind"] in ("on_topic", "partial_overlap")
|
|
|
|
|
|
def test_deflection_suppressed_for_why_questions():
|
|
"""WHY-shape questions: answer discusses causes, may or may not
|
|
echo the question's verb. Bench: pointer mode deflected 3/3 on
|
|
'why did the titanic sink?' but answer was a valid cause."""
|
|
d = diagnose_deflection(
|
|
"why did the titanic sink?",
|
|
"The Titanic struck an iceberg and sank in 1912.",
|
|
)
|
|
assert d["shape_suppressed"] is True
|
|
# Has overlap on titanic+sink, classifies as on_topic or partial.
|
|
assert d["kind"] in ("on_topic", "partial_overlap")
|
|
|
|
|
|
def test_deflection_not_suppressed_for_who_questions():
|
|
"""Mars-BDFL pattern still fires on who-shape questions."""
|
|
d = diagnose_deflection(
|
|
"who is a benevolent dictator for life for mars?",
|
|
"Guido van Rossum is a Benevolent Dictator For Life for Python.",
|
|
)
|
|
assert d["shape_suppressed"] is False
|
|
assert d["kind"] == "deflection"
|
|
assert d["subject_anchor"] == "mars"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# title-relevance sidecar
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_title_mismatch_on_qcd_cited_for_spin_glass_claim():
|
|
"""Empirical 2026-05-01: 'explain spin glass modeling, tensors?'
|
|
returned STRICT 1/1 with claim 'Spin glass modeling involves...
|
|
mathematical tools such as tensors' cited to a chunk from
|
|
Quantum_chromodynamics. Claim tokens have zero overlap with QCD
|
|
title — sidecar should flag TITLE_MISMATCH so an operator can
|
|
see the retrieval-driven hallucination signature."""
|
|
d = diagnose_title_relevance(
|
|
"Spin glass modeling involves using the concept of spin glasses, "
|
|
"which are disordered magnetic systems, to study complex systems "
|
|
"and phenomena in physics and other fields. Spin glasses are "
|
|
"characterized by random interactions. Tensors are used to "
|
|
"represent the interactions between spins.",
|
|
["Quantum_chromodynamics"],
|
|
)
|
|
assert d["kind"] == "title_mismatch"
|
|
assert d["overlap"] == []
|
|
# Claim has 'spin', 'glass', 'modeling', etc. — title has 'quantum',
|
|
# 'chromodynamics'. Zero stem overlap.
|
|
assert "spin" in d["claim_tokens"]
|
|
assert "quantum" in d["title_tokens"]
|
|
|
|
|
|
def test_title_match_on_lois_lane_cited_for_supermans_girlfriend():
|
|
"""Sanity: legit citation (Lois Lane article cited for Lois-Lane
|
|
claim) passes title-relevance even though Superman isn't in the
|
|
title."""
|
|
d = diagnose_title_relevance(
|
|
"Lois Lane is the longtime love interest of Superman.",
|
|
["Lois_Lane"],
|
|
)
|
|
assert d["kind"] == "title_match"
|
|
assert "lois" in d["overlap"]
|
|
assert "lane" in d["overlap"]
|
|
|
|
|
|
def test_title_match_underscores_normalized():
|
|
"""Wikipedia titles use underscores; normalization to spaces lets
|
|
multi-word titles tokenize correctly."""
|
|
d = diagnose_title_relevance(
|
|
"The Berlin Wall fell in 1989.",
|
|
["Berlin_Wall"],
|
|
)
|
|
assert d["kind"] == "title_match"
|
|
assert "berlin" in d["overlap"]
|
|
assert "wall" in d["overlap"]
|
|
|
|
|
|
def test_title_mismatch_no_titles():
|
|
"""Defensive: empty title list returns no_titles."""
|
|
d = diagnose_title_relevance("Some claim.", [])
|
|
assert d["kind"] == "no_titles"
|
|
|
|
|
|
def test_title_mismatch_no_claim_tokens():
|
|
"""Defensive: empty/stopword-only claim returns no_claim_tokens."""
|
|
d = diagnose_title_relevance("the a an", ["Anything"])
|
|
assert d["kind"] == "no_claim_tokens"
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
# Metaphor-deflection sidecar
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_metaphor_deflection_swallowtail_canary():
|
|
"""The original 2026-05-02 emergent log case: swallowtail butterfly
|
|
question framed metaphorically (gracefully fluttering amidst the
|
|
rockiest terrain undeterred by upbraiding winds), answer was
|
|
purely literal taxonomic (Macleay's Swallowtail found in Eastern
|
|
Australia ...). Sidecar should fire."""
|
|
q = (
|
|
"How can a swallowtail butterfly, gracefully fluttering amidst "
|
|
"the rockiest terrain, remain undeterred by the upbraiding "
|
|
"winds that seem to challenge its delicate flight?"
|
|
)
|
|
a = (
|
|
"The Macleay's Swallowtail butterfly is found in Eastern "
|
|
"Australia including the ACT, New South Wales, Queensland, "
|
|
"Victoria and Tasmania."
|
|
)
|
|
d = diagnose_metaphor_deflection(q, a)
|
|
assert d["kind"] == "metaphor_deflection"
|
|
# cue_count should pick up at least: amidst, fluttering, gracefully,
|
|
# rockiest, upbraiding (≥5 real cues; butterfly filtered).
|
|
assert d["cue_count"] >= 4
|
|
assert "amidst" in d["cue_tokens"]
|
|
assert "rockiest" in d["cue_tokens"]
|
|
assert "upbraiding" in d["cue_tokens"]
|
|
assert "gracefully" in d["cue_tokens"]
|
|
assert "butterfly" not in d["cue_tokens"] # filtered noun
|
|
assert d["answer_overlap_count"] == 0
|
|
|
|
|
|
def test_metaphor_deflection_no_signal_on_literal_question():
|
|
"""Plain factual questions don't have metaphor cues; sidecar
|
|
returns no_signal."""
|
|
d = diagnose_metaphor_deflection(
|
|
"who painted the mona lisa?",
|
|
"Leonardo da Vinci painted the Mona Lisa around 1503.",
|
|
)
|
|
assert d["kind"] == "no_signal"
|
|
assert d["cue_count"] == 0
|
|
|
|
|
|
def test_metaphor_deflection_no_signal_when_answer_engages_cues():
|
|
"""If the answer echoes any of the question's metaphor cues, the
|
|
sidecar does NOT fire — the model engaged with the framing."""
|
|
q = "gracefully amidst rockiest terrain undeterred by upbraiding winds"
|
|
a = "The bird flies gracefully amidst the rockiest terrain undeterred."
|
|
d = diagnose_metaphor_deflection(q, a)
|
|
assert d["kind"] == "no_signal"
|
|
assert d["answer_overlap_count"] >= 1
|
|
|
|
|
|
def test_metaphor_deflection_filters_common_ly_nouns():
|
|
"""Naive .endswith('ly') would pick up 'butterfly', 'family',
|
|
'italy', 'july' etc. as adverbs. The block-list filters them so
|
|
they don't inflate cue count."""
|
|
from arborist.qa.inspect import _extract_metaphor_cues
|
|
cues = _extract_metaphor_cues(
|
|
"the butterfly flew over italy in july with the family"
|
|
)
|
|
# None of the -ly-suffix nouns should appear as cues.
|
|
for noun in ("butterfly", "italy", "july", "family"):
|
|
assert noun not in cues, f"{noun} leaked through as a cue"
|
|
|
|
|
|
def test_metaphor_deflection_under_threshold_returns_no_signal():
|
|
"""Sidecar requires >=3 cue tokens to fire — a single -ly word
|
|
isn't enough signal to flag metaphor framing."""
|
|
d = diagnose_metaphor_deflection(
|
|
"What gracefully describes a circle?",
|
|
"A circle is the set of points equidistant from a center.",
|
|
)
|
|
assert d["kind"] == "no_signal"
|
|
assert d["cue_count"] < 3
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
# Wordlist configurability — supplemental dictionaries
|
|
# ─────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_register_metaphor_dictionary_unions_into_wordlist(tmp_path):
|
|
"""Custom wordlist registers, unions into the default, and the
|
|
suffix tests pick up domain-specific stems."""
|
|
import arborist.qa.inspect as m
|
|
|
|
# Reset cache so the test sees a clean slate.
|
|
saved_cache = m._english_wordlist_cache
|
|
saved_extra = list(m._extra_dict_paths)
|
|
m._english_wordlist_cache = None
|
|
m._extra_dict_paths = []
|
|
|
|
try:
|
|
wl_default = m._english_wordlist()
|
|
size_default = len(wl_default)
|
|
|
|
custom = tmp_path / "domain.txt"
|
|
custom.write_text("foofoogeneous\nzaplet\n")
|
|
m.register_metaphor_dictionary(custom)
|
|
|
|
wl_after = m._english_wordlist()
|
|
assert "foofoogeneous" in wl_after
|
|
assert "zaplet" in wl_after
|
|
assert len(wl_after) >= size_default + 2
|
|
|
|
# Suffix tests now classify domain-specific adverbs.
|
|
assert m._is_adverbial_ly("foofoogeneously") # stem in custom dict
|
|
finally:
|
|
m._english_wordlist_cache = saved_cache
|
|
m._extra_dict_paths = saved_extra
|
|
|
|
|
|
def test_arborist_metaphor_dicts_env_var_supplements(monkeypatch, tmp_path):
|
|
"""Setting ARBORIST_METAPHOR_DICTS=path1:path2 unions both into
|
|
the default wordlist on first lookup."""
|
|
import arborist.qa.inspect as m
|
|
|
|
saved_cache = m._english_wordlist_cache
|
|
m._english_wordlist_cache = None
|
|
|
|
try:
|
|
d1 = tmp_path / "d1.txt"
|
|
d1.write_text("widgetspeak\n")
|
|
d2 = tmp_path / "d2.txt"
|
|
d2.write_text("frizzlebop\n")
|
|
monkeypatch.setenv("ARBORIST_METAPHOR_DICTS", f"{d1}:{d2}")
|
|
|
|
wl = m._english_wordlist()
|
|
assert "widgetspeak" in wl
|
|
assert "frizzlebop" in wl
|
|
finally:
|
|
m._english_wordlist_cache = saved_cache
|
|
|
|
|
|
def test_register_metaphor_dictionary_idempotent(tmp_path):
|
|
"""Registering the same path twice is a no-op (cache invalidates
|
|
once, second call is a no-op since path already in list)."""
|
|
import arborist.qa.inspect as m
|
|
|
|
saved_extra = list(m._extra_dict_paths)
|
|
saved_cache = m._english_wordlist_cache
|
|
m._extra_dict_paths = []
|
|
m._english_wordlist_cache = None
|
|
|
|
try:
|
|
custom = tmp_path / "d.txt"
|
|
custom.write_text("uniquewidget\n")
|
|
m.register_metaphor_dictionary(custom)
|
|
m.register_metaphor_dictionary(custom) # second call
|
|
# Path appears once.
|
|
assert m._extra_dict_paths.count(custom) == 1
|
|
finally:
|
|
m._extra_dict_paths = saved_extra
|
|
m._english_wordlist_cache = saved_cache
|
|
|
|
|
|
# ---------------------------------------------------------------------
|
|
# Authorship warrant ladder (#000026 Phase 3) — wired into inspect()
|
|
# ---------------------------------------------------------------------
|
|
|
|
|
|
def test_inspect_includes_authorship_field(tmp_path):
|
|
"""Every inspect() result carries an `authorship` sidecar dict.
|
|
For non-authorship questions, tier is NO_AUTHORSHIP_SIGNAL."""
|
|
qa_db = tmp_path / "qa.db"
|
|
shard = tmp_path / "001.db"
|
|
DOC = "ab" * 32
|
|
_seed_doc(
|
|
shard,
|
|
document_root=DOC,
|
|
document_uri="https://example.com/x",
|
|
chunk_text="Pikachu can store electricity in its cheeks.",
|
|
)
|
|
_seed_record(
|
|
qa_db, cache_key="01" * 32,
|
|
sources=[{
|
|
"document_root": DOC, "document_uri": "https://example.com/x",
|
|
"title": "X", "shard": shard.name, "chunk_idx": 0,
|
|
}],
|
|
unverified=[],
|
|
)
|
|
result = inspect_cache_key("01" * 32, qa_db=qa_db, shards_dir=tmp_path)
|
|
assert "authorship" in result
|
|
assert result["authorship"]["tier"] == "NO_AUTHORSHIP_SIGNAL"
|
|
|
|
|
|
def test_inspect_authorship_copyright_footer_tier(tmp_path):
|
|
"""Authorship-shaped question + copyright-footer chunk text →
|
|
AUTHOR_COPYRIGHT_FOOTER tier (the canonical virt-back case)."""
|
|
qa_db = tmp_path / "qa.db"
|
|
shard = tmp_path / "001.db"
|
|
DOC = "cd" * 32
|
|
_seed_doc(
|
|
shard,
|
|
document_root=DOC,
|
|
document_uri="https://example.com/virt-back",
|
|
chunk_text="virt-back is a backup utility.\n\n© Russell Ballestrini",
|
|
)
|
|
_seed_record(
|
|
qa_db, cache_key="02" * 32,
|
|
sources=[{
|
|
"document_root": DOC,
|
|
"document_uri": "https://example.com/virt-back",
|
|
"title": "virt-back", "shard": shard.name, "chunk_idx": 0,
|
|
}],
|
|
unverified=[],
|
|
question_text="who wrote virt-back?",
|
|
answer_text="Russell Ballestrini wrote virt-back.",
|
|
)
|
|
result = inspect_cache_key("02" * 32, qa_db=qa_db, shards_dir=tmp_path)
|
|
assert result["authorship"]["tier"] == "AUTHOR_COPYRIGHT_FOOTER"
|
|
assert "Russell Ballestrini" in result["authorship"]["candidate_names"]
|
|
|
|
|
|
def test_inspect_authorship_repo_owner_tier(tmp_path):
|
|
"""Repo-URL chunks fire tier 2 (REPOSITORY_OWNER)."""
|
|
qa_db = tmp_path / "qa.db"
|
|
shard = tmp_path / "001.db"
|
|
DOC = "ef" * 32
|
|
_seed_doc(
|
|
shard,
|
|
document_root=DOC,
|
|
document_uri="https://example.com/foo",
|
|
chunk_text="See https://github.com/russellballestrini/virt-back for source.",
|
|
)
|
|
_seed_record(
|
|
qa_db, cache_key="03" * 32,
|
|
sources=[{
|
|
"document_root": DOC,
|
|
"document_uri": "https://example.com/foo",
|
|
"title": "foo", "shard": shard.name, "chunk_idx": 0,
|
|
}],
|
|
unverified=[],
|
|
question_text="who maintains virt-back?",
|
|
answer_text="russellballestrini.",
|
|
)
|
|
result = inspect_cache_key("03" * 32, qa_db=qa_db, shards_dir=tmp_path)
|
|
assert result["authorship"]["tier"] == "AUTHOR_REPOSITORY_OWNER"
|
|
|
|
|
|
# --- diagnose_coherence (#000052 §3.1) ----------------------------------
|
|
|
|
|
|
def test_coherence_phrase_component_reuse_zionist_field_case():
|
|
# The 2026-05-12 field case: a phrase defined in terms of a word
|
|
# inside it. Lexical verifier scored it EVIDENCE-WARRANTED-PARTIAL;
|
|
# deflection/title-relevance both waved it through; NLI returns
|
|
# neutral. This sidecar is the one that catches it.
|
|
d = diagnose_coherence(
|
|
"The phrase 'Zionist entity' is sometimes used as the entity, "
|
|
"referring to the State of Israel."
|
|
)
|
|
assert d["kind"] == "phrase_component_reuse"
|
|
assert d["evidence"]["reused_tokens"] == ["entity"]
|
|
assert "entity" in d["evidence"]["quoted_phrase_tokens"]
|
|
|
|
|
|
def test_coherence_circular_x_is_x():
|
|
d = diagnose_coherence("Water is water.")
|
|
assert d["kind"] == "circular"
|
|
assert d["evidence"]["subject_tokens"] == ["water"]
|
|
|
|
|
|
def test_coherence_circular_subject_subset_of_predicate():
|
|
d = diagnose_coherence(
|
|
"The entity is the entity referring to the State of Israel."
|
|
)
|
|
assert d["kind"] == "circular"
|
|
|
|
|
|
def test_coherence_vacuous_predicate_only_hypernyms():
|
|
d = diagnose_coherence("Happiness refers to a concept used in some contexts.")
|
|
assert d["kind"] == "vacuous"
|
|
d2 = diagnose_coherence("Gravity is a thing.")
|
|
assert d2["kind"] == "vacuous"
|
|
|
|
|
|
def test_coherence_ok_on_well_formed_definitions():
|
|
for ans in (
|
|
"Paris is the capital of France.",
|
|
"A poodle is a type of dog.",
|
|
"The Beatles were a band formed in Liverpool in 1960.",
|
|
"The term 'open source' refers to software whose source code is "
|
|
"publicly available.",
|
|
"The phrase 'break a leg' is used as an idiom meaning good luck.",
|
|
):
|
|
assert diagnose_coherence(ans)["kind"] == "ok", ans
|
|
|
|
|
|
def test_coherence_copula_inside_quotes_does_not_break_split():
|
|
# 'war is peace' contains "is" — the split must not fire on it.
|
|
d = diagnose_coherence(
|
|
"The phrase 'war is peace' is a slogan from the novel "
|
|
"Nineteen Eighty-Four."
|
|
)
|
|
assert d["kind"] == "ok"
|
|
|
|
|
|
def test_coherence_empty_and_pronoun_subject_are_not_flagged():
|
|
assert diagnose_coherence("")["kind"] == "empty"
|
|
assert diagnose_coherence(" \n ")["kind"] == "empty"
|
|
# pronoun subject → no real topic token → conservatively not flagged
|
|
assert diagnose_coherence("It is something used in various contexts.")[
|
|
"kind"
|
|
] == "ok"
|
|
|
|
|
|
def test_coherence_reports_all_findings_and_worst_kind():
|
|
d = diagnose_coherence(
|
|
"- The phrase 'Zionist entity' is used as the entity.\n"
|
|
"- Happiness is a concept."
|
|
)
|
|
assert d["kind"] == "phrase_component_reuse" # most severe wins
|
|
kinds = {f["kind"] for f in d["findings"]}
|
|
assert kinds == {"phrase_component_reuse", "vacuous"}
|
|
assert d["n_sentences"] == 2
|
|
|
|
|
|
def test_coherence_is_pure_no_side_effects():
|
|
# No DB, no files — diagnose_coherence is a pure function over text.
|
|
before = diagnose_coherence("Water is water.")
|
|
after = diagnose_coherence("Water is water.")
|
|
assert before == after
|
|
|
|
|
|
# --- diagnose_coherence broader fixture coverage (#000052 §3.1 round-2) -----
|
|
# Added 2026-05-13 after bench-maxing §3.1 against the 808-cell pooled
|
|
# bench-qa STRICT set (`bench/qa_results/2026-05-12T{20-53-11Z,21-58-58Z,
|
|
# 22-44-30Z}.jsonl` → 5.4% FP rate). The tests below split into:
|
|
# (a) MORE positive coverage — incoherent shapes that should still flag
|
|
# (b) REGRESSION xfail tests — exact false-positive shapes from real
|
|
# bench-qa STRICT output; track until the detector rules tighten.
|
|
# Pure lexical, no model, no I/O — runs in the default suite.
|
|
|
|
def test_coherence_more_circular_named_entity():
|
|
# "X's Y is X's Y" — pronominal possessive reuse
|
|
d = diagnose_coherence("Newton's laws are Newton's laws.")
|
|
assert d["kind"] == "circular"
|
|
|
|
|
|
def test_coherence_more_phrase_component_reuse_grammar_term():
|
|
# the "term <X>" framing — "<X>" is defined as a bare token from inside the phrase
|
|
d = diagnose_coherence(
|
|
"The term 'binary search' refers to a binary search of an ordered list."
|
|
)
|
|
assert d["kind"] == "phrase_component_reuse"
|
|
|
|
|
|
def test_coherence_more_vacuous_pure_hypernym_chain():
|
|
# multi-sentence vacuous chain
|
|
d = diagnose_coherence(
|
|
"Happiness is a thing. Joy refers to a concept. Wellbeing is something used in various contexts."
|
|
)
|
|
assert d["kind"] == "vacuous"
|
|
assert len(d["findings"]) >= 2
|
|
|
|
|
|
import pytest as _pytest
|
|
|
|
|
|
# The 5 tests below were xfail regressions documenting bench-qa STRICT
|
|
# false-positives; the rules have now been tightened in
|
|
# arborist/qa/inspect.py to fix them (§3.1 round-2 patch 2026-05-13).
|
|
|
|
def test_coherence_ok_on_short_concrete_fact_with_acronym_value():
|
|
"""Was xfail: 'The chemical symbol for gold is Au.' → vacuous.
|
|
Fix: vacuous rule now treats short acronym/symbol tokens (Au, Fe,
|
|
DNA, FBI, etc.) as content via
|
|
``_coherence_predicate_has_short_acronym_content``."""
|
|
d = diagnose_coherence("The chemical symbol for gold is Au.")
|
|
assert d["kind"] == "ok"
|
|
# element-symbol variant:
|
|
assert diagnose_coherence("Iron has the chemical symbol Fe.")["kind"] == "ok"
|
|
# all-caps acronym:
|
|
assert diagnose_coherence("The acronym for the agency is FBI.")["kind"] == "ok"
|
|
|
|
|
|
def test_coherence_ok_on_named_subject_with_repeated_head_noun():
|
|
"""Was xfail: 'X's Y was a Y ... named after X' shape.
|
|
Fix: circular now requires (predicate-all-vacuous) OR
|
|
(leads-with-subject AND non-subject differentia ≤ 2). The MJ
|
|
Restaurant predicate has 6 non-subject differentia
|
|
(restaurant, chicago, illinois, basketball, player, named) so
|
|
no longer fires."""
|
|
d = diagnose_coherence(
|
|
"Michael Jordan's Restaurant was a restaurant in Chicago, Illinois, "
|
|
"named after the basketball player Michael Jordan."
|
|
)
|
|
assert d["kind"] == "ok"
|
|
|
|
|
|
def test_coherence_ok_on_compound_noun_definition():
|
|
"""Was xfail: 'the Western X was the western half of the X'.
|
|
Same circular-differentia-cap fix as MJ."""
|
|
d = diagnose_coherence(
|
|
"The Western Roman Empire was the western half of the Roman Empire, "
|
|
"from its division by Diocletian in 285 AD until its fall."
|
|
)
|
|
assert d["kind"] == "ok"
|
|
|
|
|
|
def test_coherence_ok_on_translation_etymology_with_derivative_token():
|
|
"""Was xfail: 'The name <Phrase> is a translation … <derivative>'.
|
|
Fix: phrase_component_reuse now suppresses when the predicate
|
|
contains another quoted phrase (signals a translation /
|
|
definition / etymology chain, not circularity)."""
|
|
d = diagnose_coherence(
|
|
"The name 'Rosebud River' is a translation of the Blackfoot word "
|
|
"Akokiniskway, meaning 'the river of the roses'."
|
|
)
|
|
assert d["kind"] == "ok"
|
|
|
|
|
|
def test_coherence_ok_on_claim_lattice_truncated_bracket_tail():
|
|
"""Was xfail: claim-lattice [E\\d | …\"] artifacts parsed as
|
|
vacuous sentences. Fix: sentences matching the bracket-artifact
|
|
regex (``\\[E\\d+\\s*\\|`` opener or ``...\"]`` truncation
|
|
tail) are skipped entirely in classify."""
|
|
d = diagnose_coherence(
|
|
"Linux and BSD are both Unix-like operating systems. "
|
|
"[E1 | Linux | abcd1234: \"Linux is a Unix-like operating system kernel "
|
|
"first released by Linus Torvalds in 1991. Such a thesis was...\"]"
|
|
)
|
|
assert d["kind"] == "ok"
|
|
|
|
|
|
@_pytest.mark.xfail(reason="§3.1 false-positive: 'The term <X>' framing where the quoted phrase's head token appears in the predicate as a generic referent (idiomatic encyclopedic English). Tracked from real bench-qa STRICT regression. Borderline — the SHAPE matches phrase_component_reuse, but the idiom is legitimate.")
|
|
def test_coherence_ok_on_term_idiom():
|
|
d = diagnose_coherence(
|
|
"The term \"traditional Unix\" may be used to describe a Unix or an "
|
|
"operating system that has the characteristics of early Unix versions."
|
|
)
|
|
assert d["kind"] == "ok"
|
|
|
|
|
|
def test_coherence_pooled_bench_qa_strict_fp_rate_documented():
|
|
"""Document the load-bearing real-traffic FP rate so future work
|
|
can be measured against it. The pooled bench-qa STRICT sample
|
|
(n=1 + n=3 + n=5 ARBORIST_NLI_SHADOW=1 bench-qa runs = 808 cells)
|
|
has been the §3.1 regression target:
|
|
2026-05-13 (initial measurement, pre-patch): 44/808 = 5.4% FP
|
|
2026-05-13 (round-2 rule tightening): 9/808 = 1.1% FP
|
|
— bracket-artifact skip + circular-differentia cap +
|
|
translation-phrase-pair exception + short-acronym vacuous
|
|
escape (5 of 6 §3.1 round-2 xfail regressions fixed; the
|
|
'term <X>' idiomatic-encyclopedic-English case stays xfail).
|
|
This test asserts the *upper bound* — tightening the rules should
|
|
keep it at or below the post-patch number. If a future change
|
|
pushes it UP past 2%, this test fails loud."""
|
|
import json as _json
|
|
from pathlib import Path as _Path
|
|
# this is a slow-ish test (~1s for 808 rows of lexical regex);
|
|
# skip if the bench-qa JSONL files aren't present (e.g. fresh checkout)
|
|
files = [_Path(p) for p in [
|
|
"bench/qa_results/2026-05-12T20-53-11Z.jsonl",
|
|
"bench/qa_results/2026-05-12T21-58-58Z.jsonl",
|
|
"bench/qa_results/2026-05-12T22-44-30Z.jsonl",
|
|
]]
|
|
if not all(p.exists() for p in files):
|
|
_pytest.skip("pooled bench-qa STRICT files not present (gitignored — run ARBORIST_NLI_SHADOW=1 make bench-qa to generate)")
|
|
rows = []
|
|
for p in files:
|
|
for ln in p.read_text().splitlines():
|
|
o = _json.loads(ln.strip())
|
|
if o.get("audit_mode") == "STRICT" and o.get("answer_text"):
|
|
rows.append(o)
|
|
flagged = sum(1 for r in rows
|
|
if diagnose_coherence(r["answer_text"])["kind"] not in ("ok", "empty"))
|
|
assert len(rows) >= 800, f"pooled STRICT sample shrunk unexpectedly ({len(rows)} rows; expected ~808)"
|
|
# post round-2 rate: 9/808 = 0.011; ceiling at 0.02 leaves a tiny
|
|
# bit of headroom for fixture churn but fires loud on regressions.
|
|
assert flagged / len(rows) <= 0.02, \
|
|
f"§3.1 FP rate on real STRICT regressed: {flagged}/{len(rows)} = {flagged/len(rows):.3f}"
|