#000052 §3.1: diagnose_coherence — lexical answer-shape sidecar

Per-sentence shape check (no model) emitting kind ∈
{phrase_component_reuse, circular, vacuous, ok, empty}:
  - circular: subject content-tokens ⊆ predicate's and the predicate
    leads with a subject token ("Water is water").
  - phrase_component_reuse: subject quotes a phrase, predicate reuses
    one of that phrase's own tokens as a bare "the/a/an <token>"
    referent — the 2026-05-12 field case ("the phrase 'Zionist entity'
    is used as the entity"), a token collision the verifier +
    deflection + title-relevance all pass and NLI returns neutral on.
    Copulas inside a quoted span are skipped so 'war is peace' doesn't
    break the subject/predicate split.
  - vacuous: predicate is only placeholder hypernyms + filler ("X is
    a thing").

Conservative — no full token-salad parsing; legit definitions pass ok.
Surfaced in inspect_cache_key + the `arborist inspect` human view
(· incoherent: <kind>). Advisory only — never writes providence_cache
/ audit_events / run_dag_root; demote-only verifier hook deliberately
not wired. 9 tests; full suite green (2500 passed).
This commit is contained in:
russell@unturf.com 2026-05-12 17:19:07 -04:00
parent 42a0f933ff
commit 96ff586da1
No known key found for this signature in database
6 changed files with 420 additions and 11 deletions

View file

@ -21,6 +21,7 @@ import pytest
from arborist.qa.inspect import (
_classify_span,
_normalize,
diagnose_coherence,
diagnose_deflection,
diagnose_metaphor_deflection,
diagnose_title_relevance,
@ -852,3 +853,88 @@ def test_inspect_authorship_repo_owner_tier(tmp_path):
)
result = inspect_cache_key("03" * 32, qa_db=qa_db, shards_dir=tmp_path)
assert result["authorship"]["tier"] == "AUTHOR_REPOSITORY_OWNER"
# --- diagnose_coherence (#000052 §3.1) ----------------------------------
def test_coherence_phrase_component_reuse_zionist_field_case():
# The 2026-05-12 field case: a phrase defined in terms of a word
# inside it. Lexical verifier scored it EVIDENCE-WARRANTED-PARTIAL;
# deflection/title-relevance both waved it through; NLI returns
# neutral. This sidecar is the one that catches it.
d = diagnose_coherence(
"The phrase 'Zionist entity' is sometimes used as the entity, "
"referring to the State of Israel."
)
assert d["kind"] == "phrase_component_reuse"
assert d["evidence"]["reused_tokens"] == ["entity"]
assert "entity" in d["evidence"]["quoted_phrase_tokens"]
def test_coherence_circular_x_is_x():
d = diagnose_coherence("Water is water.")
assert d["kind"] == "circular"
assert d["evidence"]["subject_tokens"] == ["water"]
def test_coherence_circular_subject_subset_of_predicate():
d = diagnose_coherence(
"The entity is the entity referring to the State of Israel."
)
assert d["kind"] == "circular"
def test_coherence_vacuous_predicate_only_hypernyms():
d = diagnose_coherence("Happiness refers to a concept used in some contexts.")
assert d["kind"] == "vacuous"
d2 = diagnose_coherence("Gravity is a thing.")
assert d2["kind"] == "vacuous"
def test_coherence_ok_on_well_formed_definitions():
for ans in (
"Paris is the capital of France.",
"A poodle is a type of dog.",
"The Beatles were a band formed in Liverpool in 1960.",
"The term 'open source' refers to software whose source code is "
"publicly available.",
"The phrase 'break a leg' is used as an idiom meaning good luck.",
):
assert diagnose_coherence(ans)["kind"] == "ok", ans
def test_coherence_copula_inside_quotes_does_not_break_split():
# 'war is peace' contains "is" — the split must not fire on it.
d = diagnose_coherence(
"The phrase 'war is peace' is a slogan from the novel "
"Nineteen Eighty-Four."
)
assert d["kind"] == "ok"
def test_coherence_empty_and_pronoun_subject_are_not_flagged():
assert diagnose_coherence("")["kind"] == "empty"
assert diagnose_coherence(" \n ")["kind"] == "empty"
# pronoun subject → no real topic token → conservatively not flagged
assert diagnose_coherence("It is something used in various contexts.")[
"kind"
] == "ok"
def test_coherence_reports_all_findings_and_worst_kind():
d = diagnose_coherence(
"- The phrase 'Zionist entity' is used as the entity.\n"
"- Happiness is a concept."
)
assert d["kind"] == "phrase_component_reuse" # most severe wins
kinds = {f["kind"] for f in d["findings"]}
assert kinds == {"phrase_component_reuse", "vacuous"}
assert d["n_sentences"] == 2
def test_coherence_is_pure_no_side_effects():
# No DB, no files — diagnose_coherence is a pure function over text.
before = diagnose_coherence("Water is water.")
after = diagnose_coherence("Water is water.")
assert before == after