#000052 §3.1: diagnose_coherence — lexical answer-shape sidecar
Per-sentence shape check (no model) emitting kind ∈
{phrase_component_reuse, circular, vacuous, ok, empty}:
- circular: subject content-tokens ⊆ predicate's and the predicate
leads with a subject token ("Water is water").
- phrase_component_reuse: subject quotes a phrase, predicate reuses
one of that phrase's own tokens as a bare "the/a/an <token>"
referent — the 2026-05-12 field case ("the phrase 'Zionist entity'
is used as the entity"), a token collision the verifier +
deflection + title-relevance all pass and NLI returns neutral on.
Copulas inside a quoted span are skipped so 'war is peace' doesn't
break the subject/predicate split.
- vacuous: predicate is only placeholder hypernyms + filler ("X is
a thing").
Conservative — no full token-salad parsing; legit definitions pass ok.
Surfaced in inspect_cache_key + the `arborist inspect` human view
(· incoherent: <kind>). Advisory only — never writes providence_cache
/ audit_events / run_dag_root; demote-only verifier hook deliberately
not wired. 9 tests; full suite green (2500 passed).
This commit is contained in:
parent
42a0f933ff
commit
96ff586da1
6 changed files with 420 additions and 11 deletions
|
|
@ -21,6 +21,7 @@ import pytest
|
|||
from arborist.qa.inspect import (
|
||||
_classify_span,
|
||||
_normalize,
|
||||
diagnose_coherence,
|
||||
diagnose_deflection,
|
||||
diagnose_metaphor_deflection,
|
||||
diagnose_title_relevance,
|
||||
|
|
@ -852,3 +853,88 @@ def test_inspect_authorship_repo_owner_tier(tmp_path):
|
|||
)
|
||||
result = inspect_cache_key("03" * 32, qa_db=qa_db, shards_dir=tmp_path)
|
||||
assert result["authorship"]["tier"] == "AUTHOR_REPOSITORY_OWNER"
|
||||
|
||||
|
||||
# --- diagnose_coherence (#000052 §3.1) ----------------------------------
|
||||
|
||||
|
||||
def test_coherence_phrase_component_reuse_zionist_field_case():
|
||||
# The 2026-05-12 field case: a phrase defined in terms of a word
|
||||
# inside it. Lexical verifier scored it EVIDENCE-WARRANTED-PARTIAL;
|
||||
# deflection/title-relevance both waved it through; NLI returns
|
||||
# neutral. This sidecar is the one that catches it.
|
||||
d = diagnose_coherence(
|
||||
"The phrase 'Zionist entity' is sometimes used as the entity, "
|
||||
"referring to the State of Israel."
|
||||
)
|
||||
assert d["kind"] == "phrase_component_reuse"
|
||||
assert d["evidence"]["reused_tokens"] == ["entity"]
|
||||
assert "entity" in d["evidence"]["quoted_phrase_tokens"]
|
||||
|
||||
|
||||
def test_coherence_circular_x_is_x():
|
||||
d = diagnose_coherence("Water is water.")
|
||||
assert d["kind"] == "circular"
|
||||
assert d["evidence"]["subject_tokens"] == ["water"]
|
||||
|
||||
|
||||
def test_coherence_circular_subject_subset_of_predicate():
|
||||
d = diagnose_coherence(
|
||||
"The entity is the entity referring to the State of Israel."
|
||||
)
|
||||
assert d["kind"] == "circular"
|
||||
|
||||
|
||||
def test_coherence_vacuous_predicate_only_hypernyms():
|
||||
d = diagnose_coherence("Happiness refers to a concept used in some contexts.")
|
||||
assert d["kind"] == "vacuous"
|
||||
d2 = diagnose_coherence("Gravity is a thing.")
|
||||
assert d2["kind"] == "vacuous"
|
||||
|
||||
|
||||
def test_coherence_ok_on_well_formed_definitions():
|
||||
for ans in (
|
||||
"Paris is the capital of France.",
|
||||
"A poodle is a type of dog.",
|
||||
"The Beatles were a band formed in Liverpool in 1960.",
|
||||
"The term 'open source' refers to software whose source code is "
|
||||
"publicly available.",
|
||||
"The phrase 'break a leg' is used as an idiom meaning good luck.",
|
||||
):
|
||||
assert diagnose_coherence(ans)["kind"] == "ok", ans
|
||||
|
||||
|
||||
def test_coherence_copula_inside_quotes_does_not_break_split():
|
||||
# 'war is peace' contains "is" — the split must not fire on it.
|
||||
d = diagnose_coherence(
|
||||
"The phrase 'war is peace' is a slogan from the novel "
|
||||
"Nineteen Eighty-Four."
|
||||
)
|
||||
assert d["kind"] == "ok"
|
||||
|
||||
|
||||
def test_coherence_empty_and_pronoun_subject_are_not_flagged():
|
||||
assert diagnose_coherence("")["kind"] == "empty"
|
||||
assert diagnose_coherence(" \n ")["kind"] == "empty"
|
||||
# pronoun subject → no real topic token → conservatively not flagged
|
||||
assert diagnose_coherence("It is something used in various contexts.")[
|
||||
"kind"
|
||||
] == "ok"
|
||||
|
||||
|
||||
def test_coherence_reports_all_findings_and_worst_kind():
|
||||
d = diagnose_coherence(
|
||||
"- The phrase 'Zionist entity' is used as the entity.\n"
|
||||
"- Happiness is a concept."
|
||||
)
|
||||
assert d["kind"] == "phrase_component_reuse" # most severe wins
|
||||
kinds = {f["kind"] for f in d["findings"]}
|
||||
assert kinds == {"phrase_component_reuse", "vacuous"}
|
||||
assert d["n_sentences"] == 2
|
||||
|
||||
|
||||
def test_coherence_is_pure_no_side_effects():
|
||||
# No DB, no files — diagnose_coherence is a pure function over text.
|
||||
before = diagnose_coherence("Water is water.")
|
||||
after = diagnose_coherence("Water is water.")
|
||||
assert before == after
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue