russell@unturf.com
221b784a80
`arborist.qa.evidence._content_tokens` dropped every token under 4 chars, so a short all-caps acronym (CPU, GPU, DNA, FBI, USB…) never registered as a content token — which defeated Rule 8 (_claim_title_overlap / TITLE_MISMATCH), the subject-tokens-absent check (Rule 9), the bare-name-claim guard, and spotlight-excerpt token selection whenever a question/claim's topic IS an acronym. The field case: `what is a CPU?` cited to the "CPU design" article tripped TITLE_MISMATCH even though claim and title both contain "CPU". Fix: keep a token if it's an all-caps 2-3-char alpha run in the source text; everything else unchanged. The change only ever ADDS tokens, so TITLE_MISMATCH / SUBJECT_TOKENS_ABSENT / BARE_NAME_CLAIM can only stop firing, never start — monotone toward fewer spurious demotes; no STRICT→non-STRICT transition is possible from it. Versioned: `content_token_rules: "v2-acronym-aware"` added to runner.DEFAULT_POLICY + query.DEFAULT_QUERY_POLICY + keys._VERIFIER_POLICY_FIELDS → folds into verifier_policy_hash, prior cache records orphan on lookup (by design; same discipline as base_version / hyphen_fold_v1). Does NOT touch the retrieval abbreviation→expansion gap (CPU→Central processing unit — #000050 vec hybrid / concepts/ synonym edges; the root cause of the satellite retrieval). 8 new tests; full suite green (2502); bench-qa-smoke clean. Next ID 000053 -> 000054.
67 lines
2.5 KiB
Python
67 lines
2.5 KiB
Python
"""Verifier content-token rule — acronym awareness (#000053).
|
|
|
|
`_content_tokens` drops <4-char tokens *except* all-caps 2-3-char
|
|
acronyms (CPU/GPU/DNA/FBI/USB…). Pre-#000053 it dropped every short
|
|
token, so a CPU/GPU claim cited to a "CPU foo" / "GPU bar" article
|
|
tripped TITLE_MISMATCH spuriously (the shared "CPU" token didn't
|
|
register on either side).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from arborist.qa.evidence import _content_tokens
|
|
from arborist.qa.keys import _VERIFIER_POLICY_FIELDS, verifier_policy_hash
|
|
from arborist.qa.runner import DEFAULT_POLICY
|
|
from arborist.qa.query import DEFAULT_QUERY_POLICY
|
|
from arborist.qa.verify import _claim_title_overlap
|
|
|
|
|
|
def test_uppercase_acronym_survives_short_token_filter():
|
|
toks = _content_tokens("A CPU, or central processing unit, is hardware.")
|
|
assert "cpu" in toks
|
|
assert "central" in toks and "processing" in toks
|
|
|
|
|
|
def test_lowercase_short_word_still_dropped():
|
|
# only the *all-caps* form is rescued; a 3-char lowercase word stays out
|
|
toks = _content_tokens("the cat sat and ran far")
|
|
assert "cat" not in toks and "sat" not in toks and "ran" not in toks
|
|
|
|
|
|
def test_two_and_three_char_caps_only():
|
|
toks = _content_tokens("GPU DNA FBI US AI ABCD running")
|
|
assert {"gpu", "dna", "fbi", "us", "ai"} <= set(toks)
|
|
assert "abcd" in toks # 4 chars — passes the normal filter anyway
|
|
assert "running" in toks
|
|
|
|
|
|
def test_punctuation_stripped_before_acronym_check():
|
|
assert "cpu" in _content_tokens("(CPU). \"GPU,\"")
|
|
assert "gpu" in _content_tokens("(CPU). \"GPU,\"")
|
|
|
|
|
|
def test_rule8_title_overlap_now_passes_on_shared_acronym():
|
|
# the field case: a CPU claim cited to the "CPU design" article —
|
|
# they share "CPU", which now counts as a content token.
|
|
assert _claim_title_overlap(
|
|
"A CPU is the central processing unit of a computer.", "CPU design"
|
|
)
|
|
|
|
|
|
def test_rule8_still_rejects_when_no_overlap():
|
|
assert not _claim_title_overlap(
|
|
"A CPU is the central processing unit of a computer.",
|
|
"Quantum chromodynamics",
|
|
)
|
|
|
|
|
|
def test_content_token_rules_marker_is_in_policies_and_hash_field_set():
|
|
assert DEFAULT_POLICY["content_token_rules"] == "v2-acronym-aware"
|
|
assert DEFAULT_QUERY_POLICY["content_token_rules"] == "v2-acronym-aware"
|
|
assert "content_token_rules" in _VERIFIER_POLICY_FIELDS
|
|
|
|
|
|
def test_verifier_policy_hash_tracks_content_token_rules():
|
|
a = verifier_policy_hash(DEFAULT_POLICY)
|
|
b = verifier_policy_hash(dict(DEFAULT_POLICY, content_token_rules="v1"))
|
|
assert a != b
|