From 5f4f4ceb1c9277af183c5c26fb66753715a360a0 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 13 May 2026 12:38:09 -0400 Subject: [PATCH] =?UTF-8?q?#000052:=20more=20tests=20for=20=C2=A73.1=20+?= =?UTF-8?q?=20=C2=A73.2=20=E2=80=94=20bench-max=20the=20detectors=20agains?= =?UTF-8?q?t=20real=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §3.1 diagnose_coherence (now 19 tests, +10 from the parallel session's 9): - 3 more positive shapes (multi-sentence vacuous, named-entity circular, grammar-term phrase_component_reuse). - 5 xfail regression tests for SHAPES THAT FALSE-FIRE on real bench-qa STRICT data (44/808 = 5.4% FP rate measured on the pooled n=1+3+5 STRICT answers). Each xfail names the exact shape + why it should ideally be 'ok' + which rule needs tightening: * 'The chemical symbol for gold is Au.' → vacuous (short predicate) * 'Michael Jordan's Restaurant was a restaurant ... named after Michael Jordan.' → circular (named-after re-use) * 'The Western X was the western half of the X' → circular * 'The name is a translation ... of the ' → phrase_component_reuse (translation/etymology) * claim-lattice [E1 | … …"] tails → vacuous (truncated bracket fragment) * 'The term ' → phrase_component_reuse (idiomatic English) - 1 load-bearing real-traffic test: FP rate on 808-cell pooled STRICT must stay ≤ 7% (current 5.4%) — fires loud if a future change regresses it. Skips on fresh-checkout (bench/qa_results/ gitignored). §3.2 ShadowRelevance (now 20 tests, +7 from the round-1 scaffold): - Manifest tests for round-2 primary (bge-reranker-large), the size spectrum coverage (50-560MB), the candidate-bench findings block (biggest-within-family / not-across-families / deeper-not-better / capacity-floor). - Pair-kind distinction (question_answer vs claim_source recorded separately for downstream telemetry / governance hashing). - Batch-order preservation (_score_batch must return scores in input order — load-bearing for downstream zip-back). - Empty-input handling (Q empty, D empty, whitespace-only). - Zionist-entity discriminator sanity (on-topic > off-topic logit). - demote_below_score-stays-null invariant (the §7 #18→#27 discipline: no hardcoded threshold; must come from a real-traffic shadow sweep). Total: 101 passed + 6 xfailed (5 §3.1 regressions documented + 1 from parallel session). The 5 xfails are the bench-maxing receipts — they document EXACTLY which shapes §3.1 false-fires on, with the rule that needs tightening named in each reason. --- ...0052-relevance-coherence-meta-cognition.md | 2 +- tests/test_inspect.py | 123 ++++++++++++++++++ tests/test_relevance_shadow.py | 119 +++++++++++++++++ 3 files changed, 243 insertions(+), 1 deletion(-) diff --git a/docs/tickets/ticket-000052-relevance-coherence-meta-cognition.md b/docs/tickets/ticket-000052-relevance-coherence-meta-cognition.md index 123127c..c906ac9 100644 --- a/docs/tickets/ticket-000052-relevance-coherence-meta-cognition.md +++ b/docs/tickets/ticket-000052-relevance-coherence-meta-cognition.md @@ -251,7 +251,7 @@ structure verbatim. Pieces: (the Zionist-entity-style mis-cite). Both return a single relevance *logit* (higher = more relevant; **not a probability**) and a `would_demote` flag that stays `False` until a threshold is set. -- `tests/test_relevance_shadow.py` — 13 tests (pure-Python + graceful +- `tests/test_relevance_shadow.py` — 20 tests (pure-Python + graceful degradation + the Zionist sanity check); all green. **Candidate-bench round 2 (2026-05-13)** — 26-pair hand-built fixture diff --git a/tests/test_inspect.py b/tests/test_inspect.py index cf43d5c..8bdc26b 100644 --- a/tests/test_inspect.py +++ b/tests/test_inspect.py @@ -938,3 +938,126 @@ def test_coherence_is_pure_no_side_effects(): before = diagnose_coherence("Water is water.") after = diagnose_coherence("Water is water.") assert before == after + + +# --- diagnose_coherence broader fixture coverage (#000052 §3.1 round-2) ----- +# Added 2026-05-13 after bench-maxing §3.1 against the 808-cell pooled +# bench-qa STRICT set (`bench/qa_results/2026-05-12T{20-53-11Z,21-58-58Z, +# 22-44-30Z}.jsonl` → 5.4% FP rate). The tests below split into: +# (a) MORE positive coverage — incoherent shapes that should still flag +# (b) REGRESSION xfail tests — exact false-positive shapes from real +# bench-qa STRICT output; track until the detector rules tighten. +# Pure lexical, no model, no I/O — runs in the default suite. + +def test_coherence_more_circular_named_entity(): + # "X's Y is X's Y" — pronominal possessive reuse + d = diagnose_coherence("Newton's laws are Newton's laws.") + assert d["kind"] == "circular" + + +def test_coherence_more_phrase_component_reuse_grammar_term(): + # the "term " framing — "" is defined as a bare token from inside the phrase + d = diagnose_coherence( + "The term 'binary search' refers to a binary search of an ordered list." + ) + assert d["kind"] == "phrase_component_reuse" + + +def test_coherence_more_vacuous_pure_hypernym_chain(): + # multi-sentence vacuous chain + d = diagnose_coherence( + "Happiness is a thing. Joy refers to a concept. Wellbeing is something used in various contexts." + ) + assert d["kind"] == "vacuous" + assert len(d["findings"]) >= 2 + + +import pytest as _pytest + + +@_pytest.mark.xfail(reason="§3.1 false-positive: short-predicate concrete facts like 'The chemical symbol for gold is Au.' flagged as vacuous. The trailing 'Au' is a 2-char content-free token to the rule, but it IS the answer. Real bench-qa STRICT regression: this exact sentence appears 4× in the 808-cell pool, flagged each time. Tighten the vacuous rule to recognize a single-token-named entity as a sufficient predicate when it's not on the hypernym placeholder list.") +def test_coherence_ok_on_short_concrete_fact_with_acronym_value(): + d = diagnose_coherence("The chemical symbol for gold is Au.") + assert d["kind"] == "ok" + + +@_pytest.mark.xfail(reason="§3.1 false-positive: 'X's Y was a Y ... named after X' shape — subject's proper-noun tokens reappear at the predicate's end via 'named after / called'. Subject's tokens ⊆ predicate's tokens, but the predicate has real differentia. Tighten circular to require the predicate to be NEARLY-EMPTY-modulo-determiners, not just superset on head tokens. Real bench-qa STRICT regression: 'Michael Jordan's Restaurant ... named after the basketball player Michael Jordan' (the trailing 'Michael Jordan' is what trips it; the shorter variant without that trailing reference does NOT trip).") +def test_coherence_ok_on_named_subject_with_repeated_head_noun(): + d = diagnose_coherence( + "Michael Jordan's Restaurant was a restaurant in Chicago, Illinois, " + "named after the basketball player Michael Jordan." + ) + assert d["kind"] == "ok" + + +@_pytest.mark.xfail(reason="§3.1 false-positive: 'the Western X was the western half of the X' encyclopedic compound-noun shape — tracked from real bench-qa STRICT regression. The 'half/part of X' construction is a legitimate definitional relation, not a circular tautology.") +def test_coherence_ok_on_compound_noun_definition(): + d = diagnose_coherence( + "The Western Roman Empire was the western half of the Roman Empire, " + "from its division by Diocletian in 285 AD until its fall." + ) + assert d["kind"] == "ok" + + +@_pytest.mark.xfail(reason="§3.1 false-positive: a quoted phrase containing a noun whose plural/derivative appears legitimately in the predicate (translation/etymology context) — 'Rosebud River' phrase, 'roses' in the predicate. Tracked from real bench-qa STRICT regression. Tighten phrase_component_reuse to require the bare-referent token to be USED AS THE SAME REFERENT as the phrase's head, not just lexically related.") +def test_coherence_ok_on_translation_etymology_with_derivative_token(): + d = diagnose_coherence( + "The name 'Rosebud River' is a translation of the Blackfoot word " + "Akokiniskway, meaning 'the river of the roses'." + ) + assert d["kind"] == "ok" + + +@_pytest.mark.xfail(reason="§3.1 false-positive: a bracketed claim-lattice-mode answer fragment (closing `..\"]`) tail-flagged as vacuous on a truncated/list-tail sentence. Tracked from real bench-qa STRICT regression. Tighten the sentence splitter to skip bracket-fragment tails.") +def test_coherence_ok_on_claim_lattice_truncated_bracket_tail(): + # Real bench-qa STRICT shape — list-mode answer with a quoted + # excerpt that ends in `..."]`. The trailing fragment is parsed as + # a sentence and flagged as `vacuous` (predicate is empty). + d = diagnose_coherence( + "Linux and BSD are both Unix-like operating systems. " + "[E1 | Linux | abcd1234: \"Linux is a Unix-like operating system kernel " + "first released by Linus Torvalds in 1991. Such a thesis was...\"]" + ) + assert d["kind"] == "ok" + + +@_pytest.mark.xfail(reason="§3.1 false-positive: 'The term ' framing where the quoted phrase's head token appears in the predicate as a generic referent (idiomatic encyclopedic English). Tracked from real bench-qa STRICT regression. Borderline — the SHAPE matches phrase_component_reuse, but the idiom is legitimate.") +def test_coherence_ok_on_term_idiom(): + d = diagnose_coherence( + "The term \"traditional Unix\" may be used to describe a Unix or an " + "operating system that has the characteristics of early Unix versions." + ) + assert d["kind"] == "ok" + + +def test_coherence_pooled_bench_qa_strict_fp_rate_documented(): + """Document the load-bearing real-traffic FP rate so future work + can be measured against it. As of 2026-05-13, §3.1 false-fires on + 44/808 = 5.4% of pooled bench-qa STRICT answers (the data set + pooled from n=1 + n=3 + n=5 ARBORIST_NLI_SHADOW=1 bench-qa runs). + This test asserts the *upper bound* — tightening the rules should + keep it at or below this. If a future change pushes it UP, this + test will fail and the regression is loud.""" + import json as _json + from pathlib import Path as _Path + # this is a slow-ish test (~1s for 808 rows of lexical regex); + # skip if the bench-qa JSONL files aren't present (e.g. fresh checkout) + files = [_Path(p) for p in [ + "bench/qa_results/2026-05-12T20-53-11Z.jsonl", + "bench/qa_results/2026-05-12T21-58-58Z.jsonl", + "bench/qa_results/2026-05-12T22-44-30Z.jsonl", + ]] + if not all(p.exists() for p in files): + _pytest.skip("pooled bench-qa STRICT files not present (gitignored — run ARBORIST_NLI_SHADOW=1 make bench-qa to generate)") + rows = [] + for p in files: + for ln in p.read_text().splitlines(): + o = _json.loads(ln.strip()) + if o.get("audit_mode") == "STRICT" and o.get("answer_text"): + rows.append(o) + flagged = sum(1 for r in rows + if diagnose_coherence(r["answer_text"])["kind"] not in ("ok", "empty")) + assert len(rows) >= 800, f"pooled STRICT sample shrunk unexpectedly ({len(rows)} rows; expected ~808)" + # current rate: 44/808 = 0.0545; ceiling at 0.07 leaves a tiny bit of headroom for fixture churn + assert flagged / len(rows) <= 0.07, \ + f"§3.1 FP rate on real STRICT regressed: {flagged}/{len(rows)} = {flagged/len(rows):.3f}" diff --git a/tests/test_relevance_shadow.py b/tests/test_relevance_shadow.py index b8caafa..a6765da 100644 --- a/tests/test_relevance_shadow.py +++ b/tests/test_relevance_shadow.py @@ -130,3 +130,122 @@ def test_zionist_entity_field_case_when_available(): assert off.available and on.available # on-topic should score strictly higher than off-topic — that's the discriminator assert on.relevance_score > off.relevance_score + + +# --- broader §3.2 coverage (round-2 candidate-bench, 2026-05-13) ---------- +# Added after extending the size spectrum 30MB→560MB (#000052 §3.2.1 +# round-2): more tests covering pair-kind-specific behavior, batch +# correctness, and the new manifest entries. + +def test_manifest_round2_primary_is_bge_reranker_large(): + m = load_manifest() + assert m["relevance_model_version"] == "relevance-shadow-v1-bge-reranker-large" + assert m["hf_repo"] == "BAAI/bge-reranker-large" + # the round-2 candidate-bench block is recorded + assert "candidate_bench_results_2026_05_13_round2" in m + cb = m["candidate_bench_results_2026_05_13_round2"] + assert cb["rank_by_sep_margin"][0]["model"] == "bge-reranker-large" + # capacity-floor finding is documented + floor = cb["rank_by_sep_margin"][-1] + assert floor["model"] == "ms-marco-MiniLM-L-2-v2" + assert floor["margin"] < 0 # NOT clean-separable + # findings block exists with the key science + assert "findings" in cb + for key in ("biggest_is_best_within_a_family", "but_not_across_families", + "deeper_is_not_better_within_MS_MARCO_MiniLM", + "capacity_floor_between_L_2_and_L_4"): + assert key in cb["findings"] + + +def test_manifest_alternates_cover_size_spectrum(): + m = load_manifest() + sizes = sorted(a["approx_mb"] for a in m["alternates"]) + # the alternates must span the spectrum from ≤50MB (small) to ≥280MB (large) + assert sizes[0] <= 50, f"smallest alternate is {sizes[0]}MB — needs a ≤50MB datapoint" + assert sizes[-1] >= 280, f"largest alternate is {sizes[-1]}MB — needs a ≥280MB datapoint" + + +def test_pair_kind_recorded_separately_for_qa_vs_claim_source(): + # The result records WHICH method produced the score — same model, + # same primitive, but the bookkeeping must distinguish the two + # for downstream telemetry / governance hashing. + r = ShadowRelevance() + res_qa = r.check_question_answer("who painted the mona lisa?", + "Leonardo da Vinci painted the Mona Lisa.") + res_cs = r.check_claim_source("Leonardo da Vinci painted the Mona Lisa.", + "The Mona Lisa is a portrait by Leonardo da Vinci.") + assert res_qa.pair_kind == "question_answer" + assert res_cs.pair_kind == "claim_source" + + +def test_batch_score_preserves_input_order(): + """_score_batch returns scores in input order (any padding/batching + must not reorder). This is load-bearing for downstream bench + scripts that zip scores back onto input records.""" + r = ShadowRelevance() + r._ensure_loaded() + if not r.available: + pytest.skip("[nli] extra not installed") + pairs = [ + ("very short", "very short"), + ("query about the painting Mona Lisa by Leonardo da Vinci in the 16th century", + "Leonardo da Vinci painted the Mona Lisa around 1503–1519, an Italian Renaissance portrait."), + ("a", "b"), + ] + a = r._score_batch(pairs) + # scoring each pair individually should give the same answers (modulo + # padding noise, which for fp32 should be 0) + individual = [r._score_batch([p])[0] for p in pairs] + assert len(a) == 3 == len(individual) + for batched, single in zip(a, individual): + # padding can introduce tiny numerical drift on fp16/fp32; allow + # 0.05 logit slack (well below any threshold we'd ever set) + assert abs(batched - single) < 0.05, f"batched {batched} vs single {single}" + + +def test_empty_string_query_or_document_returns_empty_input(): + r = ShadowRelevance() + r._ensure_loaded() + if not r.available: + pytest.skip("[nli] extra not installed") + assert r.check_question_answer("", "some answer").reason == "empty_input" + assert r.check_claim_source("a claim", "").reason == "empty_input" + # whitespace-only too + assert r.check_question_answer(" ", "answer").reason == "empty_input" + + +def test_score_sign_distinguishes_on_topic_from_off_topic(): + """Sanity: on the Zionist field case (the motivating example), + the on-topic (claim, source) pair MUST score strictly higher than + the off-topic one. This is the load-bearing discrimination — if + this assertion fails on the manifest's primary, the whole §3.2 + approach is invalidated and we walk back.""" + r = ShadowRelevance() + r._ensure_loaded() + if not r.available: + pytest.skip("[nli] extra not installed") + on = r.check_claim_source( + "the phrase 'Zionist entity' is sometimes used as a pejorative for the State of Israel", + "Zionist entity () is a phrase sometimes used by Arabs and Muslims as a pejorative for the State of Israel.", + ) + off = r.check_claim_source( + "the phrase 'Zionist entity' is sometimes used as the entity, referring to the State of Israel", + "In grammar, a phrase is a group of words functioning as a single unit in the syntax of a sentence.", + ) + assert on.available and off.available + assert on.relevance_score > off.relevance_score + + +def test_demote_below_score_remains_null_until_bench_sets_it(): + """Load-bearing invariant from #000049 §7 #18→#27: the demote + threshold MUST be set by a real-traffic shadow sweep, not by a + candidate-bench number or literature default. If anyone sets + demote_below_score in the manifest before step 2 lands, this + test fires.""" + m = load_manifest() + assert m["demote_below_score"] is None, \ + "demote_below_score must be null until §3.2.2 step 2 sets it from a real-traffic shadow sweep" + # the round-2 candidate-bench DOES record candidate θ values, but + # explicitly notes they are not the runtime threshold + cb = m["candidate_bench_results_2026_05_13_round2"] + assert "note" in cb and "CANDIDATE-BENCH ONLY" in cb["note"]