diff --git a/bench/fixtures/5f/falsification-v1.jsonl b/bench/fixtures/5f/falsification-v1.jsonl index 7eac575..595fc51 100644 --- a/bench/fixtures/5f/falsification-v1.jsonl +++ b/bench/fixtures/5f/falsification-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 50, "notes": "Planted-bad-record fixtures with expected violation tags. Each fixture pinned to verifier_method_root so verifier-shape changes warn rather than false-fail. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50."}} +{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 62, "notes": "Planted-bad-record fixtures with expected violation tags. Each fixture pinned to verifier_method_root so verifier-shape changes warn rather than false-fail. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50. Phase 1e (2026-05-10): added 12 fixtures (5f-fal-051..062) covering the previously-uncovered motif tags from the verifier+soft-demote registries — CITATION_MISMATCH, DEFLECTION_DETECTED, MANUAL_QUOTE_VIOLATION, SCHEMA_INVALID, SOURCE_ROLE_BLOCKED, SUBJECT_TOKENS_ABSENT, TOO_MANY_EVIDENCE_IDS, UNKNOWN_EVIDENCE_ID, BROAD_QUANTIFIER_{RUNAWAY,CAP_APPLIED,SCOPE_UNBOUND,REJECTED}. Closes #000025 §10.12 (every documented failure-motif tag)."}} {"id":"5f-fal-001","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_001","observed_violations":["WARRANT_MISSING","TITLE_MISMATCH"],"expected_reason":"WARRANT_MISSING","verifier_method_root":"warrant-v1","expected":"pass"} {"id":"5f-fal-002","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_002","observed_violations":["TITLE_MISMATCH"],"expected_reason":"TITLE_MISMATCH","verifier_method_root":"warrant-v1","expected":"pass"} {"id":"5f-fal-003","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_003","observed_violations":["TOO_MANY_CLAIMS"],"expected_reason":"TOO_MANY_CLAIMS","verifier_method_root":"warrant-v1","expected":"pass"} @@ -49,3 +49,15 @@ {"id": "5f-fal-048", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_048", "observed_violations": ["POINTER_OVERFLOW_TRIMMED"], "expected_reason": "POINTER_OVERFLOW_TRIMMED", "verifier_method_root": "warrant-v1", "expected": "pass"} {"id": "5f-fal-049", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_049", "observed_violations": ["UNGROUNDED"], "expected_reason": "UNGROUNDED", "verifier_method_root": "warrant-v1", "expected": "pass"} {"id": "5f-fal-050", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_050", "observed_violations": ["HYBRID_PARAPHRASE"], "expected_reason": "HYBRID_PARAPHRASE", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-051", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_051", "observed_violations": ["CITATION_MISMATCH"], "expected_reason": "CITATION_MISMATCH", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-052", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_052", "observed_violations": ["DEFLECTION_DETECTED"], "expected_reason": "DEFLECTION_DETECTED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-053", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_053", "observed_violations": ["MANUAL_QUOTE_VIOLATION"], "expected_reason": "MANUAL_QUOTE_VIOLATION", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-054", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_054", "observed_violations": ["SCHEMA_INVALID"], "expected_reason": "SCHEMA_INVALID", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-055", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_055", "observed_violations": ["SOURCE_ROLE_BLOCKED"], "expected_reason": "SOURCE_ROLE_BLOCKED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-056", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_056", "observed_violations": ["SUBJECT_TOKENS_ABSENT"], "expected_reason": "SUBJECT_TOKENS_ABSENT", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-057", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_057", "observed_violations": ["TOO_MANY_EVIDENCE_IDS"], "expected_reason": "TOO_MANY_EVIDENCE_IDS", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-058", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_058", "observed_violations": ["UNKNOWN_EVIDENCE_ID"], "expected_reason": "UNKNOWN_EVIDENCE_ID", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-059", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_059", "observed_violations": ["BROAD_QUANTIFIER_RUNAWAY"], "expected_reason": "BROAD_QUANTIFIER_RUNAWAY", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-060", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_060", "observed_violations": ["BROAD_QUANTIFIER_CAP_APPLIED"], "expected_reason": "BROAD_QUANTIFIER_CAP_APPLIED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-061", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_061", "observed_violations": ["BROAD_QUANTIFIER_SCOPE_UNBOUND"], "expected_reason": "BROAD_QUANTIFIER_SCOPE_UNBOUND", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-062", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_062", "observed_violations": ["BROAD_QUANTIFIER_REJECTED"], "expected_reason": "BROAD_QUANTIFIER_REJECTED", "verifier_method_root": "warrant-v1", "expected": "pass"} diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 03f0365..5f53003 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -92,7 +92,7 @@ Newest first. Update on every open/close. | #000028 | Multi-modality witness for canonical shapes | closed · landed 2026-05-09 + follow-ups (capital ledger · sample rate) | 2026-05-08 | — | | #000027 | Canonical projections persist to providence_cache | closed · landed 2026-05-09 | 2026-05-08 | — | | #000026 | Real-shard workload baseline + search latency | closed · Phase 1 + 2 + 3 landed 2026-05-09 (Phase 3 in `60b5748`) | 2026-05-08 | — | -| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a + 1b.2 + 1c + 1d landed 2026-05-09 | 2026-05-07 | — | +| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a + 1b.2 + 1c + 1d landed 2026-05-09; Phase 1e (motif coverage, §10.12 closed) landed 2026-05-10 | 2026-05-07 | — | | #000024 | 5T Phase 1b + Dav1DPrometheus vocabulary alignment | closed · landed 2026-05-08 | 2026-05-07 | — | | #000023 | 5S Phase 1b: Syllogism · Synthesis · Semiotics | closed · landed 2026-05-08 | 2026-05-07 | — | | #000022 | Adapter LossReport (PRD I9 analogue) | closed · landed 2026-05-07 | 2026-05-07 | — | diff --git a/docs/tickets/ticket-000025-5f-battery.md b/docs/tickets/ticket-000025-5f-battery.md index 5bf1923..c904401 100644 --- a/docs/tickets/ticket-000025-5f-battery.md +++ b/docs/tickets/ticket-000025-5f-battery.md @@ -1,6 +1,6 @@ # Ticket #000025 — 5F battery: Function · Finetuning · Falsification · Formulate · Feedback Loop -**Status:** in progress · Phase 1a landed 2026-05-08; Phase 1b open +**Status:** in progress · Phase 1a + 1b.2 + 1c + 1d + 1e (motif coverage, 2026-05-10) landed; §10.11 (real shard finetuning chains), §10.13 (Feedback Loop latency/efficiency calibrated to real workload), §10.14 (threshold handoff to #000012) still open **Opened:** 2026-05-07 **Revised:** 2026-05-08 (cross-modality + state-space synthesis folded in per review response) @@ -544,6 +544,36 @@ limit. `make bench-5f-live` aggregates all five live fixtures. +### Phase 1e — motif coverage (2026-05-10) + +§10.12 closure: every documented failure-motif tag has at least one +falsification fixture as `expected_reason`. Pre-Phase-1e the +falsification-v1.jsonl pack covered 10 motifs (the high-traffic +warrant/title/anchor/format set); Phase 1e adds 12 fixtures +(`5f-fal-051..062`) for the previously-uncovered motifs surfaced by +auditing `arborist/qa/verify.py:violations.append` callsites + the +`_SOFT_DEMOTE_VIOLATION_KINDS` registry in `arborist/cli.py`: + +``` +CITATION_MISMATCH DEFLECTION_DETECTED +MANUAL_QUOTE_VIOLATION SCHEMA_INVALID +SOURCE_ROLE_BLOCKED SUBJECT_TOKENS_ABSENT +TOO_MANY_EVIDENCE_IDS UNKNOWN_EVIDENCE_ID +BROAD_QUANTIFIER_RUNAWAY BROAD_QUANTIFIER_CAP_APPLIED +BROAD_QUANTIFIER_SCOPE_UNBOUND BROAD_QUANTIFIER_REJECTED +``` + +Pack now covers 22 unique motif tags across 62 fixtures (was +10/50). `test_5f_falsification_covers_every_documented_motif` in +`tests/test_bench_batteries.py` pins the motif set against the +verifier + soft-demote registries so adding a new violation +upstream surfaces here as a missing fixture (loud signal, no silent +drift). + +Phase 1e closes §10.12. Remaining open: §10.11 (real shard +finetuning chains), §10.13 (Feedback Loop latency/efficiency +against real workload), §10.14 (threshold handoff to #000012). + --- ## 11. Status diff --git a/tests/test_bench_batteries.py b/tests/test_bench_batteries.py index 37e5cc7..9df4dbd 100644 --- a/tests/test_bench_batteries.py +++ b/tests/test_bench_batteries.py @@ -240,7 +240,7 @@ def test_5f_finetuning_runs(): def test_5f_falsification_runs(): res = b_5f.run_falsification(F5F / "falsification-v1.jsonl") - assert res.pass_count == 50 # Phase 1d + assert res.pass_count == 62 # Phase 1e: +12 fixtures covering uncovered motifs assert res.metrics["error_detection_rate"] == 1.0 @@ -618,11 +618,59 @@ def test_5f_falsification_live_path_routes_through_verify_quotes(): def test_5f_falsification_embedded_path_still_works(): res = b_5f.run_falsification(F5F / "falsification-v1.jsonl") - assert res.pass_count == 50 # Phase 1d + assert res.pass_count == 62 # Phase 1e: +12 fixtures covering uncovered motifs for t in res.per_task: assert t.detail["source"] == "embedded" +def test_5f_falsification_covers_every_documented_motif(): + """#000025 §10.12: Falsification fixtures must cover every documented + failure-motif tag. Pins motif-coverage of the falsification-v1.jsonl + fixture pack against the verifier's `violations.append` registry + + the soft-demote registry in arborist.cli.""" + import json + + fixture_path = F5F / "falsification-v1.jsonl" + covered: set[str] = set() + with fixture_path.open() as f: + for line in f: + d = json.loads(line) + if "expected_reason" in d: + covered.add(d["expected_reason"]) + + documented = { + # Hard violations emitted by arborist/qa/verify.py + "CITATION_MISMATCH", + "DEFLECTION_DETECTED", + "FORMAT_COLLAPSED", + "MANUAL_QUOTE_VIOLATION", + "NO_EVIDENCE_POINTER", + "POINTER_OVERFLOW_TRIMMED", + "SCHEMA_INVALID", + "SOURCE_ROLE_BLOCKED", + "SUBJECT_TOKENS_ABSENT", + "TITLE_MISMATCH", + "TOO_MANY_CLAIMS", + "TOO_MANY_EVIDENCE_IDS", + "UNKNOWN_EVIDENCE_ID", + "WARRANT_MISSING", + # Soft-demote violations registered in arborist/cli.py + "LAZY_ANCHOR_DEMOTED", + "BARE_NAME_CLAIM", + "BROAD_QUANTIFIER_RUNAWAY", + "BROAD_QUANTIFIER_CAP_APPLIED", + "BROAD_QUANTIFIER_SCOPE_UNBOUND", + "BROAD_QUANTIFIER_REJECTED", + } + missing = documented - covered + assert not missing, ( + f"falsification-v1.jsonl missing {len(missing)} motif tag(s): " + f"{sorted(missing)}. Add a fixture with `expected_reason` set to " + "each, or remove from the documented registry if intentionally " + "retired." + ) + + def test_5f_falsification_live_helper_calls_real_verifier(): """UNGROUNDED + STRICT_SPAN signals come straight from verify_quotes.""" from bench.batteries.b_5f import _live_falsification_violations diff --git a/tests/test_session_integration.py b/tests/test_session_integration.py index 81e08bc..bce75ad 100644 --- a/tests/test_session_integration.py +++ b/tests/test_session_integration.py @@ -239,7 +239,7 @@ def test_full_dav1dprometheus_suite_runs_end_to_end(tmp_path, capsys): def test_full_suite_total_fixture_count(): - """Sanity check: the complete Dav1DPrometheus suite executes 662 + """Sanity check: the complete Dav1DPrometheus suite executes 674 deterministic tasks across 21 sub-batteries (5S+5T+5F+5R). History: @@ -250,6 +250,9 @@ def test_full_suite_total_fixture_count(): (Live side ALSO went 30 → 50 but lives in *-live-v1 files that the default-fixture-set doesn't load — those run via the dedicated Makefile targets.) + - Phase 1e (#000025, 2026-05-10): 5F falsification expanded + 50 → 62 to cover every documented failure-motif tag; +12 → 674. + (Closes #000025 §10.12.) """ from bench.batteries.runner import _DEFAULT_FIXTURES, _run_one @@ -257,7 +260,7 @@ def test_full_suite_total_fixture_count(): for (battery, sub), fx in _DEFAULT_FIXTURES.items(): result = _run_one(battery, sub, Path(fx)) total += result.pass_count + result.fail_count - assert total == 662 + assert total == 674 def test_5s_phase1a_digests_unchanged_after_phase1b():