#000025 Phase 1e: 5F falsification motif coverage (§10.12 closed)
Pre-Phase-1e: falsification-v1.jsonl covered 10 motif tags across 50 fixtures (the high-traffic warrant/title/anchor/format set). Phase 1e adds 12 fixtures (5f-fal-051..062) for the previously-uncovered motifs from the verifier+soft-demote registries: CITATION_MISMATCH DEFLECTION_DETECTED MANUAL_QUOTE_VIOLATION SCHEMA_INVALID SOURCE_ROLE_BLOCKED SUBJECT_TOKENS_ABSENT TOO_MANY_EVIDENCE_IDS UNKNOWN_EVIDENCE_ID BROAD_QUANTIFIER_RUNAWAY BROAD_QUANTIFIER_CAP_APPLIED BROAD_QUANTIFIER_SCOPE_UNBOUND BROAD_QUANTIFIER_REJECTED Coverage now: 22 unique motif tags across 62 fixtures. Harness changes: - test_bench_batteries.py: bump pass_count assertion 50 → 62 in both falsification tests; add test_5f_falsification_covers_every_documented_motif that pins the motif set against the verifier+soft-demote registries so adding a new violation upstream surfaces here as a missing fixture (loud signal, no silent drift). - test_session_integration.py: bump full-suite total 662 → 674. Closes #000025 §10.12 (every documented failure-motif tag). Still open in Phase 1b: §10.11 (real shard finetuning chains), §10.13 (Feedback Loop latency/efficiency against real workload), §10.14 (threshold handoff to #000012).
This commit is contained in:
parent
30a9488578
commit
883289d00e
5 changed files with 100 additions and 7 deletions
|
|
@ -1,4 +1,4 @@
|
|||
{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 50, "notes": "Planted-bad-record fixtures with expected violation tags. Each fixture pinned to verifier_method_root so verifier-shape changes warn rather than false-fail. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50."}}
|
||||
{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 62, "notes": "Planted-bad-record fixtures with expected violation tags. Each fixture pinned to verifier_method_root so verifier-shape changes warn rather than false-fail. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50. Phase 1e (2026-05-10): added 12 fixtures (5f-fal-051..062) covering the previously-uncovered motif tags from the verifier+soft-demote registries — CITATION_MISMATCH, DEFLECTION_DETECTED, MANUAL_QUOTE_VIOLATION, SCHEMA_INVALID, SOURCE_ROLE_BLOCKED, SUBJECT_TOKENS_ABSENT, TOO_MANY_EVIDENCE_IDS, UNKNOWN_EVIDENCE_ID, BROAD_QUANTIFIER_{RUNAWAY,CAP_APPLIED,SCOPE_UNBOUND,REJECTED}. Closes #000025 §10.12 (every documented failure-motif tag)."}}
|
||||
{"id":"5f-fal-001","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_001","observed_violations":["WARRANT_MISSING","TITLE_MISMATCH"],"expected_reason":"WARRANT_MISSING","verifier_method_root":"warrant-v1","expected":"pass"}
|
||||
{"id":"5f-fal-002","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_002","observed_violations":["TITLE_MISMATCH"],"expected_reason":"TITLE_MISMATCH","verifier_method_root":"warrant-v1","expected":"pass"}
|
||||
{"id":"5f-fal-003","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_003","observed_violations":["TOO_MANY_CLAIMS"],"expected_reason":"TOO_MANY_CLAIMS","verifier_method_root":"warrant-v1","expected":"pass"}
|
||||
|
|
@ -49,3 +49,15 @@
|
|||
{"id": "5f-fal-048", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_048", "observed_violations": ["POINTER_OVERFLOW_TRIMMED"], "expected_reason": "POINTER_OVERFLOW_TRIMMED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-049", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_049", "observed_violations": ["UNGROUNDED"], "expected_reason": "UNGROUNDED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-050", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_050", "observed_violations": ["HYBRID_PARAPHRASE"], "expected_reason": "HYBRID_PARAPHRASE", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-051", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_051", "observed_violations": ["CITATION_MISMATCH"], "expected_reason": "CITATION_MISMATCH", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-052", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_052", "observed_violations": ["DEFLECTION_DETECTED"], "expected_reason": "DEFLECTION_DETECTED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-053", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_053", "observed_violations": ["MANUAL_QUOTE_VIOLATION"], "expected_reason": "MANUAL_QUOTE_VIOLATION", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-054", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_054", "observed_violations": ["SCHEMA_INVALID"], "expected_reason": "SCHEMA_INVALID", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-055", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_055", "observed_violations": ["SOURCE_ROLE_BLOCKED"], "expected_reason": "SOURCE_ROLE_BLOCKED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-056", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_056", "observed_violations": ["SUBJECT_TOKENS_ABSENT"], "expected_reason": "SUBJECT_TOKENS_ABSENT", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-057", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_057", "observed_violations": ["TOO_MANY_EVIDENCE_IDS"], "expected_reason": "TOO_MANY_EVIDENCE_IDS", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-058", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_058", "observed_violations": ["UNKNOWN_EVIDENCE_ID"], "expected_reason": "UNKNOWN_EVIDENCE_ID", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-059", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_059", "observed_violations": ["BROAD_QUANTIFIER_RUNAWAY"], "expected_reason": "BROAD_QUANTIFIER_RUNAWAY", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-060", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_060", "observed_violations": ["BROAD_QUANTIFIER_CAP_APPLIED"], "expected_reason": "BROAD_QUANTIFIER_CAP_APPLIED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-061", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_061", "observed_violations": ["BROAD_QUANTIFIER_SCOPE_UNBOUND"], "expected_reason": "BROAD_QUANTIFIER_SCOPE_UNBOUND", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-062", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_062", "observed_violations": ["BROAD_QUANTIFIER_REJECTED"], "expected_reason": "BROAD_QUANTIFIER_REJECTED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ Newest first. Update on every open/close.
|
|||
| #000028 | Multi-modality witness for canonical shapes | closed · landed 2026-05-09 + follow-ups (capital ledger · sample rate) | 2026-05-08 | — |
|
||||
| #000027 | Canonical projections persist to providence_cache | closed · landed 2026-05-09 | 2026-05-08 | — |
|
||||
| #000026 | Real-shard workload baseline + search latency | closed · Phase 1 + 2 + 3 landed 2026-05-09 (Phase 3 in `60b5748`) | 2026-05-08 | — |
|
||||
| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a + 1b.2 + 1c + 1d landed 2026-05-09 | 2026-05-07 | — |
|
||||
| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a + 1b.2 + 1c + 1d landed 2026-05-09; Phase 1e (motif coverage, §10.12 closed) landed 2026-05-10 | 2026-05-07 | — |
|
||||
| #000024 | 5T Phase 1b + Dav1DPrometheus vocabulary alignment | closed · landed 2026-05-08 | 2026-05-07 | — |
|
||||
| #000023 | 5S Phase 1b: Syllogism · Synthesis · Semiotics | closed · landed 2026-05-08 | 2026-05-07 | — |
|
||||
| #000022 | Adapter LossReport (PRD I9 analogue) | closed · landed 2026-05-07 | 2026-05-07 | — |
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Ticket #000025 — 5F battery: Function · Finetuning · Falsification · Formulate · Feedback Loop
|
||||
|
||||
**Status:** in progress · Phase 1a landed 2026-05-08; Phase 1b open
|
||||
**Status:** in progress · Phase 1a + 1b.2 + 1c + 1d + 1e (motif coverage, 2026-05-10) landed; §10.11 (real shard finetuning chains), §10.13 (Feedback Loop latency/efficiency calibrated to real workload), §10.14 (threshold handoff to #000012) still open
|
||||
**Opened:** 2026-05-07
|
||||
**Revised:** 2026-05-08 (cross-modality + state-space synthesis
|
||||
folded in per review response)
|
||||
|
|
@ -544,6 +544,36 @@ limit.
|
|||
|
||||
`make bench-5f-live` aggregates all five live fixtures.
|
||||
|
||||
### Phase 1e — motif coverage (2026-05-10)
|
||||
|
||||
§10.12 closure: every documented failure-motif tag has at least one
|
||||
falsification fixture as `expected_reason`. Pre-Phase-1e the
|
||||
falsification-v1.jsonl pack covered 10 motifs (the high-traffic
|
||||
warrant/title/anchor/format set); Phase 1e adds 12 fixtures
|
||||
(`5f-fal-051..062`) for the previously-uncovered motifs surfaced by
|
||||
auditing `arborist/qa/verify.py:violations.append` callsites + the
|
||||
`_SOFT_DEMOTE_VIOLATION_KINDS` registry in `arborist/cli.py`:
|
||||
|
||||
```
|
||||
CITATION_MISMATCH DEFLECTION_DETECTED
|
||||
MANUAL_QUOTE_VIOLATION SCHEMA_INVALID
|
||||
SOURCE_ROLE_BLOCKED SUBJECT_TOKENS_ABSENT
|
||||
TOO_MANY_EVIDENCE_IDS UNKNOWN_EVIDENCE_ID
|
||||
BROAD_QUANTIFIER_RUNAWAY BROAD_QUANTIFIER_CAP_APPLIED
|
||||
BROAD_QUANTIFIER_SCOPE_UNBOUND BROAD_QUANTIFIER_REJECTED
|
||||
```
|
||||
|
||||
Pack now covers 22 unique motif tags across 62 fixtures (was
|
||||
10/50). `test_5f_falsification_covers_every_documented_motif` in
|
||||
`tests/test_bench_batteries.py` pins the motif set against the
|
||||
verifier + soft-demote registries so adding a new violation
|
||||
upstream surfaces here as a missing fixture (loud signal, no silent
|
||||
drift).
|
||||
|
||||
Phase 1e closes §10.12. Remaining open: §10.11 (real shard
|
||||
finetuning chains), §10.13 (Feedback Loop latency/efficiency
|
||||
against real workload), §10.14 (threshold handoff to #000012).
|
||||
|
||||
---
|
||||
|
||||
## 11. Status
|
||||
|
|
|
|||
|
|
@ -240,7 +240,7 @@ def test_5f_finetuning_runs():
|
|||
|
||||
def test_5f_falsification_runs():
|
||||
res = b_5f.run_falsification(F5F / "falsification-v1.jsonl")
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
assert res.pass_count == 62 # Phase 1e: +12 fixtures covering uncovered motifs
|
||||
assert res.metrics["error_detection_rate"] == 1.0
|
||||
|
||||
|
||||
|
|
@ -618,11 +618,59 @@ def test_5f_falsification_live_path_routes_through_verify_quotes():
|
|||
|
||||
def test_5f_falsification_embedded_path_still_works():
|
||||
res = b_5f.run_falsification(F5F / "falsification-v1.jsonl")
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
assert res.pass_count == 62 # Phase 1e: +12 fixtures covering uncovered motifs
|
||||
for t in res.per_task:
|
||||
assert t.detail["source"] == "embedded"
|
||||
|
||||
|
||||
def test_5f_falsification_covers_every_documented_motif():
|
||||
"""#000025 §10.12: Falsification fixtures must cover every documented
|
||||
failure-motif tag. Pins motif-coverage of the falsification-v1.jsonl
|
||||
fixture pack against the verifier's `violations.append` registry +
|
||||
the soft-demote registry in arborist.cli."""
|
||||
import json
|
||||
|
||||
fixture_path = F5F / "falsification-v1.jsonl"
|
||||
covered: set[str] = set()
|
||||
with fixture_path.open() as f:
|
||||
for line in f:
|
||||
d = json.loads(line)
|
||||
if "expected_reason" in d:
|
||||
covered.add(d["expected_reason"])
|
||||
|
||||
documented = {
|
||||
# Hard violations emitted by arborist/qa/verify.py
|
||||
"CITATION_MISMATCH",
|
||||
"DEFLECTION_DETECTED",
|
||||
"FORMAT_COLLAPSED",
|
||||
"MANUAL_QUOTE_VIOLATION",
|
||||
"NO_EVIDENCE_POINTER",
|
||||
"POINTER_OVERFLOW_TRIMMED",
|
||||
"SCHEMA_INVALID",
|
||||
"SOURCE_ROLE_BLOCKED",
|
||||
"SUBJECT_TOKENS_ABSENT",
|
||||
"TITLE_MISMATCH",
|
||||
"TOO_MANY_CLAIMS",
|
||||
"TOO_MANY_EVIDENCE_IDS",
|
||||
"UNKNOWN_EVIDENCE_ID",
|
||||
"WARRANT_MISSING",
|
||||
# Soft-demote violations registered in arborist/cli.py
|
||||
"LAZY_ANCHOR_DEMOTED",
|
||||
"BARE_NAME_CLAIM",
|
||||
"BROAD_QUANTIFIER_RUNAWAY",
|
||||
"BROAD_QUANTIFIER_CAP_APPLIED",
|
||||
"BROAD_QUANTIFIER_SCOPE_UNBOUND",
|
||||
"BROAD_QUANTIFIER_REJECTED",
|
||||
}
|
||||
missing = documented - covered
|
||||
assert not missing, (
|
||||
f"falsification-v1.jsonl missing {len(missing)} motif tag(s): "
|
||||
f"{sorted(missing)}. Add a fixture with `expected_reason` set to "
|
||||
"each, or remove from the documented registry if intentionally "
|
||||
"retired."
|
||||
)
|
||||
|
||||
|
||||
def test_5f_falsification_live_helper_calls_real_verifier():
|
||||
"""UNGROUNDED + STRICT_SPAN signals come straight from verify_quotes."""
|
||||
from bench.batteries.b_5f import _live_falsification_violations
|
||||
|
|
|
|||
|
|
@ -239,7 +239,7 @@ def test_full_dav1dprometheus_suite_runs_end_to_end(tmp_path, capsys):
|
|||
|
||||
|
||||
def test_full_suite_total_fixture_count():
|
||||
"""Sanity check: the complete Dav1DPrometheus suite executes 662
|
||||
"""Sanity check: the complete Dav1DPrometheus suite executes 674
|
||||
deterministic tasks across 21 sub-batteries (5S+5T+5F+5R).
|
||||
|
||||
History:
|
||||
|
|
@ -250,6 +250,9 @@ def test_full_suite_total_fixture_count():
|
|||
(Live side ALSO went 30 → 50 but lives in *-live-v1 files
|
||||
that the default-fixture-set doesn't load — those run via
|
||||
the dedicated Makefile targets.)
|
||||
- Phase 1e (#000025, 2026-05-10): 5F falsification expanded
|
||||
50 → 62 to cover every documented failure-motif tag; +12 → 674.
|
||||
(Closes #000025 §10.12.)
|
||||
"""
|
||||
from bench.batteries.runner import _DEFAULT_FIXTURES, _run_one
|
||||
|
||||
|
|
@ -257,7 +260,7 @@ def test_full_suite_total_fixture_count():
|
|||
for (battery, sub), fx in _DEFAULT_FIXTURES.items():
|
||||
result = _run_one(battery, sub, Path(fx))
|
||||
total += result.pass_count + result.fail_count
|
||||
assert total == 662
|
||||
assert total == 674
|
||||
|
||||
|
||||
def test_5s_phase1a_digests_unchanged_after_phase1b():
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue