Closes the controller → 5F battery loop fox designed: #000037 Phase 1 produces FalsificationFixtureProposal records from high-divergence providence_cache rows; this commit lands the harvester that turns those proposals into a real 5F fixture pack the falsification battery exercises every test run. bench/scripts/harvest_falsification_proposals.py — reads qa.db, filters live rows with witness_divergence = (n_unverified / n_quotes) >= 0.5, stratifies by audit_mode, picks 20 HYBRID + 20 UNGROUNDED top-by-cache_key for determinism, writes embedded-mode fixtures to bench/fixtures/5f/falsification-harvested-v1.jsonl. Each fixture carries `_harvest_meta` with the source cache_key, divergence at harvest time, audit_mode at harvest time, and the ticket reference (#000037 §13 step 11). Embedded mode — uses `observed_violations` directly without calling verify_quotes again; `answer_text` preserved verbatim for debugging. Pack composition (initial harvest 2026-05-10): 20 UNGROUNDED (expected_reason: UNGROUNDED) 9 HYBRID_QUOTE (NEW motif — not in falsification-v1.jsonl) 7 HYBRID_CLAIM_LATTICE (NEW motif — not in falsification-v1.jsonl) 4 HYBRID_PARAPHRASE (already covered in v1) Two harness tests pin the pack: - test_5f_falsification_harvested_pack_runs_clean: 100% pass-rate, every fixture carries traceable _harvest_meta. - test_5f_falsification_harvested_pack_widens_motif_coverage: the HYBRID_QUOTE / HYBRID_CLAIM_LATTICE motifs surface from real corpus (loud-fail if harvest rotation drops them). Makefile: `make bench-5f-harvest` re-runs the harvester. Parameters exposed: HARVEST_QA_DB, HARVEST_OUT, HARVEST_THRESHOLD, HARVEST_SAMPLE_PER_BUCKET. Full suite: 2328 passed, 37 skipped (+2 from the two new pins).
This commit is contained in:
parent
12bf2df9d0
commit
ff1752c383
4 changed files with 373 additions and 0 deletions
|
|
@ -671,6 +671,57 @@ def test_5f_falsification_covers_every_documented_motif():
|
|||
)
|
||||
|
||||
|
||||
def test_5f_falsification_harvested_pack_runs_clean():
|
||||
"""#000037 Phase 1 → 5F loop: the harvested pack at
|
||||
falsification-harvested-v1.jsonl was generated by
|
||||
bench/scripts/harvest_falsification_proposals.py from real qa.db
|
||||
high-divergence rows. Pin: pack runs at 100% pass-rate, all
|
||||
fixtures route through the embedded runner path, every fixture
|
||||
carries the harvest-attribution metadata."""
|
||||
import json
|
||||
|
||||
fixture_path = F5F / "falsification-harvested-v1.jsonl"
|
||||
res = b_5f.run_falsification(fixture_path)
|
||||
assert res.fail_count == 0
|
||||
assert res.metrics["error_detection_rate"] == 1.0
|
||||
# Every fixture must carry the harvest metadata so future
|
||||
# regenerations are traceable to the source providence_cache
|
||||
# row.
|
||||
with fixture_path.open() as f:
|
||||
next(f) # _meta header
|
||||
for line in f:
|
||||
row = json.loads(line)
|
||||
meta = row.get("_harvest_meta", {})
|
||||
assert meta.get("source_ticket") == "#000037 §13 step 11"
|
||||
assert meta.get("harvested_from") == "providence_cache"
|
||||
assert isinstance(meta.get("witness_divergence"), (int, float))
|
||||
assert meta["witness_divergence"] >= 0.5
|
||||
assert meta.get("audit_mode_at_harvest") in ("HYBRID", "UNGROUNDED")
|
||||
|
||||
|
||||
def test_5f_falsification_harvested_pack_widens_motif_coverage():
|
||||
"""The harvested pack introduces real-corpus motif tags that the
|
||||
hand-curated falsification-v1.jsonl doesn't have. Pin: the
|
||||
HYBRID_QUOTE + HYBRID_CLAIM_LATTICE pair appear in the harvested
|
||||
pack — concrete evidence that the harvest is doing widening
|
||||
work, not just duplicating the curated set."""
|
||||
import json
|
||||
|
||||
fixture_path = F5F / "falsification-harvested-v1.jsonl"
|
||||
motifs: set[str] = set()
|
||||
with fixture_path.open() as f:
|
||||
for line in f:
|
||||
d = json.loads(line)
|
||||
if "expected_reason" in d:
|
||||
motifs.add(d["expected_reason"])
|
||||
# Harvest contains HYBRID variants surfaced from corpus that the
|
||||
# hand-curated set lacks. If the harvested DB drifts and these
|
||||
# motifs disappear, the test loud-fails — by then the harvest
|
||||
# rotation has lost signal and the pack should be regenerated.
|
||||
assert "HYBRID_QUOTE" in motifs or "HYBRID_CLAIM_LATTICE" in motifs
|
||||
assert "UNGROUNDED" in motifs
|
||||
|
||||
|
||||
def test_5f_falsification_live_helper_calls_real_verifier():
|
||||
"""UNGROUNDED + STRICT_SPAN signals come straight from verify_quotes."""
|
||||
from bench.batteries.b_5f import _live_falsification_violations
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue