russell@unturf.com
3ea27aa471
The #000025 §10.14 calibration showed _delta_5{s,t,f} mean over a battery's 5 subs, so a single-sub gain weighs 1/5 of face value (the 5× dilution). #000047 ships the knob to pick the aggregation, default unchanged. WeightSet.delta_aggregator ∈ {"mean","max","sum"} (default "mean") — a categorical field, validated in __post_init__ against DELTA_AGGREGATORS; from_dict takes it as a string. Default unchanged → ScoredFork output byte-identical → no fork_score.ESTIMATOR_VERSION bump. fork_score._aggregate(deltas, how): mean = arithmetic mean, max = max(0.0, max_i Δ_i), sum = Σ Δ_i; empty → 0.0. _delta_5s/_delta_5t/ _delta_5f take an aggregator arg (default "mean"); the 5F efficiency bonus is added after the aggregated base (aggregator-independent). fork_score passes weights.delta_aggregator. The per-sub HARD_REGRESSION_FLOOR flags are computed before aggregation, so a single-sub regression still forces REJECT under max/sum. The chosen aggregator is recorded in ScoredFork.weights["delta_aggregator"] (via WeightSet.as_dict()); fork_score_branches traceability stays via the opaque weights_id — no schema migration. bench/scripts/fivef_threshold_calibration.py gained §5 — runs the #000046 below-ceiling pack (5f/falsification at 0.333) and shows the verdict / γ·Δ5f under each aggregator; bench/results/5f-threshold- calibration-2026-05-11.md §5 is the captured record. Default stays "mean" — the conservative, noise-robust, regression-symmetric choice matching docs/bench-maxing.md's per-rate floor framing; v8 picks max/sum per-deployment. Tests: 8 new in tests/test_fork_score.py + 1 anchor in tests/test_fivef_threshold_calibration.py; tests/test_weights.py as_dict field-set test updated to include delta_aggregator; test_fork_score.py AUTOCOUNT tags (#000012 §286, warrant-substrate- cookbook.md ×2) bumped 23 → 31. #000047 closed; #000012 §8 §3 + TICKETS.md row updated. Full suite: 2330 passed, 28 skipped.
90 lines
3.3 KiB
Python
90 lines
3.3 KiB
Python
"""Tests for the #000025 §10.14 ForkScore threshold-calibration handoff
|
||
script (``bench/scripts/fivef_threshold_calibration.py``).
|
||
|
||
The script is pure measurement over the canonical 5S/5T/5F packs +
|
||
`fork_score`, so its markdown report is deterministic given fixed
|
||
fixtures + code. These tests pin the structural anchors and the
|
||
floor-constant sanity verdicts so the handoff doesn't drift silently.
|
||
|
||
The live 5F packs (``*-live-v1.jsonl``) spin up temp shards and add
|
||
~15s per run; the calibration *logic* (granularity, fork-score sanity)
|
||
doesn't depend on them, so the tests stub ``_LIVE_PACKS`` to empty.
|
||
The live runners are exercised by ``tests/test_bench_batteries.py``
|
||
and ``make bench-5f-live``.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
|
||
from bench.scripts import fivef_threshold_calibration as calib
|
||
|
||
|
||
@pytest.fixture(scope="module", autouse=True)
|
||
def _no_live_packs():
|
||
saved = calib._LIVE_PACKS
|
||
calib._LIVE_PACKS = {}
|
||
try:
|
||
yield
|
||
finally:
|
||
calib._LIVE_PACKS = saved
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def report() -> str:
|
||
return calib.build_report()
|
||
|
||
|
||
def test_report_is_deterministic():
|
||
a = calib.build_report()
|
||
b = calib.build_report()
|
||
|
||
def _strip_ts(s: str) -> str:
|
||
return "\n".join(ln for ln in s.splitlines() if not ln.startswith("**Date:**"))
|
||
|
||
assert _strip_ts(a) == _strip_ts(b)
|
||
|
||
|
||
def test_report_has_structural_anchors(report: str):
|
||
assert report.startswith("# 5S/5T/5F → v8 ForkScore threshold-calibration handoff")
|
||
assert "#000025 §10.14" in report
|
||
assert "## 1. Baseline rates + granularity" in report
|
||
assert "## 2. Identity-fork verdict (no change)" in report
|
||
assert "## 3. Floor-constant sanity checks" in report
|
||
assert "## 4. Recommendation for #000012" in report
|
||
assert "## 5. `delta_aggregator` comparison" in report
|
||
# The aggregator-comparison table names all three aggregators.
|
||
for agg in ("`mean`", "`max`", "`sum`"):
|
||
assert agg in report
|
||
for sub in (
|
||
"syntax", "semantics", "syllogism", "synthesis", "semiotics",
|
||
"transfer-learning", "triangulation", "truthtables", "transitivity", "time",
|
||
"function", "finetuning", "falsification", "formulate", "feedback-loop",
|
||
):
|
||
assert f"| {sub} |" in report
|
||
|
||
|
||
def test_identity_fork_is_marginal(report: str):
|
||
# Packs are at ceiling at HEAD → every Δ-rate term is 0 → MARGINAL.
|
||
assert "`fork_score(parent, parent)` → **MARGINAL (score +0.0000)**" in report
|
||
|
||
|
||
def test_floor_sanity_verdicts_present(report: str):
|
||
assert "child recovers **all 5** 5F subs by +0.06 | ACCEPT" in report
|
||
assert "child recovers **only 1** 5F sub by +0.06 | MARGINAL" in report
|
||
assert "child drops one 5F sub by −0.05 | REJECT" in report
|
||
assert "REGRESSION_5F:5f/falsification" in report
|
||
|
||
|
||
def test_recommendation_keeps_floors_and_flags_dilution(report: str):
|
||
assert "Keep `SIGNAL_FLOOR = 0.05` and `HARD_REGRESSION_FLOOR = 0.05`" in report
|
||
assert "5× averaging dilution" in report
|
||
assert "Ceiling saturation" in report
|
||
assert "No constant change shipped" in report
|
||
|
||
|
||
def test_cli_main_writes_report(tmp_path):
|
||
out = tmp_path / "calib.md"
|
||
rc = calib.main(["--out", str(out)])
|
||
assert rc == 0
|
||
text = out.read_text(encoding="utf-8")
|
||
assert text.startswith("# 5S/5T/5F → v8 ForkScore threshold-calibration handoff")
|