5f: efficiency metrics with explicit zero-cost guards
Per fox's 2026-05-08 review offbd99a8: implement adaptation_efficiency and feedback_efficiency in run_finetuning + run_feedback_loop with explicit sentinels — not Python floating-point accidents. _efficiency(gain, cost) helper: - cost > 0: standard ratio - cost == 0, gain > 0: EFFICIENCY_INFINITE (free improvement) - cost == 0, gain == 0: EFFICIENCY_UNDEFINED (= 0.0; no signal) - cost == 0, gain < 0: -EFFICIENCY_INFINITE (free regression) Battery-level metrics report mean_finite (computed over finite values only) + infinite_count + neg_infinite_count so the mean stays dimensionally truthful and consumers can pivot on the special buckets separately. Phase 1a cost proxies: - run_finetuning: _capital_cost_delta sums resource_budget (max_compute_ms_delta * 1e-3 + max_storage_delta_bytes / 1e6). Phase 1b.2 will replace with real capital_ledger reads. - run_feedback_loop: chain length = cost. Phase 1b.2 capital_ledger integration replaces it. Tests added (7): - _efficiency over four boundary cases - 5F finetuning + feedback_loop emit the new metrics keys - Synthetic zero-cost finetuning fixture verifies +inf path Full suite: 1110 passed, 36 skipped. Closes the one actionable from thefbd99a8review.
This commit is contained in:
parent
2af28e66af
commit
3bff55234f
3 changed files with 198 additions and 12 deletions
|
|
@ -155,15 +155,64 @@ def run_function(fixtures_path: Path) -> BatteryResult:
|
|||
# ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def _capital_cost_delta(task: dict) -> float:
|
||||
"""Sum the resource_budget deltas as a single scalar capital cost.
|
||||
|
||||
Phase 1a uses budget-as-cost: v1 fixtures don't carry actual
|
||||
measured costs, only budget caps. Phase 1b.2 will replace this
|
||||
with real values from the capital_ledger (#000020).
|
||||
|
||||
Sums ``max_compute_ms_delta`` (treated 1 ms = 1e-3 cost units) +
|
||||
``max_storage_delta_bytes`` (1 MB = 1 cost unit). Tunable later.
|
||||
"""
|
||||
budget = task.get("resource_budget", {})
|
||||
compute_ms = float(budget.get("max_compute_ms_delta", 0))
|
||||
storage_bytes = float(budget.get("max_storage_delta_bytes", 0))
|
||||
return compute_ms * 1e-3 + storage_bytes / 1e6
|
||||
|
||||
|
||||
# Sentinels for division-by-zero efficiency metrics. Per fox's
|
||||
# 2026-05-08 review of #fbd99a8: explicit values, not Python NaN/inf
|
||||
# accidents. We use Python ``float('inf')`` when cost=0 and gain>0 so
|
||||
# downstream consumers can compare; JSON serialization uses
|
||||
# ``default=str`` and renders inf as ``"inf"`` — callers checking
|
||||
# numeric ordering must handle both representations.
|
||||
EFFICIENCY_INFINITE = float("inf")
|
||||
EFFICIENCY_UNDEFINED = 0.0 # zero gain at zero cost = no efficiency observed
|
||||
|
||||
|
||||
def _efficiency(gain: float, cost: float) -> float:
|
||||
"""Compute gain/cost with explicit zero-cost guards.
|
||||
|
||||
- cost > 0 → gain / cost (standard ratio)
|
||||
- cost == 0, gain > 0 → EFFICIENCY_INFINITE (free improvement)
|
||||
- cost == 0, gain == 0 → EFFICIENCY_UNDEFINED (no signal)
|
||||
- cost == 0, gain < 0 → -EFFICIENCY_INFINITE (free regression — bad)
|
||||
"""
|
||||
if cost > 0:
|
||||
return gain / cost
|
||||
if gain > 0:
|
||||
return EFFICIENCY_INFINITE
|
||||
if gain < 0:
|
||||
return -EFFICIENCY_INFINITE
|
||||
return EFFICIENCY_UNDEFINED
|
||||
|
||||
|
||||
def run_finetuning(fixtures_path: Path) -> BatteryResult:
|
||||
"""Adaptation improvement check.
|
||||
"""Adaptation improvement check + adaptation_efficiency metric.
|
||||
|
||||
Each task asserts that ``child_measured_value >=
|
||||
parent_measured_value + expected_improvement_min``. Capital cost
|
||||
deltas are recorded but not gated in v1; v8 selection will use
|
||||
them (ticket #000012).
|
||||
parent_measured_value + expected_improvement_min``.
|
||||
|
||||
Per #000025 §5.2 and the 2026-05-08 fbd99a8 review:
|
||||
``adaptation_efficiency = Δimprovement / Δcapital_cost`` with
|
||||
explicit zero-cost guards (see :func:`_efficiency`). Hooks into
|
||||
the capital ledger (#000020) for cost-aware fitness signals
|
||||
that v8 fork-choice (#000012) will consume.
|
||||
"""
|
||||
per_task: list[TaskResult] = []
|
||||
efficiency_values: list[float] = []
|
||||
finite_efficiency_values: list[float] = []
|
||||
for task in iter_tasks(fixtures_path):
|
||||
task_id = task["id"]
|
||||
ok, reason = _carrier_check(task)
|
||||
|
|
@ -181,11 +230,18 @@ def run_finetuning(fixtures_path: Path) -> BatteryResult:
|
|||
expected = task.get("expected", "pass")
|
||||
observed = "pass" if improved else "fail"
|
||||
passed = observed == expected
|
||||
cost = _capital_cost_delta(task)
|
||||
efficiency = _efficiency(improvement, cost)
|
||||
efficiency_values.append(efficiency)
|
||||
if efficiency not in (EFFICIENCY_INFINITE, -EFFICIENCY_INFINITE):
|
||||
finite_efficiency_values.append(efficiency)
|
||||
detail = {
|
||||
"parent_measured_value": parent,
|
||||
"child_measured_value": child,
|
||||
"improvement": improvement,
|
||||
"min_improvement": min_improvement,
|
||||
"capital_cost_delta": cost,
|
||||
"adaptation_efficiency": efficiency,
|
||||
"expected": expected,
|
||||
}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
|
|
@ -195,12 +251,26 @@ def run_finetuning(fixtures_path: Path) -> BatteryResult:
|
|||
|
||||
total = len(per_task)
|
||||
rate = sum(1 for t in per_task if t.passed) / total if total else 0.0
|
||||
# Mean efficiency reported only over the finite tasks; +inf and
|
||||
# -inf participants are counted in their own buckets so the mean
|
||||
# stays a meaningful summary.
|
||||
mean_finite_efficiency = (
|
||||
sum(finite_efficiency_values) / len(finite_efficiency_values)
|
||||
if finite_efficiency_values else 0.0
|
||||
)
|
||||
inf_count = sum(1 for e in efficiency_values if e == EFFICIENCY_INFINITE)
|
||||
neg_inf_count = sum(1 for e in efficiency_values if e == -EFFICIENCY_INFINITE)
|
||||
meta = fixture_meta(fixtures_path)
|
||||
return _build_result(
|
||||
meta.get("sub_battery", "finetuning"),
|
||||
fixtures_path,
|
||||
per_task,
|
||||
{"adaptation_improvement_rate": rate},
|
||||
{
|
||||
"adaptation_improvement_rate": rate,
|
||||
"adaptation_efficiency_mean_finite": mean_finite_efficiency,
|
||||
"adaptation_efficiency_infinite_count": float(inf_count),
|
||||
"adaptation_efficiency_neg_infinite_count": float(neg_inf_count),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -329,16 +399,24 @@ def run_formulate(fixtures_path: Path) -> BatteryResult:
|
|||
|
||||
|
||||
def run_feedback_loop(fixtures_path: Path) -> BatteryResult:
|
||||
"""Integration coverage rate over operation/observation chains.
|
||||
"""Integration coverage + feedback_efficiency metric.
|
||||
|
||||
Each task carries a chain of (operation, observation) pairs. The
|
||||
last item lists ``expected_delta`` — content the post-chain
|
||||
state must contain. Pass = expected_delta appears in the
|
||||
final-step observation OR a post-step observation field.
|
||||
aggregated observation feed.
|
||||
|
||||
Per #000025 §5.5 and the 2026-05-08 fbd99a8 review:
|
||||
``feedback_efficiency = downstream_effect_count / Δcapital_cost``
|
||||
with explicit zero-cost guards (see :func:`_efficiency`). For
|
||||
Phase 1a the per-task cost is the chain length (one cost-unit
|
||||
per operation); Phase 1b.2 will use the capital_ledger directly.
|
||||
"""
|
||||
per_task: list[TaskResult] = []
|
||||
integrated = 0
|
||||
total_obs = 0
|
||||
efficiency_values: list[float] = []
|
||||
finite_efficiency_values: list[float] = []
|
||||
for task in iter_tasks(fixtures_path):
|
||||
task_id = task["id"]
|
||||
ok, reason = _carrier_check(task)
|
||||
|
|
@ -350,13 +428,9 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult:
|
|||
try:
|
||||
chain = task["chain"]
|
||||
total_obs += len(chain)
|
||||
# Aggregate everything observed across the chain.
|
||||
observed_text = " ".join(
|
||||
step.get("observation", "")
|
||||
for step in chain
|
||||
step.get("observation", "") for step in chain
|
||||
).lower()
|
||||
# Final-step expected_delta must appear in the aggregated
|
||||
# observation feed OR in a downstream-state field.
|
||||
final = chain[-1]
|
||||
delta_marker = (final.get("expected_delta") or "").lower()
|
||||
integrated_ok = bool(delta_marker) and delta_marker in observed_text
|
||||
|
|
@ -365,9 +439,19 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult:
|
|||
expected = task.get("expected", "pass")
|
||||
observed = "pass" if integrated_ok else "fail"
|
||||
passed = observed == expected
|
||||
# Per-task feedback_efficiency: one downstream effect per
|
||||
# successful chain, divided by chain-length-as-cost-proxy.
|
||||
downstream_effect = 1.0 if integrated_ok else 0.0
|
||||
cost = float(len(chain))
|
||||
efficiency = _efficiency(downstream_effect, cost)
|
||||
efficiency_values.append(efficiency)
|
||||
if efficiency not in (EFFICIENCY_INFINITE, -EFFICIENCY_INFINITE):
|
||||
finite_efficiency_values.append(efficiency)
|
||||
detail = {
|
||||
"integrated": integrated_ok,
|
||||
"delta_marker": delta_marker,
|
||||
"chain_length": len(chain),
|
||||
"feedback_efficiency": efficiency,
|
||||
}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
passed = False
|
||||
|
|
@ -377,6 +461,11 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult:
|
|||
total = len(per_task)
|
||||
rate = sum(1 for t in per_task if t.passed) / total if total else 0.0
|
||||
integration_rate = integrated / total_obs if total_obs else 0.0
|
||||
mean_finite_efficiency = (
|
||||
sum(finite_efficiency_values) / len(finite_efficiency_values)
|
||||
if finite_efficiency_values else 0.0
|
||||
)
|
||||
inf_count = sum(1 for e in efficiency_values if e == EFFICIENCY_INFINITE)
|
||||
meta = fixture_meta(fixtures_path)
|
||||
return _build_result(
|
||||
meta.get("sub_battery", "feedback-loop"),
|
||||
|
|
@ -385,6 +474,8 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult:
|
|||
{
|
||||
"integration_coverage_rate": rate,
|
||||
"downstream_effect_rate": integration_rate,
|
||||
"feedback_efficiency_mean_finite": mean_finite_efficiency,
|
||||
"feedback_efficiency_infinite_count": float(inf_count),
|
||||
},
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -238,6 +238,18 @@ adaptation_efficiency = improvement_delta / capital_cost_delta
|
|||
landed) and prepares fitness-cost weighing for v8 fork choice
|
||||
(#000012).
|
||||
|
||||
**Zero-cost guard (per 2026-05-08 fbd99a8 review):** Phase 1a
|
||||
implements explicit sentinels in :func:`bench.batteries.b_5f._efficiency`:
|
||||
|
||||
- `cost > 0` → standard ratio
|
||||
- `cost == 0, gain > 0` → `EFFICIENCY_INFINITE` (free improvement)
|
||||
- `cost == 0, gain == 0` → `EFFICIENCY_UNDEFINED` (= 0.0; no signal)
|
||||
- `cost == 0, gain < 0` → `-EFFICIENCY_INFINITE` (free regression)
|
||||
|
||||
Battery-level metrics report `mean_finite` (over finite values
|
||||
only) plus `infinite_count` and `neg_infinite_count` so consumers
|
||||
preserve dimensional truth without a single mean polluted by inf.
|
||||
|
||||
### 5.3 Falsification
|
||||
|
||||
**Definition:** does the system detect and correct errors?
|
||||
|
|
@ -348,6 +360,9 @@ feedback_efficiency = downstream_effect_count / capital_cost_delta
|
|||
```
|
||||
|
||||
`feedback_efficiency` again hooks the capital ledger (#000020).
|
||||
Same zero-cost-guard semantics as `adaptation_efficiency` — see the
|
||||
:func:`_efficiency` helper. Phase 1a uses chain length as the
|
||||
cost-proxy; Phase 1b.2 reads from the capital_ledger directly.
|
||||
|
||||
**Phase 1a uses existing surfaces only:** `memory_records`,
|
||||
`memory_branch_summaries`, `audit_events`. No new substrate work.
|
||||
|
|
|
|||
|
|
@ -255,6 +255,86 @@ def test_5f_feedback_loop_runs():
|
|||
assert res.metrics["integration_coverage_rate"] == 1.0
|
||||
|
||||
|
||||
# --- 5F efficiency-metric zero-cost guards (2026-05-08 review) ----
|
||||
|
||||
|
||||
def test_efficiency_zero_cost_positive_gain_returns_infinite():
|
||||
from bench.batteries.b_5f import EFFICIENCY_INFINITE, _efficiency
|
||||
|
||||
assert _efficiency(0.5, 0) == EFFICIENCY_INFINITE
|
||||
|
||||
|
||||
def test_efficiency_zero_cost_zero_gain_returns_zero():
|
||||
from bench.batteries.b_5f import EFFICIENCY_UNDEFINED, _efficiency
|
||||
|
||||
assert _efficiency(0, 0) == EFFICIENCY_UNDEFINED
|
||||
assert _efficiency(0, 0) == 0.0
|
||||
|
||||
|
||||
def test_efficiency_zero_cost_negative_gain_returns_neg_infinite():
|
||||
from bench.batteries.b_5f import EFFICIENCY_INFINITE, _efficiency
|
||||
|
||||
assert _efficiency(-0.3, 0) == -EFFICIENCY_INFINITE
|
||||
|
||||
|
||||
def test_efficiency_normal_ratio():
|
||||
from bench.batteries.b_5f import _efficiency
|
||||
|
||||
assert _efficiency(1.0, 4.0) == 0.25
|
||||
assert _efficiency(2.0, 1.0) == 2.0
|
||||
|
||||
|
||||
def test_5f_finetuning_emits_efficiency_metrics():
|
||||
res = b_5f.run_finetuning(F5F / "finetuning-v1.jsonl")
|
||||
assert "adaptation_efficiency_mean_finite" in res.metrics
|
||||
assert "adaptation_efficiency_infinite_count" in res.metrics
|
||||
assert "adaptation_efficiency_neg_infinite_count" in res.metrics
|
||||
# Per-task detail carries the per-task efficiency.
|
||||
for t in res.per_task:
|
||||
assert "adaptation_efficiency" in t.detail
|
||||
assert "capital_cost_delta" in t.detail
|
||||
|
||||
|
||||
def test_5f_feedback_loop_emits_efficiency_metrics():
|
||||
res = b_5f.run_feedback_loop(F5F / "feedback-loop-v1.jsonl")
|
||||
assert "feedback_efficiency_mean_finite" in res.metrics
|
||||
assert "feedback_efficiency_infinite_count" in res.metrics
|
||||
for t in res.per_task:
|
||||
assert "feedback_efficiency" in t.detail
|
||||
assert "chain_length" in t.detail
|
||||
|
||||
|
||||
def test_finetuning_zero_cost_fixture_emits_inf(tmp_path):
|
||||
"""Synthesize a fixture with zero resource_budget; assert inf emitted."""
|
||||
p = tmp_path / "ft-zero.jsonl"
|
||||
p.write_text(
|
||||
json.dumps({"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1"}}) + "\n" +
|
||||
json.dumps({
|
||||
"id": "5f-ft-zero",
|
||||
"carrier": "selfmodel_snapshot",
|
||||
"domain": "capability_transition",
|
||||
"pi_star_ref": "pi_selfmodel_v1",
|
||||
"parent_selfmodel": "P", "child_selfmodel": "C",
|
||||
"target_capability": "TEST",
|
||||
"parent_measured_value": 0.40,
|
||||
"child_measured_value": 0.60,
|
||||
"expected_improvement_min": 0.05,
|
||||
"resource_budget": {
|
||||
"max_compute_ms_delta": 0,
|
||||
"max_storage_delta_bytes": 0,
|
||||
},
|
||||
"expected": "pass",
|
||||
}) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
res = b_5f.run_finetuning(p)
|
||||
assert res.pass_count == 1
|
||||
assert res.metrics["adaptation_efficiency_infinite_count"] == 1.0
|
||||
# The single task's detail should record the +inf efficiency.
|
||||
from bench.batteries.b_5f import EFFICIENCY_INFINITE
|
||||
assert res.per_task[0].detail["adaptation_efficiency"] == EFFICIENCY_INFINITE
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Carrier rejection tests — runners fail unsupported carriers cleanly
|
||||
# ---------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue