From 3bff55234f3db2086e65e4b751c45e7fd31caef4 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 7 May 2026 20:57:34 -0400 Subject: [PATCH] 5f: efficiency metrics with explicit zero-cost guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per fox's 2026-05-08 review of fbd99a8: implement adaptation_efficiency and feedback_efficiency in run_finetuning + run_feedback_loop with explicit sentinels — not Python floating-point accidents. _efficiency(gain, cost) helper: - cost > 0: standard ratio - cost == 0, gain > 0: EFFICIENCY_INFINITE (free improvement) - cost == 0, gain == 0: EFFICIENCY_UNDEFINED (= 0.0; no signal) - cost == 0, gain < 0: -EFFICIENCY_INFINITE (free regression) Battery-level metrics report mean_finite (computed over finite values only) + infinite_count + neg_infinite_count so the mean stays dimensionally truthful and consumers can pivot on the special buckets separately. Phase 1a cost proxies: - run_finetuning: _capital_cost_delta sums resource_budget (max_compute_ms_delta * 1e-3 + max_storage_delta_bytes / 1e6). Phase 1b.2 will replace with real capital_ledger reads. - run_feedback_loop: chain length = cost. Phase 1b.2 capital_ledger integration replaces it. Tests added (7): - _efficiency over four boundary cases - 5F finetuning + feedback_loop emit the new metrics keys - Synthetic zero-cost finetuning fixture verifies +inf path Full suite: 1110 passed, 36 skipped. Closes the one actionable from the fbd99a8 review. --- bench/batteries/b_5f.py | 115 ++++++++++++++++++++--- docs/tickets/ticket-000025-5f-battery.md | 15 +++ tests/test_bench_batteries.py | 80 ++++++++++++++++ 3 files changed, 198 insertions(+), 12 deletions(-) diff --git a/bench/batteries/b_5f.py b/bench/batteries/b_5f.py index f01cf50..d8094ae 100644 --- a/bench/batteries/b_5f.py +++ b/bench/batteries/b_5f.py @@ -155,15 +155,64 @@ def run_function(fixtures_path: Path) -> BatteryResult: # --------------------------------------------------------------------- +def _capital_cost_delta(task: dict) -> float: + """Sum the resource_budget deltas as a single scalar capital cost. + + Phase 1a uses budget-as-cost: v1 fixtures don't carry actual + measured costs, only budget caps. Phase 1b.2 will replace this + with real values from the capital_ledger (#000020). + + Sums ``max_compute_ms_delta`` (treated 1 ms = 1e-3 cost units) + + ``max_storage_delta_bytes`` (1 MB = 1 cost unit). Tunable later. + """ + budget = task.get("resource_budget", {}) + compute_ms = float(budget.get("max_compute_ms_delta", 0)) + storage_bytes = float(budget.get("max_storage_delta_bytes", 0)) + return compute_ms * 1e-3 + storage_bytes / 1e6 + + +# Sentinels for division-by-zero efficiency metrics. Per fox's +# 2026-05-08 review of #fbd99a8: explicit values, not Python NaN/inf +# accidents. We use Python ``float('inf')`` when cost=0 and gain>0 so +# downstream consumers can compare; JSON serialization uses +# ``default=str`` and renders inf as ``"inf"`` — callers checking +# numeric ordering must handle both representations. +EFFICIENCY_INFINITE = float("inf") +EFFICIENCY_UNDEFINED = 0.0 # zero gain at zero cost = no efficiency observed + + +def _efficiency(gain: float, cost: float) -> float: + """Compute gain/cost with explicit zero-cost guards. + + - cost > 0 → gain / cost (standard ratio) + - cost == 0, gain > 0 → EFFICIENCY_INFINITE (free improvement) + - cost == 0, gain == 0 → EFFICIENCY_UNDEFINED (no signal) + - cost == 0, gain < 0 → -EFFICIENCY_INFINITE (free regression — bad) + """ + if cost > 0: + return gain / cost + if gain > 0: + return EFFICIENCY_INFINITE + if gain < 0: + return -EFFICIENCY_INFINITE + return EFFICIENCY_UNDEFINED + + def run_finetuning(fixtures_path: Path) -> BatteryResult: - """Adaptation improvement check. + """Adaptation improvement check + adaptation_efficiency metric. Each task asserts that ``child_measured_value >= - parent_measured_value + expected_improvement_min``. Capital cost - deltas are recorded but not gated in v1; v8 selection will use - them (ticket #000012). + parent_measured_value + expected_improvement_min``. + + Per #000025 §5.2 and the 2026-05-08 fbd99a8 review: + ``adaptation_efficiency = Δimprovement / Δcapital_cost`` with + explicit zero-cost guards (see :func:`_efficiency`). Hooks into + the capital ledger (#000020) for cost-aware fitness signals + that v8 fork-choice (#000012) will consume. """ per_task: list[TaskResult] = [] + efficiency_values: list[float] = [] + finite_efficiency_values: list[float] = [] for task in iter_tasks(fixtures_path): task_id = task["id"] ok, reason = _carrier_check(task) @@ -181,11 +230,18 @@ def run_finetuning(fixtures_path: Path) -> BatteryResult: expected = task.get("expected", "pass") observed = "pass" if improved else "fail" passed = observed == expected + cost = _capital_cost_delta(task) + efficiency = _efficiency(improvement, cost) + efficiency_values.append(efficiency) + if efficiency not in (EFFICIENCY_INFINITE, -EFFICIENCY_INFINITE): + finite_efficiency_values.append(efficiency) detail = { "parent_measured_value": parent, "child_measured_value": child, "improvement": improvement, "min_improvement": min_improvement, + "capital_cost_delta": cost, + "adaptation_efficiency": efficiency, "expected": expected, } except Exception as exc: # noqa: BLE001 @@ -195,12 +251,26 @@ def run_finetuning(fixtures_path: Path) -> BatteryResult: total = len(per_task) rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + # Mean efficiency reported only over the finite tasks; +inf and + # -inf participants are counted in their own buckets so the mean + # stays a meaningful summary. + mean_finite_efficiency = ( + sum(finite_efficiency_values) / len(finite_efficiency_values) + if finite_efficiency_values else 0.0 + ) + inf_count = sum(1 for e in efficiency_values if e == EFFICIENCY_INFINITE) + neg_inf_count = sum(1 for e in efficiency_values if e == -EFFICIENCY_INFINITE) meta = fixture_meta(fixtures_path) return _build_result( meta.get("sub_battery", "finetuning"), fixtures_path, per_task, - {"adaptation_improvement_rate": rate}, + { + "adaptation_improvement_rate": rate, + "adaptation_efficiency_mean_finite": mean_finite_efficiency, + "adaptation_efficiency_infinite_count": float(inf_count), + "adaptation_efficiency_neg_infinite_count": float(neg_inf_count), + }, ) @@ -329,16 +399,24 @@ def run_formulate(fixtures_path: Path) -> BatteryResult: def run_feedback_loop(fixtures_path: Path) -> BatteryResult: - """Integration coverage rate over operation/observation chains. + """Integration coverage + feedback_efficiency metric. Each task carries a chain of (operation, observation) pairs. The last item lists ``expected_delta`` — content the post-chain state must contain. Pass = expected_delta appears in the - final-step observation OR a post-step observation field. + aggregated observation feed. + + Per #000025 §5.5 and the 2026-05-08 fbd99a8 review: + ``feedback_efficiency = downstream_effect_count / Δcapital_cost`` + with explicit zero-cost guards (see :func:`_efficiency`). For + Phase 1a the per-task cost is the chain length (one cost-unit + per operation); Phase 1b.2 will use the capital_ledger directly. """ per_task: list[TaskResult] = [] integrated = 0 total_obs = 0 + efficiency_values: list[float] = [] + finite_efficiency_values: list[float] = [] for task in iter_tasks(fixtures_path): task_id = task["id"] ok, reason = _carrier_check(task) @@ -350,13 +428,9 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult: try: chain = task["chain"] total_obs += len(chain) - # Aggregate everything observed across the chain. observed_text = " ".join( - step.get("observation", "") - for step in chain + step.get("observation", "") for step in chain ).lower() - # Final-step expected_delta must appear in the aggregated - # observation feed OR in a downstream-state field. final = chain[-1] delta_marker = (final.get("expected_delta") or "").lower() integrated_ok = bool(delta_marker) and delta_marker in observed_text @@ -365,9 +439,19 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult: expected = task.get("expected", "pass") observed = "pass" if integrated_ok else "fail" passed = observed == expected + # Per-task feedback_efficiency: one downstream effect per + # successful chain, divided by chain-length-as-cost-proxy. + downstream_effect = 1.0 if integrated_ok else 0.0 + cost = float(len(chain)) + efficiency = _efficiency(downstream_effect, cost) + efficiency_values.append(efficiency) + if efficiency not in (EFFICIENCY_INFINITE, -EFFICIENCY_INFINITE): + finite_efficiency_values.append(efficiency) detail = { "integrated": integrated_ok, "delta_marker": delta_marker, + "chain_length": len(chain), + "feedback_efficiency": efficiency, } except Exception as exc: # noqa: BLE001 passed = False @@ -377,6 +461,11 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult: total = len(per_task) rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 integration_rate = integrated / total_obs if total_obs else 0.0 + mean_finite_efficiency = ( + sum(finite_efficiency_values) / len(finite_efficiency_values) + if finite_efficiency_values else 0.0 + ) + inf_count = sum(1 for e in efficiency_values if e == EFFICIENCY_INFINITE) meta = fixture_meta(fixtures_path) return _build_result( meta.get("sub_battery", "feedback-loop"), @@ -385,6 +474,8 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult: { "integration_coverage_rate": rate, "downstream_effect_rate": integration_rate, + "feedback_efficiency_mean_finite": mean_finite_efficiency, + "feedback_efficiency_infinite_count": float(inf_count), }, ) diff --git a/docs/tickets/ticket-000025-5f-battery.md b/docs/tickets/ticket-000025-5f-battery.md index 5091b08..058e3eb 100644 --- a/docs/tickets/ticket-000025-5f-battery.md +++ b/docs/tickets/ticket-000025-5f-battery.md @@ -238,6 +238,18 @@ adaptation_efficiency = improvement_delta / capital_cost_delta landed) and prepares fitness-cost weighing for v8 fork choice (#000012). +**Zero-cost guard (per 2026-05-08 fbd99a8 review):** Phase 1a +implements explicit sentinels in :func:`bench.batteries.b_5f._efficiency`: + +- `cost > 0` → standard ratio +- `cost == 0, gain > 0` → `EFFICIENCY_INFINITE` (free improvement) +- `cost == 0, gain == 0` → `EFFICIENCY_UNDEFINED` (= 0.0; no signal) +- `cost == 0, gain < 0` → `-EFFICIENCY_INFINITE` (free regression) + +Battery-level metrics report `mean_finite` (over finite values +only) plus `infinite_count` and `neg_infinite_count` so consumers +preserve dimensional truth without a single mean polluted by inf. + ### 5.3 Falsification **Definition:** does the system detect and correct errors? @@ -348,6 +360,9 @@ feedback_efficiency = downstream_effect_count / capital_cost_delta ``` `feedback_efficiency` again hooks the capital ledger (#000020). +Same zero-cost-guard semantics as `adaptation_efficiency` — see the +:func:`_efficiency` helper. Phase 1a uses chain length as the +cost-proxy; Phase 1b.2 reads from the capital_ledger directly. **Phase 1a uses existing surfaces only:** `memory_records`, `memory_branch_summaries`, `audit_events`. No new substrate work. diff --git a/tests/test_bench_batteries.py b/tests/test_bench_batteries.py index db00a48..16b23e3 100644 --- a/tests/test_bench_batteries.py +++ b/tests/test_bench_batteries.py @@ -255,6 +255,86 @@ def test_5f_feedback_loop_runs(): assert res.metrics["integration_coverage_rate"] == 1.0 +# --- 5F efficiency-metric zero-cost guards (2026-05-08 review) ---- + + +def test_efficiency_zero_cost_positive_gain_returns_infinite(): + from bench.batteries.b_5f import EFFICIENCY_INFINITE, _efficiency + + assert _efficiency(0.5, 0) == EFFICIENCY_INFINITE + + +def test_efficiency_zero_cost_zero_gain_returns_zero(): + from bench.batteries.b_5f import EFFICIENCY_UNDEFINED, _efficiency + + assert _efficiency(0, 0) == EFFICIENCY_UNDEFINED + assert _efficiency(0, 0) == 0.0 + + +def test_efficiency_zero_cost_negative_gain_returns_neg_infinite(): + from bench.batteries.b_5f import EFFICIENCY_INFINITE, _efficiency + + assert _efficiency(-0.3, 0) == -EFFICIENCY_INFINITE + + +def test_efficiency_normal_ratio(): + from bench.batteries.b_5f import _efficiency + + assert _efficiency(1.0, 4.0) == 0.25 + assert _efficiency(2.0, 1.0) == 2.0 + + +def test_5f_finetuning_emits_efficiency_metrics(): + res = b_5f.run_finetuning(F5F / "finetuning-v1.jsonl") + assert "adaptation_efficiency_mean_finite" in res.metrics + assert "adaptation_efficiency_infinite_count" in res.metrics + assert "adaptation_efficiency_neg_infinite_count" in res.metrics + # Per-task detail carries the per-task efficiency. + for t in res.per_task: + assert "adaptation_efficiency" in t.detail + assert "capital_cost_delta" in t.detail + + +def test_5f_feedback_loop_emits_efficiency_metrics(): + res = b_5f.run_feedback_loop(F5F / "feedback-loop-v1.jsonl") + assert "feedback_efficiency_mean_finite" in res.metrics + assert "feedback_efficiency_infinite_count" in res.metrics + for t in res.per_task: + assert "feedback_efficiency" in t.detail + assert "chain_length" in t.detail + + +def test_finetuning_zero_cost_fixture_emits_inf(tmp_path): + """Synthesize a fixture with zero resource_budget; assert inf emitted.""" + p = tmp_path / "ft-zero.jsonl" + p.write_text( + json.dumps({"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1"}}) + "\n" + + json.dumps({ + "id": "5f-ft-zero", + "carrier": "selfmodel_snapshot", + "domain": "capability_transition", + "pi_star_ref": "pi_selfmodel_v1", + "parent_selfmodel": "P", "child_selfmodel": "C", + "target_capability": "TEST", + "parent_measured_value": 0.40, + "child_measured_value": 0.60, + "expected_improvement_min": 0.05, + "resource_budget": { + "max_compute_ms_delta": 0, + "max_storage_delta_bytes": 0, + }, + "expected": "pass", + }) + "\n", + encoding="utf-8", + ) + res = b_5f.run_finetuning(p) + assert res.pass_count == 1 + assert res.metrics["adaptation_efficiency_infinite_count"] == 1.0 + # The single task's detail should record the +inf efficiency. + from bench.batteries.b_5f import EFFICIENCY_INFINITE + assert res.per_task[0].detail["adaptation_efficiency"] == EFFICIENCY_INFINITE + + # --------------------------------------------------------------------- # Carrier rejection tests — runners fail unsupported carriers cleanly # ---------------------------------------------------------------------