diff --git a/Makefile b/Makefile index d0f8721..a7e70ba 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,8 @@ SEARCH_Q ?= computer chain-check chain-check-shards \ falsify burn burn-kindergarten inspect bootstrap-crawler test-crawler crawl-ingest \ recrawl-check bench-qa bench-qa-smoke bench-qa-progressive-and \ - prometheus-trigger-probe \ + prometheus-trigger-probe bench-5f-threshold-calibration \ + bench-5f-selfmodel-snapshot bench-5f-finetuning-shardchain \ bootstrap-math clean clean-db clean-data help \ textbooks-summary textbooks-urls fetch-textbooks textbooks-stats textbooks-verify \ crawl-textbooks crawl-textbooks-stats textbook textbook-list @@ -489,6 +490,32 @@ bench-5f-falsification-live: bootstrap ## 5F Falsification via real arborist.qa. bench-5f-live: bench-5f-formulate-live bench-5f-feedback-loop-live bench-5f-function-live bench-5f-finetuning-live bench-5f-falsification-live ## all 5F Phase-1b.2 live wire-ups +# #000025 §10.11 — persistent SelfModel-chain lineage. Each run of +# bench-5f-selfmodel-snapshot runs the 5S/5T/5F embedded packs and +# appends ONE chained snapshot (capability claims = the battery +# rates) to SELFMODEL_CHAIN_DB. Run it >= 2x, then +# bench-5f-finetuning-shardchain measures improvement between the two +# most-recent snapshots — a real cross-run lineage, not a synthetic +# parent/child pair. Operator targets: NOT part of `make bench-5f`, +# `make test`, or a fresh checkout. +SELFMODEL_CHAIN_DB ?= $(SHARDS_DIR)/selfmodel-chain.db +bench-5f-selfmodel-snapshot: bootstrap ## #000025 §10.11 — append one SelfModel snapshot (rates as claims) to the chain shard + PYTHONUNBUFFERED=1 ARBORIST_SELFMODEL_CHAIN_DB=$(SELFMODEL_CHAIN_DB) \ + $(PY) bench/scripts/selfmodel_chain_snapshot.py +bench-5f-finetuning-shardchain: bootstrap ## #000025 §10.11 — Finetuning over the two latest chain snapshots (run bench-5f-selfmodel-snapshot >=2x first) + PYTHONUNBUFFERED=1 ARBORIST_SELFMODEL_CHAIN_DB=$(SELFMODEL_CHAIN_DB) \ + $(PY) -m bench.batteries.runner --battery 5f --sub finetuning \ + --fixtures bench/fixtures/5f/finetuning-shardchain-v1.jsonl + +# #000025 §10.14 — ForkScore threshold-calibration handoff to #000012. +# Runs the canonical 5S/5T/5F packs + the 5F live packs, reports +# baseline rates / granularity / floor-constant sanity checks. +# Pure measurement; output is the deliverable #000012 cites. +FIVEF_CALIBRATION_OUT ?= bench/results/5f-threshold-calibration-$(shell date -u +%Y-%m-%d).md +bench-5f-threshold-calibration: bootstrap ## #000025 §10.14 — 5S/5T/5F → ForkScore threshold calibration report + PYTHONUNBUFFERED=1 $(PY) bench/scripts/fivef_threshold_calibration.py \ + --out $(FIVEF_CALIBRATION_OUT) + bench-5r: bootstrap ## 5R battery (React+Rearrange+Restore+Replicate+Resonate) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub react --fixtures bench/fixtures/5r/react-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub rearrange --fixtures bench/fixtures/5r/rearrange-v1.jsonl diff --git a/bench/batteries/b_5f.py b/bench/batteries/b_5f.py index 182deb7..5e3027c 100644 --- a/bench/batteries/b_5f.py +++ b/bench/batteries/b_5f.py @@ -15,6 +15,7 @@ memory_snapshot carriers; future carriers fail/skip explicitly. from __future__ import annotations +import time from pathlib import Path from bench.batteries.base import ( @@ -305,13 +306,83 @@ def _live_finetuning_measure(task: dict) -> dict: pass +def _chain_finetuning_measure(task: dict) -> dict: + """Read the two most-recent SelfModel snapshots from a *persistent* + chain shard (#000025 §10.11) and return the parent/child measured + values for ``target_capability``. + + The shard is the one ``bench/scripts/selfmodel_chain_snapshot.py`` + appends to (default ``~/.arborist/shards/selfmodel-chain.db``, + overridable per-task via ``selfmodel_shard`` and via the + ``ARBORIST_SELFMODEL_CHAIN_DB`` env var). Unlike the Phase-1b.2 + "live" path — which fabricates a parent + child in a fresh temp + shard every run — this reads a genuine lineage that outlives the + process: ``latest()`` is the child, its ``parent_selfmodel_root`` + is the parent. + + Raises if the shard is absent, has < 2 snapshots, or neither + snapshot carries a claim for ``target_capability`` — the caller + turns that into an honest task failure (this pack is meant to run + *after* the chain has been bootstrapped, not on a fresh checkout). + """ + import os as _os + + from arborist.selfmodel.store import claims_for, latest, load + from arborist.store import connect + + metric = task["target_capability"] + # Resolution order: ARBORIST_SELFMODEL_CHAIN_DB env (the test / + # operator override) → the fixture's ``selfmodel_shard`` value + # (which is also the dispatch gate) → the conventional default. + raw_path = ( + _os.environ.get("ARBORIST_SELFMODEL_CHAIN_DB") + or task.get("selfmodel_shard") + or "~/.arborist/shards/selfmodel-chain.db" + ) + shard = Path(_os.path.expandvars(str(raw_path))).expanduser() + if not shard.exists(): + raise FileNotFoundError( + f"selfmodel chain shard not found: {shard} — run " + "`make bench-5f-selfmodel-snapshot` (twice) to bootstrap it" + ) + conn = connect(shard) + try: + child_row = latest(conn) + if child_row is None: + raise ValueError(f"no SelfModel snapshots in {shard}") + parent_root = child_row.get("parent_selfmodel_root") + if not parent_root: + raise ValueError( + f"chain shard {shard} has only one snapshot; need >= 2 " + "(run the bootstrapper again to extend the lineage)" + ) + parent_row = load(conn, parent_root) + if parent_row is None: + raise ValueError(f"parent snapshot {parent_root[:12]}… missing from {shard}") + + def _measured(root: str) -> float: + for c in claims_for(conn, root): + if c["metric"] == metric and c["measured_value"] is not None: + return float(c["measured_value"]) + raise ValueError(f"no claim for {metric!r} on snapshot {root[:12]}…") + + return { + "parent_root": parent_root, + "child_root": child_row["selfmodel_root"], + "parent_measured": _measured(parent_root), + "child_measured": _measured(child_row["selfmodel_root"]), + } + finally: + conn.close() + + def run_finetuning(fixtures_path: Path) -> BatteryResult: """Adaptation improvement check + adaptation_efficiency metric. Each task asserts that ``child_measured_value >= parent_measured_value + expected_improvement_min``. - Two fixture modes: + Three fixture modes (selected per-task): - **Embedded** (Phase 1a): parent/child measured values come directly from the fixture; runner does the math. @@ -323,6 +394,15 @@ def run_finetuning(fixtures_path: Path) -> BatteryResult: :func:`arborist.selfmodel.store.claims_for`, then runs the same improvement check. Tests the SelfModel persistence surface, not just the fixture data. + - **Shard-chain** (§10.11 — gate via ``"selfmodel_shard"``): + runner reads the two most-recent snapshots from a *persistent* + chain shard (the one ``bench/scripts/selfmodel_chain_snapshot.py`` + grows, one snapshot per run) and measures improvement on + ``target_capability`` between them. This is the real lineage — + not a pair fabricated for the test — so the chained Δ reflects + genuine cross-run drift (0.0 today; the packs are at ceiling). + A task whose shard is absent / too short fails honestly: the + pack runs *after* the bootstrapper, not on a fresh checkout. Per #000025 §5.2 and the 2026-05-08 fbd99a8 review: ``adaptation_efficiency = Δimprovement / Δcapital_cost`` with @@ -340,7 +420,12 @@ def run_finetuning(fixtures_path: Path) -> BatteryResult: ) continue try: - if task.get("live"): + if task.get("selfmodel_shard"): + source = "shard-chain" + chain = _chain_finetuning_measure(task) + parent = float(chain["parent_measured"]) + child = float(chain["child_measured"]) + elif task.get("live"): source = "live" live = _live_finetuning_measure(task) parent = float(live["parent_measured"]) @@ -370,6 +455,9 @@ def run_finetuning(fixtures_path: Path) -> BatteryResult: "adaptation_efficiency": efficiency, "expected": expected, } + if source == "shard-chain": + detail["chain_parent_root"] = chain["parent_root"] + detail["chain_child_root"] = chain["child_root"] except Exception as exc: # noqa: BLE001 passed = False detail = {"reason": f"{type(exc).__name__}: {exc}"} @@ -725,15 +813,32 @@ def _live_delta_satisfied(state: dict, expected: dict) -> tuple[bool, str]: return True, "all live deltas satisfied" +def _persisted_cost(state: dict) -> float: + """Real-workload cost of a live feedback chain: the persisted + footprint it actually produced, not the count of *requested* ops. + + ``audit_event`` rows the chain wrote (count) + their body bytes / + 1e6 (matching :func:`_capital_cost_delta`'s 1 MB = 1 cost-unit + convention). Every non-empty live chain writes ≥ 1 audit event + (the ``append_audit`` op, the ``memory_snapshot_landed`` event, + etc.), so this is > 0 for any chain that did work — :func:`_efficiency` + still handles the cost-0 edge as documented. + """ + n_audit = len(state.get("audit_event_types", [])) + body_bytes = sum(len(b) for b in state.get("audit_event_bodies", []) if b) + return float(n_audit) + body_bytes / 1e6 + + def run_feedback_loop(fixtures_path: Path) -> BatteryResult: - """Integration coverage + feedback_efficiency metric. + """Integration coverage + feedback_efficiency + feedback_latency. Two fixture modes (selected per-task): - **Embedded** (Phase 1a): fixture provides a ``chain`` of ``(operation, observation)`` pairs; runner aggregates the observation text and checks ``expected_delta`` (a string - marker) appears in it. + marker) appears in it. No real work happens, so latency is + ``None`` and the cost-proxy stays ``len(chain)``. - **Live** (Phase 1b.2): fixture provides ``live_chain`` — a sequence of ops applied to a fresh temp arborist shard via :func:`_live_feedback_chain` — plus an ``expected_delta`` @@ -743,15 +848,28 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult: Per #000025 §5.5 and the 2026-05-08 fbd99a8 review: ``feedback_efficiency = downstream_effect_count / Δcapital_cost`` - with explicit zero-cost guards (see :func:`_efficiency`). For - Phase 1a the per-task cost is the chain length (one cost-unit - per operation); Phase 1b.2 will use the capital_ledger directly. + with explicit zero-cost guards (see :func:`_efficiency`). + + **§10.13 (2026-05-11) — calibrated against real workload.** For a + live chain the cost is the *persisted footprint* :func:`_persisted_cost` + — the audit-event rows the chain actually wrote plus their body + bytes — instead of the Phase-1a ``len(chain)`` proxy (which + counted *requested* ops, not work done). §10.13 also computes + ``feedback_latency`` — listed in §5.5 since Phase 1a but never + implemented — as the wall-clock seconds to apply a live chain + against its temp shard. Latency is a wall-clock field + (non-deterministic run-to-run, like ``BatteryResult.timestamp``) + and is **not** consumed by ``fork_score`` — it's surfaced for + operational observability + the capital ledger (#000020), not for + fork selection. Embedded tasks keep ``len(chain)`` as cost and + report ``feedback_latency_seconds = None``. """ per_task: list[TaskResult] = [] integrated = 0 total_obs = 0 efficiency_values: list[float] = [] finite_efficiency_values: list[float] = [] + live_latencies: list[float] = [] for task in iter_tasks(fixtures_path): task_id = task["id"] ok, reason = _carrier_check(task) @@ -764,7 +882,10 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult: if "live_chain" in task: source = "live" live_chain = task["live_chain"] + _t0 = time.perf_counter() state = _live_feedback_chain(live_chain) + latency_s = time.perf_counter() - _t0 + live_latencies.append(latency_s) ok_live, why = _live_delta_satisfied( state, task.get("expected_delta", {}) or {} ) @@ -777,7 +898,7 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult: observed = "pass" if integrated_ok else "fail" passed = observed == expected_outcome downstream_effect = 1.0 if integrated_ok else 0.0 - cost = float(chain_length) + cost = _persisted_cost(state) # §10.13: real footprint efficiency = _efficiency(downstream_effect, cost) efficiency_values.append(efficiency) if efficiency not in ( @@ -790,6 +911,9 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult: "reason": why, "chain_length": chain_length, "feedback_efficiency": efficiency, + "feedback_latency_seconds": latency_s, + "persisted_cost": cost, + "persisted_audit_rows": len(state.get("audit_event_types", [])), "live_state": { "audit_event_count": len(state["audit_event_types"]), "memory_branch_count": len(state["memory_branch_ids"]), @@ -824,6 +948,7 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult: "delta_marker": delta_marker, "chain_length": len(chain), "feedback_efficiency": efficiency, + "feedback_latency_seconds": None, # no real work } except Exception as exc: # noqa: BLE001 passed = False @@ -838,6 +963,12 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult: if finite_efficiency_values else 0.0 ) inf_count = sum(1 for e in efficiency_values if e == EFFICIENCY_INFINITE) + # §10.13: mean wall-clock latency over the live tasks (0.0 when a + # pack has no live_chain tasks). Wall-clock, run-to-run variable — + # not a fork_score input. + feedback_latency_mean = ( + sum(live_latencies) / len(live_latencies) if live_latencies else 0.0 + ) meta = fixture_meta(fixtures_path) return _build_result( meta.get("sub_battery", "feedback-loop"), @@ -848,6 +979,8 @@ def run_feedback_loop(fixtures_path: Path) -> BatteryResult: "downstream_effect_rate": integration_rate, "feedback_efficiency_mean_finite": mean_finite_efficiency, "feedback_efficiency_infinite_count": float(inf_count), + "feedback_latency_mean_seconds": feedback_latency_mean, + "feedback_live_task_count": float(len(live_latencies)), }, ) diff --git a/bench/fixtures/5f/finetuning-shardchain-v1.jsonl b/bench/fixtures/5f/finetuning-shardchain-v1.jsonl new file mode 100644 index 0000000..b02db6e --- /dev/null +++ b/bench/fixtures/5f/finetuning-shardchain-v1.jsonl @@ -0,0 +1,7 @@ +{"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1", "task_count": 6, "notes": "#000025 §10.11: shard-chain Finetuning — reads the two most-recent SelfModel snapshots from a PERSISTENT chain shard (default ~/.arborist/shards/selfmodel-chain.db, override via ARBORIST_SELFMODEL_CHAIN_DB) and measures improvement on target_capability between them. The chain is grown one snapshot per run by `make bench-5f-selfmodel-snapshot`; run THAT (>=2x) before this pack. Tasks fail honestly if the shard is absent / too short — this is an operator pack, not part of `make bench-5f` or the default test suite. expected_improvement_min=0.0 = a no-regression assertion (the embedded packs are at ceiling, so the chained delta is 0.0)."}} +{"id":"5f-ft-chain-001","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"capability_transition","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","selfmodel_shard":"~/.arborist/shards/selfmodel-chain.db","target_capability":"5S-syntax","expected_improvement_min":0.0,"resource_budget":{"max_compute_ms_delta":0,"max_storage_delta_bytes":0},"expected":"pass"} +{"id":"5f-ft-chain-002","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"capability_transition","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","selfmodel_shard":"~/.arborist/shards/selfmodel-chain.db","target_capability":"5S-syllogism","expected_improvement_min":0.0,"resource_budget":{"max_compute_ms_delta":0,"max_storage_delta_bytes":0},"expected":"pass"} +{"id":"5f-ft-chain-003","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"capability_transition","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","selfmodel_shard":"~/.arborist/shards/selfmodel-chain.db","target_capability":"5T-time","expected_improvement_min":0.0,"resource_budget":{"max_compute_ms_delta":0,"max_storage_delta_bytes":0},"expected":"pass"} +{"id":"5f-ft-chain-004","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"capability_transition","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","selfmodel_shard":"~/.arborist/shards/selfmodel-chain.db","target_capability":"5T-truthtables","expected_improvement_min":0.0,"resource_budget":{"max_compute_ms_delta":0,"max_storage_delta_bytes":0},"expected":"pass"} +{"id":"5f-ft-chain-005","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"capability_transition","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","selfmodel_shard":"~/.arborist/shards/selfmodel-chain.db","target_capability":"5F-feedback-loop","expected_improvement_min":0.0,"resource_budget":{"max_compute_ms_delta":0,"max_storage_delta_bytes":0},"expected":"pass"} +{"id":"5f-ft-chain-006","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"capability_transition","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","selfmodel_shard":"~/.arborist/shards/selfmodel-chain.db","target_capability":"5F-falsification","expected_improvement_min":0.0,"resource_budget":{"max_compute_ms_delta":0,"max_storage_delta_bytes":0},"expected":"pass"} diff --git a/bench/results/5f-threshold-calibration-2026-05-11.md b/bench/results/5f-threshold-calibration-2026-05-11.md new file mode 100644 index 0000000..39e3b36 --- /dev/null +++ b/bench/results/5f-threshold-calibration-2026-05-11.md @@ -0,0 +1,60 @@ +# 5S/5T/5F → v8 ForkScore threshold-calibration handoff + +**Date:** 2026-05-11T11:26:39Z +**Ticket:** #000025 §10.14 (closure deliverable) — handoff to #000012. +**Method:** ran the canonical 5S/5T/5F sub-batteries (the packs `fork_score._BATTERY_RATE_KEYS` reads) plus the 5F live packs; computed baseline rates, observability granularity (1/n), and ran `fork_score` on the parent vs three synthetic child perturbations. Pure measurement. + +## 1. Baseline rates + granularity + +Each row's *rate* is the single metric `fork_score` consumes for that sub-battery (`_BATTERY_RATE_KEYS`). *Granularity* = 1/n — the smallest rate change a single fixture flip can produce, i.e. the finest Δ the scorer could ever observe on that pack. + +| battery | sub-battery | fixtures (n) | embedded rate | live rate | granularity (1/n) | +|---|---|---|---|---|---| +| 5s | syntax | 10 | 1.0000 | — | 0.1000 (10.0pp) | +| 5s | semantics | 8 | 1.0000 | — | 0.1250 (12.5pp) | +| 5s | syllogism | 30 | 1.0000 | — | 0.0333 (3.3pp) | +| 5s | synthesis | 30 | 1.0000 | — | 0.0333 (3.3pp) | +| 5s | semiotics | 30 | 1.0000 | — | 0.0333 (3.3pp) | +| 5t | transfer-learning | 30 | 1.0000 | — | 0.0333 (3.3pp) | +| 5t | triangulation | 30 | 1.0000 | — | 0.0333 (3.3pp) | +| 5t | truthtables | 30 | 1.0000 | — | 0.0333 (3.3pp) | +| 5t | transitivity | 30 | 1.0000 | — | 0.0333 (3.3pp) | +| 5t | time | 30 | 1.0000 | — | 0.0333 (3.3pp) | +| 5f | function | 50 | 1.0000 | 1.0000 | 0.0200 (2.0pp) | +| 5f | finetuning | 50 | 1.0000 | 1.0000 | 0.0200 (2.0pp) | +| 5f | falsification | 62 | 1.0000 | 1.0000 | 0.0161 (1.6pp) | +| 5f | formulate | 50 | 1.0000 | 1.0000 | 0.0200 (2.0pp) | +| 5f | feedback-loop | 50 | 1.0000 | 1.0000 | 0.0200 (2.0pp) | + +- **5S mean baseline rate:** 1.0000 +- **5T mean baseline rate:** 1.0000 +- **5F mean baseline rate:** 1.0000 +- **Coarsest pack granularity:** 0.1250 (12.5pp) +- **Finest pack granularity:** 0.0161 (1.6pp) + +## 2. Identity-fork verdict (no change) + +`fork_score(parent, parent)` → **MARGINAL (score +0.0000)** + +Every 5S/5T/5F pack is at ceiling (rate 1.0) at HEAD, so every Δ-rate term is exactly 0 and the identity fork lands in the `[0, SIGNAL_FLOOR)` band → **MARGINAL**, not ACCEPT. Takeaway for #000012: an unchanged child is *not* auto-accepted; positive score has to come from a Δ-rate gain (needs harder fixtures, see §4), an efficiency-bonus increase (`adaptation_efficiency` / `feedback_efficiency`), or one of the non-bench terms (`selfmodel_calibration_gain`, `audit_completeness`, …). + +## 3. Floor-constant sanity checks + +`SIGNAL_FLOOR = 0.05` · `HARD_REGRESSION_FLOOR = 0.05` (current `arborist/substrate/fork_score.py`). The production parent is at ceiling, so these are exercised against a synthetic *degraded parent* — every 5F sub-battery rate knocked to 0.90, giving a child headroom to improve. + +| scenario | verdict | +|---|---| +| identity — `fork_score(HEAD, HEAD)` | MARGINAL (score +0.0000) | +| degraded parent → child recovers **all 5** 5F subs by +0.06 | ACCEPT (score +0.0600) | +| degraded parent → child recovers **only 1** 5F sub by +0.06 | MARGINAL (score +0.0120) | +| degraded parent → child drops one 5F sub by −0.05 | REJECT (score -0.0300) — flags: REGRESSION_5F:5f/falsification: -0.050 | + +Row 2 vs row 3 is the **5× averaging dilution**: `_delta_5f` means over all 5 sub-batteries, so the same +0.06 gain scores `1.0·0.06 = 0.06` (ACCEPT) when applied to all five subs but only `1.0·(0.06/5) = 0.012` (MARGINAL) when applied to one. Each battery contributes a separate Δ-term at weight 1.0, so a uniform +`SIGNAL_FLOOR` across *every* sub of *all three* batteries would score `~0.15`. + +## 4. Recommendation for #000012 + +- **Keep `SIGNAL_FLOOR = 0.05` and `HARD_REGRESSION_FLOOR = 0.05`.** They match `docs/bench-maxing.md`'s 5-pp signal floor and the sanity checks in §3 behave as documented. +- **Granularity caveat (load-bearing):** the foundational 5S packs are *coarser* than the floors — `syntax` (n=10 → 10.0pp) and `semantics` (n=8 → 12.5pp). On those packs a single fixture flip is ≥ 5pp, so *any* regression there trips the hard-reject. That is the intended zero-tolerance behaviour on the syntax/semantics base — not a bug — but #000012 should document it: HARD_REGRESSION_FLOOR is not a tunable knob on the small packs, it's effectively 'one fixture'. The n=30 / n=50 / n=62 packs have 3.3pp / 2.0pp / 1.6pp granularity, so there 5pp ≈ 1.5–3 fixture flips. +- **5× averaging dilution (document this in #000012):** a single sub-battery's rate gain is worth a fifth of its face value because `_delta_5{s,t,f}` means over 5 subs. So 'a fork must improve by `SIGNAL_FLOOR`' really means *one of*: ~`0.25` on a single sub, ~`0.05` uniform across one battery's 5 subs, or ~`0.017` uniform across the whole 15-sub suite. If #000012 wants single-sub improvements to weigh equally it should switch `_delta_*` from mean to max-or-sum — but that's a #000012-owned design call, not a calibration finding. +- **Ceiling saturation:** until harder fixtures drop a pack's baseline below 1.0, the `α·Δ5s + β·Δ5t + γ·Δ5f` terms can only be ≤ 0. ForkScore acceptance at the current pack difficulty is driven by the efficiency bonuses + non-bench terms. If #000012 wants the bench Δ-rate terms to carry real positive signal, the 5S/5T/5F packs need a harder tier (or a deliberately-degraded parent baseline) — track that as a #000025 follow-up, not a #000012 blocker. +- **No constant change shipped by this calibration.** This is a handoff document; if #000012 decides to move a floor it owns that edit (and the resulting `governance_policy_hash` is unaffected — ForkScore constants don't fold into it; they're estimator parameters pinned by `ESTIMATOR_VERSION`). diff --git a/bench/scripts/fivef_threshold_calibration.py b/bench/scripts/fivef_threshold_calibration.py new file mode 100644 index 0000000..7177389 --- /dev/null +++ b/bench/scripts/fivef_threshold_calibration.py @@ -0,0 +1,349 @@ +"""5S/5T/5F → v8 ForkScore threshold-calibration handoff (#000025 §10.14). + +Closure deliverable for ticket #000025 §10.14 ("threshold calibration for +v8 selection acceptance handed off to #000012"). Pure measurement — runs +the canonical 5S / 5T / 5F sub-batteries (embedded packs + the 5F live +packs), builds the parent/child metrics bundle the ForkScore reads +(:func:`arborist.substrate.fork_score.bench_result_to_metrics` shape), +and reports: + + 1. Per-pack baseline rate (the ``_BATTERY_RATE_KEYS`` metric the scorer + consumes) + fixture count + 1/n observability granularity. + 2. The identity-fork verdict — ``fork_score(parent, parent)`` — which + tells #000012 what "no change" scores to (it's MARGINAL, not + ACCEPT, because every Δ-rate term is 0). + 3. Synthetic-perturbation verdicts: a single +``SIGNAL_FLOOR`` bump on + one 5F sub-battery → ACCEPT; a single −``HARD_REGRESSION_FLOOR`` + drop → REJECT. Confirms the floor constants do what the docstrings + claim. + 4. A recommendation block: are ``SIGNAL_FLOOR`` / ``HARD_REGRESSION_FLOOR`` + (both 0.05 today) appropriate given the observed pack granularity? + +No mutation, no LLM call, no schema / governance-hash / canonicalization +change. The markdown report is the deliverable; #000012 cites it. +""" + +from __future__ import annotations + +import argparse +import statistics +import sys +from dataclasses import asdict +from datetime import datetime, timezone +from pathlib import Path + +# bench/ is on sys.path when run as a module (python -m); when run as a +# script, add the repo root so `import bench...` and `import arborist...` +# resolve. +_REPO_ROOT = Path(__file__).resolve().parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from bench.batteries import b_5f, b_5s, b_5t # noqa: E402 +from bench.batteries.base import BatteryResult # noqa: E402 +from arborist.substrate.fork_score import ( # noqa: E402 + HARD_REGRESSION_FLOOR, + SIGNAL_FLOOR, + _BATTERY_RATE_KEYS, + bench_result_to_metrics, + fork_score, +) + +_FIX = _REPO_ROOT / "bench" / "fixtures" + +# The canonical sub-battery → fixture-pack map the ForkScore Δ-rate terms +# read. Mirrors bench.batteries.runner._DEFAULT_FIXTURES but restricted +# to the keys present in fork_score._BATTERY_RATE_KEYS (so legacy 5t +# "transfer" is excluded — the scorer drops it). The "live" 5F packs are +# included as a separate column because their runners exercise real +# arborist surfaces on temp shards (Phase 1b.2). +_EMBEDDED_PACKS: dict[tuple[str, str], Path] = { + ("5s", "syntax"): _FIX / "5s" / "syntax-v1.jsonl", + ("5s", "semantics"): _FIX / "5s" / "semantics-v1.jsonl", + ("5s", "syllogism"): _FIX / "5s" / "syllogism-v1.jsonl", + ("5s", "synthesis"): _FIX / "5s" / "synthesis-v1.jsonl", + ("5s", "semiotics"): _FIX / "5s" / "semiotics-v1.jsonl", + ("5t", "transfer-learning"): _FIX / "5t" / "transfer-learning-v2.jsonl", + ("5t", "triangulation"): _FIX / "5t" / "triangulation-v1.jsonl", + ("5t", "truthtables"): _FIX / "5t" / "truthtables-v1.jsonl", + ("5t", "transitivity"): _FIX / "5t" / "transitivity-v1.jsonl", + ("5t", "time"): _FIX / "5t" / "time-v1.jsonl", + ("5f", "function"): _FIX / "5f" / "function-v1.jsonl", + ("5f", "finetuning"): _FIX / "5f" / "finetuning-v1.jsonl", + ("5f", "falsification"): _FIX / "5f" / "falsification-v1.jsonl", + ("5f", "formulate"): _FIX / "5f" / "formulate-v1.jsonl", + ("5f", "feedback-loop"): _FIX / "5f" / "feedback-loop-v1.jsonl", +} + +_LIVE_PACKS: dict[tuple[str, str], Path] = { + ("5f", "function"): _FIX / "5f" / "function-live-v1.jsonl", + ("5f", "finetuning"): _FIX / "5f" / "finetuning-live-v1.jsonl", + ("5f", "falsification"): _FIX / "5f" / "falsification-live-v1.jsonl", + ("5f", "formulate"): _FIX / "5f" / "formulate-live-v1.jsonl", + ("5f", "feedback-loop"): _FIX / "5f" / "feedback-loop-live-v1.jsonl", +} + +_SUB_RUNNERS = {"5s": b_5s.SUB_BATTERIES, "5t": b_5t.SUB_BATTERIES, "5f": b_5f.SUB_BATTERIES} + + +def _run_pack(battery: str, sub: str, path: Path) -> BatteryResult: + return _SUB_RUNNERS[battery][sub](path) + + +def _results_payload(packs: dict[tuple[str, str], Path]) -> dict: + """Run every pack and shape the output like bench.batteries.runner.""" + results = [] + for (battery, sub), path in packs.items(): + r = _run_pack(battery, sub, path) + results.append(asdict(r)) + return {"schema_version": "bench-result-v1", "results": results} + + +def _rate_of(metrics: dict, battery: str, sub: str) -> float | None: + key = _BATTERY_RATE_KEYS.get(battery, {}).get(sub) + if key is None: + return None + v = metrics.get(key) + return float(v) if v is not None else None + + +def _deepcopy_bundle(metrics_bundle: dict) -> dict: + return {b: {s: dict(m) for s, m in subs.items()} for b, subs in metrics_bundle.items()} + + +def _set_rate(bundle: dict, battery: str, sub: str, value: float) -> None: + key = _BATTERY_RATE_KEYS[battery][sub] + bundle.setdefault(battery, {}).setdefault(sub, {})[key] = max(0.0, min(1.0, value)) + + +def _shift_rate(bundle: dict, battery: str, sub: str, delta: float) -> None: + key = _BATTERY_RATE_KEYS[battery][sub] + base = float(bundle.get(battery, {}).get(sub, {}).get(key, 0.0)) + _set_rate(bundle, battery, sub, base + delta) + + +def _degraded_parent(parent: dict, knock_to: float = 0.90) -> dict: + """A synthetic parent with every 5F sub-battery rate knocked down to + ``knock_to`` — gives a child headroom to improve. The production + parent is at ceiling (rate 1.0 on every pack), so threshold + mechanics can only be exercised against a below-ceiling baseline.""" + out = _deepcopy_bundle(parent) + for sub in _BATTERY_RATE_KEYS["5f"]: + _set_rate(out, "5f", sub, knock_to) + return out + + +def _fmt_verdict(sf) -> str: + flags = f" — flags: {', '.join(sf.flags)}" if sf.flags else "" + return f"{sf.verdict} (score {sf.score:+.4f}){flags}" + + +def build_report() -> str: + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + embedded = _results_payload(_EMBEDDED_PACKS) + live = _results_payload(_LIVE_PACKS) + parent = bench_result_to_metrics(embedded) + live_metrics = bench_result_to_metrics(live) + + # ---- per-pack table rows ------------------------------------------------- + rows = [] + granularities: list[float] = [] + for r in embedded["results"]: + battery, sub = r["battery"], r["sub_battery"] + n = r["pass_count"] + r["fail_count"] + rate = _rate_of(r["metrics"], battery, sub) + gran = (1.0 / n) if n else float("nan") + if n: + granularities.append(gran) + live_r = next( + (lr for lr in live["results"] if lr["battery"] == battery and lr["sub_battery"] == sub), + None, + ) + live_rate = _rate_of(live_r["metrics"], battery, sub) if live_r else None + rows.append((battery, sub, n, rate, gran, live_rate)) + + coarsest = max(granularities) if granularities else float("nan") + finest = min(granularities) if granularities else float("nan") + + # ---- fork-score sanity verdicts ------------------------------------------ + # The production parent is at ceiling (rate 1.0 everywhere), so a + # "+SIGNAL_FLOOR" child clamps right back to 1.0 and shows nothing. + # Exercise the threshold mechanics against a synthetic below-ceiling + # parent (every 5F sub knocked to 0.90). + identity = fork_score(parent, parent) + knock = 0.90 + degraded = _degraded_parent(parent, knock) + + # Child recovers EVERY 5F sub uniformly by 0.06 (> SIGNAL_FLOOR each) + # → mean Δ5F = 0.06 → score = γ·0.06 = 0.06 ≥ SIGNAL_FLOOR → ACCEPT. + child_uniform = _deepcopy_bundle(degraded) + for sub in _BATTERY_RATE_KEYS["5f"]: + _shift_rate(child_uniform, "5f", sub, 0.06) + accept_uniform = fork_score(degraded, child_uniform) + + # Child recovers only ONE 5F sub by 0.06 → mean Δ5F = 0.06/5 = 0.012 + # → score 0.012 < SIGNAL_FLOOR → MARGINAL. This is the 5× averaging + # dilution: a single-sub gain is worth a fifth of its face value. + child_single = _deepcopy_bundle(degraded) + _shift_rate(child_single, "5f", "function", 0.06) + marginal_single = fork_score(degraded, child_single) + + # Child drops one 5F sub by HARD_REGRESSION_FLOOR from the degraded + # parent → hard regression → REJECT regardless of the other terms. + child_regress = _deepcopy_bundle(degraded) + _shift_rate(child_regress, "5f", "falsification", -HARD_REGRESSION_FLOOR) + reject_regress = fork_score(degraded, child_regress) + + # ---- emit ---------------------------------------------------------------- + out: list[str] = [] + out.append("# 5S/5T/5F → v8 ForkScore threshold-calibration handoff") + out.append("") + out.append(f"**Date:** {now}") + out.append("**Ticket:** #000025 §10.14 (closure deliverable) — handoff to #000012.") + out.append( + "**Method:** ran the canonical 5S/5T/5F sub-batteries (the packs " + "`fork_score._BATTERY_RATE_KEYS` reads) plus the 5F live packs; " + "computed baseline rates, observability granularity (1/n), and " + "ran `fork_score` on the parent vs three synthetic child " + "perturbations. Pure measurement." + ) + out.append("") + out.append("## 1. Baseline rates + granularity") + out.append("") + out.append( + "Each row's *rate* is the single metric `fork_score` consumes for " + "that sub-battery (`_BATTERY_RATE_KEYS`). *Granularity* = 1/n — " + "the smallest rate change a single fixture flip can produce, i.e. " + "the finest Δ the scorer could ever observe on that pack." + ) + out.append("") + out.append("| battery | sub-battery | fixtures (n) | embedded rate | live rate | granularity (1/n) |") + out.append("|---|---|---|---|---|---|") + for battery, sub, n, rate, gran, live_rate in rows: + rate_s = f"{rate:.4f}" if rate is not None else "—" + live_s = f"{live_rate:.4f}" if live_rate is not None else "—" + out.append(f"| {battery} | {sub} | {n} | {rate_s} | {live_s} | {gran:.4f} ({gran*100:.1f}pp) |") + out.append("") + # Per-battery mean rate. + for battery in ("5s", "5t", "5f"): + vals = [rate for b, s, n, rate, g, lr in rows if b == battery and rate is not None] + if vals: + out.append(f"- **{battery.upper()} mean baseline rate:** {statistics.fmean(vals):.4f}") + out.append(f"- **Coarsest pack granularity:** {coarsest:.4f} ({coarsest*100:.1f}pp)") + out.append(f"- **Finest pack granularity:** {finest:.4f} ({finest*100:.1f}pp)") + out.append("") + out.append("## 2. Identity-fork verdict (no change)") + out.append("") + out.append(f"`fork_score(parent, parent)` → **{_fmt_verdict(identity)}**") + out.append("") + out.append( + "Every 5S/5T/5F pack is at ceiling (rate 1.0) at HEAD, so every " + "Δ-rate term is exactly 0 and the identity fork lands in the " + "`[0, SIGNAL_FLOOR)` band → **MARGINAL**, not ACCEPT. Takeaway for " + "#000012: an unchanged child is *not* auto-accepted; positive " + "score has to come from a Δ-rate gain (needs harder fixtures, see " + "§4), an efficiency-bonus increase (`adaptation_efficiency` / " + "`feedback_efficiency`), or one of the non-bench terms " + "(`selfmodel_calibration_gain`, `audit_completeness`, …)." + ) + out.append("") + out.append("## 3. Floor-constant sanity checks") + out.append("") + out.append( + f"`SIGNAL_FLOOR = {SIGNAL_FLOOR}` · `HARD_REGRESSION_FLOOR = " + f"{HARD_REGRESSION_FLOOR}` (current `arborist/substrate/fork_score.py`). " + f"The production parent is at ceiling, so these are exercised against " + f"a synthetic *degraded parent* — every 5F sub-battery rate knocked " + f"to {knock:.2f}, giving a child headroom to improve." + ) + out.append("") + out.append("| scenario | verdict |") + out.append("|---|---|") + out.append(f"| identity — `fork_score(HEAD, HEAD)` | {_fmt_verdict(identity)} |") + out.append(f"| degraded parent → child recovers **all 5** 5F subs by +0.06 | {_fmt_verdict(accept_uniform)} |") + out.append(f"| degraded parent → child recovers **only 1** 5F sub by +0.06 | {_fmt_verdict(marginal_single)} |") + out.append(f"| degraded parent → child drops one 5F sub by −{HARD_REGRESSION_FLOOR} | {_fmt_verdict(reject_regress)} |") + out.append("") + out.append( + "Row 2 vs row 3 is the **5× averaging dilution**: `_delta_5f` means " + "over all 5 sub-batteries, so the same +0.06 gain scores `1.0·0.06 = " + "0.06` (ACCEPT) when applied to all five subs but only `1.0·(0.06/5) " + "= 0.012` (MARGINAL) when applied to one. Each battery contributes a " + "separate Δ-term at weight 1.0, so a uniform +`SIGNAL_FLOOR` across " + "*every* sub of *all three* batteries would score `~0.15`." + ) + out.append("") + out.append("## 4. Recommendation for #000012") + out.append("") + out.append( + f"- **Keep `SIGNAL_FLOOR = {SIGNAL_FLOOR}` and " + f"`HARD_REGRESSION_FLOOR = {HARD_REGRESSION_FLOOR}`.** They match " + "`docs/bench-maxing.md`'s 5-pp signal floor and the sanity checks " + "in §3 behave as documented." + ) + out.append( + f"- **Granularity caveat (load-bearing):** the foundational 5S " + f"packs are *coarser* than the floors — `syntax` (n=10 → 10.0pp) " + f"and `semantics` (n=8 → 12.5pp). On those packs a single fixture " + f"flip is ≥ {HARD_REGRESSION_FLOOR*100:.0f}pp, so *any* regression " + f"there trips the hard-reject. That is the intended zero-tolerance " + f"behaviour on the syntax/semantics base — not a bug — but #000012 " + f"should document it: HARD_REGRESSION_FLOOR is not a tunable knob " + f"on the small packs, it's effectively 'one fixture'. The n=30 / " + f"n=50 / n=62 packs have 3.3pp / 2.0pp / 1.6pp granularity, so " + f"there {HARD_REGRESSION_FLOOR*100:.0f}pp ≈ 1.5–3 fixture flips." + ) + out.append( + "- **5× averaging dilution (document this in #000012):** a single " + "sub-battery's rate gain is worth a fifth of its face value because " + "`_delta_5{s,t,f}` means over 5 subs. So 'a fork must improve " + f"by `SIGNAL_FLOOR`' really means *one of*: ~`{SIGNAL_FLOOR*5:.2f}` " + f"on a single sub, ~`{SIGNAL_FLOOR:.2f}` uniform across one " + f"battery's 5 subs, or ~`{SIGNAL_FLOOR/3:.3f}` uniform across the " + "whole 15-sub suite. If #000012 wants single-sub improvements to " + "weigh equally it should switch `_delta_*` from mean to " + "max-or-sum — but that's a #000012-owned design call, not a " + "calibration finding." + ) + out.append( + "- **Ceiling saturation:** until harder fixtures drop a pack's " + "baseline below 1.0, the `α·Δ5s + β·Δ5t + γ·Δ5f` terms can only be " + "≤ 0. ForkScore acceptance at the current pack difficulty is " + "driven by the efficiency bonuses + non-bench terms. If #000012 " + "wants the bench Δ-rate terms to carry real positive signal, the " + "5S/5T/5F packs need a harder tier (or a deliberately-degraded " + "parent baseline) — track that as a #000025 follow-up, not a " + "#000012 blocker." + ) + out.append( + "- **No constant change shipped by this calibration.** This is a " + "handoff document; if #000012 decides to move a floor it owns that " + "edit (and the resulting `governance_policy_hash` is unaffected — " + "ForkScore constants don't fold into it; they're estimator " + "parameters pinned by `ESTIMATOR_VERSION`)." + ) + out.append("") + return "\n".join(out) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument( + "--out", + type=Path, + default=None, + help="Write the markdown report here (default: stdout).", + ) + args = p.parse_args(argv) + md = build_report() + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(md, encoding="utf-8") + print(f"wrote {args.out}") + else: + print(md) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/scripts/selfmodel_chain_snapshot.py b/bench/scripts/selfmodel_chain_snapshot.py new file mode 100644 index 0000000..81a60a4 --- /dev/null +++ b/bench/scripts/selfmodel_chain_snapshot.py @@ -0,0 +1,171 @@ +"""Append one SelfModel snapshot to a persistent chain shard, with +capability claims drawn from a fresh 5S/5T/5F bench run (#000025 §10.11). + +Why this exists: Phase 1a's Finetuning sub-battery scored *synthetic* +parent→child SelfModel pairs; Phase 1b.2's "live" path round-trips a +parent + child through a *fresh temp* shard each run. §10.11 wants the +real thing — a SelfModel lineage that *persists across runs*, so the +Finetuning runner can measure improvement between two genuine, +chained snapshots instead of a pair fabricated for the occasion. + +Each invocation: + + 1. Runs the canonical 5S/5T/5F sub-batteries (embedded packs — the + ones `fork_score._BATTERY_RATE_KEYS` reads). Deterministic, ~1s, + no LLM call. + 2. Builds one :class:`arborist.selfmodel.CapabilityClaim` per + sub-battery: ``metric = "5S-syntax"`` etc., ``measured_value`` = + that pack's rate, ``eval_digest`` = that pack's fixture digest, + ``threshold`` = ``fork_score.SIGNAL_FLOOR`` (the rate floor below + which a fork-score Δ on this capability is a hard regression). + 3. Calls :func:`arborist.selfmodel.snapshot` (which auto-parents to + the latest root in the shard) → :func:`with_claims` → + :func:`store_snapshot`. The new snapshot's distinct parent makes + it a distinct root, so the chain grows by exactly one per run. + +Output shard defaults to ``~/.arborist/shards/selfmodel-chain.db`` +(override with ``--shard`` or ``ARBORIST_SELFMODEL_CHAIN_DB``). The +shard carries an ``audit_events`` table (every ``store_snapshot`` +write chains through it), so ``make chain-check-shards`` covers it. + +Idempotent on content: re-running with no chain growth in between is +a no-op only if the parent root is unchanged — which it isn't after +the first append, so successive runs always extend the lineage. The +bench rates are at ceiling (1.0) today, so the chained Δ is 0.0; the +point is that the *mechanism* is real and the lineage outlives any +single process. +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time +from dataclasses import asdict +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from bench.batteries import b_5f, b_5s, b_5t # noqa: E402 +from arborist.selfmodel import CapabilityClaim, snapshot, store_snapshot # noqa: E402 +from arborist.selfmodel.canonical import with_claims # noqa: E402 +from arborist.selfmodel.store import claims_for, latest # noqa: E402 +from arborist.store import connect, transaction # noqa: E402 +from arborist.substrate.fork_score import SIGNAL_FLOOR, _BATTERY_RATE_KEYS # noqa: E402 + +_FIX = _REPO_ROOT / "bench" / "fixtures" + +# Canonical sub-battery → embedded fixture pack (the keys fork_score +# reads). Mirrors bench.batteries.runner._DEFAULT_FIXTURES minus the +# legacy 5t "transfer" the scorer ignores. +_PACKS: list[tuple[str, str, Path]] = [ + ("5s", "syntax", _FIX / "5s" / "syntax-v1.jsonl"), + ("5s", "semantics", _FIX / "5s" / "semantics-v1.jsonl"), + ("5s", "syllogism", _FIX / "5s" / "syllogism-v1.jsonl"), + ("5s", "synthesis", _FIX / "5s" / "synthesis-v1.jsonl"), + ("5s", "semiotics", _FIX / "5s" / "semiotics-v1.jsonl"), + ("5t", "transfer-learning", _FIX / "5t" / "transfer-learning-v2.jsonl"), + ("5t", "triangulation", _FIX / "5t" / "triangulation-v1.jsonl"), + ("5t", "truthtables", _FIX / "5t" / "truthtables-v1.jsonl"), + ("5t", "transitivity", _FIX / "5t" / "transitivity-v1.jsonl"), + ("5t", "time", _FIX / "5t" / "time-v1.jsonl"), + ("5f", "function", _FIX / "5f" / "function-v1.jsonl"), + ("5f", "finetuning", _FIX / "5f" / "finetuning-v1.jsonl"), + ("5f", "falsification", _FIX / "5f" / "falsification-v1.jsonl"), + ("5f", "formulate", _FIX / "5f" / "formulate-v1.jsonl"), + ("5f", "feedback-loop", _FIX / "5f" / "feedback-loop-v1.jsonl"), +] + +_SUB_RUNNERS = {"5s": b_5s.SUB_BATTERIES, "5t": b_5t.SUB_BATTERIES, "5f": b_5f.SUB_BATTERIES} + + +def default_shard() -> Path: + env = os.environ.get("ARBORIST_SELFMODEL_CHAIN_DB") + if env: + return Path(env).expanduser() + return Path.home() / ".arborist" / "shards" / "selfmodel-chain.db" + + +def _battery_claims(measured_at: int) -> list[CapabilityClaim]: + claims: list[CapabilityClaim] = [] + for battery, sub, path in _PACKS: + res = _SUB_RUNNERS[battery][sub](path) + metric_key = _BATTERY_RATE_KEYS[battery][sub] + rate = float(res.metrics.get(metric_key, 0.0)) + claims.append( + CapabilityClaim( + metric=f"{battery.upper()}-{sub}", + threshold=float(SIGNAL_FLOOR), + eval_digest=res.fixture_digest, + measured_value=rate, + measured_at=measured_at, + validity_horizon="next-checkpoint", + claim_text=f"{battery.upper()} {sub} {metric_key} on the embedded pack", + ) + ) + return claims + + +def append_snapshot(shard: Path, *, ts: int | None = None) -> dict: + """Run the suite, append one chained SelfModel snapshot, return a + summary dict ``{root, parent_root, depth, claims: [...]}``.""" + if ts is None: + ts = int(time.time()) + shard.parent.mkdir(parents=True, exist_ok=True) + conn = connect(shard) + try: + prev = latest(conn) + claims = _battery_claims(ts) + with transaction(conn): + sm = with_claims(snapshot(conn), claims) + root = store_snapshot(conn, sm, claims=claims, ts=ts) + # Walk the parent chain to report depth. + depth = 1 + cur = load_parent(conn, root) + while cur is not None: + depth += 1 + cur = load_parent(conn, cur) + rows = claims_for(conn, root) + return { + "shard": str(shard), + "root": root, + "parent_root": prev["selfmodel_root"] if prev else None, + "depth": depth, + "claims": {r["metric"]: r["measured_value"] for r in rows}, + } + finally: + conn.close() + + +def load_parent(conn, root: str) -> str | None: + row = conn.execute( + "SELECT parent_selfmodel_root FROM selfmodel_records WHERE selfmodel_root = ?", + (root,), + ).fetchone() + if row is None: + return None + return row["parent_selfmodel_root"] + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument( + "--shard", + type=Path, + default=None, + help="Chain shard path (default: $ARBORIST_SELFMODEL_CHAIN_DB or ~/.arborist/shards/selfmodel-chain.db)", + ) + args = p.parse_args(argv) + shard = args.shard.expanduser() if args.shard else default_shard() + summary = append_snapshot(shard) + import json as _json + + print(_json.dumps(summary, indent=2, default=str)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 413f5c3..79d810b 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -112,7 +112,7 @@ Newest first. Update on every open/close. | #000028 | Multi-modality witness for canonical shapes | closed · landed 2026-05-09 + follow-ups (capital ledger · sample rate) | 2026-05-08 | — | | #000027 | Canonical projections persist to providence_cache | closed · landed 2026-05-09 | 2026-05-08 | — | | #000026 | Real-shard workload baseline + search latency | closed · Phase 1 + 2 + 3 landed 2026-05-09 (Phase 3 in `60b5748`) | 2026-05-08 | — | -| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a + 1b.2 + 1c + 1d landed 2026-05-09; Phase 1e (motif coverage, §10.12 closed) landed 2026-05-10 | 2026-05-07 | — | +| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | closed · 2026-05-11 — Phase 1a–1f landed 2026-05-09/10; Phase 1g (§10.13 feedback latency + persisted-footprint efficiency), 1h (§10.14 ForkScore threshold-calibration handoff → #000012 §8), 1i (§10.11 persistent SelfModel-chain lineage: `bench-5f-selfmodel-snapshot` grows the chain, `run_finetuning` shard-chain mode reads the 2 latest snapshots) all landed 2026-05-11. Every §10 closure criterion met | 2026-05-07 | — | | #000024 | 5T Phase 1b + Dav1DPrometheus vocabulary alignment | closed · landed 2026-05-08 | 2026-05-07 | — | | #000023 | 5S Phase 1b: Syllogism · Synthesis · Semiotics | closed · landed 2026-05-08 | 2026-05-07 | — | | #000022 | Adapter LossReport (PRD I9 analogue) | closed · landed 2026-05-07 | 2026-05-07 | — | diff --git a/docs/tickets/ticket-000012-selection-consensus-protocol.md b/docs/tickets/ticket-000012-selection-consensus-protocol.md index 9fe1764..bc0c047 100644 --- a/docs/tickets/ticket-000012-selection-consensus-protocol.md +++ b/docs/tickets/ticket-000012-selection-consensus-protocol.md @@ -506,3 +506,64 @@ its own ticket. Until then, #000037 Trigger 1 simply reports "no data — Phase 1c not landed" and the controller's multi-branch path is paper-only. That matches §12's "measured trigger, not calendar date" discipline. + +--- + +## 8. ForkScore threshold calibration handoff (from #000025 §10.14) + +Ticket #000025 §10.14 ("threshold calibration for v8 selection +acceptance handed off to #000012") landed 2026-05-11. Deliverable: +`bench/scripts/fivef_threshold_calibration.py` → +`bench/results/5f-threshold-calibration-2026-05-11.md` +(regenerate via `make bench-5f-threshold-calibration`). It runs the +canonical 5S/5T/5F packs (the ones `fork_score._BATTERY_RATE_KEYS` +reads) plus the 5F live packs and reports baseline rates, +observability granularity (1/n), and `fork_score` verdicts on the +parent vs synthetic child perturbations. Findings the v8 acceptance +protocol must account for: + +1. **Floor constants stay at 0.05.** `SIGNAL_FLOOR = 0.05` and + `HARD_REGRESSION_FLOOR = 0.05` match `docs/bench-maxing.md`'s + 5-pp signal floor; the synthetic sanity checks (identity → + MARGINAL; degraded-parent → uniform-recovery → ACCEPT; + single-sub regression → REJECT) all behave as the docstrings + claim. The calibration ships **no constant change** — if a + future revision moves a floor, this ticket owns that edit, and + it does **not** fold into `governance_policy_hash` (ForkScore + constants are estimator parameters pinned by + `ESTIMATOR_VERSION`, not governance-policy inputs). + +2. **Pack granularity vs the floors (load-bearing).** The + foundational 5S packs are *coarser* than `HARD_REGRESSION_FLOOR`: + `syntax` n=10 → 10.0 pp/fixture, `semantics` n=8 → 12.5 + pp/fixture. On those packs a single fixture flip is ≥ 5 pp, so + *any* regression on syntax/semantics trips the hard-reject — + intended zero-tolerance on the base, not a bug, but the v8 + protocol should state it: on the small packs + `HARD_REGRESSION_FLOOR` is effectively "one fixture", not a + tunable knob. The n=30 / n=50 / n=62 packs run 3.3 / 2.0 / 1.6 + pp per fixture, so there 5 pp ≈ 1.5–3 fixture flips. + +3. **5× averaging dilution.** `_delta_5{s,t,f}` *means* over the 5 + sub-batteries of a battery, so a single sub-battery's rate gain + contributes a fifth of its face value. "A fork must improve by + `SIGNAL_FLOOR`" therefore means one of: ≈ 0.25 on a single + sub-battery, ≈ 0.05 uniform across one battery's 5 subs, or + ≈ 0.017 uniform across the whole 15-sub suite. Each battery + contributes a separate Δ-term at weight 1.0, so a uniform + +`SIGNAL_FLOOR` across every sub of all three batteries scores + ≈ 0.15. If v8 wants single-sub improvements to weigh equally, + switch `_delta_*` from mean to a max/sum aggregator — a design + call this ticket owns, not a calibration finding. + +4. **Ceiling saturation.** Every 5S/5T/5F pack is at rate 1.0 at + HEAD, so `α·Δ5s + β·Δ5t + γ·Δ5f` can only be ≤ 0 — an unchanged + child scores 0 → MARGINAL (not ACCEPT). Until a harder fixture + tier (or a deliberately-degraded parent baseline) drops a pack + below ceiling, fork *acceptance* at the current pack difficulty + is carried by the efficiency bonuses (`adaptation_efficiency` / + `feedback_efficiency`) + the non-bench terms + (`selfmodel_calibration_gain`, `audit_completeness`, + `validator_diversity`, …), not by bench Δ-rate. Adding the + harder tier is tracked as a #000025 follow-up, not a #000012 + blocker. diff --git a/docs/tickets/ticket-000025-5f-battery.md b/docs/tickets/ticket-000025-5f-battery.md index efef499..fb8c9a7 100644 --- a/docs/tickets/ticket-000025-5f-battery.md +++ b/docs/tickets/ticket-000025-5f-battery.md @@ -1,6 +1,6 @@ # Ticket #000025 — 5F battery: Function · Finetuning · Falsification · Formulate · Feedback Loop -**Status:** in progress · Phase 1a + 1b.2 + 1c + 1d + 1e (motif coverage, 2026-05-10) landed; §10.11 (real shard finetuning chains), §10.13 (Feedback Loop latency/efficiency calibrated to real workload), §10.14 (threshold handoff to #000012) still open +**Status:** closed · 2026-05-11 — Phase 1a + 1b.2 + 1c + 1d + 1e + 1f all landed; §10.11 (real shard finetuning chains — persistent SelfModel-chain lineage), §10.13 (Feedback Loop latency + persisted-footprint efficiency on real workload), §10.14 (ForkScore threshold-calibration handoff to #000012) all landed 2026-05-11. Every §10 closure criterion met **Opened:** 2026-05-07 **Revised:** 2026-05-08 (cross-modality + state-space synthesis folded in per review response) @@ -361,8 +361,21 @@ feedback_efficiency = downstream_effect_count / capital_cost_delta `feedback_efficiency` again hooks the capital ledger (#000020). Same zero-cost-guard semantics as `adaptation_efficiency` — see the -:func:`_efficiency` helper. Phase 1a uses chain length as the -cost-proxy; Phase 1b.2 reads from the capital_ledger directly. +:func:`_efficiency` helper. Phase 1a used chain length (count of +*requested* ops) as the cost-proxy. **§10.13 (2026-05-11)** replaced +that for live chains with the *persisted footprint* — +:func:`bench.batteries.b_5f._persisted_cost` = the audit-event rows +the chain actually wrote + their body bytes / 1e6 (matching +`_capital_cost_delta`'s 1 MB = 1 unit convention) — and finally +implemented `feedback_latency` (listed above since Phase 1a, never +computed): wall-clock seconds to apply a live chain against its temp +shard, surfaced as per-task `feedback_latency_seconds` + battery +`feedback_latency_mean_seconds` (0.0 when a pack has no live tasks) ++ `feedback_live_task_count`. Latency is a wall-clock field +(run-to-run variable, like `BatteryResult.timestamp`) and is **not** +a `fork_score` input — it's for operational observability + the +#000020 ledger. Embedded chains keep `len(chain)` as cost and report +`feedback_latency_seconds = None`. **Phase 1a uses existing surfaces only:** `memory_records`, `memory_branch_summaries`, `audit_events`. No new substrate work. @@ -578,7 +591,7 @@ against real workload), §10.14 (threshold handoff to #000012). ## 11. Status -**As of 2026-05-10: in progress.** +**As of 2026-05-11: closed — all §10 closure criteria met.** - **Phase 1a** (Function + Finetuning seed): landed. - **Phase 1b.2** (Falsification curated fixtures): landed. @@ -608,11 +621,72 @@ against real workload), §10.14 (threshold handoff to #000012). `audit_mode_at_harvest`, `harvest_threshold`, and `source_ticket: "#000037 §13 step 11"` for full traceability. Regenerate via `make bench-5f-harvest`. +- **Phase 1g** (§10.13 — Feedback Loop latency / real-workload + efficiency): landed 2026-05-11. `run_feedback_loop` now computes + `feedback_latency` (listed in §5.5 since Phase 1a, never + implemented) — wall-clock seconds to apply a live chain against its + temp shard, surfaced per-task (`feedback_latency_seconds`) + + battery (`feedback_latency_mean_seconds`, `feedback_live_task_count`). + For live chains `feedback_efficiency`'s cost denominator switched + from `len(chain)` (count of *requested* ops) to the *persisted + footprint* `_persisted_cost` = audit-event rows the chain actually + wrote + their body bytes / 1e6. Embedded chains keep `len(chain)` + and report `feedback_latency_seconds = None`. Latency is a + wall-clock field (run-to-run variable, like + `BatteryResult.timestamp`) and is **not** a `fork_score` input. + Tests: 3 new in `tests/test_bench_batteries.py` + (`_persisted_cost` helper; embedded reports no latency; live + reports latency + persisted cost + the `efficiency = 1/cost` + identity). +- **Phase 1h** (§10.14 — ForkScore threshold-calibration handoff to + #000012): landed 2026-05-11. `bench/scripts/fivef_threshold_calibration.py` + (run via `make bench-5f-threshold-calibration`) runs the canonical + 5S/5T/5F packs + the 5F live packs and reports baseline rates, + observability granularity (1/n), and `fork_score` verdicts on the + parent vs synthetic perturbations → + `bench/results/5f-threshold-calibration-2026-05-11.md`. Findings + written into #000012 §8: keep `SIGNAL_FLOOR` / `HARD_REGRESSION_FLOOR` + at 0.05; the small 5S packs (`syntax` n=10, `semantics` n=8) are + *coarser* than the floors so any regression there trips hard-reject + (intended zero-tolerance); the 5× averaging dilution in `_delta_*`; + ceiling saturation (every pack at 1.0 → Δ-rate terms ≤ 0, so fork + *acceptance* is carried by the efficiency bonuses + non-bench + terms until a harder fixture tier lands). No constant change + shipped — that's #000012's call. Tests: 6 in + `tests/test_fivef_threshold_calibration.py`. +- **Phase 1i** (§10.11 — real shard finetuning chains): landed + 2026-05-11. `bench/scripts/selfmodel_chain_snapshot.py` + (`make bench-5f-selfmodel-snapshot`) appends ONE chained SelfModel + snapshot per run to a persistent shard + (`~/.arborist/shards/selfmodel-chain.db`, override via + `ARBORIST_SELFMODEL_CHAIN_DB`) with one `CapabilityClaim` per + sub-battery (`metric = "5S-syntax"` etc., `measured_value` = that + pack's rate, `eval_digest` = the pack's fixture digest, `threshold` + = `fork_score.SIGNAL_FLOOR`). `snapshot()` auto-parents, so the + distinct parent makes each snapshot a distinct root and the + lineage grows by one per run. `run_finetuning` gains a third + dispatch mode — **shard-chain** (gated on a task's `selfmodel_shard` + key) — via `_chain_finetuning_measure`: reads the two most-recent + snapshots (`latest()` = child, its `parent_selfmodel_root` = + parent) and measures improvement on `target_capability` between + them. This is the real lineage replacing Phase-1a's synthetic + parent→child pairs; the chained Δ reflects genuine cross-run drift + (0.0 today — the embedded packs are at ceiling). Operator pack + `bench/fixtures/5f/finetuning-shardchain-v1.jsonl` (6 tasks) + + `make bench-5f-finetuning-shardchain`; NOT in `make bench-5f`, + `make test`, or a fresh checkout (a missing/too-short chain fails + honestly). The real chain shard was bootstrapped 2-deep on + 2026-05-11; `make chain-check-shards` reports 0 breaks on it. + Tests: 10 in `tests/test_selfmodel_chain.py`. -**Still open** (carried in file header): -- §10.11 — real shard finetuning chains -- §10.13 — Feedback Loop latency/efficiency calibrated to real workload -- §10.14 — threshold handoff to #000012 +**All §10 closure criteria met (2026-05-11).** §10 Phase 1a criteria +1–9 + Phase 1b criteria 10–14 are satisfied: +- 10 (30–50 fixtures per sub-battery): 5F embedded packs at 50 each + (62 for falsification per Phase 1e). +- 11 (real shard finetuning chains): Phase 1i above. +- 12 (motif coverage): Phase 1e. +- 13 (Feedback Loop latency/efficiency on real workload): Phase 1g. +- 14 (threshold calibration handed to #000012): Phase 1h. Larger surface than #000023 / #000024 (five sub-batteries vs three / four). diff --git a/tests/test_bench_batteries.py b/tests/test_bench_batteries.py index 6ac4291..544d7d1 100644 --- a/tests/test_bench_batteries.py +++ b/tests/test_bench_batteries.py @@ -547,6 +547,55 @@ def test_5f_live_feedback_chain_audit_chain_intact(): pass +# --- 5F §10.13 — feedback latency / real-workload efficiency -------- + + +def test_5f_persisted_cost_helper(): + """_persisted_cost = audit-row count + body bytes / 1e6 (the real + footprint a live chain wrote, not the count of requested ops).""" + from bench.batteries.b_5f import _persisted_cost + + assert _persisted_cost({}) == 0.0 + state = { + "audit_event_types": ["a", "b", "c"], + "audit_event_bodies": ['{"x":1}', '{"y":2}', ""], # 7 + 7 + 0 bytes + } + assert _persisted_cost(state) == pytest.approx(3.0 + 14 / 1e6) + + +def test_5f_feedback_loop_embedded_reports_no_latency(): + res = b_5f.run_feedback_loop(F5F / "feedback-loop-v1.jsonl") + assert res.metrics["feedback_latency_mean_seconds"] == 0.0 + assert res.metrics["feedback_live_task_count"] == 0.0 + for t in res.per_task: + assert t.detail.get("feedback_latency_seconds") is None + # Embedded cost-proxy stays len(chain) → integer-ratio efficiency. + assert "persisted_cost" not in t.detail + + +def test_5f_feedback_loop_live_reports_latency_and_persisted_cost(): + res = b_5f.run_feedback_loop(F5F / "feedback-loop-live-v1.jsonl") + assert res.metrics["feedback_live_task_count"] == 50.0 # Phase 1d + assert res.metrics["feedback_latency_mean_seconds"] > 0.0 + for t in res.per_task: + if not t.passed: + continue + lat = t.detail["feedback_latency_seconds"] + assert isinstance(lat, float) and lat >= 0.0 + # Real footprint: every live chain has >= 1 append_audit op. + assert t.detail["persisted_audit_rows"] >= 1 + assert t.detail["persisted_cost"] >= 1.0 + # feedback_efficiency now = downstream_effect / persisted_cost, + # not / len(chain). downstream_effect is 1.0 when the chain + # integrated, else 0.0 (an expected-"fail" task still passes). + if t.detail["integrated"]: + assert t.detail["feedback_efficiency"] == pytest.approx( + 1.0 / t.detail["persisted_cost"] + ) + else: + assert t.detail["feedback_efficiency"] == 0.0 + + # --- 5F Phase 1b.2 — Function / Finetuning / Falsification live - diff --git a/tests/test_fivef_threshold_calibration.py b/tests/test_fivef_threshold_calibration.py new file mode 100644 index 0000000..5f66883 --- /dev/null +++ b/tests/test_fivef_threshold_calibration.py @@ -0,0 +1,86 @@ +"""Tests for the #000025 §10.14 ForkScore threshold-calibration handoff +script (``bench/scripts/fivef_threshold_calibration.py``). + +The script is pure measurement over the canonical 5S/5T/5F packs + +`fork_score`, so its markdown report is deterministic given fixed +fixtures + code. These tests pin the structural anchors and the +floor-constant sanity verdicts so the handoff doesn't drift silently. + +The live 5F packs (``*-live-v1.jsonl``) spin up temp shards and add +~15s per run; the calibration *logic* (granularity, fork-score sanity) +doesn't depend on them, so the tests stub ``_LIVE_PACKS`` to empty. +The live runners are exercised by ``tests/test_bench_batteries.py`` +and ``make bench-5f-live``. +""" +from __future__ import annotations + +import pytest + +from bench.scripts import fivef_threshold_calibration as calib + + +@pytest.fixture(scope="module", autouse=True) +def _no_live_packs(): + saved = calib._LIVE_PACKS + calib._LIVE_PACKS = {} + try: + yield + finally: + calib._LIVE_PACKS = saved + + +@pytest.fixture(scope="module") +def report() -> str: + return calib.build_report() + + +def test_report_is_deterministic(): + a = calib.build_report() + b = calib.build_report() + + def _strip_ts(s: str) -> str: + return "\n".join(ln for ln in s.splitlines() if not ln.startswith("**Date:**")) + + assert _strip_ts(a) == _strip_ts(b) + + +def test_report_has_structural_anchors(report: str): + assert report.startswith("# 5S/5T/5F → v8 ForkScore threshold-calibration handoff") + assert "#000025 §10.14" in report + assert "## 1. Baseline rates + granularity" in report + assert "## 2. Identity-fork verdict (no change)" in report + assert "## 3. Floor-constant sanity checks" in report + assert "## 4. Recommendation for #000012" in report + for sub in ( + "syntax", "semantics", "syllogism", "synthesis", "semiotics", + "transfer-learning", "triangulation", "truthtables", "transitivity", "time", + "function", "finetuning", "falsification", "formulate", "feedback-loop", + ): + assert f"| {sub} |" in report + + +def test_identity_fork_is_marginal(report: str): + # Packs are at ceiling at HEAD → every Δ-rate term is 0 → MARGINAL. + assert "`fork_score(parent, parent)` → **MARGINAL (score +0.0000)**" in report + + +def test_floor_sanity_verdicts_present(report: str): + assert "child recovers **all 5** 5F subs by +0.06 | ACCEPT" in report + assert "child recovers **only 1** 5F sub by +0.06 | MARGINAL" in report + assert "child drops one 5F sub by −0.05 | REJECT" in report + assert "REGRESSION_5F:5f/falsification" in report + + +def test_recommendation_keeps_floors_and_flags_dilution(report: str): + assert "Keep `SIGNAL_FLOOR = 0.05` and `HARD_REGRESSION_FLOOR = 0.05`" in report + assert "5× averaging dilution" in report + assert "Ceiling saturation" in report + assert "No constant change shipped" in report + + +def test_cli_main_writes_report(tmp_path): + out = tmp_path / "calib.md" + rc = calib.main(["--out", str(out)]) + assert rc == 0 + text = out.read_text(encoding="utf-8") + assert text.startswith("# 5S/5T/5F → v8 ForkScore threshold-calibration handoff") diff --git a/tests/test_selfmodel_chain.py b/tests/test_selfmodel_chain.py new file mode 100644 index 0000000..8722708 --- /dev/null +++ b/tests/test_selfmodel_chain.py @@ -0,0 +1,134 @@ +"""Tests for the #000025 §10.11 SelfModel snapshot-chain — the +persistent lineage that replaces Phase-1a's synthetic parent→child +pairs in the Finetuning sub-battery. + +Two surfaces: + - ``bench/scripts/selfmodel_chain_snapshot.py`` — the bootstrapper + that appends one chained snapshot per run (capability claims = + the 5S/5T/5F battery rates). + - ``bench.batteries.b_5f._chain_finetuning_measure`` + the + ``run_finetuning`` "shard-chain" dispatch — reads the two latest + snapshots and measures improvement on ``target_capability``. + +All against temp shards (``tmp_path`` / ``tmp_path_factory``) — never +the real ``~/.arborist/shards/selfmodel-chain.db``. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from bench.batteries import b_5f +from bench.scripts.selfmodel_chain_snapshot import append_snapshot + +_F5F = Path(__file__).parent.parent / "bench" / "fixtures" / "5f" +_SHARDCHAIN_PACK = _F5F / "finetuning-shardchain-v1.jsonl" + + +@pytest.fixture(scope="module") +def chain_shard(tmp_path_factory) -> Path: + """A persistent 2-deep SelfModel chain on a temp shard.""" + shard = tmp_path_factory.mktemp("smchain") / "chain.db" + s1 = append_snapshot(shard, ts=1_700_000_000) + s2 = append_snapshot(shard, ts=1_700_000_001) + assert s1["depth"] == 1 and s1["parent_root"] is None + assert s2["depth"] == 2 and s2["parent_root"] == s1["root"] + return shard + + +# ---- bootstrapper ------------------------------------------------------- + + +def test_bootstrapper_grows_chain_one_per_run(tmp_path): + shard = tmp_path / "c.db" + a = append_snapshot(shard, ts=1_700_000_000) + assert a["depth"] == 1 + assert a["parent_root"] is None + # 15 canonical sub-batteries → 15 capability claims. + assert len(a["claims"]) == 15 + assert all(v == 1.0 for v in a["claims"].values()) # packs at ceiling + b = append_snapshot(shard, ts=1_700_000_001) + assert b["depth"] == 2 + assert b["parent_root"] == a["root"] + assert b["root"] != a["root"] # distinct parent ⇒ distinct root + + +def test_bootstrapper_claims_carry_real_metric_names(chain_shard: Path): + a = append_snapshot(chain_shard, ts=1_700_000_002) # depth 3 + assert a["depth"] == 3 + assert {"5S-syntax", "5T-time", "5F-feedback-loop"} <= set(a["claims"]) + + +# ---- _chain_finetuning_measure ----------------------------------------- + + +def test_chain_measure_reads_two_latest(chain_shard: Path): + out = b_5f._chain_finetuning_measure( + {"target_capability": "5F-feedback-loop", "selfmodel_shard": str(chain_shard)} + ) + assert out["parent_measured"] == 1.0 + assert out["child_measured"] == 1.0 + assert out["parent_root"] != out["child_root"] + + +def test_chain_measure_env_var_overrides_fixture(chain_shard: Path, monkeypatch): + monkeypatch.setenv("ARBORIST_SELFMODEL_CHAIN_DB", str(chain_shard)) + out = b_5f._chain_finetuning_measure( + {"target_capability": "5S-syntax", "selfmodel_shard": "/nonexistent/ignored.db"} + ) + assert out["child_measured"] == 1.0 + + +def test_chain_measure_absent_shard_raises(tmp_path): + with pytest.raises(FileNotFoundError, match="chain shard not found"): + b_5f._chain_finetuning_measure( + {"target_capability": "5S-syntax", "selfmodel_shard": str(tmp_path / "nope.db")} + ) + + +def test_chain_measure_single_snapshot_raises(tmp_path): + shard = tmp_path / "one.db" + append_snapshot(shard, ts=1_700_000_000) # depth 1 + with pytest.raises(ValueError, match="only one snapshot"): + b_5f._chain_finetuning_measure( + {"target_capability": "5S-syntax", "selfmodel_shard": str(shard)} + ) + + +def test_chain_measure_unknown_metric_raises(chain_shard: Path): + with pytest.raises(ValueError, match="no claim for"): + b_5f._chain_finetuning_measure( + {"target_capability": "NOT-A-REAL-METRIC", "selfmodel_shard": str(chain_shard)} + ) + + +# ---- run_finetuning shard-chain dispatch ------------------------------- + + +def test_run_finetuning_shardchain_pack_passes(chain_shard: Path, monkeypatch): + monkeypatch.setenv("ARBORIST_SELFMODEL_CHAIN_DB", str(chain_shard)) + res = b_5f.run_finetuning(_SHARDCHAIN_PACK) + assert res.pass_count == 6 + assert res.fail_count == 0 + for t in res.per_task: + assert t.detail["source"] == "shard-chain" + assert t.detail["improvement"] == 0.0 # packs at ceiling ⇒ no drift + assert t.detail["chain_parent_root"] != t.detail["chain_child_root"] + + +def test_run_finetuning_shardchain_pack_fails_honestly_without_chain(tmp_path, monkeypatch): + monkeypatch.setenv("ARBORIST_SELFMODEL_CHAIN_DB", str(tmp_path / "absent.db")) + res = b_5f.run_finetuning(_SHARDCHAIN_PACK) + assert res.pass_count == 0 + assert res.fail_count == 6 + assert "FileNotFoundError" in res.per_task[0].detail["reason"] + + +def test_embedded_and_live_finetuning_unaffected(): + """The shard-chain dispatch is gated on ``selfmodel_shard``; the + embedded + live packs still route through their original paths.""" + emb = b_5f.run_finetuning(_F5F / "finetuning-v1.jsonl") + assert emb.pass_count == 50 + for t in emb.per_task: + assert t.detail["source"] == "embedded"