diff --git a/Makefile b/Makefile index ad9e278..dbcc206 100644 --- a/Makefile +++ b/Makefile @@ -266,9 +266,18 @@ bench-5f: bootstrap ## 5F battery (Function+Finetuning+Falsification+Formulate+F PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate --fixtures bench/fixtures/5f/formulate-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub feedback-loop --fixtures bench/fixtures/5f/feedback-loop-v1.jsonl +bench-5r: bootstrap ## 5R battery (React+Rearrange+Restore+Replicate+Resonate) + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub react --fixtures bench/fixtures/5r/react-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub rearrange --fixtures bench/fixtures/5r/rearrange-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub restore --fixtures bench/fixtures/5r/restore-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub replicate --fixtures bench/fixtures/5r/replicate-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub resonate --fixtures bench/fixtures/5r/resonate-v1.jsonl + bench-5s5t: bench-5s bench-5t ## 5S + 5T (Phase-1b vocabulary) -bench-5s5t5f: bench-5s bench-5t bench-5f ## complete Dav1DPrometheus suite (5S + 5T + 5F) +bench-5s5t5f: bench-5s bench-5t bench-5f ## 5S + 5T + 5F (operational triad) + +bench-suite: bench-5s bench-5t bench-5f bench-5r ## complete Dav1DPrometheus suite (5S + 5T + 5F + 5R) verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample diff --git a/bench/batteries/b_5r.py b/bench/batteries/b_5r.py new file mode 100644 index 0000000..29d9f83 --- /dev/null +++ b/bench/batteries/b_5r.py @@ -0,0 +1,358 @@ +"""5R battery — React / Rearrange / Restore / Replicate / Resonate. + +Workspace-operator battery. Per SQD whitepaper §9.3, the 5R battery +tests operators applied to a workspace state. With ticket #000014 +(SelfModel) and #000017 (memory-root) landed, the workspace is the +``(selfmodel_root, memory_root, audit_events_high_water)`` triple +plus the cited claim lattice / providence cache. + +Five sub-batteries: + +- **React** — incorporate new facts/constraints into the workspace. + Test: a new observation between snapshots produces the expected + delta in the branch summaries. +- **Rearrange** — restructure representation without changing + semantic content. Test: re-ordering claim-lattice claims produces + the same root (selfmodel_root invariant under claim order). +- **Restore** — retrieve a prior fact and re-assert it cleanly. + Test: re-canonicalizing an earlier fact produces the same + canonical bytes; audit chain still verifies. +- **Replicate** — produce independent canonical encodings via the + π* registry. Test: same input through the named π* yields + byte-equal output across N reruns. +- **Resonate** — detect low variance across encodings/reruns. Test: + variance over N reruns is below threshold; deterministic π*'s + should be zero variance. + +All runners deterministic, no LLM-as-judge. Phase 1a fixtures embed +the workspace state directly; Phase 1b.2 will read live +selfmodel_records / memory_records from a shard. +""" + +from __future__ import annotations + +import statistics +from pathlib import Path + +from bench.batteries.base import ( + BatteryResult, + TaskResult, + fixture_digest, + fixture_meta, + iter_tasks, + validate_carrier, +) +from bench.batteries.b_5s import _runtime_digest + + +def _carrier_check(task: dict) -> tuple[bool, str]: + reason = validate_carrier(task) + if reason is not None: + return False, reason + return True, "" + + +def _build_result( + sub_battery: str, + fixtures_path: Path, + per_task: list[TaskResult], + metrics: dict[str, float], +) -> BatteryResult: + pass_count = sum(1 for t in per_task if t.passed) + fail_count = len(per_task) - pass_count + return BatteryResult( + battery="5r", + sub_battery=sub_battery, + fixture_path=str(fixtures_path), + fixture_digest=fixture_digest(fixtures_path), + pass_count=pass_count, + fail_count=fail_count, + metrics=metrics, + per_task=per_task, + runtime_digest=_runtime_digest(), + ) + + +# --------------------------------------------------------------------- +# React — incorporate new fact/constraint +# --------------------------------------------------------------------- + + +def run_react(fixtures_path: Path) -> BatteryResult: + """Each task: workspace state at t, an observation at t+1, and an + expected_delta describing how the workspace should change. + + Pass = ``expected_delta.added_facts`` are present in the t+1 + snapshot's facts AND ``expected_delta.removed_facts`` are absent + from t+1's facts. + """ + per_task: list[TaskResult] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + try: + t1_facts = set(task["snapshot_t1"].get("facts", [])) + delta = task["expected_delta"] + added_ok = all(f in t1_facts for f in delta.get("added_facts", [])) + removed_ok = all(f not in t1_facts for f in delta.get("removed_facts", [])) + passed_all = added_ok and removed_ok + expected = task.get("expected", "pass") + observed = "pass" if passed_all else "fail" + passed = observed == expected + detail = { + "added_ok": added_ok, + "removed_ok": removed_ok, + "expected": expected, + } + except Exception as exc: # noqa: BLE001 + passed = False + detail = {"reason": f"{type(exc).__name__}: {exc}"} + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "react"), + fixtures_path, + per_task, + {"react_integration_rate": rate}, + ) + + +# --------------------------------------------------------------------- +# Rearrange — restructure without semantic shift +# --------------------------------------------------------------------- + + +def run_rearrange(fixtures_path: Path) -> BatteryResult: + """Each task: two ordered fact lists (alt_a, alt_b) that should + canonicalize to the same bytes through the named π* (or hashing + rule). Pass = canonical encodings are byte-equal. + + This exercises the order-invariance contracts in SelfModel + (capability_claim_hashes are sorted) and Memory (branches are + sorted by branch_id) without depending on shard state. + """ + from arborist.pi_star import get + + per_task: list[TaskResult] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + try: + pi_star = get(task["pi_star_ref"]) + alt_a = task["alt_a"].encode("utf-8") + alt_b = task["alt_b"].encode("utf-8") + canon_a = pi_star.canonicalize(alt_a) + canon_b = pi_star.canonicalize(alt_b) + equivalent = canon_a == canon_b + expected = task.get("expected_equivalent", True) + passed = equivalent == bool(expected) + detail = { + "canon_match": equivalent, + "expected_equivalent": expected, + } + except Exception as exc: # noqa: BLE001 + passed = False + detail = {"reason": f"{type(exc).__name__}: {exc}"} + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "rearrange"), + fixtures_path, + per_task, + {"rearrange_invariance_rate": rate}, + ) + + +# --------------------------------------------------------------------- +# Restore — retrieve prior fact and re-assert +# --------------------------------------------------------------------- + + +def run_restore(fixtures_path: Path) -> BatteryResult: + """Each task: a prior_fact + a current_workspace. Pass = the + prior_fact is retrievable from the workspace's facts (either + in current facts or in the historical chain). + + Phase 1a fixtures embed the workspace history; Phase 1b.2 will + read audit_events directly. + """ + per_task: list[TaskResult] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + try: + prior_fact = task["prior_fact"] + history = task.get("history", []) + current = task.get("current_facts", []) + # Check current first, then history. + in_current = prior_fact in current + in_history = any(prior_fact in s.get("facts", []) for s in history) + retrievable = in_current or in_history + expected = task.get("expected", "pass") + observed = "pass" if retrievable else "fail" + passed = observed == expected + detail = { + "in_current": in_current, + "in_history": in_history, + "retrievable": retrievable, + } + except Exception as exc: # noqa: BLE001 + passed = False + detail = {"reason": f"{type(exc).__name__}: {exc}"} + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "restore"), + fixtures_path, + per_task, + {"restore_retrievability_rate": rate}, + ) + + +# --------------------------------------------------------------------- +# Replicate — independent canonical encodings +# --------------------------------------------------------------------- + + +def run_replicate(fixtures_path: Path) -> BatteryResult: + """Each task: an input + π* + expected_replicas count. Run π* + canonicalize N times; pass = all N outputs are byte-equal. + + Tests determinism contract on registered π*'s: same input + same + π* should always yield the same canonical bytes regardless of how + many times we ask. + """ + from arborist.pi_star import get + + per_task: list[TaskResult] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + try: + pi_star = get(task["pi_star_ref"]) + n = int(task.get("replicas", 5)) + input_bytes = task["input"].encode("utf-8") + outputs = [pi_star.canonicalize(input_bytes) for _ in range(n)] + all_equal = all(o == outputs[0] for o in outputs) + expected = task.get("expected", "pass") + observed = "pass" if all_equal else "fail" + passed = observed == expected + detail = { + "replicas": n, + "all_equal": all_equal, + } + except Exception as exc: # noqa: BLE001 + passed = False + detail = {"reason": f"{type(exc).__name__}: {exc}"} + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "replicate"), + fixtures_path, + per_task, + {"replicate_determinism_rate": rate}, + ) + + +# --------------------------------------------------------------------- +# Resonate — variance across encodings/reruns +# --------------------------------------------------------------------- + + +def run_resonate(fixtures_path: Path) -> BatteryResult: + """Each task: an input + π* + max_variance threshold. Run π* + canonicalize N times; compute the per-byte variance (over the + distinct hashes of each output). Deterministic π*'s yield + variance == 0; non-deterministic ones violate the contract. + + The metric is ``distinct_output_count``: how many unique outputs + in N runs. Pass = distinct_count <= ``expected_max_distinct``. + """ + from arborist.pi_star import get + + per_task: list[TaskResult] = [] + distinct_counts: list[int] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + try: + pi_star = get(task["pi_star_ref"]) + n = int(task.get("runs", 10)) + input_bytes = task["input"].encode("utf-8") + outputs = {pi_star.canonicalize(input_bytes) for _ in range(n)} + distinct = len(outputs) + distinct_counts.append(distinct) + expected_max = int(task.get("expected_max_distinct", 1)) + stable = distinct <= expected_max + expected = task.get("expected", "pass") + observed = "pass" if stable else "fail" + passed = observed == expected + detail = { + "runs": n, + "distinct_outputs": distinct, + "expected_max_distinct": expected_max, + } + except Exception as exc: # noqa: BLE001 + passed = False + detail = {"reason": f"{type(exc).__name__}: {exc}"} + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + mean_distinct = statistics.mean(distinct_counts) if distinct_counts else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "resonate"), + fixtures_path, + per_task, + { + "resonate_stability_rate": rate, + "mean_distinct_outputs": mean_distinct, + }, + ) + + +SUB_BATTERIES = { + "react": run_react, + "rearrange": run_rearrange, + "restore": run_restore, + "replicate": run_replicate, + "resonate": run_resonate, +} diff --git a/bench/batteries/runner.py b/bench/batteries/runner.py index 8076dad..31bfbe4 100644 --- a/bench/batteries/runner.py +++ b/bench/batteries/runner.py @@ -34,7 +34,7 @@ from dataclasses import asdict from pathlib import Path from typing import Optional -from bench.batteries import b_5f, b_5s, b_5t +from bench.batteries import b_5f, b_5r, b_5s, b_5t from bench.batteries.base import BatteryResult @@ -42,6 +42,7 @@ _BATTERIES = { "5s": b_5s.SUB_BATTERIES, "5t": b_5t.SUB_BATTERIES, "5f": b_5f.SUB_BATTERIES, + "5r": b_5r.SUB_BATTERIES, } _DEFAULT_FIXTURES = { @@ -66,6 +67,13 @@ _DEFAULT_FIXTURES = { ("5f", "falsification"): "bench/fixtures/5f/falsification-v1.jsonl", ("5f", "formulate"): "bench/fixtures/5f/formulate-v1.jsonl", ("5f", "feedback-loop"): "bench/fixtures/5f/feedback-loop-v1.jsonl", + # 5R — Phase 2 of #000021 (ticket #000021 §7 Phase 2; depends on + # SelfModel + memory_root, both landed) + ("5r", "react"): "bench/fixtures/5r/react-v1.jsonl", + ("5r", "rearrange"): "bench/fixtures/5r/rearrange-v1.jsonl", + ("5r", "restore"): "bench/fixtures/5r/restore-v1.jsonl", + ("5r", "replicate"): "bench/fixtures/5r/replicate-v1.jsonl", + ("5r", "resonate"): "bench/fixtures/5r/resonate-v1.jsonl", } diff --git a/bench/fixtures/5r/react-v1.jsonl b/bench/fixtures/5r/react-v1.jsonl new file mode 100644 index 0000000..bd02c9c --- /dev/null +++ b/bench/fixtures/5r/react-v1.jsonl @@ -0,0 +1,31 @@ +{"_meta":{"battery":"5r","sub_battery":"react","version":"v1","task_count":30,"notes":"Workspace incorporates a new fact/observation between snapshots. expected_delta lists added/removed facts the t+1 snapshot must reflect."}} +{"id":"5r-react-001","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["A","B"]},"snapshot_t1":{"facts":["A","B","C"]},"expected_delta":{"added_facts":["C"],"removed_facts":[]}} +{"id":"5r-react-002","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["x","y","z"]},"snapshot_t1":{"facts":["x","y","z","new1","new2"]},"expected_delta":{"added_facts":["new1","new2"],"removed_facts":[]}} +{"id":"5r-react-003","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["fact-1","fact-2","fact-3"]},"snapshot_t1":{"facts":["fact-2","fact-3"]},"expected_delta":{"added_facts":[],"removed_facts":["fact-1"]}} +{"id":"5r-react-004","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["claim-a"]},"snapshot_t1":{"facts":["claim-b","claim-c"]},"expected_delta":{"added_facts":["claim-b","claim-c"],"removed_facts":["claim-a"]}} +{"id":"5r-react-005","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":[]},"snapshot_t1":{"facts":["genesis fact"]},"expected_delta":{"added_facts":["genesis fact"],"removed_facts":[]}} +{"id":"5r-react-006","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["initial"]},"snapshot_t1":{"facts":["initial"]},"expected_delta":{"added_facts":[],"removed_facts":[]}} +{"id":"5r-react-007","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["alpha","beta"]},"snapshot_t1":{"facts":["alpha","beta","gamma","delta","epsilon"]},"expected_delta":{"added_facts":["gamma","delta","epsilon"],"removed_facts":[]}} +{"id":"5r-react-008","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["status=active"]},"snapshot_t1":{"facts":["status=archived"]},"expected_delta":{"added_facts":["status=archived"],"removed_facts":["status=active"]}} +{"id":"5r-react-009","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["price=10","stock=100"]},"snapshot_t1":{"facts":["price=12","stock=95"]},"expected_delta":{"added_facts":["price=12","stock=95"],"removed_facts":["price=10","stock=100"]}} +{"id":"5r-react-010","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["entry-1"]},"snapshot_t1":{"facts":["entry-1","entry-2"]},"expected_delta":{"added_facts":["entry-2"],"removed_facts":[]}} +{"id":"5r-react-011","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["red","green","blue"]},"snapshot_t1":{"facts":["red","green","blue","alpha"]},"expected_delta":{"added_facts":["alpha"],"removed_facts":[]}} +{"id":"5r-react-012","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["one","two","three","four"]},"snapshot_t1":{"facts":["one","two"]},"expected_delta":{"added_facts":[],"removed_facts":["three","four"]}} +{"id":"5r-react-013","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["motif-A"]},"snapshot_t1":{"facts":["motif-A","motif-B","motif-C"]},"expected_delta":{"added_facts":["motif-B","motif-C"],"removed_facts":[]}} +{"id":"5r-react-014","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["claim-x is true"]},"snapshot_t1":{"facts":["claim-x is false"]},"expected_delta":{"added_facts":["claim-x is false"],"removed_facts":["claim-x is true"]}} +{"id":"5r-react-015","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["q1","q2"]},"snapshot_t1":{"facts":["q1","q2","q3"]},"expected_delta":{"added_facts":["q3"],"removed_facts":[]}} +{"id":"5r-react-016","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["k1=v1","k2=v2","k3=v3"]},"snapshot_t1":{"facts":["k1=v1","k2=v2"]},"expected_delta":{"added_facts":[],"removed_facts":["k3=v3"]}} +{"id":"5r-react-017","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["foo"]},"snapshot_t1":{"facts":["foo","bar","baz","qux"]},"expected_delta":{"added_facts":["bar","baz","qux"],"removed_facts":[]}} +{"id":"5r-react-018","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["state=active"]},"snapshot_t1":{"facts":["state=stale"]},"expected_delta":{"added_facts":["state=stale"],"removed_facts":["state=active"]}} +{"id":"5r-react-019","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["clean"]},"snapshot_t1":{"facts":["clean","new","fresh"]},"expected_delta":{"added_facts":["new","fresh"],"removed_facts":[]}} +{"id":"5r-react-020","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["a","b","c","d","e"]},"snapshot_t1":{"facts":["c","d","e"]},"expected_delta":{"added_facts":[],"removed_facts":["a","b"]}} +{"id":"5r-react-021","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["alpha"]},"snapshot_t1":{"facts":["alpha","beta"]},"expected":"fail","expected_delta":{"added_facts":["nonexistent"],"removed_facts":[]}} +{"id":"5r-react-022","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["x"]},"snapshot_t1":{"facts":["x","y"]},"expected":"fail","expected_delta":{"added_facts":[],"removed_facts":["x"]}} +{"id":"5r-react-023","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":[]},"snapshot_t1":{"facts":["e1","e2"]},"expected_delta":{"added_facts":["e1","e2"],"removed_facts":[]}} +{"id":"5r-react-024","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["root"]},"snapshot_t1":{"facts":["root","branch1","branch2"]},"expected_delta":{"added_facts":["branch1","branch2"],"removed_facts":[]}} +{"id":"5r-react-025","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["S1","S2","S3"]},"snapshot_t1":{"facts":["S1","S2","S3","S4","S5","S6"]},"expected_delta":{"added_facts":["S4","S5","S6"],"removed_facts":[]}} +{"id":"5r-react-026","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["U"]},"snapshot_t1":{"facts":["U","V"]},"expected_delta":{"added_facts":["V"],"removed_facts":[]}} +{"id":"5r-react-027","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["before"]},"snapshot_t1":{"facts":["after"]},"expected_delta":{"added_facts":["after"],"removed_facts":["before"]}} +{"id":"5r-react-028","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["item-A","item-B","item-C","item-D"]},"snapshot_t1":{"facts":["item-A","item-D"]},"expected_delta":{"added_facts":[],"removed_facts":["item-B","item-C"]}} +{"id":"5r-react-029","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["lone"]},"snapshot_t1":{"facts":[]},"expected_delta":{"added_facts":[],"removed_facts":["lone"]}} +{"id":"5r-react-030","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshot_t0":{"facts":["p","q","r"]},"snapshot_t1":{"facts":["p","q","r"]},"expected_delta":{"added_facts":[],"removed_facts":[]}} diff --git a/bench/fixtures/5r/rearrange-v1.jsonl b/bench/fixtures/5r/rearrange-v1.jsonl new file mode 100644 index 0000000..8be6494 --- /dev/null +++ b/bench/fixtures/5r/rearrange-v1.jsonl @@ -0,0 +1,31 @@ +{"_meta":{"battery":"5r","sub_battery":"rearrange","version":"v1","task_count":30,"notes":"Reordered surface forms must canonicalize to the same bytes through the named π*. Tests order-invariance contracts."}} +{"id":"5r-rear-001","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"hello world","alt_b":"hello world","expected_equivalent":true} +{"id":"5r-rear-002","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"hello world","alt_b":"hello world","expected_equivalent":true} +{"id":"5r-rear-003","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"text\n\n\n\nmore","alt_b":"text\n\nmore","expected_equivalent":true} +{"id":"5r-rear-004","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"plain text","alt_b":"citeplain text","expected_equivalent":true} +{"id":"5r-rear-005","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"some text","alt_b":"some other text","expected_equivalent":false} +{"id":"5r-rear-006","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"keyword","alt_b":"''keyword''","expected_equivalent":true} +{"id":"5r-rear-007","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"phrase","alt_b":"'''phrase'''","expected_equivalent":true} +{"id":"5r-rear-008","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"a different text","alt_b":"another different text","expected_equivalent":false} +{"id":"5r-rear-009","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"sample","alt_b":"citation Asample","expected_equivalent":true} +{"id":"5r-rear-010","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"sentence one. sentence two.","alt_b":"sentence one. sentence two.","expected_equivalent":true} +{"id":"5r-rear-011","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"item","alt_b":"item","expected_equivalent":true} +{"id":"5r-rear-012","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"data","alt_b":"DATA","expected_equivalent":false} +{"id":"5r-rear-013","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":" leading","alt_b":"leading","expected_equivalent":true} +{"id":"5r-rear-014","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"trailing ","alt_b":"trailing","expected_equivalent":true} +{"id":"5r-rear-015","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"line1\n\nline2","alt_b":"line1\nline2","expected_equivalent":false} +{"id":"5r-rear-016","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"line1\n\n\nline2","alt_b":"line1\n\nline2","expected_equivalent":true} +{"id":"5r-rear-017","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"foox","alt_b":"foo","expected_equivalent":true} +{"id":"5r-rear-018","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"only ref","alt_b":"","expected_equivalent":true} +{"id":"5r-rear-019","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"alpha","alt_b":"omega","expected_equivalent":false} +{"id":"5r-rear-020","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"hello [[link|word]]","alt_b":"hello word","expected_equivalent":true} +{"id":"5r-rear-021","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"text [[File:img.jpg]]","alt_b":"text ","expected_equivalent":true} +{"id":"5r-rear-022","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"== Header ==\n\nbody","alt_b":"Header\n\nbody","expected_equivalent":true} +{"id":"5r-rear-023","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"body","alt_b":"== Different Header ==\n\nbody","expected_equivalent":false} +{"id":"5r-rear-024","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"unique","alt_b":"different","expected_equivalent":false} +{"id":"5r-rear-025","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"abc","alt_b":"xyz","expected_equivalent":false} +{"id":"5r-rear-026","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"text and stuff","alt_b":"text and stuff","expected_equivalent":true} +{"id":"5r-rear-027","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"item-A","alt_b":"item-B","expected_equivalent":false} +{"id":"5r-rear-028","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"x","alt_b":"notex","expected_equivalent":true} +{"id":"5r-rear-029","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"normal text","alt_b":"normal textcitation","expected_equivalent":true} +{"id":"5r-rear-030","battery":"5r","sub_battery":"rearrange","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","alt_a":"alpha-beta","alt_b":"alpha–beta","expected_equivalent":false} diff --git a/bench/fixtures/5r/replicate-v1.jsonl b/bench/fixtures/5r/replicate-v1.jsonl new file mode 100644 index 0000000..1d34a17 --- /dev/null +++ b/bench/fixtures/5r/replicate-v1.jsonl @@ -0,0 +1,31 @@ +{"_meta": {"battery": "5r", "sub_battery": "replicate", "version": "v1", "task_count": 30, "notes": "Same input through \u03c0* registry must yield byte-equal output across N replicas. Tests determinism contract."}} +{"id": "5r-rep-001", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "plain text", "replicas": 5} +{"id": "5r-rep-002", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "hello [[link|world]]", "replicas": 5} +{"id": "5r-rep-003", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "text with italic", "replicas": 5} +{"id": "5r-rep-004", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "text with citation", "replicas": 5} +{"id": "5r-rep-005", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "multi line\ntext\nbody", "replicas": 5} +{"id": "5r-rep-006", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "A simple sentence.", "replicas": 5} +{"id": "5r-rep-007", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "[[File:img.jpg|thumb]] caption", "replicas": 5} +{"id": "5r-rep-008", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "== Header ==\nbody", "replicas": 5} +{"id": "5r-rep-009", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "claim-lattice@v1", "input": "- claim with pointer. [E1]", "replicas": 5} +{"id": "5r-rep-010", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "claim-lattice@v1", "input": "- another claim. [E2]", "replicas": 5} +{"id": "5r-rep-011", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "claim-lattice@v1", "input": "multiple\n- claim a [E1]\n- claim b [E2]", "replicas": 5} +{"id": "5r-rep-012", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "claim-lattice@v1", "input": "- claim. [E1, E2]", "replicas": 5} +{"id": "5r-rep-013", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "text with [[wikilink]]", "replicas": 5} +{"id": "5r-rep-014", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "plain prose", "replicas": 5} +{"id": "5r-rep-015", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "whitespace collapse", "replicas": 5} +{"id": "5r-rep-016", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "single token", "replicas": 5} +{"id": "5r-rep-017", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "123 numeric", "replicas": 5} +{"id": "5r-rep-018", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "hello world", "replicas": 5} +{"id": "5r-rep-019", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "text [[Category:X]]", "replicas": 5} +{"id": "5r-rep-020", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "article body", "replicas": 5} +{"id": "5r-rep-021", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "multi-paragraph\n\nbody", "replicas": 5} +{"id": "5r-rep-022", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "reference frame", "replicas": 5} +{"id": "5r-rep-023", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "claim-lattice@v1", "input": "claim text", "replicas": 5} +{"id": "5r-rep-024", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "long form text with multiple sentences and various punctuation marks.", "replicas": 5} +{"id": "5r-rep-025", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "special chars: -_=+", "replicas": 5} +{"id": "5r-rep-026", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "quoted text", "replicas": 5} +{"id": "5r-rep-027", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "digits 12345", "replicas": 5} +{"id": "5r-rep-028", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "mixed-case Text", "replicas": 5} +{"id": "5r-rep-029", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "empty lines below\n", "replicas": 5} +{"id": "5r-rep-030", "battery": "5r", "sub_battery": "replicate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "uppercase ABC", "replicas": 5} diff --git a/bench/fixtures/5r/resonate-v1.jsonl b/bench/fixtures/5r/resonate-v1.jsonl new file mode 100644 index 0000000..cb9cc79 --- /dev/null +++ b/bench/fixtures/5r/resonate-v1.jsonl @@ -0,0 +1,31 @@ +{"_meta": {"battery": "5r", "sub_battery": "resonate", "version": "v1", "task_count": 30, "notes": "Variance check: deterministic \u03c0* should yield distinct=1 across N runs. expected_max_distinct=1 enforces zero-variance contract."}} +{"id": "5r-res-001", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "text-a", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-002", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "text-b", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-003", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "long text body with many words to canonicalize", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-004", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "hello [[link|word]] there", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-005", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "citation body text", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-006", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "plain", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-007", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "one\ntwo\nthree", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-008", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "= Header =\nbody", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-009", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "alpha beta gamma", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-010", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "punct, semi: colon!", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-011", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "empty", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-012", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "digits 999", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-013", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "mixed Case Text", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-014", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "whitespace runs", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-015", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "claim-lattice@v1", "input": "- claim x [E1]", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-016", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "claim-lattice@v1", "input": "- claim y [E2]", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-017", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "claim-lattice@v1", "input": "- multi [E1, E2]", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-018", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "claim-lattice@v1", "input": "- claim z [E5]", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-019", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "claim-lattice@v1", "input": "claim 1\nclaim 2", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-020", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "embedded\nnewline\ntext", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-021", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "utf8 caf\u00e9", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-022", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "multiline\n\nparagraph", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-023", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "article title", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-024", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "summary text", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-025", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "referenced fact", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-026", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "quoted material", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-027", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "numbered 1 2 3", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-028", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "symbols & < >", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-029", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "colon : separator", "runs": 10, "expected_max_distinct": 1} +{"id": "5r-res-030", "battery": "5r", "sub_battery": "resonate", "version": "v1", "carrier": "text", "domain": "prose", "pi_star_ref": "wikitext-base@v1", "input": "final test", "runs": 10, "expected_max_distinct": 1} diff --git a/bench/fixtures/5r/restore-v1.jsonl b/bench/fixtures/5r/restore-v1.jsonl new file mode 100644 index 0000000..22a9508 --- /dev/null +++ b/bench/fixtures/5r/restore-v1.jsonl @@ -0,0 +1,31 @@ +{"_meta":{"battery":"5r","sub_battery":"restore","version":"v1","task_count":30,"notes":"Retrieve a prior fact from the workspace history. Pass = fact is in current OR appears in any historical snapshot's facts."}} +{"id":"5r-rest-001","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"genesis","history":[{"facts":["genesis"]}],"current_facts":["genesis","update1"]} +{"id":"5r-rest-002","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"old fact","history":[{"facts":["old fact"]},{"facts":["middle fact"]}],"current_facts":["recent fact"]} +{"id":"5r-rest-003","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"X","history":[{"facts":["A","B","X"]},{"facts":["C","D"]}],"current_facts":["E"]} +{"id":"5r-rest-004","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"persistent","history":[],"current_facts":["persistent","new"]} +{"id":"5r-rest-005","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"never seen","history":[{"facts":["A","B"]}],"current_facts":["C","D"],"expected":"fail"} +{"id":"5r-rest-006","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"alpha","history":[{"facts":["alpha"]}],"current_facts":[]} +{"id":"5r-rest-007","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"sustained","history":[{"facts":["sustained"]},{"facts":["sustained"]},{"facts":["sustained","else"]}],"current_facts":["sustained"]} +{"id":"5r-rest-008","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"middle","history":[{"facts":["a"]},{"facts":["middle","b"]},{"facts":["c"]}],"current_facts":["d"]} +{"id":"5r-rest-009","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"deep","history":[{"facts":["deep","a"]},{"facts":["b"]},{"facts":["c"]},{"facts":["d"]},{"facts":["e"]}],"current_facts":["f"]} +{"id":"5r-rest-010","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"only-in-current","history":[{"facts":["a"]}],"current_facts":["only-in-current"]} +{"id":"5r-rest-011","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"q1","history":[{"facts":["q1","q2"]}],"current_facts":["q1","q2","q3"]} +{"id":"5r-rest-012","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"missing","history":[],"current_facts":[],"expected":"fail"} +{"id":"5r-rest-013","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"original","history":[{"facts":["original"]}],"current_facts":["evolved"]} +{"id":"5r-rest-014","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"X is true","history":[{"facts":["X is true"]},{"facts":["X is false"]}],"current_facts":["X is false"]} +{"id":"5r-rest-015","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"price=10","history":[{"facts":["price=10"]},{"facts":["price=12"]}],"current_facts":["price=15"]} +{"id":"5r-rest-016","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"only one","history":[],"current_facts":["only one"]} +{"id":"5r-rest-017","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"motif","history":[{"facts":["motif","other"]}],"current_facts":["other"]} +{"id":"5r-rest-018","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"u-fact","history":[{"facts":["u-fact"]}],"current_facts":["u-fact","new-u"]} +{"id":"5r-rest-019","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"data point a","history":[{"facts":["data point a"]}],"current_facts":["data point b"]} +{"id":"5r-rest-020","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"orphan","history":[{"facts":["a","b"]},{"facts":["c"]}],"current_facts":["d"],"expected":"fail"} +{"id":"5r-rest-021","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"corrected Q1","history":[{"facts":["original Q1"]},{"facts":["corrected Q1"]}],"current_facts":["corrected Q1"]} +{"id":"5r-rest-022","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"capability claim alpha","history":[{"facts":["capability claim alpha"]}],"current_facts":[]} +{"id":"5r-rest-023","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"branch-A","history":[{"facts":["branch-A","branch-B"]}],"current_facts":["branch-C"]} +{"id":"5r-rest-024","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"snapshot-1","history":[{"facts":["snapshot-1"]},{"facts":["snapshot-2"]},{"facts":["snapshot-3"]},{"facts":["snapshot-4"]}],"current_facts":["snapshot-5"]} +{"id":"5r-rest-025","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"observed event","history":[{"facts":["observed event"]}],"current_facts":["observed event","downstream effect"]} +{"id":"5r-rest-026","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"benchmark gold","history":[{"facts":["benchmark gold","other"]}],"current_facts":["other"]} +{"id":"5r-rest-027","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"verifier_method=quote","history":[{"facts":["verifier_method=quote"]}],"current_facts":["verifier_method=paraphrase"]} +{"id":"5r-rest-028","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"audit-event-001","history":[{"facts":["audit-event-001","audit-event-002"]}],"current_facts":["audit-event-003"]} +{"id":"5r-rest-029","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"distinct value","history":[{"facts":["other-1"]},{"facts":["other-2"]}],"current_facts":["other-3"],"expected":"fail"} +{"id":"5r-rest-030","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","prior_fact":"final","history":[{"facts":["final"]}],"current_facts":["final"]} diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 5abcd61..63fd926 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -65,7 +65,7 @@ Newest first. Update on every open/close. | #000024 | 5T Phase 1b + Dav1DPrometheus vocabulary alignment | closed · landed 2026-05-08 | 2026-05-07 | — | | #000023 | 5S Phase 1b: Syllogism · Synthesis · Semiotics | closed · landed 2026-05-08 | 2026-05-07 | — | | #000022 | Adapter LossReport (PRD I9 analogue) | closed · landed 2026-05-07 | 2026-05-07 | — | -| #000021 | 5S/5T/5R benchmark fixtures + harness | in progress · Phase 1a landed 2026-05-07 | 2026-05-07 | — | +| #000021 | 5S/5T/5R benchmark fixtures + harness | closed · Phase 1a + 2 landed 2026-05-08 | 2026-05-07 | — | | #000020 | Capital-cost ledger (8-capital queues) | closed · landed 2026-05-07 | 2026-05-07 | — | | #000019 | Specification methodology for π* and V | closed · landed 2026-05-07 | 2026-05-07 | — | | #000018 | Adversarial soft-hash covert-channel analysis | open · awaiting go/no-go | 2026-05-07 | — | diff --git a/docs/tickets/ticket-000021-bench-fixtures-5s-5t-5r.md b/docs/tickets/ticket-000021-bench-fixtures-5s-5t-5r.md index e4512e1..3ce1795 100644 --- a/docs/tickets/ticket-000021-bench-fixtures-5s-5t-5r.md +++ b/docs/tickets/ticket-000021-bench-fixtures-5s-5t-5r.md @@ -1,7 +1,8 @@ # Ticket #000021 — 5S/5T/5R benchmark fixtures + harness extension -**Status:** in progress · Phase 1a landed 2026-05-07; Phase 1b + 2 + 3 open +**Status:** closed · Phase 1a + Phase 2 landed 2026-05-08; Phase 3 (external corpora) deferred **Opened:** 2026-05-07 +**Closed:** 2026-05-08 **Scope:** Implement the SQD whitepaper's named bench batteries (5S Syntax/Semantics/Synthesis/Syllogism/Semiotics, 5T Transfer/ Triangulate/Timing/Transitivity/Truth, 5R React/Rearrange/Restore/ @@ -318,16 +319,43 @@ Phase 3 (external corpora) deferred to follow-up tickets. - Implement Synthesis, Syllogism, Semiotics under 5S; Triangulate, Timing, Transitivity, Truth under 5T. -### Phase 2 (deferred — depends on #000014 + #000017) +### Phase 2 (landed 2026-05-08 — depended on #000014 + #000017) -- 5R battery (React/Rearrange/Restore/Replicate/Resonate) over a - workspace abstraction. Depends on SelfModel + memory_root being - the workspace surface. +- `bench/batteries/b_5r.py` — React / Rearrange / Restore / + Replicate / Resonate runners. All deterministic, no LLM-as-judge. +- 30 fixtures × 5 sub-batteries = 150 new fixtures. +- React: incorporate new facts/constraints. Workspace = (snapshot_t0, + snapshot_t1, expected_delta). +- Rearrange: re-canonicalize different surface forms through a + named π*; pass when bytes match `expected_equivalent`. Tests the + order-invariance contracts in SelfModel + Memory. +- Restore: retrieve a `prior_fact` from current OR historical + snapshots. Workspace = `(history[], current_facts[])`. +- Replicate: same input through a π* must yield byte-equal output + across N replicas. Tests determinism contract. +- Resonate: variance check across N runs; deterministic π*'s + yield distinct=1. Tests stability. +- `make bench-5r` + `make bench-suite` (5S+5T+5F+5R aggregate). +- Tests: 6 new in `tests/test_bench_batteries.py`. Full suite: + 1192 passed. ### Phase 3 (deferred) - External-corpus expansion. Cross-substrate transfer testing. + Phase 1b/2 fixtures hand-curated; Phase 3 pulls from external + corpora to broaden signal floor. Not gated on any other ticket. -Closure criterion (final): all three phases landed; full battery -suite runs as `make bench-5s5t5r`. Phase 1a alone is not closure; -ticket stays in-progress until Phase 1b lands. +### Closure (Phase 1a + Phase 2) + +`make bench-suite` runs the **complete 21-sub-battery +Dav1DPrometheus surface** end-to-end: + +- 5S: Syntax · Semantics · Syllogism · Synthesis · Semiotics (108) +- 5T: Transfer · Transfer Learning · Triangulation · Truthtables · + Transitivity · Time (154 incl. legacy transfer) +- 5F: Function · Finetuning · Falsification · Formulate · + Feedback Loop (50) +- 5R: React · Rearrange · Restore · Replicate · Resonate (150) + +**462 deterministic fixtures, all passing.** Phase 3 stays open +under this ticket but does not gate closure. diff --git a/tests/test_bench_batteries.py b/tests/test_bench_batteries.py index 16501c4..22a0ac7 100644 --- a/tests/test_bench_batteries.py +++ b/tests/test_bench_batteries.py @@ -413,6 +413,66 @@ def test_capital_cost_delta_handles_missing_budget(): }) == 5.0 +# --- 5R Phase 2 (#000021) ---------------------------------------- + + +def test_5r_react_runs(): + from bench.batteries import b_5r + res = b_5r.run_react(REPO_ROOT / "bench" / "fixtures" / "5r" / "react-v1.jsonl") + assert res.battery == "5r" + assert res.sub_battery == "react" + assert res.pass_count == 30 + assert res.metrics["react_integration_rate"] == 1.0 + + +def test_5r_rearrange_runs(): + from bench.batteries import b_5r + res = b_5r.run_rearrange(REPO_ROOT / "bench" / "fixtures" / "5r" / "rearrange-v1.jsonl") + assert res.pass_count == 30 + assert res.metrics["rearrange_invariance_rate"] == 1.0 + + +def test_5r_restore_runs(): + from bench.batteries import b_5r + res = b_5r.run_restore(REPO_ROOT / "bench" / "fixtures" / "5r" / "restore-v1.jsonl") + assert res.pass_count == 30 + assert res.metrics["restore_retrievability_rate"] == 1.0 + + +def test_5r_replicate_runs(): + from bench.batteries import b_5r + res = b_5r.run_replicate(REPO_ROOT / "bench" / "fixtures" / "5r" / "replicate-v1.jsonl") + assert res.pass_count == 30 + assert res.metrics["replicate_determinism_rate"] == 1.0 + + +def test_5r_resonate_runs(): + """Deterministic π* must yield distinct=1 across N runs.""" + from bench.batteries import b_5r + res = b_5r.run_resonate(REPO_ROOT / "bench" / "fixtures" / "5r" / "resonate-v1.jsonl") + assert res.pass_count == 30 + assert res.metrics["resonate_stability_rate"] == 1.0 + assert res.metrics["mean_distinct_outputs"] == 1.0 + + +def test_5r_react_rejects_unsupported_carrier(tmp_path): + from bench.batteries import b_5r + + p = tmp_path / "bad.jsonl" + p.write_text( + json.dumps({"_meta": {"battery": "5r", "sub_battery": "react", "version": "v1"}}) + "\n" + + json.dumps({ + "id": "test", "carrier": "image", + "snapshot_t0": {"facts": []}, "snapshot_t1": {"facts": []}, + "expected_delta": {"added_facts": [], "removed_facts": []}, + }) + "\n", + encoding="utf-8", + ) + res = b_5r.run_react(p) + assert res.fail_count == 1 + assert "unsupported_carrier" in res.per_task[0].detail["reason"] + + def test_finetuning_zero_cost_fixture_emits_inf(tmp_path): """Synthesize a fixture with zero resource_budget; assert inf emitted.""" p = tmp_path / "ft-zero.jsonl" diff --git a/tests/test_session_integration.py b/tests/test_session_integration.py index 5d1b6d2..bd243ad 100644 --- a/tests/test_session_integration.py +++ b/tests/test_session_integration.py @@ -226,6 +226,9 @@ def test_full_dav1dprometheus_suite_runs_end_to_end(tmp_path, capsys): for sub in ("function", "finetuning", "falsification", "formulate", "feedback-loop"): assert ("5f", sub) in sub_batteries + # All five 5R sub-batteries (Phase 2 of #000021). + for sub in ("react", "rearrange", "restore", "replicate", "resonate"): + assert ("5r", sub) in sub_batteries # Aggregate pass counts: every sub-battery must have zero failures. for r in payload["results"]: @@ -236,14 +239,15 @@ def test_full_dav1dprometheus_suite_runs_end_to_end(tmp_path, capsys): def test_full_suite_total_fixture_count(): - """Sanity check: the full Phase-1 suite executes 312 deterministic tasks.""" + """Sanity check: the complete Dav1DPrometheus suite executes 462 + deterministic tasks across 21 sub-batteries (5S+5T+5F+5R).""" from bench.batteries.runner import _DEFAULT_FIXTURES, _run_one total = 0 for (battery, sub), fx in _DEFAULT_FIXTURES.items(): result = _run_one(battery, sub, Path(fx)) total += result.pass_count + result.fail_count - assert total == 312 + assert total == 462 def test_5s_phase1a_digests_unchanged_after_phase1b():