diff --git a/Makefile b/Makefile index 86aa3f4..73ea5f3 100644 --- a/Makefile +++ b/Makefile @@ -242,6 +242,22 @@ bench-emergent-pending: bootstrap ## print log entries awaiting teacher review $(PY) scripts/bench_emergent.py --print-pending +bench-5s: bootstrap ## 5S battery (Syntax + Semantics) on Phase-1a seed fixtures + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner \ + --battery 5s --sub syntax \ + --fixtures bench/fixtures/5s/syntax-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner \ + --battery 5s --sub semantics \ + --fixtures bench/fixtures/5s/semantics-v1.jsonl + +bench-5t: bootstrap ## 5T battery (Transfer) on Phase-1a seed fixture + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner \ + --battery 5t --sub transfer \ + --fixtures bench/fixtures/5t/transfer-v1.jsonl + +bench-5s5t: bench-5s bench-5t ## both Phase-1a batteries (alias) + + verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample $(ARBORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N) diff --git a/bench/batteries/__init__.py b/bench/batteries/__init__.py new file mode 100644 index 0000000..52e7f9a --- /dev/null +++ b/bench/batteries/__init__.py @@ -0,0 +1,13 @@ +"""5S/5T/5R battery harness (ticket #000021). + +Phase 1a (this landing): 5S Syntax + 5S Semantics + 5T Transfer +runners with seed fixtures. Phase 1b (full fixture authoring) and +Phase 2 (5R battery, depends on tickets #000014 + #000017) land +in follow-up tickets. +""" + +from __future__ import annotations + +from bench.batteries.base import Battery, BatteryResult, TaskResult + +__all__ = ["Battery", "BatteryResult", "TaskResult"] diff --git a/bench/batteries/b_5s.py b/bench/batteries/b_5s.py new file mode 100644 index 0000000..fb5a2a1 --- /dev/null +++ b/bench/batteries/b_5s.py @@ -0,0 +1,163 @@ +"""5S battery — Syntax / Semantics / Synthesis / Syllogism / Semiotics. + +Phase 1a implements Syntax and Semantics against arborist's π* +registry. The other three sub-batteries are reserved as stubs that +return zero-task BatteryResults; full fixtures + runners land in +follow-up tickets. +""" + +from __future__ import annotations + +from pathlib import Path + +from bench.batteries.base import ( + BatteryResult, + TaskResult, + fixture_digest, + fixture_meta, + iter_tasks, +) + + +def _runtime_digest() -> str: + """SHA-256 fingerprint of the active π* registry. + + Includes name@version of every registered π* so a registry change + surfaces in the runtime_digest. + """ + import hashlib + + from arborist.pi_star import list_keys + + payload = "\n".join(list_keys()).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def run_syntax(fixtures_path: Path) -> BatteryResult: + """Each task asserts that the named π* canonicalizes the input + without raising. expected="pass" is the only supported expected; + other values fail loudly.""" + from arborist.pi_star import get + + per_task: list[TaskResult] = [] + pass_count = 0 + fail_count = 0 + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok = False + detail: dict = {} + try: + pi_star = get(task["pi_star"]) + pi_star.canonicalize(task["input"].encode("utf-8")) + ok = task.get("expected", "pass") == "pass" + if not ok: + detail["reason"] = "canonicalize succeeded but expected != pass" + except NotImplementedError: + detail["reason"] = "pi_star is a stub" + ok = False + except Exception as exc: # noqa: BLE001 + detail["reason"] = f"{type(exc).__name__}: {exc}" + ok = False + if ok: + pass_count += 1 + else: + fail_count += 1 + per_task.append(TaskResult(task_id=task_id, passed=ok, detail=detail)) + + total = pass_count + fail_count + pass_rate = pass_count / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return BatteryResult( + battery="5s", + sub_battery=meta.get("sub_battery", "syntax"), + fixture_path=str(fixtures_path), + fixture_digest=fixture_digest(fixtures_path), + pass_count=pass_count, + fail_count=fail_count, + metrics={"parse_pass_rate": pass_rate}, + per_task=per_task, + runtime_digest=_runtime_digest(), + ) + + +def run_semantics(fixtures_path: Path) -> BatteryResult: + """Each task: two inputs + a π*; passes if canonicalize agrees with + expected_equivalent.""" + from arborist.pi_star import get + + per_task: list[TaskResult] = [] + pass_count = 0 + fail_count = 0 + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok = False + detail: dict = {} + try: + pi_star = get(task["pi_star"]) + ca = pi_star.canonicalize(task["input_a"].encode("utf-8")) + cb = pi_star.canonicalize(task["input_b"].encode("utf-8")) + equivalent = ca == cb + ok = equivalent == bool(task["expected_equivalent"]) + if not ok: + detail["reason"] = ( + f"observed_equivalent={equivalent} " + f"expected={task['expected_equivalent']}" + ) + except Exception as exc: # noqa: BLE001 + detail["reason"] = f"{type(exc).__name__}: {exc}" + ok = False + if ok: + pass_count += 1 + else: + fail_count += 1 + per_task.append(TaskResult(task_id=task_id, passed=ok, detail=detail)) + + total = pass_count + fail_count + rate = pass_count / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return BatteryResult( + battery="5s", + sub_battery=meta.get("sub_battery", "semantics"), + fixture_path=str(fixtures_path), + fixture_digest=fixture_digest(fixtures_path), + pass_count=pass_count, + fail_count=fail_count, + metrics={"equivalence_recovery_rate": rate}, + per_task=per_task, + runtime_digest=_runtime_digest(), + ) + + +def _stub_result(name: str, fixtures_path: Path) -> BatteryResult: + return BatteryResult( + battery="5s", + sub_battery=name, + fixture_path=str(fixtures_path), + fixture_digest="", + pass_count=0, + fail_count=0, + metrics={}, + per_task=[], + runtime_digest="", + ) + + +def run_synthesis(fixtures_path: Path) -> BatteryResult: + return _stub_result("synthesis", fixtures_path) + + +def run_syllogism(fixtures_path: Path) -> BatteryResult: + return _stub_result("syllogism", fixtures_path) + + +def run_semiotics(fixtures_path: Path) -> BatteryResult: + return _stub_result("semiotics", fixtures_path) + + +SUB_BATTERIES = { + "syntax": run_syntax, + "semantics": run_semantics, + "synthesis": run_synthesis, + "syllogism": run_syllogism, + "semiotics": run_semiotics, +} diff --git a/bench/batteries/b_5t.py b/bench/batteries/b_5t.py new file mode 100644 index 0000000..14c81d0 --- /dev/null +++ b/bench/batteries/b_5t.py @@ -0,0 +1,105 @@ +"""5T battery — Transfer / Triangulate / Timing / Transitivity / Truth. + +Phase 1a implements Transfer (paraphrase-invariance under a π*). +The other four sub-batteries are reserved as stubs. +""" + +from __future__ import annotations + +from pathlib import Path + +from bench.batteries.base import ( + BatteryResult, + TaskResult, + fixture_digest, + fixture_meta, + iter_tasks, +) +from bench.batteries.b_5s import _runtime_digest + + +def run_transfer(fixtures_path: Path) -> BatteryResult: + """Each task: π* + two surface variants → does canonicalize + collapse them as expected_equivalent says it should?""" + from arborist.pi_star import get + + per_task: list[TaskResult] = [] + pass_count = 0 + fail_count = 0 + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok = False + detail: dict = {} + try: + pi_star = get(task["pi_star"]) + ca = pi_star.canonicalize(task["input_a"].encode("utf-8")) + cb = pi_star.canonicalize(task["input_b"].encode("utf-8")) + equivalent = ca == cb + ok = equivalent == bool(task["expected_equivalent"]) + if not ok: + detail["reason"] = ( + f"observed_equivalent={equivalent} " + f"expected={task['expected_equivalent']}" + ) + except Exception as exc: # noqa: BLE001 + detail["reason"] = f"{type(exc).__name__}: {exc}" + ok = False + if ok: + pass_count += 1 + else: + fail_count += 1 + per_task.append(TaskResult(task_id=task_id, passed=ok, detail=detail)) + + total = pass_count + fail_count + rate = pass_count / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return BatteryResult( + battery="5t", + sub_battery=meta.get("sub_battery", "transfer"), + fixture_path=str(fixtures_path), + fixture_digest=fixture_digest(fixtures_path), + pass_count=pass_count, + fail_count=fail_count, + metrics={"transfer_pass_rate": rate}, + per_task=per_task, + runtime_digest=_runtime_digest(), + ) + + +def _stub_result(name: str, fixtures_path: Path) -> BatteryResult: + return BatteryResult( + battery="5t", + sub_battery=name, + fixture_path=str(fixtures_path), + fixture_digest="", + pass_count=0, + fail_count=0, + metrics={}, + per_task=[], + runtime_digest="", + ) + + +def run_triangulate(fixtures_path: Path) -> BatteryResult: + return _stub_result("triangulate", fixtures_path) + + +def run_timing(fixtures_path: Path) -> BatteryResult: + return _stub_result("timing", fixtures_path) + + +def run_transitivity(fixtures_path: Path) -> BatteryResult: + return _stub_result("transitivity", fixtures_path) + + +def run_truth(fixtures_path: Path) -> BatteryResult: + return _stub_result("truth", fixtures_path) + + +SUB_BATTERIES = { + "transfer": run_transfer, + "triangulate": run_triangulate, + "timing": run_timing, + "transitivity": run_transitivity, + "truth": run_truth, +} diff --git a/bench/batteries/base.py b/bench/batteries/base.py new file mode 100644 index 0000000..1ab6579 --- /dev/null +++ b/bench/batteries/base.py @@ -0,0 +1,92 @@ +"""Battery base classes. + +A Battery runs a fixture file end-to-end against a specified runtime +(usually arborist's existing surface) and emits a deterministic +:class:`BatteryResult`. Determinism: same fixture digest + same code +digest → identical result (modulo wall-clock fields). +""" + +from __future__ import annotations + +import hashlib +import json +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Iterable, Optional, Protocol + + +@dataclass +class TaskResult: + """Per-task outcome inside a battery run.""" + + task_id: str + passed: bool + detail: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class BatteryResult: + """End-to-end battery result; deterministic given fixed inputs.""" + + battery: str + sub_battery: str + fixture_path: str + fixture_digest: str + pass_count: int + fail_count: int + metrics: dict[str, float] = field(default_factory=dict) + per_task: list[TaskResult] = field(default_factory=list) + runtime_digest: str = "" + timestamp: int = field(default_factory=lambda: int(time.time())) + + def to_dict(self) -> dict: + d = asdict(self) + return d + + +class Battery(Protocol): + """One sub-battery (e.g., 5S Syntax).""" + + name: str + + def run(self, fixtures_path: Path) -> BatteryResult: + ... + + +def fixture_digest(fixtures_path: Path) -> str: + """SHA-256 over the raw fixture-file bytes. + + Used as the canonical identity of a battery run; bench results + cite this digest so cross-run comparisons are unambiguous. + """ + return hashlib.sha256( + Path(fixtures_path).read_bytes() + ).hexdigest() + + +def iter_tasks(fixtures_path: Path) -> Iterable[dict]: + """Yield non-meta task dicts from a JSONL fixture file.""" + with open(fixtures_path, "r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + obj = json.loads(line) + if "_meta" in obj: + continue + yield obj + + +def fixture_meta(fixtures_path: Path) -> dict: + """Return the ``_meta`` dict if present at the top of the JSONL.""" + with open(fixtures_path, "r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + obj = json.loads(line) + if "_meta" in obj: + return obj["_meta"] + return {} + return {} diff --git a/bench/batteries/runner.py b/bench/batteries/runner.py new file mode 100644 index 0000000..be2af7f --- /dev/null +++ b/bench/batteries/runner.py @@ -0,0 +1,114 @@ +"""CLI entry: ``python -m bench.batteries.runner``. + +Runs one or more battery sub-batteries against their fixture files +and emits a JSON result on stdout (or to ``--out`` if provided). + +Usage examples +-------------- + +:: + + # Run 5S Syntax + Semantics with the seed fixtures. + python -m bench.batteries.runner \\ + --battery 5s --sub syntax --fixtures bench/fixtures/5s/syntax-v1.jsonl + + python -m bench.batteries.runner \\ + --battery 5s --sub semantics \\ + --fixtures bench/fixtures/5s/semantics-v1.jsonl + + # Run 5T Transfer. + python -m bench.batteries.runner \\ + --battery 5t --sub transfer \\ + --fixtures bench/fixtures/5t/transfer-v1.jsonl + + # All Phase-1a fixtures, written to a results file. + python -m bench.batteries.runner --all --out bench/results/run.json +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import asdict +from pathlib import Path +from typing import Optional + +from bench.batteries import b_5s, b_5t +from bench.batteries.base import BatteryResult + + +_BATTERIES = { + "5s": b_5s.SUB_BATTERIES, + "5t": b_5t.SUB_BATTERIES, +} + +_DEFAULT_FIXTURES = { + ("5s", "syntax"): "bench/fixtures/5s/syntax-v1.jsonl", + ("5s", "semantics"): "bench/fixtures/5s/semantics-v1.jsonl", + ("5t", "transfer"): "bench/fixtures/5t/transfer-v1.jsonl", +} + + +def _run_one(battery: str, sub: str, fixtures: Path) -> BatteryResult: + if battery not in _BATTERIES: + raise SystemExit(f"unknown battery {battery!r}") + if sub not in _BATTERIES[battery]: + raise SystemExit(f"unknown sub-battery {sub!r} for {battery}") + return _BATTERIES[battery][sub](fixtures) + + +def _emit(results: list[BatteryResult], out: Optional[str]) -> None: + payload = { + "schema_version": "bench-result-v1", + "results": [asdict(r) for r in results], + } + encoded = json.dumps(payload, indent=2, ensure_ascii=False, default=str) + if out: + Path(out).parent.mkdir(parents=True, exist_ok=True) + Path(out).write_text(encoded + "\n", encoding="utf-8") + print(f"wrote {out}") + else: + print(encoded) + + +def main(argv: Optional[list[str]] = None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--battery", choices=sorted(_BATTERIES), default=None) + p.add_argument("--sub", default=None, help="sub-battery name") + p.add_argument("--fixtures", default=None, help="fixture JSONL path") + p.add_argument( + "--all", + dest="run_all", + action="store_true", + help="run all Phase-1a sub-batteries with default fixtures", + ) + p.add_argument("--out", default=None, help="write JSON to this path") + args = p.parse_args(argv) + + results: list[BatteryResult] = [] + if args.run_all: + for (battery, sub), fixture_path in _DEFAULT_FIXTURES.items(): + results.append(_run_one(battery, sub, Path(fixture_path))) + else: + if not args.battery or not args.sub: + p.error("--battery and --sub are required unless --all is given") + if args.fixtures: + fixtures = Path(args.fixtures) + else: + key = (args.battery, args.sub) + if key not in _DEFAULT_FIXTURES: + p.error( + f"no default fixture for ({args.battery},{args.sub}); " + "pass --fixtures explicitly" + ) + fixtures = Path(_DEFAULT_FIXTURES[key]) + results.append(_run_one(args.battery, args.sub, fixtures)) + + _emit(results, args.out) + failed = sum(r.fail_count for r in results) + return 1 if failed else 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/bench/fixtures/5s/semantics-v1.jsonl b/bench/fixtures/5s/semantics-v1.jsonl new file mode 100644 index 0000000..7c837b1 --- /dev/null +++ b/bench/fixtures/5s/semantics-v1.jsonl @@ -0,0 +1,9 @@ +{"_meta":{"battery":"5s","sub_battery":"semantics","version":"v1","task_count":8}} +{"id":"5s-sem-001","kind":"equivalence","pi_star":"wikitext-base@v1","input_a":"hello world","input_b":"hello world","expected_equivalent":true} +{"id":"5s-sem-002","kind":"equivalence","pi_star":"wikitext-base@v1","input_a":"hello world","input_b":"hello world","expected_equivalent":true} +{"id":"5s-sem-003","kind":"equivalence","pi_star":"wikitext-base@v1","input_a":"hello world","input_b":"goodbye world","expected_equivalent":false} +{"id":"5s-sem-004","kind":"equivalence","pi_star":"wikitext-base@v1","input_a":"[[link|word]]","input_b":"word","expected_equivalent":true} +{"id":"5s-sem-005","kind":"equivalence","pi_star":"wikitext-base@v1","input_a":"citetext","input_b":"text","expected_equivalent":true} +{"id":"5s-sem-006","kind":"equivalence","pi_star":"claim-lattice@v1","input_a":"- A claim. [E1]","input_b":"- A claim. [E1]","expected_equivalent":true} +{"id":"5s-sem-007","kind":"equivalence","pi_star":"claim-lattice@v1","input_a":"- A claim. [E1]","input_b":"- A different claim. [E1]","expected_equivalent":false} +{"id":"5s-sem-008","kind":"equivalence","pi_star":"claim-lattice@v1","input_a":"- A claim. [E1]","input_b":"- A claim. [E2]","expected_equivalent":false} diff --git a/bench/fixtures/5s/syntax-v1.jsonl b/bench/fixtures/5s/syntax-v1.jsonl new file mode 100644 index 0000000..da87ebe --- /dev/null +++ b/bench/fixtures/5s/syntax-v1.jsonl @@ -0,0 +1,11 @@ +{"_meta":{"battery":"5s","sub_battery":"syntax","version":"v1","task_count":10}} +{"id":"5s-syn-001","kind":"parse-pass","pi_star":"wikitext-base@v1","input":"Plain text.","expected":"pass"} +{"id":"5s-syn-002","kind":"parse-pass","pi_star":"wikitext-base@v1","input":"[[Wikilink]]","expected":"pass"} +{"id":"5s-syn-003","kind":"parse-pass","pi_star":"wikitext-base@v1","input":"''italic''","expected":"pass"} +{"id":"5s-syn-004","kind":"parse-pass","pi_star":"wikitext-base@v1","input":"'''bold'''","expected":"pass"} +{"id":"5s-syn-005","kind":"parse-pass","pi_star":"wikitext-base@v1","input":"== Header ==","expected":"pass"} +{"id":"5s-syn-006","kind":"parse-pass","pi_star":"claim-lattice@v1","input":"- A claim. [E1]","expected":"pass"} +{"id":"5s-syn-007","kind":"parse-pass","pi_star":"claim-lattice@v1","input":"- Another claim. [E2]","expected":"pass"} +{"id":"5s-syn-008","kind":"parse-pass","pi_star":"claim-lattice@v1","input":"","expected":"pass"} +{"id":"5s-syn-009","kind":"parse-pass","pi_star":"wikitext-base@v1","input":"citation dropped","expected":"pass"} +{"id":"5s-syn-010","kind":"parse-pass","pi_star":"wikitext-base@v1","input":"[[File:img.jpg|thumb]]","expected":"pass"} diff --git a/bench/fixtures/5t/transfer-v1.jsonl b/bench/fixtures/5t/transfer-v1.jsonl new file mode 100644 index 0000000..394d94f --- /dev/null +++ b/bench/fixtures/5t/transfer-v1.jsonl @@ -0,0 +1,5 @@ +{"_meta":{"battery":"5t","sub_battery":"transfer","version":"v1","task_count":4,"note":"Phase 1 seed; full fixtures land in follow-up. Each task names a π* and asserts that canonicalize-then-canonicalize equivalence is preserved across paraphrase shapes drawn from a different surface form."}} +{"id":"5t-xfer-001","kind":"paraphrase-invariance","pi_star":"wikitext-base@v1","input_a":"hello [[world]]","input_b":"hello [[world|world]]","expected_equivalent":true} +{"id":"5t-xfer-002","kind":"paraphrase-invariance","pi_star":"wikitext-base@v1","input_a":"text\n\n\n\nmore","input_b":"text\n\nmore","expected_equivalent":true} +{"id":"5t-xfer-003","kind":"paraphrase-invariance","pi_star":"claim-lattice@v1","input_a":"- A. [E1]\n- B. [E2]","input_b":"- A. [E1]\n- B. [E2]\n","expected_equivalent":true} +{"id":"5t-xfer-004","kind":"paraphrase-invariance","pi_star":"claim-lattice@v1","input_a":"- A. [E1]","input_b":"- A. [E1, E2]","expected_equivalent":false} diff --git a/docs/TICKETS.md b/docs/TICKETS.md index cf49165..228ddca 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -58,7 +58,7 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| | #000022 | Adapter LossReport (PRD I9 analogue) | open · awaiting go/no-go | 2026-05-07 | — | -| #000021 | 5S/5T/5R benchmark fixtures + harness | open · awaiting go/no-go | 2026-05-07 | — | +| #000021 | 5S/5T/5R benchmark fixtures + harness | in progress · Phase 1a landed 2026-05-07 | 2026-05-07 | — | | #000020 | Capital-cost ledger (8-capital queues) | closed · landed 2026-05-07 | 2026-05-07 | — | | #000019 | Specification methodology for π* and V | closed · landed 2026-05-07 | 2026-05-07 | — | | #000018 | Adversarial soft-hash covert-channel analysis | open · awaiting go/no-go | 2026-05-07 | — | diff --git a/docs/tickets/ticket-000021-bench-fixtures-5s-5t-5r.md b/docs/tickets/ticket-000021-bench-fixtures-5s-5t-5r.md index 52e5576..e4512e1 100644 --- a/docs/tickets/ticket-000021-bench-fixtures-5s-5t-5r.md +++ b/docs/tickets/ticket-000021-bench-fixtures-5s-5t-5r.md @@ -1,6 +1,6 @@ # Ticket #000021 — 5S/5T/5R benchmark fixtures + harness extension -**Status:** open · awaiting go/no-go +**Status:** in progress · Phase 1a landed 2026-05-07; Phase 1b + 2 + 3 open **Opened:** 2026-05-07 **Scope:** Implement the SQD whitepaper's named bench batteries (5S Syntax/Semantics/Synthesis/Syllogism/Semiotics, 5T Transfer/ @@ -290,12 +290,44 @@ bench-5s5t: bench-5s bench-5t ## 7. Status -**Open · awaiting go/no-go.** Mid-sized implementation surface -(~600 lines code + 200-1000 fixtures). Recommended Phase 1 scope -keeps it shippable in a single session for a focused operator; -Phase 2/3 are future tickets. +**In progress · Phase 1a landed 2026-05-07.** Skeleton + seed +fixtures ship; full Phase 1 fixture authoring + Phase 2 (5R) + +Phase 3 (external corpora) deferred to follow-up tickets. -Closure criterion: `bench/fixtures/5s/`, `bench/fixtures/5t/`, -`bench/batteries/{b_5s,b_5t,runner}.py` land; `make bench-5s5t` -runs end-to-end against the existing shards and produces a JSON -result file with all 10 battery slots populated. +### Phase 1a (landed) + +- `bench/batteries/{base,b_5s,b_5t,runner}.py` — Battery protocol, + BatteryResult, fixture digest helpers, runner CLI. +- `bench/fixtures/5s/syntax-v1.jsonl` — 10 seed tasks against + `wikitext-base@v1` and `claim-lattice@v1`. All pass. +- `bench/fixtures/5s/semantics-v1.jsonl` — 8 seed tasks + (equivalence assertions). All pass. +- `bench/fixtures/5t/transfer-v1.jsonl` — 4 seed tasks + (paraphrase-invariance). All pass. +- Battery runners for **5S Syntax**, **5S Semantics**, and **5T + Transfer** are functional. Other 5S/5T sub-batteries are stubs + returning zero-task BatteryResults. +- Makefile targets `bench-5s`, `bench-5t`, `bench-5s5t`. +- Tests: `tests/test_bench_batteries.py` — 17 cases. Full suite: + 1076 passed, 36 skipped. + +### Phase 1b (deferred) + +- Expand seed fixtures to 50-200 per sub-battery so signal floor + is meaningful (5pp tolerance over n≥3 runs). +- Implement Synthesis, Syllogism, Semiotics under 5S; Triangulate, + Timing, Transitivity, Truth under 5T. + +### Phase 2 (deferred — depends on #000014 + #000017) + +- 5R battery (React/Rearrange/Restore/Replicate/Resonate) over a + workspace abstraction. Depends on SelfModel + memory_root being + the workspace surface. + +### Phase 3 (deferred) + +- External-corpus expansion. Cross-substrate transfer testing. + +Closure criterion (final): all three phases landed; full battery +suite runs as `make bench-5s5t5r`. Phase 1a alone is not closure; +ticket stays in-progress until Phase 1b lands. diff --git a/tests/test_bench_batteries.py b/tests/test_bench_batteries.py new file mode 100644 index 0000000..162a02b --- /dev/null +++ b/tests/test_bench_batteries.py @@ -0,0 +1,156 @@ +"""5S/5T battery harness tests (ticket #000021 Phase 1a). + +Covers: +- Fixture digest is stable across reads +- 5S Syntax produces a non-empty BatteryResult on seed fixtures +- 5S Semantics passes/fails as expected on seed fixtures +- 5T Transfer passes/fails as expected on seed fixtures +- Stub sub-batteries return zero-task results without error +- Runner produces JSON-serializable output +""" + +from __future__ import annotations + +import json +from dataclasses import asdict +from pathlib import Path + +import pytest + +from bench.batteries import b_5s, b_5t +from bench.batteries.base import ( + BatteryResult, + fixture_digest, + fixture_meta, + iter_tasks, +) + + +REPO_ROOT = Path(__file__).resolve().parent.parent +SYNTAX_FX = REPO_ROOT / "bench" / "fixtures" / "5s" / "syntax-v1.jsonl" +SEM_FX = REPO_ROOT / "bench" / "fixtures" / "5s" / "semantics-v1.jsonl" +XFER_FX = REPO_ROOT / "bench" / "fixtures" / "5t" / "transfer-v1.jsonl" + + +def test_fixture_digest_stable(): + a = fixture_digest(SYNTAX_FX) + b = fixture_digest(SYNTAX_FX) + assert a == b + assert len(a) == 64 + + +def test_fixture_meta_round_trip(): + m = fixture_meta(SYNTAX_FX) + assert m["battery"] == "5s" + assert m["sub_battery"] == "syntax" + assert m["version"] == "v1" + + +def test_iter_tasks_skips_meta(): + tasks = list(iter_tasks(SYNTAX_FX)) + assert all("_meta" not in t for t in tasks) + assert len(tasks) == 10 + + +def test_5s_syntax_runs(): + res = b_5s.run_syntax(SYNTAX_FX) + assert isinstance(res, BatteryResult) + assert res.battery == "5s" + assert res.sub_battery == "syntax" + assert res.pass_count + res.fail_count == 10 + # All seed fixtures should pass — they're chosen for active π*'s. + assert res.pass_count == 10 + assert res.metrics["parse_pass_rate"] == 1.0 + assert len(res.runtime_digest) == 64 + + +def test_5s_semantics_runs(): + res = b_5s.run_semantics(SEM_FX) + assert res.battery == "5s" + assert res.sub_battery == "semantics" + # Seed fixtures are hand-curated to pass under current π*'s. + assert res.pass_count == res.pass_count + res.fail_count + assert res.metrics["equivalence_recovery_rate"] == 1.0 + + +def test_5t_transfer_runs(): + res = b_5t.run_transfer(XFER_FX) + assert res.battery == "5t" + assert res.sub_battery == "transfer" + assert res.pass_count + res.fail_count >= 1 + + +@pytest.mark.parametrize( + "fn", + [ + b_5s.run_synthesis, + b_5s.run_syllogism, + b_5s.run_semiotics, + b_5t.run_triangulate, + b_5t.run_timing, + b_5t.run_transitivity, + b_5t.run_truth, + ], +) +def test_stub_sub_batteries_return_zero(fn, tmp_path): + # Stub takes any path; doesn't read it. + res = fn(tmp_path / "missing.jsonl") + assert res.pass_count == 0 + assert res.fail_count == 0 + + +def test_battery_result_is_json_serializable(): + res = b_5s.run_syntax(SYNTAX_FX) + payload = json.dumps(asdict(res), default=str) + assert "syntax" in payload + + +def test_runner_main_smoke(tmp_path, capsys): + """Runner produces JSON on stdout when --out is omitted.""" + from bench.batteries.runner import main + + rc = main( + [ + "--battery", + "5s", + "--sub", + "syntax", + "--fixtures", + str(SYNTAX_FX), + ] + ) + out = capsys.readouterr().out + payload = json.loads(out) + assert payload["schema_version"] == "bench-result-v1" + assert payload["results"][0]["battery"] == "5s" + assert rc == 0 + + +def test_runner_writes_to_file(tmp_path): + from bench.batteries.runner import main + + out_path = tmp_path / "result.json" + rc = main( + [ + "--battery", + "5s", + "--sub", + "semantics", + "--fixtures", + str(SEM_FX), + "--out", + str(out_path), + ] + ) + assert rc == 0 + payload = json.loads(out_path.read_text()) + assert payload["results"][0]["sub_battery"] == "semantics" + + +def test_runner_all_runs_phase_1a(): + """--all runs all three default fixtures and exits 0 when all pass.""" + from bench.batteries.runner import main + + rc = main(["--all"]) + # All seed fixtures are constructed to pass; rc=0 expected. + assert rc == 0