From ba653755e4f03b22f8a82c6f8be2122017d44eb3 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 8 May 2026 08:20:51 -0400 Subject: [PATCH] =?UTF-8?q?5f:=20Phase=201b.2=20=E2=80=94=20Formulate=20ru?= =?UTF-8?q?nner=20wires=20to=20arborist.qa.parse=5Fclaims=20(live)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First sub-battery to bridge from synthetic gold output to actual organism behavior. run_formulate now supports two fixture modes selected per-task: - Embedded (Phase 1a): produced_lattice in the fixture. 10 seed fixtures continue to pass via this path. - Live (Phase 1b.2): only input_text in the fixture; runner calls arborist.qa.parse_claims.parse_pointer_claims(input_text) and matches the live output against expected_lattice. Embedded takes precedence if both fields are present. Per-task detail.source ("embedded" | "live") surfaces in bench output so synthetic vs live signal is distinguishable. Surface: - bench/batteries/b_5f.py — _live_produced_lattice helper + two-mode dispatch in run_formulate - bench/fixtures/5f/formulate-live-v1.jsonl — 15 live-mode fixtures with input_text + expected_lattice (no produced_lattice) - Makefile: bench-5f-formulate-live target - Tests: 4 new in tests/test_bench_batteries.py - live path routes through real parser, all 15 pass - embedded path still works (10 Phase-1a fixtures) - _live_produced_lattice helper directly verifies parse output - fixture missing both fields fails cleanly with explanatory reason Phase 1a fixture digests unchanged. _DEFAULT_FIXTURES still points at formulate-v1.jsonl so `runner --all` behavior is identical; live-mode fixtures invoked via explicit --fixtures path. Full suite: 1196 passed, 36 skipped. Pattern set. Function/Finetuning/Falsification/Feedback Loop follow in subsequent commits. --- Makefile | 4 ++ bench/batteries/b_5f.py | 51 ++++++++++++++++++--- bench/fixtures/5f/formulate-live-v1.jsonl | 16 +++++++ docs/tickets/ticket-000025-5f-battery.md | 20 +++++++++ tests/test_bench_batteries.py | 55 +++++++++++++++++++++++ 5 files changed, 141 insertions(+), 5 deletions(-) create mode 100644 bench/fixtures/5f/formulate-live-v1.jsonl diff --git a/Makefile b/Makefile index dbcc206..53400e9 100644 --- a/Makefile +++ b/Makefile @@ -266,6 +266,10 @@ bench-5f: bootstrap ## 5F battery (Function+Finetuning+Falsification+Formulate+F PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate --fixtures bench/fixtures/5f/formulate-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub feedback-loop --fixtures bench/fixtures/5f/feedback-loop-v1.jsonl +bench-5f-formulate-live: bootstrap ## 5F Formulate via live arborist.qa.parse_claims (Phase 1b.2) + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate \ + --fixtures bench/fixtures/5f/formulate-live-v1.jsonl + bench-5r: bootstrap ## 5R battery (React+Rearrange+Restore+Replicate+Resonate) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub react --fixtures bench/fixtures/5r/react-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub rearrange --fixtures bench/fixtures/5r/rearrange-v1.jsonl diff --git a/bench/batteries/b_5f.py b/bench/batteries/b_5f.py index d8094ae..3a69a6f 100644 --- a/bench/batteries/b_5f.py +++ b/bench/batteries/b_5f.py @@ -330,14 +330,44 @@ def run_falsification(fixtures_path: Path) -> BatteryResult: # --------------------------------------------------------------------- +def _live_produced_lattice(input_text: str) -> dict: + """Derive a ``produced_lattice`` by routing ``input_text`` through + arborist's actual claim-lattice parser. Phase 1b.2 entry point — + moves the runner from synthetic gold to live organism output. + + Returns the same shape Phase 1a fixtures embed: + ``{"claims": [{"claim_text": ..., "pointer_ids": [...]}, ...]}``. + """ + from arborist.qa.parse_claims import parse_pointer_claims + + parsed = parse_pointer_claims(input_text) + return { + "claims": [ + {"claim_text": c.claim_text, "pointer_ids": list(c.pointer_ids)} + for c in parsed + ], + } + + def run_formulate(fixtures_path: Path) -> BatteryResult: """Structural match against expected claim lattice. Match policy: claim count + sorted-approx claim text + exact - pointer-ID-set. NOT exact-string. The fixture provides - ``produced_lattice`` (the system's output) and - ``expected_lattice`` (the gold). v1 fixtures embed produced - output; Phase 1b.2 will call arborist.qa.parse_claims. + pointer-ID-set. NOT exact-string. + + Two fixture modes (selected per-task): + + - **Embedded** (Phase 1a): fixture provides ``produced_lattice`` + directly. Used for the 10 seed fixtures landed under #000025. + - **Live** (Phase 1b.2): fixture provides ``input_text`` only; + runner calls + :func:`arborist.qa.parse_claims.parse_pointer_claims` and + tests the actual organism's parser against + ``expected_lattice``. + + Embedded takes precedence if both fields are present. + Per-task ``detail.source`` reports which path ran, so the bench + output distinguishes synthetic from live signal. """ per_task: list[TaskResult] = [] for task in iter_tasks(fixtures_path): @@ -349,7 +379,17 @@ def run_formulate(fixtures_path: Path) -> BatteryResult: ) continue try: - produced = task["produced_lattice"] + if "produced_lattice" in task: + produced = task["produced_lattice"] + source = "embedded" + elif "input_text" in task: + produced = _live_produced_lattice(task["input_text"]) + source = "live" + else: + raise KeyError( + "task needs 'produced_lattice' (embedded) or " + "'input_text' (live derivation)" + ) expected = task["expected_lattice"] count_ok = len(produced.get("claims", [])) == expected.get("claim_count") order_ok = True @@ -373,6 +413,7 @@ def run_formulate(fixtures_path: Path) -> BatteryResult: observed = "pass" if passed_all else "fail" passed = observed == expected_outcome detail = { + "source": source, "count_ok": count_ok, "pointer_ok": pointer_ok, "text_ok": text_ok, diff --git a/bench/fixtures/5f/formulate-live-v1.jsonl b/bench/fixtures/5f/formulate-live-v1.jsonl new file mode 100644 index 0000000..2e54c98 --- /dev/null +++ b/bench/fixtures/5f/formulate-live-v1.jsonl @@ -0,0 +1,16 @@ +{"_meta":{"battery":"5f","sub_battery":"formulate","version":"v1","task_count":15,"notes":"Phase 1b.2 live-derived fixtures: input_text routes through arborist.qa.parse_claims.parse_pointer_claims; expected_lattice is the gold the live parser must produce. No produced_lattice embedded — runner derives it."}} +{"id":"5f-form-live-001","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- One claim. [E1]\n- Two claim. [E2]","expected_lattice":{"claim_count":2,"claims":[{"claim_text":"One claim.","pointer_ids":["E1"]},{"claim_text":"Two claim.","pointer_ids":["E2"]}]},"expected":"pass"} +{"id":"5f-form-live-002","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Multi pointer claim. [E1, E2]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"Multi pointer claim.","pointer_ids":["E1","E2"]}]},"expected":"pass"} +{"id":"5f-form-live-003","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- A claim. [E1]\n- Another. [E2]\n- Third. [E3]","expected_lattice":{"claim_count":3,"claims":[{"claim_text":"A claim.","pointer_ids":["E1"]},{"claim_text":"Another.","pointer_ids":["E2"]},{"claim_text":"Third.","pointer_ids":["E3"]}]},"expected":"pass"} +{"id":"5f-form-live-004","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"The system has one claim. [E1]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"The system has one claim.","pointer_ids":["E1"]}]},"expected":"pass"} +{"id":"5f-form-live-005","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- claim alpha. [E1]\n- claim beta. [E2]\n- claim gamma. [E1, E3]","expected_lattice":{"claim_count":3,"claims":[{"claim_text":"claim alpha.","pointer_ids":["E1"]},{"claim_text":"claim beta.","pointer_ids":["E2"]},{"claim_text":"claim gamma.","pointer_ids":["E1","E3"]}]},"expected":"pass"} +{"id":"5f-form-live-006","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- The release date was July 3 1985. [E1]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"The release date was July 3 1985.","pointer_ids":["E1"]}]},"expected":"pass"} +{"id":"5f-form-live-007","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Penicillin was discovered in 1928. [E1]\n- Alexander Fleming discovered it. [E1, E2]","expected_lattice":{"claim_count":2,"claims":[{"claim_text":"Penicillin was discovered in 1928.","pointer_ids":["E1"]},{"claim_text":"Alexander Fleming discovered it.","pointer_ids":["E1","E2"]}]},"expected":"pass"} +{"id":"5f-form-live-008","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Mount Everest is the tallest mountain. [E5]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"Mount Everest is the tallest mountain.","pointer_ids":["E5"]}]},"expected":"pass"} +{"id":"5f-form-live-009","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Water freezes at zero degrees Celsius. [E1]\n- The boiling point at sea level is one hundred. [E1]","expected_lattice":{"claim_count":2,"claims":[{"claim_text":"Water freezes at zero degrees Celsius.","pointer_ids":["E1"]},{"claim_text":"The boiling point at sea level is one hundred.","pointer_ids":["E1"]}]},"expected":"pass"} +{"id":"5f-form-live-010","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Lone claim with two pointers. [E1, E2]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"Lone claim with two pointers.","pointer_ids":["E1","E2"]}]},"expected":"pass"} +{"id":"5f-form-live-011","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- DNA has a double helix structure. [E1]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"DNA has a double helix structure.","pointer_ids":["E1"]}]},"expected":"pass"} +{"id":"5f-form-live-012","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Hydrogen has atomic number 1. [E1]\n- Helium has atomic number 2. [E2]\n- Lithium has atomic number 3. [E3]\n- Beryllium has atomic number 4. [E4]","expected_lattice":{"claim_count":4,"claims":[{"claim_text":"Hydrogen has atomic number 1.","pointer_ids":["E1"]},{"claim_text":"Helium has atomic number 2.","pointer_ids":["E2"]},{"claim_text":"Lithium has atomic number 3.","pointer_ids":["E3"]},{"claim_text":"Beryllium has atomic number 4.","pointer_ids":["E4"]}]},"expected":"pass"} +{"id":"5f-form-live-013","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- The Pacific Ocean is the largest ocean. [E10]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"The Pacific Ocean is the largest ocean.","pointer_ids":["E10"]}]},"expected":"pass"} +{"id":"5f-form-live-014","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Goal-supporting claim. [E1]\n- Counter-evidence claim. [E2]","expected_lattice":{"claim_count":2,"claims":[{"claim_text":"Goal-supporting claim.","pointer_ids":["E1"]},{"claim_text":"Counter-evidence claim.","pointer_ids":["E2"]}]},"expected":"pass"} +{"id":"5f-form-live-015","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Claim with multiple pointers. [E1, E2, E3, E4]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"Claim with multiple pointers.","pointer_ids":["E1","E2","E3","E4"]}]},"expected":"pass"} diff --git a/docs/tickets/ticket-000025-5f-battery.md b/docs/tickets/ticket-000025-5f-battery.md index 058e3eb..70a2bea 100644 --- a/docs/tickets/ticket-000025-5f-battery.md +++ b/docs/tickets/ticket-000025-5f-battery.md @@ -517,6 +517,26 @@ test_5f_feedback_loop_observation_affects_downstream_snapshot 14. Threshold calibration for v8 selection acceptance handed off to ticket #000012. +### Phase 1b.2 — first live wire-up landed 2026-05-08 + +`run_formulate` now supports two fixture modes: + +- **Embedded** (Phase 1a): fixture provides `produced_lattice` + directly. The 10 seed fixtures continue to pass. +- **Live**: fixture provides `input_text` only; runner calls + `arborist.qa.parse_claims.parse_pointer_claims` on it and + matches the live output against `expected_lattice`. + +`bench/fixtures/5f/formulate-live-v1.jsonl` ships 15 live-mode +fixtures. `make bench-5f-formulate-live` runs them. Per-task +`detail.source` reports `"embedded"` or `"live"` so bench output +distinguishes synthetic from live signal. + +This is the **first sub-battery to bridge from synthetic gold to +actual organism behavior** — proves the Phase 1b.2 pattern. The +remaining sub-batteries (Function, Finetuning, Falsification, +Feedback Loop) follow the same pattern in subsequent commits. + --- ## 11. Status diff --git a/tests/test_bench_batteries.py b/tests/test_bench_batteries.py index 22a0ac7..5c72ba3 100644 --- a/tests/test_bench_batteries.py +++ b/tests/test_bench_batteries.py @@ -413,6 +413,61 @@ def test_capital_cost_delta_handles_missing_budget(): }) == 5.0 +# --- 5F Phase 1b.2 — Formulate live wire-up --------------------- + + +def test_5f_formulate_live_path_routes_through_parse_claims(): + """The new formulate-live-v1.jsonl fixtures use input_text only; + runner derives produced_lattice via parse_pointer_claims and + matches against expected_lattice.""" + res = b_5f.run_formulate(F5F / "formulate-live-v1.jsonl") + assert res.pass_count == 15 + assert res.metrics["structural_match_rate"] == 1.0 + # Every task ran through the live path. + for t in res.per_task: + assert t.detail["source"] == "live" + + +def test_5f_formulate_embedded_path_still_works(): + """Phase 1a fixtures (embedded produced_lattice) keep working + after the Phase 1b.2 wire-up. Backward compat invariant.""" + res = b_5f.run_formulate(F5F / "formulate-v1.jsonl") + assert res.pass_count == 10 + for t in res.per_task: + assert t.detail["source"] == "embedded" + + +def test_5f_formulate_live_helper_uses_real_arborist_parser(tmp_path): + """The live path actually calls + arborist.qa.parse_claims.parse_pointer_claims — not a stub.""" + from bench.batteries.b_5f import _live_produced_lattice + + out = _live_produced_lattice("- Hello world. [E1]\n- Second one. [E2]") + assert out["claims"][0]["claim_text"] == "Hello world." + assert out["claims"][0]["pointer_ids"] == ["E1"] + assert out["claims"][1]["claim_text"] == "Second one." + assert out["claims"][1]["pointer_ids"] == ["E2"] + + +def test_5f_formulate_rejects_fixture_with_neither_field(tmp_path): + """Task without produced_lattice OR input_text → fails cleanly.""" + p = tmp_path / "bad.jsonl" + p.write_text( + json.dumps({"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1"}}) + "\n" + + json.dumps({ + "id": "test", + "carrier": "text", + "domain": "claim_lattice", + "pi_star_ref": "claim-lattice@v1", + "expected_lattice": {"claim_count": 0, "claims": []}, + }) + "\n", + encoding="utf-8", + ) + res = b_5f.run_formulate(p) + assert res.fail_count == 1 + assert "produced_lattice" in res.per_task[0].detail["reason"] + + # --- 5R Phase 2 (#000021) ----------------------------------------