5f: Phase 1b.2 — Formulate runner wires to arborist.qa.parse_claims (live)

First sub-battery to bridge from synthetic gold output to actual organism behavior. run_formulate now supports two fixture modes selected per-task: - Embedded (Phase 1a): produced_lattice in the fixture. 10 seed fixtures continue to pass via this path. - Live (Phase 1b.2): only input_text in the fixture; runner calls arborist.qa.parse_claims.parse_pointer_claims(input_text) and matches the live output against expected_lattice. Embedded takes precedence if both fields are present. Per-task detail.source ("embedded" | "live") surfaces in bench output so synthetic vs live signal is distinguishable. Surface: - bench/batteries/b_5f.py — _live_produced_lattice helper + two-mode dispatch in run_formulate - bench/fixtures/5f/formulate-live-v1.jsonl — 15 live-mode fixtures with input_text + expected_lattice (no produced_lattice) - Makefile: bench-5f-formulate-live target - Tests: 4 new in tests/test_bench_batteries.py - live path routes through real parser, all 15 pass - embedded path still works (10 Phase-1a fixtures) - _live_produced_lattice helper directly verifies parse output - fixture missing both fields fails cleanly with explanatory reason Phase 1a fixture digests unchanged. _DEFAULT_FIXTURES still points at formulate-v1.jsonl so `runner --all` behavior is identical; live-mode fixtures invoked via explicit --fixtures path. Full suite: 1196 passed, 36 skipped. Pattern set. Function/Finetuning/Falsification/Feedback Loop follow in subsequent commits.
2026-05-08 08:20:51 -04:00 · 2026-05-08 08:20:51 -04:00 · ba653755e4
commit ba653755e4
parent 6e20c792c4
5 changed files with 141 additions and 5 deletions
--- a/4
+++ b/4
@ -266,6 +266,10 @@ bench-5f: bootstrap ## 5F battery (Function+Finetuning+Falsification+Formulate+F
 	PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate      --fixtures bench/fixtures/5f/formulate-v1.jsonl
 	PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub feedback-loop  --fixtures bench/fixtures/5f/feedback-loop-v1.jsonl

+bench-5f-formulate-live: bootstrap ## 5F Formulate via live arborist.qa.parse_claims (Phase 1b.2)
+	PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate \
+	    --fixtures bench/fixtures/5f/formulate-live-v1.jsonl
+
 bench-5r: bootstrap ## 5R battery (React+Rearrange+Restore+Replicate+Resonate)
 	PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub react      --fixtures bench/fixtures/5r/react-v1.jsonl
 	PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub rearrange  --fixtures bench/fixtures/5r/rearrange-v1.jsonl
--- a/bench/batteries/b_5f.py
+++ b/bench/batteries/b_5f.py
@ -330,14 +330,44 @@ def run_falsification(fixtures_path: Path) -> BatteryResult:
 # ---------------------------------------------------------------------


+def _live_produced_lattice(input_text: str) -> dict:
+    """Derive a ``produced_lattice`` by routing ``input_text`` through
+    arborist's actual claim-lattice parser. Phase 1b.2 entry point —
+    moves the runner from synthetic gold to live organism output.
+
+    Returns the same shape Phase 1a fixtures embed:
+    ``{"claims": [{"claim_text": ..., "pointer_ids": [...]}, ...]}``.
+    """
+    from arborist.qa.parse_claims import parse_pointer_claims
+
+    parsed = parse_pointer_claims(input_text)
+    return {
+        "claims": [
+            {"claim_text": c.claim_text, "pointer_ids": list(c.pointer_ids)}
+            for c in parsed
+        ],
+    }
+
+
 def run_formulate(fixtures_path: Path) -> BatteryResult:
    """Structural match against expected claim lattice.

    Match policy: claim count + sorted-approx claim text + exact
-    pointer-ID-set. NOT exact-string. The fixture provides
-    ``produced_lattice`` (the system's output) and
-    ``expected_lattice`` (the gold). v1 fixtures embed produced
-    output; Phase 1b.2 will call arborist.qa.parse_claims.
+    pointer-ID-set. NOT exact-string.
+
+    Two fixture modes (selected per-task):
+
+    - **Embedded** (Phase 1a): fixture provides ``produced_lattice``
+      directly. Used for the 10 seed fixtures landed under #000025.
+    - **Live** (Phase 1b.2): fixture provides ``input_text`` only;
+      runner calls
+      :func:`arborist.qa.parse_claims.parse_pointer_claims` and
+      tests the actual organism's parser against
+      ``expected_lattice``.
+
+    Embedded takes precedence if both fields are present.
+    Per-task ``detail.source`` reports which path ran, so the bench
+    output distinguishes synthetic from live signal.
    """
    per_task: list[TaskResult] = []
    for task in iter_tasks(fixtures_path):
@ -349,7 +379,17 @@ def run_formulate(fixtures_path: Path) -> BatteryResult:
            )
            continue
        try:
-            produced = task["produced_lattice"]
+            if "produced_lattice" in task:
+                produced = task["produced_lattice"]
+                source = "embedded"
+            elif "input_text" in task:
+                produced = _live_produced_lattice(task["input_text"])
+                source = "live"
+            else:
+                raise KeyError(
+                    "task needs 'produced_lattice' (embedded) or "
+                    "'input_text' (live derivation)"
+                )
            expected = task["expected_lattice"]
            count_ok = len(produced.get("claims", [])) == expected.get("claim_count")
            order_ok = True
@ -373,6 +413,7 @@ def run_formulate(fixtures_path: Path) -> BatteryResult:
            observed = "pass" if passed_all else "fail"
            passed = observed == expected_outcome
            detail = {
+                "source": source,
                "count_ok": count_ok,
                "pointer_ok": pointer_ok,
                "text_ok": text_ok,
--- a/bench/fixtures/5f/formulate-live-v1.jsonl
+++ b/bench/fixtures/5f/formulate-live-v1.jsonl
@ -0,0 +1,16 @@
+{"_meta":{"battery":"5f","sub_battery":"formulate","version":"v1","task_count":15,"notes":"Phase 1b.2 live-derived fixtures: input_text routes through arborist.qa.parse_claims.parse_pointer_claims; expected_lattice is the gold the live parser must produce. No produced_lattice embedded — runner derives it."}}
+{"id":"5f-form-live-001","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- One claim. [E1]\n- Two claim. [E2]","expected_lattice":{"claim_count":2,"claims":[{"claim_text":"One claim.","pointer_ids":["E1"]},{"claim_text":"Two claim.","pointer_ids":["E2"]}]},"expected":"pass"}
+{"id":"5f-form-live-002","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Multi pointer claim. [E1, E2]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"Multi pointer claim.","pointer_ids":["E1","E2"]}]},"expected":"pass"}
+{"id":"5f-form-live-003","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- A claim. [E1]\n- Another. [E2]\n- Third. [E3]","expected_lattice":{"claim_count":3,"claims":[{"claim_text":"A claim.","pointer_ids":["E1"]},{"claim_text":"Another.","pointer_ids":["E2"]},{"claim_text":"Third.","pointer_ids":["E3"]}]},"expected":"pass"}
+{"id":"5f-form-live-004","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"The system has one claim. [E1]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"The system has one claim.","pointer_ids":["E1"]}]},"expected":"pass"}
+{"id":"5f-form-live-005","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- claim alpha. [E1]\n- claim beta. [E2]\n- claim gamma. [E1, E3]","expected_lattice":{"claim_count":3,"claims":[{"claim_text":"claim alpha.","pointer_ids":["E1"]},{"claim_text":"claim beta.","pointer_ids":["E2"]},{"claim_text":"claim gamma.","pointer_ids":["E1","E3"]}]},"expected":"pass"}
+{"id":"5f-form-live-006","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- The release date was July 3 1985. [E1]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"The release date was July 3 1985.","pointer_ids":["E1"]}]},"expected":"pass"}
+{"id":"5f-form-live-007","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Penicillin was discovered in 1928. [E1]\n- Alexander Fleming discovered it. [E1, E2]","expected_lattice":{"claim_count":2,"claims":[{"claim_text":"Penicillin was discovered in 1928.","pointer_ids":["E1"]},{"claim_text":"Alexander Fleming discovered it.","pointer_ids":["E1","E2"]}]},"expected":"pass"}
+{"id":"5f-form-live-008","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Mount Everest is the tallest mountain. [E5]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"Mount Everest is the tallest mountain.","pointer_ids":["E5"]}]},"expected":"pass"}
+{"id":"5f-form-live-009","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Water freezes at zero degrees Celsius. [E1]\n- The boiling point at sea level is one hundred. [E1]","expected_lattice":{"claim_count":2,"claims":[{"claim_text":"Water freezes at zero degrees Celsius.","pointer_ids":["E1"]},{"claim_text":"The boiling point at sea level is one hundred.","pointer_ids":["E1"]}]},"expected":"pass"}
+{"id":"5f-form-live-010","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Lone claim with two pointers. [E1, E2]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"Lone claim with two pointers.","pointer_ids":["E1","E2"]}]},"expected":"pass"}
+{"id":"5f-form-live-011","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- DNA has a double helix structure. [E1]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"DNA has a double helix structure.","pointer_ids":["E1"]}]},"expected":"pass"}
+{"id":"5f-form-live-012","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Hydrogen has atomic number 1. [E1]\n- Helium has atomic number 2. [E2]\n- Lithium has atomic number 3. [E3]\n- Beryllium has atomic number 4. [E4]","expected_lattice":{"claim_count":4,"claims":[{"claim_text":"Hydrogen has atomic number 1.","pointer_ids":["E1"]},{"claim_text":"Helium has atomic number 2.","pointer_ids":["E2"]},{"claim_text":"Lithium has atomic number 3.","pointer_ids":["E3"]},{"claim_text":"Beryllium has atomic number 4.","pointer_ids":["E4"]}]},"expected":"pass"}
+{"id":"5f-form-live-013","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- The Pacific Ocean is the largest ocean. [E10]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"The Pacific Ocean is the largest ocean.","pointer_ids":["E10"]}]},"expected":"pass"}
+{"id":"5f-form-live-014","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Goal-supporting claim. [E1]\n- Counter-evidence claim. [E2]","expected_lattice":{"claim_count":2,"claims":[{"claim_text":"Goal-supporting claim.","pointer_ids":["E1"]},{"claim_text":"Counter-evidence claim.","pointer_ids":["E2"]}]},"expected":"pass"}
+{"id":"5f-form-live-015","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Claim with multiple pointers. [E1, E2, E3, E4]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"Claim with multiple pointers.","pointer_ids":["E1","E2","E3","E4"]}]},"expected":"pass"}
--- a/docs/tickets/ticket-000025-5f-battery.md
+++ b/docs/tickets/ticket-000025-5f-battery.md
@ -517,6 +517,26 @@ test_5f_feedback_loop_observation_affects_downstream_snapshot
 14. Threshold calibration for v8 selection acceptance handed off
    to ticket #000012.

+### Phase 1b.2 — first live wire-up landed 2026-05-08
+
+`run_formulate` now supports two fixture modes:
+
+- **Embedded** (Phase 1a): fixture provides `produced_lattice`
+  directly. The 10 seed fixtures continue to pass.
+- **Live**: fixture provides `input_text` only; runner calls
+  `arborist.qa.parse_claims.parse_pointer_claims` on it and
+  matches the live output against `expected_lattice`.
+
+`bench/fixtures/5f/formulate-live-v1.jsonl` ships 15 live-mode
+fixtures. `make bench-5f-formulate-live` runs them. Per-task
+`detail.source` reports `"embedded"` or `"live"` so bench output
+distinguishes synthetic from live signal.
+
+This is the **first sub-battery to bridge from synthetic gold to
+actual organism behavior** — proves the Phase 1b.2 pattern. The
+remaining sub-batteries (Function, Finetuning, Falsification,
+Feedback Loop) follow the same pattern in subsequent commits.
+
 ---

 ## 11. Status
--- a/tests/test_bench_batteries.py
+++ b/tests/test_bench_batteries.py
@ -413,6 +413,61 @@ def test_capital_cost_delta_handles_missing_budget():
    }) == 5.0


+# --- 5F Phase 1b.2 — Formulate live wire-up ---------------------
+
+
+def test_5f_formulate_live_path_routes_through_parse_claims():
+    """The new formulate-live-v1.jsonl fixtures use input_text only;
+    runner derives produced_lattice via parse_pointer_claims and
+    matches against expected_lattice."""
+    res = b_5f.run_formulate(F5F / "formulate-live-v1.jsonl")
+    assert res.pass_count == 15
+    assert res.metrics["structural_match_rate"] == 1.0
+    # Every task ran through the live path.
+    for t in res.per_task:
+        assert t.detail["source"] == "live"
+
+
+def test_5f_formulate_embedded_path_still_works():
+    """Phase 1a fixtures (embedded produced_lattice) keep working
+    after the Phase 1b.2 wire-up. Backward compat invariant."""
+    res = b_5f.run_formulate(F5F / "formulate-v1.jsonl")
+    assert res.pass_count == 10
+    for t in res.per_task:
+        assert t.detail["source"] == "embedded"
+
+
+def test_5f_formulate_live_helper_uses_real_arborist_parser(tmp_path):
+    """The live path actually calls
+    arborist.qa.parse_claims.parse_pointer_claims — not a stub."""
+    from bench.batteries.b_5f import _live_produced_lattice
+
+    out = _live_produced_lattice("- Hello world. [E1]\n- Second one. [E2]")
+    assert out["claims"][0]["claim_text"] == "Hello world."
+    assert out["claims"][0]["pointer_ids"] == ["E1"]
+    assert out["claims"][1]["claim_text"] == "Second one."
+    assert out["claims"][1]["pointer_ids"] == ["E2"]
+
+
+def test_5f_formulate_rejects_fixture_with_neither_field(tmp_path):
+    """Task without produced_lattice OR input_text → fails cleanly."""
+    p = tmp_path / "bad.jsonl"
+    p.write_text(
+        json.dumps({"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1"}}) + "\n" +
+        json.dumps({
+            "id": "test",
+            "carrier": "text",
+            "domain": "claim_lattice",
+            "pi_star_ref": "claim-lattice@v1",
+            "expected_lattice": {"claim_count": 0, "claims": []},
+        }) + "\n",
+        encoding="utf-8",
+    )
+    res = b_5f.run_formulate(p)
+    assert res.fail_count == 1
+    assert "produced_lattice" in res.per_task[0].detail["reason"]
+
+
 # --- 5R Phase 2 (#000021) ----------------------------------------