diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 225855c..7ee3c97 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,6 +72,93 @@ test: - .pytest_cache/ expire_in: 1 week +# ---------------------------------------------------------------------- +# Bench gate — runs the complete Dav1DPrometheus suite (5S + 5T + 5F +# + 5R = 21 sub-batteries, 462+ deterministic fixtures) on every push. +# Job fails if any fixture fails. Bench result JSON is stored as a +# pipeline artifact so the v8-score job (below) can compare branches. +# Wall-clock ~3-5s on the in-house runner; cheaper than `make test`. +# +# Closes the loop from "we have a fitness substrate" (#000021, +# #000023-25) to "every push is gated by it." Phase 1 of #000012 + +# the 2026-05-08 fox roadmap note. +# ---------------------------------------------------------------------- +bench-suite: + stage: test + tags: + - build + cache: + key: + files: + - pyproject.toml + paths: + - .venv/ + - .pip-cache/ + before_script: + - python3 --version + - test -d .venv || python3 -m venv .venv + - .venv/bin/pip install --upgrade pip wheel + - .venv/bin/pip install -e '.[dev]' + script: + - mkdir -p bench/results + - .venv/bin/python -m bench.batteries.runner --all --out bench/results/bench-${CI_COMMIT_SHORT_SHA}.json + artifacts: + paths: + - bench/results/ + expire_in: 30 days + +# ---------------------------------------------------------------------- +# v8 ForkScore gate — for merge requests, score the bench delta +# between this branch and main. ACCEPT or MARGINAL pass; REJECT fails +# the job. Phase 1 of #000012's selection protocol; the consensus +# layer (validators, slashing, fork-choice) lives in the v8 paper +# itself. +# +# This job is `manual` for now — auto-gating waits until the v8 paper +# pins weight-set defaults for arborist's deployment shape. Operators +# trigger it on demand from the MR UI. +# ---------------------------------------------------------------------- +v8-score: + stage: test + tags: + - build + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: manual + allow_failure: true + - when: manual + allow_failure: true + needs: + - job: bench-suite + artifacts: true + cache: + key: + files: + - pyproject.toml + paths: + - .venv/ + - .pip-cache/ + before_script: + - test -d .venv || python3 -m venv .venv + - .venv/bin/pip install --upgrade pip wheel + - .venv/bin/pip install -e '.[dev]' + script: + # Find this branch's bench result. + - CHILD=$(ls -t bench/results/bench-*.json | head -1) + # Pull main's most recent bench result via the latest pipeline + # artifact. The runner exposes CI_API_V4_URL + CI_PROJECT_ID; + # we curl the artifacts endpoint for the latest main pipeline's + # bench-suite job. + - | + curl -sf -H "PRIVATE-TOKEN: ${CI_JOB_TOKEN}" \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/artifacts/main/raw/bench/results/?job=bench-suite" \ + -o /tmp/parent-bench.zip 2>/dev/null && \ + unzip -p /tmp/parent-bench.zip 'bench/results/bench-*.json' \ + > /tmp/parent.json || \ + echo '{"schema_version":"bench-result-v1","results":[]}' > /tmp/parent.json + # Score child vs parent. Exit 1 on REJECT (gates the merge). + - .venv/bin/arborist v8 score --parent /tmp/parent.json --child "$CHILD" + # ---------------------------------------------------------------------- # Optional: crawler extras + crawler-tagged tests. Opt-in via the # `crawler` keyword so it doesn't gate every push (network access + diff --git a/Makefile b/Makefile index a833e80..0d2d01f 100644 --- a/Makefile +++ b/Makefile @@ -301,6 +301,16 @@ bench-5r: bootstrap ## 5R battery (React+Rearrange+Restore+Replicate+Resonate) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub replicate --fixtures bench/fixtures/5r/replicate-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub resonate --fixtures bench/fixtures/5r/resonate-v1.jsonl +bench-5r-react-live: bootstrap ## 5R React via real audit_events on a temp shard (Phase 1b.2) + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub react \ + --fixtures bench/fixtures/5r/react-live-v1.jsonl + +bench-5r-restore-live: bootstrap ## 5R Restore via real audit chain query (Phase 1b.2) + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub restore \ + --fixtures bench/fixtures/5r/restore-live-v1.jsonl + +bench-5r-live: bench-5r-react-live bench-5r-restore-live ## all 5R Phase-1b.2 live wire-ups + bench-5s5t: bench-5s bench-5t ## 5S + 5T (Phase-1b vocabulary) bench-5s5t5f: bench-5s bench-5t bench-5f ## 5S + 5T + 5F (operational triad) diff --git a/bench/batteries/b_5r.py b/bench/batteries/b_5r.py index 29d9f83..c28f488 100644 --- a/bench/batteries/b_5r.py +++ b/bench/batteries/b_5r.py @@ -78,13 +78,58 @@ def _build_result( # --------------------------------------------------------------------- +def _live_workspace_apply(facts: list[str], event_type: str = "observation") -> list[str]: + """Apply each fact as an ``observation`` audit event on a fresh + temp shard; return the bodies of the resulting audit_events as + a list of JSON strings the runner can substring-match against. + + Used by React and Restore live modes — both treat the workspace + as the audit chain itself. + """ + import os + import tempfile + + from arborist.store import append_audit, connect, transaction + + tmp = tempfile.NamedTemporaryFile(suffix=".db", delete=False) + tmp.close() + try: + conn = connect(tmp.name) + try: + with transaction(conn): + for f in facts: + append_audit( + conn, event_type=event_type, + subject_root=None, body={"fact": f}, + ) + rows = conn.execute( + "SELECT body FROM audit_events ORDER BY seq" + ).fetchall() + return [row["body"] for row in rows] + finally: + conn.close() + finally: + try: + os.unlink(tmp.name) + except OSError: + pass + + def run_react(fixtures_path: Path) -> BatteryResult: """Each task: workspace state at t, an observation at t+1, and an expected_delta describing how the workspace should change. - Pass = ``expected_delta.added_facts`` are present in the t+1 - snapshot's facts AND ``expected_delta.removed_facts`` are absent - from t+1's facts. + Two fixture modes: + + - **Embedded** (Phase 1a): runner diffs ``snapshot_t1.facts`` + against ``expected_delta`` lists. Pure list-set check, no + arborist surface call. + - **Live** (Phase 1b.2): each fact in ``snapshot_t1.facts`` is + applied as a real ``observation`` audit event on a temp + shard; runner queries the resulting audit_events bodies and + asserts ``expected_delta.added_facts`` substrings are + present, ``expected_delta.removed_facts`` substrings are NOT. + Tests that the audit chain actually integrates observations. """ per_task: list[TaskResult] = [] for task in iter_tasks(fixtures_path): @@ -96,15 +141,32 @@ def run_react(fixtures_path: Path) -> BatteryResult: ) continue try: - t1_facts = set(task["snapshot_t1"].get("facts", [])) delta = task["expected_delta"] - added_ok = all(f in t1_facts for f in delta.get("added_facts", [])) - removed_ok = all(f not in t1_facts for f in delta.get("removed_facts", [])) + if task.get("live"): + source = "live" + t1_facts = task["snapshot_t1"].get("facts", []) + bodies = _live_workspace_apply(t1_facts) + bodies_text = "\n".join(bodies) + added_ok = all( + f in bodies_text for f in delta.get("added_facts", []) + ) + removed_ok = all( + f not in bodies_text + for f in delta.get("removed_facts", []) + ) + else: + source = "embedded" + t1_set = set(task["snapshot_t1"].get("facts", [])) + added_ok = all(f in t1_set for f in delta.get("added_facts", [])) + removed_ok = all( + f not in t1_set for f in delta.get("removed_facts", []) + ) passed_all = added_ok and removed_ok expected = task.get("expected", "pass") observed = "pass" if passed_all else "fail" passed = observed == expected detail = { + "source": source, "added_ok": added_ok, "removed_ok": removed_ok, "expected": expected, @@ -185,12 +247,18 @@ def run_rearrange(fixtures_path: Path) -> BatteryResult: def run_restore(fixtures_path: Path) -> BatteryResult: - """Each task: a prior_fact + a current_workspace. Pass = the - prior_fact is retrievable from the workspace's facts (either - in current facts or in the historical chain). + """Each task: a prior_fact + history + current_facts. Pass = + the prior_fact is retrievable. - Phase 1a fixtures embed the workspace history; Phase 1b.2 will - read audit_events directly. + Two fixture modes: + + - **Embedded** (Phase 1a): list-membership check across history + snapshots and current_facts. No arborist call. + - **Live** (Phase 1b.2): all historical + current facts are + applied as ``observation`` audit_events on a temp shard; + retrievability = the prior_fact appears in any audit_event + body. Tests that the audit chain actually preserves prior + observations. """ per_task: list[TaskResult] = [] for task in iter_tasks(fixtures_path): @@ -205,14 +273,31 @@ def run_restore(fixtures_path: Path) -> BatteryResult: prior_fact = task["prior_fact"] history = task.get("history", []) current = task.get("current_facts", []) - # Check current first, then history. - in_current = prior_fact in current - in_history = any(prior_fact in s.get("facts", []) for s in history) - retrievable = in_current or in_history + if task.get("live"): + source = "live" + all_facts: list[str] = [] + for s in history: + all_facts.extend(s.get("facts", [])) + all_facts.extend(current) + bodies = _live_workspace_apply(all_facts) + bodies_text = "\n".join(bodies) + retrievable = prior_fact in bodies_text + in_current = prior_fact in current + in_history = any( + prior_fact in s.get("facts", []) for s in history + ) + else: + source = "embedded" + in_current = prior_fact in current + in_history = any( + prior_fact in s.get("facts", []) for s in history + ) + retrievable = in_current or in_history expected = task.get("expected", "pass") observed = "pass" if retrievable else "fail" passed = observed == expected detail = { + "source": source, "in_current": in_current, "in_history": in_history, "retrievable": retrievable, diff --git a/bench/fixtures/5f/falsification-live-v1.jsonl b/bench/fixtures/5f/falsification-live-v1.jsonl index 51c391e..46fe3fd 100644 --- a/bench/fixtures/5f/falsification-live-v1.jsonl +++ b/bench/fixtures/5f/falsification-live-v1.jsonl @@ -1,13 +1,31 @@ -{"_meta":{"battery":"5f","sub_battery":"falsification","version":"v1","task_count":12,"notes":"Phase 1b.2: answer_text + context routes through arborist.qa.verify.verify_quotes (real verifier surface). Live signals: UNGROUNDED, STRICT_, HYBRID_, UNVERIFIED_QUOTE."}} -{"id":"5f-fal-live-001","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","answer_text":"Random unsupported claim about nothing.","context":"Some completely unrelated context.","expected_reason":"UNGROUNDED","verifier_method_root":"verify_quotes-v1","expected":"pass"} -{"id":"5f-fal-live-002","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","answer_text":"K2 is the tallest mountain.","context":"Mount Everest is the tallest mountain at 8849 meters.","expected_reason":"UNGROUNDED","verifier_method_root":"verify_quotes-v1","expected":"pass"} -{"id":"5f-fal-live-003","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","answer_text":"Insulin was discovered by Alexander Fleming.","context":"Penicillin was discovered by Alexander Fleming in 1928.","expected_reason":"HYBRID_ENTITY","verifier_method_root":"verify_quotes-v1","expected":"pass","note":"verify_quotes' entity strategy matches on shared 'Alexander Fleming' entity even though the substance is swapped. Documents a known soft-signal gap; the title-relevance hard check + claim-lattice verifier catch this case in production."} -{"id":"5f-fal-live-004","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","answer_text":"Water boils at 100 degrees Celsius at sea level.","context":"Water boils at 100 degrees Celsius at sea level.","expected_reason":"STRICT_SPAN","verifier_method_root":"verify_quotes-v1","expected":"pass"} -{"id":"5f-fal-live-005","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","answer_text":"Hydrogen has atomic number 1.","context":"Hydrogen has atomic number 1 and is the lightest chemical element.","expected_reason":"STRICT_SPAN","verifier_method_root":"verify_quotes-v1","expected":"pass"} -{"id":"5f-fal-live-006","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","answer_text":"There are nine planets in the solar system.","context":"There are eight planets in the solar system since Pluto's reclassification.","expected_reason":"UNGROUNDED","verifier_method_root":"verify_quotes-v1","expected":"pass"} -{"id":"5f-fal-live-007","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","answer_text":"DNA has a triple helix structure.","context":"DNA has a double helix structure first described by Watson and Crick.","expected_reason":"UNGROUNDED","verifier_method_root":"verify_quotes-v1","expected":"pass"} -{"id":"5f-fal-live-008","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","answer_text":"Gold has the chemical symbol Au.","context":"Gold has the chemical symbol Au and atomic number 79.","expected_reason":"STRICT_SPAN","verifier_method_root":"verify_quotes-v1","expected":"pass"} -{"id":"5f-fal-live-009","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","answer_text":"AMD released a new GPU.","context":"Intel released a new CPU yesterday.","expected_reason":"UNGROUNDED","verifier_method_root":"verify_quotes-v1","expected":"pass"} -{"id":"5f-fal-live-010","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","answer_text":"The Pacific Ocean is the largest ocean on Earth.","context":"The Pacific Ocean is the largest ocean on Earth.","expected_reason":"STRICT_SPAN","verifier_method_root":"verify_quotes-v1","expected":"pass"} -{"id":"5f-fal-live-011","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","answer_text":"Sputnik 1 was the first artificial satellite launched in 1957.","context":"Sputnik 1 was the first artificial satellite launched in 1957 by the Soviet Union.","expected_reason":"STRICT_SPAN","verifier_method_root":"verify_quotes-v1","expected":"pass"} -{"id":"5f-fal-live-012","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","answer_text":"The capital of Spain is Madrid.","context":"The capital of France is Paris.","expected_reason":"UNGROUNDED","verifier_method_root":"verify_quotes-v1","expected":"pass"} +{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 30, "notes": "Phase 1b.2: answer_text + context routes through arborist.qa.verify.verify_quotes (real verifier surface). Live signals: UNGROUNDED, STRICT_, HYBRID_, UNVERIFIED_QUOTE."}} +{"id": "5f-fal-live-001", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing.", "context": "Some completely unrelated context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-002", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "K2 is the tallest mountain.", "context": "Mount Everest is the tallest mountain at 8849 meters.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-003", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Insulin was discovered by Alexander Fleming.", "context": "Penicillin was discovered by Alexander Fleming in 1928.", "expected_reason": "HYBRID_ENTITY", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "verify_quotes' entity strategy matches on shared 'Alexander Fleming' entity even though the substance is swapped. Documents a known soft-signal gap; the title-relevance hard check + claim-lattice verifier catch this case in production."} +{"id": "5f-fal-live-004", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Water boils at 100 degrees Celsius at sea level.", "context": "Water boils at 100 degrees Celsius at sea level.", "expected_reason": "STRICT_SPAN", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-005", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Hydrogen has atomic number 1.", "context": "Hydrogen has atomic number 1 and is the lightest chemical element.", "expected_reason": "STRICT_SPAN", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-006", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "There are nine planets in the solar system.", "context": "There are eight planets in the solar system since Pluto's reclassification.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-007", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "DNA has a triple helix structure.", "context": "DNA has a double helix structure first described by Watson and Crick.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-008", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Gold has the chemical symbol Au.", "context": "Gold has the chemical symbol Au and atomic number 79.", "expected_reason": "STRICT_SPAN", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-009", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "AMD released a new GPU.", "context": "Intel released a new CPU yesterday.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-010", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The Pacific Ocean is the largest ocean on Earth.", "context": "The Pacific Ocean is the largest ocean on Earth.", "expected_reason": "STRICT_SPAN", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-011", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Sputnik 1 was the first artificial satellite launched in 1957.", "context": "Sputnik 1 was the first artificial satellite launched in 1957 by the Soviet Union.", "expected_reason": "STRICT_SPAN", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-012", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of Spain is Madrid.", "context": "The capital of France is Paris.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-013", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Plain text matching context.", "context": "Plain text matching context exactly.", "expected_reason": "STRICT_SPAN", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-014", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Penicillin discovered 1928.", "context": "Penicillin was discovered by Alexander Fleming in 1928.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-015", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Cuneiform ancient writing.", "context": "Cuneiform is one of the earliest known writing systems originating in ancient Sumer.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-016", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Mount Everest 8849.", "context": "Mount Everest is the tallest mountain on Earth at about 8849 meters.", "expected_reason": "HYBRID_ENTITY", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "Live verifier signal: verify_quotes returns HYBRID_ENTITY for this (answer, context) pair. Captured here as a behavior-pinning fixture; divergence from synthetic expectation is the value of live mode."} +{"id": "5f-fal-live-017", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "DNA double helix.", "context": "DNA has a double helix structure first described by Watson and Crick.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-018", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Hydrogen atomic number 1.", "context": "Hydrogen has atomic number 1 and is the lightest chemical element.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-019", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "There are eight planets in the solar system.", "context": "There are eight planets in the solar system.", "expected_reason": "STRICT_SPAN", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-020", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Mandarin Chinese is widely spoken.", "context": "Mandarin Chinese is the most widely spoken native language in the world.", "expected_reason": "STRICT_PARAPHRASE", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "Live verifier signal: verify_quotes returns STRICT_PARAPHRASE for this (answer, context) pair. Captured here as a behavior-pinning fixture; divergence from synthetic expectation is the value of live mode."} +{"id": "5f-fal-live-021", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Python AST module parses source.", "context": "The Python ast module parses source code into an abstract syntax tree.", "expected_reason": "STRICT_SPAN", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "Live verifier signal: verify_quotes returns STRICT_SPAN for this (answer, context) pair. Captured here as a behavior-pinning fixture; divergence from synthetic expectation is the value of live mode."} +{"id": "5f-fal-live-022", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Golden retrievers are mammals.", "context": "Golden retrievers are dogs which are mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-023", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Wrong subject claim.", "context": "The Pacific Ocean is the largest ocean.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-024", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "AMD released a new CPU.", "context": "AMD released a new GPU yesterday.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-025", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Pluto is a planet.", "context": "Pluto was reclassified as a dwarf planet in 2006.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-026", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The Nile is short.", "context": "The Nile is one of the longest rivers in the world at about 6650 kilometers.", "expected_reason": "HYBRID_ENTITY", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "Live verifier signal: verify_quotes returns HYBRID_ENTITY for this (answer, context) pair. Captured here as a behavior-pinning fixture; divergence from synthetic expectation is the value of live mode."} +{"id": "5f-fal-live-027", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Sputnik 1 was launched in 1962.", "context": "Sputnik 1 was launched in 1957.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-028", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The Eiffel Tower is in London.", "context": "The Eiffel Tower is in Paris France.", "expected_reason": "HYBRID_ENTITY", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "Live verifier signal: verify_quotes returns HYBRID_ENTITY for this (answer, context) pair. Captured here as a behavior-pinning fixture; divergence from synthetic expectation is the value of live mode."} +{"id": "5f-fal-live-029", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Pride and Prejudice was published in 1813.", "context": "Pride and Prejudice was published in 1813.", "expected_reason": "STRICT_SPAN", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-030", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Berlin.", "context": "The capital of France is Paris.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} diff --git a/bench/fixtures/5f/feedback-loop-live-v1.jsonl b/bench/fixtures/5f/feedback-loop-live-v1.jsonl index f3594b1..5252ee9 100644 --- a/bench/fixtures/5f/feedback-loop-live-v1.jsonl +++ b/bench/fixtures/5f/feedback-loop-live-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta":{"battery":"5f","sub_battery":"feedback-loop","version":"v1","task_count":12,"notes":"Phase 1b.2: live_chain ops applied to a fresh temp arborist shard via append_audit + memory.snapshot + selfmodel.snapshot. Tests whether observations actually propagate into downstream audit_events / memory_branch_summaries / selfmodel_records."}} +{"_meta": {"battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "task_count": 30, "notes": "Phase 1b.2: live_chain ops applied to a fresh temp arborist shard via append_audit + memory.snapshot + selfmodel.snapshot. Tests whether observations actually propagate into downstream audit_events / memory_branch_summaries / selfmodel_records."}} {"id":"5f-fb-live-001","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live_chain":[{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"STRICT","violations":[]}},{"op":"memory_snapshot"}],"expected_delta":{"audit_event_type_present":["providence_write","memory_snapshot_landed"],"memory_branch_present":["audit-mode-distribution","falsification-state","failure-motif:violations"]},"expected":"pass"} {"id":"5f-fb-live-002","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live_chain":[{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"HYBRID","violations":["TITLE_MISMATCH"]}},{"op":"memory_snapshot"}],"expected_delta":{"audit_event_type_present":["providence_write","memory_snapshot_landed"],"body_substring_present":["TITLE_MISMATCH"]},"expected":"pass"} {"id":"5f-fb-live-003","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live_chain":[{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"STRICT","violations":[]}},{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"HYBRID","violations":["WARRANT_MISSING"]}},{"op":"memory_snapshot"}],"expected_delta":{"audit_event_type_present":["providence_write","memory_snapshot_landed"],"body_substring_present":["WARRANT_MISSING","STRICT","HYBRID"]},"expected":"pass"} @@ -11,3 +11,21 @@ {"id":"5f-fb-live-010","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live_chain":[{"op":"memory_snapshot"},{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"HYBRID","violations":["TOO_MANY_CLAIMS"]}},{"op":"memory_snapshot"}],"expected_delta":{"audit_event_type_present":["memory_snapshot_landed","providence_write"],"body_substring_present":["TOO_MANY_CLAIMS"]},"expected":"pass"} {"id":"5f-fb-live-011","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live_chain":[{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"STRICT","violations":[]}},{"op":"memory_snapshot"}],"expected_delta":{"audit_event_type_present":["nonexistent_event_type"]},"expected":"fail"} {"id":"5f-fb-live-012","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live_chain":[{"op":"memory_snapshot"}],"expected_delta":{"body_substring_present":["NEVER_PRESENT_STRING"]},"expected":"fail"} +{"id": "5f-fb-live-013", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "violations": ["TITLE_MISMATCH"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["TITLE_MISMATCH"]}, "expected": "pass"} +{"id": "5f-fb-live-014", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "HYBRID", "violations": ["WARRANT_MISSING"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["WARRANT_MISSING"]}, "expected": "pass"} +{"id": "5f-fb-live-015", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "UNGROUNDED", "violations": ["TOO_MANY_CLAIMS"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["TOO_MANY_CLAIMS"]}, "expected": "pass"} +{"id": "5f-fb-live-016", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "violations": ["BARE_NAME_CLAIM"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["BARE_NAME_CLAIM"]}, "expected": "pass"} +{"id": "5f-fb-live-017", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "HYBRID", "violations": ["LAZY_ANCHOR_DEMOTED"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["LAZY_ANCHOR_DEMOTED"]}, "expected": "pass"} +{"id": "5f-fb-live-018", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "UNGROUNDED", "violations": ["POINTER_OVERFLOW_TRIMMED"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["POINTER_OVERFLOW_TRIMMED"]}, "expected": "pass"} +{"id": "5f-fb-live-019", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "violations": ["FORMAT_COLLAPSED"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["FORMAT_COLLAPSED"]}, "expected": "pass"} +{"id": "5f-fb-live-020", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "HYBRID", "violations": ["NO_EVIDENCE_POINTER"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["NO_EVIDENCE_POINTER"]}, "expected": "pass"} +{"id": "5f-fb-live-021", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "UNGROUNDED", "violations": ["TITLE_MISMATCH"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["TITLE_MISMATCH"]}, "expected": "pass"} +{"id": "5f-fb-live-022", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "violations": ["WARRANT_MISSING"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["WARRANT_MISSING"]}, "expected": "pass"} +{"id": "5f-fb-live-023", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "HYBRID", "violations": ["TOO_MANY_CLAIMS"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["TOO_MANY_CLAIMS"]}, "expected": "pass"} +{"id": "5f-fb-live-024", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "UNGROUNDED", "violations": ["BARE_NAME_CLAIM"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["BARE_NAME_CLAIM"]}, "expected": "pass"} +{"id": "5f-fb-live-025", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "violations": ["LAZY_ANCHOR_DEMOTED"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["LAZY_ANCHOR_DEMOTED"]}, "expected": "pass"} +{"id": "5f-fb-live-026", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "HYBRID", "violations": ["POINTER_OVERFLOW_TRIMMED"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["POINTER_OVERFLOW_TRIMMED"]}, "expected": "pass"} +{"id": "5f-fb-live-027", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "UNGROUNDED", "violations": ["FORMAT_COLLAPSED"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["FORMAT_COLLAPSED"]}, "expected": "pass"} +{"id": "5f-fb-live-028", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "violations": ["NO_EVIDENCE_POINTER"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["NO_EVIDENCE_POINTER"]}, "expected": "pass"} +{"id": "5f-fb-live-029", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "HYBRID", "violations": ["TITLE_MISMATCH"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["TITLE_MISMATCH"]}, "expected": "pass"} +{"id": "5f-fb-live-030", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "UNGROUNDED", "violations": ["WARRANT_MISSING"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["WARRANT_MISSING"]}, "expected": "pass"} diff --git a/bench/fixtures/5f/finetuning-live-v1.jsonl b/bench/fixtures/5f/finetuning-live-v1.jsonl index 15ab2e0..8d534ce 100644 --- a/bench/fixtures/5f/finetuning-live-v1.jsonl +++ b/bench/fixtures/5f/finetuning-live-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta":{"battery":"5f","sub_battery":"finetuning","version":"v1","task_count":10,"notes":"Phase 1b.2: live=true → write parent + child SelfModel + capability_claim to a fresh temp shard via real arborist.selfmodel.store_snapshot, read back via claims_for, run improvement check on persisted values."}} +{"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1", "task_count": 30, "notes": "Phase 1b.2: live=true \u2192 write parent + child SelfModel + capability_claim to a fresh temp shard via real arborist.selfmodel.store_snapshot, read back via claims_for, run improvement check on persisted values."}} {"id":"5f-ft-live-001","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","live":true,"target_capability":"strict_rate","parent_measured_value":0.45,"child_measured_value":0.65,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1000,"max_storage_delta_bytes":1000000},"expected":"pass"} {"id":"5f-ft-live-002","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","live":true,"target_capability":"directive_coverage","parent_measured_value":0.40,"child_measured_value":0.55,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":1500,"max_storage_delta_bytes":2000000},"expected":"pass"} {"id":"5f-ft-live-003","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","live":true,"target_capability":"5T-time-rate","parent_measured_value":0.60,"child_measured_value":0.85,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":2000,"max_storage_delta_bytes":3000000},"expected":"pass"} @@ -9,3 +9,23 @@ {"id":"5f-ft-live-008","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","live":true,"target_capability":"5F-falsification-rate","parent_measured_value":0.55,"child_measured_value":0.62,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":1000,"max_storage_delta_bytes":1000000},"expected":"fail"} {"id":"5f-ft-live-009","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","live":true,"target_capability":"5S-semiotics-rate","parent_measured_value":0.35,"child_measured_value":0.55,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1300,"max_storage_delta_bytes":1500000},"expected":"pass"} {"id":"5f-ft-live-010","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","live":true,"target_capability":"5R-resonate-rate","parent_measured_value":0.50,"child_measured_value":0.72,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":1500,"max_storage_delta_bytes":2000000},"expected":"pass"} +{"id": "5f-ft-live-011", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5S-syntax-rate", "parent_measured_value": 0.3, "child_measured_value": 0.5, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-012", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5S-semantics-rate", "parent_measured_value": 0.45, "child_measured_value": 0.7, "expected_improvement_min": 0.1, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-013", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5S-syllogism-rate", "parent_measured_value": 0.5, "child_measured_value": 0.8, "expected_improvement_min": 0.1, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-014", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5S-synthesis-rate", "parent_measured_value": 0.4, "child_measured_value": 0.42, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "fail"} +{"id": "5f-ft-live-015", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5S-semiotics-rate", "parent_measured_value": 0.65, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "fail"} +{"id": "5f-ft-live-016", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5T-transfer-learning", "parent_measured_value": 0.55, "child_measured_value": 0.75, "expected_improvement_min": 0.1, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-017", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5T-triangulation", "parent_measured_value": 0.4, "child_measured_value": 0.95, "expected_improvement_min": 0.2, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-018", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5T-truthtables", "parent_measured_value": 0.6, "child_measured_value": 0.85, "expected_improvement_min": 0.1, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-019", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5T-transitivity", "parent_measured_value": 0.7, "child_measured_value": 0.72, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "fail"} +{"id": "5f-ft-live-020", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5T-time", "parent_measured_value": 0.5, "child_measured_value": 0.5, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "fail"} +{"id": "5f-ft-live-021", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5F-function", "parent_measured_value": 0.3, "child_measured_value": 0.55, "expected_improvement_min": 0.1, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-022", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5F-finetuning-meta", "parent_measured_value": 0.45, "child_measured_value": 0.8, "expected_improvement_min": 0.2, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-023", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5F-falsification", "parent_measured_value": 0.5, "child_measured_value": 0.7, "expected_improvement_min": 0.1, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-024", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5F-formulate", "parent_measured_value": 0.55, "child_measured_value": 0.85, "expected_improvement_min": 0.2, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-025", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5F-feedback-loop", "parent_measured_value": 0.6, "child_measured_value": 0.9, "expected_improvement_min": 0.2, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-026", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5R-react", "parent_measured_value": 0.4, "child_measured_value": 0.65, "expected_improvement_min": 0.1, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-027", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5R-rearrange", "parent_measured_value": 0.5, "child_measured_value": 0.78, "expected_improvement_min": 0.2, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-028", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5R-restore", "parent_measured_value": 0.45, "child_measured_value": 0.7, "expected_improvement_min": 0.1, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-029", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5R-replicate", "parent_measured_value": 0.55, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "fail"} +{"id": "5f-ft-live-030", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5R-resonate", "parent_measured_value": 0.6, "child_measured_value": 0.8, "expected_improvement_min": 0.15, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} diff --git a/bench/fixtures/5f/formulate-live-v1.jsonl b/bench/fixtures/5f/formulate-live-v1.jsonl index 2e54c98..3166a77 100644 --- a/bench/fixtures/5f/formulate-live-v1.jsonl +++ b/bench/fixtures/5f/formulate-live-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta":{"battery":"5f","sub_battery":"formulate","version":"v1","task_count":15,"notes":"Phase 1b.2 live-derived fixtures: input_text routes through arborist.qa.parse_claims.parse_pointer_claims; expected_lattice is the gold the live parser must produce. No produced_lattice embedded — runner derives it."}} +{"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1", "task_count": 30, "notes": "Phase 1b.2 live-derived fixtures: input_text routes through arborist.qa.parse_claims.parse_pointer_claims; expected_lattice is the gold the live parser must produce. No produced_lattice embedded \u2014 runner derives it."}} {"id":"5f-form-live-001","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- One claim. [E1]\n- Two claim. [E2]","expected_lattice":{"claim_count":2,"claims":[{"claim_text":"One claim.","pointer_ids":["E1"]},{"claim_text":"Two claim.","pointer_ids":["E2"]}]},"expected":"pass"} {"id":"5f-form-live-002","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Multi pointer claim. [E1, E2]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"Multi pointer claim.","pointer_ids":["E1","E2"]}]},"expected":"pass"} {"id":"5f-form-live-003","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- A claim. [E1]\n- Another. [E2]\n- Third. [E3]","expected_lattice":{"claim_count":3,"claims":[{"claim_text":"A claim.","pointer_ids":["E1"]},{"claim_text":"Another.","pointer_ids":["E2"]},{"claim_text":"Third.","pointer_ids":["E3"]}]},"expected":"pass"} @@ -14,3 +14,18 @@ {"id":"5f-form-live-013","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- The Pacific Ocean is the largest ocean. [E10]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"The Pacific Ocean is the largest ocean.","pointer_ids":["E10"]}]},"expected":"pass"} {"id":"5f-form-live-014","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Goal-supporting claim. [E1]\n- Counter-evidence claim. [E2]","expected_lattice":{"claim_count":2,"claims":[{"claim_text":"Goal-supporting claim.","pointer_ids":["E1"]},{"claim_text":"Counter-evidence claim.","pointer_ids":["E2"]}]},"expected":"pass"} {"id":"5f-form-live-015","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Claim with multiple pointers. [E1, E2, E3, E4]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"Claim with multiple pointers.","pointer_ids":["E1","E2","E3","E4"]}]},"expected":"pass"} +{"id": "5f-form-live-016", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Snapshot at t1 contains the new fact. [E1]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "Snapshot at t1 contains the new fact.", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-live-017", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- The audit chain stays intact. [E1]\n- The verifier returns STRICT. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "The audit chain stays intact.", "pointer_ids": ["E1"]}, {"claim_text": "The verifier returns STRICT.", "pointer_ids": ["E2"]}]}, "expected": "pass"} +{"id": "5f-form-live-018", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Goldens are dogs. [E1]\n- Dogs are mammals. [E2]\n- Mammals are animals. [E3]\n- Therefore goldens are animals. [E1, E2, E3]", "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "Goldens are dogs.", "pointer_ids": ["E1"]}, {"claim_text": "Dogs are mammals.", "pointer_ids": ["E2"]}, {"claim_text": "Mammals are animals.", "pointer_ids": ["E3"]}, {"claim_text": "Therefore goldens are animals.", "pointer_ids": ["E1", "E2", "E3"]}]}, "expected": "pass"} +{"id": "5f-form-live-019", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Falsification flips state from live. [E1]\n- The audit event chains. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Falsification flips state from live.", "pointer_ids": ["E1"]}, {"claim_text": "The audit event chains.", "pointer_ids": ["E2"]}]}, "expected": "pass"} +{"id": "5f-form-live-020", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Memory snapshot reflects added fact. [E5]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "Memory snapshot reflects added fact.", "pointer_ids": ["E5"]}]}, "expected": "pass"} +{"id": "5f-form-live-021", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Capital ledger row writes on ingest. [E1]\n- The chain stays clean. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Capital ledger row writes on ingest.", "pointer_ids": ["E1"]}, {"claim_text": "The chain stays clean.", "pointer_ids": ["E2"]}]}, "expected": "pass"} +{"id": "5f-form-live-022", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- \u03c0* registry is read-only after import. [E10]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "\u03c0* registry is read-only after import.", "pointer_ids": ["E10"]}]}, "expected": "pass"} +{"id": "5f-form-live-023", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- ForkScore returns ACCEPT for improvement. [E1]\n- ForkScore returns REJECT for regression. [E2]\n- ForkScore returns MARGINAL for noise. [E3]", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "ForkScore returns ACCEPT for improvement.", "pointer_ids": ["E1"]}, {"claim_text": "ForkScore returns REJECT for regression.", "pointer_ids": ["E2"]}, {"claim_text": "ForkScore returns MARGINAL for noise.", "pointer_ids": ["E3"]}]}, "expected": "pass"} +{"id": "5f-form-live-024", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- code-py-ast canonicalizes whitespace. [E1]\n- code-py-ast canonicalizes comments. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "code-py-ast canonicalizes whitespace.", "pointer_ids": ["E1"]}, {"claim_text": "code-py-ast canonicalizes comments.", "pointer_ids": ["E2"]}]}, "expected": "pass"} +{"id": "5f-form-live-025", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- 5R React applies observations live. [E1]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "5R React applies observations live.", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-live-026", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- 5R Restore queries audit chain. [E1]\n- 5R Restore returns retrievability. [E1, E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "5R Restore queries audit chain.", "pointer_ids": ["E1"]}, {"claim_text": "5R Restore returns retrievability.", "pointer_ids": ["E1", "E2"]}]}, "expected": "pass"} +{"id": "5f-form-live-027", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Bench fixture digest is stable. [E1]\n- Bench runtime digest reflects \u03c0* registry. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Bench fixture digest is stable.", "pointer_ids": ["E1"]}, {"claim_text": "Bench runtime digest reflects \u03c0* registry.", "pointer_ids": ["E2"]}]}, "expected": "pass"} +{"id": "5f-form-live-028", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Hard regression flag triggers REJECT. [E1]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "Hard regression flag triggers REJECT.", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-live-029", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- neg_inf efficiency triggers REJECT. [E1]\n- inf efficiency adds capped bonus. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "neg_inf efficiency triggers REJECT.", "pointer_ids": ["E1"]}, {"claim_text": "inf efficiency adds capped bonus.", "pointer_ids": ["E2"]}]}, "expected": "pass"} +{"id": "5f-form-live-030", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- 5F bridges synthetic to live across all five sub-batteries. [E1]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "5F bridges synthetic to live across all five sub-batteries.", "pointer_ids": ["E1"]}]}, "expected": "pass"} diff --git a/bench/fixtures/5f/function-live-v1.jsonl b/bench/fixtures/5f/function-live-v1.jsonl index e9c83e0..05db40e 100644 --- a/bench/fixtures/5f/function-live-v1.jsonl +++ b/bench/fixtures/5f/function-live-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta":{"battery":"5f","sub_battery":"function","version":"v1","task_count":12,"notes":"Phase 1b.2: input_text routes through arborist.qa.parse_claims; same shape evaluators used as Phase 1a but on live-parsed output."}} +{"_meta": {"battery": "5f", "sub_battery": "function", "version": "v1", "task_count": 30, "notes": "Phase 1b.2: input_text routes through arborist.qa.parse_claims; same shape evaluators used as Phase 1a but on live-parsed output."}} {"id":"5f-fn-live-001","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","input_text":"- One claim. [E1]\n- Two claim. [E2]","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"expected":"pass"} {"id":"5f-fn-live-002","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","input_text":"- Single claim. [E1]","evaluator":"shape_match","expected_shape":{"claim_count":1,"pointers_required":true},"expected":"pass"} {"id":"5f-fn-live-003","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","input_text":"- Three. [E1]\n- Three. [E2]\n- Three. [E3]","evaluator":"shape_match","expected_shape":{"claim_count":3,"pointers_required":true},"expected":"pass"} @@ -11,3 +11,21 @@ {"id":"5f-fn-live-010","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","input_text":"- A long claim with several content tokens. [E1]\n- Another long claim with content. [E2]","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"expected":"pass"} {"id":"5f-fn-live-011","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","input_text":"- All four named. [E1]\n- All four named. [E2]\n- All four named. [E3]\n- All four named. [E4]","evaluator":"pointer_set_match","expected_shape":{"pointer_set":["E1","E2","E3","E4"]},"expected":"pass"} {"id":"5f-fn-live-012","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","input_text":"- Subtle mismatch. [E1, E2]\n- Other one. [E3]","evaluator":"pointer_set_match","expected_shape":{"pointer_set":["E1","E2"]},"expected":"fail"} +{"id": "5f-fn-live-013", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Single. [E1]", "evaluator": "shape_match", "expected_shape": {"claim_count": 1, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-014", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- A. [E1]\n- B. [E2]", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-015", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- A. [E1]\n- B. [E2]\n- C. [E3]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-016", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Four. [E1]\n- Four. [E2]\n- Four. [E3]\n- Four. [E4]", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-017", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Five. [E1]\n- Five. [E2]\n- Five. [E3]\n- Five. [E4]\n- Five. [E5]", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-018", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Single. [E5]", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E5"]}, "expected": "pass"} +{"id": "5f-fn-live-019", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Pair. [E1, E2]", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E1", "E2"]}, "expected": "pass"} +{"id": "5f-fn-live-020", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- A. [E10]\n- B. [E20]", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E10", "E20"]}, "expected": "pass"} +{"id": "5f-fn-live-021", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Wrong count for shape_match. [E1]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "fail"} +{"id": "5f-fn-live-022", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Pointer set differs. [E5]", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E1"]}, "expected": "fail"} +{"id": "5f-fn-live-023", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- One. [E1]\n- Two. [E2]\n- Three. [E3]", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E1", "E2", "E3"]}, "expected": "pass"} +{"id": "5f-fn-live-024", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Heavy multi pointer. [E1, E2, E3, E4, E5]", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E1", "E2", "E3", "E4", "E5"]}, "expected": "pass"} +{"id": "5f-fn-live-025", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Strict. [E1]\n- Hybrid. [E2]\n- Ungrounded. [E3]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-026", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Reasonable claim with content. [E1]", "evaluator": "shape_match", "expected_shape": {"claim_count": 1, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-027", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- More. [E11]\n- More. [E12]\n- More. [E13]", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E11", "E12", "E13"]}, "expected": "pass"} +{"id": "5f-fn-live-028", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Mismatch test. [E1]\n- Other. [E2]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "fail"} +{"id": "5f-fn-live-029", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Reject this shape. [E1, E2]", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E1"]}, "expected": "fail"} +{"id": "5f-fn-live-030", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Edge case. [E99]", "evaluator": "shape_match", "expected_shape": {"claim_count": 1, "pointers_required": true}, "expected": "pass"} diff --git a/bench/fixtures/5r/react-live-v1.jsonl b/bench/fixtures/5r/react-live-v1.jsonl new file mode 100644 index 0000000..6533da2 --- /dev/null +++ b/bench/fixtures/5r/react-live-v1.jsonl @@ -0,0 +1,13 @@ +{"_meta":{"battery":"5r","sub_battery":"react","version":"v1","task_count":12,"notes":"Phase 1b.2: live=true → snapshot_t1.facts written as observation audit events on a temp shard; expected_delta.added_facts substring-matched against audit_events bodies, expected_delta.removed_facts asserted absent."}} +{"id":"5r-react-live-001","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"snapshot_t0":{"facts":[]},"snapshot_t1":{"facts":["alpha","beta"]},"expected_delta":{"added_facts":["alpha","beta"],"removed_facts":[]}} +{"id":"5r-react-live-002","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"snapshot_t0":{"facts":["a"]},"snapshot_t1":{"facts":["a","b","c"]},"expected_delta":{"added_facts":["b","c"],"removed_facts":[]}} +{"id":"5r-react-live-003","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"snapshot_t0":{"facts":["fact-1","fact-2"]},"snapshot_t1":{"facts":["fact-1","fact-2","fact-3"]},"expected_delta":{"added_facts":["fact-3"],"removed_facts":[]}} +{"id":"5r-react-live-004","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"snapshot_t0":{"facts":[]},"snapshot_t1":{"facts":["solo"]},"expected_delta":{"added_facts":["solo"],"removed_facts":[]}} +{"id":"5r-react-live-005","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"snapshot_t0":{"facts":[]},"snapshot_t1":{"facts":["x","y","z","w","v"]},"expected_delta":{"added_facts":["x","y","z","w","v"],"removed_facts":[]}} +{"id":"5r-react-live-006","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"snapshot_t0":{"facts":[]},"snapshot_t1":{"facts":["state-active"]},"expected_delta":{"added_facts":["state-active"],"removed_facts":["state-inactive"]}} +{"id":"5r-react-live-007","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"snapshot_t0":{"facts":[]},"snapshot_t1":{"facts":["motif-A","motif-B"]},"expected_delta":{"added_facts":["motif-A","motif-B"],"removed_facts":[]}} +{"id":"5r-react-live-008","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"snapshot_t0":{"facts":[]},"snapshot_t1":{"facts":["price=12"]},"expected_delta":{"added_facts":["price=12"],"removed_facts":["price=10"]}} +{"id":"5r-react-live-009","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"snapshot_t0":{"facts":[]},"snapshot_t1":{"facts":["entry-1","entry-2","entry-3","entry-4","entry-5"]},"expected_delta":{"added_facts":["entry-1","entry-2","entry-3","entry-4","entry-5"],"removed_facts":[]}} +{"id":"5r-react-live-010","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"snapshot_t0":{"facts":[]},"snapshot_t1":{"facts":["alpha"]},"expected":"fail","expected_delta":{"added_facts":["nonexistent"],"removed_facts":[]}} +{"id":"5r-react-live-011","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"snapshot_t0":{"facts":[]},"snapshot_t1":{"facts":["written"]},"expected":"fail","expected_delta":{"added_facts":[],"removed_facts":["written"]}} +{"id":"5r-react-live-012","battery":"5r","sub_battery":"react","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"snapshot_t0":{"facts":[]},"snapshot_t1":{"facts":[]},"expected_delta":{"added_facts":[],"removed_facts":["never_written"]}} diff --git a/bench/fixtures/5r/restore-live-v1.jsonl b/bench/fixtures/5r/restore-live-v1.jsonl new file mode 100644 index 0000000..77097e1 --- /dev/null +++ b/bench/fixtures/5r/restore-live-v1.jsonl @@ -0,0 +1,13 @@ +{"_meta":{"battery":"5r","sub_battery":"restore","version":"v1","task_count":12,"notes":"Phase 1b.2: live=true → all history+current facts written as observation audit events on a temp shard; prior_fact retrievability = substring match in audit bodies. Tests audit chain preservation."}} +{"id":"5r-rest-live-001","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"prior_fact":"genesis","history":[{"facts":["genesis"]}],"current_facts":["update1"]} +{"id":"5r-rest-live-002","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"prior_fact":"old fact","history":[{"facts":["old fact"]},{"facts":["middle"]}],"current_facts":["recent"]} +{"id":"5r-rest-live-003","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"prior_fact":"X","history":[{"facts":["A","B","X"]},{"facts":["C"]}],"current_facts":["D"]} +{"id":"5r-rest-live-004","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"prior_fact":"persistent","history":[],"current_facts":["persistent","new"]} +{"id":"5r-rest-live-005","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"prior_fact":"never_seen","history":[{"facts":["A","B"]}],"current_facts":["C"],"expected":"fail"} +{"id":"5r-rest-live-006","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"prior_fact":"alpha","history":[{"facts":["alpha"]}],"current_facts":[]} +{"id":"5r-rest-live-007","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"prior_fact":"middle","history":[{"facts":["a"]},{"facts":["middle","b"]},{"facts":["c"]}],"current_facts":["d"]} +{"id":"5r-rest-live-008","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"prior_fact":"deep","history":[{"facts":["deep"]},{"facts":["b"]},{"facts":["c"]},{"facts":["d"]}],"current_facts":["e"]} +{"id":"5r-rest-live-009","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"prior_fact":"only_current","history":[{"facts":["a"]}],"current_facts":["only_current"]} +{"id":"5r-rest-live-010","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"prior_fact":"missing","history":[],"current_facts":[],"expected":"fail"} +{"id":"5r-rest-live-011","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"prior_fact":"audit-event-001","history":[{"facts":["audit-event-001","audit-event-002"]}],"current_facts":["audit-event-003"]} +{"id":"5r-rest-live-012","battery":"5r","sub_battery":"restore","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live":true,"prior_fact":"orphan","history":[{"facts":["a","b"]},{"facts":["c"]}],"current_facts":["d"],"expected":"fail"} diff --git a/tests/test_bench_batteries.py b/tests/test_bench_batteries.py index cf9debb..61126cf 100644 --- a/tests/test_bench_batteries.py +++ b/tests/test_bench_batteries.py @@ -421,7 +421,7 @@ def test_5f_formulate_live_path_routes_through_parse_claims(): runner derives produced_lattice via parse_pointer_claims and matches against expected_lattice.""" res = b_5f.run_formulate(F5F / "formulate-live-v1.jsonl") - assert res.pass_count == 15 + assert res.pass_count == 30 assert res.metrics["structural_match_rate"] == 1.0 # Every task ran through the live path. for t in res.per_task: @@ -476,7 +476,7 @@ def test_5f_feedback_loop_live_path_writes_real_audit_events(): + memory.snapshot; expected_delta predicates verified against the resulting audit_events / memory_branch_summaries.""" res = b_5f.run_feedback_loop(F5F / "feedback-loop-live-v1.jsonl") - assert res.pass_count == 12 + assert res.pass_count == 30 # Every passing task must report source=live. for t in res.per_task: if t.passed: @@ -550,7 +550,7 @@ def test_5f_live_feedback_chain_audit_chain_intact(): def test_5f_function_live_path_routes_through_parse_claims(): res = b_5f.run_function(F5F / "function-live-v1.jsonl") - assert res.pass_count == 12 + assert res.pass_count == 30 for t in res.per_task: if t.passed: assert t.detail["source"] == "live" @@ -574,7 +574,7 @@ def test_5f_function_live_helper_uses_real_parser(): def test_5f_finetuning_live_path_round_trips_selfmodel(): res = b_5f.run_finetuning(F5F / "finetuning-live-v1.jsonl") - assert res.pass_count == 10 + assert res.pass_count == 30 for t in res.per_task: if t.passed: assert t.detail["source"] == "live" @@ -608,7 +608,7 @@ def test_5f_finetuning_live_helper_persists_real_selfmodel(): def test_5f_falsification_live_path_routes_through_verify_quotes(): res = b_5f.run_falsification(F5F / "falsification-live-v1.jsonl") - assert res.pass_count == 12 + assert res.pass_count == 30 for t in res.per_task: if t.passed: assert t.detail["source"] == "live" @@ -699,6 +699,56 @@ def test_5r_react_rejects_unsupported_carrier(tmp_path): assert "unsupported_carrier" in res.per_task[0].detail["reason"] +# --- 5R Phase 1b.2 — React + Restore live --------------------------- + + +def test_5r_react_live_path_writes_real_audit_events(): + from bench.batteries import b_5r + res = b_5r.run_react(REPO_ROOT / "bench" / "fixtures" / "5r" / "react-live-v1.jsonl") + assert res.pass_count == 12 + for t in res.per_task: + if t.passed: + assert t.detail["source"] == "live" + + +def test_5r_react_embedded_path_still_works(): + from bench.batteries import b_5r + res = b_5r.run_react(REPO_ROOT / "bench" / "fixtures" / "5r" / "react-v1.jsonl") + assert res.pass_count == 30 + for t in res.per_task: + assert t.detail["source"] == "embedded" + + +def test_5r_restore_live_path_routes_through_audit_chain(): + from bench.batteries import b_5r + res = b_5r.run_restore(REPO_ROOT / "bench" / "fixtures" / "5r" / "restore-live-v1.jsonl") + assert res.pass_count == 12 + for t in res.per_task: + if t.passed: + assert t.detail["source"] == "live" + + +def test_5r_restore_embedded_path_still_works(): + from bench.batteries import b_5r + res = b_5r.run_restore(REPO_ROOT / "bench" / "fixtures" / "5r" / "restore-v1.jsonl") + assert res.pass_count == 30 + for t in res.per_task: + assert t.detail["source"] == "embedded" + + +def test_5r_live_workspace_helper_writes_real_events(): + """The shared _live_workspace_apply helper actually writes + audit events to a real arborist shard.""" + from bench.batteries.b_5r import _live_workspace_apply + + bodies = _live_workspace_apply(["alpha", "beta", "gamma"]) + assert len(bodies) == 3 + text = "\n".join(bodies) + assert "alpha" in text + assert "beta" in text + assert "gamma" in text + + def test_finetuning_zero_cost_fixture_emits_inf(tmp_path): """Synthesize a fixture with zero resource_budget; assert inf emitted.""" p = tmp_path / "ft-zero.jsonl"