From 2af28e66af0209a9a4fb475ce40794bf6ddd89dd Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 7 May 2026 20:14:44 -0400 Subject: [PATCH] =?UTF-8?q?bench:=20land=20#000023=20+=20#000024=20+=20#00?= =?UTF-8?q?0025=20(Phase=201a/1b=20=E2=80=94=20Dav1DPrometheus=20suite)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements three coupled tickets in one push: complete the 5S battery, align 5T to Dav1DPrometheus vocabulary + complete it, open 5F battery. All runners deterministic, no LLM-as-judge anywhere. #000023 — 5S Phase 1b (closed) run_syllogism + run_synthesis + run_semiotics replace stubs. 30 deterministic fixtures each (90 total, all passing). Carrier metadata mandatory; unsupported carriers fail explicitly. Syllogism kernel handles categorical_transitivity, chain_3, invalid_converse, missing_premise. Order-agnostic over premise permutations. Synthesis uses content-token subset check (stopwords removed) — catches single-token entity swaps that lax overlap missed. Semiotics validates synonym-swap invariance via π* canonicalize + re-substitution. Hidden-channel work stays defensive only. #000024 — 5T Phase 1b (closed) Vocabulary aligned: Transfer→transfer-learning, Truth→truthtables, Timing→time. Legacy transfer-v1.jsonl + run_transfer kept intact. run_transfer_learning, run_triangulation, run_truthtables, run_transitivity, run_time replace stubs. 30 fixtures each × 5 = 150 total (155 with legacy transfer). Triangulation runs 4 strategies (substring, token_subset, token_overlap, entity_match) and gates on agreement threshold. Truthtables: deterministic propositional evaluator with recursive-descent parser; supports AND/OR/NOT/XOR/IMPL/IFF; capped at N=4 variables. Transitivity: typed-relation whitelist (implies, subset_of, ancestor_of, before, less_than). BFS path walk; mixed/unknown relations fail by construction. Time: synthetic memory-snapshot chains test preservation, stale-marking, current_root tracking. Phase 1b.2 will read real memory_records. #000025 — 5F Phase 1a (in progress; Phase 1b expansion still open) New bench/batteries/b_5f.py with five sub-batteries: - Function (shape_match / pointer_set_match / threshold_on_metric) - Finetuning (parent→child SelfModel improvement check) - Falsification (planted-error detection rate, verifier_method_root pinned) - Formulate (structural lattice match: claim count + sorted-approx text + exact pointer-ID-set; not exact-string) - Feedback Loop (operation/observation chain → expected_delta in aggregated observation feed) 10 deterministic seed fixtures per sub-battery (50 total). feedback_efficiency / adaptation_efficiency hooks stub the capital- ledger integration for v8 fork-choice. Cross-cutting: - bench/batteries/base.py: PHASE_1_CARRIERS whitelist + validate_carrier helper. Backward-compatible with Phase 1a fixtures that lack carrier field (defaults to "text"). - runner.py registers all 16 sub-batteries in _DEFAULT_FIXTURES so `--all` runs the entire Dav1DPrometheus suite. - Makefile: bench-5s / bench-5t / bench-5f / bench-5s5t5f targets. bench-5t-legacy preserves Phase 1a access. - tests/test_bench_batteries.py: 29 tests (was 17). Phase 1a digest stability checks, sub-battery smoke tests, carrier rejection tests, transitivity-whitelist test, truthtables N>4 cap test. Bench summary across the full suite: 5s syntax/semantics/syllogism/synthesis/semiotics 108/108 pass 5t transfer/transfer-learning/triangulation/truthtables/transitivity/time 154/154 pass 5f function/finetuning/falsification/formulate/feedback-loop 50/50 pass TOTAL: 312 fixtures across 16 sub-batteries — 100% pass. Full test suite: 1103 passed, 36 skipped. Source: Legally Unprecedented Dav1DPrometheus (BasementAGI host). Honoring his framework. The complete state-space synthesis (SQD + v7 + 5S/5T/5F) is now executable infrastructure, not metaphor. --- Makefile | 37 +- bench/batteries/b_5f.py | 398 +++++++++++ bench/batteries/b_5s.py | 489 +++++++++++++- bench/batteries/b_5t.py | 639 ++++++++++++++++-- bench/batteries/base.py | 42 ++ bench/batteries/runner.py | 21 +- bench/fixtures/5f/falsification-v1.jsonl | 11 + bench/fixtures/5f/feedback-loop-v1.jsonl | 11 + bench/fixtures/5f/finetuning-v1.jsonl | 11 + bench/fixtures/5f/formulate-v1.jsonl | 11 + bench/fixtures/5f/function-v1.jsonl | 11 + bench/fixtures/5s/semiotics-v1.jsonl | 31 + bench/fixtures/5s/syllogism-v1.jsonl | 31 + bench/fixtures/5s/synthesis-v1.jsonl | 31 + bench/fixtures/5t/time-v1.jsonl | 31 + bench/fixtures/5t/transfer-learning-v2.jsonl | 31 + bench/fixtures/5t/transitivity-v1.jsonl | 31 + bench/fixtures/5t/triangulation-v1.jsonl | 31 + bench/fixtures/5t/truthtables-v1.jsonl | 31 + docs/TICKETS.md | 6 +- ...-phase-1b-syllogism-synthesis-semiotics.md | 3 +- .../ticket-000024-5t-phase-1b-and-rename.md | 3 +- docs/tickets/ticket-000025-5f-battery.md | 2 +- tests/test_bench_batteries.py | 372 +++++++--- 24 files changed, 2127 insertions(+), 188 deletions(-) create mode 100644 bench/batteries/b_5f.py create mode 100644 bench/fixtures/5f/falsification-v1.jsonl create mode 100644 bench/fixtures/5f/feedback-loop-v1.jsonl create mode 100644 bench/fixtures/5f/finetuning-v1.jsonl create mode 100644 bench/fixtures/5f/formulate-v1.jsonl create mode 100644 bench/fixtures/5f/function-v1.jsonl create mode 100644 bench/fixtures/5s/semiotics-v1.jsonl create mode 100644 bench/fixtures/5s/syllogism-v1.jsonl create mode 100644 bench/fixtures/5s/synthesis-v1.jsonl create mode 100644 bench/fixtures/5t/time-v1.jsonl create mode 100644 bench/fixtures/5t/transfer-learning-v2.jsonl create mode 100644 bench/fixtures/5t/transitivity-v1.jsonl create mode 100644 bench/fixtures/5t/triangulation-v1.jsonl create mode 100644 bench/fixtures/5t/truthtables-v1.jsonl diff --git a/Makefile b/Makefile index 73ea5f3..ad9e278 100644 --- a/Makefile +++ b/Makefile @@ -242,20 +242,33 @@ bench-emergent-pending: bootstrap ## print log entries awaiting teacher review $(PY) scripts/bench_emergent.py --print-pending -bench-5s: bootstrap ## 5S battery (Syntax + Semantics) on Phase-1a seed fixtures - PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner \ - --battery 5s --sub syntax \ - --fixtures bench/fixtures/5s/syntax-v1.jsonl - PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner \ - --battery 5s --sub semantics \ - --fixtures bench/fixtures/5s/semantics-v1.jsonl +bench-5s: bootstrap ## 5S battery (Syntax+Semantics+Syllogism+Synthesis+Semiotics) + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax --fixtures bench/fixtures/5s/syntax-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics --fixtures bench/fixtures/5s/semantics-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syllogism --fixtures bench/fixtures/5s/syllogism-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub synthesis --fixtures bench/fixtures/5s/synthesis-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semiotics --fixtures bench/fixtures/5s/semiotics-v1.jsonl -bench-5t: bootstrap ## 5T battery (Transfer) on Phase-1a seed fixture - PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner \ - --battery 5t --sub transfer \ - --fixtures bench/fixtures/5t/transfer-v1.jsonl +bench-5t: bootstrap ## 5T battery (Transfer Learning+Triangulation+Truthtables+Transitivity+Time) + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub transfer-learning --fixtures bench/fixtures/5t/transfer-learning-v2.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub triangulation --fixtures bench/fixtures/5t/triangulation-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub truthtables --fixtures bench/fixtures/5t/truthtables-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub transitivity --fixtures bench/fixtures/5t/transitivity-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub time --fixtures bench/fixtures/5t/time-v1.jsonl -bench-5s5t: bench-5s bench-5t ## both Phase-1a batteries (alias) +bench-5t-legacy: bootstrap ## 5T legacy SQD-name (transfer-v1) for Phase-1a digest stability + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub transfer --fixtures bench/fixtures/5t/transfer-v1.jsonl + +bench-5f: bootstrap ## 5F battery (Function+Finetuning+Falsification+Formulate+Feedback Loop) + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub function --fixtures bench/fixtures/5f/function-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub finetuning --fixtures bench/fixtures/5f/finetuning-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub falsification --fixtures bench/fixtures/5f/falsification-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate --fixtures bench/fixtures/5f/formulate-v1.jsonl + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub feedback-loop --fixtures bench/fixtures/5f/feedback-loop-v1.jsonl + +bench-5s5t: bench-5s bench-5t ## 5S + 5T (Phase-1b vocabulary) + +bench-5s5t5f: bench-5s bench-5t bench-5f ## complete Dav1DPrometheus suite (5S + 5T + 5F) verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample diff --git a/bench/batteries/b_5f.py b/bench/batteries/b_5f.py new file mode 100644 index 0000000..f01cf50 --- /dev/null +++ b/bench/batteries/b_5f.py @@ -0,0 +1,398 @@ +"""5F battery — Function / Finetuning / Falsification / Formulate / +Feedback Loop (ticket #000025). + +Operational-quality axis from Dav1DPrometheus's framework. 5S +evaluates linguistic/sign-structure precision; 5T evaluates +temporal/cross-reasoning/logical coverage; 5F evaluates whether the +organism *acts, adapts, falsifies, formulates, and integrates +feedback* — the dimensions that turn a state-space into an adaptive +state-space. + +All runners deterministic, no LLM-as-judge. Phase 1a fixtures use +text / claim_lattice / providence_record / selfmodel_snapshot / +memory_snapshot carriers; future carriers fail/skip explicitly. +""" + +from __future__ import annotations + +from pathlib import Path + +from bench.batteries.base import ( + BatteryResult, + TaskResult, + fixture_digest, + fixture_meta, + iter_tasks, + validate_carrier, +) +from bench.batteries.b_5s import _content_tokens, _runtime_digest + + +def _carrier_check(task: dict) -> tuple[bool, str]: + reason = validate_carrier(task) + if reason is not None: + return False, reason + return True, "" + + +def _build_result( + sub_battery: str, + fixtures_path: Path, + per_task: list[TaskResult], + metrics: dict[str, float], +) -> BatteryResult: + pass_count = sum(1 for t in per_task if t.passed) + fail_count = len(per_task) - pass_count + return BatteryResult( + battery="5f", + sub_battery=sub_battery, + fixture_path=str(fixtures_path), + fixture_digest=fixture_digest(fixtures_path), + pass_count=pass_count, + fail_count=fail_count, + metrics=metrics, + per_task=per_task, + runtime_digest=_runtime_digest(), + ) + + +# --------------------------------------------------------------------- +# Function — does the system perform the designated operation? +# --------------------------------------------------------------------- + + +def _eval_shape_match(task: dict) -> tuple[bool, str]: + """Check that the produced output shape matches the expected + shape declaration. + + Phase 1a fixtures embed both the produced output and the + expected_shape. Real production wiring (Phase 1b) will call + arborist.qa.runner.ask and use the response shape. + """ + produced = task.get("produced_output", {}) + shape = task["expected_shape"] + claim_count_ok = len(produced.get("claims", [])) == shape.get("claim_count") + pointers_ok = True + if shape.get("pointers_required"): + pointers_ok = all( + isinstance(c.get("pointer_ids", []), list) + and len(c.get("pointer_ids", [])) > 0 + for c in produced.get("claims", []) + ) + return claim_count_ok and pointers_ok, "shape_match" + + +def _eval_pointer_set_match(task: dict) -> tuple[bool, str]: + produced = set() + for c in task.get("produced_output", {}).get("claims", []): + for p in c.get("pointer_ids", []): + produced.add(p) + expected = set(task["expected_shape"]["pointer_set"]) + return produced == expected, f"pointer_set_match expected={sorted(expected)} got={sorted(produced)}" + + +def _eval_threshold_on_metric(task: dict) -> tuple[bool, str]: + metric = task["expected_shape"]["metric"] + threshold = task["expected_shape"]["threshold"] + measured = task.get("produced_output", {}).get(metric) + if measured is None: + return False, f"metric {metric!r} not in produced_output" + return measured >= threshold, f"{metric}={measured} threshold={threshold}" + + +_FUNCTION_EVALUATORS = { + "shape_match": _eval_shape_match, + "pointer_set_match": _eval_pointer_set_match, + "threshold_on_metric": _eval_threshold_on_metric, +} + + +def run_function(fixtures_path: Path) -> BatteryResult: + """Function pass-rate over deterministic evaluator types.""" + per_task: list[TaskResult] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + evaluator = task.get("evaluator") + fn = _FUNCTION_EVALUATORS.get(evaluator) + if fn is None: + per_task.append( + TaskResult(task_id=task_id, passed=False, + detail={"reason": f"unknown_evaluator: {evaluator!r}"}) + ) + continue + try: + valid, why = fn(task) + expected = task.get("expected", "pass") + observed = "pass" if valid else "fail" + passed = observed == expected + except Exception as exc: # noqa: BLE001 + passed = False + why = f"{type(exc).__name__}: {exc}" + per_task.append(TaskResult( + task_id=task_id, passed=passed, + detail={"evaluator": evaluator, "reason": why}, + )) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "function"), + fixtures_path, + per_task, + {"function_pass_rate": rate}, + ) + + +# --------------------------------------------------------------------- +# Finetuning — does the organism improve parent→child? +# --------------------------------------------------------------------- + + +def run_finetuning(fixtures_path: Path) -> BatteryResult: + """Adaptation improvement check. + + Each task asserts that ``child_measured_value >= + parent_measured_value + expected_improvement_min``. Capital cost + deltas are recorded but not gated in v1; v8 selection will use + them (ticket #000012). + """ + per_task: list[TaskResult] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + try: + parent = float(task["parent_measured_value"]) + child = float(task["child_measured_value"]) + min_improvement = float(task.get("expected_improvement_min", 0.05)) + improvement = child - parent + improved = improvement >= min_improvement + expected = task.get("expected", "pass") + observed = "pass" if improved else "fail" + passed = observed == expected + detail = { + "parent_measured_value": parent, + "child_measured_value": child, + "improvement": improvement, + "min_improvement": min_improvement, + "expected": expected, + } + except Exception as exc: # noqa: BLE001 + passed = False + detail = {"reason": f"{type(exc).__name__}: {exc}"} + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "finetuning"), + fixtures_path, + per_task, + {"adaptation_improvement_rate": rate}, + ) + + +# --------------------------------------------------------------------- +# Falsification — does the organism detect and correct errors? +# --------------------------------------------------------------------- + + +def run_falsification(fixtures_path: Path) -> BatteryResult: + """Planted-error detection rate. + + Each fixture lists a planted-bad providence record's + ``observed_violations`` list and an ``expected_reason`` (the tag + the verifier should surface). Test passes if expected_reason + appears in observed_violations. + """ + per_task: list[TaskResult] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + try: + observed = set(task.get("observed_violations", [])) + expected_reason = task["expected_reason"] + expected_outcome = task.get("expected", "pass") + detected = expected_reason in observed + observed_outcome = "pass" if detected else "fail" + passed = observed_outcome == expected_outcome + detail = { + "expected_reason": expected_reason, + "observed_violations": sorted(observed), + "detected": detected, + "verifier_method_root": task.get("verifier_method_root"), + } + except Exception as exc: # noqa: BLE001 + passed = False + detail = {"reason": f"{type(exc).__name__}: {exc}"} + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "falsification"), + fixtures_path, + per_task, + {"error_detection_rate": rate}, + ) + + +# --------------------------------------------------------------------- +# Formulate — turn unstructured input into structured output. +# --------------------------------------------------------------------- + + +def run_formulate(fixtures_path: Path) -> BatteryResult: + """Structural match against expected claim lattice. + + Match policy: claim count + sorted-approx claim text + exact + pointer-ID-set. NOT exact-string. The fixture provides + ``produced_lattice`` (the system's output) and + ``expected_lattice`` (the gold). v1 fixtures embed produced + output; Phase 1b.2 will call arborist.qa.parse_claims. + """ + per_task: list[TaskResult] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + try: + produced = task["produced_lattice"] + expected = task["expected_lattice"] + count_ok = len(produced.get("claims", [])) == expected.get("claim_count") + order_ok = True + pointer_ok = True + text_ok = True + for i, exp in enumerate(expected.get("claims", [])): + if i >= len(produced.get("claims", [])): + count_ok = False + break + got = produced["claims"][i] + # Pointer-set exact match. + if set(got.get("pointer_ids", [])) != set(exp.get("pointer_ids", [])): + pointer_ok = False + # Sorted-approx claim text — content-token overlap ≥ 0.7. + got_t = _content_tokens(got.get("claim_text", "")) + exp_t = _content_tokens(exp.get("claim_text", "")) + if not exp_t or len(got_t & exp_t) / len(exp_t) < 0.7: + text_ok = False + passed_all = count_ok and order_ok and pointer_ok and text_ok + expected_outcome = task.get("expected", "pass") + observed = "pass" if passed_all else "fail" + passed = observed == expected_outcome + detail = { + "count_ok": count_ok, + "pointer_ok": pointer_ok, + "text_ok": text_ok, + } + except Exception as exc: # noqa: BLE001 + passed = False + detail = {"reason": f"{type(exc).__name__}: {exc}"} + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "formulate"), + fixtures_path, + per_task, + {"structural_match_rate": rate}, + ) + + +# --------------------------------------------------------------------- +# Feedback Loop — does observation at t affect state at t+1? +# --------------------------------------------------------------------- + + +def run_feedback_loop(fixtures_path: Path) -> BatteryResult: + """Integration coverage rate over operation/observation chains. + + Each task carries a chain of (operation, observation) pairs. The + last item lists ``expected_delta`` — content the post-chain + state must contain. Pass = expected_delta appears in the + final-step observation OR a post-step observation field. + """ + per_task: list[TaskResult] = [] + integrated = 0 + total_obs = 0 + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + try: + chain = task["chain"] + total_obs += len(chain) + # Aggregate everything observed across the chain. + observed_text = " ".join( + step.get("observation", "") + for step in chain + ).lower() + # Final-step expected_delta must appear in the aggregated + # observation feed OR in a downstream-state field. + final = chain[-1] + delta_marker = (final.get("expected_delta") or "").lower() + integrated_ok = bool(delta_marker) and delta_marker in observed_text + if integrated_ok: + integrated += 1 + expected = task.get("expected", "pass") + observed = "pass" if integrated_ok else "fail" + passed = observed == expected + detail = { + "integrated": integrated_ok, + "delta_marker": delta_marker, + } + except Exception as exc: # noqa: BLE001 + passed = False + detail = {"reason": f"{type(exc).__name__}: {exc}"} + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + integration_rate = integrated / total_obs if total_obs else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "feedback-loop"), + fixtures_path, + per_task, + { + "integration_coverage_rate": rate, + "downstream_effect_rate": integration_rate, + }, + ) + + +SUB_BATTERIES = { + "function": run_function, + "finetuning": run_finetuning, + "falsification": run_falsification, + "formulate": run_formulate, + "feedback-loop": run_feedback_loop, +} diff --git a/bench/batteries/b_5s.py b/bench/batteries/b_5s.py index fb5a2a1..cbe4a4b 100644 --- a/bench/batteries/b_5s.py +++ b/bench/batteries/b_5s.py @@ -1,9 +1,14 @@ -"""5S battery — Syntax / Semantics / Synthesis / Syllogism / Semiotics. +"""5S battery — Syntax / Semantics / Syllogism / Synthesis / Semiotics. -Phase 1a implements Syntax and Semantics against arborist's π* -registry. The other three sub-batteries are reserved as stubs that -return zero-task BatteryResults; full fixtures + runners land in -follow-up tickets. +Phase 1a (landed in ticket #000021): Syntax + Semantics over the +existing π* registry. + +Phase 1b (ticket #000023, this file): Syllogism, Synthesis, +Semiotics — all carrier-aware (text / claim_lattice / prose carriers +in v1; future-carrier hooks via fixture metadata that v1 rejects +explicitly). + +No LLM-as-judge anywhere. All runners deterministic. """ from __future__ import annotations @@ -16,6 +21,7 @@ from bench.batteries.base import ( fixture_digest, fixture_meta, iter_tasks, + validate_carrier, ) @@ -33,6 +39,19 @@ def _runtime_digest() -> str: return hashlib.sha256(payload).hexdigest() +def _carrier_check(task: dict) -> tuple[bool, str]: + """Return (ok, reason). ok=False means task fails immediately.""" + reason = validate_carrier(task) + if reason is not None: + return False, reason + return True, "" + + +# --------------------------------------------------------------------- +# Phase 1a: Syntax + Semantics (landed in ticket #000021) +# --------------------------------------------------------------------- + + def run_syntax(fixtures_path: Path) -> BatteryResult: """Each task asserts that the named π* canonicalizes the input without raising. expected="pass" is the only supported expected; @@ -44,7 +63,13 @@ def run_syntax(fixtures_path: Path) -> BatteryResult: fail_count = 0 for task in iter_tasks(fixtures_path): task_id = task["id"] - ok = False + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + fail_count += 1 + continue detail: dict = {} try: pi_star = get(task["pi_star"]) @@ -90,7 +115,13 @@ def run_semantics(fixtures_path: Path) -> BatteryResult: fail_count = 0 for task in iter_tasks(fixtures_path): task_id = task["id"] - ok = False + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + fail_count += 1 + continue detail: dict = {} try: pi_star = get(task["pi_star"]) @@ -128,36 +159,446 @@ def run_semantics(fixtures_path: Path) -> BatteryResult: ) -def _stub_result(name: str, fixtures_path: Path) -> BatteryResult: - return BatteryResult( - battery="5s", - sub_battery=name, - fixture_path=str(fixtures_path), - fixture_digest="", - pass_count=0, - fail_count=0, - metrics={}, - per_task=[], - runtime_digest="", - ) +# --------------------------------------------------------------------- +# Phase 1b: Syllogism (ticket #000023) +# --------------------------------------------------------------------- + +# Whitelist of syllogistic rules the deterministic runner can license. +# Each rule has a small kernel that returns (valid: bool, reason: str). +def _syllogism_rule_categorical_transitivity( + premises: list[dict], candidate: dict +) -> tuple[bool, str]: + """All A are B; All B are C ⇒ All A are C. + + Premise text shape: ``All are .`` (also accepts "is"). + Order-agnostic: premises may appear in either chain order so a + natural English ordering ("All mammals are animals; All dogs are + mammals") and the textbook ordering ("All dogs are mammals; All + mammals are animals") both validate. + """ + if len(premises) != 2: + return False, "categorical_transitivity needs exactly 2 premises" + p1 = _parse_categorical(premises[0]["claim"]) + p2 = _parse_categorical(premises[1]["claim"]) + c = _parse_categorical(candidate["claim"]) + if p1 is None or p2 is None or c is None: + return False, "claim does not match 'All X are Y.' pattern" + # Try both orderings so fixture authors can write premises in + # natural English order (general first) or textbook order + # (specific first). + for first, second in [(p1, p2), (p2, p1)]: + if first[1] == second[0]: + if c[0] == first[0] and c[1] == second[1]: + return True, "" + return False, "candidate endpoints do not match chain" + return False, f"premises do not chain: {p1} vs {p2}" -def run_synthesis(fixtures_path: Path) -> BatteryResult: - return _stub_result("synthesis", fixtures_path) +def _parse_categorical(text: str) -> tuple[str, str] | None: + """Parse 'All are .' or 'All is .' into (X, Y).""" + s = text.strip().rstrip(".") + lower = s.lower() + if not lower.startswith("all "): + return None + rest = s[4:] + for connector in (" are ", " is "): + parts = rest.split(connector, 1) + if len(parts) == 2: + return parts[0].strip(), parts[1].strip() + return None + + +def _syllogism_rule_invalid_converse( + premises: list[dict], candidate: dict +) -> tuple[bool, str]: + """Candidate is the converse of a premise. Must FAIL. + + A fixture tagged ``rule="invalid_converse"`` with ``expected="fail"`` + asserts that the runner correctly identifies an invalid step. + """ + if len(premises) < 1: + return False, "invalid_converse needs at least 1 premise" + c = _parse_categorical(candidate["claim"]) + if c is None: + return False, "candidate not a categorical claim" + for prem in premises: + p = _parse_categorical(prem["claim"]) + if p is None: + continue + if p[0] == c[1] and p[1] == c[0]: + # Converse detected — invalid step. + return False, "invalid converse of premise" + # Not actually a converse — the fixture is mis-tagged. + return False, "no converse pair detected; fixture mis-tagged" + + +def _syllogism_rule_missing_premise( + premises: list[dict], candidate: dict +) -> tuple[bool, str]: + """Candidate references evidence not present in premises. Must FAIL.""" + available_flat: set[str] = set() + for p in premises: + for i in p.get("evidence_ids", []): + available_flat.add(i) + used = set(candidate.get("uses", [])) + missing = used - available_flat + if missing: + return False, f"candidate references missing evidence ids: {sorted(missing)}" + return True, "all evidence present" + + +def _syllogism_rule_chain_3( + premises: list[dict], candidate: dict +) -> tuple[bool, str]: + """Three-step transitive chain: A→B, B→C, C→D ⇒ A→D. + + Order-agnostic over premise permutations. + """ + from itertools import permutations + + if len(premises) != 3: + return False, "chain_3 needs exactly 3 premises" + parsed = [_parse_categorical(p["claim"]) for p in premises] + c = _parse_categorical(candidate["claim"]) + if any(x is None for x in parsed) or c is None: + return False, "claim parse failure" + for ordering in permutations(parsed): + a, b, d = ordering + if a[1] == b[0] and b[1] == d[0]: + if c[0] == a[0] and c[1] == d[1]: + return True, "" + return False, "no valid chain ordering matches candidate endpoints" + + +_SYLLOGISM_RULES = { + "categorical_transitivity": _syllogism_rule_categorical_transitivity, + "invalid_converse": _syllogism_rule_invalid_converse, + "missing_premise": _syllogism_rule_missing_premise, + "chain_3": _syllogism_rule_chain_3, +} def run_syllogism(fixtures_path: Path) -> BatteryResult: - return _stub_result("syllogism", fixtures_path) + """Stepwise entailment validity. Deterministic, no LLM. + + Each task names a ``rule`` from :data:`_SYLLOGISM_RULES`. The + runner applies the rule's kernel; the task passes if the kernel's + verdict matches ``expected`` (``"pass"`` ⇒ valid step; + ``"fail"`` ⇒ invalid step correctly identified). + """ + per_task: list[TaskResult] = [] + pass_count = 0 + fail_count = 0 + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + fail_count += 1 + continue + rule = task.get("rule") + kernel = _SYLLOGISM_RULES.get(rule) + if kernel is None: + per_task.append( + TaskResult( + task_id=task_id, + passed=False, + detail={"reason": f"unknown_rule: {rule!r}"}, + ) + ) + fail_count += 1 + continue + expected = task.get("expected", "pass") + valid, why = kernel(task["premises"], task["candidate_step"]) + observed = "pass" if valid else "fail" + passed = observed == expected + detail = { + "rule": rule, + "expected": expected, + "observed": observed, + "kernel_reason": why, + } + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + if passed: + pass_count += 1 + else: + fail_count += 1 + + total = pass_count + fail_count + rate = pass_count / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return BatteryResult( + battery="5s", + sub_battery=meta.get("sub_battery", "syllogism"), + fixture_path=str(fixtures_path), + fixture_digest=fixture_digest(fixtures_path), + pass_count=pass_count, + fail_count=fail_count, + metrics={"step_validity_rate": rate}, + per_task=per_task, + runtime_digest=_runtime_digest(), + ) + + +# --------------------------------------------------------------------- +# Phase 1b: Synthesis (ticket #000023) +# --------------------------------------------------------------------- + + +def _synthesis_check(task: dict) -> tuple[bool, str]: + """Validate that the expected_derivation is supported by the fact_set. + + Each derivation step lists ``uses`` (fact IDs from the fact_set). + A derivation is valid iff every ``uses`` ID exists in the + fact_set AND each derivation claim's text appears as a substring + of at least one cited fact's text (lower-cased, whitespace- + normalized). This is structural support, not semantic entailment; + enough to drive Phase 1b without an LLM judge. + """ + facts = {f["id"]: f["text"] for f in task.get("fact_set", [])} + derivation = task.get("expected_derivation", []) + for step in derivation: + uses = step.get("uses", []) + for fid in uses: + if fid not in facts: + return False, f"derivation step uses unknown fact id {fid!r}" + claim_text = _normalize(step["claim"]) + any_support = False + for fid in uses: + fact_text = _normalize(facts[fid]) + if claim_text in fact_text or _token_subset(claim_text, fact_text): + any_support = True + break + if not any_support: + return False, f"step {step['claim']!r} not supported by cited facts" + return True, "all steps supported" + + +_STOPWORDS = frozenset({ + "a", "an", "the", "is", "are", "was", "were", "be", "been", "being", + "of", "on", "in", "to", "for", "with", "by", "and", "or", "as", + "that", "which", "who", "whom", "whose", "this", "these", "those", + "has", "have", "had", "do", "does", "did", "from", "at", "into", + "over", "under", "above", "below", "per", +}) + + +def _normalize(s: str) -> str: + return " ".join(s.lower().strip().split()) + + +def _content_tokens(s: str) -> set[str]: + """Lower-cased tokens with stopwords removed, punctuation stripped.""" + out = set() + for tok in s.lower().split(): + clean = tok.strip(".,;:!?\"'()[]{}") + if clean and clean not in _STOPWORDS: + out.add(clean) + return out + + +def _token_subset(a: str, b: str) -> bool: + """True if every content token in a appears in b. + + Stricter than the previous overlap-percentage check — catches + single-token entity swaps (e.g., 'K2' vs 'Mount Everest'). + """ + a_tokens = _content_tokens(a) + b_tokens = _content_tokens(b) + if not a_tokens: + return False + return a_tokens.issubset(b_tokens) + + +def run_synthesis(fixtures_path: Path) -> BatteryResult: + """Goal + fact_set + expected_derivation → pass/fail. + + Pure structural check, no LLM, no retrieval. Phase 1b stays + deterministic. + """ + per_task: list[TaskResult] = [] + pass_count = 0 + fail_count = 0 + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + fail_count += 1 + continue + expected = task.get("expected", "pass") + valid, why = _synthesis_check(task) + observed = "pass" if valid else "fail" + passed = observed == expected + detail = { + "expected": expected, + "observed": observed, + "reason": why, + } + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + if passed: + pass_count += 1 + else: + fail_count += 1 + + total = pass_count + fail_count + rate = pass_count / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return BatteryResult( + battery="5s", + sub_battery=meta.get("sub_battery", "synthesis"), + fixture_path=str(fixtures_path), + fixture_digest=fixture_digest(fixtures_path), + pass_count=pass_count, + fail_count=fail_count, + metrics={"derivation_pass_rate": rate}, + per_task=per_task, + runtime_digest=_runtime_digest(), + ) + + +# --------------------------------------------------------------------- +# Phase 1b: Semiotics (ticket #000023) +# --------------------------------------------------------------------- + + +def _semiotic_swap_check(task: dict) -> tuple[bool, str]: + """Apply ``swap.from`` → ``swap.to`` in source_text; canonicalize + both via the named π*; compare. + + For a synonym swap with ``expected_invariant=True``: the canonical + forms should differ ONLY in the swapped token (the rest is + identical). For a non-equivalent swap with + ``expected_invariant=False``: canonical forms should differ + semantically (we approximate this as: the swapped strings remain + different after canonicalization, AND the invariant_fields would + not all match — Phase 1b only checks the surface canonicalization). + """ + from arborist.pi_star import get + + pi_star = get(task["pi_star_ref"]) + src = task["source_text"].encode("utf-8") + swap = task["swap"] + swapped = task["source_text"].replace(swap["from"], swap["to"]).encode("utf-8") + canon_src = pi_star.canonicalize(src) + canon_swapped = pi_star.canonicalize(swapped) + + if task["expected_invariant"]: + # For Phase 1b: a synonym swap should yield a canonical-form + # change limited to the swap. We assert the two forms differ + # by exactly the swap (the canonicalizer should preserve the + # rest of the prose). The structural check is: every word in + # canon_src that isn't in `swap.from` survives in canon_swapped. + return _swap_preserves_invariant( + canon_src, canon_swapped, swap["from"], swap["to"] + ) + # expected_invariant=False: a non-synonym swap should produce + # canonical forms that differ in ways NOT explainable solely by + # the swap. The simplest deterministic check: swap was applied + # and canonical forms are not byte-equal (they wouldn't be — the + # swap text is different). We assert the swap WAS detected as + # non-trivial — i.e., the canonical forms DIFFER even after + # accounting for the swap text. A correct non-equivalent fixture + # has distinct canonical "essence" beyond the surface swap. + return _swap_changes_essence( + canon_src, canon_swapped, swap["from"], swap["to"] + ) + + +def _swap_preserves_invariant( + canon_src: bytes, canon_swapped: bytes, frm: str, to: str +) -> tuple[bool, str]: + """Does the swap produce only the expected substitution?""" + # Re-substitute in canonical form: replacing `to` with `frm` in + # canon_swapped should reproduce canon_src. + rebuilt = canon_swapped.decode("utf-8").replace(to, frm).encode("utf-8") + if rebuilt == canon_src: + return True, "synonym swap preserved invariant" + return False, "swap changed more than the substituted token" + + +def _swap_changes_essence( + canon_src: bytes, canon_swapped: bytes, frm: str, to: str +) -> tuple[bool, str]: + """Does the swap change more than just the surface token? + + Re-substituting `to` → `frm` should NOT yield canon_src for a + non-synonym swap (the meaning shift surfaces somewhere). Phase + 1b uses the cheap surface heuristic: forms differ even after + re-substitution. + """ + rebuilt = canon_swapped.decode("utf-8").replace(to, frm).encode("utf-8") + if rebuilt != canon_src: + return True, "non-equivalent swap detected" + return False, "swap was actually a synonym; fixture mis-tagged" def run_semiotics(fixtures_path: Path) -> BatteryResult: - return _stub_result("semiotics", fixtures_path) + """Invariance under controlled sign substitution. + + v1 fixtures use ``concept_relations.relation_kind='synonym'`` for + positives and explicit non-synonyms for negatives. Hidden-channel + / steganography fixtures are reserved for Phase 2 — defensive + detection only, never generation. + """ + per_task: list[TaskResult] = [] + pass_count = 0 + fail_count = 0 + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + fail_count += 1 + continue + try: + valid, why = _semiotic_swap_check(task) + expected = task["expected_invariant"] + observed = valid + passed = observed == expected + except Exception as exc: # noqa: BLE001 + valid = False + why = f"{type(exc).__name__}: {exc}" + passed = False + + detail = { + "expected_invariant": task["expected_invariant"], + "observed_invariant": valid, + "reason": why, + "relation_kind": task.get("swap", {}).get("relation_kind"), + } + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + if passed: + pass_count += 1 + else: + fail_count += 1 + + total = pass_count + fail_count + rate = pass_count / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return BatteryResult( + battery="5s", + sub_battery=meta.get("sub_battery", "semiotics"), + fixture_path=str(fixtures_path), + fixture_digest=fixture_digest(fixtures_path), + pass_count=pass_count, + fail_count=fail_count, + metrics={"invariance_under_swap": rate}, + per_task=per_task, + runtime_digest=_runtime_digest(), + ) SUB_BATTERIES = { "syntax": run_syntax, "semantics": run_semantics, - "synthesis": run_synthesis, "syllogism": run_syllogism, + "synthesis": run_synthesis, "semiotics": run_semiotics, } diff --git a/bench/batteries/b_5t.py b/bench/batteries/b_5t.py index 14c81d0..39afda0 100644 --- a/bench/batteries/b_5t.py +++ b/bench/batteries/b_5t.py @@ -1,12 +1,29 @@ -"""5T battery — Transfer / Triangulate / Timing / Transitivity / Truth. +"""5T battery — Transitivity / Transfer Learning / Triangulation / +Truthtables / Time. -Phase 1a implements Transfer (paraphrase-invariance under a π*). -The other four sub-batteries are reserved as stubs. +Phase 1a (landed in ticket #000021): Transfer (legacy SQD-whitepaper +name). Stays intact for back-compat. + +Phase 1b (ticket #000024, this file): +- Vocabulary aligned to Dav1DPrometheus: ``transfer-learning``, + ``triangulation``, ``truthtables``, ``transitivity``, ``time``. +- ``transfer-v1`` digest stays pinned; new ``transfer-learning-v2`` + is the canonical name going forward. +- Five non-stub runners over text / claim_lattice / propositional / + memory-root carriers. + +No LLM-as-judge anywhere. All runners deterministic. Cross-modality +discipline: fixtures may carry future-modality hooks; v1 carriers +limited to the Phase-1 whitelist in ``base.PHASE_1_CARRIERS``. """ from __future__ import annotations +import json +import re +from itertools import permutations from pathlib import Path +from typing import Iterable from bench.batteries.base import ( BatteryResult, @@ -14,21 +31,62 @@ from bench.batteries.base import ( fixture_digest, fixture_meta, iter_tasks, + validate_carrier, ) from bench.batteries.b_5s import _runtime_digest +def _carrier_check(task: dict) -> tuple[bool, str]: + reason = validate_carrier(task) + if reason is not None: + return False, reason + return True, "" + + +def _build_result( + sub_battery: str, + fixtures_path: Path, + per_task: list[TaskResult], + metrics: dict[str, float], +) -> BatteryResult: + pass_count = sum(1 for t in per_task if t.passed) + fail_count = len(per_task) - pass_count + return BatteryResult( + battery="5t", + sub_battery=sub_battery, + fixture_path=str(fixtures_path), + fixture_digest=fixture_digest(fixtures_path), + pass_count=pass_count, + fail_count=fail_count, + metrics=metrics, + per_task=per_task, + runtime_digest=_runtime_digest(), + ) + + +# --------------------------------------------------------------------- +# Phase 1a: Transfer (legacy) +# --------------------------------------------------------------------- + + def run_transfer(fixtures_path: Path) -> BatteryResult: - """Each task: π* + two surface variants → does canonicalize - collapse them as expected_equivalent says it should?""" + """Legacy SQD-whitepaper Transfer runner. + + Each task: π* + two surface variants → does canonicalize collapse + them as expected_equivalent says? Kept untouched for Phase 1a + digest stability. + """ from arborist.pi_star import get per_task: list[TaskResult] = [] - pass_count = 0 - fail_count = 0 for task in iter_tasks(fixtures_path): task_id = task["id"] - ok = False + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue detail: dict = {} try: pi_star = get(task["pi_star"]) @@ -44,62 +102,551 @@ def run_transfer(fixtures_path: Path) -> BatteryResult: except Exception as exc: # noqa: BLE001 detail["reason"] = f"{type(exc).__name__}: {exc}" ok = False - if ok: - pass_count += 1 - else: - fail_count += 1 per_task.append(TaskResult(task_id=task_id, passed=ok, detail=detail)) - total = pass_count + fail_count - rate = pass_count / total if total else 0.0 + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 meta = fixture_meta(fixtures_path) - return BatteryResult( - battery="5t", - sub_battery=meta.get("sub_battery", "transfer"), - fixture_path=str(fixtures_path), - fixture_digest=fixture_digest(fixtures_path), - pass_count=pass_count, - fail_count=fail_count, - metrics={"transfer_pass_rate": rate}, - per_task=per_task, - runtime_digest=_runtime_digest(), + return _build_result( + meta.get("sub_battery", "transfer"), + fixtures_path, + per_task, + {"transfer_pass_rate": rate}, ) -def _stub_result(name: str, fixtures_path: Path) -> BatteryResult: - return BatteryResult( - battery="5t", - sub_battery=name, - fixture_path=str(fixtures_path), - fixture_digest="", - pass_count=0, - fail_count=0, - metrics={}, - per_task=[], - runtime_digest="", +# --------------------------------------------------------------------- +# Phase 1b: Transfer Learning (Dav1DPrometheus canonical) +# --------------------------------------------------------------------- + + +def run_transfer_learning(fixtures_path: Path) -> BatteryResult: + """Pattern transfer across tasks/domains. + + Each task: a source-task pattern + a target-task pattern + an + expected_transfer flag. Runner asserts that the target task + can be expressed under the same canonical structure as the + source — for v1, this means the target's content tokens are a + superset of the source's content tokens at the *pattern* level + (not the example level). + """ + per_task: list[TaskResult] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + try: + src_pattern = task["source_task"]["pattern"] + tgt_pattern = task["target_task"]["pattern"] + expected = bool(task["target_task"].get("expected_transfer", True)) + structural_match = _patterns_share_structure(src_pattern, tgt_pattern) + passed = structural_match == expected + detail = { + "expected_transfer": expected, + "observed_transfer": structural_match, + } + except Exception as exc: # noqa: BLE001 + passed = False + detail = {"reason": f"{type(exc).__name__}: {exc}"} + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "transfer-learning"), + fixtures_path, + per_task, + {"transfer_learning_success_rate": rate}, ) -def run_triangulate(fixtures_path: Path) -> BatteryResult: - return _stub_result("triangulate", fixtures_path) +def _patterns_share_structure(a: str, b: str) -> bool: + """Two patterns share structure iff they share ≥ 50% of content + tokens AND have similar length (within 50% of each other).""" + from bench.batteries.b_5s import _content_tokens + + a_tokens = _content_tokens(a) + b_tokens = _content_tokens(b) + if not a_tokens or not b_tokens: + return False + overlap = len(a_tokens & b_tokens) / max(len(a_tokens), len(b_tokens)) + len_ratio = min(len(a), len(b)) / max(len(a), len(b)) + return overlap >= 0.5 and len_ratio >= 0.5 -def run_timing(fixtures_path: Path) -> BatteryResult: - return _stub_result("timing", fixtures_path) +# --------------------------------------------------------------------- +# Phase 1b: Triangulation +# --------------------------------------------------------------------- + +# Independent verifier strategies that agree (or don't) on whether a +# claim is supported by evidence text. Each is deterministic. + +def _strat_substring(claim: str, evidence: str) -> bool: + return claim.strip().lower() in evidence.lower() + + +def _strat_token_subset(claim: str, evidence: str) -> bool: + from bench.batteries.b_5s import _content_tokens + + a = _content_tokens(claim) + b = _content_tokens(evidence) + return bool(a) and a.issubset(b) + + +def _strat_token_overlap(claim: str, evidence: str, threshold: float = 0.6) -> bool: + from bench.batteries.b_5s import _content_tokens + + a = _content_tokens(claim) + b = _content_tokens(evidence) + if not a: + return False + return len(a & b) / len(a) >= threshold + + +def _strat_entity_match(claim: str, evidence: str) -> bool: + """Capitalized tokens (proper nouns) in claim must appear in evidence.""" + entities = { + tok.strip(".,;:!?") + for tok in claim.split() + if tok and tok[0].isupper() and tok.lower() not in {"a", "an", "the", "i"} + } + if not entities: + return False + e_low = evidence.lower() + return all(e.lower() in e_low for e in entities) + + +_TRIANGULATION_STRATEGIES = { + "substring": _strat_substring, + "token_subset": _strat_token_subset, + "token_overlap": _strat_token_overlap, + "entity_match": _strat_entity_match, +} + + +def run_triangulation(fixtures_path: Path) -> BatteryResult: + """Independent-strategy agreement on (claim, evidence) pairs. + + Each task names a list of strategies + an ``expected_agreement_min``. + Runner runs each strategy, computes pass-rate across them, and + accepts the task iff agreement rate ≥ threshold. + """ + per_task: list[TaskResult] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + strategies = task.get("strategies", list(_TRIANGULATION_STRATEGIES)) + claim = task["claim"] + evidence = task["evidence"] + results = {} + verdicts: list[bool] = [] + for s in strategies: + fn = _TRIANGULATION_STRATEGIES.get(s) + if fn is None: + continue + v = fn(claim, evidence) + results[s] = v + verdicts.append(v) + if not verdicts: + per_task.append( + TaskResult(task_id=task_id, passed=False, + detail={"reason": "no recognized strategies"}) + ) + continue + agreement = sum(verdicts) / len(verdicts) + threshold = float(task.get("expected_agreement_min", 0.75)) + expected = task.get("expected", "pass") + # expected="pass" → claim should triangulate; agreement>=threshold. + # expected="fail" → claim should NOT triangulate; agreement= threshold + else: + passed = agreement < threshold + per_task.append( + TaskResult( + task_id=task_id, + passed=passed, + detail={ + "strategy_results": results, + "agreement": agreement, + "threshold": threshold, + "expected": expected, + }, + ) + ) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "triangulation"), + fixtures_path, + per_task, + {"triangulation_agreement_rate": rate}, + ) + + +# --------------------------------------------------------------------- +# Phase 1b: Truthtables (N=2..4) +# --------------------------------------------------------------------- + + +def _eval_propositional(expr: str, env: dict[str, bool]) -> bool: + """Deterministic propositional evaluator for AND/OR/NOT/XOR. + + Recognizes tokens A-Z (variable), AND, OR, NOT, XOR, IMPL, IFF, + parentheses. No LLM. Operator precedence: NOT > AND > OR/XOR/IMPL/IFF + (left-to-right within same precedence). Parens override. + + Implementation: tokenize → recursive-descent parse → evaluate. + """ + tokens = _tokenize_propositional(expr) + pos = [0] + + def expect(t: str) -> None: + if pos[0] >= len(tokens) or tokens[pos[0]] != t: + raise ValueError(f"expected {t!r} at {pos[0]}") + pos[0] += 1 + + def peek() -> str | None: + if pos[0] >= len(tokens): + return None + return tokens[pos[0]] + + def parse_atom() -> bool: + t = peek() + if t is None: + raise ValueError("unexpected end of expression") + if t == "(": + pos[0] += 1 + v = parse_or() + expect(")") + return v + if t == "NOT": + pos[0] += 1 + return not parse_atom() + if t in ("TRUE", "FALSE"): + pos[0] += 1 + return t == "TRUE" + if len(t) == 1 and t.isupper() and t.isalpha(): + if t not in env: + raise ValueError(f"variable {t!r} not in env") + pos[0] += 1 + return env[t] + raise ValueError(f"unexpected token {t!r}") + + def parse_and() -> bool: + left = parse_atom() + while peek() == "AND": + pos[0] += 1 + right = parse_atom() + left = left and right + return left + + def parse_or() -> bool: + left = parse_and() + while peek() in ("OR", "XOR", "IMPL", "IFF"): + op = peek() + pos[0] += 1 + right = parse_and() + if op == "OR": + left = left or right + elif op == "XOR": + left = left != right + elif op == "IMPL": + left = (not left) or right + else: # IFF + left = left == right + return left + + val = parse_or() + if pos[0] != len(tokens): + raise ValueError(f"unconsumed tokens at {pos[0]}: {tokens[pos[0]:]!r}") + return val + + +def _tokenize_propositional(expr: str) -> list[str]: + """Split an expression into tokens, treating AND/OR/NOT/XOR/IMPL/IFF as keywords.""" + out: list[str] = [] + i = 0 + while i < len(expr): + c = expr[i] + if c.isspace(): + i += 1 + continue + if c in "()": + out.append(c) + i += 1 + continue + if c.isalpha(): + j = i + while j < len(expr) and expr[j].isalpha(): + j += 1 + tok = expr[i:j] + up = tok.upper() + if up in ("AND", "OR", "NOT", "XOR", "IMPL", "IFF", "TRUE", "FALSE"): + out.append(up) + elif len(tok) == 1: + out.append(tok.upper()) + else: + raise ValueError(f"unrecognized token: {tok!r}") + i = j + continue + raise ValueError(f"unexpected char: {c!r}") + return out + + +def run_truthtables(fixtures_path: Path) -> BatteryResult: + """Exhaustive propositional truth-table evaluation; N=2..4. + + Each task lists `variables` (≤4), an `expression`, and a `rows` + array enumerating expected outputs. Runner evaluates the + expression on every row and fails the task if any row mismatches. + """ + per_task: list[TaskResult] = [] + coverage_total = 0 + coverage_correct = 0 + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + variables = task["variables"] + if len(variables) > 4: + per_task.append( + TaskResult(task_id=task_id, passed=False, + detail={"reason": f"N={len(variables)} exceeds cap of 4"}) + ) + continue + rows = task["rows"] + coverage_total += len(rows) + all_correct = True + first_mismatch = None + for row in rows: + try: + got = _eval_propositional(task["expression"], row["inputs"]) + except Exception as exc: # noqa: BLE001 + all_correct = False + first_mismatch = { + "inputs": row["inputs"], + "error": f"{type(exc).__name__}: {exc}", + } + break + if got == bool(row["expected"]): + coverage_correct += 1 + else: + all_correct = False + if first_mismatch is None: + first_mismatch = { + "inputs": row["inputs"], + "expected": row["expected"], + "got": got, + } + detail: dict = {"row_count": len(rows)} + if first_mismatch is not None: + detail["first_mismatch"] = first_mismatch + per_task.append(TaskResult(task_id=task_id, passed=all_correct, detail=detail)) + + total = len(per_task) + pass_rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + row_acc = coverage_correct / coverage_total if coverage_total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "truthtables"), + fixtures_path, + per_task, + { + "truth_table_coverage_rate": pass_rate, + "row_accuracy_rate": row_acc, + }, + ) + + +# --------------------------------------------------------------------- +# Phase 1b: Transitivity (typed-relation whitelist) +# --------------------------------------------------------------------- + +# Per ticket #000024 §4.3: only relations on this whitelist license +# transitive closure. "related_to" / "causes" without domain +# constraints are NOT transitive. +_TRANSITIVE_RELATIONS = frozenset({ + "implies", + "subset_of", + "ancestor_of", + "before", + "less_than", +}) + + +def _walk_relation_path( + edges: list[dict], start: str, end: str, relation: str +) -> bool: + """BFS over edges of the given relation only. Returns True iff a + path from start to end exists in the relation-restricted subgraph. + """ + if relation not in _TRANSITIVE_RELATIONS: + return False + adj: dict[str, set[str]] = {} + for e in edges: + if e.get("relation") != relation: + continue + adj.setdefault(e["from"], set()).add(e["to"]) + if start == end: + return True + visited = {start} + queue = [start] + while queue: + node = queue.pop(0) + for nxt in adj.get(node, ()): + if nxt == end: + return True + if nxt not in visited: + visited.add(nxt) + queue.append(nxt) + return False def run_transitivity(fixtures_path: Path) -> BatteryResult: - return _stub_result("transitivity", fixtures_path) + """Typed-relation chain pass-rate. + + Each task: typed edges + a query (from, to, relation). Runner + walks the edge subgraph for that relation and asserts whether a + path exists. Mixed-relation chains never pass — runner only + considers edges of the queried relation. Non-transitive + relations always fail by construction. + """ + per_task: list[TaskResult] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + try: + q = task["query"] + edges = task["edges"] + path_exists = _walk_relation_path(edges, q["from"], q["to"], q["relation"]) + observed = "pass" if path_exists else "fail" + expected = task.get("expected", "pass") + passed = observed == expected + detail = { + "expected": expected, + "observed": observed, + "relation": q["relation"], + "transitive": q["relation"] in _TRANSITIVE_RELATIONS, + } + except Exception as exc: # noqa: BLE001 + passed = False + detail = {"reason": f"{type(exc).__name__}: {exc}"} + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + + total = len(per_task) + rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "transitivity"), + fixtures_path, + per_task, + {"full_chain_pass_rate": rate}, + ) -def run_truth(fixtures_path: Path) -> BatteryResult: - return _stub_result("truth", fixtures_path) +# --------------------------------------------------------------------- +# Phase 1b: Time (memory-root snapshots) +# --------------------------------------------------------------------- +def run_time(fixtures_path: Path) -> BatteryResult: + """Temporal-context preservation across snapshots. + + Each task: an ordered list of snapshots (each with a memory_root, + facts, optional stale-marker) + an `expected` block stating which + facts should preserve, which should mark stale, and what the + final root is. Runner deterministically validates these + properties from the fixture itself — no shard reads required for + Phase 1b. Real shard-driven Time runs ship in Phase 1b.2 once + arborist's memory_records is populated by a real workload. + """ + per_task: list[TaskResult] = [] + for task in iter_tasks(fixtures_path): + task_id = task["id"] + ok, reason = _carrier_check(task) + if not ok: + per_task.append( + TaskResult(task_id=task_id, passed=False, detail={"reason": reason}) + ) + continue + try: + snapshots = task["snapshots"] + expected = task["expected"] + # All facts seen across the whole chain. + all_facts: set[str] = set() + for s in snapshots: + all_facts.update(s.get("facts", [])) + all_facts.update(s.get("stale_facts", [])) + # Preservation check: every fact named in expected.preserved_facts + # must appear in some snapshot. + preserved_ok = all( + f in all_facts + for f in expected.get("preserved_facts", []) + ) + # Stale check: every fact in expected.marks_stale must appear + # at least once and the FINAL snapshot's stale_facts must + # include it. + final_stale = set(snapshots[-1].get("stale_facts", [])) + stale_ok = all( + f in final_stale + for f in expected.get("marks_stale", []) + ) + # Current root check. + current_root_ok = ( + snapshots[-1].get("memory_root") == expected["current_root"] + ) + passed = preserved_ok and stale_ok and current_root_ok + detail = { + "preserved_ok": preserved_ok, + "stale_ok": stale_ok, + "current_root_ok": current_root_ok, + } + except Exception as exc: # noqa: BLE001 + passed = False + detail = {"reason": f"{type(exc).__name__}: {exc}"} + per_task.append(TaskResult(task_id=task_id, passed=passed, detail=detail)) + + total = len(per_task) + pres_rate = sum(1 for t in per_task if t.passed) / total if total else 0.0 + meta = fixture_meta(fixtures_path) + return _build_result( + meta.get("sub_battery", "time"), + fixtures_path, + per_task, + {"temporal_context_preservation_rate": pres_rate}, + ) + + +# Triangulate / Timing alias names from Phase 1a stubs are removed — +# canonical Dav1DPrometheus names below. + SUB_BATTERIES = { - "transfer": run_transfer, - "triangulate": run_triangulate, - "timing": run_timing, + "transfer": run_transfer, # legacy SQD name (Phase 1a) + "transfer-learning": run_transfer_learning, # canonical Dav1DPrometheus + "triangulation": run_triangulation, + "truthtables": run_truthtables, "transitivity": run_transitivity, - "truth": run_truth, + "time": run_time, } diff --git a/bench/batteries/base.py b/bench/batteries/base.py index 1ab6579..69ea2b2 100644 --- a/bench/batteries/base.py +++ b/bench/batteries/base.py @@ -90,3 +90,45 @@ def fixture_meta(fixtures_path: Path) -> dict: return obj["_meta"] return {} return {} + + +# Phase-1 carrier whitelist (per tickets #000023/#000024/#000025). +# Tasks lacking ``carrier`` default to "text" for backward compat with +# Phase-1a fixtures (syntax-v1, semantics-v1, transfer-v1) — those +# digests stay pinned and the carrier field is purely additive. +PHASE_1_CARRIERS = frozenset({ + "text", + "claim_lattice", + "prose", + "memory_snapshot", + "selfmodel_snapshot", + "providence_record", + "audit_event", + "verifier_strategies", + "propositional_logic", + "relation_graph", + "qa_answer", + "capability_transition", + "memory_root", +}) + + +def validate_carrier(task: dict) -> Optional[str]: + """Validate the task's ``carrier`` field against the Phase-1 whitelist. + + Returns ``None`` if the carrier is allowed (or absent — defaults + to ``"text"`` for backward compat). Returns a reason string if + the carrier is set to something Phase-1 doesn't support, so the + caller can mark the task ``failed`` with + ``reason="unsupported_carrier"`` per the cross-modality + discipline. + """ + carrier = task.get("carrier") + if carrier is None: + # Phase-1a fixtures have no carrier field; default to "text". + return None + if not isinstance(carrier, str): + return f"unsupported_carrier: non-string carrier {carrier!r}" + if carrier in PHASE_1_CARRIERS: + return None + return f"unsupported_carrier: {carrier!r} not in Phase-1 whitelist" diff --git a/bench/batteries/runner.py b/bench/batteries/runner.py index be2af7f..8076dad 100644 --- a/bench/batteries/runner.py +++ b/bench/batteries/runner.py @@ -34,19 +34,38 @@ from dataclasses import asdict from pathlib import Path from typing import Optional -from bench.batteries import b_5s, b_5t +from bench.batteries import b_5f, b_5s, b_5t from bench.batteries.base import BatteryResult _BATTERIES = { "5s": b_5s.SUB_BATTERIES, "5t": b_5t.SUB_BATTERIES, + "5f": b_5f.SUB_BATTERIES, } _DEFAULT_FIXTURES = { + # 5S — Phase 1a ("5s", "syntax"): "bench/fixtures/5s/syntax-v1.jsonl", ("5s", "semantics"): "bench/fixtures/5s/semantics-v1.jsonl", + # 5S — Phase 1b (ticket #000023) + ("5s", "syllogism"): "bench/fixtures/5s/syllogism-v1.jsonl", + ("5s", "synthesis"): "bench/fixtures/5s/synthesis-v1.jsonl", + ("5s", "semiotics"): "bench/fixtures/5s/semiotics-v1.jsonl", + # 5T — Phase 1a (legacy SQD vocabulary) ("5t", "transfer"): "bench/fixtures/5t/transfer-v1.jsonl", + # 5T — Phase 1b (Dav1DPrometheus vocabulary, ticket #000024) + ("5t", "transfer-learning"): "bench/fixtures/5t/transfer-learning-v2.jsonl", + ("5t", "triangulation"): "bench/fixtures/5t/triangulation-v1.jsonl", + ("5t", "truthtables"): "bench/fixtures/5t/truthtables-v1.jsonl", + ("5t", "transitivity"): "bench/fixtures/5t/transitivity-v1.jsonl", + ("5t", "time"): "bench/fixtures/5t/time-v1.jsonl", + # 5F — Phase 1a (ticket #000025) + ("5f", "function"): "bench/fixtures/5f/function-v1.jsonl", + ("5f", "finetuning"): "bench/fixtures/5f/finetuning-v1.jsonl", + ("5f", "falsification"): "bench/fixtures/5f/falsification-v1.jsonl", + ("5f", "formulate"): "bench/fixtures/5f/formulate-v1.jsonl", + ("5f", "feedback-loop"): "bench/fixtures/5f/feedback-loop-v1.jsonl", } diff --git a/bench/fixtures/5f/falsification-v1.jsonl b/bench/fixtures/5f/falsification-v1.jsonl new file mode 100644 index 0000000..20e9a69 --- /dev/null +++ b/bench/fixtures/5f/falsification-v1.jsonl @@ -0,0 +1,11 @@ +{"_meta":{"battery":"5f","sub_battery":"falsification","version":"v1","task_count":10,"notes":"Planted-bad-record fixtures with expected violation tags. Each fixture pinned to verifier_method_root so verifier-shape changes warn rather than false-fail."}} +{"id":"5f-fal-001","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_001","observed_violations":["WARRANT_MISSING","TITLE_MISMATCH"],"expected_reason":"WARRANT_MISSING","verifier_method_root":"warrant-v1","expected":"pass"} +{"id":"5f-fal-002","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_002","observed_violations":["TITLE_MISMATCH"],"expected_reason":"TITLE_MISMATCH","verifier_method_root":"warrant-v1","expected":"pass"} +{"id":"5f-fal-003","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_003","observed_violations":["TOO_MANY_CLAIMS"],"expected_reason":"TOO_MANY_CLAIMS","verifier_method_root":"warrant-v1","expected":"pass"} +{"id":"5f-fal-004","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_004","observed_violations":["BARE_NAME_CLAIM"],"expected_reason":"BARE_NAME_CLAIM","verifier_method_root":"warrant-v1","expected":"pass"} +{"id":"5f-fal-005","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_005","observed_violations":["LAZY_ANCHOR_DEMOTED","TITLE_MISMATCH"],"expected_reason":"LAZY_ANCHOR_DEMOTED","verifier_method_root":"warrant-v1","expected":"pass"} +{"id":"5f-fal-006","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_006","observed_violations":["POINTER_OVERFLOW_TRIMMED"],"expected_reason":"POINTER_OVERFLOW_TRIMMED","verifier_method_root":"warrant-v1","expected":"pass"} +{"id":"5f-fal-007","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_007","observed_violations":[],"expected_reason":"WARRANT_MISSING","verifier_method_root":"warrant-v1","expected":"fail"} +{"id":"5f-fal-008","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_008","observed_violations":["UNRELATED_TAG"],"expected_reason":"WARRANT_MISSING","verifier_method_root":"warrant-v1","expected":"fail"} +{"id":"5f-fal-009","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_009","observed_violations":["FORMAT_COLLAPSED","WARRANT_MISSING"],"expected_reason":"FORMAT_COLLAPSED","verifier_method_root":"warrant-v1","expected":"pass"} +{"id":"5f-fal-010","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_010","observed_violations":["NO_EVIDENCE_POINTER"],"expected_reason":"NO_EVIDENCE_POINTER","verifier_method_root":"warrant-v1","expected":"pass"} diff --git a/bench/fixtures/5f/feedback-loop-v1.jsonl b/bench/fixtures/5f/feedback-loop-v1.jsonl new file mode 100644 index 0000000..2d95ecc --- /dev/null +++ b/bench/fixtures/5f/feedback-loop-v1.jsonl @@ -0,0 +1,11 @@ +{"_meta":{"battery":"5f","sub_battery":"feedback-loop","version":"v1","task_count":10,"notes":"Operation/observation chains. Final-step expected_delta must appear in aggregated observation feed; tests integration of observations into downstream state."}} +{"id":"5f-fb-001","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"snapshot","observation":"claim X is stale"},{"operation":"update_memory","observation":"memory branch marks X stale"},{"operation":"snapshot","observation":"X appears in stale set","expected_delta":"X appears in stale set"}],"expected":"pass"} +{"id":"5f-fb-002","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"new fact alpha learned"},{"operation":"update_memory","observation":"memory branch records alpha"},{"operation":"snapshot","observation":"alpha appears in current memory","expected_delta":"alpha appears in current memory"}],"expected":"pass"} +{"id":"5f-fb-003","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"verifier flagged warrant_missing"},{"operation":"falsify","observation":"providence record marked stale"},{"operation":"snapshot","observation":"stale record appears in falsification log","expected_delta":"stale record appears in falsification log"}],"expected":"pass"} +{"id":"5f-fb-004","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"snapshot","observation":"baseline observed"},{"operation":"observe","observation":"motif title_mismatch occurred"},{"operation":"update_memory","observation":"motif counter incremented","expected_delta":"motif counter incremented"}],"expected":"pass"} +{"id":"5f-fb-005","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"capability claim falsified"},{"operation":"selfmodel_update","observation":"selfmodel state flipped to falsified"},{"operation":"snapshot","observation":"falsified state recorded in selfmodel_records","expected_delta":"falsified state recorded in selfmodel_records"}],"expected":"pass"} +{"id":"5f-fb-006","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"stale data point"},{"operation":"no_action","observation":"nothing changed"},{"operation":"snapshot","observation":"unrelated content","expected_delta":"data point appears in stale set"}],"expected":"fail"} +{"id":"5f-fb-007","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"snapshot","observation":"empty"},{"operation":"observe","observation":"event delta_event happened"},{"operation":"snapshot","observation":"delta_event recorded","expected_delta":"delta_event recorded"}],"expected":"pass"} +{"id":"5f-fb-008","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"correction applied to claim Q1"},{"operation":"update_memory","observation":"correction logged"},{"operation":"snapshot","observation":"corrected Q1 visible","expected_delta":"corrected Q1 visible"}],"expected":"pass"} +{"id":"5f-fb-009","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"silent failure unobserved"},{"operation":"snapshot","observation":"chain produced no signal","expected_delta":"silent failure unobserved"}],"expected":"pass"} +{"id":"5f-fb-010","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"ingest succeeded"},{"operation":"update_memory","observation":"ingest event in audit chain"},{"operation":"snapshot","observation":"ingest visible at high water mark","expected_delta":"ingest visible at high water mark"}],"expected":"pass"} diff --git a/bench/fixtures/5f/finetuning-v1.jsonl b/bench/fixtures/5f/finetuning-v1.jsonl new file mode 100644 index 0000000..93e0000 --- /dev/null +++ b/bench/fixtures/5f/finetuning-v1.jsonl @@ -0,0 +1,11 @@ +{"_meta":{"battery":"5f","sub_battery":"finetuning","version":"v1","task_count":10,"notes":"Synthetic parent→child SelfModel measured-value pairs. Phase 1b.2 will pull from real shard SelfModel chains."}} +{"id":"5f-ft-001","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_001","child_selfmodel":"SM_CHILD_001","target_capability":"CAP-5S-SYLLOGISM","parent_measured_value":0.45,"child_measured_value":0.65,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1000,"max_storage_delta_bytes":1000000},"expected":"pass"} +{"id":"5f-ft-002","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_002","child_selfmodel":"SM_CHILD_002","target_capability":"CAP-5T-TIME","parent_measured_value":0.40,"child_measured_value":0.55,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1500,"max_storage_delta_bytes":2000000},"expected":"pass"} +{"id":"5f-ft-003","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_003","child_selfmodel":"SM_CHILD_003","target_capability":"CAP-5F-FORMULATE","parent_measured_value":0.60,"child_measured_value":0.85,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":2000,"max_storage_delta_bytes":3000000},"expected":"pass"} +{"id":"5f-ft-004","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_004","child_selfmodel":"SM_CHILD_004","target_capability":"CAP-5T-TRUTHTABLES","parent_measured_value":0.50,"child_measured_value":0.50,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":500,"max_storage_delta_bytes":500000},"expected":"fail"} +{"id":"5f-ft-005","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_005","child_selfmodel":"SM_CHILD_005","target_capability":"CAP-5F-FUNCTION","parent_measured_value":0.70,"child_measured_value":0.65,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1000,"max_storage_delta_bytes":1000000},"expected":"fail"} +{"id":"5f-ft-006","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_006","child_selfmodel":"SM_CHILD_006","target_capability":"CAP-5S-SYNTHESIS","parent_measured_value":0.30,"child_measured_value":0.45,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":1200,"max_storage_delta_bytes":1500000},"expected":"pass"} +{"id":"5f-ft-007","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_007","child_selfmodel":"SM_CHILD_007","target_capability":"CAP-5T-TRIANGULATION","parent_measured_value":0.65,"child_measured_value":0.78,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":800,"max_storage_delta_bytes":800000},"expected":"pass"} +{"id":"5f-ft-008","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_008","child_selfmodel":"SM_CHILD_008","target_capability":"CAP-5F-FALSIFICATION","parent_measured_value":0.55,"child_measured_value":0.62,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":1000,"max_storage_delta_bytes":1000000},"expected":"fail"} +{"id":"5f-ft-009","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_009","child_selfmodel":"SM_CHILD_009","target_capability":"CAP-5S-SEMIOTICS","parent_measured_value":0.35,"child_measured_value":0.55,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1300,"max_storage_delta_bytes":1500000},"expected":"pass"} +{"id":"5f-ft-010","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_010","child_selfmodel":"SM_CHILD_010","target_capability":"CAP-5F-FEEDBACK","parent_measured_value":0.50,"child_measured_value":0.72,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":1500,"max_storage_delta_bytes":2000000},"expected":"pass"} diff --git a/bench/fixtures/5f/formulate-v1.jsonl b/bench/fixtures/5f/formulate-v1.jsonl new file mode 100644 index 0000000..43c674c --- /dev/null +++ b/bench/fixtures/5f/formulate-v1.jsonl @@ -0,0 +1,11 @@ +{"_meta":{"battery":"5f","sub_battery":"formulate","version":"v1","task_count":10,"notes":"Phase 1a fixtures embed produced + expected lattices for structural matching. Phase 1b.2 will route through arborist.qa.parse_claims."}} +{"id":"5f-form-001","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"The system saved one claim with one pointer.","produced_lattice":{"claims":[{"claim_text":"The system saved one claim with one pointer.","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"The system saved one claim with one pointer.","pointer_ids":["E1"]}]},"expected":"pass"} +{"id":"5f-form-002","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Two claims with two pointers each.","produced_lattice":{"claims":[{"claim_text":"first claim with pointers","pointer_ids":["E1","E2"]},{"claim_text":"second claim with pointers","pointer_ids":["E3","E4"]}]},"expected_lattice":{"claim_count":2,"claims":[{"claim_text":"first claim with pointers","pointer_ids":["E1","E2"]},{"claim_text":"second claim with pointers","pointer_ids":["E3","E4"]}]},"expected":"pass"} +{"id":"5f-form-003","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Detect missing pointer claim.","produced_lattice":{"claims":[{"claim_text":"a claim","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"a claim","pointer_ids":["E1","E2"]}]},"expected":"fail"} +{"id":"5f-form-004","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Detect wrong claim count.","produced_lattice":{"claims":[{"claim_text":"a","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":2,"claims":[{"claim_text":"a","pointer_ids":["E1"]},{"claim_text":"b","pointer_ids":["E2"]}]},"expected":"fail"} +{"id":"5f-form-005","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Three sequential claims preserved.","produced_lattice":{"claims":[{"claim_text":"alpha beta gamma","pointer_ids":["E1"]},{"claim_text":"delta epsilon zeta","pointer_ids":["E2"]},{"claim_text":"eta theta iota","pointer_ids":["E3"]}]},"expected_lattice":{"claim_count":3,"claims":[{"claim_text":"alpha beta gamma","pointer_ids":["E1"]},{"claim_text":"delta epsilon zeta","pointer_ids":["E2"]},{"claim_text":"eta theta iota","pointer_ids":["E3"]}]},"expected":"pass"} +{"id":"5f-form-006","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Sorted-approx text match works under reordered tokens.","produced_lattice":{"claims":[{"claim_text":"world hello there","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"hello world there","pointer_ids":["E1"]}]},"expected":"pass"} +{"id":"5f-form-007","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Detect text drift below 70% overlap.","produced_lattice":{"claims":[{"claim_text":"completely different sentence","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"the original gold reference text","pointer_ids":["E1"]}]},"expected":"fail"} +{"id":"5f-form-008","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Two claims with shared pointer.","produced_lattice":{"claims":[{"claim_text":"first observation","pointer_ids":["E1"]},{"claim_text":"second observation","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":2,"claims":[{"claim_text":"first observation","pointer_ids":["E1"]},{"claim_text":"second observation","pointer_ids":["E1"]}]},"expected":"pass"} +{"id":"5f-form-009","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Detect pointer-id mismatch between produced and expected.","produced_lattice":{"claims":[{"claim_text":"sentence text","pointer_ids":["E5"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"sentence text","pointer_ids":["E1"]}]},"expected":"fail"} +{"id":"5f-form-010","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Multi-pointer claim with full pointer set match.","produced_lattice":{"claims":[{"claim_text":"complex multi-source claim","pointer_ids":["E1","E2","E3"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"complex multi-source claim","pointer_ids":["E1","E2","E3"]}]},"expected":"pass"} diff --git a/bench/fixtures/5f/function-v1.jsonl b/bench/fixtures/5f/function-v1.jsonl new file mode 100644 index 0000000..61fc1f8 --- /dev/null +++ b/bench/fixtures/5f/function-v1.jsonl @@ -0,0 +1,11 @@ +{"_meta":{"battery":"5f","sub_battery":"function","version":"v1","task_count":10,"notes":"Phase 1a: ten seed tasks across shape_match/pointer_set_match/threshold_on_metric evaluators. Phase 1b expands to 30+."}} +{"id":"5f-fn-001","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Return exactly two claim-lattice claims with evidence pointers.","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"first claim","pointer_ids":["E1"]},{"claim_text":"second claim","pointer_ids":["E2"]}]},"expected":"pass"} +{"id":"5f-fn-002","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Return exactly three claims with pointers.","evaluator":"shape_match","expected_shape":{"claim_count":3,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"a","pointer_ids":["E1"]},{"claim_text":"b","pointer_ids":["E2"]},{"claim_text":"c","pointer_ids":["E3"]}]},"expected":"pass"} +{"id":"5f-fn-003","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Detect missing pointers in claims.","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"a","pointer_ids":[]},{"claim_text":"b","pointer_ids":["E2"]}]},"expected":"fail"} +{"id":"5f-fn-004","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Detect wrong claim count.","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"a","pointer_ids":["E1"]}]},"expected":"fail"} +{"id":"5f-fn-005","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Verify pointer set matches.","evaluator":"pointer_set_match","expected_shape":{"pointer_set":["E1","E2","E3"]},"produced_output":{"claims":[{"claim_text":"x","pointer_ids":["E1","E2"]},{"claim_text":"y","pointer_ids":["E3"]}]},"expected":"pass"} +{"id":"5f-fn-006","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Detect pointer-set mismatch.","evaluator":"pointer_set_match","expected_shape":{"pointer_set":["E1","E2"]},"produced_output":{"claims":[{"claim_text":"x","pointer_ids":["E1","E5"]}]},"expected":"fail"} +{"id":"5f-fn-007","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Verify directive coverage threshold.","evaluator":"threshold_on_metric","expected_shape":{"metric":"directive_coverage","threshold":0.8},"produced_output":{"directive_coverage":0.95},"expected":"pass"} +{"id":"5f-fn-008","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Verify strict-rate threshold.","evaluator":"threshold_on_metric","expected_shape":{"metric":"strict_rate","threshold":0.5},"produced_output":{"strict_rate":0.62},"expected":"pass"} +{"id":"5f-fn-009","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Detect failed metric threshold.","evaluator":"threshold_on_metric","expected_shape":{"metric":"strict_rate","threshold":0.5},"produced_output":{"strict_rate":0.31},"expected":"fail"} +{"id":"5f-fn-010","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Verify pointer set with single pointer.","evaluator":"pointer_set_match","expected_shape":{"pointer_set":["E1"]},"produced_output":{"claims":[{"claim_text":"single","pointer_ids":["E1"]}]},"expected":"pass"} diff --git a/bench/fixtures/5s/semiotics-v1.jsonl b/bench/fixtures/5s/semiotics-v1.jsonl new file mode 100644 index 0000000..e9ee993 --- /dev/null +++ b/bench/fixtures/5s/semiotics-v1.jsonl @@ -0,0 +1,31 @@ +{"_meta":{"battery":"5s","sub_battery":"semiotics","version":"v1","task_count":30,"notes":"Synonym-swap invariance under wikitext-base@v1 canonicalization. Synonym swaps (relation_kind=synonym) preserve invariant; non-equivalent swaps must fail. No hidden-channel generation; defensive detection reserved for Phase 2."}} +{"id":"5s-sem-001","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"AMD released a new GPU.","swap":{"from":"AMD","to":"AMD","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-002","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"NYC has eight million residents.","swap":{"from":"NYC","to":"NYC","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-003","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"USA was founded in 1776.","swap":{"from":"USA","to":"USA","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-004","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"NASA launched a probe.","swap":{"from":"NASA","to":"NASA","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-005","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"UK joined the United Nations.","swap":{"from":"UK","to":"UK","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-006","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"EU members coordinate trade policy.","swap":{"from":"EU","to":"EU","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-007","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"DNA is a molecule.","swap":{"from":"DNA","to":"DNA","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-008","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"GPU drivers were updated.","swap":{"from":"GPU","to":"GPU","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-009","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"CPU clock speed increased.","swap":{"from":"CPU","to":"CPU","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-010","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"FBI agents arrested a suspect.","swap":{"from":"FBI","to":"FBI","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-011","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"NYC traffic was heavy yesterday.","swap":{"from":"NYC","to":"New York City","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-012","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"AMD released the chip.","swap":{"from":"AMD","to":"Advanced Micro Devices","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-013","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"USA signed the treaty.","swap":{"from":"USA","to":"United States of America","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-014","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"NASA published research.","swap":{"from":"NASA","to":"the National Aeronautics and Space Administration","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-015","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"UK negotiated trade deals.","swap":{"from":"UK","to":"United Kingdom","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-016","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"EU passed legislation.","swap":{"from":"EU","to":"European Union","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-017","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"FBI agents arrested a suspect today.","swap":{"from":"FBI","to":"Federal Bureau of Investigation","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-018","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"GPU drivers received an update.","swap":{"from":"GPU","to":"graphics processing unit","relation_kind":"synonym"},"invariant_fields":["entity","predicate","object"],"expected_invariant":true} +{"id":"5s-sem-019","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"AMD released a new GPU.","swap":{"from":"AMD","to":"Intel","relation_kind":"rivalry"},"invariant_fields":["entity"],"expected_invariant":false} +{"id":"5s-sem-020","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"NYC traffic was heavy.","swap":{"from":"NYC","to":"Tokyo","relation_kind":"non_equivalent"},"invariant_fields":["entity"],"expected_invariant":false} +{"id":"5s-sem-021","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"USA signed the treaty.","swap":{"from":"USA","to":"China","relation_kind":"non_equivalent"},"invariant_fields":["entity"],"expected_invariant":false} +{"id":"5s-sem-022","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"NASA published research.","swap":{"from":"NASA","to":"SpaceX","relation_kind":"non_equivalent"},"invariant_fields":["entity"],"expected_invariant":false} +{"id":"5s-sem-023","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"UK negotiated deals.","swap":{"from":"UK","to":"France","relation_kind":"non_equivalent"},"invariant_fields":["entity"],"expected_invariant":false} +{"id":"5s-sem-024","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"EU passed legislation.","swap":{"from":"EU","to":"ASEAN","relation_kind":"non_equivalent"},"invariant_fields":["entity"],"expected_invariant":false} +{"id":"5s-sem-025","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"DNA is a molecule.","swap":{"from":"DNA","to":"RNA","relation_kind":"non_equivalent"},"invariant_fields":["entity"],"expected_invariant":false} +{"id":"5s-sem-026","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"GPU drivers were updated.","swap":{"from":"GPU","to":"CPU","relation_kind":"non_equivalent"},"invariant_fields":["entity"],"expected_invariant":false} +{"id":"5s-sem-027","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"FBI agents arrested a suspect.","swap":{"from":"FBI","to":"CIA","relation_kind":"non_equivalent"},"invariant_fields":["entity"],"expected_invariant":false} +{"id":"5s-sem-028","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"CPU clock speed increased.","swap":{"from":"CPU","to":"RAM","relation_kind":"non_equivalent"},"invariant_fields":["entity"],"expected_invariant":false} +{"id":"5s-sem-029","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"Apple released a new phone.","swap":{"from":"Apple","to":"Banana","relation_kind":"non_equivalent"},"invariant_fields":["entity"],"expected_invariant":false} +{"id":"5s-sem-030","battery":"5s","sub_battery":"semiotics","version":"v1","carrier":"text","domain":"prose","pi_star_ref":"wikitext-base@v1","source_text":"Google launched a service.","swap":{"from":"Google","to":"Microsoft","relation_kind":"non_equivalent"},"invariant_fields":["entity"],"expected_invariant":false} diff --git a/bench/fixtures/5s/syllogism-v1.jsonl b/bench/fixtures/5s/syllogism-v1.jsonl new file mode 100644 index 0000000..2055e3e --- /dev/null +++ b/bench/fixtures/5s/syllogism-v1.jsonl @@ -0,0 +1,31 @@ +{"_meta":{"battery":"5s","sub_battery":"syllogism","version":"v1","task_count":30,"notes":"Deterministic categorical-syllogism fixtures. No LLM-as-judge. Carrier metadata included per ticket #000023 cross-modality discipline."}} +{"id":"5s-syl-001","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All mammals are animals.","evidence_ids":["E1"]},{"claim":"All dogs are mammals.","evidence_ids":["E2"]}],"candidate_step":{"claim":"All dogs are animals.","uses":["E1","E2"]},"rule":"categorical_transitivity","expected":"pass"} +{"id":"5s-syl-002","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All birds are animals.","evidence_ids":["E1"]},{"claim":"All sparrows are birds.","evidence_ids":["E2"]}],"candidate_step":{"claim":"All sparrows are animals.","uses":["E1","E2"]},"rule":"categorical_transitivity","expected":"pass"} +{"id":"5s-syl-003","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All squares are rectangles.","evidence_ids":["E1"]},{"claim":"All rectangles are quadrilaterals.","evidence_ids":["E2"]}],"candidate_step":{"claim":"All squares are quadrilaterals.","uses":["E1","E2"]},"rule":"categorical_transitivity","expected":"pass"} +{"id":"5s-syl-004","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All vehicles are machines.","evidence_ids":["E1"]},{"claim":"All cars are vehicles.","evidence_ids":["E2"]}],"candidate_step":{"claim":"All cars are machines.","uses":["E1","E2"]},"rule":"categorical_transitivity","expected":"pass"} +{"id":"5s-syl-005","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All planets are celestial bodies.","evidence_ids":["E1"]},{"claim":"All gas giants are planets.","evidence_ids":["E2"]}],"candidate_step":{"claim":"All gas giants are celestial bodies.","uses":["E1","E2"]},"rule":"categorical_transitivity","expected":"pass"} +{"id":"5s-syl-006","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All trees are plants.","evidence_ids":["E1"]},{"claim":"All oaks are trees.","evidence_ids":["E2"]}],"candidate_step":{"claim":"All oaks are plants.","uses":["E1","E2"]},"rule":"categorical_transitivity","expected":"pass"} +{"id":"5s-syl-007","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All instruments are tools.","evidence_ids":["E1"]},{"claim":"All hammers are instruments.","evidence_ids":["E2"]}],"candidate_step":{"claim":"All hammers are tools.","uses":["E1","E2"]},"rule":"categorical_transitivity","expected":"pass"} +{"id":"5s-syl-008","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All metals are conductors.","evidence_ids":["E1"]},{"claim":"All copper wires are metals.","evidence_ids":["E2"]}],"candidate_step":{"claim":"All copper wires are conductors.","uses":["E1","E2"]},"rule":"categorical_transitivity","expected":"pass"} +{"id":"5s-syl-009","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All elephants are mammals.","evidence_ids":["E1"]},{"claim":"All African elephants are elephants.","evidence_ids":["E2"]}],"candidate_step":{"claim":"All African elephants are mammals.","uses":["E1","E2"]},"rule":"categorical_transitivity","expected":"pass"} +{"id":"5s-syl-010","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All books are publications.","evidence_ids":["E1"]},{"claim":"All novels are books.","evidence_ids":["E2"]}],"candidate_step":{"claim":"All novels are publications.","uses":["E1","E2"]},"rule":"categorical_transitivity","expected":"pass"} +{"id":"5s-syl-011","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All sonnets are poems.","evidence_ids":["E1"]},{"claim":"All Shakespearean sonnets are sonnets.","evidence_ids":["E2"]}],"candidate_step":{"claim":"All Shakespearean sonnets are poems.","uses":["E1","E2"]},"rule":"categorical_transitivity","expected":"pass"} +{"id":"5s-syl-012","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All triangles are polygons.","evidence_ids":["E1"]},{"claim":"All equilateral triangles are triangles.","evidence_ids":["E2"]}],"candidate_step":{"claim":"All equilateral triangles are polygons.","uses":["E1","E2"]},"rule":"categorical_transitivity","expected":"pass"} +{"id":"5s-syl-013","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All sedans are cars.","evidence_ids":["E1"]},{"claim":"All cars are motor vehicles.","evidence_ids":["E2"]},{"claim":"All motor vehicles are transportation.","evidence_ids":["E3"]}],"candidate_step":{"claim":"All sedans are transportation.","uses":["E1","E2","E3"]},"rule":"chain_3","expected":"pass"} +{"id":"5s-syl-014","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All goldens are retrievers.","evidence_ids":["E1"]},{"claim":"All retrievers are dogs.","evidence_ids":["E2"]},{"claim":"All dogs are mammals.","evidence_ids":["E3"]}],"candidate_step":{"claim":"All goldens are mammals.","uses":["E1","E2","E3"]},"rule":"chain_3","expected":"pass"} +{"id":"5s-syl-015","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All quartz crystals are silicates.","evidence_ids":["E1"]},{"claim":"All silicates are minerals.","evidence_ids":["E2"]},{"claim":"All minerals are inorganic.","evidence_ids":["E3"]}],"candidate_step":{"claim":"All quartz crystals are inorganic.","uses":["E1","E2","E3"]},"rule":"chain_3","expected":"pass"} +{"id":"5s-syl-016","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All laptops are computers.","evidence_ids":["E1"]},{"claim":"All computers are electronics.","evidence_ids":["E2"]},{"claim":"All electronics are manufactured goods.","evidence_ids":["E3"]}],"candidate_step":{"claim":"All laptops are manufactured goods.","uses":["E1","E2","E3"]},"rule":"chain_3","expected":"pass"} +{"id":"5s-syl-017","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All apples are fruits.","evidence_ids":["E1"]},{"claim":"All fruits are produce.","evidence_ids":["E2"]},{"claim":"All produce is food.","evidence_ids":["E3"]}],"candidate_step":{"claim":"All apples are food.","uses":["E1","E2","E3"]},"rule":"chain_3","expected":"pass"} +{"id":"5s-syl-018","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All Tuesdays are weekdays.","evidence_ids":["E1"]},{"claim":"All weekdays are days.","evidence_ids":["E2"]},{"claim":"All days are time periods.","evidence_ids":["E3"]}],"candidate_step":{"claim":"All Tuesdays are time periods.","uses":["E1","E2","E3"]},"rule":"chain_3","expected":"pass"} +{"id":"5s-syl-019","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All dogs are animals.","evidence_ids":["E1"]}],"candidate_step":{"claim":"All animals are dogs.","uses":["E1"]},"rule":"invalid_converse","expected":"fail"} +{"id":"5s-syl-020","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All squares are rectangles.","evidence_ids":["E1"]}],"candidate_step":{"claim":"All rectangles are squares.","uses":["E1"]},"rule":"invalid_converse","expected":"fail"} +{"id":"5s-syl-021","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All ravens are birds.","evidence_ids":["E1"]}],"candidate_step":{"claim":"All birds are ravens.","uses":["E1"]},"rule":"invalid_converse","expected":"fail"} +{"id":"5s-syl-022","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All electric cars are vehicles.","evidence_ids":["E1"]}],"candidate_step":{"claim":"All vehicles are electric cars.","uses":["E1"]},"rule":"invalid_converse","expected":"fail"} +{"id":"5s-syl-023","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All violins are stringed instruments.","evidence_ids":["E1"]}],"candidate_step":{"claim":"All stringed instruments are violins.","uses":["E1"]},"rule":"invalid_converse","expected":"fail"} +{"id":"5s-syl-024","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All emeralds are gemstones.","evidence_ids":["E1"]}],"candidate_step":{"claim":"All gemstones are emeralds.","uses":["E1"]},"rule":"invalid_converse","expected":"fail"} +{"id":"5s-syl-025","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All cats are mammals.","evidence_ids":["E1"]}],"candidate_step":{"claim":"All cats are animals.","uses":["E1","E2"]},"rule":"missing_premise","expected":"fail"} +{"id":"5s-syl-026","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All trout are fish.","evidence_ids":["E1"]}],"candidate_step":{"claim":"All trout are vertebrates.","uses":["E1","E5"]},"rule":"missing_premise","expected":"fail"} +{"id":"5s-syl-027","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All boats are watercraft.","evidence_ids":["E1"]}],"candidate_step":{"claim":"All boats are vehicles.","uses":["E1","E2","E3"]},"rule":"missing_premise","expected":"fail"} +{"id":"5s-syl-028","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All sparrows are birds.","evidence_ids":["E1"]}],"candidate_step":{"claim":"All sparrows are animals.","uses":["E1","E9"]},"rule":"missing_premise","expected":"fail"} +{"id":"5s-syl-029","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All hexagons are polygons.","evidence_ids":["E1"]}],"candidate_step":{"claim":"All hexagons are shapes.","uses":["E1","E_missing"]},"rule":"missing_premise","expected":"fail"} +{"id":"5s-syl-030","battery":"5s","sub_battery":"syllogism","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","premises":[{"claim":"All koalas are marsupials.","evidence_ids":["E1"]}],"candidate_step":{"claim":"All koalas are mammals.","uses":["E1","E_unknown"]},"rule":"missing_premise","expected":"fail"} diff --git a/bench/fixtures/5s/synthesis-v1.jsonl b/bench/fixtures/5s/synthesis-v1.jsonl new file mode 100644 index 0000000..a27bd26 --- /dev/null +++ b/bench/fixtures/5s/synthesis-v1.jsonl @@ -0,0 +1,31 @@ +{"_meta":{"battery":"5s","sub_battery":"synthesis","version":"v1","task_count":30,"notes":"Goal + fact-set + expected derivation. Structural support check (token-overlap or substring); no LLM-as-judge. Carrier metadata included per ticket #000023."}} +{"id":"5s-syn-001","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the release-date claim is supported.","fact_set":[{"id":"F1","text":"Back to the Future was released in theaters on July 3, 1985."}],"expected_derivation":[{"claim":"Back to the Future was released on July 3, 1985.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-002","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the discovery-year claim.","fact_set":[{"id":"F1","text":"Penicillin was discovered by Alexander Fleming in 1928."}],"expected_derivation":[{"claim":"Penicillin was discovered in 1928.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-003","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the founding-year claim.","fact_set":[{"id":"F1","text":"The company AMD was founded in 1969."}],"expected_derivation":[{"claim":"AMD was founded in 1969.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-004","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the chemical-formula claim.","fact_set":[{"id":"F1","text":"Water has the chemical formula H2O."}],"expected_derivation":[{"claim":"Water has chemical formula H2O.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-005","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the speed-of-light claim.","fact_set":[{"id":"F1","text":"The speed of light in vacuum is approximately 299792458 meters per second."}],"expected_derivation":[{"claim":"The speed of light is approximately 299792458 meters per second.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-006","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the capital-city claim.","fact_set":[{"id":"F1","text":"The capital of France is Paris."}],"expected_derivation":[{"claim":"The capital of France is Paris.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-007","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the publication-year claim.","fact_set":[{"id":"F1","text":"Pride and Prejudice by Jane Austen was published in 1813."}],"expected_derivation":[{"claim":"Pride and Prejudice was published in 1813.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-008","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the inventor claim.","fact_set":[{"id":"F1","text":"The telephone was invented by Alexander Graham Bell in 1876."}],"expected_derivation":[{"claim":"The telephone was invented by Alexander Graham Bell.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-009","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the planet-count claim.","fact_set":[{"id":"F1","text":"There are eight planets in the solar system."}],"expected_derivation":[{"claim":"There are eight planets in the solar system.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-010","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the ocean claim.","fact_set":[{"id":"F1","text":"The Pacific Ocean is the largest ocean on Earth."}],"expected_derivation":[{"claim":"The Pacific Ocean is the largest ocean on Earth.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-011","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the mountain claim.","fact_set":[{"id":"F1","text":"Mount Everest is the tallest mountain on Earth at 8849 meters."}],"expected_derivation":[{"claim":"Mount Everest is the tallest mountain on Earth.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-012","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the temperature claim.","fact_set":[{"id":"F1","text":"Water freezes at 0 degrees Celsius at standard atmospheric pressure."}],"expected_derivation":[{"claim":"Water freezes at 0 degrees Celsius.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-013","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the language claim.","fact_set":[{"id":"F1","text":"Mandarin Chinese is the most widely spoken native language in the world."}],"expected_derivation":[{"claim":"Mandarin Chinese is the most widely spoken native language in the world.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-014","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the metal claim.","fact_set":[{"id":"F1","text":"Gold has the chemical symbol Au and atomic number 79."}],"expected_derivation":[{"claim":"Gold has chemical symbol Au.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-015","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the orbit claim.","fact_set":[{"id":"F1","text":"The Moon orbits the Earth at an average distance of 384400 kilometers."}],"expected_derivation":[{"claim":"The Moon orbits the Earth.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-016","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the species claim.","fact_set":[{"id":"F1","text":"The blue whale is the largest animal known to have ever existed."}],"expected_derivation":[{"claim":"The blue whale is the largest animal known to have ever existed.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-017","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the river claim.","fact_set":[{"id":"F1","text":"The Nile is one of the longest rivers in the world at about 6650 kilometers."}],"expected_derivation":[{"claim":"The Nile is one of the longest rivers in the world.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-018","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the element claim.","fact_set":[{"id":"F1","text":"Hydrogen has atomic number 1 and is the lightest chemical element."}],"expected_derivation":[{"claim":"Hydrogen has atomic number 1.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-019","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the structure claim.","fact_set":[{"id":"F1","text":"DNA has a double-helix structure described by Watson and Crick in 1953."}],"expected_derivation":[{"claim":"DNA has a double-helix structure.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-020","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the protocol claim.","fact_set":[{"id":"F1","text":"HTTP is a protocol used to transmit data on the World Wide Web."}],"expected_derivation":[{"claim":"HTTP is a protocol used to transmit data on the World Wide Web.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-021","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the satellite claim.","fact_set":[{"id":"F1","text":"Sputnik 1 was the first artificial satellite launched into Earth orbit in 1957."}],"expected_derivation":[{"claim":"Sputnik 1 was the first artificial satellite.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-022","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the chemical claim.","fact_set":[{"id":"F1","text":"Oxygen has the chemical symbol O and atomic number 8."}],"expected_derivation":[{"claim":"Oxygen has chemical symbol O.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-023","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the city-elevation claim.","fact_set":[{"id":"F1","text":"La Paz Bolivia sits at an elevation of approximately 3640 meters above sea level."}],"expected_derivation":[{"claim":"La Paz Bolivia sits at an elevation of approximately 3640 meters above sea level.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-024","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the painting claim.","fact_set":[{"id":"F1","text":"Leonardo da Vinci painted the Mona Lisa in the early sixteenth century."}],"expected_derivation":[{"claim":"Leonardo da Vinci painted the Mona Lisa.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-025","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Verify the writing-system claim.","fact_set":[{"id":"F1","text":"Cuneiform is one of the earliest known writing systems originating in ancient Sumer."}],"expected_derivation":[{"claim":"Cuneiform is one of the earliest known writing systems.","uses":["F1"]}],"expected":"pass"} +{"id":"5s-syn-026","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Reject derivation that cites a missing fact.","fact_set":[{"id":"F1","text":"The capital of France is Paris."}],"expected_derivation":[{"claim":"The capital of Spain is Madrid.","uses":["F2"]}],"expected":"fail"} +{"id":"5s-syn-027","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Reject claim not supported by cited fact.","fact_set":[{"id":"F1","text":"Mount Everest is the tallest mountain on Earth."}],"expected_derivation":[{"claim":"K2 is the tallest mountain on Earth.","uses":["F1"]}],"expected":"fail"} +{"id":"5s-syn-028","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Reject claim not supported by cited fact.","fact_set":[{"id":"F1","text":"Penicillin was discovered by Alexander Fleming in 1928."}],"expected_derivation":[{"claim":"Insulin was discovered by Alexander Fleming in 1928.","uses":["F1"]}],"expected":"fail"} +{"id":"5s-syn-029","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Reject derivation citing unknown fact id.","fact_set":[{"id":"F1","text":"Water has the chemical formula H2O."}],"expected_derivation":[{"claim":"Water has the chemical formula H2O.","uses":["F_unknown"]}],"expected":"fail"} +{"id":"5s-syn-030","battery":"5s","sub_battery":"synthesis","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","goal":"Reject claim totally unrelated to cited fact.","fact_set":[{"id":"F1","text":"Hydrogen has atomic number 1 and is the lightest chemical element."}],"expected_derivation":[{"claim":"Helium balloons float because helium is lighter than air.","uses":["F1"]}],"expected":"fail"} diff --git a/bench/fixtures/5t/time-v1.jsonl b/bench/fixtures/5t/time-v1.jsonl new file mode 100644 index 0000000..de25df9 --- /dev/null +++ b/bench/fixtures/5t/time-v1.jsonl @@ -0,0 +1,31 @@ +{"_meta":{"battery":"5t","sub_battery":"time","version":"v1","task_count":30,"notes":"Synthetic memory-snapshot chains. Tests preservation, stale-marking, and current_root tracking deterministically. Phase 1b uses fixture-embedded snapshots; Phase 1b.2 will read real memory_records once arborist has populated them."}} +{"id":"5t-time-001","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"M0","facts":["X is true at t0"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"M1","facts":["X is revised at t1"],"stale_facts":["X is true at t0"]}],"expected":{"preserved_facts":["X is true at t0"],"marks_stale":["X is true at t0"],"current_root":"M1"}} +{"id":"5t-time-002","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"M0","facts":["A","B"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"M1","facts":["A","C"],"stale_facts":["B"]}],"expected":{"preserved_facts":["A","B"],"marks_stale":["B"],"current_root":"M1"}} +{"id":"5t-time-003","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"M0","facts":["fact-1"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"M1","facts":["fact-1","fact-2"],"stale_facts":[]},{"snapshot_id":"S2","memory_root":"M2","facts":["fact-1","fact-2","fact-3"],"stale_facts":[]}],"expected":{"preserved_facts":["fact-1","fact-2"],"marks_stale":[],"current_root":"M2"}} +{"id":"5t-time-004","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"R0","facts":["P","Q"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"R1","facts":["P","Q","R"],"stale_facts":[]}],"expected":{"preserved_facts":["P","Q"],"marks_stale":[],"current_root":"R1"}} +{"id":"5t-time-005","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"H0","facts":["alpha"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"H1","facts":["alpha","beta"],"stale_facts":[]},{"snapshot_id":"S2","memory_root":"H2","facts":["beta","gamma"],"stale_facts":["alpha"]}],"expected":{"preserved_facts":["alpha","beta"],"marks_stale":["alpha"],"current_root":"H2"}} +{"id":"5t-time-006","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"K0","facts":["red"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"K1","facts":["red","blue"],"stale_facts":[]}],"expected":{"preserved_facts":["red"],"marks_stale":[],"current_root":"K1"}} +{"id":"5t-time-007","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"M0","facts":["claim-a"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"M1","facts":["claim-a","claim-b","claim-c"],"stale_facts":[]}],"expected":{"preserved_facts":["claim-a"],"marks_stale":[],"current_root":"M1"}} +{"id":"5t-time-008","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"N0","facts":["one","two","three"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"N1","facts":["one","two","three","four"],"stale_facts":[]},{"snapshot_id":"S2","memory_root":"N2","facts":["four","five"],"stale_facts":["one","two","three"]}],"expected":{"preserved_facts":["one","two","three","four"],"marks_stale":["one","two","three"],"current_root":"N2"}} +{"id":"5t-time-009","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"O0","facts":["initial"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"O1","facts":["initial","added"],"stale_facts":[]},{"snapshot_id":"S2","memory_root":"O2","facts":["initial","added","more"],"stale_facts":[]}],"expected":{"preserved_facts":["initial","added"],"marks_stale":[],"current_root":"O2"}} +{"id":"5t-time-010","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"T0","facts":["claim-x is true"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"T1","facts":["claim-x is false"],"stale_facts":["claim-x is true"]}],"expected":{"preserved_facts":["claim-x is true"],"marks_stale":["claim-x is true"],"current_root":"T1"}} +{"id":"5t-time-011","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"V0","facts":[],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"V1","facts":["new fact"],"stale_facts":[]}],"expected":{"preserved_facts":[],"marks_stale":[],"current_root":"V1"}} +{"id":"5t-time-012","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"L0","facts":["only fact"],"stale_facts":[]}],"expected":{"preserved_facts":["only fact"],"marks_stale":[],"current_root":"L0"}} +{"id":"5t-time-013","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"D0","facts":["data point a"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"D1","facts":["data point a","data point b"],"stale_facts":[]},{"snapshot_id":"S2","memory_root":"D2","facts":["data point a","data point b","data point c"],"stale_facts":[]},{"snapshot_id":"S3","memory_root":"D3","facts":["data point a","data point b","data point c","data point d"],"stale_facts":[]}],"expected":{"preserved_facts":["data point a","data point b","data point c"],"marks_stale":[],"current_root":"D3"}} +{"id":"5t-time-014","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"P0","facts":["earth is round","sky is blue"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"P1","facts":["earth is round","sky is blue","grass is green"],"stale_facts":[]}],"expected":{"preserved_facts":["earth is round","sky is blue"],"marks_stale":[],"current_root":"P1"}} +{"id":"5t-time-015","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"S0root","facts":["claim alpha"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"S1root","facts":["claim alpha","claim beta"],"stale_facts":[]}],"expected":{"preserved_facts":["claim alpha"],"marks_stale":[],"current_root":"S1root"}} +{"id":"5t-time-016","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"R0","facts":["first observation"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"R1","facts":["first observation revised"],"stale_facts":["first observation"]}],"expected":{"preserved_facts":["first observation"],"marks_stale":["first observation"],"current_root":"R1"}} +{"id":"5t-time-017","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"E0","facts":["entry-1","entry-2"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"E1","facts":["entry-1","entry-2","entry-3","entry-4"],"stale_facts":[]}],"expected":{"preserved_facts":["entry-1","entry-2"],"marks_stale":[],"current_root":"E1"}} +{"id":"5t-time-018","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"G0","facts":["genesis"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"G1","facts":["genesis","update1"],"stale_facts":[]},{"snapshot_id":"S2","memory_root":"G2","facts":["genesis","update1","update2"],"stale_facts":[]},{"snapshot_id":"S3","memory_root":"G3","facts":["genesis","update1","update2","update3"],"stale_facts":[]}],"expected":{"preserved_facts":["genesis","update1","update2"],"marks_stale":[],"current_root":"G3"}} +{"id":"5t-time-019","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"F0","facts":["x=1","y=2"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"F1","facts":["x=2","y=2"],"stale_facts":["x=1"]}],"expected":{"preserved_facts":["y=2","x=1"],"marks_stale":["x=1"],"current_root":"F1"}} +{"id":"5t-time-020","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"H0","facts":["price=10"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"H1","facts":["price=12"],"stale_facts":["price=10"]},{"snapshot_id":"S2","memory_root":"H2","facts":["price=15"],"stale_facts":["price=10","price=12"]}],"expected":{"preserved_facts":["price=10","price=12"],"marks_stale":["price=10","price=12"],"current_root":"H2"}} +{"id":"5t-time-021","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"A0","facts":["status=active"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"A1","facts":["status=archived"],"stale_facts":["status=active"]}],"expected":{"preserved_facts":["status=active"],"marks_stale":["status=active"],"current_root":"A1"}} +{"id":"5t-time-022","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"B0","facts":["fact-1","fact-2","fact-3"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"B1","facts":["fact-1","fact-2"],"stale_facts":["fact-3"]}],"expected":{"preserved_facts":["fact-1","fact-2","fact-3"],"marks_stale":["fact-3"],"current_root":"B1"}} +{"id":"5t-time-023","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"C0","facts":["motif-A"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"C1","facts":["motif-A","motif-B"],"stale_facts":[]}],"expected":{"preserved_facts":["motif-A"],"marks_stale":[],"current_root":"C1"}} +{"id":"5t-time-024","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"Q0","facts":["q1"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"Q1","facts":["q1","q2"],"stale_facts":[]},{"snapshot_id":"S2","memory_root":"Q2","facts":["q1","q2","q3"],"stale_facts":[]},{"snapshot_id":"S3","memory_root":"Q3","facts":["q1","q2","q3","q4"],"stale_facts":[]},{"snapshot_id":"S4","memory_root":"Q4","facts":["q1","q2","q3","q4","q5"],"stale_facts":[]}],"expected":{"preserved_facts":["q1","q2","q3","q4"],"marks_stale":[],"current_root":"Q4"}} +{"id":"5t-time-025","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"U0","facts":["u-fact"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"U1","facts":["u-fact","new-u"],"stale_facts":[]}],"expected":{"preserved_facts":["u-fact"],"marks_stale":[],"current_root":"U1"}} +{"id":"5t-time-026","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"W0","facts":["a","b","c"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"W1","facts":["a","b","c","d"],"stale_facts":[]}],"expected":{"preserved_facts":["a","b","c"],"marks_stale":[],"current_root":"W1"}} +{"id":"5t-time-027","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"Y0","facts":["state-stable"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"Y1","facts":["state-changed"],"stale_facts":["state-stable"]}],"expected":{"preserved_facts":["state-stable"],"marks_stale":["state-stable"],"current_root":"Y1"}} +{"id":"5t-time-028","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"Z0","facts":["fact-1"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"Z1","facts":["fact-1","fact-2"],"stale_facts":[]},{"snapshot_id":"S2","memory_root":"Z2","facts":["fact-1","fact-2","fact-3"],"stale_facts":[]},{"snapshot_id":"S3","memory_root":"Z3","facts":["fact-2","fact-3","fact-4"],"stale_facts":["fact-1"]}],"expected":{"preserved_facts":["fact-1","fact-2","fact-3"],"marks_stale":["fact-1"],"current_root":"Z3"}} +{"id":"5t-time-029","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"AA0","facts":["original"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"AA1","facts":["original","extension"],"stale_facts":[]}],"expected":{"preserved_facts":["original"],"marks_stale":[],"current_root":"AA1"}} +{"id":"5t-time-030","battery":"5t","sub_battery":"time","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","snapshots":[{"snapshot_id":"S0","memory_root":"BB0","facts":["alpha","beta","gamma","delta"],"stale_facts":[]},{"snapshot_id":"S1","memory_root":"BB1","facts":["alpha","beta","gamma","delta","epsilon"],"stale_facts":[]}],"expected":{"preserved_facts":["alpha","beta","gamma","delta"],"marks_stale":[],"current_root":"BB1"}} diff --git a/bench/fixtures/5t/transfer-learning-v2.jsonl b/bench/fixtures/5t/transfer-learning-v2.jsonl new file mode 100644 index 0000000..1c4a388 --- /dev/null +++ b/bench/fixtures/5t/transfer-learning-v2.jsonl @@ -0,0 +1,31 @@ +{"_meta":{"battery":"5t","sub_battery":"transfer-learning","version":"v2","task_count":30,"notes":"Pattern transfer across tasks/domains. Source pattern + target pattern + expected_transfer. Carrier metadata per ticket #000024."}} +{"id":"5t-tl-001","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"A supports B when cited span contains anchor B."},"target_task":{"pattern":"A supports B when cited span contains anchor B for date claims.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-002","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Verify entity name match between claim and source title."},"target_task":{"pattern":"Verify entity name match between claim and source title across abbreviation variants.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-003","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Reject answer when claim text shares zero stems with cited title."},"target_task":{"pattern":"Reject answer when claim text shares zero stems with cited title for biography pages.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-004","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Match release date format YYYY-MM-DD against ISO date in evidence."},"target_task":{"pattern":"Match release date format YYYY-MM-DD against ISO date in evidence for software products.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-005","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Locate cause-effect relations using a verb-pair pattern."},"target_task":{"pattern":"Locate cause-effect relations using a verb-pair pattern within scientific text.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-006","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Detect quantifier scope using broad-quantifier preflight tokens."},"target_task":{"pattern":"Detect quantifier scope using broad-quantifier preflight tokens for survey questions.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-007","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Stem-fold possessives and plurals before token comparison."},"target_task":{"pattern":"Stem-fold possessives and plurals before token comparison in title-relevance checks.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-008","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Promote phrase-pattern matches in retrieval ranking."},"target_task":{"pattern":"Promote phrase-pattern matches in retrieval ranking for allusion-style queries.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-009","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Apply trailing-citation strip before substring testing."},"target_task":{"pattern":"Apply trailing-citation strip before substring testing within Wikipedia prose.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-010","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Compute body-coverage square-root rerank to counter BM25 short-doc bias."},"target_task":{"pattern":"Compute body-coverage square-root rerank to counter BM25 short-doc bias for long-form articles.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-011","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Filter retrieved documents by title-token overlap."},"target_task":{"pattern":"Filter retrieved documents by title-token overlap for narrow specialist topics.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-012","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Apply rivalry exclusion to keep AMD docs out of Intel queries."},"target_task":{"pattern":"Apply rivalry exclusion to keep AMD docs out of Intel queries even with synonym expansion.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-013","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Drop template phrase stopwords before retrieval-token formation."},"target_task":{"pattern":"Drop template phrase stopwords before retrieval-token formation for instructional questions.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-014","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Cap claim count at twelve to prevent runaway answers."},"target_task":{"pattern":"Cap claim count at twelve to prevent runaway answers in broad survey questions.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-015","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Bind retrieval keywords into the run-DAG via retrieval_plan_hash."},"target_task":{"pattern":"Bind retrieval keywords into the run-DAG via retrieval_plan_hash across operator-augmented queries.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-016","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Verify quote substrings in cited evidence under wikitext-base@v1."},"target_task":{"pattern":"Run apriori multistep proof on a unrelated codebase audit.","expected_transfer":false},"expected":"pass"} +{"id":"5t-tl-017","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Extract entity proximity clusters from prose."},"target_task":{"pattern":"Schedule downtime for kernel patch rollouts.","expected_transfer":false},"expected":"pass"} +{"id":"5t-tl-018","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Compute SHA-256 over canonical JSON for audit-event chaining."},"target_task":{"pattern":"Reconcile bank statements at end of fiscal quarter.","expected_transfer":false},"expected":"pass"} +{"id":"5t-tl-019","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Run paraphrase-strategy verifier with token-coverage threshold."},"target_task":{"pattern":"Render thumbnails for image-gallery web pages.","expected_transfer":false},"expected":"pass"} +{"id":"5t-tl-020","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Build claim lattice from answer text with pointer IDs."},"target_task":{"pattern":"Brew coffee using a French press at 95C water.","expected_transfer":false},"expected":"pass"} +{"id":"5t-tl-021","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Detect title mismatch when claim shares zero stems with cited title."},"target_task":{"pattern":"Sort customer reviews by recency.","expected_transfer":false},"expected":"pass"} +{"id":"5t-tl-022","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Apply soft preflight hint sidecar to broad-quantifier questions."},"target_task":{"pattern":"Compress JPEG images for web delivery.","expected_transfer":false},"expected":"pass"} +{"id":"5t-tl-023","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Map answer mode to per-mode max context character cap."},"target_task":{"pattern":"Schedule weekly garbage pickup routes.","expected_transfer":false},"expected":"pass"} +{"id":"5t-tl-024","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Verify quote substrings in cited evidence."},"target_task":{"pattern":"Verify quote substrings in cited evidence with cross-doc context.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-025","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Score answers by directive coverage across D1 through D8."},"target_task":{"pattern":"Score answers by directive coverage across D1 through D8 in lattice mode.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-026","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Detect deflection via subject-anchor heuristic on last content token."},"target_task":{"pattern":"Detect deflection via subject-anchor heuristic on last content token for what-questions.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-027","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Resolve unresolved forward links by document URI lookup."},"target_task":{"pattern":"Resolve unresolved forward links by document URI lookup at ingest-batch boundary.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-028","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Re-derive concept_relations on backfill without invalidating cache_keys."},"target_task":{"pattern":"Re-derive concept_relations on backfill without invalidating cache_keys for mesh-distributed shards.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-029","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Falsify providence records under verifier upgrade with stated reason."},"target_task":{"pattern":"Falsify providence records under verifier upgrade with stated reason and audit event chain.","expected_transfer":true},"expected":"pass"} +{"id":"5t-tl-030","battery":"5t","sub_battery":"transfer-learning","version":"v2","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","source_task":{"pattern":"Snapshot SelfModel root with capability claim hashes folded in."},"target_task":{"pattern":"Snapshot SelfModel root with capability claim hashes folded in across checkpoint cadence.","expected_transfer":true},"expected":"pass"} diff --git a/bench/fixtures/5t/transitivity-v1.jsonl b/bench/fixtures/5t/transitivity-v1.jsonl new file mode 100644 index 0000000..7174c75 --- /dev/null +++ b/bench/fixtures/5t/transitivity-v1.jsonl @@ -0,0 +1,31 @@ +{"_meta":{"battery":"5t","sub_battery":"transitivity","version":"v1","task_count":30,"notes":"Typed-relation chains. Whitelist: implies, subset_of, ancestor_of, before, less_than. Mixed-relation or non-whitelisted relations always fail by construction."}} +{"id":"5t-trn-001","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"A","to":"B","relation":"implies"},{"from":"B","to":"C","relation":"implies"}],"query":{"from":"A","to":"C","relation":"implies"},"expected":"pass"} +{"id":"5t-trn-002","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"X","to":"Y","relation":"implies"},{"from":"Y","to":"Z","relation":"implies"},{"from":"Z","to":"W","relation":"implies"}],"query":{"from":"X","to":"W","relation":"implies"},"expected":"pass"} +{"id":"5t-trn-003","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"squares","to":"rectangles","relation":"subset_of"},{"from":"rectangles","to":"quadrilaterals","relation":"subset_of"}],"query":{"from":"squares","to":"quadrilaterals","relation":"subset_of"},"expected":"pass"} +{"id":"5t-trn-004","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"sparrows","to":"birds","relation":"subset_of"},{"from":"birds","to":"animals","relation":"subset_of"}],"query":{"from":"sparrows","to":"animals","relation":"subset_of"},"expected":"pass"} +{"id":"5t-trn-005","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"oaks","to":"trees","relation":"subset_of"},{"from":"trees","to":"plants","relation":"subset_of"},{"from":"plants","to":"organisms","relation":"subset_of"}],"query":{"from":"oaks","to":"organisms","relation":"subset_of"},"expected":"pass"} +{"id":"5t-trn-006","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"alice","to":"bob","relation":"ancestor_of"},{"from":"bob","to":"carol","relation":"ancestor_of"}],"query":{"from":"alice","to":"carol","relation":"ancestor_of"},"expected":"pass"} +{"id":"5t-trn-007","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"grandma","to":"mother","relation":"ancestor_of"},{"from":"mother","to":"daughter","relation":"ancestor_of"},{"from":"daughter","to":"granddaughter","relation":"ancestor_of"}],"query":{"from":"grandma","to":"granddaughter","relation":"ancestor_of"},"expected":"pass"} +{"id":"5t-trn-008","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"event1","to":"event2","relation":"before"},{"from":"event2","to":"event3","relation":"before"}],"query":{"from":"event1","to":"event3","relation":"before"},"expected":"pass"} +{"id":"5t-trn-009","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"jurassic","to":"cretaceous","relation":"before"},{"from":"cretaceous","to":"paleogene","relation":"before"},{"from":"paleogene","to":"neogene","relation":"before"}],"query":{"from":"jurassic","to":"neogene","relation":"before"},"expected":"pass"} +{"id":"5t-trn-010","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"two","to":"three","relation":"less_than"},{"from":"three","to":"four","relation":"less_than"}],"query":{"from":"two","to":"four","relation":"less_than"},"expected":"pass"} +{"id":"5t-trn-011","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"five","to":"ten","relation":"less_than"},{"from":"ten","to":"twenty","relation":"less_than"},{"from":"twenty","to":"forty","relation":"less_than"}],"query":{"from":"five","to":"forty","relation":"less_than"},"expected":"pass"} +{"id":"5t-trn-012","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"P","to":"Q","relation":"implies"},{"from":"Q","to":"R","relation":"implies"},{"from":"R","to":"S","relation":"implies"},{"from":"S","to":"T","relation":"implies"}],"query":{"from":"P","to":"T","relation":"implies"},"expected":"pass"} +{"id":"5t-trn-013","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"mammals","to":"vertebrates","relation":"subset_of"},{"from":"dogs","to":"mammals","relation":"subset_of"}],"query":{"from":"dogs","to":"vertebrates","relation":"subset_of"},"expected":"pass"} +{"id":"5t-trn-014","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"poodle","to":"dog","relation":"subset_of"},{"from":"dog","to":"mammal","relation":"subset_of"},{"from":"mammal","to":"animal","relation":"subset_of"}],"query":{"from":"poodle","to":"animal","relation":"subset_of"},"expected":"pass"} +{"id":"5t-trn-015","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"dawn","to":"morning","relation":"before"},{"from":"morning","to":"noon","relation":"before"},{"from":"noon","to":"evening","relation":"before"}],"query":{"from":"dawn","to":"evening","relation":"before"},"expected":"pass"} +{"id":"5t-trn-016","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"A","to":"B","relation":"implies"},{"from":"C","to":"D","relation":"implies"}],"query":{"from":"A","to":"D","relation":"implies"},"expected":"fail"} +{"id":"5t-trn-017","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"A","to":"B","relation":"related_to"},{"from":"B","to":"C","relation":"causes"}],"query":{"from":"A","to":"C","relation":"causes"},"expected":"fail"} +{"id":"5t-trn-018","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"alice","to":"bob","relation":"likes"},{"from":"bob","to":"carol","relation":"likes"}],"query":{"from":"alice","to":"carol","relation":"likes"},"expected":"fail"} +{"id":"5t-trn-019","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"X","to":"Y","relation":"related_to"},{"from":"Y","to":"Z","relation":"related_to"}],"query":{"from":"X","to":"Z","relation":"related_to"},"expected":"fail"} +{"id":"5t-trn-020","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"P","to":"Q","relation":"causes"},{"from":"Q","to":"R","relation":"causes"}],"query":{"from":"P","to":"R","relation":"causes"},"expected":"fail"} +{"id":"5t-trn-021","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"A","to":"B","relation":"implies"},{"from":"B","to":"C","relation":"subset_of"}],"query":{"from":"A","to":"C","relation":"implies"},"expected":"fail"} +{"id":"5t-trn-022","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"a","to":"b","relation":"subset_of"},{"from":"b","to":"c","relation":"ancestor_of"}],"query":{"from":"a","to":"c","relation":"subset_of"},"expected":"fail"} +{"id":"5t-trn-023","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"a","to":"b","relation":"before"},{"from":"b","to":"c","relation":"after"}],"query":{"from":"a","to":"c","relation":"before"},"expected":"fail"} +{"id":"5t-trn-024","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"A","to":"B","relation":"likes"},{"from":"B","to":"C","relation":"likes"},{"from":"C","to":"D","relation":"likes"}],"query":{"from":"A","to":"D","relation":"likes"},"expected":"fail"} +{"id":"5t-trn-025","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"M","to":"N","relation":"unknown_relation"},{"from":"N","to":"O","relation":"unknown_relation"}],"query":{"from":"M","to":"O","relation":"unknown_relation"},"expected":"fail"} +{"id":"5t-trn-026","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"node1","to":"node2","relation":"adjacent"},{"from":"node2","to":"node3","relation":"adjacent"}],"query":{"from":"node1","to":"node3","relation":"adjacent"},"expected":"fail"} +{"id":"5t-trn-027","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"a","to":"b","relation":"implies"},{"from":"c","to":"d","relation":"implies"},{"from":"e","to":"f","relation":"implies"}],"query":{"from":"a","to":"f","relation":"implies"},"expected":"fail"} +{"id":"5t-trn-028","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"A","to":"B","relation":"less_than"},{"from":"C","to":"D","relation":"less_than"}],"query":{"from":"A","to":"D","relation":"less_than"},"expected":"fail"} +{"id":"5t-trn-029","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"A","to":"B","relation":"implies"},{"from":"B","to":"C","relation":"implies"},{"from":"D","to":"E","relation":"implies"},{"from":"E","to":"F","relation":"implies"}],"query":{"from":"A","to":"F","relation":"implies"},"expected":"fail"} +{"id":"5t-trn-030","battery":"5t","sub_battery":"transitivity","version":"v1","carrier":"claim_lattice","domain":"relation_graph","pi_star_ref":"pi_relation_graph_v1","edges":[{"from":"X","to":"Y","relation":"correlated_with"},{"from":"Y","to":"Z","relation":"correlated_with"}],"query":{"from":"X","to":"Z","relation":"correlated_with"},"expected":"fail"} diff --git a/bench/fixtures/5t/triangulation-v1.jsonl b/bench/fixtures/5t/triangulation-v1.jsonl new file mode 100644 index 0000000..5c00271 --- /dev/null +++ b/bench/fixtures/5t/triangulation-v1.jsonl @@ -0,0 +1,31 @@ +{"_meta":{"battery":"5t","sub_battery":"triangulation","version":"v1","task_count":30,"notes":"Independent strategies (substring, token_subset, token_overlap, entity_match). expected=pass means agreement>=threshold; expected=fail means agreement= 1 + assert res.sub_battery == "transfer-learning" + assert res.pass_count == 30 + assert res.metrics["transfer_learning_success_rate"] == 1.0 -@pytest.mark.parametrize( - "fn", - [ - b_5s.run_synthesis, - b_5s.run_syllogism, - b_5s.run_semiotics, - b_5t.run_triangulate, - b_5t.run_timing, - b_5t.run_transitivity, - b_5t.run_truth, - ], -) -def test_stub_sub_batteries_return_zero(fn, tmp_path): - # Stub takes any path; doesn't read it. - res = fn(tmp_path / "missing.jsonl") +def test_5t_triangulation_runs(): + res = b_5t.run_triangulation(F5T / "triangulation-v1.jsonl") + assert res.pass_count == 30 + assert res.metrics["triangulation_agreement_rate"] == 1.0 + + +def test_5t_truthtables_runs(): + res = b_5t.run_truthtables(F5T / "truthtables-v1.jsonl") + assert res.pass_count == 30 + assert res.metrics["truth_table_coverage_rate"] == 1.0 + + +def test_5t_truthtables_caps_at_n_4(): + """Per ticket #000024 §4.4: N > 4 must be rejected.""" + import tempfile + + bad = tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) + bad.write(json.dumps({"_meta": {"battery": "5t", "sub_battery": "truthtables", "version": "v1"}}) + "\n") + bad.write(json.dumps({ + "id": "5t-tt-toobig", + "carrier": "claim_lattice", + "domain": "propositional_logic", + "pi_star_ref": "pi_truth_table_v1", + "variables": ["A", "B", "C", "D", "E"], + "expression": "A AND B AND C AND D AND E", + "rows": [{"inputs": {v: False for v in "ABCDE"}, "expected": False}], + }) + "\n") + bad.close() + res = b_5t.run_truthtables(Path(bad.name)) assert res.pass_count == 0 - assert res.fail_count == 0 + assert res.fail_count == 1 + assert "exceeds cap" in res.per_task[0].detail["reason"] -def test_battery_result_is_json_serializable(): - res = b_5s.run_syntax(SYNTAX_FX) - payload = json.dumps(asdict(res), default=str) - assert "syntax" in payload +def test_5t_transitivity_runs(): + res = b_5t.run_transitivity(F5T / "transitivity-v1.jsonl") + assert res.pass_count == 30 + assert res.metrics["full_chain_pass_rate"] == 1.0 -def test_runner_main_smoke(tmp_path, capsys): - """Runner produces JSON on stdout when --out is omitted.""" +def test_5t_transitivity_rejects_non_whitelisted_relation(): + """Non-whitelisted relations always fail by construction.""" + import tempfile + + bad = tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) + bad.write(json.dumps({"_meta": {"battery": "5t", "sub_battery": "transitivity", "version": "v1"}}) + "\n") + bad.write(json.dumps({ + "id": "test", + "carrier": "claim_lattice", + "domain": "relation_graph", + "pi_star_ref": "pi_relation_graph_v1", + "edges": [ + {"from": "A", "to": "B", "relation": "related_to"}, + {"from": "B", "to": "C", "relation": "related_to"}, + ], + "query": {"from": "A", "to": "C", "relation": "related_to"}, + "expected": "pass", # but it can't possibly pass on a non-transitive relation + }) + "\n") + bad.close() + res = b_5t.run_transitivity(Path(bad.name)) + # observed=fail because not in whitelist; expected=pass; mismatch. + assert res.fail_count == 1 + + +def test_5t_time_runs_against_synthetic_snapshots(): + res = b_5t.run_time(F5T / "time-v1.jsonl") + assert res.pass_count == 30 + assert res.metrics["temporal_context_preservation_rate"] == 1.0 + + +# --------------------------------------------------------------------- +# 5F Phase 1a — Function / Finetuning / Falsification / Formulate / +# Feedback Loop +# --------------------------------------------------------------------- + + +def test_5f_function_runs(): + res = b_5f.run_function(F5F / "function-v1.jsonl") + assert res.battery == "5f" + assert res.sub_battery == "function" + assert res.pass_count == 10 + assert res.metrics["function_pass_rate"] == 1.0 + + +def test_5f_finetuning_runs(): + res = b_5f.run_finetuning(F5F / "finetuning-v1.jsonl") + assert res.pass_count == 10 + assert res.metrics["adaptation_improvement_rate"] == 1.0 + + +def test_5f_falsification_runs(): + res = b_5f.run_falsification(F5F / "falsification-v1.jsonl") + assert res.pass_count == 10 + assert res.metrics["error_detection_rate"] == 1.0 + + +def test_5f_formulate_runs(): + res = b_5f.run_formulate(F5F / "formulate-v1.jsonl") + assert res.pass_count == 10 + assert res.metrics["structural_match_rate"] == 1.0 + + +def test_5f_feedback_loop_runs(): + res = b_5f.run_feedback_loop(F5F / "feedback-loop-v1.jsonl") + assert res.pass_count == 10 + assert res.metrics["integration_coverage_rate"] == 1.0 + + +# --------------------------------------------------------------------- +# Carrier rejection tests — runners fail unsupported carriers cleanly +# --------------------------------------------------------------------- + + +def _write_unsupported_fixture(path: Path, sub: str, battery: str = "5s") -> None: + """Write a fixture with an unsupported carrier.""" + path.write_text( + json.dumps({"_meta": {"battery": battery, "sub_battery": sub, "version": "v1"}}) + "\n" + + json.dumps({ + "id": "test", + "carrier": "image", # not in Phase-1 whitelist + "domain": "scene_graph", + "pi_star_ref": "image-base@v1", + # the rest doesn't matter — runner rejects on carrier check + "input": "x", + "premises": [], "candidate_step": {"claim": "x", "uses": []}, + "rule": "categorical_transitivity", + "expected": "pass", + }) + "\n", + encoding="utf-8", + ) + + +def test_5s_syllogism_rejects_unsupported_carrier(tmp_path): + p = tmp_path / "bad.jsonl" + _write_unsupported_fixture(p, "syllogism") + res = b_5s.run_syllogism(p) + assert res.fail_count == 1 + assert "unsupported_carrier" in res.per_task[0].detail["reason"] + + +def test_5t_transfer_learning_rejects_unsupported_carrier(tmp_path): + p = tmp_path / "bad.jsonl" + _write_unsupported_fixture(p, "transfer-learning", battery="5t") + res = b_5t.run_transfer_learning(p) + assert res.fail_count == 1 + assert "unsupported_carrier" in res.per_task[0].detail["reason"] + + +def test_5f_function_rejects_unsupported_carrier(tmp_path): + p = tmp_path / "bad.jsonl" + _write_unsupported_fixture(p, "function", battery="5f") + res = b_5f.run_function(p) + assert res.fail_count == 1 + assert "unsupported_carrier" in res.per_task[0].detail["reason"] + + +# --------------------------------------------------------------------- +# Runner CLI (existing test from Phase 1a) +# --------------------------------------------------------------------- + + +def test_runner_main_smoke(capsys): from bench.batteries.runner import main - rc = main( - [ - "--battery", - "5s", - "--sub", - "syntax", - "--fixtures", - str(SYNTAX_FX), - ] - ) + rc = main([ + "--battery", "5s", "--sub", "syntax", + "--fixtures", str(F5S / "syntax-v1.jsonl"), + ]) out = capsys.readouterr().out payload = json.loads(out) assert payload["schema_version"] == "bench-result-v1" @@ -126,31 +322,9 @@ def test_runner_main_smoke(tmp_path, capsys): assert rc == 0 -def test_runner_writes_to_file(tmp_path): - from bench.batteries.runner import main - - out_path = tmp_path / "result.json" - rc = main( - [ - "--battery", - "5s", - "--sub", - "semantics", - "--fixtures", - str(SEM_FX), - "--out", - str(out_path), - ] - ) - assert rc == 0 - payload = json.loads(out_path.read_text()) - assert payload["results"][0]["sub_battery"] == "semantics" - - -def test_runner_all_runs_phase_1a(): - """--all runs all three default fixtures and exits 0 when all pass.""" +def test_runner_all_runs_full_suite(): + """--all runs every battery in _DEFAULT_FIXTURES; rc=0 since all pass.""" from bench.batteries.runner import main rc = main(["--all"]) - # All seed fixtures are constructed to pass; rc=0 expected. assert rc == 0