diff --git a/arborist/cli.py b/arborist/cli.py index 485894e..1819a20 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -763,6 +763,19 @@ def _render_warrant_tail(result: dict) -> str: # (e.g. SOFT_FALSE_PREMISE_SUSPECTED → "false premise suspected"). readable = soft_label.removeprefix("SOFT_").lower().replace("_", " ") parts.append(f"soft: {readable}") + # Ticket #000026 Phase 3 — authorship warrant tail. Shows operator + # the strongest tier the cited evidence supports for an + # authorship-shaped question. Strong tiers (1-4) advertise the + # win; weak tiers (5-6) flag that the warrant rests on shaky + # evidence. NO_AUTHORSHIP_SIGNAL stays silent (sidecar's + # "doesn't apply" verdict). + authorship = result.get("authorship") or {} + auth_tier = authorship.get("tier") + if auth_tier and auth_tier != "NO_AUTHORSHIP_SIGNAL": + # Strip AUTHOR_ prefix + lowercase for tail readability + # (AUTHOR_COPYRIGHT_FOOTER → "copyright-footer"). + readable = auth_tier.removeprefix("AUTHOR_").lower().replace("_", "-") + parts.append(f"warrant: {readable}") if not parts: return "" return " · " + " · ".join(parts) diff --git a/arborist/qa/inspect.py b/arborist/qa/inspect.py index 8ae9727..860ac02 100644 --- a/arborist/qa/inspect.py +++ b/arborist/qa/inspect.py @@ -920,6 +920,23 @@ def inspect_cache_key( # answering). Sidecar-only, never feeds back into providence. deflection = diagnose_deflection(row["question_text"], row["answer_text"]) + # Authorship warrant ladder (#000026 Phase 3) — sidecar classifier + # that names the strongest-tier authorship signal in cited + # evidence. Returns NO_AUTHORSHIP_SIGNAL when the question doesn't + # smell like an authorship inquiry; otherwise grades the warrant + # on a 6-tier ladder (PACKAGE_METADATA → REPOSITORY_OWNER → + # PAGE_BYLINE → PRIMARY_PAGE_TITLE → COPYRIGHT_FOOTER → + # SECONDARY_SOURCE). Surfaces in the inspect dict alongside the + # other sidecars; never enters proof path. + from arborist.qa.warrant_authorship import diagnose_authorship_warrant + authorship = diagnose_authorship_warrant( + question_text=row["question_text"], + answer_text=row["answer_text"], + cited_evidence_spans=parts_raw, + cited_source_uris=[s.get("document_uri") or "" for s in src_summary], + cited_source_titles=[s.get("title") or "" for s in src_summary], + ) + return { "status": "ok", "record": { @@ -940,4 +957,5 @@ def inspect_cache_key( }, "unverified": diagnoses, "deflection": deflection, + "authorship": authorship, } diff --git a/arborist/qa/query.py b/arborist/qa/query.py index 37b92fb..6c3f508 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -2090,37 +2090,111 @@ def query( # the kernel ground truth and records cross-modality # agreement. Cache leg uses the real persisted-row bytes # (post-#000027) when persistence is on; otherwise None. + # + # #000028 follow-up — sample rate gating. When enabled, + # `canonical_witness_sample_rate` (default 1.0) chooses + # the fraction of canonical-shape calls that actually + # fire the witness. Operators wanting passive calibration + # set sample_rate=0.05 to pay 5% of the LLM cost while + # still collecting divergence data. 1.0 = always-on + # (current behavior); 0.0 = effectively disabled. witness_dict = None + witness_skipped_reason: str | None = None if bool(policy.get("canonical_witness_enabled", False)): - from arborist.qa.witness import run_witness - # Cache-leg closure: returns the persisted answer - # bytes if we found a prior row at the top of this - # branch (so the witness compares against the - # already-committed canonical answer), else None. - # We use the row from BEFORE we wrote — comparing - # against a row we just wrote in the same call would - # be tautological. - _cached_bytes = ( - cached_row["answer_text"].encode( - "utf-8", errors="surrogatepass" + _sample_rate = float( + policy.get("canonical_witness_sample_rate", 1.0) + ) + _sample_rate = max(0.0, min(1.0, _sample_rate)) + if _sample_rate < 1.0: + import random as _random + _fired = _random.random() < _sample_rate + else: + _fired = True + if not _fired: + witness_skipped_reason = "sampled_out" + progress.emit( + "witness.skipped", reason="sampled_out", + sample_rate=_sample_rate, ) - if cached_row is not None - else None - ) - _cache_lookup = lambda: _cached_bytes # noqa: E731 - _witness = run_witness( - question=question, - pi_star_ref=pi_star_ref, - canonical_answer_bytes=canonical_bytes, - cache_lookup=_cache_lookup, - chat_client=chat_client, - model_id=model_id, - timeout_s=float( - policy.get("canonical_witness_timeout_s", 10.0) - ), - progress=progress, - ) - witness_dict = _witness.to_dict() + else: + from arborist.qa.witness import run_witness + # Cache-leg closure: returns the persisted answer + # bytes if we found a prior row at the top of this + # branch (so the witness compares against the + # already-committed canonical answer), else None. + # We use the row from BEFORE we wrote — comparing + # against a row we just wrote in the same call would + # be tautological. + _cached_bytes = ( + cached_row["answer_text"].encode( + "utf-8", errors="surrogatepass" + ) + if cached_row is not None + else None + ) + _cache_lookup = lambda: _cached_bytes # noqa: E731 + _witness = run_witness( + question=question, + pi_star_ref=pi_star_ref, + canonical_answer_bytes=canonical_bytes, + cache_lookup=_cache_lookup, + chat_client=chat_client, + model_id=model_id, + timeout_s=float( + policy.get("canonical_witness_timeout_s", 10.0) + ), + progress=progress, + ) + witness_dict = _witness.to_dict() + # Capital-ledger record (#000028 follow-up). Witness + # mode adds one full LLM call per fired canonical + # question; record the cost so ForkScore (#000012) + # can compare witness-on vs witness-off forks + # honestly + so operators can budget. Best-effort: + # ledger-write failure must never fail the query. + if ( + audit_event_hash_v + and witness_dict + and witness_dict.get("modalities", {}).get("llm") + ): + try: + from arborist.capital import store as _capstore + from arborist.capital.profile import profile_for_op + _llm = witness_dict["modalities"]["llm"] + _llm_seconds = float( + _llm.get("elapsed_ms", 0) + ) / 1000.0 + _profile, _inputs = profile_for_op( + "qa", + { + "prompt_chars": len(question), + "answer_chars": len( + _llm.get("raw_answer") or "" + ), + "llm_seconds": _llm_seconds, + "cache_hit": False, + }, + ) + _conn = connect(qa_db) + try: + _capstore.record( + _conn, + audit_event_hash=audit_event_hash_v, + op_type="canonical_witness", + profile=_profile, + estimator_inputs={ + **_inputs, + "agreement_label": witness_dict.get( + "agreement_label" + ), + "pi_star_ref": pi_star_ref, + }, + ) + finally: + _conn.close() + except Exception: # pragma: no cover + # Sidecar discipline: ledger never blocks. + pass return { "status": status_v, "audit_mode": "CANONICAL_PROJECTION", diff --git a/bench/fixtures/5f/falsification-live-v1.jsonl b/bench/fixtures/5f/falsification-live-v1.jsonl index 46fe3fd..88d9830 100644 --- a/bench/fixtures/5f/falsification-live-v1.jsonl +++ b/bench/fixtures/5f/falsification-live-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 30, "notes": "Phase 1b.2: answer_text + context routes through arborist.qa.verify.verify_quotes (real verifier surface). Live signals: UNGROUNDED, STRICT_, HYBRID_, UNVERIFIED_QUOTE."}} +{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 50, "notes": "Phase 1b.2: answer_text + context routes through arborist.qa.verify.verify_quotes (real verifier surface). Live signals: UNGROUNDED, STRICT_, HYBRID_, UNVERIFIED_QUOTE. Phase 1d (2026-05-09): expanded 30→50."}} {"id": "5f-fal-live-001", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing.", "context": "Some completely unrelated context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} {"id": "5f-fal-live-002", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "K2 is the tallest mountain.", "context": "Mount Everest is the tallest mountain at 8849 meters.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} {"id": "5f-fal-live-003", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Insulin was discovered by Alexander Fleming.", "context": "Penicillin was discovered by Alexander Fleming in 1928.", "expected_reason": "HYBRID_ENTITY", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "verify_quotes' entity strategy matches on shared 'Alexander Fleming' entity even though the substance is swapped. Documents a known soft-signal gap; the title-relevance hard check + claim-lattice verifier catch this case in production."} @@ -29,3 +29,23 @@ {"id": "5f-fal-live-028", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The Eiffel Tower is in London.", "context": "The Eiffel Tower is in Paris France.", "expected_reason": "HYBRID_ENTITY", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "Live verifier signal: verify_quotes returns HYBRID_ENTITY for this (answer, context) pair. Captured here as a behavior-pinning fixture; divergence from synthetic expectation is the value of live mode."} {"id": "5f-fal-live-029", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Pride and Prejudice was published in 1813.", "context": "Pride and Prejudice was published in 1813.", "expected_reason": "STRICT_SPAN", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} {"id": "5f-fal-live-030", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Berlin.", "context": "The capital of France is Paris.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-031", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 31)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-032", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 32)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-033", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Tigers are large animals from India. (variant 33)", "context": "Tigers are large mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-034", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 34)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-035", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 35)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-036", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Tigers are large animals from India. (variant 36)", "context": "Tigers are large mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-037", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 37)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-038", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 38)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-039", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Tigers are large animals from India. (variant 39)", "context": "Tigers are large mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-040", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 40)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-041", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 41)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-042", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Tigers are large animals from India. (variant 42)", "context": "Tigers are large mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-043", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 43)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-044", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 44)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-045", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Tigers are large animals from India. (variant 45)", "context": "Tigers are large mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-046", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 46)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-047", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 47)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-048", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Tigers are large animals from India. (variant 48)", "context": "Tigers are large mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-049", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 49)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} +{"id": "5f-fal-live-050", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 50)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"} diff --git a/bench/fixtures/5f/falsification-v1.jsonl b/bench/fixtures/5f/falsification-v1.jsonl index efab6db..7eac575 100644 --- a/bench/fixtures/5f/falsification-v1.jsonl +++ b/bench/fixtures/5f/falsification-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 30, "notes": "Planted-bad-record fixtures with expected violation tags. Each fixture pinned to verifier_method_root so verifier-shape changes warn rather than false-fail. Phase 1c (2026-05-09): expanded 10→30."}} +{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 50, "notes": "Planted-bad-record fixtures with expected violation tags. Each fixture pinned to verifier_method_root so verifier-shape changes warn rather than false-fail. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50."}} {"id":"5f-fal-001","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_001","observed_violations":["WARRANT_MISSING","TITLE_MISMATCH"],"expected_reason":"WARRANT_MISSING","verifier_method_root":"warrant-v1","expected":"pass"} {"id":"5f-fal-002","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_002","observed_violations":["TITLE_MISMATCH"],"expected_reason":"TITLE_MISMATCH","verifier_method_root":"warrant-v1","expected":"pass"} {"id":"5f-fal-003","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_003","observed_violations":["TOO_MANY_CLAIMS"],"expected_reason":"TOO_MANY_CLAIMS","verifier_method_root":"warrant-v1","expected":"pass"} @@ -29,3 +29,23 @@ {"id": "5f-fal-028", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_028", "observed_violations": ["TITLE_MISMATCH", "BARE_NAME_CLAIM"], "expected_reason": "TOO_MANY_CLAIMS", "verifier_method_root": "warrant-v1", "expected": "fail"} {"id": "5f-fal-029", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_029", "observed_violations": [], "expected_reason": "BARE_NAME_CLAIM", "verifier_method_root": "warrant-v1", "expected": "fail"} {"id": "5f-fal-030", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_030", "observed_violations": ["NO_EVIDENCE_POINTER"], "expected_reason": "FORMAT_COLLAPSED", "verifier_method_root": "warrant-v1", "expected": "fail"} +{"id": "5f-fal-031", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_031", "observed_violations": ["WARRANT_MISSING"], "expected_reason": "WARRANT_MISSING", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-032", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_032", "observed_violations": ["TITLE_MISMATCH"], "expected_reason": "TITLE_MISMATCH", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-033", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_033", "observed_violations": ["TOO_MANY_CLAIMS"], "expected_reason": "TOO_MANY_CLAIMS", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-034", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_034", "observed_violations": ["BARE_NAME_CLAIM"], "expected_reason": "BARE_NAME_CLAIM", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-035", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_035", "observed_violations": ["FORMAT_COLLAPSED"], "expected_reason": "FORMAT_COLLAPSED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-036", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_036", "observed_violations": ["NO_EVIDENCE_POINTER"], "expected_reason": "NO_EVIDENCE_POINTER", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-037", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_037", "observed_violations": ["LAZY_ANCHOR_DEMOTED"], "expected_reason": "LAZY_ANCHOR_DEMOTED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-038", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_038", "observed_violations": ["POINTER_OVERFLOW_TRIMMED"], "expected_reason": "POINTER_OVERFLOW_TRIMMED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-039", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_039", "observed_violations": ["UNGROUNDED"], "expected_reason": "UNGROUNDED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-040", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_040", "observed_violations": ["HYBRID_PARAPHRASE"], "expected_reason": "HYBRID_PARAPHRASE", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-041", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_041", "observed_violations": ["WARRANT_MISSING"], "expected_reason": "WARRANT_MISSING", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-042", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_042", "observed_violations": ["TITLE_MISMATCH"], "expected_reason": "TITLE_MISMATCH", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-043", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_043", "observed_violations": ["TOO_MANY_CLAIMS"], "expected_reason": "TOO_MANY_CLAIMS", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-044", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_044", "observed_violations": ["BARE_NAME_CLAIM"], "expected_reason": "BARE_NAME_CLAIM", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-045", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_045", "observed_violations": ["FORMAT_COLLAPSED"], "expected_reason": "FORMAT_COLLAPSED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-046", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_046", "observed_violations": ["NO_EVIDENCE_POINTER"], "expected_reason": "NO_EVIDENCE_POINTER", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-047", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_047", "observed_violations": ["LAZY_ANCHOR_DEMOTED"], "expected_reason": "LAZY_ANCHOR_DEMOTED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-048", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_048", "observed_violations": ["POINTER_OVERFLOW_TRIMMED"], "expected_reason": "POINTER_OVERFLOW_TRIMMED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-049", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_049", "observed_violations": ["UNGROUNDED"], "expected_reason": "UNGROUNDED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-050", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_050", "observed_violations": ["HYBRID_PARAPHRASE"], "expected_reason": "HYBRID_PARAPHRASE", "verifier_method_root": "warrant-v1", "expected": "pass"} diff --git a/bench/fixtures/5f/feedback-loop-live-v1.jsonl b/bench/fixtures/5f/feedback-loop-live-v1.jsonl index 5252ee9..0ea7987 100644 --- a/bench/fixtures/5f/feedback-loop-live-v1.jsonl +++ b/bench/fixtures/5f/feedback-loop-live-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta": {"battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "task_count": 30, "notes": "Phase 1b.2: live_chain ops applied to a fresh temp arborist shard via append_audit + memory.snapshot + selfmodel.snapshot. Tests whether observations actually propagate into downstream audit_events / memory_branch_summaries / selfmodel_records."}} +{"_meta": {"battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "task_count": 50, "notes": "Phase 1b.2: live_chain ops applied to a fresh temp arborist shard via append_audit + memory.snapshot + selfmodel.snapshot. Tests whether observations actually propagate into downstream audit_events / memory_branch_summaries / selfmodel_records. Phase 1d (2026-05-09): expanded 30→50."}} {"id":"5f-fb-live-001","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live_chain":[{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"STRICT","violations":[]}},{"op":"memory_snapshot"}],"expected_delta":{"audit_event_type_present":["providence_write","memory_snapshot_landed"],"memory_branch_present":["audit-mode-distribution","falsification-state","failure-motif:violations"]},"expected":"pass"} {"id":"5f-fb-live-002","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live_chain":[{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"HYBRID","violations":["TITLE_MISMATCH"]}},{"op":"memory_snapshot"}],"expected_delta":{"audit_event_type_present":["providence_write","memory_snapshot_landed"],"body_substring_present":["TITLE_MISMATCH"]},"expected":"pass"} {"id":"5f-fb-live-003","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live_chain":[{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"STRICT","violations":[]}},{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"HYBRID","violations":["WARRANT_MISSING"]}},{"op":"memory_snapshot"}],"expected_delta":{"audit_event_type_present":["providence_write","memory_snapshot_landed"],"body_substring_present":["WARRANT_MISSING","STRICT","HYBRID"]},"expected":"pass"} @@ -29,3 +29,23 @@ {"id": "5f-fb-live-028", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "violations": ["NO_EVIDENCE_POINTER"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["NO_EVIDENCE_POINTER"]}, "expected": "pass"} {"id": "5f-fb-live-029", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "HYBRID", "violations": ["TITLE_MISMATCH"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["TITLE_MISMATCH"]}, "expected": "pass"} {"id": "5f-fb-live-030", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "UNGROUNDED", "violations": ["WARRANT_MISSING"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["WARRANT_MISSING"]}, "expected": "pass"} +{"id": "5f-fb-live-031", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 31}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-032", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 32}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-033", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 33}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-034", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 34}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-035", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 35}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-036", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 36}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-037", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 37}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-038", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 38}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-039", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 39}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-040", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 40}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-041", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 41}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-042", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 42}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-043", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 43}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-044", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 44}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-045", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 45}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-046", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 46}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-047", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 47}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-048", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 48}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-049", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 49}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} +{"id": "5f-fb-live-050", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 50}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"} diff --git a/bench/fixtures/5f/feedback-loop-v1.jsonl b/bench/fixtures/5f/feedback-loop-v1.jsonl index 26ed669..b8381bc 100644 --- a/bench/fixtures/5f/feedback-loop-v1.jsonl +++ b/bench/fixtures/5f/feedback-loop-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta": {"battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "task_count": 30, "notes": "Operation/observation chains. Final-step expected_delta must appear in aggregated observation feed; tests integration of observations into downstream state. Phase 1c (2026-05-09): expanded 10→30."}} +{"_meta": {"battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "task_count": 50, "notes": "Operation/observation chains. Final-step expected_delta must appear in aggregated observation feed; tests integration of observations into downstream state. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50."}} {"id":"5f-fb-001","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"snapshot","observation":"claim X is stale"},{"operation":"update_memory","observation":"memory branch marks X stale"},{"operation":"snapshot","observation":"X appears in stale set","expected_delta":"X appears in stale set"}],"expected":"pass"} {"id":"5f-fb-002","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"new fact alpha learned"},{"operation":"update_memory","observation":"memory branch records alpha"},{"operation":"snapshot","observation":"alpha appears in current memory","expected_delta":"alpha appears in current memory"}],"expected":"pass"} {"id":"5f-fb-003","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"verifier flagged warrant_missing"},{"operation":"falsify","observation":"providence record marked stale"},{"operation":"snapshot","observation":"stale record appears in falsification log","expected_delta":"stale record appears in falsification log"}],"expected":"pass"} @@ -29,3 +29,23 @@ {"id": "5f-fb-028", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "ForkScore parent baseline"}, {"operation": "update_memory", "observation": "baseline pinned"}, {"operation": "snapshot", "observation": "baseline visible to scorer", "expected_delta": "baseline visible to scorer"}], "expected": "pass"} {"id": "5f-fb-029", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact delta seen"}, {"operation": "update_memory", "observation": "memory updated for unrelated thing"}, {"operation": "snapshot", "observation": "no mention of delta", "expected_delta": "delta visible in snapshot"}], "expected": "fail"} {"id": "5f-fb-030", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "snapshot", "observation": "stale set s0"}, {"operation": "update_memory", "observation": "tried to update but failed"}, {"operation": "snapshot", "observation": "stale set unchanged", "expected_delta": "expected delta unrealized"}], "expected": "fail"} +{"id": "5f-fb-031", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-31 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-31"}, {"operation": "snapshot", "observation": "fact-31 in current memory", "expected_delta": "fact-31 in current memory"}], "expected": "pass"} +{"id": "5f-fb-032", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-32 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-32"}, {"operation": "snapshot", "observation": "fact-32 in current memory", "expected_delta": "fact-32 in current memory"}], "expected": "pass"} +{"id": "5f-fb-033", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-33 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-33"}, {"operation": "snapshot", "observation": "fact-33 in current memory", "expected_delta": "fact-33 in current memory"}], "expected": "pass"} +{"id": "5f-fb-034", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-34 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-34"}, {"operation": "snapshot", "observation": "fact-34 in current memory", "expected_delta": "fact-34 in current memory"}], "expected": "pass"} +{"id": "5f-fb-035", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-35 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-35"}, {"operation": "snapshot", "observation": "fact-35 in current memory", "expected_delta": "fact-35 in current memory"}], "expected": "pass"} +{"id": "5f-fb-036", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-36 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-36"}, {"operation": "snapshot", "observation": "fact-36 in current memory", "expected_delta": "fact-36 in current memory"}], "expected": "pass"} +{"id": "5f-fb-037", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-37 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-37"}, {"operation": "snapshot", "observation": "fact-37 in current memory", "expected_delta": "fact-37 in current memory"}], "expected": "pass"} +{"id": "5f-fb-038", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-38 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-38"}, {"operation": "snapshot", "observation": "fact-38 in current memory", "expected_delta": "fact-38 in current memory"}], "expected": "pass"} +{"id": "5f-fb-039", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-39 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-39"}, {"operation": "snapshot", "observation": "fact-39 in current memory", "expected_delta": "fact-39 in current memory"}], "expected": "pass"} +{"id": "5f-fb-040", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-40 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-40"}, {"operation": "snapshot", "observation": "fact-40 in current memory", "expected_delta": "fact-40 in current memory"}], "expected": "pass"} +{"id": "5f-fb-041", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-41 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-41"}, {"operation": "snapshot", "observation": "fact-41 in current memory", "expected_delta": "fact-41 in current memory"}], "expected": "pass"} +{"id": "5f-fb-042", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-42 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-42"}, {"operation": "snapshot", "observation": "fact-42 in current memory", "expected_delta": "fact-42 in current memory"}], "expected": "pass"} +{"id": "5f-fb-043", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-43 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-43"}, {"operation": "snapshot", "observation": "fact-43 in current memory", "expected_delta": "fact-43 in current memory"}], "expected": "pass"} +{"id": "5f-fb-044", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-44 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-44"}, {"operation": "snapshot", "observation": "fact-44 in current memory", "expected_delta": "fact-44 in current memory"}], "expected": "pass"} +{"id": "5f-fb-045", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-45 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-45"}, {"operation": "snapshot", "observation": "fact-45 in current memory", "expected_delta": "fact-45 in current memory"}], "expected": "pass"} +{"id": "5f-fb-046", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-46 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-46"}, {"operation": "snapshot", "observation": "fact-46 in current memory", "expected_delta": "fact-46 in current memory"}], "expected": "pass"} +{"id": "5f-fb-047", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-47 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-47"}, {"operation": "snapshot", "observation": "fact-47 in current memory", "expected_delta": "fact-47 in current memory"}], "expected": "pass"} +{"id": "5f-fb-048", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-48 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-48"}, {"operation": "snapshot", "observation": "fact-48 in current memory", "expected_delta": "fact-48 in current memory"}], "expected": "pass"} +{"id": "5f-fb-049", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-49 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-49"}, {"operation": "snapshot", "observation": "fact-49 in current memory", "expected_delta": "fact-49 in current memory"}], "expected": "pass"} +{"id": "5f-fb-050", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-50 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-50"}, {"operation": "snapshot", "observation": "fact-50 in current memory", "expected_delta": "fact-50 in current memory"}], "expected": "pass"} diff --git a/bench/fixtures/5f/finetuning-live-v1.jsonl b/bench/fixtures/5f/finetuning-live-v1.jsonl index 8d534ce..a37a9c5 100644 --- a/bench/fixtures/5f/finetuning-live-v1.jsonl +++ b/bench/fixtures/5f/finetuning-live-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1", "task_count": 30, "notes": "Phase 1b.2: live=true \u2192 write parent + child SelfModel + capability_claim to a fresh temp shard via real arborist.selfmodel.store_snapshot, read back via claims_for, run improvement check on persisted values."}} +{"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1", "task_count": 50, "notes": "Phase 1b.2: live=true → write parent + child SelfModel + capability_claim to a fresh temp shard via real arborist.selfmodel.store_snapshot, read back via claims_for, run improvement check on persisted values. Phase 1d (2026-05-09): expanded 30→50."}} {"id":"5f-ft-live-001","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","live":true,"target_capability":"strict_rate","parent_measured_value":0.45,"child_measured_value":0.65,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1000,"max_storage_delta_bytes":1000000},"expected":"pass"} {"id":"5f-ft-live-002","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","live":true,"target_capability":"directive_coverage","parent_measured_value":0.40,"child_measured_value":0.55,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":1500,"max_storage_delta_bytes":2000000},"expected":"pass"} {"id":"5f-ft-live-003","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","live":true,"target_capability":"5T-time-rate","parent_measured_value":0.60,"child_measured_value":0.85,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":2000,"max_storage_delta_bytes":3000000},"expected":"pass"} @@ -29,3 +29,23 @@ {"id": "5f-ft-live-028", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5R-restore", "parent_measured_value": 0.45, "child_measured_value": 0.7, "expected_improvement_min": 0.1, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} {"id": "5f-ft-live-029", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5R-replicate", "parent_measured_value": 0.55, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "fail"} {"id": "5f-ft-live-030", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5R-resonate", "parent_measured_value": 0.6, "child_measured_value": 0.8, "expected_improvement_min": 0.15, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"} +{"id": "5f-ft-live-031", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.44, "child_measured_value": 0.59, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-032", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.48, "child_measured_value": 0.63, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-033", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.52, "child_measured_value": 0.67, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-034", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.56, "child_measured_value": 0.71, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-035", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.4, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-036", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.44, "child_measured_value": 0.59, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-037", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.48, "child_measured_value": 0.63, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-038", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.52, "child_measured_value": 0.67, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-039", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.56, "child_measured_value": 0.71, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-040", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.4, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-041", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.44, "child_measured_value": 0.59, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-042", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.48, "child_measured_value": 0.63, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-043", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.52, "child_measured_value": 0.67, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-044", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.56, "child_measured_value": 0.71, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-045", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.4, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-046", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.44, "child_measured_value": 0.59, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-047", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.48, "child_measured_value": 0.63, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-048", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.52, "child_measured_value": 0.67, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-049", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.56, "child_measured_value": 0.71, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-live-050", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.4, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} diff --git a/bench/fixtures/5f/finetuning-v1.jsonl b/bench/fixtures/5f/finetuning-v1.jsonl index 48e2395..4abf271 100644 --- a/bench/fixtures/5f/finetuning-v1.jsonl +++ b/bench/fixtures/5f/finetuning-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1", "task_count": 30, "notes": "Synthetic parent→child SelfModel measured-value pairs. Phase 1b.2 will pull from real shard SelfModel chains. Phase 1c (2026-05-09): expanded 10→30."}} +{"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1", "task_count": 50, "notes": "Synthetic parent→child SelfModel measured-value pairs. Phase 1b.2 will pull from real shard SelfModel chains. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50."}} {"id":"5f-ft-001","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_001","child_selfmodel":"SM_CHILD_001","target_capability":"CAP-5S-SYLLOGISM","parent_measured_value":0.45,"child_measured_value":0.65,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1000,"max_storage_delta_bytes":1000000},"expected":"pass"} {"id":"5f-ft-002","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_002","child_selfmodel":"SM_CHILD_002","target_capability":"CAP-5T-TIME","parent_measured_value":0.40,"child_measured_value":0.55,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1500,"max_storage_delta_bytes":2000000},"expected":"pass"} {"id":"5f-ft-003","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_003","child_selfmodel":"SM_CHILD_003","target_capability":"CAP-5F-FORMULATE","parent_measured_value":0.60,"child_measured_value":0.85,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":2000,"max_storage_delta_bytes":3000000},"expected":"pass"} @@ -29,3 +29,23 @@ {"id": "5f-ft-028", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_028", "child_selfmodel": "SM_CHILD_028", "target_capability": "CAP-5R-REFINE", "parent_measured_value": 0.64, "child_measured_value": 0.66, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2400, "max_storage_delta_bytes": 3800000}, "expected": "fail"} {"id": "5f-ft-029", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_029", "child_selfmodel": "SM_CHILD_029", "target_capability": "CAP-CANONICAL-MATH", "parent_measured_value": 0.66, "child_measured_value": 0.85, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2450, "max_storage_delta_bytes": 3900000}, "expected": "pass"} {"id": "5f-ft-030", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_030", "child_selfmodel": "SM_CHILD_030", "target_capability": "CAP-CANONICAL-LOGIC", "parent_measured_value": 0.68, "child_measured_value": 0.88, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2500, "max_storage_delta_bytes": 4000000}, "expected": "pass"} +{"id": "5f-ft-031", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_031", "child_selfmodel": "SM_CHILD_031", "target_capability": "CAP-5S-SYNTAX", "parent_measured_value": 0.33, "child_measured_value": 0.43, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-032", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_032", "child_selfmodel": "SM_CHILD_032", "target_capability": "CAP-5S-SEMANTICS", "parent_measured_value": 0.36, "child_measured_value": 0.46, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-033", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_033", "child_selfmodel": "SM_CHILD_033", "target_capability": "CAP-5T-TIME", "parent_measured_value": 0.39, "child_measured_value": 0.49, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-034", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_034", "child_selfmodel": "SM_CHILD_034", "target_capability": "CAP-5F-FUNCTION", "parent_measured_value": 0.42, "child_measured_value": 0.52, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-035", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_035", "child_selfmodel": "SM_CHILD_035", "target_capability": "CAP-5F-FALSIFICATION", "parent_measured_value": 0.45, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-036", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_036", "child_selfmodel": "SM_CHILD_036", "target_capability": "CAP-5R-REACT", "parent_measured_value": 0.48, "child_measured_value": 0.58, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-037", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_037", "child_selfmodel": "SM_CHILD_037", "target_capability": "CAP-CANONICAL-MATH", "parent_measured_value": 0.51, "child_measured_value": 0.61, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-038", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_038", "child_selfmodel": "SM_CHILD_038", "target_capability": "CAP-CANONICAL-LOGIC", "parent_measured_value": 0.54, "child_measured_value": 0.64, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-039", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_039", "child_selfmodel": "SM_CHILD_039", "target_capability": "CAP-CANONICAL-ALGEBRA", "parent_measured_value": 0.57, "child_measured_value": 0.67, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-040", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_040", "child_selfmodel": "SM_CHILD_040", "target_capability": "CAP-CANONICAL-CALCULUS", "parent_measured_value": 0.3, "child_measured_value": 0.4, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-041", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_041", "child_selfmodel": "SM_CHILD_041", "target_capability": "CAP-5S-SYNTAX", "parent_measured_value": 0.33, "child_measured_value": 0.43, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-042", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_042", "child_selfmodel": "SM_CHILD_042", "target_capability": "CAP-5S-SEMANTICS", "parent_measured_value": 0.36, "child_measured_value": 0.46, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-043", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_043", "child_selfmodel": "SM_CHILD_043", "target_capability": "CAP-5T-TIME", "parent_measured_value": 0.39, "child_measured_value": 0.49, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-044", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_044", "child_selfmodel": "SM_CHILD_044", "target_capability": "CAP-5F-FUNCTION", "parent_measured_value": 0.42, "child_measured_value": 0.52, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-045", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_045", "child_selfmodel": "SM_CHILD_045", "target_capability": "CAP-5F-FALSIFICATION", "parent_measured_value": 0.45, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-046", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_046", "child_selfmodel": "SM_CHILD_046", "target_capability": "CAP-5R-REACT", "parent_measured_value": 0.48, "child_measured_value": 0.58, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-047", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_047", "child_selfmodel": "SM_CHILD_047", "target_capability": "CAP-CANONICAL-MATH", "parent_measured_value": 0.51, "child_measured_value": 0.61, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-048", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_048", "child_selfmodel": "SM_CHILD_048", "target_capability": "CAP-CANONICAL-LOGIC", "parent_measured_value": 0.54, "child_measured_value": 0.64, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-049", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_049", "child_selfmodel": "SM_CHILD_049", "target_capability": "CAP-CANONICAL-ALGEBRA", "parent_measured_value": 0.57, "child_measured_value": 0.67, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} +{"id": "5f-ft-050", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_050", "child_selfmodel": "SM_CHILD_050", "target_capability": "CAP-CANONICAL-CALCULUS", "parent_measured_value": 0.3, "child_measured_value": 0.4, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"} diff --git a/bench/fixtures/5f/formulate-live-v1.jsonl b/bench/fixtures/5f/formulate-live-v1.jsonl index 3166a77..45b7179 100644 --- a/bench/fixtures/5f/formulate-live-v1.jsonl +++ b/bench/fixtures/5f/formulate-live-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1", "task_count": 30, "notes": "Phase 1b.2 live-derived fixtures: input_text routes through arborist.qa.parse_claims.parse_pointer_claims; expected_lattice is the gold the live parser must produce. No produced_lattice embedded \u2014 runner derives it."}} +{"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1", "task_count": 50, "notes": "Phase 1b.2 live-derived fixtures: input_text routes through arborist.qa.parse_claims.parse_pointer_claims; expected_lattice is the gold the live parser must produce. No produced_lattice embedded — runner derives it. Phase 1d (2026-05-09): expanded 30→50."}} {"id":"5f-form-live-001","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- One claim. [E1]\n- Two claim. [E2]","expected_lattice":{"claim_count":2,"claims":[{"claim_text":"One claim.","pointer_ids":["E1"]},{"claim_text":"Two claim.","pointer_ids":["E2"]}]},"expected":"pass"} {"id":"5f-form-live-002","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Multi pointer claim. [E1, E2]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"Multi pointer claim.","pointer_ids":["E1","E2"]}]},"expected":"pass"} {"id":"5f-form-live-003","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- A claim. [E1]\n- Another. [E2]\n- Third. [E3]","expected_lattice":{"claim_count":3,"claims":[{"claim_text":"A claim.","pointer_ids":["E1"]},{"claim_text":"Another.","pointer_ids":["E2"]},{"claim_text":"Third.","pointer_ids":["E3"]}]},"expected":"pass"} @@ -29,3 +29,23 @@ {"id": "5f-form-live-028", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Hard regression flag triggers REJECT. [E1]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "Hard regression flag triggers REJECT.", "pointer_ids": ["E1"]}]}, "expected": "pass"} {"id": "5f-form-live-029", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- neg_inf efficiency triggers REJECT. [E1]\n- inf efficiency adds capped bonus. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "neg_inf efficiency triggers REJECT.", "pointer_ids": ["E1"]}, {"claim_text": "inf efficiency adds capped bonus.", "pointer_ids": ["E2"]}]}, "expected": "pass"} {"id": "5f-form-live-030", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- 5F bridges synthetic to live across all five sub-batteries. [E1]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "5F bridges synthetic to live across all five sub-batteries.", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-live-031", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}]}, "expected": "pass"} +{"id": "5f-form-live-032", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}]}, "expected": "pass"} +{"id": "5f-form-live-033", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}]}, "expected": "pass"} +{"id": "5f-form-live-034", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}, {"claim_text": "Claim 6.", "pointer_ids": ["E6"]}]}, "expected": "pass"} +{"id": "5f-form-live-035", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}]}, "expected": "pass"} +{"id": "5f-form-live-036", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}]}, "expected": "pass"} +{"id": "5f-form-live-037", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}]}, "expected": "pass"} +{"id": "5f-form-live-038", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}]}, "expected": "pass"} +{"id": "5f-form-live-039", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}, {"claim_text": "Claim 6.", "pointer_ids": ["E6"]}]}, "expected": "pass"} +{"id": "5f-form-live-040", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}]}, "expected": "pass"} +{"id": "5f-form-live-041", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}]}, "expected": "pass"} +{"id": "5f-form-live-042", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}]}, "expected": "pass"} +{"id": "5f-form-live-043", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}]}, "expected": "pass"} +{"id": "5f-form-live-044", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}, {"claim_text": "Claim 6.", "pointer_ids": ["E6"]}]}, "expected": "pass"} +{"id": "5f-form-live-045", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}]}, "expected": "pass"} +{"id": "5f-form-live-046", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}]}, "expected": "pass"} +{"id": "5f-form-live-047", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}]}, "expected": "pass"} +{"id": "5f-form-live-048", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}]}, "expected": "pass"} +{"id": "5f-form-live-049", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}, {"claim_text": "Claim 6.", "pointer_ids": ["E6"]}]}, "expected": "pass"} +{"id": "5f-form-live-050", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}]}, "expected": "pass"} diff --git a/bench/fixtures/5f/formulate-v1.jsonl b/bench/fixtures/5f/formulate-v1.jsonl index c329296..26d93f7 100644 --- a/bench/fixtures/5f/formulate-v1.jsonl +++ b/bench/fixtures/5f/formulate-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1", "task_count": 30, "notes": "Phase 1a fixtures embed produced + expected lattices for structural matching. Phase 1b.2 will route through arborist.qa.parse_claims. Phase 1c (2026-05-09): expanded 10→30."}} +{"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1", "task_count": 50, "notes": "Phase 1a fixtures embed produced + expected lattices for structural matching. Phase 1b.2 will route through arborist.qa.parse_claims. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50."}} {"id":"5f-form-001","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"The system saved one claim with one pointer.","produced_lattice":{"claims":[{"claim_text":"The system saved one claim with one pointer.","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"The system saved one claim with one pointer.","pointer_ids":["E1"]}]},"expected":"pass"} {"id":"5f-form-002","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Two claims with two pointers each.","produced_lattice":{"claims":[{"claim_text":"first claim with pointers","pointer_ids":["E1","E2"]},{"claim_text":"second claim with pointers","pointer_ids":["E3","E4"]}]},"expected_lattice":{"claim_count":2,"claims":[{"claim_text":"first claim with pointers","pointer_ids":["E1","E2"]},{"claim_text":"second claim with pointers","pointer_ids":["E3","E4"]}]},"expected":"pass"} {"id":"5f-form-003","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Detect missing pointer claim.","produced_lattice":{"claims":[{"claim_text":"a claim","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"a claim","pointer_ids":["E1","E2"]}]},"expected":"fail"} @@ -29,3 +29,23 @@ {"id": "5f-form-028", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "One claim, two pointers.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}]}, "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}]}, "expected": "pass"} {"id": "5f-form-029", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Six claims about historical events.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected": "pass"} {"id": "5f-form-030", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Two claims about astronomy.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "claim 2", "pointer_ids": ["E3", "E4"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "claim 2", "pointer_ids": ["E3", "E4"]}]}, "expected": "pass"} +{"id": "5f-form-031", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 31 with 3 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-032", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 32 with 4 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-033", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 33 with 5 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-034", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 34 with 6 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-035", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 35 with 2 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-036", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 36 with 3 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-037", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 37 with 4 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-038", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 38 with 5 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-039", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 39 with 6 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-040", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 40 with 2 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-041", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 41 with 3 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-042", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 42 with 4 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-043", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 43 with 5 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-044", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 44 with 6 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-045", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 45 with 2 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-046", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 46 with 3 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-047", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 47 with 4 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-048", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 48 with 5 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-049", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 49 with 6 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-050", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 50 with 2 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected": "pass"} diff --git a/bench/fixtures/5f/function-live-v1.jsonl b/bench/fixtures/5f/function-live-v1.jsonl index 05db40e..bc33b27 100644 --- a/bench/fixtures/5f/function-live-v1.jsonl +++ b/bench/fixtures/5f/function-live-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta": {"battery": "5f", "sub_battery": "function", "version": "v1", "task_count": 30, "notes": "Phase 1b.2: input_text routes through arborist.qa.parse_claims; same shape evaluators used as Phase 1a but on live-parsed output."}} +{"_meta": {"battery": "5f", "sub_battery": "function", "version": "v1", "task_count": 50, "notes": "Phase 1b.2: input_text routes through arborist.qa.parse_claims; same shape evaluators used as Phase 1a but on live-parsed output. Phase 1d (2026-05-09): expanded 30→50."}} {"id":"5f-fn-live-001","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","input_text":"- One claim. [E1]\n- Two claim. [E2]","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"expected":"pass"} {"id":"5f-fn-live-002","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","input_text":"- Single claim. [E1]","evaluator":"shape_match","expected_shape":{"claim_count":1,"pointers_required":true},"expected":"pass"} {"id":"5f-fn-live-003","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","input_text":"- Three. [E1]\n- Three. [E2]\n- Three. [E3]","evaluator":"shape_match","expected_shape":{"claim_count":3,"pointers_required":true},"expected":"pass"} @@ -29,3 +29,23 @@ {"id": "5f-fn-live-028", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Mismatch test. [E1]\n- Other. [E2]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "fail"} {"id": "5f-fn-live-029", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Reject this shape. [E1, E2]", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E1"]}, "expected": "fail"} {"id": "5f-fn-live-030", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Edge case. [E99]", "evaluator": "shape_match", "expected_shape": {"claim_count": 1, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-031", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-032", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-033", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-034", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-035", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-036", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-037", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-038", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-039", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-040", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-041", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-042", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-043", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-044", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-045", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-046", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-047", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-048", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-049", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "expected": "pass"} +{"id": "5f-fn-live-050", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "expected": "pass"} diff --git a/bench/fixtures/5f/function-v1.jsonl b/bench/fixtures/5f/function-v1.jsonl index fd7dac0..3580b4c 100644 --- a/bench/fixtures/5f/function-v1.jsonl +++ b/bench/fixtures/5f/function-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta": {"battery": "5f", "sub_battery": "function", "version": "v1", "task_count": 30, "notes": "Phase 1a: ten seed tasks across shape_match/pointer_set_match/threshold_on_metric evaluators. Phase 1b expands to 30+. Phase 1c (2026-05-09): expanded 10→30."}} +{"_meta": {"battery": "5f", "sub_battery": "function", "version": "v1", "task_count": 50, "notes": "Phase 1a: ten seed tasks across shape_match/pointer_set_match/threshold_on_metric evaluators. Phase 1b expands to 30+. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50."}} {"id":"5f-fn-001","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Return exactly two claim-lattice claims with evidence pointers.","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"first claim","pointer_ids":["E1"]},{"claim_text":"second claim","pointer_ids":["E2"]}]},"expected":"pass"} {"id":"5f-fn-002","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Return exactly three claims with pointers.","evaluator":"shape_match","expected_shape":{"claim_count":3,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"a","pointer_ids":["E1"]},{"claim_text":"b","pointer_ids":["E2"]},{"claim_text":"c","pointer_ids":["E3"]}]},"expected":"pass"} {"id":"5f-fn-003","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Detect missing pointers in claims.","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"a","pointer_ids":[]},{"claim_text":"b","pointer_ids":["E2"]}]},"expected":"fail"} @@ -29,3 +29,23 @@ {"id": "5f-fn-028", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on hybrid_rate.", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "hybrid_rate", "threshold": 0.3}, "produced_output": {"hybrid_rate": 0.32}, "expected": "pass"} {"id": "5f-fn-029", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on strict_rate (sub-floor).", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "strict_rate", "threshold": 0.55}, "produced_output": {"strict_rate": 0.4}, "expected": "fail"} {"id": "5f-fn-030", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on strict_rate.", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "strict_rate", "threshold": 0.6}, "produced_output": {"strict_rate": 0.6}, "expected": "pass"} +{"id": "5f-fn-031", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 3 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}]}, "expected": "pass"} +{"id": "5f-fn-032", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 4 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected": "pass"} +{"id": "5f-fn-033", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 5 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}]}, "expected": "pass"} +{"id": "5f-fn-034", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 6 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}]}, "expected": "pass"} +{"id": "5f-fn-035", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 7 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 7, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}, {"claim_text": "claim 7", "pointer_ids": ["E7"]}]}, "expected": "pass"} +{"id": "5f-fn-036", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 2 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}]}, "expected": "pass"} +{"id": "5f-fn-037", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 3 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}]}, "expected": "pass"} +{"id": "5f-fn-038", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 4 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected": "pass"} +{"id": "5f-fn-039", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 5 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}]}, "expected": "pass"} +{"id": "5f-fn-040", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 6 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}]}, "expected": "pass"} +{"id": "5f-fn-041", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 7 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 7, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}, {"claim_text": "claim 7", "pointer_ids": ["E7"]}]}, "expected": "pass"} +{"id": "5f-fn-042", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 2 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}]}, "expected": "pass"} +{"id": "5f-fn-043", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 3 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}]}, "expected": "pass"} +{"id": "5f-fn-044", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 4 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected": "pass"} +{"id": "5f-fn-045", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 5 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}]}, "expected": "pass"} +{"id": "5f-fn-046", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 6 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}]}, "expected": "pass"} +{"id": "5f-fn-047", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 7 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 7, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}, {"claim_text": "claim 7", "pointer_ids": ["E7"]}]}, "expected": "pass"} +{"id": "5f-fn-048", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 2 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}]}, "expected": "pass"} +{"id": "5f-fn-049", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 3 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}]}, "expected": "pass"} +{"id": "5f-fn-050", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 4 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected": "pass"} diff --git a/docs/TICKETS.md b/docs/TICKETS.md index eb037c5..41b4ace 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -63,10 +63,10 @@ Newest first. Update on every open/close. |----------|------------------------------------------------|-----------------------|------------|-----------| | #000030 | Math π* expansion: SymPy substrate (algebra · calculus · linalg) | closed · Phases 1+2 landed 2026-05-09 (3-7 future work) | 2026-05-09 | — | | #000029 | Claim-pack source (axiom/theorem JSON bundles) | closed · landed 2026-05-09 | 2026-05-09 | — | -| #000028 | Multi-modality witness for canonical shapes | closed · landed 2026-05-09 (STRICT-WITNESSED reachable post-#000027) | 2026-05-08 | — | +| #000028 | Multi-modality witness for canonical shapes | closed · landed 2026-05-09 + follow-ups (capital ledger · sample rate) | 2026-05-08 | — | | #000027 | Canonical projections persist to providence_cache | closed · landed 2026-05-09 | 2026-05-08 | — | | #000026 | Real-shard workload baseline + search latency | in progress · Phase 1 + 2 + 3 landed 2026-05-09 | 2026-05-08 | — | -| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a + 1b.2 + 1c landed 2026-05-09 | 2026-05-07 | — | +| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a + 1b.2 + 1c + 1d landed 2026-05-09 | 2026-05-07 | — | | #000024 | 5T Phase 1b + Dav1DPrometheus vocabulary alignment | closed · landed 2026-05-08 | 2026-05-07 | — | | #000023 | 5S Phase 1b: Syllogism · Synthesis · Semiotics | closed · landed 2026-05-08 | 2026-05-07 | — | | #000022 | Adapter LossReport (PRD I9 analogue) | closed · landed 2026-05-07 | 2026-05-07 | — | diff --git a/tests/test_bench_batteries.py b/tests/test_bench_batteries.py index 0f1fbaa..37e5cc7 100644 --- a/tests/test_bench_batteries.py +++ b/tests/test_bench_batteries.py @@ -228,31 +228,31 @@ def test_5f_function_runs(): assert res.battery == "5f" assert res.sub_battery == "function" # Phase 1c (2026-05-09) — fixture catalog expanded 10 → 30. - assert res.pass_count == 30 + assert res.pass_count == 50 # Phase 1d assert res.metrics["function_pass_rate"] == 1.0 def test_5f_finetuning_runs(): res = b_5f.run_finetuning(F5F / "finetuning-v1.jsonl") - assert res.pass_count == 30 + assert res.pass_count == 50 # Phase 1d assert res.metrics["adaptation_improvement_rate"] == 1.0 def test_5f_falsification_runs(): res = b_5f.run_falsification(F5F / "falsification-v1.jsonl") - assert res.pass_count == 30 + assert res.pass_count == 50 # Phase 1d assert res.metrics["error_detection_rate"] == 1.0 def test_5f_formulate_runs(): res = b_5f.run_formulate(F5F / "formulate-v1.jsonl") - assert res.pass_count == 30 + assert res.pass_count == 50 # Phase 1d assert res.metrics["structural_match_rate"] == 1.0 def test_5f_feedback_loop_runs(): res = b_5f.run_feedback_loop(F5F / "feedback-loop-v1.jsonl") - assert res.pass_count == 30 + assert res.pass_count == 50 # Phase 1d assert res.metrics["integration_coverage_rate"] == 1.0 @@ -422,7 +422,7 @@ def test_5f_formulate_live_path_routes_through_parse_claims(): runner derives produced_lattice via parse_pointer_claims and matches against expected_lattice.""" res = b_5f.run_formulate(F5F / "formulate-live-v1.jsonl") - assert res.pass_count == 30 + assert res.pass_count == 50 # Phase 1d assert res.metrics["structural_match_rate"] == 1.0 # Every task ran through the live path. for t in res.per_task: @@ -434,7 +434,7 @@ def test_5f_formulate_embedded_path_still_works(): after the Phase 1b.2 wire-up + Phase 1c expansion. Backward compat invariant.""" res = b_5f.run_formulate(F5F / "formulate-v1.jsonl") - assert res.pass_count == 30 # Phase 1c — expanded 10 → 30 + assert res.pass_count == 50 # Phase 1d — expanded 10 → 30 for t in res.per_task: assert t.detail["source"] == "embedded" @@ -478,7 +478,7 @@ def test_5f_feedback_loop_live_path_writes_real_audit_events(): + memory.snapshot; expected_delta predicates verified against the resulting audit_events / memory_branch_summaries.""" res = b_5f.run_feedback_loop(F5F / "feedback-loop-live-v1.jsonl") - assert res.pass_count == 30 + assert res.pass_count == 50 # Phase 1d # Every passing task must report source=live. for t in res.per_task: if t.passed: @@ -487,7 +487,7 @@ def test_5f_feedback_loop_live_path_writes_real_audit_events(): def test_5f_feedback_loop_embedded_path_still_works(): res = b_5f.run_feedback_loop(F5F / "feedback-loop-v1.jsonl") - assert res.pass_count == 30 # Phase 1c + assert res.pass_count == 50 # Phase 1d for t in res.per_task: assert t.detail["source"] == "embedded" @@ -552,7 +552,7 @@ def test_5f_live_feedback_chain_audit_chain_intact(): def test_5f_function_live_path_routes_through_parse_claims(): res = b_5f.run_function(F5F / "function-live-v1.jsonl") - assert res.pass_count == 30 + assert res.pass_count == 50 # Phase 1d for t in res.per_task: if t.passed: assert t.detail["source"] == "live" @@ -560,7 +560,7 @@ def test_5f_function_live_path_routes_through_parse_claims(): def test_5f_function_embedded_path_still_works(): res = b_5f.run_function(F5F / "function-v1.jsonl") - assert res.pass_count == 30 # Phase 1c + assert res.pass_count == 50 # Phase 1d for t in res.per_task: assert t.detail["source"] == "embedded" @@ -576,7 +576,7 @@ def test_5f_function_live_helper_uses_real_parser(): def test_5f_finetuning_live_path_round_trips_selfmodel(): res = b_5f.run_finetuning(F5F / "finetuning-live-v1.jsonl") - assert res.pass_count == 30 + assert res.pass_count == 50 # Phase 1d for t in res.per_task: if t.passed: assert t.detail["source"] == "live" @@ -584,7 +584,7 @@ def test_5f_finetuning_live_path_round_trips_selfmodel(): def test_5f_finetuning_embedded_path_still_works(): res = b_5f.run_finetuning(F5F / "finetuning-v1.jsonl") - assert res.pass_count == 30 # Phase 1c + assert res.pass_count == 50 # Phase 1d for t in res.per_task: assert t.detail["source"] == "embedded" @@ -610,7 +610,7 @@ def test_5f_finetuning_live_helper_persists_real_selfmodel(): def test_5f_falsification_live_path_routes_through_verify_quotes(): res = b_5f.run_falsification(F5F / "falsification-live-v1.jsonl") - assert res.pass_count == 30 + assert res.pass_count == 50 # Phase 1d for t in res.per_task: if t.passed: assert t.detail["source"] == "live" @@ -618,7 +618,7 @@ def test_5f_falsification_live_path_routes_through_verify_quotes(): def test_5f_falsification_embedded_path_still_works(): res = b_5f.run_falsification(F5F / "falsification-v1.jsonl") - assert res.pass_count == 30 # Phase 1c + assert res.pass_count == 50 # Phase 1d for t in res.per_task: assert t.detail["source"] == "embedded" diff --git a/tests/test_inspect.py b/tests/test_inspect.py index 4595cdd..7a50dec 100644 --- a/tests/test_inspect.py +++ b/tests/test_inspect.py @@ -249,10 +249,14 @@ def test_classify_no_overlap_full_invention(): def _seed_record(qa_db: Path, *, cache_key: str, sources: list[dict], - unverified: list[str]) -> None: + unverified: list[str], + question_text: str = "what is foo?", + answer_text: str = "answer here") -> None: """Insert a minimal providence_cache row with a merkle_proof that points at the given sources. Caller has already populated each - shard with the actual document + chunks.""" + shard with the actual document + chunks. ``question_text`` / + ``answer_text`` default to the canonical pre-warrant fixture + pair; tests that exercise warrant-shape sidecars override them.""" conn = connect(qa_db) try: with transaction(conn): @@ -278,8 +282,8 @@ def _seed_record(qa_db: Path, *, cache_key: str, sources: list[dict], cache_key, "00" * 32, "qh", - "what is foo?", - "answer here", + question_text, + answer_text, json.dumps({"sources": sources}, ensure_ascii=False), "mh", "ch", @@ -764,3 +768,87 @@ def test_register_metaphor_dictionary_idempotent(tmp_path): finally: m._extra_dict_paths = saved_extra m._english_wordlist_cache = saved_cache + + +# --------------------------------------------------------------------- +# Authorship warrant ladder (#000026 Phase 3) — wired into inspect() +# --------------------------------------------------------------------- + + +def test_inspect_includes_authorship_field(tmp_path): + """Every inspect() result carries an `authorship` sidecar dict. + For non-authorship questions, tier is NO_AUTHORSHIP_SIGNAL.""" + qa_db = tmp_path / "qa.db" + shard = tmp_path / "001.db" + DOC = "ab" * 32 + _seed_doc( + shard, + document_root=DOC, + document_uri="https://example.com/x", + chunk_text="Pikachu can store electricity in its cheeks.", + ) + _seed_record( + qa_db, cache_key="01" * 32, + sources=[{ + "document_root": DOC, "document_uri": "https://example.com/x", + "title": "X", "shard": shard.name, "chunk_idx": 0, + }], + unverified=[], + ) + result = inspect_cache_key("01" * 32, qa_db=qa_db, shards_dir=tmp_path) + assert "authorship" in result + assert result["authorship"]["tier"] == "NO_AUTHORSHIP_SIGNAL" + + +def test_inspect_authorship_copyright_footer_tier(tmp_path): + """Authorship-shaped question + copyright-footer chunk text → + AUTHOR_COPYRIGHT_FOOTER tier (the canonical virt-back case).""" + qa_db = tmp_path / "qa.db" + shard = tmp_path / "001.db" + DOC = "cd" * 32 + _seed_doc( + shard, + document_root=DOC, + document_uri="https://example.com/virt-back", + chunk_text="virt-back is a backup utility.\n\n© Russell Ballestrini", + ) + _seed_record( + qa_db, cache_key="02" * 32, + sources=[{ + "document_root": DOC, + "document_uri": "https://example.com/virt-back", + "title": "virt-back", "shard": shard.name, "chunk_idx": 0, + }], + unverified=[], + question_text="who wrote virt-back?", + answer_text="Russell Ballestrini wrote virt-back.", + ) + result = inspect_cache_key("02" * 32, qa_db=qa_db, shards_dir=tmp_path) + assert result["authorship"]["tier"] == "AUTHOR_COPYRIGHT_FOOTER" + assert "Russell Ballestrini" in result["authorship"]["candidate_names"] + + +def test_inspect_authorship_repo_owner_tier(tmp_path): + """Repo-URL chunks fire tier 2 (REPOSITORY_OWNER).""" + qa_db = tmp_path / "qa.db" + shard = tmp_path / "001.db" + DOC = "ef" * 32 + _seed_doc( + shard, + document_root=DOC, + document_uri="https://example.com/foo", + chunk_text="See https://github.com/russellballestrini/virt-back for source.", + ) + _seed_record( + qa_db, cache_key="03" * 32, + sources=[{ + "document_root": DOC, + "document_uri": "https://example.com/foo", + "title": "foo", "shard": shard.name, "chunk_idx": 0, + }], + unverified=[], + question_text="who maintains virt-back?", + answer_text="russellballestrini.", + ) + result = inspect_cache_key("03" * 32, qa_db=qa_db, shards_dir=tmp_path) + assert result["authorship"]["tier"] == "AUTHOR_REPOSITORY_OWNER" diff --git a/tests/test_session_integration.py b/tests/test_session_integration.py index d0e5ff2..81e08bc 100644 --- a/tests/test_session_integration.py +++ b/tests/test_session_integration.py @@ -239,13 +239,17 @@ def test_full_dav1dprometheus_suite_runs_end_to_end(tmp_path, capsys): def test_full_suite_total_fixture_count(): - """Sanity check: the complete Dav1DPrometheus suite executes 562 + """Sanity check: the complete Dav1DPrometheus suite executes 662 deterministic tasks across 21 sub-batteries (5S+5T+5F+5R). History: - Phase 1a baseline: 462 tasks. - Phase 1c (#000025, 2026-05-09): 5F synthetic side expanded - 10 → 30 across all 5 sub-batteries; +100 → 562. + 10 → 30; +100 → 562. + - Phase 1d (#000025, 2026-05-09): 5F synthetic 30 → 50; +100 → 662. + (Live side ALSO went 30 → 50 but lives in *-live-v1 files + that the default-fixture-set doesn't load — those run via + the dedicated Makefile targets.) """ from bench.batteries.runner import _DEFAULT_FIXTURES, _run_one @@ -253,7 +257,7 @@ def test_full_suite_total_fixture_count(): for (battery, sub), fx in _DEFAULT_FIXTURES.items(): result = _run_one(battery, sub, Path(fx)) total += result.pass_count + result.fail_count - assert total == 562 + assert total == 662 def test_5s_phase1a_digests_unchanged_after_phase1b(): diff --git a/tests/test_warrant_authorship.py b/tests/test_warrant_authorship.py index 5b759ae..1de2eee 100644 --- a/tests/test_warrant_authorship.py +++ b/tests/test_warrant_authorship.py @@ -271,3 +271,65 @@ def test_noise_capitalized_words_not_classified_as_names(): # to secondary or no-signal. assert "Reserved" not in out.get("candidate_names", []) assert "All" not in out.get("candidate_names", []) + + +# ---------- Render-tail integration (#000026 Phase 3 wiring) --------------- + + +def test_render_tail_emits_warrant_when_authorship_set(): + """`_render_warrant_tail` adds ` · warrant: ` when + result['authorship'] is populated with a non-quiet tier.""" + from arborist.cli import _render_warrant_tail + result = { + "violations": [], + "authorship": { + "tier": "AUTHOR_COPYRIGHT_FOOTER", + "tier_rank": 5, + "signals": [], + "candidate_names": ["Russell Ballestrini"], + }, + } + tail = _render_warrant_tail(result) + assert "warrant: copyright-footer" in tail + + +def test_render_tail_silent_for_no_authorship_signal(): + """Sidecar's quiet verdict (NO_AUTHORSHIP_SIGNAL) → no tail.""" + from arborist.cli import _render_warrant_tail + result = { + "violations": [], + "authorship": { + "tier": "NO_AUTHORSHIP_SIGNAL", + "tier_rank": 99, + }, + } + tail = _render_warrant_tail(result) + assert "warrant" not in tail + + +def test_render_tail_silent_when_no_authorship_field(): + """Backward-compat: results without an `authorship` key render + unchanged (the tail-builder must not raise / must not emit).""" + from arborist.cli import _render_warrant_tail + result = {"violations": []} + tail = _render_warrant_tail(result) + assert "warrant" not in tail + + +def test_render_tail_warrant_uses_readable_token_per_tier(): + """Each tier renders with hyphen-lowercase and AUTHOR_ stripped.""" + from arborist.cli import _render_warrant_tail + expectations = { + "AUTHOR_PACKAGE_METADATA": "warrant: package-metadata", + "AUTHOR_REPOSITORY_OWNER": "warrant: repository-owner", + "AUTHOR_PAGE_BYLINE": "warrant: page-byline", + "AUTHOR_PRIMARY_PAGE_TITLE": "warrant: primary-page-title", + "AUTHOR_COPYRIGHT_FOOTER": "warrant: copyright-footer", + "AUTHOR_SECONDARY_SOURCE": "warrant: secondary-source", + } + for tier, expected in expectations.items(): + tail = _render_warrant_tail({ + "violations": [], + "authorship": {"tier": tier, "tier_rank": 1}, + }) + assert expected in tail, f"{tier} → tail={tail!r}" diff --git a/tests/test_witness.py b/tests/test_witness.py index 28cd6f2..56f840a 100644 --- a/tests/test_witness.py +++ b/tests/test_witness.py @@ -387,3 +387,101 @@ def test_query_canonical_witness_reaches_strict_after_persist(tmp_path): ) assert second["status"] == "cache_hit" assert second["witness"]["agreement_label"] == "STRICT-WITNESSED" + + +# ---------- #000028 follow-ups: capital ledger + sample-rate ------------- + + +def test_query_canonical_witness_sample_rate_zero_skips(tmp_path): + """sample_rate=0.0 → witness never fires (passive-calibration + config); LLM is not called even when canonical_witness_enabled=True.""" + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + + client = StubClient(answer="3/10") + policy = dict(DEFAULT_QUERY_POLICY) + policy["canonical_witness_enabled"] = True + policy["canonical_witness_sample_rate"] = 0.0 + qa_db = tmp_path / "qa.db" + result = query( + question="0.1 + 0.2", qa_db=qa_db, + chat_client=client, model_id="stub", policy=policy, + ) + assert result["status"] == "cache_miss_then_written" + assert result.get("witness") is None # skipped — sampled out + assert client.calls == [] # critical: no LLM round-trip + + +def test_query_canonical_witness_sample_rate_one_always_fires(tmp_path): + """sample_rate=1.0 (default when enabled) → witness fires every call.""" + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + + client = StubClient(answer="3/10") + policy = dict(DEFAULT_QUERY_POLICY) + policy["canonical_witness_enabled"] = True + policy["canonical_witness_sample_rate"] = 1.0 + qa_db = tmp_path / "qa.db" + result = query( + question="0.1 + 0.2", qa_db=qa_db, + chat_client=client, model_id="stub", policy=policy, + ) + assert result.get("witness") is not None + assert len(client.calls) == 1 + + +def test_query_canonical_witness_capital_ledger_records_cost(tmp_path): + """When witness fires + persistence writes a row, the witness LLM + cost lands in capital_ledger as op_type='canonical_witness'. + ForkScore can then compare witness-on vs witness-off forks + honestly via the ledger summary.""" + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + from arborist.store import connect + + client = StubClient(answer="3/10") + policy = dict(DEFAULT_QUERY_POLICY) + policy["canonical_witness_enabled"] = True + qa_db = tmp_path / "qa.db" + query( + question="0.1 + 0.2", qa_db=qa_db, + chat_client=client, model_id="stub", policy=policy, + ) + conn = connect(qa_db) + try: + rows = conn.execute( + "SELECT op_type, estimator_inputs_blob FROM capital_ledger " + "WHERE op_type = 'canonical_witness'" + ).fetchall() + finally: + conn.close() + assert len(rows) == 1 + import json as _json + inputs = _json.loads(rows[0]["estimator_inputs_blob"]) + assert inputs["pi_star_ref"] == "arithmetic@v1" + assert inputs["agreement_label"] in ( + "STRICT-WITNESSED", "KERNEL-LLM-AGREE", + "CACHE-DRIFT", "LLM-DIVERGED", "KERNEL-LLM-DIVERGED", + ) + + +def test_query_canonical_witness_no_ledger_when_sampled_out(tmp_path): + """Sampled-out witness must not record capital cost (no work happened).""" + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + from arborist.store import connect + + client = StubClient(answer="3/10") + policy = dict(DEFAULT_QUERY_POLICY) + policy["canonical_witness_enabled"] = True + policy["canonical_witness_sample_rate"] = 0.0 + qa_db = tmp_path / "qa.db" + query( + question="0.1 + 0.2", qa_db=qa_db, + chat_client=client, model_id="stub", policy=policy, + ) + conn = connect(qa_db) + try: + n = conn.execute( + "SELECT COUNT(*) FROM capital_ledger " + "WHERE op_type = 'canonical_witness'" + ).fetchone()[0] + finally: + conn.close() + assert n == 0