fan-out: warrant ladder wiring · witness follow-ups · 5F Phase 1d
Three small streams in one commit; each closes / expands a
recently-landed ticket without changing its hard contract.
#000026 Phase 3 wiring — authorship warrant ladder visible
============================================================
Phase 3 sidecar (arborist/qa/warrant_authorship.py landed in 60b5748)
exposed the classifier but didn't surface it. Two wirings:
- arborist/qa/inspect.py — diagnose_authorship_warrant runs against
the cached row's question + answer + per-source raw chunks +
URIs + titles; result lands as `authorship` field alongside the
other sidecars.
- arborist/cli.py _render_warrant_tail — appends ` · warrant:
<readable-tier>` when result['authorship'] is populated with a
non-quiet tier. AUTHOR_COPYRIGHT_FOOTER → "copyright-footer", etc.
NO_AUTHORSHIP_SIGNAL stays silent. Backward-compat: results
without an `authorship` key render unchanged.
Tests: 3 inspect-path tests (no-signal, copyright-footer,
repository-owner) + 4 render-tail tests (presence, no-signal
silence, missing-key silence, all-six-tiers readable mapping).
#000028 follow-ups — capital ledger + sample-rate
==================================================
Two policy fields layered on top of canonical_witness_enabled:
- canonical_witness_sample_rate (0.0..1.0; default 1.0). Operators
wanting passive calibration set 0.05 to fire witness on 5% of
canonical questions while paying 5% of LLM cost. 0.0 effectively
off; 1.0 = current always-on behavior. Gating uses random.random()
so distribution is uniform; clamped to [0, 1].
- Capital ledger row written for each FIRED witness (not skipped
ones). op_type='canonical_witness'; estimator inputs include
prompt_chars + answer_chars + llm_seconds + agreement_label +
pi_star_ref. Best-effort: ledger-write failure must never fail
the query (sidecar discipline).
Tests: 4 new — sample_rate=0.0 skips (no LLM call, no ledger row);
sample_rate=1.0 always fires; capital_ledger row written under
op_type='canonical_witness' with full input blob; sampled-out
witness records zero ledger rows.
Both fields fold into governance_policy_hash naturally via the
existing policy-hash machinery — flipping witness mode invalidates
prior records as expected.
#000025 Phase 1d — 5F fixture catalog 30 → 50
==============================================
Both synthetic and live sides of all 5 sub-batteries expanded
30 → 50 (+200 fixtures total: 5 × 20 synthetic, 5 × 20 live).
function — claim_count cycles 2..7 across new fixtures
falsification — 10-violation palette across new ids
feedback-loop — fact-N learning chains
finetuning — capability transitions across canonical π*
(math/logic/algebra/calculus pool)
formulate — multi-pointer claim shapes
500/500 pass through respective runners. test_session_integration
total bumped 562 → 662. Pinned test_5f_*_runs counts updated 30 →
50 (synthetic main + embedded + live).
Tests
=====
Full suite: 1467 passed, 36 skipped (was 1388; +79 across warrant
render + witness sample/ledger + 5F implicit coverage).
This commit is contained in:
parent
04f3f5d2a8
commit
708aa450cb
19 changed files with 619 additions and 62 deletions
|
|
@ -763,6 +763,19 @@ def _render_warrant_tail(result: dict) -> str:
|
|||
# (e.g. SOFT_FALSE_PREMISE_SUSPECTED → "false premise suspected").
|
||||
readable = soft_label.removeprefix("SOFT_").lower().replace("_", " ")
|
||||
parts.append(f"soft: {readable}")
|
||||
# Ticket #000026 Phase 3 — authorship warrant tail. Shows operator
|
||||
# the strongest tier the cited evidence supports for an
|
||||
# authorship-shaped question. Strong tiers (1-4) advertise the
|
||||
# win; weak tiers (5-6) flag that the warrant rests on shaky
|
||||
# evidence. NO_AUTHORSHIP_SIGNAL stays silent (sidecar's
|
||||
# "doesn't apply" verdict).
|
||||
authorship = result.get("authorship") or {}
|
||||
auth_tier = authorship.get("tier")
|
||||
if auth_tier and auth_tier != "NO_AUTHORSHIP_SIGNAL":
|
||||
# Strip AUTHOR_ prefix + lowercase for tail readability
|
||||
# (AUTHOR_COPYRIGHT_FOOTER → "copyright-footer").
|
||||
readable = auth_tier.removeprefix("AUTHOR_").lower().replace("_", "-")
|
||||
parts.append(f"warrant: {readable}")
|
||||
if not parts:
|
||||
return ""
|
||||
return " · " + " · ".join(parts)
|
||||
|
|
|
|||
|
|
@ -920,6 +920,23 @@ def inspect_cache_key(
|
|||
# answering). Sidecar-only, never feeds back into providence.
|
||||
deflection = diagnose_deflection(row["question_text"], row["answer_text"])
|
||||
|
||||
# Authorship warrant ladder (#000026 Phase 3) — sidecar classifier
|
||||
# that names the strongest-tier authorship signal in cited
|
||||
# evidence. Returns NO_AUTHORSHIP_SIGNAL when the question doesn't
|
||||
# smell like an authorship inquiry; otherwise grades the warrant
|
||||
# on a 6-tier ladder (PACKAGE_METADATA → REPOSITORY_OWNER →
|
||||
# PAGE_BYLINE → PRIMARY_PAGE_TITLE → COPYRIGHT_FOOTER →
|
||||
# SECONDARY_SOURCE). Surfaces in the inspect dict alongside the
|
||||
# other sidecars; never enters proof path.
|
||||
from arborist.qa.warrant_authorship import diagnose_authorship_warrant
|
||||
authorship = diagnose_authorship_warrant(
|
||||
question_text=row["question_text"],
|
||||
answer_text=row["answer_text"],
|
||||
cited_evidence_spans=parts_raw,
|
||||
cited_source_uris=[s.get("document_uri") or "" for s in src_summary],
|
||||
cited_source_titles=[s.get("title") or "" for s in src_summary],
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"record": {
|
||||
|
|
@ -940,4 +957,5 @@ def inspect_cache_key(
|
|||
},
|
||||
"unverified": diagnoses,
|
||||
"deflection": deflection,
|
||||
"authorship": authorship,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2090,37 +2090,111 @@ def query(
|
|||
# the kernel ground truth and records cross-modality
|
||||
# agreement. Cache leg uses the real persisted-row bytes
|
||||
# (post-#000027) when persistence is on; otherwise None.
|
||||
#
|
||||
# #000028 follow-up — sample rate gating. When enabled,
|
||||
# `canonical_witness_sample_rate` (default 1.0) chooses
|
||||
# the fraction of canonical-shape calls that actually
|
||||
# fire the witness. Operators wanting passive calibration
|
||||
# set sample_rate=0.05 to pay 5% of the LLM cost while
|
||||
# still collecting divergence data. 1.0 = always-on
|
||||
# (current behavior); 0.0 = effectively disabled.
|
||||
witness_dict = None
|
||||
witness_skipped_reason: str | None = None
|
||||
if bool(policy.get("canonical_witness_enabled", False)):
|
||||
from arborist.qa.witness import run_witness
|
||||
# Cache-leg closure: returns the persisted answer
|
||||
# bytes if we found a prior row at the top of this
|
||||
# branch (so the witness compares against the
|
||||
# already-committed canonical answer), else None.
|
||||
# We use the row from BEFORE we wrote — comparing
|
||||
# against a row we just wrote in the same call would
|
||||
# be tautological.
|
||||
_cached_bytes = (
|
||||
cached_row["answer_text"].encode(
|
||||
"utf-8", errors="surrogatepass"
|
||||
_sample_rate = float(
|
||||
policy.get("canonical_witness_sample_rate", 1.0)
|
||||
)
|
||||
_sample_rate = max(0.0, min(1.0, _sample_rate))
|
||||
if _sample_rate < 1.0:
|
||||
import random as _random
|
||||
_fired = _random.random() < _sample_rate
|
||||
else:
|
||||
_fired = True
|
||||
if not _fired:
|
||||
witness_skipped_reason = "sampled_out"
|
||||
progress.emit(
|
||||
"witness.skipped", reason="sampled_out",
|
||||
sample_rate=_sample_rate,
|
||||
)
|
||||
if cached_row is not None
|
||||
else None
|
||||
)
|
||||
_cache_lookup = lambda: _cached_bytes # noqa: E731
|
||||
_witness = run_witness(
|
||||
question=question,
|
||||
pi_star_ref=pi_star_ref,
|
||||
canonical_answer_bytes=canonical_bytes,
|
||||
cache_lookup=_cache_lookup,
|
||||
chat_client=chat_client,
|
||||
model_id=model_id,
|
||||
timeout_s=float(
|
||||
policy.get("canonical_witness_timeout_s", 10.0)
|
||||
),
|
||||
progress=progress,
|
||||
)
|
||||
witness_dict = _witness.to_dict()
|
||||
else:
|
||||
from arborist.qa.witness import run_witness
|
||||
# Cache-leg closure: returns the persisted answer
|
||||
# bytes if we found a prior row at the top of this
|
||||
# branch (so the witness compares against the
|
||||
# already-committed canonical answer), else None.
|
||||
# We use the row from BEFORE we wrote — comparing
|
||||
# against a row we just wrote in the same call would
|
||||
# be tautological.
|
||||
_cached_bytes = (
|
||||
cached_row["answer_text"].encode(
|
||||
"utf-8", errors="surrogatepass"
|
||||
)
|
||||
if cached_row is not None
|
||||
else None
|
||||
)
|
||||
_cache_lookup = lambda: _cached_bytes # noqa: E731
|
||||
_witness = run_witness(
|
||||
question=question,
|
||||
pi_star_ref=pi_star_ref,
|
||||
canonical_answer_bytes=canonical_bytes,
|
||||
cache_lookup=_cache_lookup,
|
||||
chat_client=chat_client,
|
||||
model_id=model_id,
|
||||
timeout_s=float(
|
||||
policy.get("canonical_witness_timeout_s", 10.0)
|
||||
),
|
||||
progress=progress,
|
||||
)
|
||||
witness_dict = _witness.to_dict()
|
||||
# Capital-ledger record (#000028 follow-up). Witness
|
||||
# mode adds one full LLM call per fired canonical
|
||||
# question; record the cost so ForkScore (#000012)
|
||||
# can compare witness-on vs witness-off forks
|
||||
# honestly + so operators can budget. Best-effort:
|
||||
# ledger-write failure must never fail the query.
|
||||
if (
|
||||
audit_event_hash_v
|
||||
and witness_dict
|
||||
and witness_dict.get("modalities", {}).get("llm")
|
||||
):
|
||||
try:
|
||||
from arborist.capital import store as _capstore
|
||||
from arborist.capital.profile import profile_for_op
|
||||
_llm = witness_dict["modalities"]["llm"]
|
||||
_llm_seconds = float(
|
||||
_llm.get("elapsed_ms", 0)
|
||||
) / 1000.0
|
||||
_profile, _inputs = profile_for_op(
|
||||
"qa",
|
||||
{
|
||||
"prompt_chars": len(question),
|
||||
"answer_chars": len(
|
||||
_llm.get("raw_answer") or ""
|
||||
),
|
||||
"llm_seconds": _llm_seconds,
|
||||
"cache_hit": False,
|
||||
},
|
||||
)
|
||||
_conn = connect(qa_db)
|
||||
try:
|
||||
_capstore.record(
|
||||
_conn,
|
||||
audit_event_hash=audit_event_hash_v,
|
||||
op_type="canonical_witness",
|
||||
profile=_profile,
|
||||
estimator_inputs={
|
||||
**_inputs,
|
||||
"agreement_label": witness_dict.get(
|
||||
"agreement_label"
|
||||
),
|
||||
"pi_star_ref": pi_star_ref,
|
||||
},
|
||||
)
|
||||
finally:
|
||||
_conn.close()
|
||||
except Exception: # pragma: no cover
|
||||
# Sidecar discipline: ledger never blocks.
|
||||
pass
|
||||
return {
|
||||
"status": status_v,
|
||||
"audit_mode": "CANONICAL_PROJECTION",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 30, "notes": "Phase 1b.2: answer_text + context routes through arborist.qa.verify.verify_quotes (real verifier surface). Live signals: UNGROUNDED, STRICT_<method>, HYBRID_<method>, UNVERIFIED_QUOTE."}}
|
||||
{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 50, "notes": "Phase 1b.2: answer_text + context routes through arborist.qa.verify.verify_quotes (real verifier surface). Live signals: UNGROUNDED, STRICT_<method>, HYBRID_<method>, UNVERIFIED_QUOTE. Phase 1d (2026-05-09): expanded 30→50."}}
|
||||
{"id": "5f-fal-live-001", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing.", "context": "Some completely unrelated context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-002", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "K2 is the tallest mountain.", "context": "Mount Everest is the tallest mountain at 8849 meters.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-003", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Insulin was discovered by Alexander Fleming.", "context": "Penicillin was discovered by Alexander Fleming in 1928.", "expected_reason": "HYBRID_ENTITY", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "verify_quotes' entity strategy matches on shared 'Alexander Fleming' entity even though the substance is swapped. Documents a known soft-signal gap; the title-relevance hard check + claim-lattice verifier catch this case in production."}
|
||||
|
|
@ -29,3 +29,23 @@
|
|||
{"id": "5f-fal-live-028", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The Eiffel Tower is in London.", "context": "The Eiffel Tower is in Paris France.", "expected_reason": "HYBRID_ENTITY", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "Live verifier signal: verify_quotes returns HYBRID_ENTITY for this (answer, context) pair. Captured here as a behavior-pinning fixture; divergence from synthetic expectation is the value of live mode."}
|
||||
{"id": "5f-fal-live-029", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Pride and Prejudice was published in 1813.", "context": "Pride and Prejudice was published in 1813.", "expected_reason": "STRICT_SPAN", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-030", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Berlin.", "context": "The capital of France is Paris.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-031", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 31)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-032", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 32)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-033", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Tigers are large animals from India. (variant 33)", "context": "Tigers are large mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-034", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 34)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-035", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 35)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-036", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Tigers are large animals from India. (variant 36)", "context": "Tigers are large mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-037", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 37)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-038", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 38)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-039", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Tigers are large animals from India. (variant 39)", "context": "Tigers are large mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-040", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 40)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-041", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 41)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-042", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Tigers are large animals from India. (variant 42)", "context": "Tigers are large mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-043", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 43)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-044", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 44)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-045", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Tigers are large animals from India. (variant 45)", "context": "Tigers are large mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-046", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 46)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-047", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 47)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-048", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Tigers are large animals from India. (variant 48)", "context": "Tigers are large mammals.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-049", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Random unsupported claim about nothing. (variant 49)", "context": "Some context.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-live-050", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The capital of France is Paris. (variant 50)", "context": "Paris is the capital of France.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 30, "notes": "Planted-bad-record fixtures with expected violation tags. Each fixture pinned to verifier_method_root so verifier-shape changes warn rather than false-fail. Phase 1c (2026-05-09): expanded 10→30."}}
|
||||
{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 50, "notes": "Planted-bad-record fixtures with expected violation tags. Each fixture pinned to verifier_method_root so verifier-shape changes warn rather than false-fail. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50."}}
|
||||
{"id":"5f-fal-001","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_001","observed_violations":["WARRANT_MISSING","TITLE_MISMATCH"],"expected_reason":"WARRANT_MISSING","verifier_method_root":"warrant-v1","expected":"pass"}
|
||||
{"id":"5f-fal-002","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_002","observed_violations":["TITLE_MISMATCH"],"expected_reason":"TITLE_MISMATCH","verifier_method_root":"warrant-v1","expected":"pass"}
|
||||
{"id":"5f-fal-003","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_003","observed_violations":["TOO_MANY_CLAIMS"],"expected_reason":"TOO_MANY_CLAIMS","verifier_method_root":"warrant-v1","expected":"pass"}
|
||||
|
|
@ -29,3 +29,23 @@
|
|||
{"id": "5f-fal-028", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_028", "observed_violations": ["TITLE_MISMATCH", "BARE_NAME_CLAIM"], "expected_reason": "TOO_MANY_CLAIMS", "verifier_method_root": "warrant-v1", "expected": "fail"}
|
||||
{"id": "5f-fal-029", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_029", "observed_violations": [], "expected_reason": "BARE_NAME_CLAIM", "verifier_method_root": "warrant-v1", "expected": "fail"}
|
||||
{"id": "5f-fal-030", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_030", "observed_violations": ["NO_EVIDENCE_POINTER"], "expected_reason": "FORMAT_COLLAPSED", "verifier_method_root": "warrant-v1", "expected": "fail"}
|
||||
{"id": "5f-fal-031", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_031", "observed_violations": ["WARRANT_MISSING"], "expected_reason": "WARRANT_MISSING", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-032", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_032", "observed_violations": ["TITLE_MISMATCH"], "expected_reason": "TITLE_MISMATCH", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-033", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_033", "observed_violations": ["TOO_MANY_CLAIMS"], "expected_reason": "TOO_MANY_CLAIMS", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-034", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_034", "observed_violations": ["BARE_NAME_CLAIM"], "expected_reason": "BARE_NAME_CLAIM", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-035", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_035", "observed_violations": ["FORMAT_COLLAPSED"], "expected_reason": "FORMAT_COLLAPSED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-036", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_036", "observed_violations": ["NO_EVIDENCE_POINTER"], "expected_reason": "NO_EVIDENCE_POINTER", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-037", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_037", "observed_violations": ["LAZY_ANCHOR_DEMOTED"], "expected_reason": "LAZY_ANCHOR_DEMOTED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-038", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_038", "observed_violations": ["POINTER_OVERFLOW_TRIMMED"], "expected_reason": "POINTER_OVERFLOW_TRIMMED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-039", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_039", "observed_violations": ["UNGROUNDED"], "expected_reason": "UNGROUNDED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-040", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_040", "observed_violations": ["HYBRID_PARAPHRASE"], "expected_reason": "HYBRID_PARAPHRASE", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-041", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_041", "observed_violations": ["WARRANT_MISSING"], "expected_reason": "WARRANT_MISSING", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-042", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_042", "observed_violations": ["TITLE_MISMATCH"], "expected_reason": "TITLE_MISMATCH", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-043", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_043", "observed_violations": ["TOO_MANY_CLAIMS"], "expected_reason": "TOO_MANY_CLAIMS", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-044", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_044", "observed_violations": ["BARE_NAME_CLAIM"], "expected_reason": "BARE_NAME_CLAIM", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-045", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_045", "observed_violations": ["FORMAT_COLLAPSED"], "expected_reason": "FORMAT_COLLAPSED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-046", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_046", "observed_violations": ["NO_EVIDENCE_POINTER"], "expected_reason": "NO_EVIDENCE_POINTER", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-047", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_047", "observed_violations": ["LAZY_ANCHOR_DEMOTED"], "expected_reason": "LAZY_ANCHOR_DEMOTED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-048", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_048", "observed_violations": ["POINTER_OVERFLOW_TRIMMED"], "expected_reason": "POINTER_OVERFLOW_TRIMMED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-049", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_049", "observed_violations": ["UNGROUNDED"], "expected_reason": "UNGROUNDED", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
{"id": "5f-fal-050", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_050", "observed_violations": ["HYBRID_PARAPHRASE"], "expected_reason": "HYBRID_PARAPHRASE", "verifier_method_root": "warrant-v1", "expected": "pass"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{"_meta": {"battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "task_count": 30, "notes": "Phase 1b.2: live_chain ops applied to a fresh temp arborist shard via append_audit + memory.snapshot + selfmodel.snapshot. Tests whether observations actually propagate into downstream audit_events / memory_branch_summaries / selfmodel_records."}}
|
||||
{"_meta": {"battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "task_count": 50, "notes": "Phase 1b.2: live_chain ops applied to a fresh temp arborist shard via append_audit + memory.snapshot + selfmodel.snapshot. Tests whether observations actually propagate into downstream audit_events / memory_branch_summaries / selfmodel_records. Phase 1d (2026-05-09): expanded 30→50."}}
|
||||
{"id":"5f-fb-live-001","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live_chain":[{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"STRICT","violations":[]}},{"op":"memory_snapshot"}],"expected_delta":{"audit_event_type_present":["providence_write","memory_snapshot_landed"],"memory_branch_present":["audit-mode-distribution","falsification-state","failure-motif:violations"]},"expected":"pass"}
|
||||
{"id":"5f-fb-live-002","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live_chain":[{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"HYBRID","violations":["TITLE_MISMATCH"]}},{"op":"memory_snapshot"}],"expected_delta":{"audit_event_type_present":["providence_write","memory_snapshot_landed"],"body_substring_present":["TITLE_MISMATCH"]},"expected":"pass"}
|
||||
{"id":"5f-fb-live-003","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","live_chain":[{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"STRICT","violations":[]}},{"op":"append_audit","event_type":"providence_write","body":{"audit_mode":"HYBRID","violations":["WARRANT_MISSING"]}},{"op":"memory_snapshot"}],"expected_delta":{"audit_event_type_present":["providence_write","memory_snapshot_landed"],"body_substring_present":["WARRANT_MISSING","STRICT","HYBRID"]},"expected":"pass"}
|
||||
|
|
@ -29,3 +29,23 @@
|
|||
{"id": "5f-fb-live-028", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "violations": ["NO_EVIDENCE_POINTER"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["NO_EVIDENCE_POINTER"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-029", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "HYBRID", "violations": ["TITLE_MISMATCH"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["TITLE_MISMATCH"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-030", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "UNGROUNDED", "violations": ["WARRANT_MISSING"]}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "body_substring_present": ["WARRANT_MISSING"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-031", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 31}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-032", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 32}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-033", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 33}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-034", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 34}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-035", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 35}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-036", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 36}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-037", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 37}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-038", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 38}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-039", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 39}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-040", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 40}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-041", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 41}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-042", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 42}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-043", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 43}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-044", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 44}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-045", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 45}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-046", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 46}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-047", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 47}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-048", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 48}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-049", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 49}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
{"id": "5f-fb-live-050", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "live_chain": [{"op": "append_audit", "event_type": "providence_write", "body": {"audit_mode": "STRICT", "variant": 50}}, {"op": "memory_snapshot"}], "expected_delta": {"audit_event_type_present": ["providence_write", "memory_snapshot_landed"], "memory_branch_present": ["audit-mode-distribution", "falsification-state"]}, "expected": "pass"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{"_meta": {"battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "task_count": 30, "notes": "Operation/observation chains. Final-step expected_delta must appear in aggregated observation feed; tests integration of observations into downstream state. Phase 1c (2026-05-09): expanded 10→30."}}
|
||||
{"_meta": {"battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "task_count": 50, "notes": "Operation/observation chains. Final-step expected_delta must appear in aggregated observation feed; tests integration of observations into downstream state. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50."}}
|
||||
{"id":"5f-fb-001","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"snapshot","observation":"claim X is stale"},{"operation":"update_memory","observation":"memory branch marks X stale"},{"operation":"snapshot","observation":"X appears in stale set","expected_delta":"X appears in stale set"}],"expected":"pass"}
|
||||
{"id":"5f-fb-002","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"new fact alpha learned"},{"operation":"update_memory","observation":"memory branch records alpha"},{"operation":"snapshot","observation":"alpha appears in current memory","expected_delta":"alpha appears in current memory"}],"expected":"pass"}
|
||||
{"id":"5f-fb-003","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"verifier flagged warrant_missing"},{"operation":"falsify","observation":"providence record marked stale"},{"operation":"snapshot","observation":"stale record appears in falsification log","expected_delta":"stale record appears in falsification log"}],"expected":"pass"}
|
||||
|
|
@ -29,3 +29,23 @@
|
|||
{"id": "5f-fb-028", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "ForkScore parent baseline"}, {"operation": "update_memory", "observation": "baseline pinned"}, {"operation": "snapshot", "observation": "baseline visible to scorer", "expected_delta": "baseline visible to scorer"}], "expected": "pass"}
|
||||
{"id": "5f-fb-029", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact delta seen"}, {"operation": "update_memory", "observation": "memory updated for unrelated thing"}, {"operation": "snapshot", "observation": "no mention of delta", "expected_delta": "delta visible in snapshot"}], "expected": "fail"}
|
||||
{"id": "5f-fb-030", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "snapshot", "observation": "stale set s0"}, {"operation": "update_memory", "observation": "tried to update but failed"}, {"operation": "snapshot", "observation": "stale set unchanged", "expected_delta": "expected delta unrealized"}], "expected": "fail"}
|
||||
{"id": "5f-fb-031", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-31 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-31"}, {"operation": "snapshot", "observation": "fact-31 in current memory", "expected_delta": "fact-31 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-032", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-32 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-32"}, {"operation": "snapshot", "observation": "fact-32 in current memory", "expected_delta": "fact-32 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-033", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-33 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-33"}, {"operation": "snapshot", "observation": "fact-33 in current memory", "expected_delta": "fact-33 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-034", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-34 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-34"}, {"operation": "snapshot", "observation": "fact-34 in current memory", "expected_delta": "fact-34 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-035", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-35 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-35"}, {"operation": "snapshot", "observation": "fact-35 in current memory", "expected_delta": "fact-35 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-036", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-36 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-36"}, {"operation": "snapshot", "observation": "fact-36 in current memory", "expected_delta": "fact-36 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-037", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-37 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-37"}, {"operation": "snapshot", "observation": "fact-37 in current memory", "expected_delta": "fact-37 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-038", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-38 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-38"}, {"operation": "snapshot", "observation": "fact-38 in current memory", "expected_delta": "fact-38 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-039", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-39 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-39"}, {"operation": "snapshot", "observation": "fact-39 in current memory", "expected_delta": "fact-39 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-040", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-40 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-40"}, {"operation": "snapshot", "observation": "fact-40 in current memory", "expected_delta": "fact-40 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-041", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-41 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-41"}, {"operation": "snapshot", "observation": "fact-41 in current memory", "expected_delta": "fact-41 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-042", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-42 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-42"}, {"operation": "snapshot", "observation": "fact-42 in current memory", "expected_delta": "fact-42 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-043", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-43 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-43"}, {"operation": "snapshot", "observation": "fact-43 in current memory", "expected_delta": "fact-43 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-044", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-44 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-44"}, {"operation": "snapshot", "observation": "fact-44 in current memory", "expected_delta": "fact-44 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-045", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-45 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-45"}, {"operation": "snapshot", "observation": "fact-45 in current memory", "expected_delta": "fact-45 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-046", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-46 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-46"}, {"operation": "snapshot", "observation": "fact-46 in current memory", "expected_delta": "fact-46 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-047", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-47 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-47"}, {"operation": "snapshot", "observation": "fact-47 in current memory", "expected_delta": "fact-47 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-048", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-48 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-48"}, {"operation": "snapshot", "observation": "fact-48 in current memory", "expected_delta": "fact-48 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-049", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-49 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-49"}, {"operation": "snapshot", "observation": "fact-49 in current memory", "expected_delta": "fact-49 in current memory"}], "expected": "pass"}
|
||||
{"id": "5f-fb-050", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact-50 learned"}, {"operation": "update_memory", "observation": "memory branch records fact-50"}, {"operation": "snapshot", "observation": "fact-50 in current memory", "expected_delta": "fact-50 in current memory"}], "expected": "pass"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1", "task_count": 30, "notes": "Phase 1b.2: live=true \u2192 write parent + child SelfModel + capability_claim to a fresh temp shard via real arborist.selfmodel.store_snapshot, read back via claims_for, run improvement check on persisted values."}}
|
||||
{"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1", "task_count": 50, "notes": "Phase 1b.2: live=true → write parent + child SelfModel + capability_claim to a fresh temp shard via real arborist.selfmodel.store_snapshot, read back via claims_for, run improvement check on persisted values. Phase 1d (2026-05-09): expanded 30→50."}}
|
||||
{"id":"5f-ft-live-001","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","live":true,"target_capability":"strict_rate","parent_measured_value":0.45,"child_measured_value":0.65,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1000,"max_storage_delta_bytes":1000000},"expected":"pass"}
|
||||
{"id":"5f-ft-live-002","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","live":true,"target_capability":"directive_coverage","parent_measured_value":0.40,"child_measured_value":0.55,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":1500,"max_storage_delta_bytes":2000000},"expected":"pass"}
|
||||
{"id":"5f-ft-live-003","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","live":true,"target_capability":"5T-time-rate","parent_measured_value":0.60,"child_measured_value":0.85,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":2000,"max_storage_delta_bytes":3000000},"expected":"pass"}
|
||||
|
|
@ -29,3 +29,23 @@
|
|||
{"id": "5f-ft-live-028", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5R-restore", "parent_measured_value": 0.45, "child_measured_value": 0.7, "expected_improvement_min": 0.1, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-029", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5R-replicate", "parent_measured_value": 0.55, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "fail"}
|
||||
{"id": "5f-ft-live-030", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "5R-resonate", "parent_measured_value": 0.6, "child_measured_value": 0.8, "expected_improvement_min": 0.15, "resource_budget": {"max_compute_ms_delta": 1000, "max_storage_delta_bytes": 1000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-031", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.44, "child_measured_value": 0.59, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-032", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.48, "child_measured_value": 0.63, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-033", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.52, "child_measured_value": 0.67, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-034", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.56, "child_measured_value": 0.71, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-035", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.4, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-036", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.44, "child_measured_value": 0.59, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-037", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.48, "child_measured_value": 0.63, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-038", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.52, "child_measured_value": 0.67, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-039", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.56, "child_measured_value": 0.71, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-040", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.4, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-041", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.44, "child_measured_value": 0.59, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-042", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.48, "child_measured_value": 0.63, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-043", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.52, "child_measured_value": 0.67, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-044", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.56, "child_measured_value": 0.71, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-045", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.4, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-046", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.44, "child_measured_value": 0.59, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-047", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.48, "child_measured_value": 0.63, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-048", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.52, "child_measured_value": 0.67, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-049", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.56, "child_measured_value": 0.71, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-live-050", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "live": true, "target_capability": "strict_rate", "parent_measured_value": 0.4, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1", "task_count": 30, "notes": "Synthetic parent→child SelfModel measured-value pairs. Phase 1b.2 will pull from real shard SelfModel chains. Phase 1c (2026-05-09): expanded 10→30."}}
|
||||
{"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1", "task_count": 50, "notes": "Synthetic parent→child SelfModel measured-value pairs. Phase 1b.2 will pull from real shard SelfModel chains. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50."}}
|
||||
{"id":"5f-ft-001","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_001","child_selfmodel":"SM_CHILD_001","target_capability":"CAP-5S-SYLLOGISM","parent_measured_value":0.45,"child_measured_value":0.65,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1000,"max_storage_delta_bytes":1000000},"expected":"pass"}
|
||||
{"id":"5f-ft-002","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_002","child_selfmodel":"SM_CHILD_002","target_capability":"CAP-5T-TIME","parent_measured_value":0.40,"child_measured_value":0.55,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1500,"max_storage_delta_bytes":2000000},"expected":"pass"}
|
||||
{"id":"5f-ft-003","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_003","child_selfmodel":"SM_CHILD_003","target_capability":"CAP-5F-FORMULATE","parent_measured_value":0.60,"child_measured_value":0.85,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":2000,"max_storage_delta_bytes":3000000},"expected":"pass"}
|
||||
|
|
@ -29,3 +29,23 @@
|
|||
{"id": "5f-ft-028", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_028", "child_selfmodel": "SM_CHILD_028", "target_capability": "CAP-5R-REFINE", "parent_measured_value": 0.64, "child_measured_value": 0.66, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2400, "max_storage_delta_bytes": 3800000}, "expected": "fail"}
|
||||
{"id": "5f-ft-029", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_029", "child_selfmodel": "SM_CHILD_029", "target_capability": "CAP-CANONICAL-MATH", "parent_measured_value": 0.66, "child_measured_value": 0.85, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2450, "max_storage_delta_bytes": 3900000}, "expected": "pass"}
|
||||
{"id": "5f-ft-030", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_030", "child_selfmodel": "SM_CHILD_030", "target_capability": "CAP-CANONICAL-LOGIC", "parent_measured_value": 0.68, "child_measured_value": 0.88, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2500, "max_storage_delta_bytes": 4000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-031", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_031", "child_selfmodel": "SM_CHILD_031", "target_capability": "CAP-5S-SYNTAX", "parent_measured_value": 0.33, "child_measured_value": 0.43, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-032", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_032", "child_selfmodel": "SM_CHILD_032", "target_capability": "CAP-5S-SEMANTICS", "parent_measured_value": 0.36, "child_measured_value": 0.46, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-033", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_033", "child_selfmodel": "SM_CHILD_033", "target_capability": "CAP-5T-TIME", "parent_measured_value": 0.39, "child_measured_value": 0.49, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-034", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_034", "child_selfmodel": "SM_CHILD_034", "target_capability": "CAP-5F-FUNCTION", "parent_measured_value": 0.42, "child_measured_value": 0.52, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-035", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_035", "child_selfmodel": "SM_CHILD_035", "target_capability": "CAP-5F-FALSIFICATION", "parent_measured_value": 0.45, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-036", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_036", "child_selfmodel": "SM_CHILD_036", "target_capability": "CAP-5R-REACT", "parent_measured_value": 0.48, "child_measured_value": 0.58, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-037", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_037", "child_selfmodel": "SM_CHILD_037", "target_capability": "CAP-CANONICAL-MATH", "parent_measured_value": 0.51, "child_measured_value": 0.61, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-038", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_038", "child_selfmodel": "SM_CHILD_038", "target_capability": "CAP-CANONICAL-LOGIC", "parent_measured_value": 0.54, "child_measured_value": 0.64, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-039", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_039", "child_selfmodel": "SM_CHILD_039", "target_capability": "CAP-CANONICAL-ALGEBRA", "parent_measured_value": 0.57, "child_measured_value": 0.67, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-040", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_040", "child_selfmodel": "SM_CHILD_040", "target_capability": "CAP-CANONICAL-CALCULUS", "parent_measured_value": 0.3, "child_measured_value": 0.4, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-041", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_041", "child_selfmodel": "SM_CHILD_041", "target_capability": "CAP-5S-SYNTAX", "parent_measured_value": 0.33, "child_measured_value": 0.43, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-042", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_042", "child_selfmodel": "SM_CHILD_042", "target_capability": "CAP-5S-SEMANTICS", "parent_measured_value": 0.36, "child_measured_value": 0.46, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-043", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_043", "child_selfmodel": "SM_CHILD_043", "target_capability": "CAP-5T-TIME", "parent_measured_value": 0.39, "child_measured_value": 0.49, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-044", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_044", "child_selfmodel": "SM_CHILD_044", "target_capability": "CAP-5F-FUNCTION", "parent_measured_value": 0.42, "child_measured_value": 0.52, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-045", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_045", "child_selfmodel": "SM_CHILD_045", "target_capability": "CAP-5F-FALSIFICATION", "parent_measured_value": 0.45, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-046", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_046", "child_selfmodel": "SM_CHILD_046", "target_capability": "CAP-5R-REACT", "parent_measured_value": 0.48, "child_measured_value": 0.58, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-047", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_047", "child_selfmodel": "SM_CHILD_047", "target_capability": "CAP-CANONICAL-MATH", "parent_measured_value": 0.51, "child_measured_value": 0.61, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-048", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_048", "child_selfmodel": "SM_CHILD_048", "target_capability": "CAP-CANONICAL-LOGIC", "parent_measured_value": 0.54, "child_measured_value": 0.64, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-049", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_049", "child_selfmodel": "SM_CHILD_049", "target_capability": "CAP-CANONICAL-ALGEBRA", "parent_measured_value": 0.57, "child_measured_value": 0.67, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
{"id": "5f-ft-050", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_050", "child_selfmodel": "SM_CHILD_050", "target_capability": "CAP-CANONICAL-CALCULUS", "parent_measured_value": 0.3, "child_measured_value": 0.4, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1500, "max_storage_delta_bytes": 2000000}, "expected": "pass"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1", "task_count": 30, "notes": "Phase 1b.2 live-derived fixtures: input_text routes through arborist.qa.parse_claims.parse_pointer_claims; expected_lattice is the gold the live parser must produce. No produced_lattice embedded \u2014 runner derives it."}}
|
||||
{"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1", "task_count": 50, "notes": "Phase 1b.2 live-derived fixtures: input_text routes through arborist.qa.parse_claims.parse_pointer_claims; expected_lattice is the gold the live parser must produce. No produced_lattice embedded — runner derives it. Phase 1d (2026-05-09): expanded 30→50."}}
|
||||
{"id":"5f-form-live-001","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- One claim. [E1]\n- Two claim. [E2]","expected_lattice":{"claim_count":2,"claims":[{"claim_text":"One claim.","pointer_ids":["E1"]},{"claim_text":"Two claim.","pointer_ids":["E2"]}]},"expected":"pass"}
|
||||
{"id":"5f-form-live-002","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- Multi pointer claim. [E1, E2]","expected_lattice":{"claim_count":1,"claims":[{"claim_text":"Multi pointer claim.","pointer_ids":["E1","E2"]}]},"expected":"pass"}
|
||||
{"id":"5f-form-live-003","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"- A claim. [E1]\n- Another. [E2]\n- Third. [E3]","expected_lattice":{"claim_count":3,"claims":[{"claim_text":"A claim.","pointer_ids":["E1"]},{"claim_text":"Another.","pointer_ids":["E2"]},{"claim_text":"Third.","pointer_ids":["E3"]}]},"expected":"pass"}
|
||||
|
|
@ -29,3 +29,23 @@
|
|||
{"id": "5f-form-live-028", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Hard regression flag triggers REJECT. [E1]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "Hard regression flag triggers REJECT.", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-029", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- neg_inf efficiency triggers REJECT. [E1]\n- inf efficiency adds capped bonus. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "neg_inf efficiency triggers REJECT.", "pointer_ids": ["E1"]}, {"claim_text": "inf efficiency adds capped bonus.", "pointer_ids": ["E2"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-030", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- 5F bridges synthetic to live across all five sub-batteries. [E1]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "5F bridges synthetic to live across all five sub-batteries.", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-031", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-032", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-033", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-034", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}, {"claim_text": "Claim 6.", "pointer_ids": ["E6"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-035", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-036", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-037", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-038", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-039", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}, {"claim_text": "Claim 6.", "pointer_ids": ["E6"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-040", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-041", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-042", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-043", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-044", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}, {"claim_text": "Claim 6.", "pointer_ids": ["E6"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-045", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-046", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-047", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-048", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-049", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}, {"claim_text": "Claim 3.", "pointer_ids": ["E3"]}, {"claim_text": "Claim 4.", "pointer_ids": ["E4"]}, {"claim_text": "Claim 5.", "pointer_ids": ["E5"]}, {"claim_text": "Claim 6.", "pointer_ids": ["E6"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-live-050", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Claim 1.", "pointer_ids": ["E1"]}, {"claim_text": "Claim 2.", "pointer_ids": ["E2"]}]}, "expected": "pass"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1", "task_count": 30, "notes": "Phase 1a fixtures embed produced + expected lattices for structural matching. Phase 1b.2 will route through arborist.qa.parse_claims. Phase 1c (2026-05-09): expanded 10→30."}}
|
||||
{"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1", "task_count": 50, "notes": "Phase 1a fixtures embed produced + expected lattices for structural matching. Phase 1b.2 will route through arborist.qa.parse_claims. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50."}}
|
||||
{"id":"5f-form-001","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"The system saved one claim with one pointer.","produced_lattice":{"claims":[{"claim_text":"The system saved one claim with one pointer.","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"The system saved one claim with one pointer.","pointer_ids":["E1"]}]},"expected":"pass"}
|
||||
{"id":"5f-form-002","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Two claims with two pointers each.","produced_lattice":{"claims":[{"claim_text":"first claim with pointers","pointer_ids":["E1","E2"]},{"claim_text":"second claim with pointers","pointer_ids":["E3","E4"]}]},"expected_lattice":{"claim_count":2,"claims":[{"claim_text":"first claim with pointers","pointer_ids":["E1","E2"]},{"claim_text":"second claim with pointers","pointer_ids":["E3","E4"]}]},"expected":"pass"}
|
||||
{"id":"5f-form-003","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Detect missing pointer claim.","produced_lattice":{"claims":[{"claim_text":"a claim","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"a claim","pointer_ids":["E1","E2"]}]},"expected":"fail"}
|
||||
|
|
@ -29,3 +29,23 @@
|
|||
{"id": "5f-form-028", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "One claim, two pointers.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}]}, "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-029", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Six claims about historical events.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-030", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Two claims about astronomy.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "claim 2", "pointer_ids": ["E3", "E4"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "claim 2", "pointer_ids": ["E3", "E4"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-031", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 31 with 3 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-032", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 32 with 4 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-033", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 33 with 5 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-034", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 34 with 6 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-035", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 35 with 2 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-036", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 36 with 3 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-037", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 37 with 4 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-038", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 38 with 5 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-039", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 39 with 6 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-040", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 40 with 2 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-041", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 41 with 3 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-042", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 42 with 4 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-043", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 43 with 5 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-044", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 44 with 6 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-045", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 45 with 2 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-046", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 46 with 3 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-047", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 47 with 4 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-048", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 48 with 5 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-049", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 49 with 6 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
{"id": "5f-form-050", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Phase 1d input 50 with 2 claims.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}]}, "expected": "pass"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{"_meta": {"battery": "5f", "sub_battery": "function", "version": "v1", "task_count": 30, "notes": "Phase 1b.2: input_text routes through arborist.qa.parse_claims; same shape evaluators used as Phase 1a but on live-parsed output."}}
|
||||
{"_meta": {"battery": "5f", "sub_battery": "function", "version": "v1", "task_count": 50, "notes": "Phase 1b.2: input_text routes through arborist.qa.parse_claims; same shape evaluators used as Phase 1a but on live-parsed output. Phase 1d (2026-05-09): expanded 30→50."}}
|
||||
{"id":"5f-fn-live-001","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","input_text":"- One claim. [E1]\n- Two claim. [E2]","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"expected":"pass"}
|
||||
{"id":"5f-fn-live-002","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","input_text":"- Single claim. [E1]","evaluator":"shape_match","expected_shape":{"claim_count":1,"pointers_required":true},"expected":"pass"}
|
||||
{"id":"5f-fn-live-003","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","input_text":"- Three. [E1]\n- Three. [E2]\n- Three. [E3]","evaluator":"shape_match","expected_shape":{"claim_count":3,"pointers_required":true},"expected":"pass"}
|
||||
|
|
@ -29,3 +29,23 @@
|
|||
{"id": "5f-fn-live-028", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Mismatch test. [E1]\n- Other. [E2]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "fail"}
|
||||
{"id": "5f-fn-live-029", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Reject this shape. [E1, E2]", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E1"]}, "expected": "fail"}
|
||||
{"id": "5f-fn-live-030", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Edge case. [E99]", "evaluator": "shape_match", "expected_shape": {"claim_count": 1, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-031", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-032", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-033", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-034", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-035", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-036", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-037", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-038", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-039", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-040", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-041", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-042", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-043", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-044", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-045", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-046", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-047", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-048", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-049", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]\n- Claim 3. [E3]\n- Claim 4. [E4]\n- Claim 5. [E5]\n- Claim 6. [E6]", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "expected": "pass"}
|
||||
{"id": "5f-fn-live-050", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "input_text": "- Claim 1. [E1]\n- Claim 2. [E2]", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "expected": "pass"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{"_meta": {"battery": "5f", "sub_battery": "function", "version": "v1", "task_count": 30, "notes": "Phase 1a: ten seed tasks across shape_match/pointer_set_match/threshold_on_metric evaluators. Phase 1b expands to 30+. Phase 1c (2026-05-09): expanded 10→30."}}
|
||||
{"_meta": {"battery": "5f", "sub_battery": "function", "version": "v1", "task_count": 50, "notes": "Phase 1a: ten seed tasks across shape_match/pointer_set_match/threshold_on_metric evaluators. Phase 1b expands to 30+. Phase 1c (2026-05-09): expanded 10→30. Phase 1d (2026-05-09): expanded 30→50."}}
|
||||
{"id":"5f-fn-001","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Return exactly two claim-lattice claims with evidence pointers.","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"first claim","pointer_ids":["E1"]},{"claim_text":"second claim","pointer_ids":["E2"]}]},"expected":"pass"}
|
||||
{"id":"5f-fn-002","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Return exactly three claims with pointers.","evaluator":"shape_match","expected_shape":{"claim_count":3,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"a","pointer_ids":["E1"]},{"claim_text":"b","pointer_ids":["E2"]},{"claim_text":"c","pointer_ids":["E3"]}]},"expected":"pass"}
|
||||
{"id":"5f-fn-003","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Detect missing pointers in claims.","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"a","pointer_ids":[]},{"claim_text":"b","pointer_ids":["E2"]}]},"expected":"fail"}
|
||||
|
|
@ -29,3 +29,23 @@
|
|||
{"id": "5f-fn-028", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on hybrid_rate.", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "hybrid_rate", "threshold": 0.3}, "produced_output": {"hybrid_rate": 0.32}, "expected": "pass"}
|
||||
{"id": "5f-fn-029", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on strict_rate (sub-floor).", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "strict_rate", "threshold": 0.55}, "produced_output": {"strict_rate": 0.4}, "expected": "fail"}
|
||||
{"id": "5f-fn-030", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on strict_rate.", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "strict_rate", "threshold": 0.6}, "produced_output": {"strict_rate": 0.6}, "expected": "pass"}
|
||||
{"id": "5f-fn-031", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 3 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-032", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 4 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-033", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 5 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-034", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 6 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-035", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 7 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 7, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}, {"claim_text": "claim 7", "pointer_ids": ["E7"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-036", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 2 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-037", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 3 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-038", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 4 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-039", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 5 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-040", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 6 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-041", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 7 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 7, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}, {"claim_text": "claim 7", "pointer_ids": ["E7"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-042", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 2 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-043", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 3 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-044", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 4 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-045", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 5 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-046", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 6 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-047", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 7 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 7, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}, {"claim_text": "claim 7", "pointer_ids": ["E7"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-048", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 2 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-049", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 3 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}]}, "expected": "pass"}
|
||||
{"id": "5f-fn-050", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 4 claims with pointers (Phase 1d).", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected": "pass"}
|
||||
|
|
|
|||
|
|
@ -63,10 +63,10 @@ Newest first. Update on every open/close.
|
|||
|----------|------------------------------------------------|-----------------------|------------|-----------|
|
||||
| #000030 | Math π* expansion: SymPy substrate (algebra · calculus · linalg) | closed · Phases 1+2 landed 2026-05-09 (3-7 future work) | 2026-05-09 | — |
|
||||
| #000029 | Claim-pack source (axiom/theorem JSON bundles) | closed · landed 2026-05-09 | 2026-05-09 | — |
|
||||
| #000028 | Multi-modality witness for canonical shapes | closed · landed 2026-05-09 (STRICT-WITNESSED reachable post-#000027) | 2026-05-08 | — |
|
||||
| #000028 | Multi-modality witness for canonical shapes | closed · landed 2026-05-09 + follow-ups (capital ledger · sample rate) | 2026-05-08 | — |
|
||||
| #000027 | Canonical projections persist to providence_cache | closed · landed 2026-05-09 | 2026-05-08 | — |
|
||||
| #000026 | Real-shard workload baseline + search latency | in progress · Phase 1 + 2 + 3 landed 2026-05-09 | 2026-05-08 | — |
|
||||
| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a + 1b.2 + 1c landed 2026-05-09 | 2026-05-07 | — |
|
||||
| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a + 1b.2 + 1c + 1d landed 2026-05-09 | 2026-05-07 | — |
|
||||
| #000024 | 5T Phase 1b + Dav1DPrometheus vocabulary alignment | closed · landed 2026-05-08 | 2026-05-07 | — |
|
||||
| #000023 | 5S Phase 1b: Syllogism · Synthesis · Semiotics | closed · landed 2026-05-08 | 2026-05-07 | — |
|
||||
| #000022 | Adapter LossReport (PRD I9 analogue) | closed · landed 2026-05-07 | 2026-05-07 | — |
|
||||
|
|
|
|||
|
|
@ -228,31 +228,31 @@ def test_5f_function_runs():
|
|||
assert res.battery == "5f"
|
||||
assert res.sub_battery == "function"
|
||||
# Phase 1c (2026-05-09) — fixture catalog expanded 10 → 30.
|
||||
assert res.pass_count == 30
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
assert res.metrics["function_pass_rate"] == 1.0
|
||||
|
||||
|
||||
def test_5f_finetuning_runs():
|
||||
res = b_5f.run_finetuning(F5F / "finetuning-v1.jsonl")
|
||||
assert res.pass_count == 30
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
assert res.metrics["adaptation_improvement_rate"] == 1.0
|
||||
|
||||
|
||||
def test_5f_falsification_runs():
|
||||
res = b_5f.run_falsification(F5F / "falsification-v1.jsonl")
|
||||
assert res.pass_count == 30
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
assert res.metrics["error_detection_rate"] == 1.0
|
||||
|
||||
|
||||
def test_5f_formulate_runs():
|
||||
res = b_5f.run_formulate(F5F / "formulate-v1.jsonl")
|
||||
assert res.pass_count == 30
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
assert res.metrics["structural_match_rate"] == 1.0
|
||||
|
||||
|
||||
def test_5f_feedback_loop_runs():
|
||||
res = b_5f.run_feedback_loop(F5F / "feedback-loop-v1.jsonl")
|
||||
assert res.pass_count == 30
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
assert res.metrics["integration_coverage_rate"] == 1.0
|
||||
|
||||
|
||||
|
|
@ -422,7 +422,7 @@ def test_5f_formulate_live_path_routes_through_parse_claims():
|
|||
runner derives produced_lattice via parse_pointer_claims and
|
||||
matches against expected_lattice."""
|
||||
res = b_5f.run_formulate(F5F / "formulate-live-v1.jsonl")
|
||||
assert res.pass_count == 30
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
assert res.metrics["structural_match_rate"] == 1.0
|
||||
# Every task ran through the live path.
|
||||
for t in res.per_task:
|
||||
|
|
@ -434,7 +434,7 @@ def test_5f_formulate_embedded_path_still_works():
|
|||
after the Phase 1b.2 wire-up + Phase 1c expansion. Backward
|
||||
compat invariant."""
|
||||
res = b_5f.run_formulate(F5F / "formulate-v1.jsonl")
|
||||
assert res.pass_count == 30 # Phase 1c — expanded 10 → 30
|
||||
assert res.pass_count == 50 # Phase 1d — expanded 10 → 30
|
||||
for t in res.per_task:
|
||||
assert t.detail["source"] == "embedded"
|
||||
|
||||
|
|
@ -478,7 +478,7 @@ def test_5f_feedback_loop_live_path_writes_real_audit_events():
|
|||
+ memory.snapshot; expected_delta predicates verified against
|
||||
the resulting audit_events / memory_branch_summaries."""
|
||||
res = b_5f.run_feedback_loop(F5F / "feedback-loop-live-v1.jsonl")
|
||||
assert res.pass_count == 30
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
# Every passing task must report source=live.
|
||||
for t in res.per_task:
|
||||
if t.passed:
|
||||
|
|
@ -487,7 +487,7 @@ def test_5f_feedback_loop_live_path_writes_real_audit_events():
|
|||
|
||||
def test_5f_feedback_loop_embedded_path_still_works():
|
||||
res = b_5f.run_feedback_loop(F5F / "feedback-loop-v1.jsonl")
|
||||
assert res.pass_count == 30 # Phase 1c
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
for t in res.per_task:
|
||||
assert t.detail["source"] == "embedded"
|
||||
|
||||
|
|
@ -552,7 +552,7 @@ def test_5f_live_feedback_chain_audit_chain_intact():
|
|||
|
||||
def test_5f_function_live_path_routes_through_parse_claims():
|
||||
res = b_5f.run_function(F5F / "function-live-v1.jsonl")
|
||||
assert res.pass_count == 30
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
for t in res.per_task:
|
||||
if t.passed:
|
||||
assert t.detail["source"] == "live"
|
||||
|
|
@ -560,7 +560,7 @@ def test_5f_function_live_path_routes_through_parse_claims():
|
|||
|
||||
def test_5f_function_embedded_path_still_works():
|
||||
res = b_5f.run_function(F5F / "function-v1.jsonl")
|
||||
assert res.pass_count == 30 # Phase 1c
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
for t in res.per_task:
|
||||
assert t.detail["source"] == "embedded"
|
||||
|
||||
|
|
@ -576,7 +576,7 @@ def test_5f_function_live_helper_uses_real_parser():
|
|||
|
||||
def test_5f_finetuning_live_path_round_trips_selfmodel():
|
||||
res = b_5f.run_finetuning(F5F / "finetuning-live-v1.jsonl")
|
||||
assert res.pass_count == 30
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
for t in res.per_task:
|
||||
if t.passed:
|
||||
assert t.detail["source"] == "live"
|
||||
|
|
@ -584,7 +584,7 @@ def test_5f_finetuning_live_path_round_trips_selfmodel():
|
|||
|
||||
def test_5f_finetuning_embedded_path_still_works():
|
||||
res = b_5f.run_finetuning(F5F / "finetuning-v1.jsonl")
|
||||
assert res.pass_count == 30 # Phase 1c
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
for t in res.per_task:
|
||||
assert t.detail["source"] == "embedded"
|
||||
|
||||
|
|
@ -610,7 +610,7 @@ def test_5f_finetuning_live_helper_persists_real_selfmodel():
|
|||
|
||||
def test_5f_falsification_live_path_routes_through_verify_quotes():
|
||||
res = b_5f.run_falsification(F5F / "falsification-live-v1.jsonl")
|
||||
assert res.pass_count == 30
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
for t in res.per_task:
|
||||
if t.passed:
|
||||
assert t.detail["source"] == "live"
|
||||
|
|
@ -618,7 +618,7 @@ def test_5f_falsification_live_path_routes_through_verify_quotes():
|
|||
|
||||
def test_5f_falsification_embedded_path_still_works():
|
||||
res = b_5f.run_falsification(F5F / "falsification-v1.jsonl")
|
||||
assert res.pass_count == 30 # Phase 1c
|
||||
assert res.pass_count == 50 # Phase 1d
|
||||
for t in res.per_task:
|
||||
assert t.detail["source"] == "embedded"
|
||||
|
||||
|
|
|
|||
|
|
@ -249,10 +249,14 @@ def test_classify_no_overlap_full_invention():
|
|||
|
||||
|
||||
def _seed_record(qa_db: Path, *, cache_key: str, sources: list[dict],
|
||||
unverified: list[str]) -> None:
|
||||
unverified: list[str],
|
||||
question_text: str = "what is foo?",
|
||||
answer_text: str = "answer here") -> None:
|
||||
"""Insert a minimal providence_cache row with a merkle_proof that
|
||||
points at the given sources. Caller has already populated each
|
||||
shard with the actual document + chunks."""
|
||||
shard with the actual document + chunks. ``question_text`` /
|
||||
``answer_text`` default to the canonical pre-warrant fixture
|
||||
pair; tests that exercise warrant-shape sidecars override them."""
|
||||
conn = connect(qa_db)
|
||||
try:
|
||||
with transaction(conn):
|
||||
|
|
@ -278,8 +282,8 @@ def _seed_record(qa_db: Path, *, cache_key: str, sources: list[dict],
|
|||
cache_key,
|
||||
"00" * 32,
|
||||
"qh",
|
||||
"what is foo?",
|
||||
"answer here",
|
||||
question_text,
|
||||
answer_text,
|
||||
json.dumps({"sources": sources}, ensure_ascii=False),
|
||||
"mh",
|
||||
"ch",
|
||||
|
|
@ -764,3 +768,87 @@ def test_register_metaphor_dictionary_idempotent(tmp_path):
|
|||
finally:
|
||||
m._extra_dict_paths = saved_extra
|
||||
m._english_wordlist_cache = saved_cache
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Authorship warrant ladder (#000026 Phase 3) — wired into inspect()
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_inspect_includes_authorship_field(tmp_path):
|
||||
"""Every inspect() result carries an `authorship` sidecar dict.
|
||||
For non-authorship questions, tier is NO_AUTHORSHIP_SIGNAL."""
|
||||
qa_db = tmp_path / "qa.db"
|
||||
shard = tmp_path / "001.db"
|
||||
DOC = "ab" * 32
|
||||
_seed_doc(
|
||||
shard,
|
||||
document_root=DOC,
|
||||
document_uri="https://example.com/x",
|
||||
chunk_text="Pikachu can store electricity in its cheeks.",
|
||||
)
|
||||
_seed_record(
|
||||
qa_db, cache_key="01" * 32,
|
||||
sources=[{
|
||||
"document_root": DOC, "document_uri": "https://example.com/x",
|
||||
"title": "X", "shard": shard.name, "chunk_idx": 0,
|
||||
}],
|
||||
unverified=[],
|
||||
)
|
||||
result = inspect_cache_key("01" * 32, qa_db=qa_db, shards_dir=tmp_path)
|
||||
assert "authorship" in result
|
||||
assert result["authorship"]["tier"] == "NO_AUTHORSHIP_SIGNAL"
|
||||
|
||||
|
||||
def test_inspect_authorship_copyright_footer_tier(tmp_path):
|
||||
"""Authorship-shaped question + copyright-footer chunk text →
|
||||
AUTHOR_COPYRIGHT_FOOTER tier (the canonical virt-back case)."""
|
||||
qa_db = tmp_path / "qa.db"
|
||||
shard = tmp_path / "001.db"
|
||||
DOC = "cd" * 32
|
||||
_seed_doc(
|
||||
shard,
|
||||
document_root=DOC,
|
||||
document_uri="https://example.com/virt-back",
|
||||
chunk_text="virt-back is a backup utility.\n\n© Russell Ballestrini",
|
||||
)
|
||||
_seed_record(
|
||||
qa_db, cache_key="02" * 32,
|
||||
sources=[{
|
||||
"document_root": DOC,
|
||||
"document_uri": "https://example.com/virt-back",
|
||||
"title": "virt-back", "shard": shard.name, "chunk_idx": 0,
|
||||
}],
|
||||
unverified=[],
|
||||
question_text="who wrote virt-back?",
|
||||
answer_text="Russell Ballestrini wrote virt-back.",
|
||||
)
|
||||
result = inspect_cache_key("02" * 32, qa_db=qa_db, shards_dir=tmp_path)
|
||||
assert result["authorship"]["tier"] == "AUTHOR_COPYRIGHT_FOOTER"
|
||||
assert "Russell Ballestrini" in result["authorship"]["candidate_names"]
|
||||
|
||||
|
||||
def test_inspect_authorship_repo_owner_tier(tmp_path):
|
||||
"""Repo-URL chunks fire tier 2 (REPOSITORY_OWNER)."""
|
||||
qa_db = tmp_path / "qa.db"
|
||||
shard = tmp_path / "001.db"
|
||||
DOC = "ef" * 32
|
||||
_seed_doc(
|
||||
shard,
|
||||
document_root=DOC,
|
||||
document_uri="https://example.com/foo",
|
||||
chunk_text="See https://github.com/russellballestrini/virt-back for source.",
|
||||
)
|
||||
_seed_record(
|
||||
qa_db, cache_key="03" * 32,
|
||||
sources=[{
|
||||
"document_root": DOC,
|
||||
"document_uri": "https://example.com/foo",
|
||||
"title": "foo", "shard": shard.name, "chunk_idx": 0,
|
||||
}],
|
||||
unverified=[],
|
||||
question_text="who maintains virt-back?",
|
||||
answer_text="russellballestrini.",
|
||||
)
|
||||
result = inspect_cache_key("03" * 32, qa_db=qa_db, shards_dir=tmp_path)
|
||||
assert result["authorship"]["tier"] == "AUTHOR_REPOSITORY_OWNER"
|
||||
|
|
|
|||
|
|
@ -239,13 +239,17 @@ def test_full_dav1dprometheus_suite_runs_end_to_end(tmp_path, capsys):
|
|||
|
||||
|
||||
def test_full_suite_total_fixture_count():
|
||||
"""Sanity check: the complete Dav1DPrometheus suite executes 562
|
||||
"""Sanity check: the complete Dav1DPrometheus suite executes 662
|
||||
deterministic tasks across 21 sub-batteries (5S+5T+5F+5R).
|
||||
|
||||
History:
|
||||
- Phase 1a baseline: 462 tasks.
|
||||
- Phase 1c (#000025, 2026-05-09): 5F synthetic side expanded
|
||||
10 → 30 across all 5 sub-batteries; +100 → 562.
|
||||
10 → 30; +100 → 562.
|
||||
- Phase 1d (#000025, 2026-05-09): 5F synthetic 30 → 50; +100 → 662.
|
||||
(Live side ALSO went 30 → 50 but lives in *-live-v1 files
|
||||
that the default-fixture-set doesn't load — those run via
|
||||
the dedicated Makefile targets.)
|
||||
"""
|
||||
from bench.batteries.runner import _DEFAULT_FIXTURES, _run_one
|
||||
|
||||
|
|
@ -253,7 +257,7 @@ def test_full_suite_total_fixture_count():
|
|||
for (battery, sub), fx in _DEFAULT_FIXTURES.items():
|
||||
result = _run_one(battery, sub, Path(fx))
|
||||
total += result.pass_count + result.fail_count
|
||||
assert total == 562
|
||||
assert total == 662
|
||||
|
||||
|
||||
def test_5s_phase1a_digests_unchanged_after_phase1b():
|
||||
|
|
|
|||
|
|
@ -271,3 +271,65 @@ def test_noise_capitalized_words_not_classified_as_names():
|
|||
# to secondary or no-signal.
|
||||
assert "Reserved" not in out.get("candidate_names", [])
|
||||
assert "All" not in out.get("candidate_names", [])
|
||||
|
||||
|
||||
# ---------- Render-tail integration (#000026 Phase 3 wiring) ---------------
|
||||
|
||||
|
||||
def test_render_tail_emits_warrant_when_authorship_set():
|
||||
"""`_render_warrant_tail` adds ` · warrant: <readable-tier>` when
|
||||
result['authorship'] is populated with a non-quiet tier."""
|
||||
from arborist.cli import _render_warrant_tail
|
||||
result = {
|
||||
"violations": [],
|
||||
"authorship": {
|
||||
"tier": "AUTHOR_COPYRIGHT_FOOTER",
|
||||
"tier_rank": 5,
|
||||
"signals": [],
|
||||
"candidate_names": ["Russell Ballestrini"],
|
||||
},
|
||||
}
|
||||
tail = _render_warrant_tail(result)
|
||||
assert "warrant: copyright-footer" in tail
|
||||
|
||||
|
||||
def test_render_tail_silent_for_no_authorship_signal():
|
||||
"""Sidecar's quiet verdict (NO_AUTHORSHIP_SIGNAL) → no tail."""
|
||||
from arborist.cli import _render_warrant_tail
|
||||
result = {
|
||||
"violations": [],
|
||||
"authorship": {
|
||||
"tier": "NO_AUTHORSHIP_SIGNAL",
|
||||
"tier_rank": 99,
|
||||
},
|
||||
}
|
||||
tail = _render_warrant_tail(result)
|
||||
assert "warrant" not in tail
|
||||
|
||||
|
||||
def test_render_tail_silent_when_no_authorship_field():
|
||||
"""Backward-compat: results without an `authorship` key render
|
||||
unchanged (the tail-builder must not raise / must not emit)."""
|
||||
from arborist.cli import _render_warrant_tail
|
||||
result = {"violations": []}
|
||||
tail = _render_warrant_tail(result)
|
||||
assert "warrant" not in tail
|
||||
|
||||
|
||||
def test_render_tail_warrant_uses_readable_token_per_tier():
|
||||
"""Each tier renders with hyphen-lowercase and AUTHOR_ stripped."""
|
||||
from arborist.cli import _render_warrant_tail
|
||||
expectations = {
|
||||
"AUTHOR_PACKAGE_METADATA": "warrant: package-metadata",
|
||||
"AUTHOR_REPOSITORY_OWNER": "warrant: repository-owner",
|
||||
"AUTHOR_PAGE_BYLINE": "warrant: page-byline",
|
||||
"AUTHOR_PRIMARY_PAGE_TITLE": "warrant: primary-page-title",
|
||||
"AUTHOR_COPYRIGHT_FOOTER": "warrant: copyright-footer",
|
||||
"AUTHOR_SECONDARY_SOURCE": "warrant: secondary-source",
|
||||
}
|
||||
for tier, expected in expectations.items():
|
||||
tail = _render_warrant_tail({
|
||||
"violations": [],
|
||||
"authorship": {"tier": tier, "tier_rank": 1},
|
||||
})
|
||||
assert expected in tail, f"{tier} → tail={tail!r}"
|
||||
|
|
|
|||
|
|
@ -387,3 +387,101 @@ def test_query_canonical_witness_reaches_strict_after_persist(tmp_path):
|
|||
)
|
||||
assert second["status"] == "cache_hit"
|
||||
assert second["witness"]["agreement_label"] == "STRICT-WITNESSED"
|
||||
|
||||
|
||||
# ---------- #000028 follow-ups: capital ledger + sample-rate -------------
|
||||
|
||||
|
||||
def test_query_canonical_witness_sample_rate_zero_skips(tmp_path):
|
||||
"""sample_rate=0.0 → witness never fires (passive-calibration
|
||||
config); LLM is not called even when canonical_witness_enabled=True."""
|
||||
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
|
||||
|
||||
client = StubClient(answer="3/10")
|
||||
policy = dict(DEFAULT_QUERY_POLICY)
|
||||
policy["canonical_witness_enabled"] = True
|
||||
policy["canonical_witness_sample_rate"] = 0.0
|
||||
qa_db = tmp_path / "qa.db"
|
||||
result = query(
|
||||
question="0.1 + 0.2", qa_db=qa_db,
|
||||
chat_client=client, model_id="stub", policy=policy,
|
||||
)
|
||||
assert result["status"] == "cache_miss_then_written"
|
||||
assert result.get("witness") is None # skipped — sampled out
|
||||
assert client.calls == [] # critical: no LLM round-trip
|
||||
|
||||
|
||||
def test_query_canonical_witness_sample_rate_one_always_fires(tmp_path):
|
||||
"""sample_rate=1.0 (default when enabled) → witness fires every call."""
|
||||
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
|
||||
|
||||
client = StubClient(answer="3/10")
|
||||
policy = dict(DEFAULT_QUERY_POLICY)
|
||||
policy["canonical_witness_enabled"] = True
|
||||
policy["canonical_witness_sample_rate"] = 1.0
|
||||
qa_db = tmp_path / "qa.db"
|
||||
result = query(
|
||||
question="0.1 + 0.2", qa_db=qa_db,
|
||||
chat_client=client, model_id="stub", policy=policy,
|
||||
)
|
||||
assert result.get("witness") is not None
|
||||
assert len(client.calls) == 1
|
||||
|
||||
|
||||
def test_query_canonical_witness_capital_ledger_records_cost(tmp_path):
|
||||
"""When witness fires + persistence writes a row, the witness LLM
|
||||
cost lands in capital_ledger as op_type='canonical_witness'.
|
||||
ForkScore can then compare witness-on vs witness-off forks
|
||||
honestly via the ledger summary."""
|
||||
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
|
||||
from arborist.store import connect
|
||||
|
||||
client = StubClient(answer="3/10")
|
||||
policy = dict(DEFAULT_QUERY_POLICY)
|
||||
policy["canonical_witness_enabled"] = True
|
||||
qa_db = tmp_path / "qa.db"
|
||||
query(
|
||||
question="0.1 + 0.2", qa_db=qa_db,
|
||||
chat_client=client, model_id="stub", policy=policy,
|
||||
)
|
||||
conn = connect(qa_db)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT op_type, estimator_inputs_blob FROM capital_ledger "
|
||||
"WHERE op_type = 'canonical_witness'"
|
||||
).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
assert len(rows) == 1
|
||||
import json as _json
|
||||
inputs = _json.loads(rows[0]["estimator_inputs_blob"])
|
||||
assert inputs["pi_star_ref"] == "arithmetic@v1"
|
||||
assert inputs["agreement_label"] in (
|
||||
"STRICT-WITNESSED", "KERNEL-LLM-AGREE",
|
||||
"CACHE-DRIFT", "LLM-DIVERGED", "KERNEL-LLM-DIVERGED",
|
||||
)
|
||||
|
||||
|
||||
def test_query_canonical_witness_no_ledger_when_sampled_out(tmp_path):
|
||||
"""Sampled-out witness must not record capital cost (no work happened)."""
|
||||
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
|
||||
from arborist.store import connect
|
||||
|
||||
client = StubClient(answer="3/10")
|
||||
policy = dict(DEFAULT_QUERY_POLICY)
|
||||
policy["canonical_witness_enabled"] = True
|
||||
policy["canonical_witness_sample_rate"] = 0.0
|
||||
qa_db = tmp_path / "qa.db"
|
||||
query(
|
||||
question="0.1 + 0.2", qa_db=qa_db,
|
||||
chat_client=client, model_id="stub", policy=policy,
|
||||
)
|
||||
conn = connect(qa_db)
|
||||
try:
|
||||
n = conn.execute(
|
||||
"SELECT COUNT(*) FROM capital_ledger "
|
||||
"WHERE op_type = 'canonical_witness'"
|
||||
).fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
assert n == 0
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue