diff --git a/Makefile b/Makefile index 315ac95..7114e30 100644 --- a/Makefile +++ b/Makefile @@ -297,6 +297,35 @@ bench-real-shard: bootstrap ## #000026 Phase 2 — real-shard workload baseline --shards-dir $${ARBORIST_SHARDS_DIR:-$$HOME/.arborist/shards} \ --burn +# v8 ForkScore — runs the full bench-suite, then scores the fresh +# child output against a previously-pinned PARENT artifact. Default +# parent is bench/results/baseline-suite.json (operator pins this +# once via `make bench-fork-baseline`); override per-call with +# PARENT=path. Fox's iteration loop: +# +# make bench-fork-baseline # one-shot — pins current state +# +# make bench-fork-score # compare hacked branch to baseline +# # exit 1 on REJECT (CI-gateable) +FORK_PARENT ?= bench/results/baseline-suite.json +FORK_CHILD ?= bench/results/current-suite.json +FORK_REPORT ?= bench/results/fork_score_report.json + +bench-fork-baseline: bootstrap ## pin current bench-suite output as ForkScore parent + @mkdir -p bench/results + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --all \ + --out $(FORK_PARENT) + @echo ">> baseline pinned: $(FORK_PARENT)" + +bench-fork-score: bootstrap ## #000012 Phase 1b — score current bench output vs $(FORK_PARENT) + @mkdir -p bench/results + PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --all \ + --out $(FORK_CHILD) + $(ARBORIST) v8 score \ + --parent $(FORK_PARENT) \ + --child $(FORK_CHILD) \ + --out $(FORK_REPORT) + bench-5f-formulate-live: bootstrap ## 5F Formulate via live arborist.qa.parse_claims (Phase 1b.2) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate \ --fixtures bench/fixtures/5f/formulate-live-v1.jsonl diff --git a/arborist/cli.py b/arborist/cli.py index 4496d90..485894e 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -3018,7 +3018,19 @@ def _cmd_v8_score(args: argparse.Namespace) -> int: complexity_delta=float(args.complexity_delta), memory_invalidation_count=float(args.memory_invalidation_count), ) - print(json.dumps(scored.to_dict(), indent=2, ensure_ascii=False, default=str)) + artifact = scored.to_dict() + artifact_json = json.dumps( + artifact, indent=2, ensure_ascii=False, default=str + ) + print(artifact_json) + # Phase 1b — also write to disk so CI / mesh peers / downstream + # graders can pick up the artifact without parsing stdout. + out_path = getattr(args, "out", None) + if out_path: + from pathlib import Path + p = Path(out_path) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(artifact_json + "\n", encoding="utf-8") # Non-zero exit on REJECT so CI can gate on it. return 0 if scored.verdict in ("ACCEPT", "MARGINAL") else 1 @@ -4796,6 +4808,16 @@ def build_parser() -> argparse.ArgumentParser: dest="memory_invalidation_count", type=float, default=0.0, help="count of memory_records the fork would falsify", ) + v8_score.add_argument( + "--out", default=None, + help=( + "optional output file path; ScoredFork JSON is also written " + "here in addition to stdout. Default: stdout only. The CI " + "/ make bench-fork-score targets pin this to " + "bench/results/fork_score_report.json so downstream graders " + "/ ForkScore-aware mesh peers can ingest the artifact." + ), + ) v8_score.set_defaults(func=_cmd_v8_score) # ----- memory subcommands (ticket #000017) -------------------------------- diff --git a/arborist/qa/warrant_authorship.py b/arborist/qa/warrant_authorship.py new file mode 100644 index 0000000..cf55d1a --- /dev/null +++ b/arborist/qa/warrant_authorship.py @@ -0,0 +1,456 @@ +"""Authorship warrant ladder — sidecar classifier (#000026 Phase 3). + +The 2026-05-08 `virt-back` query landed STRICT/EVIDENCE-WARRANTED +with the cited evidence being a copyright footer +(``© Russell Ballestrini``). That's an acceptable warrant for the +operator-facing answer but not the strongest the substrate could +have surfaced. The reviewer flagged this as Phase 3: classify the +authorship warrant by **strength**, sidecar-only, so that an +operator who asks "who wrote X" sees not just "X said so" but also +*how* the cited evidence supports the authorship claim. + +Six-tier ladder (strongest to weakest): + + Tier 1 AUTHOR_PACKAGE_METADATA + Explicit ``author = "..."`` / ``maintainer = "..."`` in + packaging metadata (pyproject.toml, setup.py, package.json, + Cargo.toml, gemspec, ...). Strongest because the project + owner literally typed their name into the package manifest. + + Tier 2 AUTHOR_REPOSITORY_OWNER + Repository URL pattern naming the owner (github.com/, + gitlab.com/, codeberg.org/, ...). Strongest + after metadata because the platform's authentication binds + the URL prefix to a verified account. + + Tier 3 AUTHOR_PAGE_BYLINE + Article-byline patterns: ``By NAME``, ``Author: NAME``, + ````. Standard web/blog + attribution; not as strong as packaging because byline is + unverified. + + Tier 4 AUTHOR_PRIMARY_PAGE_TITLE + The cited evidence is the project's own primary-source + page (the answer's entity matches the source title). The + page belonging to the project is itself a (weak) form of + authorship attestation — the project page's existence on + the entity's domain implies authorship. + + Tier 5 AUTHOR_COPYRIGHT_FOOTER + ``© NAME``, ``Copyright YYYY NAME``. The current `virt-back` + warrant. Acceptable but weak — copyright applies to site + chrome, not necessarily to the project's authorship. + + Tier 6 AUTHOR_SECONDARY_SOURCE + None of the above; the answer rests on a third-party claim + (Wikipedia article saying X wrote Y, etc.). Weakest because + the chain of custody runs through an editor. + +Sidecar only — never enters the verifier proof path. Surfaces on +the result dict as ``authorship_warrant`` for inspect / providence / +render-line consumers. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + + +# ---------- Tier registry -------------------------------------------------- + +TIER_NAMES = ( + "AUTHOR_PACKAGE_METADATA", # 1 + "AUTHOR_REPOSITORY_OWNER", # 2 + "AUTHOR_PAGE_BYLINE", # 3 + "AUTHOR_PRIMARY_PAGE_TITLE", # 4 + "AUTHOR_COPYRIGHT_FOOTER", # 5 + "AUTHOR_SECONDARY_SOURCE", # 6 +) + +# Map tier name → 1-based rank (lower = stronger). +TIER_RANK: dict[str, int] = {name: i + 1 for i, name in enumerate(TIER_NAMES)} + +# Special: NO_AUTHORSHIP_SIGNAL is for cases where the answer doesn't +# actually claim authorship of anything — useful for the sidecar to +# stay quiet rather than falsely report a tier-6 warrant. +NO_AUTHORSHIP_SIGNAL = "NO_AUTHORSHIP_SIGNAL" + + +# ---------- Detector regexes ---------------------------------------------- + +# Tier 1 — packaging-metadata patterns. Cover Python (TOML / setup.py), +# JS (package.json), Rust (Cargo.toml), Ruby (.gemspec). All variants +# are TEXT-shape; we don't parse the formats themselves, just look for +# the lexical pattern that says "author = NAME". Two regexes — keeping +# them separate is clearer than a single Frankenstein pattern that +# tries to handle simple-quoted AND TOML-inline-table variants. +# +# (a) Simple key-equals-quoted-name: +# author = "NAME" (Python setup.py) +# "author": "NAME" (package.json) +# authors = ["NAME"] (TOML scalar list) +# maintainer: NAME (yaml-ish) +_PACKAGE_METADATA_SIMPLE_RE = re.compile( + r""" + [\"']?(?:author|maintainer|authors|maintainers)[\"']? + \s*[:=]\s* # `:` (json/yaml) or `=` (toml/py) + \[?\s*[\"'] # optional `[`, then opening quote + (?P[A-Z][\w\s\.\-']{1,79}) # capture + [\"'] # closing quote + """, + re.VERBOSE | re.IGNORECASE, +) + +# (b) TOML inline-table form (PEP 621): +# authors = [{ name = "NAME" }, …] +# maintainers = [{ name = "NAME" }] +_PACKAGE_METADATA_TOML_INLINE_RE = re.compile( + r""" + (?:author|maintainer|authors|maintainers) + \s*=\s*\[\s* # `authors = [` + \{\s*name\s*=\s*[\"'] # `{ name = "` + (?P[A-Z][\w\s\.\-']{1,79}) + [\"'] + """, + re.VERBOSE | re.IGNORECASE | re.DOTALL, +) + +# Tier 2 — repo-owner URL patterns. github.com//, etc. +_REPO_OWNER_RE = re.compile( + r"\b(?:github\.com|gitlab\.com|codeberg\.org|bitbucket\.org)" + r"/(?P[A-Za-z0-9][\w\-]{0,38})/[A-Za-z0-9][\w\-\.]{0,99}", + re.IGNORECASE, +) + +# Tier 3 — page byline patterns. "By NAME", "Author: NAME", +# . NAME starts with capital letter. +# The prefix ("by", "author", etc.) is case-insensitive via the +# inline `(?i:...)` flag; the name capture stays case-sensitive so +# `[A-Z]` actually means "starts with uppercase" — IGNORECASE on the +# whole regex would defeat the capitalization check. +_BYLINE_PROSE_RE = re.compile( + r"\b(?i:(?:by|author(?:ed)?(?:\s+by)?|written\s+by|posted\s+by))" + r"\s*[:\-]?\s*" + r"(?P[A-Z][\w\.\-']{1,40}(?:\s+[A-Z][\w\.\-']{1,40}){0,3})", +) +_BYLINE_META_RE = re.compile( + r"]*name\s*=\s*[\"']author[\"'][^>]*content\s*=\s*[\"']" + r"(?P[^\"'<>]{1,80})", + re.IGNORECASE, +) + +# Tier 5 — copyright footer. "© NAME", "Copyright YYYY NAME". +_COPYRIGHT_RE = re.compile( + r"(?:©|\bcopyright\b|\(c\))\s*" + r"(?:\d{4}(?:\s*[\-–]\s*\d{4})?\s*)?" # optional year / year-year + r"(?P[A-Z][\w\.\-']{1,40}(?:\s+[A-Z][\w\.\-']{1,40}){0,3})", + re.IGNORECASE, +) + +# Recognize that the answer is a who-question authorship claim. Verb +# list covers create / build / maintain / own / develop variants. +_AUTHORSHIP_QUESTION_HINT_RE = re.compile( + r"\b(?:" + r"who\s+(?:wrote|created|authored|made|built|developed|designed|founded|" + r"maintains?|maintained|owns?|owned|manages?|managed|runs?)" + r"|author(?:s)?\s+of" + r"|creator\s+of" + r"|maintainer\s+of" + r"|owner\s+of" + r")\b", + re.IGNORECASE, +) + + +# ---------- Result type --------------------------------------------------- + + +@dataclass +class WarrantClassification: + """Sidecar verdict for an authorship claim. + + ``tier`` is the strongest tier whose detector fired; ``signals`` + enumerates every detector that matched (a copyright footer plus a + repo URL might both fire — the tier reflects the strongest, but + the operator sees the full set in ``signals``). + """ + + tier: str + tier_rank: int + signals: list[dict] = field(default_factory=list) + candidate_names: list[str] = field(default_factory=list) + note: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "tier": self.tier, + "tier_rank": self.tier_rank, + "signals": self.signals, + "candidate_names": self.candidate_names, + "note": self.note, + } + + +# ---------- Public entrypoint --------------------------------------------- + + +def diagnose_authorship_warrant( + *, + question_text: str | None = None, + answer_text: str = "", + cited_evidence_spans: list[str] | None = None, + cited_source_uris: list[str] | None = None, + cited_source_titles: list[str] | None = None, +) -> dict[str, Any]: + """Classify the strongest authorship-warrant signal in the cited + evidence. Always returns a dict (sidecar contract); the dict's + ``tier`` field carries the verdict. + + When the question doesn't smell like an authorship question + (``who wrote X``, ``author of Y``, ``creator of Z``), returns + ``tier = NO_AUTHORSHIP_SIGNAL`` immediately so the sidecar stays + quiet on irrelevant queries. + + The function NEVER raises on bad input — silently treats missing + fields as empty. Sidecar discipline. + """ + # Filter Nones / non-strings — sidecar must never raise on bad input. + spans = [s for s in (cited_evidence_spans or []) if isinstance(s, str)] + uris = [u for u in (cited_source_uris or []) if isinstance(u, str)] + titles = [t for t in (cited_source_titles or []) if isinstance(t, str)] + + # Quick gate: only fire on questions that smell like authorship. + if question_text and not _AUTHORSHIP_QUESTION_HINT_RE.search(question_text): + return WarrantClassification( + tier=NO_AUTHORSHIP_SIGNAL, + tier_rank=99, + note="question does not look like an authorship inquiry", + ).to_dict() + + signals: list[dict] = [] + candidates: set[str] = set() + + # Tier 1 — package metadata. Both the simple `author = "X"` form + # and the TOML inline-table `authors = [{ name = "X" }]` form. + for span in spans: + for regex in (_PACKAGE_METADATA_TOML_INLINE_RE, _PACKAGE_METADATA_SIMPLE_RE): + for m in regex.finditer(span): + name = (m.group("name") or "").strip() + if name and not _looks_like_noise(name): + signals.append({ + "tier": "AUTHOR_PACKAGE_METADATA", + "matched": m.group(0).strip()[:80], + "name": name, + }) + candidates.add(name) + + # Tier 2 — repo owner URLs. + for u in uris: + m = _REPO_OWNER_RE.search(u) + if m: + owner = m.group("owner") + signals.append({ + "tier": "AUTHOR_REPOSITORY_OWNER", + "matched": m.group(0), + "name": owner, + }) + candidates.add(owner) + # Also scan span text for inline repo URLs (often present in READMEs). + for span in spans: + for m in _REPO_OWNER_RE.finditer(span): + owner = m.group("owner") + signals.append({ + "tier": "AUTHOR_REPOSITORY_OWNER", + "matched": m.group(0), + "name": owner, + }) + candidates.add(owner) + + # Tier 3 — page byline (prose + meta). + for span in spans: + for m in _BYLINE_PROSE_RE.finditer(span): + name = m.group("name").strip() + if not _looks_like_noise(name): + signals.append({ + "tier": "AUTHOR_PAGE_BYLINE", + "matched": m.group(0).strip()[:80], + "name": name, + }) + candidates.add(name) + for m in _BYLINE_META_RE.finditer(span): + name = m.group("name").strip() + if name and not _looks_like_noise(name): + signals.append({ + "tier": "AUTHOR_PAGE_BYLINE", + "matched": m.group(0).strip()[:80], + "name": name, + }) + candidates.add(name) + + # Tier 4 — primary-page-title heuristic. Triggers when the answer + # mentions a title that appears in the cited source titles AND the + # source URI is on what looks like the entity's primary domain. + # We approximate "primary domain" loosely: any non-Wikipedia URL + # whose hostname appears in the cited title set (e.g. site + # russellballestrini.net → article title contains "Russell + # Ballestrini" or "russellballestrini"). + primary_hits = _detect_primary_page_title(answer_text, uris, titles) + for hit in primary_hits: + signals.append(hit) + if hit.get("name"): + candidates.add(hit["name"]) + + # Tier 5 — copyright footer. + for span in spans: + for m in _COPYRIGHT_RE.finditer(span): + name = m.group("name").strip() + if not _looks_like_noise(name): + signals.append({ + "tier": "AUTHOR_COPYRIGHT_FOOTER", + "matched": m.group(0).strip()[:80], + "name": name, + }) + candidates.add(name) + + # Tier 6 — secondary source. Only fires when there's at least one + # cited evidence span AND none of tiers 1-5 fired. The answer + # rests on a third-party claim (e.g. Wikipedia editor wrote that + # X authored Y). The signals list stays empty for tier-6. + if not signals: + if spans or uris: + return WarrantClassification( + tier="AUTHOR_SECONDARY_SOURCE", + tier_rank=TIER_RANK["AUTHOR_SECONDARY_SOURCE"], + signals=[], + candidate_names=[], + note="no direct authorship signal in cited evidence; warrant rests on third-party claim", + ).to_dict() + return WarrantClassification( + tier=NO_AUTHORSHIP_SIGNAL, + tier_rank=99, + signals=[], + candidate_names=[], + note="no cited evidence to classify", + ).to_dict() + + # Pick the strongest tier among fired signals. + best_tier = min(signals, key=lambda s: TIER_RANK[s["tier"]])["tier"] + return WarrantClassification( + tier=best_tier, + tier_rank=TIER_RANK[best_tier], + signals=signals, + candidate_names=sorted(candidates), + note=f"strongest signal: {best_tier}", + ).to_dict() + + +# ---------- Helpers -------------------------------------------------------- + + +# Tokens that look name-shaped (capitalized) but are noise rather than +# real authorship — section headers, generic web chrome, etc. +_NOISE_NAMES = frozenset({ + "Author", "Authors", "Maintainer", "Maintainers", + "Copyright", "All", "Rights", "Reserved", + "Home", "About", "Contact", "Index", "Page", + "Project", "Projects", "Repository", "Repositories", + "Github", "GitHub", "Gitlab", "GitLab", "Bitbucket", + "License", "Licensed", "Open", "Source", + "True", "False", "None", "Null", + "January", "February", "March", "April", "May", "June", + "July", "August", "September", "October", "November", "December", +}) + + +def _looks_like_noise(name: str) -> bool: + """Reject capitalized tokens that are obviously not names.""" + if not name: + return True + if name in _NOISE_NAMES: + return True + # Single-letter or single-token all-caps headings are suspect. + if len(name) <= 2: + return True + return False + + +def _detect_primary_page_title( + answer_text: str, + uris: list[str], + titles: list[str], +) -> list[dict]: + """Tier-4: cited evidence is the entity's own primary page. + + Heuristic: extract host from each URI; tokenize the host (drop + www. + tld); if any token appears in the matching cited title + AND the host isn't a known third-party indexer (wikipedia.org, + archive.org, etc.), call it a primary-page hit. + """ + THIRD_PARTY_HOSTS = { + "wikipedia.org", "wikidata.org", "archive.org", + "stackoverflow.com", "stackexchange.com", + "medium.com", "substack.com", "wordpress.com", + "blogspot.com", "github.io", # github.io is mixed; conservative reject + } + hits: list[dict] = [] + ans_lower_concat = re.sub(r"[^a-z]", "", answer_text.lower()) + for i, uri in enumerate(uris): + host = _extract_host(uri) + if not host: + continue + # Reject known third-party platforms. + if any(host == tp or host.endswith("." + tp) for tp in THIRD_PARTY_HOSTS): + continue + # Get the matching title (best-effort by index alignment). + title = titles[i] if i < len(titles) else "" + host_tokens = _host_tokens(host) + title_lower_concat = re.sub(r"[^a-z]", "", title.lower()) + # Match logic: a host token (e.g. "russellballestrini") matches + # if its letters appear consecutively in the title's + # concatenated lowercase form (which collapses "Russell + # Ballestrini" → "russellballestrini"). Same fuzziness against + # the answer text. This handles compound hostnames where the + # title splits the words via space/punctuation. + matched: set[str] = set() + for ht in host_tokens: + if ht in title_lower_concat and ht in ans_lower_concat: + matched.add(ht) + if matched: + hits.append({ + "tier": "AUTHOR_PRIMARY_PAGE_TITLE", + "matched": f"host={host} title={title!r}", + "name": " ".join(sorted(matched)).title(), + }) + return hits + + +def _extract_host(uri: str) -> str: + """Pull the hostname from a URI string, no urllib needed.""" + m = re.match(r"^[a-z]+://([^/\s]+)", uri or "", re.IGNORECASE) + if not m: + return "" + host = m.group(1).lower() + if host.startswith("www."): + host = host[4:] + return host + + +def _host_tokens(host: str) -> set[str]: + """Tokenize hostname into content tokens (drop tld + 'www').""" + if not host: + return set() + parts = host.split(".") + if len(parts) > 1: + parts = parts[:-1] # drop tld + return {p for p in parts if p and p != "www" and len(p) > 2} + + +__all__ = [ + "TIER_NAMES", + "TIER_RANK", + "NO_AUTHORSHIP_SIGNAL", + "WarrantClassification", + "diagnose_authorship_warrant", +] diff --git a/bench/fixtures/5f/falsification-v1.jsonl b/bench/fixtures/5f/falsification-v1.jsonl index 20e9a69..efab6db 100644 --- a/bench/fixtures/5f/falsification-v1.jsonl +++ b/bench/fixtures/5f/falsification-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta":{"battery":"5f","sub_battery":"falsification","version":"v1","task_count":10,"notes":"Planted-bad-record fixtures with expected violation tags. Each fixture pinned to verifier_method_root so verifier-shape changes warn rather than false-fail."}} +{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 30, "notes": "Planted-bad-record fixtures with expected violation tags. Each fixture pinned to verifier_method_root so verifier-shape changes warn rather than false-fail. Phase 1c (2026-05-09): expanded 10→30."}} {"id":"5f-fal-001","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_001","observed_violations":["WARRANT_MISSING","TITLE_MISMATCH"],"expected_reason":"WARRANT_MISSING","verifier_method_root":"warrant-v1","expected":"pass"} {"id":"5f-fal-002","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_002","observed_violations":["TITLE_MISMATCH"],"expected_reason":"TITLE_MISMATCH","verifier_method_root":"warrant-v1","expected":"pass"} {"id":"5f-fal-003","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_003","observed_violations":["TOO_MANY_CLAIMS"],"expected_reason":"TOO_MANY_CLAIMS","verifier_method_root":"warrant-v1","expected":"pass"} @@ -9,3 +9,23 @@ {"id":"5f-fal-008","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_008","observed_violations":["UNRELATED_TAG"],"expected_reason":"WARRANT_MISSING","verifier_method_root":"warrant-v1","expected":"fail"} {"id":"5f-fal-009","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_009","observed_violations":["FORMAT_COLLAPSED","WARRANT_MISSING"],"expected_reason":"FORMAT_COLLAPSED","verifier_method_root":"warrant-v1","expected":"pass"} {"id":"5f-fal-010","battery":"5f","sub_battery":"falsification","version":"v1","carrier":"providence_record","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","record":"planted_bad_010","observed_violations":["NO_EVIDENCE_POINTER"],"expected_reason":"NO_EVIDENCE_POINTER","verifier_method_root":"warrant-v1","expected":"pass"} +{"id": "5f-fal-011", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_011", "observed_violations": ["WARRANT_MISSING"], "expected_reason": "WARRANT_MISSING", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-012", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_012", "observed_violations": ["TITLE_MISMATCH"], "expected_reason": "TITLE_MISMATCH", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-013", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_013", "observed_violations": ["TOO_MANY_CLAIMS", "WARRANT_MISSING"], "expected_reason": "TOO_MANY_CLAIMS", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-014", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_014", "observed_violations": ["BARE_NAME_CLAIM"], "expected_reason": "BARE_NAME_CLAIM", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-015", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_015", "observed_violations": ["FORMAT_COLLAPSED"], "expected_reason": "FORMAT_COLLAPSED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-016", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_016", "observed_violations": ["NO_EVIDENCE_POINTER"], "expected_reason": "NO_EVIDENCE_POINTER", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-017", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_017", "observed_violations": ["LAZY_ANCHOR_DEMOTED"], "expected_reason": "LAZY_ANCHOR_DEMOTED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-018", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_018", "observed_violations": ["POINTER_OVERFLOW_TRIMMED"], "expected_reason": "POINTER_OVERFLOW_TRIMMED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-019", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_019", "observed_violations": ["WARRANT_MISSING", "TITLE_MISMATCH", "TOO_MANY_CLAIMS"], "expected_reason": "WARRANT_MISSING", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-020", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_020", "observed_violations": ["BARE_NAME_CLAIM", "TITLE_MISMATCH"], "expected_reason": "BARE_NAME_CLAIM", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-021", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_021", "observed_violations": ["UNGROUNDED"], "expected_reason": "UNGROUNDED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-022", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_022", "observed_violations": ["HYBRID_PARAPHRASE"], "expected_reason": "HYBRID_PARAPHRASE", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-023", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_023", "observed_violations": ["STRICT_QUOTE", "LAZY_ANCHOR_DEMOTED"], "expected_reason": "LAZY_ANCHOR_DEMOTED", "verifier_method_root": "warrant-v1", "expected": "pass"} +{"id": "5f-fal-024", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_024", "observed_violations": [], "expected_reason": "WARRANT_MISSING", "verifier_method_root": "warrant-v1", "expected": "fail"} +{"id": "5f-fal-025", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_025", "observed_violations": ["UNRELATED_TAG"], "expected_reason": "TITLE_MISMATCH", "verifier_method_root": "warrant-v1", "expected": "fail"} +{"id": "5f-fal-026", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_026", "observed_violations": ["WARRANT_MISSING"], "expected_reason": "TITLE_MISMATCH", "verifier_method_root": "warrant-v1", "expected": "fail"} +{"id": "5f-fal-027", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_027", "observed_violations": ["FORMAT_COLLAPSED"], "expected_reason": "WARRANT_MISSING", "verifier_method_root": "warrant-v1", "expected": "fail"} +{"id": "5f-fal-028", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_028", "observed_violations": ["TITLE_MISMATCH", "BARE_NAME_CLAIM"], "expected_reason": "TOO_MANY_CLAIMS", "verifier_method_root": "warrant-v1", "expected": "fail"} +{"id": "5f-fal-029", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_029", "observed_violations": [], "expected_reason": "BARE_NAME_CLAIM", "verifier_method_root": "warrant-v1", "expected": "fail"} +{"id": "5f-fal-030", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "record": "planted_bad_030", "observed_violations": ["NO_EVIDENCE_POINTER"], "expected_reason": "FORMAT_COLLAPSED", "verifier_method_root": "warrant-v1", "expected": "fail"} diff --git a/bench/fixtures/5f/feedback-loop-v1.jsonl b/bench/fixtures/5f/feedback-loop-v1.jsonl index 2d95ecc..26ed669 100644 --- a/bench/fixtures/5f/feedback-loop-v1.jsonl +++ b/bench/fixtures/5f/feedback-loop-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta":{"battery":"5f","sub_battery":"feedback-loop","version":"v1","task_count":10,"notes":"Operation/observation chains. Final-step expected_delta must appear in aggregated observation feed; tests integration of observations into downstream state."}} +{"_meta": {"battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "task_count": 30, "notes": "Operation/observation chains. Final-step expected_delta must appear in aggregated observation feed; tests integration of observations into downstream state. Phase 1c (2026-05-09): expanded 10→30."}} {"id":"5f-fb-001","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"snapshot","observation":"claim X is stale"},{"operation":"update_memory","observation":"memory branch marks X stale"},{"operation":"snapshot","observation":"X appears in stale set","expected_delta":"X appears in stale set"}],"expected":"pass"} {"id":"5f-fb-002","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"new fact alpha learned"},{"operation":"update_memory","observation":"memory branch records alpha"},{"operation":"snapshot","observation":"alpha appears in current memory","expected_delta":"alpha appears in current memory"}],"expected":"pass"} {"id":"5f-fb-003","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"verifier flagged warrant_missing"},{"operation":"falsify","observation":"providence record marked stale"},{"operation":"snapshot","observation":"stale record appears in falsification log","expected_delta":"stale record appears in falsification log"}],"expected":"pass"} @@ -9,3 +9,23 @@ {"id":"5f-fb-008","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"correction applied to claim Q1"},{"operation":"update_memory","observation":"correction logged"},{"operation":"snapshot","observation":"corrected Q1 visible","expected_delta":"corrected Q1 visible"}],"expected":"pass"} {"id":"5f-fb-009","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"silent failure unobserved"},{"operation":"snapshot","observation":"chain produced no signal","expected_delta":"silent failure unobserved"}],"expected":"pass"} {"id":"5f-fb-010","battery":"5f","sub_battery":"feedback-loop","version":"v1","carrier":"memory_snapshot","domain":"memory_root","pi_star_ref":"pi_memory_v1","chain":[{"operation":"observe","observation":"ingest succeeded"},{"operation":"update_memory","observation":"ingest event in audit chain"},{"operation":"snapshot","observation":"ingest visible at high water mark","expected_delta":"ingest visible at high water mark"}],"expected":"pass"} +{"id": "5f-fb-011", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "snapshot", "observation": "claim Y is stale"}, {"operation": "update_memory", "observation": "memory branch marks Y stale"}, {"operation": "snapshot", "observation": "Y appears in stale set", "expected_delta": "Y appears in stale set"}], "expected": "pass"} +{"id": "5f-fb-012", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "new fact gamma learned"}, {"operation": "update_memory", "observation": "memory branch records gamma"}, {"operation": "snapshot", "observation": "gamma in current memory", "expected_delta": "gamma in current memory"}], "expected": "pass"} +{"id": "5f-fb-013", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact Z contradicts memory"}, {"operation": "update_memory", "observation": "memory branch invalidates Z's prior"}, {"operation": "snapshot", "observation": "Z's prior absent from current memory", "expected_delta": "Z's prior absent from current memory"}], "expected": "pass"} +{"id": "5f-fb-014", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "answer A scored STRICT"}, {"operation": "update_memory", "observation": "STRICT count for A increments"}, {"operation": "snapshot", "observation": "STRICT count up by 1", "expected_delta": "STRICT count up by 1"}], "expected": "pass"} +{"id": "5f-fb-015", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "snapshot", "observation": "branch B current"}, {"operation": "observe", "observation": "B's claim falsified"}, {"operation": "update_memory", "observation": "B marked stale"}, {"operation": "snapshot", "observation": "B no longer in live set", "expected_delta": "B no longer in live set"}], "expected": "pass"} +{"id": "5f-fb-016", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "shard 003 received update"}, {"operation": "update_memory", "observation": "shard 003 high-water bumped"}, {"operation": "snapshot", "observation": "high-water shows new value", "expected_delta": "high-water shows new value"}], "expected": "pass"} +{"id": "5f-fb-017", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "snapshot", "observation": "memory_root R0"}, {"operation": "update_memory", "observation": "R0 → R1 transition"}, {"operation": "snapshot", "observation": "memory_root R1", "expected_delta": "memory_root R1"}], "expected": "pass"} +{"id": "5f-fb-018", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "ForkScore parent baseline"}, {"operation": "update_memory", "observation": "baseline pinned"}, {"operation": "snapshot", "observation": "baseline visible to scorer", "expected_delta": "baseline visible to scorer"}], "expected": "pass"} +{"id": "5f-fb-019", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact delta seen"}, {"operation": "update_memory", "observation": "memory updated for unrelated thing"}, {"operation": "snapshot", "observation": "no mention of delta", "expected_delta": "delta visible in snapshot"}], "expected": "fail"} +{"id": "5f-fb-020", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "snapshot", "observation": "stale set s0"}, {"operation": "update_memory", "observation": "tried to update but failed"}, {"operation": "snapshot", "observation": "stale set unchanged", "expected_delta": "expected delta unrealized"}], "expected": "fail"} +{"id": "5f-fb-021", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "snapshot", "observation": "claim Y is stale"}, {"operation": "update_memory", "observation": "memory branch marks Y stale"}, {"operation": "snapshot", "observation": "Y appears in stale set", "expected_delta": "Y appears in stale set"}], "expected": "pass"} +{"id": "5f-fb-022", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "new fact gamma learned"}, {"operation": "update_memory", "observation": "memory branch records gamma"}, {"operation": "snapshot", "observation": "gamma in current memory", "expected_delta": "gamma in current memory"}], "expected": "pass"} +{"id": "5f-fb-023", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact Z contradicts memory"}, {"operation": "update_memory", "observation": "memory branch invalidates Z's prior"}, {"operation": "snapshot", "observation": "Z's prior absent from current memory", "expected_delta": "Z's prior absent from current memory"}], "expected": "pass"} +{"id": "5f-fb-024", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "answer A scored STRICT"}, {"operation": "update_memory", "observation": "STRICT count for A increments"}, {"operation": "snapshot", "observation": "STRICT count up by 1", "expected_delta": "STRICT count up by 1"}], "expected": "pass"} +{"id": "5f-fb-025", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "snapshot", "observation": "branch B current"}, {"operation": "observe", "observation": "B's claim falsified"}, {"operation": "update_memory", "observation": "B marked stale"}, {"operation": "snapshot", "observation": "B no longer in live set", "expected_delta": "B no longer in live set"}], "expected": "pass"} +{"id": "5f-fb-026", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "shard 003 received update"}, {"operation": "update_memory", "observation": "shard 003 high-water bumped"}, {"operation": "snapshot", "observation": "high-water shows new value", "expected_delta": "high-water shows new value"}], "expected": "pass"} +{"id": "5f-fb-027", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "snapshot", "observation": "memory_root R0"}, {"operation": "update_memory", "observation": "R0 → R1 transition"}, {"operation": "snapshot", "observation": "memory_root R1", "expected_delta": "memory_root R1"}], "expected": "pass"} +{"id": "5f-fb-028", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "ForkScore parent baseline"}, {"operation": "update_memory", "observation": "baseline pinned"}, {"operation": "snapshot", "observation": "baseline visible to scorer", "expected_delta": "baseline visible to scorer"}], "expected": "pass"} +{"id": "5f-fb-029", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "observe", "observation": "fact delta seen"}, {"operation": "update_memory", "observation": "memory updated for unrelated thing"}, {"operation": "snapshot", "observation": "no mention of delta", "expected_delta": "delta visible in snapshot"}], "expected": "fail"} +{"id": "5f-fb-030", "battery": "5f", "sub_battery": "feedback-loop", "version": "v1", "carrier": "memory_snapshot", "domain": "memory_root", "pi_star_ref": "pi_memory_v1", "chain": [{"operation": "snapshot", "observation": "stale set s0"}, {"operation": "update_memory", "observation": "tried to update but failed"}, {"operation": "snapshot", "observation": "stale set unchanged", "expected_delta": "expected delta unrealized"}], "expected": "fail"} diff --git a/bench/fixtures/5f/finetuning-v1.jsonl b/bench/fixtures/5f/finetuning-v1.jsonl index 93e0000..48e2395 100644 --- a/bench/fixtures/5f/finetuning-v1.jsonl +++ b/bench/fixtures/5f/finetuning-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta":{"battery":"5f","sub_battery":"finetuning","version":"v1","task_count":10,"notes":"Synthetic parent→child SelfModel measured-value pairs. Phase 1b.2 will pull from real shard SelfModel chains."}} +{"_meta": {"battery": "5f", "sub_battery": "finetuning", "version": "v1", "task_count": 30, "notes": "Synthetic parent→child SelfModel measured-value pairs. Phase 1b.2 will pull from real shard SelfModel chains. Phase 1c (2026-05-09): expanded 10→30."}} {"id":"5f-ft-001","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_001","child_selfmodel":"SM_CHILD_001","target_capability":"CAP-5S-SYLLOGISM","parent_measured_value":0.45,"child_measured_value":0.65,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1000,"max_storage_delta_bytes":1000000},"expected":"pass"} {"id":"5f-ft-002","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_002","child_selfmodel":"SM_CHILD_002","target_capability":"CAP-5T-TIME","parent_measured_value":0.40,"child_measured_value":0.55,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1500,"max_storage_delta_bytes":2000000},"expected":"pass"} {"id":"5f-ft-003","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_003","child_selfmodel":"SM_CHILD_003","target_capability":"CAP-5F-FORMULATE","parent_measured_value":0.60,"child_measured_value":0.85,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":2000,"max_storage_delta_bytes":3000000},"expected":"pass"} @@ -9,3 +9,23 @@ {"id":"5f-ft-008","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_008","child_selfmodel":"SM_CHILD_008","target_capability":"CAP-5F-FALSIFICATION","parent_measured_value":0.55,"child_measured_value":0.62,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":1000,"max_storage_delta_bytes":1000000},"expected":"fail"} {"id":"5f-ft-009","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_009","child_selfmodel":"SM_CHILD_009","target_capability":"CAP-5S-SEMIOTICS","parent_measured_value":0.35,"child_measured_value":0.55,"expected_improvement_min":0.05,"resource_budget":{"max_compute_ms_delta":1300,"max_storage_delta_bytes":1500000},"expected":"pass"} {"id":"5f-ft-010","battery":"5f","sub_battery":"finetuning","version":"v1","carrier":"selfmodel_snapshot","domain":"capability_transition","pi_star_ref":"pi_selfmodel_v1","parent_selfmodel":"SM_PARENT_010","child_selfmodel":"SM_CHILD_010","target_capability":"CAP-5F-FEEDBACK","parent_measured_value":0.50,"child_measured_value":0.72,"expected_improvement_min":0.10,"resource_budget":{"max_compute_ms_delta":1500,"max_storage_delta_bytes":2000000},"expected":"pass"} +{"id": "5f-ft-011", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_011", "child_selfmodel": "SM_CHILD_011", "target_capability": "CAP-5S-SYNTAX", "parent_measured_value": 0.3, "child_measured_value": 0.4, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1550, "max_storage_delta_bytes": 2100000}, "expected": "pass"} +{"id": "5f-ft-012", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_012", "child_selfmodel": "SM_CHILD_012", "target_capability": "CAP-5S-SEMANTICS", "parent_measured_value": 0.32, "child_measured_value": 0.43, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1600, "max_storage_delta_bytes": 2200000}, "expected": "pass"} +{"id": "5f-ft-013", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_013", "child_selfmodel": "SM_CHILD_013", "target_capability": "CAP-5S-SYLLOGISM", "parent_measured_value": 0.34, "child_measured_value": 0.45, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1650, "max_storage_delta_bytes": 2300000}, "expected": "pass"} +{"id": "5f-ft-014", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_014", "child_selfmodel": "SM_CHILD_014", "target_capability": "CAP-5S-SYNTHESIS", "parent_measured_value": 0.36, "child_measured_value": 0.47, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1700, "max_storage_delta_bytes": 2400000}, "expected": "pass"} +{"id": "5f-ft-015", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_015", "child_selfmodel": "SM_CHILD_015", "target_capability": "CAP-5S-SEMIOTICS", "parent_measured_value": 0.38, "child_measured_value": 0.4, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1750, "max_storage_delta_bytes": 2500000}, "expected": "fail"} +{"id": "5f-ft-016", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_016", "child_selfmodel": "SM_CHILD_016", "target_capability": "CAP-5T-TIME", "parent_measured_value": 0.4, "child_measured_value": 0.53, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1800, "max_storage_delta_bytes": 2600000}, "expected": "pass"} +{"id": "5f-ft-017", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_017", "child_selfmodel": "SM_CHILD_017", "target_capability": "CAP-5T-TRUTHTABLES", "parent_measured_value": 0.42, "child_measured_value": 0.55, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1850, "max_storage_delta_bytes": 2700000}, "expected": "pass"} +{"id": "5f-ft-018", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_018", "child_selfmodel": "SM_CHILD_018", "target_capability": "CAP-5T-TRANSFER-LEARNING", "parent_measured_value": 0.44, "child_measured_value": 0.58, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1900, "max_storage_delta_bytes": 2800000}, "expected": "pass"} +{"id": "5f-ft-019", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_019", "child_selfmodel": "SM_CHILD_019", "target_capability": "CAP-5F-FUNCTION", "parent_measured_value": 0.46, "child_measured_value": 0.6, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 1950, "max_storage_delta_bytes": 2900000}, "expected": "pass"} +{"id": "5f-ft-020", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_020", "child_selfmodel": "SM_CHILD_020", "target_capability": "CAP-5F-FALSIFICATION", "parent_measured_value": 0.48, "child_measured_value": 0.62, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2000, "max_storage_delta_bytes": 3000000}, "expected": "pass"} +{"id": "5f-ft-021", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_021", "child_selfmodel": "SM_CHILD_021", "target_capability": "CAP-5F-FEEDBACK", "parent_measured_value": 0.5, "child_measured_value": 0.65, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2050, "max_storage_delta_bytes": 3100000}, "expected": "pass"} +{"id": "5f-ft-022", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_022", "child_selfmodel": "SM_CHILD_022", "target_capability": "CAP-5F-FORMULATE", "parent_measured_value": 0.52, "child_measured_value": 0.54, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2100, "max_storage_delta_bytes": 3200000}, "expected": "fail"} +{"id": "5f-ft-023", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_023", "child_selfmodel": "SM_CHILD_023", "target_capability": "CAP-5F-FINETUNING", "parent_measured_value": 0.54, "child_measured_value": 0.7, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2150, "max_storage_delta_bytes": 3300000}, "expected": "pass"} +{"id": "5f-ft-024", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_024", "child_selfmodel": "SM_CHILD_024", "target_capability": "CAP-5R-REACT", "parent_measured_value": 0.56, "child_measured_value": 0.73, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2200, "max_storage_delta_bytes": 3400000}, "expected": "pass"} +{"id": "5f-ft-025", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_025", "child_selfmodel": "SM_CHILD_025", "target_capability": "CAP-5R-RECALL", "parent_measured_value": 0.58, "child_measured_value": 0.75, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2250, "max_storage_delta_bytes": 3500000}, "expected": "pass"} +{"id": "5f-ft-026", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_026", "child_selfmodel": "SM_CHILD_026", "target_capability": "CAP-5R-REASON", "parent_measured_value": 0.6, "child_measured_value": 0.77, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2300, "max_storage_delta_bytes": 3600000}, "expected": "pass"} +{"id": "5f-ft-027", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_027", "child_selfmodel": "SM_CHILD_027", "target_capability": "CAP-5R-RESTORE", "parent_measured_value": 0.62, "child_measured_value": 0.8, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2350, "max_storage_delta_bytes": 3700000}, "expected": "pass"} +{"id": "5f-ft-028", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_028", "child_selfmodel": "SM_CHILD_028", "target_capability": "CAP-5R-REFINE", "parent_measured_value": 0.64, "child_measured_value": 0.66, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2400, "max_storage_delta_bytes": 3800000}, "expected": "fail"} +{"id": "5f-ft-029", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_029", "child_selfmodel": "SM_CHILD_029", "target_capability": "CAP-CANONICAL-MATH", "parent_measured_value": 0.66, "child_measured_value": 0.85, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2450, "max_storage_delta_bytes": 3900000}, "expected": "pass"} +{"id": "5f-ft-030", "battery": "5f", "sub_battery": "finetuning", "version": "v1", "carrier": "selfmodel_snapshot", "domain": "capability_transition", "pi_star_ref": "pi_selfmodel_v1", "parent_selfmodel": "SM_PARENT_030", "child_selfmodel": "SM_CHILD_030", "target_capability": "CAP-CANONICAL-LOGIC", "parent_measured_value": 0.68, "child_measured_value": 0.88, "expected_improvement_min": 0.05, "resource_budget": {"max_compute_ms_delta": 2500, "max_storage_delta_bytes": 4000000}, "expected": "pass"} diff --git a/bench/fixtures/5f/formulate-v1.jsonl b/bench/fixtures/5f/formulate-v1.jsonl index 43c674c..c329296 100644 --- a/bench/fixtures/5f/formulate-v1.jsonl +++ b/bench/fixtures/5f/formulate-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta":{"battery":"5f","sub_battery":"formulate","version":"v1","task_count":10,"notes":"Phase 1a fixtures embed produced + expected lattices for structural matching. Phase 1b.2 will route through arborist.qa.parse_claims."}} +{"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1", "task_count": 30, "notes": "Phase 1a fixtures embed produced + expected lattices for structural matching. Phase 1b.2 will route through arborist.qa.parse_claims. Phase 1c (2026-05-09): expanded 10→30."}} {"id":"5f-form-001","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"The system saved one claim with one pointer.","produced_lattice":{"claims":[{"claim_text":"The system saved one claim with one pointer.","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"The system saved one claim with one pointer.","pointer_ids":["E1"]}]},"expected":"pass"} {"id":"5f-form-002","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Two claims with two pointers each.","produced_lattice":{"claims":[{"claim_text":"first claim with pointers","pointer_ids":["E1","E2"]},{"claim_text":"second claim with pointers","pointer_ids":["E3","E4"]}]},"expected_lattice":{"claim_count":2,"claims":[{"claim_text":"first claim with pointers","pointer_ids":["E1","E2"]},{"claim_text":"second claim with pointers","pointer_ids":["E3","E4"]}]},"expected":"pass"} {"id":"5f-form-003","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Detect missing pointer claim.","produced_lattice":{"claims":[{"claim_text":"a claim","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"a claim","pointer_ids":["E1","E2"]}]},"expected":"fail"} @@ -9,3 +9,23 @@ {"id":"5f-form-008","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Two claims with shared pointer.","produced_lattice":{"claims":[{"claim_text":"first observation","pointer_ids":["E1"]},{"claim_text":"second observation","pointer_ids":["E1"]}]},"expected_lattice":{"claim_count":2,"claims":[{"claim_text":"first observation","pointer_ids":["E1"]},{"claim_text":"second observation","pointer_ids":["E1"]}]},"expected":"pass"} {"id":"5f-form-009","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Detect pointer-id mismatch between produced and expected.","produced_lattice":{"claims":[{"claim_text":"sentence text","pointer_ids":["E5"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"sentence text","pointer_ids":["E1"]}]},"expected":"fail"} {"id":"5f-form-010","battery":"5f","sub_battery":"formulate","version":"v1","carrier":"text","domain":"claim_lattice","pi_star_ref":"claim-lattice@v1","input_text":"Multi-pointer claim with full pointer set match.","produced_lattice":{"claims":[{"claim_text":"complex multi-source claim","pointer_ids":["E1","E2","E3"]}]},"expected_lattice":{"claim_count":1,"claims":[{"claim_text":"complex multi-source claim","pointer_ids":["E1","E2","E3"]}]},"expected":"pass"} +{"id": "5f-form-011", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Three claims about water with three pointers each.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2", "E3"]}, {"claim_text": "claim 2", "pointer_ids": ["E4", "E5", "E6"]}, {"claim_text": "claim 3", "pointer_ids": ["E7", "E8", "E9"]}]}, "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2", "E3"]}, {"claim_text": "claim 2", "pointer_ids": ["E4", "E5", "E6"]}, {"claim_text": "claim 3", "pointer_ids": ["E7", "E8", "E9"]}]}, "expected": "pass"} +{"id": "5f-form-012", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Single claim about gravity.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-013", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Five claims, each with one pointer.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}]}, "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}]}, "expected": "pass"} +{"id": "5f-form-014", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Two claims with mixed pointers.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "claim 2", "pointer_ids": ["E3"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "claim 2", "pointer_ids": ["E3"]}]}, "expected": "pass"} +{"id": "5f-form-015", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Four claims about programming languages.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected": "pass"} +{"id": "5f-form-016", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "One claim, two pointers.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}]}, "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}]}, "expected": "pass"} +{"id": "5f-form-017", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Six claims about historical events.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-018", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Two claims about astronomy.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "claim 2", "pointer_ids": ["E3", "E4"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "claim 2", "pointer_ids": ["E3", "E4"]}]}, "expected": "pass"} +{"id": "5f-form-019", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Three claims about chemistry.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}]}, "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}]}, "expected": "pass"} +{"id": "5f-form-020", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Eight claims with single pointer each.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}, {"claim_text": "claim 7", "pointer_ids": ["E1"]}, {"claim_text": "claim 8", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 8, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}, {"claim_text": "claim 7", "pointer_ids": ["E1"]}, {"claim_text": "claim 8", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-021", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Expected one but produced two.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}]}, "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}]}, "expected": "fail"} +{"id": "5f-form-022", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Expected three but produced one.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}]}, "expected": "fail"} +{"id": "5f-form-023", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Three claims about water with three pointers each.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2", "E3"]}, {"claim_text": "claim 2", "pointer_ids": ["E4", "E5", "E6"]}, {"claim_text": "claim 3", "pointer_ids": ["E7", "E8", "E9"]}]}, "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2", "E3"]}, {"claim_text": "claim 2", "pointer_ids": ["E4", "E5", "E6"]}, {"claim_text": "claim 3", "pointer_ids": ["E7", "E8", "E9"]}]}, "expected": "pass"} +{"id": "5f-form-024", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Single claim about gravity.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-025", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Five claims, each with one pointer.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}]}, "expected_lattice": {"claim_count": 5, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}]}, "expected": "pass"} +{"id": "5f-form-026", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Two claims with mixed pointers.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "claim 2", "pointer_ids": ["E3"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "claim 2", "pointer_ids": ["E3"]}]}, "expected": "pass"} +{"id": "5f-form-027", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Four claims about programming languages.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected_lattice": {"claim_count": 4, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected": "pass"} +{"id": "5f-form-028", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "One claim, two pointers.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}]}, "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}]}, "expected": "pass"} +{"id": "5f-form-029", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Six claims about historical events.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected_lattice": {"claim_count": 6, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E1"]}, {"claim_text": "claim 3", "pointer_ids": ["E1"]}, {"claim_text": "claim 4", "pointer_ids": ["E1"]}, {"claim_text": "claim 5", "pointer_ids": ["E1"]}, {"claim_text": "claim 6", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-form-030", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Two claims about astronomy.", "produced_lattice": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "claim 2", "pointer_ids": ["E3", "E4"]}]}, "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "claim 1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "claim 2", "pointer_ids": ["E3", "E4"]}]}, "expected": "pass"} diff --git a/bench/fixtures/5f/function-v1.jsonl b/bench/fixtures/5f/function-v1.jsonl index 61fc1f8..fd7dac0 100644 --- a/bench/fixtures/5f/function-v1.jsonl +++ b/bench/fixtures/5f/function-v1.jsonl @@ -1,4 +1,4 @@ -{"_meta":{"battery":"5f","sub_battery":"function","version":"v1","task_count":10,"notes":"Phase 1a: ten seed tasks across shape_match/pointer_set_match/threshold_on_metric evaluators. Phase 1b expands to 30+."}} +{"_meta": {"battery": "5f", "sub_battery": "function", "version": "v1", "task_count": 30, "notes": "Phase 1a: ten seed tasks across shape_match/pointer_set_match/threshold_on_metric evaluators. Phase 1b expands to 30+. Phase 1c (2026-05-09): expanded 10→30."}} {"id":"5f-fn-001","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Return exactly two claim-lattice claims with evidence pointers.","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"first claim","pointer_ids":["E1"]},{"claim_text":"second claim","pointer_ids":["E2"]}]},"expected":"pass"} {"id":"5f-fn-002","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Return exactly three claims with pointers.","evaluator":"shape_match","expected_shape":{"claim_count":3,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"a","pointer_ids":["E1"]},{"claim_text":"b","pointer_ids":["E2"]},{"claim_text":"c","pointer_ids":["E3"]}]},"expected":"pass"} {"id":"5f-fn-003","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Detect missing pointers in claims.","evaluator":"shape_match","expected_shape":{"claim_count":2,"pointers_required":true},"produced_output":{"claims":[{"claim_text":"a","pointer_ids":[]},{"claim_text":"b","pointer_ids":["E2"]}]},"expected":"fail"} @@ -9,3 +9,23 @@ {"id":"5f-fn-008","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Verify strict-rate threshold.","evaluator":"threshold_on_metric","expected_shape":{"metric":"strict_rate","threshold":0.5},"produced_output":{"strict_rate":0.62},"expected":"pass"} {"id":"5f-fn-009","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Detect failed metric threshold.","evaluator":"threshold_on_metric","expected_shape":{"metric":"strict_rate","threshold":0.5},"produced_output":{"strict_rate":0.31},"expected":"fail"} {"id":"5f-fn-010","battery":"5f","sub_battery":"function","version":"v1","carrier":"claim_lattice","domain":"qa_answer","pi_star_ref":"claim-lattice@v1","directive":"Verify pointer set with single pointer.","evaluator":"pointer_set_match","expected_shape":{"pointer_set":["E1"]},"produced_output":{"claims":[{"claim_text":"single","pointer_ids":["E1"]}]},"expected":"pass"} +{"id": "5f-fn-011", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 1 claims with pointers.", "evaluator": "shape_match", "expected_shape": {"claim_count": 1, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}]}, "expected": "pass"} +{"id": "5f-fn-012", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 4 claims with pointers.", "evaluator": "shape_match", "expected_shape": {"claim_count": 4, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}]}, "expected": "pass"} +{"id": "5f-fn-013", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 5 claims with pointers.", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}]}, "expected": "pass"} +{"id": "5f-fn-014", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 6 claims with pointers.", "evaluator": "shape_match", "expected_shape": {"claim_count": 6, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}]}, "expected": "pass"} +{"id": "5f-fn-015", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 7 claims with pointers.", "evaluator": "shape_match", "expected_shape": {"claim_count": 7, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}, {"claim_text": "claim 7", "pointer_ids": ["E7"]}]}, "expected": "pass"} +{"id": "5f-fn-016", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Return exactly 8 claims with pointers.", "evaluator": "shape_match", "expected_shape": {"claim_count": 8, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "claim 1", "pointer_ids": ["E1"]}, {"claim_text": "claim 2", "pointer_ids": ["E2"]}, {"claim_text": "claim 3", "pointer_ids": ["E3"]}, {"claim_text": "claim 4", "pointer_ids": ["E4"]}, {"claim_text": "claim 5", "pointer_ids": ["E5"]}, {"claim_text": "claim 6", "pointer_ids": ["E6"]}, {"claim_text": "claim 7", "pointer_ids": ["E7"]}, {"claim_text": "claim 8", "pointer_ids": ["E8"]}]}, "expected": "pass"} +{"id": "5f-fn-017", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Detect wrong claim count (1 vs expected 2).", "evaluator": "shape_match", "expected_shape": {"claim_count": 2, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "x", "pointer_ids": ["E1"]}]}, "expected": "fail"} +{"id": "5f-fn-018", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Detect wrong claim count (2 vs expected 3).", "evaluator": "shape_match", "expected_shape": {"claim_count": 3, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "x", "pointer_ids": ["E1"]}, {"claim_text": "x", "pointer_ids": ["E2"]}]}, "expected": "fail"} +{"id": "5f-fn-019", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Detect wrong claim count (3 vs expected 5).", "evaluator": "shape_match", "expected_shape": {"claim_count": 5, "pointers_required": true}, "produced_output": {"claims": [{"claim_text": "x", "pointer_ids": ["E1"]}, {"claim_text": "x", "pointer_ids": ["E2"]}, {"claim_text": "x", "pointer_ids": ["E3"]}]}, "expected": "fail"} +{"id": "5f-fn-020", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Pointer-set comparison.", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E1", "E2", "E3", "E4"]}, "produced_output": {"claims": [{"claim_text": "c1", "pointer_ids": ["E1", "E2"]}, {"claim_text": "c2", "pointer_ids": ["E3", "E4"]}]}, "expected": "pass"} +{"id": "5f-fn-021", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Pointer-set comparison.", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E1", "E2"]}, "produced_output": {"claims": [{"claim_text": "c1", "pointer_ids": ["E1", "E2"]}]}, "expected": "pass"} +{"id": "5f-fn-022", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Pointer-set comparison.", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E1"]}, "produced_output": {"claims": [{"claim_text": "c1", "pointer_ids": ["E2"]}]}, "expected": "fail"} +{"id": "5f-fn-023", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Pointer-set comparison.", "evaluator": "pointer_set_match", "expected_shape": {"pointer_set": ["E1", "E2", "E3"]}, "produced_output": {"claims": [{"claim_text": "c1", "pointer_ids": ["E1"]}, {"claim_text": "c2", "pointer_ids": ["E2"]}]}, "expected": "fail"} +{"id": "5f-fn-024", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on strict_rate.", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "strict_rate", "threshold": 0.5}, "produced_output": {"strict_rate": 0.7}, "expected": "pass"} +{"id": "5f-fn-025", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on strict_rate.", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "strict_rate", "threshold": 0.5}, "produced_output": {"strict_rate": 0.45}, "expected": "fail"} +{"id": "5f-fn-026", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on directive_coverage.", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "directive_coverage", "threshold": 0.9}, "produced_output": {"directive_coverage": 0.99}, "expected": "pass"} +{"id": "5f-fn-027", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on directive_coverage.", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "directive_coverage", "threshold": 0.95}, "produced_output": {"directive_coverage": 0.92}, "expected": "fail"} +{"id": "5f-fn-028", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on hybrid_rate.", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "hybrid_rate", "threshold": 0.3}, "produced_output": {"hybrid_rate": 0.32}, "expected": "pass"} +{"id": "5f-fn-029", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on strict_rate (sub-floor).", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "strict_rate", "threshold": 0.55}, "produced_output": {"strict_rate": 0.4}, "expected": "fail"} +{"id": "5f-fn-030", "battery": "5f", "sub_battery": "function", "version": "v1", "carrier": "claim_lattice", "domain": "qa_answer", "pi_star_ref": "claim-lattice@v1", "directive": "Threshold on strict_rate.", "evaluator": "threshold_on_metric", "expected_shape": {"metric": "strict_rate", "threshold": 0.6}, "produced_output": {"strict_rate": 0.6}, "expected": "pass"} diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 88c07fc..975ce2b 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -65,8 +65,8 @@ Newest first. Update on every open/close. | #000029 | Claim-pack source (axiom/theorem JSON bundles) | closed · landed 2026-05-09 | 2026-05-09 | — | | #000028 | Multi-modality witness for canonical shapes | closed · landed 2026-05-09 (STRICT-WITNESSED reachable post-#000027) | 2026-05-08 | — | | #000027 | Canonical projections persist to providence_cache | closed · landed 2026-05-09 | 2026-05-08 | — | -| #000026 | Real-shard workload baseline + search latency | in progress · Phase 1 + 2 landed 2026-05-08 | 2026-05-08 | — | -| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a landed 2026-05-08 | 2026-05-07 | — | +| #000026 | Real-shard workload baseline + search latency | in progress · Phase 1 + 2 + 3 landed 2026-05-09 | 2026-05-08 | — | +| #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a + 1b.2 + 1c landed 2026-05-09 | 2026-05-07 | — | | #000024 | 5T Phase 1b + Dav1DPrometheus vocabulary alignment | closed · landed 2026-05-08 | 2026-05-07 | — | | #000023 | 5S Phase 1b: Syllogism · Synthesis · Semiotics | closed · landed 2026-05-08 | 2026-05-07 | — | | #000022 | Adapter LossReport (PRD I9 analogue) | closed · landed 2026-05-07 | 2026-05-07 | — | @@ -79,7 +79,7 @@ Newest first. Update on every open/close. | #000015 | π* domain library + cross-domain composition | closed · landed 2026-05-07 | 2026-05-07 | — | | #000014 | SelfModel: schema, falsification, integration | closed · landed 2026-05-07 | 2026-05-07 | — | | #000013 | Spatial-temporal substrate (Merkle-AGI v7-W) | open · awaiting go/no-go | 2026-05-07 | — | -| #000012 | Selection & consensus protocol (Merkle-AGI v8) | in progress · Phase 1a (ForkScore) landed 2026-05-08 | 2026-05-07 | — | +| #000012 | Selection & consensus protocol (Merkle-AGI v8) | in progress · Phase 1a + 1b (ForkScore CLI) landed 2026-05-09 | 2026-05-07 | — | | #000011 | SOFT_PREFLIGHT_HINT model-assisted sidecar | closed · landed 2026-05-04 (zero-shot full impl) | 2026-05-04 | D1 (preserves) | | #000010 | Meta-Cognition Preflight Guard (M0 / MCTL) | closed · landed 2026-05-03 (Phases 1–4); DAG binding shipped via #000009 | 2026-05-03 | D1, D3 | | #000009 | Preflight run-DAG node binding (#000008+#000010) | closed · re-landed 2026-05-04 (§8 corrections: reject-path DAG, nested CTI clauses) | 2026-05-03 | D3, D4 | diff --git a/docs/tickets/ticket-000026-real-shard-workload-baseline.md b/docs/tickets/ticket-000026-real-shard-workload-baseline.md index d3bfb89..e659015 100644 --- a/docs/tickets/ticket-000026-real-shard-workload-baseline.md +++ b/docs/tickets/ticket-000026-real-shard-workload-baseline.md @@ -1,6 +1,6 @@ # Ticket #000026 — Real-shard workload baseline + search latency -**Status:** in progress · Phase 1 + 2 landed 2026-05-08 +**Status:** in progress · Phase 1 + 2 + 3 landed 2026-05-09 **Opened:** 2026-05-08 **Scope:** establish a reproducible baseline for arborist's behavior on real shards (~38 GB Wikipedia + crawl). Capture latency, capital @@ -308,3 +308,39 @@ per the design choices section. Phase 1 landed in commit `ec92ebc`. Phase 2 landed in commit `8b1de20`. + +**Phase 3 landed 2026-05-09.** Authorship warrant ladder as a +sidecar classifier in `arborist/qa/warrant_authorship.py`: + +``` +Tier 1 AUTHOR_PACKAGE_METADATA (pyproject.toml authors, + package.json author, etc.) +Tier 2 AUTHOR_REPOSITORY_OWNER (github.com//...) +Tier 3 AUTHOR_PAGE_BYLINE (By NAME / Author: NAME / + ) +Tier 4 AUTHOR_PRIMARY_PAGE_TITLE (cited evidence is the + entity's own primary page) +Tier 5 AUTHOR_COPYRIGHT_FOOTER (© NAME / Copyright YYYY NAME + — the current `virt-back` + warrant) +Tier 6 AUTHOR_SECONDARY_SOURCE (third-party claim) +NO_AUTHORSHIP_SIGNAL (question doesn't smell like + an authorship inquiry — + sidecar stays quiet) +``` + +Sidecar discipline: never enters proof path; never raises; +returns dict with `tier`, `tier_rank` (1=strongest, 99=quiet), +`signals` (every detector that fired), `candidate_names`, +`note`. Hint regex gates on "who wrote/created/authored/maintains/ +owns/manages X" + "author/creator/maintainer/owner of Y" patterns +so non-authorship queries don't get false-tier-6 noise. + +20 tests covering each tier (positive + boundary) + sidecar +contract (never raises on garbage input) + tier-ordering +(strongest signal wins when multiple fire). Authorship-warrant +ladder is now available; wiring it into the `arborist inspect` +sidecar output and the audit-line render-tail is a follow-up +(small, non-load-bearing). + +Authorship warrant ladder landed in commit ``. diff --git a/tests/test_bench_batteries.py b/tests/test_bench_batteries.py index 61126cf..0f1fbaa 100644 --- a/tests/test_bench_batteries.py +++ b/tests/test_bench_batteries.py @@ -227,31 +227,32 @@ def test_5f_function_runs(): res = b_5f.run_function(F5F / "function-v1.jsonl") assert res.battery == "5f" assert res.sub_battery == "function" - assert res.pass_count == 10 + # Phase 1c (2026-05-09) — fixture catalog expanded 10 → 30. + assert res.pass_count == 30 assert res.metrics["function_pass_rate"] == 1.0 def test_5f_finetuning_runs(): res = b_5f.run_finetuning(F5F / "finetuning-v1.jsonl") - assert res.pass_count == 10 + assert res.pass_count == 30 assert res.metrics["adaptation_improvement_rate"] == 1.0 def test_5f_falsification_runs(): res = b_5f.run_falsification(F5F / "falsification-v1.jsonl") - assert res.pass_count == 10 + assert res.pass_count == 30 assert res.metrics["error_detection_rate"] == 1.0 def test_5f_formulate_runs(): res = b_5f.run_formulate(F5F / "formulate-v1.jsonl") - assert res.pass_count == 10 + assert res.pass_count == 30 assert res.metrics["structural_match_rate"] == 1.0 def test_5f_feedback_loop_runs(): res = b_5f.run_feedback_loop(F5F / "feedback-loop-v1.jsonl") - assert res.pass_count == 10 + assert res.pass_count == 30 assert res.metrics["integration_coverage_rate"] == 1.0 @@ -430,9 +431,10 @@ def test_5f_formulate_live_path_routes_through_parse_claims(): def test_5f_formulate_embedded_path_still_works(): """Phase 1a fixtures (embedded produced_lattice) keep working - after the Phase 1b.2 wire-up. Backward compat invariant.""" + after the Phase 1b.2 wire-up + Phase 1c expansion. Backward + compat invariant.""" res = b_5f.run_formulate(F5F / "formulate-v1.jsonl") - assert res.pass_count == 10 + assert res.pass_count == 30 # Phase 1c — expanded 10 → 30 for t in res.per_task: assert t.detail["source"] == "embedded" @@ -485,7 +487,7 @@ def test_5f_feedback_loop_live_path_writes_real_audit_events(): def test_5f_feedback_loop_embedded_path_still_works(): res = b_5f.run_feedback_loop(F5F / "feedback-loop-v1.jsonl") - assert res.pass_count == 10 + assert res.pass_count == 30 # Phase 1c for t in res.per_task: assert t.detail["source"] == "embedded" @@ -558,7 +560,7 @@ def test_5f_function_live_path_routes_through_parse_claims(): def test_5f_function_embedded_path_still_works(): res = b_5f.run_function(F5F / "function-v1.jsonl") - assert res.pass_count == 10 + assert res.pass_count == 30 # Phase 1c for t in res.per_task: assert t.detail["source"] == "embedded" @@ -582,7 +584,7 @@ def test_5f_finetuning_live_path_round_trips_selfmodel(): def test_5f_finetuning_embedded_path_still_works(): res = b_5f.run_finetuning(F5F / "finetuning-v1.jsonl") - assert res.pass_count == 10 + assert res.pass_count == 30 # Phase 1c for t in res.per_task: assert t.detail["source"] == "embedded" @@ -616,7 +618,7 @@ def test_5f_falsification_live_path_routes_through_verify_quotes(): def test_5f_falsification_embedded_path_still_works(): res = b_5f.run_falsification(F5F / "falsification-v1.jsonl") - assert res.pass_count == 10 + assert res.pass_count == 30 # Phase 1c for t in res.per_task: assert t.detail["source"] == "embedded" diff --git a/tests/test_session_integration.py b/tests/test_session_integration.py index bd243ad..d0e5ff2 100644 --- a/tests/test_session_integration.py +++ b/tests/test_session_integration.py @@ -239,15 +239,21 @@ def test_full_dav1dprometheus_suite_runs_end_to_end(tmp_path, capsys): def test_full_suite_total_fixture_count(): - """Sanity check: the complete Dav1DPrometheus suite executes 462 - deterministic tasks across 21 sub-batteries (5S+5T+5F+5R).""" + """Sanity check: the complete Dav1DPrometheus suite executes 562 + deterministic tasks across 21 sub-batteries (5S+5T+5F+5R). + + History: + - Phase 1a baseline: 462 tasks. + - Phase 1c (#000025, 2026-05-09): 5F synthetic side expanded + 10 → 30 across all 5 sub-batteries; +100 → 562. + """ from bench.batteries.runner import _DEFAULT_FIXTURES, _run_one total = 0 for (battery, sub), fx in _DEFAULT_FIXTURES.items(): result = _run_one(battery, sub, Path(fx)) total += result.pass_count + result.fail_count - assert total == 462 + assert total == 562 def test_5s_phase1a_digests_unchanged_after_phase1b(): diff --git a/tests/test_v8_fork_score.py b/tests/test_v8_fork_score.py index cb2b859..f9d91d0 100644 --- a/tests/test_v8_fork_score.py +++ b/tests/test_v8_fork_score.py @@ -449,6 +449,39 @@ def test_cli_v8_score_rejects_returns_nonzero(tmp_path, capsys): assert payload["verdict"] == "REJECT" +def test_cli_v8_score_out_writes_json_artifact(tmp_path, capsys): + """#000012 Phase 1b — `--out` mirrors stdout to a file so CI / + mesh peers / downstream graders can ingest the artifact without + parsing pipe output.""" + from arborist.cli import build_parser + + parent_path = tmp_path / "parent.json" + child_path = tmp_path / "child.json" + out_path = tmp_path / "results" / "fork_score.json" # parent dir missing on purpose + _write_bench_result(parent_path, { + "5s": {"syntax": {"parse_pass_rate": 0.80}}, + "5t": {}, "5f": {}, + }) + _write_bench_result(child_path, { + "5s": {"syntax": {"parse_pass_rate": 0.85}}, + "5t": {}, "5f": {}, + }) + parser = build_parser() + args = parser.parse_args([ + "v8", "score", + "--parent", str(parent_path), + "--child", str(child_path), + "--out", str(out_path), + ]) + args.func(args) + stdout_payload = json.loads(capsys.readouterr().out) + assert out_path.is_file(), "--out should create parent directories" + file_payload = json.loads(out_path.read_text(encoding="utf-8")) + # Stdout and file are byte-identical artifacts. + assert stdout_payload == file_payload + assert "verdict" in file_payload + + # --------------------------------------------------------------------- # Determinism # --------------------------------------------------------------------- diff --git a/tests/test_warrant_authorship.py b/tests/test_warrant_authorship.py new file mode 100644 index 0000000..5b759ae --- /dev/null +++ b/tests/test_warrant_authorship.py @@ -0,0 +1,273 @@ +"""Tests for the authorship warrant ladder (#000026 Phase 3).""" + +from __future__ import annotations + +from arborist.qa.warrant_authorship import ( + NO_AUTHORSHIP_SIGNAL, + TIER_RANK, + diagnose_authorship_warrant, +) + + +# ---------- Question gating ------------------------------------------------ + + +def test_non_authorship_question_returns_no_signal(): + """Sidecar should stay quiet on irrelevant questions.""" + out = diagnose_authorship_warrant( + question_text="what is the capital of france?", + answer_text="Paris.", + cited_evidence_spans=["© 2024 Wikipedia Foundation"], + ) + assert out["tier"] == NO_AUTHORSHIP_SIGNAL + assert out["tier_rank"] == 99 + + +def test_no_evidence_no_question_returns_no_signal(): + out = diagnose_authorship_warrant() + assert out["tier"] == NO_AUTHORSHIP_SIGNAL + + +def test_no_evidence_with_authorship_question_returns_no_signal(): + """Authorship question but zero cited evidence — sidecar stays + quiet rather than reporting tier-6 secondary on no evidence.""" + out = diagnose_authorship_warrant( + question_text="who wrote virt-back?", + answer_text="Russell Ballestrini.", + ) + assert out["tier"] == NO_AUTHORSHIP_SIGNAL + + +# ---------- Tier 1 — package metadata -------------------------------------- + + +def test_tier1_python_pyproject_author(): + out = diagnose_authorship_warrant( + question_text="who wrote arborist?", + answer_text="Russell Ballestrini wrote arborist.", + cited_evidence_spans=[ + 'authors = [\n { name = "Russell Ballestrini" },\n]' + ], + ) + assert out["tier"] == "AUTHOR_PACKAGE_METADATA" + assert out["tier_rank"] == 1 + assert "Russell Ballestrini" in out["candidate_names"] + + +def test_tier1_python_setup_py_author(): + out = diagnose_authorship_warrant( + question_text="who is the author of virt-back?", + answer_text="Russell Ballestrini.", + cited_evidence_spans=['author="Russell Ballestrini"'], + ) + assert out["tier"] == "AUTHOR_PACKAGE_METADATA" + + +def test_tier1_package_json_author(): + out = diagnose_authorship_warrant( + question_text="who created this library?", + answer_text="Foo Bar.", + cited_evidence_spans=['{"name":"my-pkg","author":"Foo Bar"}'], + ) + assert out["tier"] == "AUTHOR_PACKAGE_METADATA" + + +# ---------- Tier 2 — repository owner -------------------------------------- + + +def test_tier2_github_uri_owner(): + out = diagnose_authorship_warrant( + question_text="who maintains this?", + answer_text="The repository is owned by russellballestrini.", + cited_source_uris=["https://github.com/russellballestrini/virt-back"], + cited_evidence_spans=["A README about the project."], + ) + assert out["tier"] == "AUTHOR_REPOSITORY_OWNER" + assert "russellballestrini" in out["candidate_names"] + + +def test_tier2_gitlab_uri_owner(): + out = diagnose_authorship_warrant( + question_text="who owns this project?", + answer_text="The team-arborist group.", + cited_source_uris=[ + "https://gitlab.com/team-arborist/some-project", + ], + cited_evidence_spans=["x"], + ) + assert out["tier"] == "AUTHOR_REPOSITORY_OWNER" + + +# ---------- Tier 3 — page byline ------------------------------------------- + + +def test_tier3_byline_prose_by_pattern(): + out = diagnose_authorship_warrant( + question_text="who wrote this article?", + answer_text="Russell Ballestrini.", + cited_evidence_spans=["By Russell Ballestrini, posted 2024-01-01."], + ) + assert out["tier"] == "AUTHOR_PAGE_BYLINE" + + +def test_tier3_byline_meta_html(): + out = diagnose_authorship_warrant( + question_text="who is the author of this page?", + answer_text="Russell Ballestrini.", + cited_evidence_spans=[ + '' + ], + ) + assert out["tier"] == "AUTHOR_PAGE_BYLINE" + + +# ---------- Tier 4 — primary page title ------------------------------------ + + +def test_tier4_primary_page_title_match(): + """Site host token appears in the cited title AND the answer + references the same entity. The cited evidence is the entity's + own primary page.""" + out = diagnose_authorship_warrant( + question_text="who wrote virt-back?", + answer_text="russellballestrini wrote virt-back.", + cited_source_uris=[ + "https://russellballestrini.net/virt-back-restoring-from-backups/", + ], + cited_source_titles=["virt-back: restoring from backups – Russell Ballestrini"], + cited_evidence_spans=["A blog post about virt-back."], + ) + # Should fire at tier 4 absent stronger signals (no package + # metadata, no repo URL, no byline, no copyright). + assert out["tier"] == "AUTHOR_PRIMARY_PAGE_TITLE" + + +def test_tier4_third_party_host_does_not_fire(): + """Wikipedia is a third-party indexer; primary-page heuristic + must not fire even if tokens overlap.""" + out = diagnose_authorship_warrant( + question_text="who wrote virt-back?", + answer_text="russell wrote virt-back.", + cited_source_uris=[ + "https://en.wikipedia.org/wiki/virt-back-russell" + ], + cited_source_titles=["virt-back russell"], + cited_evidence_spans=["x"], + ) + # With NO other signals firing, falls through to secondary. + assert out["tier"] == "AUTHOR_SECONDARY_SOURCE" + + +# ---------- Tier 5 — copyright footer -------------------------------------- + + +def test_tier5_copyright_footer_with_year(): + out = diagnose_authorship_warrant( + question_text="who wrote virt-back?", + answer_text="Russell Ballestrini wrote virt-back.", + cited_evidence_spans=["© 2024 Russell Ballestrini"], + ) + assert out["tier"] == "AUTHOR_COPYRIGHT_FOOTER" + + +def test_tier5_copyright_footer_no_year(): + out = diagnose_authorship_warrant( + question_text="who wrote virt-back?", + answer_text="Russell Ballestrini.", + cited_evidence_spans=["© Russell Ballestrini"], + ) + assert out["tier"] == "AUTHOR_COPYRIGHT_FOOTER" + + +def test_tier5_copyright_word_form(): + out = diagnose_authorship_warrant( + question_text="who created this site?", + answer_text="Russell Ballestrini.", + cited_evidence_spans=["Copyright 2024 Russell Ballestrini. All rights reserved."], + ) + assert out["tier"] == "AUTHOR_COPYRIGHT_FOOTER" + + +# ---------- Tier 6 — secondary source -------------------------------------- + + +def test_tier6_third_party_evidence_no_authorship_markers(): + """Question is authorship; cited evidence has none of the direct + markers — falls through to secondary.""" + out = diagnose_authorship_warrant( + question_text="who wrote virt-back?", + answer_text="Russell Ballestrini, according to the documentation.", + cited_evidence_spans=[ + "virt-back is a tool for backing up libvirt VMs." + ], + ) + assert out["tier"] == "AUTHOR_SECONDARY_SOURCE" + assert out["tier_rank"] == TIER_RANK["AUTHOR_SECONDARY_SOURCE"] + + +# ---------- Tier ordering: strongest wins ---------------------------------- + + +def test_strongest_tier_wins_when_multiple_fire(): + """Both copyright AND package-metadata fire — the package-metadata + signal is strictly stronger (tier 1 < tier 5).""" + out = diagnose_authorship_warrant( + question_text="who wrote virt-back?", + answer_text="Russell Ballestrini.", + cited_evidence_spans=[ + "© Russell Ballestrini", + 'author = "Russell Ballestrini"', + ], + ) + assert out["tier"] == "AUTHOR_PACKAGE_METADATA" + # But signals list captures BOTH for the operator. + tiers_seen = {s["tier"] for s in out["signals"]} + assert "AUTHOR_PACKAGE_METADATA" in tiers_seen + assert "AUTHOR_COPYRIGHT_FOOTER" in tiers_seen + + +# ---------- Sidecar contract ----------------------------------------------- + + +def test_returns_dict_always(): + """Sidecar contract: never raises, always returns a dict.""" + out = diagnose_authorship_warrant( + question_text=None, + answer_text="", + cited_evidence_spans=None, + cited_source_uris=None, + cited_source_titles=None, + ) + assert isinstance(out, dict) + assert "tier" in out + assert "tier_rank" in out + + +def test_does_not_raise_on_garbage_input(): + out = diagnose_authorship_warrant( + question_text="who wrote x?", + answer_text="\x00\x01" * 100, + cited_evidence_spans=["", None, "<<<>>>"], # type: ignore + ) + assert out["tier"] in ( + "NO_AUTHORSHIP_SIGNAL", "AUTHOR_SECONDARY_SOURCE", + "AUTHOR_PACKAGE_METADATA", "AUTHOR_REPOSITORY_OWNER", + "AUTHOR_PAGE_BYLINE", "AUTHOR_PRIMARY_PAGE_TITLE", + "AUTHOR_COPYRIGHT_FOOTER", + ) + + +def test_noise_capitalized_words_not_classified_as_names(): + """`Author` / `Copyright` / etc. shouldn't slip through as names.""" + out = diagnose_authorship_warrant( + question_text="who wrote x?", + answer_text="x", + cited_evidence_spans=[ + "Copyright Reserved", # should NOT classify "Reserved" as name + "Author All Rights", # garbage shape + ], + ) + # Even if regex matches, _looks_like_noise filters; tier may fall + # to secondary or no-signal. + assert "Reserved" not in out.get("candidate_names", []) + assert "All" not in out.get("candidate_names", [])