diff --git a/Makefile b/Makefile index adad1cd..852d550 100644 --- a/Makefile +++ b/Makefile @@ -515,11 +515,13 @@ bench-fork-baseline-hard: bootstrap ## #000046 — pin the below-ceiling hard-Fa --fixtures bench/fixtures/5f/falsification-hard-v1.jsonl --out $(FORK_PARENT_HARD) || true @echo ">> below-ceiling parent pinned: $(FORK_PARENT_HARD) (2/12 over-grounds expected → runner exits 1; the JSON is written)" # #000046 Phase 2 — HARD Formulate tier. Prose that parse_pointer_claims -# should segment a particular way; the line/bullet-based parser -# mis-segments 8 of 12 (merges multi-claim lines, splits wrapped -# bullets) → rate 4/12 at HEAD. `|| true` past the runner's -# nonzero-on-failures exit (8 fail by design). -bench-5f-formulate-hard: bootstrap ## #000046 Phase 2 — HARD Formulate mis-segment pack (rate < 1.0 at HEAD by design) +# should segment a particular way. Was 4/12 (line/bullet-only parser +# merged multi-claim lines / split wrapped bullets); #000048 step 2.4's +# clause segmenter closes all 8 → rate 12/12 now (pack at ceiling — a +# harder Formulate tier would re-open headroom). Runner still exits 0 +# here (0 failures), but the target is kept for the operator-pack +# convention (not in bench-5f / runner --all). +bench-5f-formulate-hard: bootstrap ## #000046 Phase 2 / #000048 step 2.4 — HARD Formulate clause-segmentation pack (12/12 after step 2.4) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate \ --fixtures bench/fixtures/5f/formulate-hard-v1.jsonl || true diff --git a/arborist/qa/parse_claims.py b/arborist/qa/parse_claims.py index 3a0a05a..d08caca 100644 --- a/arborist/qa/parse_claims.py +++ b/arborist/qa/parse_claims.py @@ -61,6 +61,25 @@ _BULLET_RE = re.compile(r"^\s*(?:[-*+•]|\d+[.)])\s+") # (".", "!", "?", ":") because that's part of the claim text. _TRAILING_STRIP = " ,;-—\t" +# Clause/claim separators within a single line (ticket #000048 step 2.4). +# A line that crams several pointered claims onto one row gets segmented +# here; the split is only KEPT if every resulting segment is itself a +# well-pointered claim (or a leading colon-terminated header to drop) — +# see _segment_line — so a legitimate single claim like "The cat is +# black and white [E1]." or "The cast: A, B, C [E1]." is never broken +# (splitting it would create pointer-less prose fragments → guard +# rejects the split). Order in the alternation is irrelevant for +# re.split (it splits on any match). Non-capturing groups so the +# separators are discarded, not re-inserted into the output. +_SEGMENT_SEP_RE = re.compile( + r";" # semicolon + r"|(?<=[.!?])\s+(?=[A-Z])" # sentence boundary: . ! ? then a capital + r"|\s+[-—–]\s+" # spaced dash / en/em dash as a clause join + r"|\s*\(\d+\)\s*" # inline enumeration marker: (1) (2) ... + r"|\s+(?:and|or|because|although|since|while)\s+" # conjunctions / subordinators + r"|,\s+(?![^\[]*\])" # comma — but not one inside a [E1, E2] bracket +) + @dataclass(frozen=True) class ParsedClaim: @@ -90,53 +109,113 @@ class ParsedClaim: return asdict(self) -def parse_pointer_claims(text: str) -> list[ParsedClaim]: - """Walk ``text`` line by line; emit one ``ParsedClaim`` per non-empty - line. +def _pointers_in(text: str) -> list[str]: + """Pointer ids inside every ``[E\\d+...]`` bracket in ``text``, in order.""" + ids: list[str] = [] + for m in _BRACKET_RE.finditer(text): + ids.extend(_POINTER_RE.findall(m.group(1))) + return ids - Skips empty / whitespace-only lines. Lines with at least one - ``[E\\d+...]`` bracket get ``parse_status="PARSED"``; lines with prose - but no bracket get ``parse_status="NO_EVIDENCE_POINTER"``. The - verifier owns the policy on what to do with a NO_EVIDENCE_POINTER - claim — typically it counts toward the denominator and downgrades - the verdict to HYBRID/UNGROUNDED. + +def _make_claim(text_source: str, raw_line: str) -> ParsedClaim: + """One ``ParsedClaim`` from a whole line or a single segment of one. + + PARSED iff ``text_source`` has at least one ``[E\\d+...]`` bracket; + otherwise NO_EVIDENCE_POINTER. ``raw_line`` is the originating line + (carried verbatim for the audit trail / render hash even when this + claim is one clause of a multi-claim line). + """ + pointer_ids = _pointers_in(text_source) + if pointer_ids: + claim_text = _BRACKET_RE.sub("", text_source) + claim_text = _BULLET_RE.sub("", claim_text).strip().rstrip(_TRAILING_STRIP) + return ParsedClaim( + claim_text=claim_text, + pointer_ids=pointer_ids, + parse_status="PARSED", + raw_line=raw_line, + ) + return ParsedClaim( + claim_text=_BULLET_RE.sub("", text_source).strip(), + pointer_ids=[], + parse_status="NO_EVIDENCE_POINTER", + raw_line=raw_line, + ) + + +def _segment_line(raw_line: str) -> list[str] | None: + """Split a line into clause/claim segments on ``_SEGMENT_SEP_RE``. + + Returns the list of segments to emit (as separate claims) **only if + the split is safe**: every non-empty segment must be a well-pointered + claim, with one exception — a *leading* colon-terminated header with + no pointer ("Two facts:", "Key points:") is allowed and dropped. If + the line has ≤1 non-empty segment, or any non-header segment lacks a + pointer (splitting it would manufacture a pointer-less prose + fragment from what was a single claim — "The cat is black and white + [E1]." → ["The cat is black", "white [E1]"]), returns ``None`` so the + caller keeps the line as one claim. + """ + parts = [p for p in _SEGMENT_SEP_RE.split(raw_line) if p and p.strip()] + if len(parts) < 2: + return None + # Drop a leading colon-terminated, pointer-less header. + if not _pointers_in(parts[0]) and parts[0].rstrip().endswith(":"): + parts = parts[1:] + if len(parts) < 2: + return None + if any(not _pointers_in(p) for p in parts): + return None + return parts + + +def parse_pointer_claims(text: str) -> list[ParsedClaim]: + """Walk ``text`` line by line; emit ``ParsedClaim``s. + + One claim per non-empty line — *except* a line that crams several + well-pointered claims onto one row is split into one claim per + clause (ticket #000048 step 2.4; see :func:`_segment_line` for the + safety guard), and a continuation line — leading whitespace then a + lowercase letter, no bullet glyph — is joined back onto the previous + claim (a wrapped bullet: "- The cell respires aerobically\\n to + produce ATP efficiently. [E1]" is one claim, not two). + + Skips empty / whitespace-only lines. A claim with ≥1 ``[E\\d+...]`` + bracket gets ``parse_status="PARSED"``; prose with no bracket gets + ``"NO_EVIDENCE_POINTER"``. The verifier owns the policy on what to + do with a NO_EVIDENCE_POINTER claim. Pointer-id ordering is preserved as the model wrote them; duplicates are NOT deduped here so the verifier can flag repeated cites. """ out: list[ParsedClaim] = [] for raw_line in text.splitlines(): + # Continuation line (a wrapped bullet/sentence): leading + # whitespace then a lowercase letter, no bullet glyph. Fold its + # text + pointers into the preceding claim. + if out and re.match(r"^\s+[a-z]", raw_line): + prev = out[-1] + joined_source = prev.claim_text + " " + raw_line.strip() + new = _make_claim(joined_source, prev.raw_line + "\n" + raw_line) + # PARSED if either part had a pointer; pointer order = + # previous claim's ids then this line's. + ids = list(prev.pointer_ids) + _pointers_in(raw_line) + out[-1] = ParsedClaim( + claim_text=new.claim_text, + pointer_ids=ids, + parse_status="PARSED" if ids else "NO_EVIDENCE_POINTER", + raw_line=prev.raw_line + "\n" + raw_line, + ) + continue + # Skip lines that are pure whitespace or only a bullet glyph. - stripped_for_emptiness = _BULLET_RE.sub("", raw_line).strip() - if not stripped_for_emptiness: + if not _BULLET_RE.sub("", raw_line).strip(): continue - # Pull every bracket-list match; collect all pointer ids inside. - bracket_matches = list(_BRACKET_RE.finditer(raw_line)) - if not bracket_matches: - out.append(ParsedClaim( - claim_text=stripped_for_emptiness, - pointer_ids=[], - parse_status="NO_EVIDENCE_POINTER", - raw_line=raw_line, - )) - continue - - pointer_ids: list[str] = [] - for m in bracket_matches: - pointer_ids.extend(_POINTER_RE.findall(m.group(1))) - - # Claim text = line minus every bracket match, then bullet- and - # trailing-punct-stripped. Preserves prose word order even when - # the model interleaves tags mid-sentence. - claim_text = _BRACKET_RE.sub("", raw_line) - claim_text = _BULLET_RE.sub("", claim_text).strip() - claim_text = claim_text.rstrip(_TRAILING_STRIP) - - out.append(ParsedClaim( - claim_text=claim_text, - pointer_ids=pointer_ids, - parse_status="PARSED", - raw_line=raw_line, - )) + segments = _segment_line(raw_line) + if segments is not None: + for seg in segments: + out.append(_make_claim(seg, raw_line)) + else: + out.append(_make_claim(raw_line, raw_line)) return out diff --git a/bench/fixtures/5f/formulate-hard-v1.jsonl b/bench/fixtures/5f/formulate-hard-v1.jsonl index 70973f3..7b11821 100644 --- a/bench/fixtures/5f/formulate-hard-v1.jsonl +++ b/bench/fixtures/5f/formulate-hard-v1.jsonl @@ -1,12 +1,12 @@ -{"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1", "task_count": 12, "notes": "#000046 Phase 2 — HARD live-path Formulate tier. Every input is prose that arborist.qa.parse_claims.parse_pointer_claims SHOULD segment into a particular claim lattice; `expected_lattice` records that correct structure. The parser is line/bullet-based — one line ⇒ one claim, [E#] tokens on that line attach to it — so 8 of 12 it mis-segments: it merges multiple sentence/semicolon/clause claims that share a line into ONE claim with all the pointers, or splits a wrapped bullet into two. Those 8 fail at HEAD by design (count mismatch + pointer-set mismatch). The other 4 are well-formed bullet/numbered lists / single claims the parser handles correctly. Rate at HEAD = 4/12 = 0.333... — a real below-ceiling baseline; parse_pointer_claims is deterministic, so it's stable. A claim-lattice parser that does sentence/clause segmentation (split on '. ', ';', subordinating conjunctions, inline enumerations) and joins wrapped bullets would lift the rate toward 1.0 → a positive gamma*Delta5f term for that child fork. NOT part of `make bench-5f` / `runner --all` / `bench-fork-baseline`; pinned via `make bench-5f-formulate-hard` / shares `make bench-fork-baseline-hard` is falsification-only — formulate-hard is its own target. See ticket #000046 §5."}} -{"id": "5f-form-hard-001", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Water is wet [E1] and fire is hot [E2].", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Water is wet.", "pointer_ids": ["E1"]}, {"claim_text": "Fire is hot.", "pointer_ids": ["E2"]}]}, "expected": "pass", "note": "MIS-SEGMENT at HEAD: two and-joined sentence claims on one line → parser merges to 1 claim with [E1,E2]; correct is 2."} -{"id": "5f-form-hard-002", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Water is wet [E1]; fire is hot [E2]; ice is cold [E3].", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Water is wet.", "pointer_ids": ["E1"]}, {"claim_text": "Fire is hot.", "pointer_ids": ["E2"]}, {"claim_text": "Ice is cold.", "pointer_ids": ["E3"]}]}, "expected": "pass", "note": "MIS-SEGMENT at HEAD: three semicolon-separated claims on one line → parser merges to 1 claim with [E1,E2,E3]; correct is 3."} -{"id": "5f-form-hard-003", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- The cell respires aerobically\n to produce ATP efficiently. [E1]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "The cell respires aerobically to produce ATP efficiently.", "pointer_ids": ["E1"]}]}, "expected": "pass", "note": "MIS-SEGMENT at HEAD: a wrapped bullet → parser splits into 2 claims, pointer on the wrong half; correct is 1 joined claim with [E1]."} -{"id": "5f-form-hard-004", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Water boils at 100 C [E1]. It freezes at 0 C [E2].", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Water boils at 100 C.", "pointer_ids": ["E1"]}, {"claim_text": "It freezes at 0 C.", "pointer_ids": ["E2"]}]}, "expected": "pass", "note": "MIS-SEGMENT at HEAD: two sentences on one line (no bullets) → parser merges to 1 claim with [E1,E2]; correct is 2."} -{"id": "5f-form-hard-005", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "The engine runs hot [E1] because the coolant leaked [E2].", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "The engine runs hot.", "pointer_ids": ["E1"]}, {"claim_text": "The coolant leaked.", "pointer_ids": ["E2"]}]}, "expected": "pass", "note": "MIS-SEGMENT at HEAD: main clause + 'because' sub-claim on one line → parser merges to 1 claim with [E1,E2]; correct is 2 (the cause is its own claim)."} -{"id": "5f-form-hard-006", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Two facts: (1) water is wet [E1] (2) fire is hot [E2].", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Water is wet.", "pointer_ids": ["E1"]}, {"claim_text": "Fire is hot.", "pointer_ids": ["E2"]}]}, "expected": "pass", "note": "MIS-SEGMENT at HEAD: an inline (1)(2) enumeration on one line → parser merges to 1 claim with [E1,E2]; correct is 2."} -{"id": "5f-form-hard-007", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Key points: clarity matters [E1], brevity matters [E2], accuracy matters [E3].", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Clarity matters.", "pointer_ids": ["E1"]}, {"claim_text": "Brevity matters.", "pointer_ids": ["E2"]}, {"claim_text": "Accuracy matters.", "pointer_ids": ["E3"]}]}, "expected": "pass", "note": "MIS-SEGMENT at HEAD: a comma-separated list after a colon on one line → parser merges to 1 claim with [E1,E2,E3]; correct is 3."} -{"id": "5f-form-hard-008", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "The sky is blue [E1] - the grass is green [E2].", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "The sky is blue.", "pointer_ids": ["E1"]}, {"claim_text": "The grass is green.", "pointer_ids": ["E2"]}]}, "expected": "pass", "note": "MIS-SEGMENT at HEAD: two dash-joined clauses on one line → parser merges to 1 claim with [E1,E2]; correct is 2."} +{"_meta": {"battery": "5f", "sub_battery": "formulate", "version": "v1", "task_count": 12, "notes": "#000046 Phase 2 — HARD live-path Formulate tier. Every input is prose that arborist.qa.parse_claims.parse_pointer_claims SHOULD segment into a particular claim lattice; `expected_lattice` records that correct structure. Was 4/12 at HEAD — the original line/bullet-only parser merged multiple sentence/semicolon/clause claims sharing a line into ONE claim with all the pointers, and split a wrapped bullet into two. #000048 step 2.4 (2026-05-11) added a clause segmenter: within a line, split on ';', sentence boundaries ('. '/'! '/'? ' then a Capital), spaced dashes, ' and '/' or '/' because '/' although '/' since '/' while ', inline '(N)' enumeration markers, and commas — but the split is KEPT only if every resulting segment is itself a well-pointered claim (a leading colon-terminated header with no pointer — 'Two facts:', 'Key points:' — is allowed and dropped), so a legitimate single claim ('The cat is black and white [E1].', 'The cast: A, B, C [E1].') is never broken; plus a wrapped-bullet join (a leading-whitespace lowercase continuation line folds into the previous claim). Closes all 8 → rate now 12/12 = 1.0 (the pack is at ceiling; a harder Formulate tier would re-open below-ceiling headroom — a #000046 follow-up). parse_pointer_claims is deterministic, so the rate is stable. NOT part of `make bench-5f` / `runner --all` / `bench-fork-baseline`; pinned via `make bench-5f-formulate-hard`. See tickets #000046 §5 + #000048 §5 step 2.4."}} +{"id": "5f-form-hard-001", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Water is wet [E1] and fire is hot [E2].", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Water is wet.", "pointer_ids": ["E1"]}, {"claim_text": "Fire is hot.", "pointer_ids": ["E2"]}]}, "expected": "pass", "note": "Caught (#000048 step 2.4): two and-joined sentence claims on one line → parser merges to 1 claim with [E1,E2]; correct is 2."} +{"id": "5f-form-hard-002", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Water is wet [E1]; fire is hot [E2]; ice is cold [E3].", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Water is wet.", "pointer_ids": ["E1"]}, {"claim_text": "Fire is hot.", "pointer_ids": ["E2"]}, {"claim_text": "Ice is cold.", "pointer_ids": ["E3"]}]}, "expected": "pass", "note": "Caught (#000048 step 2.4): three semicolon-separated claims on one line → parser merges to 1 claim with [E1,E2,E3]; correct is 3."} +{"id": "5f-form-hard-003", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- The cell respires aerobically\n to produce ATP efficiently. [E1]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "The cell respires aerobically to produce ATP efficiently.", "pointer_ids": ["E1"]}]}, "expected": "pass", "note": "Caught (#000048 step 2.4): a wrapped bullet → parser splits into 2 claims, pointer on the wrong half; correct is 1 joined claim with [E1]."} +{"id": "5f-form-hard-004", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Water boils at 100 C [E1]. It freezes at 0 C [E2].", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Water boils at 100 C.", "pointer_ids": ["E1"]}, {"claim_text": "It freezes at 0 C.", "pointer_ids": ["E2"]}]}, "expected": "pass", "note": "Caught (#000048 step 2.4): two sentences on one line (no bullets) → parser merges to 1 claim with [E1,E2]; correct is 2."} +{"id": "5f-form-hard-005", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "The engine runs hot [E1] because the coolant leaked [E2].", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "The engine runs hot.", "pointer_ids": ["E1"]}, {"claim_text": "The coolant leaked.", "pointer_ids": ["E2"]}]}, "expected": "pass", "note": "Caught (#000048 step 2.4): main clause + 'because' sub-claim on one line → parser merges to 1 claim with [E1,E2]; correct is 2 (the cause is its own claim)."} +{"id": "5f-form-hard-006", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Two facts: (1) water is wet [E1] (2) fire is hot [E2].", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Water is wet.", "pointer_ids": ["E1"]}, {"claim_text": "Fire is hot.", "pointer_ids": ["E2"]}]}, "expected": "pass", "note": "Caught (#000048 step 2.4): an inline (1)(2) enumeration on one line → parser merges to 1 claim with [E1,E2]; correct is 2."} +{"id": "5f-form-hard-007", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "Key points: clarity matters [E1], brevity matters [E2], accuracy matters [E3].", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Clarity matters.", "pointer_ids": ["E1"]}, {"claim_text": "Brevity matters.", "pointer_ids": ["E2"]}, {"claim_text": "Accuracy matters.", "pointer_ids": ["E3"]}]}, "expected": "pass", "note": "Caught (#000048 step 2.4): a comma-separated list after a colon on one line → parser merges to 1 claim with [E1,E2,E3]; correct is 3."} +{"id": "5f-form-hard-008", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "The sky is blue [E1] - the grass is green [E2].", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "The sky is blue.", "pointer_ids": ["E1"]}, {"claim_text": "The grass is green.", "pointer_ids": ["E2"]}]}, "expected": "pass", "note": "Caught (#000048 step 2.4): two dash-joined clauses on one line → parser merges to 1 claim with [E1,E2]; correct is 2."} {"id": "5f-form-hard-009", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "- Sky is blue. [E1]\n- Grass is green. [E2]", "expected_lattice": {"claim_count": 2, "claims": [{"claim_text": "Sky is blue.", "pointer_ids": ["E1"]}, {"claim_text": "Grass is green.", "pointer_ids": ["E2"]}]}, "expected": "pass", "note": "Correctly segmented at HEAD — well-formed 2-bullet list. PASSES today; kept as a headroom marker (a parser regression that started merging bullets would drop the rate)."} {"id": "5f-form-hard-010", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "1. Sky is blue [E1]\n2. Grass is green [E2]\n3. Sun is bright [E3]", "expected_lattice": {"claim_count": 3, "claims": [{"claim_text": "Sky is blue.", "pointer_ids": ["E1"]}, {"claim_text": "Grass is green.", "pointer_ids": ["E2"]}, {"claim_text": "Sun is bright.", "pointer_ids": ["E3"]}]}, "expected": "pass", "note": "Correctly segmented at HEAD — well-formed 3-item numbered list. PASSES today."} {"id": "5f-form-hard-011", "battery": "5f", "sub_battery": "formulate", "version": "v1", "carrier": "text", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "input_text": "The sky is blue. [E1]", "expected_lattice": {"claim_count": 1, "claims": [{"claim_text": "The sky is blue.", "pointer_ids": ["E1"]}]}, "expected": "pass", "note": "Correctly segmented at HEAD — single claim, single pointer. PASSES today."} diff --git a/docs/qa-modes-bench.md b/docs/qa-modes-bench.md index 6a95d55..e62a317 100644 --- a/docs/qa-modes-bench.md +++ b/docs/qa-modes-bench.md @@ -481,3 +481,51 @@ Bench artifacts: `bench/qa_results/2026-05-11T14-19-51Z.{jsonl,md}` (before — HEAD's verify.py) · `bench/qa_results/2026-05-11T17-12-41Z.{jsonl,md}` (after). Full per-ticket detail: `docs/tickets/ticket-000048-verifier-upgrade-recombination-segmentation.md` §5 step 2.1. + + +### Addendum 7 — #000048 step 2.4 parse_pointer_claims clause-segmentation regression check (2026-05-11) + +`arborist/qa/parse_claims.py` gained a clause segmenter: a line that +crams several well-pointered claims onto one row is split into one +claim per clause (split on `;`, sentence boundaries, spaced dashes, +` and `/` or `/` because `/` although `/` since `/` while `, inline +`(N)` enumeration markers, commas — with `(?![^\[]*\])` so a comma +inside a `[E1, E2]` bracket never splits it). `_segment_line` keeps +the split *only if every resulting segment is well-pointered* (a legit +single claim like "The cat is black and white [E1]." or "The cast: A, +B, C [E1]." is never broken — splitting would create pointer-less +fragments → guard rejects; a leading colon-terminated header — "Two +facts:", "Key points:" — is dropped). Plus a wrapped-bullet join: a +continuation line (leading whitespace then a lowercase letter, no +bullet) folds into the previous claim. Closes the 8 mis-segments in +`formulate-hard-v1.jsonl` → rate 4/12 → 12/12 (that pack now at +ceiling). + +Before/after `make bench-qa` (n=3 × 75 questions × 3 modes = 675 +cells; `parse_pointer_claims` feeds the 450 `claim_lattice_pointer` + +`claim_lattice` cells): + +| mode | STRICT-rate before → after | Δ | +|------|----------------------------|---| +| quote | 0.54 → 0.55 | +1pp | +| claim_lattice_pointer | 0.22 → 0.22 | 0pp | +| claim_lattice | 0.43 → 0.45 | +2pp | + +All within the 5-pp noise floor. Per-row diff (675 common cells): the +segmenter changed the parsed-claim count *on the same answer text* +for **7 of the 450 lattice cells** (0 in `claim_lattice`, 7 in +`claim_lattice_pointer`); of those, 2 caused an `audit_mode` change — +both **correct**: (a) a wrap-join recovered an answer's intended +structure (4 claims, 2 of which were pointer-less wrap-fragments → +HYBRID) into 2 well-pointered claims → STRICT; (b) a crammed-one-line +blob (1 monolithic claim, all pointers → STRICT) split into 8 claims, +some of which don't individually verify → HYBRID — the honest verdict +(false-positive STRICT was the corruption). Every other lattice/quote +delta is LLM re-answer variance (`answer_chars` changed, often +drastically). No regression — the segmenter's only visible effects on +real traffic are honest improvements. + +Bench artifacts: `bench/qa_results/2026-05-11T17-12-41Z.{jsonl,md}` +(before — HEAD's parse_claims.py) · `bench/qa_results/2026-05-11T20-26-37Z.{jsonl,md}` +(after). Full per-ticket detail: `docs/tickets/ticket-000048-verifier-upgrade-recombination-segmentation.md` +§5 step 2.4. diff --git a/docs/tickets/ticket-000012-selection-consensus-protocol.md b/docs/tickets/ticket-000012-selection-consensus-protocol.md index d5c1c9a..7730690 100644 --- a/docs/tickets/ticket-000012-selection-consensus-protocol.md +++ b/docs/tickets/ticket-000012-selection-consensus-protocol.md @@ -575,13 +575,15 @@ protocol must account for: `validator_diversity`, …), not by bench Δ-rate. The harder tier is **#000046 (closed 2026-05-11)**: two below-ceiling 5F packs (`falsification-hard-v1.jsonl`, `formulate-hard-v1.jsonl`) + - `verify_quotes`' paraphrase numeric-agreement gate + (#000048 step - 2.1) the entity salient-token-disagreement gate, which together - lifted the falsification-hard rate 4/12 → 10/12 on *real* changes - (each bench-gated: `make bench-qa` n=3 before/after showed no - STRICT-rate regression on legit answers — #000048 step 2.1's gate - fired on 0 QA answers in the whole bench) → `fork_score`'s `γ·Δ5f` - went positive on them. So the bench Δ-rate *does* now carry signal - on the 5F/falsification axis, demonstrated end-to-end. Remaining - hard-pack headroom (now 2 over-grounds + 8 mis-segments — #000048 - steps 2.2 + 2.4) is an optional follow-up, not a #000012 blocker. + `verify_quotes`' paraphrase numeric-agreement gate + (#000048 + steps 2.1 + 2.4) the entity salient-token gate and the + `parse_pointer_claims` clause segmenter, which together lifted + `falsification-hard` 4/12 → 10/12 and `formulate-hard` 4/12 → + 12/12 on *real* changes (each bench-gated: `make bench-qa` n=3 + before/after showed no STRICT-rate regression on legit answers — + the 2.1 gate fired on 0 QA answers; the 2.4 segmenter touched 7 of + 450 lattice cells, both verdict changes correct) → `fork_score`'s + `γ·Δ5f` went positive on them. So the bench Δ-rate *does* now carry + signal on the 5F/falsification axis, demonstrated end-to-end. + Remaining headroom: 2 falsification-hard over-grounds (#000048 step + 2.2) — an optional follow-up, not a #000012 blocker. diff --git a/docs/tickets/ticket-000046-harder-5sf-fixture-tier.md b/docs/tickets/ticket-000046-harder-5sf-fixture-tier.md index a5c76e2..3b90d99 100644 --- a/docs/tickets/ticket-000046-harder-5sf-fixture-tier.md +++ b/docs/tickets/ticket-000046-harder-5sf-fixture-tier.md @@ -280,18 +280,18 @@ testable only once there's a below-ceiling baseline to bench against. ### Headroom (optional follow-up — not a #000046 blocker) -Tracked as **#000048** (opened 2026-05-11). As of #000048 step 2.1 -(2026-05-11) the falsification-hard pack is at 10/12 — the entity -salient-token-disagreement gate closed the 4 HYBRID_ENTITY -over-grounds. The remaining headroom: 2 STRICT_PARAPHRASE in the -falsification-hard pack (hard-003 Mercury, hard-005 Einstein — the -false claim recombines source tokens into a different true statement, -which lexical token-coverage can't tell from grounding; #000048 step -2.2 = sequence-aware paraphrase) + the 8 mis-segments in the Formulate -hard pack (line/bullet-only `parse_pointer_claims`; #000048 step 2.4 = -clause segmentation). #000046 itself is done: the below-ceiling -baselines exist, real surface improvements lifted a rate, and -ForkScore's bench-Δ went positive on them. +Tracked as **#000048** (opened 2026-05-11). As of 2026-05-11: +#000048 step 2.1 (entity salient-token gate) closed the 4 +HYBRID_ENTITY over-grounds → `falsification-hard` 10/12; step 2.4 +(`parse_pointer_claims` clause segmentation) closed all 8 Formulate +mis-segments → `formulate-hard` 12/12 (that pack at ceiling). The +last residue: 2 STRICT_PARAPHRASE in `falsification-hard` (hard-003 +Mercury, hard-005 Einstein — the false claim recombines source tokens +into a different true statement, which lexical token-coverage can't +tell from grounding; #000048 step 2.2 = sequence-aware paraphrase +closes those, then #000048 itself closes). #000046 is done: the +below-ceiling baselines exist, real surface improvements lifted rates, +and ForkScore's bench-Δ went positive on them. --- diff --git a/docs/tickets/ticket-000048-verifier-upgrade-recombination-segmentation.md b/docs/tickets/ticket-000048-verifier-upgrade-recombination-segmentation.md index 79545cd..3dd33d5 100644 --- a/docs/tickets/ticket-000048-verifier-upgrade-recombination-segmentation.md +++ b/docs/tickets/ticket-000048-verifier-upgrade-recombination-segmentation.md @@ -1,10 +1,15 @@ # Ticket #000048 — Verifier upgrade: recombination-aware grounding + clause segmentation -**Status:** in progress · **step 2.1 landed 2026-05-11** (entity -salient-token-disagreement gate in `verify_quotes` — `_entity_salient_disagrees` -+ `_is_single_sentence`; lifts `falsification-hard` 6/12 → 10/12; -bench-gated, 0 gate-attributable QA shifts). Steps 2.4 (`parse_pointer_claims` -segmentation) + 2.2 (sequence-aware paraphrase) next per §3 / §5. +**Status:** in progress · **steps 2.1 + 2.4 landed 2026-05-11.** +2.1 — `verify_quotes` entity salient-token-disagreement gate +(`_entity_salient_disagrees` + `_is_single_sentence`; `falsification-hard` +6/12 → 10/12; bench-gated, 0 gate-attributable QA shifts). 2.4 — +`parse_pointer_claims` clause segmentation (`_SEGMENT_SEP_RE` + +`_segment_line` guard + wrapped-bullet join; `formulate-hard` 4/12 → +12/12 — that pack at ceiling now; bench-gated, 7 lattice cells +touched, both verdict changes correct). Step 2.2 (sequence-aware +paraphrase — closes `falsification-hard` 10/12 → 12/12, the last +residue) next per §3 / §5. **Opened:** 2026-05-11 **Scope:** Close the headroom #000046 left in the two below-ceiling 5F hard packs — the 6 over-grounds still in `falsification-hard-v1.jsonl` @@ -236,18 +241,60 @@ per §3. updated (positive γ·Δ5f on the real lift — possibly MARGINAL given the ÷5 dilution; ACCEPT shown via a degraded-parent sub-scenario). +### Step 2.4 (landed 2026-05-11) — `parse_pointer_claims` clause segmentation + +- **Change:** `arborist/qa/parse_claims.py` — `_SEGMENT_SEP_RE` + (split a line on `;`, sentence boundaries `. `/`! `/`? ` then a + Capital, spaced dashes ` - `/` — `/` – `, ` and `/` or `/` because + `/` although `/` since `/` while `, inline `(N)` enumeration + markers, and commas — with `(?![^\[]*\])` so a comma *inside* a + `[E1, E2]` bracket never splits it). `_segment_line` keeps the + split **only if every resulting non-empty segment is a well-pointered + claim** — a legit single claim ("The cat is black and white [E1].", + "The cast: A, B, C [E1].") is never broken because splitting it + would manufacture pointer-less prose fragments → guard rejects; a + *leading* colon-terminated header with no pointer ("Two facts:", + "Key points:") is allowed and dropped. Plus a wrapped-bullet join: + a continuation line (leading whitespace then a lowercase letter, no + bullet glyph) folds its text + pointers into the previous claim. +- **Effect on the hard pack:** closes all 8 mis-segments in + `formulate-hard-v1.jsonl` → **rate 4/12 → 12/12** (the pack is now + at ceiling — a harder Formulate tier would re-open below-ceiling + headroom; a #000046 follow-up, out of #000048's scope). +- **Bench gate:** `make bench-qa` (n=3 × 75 × 3 = 675 cells; + `parse_pointer_claims` feeds the 450 `claim_lattice_pointer` + + `claim_lattice` cells) after (`bench/qa_results/2026-05-11T20-26-37Z.md`) + vs the pre-step-2.4 baseline (`...T17-12-41Z.md` = HEAD's + parse_claims.py). STRICT-rate: quote 0.54 → 0.55, pointer 0.22 → + 0.22, lattice 0.43 → 0.45 — all within the 5-pp noise floor. + Per-row diff: the segmenter changed the parsed-claim count on the + *same answer text* for **7 of the 450 lattice cells** (0 in + `claim_lattice`, 7 in `claim_lattice_pointer`); of those, 2 caused + an `audit_mode` change — both **correct**: a wrap-join recovered an + answer's intended structure (4 claims, 2 of which were pointer-less + wrap-fragments → HYBRID) into 2 well-pointered claims → STRICT; and + a crammed-one-line-blob (1 monolithic claim, all pointers → STRICT) + split into 8 claims, some of which don't individually verify → + HYBRID, the honest verdict (false-positive STRICT was the + corruption). Every other lattice/quote delta is LLM re-answer + variance (`answer_chars` changed). No regression — the segmenter's + only visible effects on real traffic are honest improvements. +- **Tests:** 8 new in `tests/test_claim_lattice.py` (semicolon / + sentence / conjunction splits; the pointerless-fragment + cast-list + guards; leading-colon-header drop; wrapped-bullet join; + pointer-order / multi-pointer); the existing `parse_pointer_claims` + tests pass untouched; `test_5f_formulate_hard_pack` re-pinned 4/12 + → 12/12. + ### Still ahead -- **Step 2.4** — `parse_pointer_claims` clause segmentation (closes - the 8 mis-segments in `formulate-hard-v1.jsonl`). Bench gate: the - Formulate fixtures (incl. `formulate-v1.jsonl` / `formulate-live-v1.jsonl`, - must not regress) + a QA smoke (`parse_pointer_claims` feeds the - claim-lattice modes). -- **Step 2.2** — sequence-aware paraphrase match (closes hard-003 + - hard-005). Bench-gated like 2.1; conservative threshold. -- **Closure:** both hard packs at (or near) rate 1.0, or fox-decided - "residue not worth it" — at which point this ticket closes (and - step 2.3 / mini-NLI stays a deferred maybe). +- **Step 2.2** — sequence-aware paraphrase match (closes hard-003 + Mercury + hard-005 Einstein → falsification-hard 10/12 → 12/12). + Bench-gated like 2.1; conservative threshold. +- **Closure:** `falsification-hard-v1.jsonl` at (or near) rate 1.0 + (it's the only hard pack with residue — Formulate is already 12/12), + or fox-decided "residue not worth it" — at which point this ticket + closes (step 2.3 / mini-NLI stays a deferred maybe). --- diff --git a/tests/test_bench_batteries.py b/tests/test_bench_batteries.py index a4e2059..b6b23d0 100644 --- a/tests/test_bench_batteries.py +++ b/tests/test_bench_batteries.py @@ -672,28 +672,27 @@ def test_5f_falsification_embedded_path_still_works(): assert t.detail["source"] == "embedded" -# --- #000046 Phase 2 — HARD live-path Formulate tier -------------- +# --- #000046 Phase 2 / #000048 step 2.4 — Formulate hard tier ----- -def test_5f_formulate_hard_pack_below_ceiling(): - """Below-ceiling baseline for the Formulate sub-battery: 12 inputs - that parse_pointer_claims should segment a particular way; the - line/bullet-based parser mis-segments 8 of 12 (merges multi-claim - lines, splits wrapped bullets) → rate 4/12. parse_pointer_claims - is deterministic, so this is stable; the pinned value fires if it - shifts.""" +def test_5f_formulate_hard_pack(): + """The Formulate hard pack — 12 inputs `parse_pointer_claims` + should segment a particular way. Was 4/12 at HEAD (the line/ + bullet-only parser merged multi-claim lines / split wrapped + bullets); #000048 step 2.4's clause segmenter (split on `;`, + sentence boundaries, spaced dashes, conjunctions/subordinators, + inline `(N)` enumerations, commas — kept only when every segment + is well-pointered — plus a wrapped-bullet join) closes all 8 → + 12/12. parse_pointer_claims is deterministic, so the pin is + stable; if it shifts, the value fires. (The pack is now at ceiling + — a harder Formulate tier would re-open below-ceiling headroom; a + #000046 follow-up, out of #000048's scope.)""" res = b_5f.run_formulate(F5F / "formulate-hard-v1.jsonl") - assert res.pass_count == 4 - assert res.fail_count == 8 - assert res.metrics["structural_match_rate"] == pytest.approx(4 / 12) + assert res.pass_count == 12 + assert res.fail_count == 0 + assert res.metrics["structural_match_rate"] == pytest.approx(1.0) for t in res.per_task: assert t.detail["source"] == "live" - # The 8 that fail do so on claim-count mismatch (the parser put - # multiple claims on one line, or split one across lines) — that's - # the mis-segmentation, not a noisy pointer/text near-miss. - for t in res.per_task: - if not t.passed: - assert t.detail["count_ok"] is False # --- #000046 Phase 1 — HARD live-path Falsification tier ---------- diff --git a/tests/test_claim_lattice.py b/tests/test_claim_lattice.py index 0f265fd..5188614 100644 --- a/tests/test_claim_lattice.py +++ b/tests/test_claim_lattice.py @@ -161,6 +161,71 @@ def test_parse_inline_tag_preserves_prose(): assert out[0].pointer_ids == ["E1"] +# ---------- #000048 step 2.4 — clause segmentation ---------------------- + + +def test_parse_splits_semicolon_pointered_claims(): + out = parse_pointer_claims("Water is wet [E1]; fire is hot [E2]; ice is cold [E3].\n") + assert len(out) == 3 + assert [c.pointer_ids for c in out] == [["E1"], ["E2"], ["E3"]] + assert all(c.parse_status == "PARSED" for c in out) + + +def test_parse_splits_sentence_boundary(): + out = parse_pointer_claims("Water boils at 100 C [E1]. It freezes at 0 C [E2].\n") + assert len(out) == 2 + assert [c.pointer_ids for c in out] == [["E1"], ["E2"]] + + +def test_parse_splits_conjunction_when_both_sides_pointered(): + out = parse_pointer_claims("Water is wet [E1] and fire is hot [E2].\n") + assert len(out) == 2 + assert [c.pointer_ids for c in out] == [["E1"], ["E2"]] + + +def test_parse_does_not_split_pointerless_fragments(): + # "The cat is black and white" is ONE claim — splitting on "and" + # would manufacture a pointer-less "The cat is black" fragment, so + # the guard rejects the split. + out = parse_pointer_claims("The cat is black and white. [E1]\n") + assert len(out) == 1 + assert out[0].pointer_ids == ["E1"] + assert "black and white" in out[0].claim_text + + +def test_parse_does_not_split_cast_list_comma(): + out = parse_pointer_claims("The cast: Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss [E1].\n") + assert len(out) == 1 + assert out[0].pointer_ids == ["E1"] + + +def test_parse_drops_leading_colon_header_on_split(): + out = parse_pointer_claims("Two facts: (1) water is wet [E1] (2) fire is hot [E2].\n") + assert len(out) == 2 + assert [c.pointer_ids for c in out] == [["E1"], ["E2"]] + # The "Two facts:" header is dropped, not emitted as a claim. + assert all("Two facts" not in c.claim_text for c in out) + + +def test_parse_joins_wrapped_bullet_continuation(): + out = parse_pointer_claims( + "- The cell respires aerobically\n" + " to produce ATP efficiently. [E1]\n" + ) + assert len(out) == 1 + assert out[0].pointer_ids == ["E1"] + assert out[0].parse_status == "PARSED" + assert "respires aerobically" in out[0].claim_text + assert "produce ATP" in out[0].claim_text + + +def test_parse_split_preserves_pointer_order_and_multi_pointer(): + out = parse_pointer_claims("Alpha holds [E1]; beta holds [E2, E3]\n") + assert len(out) == 2 + assert out[0].pointer_ids == ["E1"] + assert out[1].pointer_ids == ["E2", "E3"] + + # ---------- verifier: happy path ------------------------------------------