diff --git a/Makefile b/Makefile index 8cc2952..7c4b64e 100644 --- a/Makefile +++ b/Makefile @@ -493,12 +493,13 @@ bench-5f-live: bench-5f-formulate-live bench-5f-feedback-loop-live bench-5f-func # #000046 Phase 1 — HARD live-path Falsification tier. Every fixture is # a near-miss the correct verdict is UNGROUNDED on; verify_quotes -# over-grounds 8 of 12 today → rate 4/12 = 0.333 at HEAD (a real -# below-ceiling baseline). NOT part of bench-5f / bench-fork-baseline. -# bench-fork-baseline-hard pins the below-ceiling rate so a future -# verify_quotes tightening shows up as a positive gamma*Delta5f term. +# over-grounds 6 of 12 today → rate 6/12 = 0.5 at HEAD (the paraphrase +# numeric-agreement gate, #000046 closure, caught the other 2). A real +# below-ceiling baseline. NOT part of bench-5f / bench-fork-baseline. +# bench-fork-baseline-hard pins the rate so a future verifier upgrade +# shows up as a larger positive gamma*Delta5f term. FORK_PARENT_HARD ?= bench/results/baseline-falsification-hard.json -# The hard pack has its expected failures (8/12 over-grounds), so the +# The hard pack has its expected failures (6/12 over-grounds), so the # runner exits 1 — that's the below-ceiling baseline working as # designed, not a build error. `|| true` keeps the make target green; # the JSON / report is written regardless (the --out write precedes @@ -510,7 +511,7 @@ bench-fork-baseline-hard: bootstrap ## #000046 — pin the below-ceiling hard-Fa @mkdir -p bench/results PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub falsification \ --fixtures bench/fixtures/5f/falsification-hard-v1.jsonl --out $(FORK_PARENT_HARD) || true - @echo ">> below-ceiling parent pinned: $(FORK_PARENT_HARD) (8/12 over-grounds expected → runner exits 1; the JSON is written)" + @echo ">> below-ceiling parent pinned: $(FORK_PARENT_HARD) (6/12 over-grounds expected → runner exits 1; the JSON is written)" # #000046 Phase 2 — HARD Formulate tier. Prose that parse_pointer_claims # should segment a particular way; the line/bullet-based parser # mis-segments 8 of 12 (merges multi-claim lines, splits wrapped diff --git a/arborist/qa/verify.py b/arborist/qa/verify.py index d278fb3..54461d3 100644 --- a/arborist/qa/verify.py +++ b/arborist/qa/verify.py @@ -366,6 +366,32 @@ def _token_coverage( _TOKEN_PUNCT_STRIP = ".,;:!?\"()[]{}" +# Digit-run extractor for the paraphrase numeric-agreement gate. A run is +# one or more digits with optional *internal* thousands-comma separators +# ("8,849"); the comma is stripped for comparison so "8,849" and "8849" +# collapse, but "300" stays distinct from "300000" (← "300,000"). Decimal +# fractions split at the dot ("3.14" → {"3", "14"}) — fine for the +# near-miss patterns this gate targets (years / counts / magnitudes). +_DIGIT_RUN_RE = re.compile(r"\d[\d,]*\d|\d") + + +def _numeric_signature(text: str) -> frozenset[str]: + """Comma-stripped digit-runs in ``text``. + + Used by :func:`_check_each_with_paraphrase`: a span that token-covers + the source but asserts a *number the source does not contain* is not + paraphrase-grounded — it's a near-miss the lexical coverage check + can't see ("Water boils at 50 degrees" vs source "...100 degrees..." + token-covers 100% because "50"/"100" aren't content tokens). The gate + is conservative on legitimate paraphrases: a number written with or + without thousands commas matches; a number absent from the source + (a rounding, a magnitude error, a swapped year) does not — and a + rounding-paraphrase demoting from paraphrase-grounded to unverified + is the honest call (it isn't a verbatim grounding). + """ + return frozenset(m.group(0).replace(",", "") for m in _DIGIT_RUN_RE.finditer(text)) + + # Common English stopwords of length >= 4 chars. Hand-curated rather # than imported from NLTK to keep arborist dependency-light and the # behavior pinned to a known set. Tokens are normalized form @@ -479,11 +505,21 @@ def _check_each_with_paraphrase( - if its normalized form is a substring of ``norm_ctx``: STRICT (verbatim verified) - else if its meaningful-token coverage in ``norm_ctx`` is - ``>= paraphrase_coverage`` and it has at least - ``paraphrase_min_tokens`` content tokens: PARAPHRASE - (token-overlap verified) + ``>= paraphrase_coverage`` AND it has at least + ``paraphrase_min_tokens`` content tokens AND every digit-run + it asserts appears (comma-normalized) in ``norm_ctx``: + PARAPHRASE (token-overlap verified) - else: UNVERIFIED + The numeric-agreement gate (#000046) catches the near-miss the + coverage check is blind to: a span whose content tokens are all + present but whose *number* is wrong — "Water boils at 50 degrees" + against a source that says 100 token-covers 100% because + "50"/"100" aren't >4-char content tokens. A rounding-paraphrase + that drops to UNVERIFIED here is the honest verdict (it isn't a + verbatim grounding); a number written with vs without thousands + commas still matches. + Returns ``(strict_verified, paraphrase_verified, unverified)``. Order within each bucket preserves input order. @@ -494,6 +530,7 @@ def _check_each_with_paraphrase( strict: list[str] = [] paraphrase: list[str] = [] unverified: list[str] = [] + ctx_numbers = _numeric_signature(norm_ctx) for it in items: if _normalize(it) in norm_ctx: strict.append(it) @@ -506,7 +543,8 @@ def _check_each_with_paraphrase( unverified.append(it) continue cov, n_tok = _token_coverage(it, norm_ctx) - if n_tok >= paraphrase_min_tokens and cov >= paraphrase_coverage: + numbers_ok = _numeric_signature(_normalize(it)) <= ctx_numbers + if n_tok >= paraphrase_min_tokens and cov >= paraphrase_coverage and numbers_ok: paraphrase.append(it) else: unverified.append(it) diff --git a/bench/fixtures/5f/falsification-hard-v1.jsonl b/bench/fixtures/5f/falsification-hard-v1.jsonl index e817e09..db84830 100644 --- a/bench/fixtures/5f/falsification-hard-v1.jsonl +++ b/bench/fixtures/5f/falsification-hard-v1.jsonl @@ -1,11 +1,11 @@ -{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 12, "notes": "#000046 — HARD live-path Falsification tier. Every fixture is a near-miss: the claim is FALSE / unsupported, so the correct verdict is UNGROUNDED, and that is what `expected_reason` records. 8 of 12 are currently OVER-GROUNDED by arborist.qa.verify.verify_quotes — its paraphrase token-coverage strategy or entity-proximity strategy matches on incidental overlap (shared entities, shared numbers, the same key terms stated in the opposite direction) and returns STRICT_PARAPHRASE / HYBRID_ENTITY instead of UNGROUNDED — so those tasks FAIL at HEAD by design. The other 4 the verifier handles correctly. Rate at HEAD = 4/12 = 0.333... — a real below-ceiling baseline. A tightening of verify_quotes (stricter entity strategy / higher paraphrase coverage threshold / wider title-relevance check) lifts the rate toward 1.0 → a positive gamma*Delta5f term in fork_score. verify_quotes is pure-lexical / deterministic, so the rate is stable. NOT part of `make bench-5f`, `make bench-5s5t5f`, `bench-fork-baseline`, or `runner --all`; pinned separately via `make bench-fork-baseline-hard`. Pre-documented gap this builds around: 5f-fal-live-003 in falsification-live-v1.jsonl. See ticket #000046."}} +{"_meta": {"battery": "5f", "sub_battery": "falsification", "version": "v1", "task_count": 12, "notes": "#000046 — HARD live-path Falsification tier. Every fixture is a near-miss: the claim is FALSE / unsupported, so the correct verdict is UNGROUNDED, and that is what `expected_reason` records. As of #000046 closure (2026-05-11), verify_quotes' paraphrase numeric-agreement gate catches the 2 magnitude/year over-grounds (50-vs-100, 300-vs-300,000) → those now correctly land UNGROUNDED. The remaining 6 still over-ground: 4 via the entity-proximity strategy (matches on a shared proper noun while the answer's other salient term is wrong → HYBRID_ENTITY) and 2 via paraphrase token-coverage on a recombined-no-number claim (\"Mercury is the largest\" vs source \"Jupiter is the largest; Mercury is the smallest\" — all content tokens present, just recombined → STRICT_PARAPHRASE). The other 4 the verifier always handled correctly. Rate at HEAD = 6/12 = 0.5 — a real below-ceiling baseline. A further verifier upgrade (order/dependency-aware match or a contradiction check, vs the lexical token-coverage that can't see recombination) lifts the rate toward 1.0 → a larger positive gamma*Delta5f term in fork_score. verify_quotes is pure-lexical / deterministic, so the rate is stable. NOT part of `make bench-5f`, `make bench-5s5t5f`, `bench-fork-baseline`, or `runner --all`; pinned separately via `make bench-fork-baseline-hard`. Pre-documented gap this builds around: 5f-fal-live-003 in falsification-live-v1.jsonl. See ticket #000046."}} {"id": "5f-fal-hard-001", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Insulin was discovered by Alexander Fleming.", "context": "Penicillin was discovered by Alexander Fleming in 1928.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "OVER-GROUND at HEAD: HYBRID_ENTITY — substance swapped, but entity strategy matches on shared 'Alexander Fleming'."} {"id": "5f-fal-hard-002", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The Eiffel Tower is in Berlin.", "context": "The Eiffel Tower is in Paris, designed by Gustave Eiffel.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "OVER-GROUND at HEAD: HYBRID_ENTITY — city swapped, entity strategy matches on 'Eiffel Tower'/'Eiffel'."} {"id": "5f-fal-hard-003", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Mercury is the largest planet in the Solar System.", "context": "Jupiter is the largest planet in the Solar System; Mercury is the smallest.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "OVER-GROUND at HEAD: STRICT_PARAPHRASE — verifier claims STRICT on a false statement; paraphrase token-coverage matches 'largest planet in the Solar System'."} -{"id": "5f-fal-hard-004", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Water boils at 50 degrees Celsius at sea level.", "context": "Water boils at 100 degrees Celsius at sea level.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "OVER-GROUND at HEAD: STRICT_PARAPHRASE — number swapped (50 vs 100), but the rest of the sentence token-covers."} +{"id": "5f-fal-hard-004", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Water boils at 50 degrees Celsius at sea level.", "context": "Water boils at 100 degrees Celsius at sea level.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "Caught (#000046 closure): the paraphrase numeric-agreement gate sees 50 ∉ {100} → UNGROUNDED. PASSES today."} {"id": "5f-fal-hard-005", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Albert Einstein developed the theory of evolution.", "context": "Albert Einstein developed the theory of relativity; Charles Darwin developed evolution.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "OVER-GROUND at HEAD: STRICT_PARAPHRASE — 'evolution' vs 'relativity' swapped; 'Albert Einstein developed the theory of' token-covers + 'evolution' appears in context (attributed to Darwin)."} {"id": "5f-fal-hard-006", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The French Revolution began in 1889.", "context": "The French Revolution began in 1789, with the storming of the Bastille.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "OVER-GROUND at HEAD: HYBRID_ENTITY — year off by a century (1889 vs 1789); entity strategy matches on 'French Revolution'."} -{"id": "5f-fal-hard-007", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The speed of light is approximately 300 kilometers per second.", "context": "The speed of light is approximately 300,000 kilometers per second in a vacuum.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "OVER-GROUND at HEAD: STRICT_PARAPHRASE — magnitude wrong by 1000x (300 vs 300,000); 'the speed of light is approximately 300 kilometers per second' token-covers."} +{"id": "5f-fal-hard-007", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The speed of light is approximately 300 kilometers per second.", "context": "The speed of light is approximately 300,000 kilometers per second in a vacuum.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "Caught (#000046 closure): the paraphrase numeric-agreement gate sees 300 ∉ {300000} (← '300,000' comma-stripped) → UNGROUNDED. PASSES today."} {"id": "5f-fal-hard-008", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The Titanic sank in the Pacific Ocean.", "context": "The Titanic sank in the North Atlantic Ocean in 1912.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "OVER-GROUND at HEAD: HYBRID_ENTITY — ocean swapped (Pacific vs North Atlantic); entity strategy matches on 'Titanic'."} {"id": "5f-fal-hard-009", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "Shakespeare wrote War and Peace.", "context": "Tolstoy wrote War and Peace; Shakespeare wrote Hamlet.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "Correctly UNGROUNDED at HEAD — verifier does NOT over-ground this near-miss. PASSES today; kept as headroom marker (a regression that started over-grounding this would drop the rate)."} {"id": "5f-fal-hard-010", "battery": "5f", "sub_battery": "falsification", "version": "v1", "carrier": "providence_record", "domain": "claim_lattice", "pi_star_ref": "claim-lattice@v1", "answer_text": "The mitochondria is the powerhouse of the nucleus.", "context": "The mitochondria is the powerhouse of the cell.", "expected_reason": "UNGROUNDED", "verifier_method_root": "verify_quotes-v1", "expected": "pass", "note": "Correctly UNGROUNDED at HEAD — 'nucleus' vs 'cell' swap is caught. PASSES today."} diff --git a/bench/results/baseline-falsification-hard.json b/bench/results/baseline-falsification-hard.json index b0ccf18..037104c 100644 --- a/bench/results/baseline-falsification-hard.json +++ b/bench/results/baseline-falsification-hard.json @@ -5,11 +5,11 @@ "battery": "5f", "sub_battery": "falsification", "fixture_path": "bench/fixtures/5f/falsification-hard-v1.jsonl", - "fixture_digest": "6da602f624be3618d893201f4c47b8c3b411b7c3b5865f49655dc95f46968c43", - "pass_count": 4, - "fail_count": 8, + "fixture_digest": "519524e6ceab694d14971832e4cd676c952f032bb05580637d9185c2b6e22096", + "pass_count": 6, + "fail_count": 6, "metrics": { - "error_detection_rate": 0.3333333333333333 + "error_detection_rate": 0.5 }, "per_task": [ { @@ -53,14 +53,14 @@ }, { "task_id": "5f-fal-hard-004", - "passed": false, + "passed": true, "detail": { "source": "live", "expected_reason": "UNGROUNDED", "observed_violations": [ - "STRICT_PARAPHRASE" + "UNGROUNDED" ], - "detected": false, + "detected": true, "verifier_method_root": "verify_quotes-v1" } }, @@ -92,14 +92,14 @@ }, { "task_id": "5f-fal-hard-007", - "passed": false, + "passed": true, "detail": { "source": "live", "expected_reason": "UNGROUNDED", "observed_violations": [ - "STRICT_PARAPHRASE" + "UNGROUNDED" ], - "detected": false, + "detected": true, "verifier_method_root": "verify_quotes-v1" } }, @@ -171,7 +171,7 @@ } ], "runtime_digest": "da5203214469cc5e682dd8c9edfcb0c95a758e1ba78567b4577b2c598c307951", - "timestamp": 1778501548 + "timestamp": 1778507146 } ] } diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 4f60f72..1f8bd56 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -93,7 +93,7 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| | #000047 | ForkScore `_delta_*` aggregator (mean vs max vs sum) | **closed · 2026-05-11** — Option D: `WeightSet.delta_aggregator` ∈ {`mean`,`max`,`sum`} (default `mean` unchanged → no `ESTIMATOR_VERSION` bump), `fork_score._delta_5{s,t,f}` dispatch via `_aggregate`, recorded in `ScoredFork.weights`, per-sub `HARD_REGRESSION_FLOOR` flags aggregator-independent; bench data behind keeping `mean` in `5f-threshold-calibration-2026-05-11.md` §5; 8+1 tests. #000012-revision / #000025 §10.14 follow-up | 2026-05-11 | — | -| #000046 | Harder 5S/5T/5F fixture tier (below-ceiling baselines) | in progress · **Phase 1 + Phase 2 landed 2026-05-11** — `falsification-hard-v1.jsonl` (12 near-misses, `verify_quotes` over-grounds 8 → rate 4/12; `make bench-5f-falsification-hard` / `bench-fork-baseline-hard`, worked-example test `fork_score` γ·Δ5f → positive) + `formulate-hard-v1.jsonl` (12 mis-segment inputs, `parse_pointer_claims` mis-segments 8 → rate 4/12; `make bench-5f-formulate-hard`). Two below-ceiling 5F subs now exist. Closure pending an actual surface improvement (`verify_quotes` tightening — bench-gated — or `parse_pointer_claims` segmentation) that lifts a rate. #000025 §10.14 follow-up; #000047 closed | 2026-05-11 | — | +| #000046 | Harder 5S/5T/5F fixture tier (below-ceiling baselines) | **closed · 2026-05-11** — Phase 1 `falsification-hard-v1.jsonl` (12 near-misses) + Phase 2 `formulate-hard-v1.jsonl` (12 mis-segments, rate 4/12) + Phase 3 `verify_quotes` paraphrase numeric-agreement gate (`_numeric_signature`; demotes a token-covering span asserting a digit-number the source lacks modulo thousands-comma) → falsification-hard rate 4/12 → 6/12 on a real change; bench-gated (`make bench-qa` n=3×75×3 before/after — no STRICT-rate regression on legit answers; only gate-caused QA shift was correctly demoting a fictional-year claim STRICT→HYBRID); `fork_score` γ·Δ5f went positive on it. Headroom (6 over-grounds + 8 mis-segments) = optional bigger-verifier-upgrade follow-up. `make bench-5f-falsification-hard` / `bench-5f-formulate-hard` / `bench-fork-baseline-hard`; 7+ tests. #000025 §10.14 follow-up; #000047 closed | 2026-05-11 | — | | #000045 | Prometheus-Σ Phase 3 sleep-sweep scheduler (gating ticket) | open · doc-only scaffold 2026-05-10; pins 8 governance parameters + 4 retrigger gates; opens implementation only after one retrigger fires | 2026-05-10 | — | | #000044 | AUTOCOUNT doc-drift discipline | closed · landed across `fc5ba50` / `03c0f6a` / `6c6defb` / `f5dbfab` / `3b30126` 2026-05-10 (mechanism + 4 metrics + 54 tags across 7 doc files; harness catches drift at test time, refresh is 60-second turnaround) | 2026-05-10 | — | | #000043 | Empirical tightening of T3 bound constants (#000036 Phase 2) | open · awaiting go/no-go (parks until v7 deployment surfaces measurement data) | 2026-05-10 | — | diff --git a/docs/qa-modes-bench.md b/docs/qa-modes-bench.md index dc75532..2dc09eb 100644 --- a/docs/qa-modes-bench.md +++ b/docs/qa-modes-bench.md @@ -407,3 +407,38 @@ correct; operators wanting strictness opt in via `--block-on-contradiction`. Bench artifact: `bench/qa_results/2026-05-04T02-18-42Z.{jsonl,md}`. + + +### Addendum 5 — #000046 Phase 3 verifier numeric-gate regression check (2026-05-11) + +`arborist/qa/verify.py` gained a paraphrase numeric-agreement gate +(`_numeric_signature` + a check in `_check_each_with_paraphrase`): +a span that token-covers the source ≥ `paraphrase_coverage` but +asserts a digit-number the source lacks (modulo thousands-comma) is +no longer paraphrase-grounded — it goes to `unverified`. Closes +#000046 by lifting the `falsification-hard-v1.jsonl` rate 4/12 → 6/12 +(`5f-fal-hard-004` 50-vs-100 and `-007` 300-vs-300,000 → UNGROUNDED). + +Before/after `make bench-qa` (n=3 × 75 questions × 3 modes = 675 +cells): + +| mode | STRICT-rate before → after | Δ | +|------|----------------------------|---| +| quote | 0.53 → 0.50 | −3pp | +| claim_lattice_pointer | 0.23 → 0.25 | +2pp | +| claim_lattice | 0.44 → 0.45 | +1pp | + +All within the 5-pp noise floor. Per-row diff (675 common cells, 74 +changed audit_mode): the *only* clearly gate-attributable QA shift +was the fictional "our cold fusion breakthrough" year-claim demoting +STRICT → HYBRID across all 3 samples — a **correct** demotion (the +year isn't grounded). Every other transition was `quote→quote` / +`claim_lattice→claim_lattice` LLM re-answer variance — the answer +text changed on the re-run, not the verifier (the gate touches only +the paraphrase fallback, never the verbatim/span/entity/claim-lattice +paths). No regression on legitimate answers; the gate ships. + +Bench artifacts: `bench/qa_results/2026-05-11T13-42-38Z.{jsonl,md}` +(before) · `bench/qa_results/2026-05-11T14-19-51Z.{jsonl,md}` (after). +Full per-ticket detail: `docs/tickets/ticket-000046-harder-5sf-fixture-tier.md` +§5 Phase 3. diff --git a/docs/tickets/ticket-000012-selection-consensus-protocol.md b/docs/tickets/ticket-000012-selection-consensus-protocol.md index 3527913..a74bdd2 100644 --- a/docs/tickets/ticket-000012-selection-consensus-protocol.md +++ b/docs/tickets/ticket-000012-selection-consensus-protocol.md @@ -573,12 +573,14 @@ protocol must account for: `feedback_efficiency`) + the non-bench terms (`selfmodel_calibration_gain`, `audit_completeness`, `validator_diversity`, …), not by bench Δ-rate. The harder tier - is tracked as **#000046** (a #000025 follow-up); its **Phase 1 - landed 2026-05-11** — `bench/fixtures/5f/falsification-hard-v1.jsonl` - (12 near-misses, rate 4/12 ≈ 0.333 at HEAD, `verify_quotes` - over-grounds 8) is the first below-ceiling pack, with a - worked-example test showing `fork_score`'s `γ·Δ5f` going positive - on a lift to 1.0. So the bench Δ-rate *can* now carry signal on - the 5F/falsification axis; closing #000046 needs an actual - `verify_quotes` tightening to lift the rate. Not a #000012 - blocker either way. + is **#000046 (closed 2026-05-11)**: two below-ceiling 5F packs + (`falsification-hard-v1.jsonl`, `formulate-hard-v1.jsonl`) + + `verify_quotes`' paraphrase numeric-agreement gate, which lifted + the falsification-hard rate 4/12 → 6/12 on a *real* change + (bench-gated: `make bench-qa` n=3 before/after showed no + STRICT-rate regression on legit answers) → `fork_score`'s `γ·Δ5f` + went positive on it. So the bench Δ-rate *does* now carry signal + on the 5F/falsification axis, demonstrated end-to-end. Remaining + hard-pack headroom (6 over-grounds + 8 mis-segments) is an + optional bigger-verifier-upgrade follow-up, not a #000012 + blocker. diff --git a/docs/tickets/ticket-000046-harder-5sf-fixture-tier.md b/docs/tickets/ticket-000046-harder-5sf-fixture-tier.md index 6860cd8..f566867 100644 --- a/docs/tickets/ticket-000046-harder-5sf-fixture-tier.md +++ b/docs/tickets/ticket-000046-harder-5sf-fixture-tier.md @@ -1,19 +1,26 @@ # Ticket #000046 — Harder 5S/5T/5F fixture tier (below-ceiling baselines) -**Status:** in progress · **Phase 1 + Phase 2 landed 2026-05-11.** -Phase 1 (Option A) — 5F Falsification hard pack -`bench/fixtures/5f/falsification-hard-v1.jsonl` (12 near-misses, rate -4/12 ≈ 0.333 at HEAD, stable; `make bench-5f-falsification-hard` / -`make bench-fork-baseline-hard`; worked-example test: -`fork_score`'s γ·Δ5f goes positive on a lift to 1.0). Phase 2 — -5F Formulate hard pack `bench/fixtures/5f/formulate-hard-v1.jsonl` -(12 mis-segment inputs, rate 4/12 at HEAD, stable; `make -bench-5f-formulate-hard`). Two below-ceiling 5F sub-batteries now -exist. **Closure pending an actual surface improvement** that lifts -either rate — `verify_quotes` tightening (falsification) or -`parse_pointer_claims` sentence/clause segmentation (formulate) — -each a separate, larger task touching the live surface, not the -bench. See §5 + §7. +**Status:** **closed · 2026-05-11.** Phase 1 (Option A) — 5F +Falsification hard pack `bench/fixtures/5f/falsification-hard-v1.jsonl` +(12 near-misses; `make bench-5f-falsification-hard` / +`make bench-fork-baseline-hard`). Phase 2 — 5F Formulate hard pack +`bench/fixtures/5f/formulate-hard-v1.jsonl` (12 mis-segment inputs, +rate 4/12, stable; `make bench-5f-formulate-hard`). Phase 3 (the +closure) — a **real `verify_quotes` tightening**: a paraphrase +numeric-agreement gate (`_numeric_signature` + a check in +`_check_each_with_paraphrase`) that demotes a token-covering span +asserting a digit-number the source lacks (modulo thousands-comma). +It lifted the falsification-hard rate **4/12 → 6/12 = 0.5** and was +bench-gated — `make bench-qa` n=3 × 75 × 3 before/after showed no +STRICT-rate regression on legitimate answers (the only gate-caused +QA shift was correctly demoting a fictional-year claim STRICT → +HYBRID; all other deltas were within the 5-pp noise floor and were +LLM re-answer variance). `fork_score`'s γ·Δ5f went positive on that +real improvement → the loop is closed end-to-end. Headroom remains +(6 over-grounds still in the hard pack — 4 HYBRID_ENTITY + 2 +recombined-no-number STRICT_PARAPHRASE — plus the Formulate pack's +8); a bigger order/dependency-aware verifier upgrade would close +those, tracked separately if wanted (see §5). See §5 + §7. **Opened:** 2026-05-11 **Scope:** Give the 5S/5T/5F bench packs *headroom* — a fixture tier the runtime does **not** pass at rate 1.0 — so the ForkScore Δ-rate @@ -223,38 +230,96 @@ testable only once there's a below-ceiling baseline to bench against. `test_5f_formulate_hard_pack_below_ceiling` (pins rate 4/12, source=live, the 8 fails are claim-count mis-segments). -### Still open (the actual closure) +### Phase 3 (the closure) — `verify_quotes` paraphrase numeric-agreement gate -Both phases *built the below-ceiling baselines* and proved the loop -with synthetic children at rate 1.0. Closing the ticket needs a -**real surface improvement** that lifts one of the rates: a -`verify_quotes` tightening (stricter entity strategy / higher -paraphrase coverage threshold / wider title-relevance check — -falsification) or `parse_pointer_claims` sentence/clause segmentation -(formulate) — each a separate, larger task touching the live surface, -not the bench, and (for `verify_quotes`) a bench-gated change (a QA -sweep must show STRICT-rate doesn't regress on legitimate answers). -When one lands, this ticket closes with the rate-improvement → -ForkScore Δ demonstrated on an *actual* change. Until then: Phases 1 -+ 2 stand, the two hard packs pin the gaps. +- **Change:** `arborist/qa/verify.py` — + `_numeric_signature(text)` (comma-stripped digit-runs: `8,849` and + `8849` collapse; `300` stays distinct from `300000` ← `300,000`) + + a gate in `_check_each_with_paraphrase`: a span that token-covers + the source ≥ `paraphrase_coverage` but asserts a digit-number the + source lacks is **not** paraphrase-grounded — it goes to + `unverified`. Catches the near-miss the lexical coverage check is + blind to ("Water boils at 50 degrees" against a source saying 100 + token-covers 100% because `50`/`100` aren't > 4-char content + tokens). The gate is **narrow by construction**: only fires on the + paraphrase fallback (verbatim / span / entity paths untouched), and + only on a digit-number, modulo thousands-comma. A rounding- + paraphrase demoting here is the honest verdict — it isn't a + verbatim grounding. +- **Effect on the hard pack:** lifts `5f-fal-hard-004` (50 vs 100) + and `-007` (300 vs 300,000) to UNGROUNDED → **falsification-hard + rate 4/12 → 6/12 = 0.5**. The other 6 over-grounds (4 HYBRID_ENTITY + + 2 recombined-no-number STRICT_PARAPHRASE — "Mercury is the + largest" vs "Jupiter is the largest; Mercury is the smallest") are + unaffected; the Formulate hard pack is unaffected (it exercises + `parse_pointer_claims`, not `verify_quotes`). +- **Bench gate:** `make bench-qa` (n=3 × 75 questions × 3 modes = 675 + cells) before (`bench/qa_results/2026-05-11T13-42-38Z.md`) and + after (`...T14-19-51Z.md`). STRICT-rate: quote 0.53 → 0.50, pointer + 0.23 → 0.25, lattice 0.44 → 0.45 — all within the 5-pp noise floor. + Per-row diff: the *only* clearly gate-attributable QA shift was the + fictional "our cold fusion breakthrough" year-claim demoting STRICT + → HYBRID (×3 samples) — a **correct** demotion (the year isn't + grounded); every other transition was `quote→quote` / `claim_lattice + →claim_lattice` LLM re-answer variance, which the gate doesn't + touch. No regression on legitimate answers. +- **Worked example / the loop:** `fork_score` on the real improvement + — parent `{5f/falsification: 4/12}` → child `{5f/falsification: + 6/12}` → `γ·Δ5f = (1/6)/5 ≈ +0.033 > 0` (positive, just under + ACCEPT given the ÷5 dilution — a *single* improvement of this size + is MARGINAL by design; the test pins the full-lift-to-1.0 case at + ACCEPT). The bench Δ-rate carries signal on the 5F/falsification + axis now, on a real change. +- **Tests:** `tests/test_verify.py` — `test_numeric_signature_comma_normalized`, + `test_numeric_signature_subset_matches_comma_variant`, + `test_paraphrase_numeric_disagreement_rejected`, + `test_paraphrase_numeric_gate_is_narrow_no_number_unaffected`, + `test_paraphrase_number_present_in_source_still_verifies`; + `tests/test_bench_batteries.py` `test_5f_falsification_hard_pack_below_ceiling` + re-pinned 4/12 → 6/12. + +### Headroom (optional follow-up — not a #000046 blocker) + +6 over-grounds remain in the falsification-hard pack (4 HYBRID_ENTITY +where the entity-proximity strategy matches on a shared proper noun +while the answer's other salient term is wrong; 2 STRICT_PARAPHRASE +where the false claim recombines source tokens into a different true +statement — "Mercury is the largest" — and lexical token-coverage +genuinely can't tell recombination from grounding), plus the 8 +mis-segments in the Formulate hard pack. Closing those needs a +bigger verifier upgrade — an order/dependency-aware match or a +contradiction check for `verify_quotes`, and sentence/clause +segmentation for `parse_pointer_claims` — each its own bench-gated +ticket if/when fox wants the headroom closed. #000046 itself is done: +the below-ceiling baselines exist, a real surface improvement lifted +a rate, and ForkScore's bench-Δ went positive on it. --- ## 6. References - `bench/fixtures/5f/falsification-hard-v1.jsonl` — the Phase-1 hard - pack (12 near-misses, rate 4/12 at HEAD). `make - bench-5f-falsification-hard` runs it; `make bench-fork-baseline-hard` - pins it to `bench/results/baseline-falsification-hard.json`. + pack (12 near-misses, rate 6/12 after the Phase-3 numeric gate). + `make bench-5f-falsification-hard` runs it; `make + bench-fork-baseline-hard` pins it to + `bench/results/baseline-falsification-hard.json`. - `bench/fixtures/5f/formulate-hard-v1.jsonl` — the Phase-2 hard pack (12 mis-segment inputs, rate 4/12 at HEAD). `make bench-5f-formulate-hard` runs it. +- `arborist/qa/verify.py` — `_numeric_signature` + the + `_check_each_with_paraphrase` numeric-agreement gate (Phase 3). - `arborist/qa/parse_claims.py` (`parse_pointer_claims`) — the line/bullet-based parser the Phase-2 pack exercises. +- `bench/qa_results/2026-05-11T13-42-38Z.md` (before) + + `bench/qa_results/2026-05-11T14-19-51Z.md` (after) — the QA-bench + gate for the Phase-3 verifier change. +- `tests/test_verify.py` — the 5 numeric-gate tests (`test_numeric_signature_*`, + `test_paraphrase_numeric_disagreement_rejected`, + `test_paraphrase_numeric_gate_is_narrow_no_number_unaffected`, + `test_paraphrase_number_present_in_source_still_verifies`). - `tests/test_bench_batteries.py` — - `test_5f_falsification_hard_pack_below_ceiling`, - `test_5f_formulate_hard_pack_below_ceiling` (pin the rates + - the over-ground / mis-segment distinctions) + + `test_5f_falsification_hard_pack_below_ceiling` (6/12 after Phase 3), + `test_5f_formulate_hard_pack_below_ceiling` (4/12) + `test_fork_score_positive_gamma_5f_on_hard_falsification_improvement` (the worked example). - `bench/scripts/fivef_threshold_calibration.py` + diff --git a/tests/test_bench_batteries.py b/tests/test_bench_batteries.py index 70df162..888bf6f 100644 --- a/tests/test_bench_batteries.py +++ b/tests/test_bench_batteries.py @@ -701,15 +701,19 @@ def test_5f_formulate_hard_pack_below_ceiling(): def test_5f_falsification_hard_pack_below_ceiling(): """The hard pack is a deliberate below-ceiling baseline: 12 - near-misses whose correct verdict is UNGROUNDED, 8 of which - verify_quotes over-grounds at HEAD → rate 4/12. verify_quotes is + near-misses whose correct verdict is UNGROUNDED. As of #000046 + closure (2026-05-11), verify_quotes' paraphrase numeric-agreement + gate catches the 2 magnitude/year over-grounds (50-vs-100, + 300-vs-300,000) → 6/12 pass; the remaining 6 over-grounds (4 + HYBRID_ENTITY, 2 STRICT_PARAPHRASE without a number) are the + headroom a future verifier upgrade closes. verify_quotes is deterministic, so this is stable; if it shifts, the pinned value here fires (loud signal — either the verifier got better/worse, or the pack drifted).""" res = b_5f.run_falsification(F5F / "falsification-hard-v1.jsonl") - assert res.pass_count == 4 - assert res.fail_count == 8 - assert res.metrics["error_detection_rate"] == pytest.approx(4 / 12) + assert res.pass_count == 6 + assert res.fail_count == 6 + assert res.metrics["error_detection_rate"] == pytest.approx(6 / 12) # All tasks route the live verifier; every fixture asserts UNGROUNDED. for t in res.per_task: assert t.detail["source"] == "live" diff --git a/tests/test_verify.py b/tests/test_verify.py index fcff443..0cb78bf 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -13,6 +13,7 @@ Tests cover the full trichotomy under each path plus regressions: from __future__ import annotations from arborist.qa.verify import ( + _numeric_signature, extract_claim_spans, extract_proper_nouns, extract_quotes, @@ -550,3 +551,67 @@ def test_wikitext_strip_does_not_rescue_genuine_hallucination(): assert v["n_verified"] == 0 assert v["audit_mode"] == "UNGROUNDED" assert len(v["unverified_quotes"]) == 1 + + +# --- #000046 — paraphrase numeric-agreement gate ----------------- + + +def test_numeric_signature_comma_normalized(): + assert _numeric_signature("8,849 meters") == frozenset({"8849"}) + assert _numeric_signature("300,000 km") == frozenset({"300000"}) + assert _numeric_signature("the year 1789, the year 1889") == frozenset({"1789", "1889"}) + assert _numeric_signature("no digit numbers at all") == frozenset() + # decimals split at the dot — fine for the near-miss patterns we target + assert _numeric_signature("pi is 3.14 plus 2") == frozenset({"3", "14", "2"}) + + +def test_numeric_signature_subset_matches_comma_variant(): + # a span number written without a thousands comma is "in" a source + # number written with one (both collapse), so legitimate + # number-paraphrases still verify. + assert _numeric_signature("8849") <= _numeric_signature("Mount Everest is 8,849 m tall") + + +def test_paraphrase_numeric_disagreement_rejected(): + # All content tokens of "Water boils at 50 degrees Celsius at sea + # level" are in the source (which says 100), so token coverage is + # 100% — but the number is wrong. The numeric-agreement gate + # demotes it from paraphrase-STRICT to UNGROUNDED. + v = verify_quotes( + "Water boils at 50 degrees Celsius at sea level.", + "Water boils at 100 degrees Celsius at sea level.", + ) + assert v["audit_mode"] == "UNGROUNDED" + # The magnitude case too: 300 vs 300,000 (1000x off). + v2 = verify_quotes( + "The speed of light is approximately 300 kilometers per second.", + "The speed of light is approximately 300,000 kilometers per second in a vacuum.", + ) + assert v2["audit_mode"] == "UNGROUNDED" + + +def test_paraphrase_numeric_gate_is_narrow_no_number_unaffected(): + # The gate fires ONLY on a digit-number the source lacks. A + # recombined-but-no-number near-miss ("Mercury is the largest" vs + # source "Jupiter is the largest; Mercury is the smallest") is NOT + # caught — that's the headroom a future verifier upgrade closes, + # not this gate's job. Pinning the narrowness so a later widening + # is a deliberate choice. + v = verify_quotes( + "Mercury is the largest planet in the Solar System.", + "Jupiter is the largest planet in the Solar System; Mercury is the smallest.", + ) + assert v["audit_mode"] == "STRICT" + assert v["verifier_method"] == "paraphrase" + + +def test_paraphrase_number_present_in_source_still_verifies(): + # A genuine paraphrase whose number IS in the source verifies as + # before — the gate doesn't touch it. + v = verify_quotes( + "The French Revolution started in 1789.", + "The French Revolution began in 1789, with the storming of the Bastille.", + ) + # quote/span/entity may classify first; what matters is it's not + # demoted to UNGROUNDED by a spurious numeric mismatch. + assert v["audit_mode"] in ("STRICT", "HYBRID")