inspect: interior_elision diagnosis — model dropped a (...) aside from source
Fox 2026-04-29 (Clark Kent / Superman query): the verifier flagged a
quoted span as unverified and the sidecar diagnosed it as 'paraphrase'
with token_coverage=1.0. Investigation showed the source actually said
`Clark Joseph Kent (middle name is also Jerome according to some
versions) is a fictional character...` and the model elided the
parenthetical aside, quoting `Clark Joseph Kent is a fictional
character...`. Every word in source, but the sequence has a 60-char
gap where the aside was. Distinct failure mode from real paraphrase
(token reordering) and trailing_artifact (model APPENDS).
Sidecar refinement, no verifier change. The binary verifier discipline
(memory rule `feedback_verifier_no_diagnostics`) holds — quote strategy
is still verbatim-only; soft signal stays in the inspect verb.
Algorithm walks every `(` in base. For each open paren at position P:
the longest k where base[:P] ends with span[:k] is the model's prefix;
the span tail (≥20 chars) must then match the source after the close
paren. First paren that satisfies both checks wins.
CLI human-render shows `matched: N prefix + M suffix chars
(parenthetical aside dropped)` plus the dropped_aside text so an
operator can decide at a glance whether the elision is benign.
Live record now reads:
[1] interior_elision
matched: 17 prefix + 138 suffix chars (parenthetical aside dropped)
dropped_aside: middle name is also jerome according to some versions
439 tests pass (sidecar +2, full suite still green).
This commit is contained in:
parent
b48646bc0c
commit
1e8d2d2dba
2 changed files with 134 additions and 0 deletions
|
|
@ -37,6 +37,85 @@ def _normalize(s: str) -> str:
|
|||
return s.lower()
|
||||
|
||||
|
||||
_INTERIOR_ELISION_MIN_PREFIX = 12
|
||||
_INTERIOR_ELISION_MIN_SUFFIX = 20
|
||||
_INTERIOR_ELISION_MAX_ASIDE = 250
|
||||
|
||||
|
||||
def _try_interior_elision(
|
||||
nspan: str, norm_base_ctx: str
|
||||
) -> dict[str, Any] | None:
|
||||
"""Probe whether the span is source minus a single ``(...)`` aside.
|
||||
|
||||
Pattern: source has ``A + (X) + B``; model wrote ``A + B``. Common on
|
||||
Wikipedia leads where parentheticals interrupt the prose flow
|
||||
(e.g. ``"Clark Joseph Kent (middle name is also Jerome ...) is a
|
||||
fictional character..."`` → model elides the parenthetical for
|
||||
readability).
|
||||
|
||||
Walk every ``(`` in base. For each open paren at position P:
|
||||
longest k where ``base[:P]`` ends with ``nspan[:k]`` is the model's
|
||||
prefix; the span tail (≥ ``_INTERIOR_ELISION_MIN_SUFFIX`` chars)
|
||||
must then match the source after the close paren. First paren that
|
||||
satisfies both checks wins. Returns ``None`` if no paren in base
|
||||
matches the pattern — caller falls through to ``trailing_artifact``
|
||||
/ paraphrase detection.
|
||||
|
||||
Conservative: BOTH a meaningful prefix AND a meaningful suffix must
|
||||
match. A model that drops the aside AND rewords the rest will fail
|
||||
the suffix check & fall through to paraphrase, where it belongs.
|
||||
|
||||
Distinct from ``trailing_artifact`` (model APPENDED a parenthetical
|
||||
that's not in source) and from ``paraphrase`` (token-coverage but
|
||||
different sequence). Interior_elision is "model dropped one aside
|
||||
that source carries" — content faithful, formatting condensed.
|
||||
"""
|
||||
if len(nspan) < _INTERIOR_ELISION_MIN_PREFIX + _INTERIOR_ELISION_MIN_SUFFIX:
|
||||
return None
|
||||
|
||||
pos = 0
|
||||
while True:
|
||||
paren_open = norm_base_ctx.find("(", pos)
|
||||
if paren_open < 0:
|
||||
return None
|
||||
pos = paren_open + 1
|
||||
# base[:paren_open] must end with span[:k] for some k. Trim a
|
||||
# trailing space so `Foo (aside)` matches when span has no
|
||||
# trailing space after `Foo`.
|
||||
b_before = norm_base_ctx[:paren_open].rstrip(" ")
|
||||
if len(b_before) < _INTERIOR_ELISION_MIN_PREFIX:
|
||||
continue
|
||||
max_k = min(len(b_before), len(nspan) - _INTERIOR_ELISION_MIN_SUFFIX)
|
||||
if max_k < _INTERIOR_ELISION_MIN_PREFIX:
|
||||
continue
|
||||
# Longest k ≤ max_k where b_before.endswith(nspan[:k]).
|
||||
# Linear sweep from longest down; first hit wins.
|
||||
found_k = 0
|
||||
for k in range(max_k, _INTERIOR_ELISION_MIN_PREFIX - 1, -1):
|
||||
if b_before.endswith(nspan[:k]):
|
||||
found_k = k
|
||||
break
|
||||
if found_k == 0:
|
||||
continue
|
||||
paren_close = norm_base_ctx.find(")", paren_open + 1)
|
||||
if paren_close < 0 or paren_close - paren_open > _INTERIOR_ELISION_MAX_ASIDE:
|
||||
continue
|
||||
aside = norm_base_ctx[paren_open + 1 : paren_close]
|
||||
span_tail = nspan[found_k:].lstrip()
|
||||
if len(span_tail) < _INTERIOR_ELISION_MIN_SUFFIX:
|
||||
continue
|
||||
src_after = norm_base_ctx[paren_close + 1 : paren_close + 1 + len(span_tail) + 16]
|
||||
src_after = src_after.lstrip()
|
||||
if not src_after.startswith(span_tail):
|
||||
continue
|
||||
return {
|
||||
"diagnosis": "interior_elision",
|
||||
"matched_prefix_chars": found_k,
|
||||
"matched_suffix_chars": len(span_tail),
|
||||
"dropped_aside": aside.strip(),
|
||||
}
|
||||
|
||||
|
||||
def _classify_span(span: str, norm_base_ctx: str, norm_raw_ctx: str) -> dict[str, Any]:
|
||||
"""Classify one unverified span against the canonical context.
|
||||
|
||||
|
|
@ -46,6 +125,10 @@ def _classify_span(span: str, norm_base_ctx: str, norm_raw_ctx: str) -> dict[str
|
|||
this — likely a verifier or canonicalization bug).
|
||||
- ``verbatim_in_raw_only``: matches raw wikitext but not the base
|
||||
form (wikitext-strip edge case).
|
||||
- ``interior_elision``: source has ``A + (aside) + B``, model wrote
|
||||
``A + B``. Content faithful to the main thread; the parenthetical
|
||||
aside got dropped for prose flow. Distinct from trailing_artifact
|
||||
(which is model-APPENDED text) and paraphrase (different sequence).
|
||||
- ``trailing_artifact``: a prefix of length >= 60 chars matches; the
|
||||
tail (often a model-added citation like ``(Source: ...)``) doesn't.
|
||||
- ``paraphrase``: query tokens (>4 chars) all present in base context
|
||||
|
|
@ -69,6 +152,15 @@ def _classify_span(span: str, norm_base_ctx: str, norm_raw_ctx: str) -> dict[str
|
|||
if nspan[:plen] in norm_base_ctx:
|
||||
longest_prefix_match = plen
|
||||
break
|
||||
|
||||
# Interior_elision probe runs BEFORE trailing_artifact: if the span
|
||||
# is source-minus-one-parenthetical, that's a more specific finding
|
||||
# than "tail doesn't match." Falls through if no prefix+`(...)`+suffix
|
||||
# pattern matches in source.
|
||||
elision = _try_interior_elision(nspan, norm_base_ctx)
|
||||
if elision is not None:
|
||||
return elision
|
||||
|
||||
if longest_prefix_match >= 60:
|
||||
tail = span[longest_prefix_match:]
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -70,6 +70,48 @@ def test_classify_trailing_artifact_citation_appended():
|
|||
assert "Source:" in out["trailing_artifact"]
|
||||
|
||||
|
||||
def test_classify_interior_elision_drops_parenthetical_aside():
|
||||
"""Fox 2026-04-29 catch (Clark Kent): source has
|
||||
`Clark Joseph Kent (middle name is also Jerome ...) is a fictional
|
||||
character...`; model quoted `Clark Joseph Kent is a fictional
|
||||
character...` — every token in source, but a `(...)` aside got
|
||||
elided for prose flow. Distinct from trailing_artifact (model
|
||||
APPENDS) and paraphrase (different sequence). Sidecar must
|
||||
surface this as `interior_elision` with the dropped aside reported."""
|
||||
base = (
|
||||
"Clark Joseph Kent (middle name is also Jerome according to some "
|
||||
"versions) is a fictional character created by Jerry Siegel and "
|
||||
"Joe Shuster. He serves as the civilian and secret identity of "
|
||||
"the superhero Superman."
|
||||
)
|
||||
span = (
|
||||
"Clark Joseph Kent is a fictional character created by Jerry Siegel "
|
||||
"and Joe Shuster. He serves as the civilian and secret identity of "
|
||||
"the superhero Superman."
|
||||
)
|
||||
out = _classify_span(span, _norm(base), _norm(base))
|
||||
assert out["diagnosis"] == "interior_elision"
|
||||
assert out["matched_prefix_chars"] >= 17 # "clark joseph kent"
|
||||
assert out["matched_suffix_chars"] >= 60
|
||||
assert "jerome" in out["dropped_aside"].lower()
|
||||
|
||||
|
||||
def test_classify_interior_elision_falls_through_when_suffix_doesnt_match():
|
||||
"""Probe is conservative: suffix must land verbatim after the close
|
||||
paren. A model that drops the aside AND rewords the rest should NOT
|
||||
classify as interior_elision."""
|
||||
base = (
|
||||
"Clark Joseph Kent (middle name is Jerome) is a fictional character "
|
||||
"created by Jerry Siegel and Joe Shuster."
|
||||
)
|
||||
span = (
|
||||
"Clark Joseph Kent is the most famous superhero in comics history "
|
||||
"and was invented by Jerry Siegel."
|
||||
)
|
||||
out = _classify_span(span, _norm(base), _norm(base))
|
||||
assert out["diagnosis"] != "interior_elision"
|
||||
|
||||
|
||||
def test_classify_paraphrase_high_token_coverage():
|
||||
"""Tokens all present, sequence different — model rewrote the source."""
|
||||
base = (
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue