diff --git a/aborist/cli.py b/aborist/cli.py index b09e851..57c6317 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -549,6 +549,14 @@ def _render_inspect_human(result: dict) -> str: if diag == "trailing_artifact": lines.append(f" matched_prefix_chars: {d.get('matched_prefix_chars')}") lines.append(f" trailing_artifact: {_short(d.get('trailing_artifact', ''), 100)}") + elif diag == "synthetic_elision_inside_quote": + lines.append( + f" [...] inserted by model — " + f"{d.get('prefix_chars', 0)} prefix chars " + f"({'in source' if d.get('prefix_in_source') else 'NOT in source'}), " + f"{d.get('suffix_chars', 0)} suffix chars " + f"({'in source' if d.get('suffix_in_source') else 'NOT in source'})" + ) elif diag == "interior_elision": lines.append( f" matched: {d.get('matched_prefix_chars')} prefix + " @@ -2848,6 +2856,7 @@ def build_parser() -> argparse.ArgumentParser: "sidecar diagnostic for a providence_cache record — pulls " "source chunks and classifies each unverified span " "(paraphrase / trailing_artifact / interior_elision / " + "synthetic_elision_inside_quote / " "no_overlap). Read-only, " "no audit events, no v9.8 field changes." ), diff --git a/aborist/qa/inspect.py b/aborist/qa/inspect.py index f07a86e..96ba5dd 100644 --- a/aborist/qa/inspect.py +++ b/aborist/qa/inspect.py @@ -42,6 +42,43 @@ _INTERIOR_ELISION_MIN_SUFFIX = 20 _INTERIOR_ELISION_MAX_ASIDE = 250 +def _try_synthetic_elision( + nspan: str, norm_base_ctx: str +) -> dict[str, Any] | None: + """Probe whether the model wrote ``[...]`` inside a quoted span. + + Distinct from ``interior_elision`` (model dropped a real ``(...)`` + aside that source carries) — synthetic elision means the model + INSERTED a literal ``[...]`` ellipsis marker between fragments of + its quote, signaling that it itself skipped content. The substring + test fails because ``[...]`` isn't in source. + + Per the corrected verifier rule (fox 2026-04-30): ``[...]`` inside + ``"..."`` is a quote-integrity failure. The verifier rejects (binary) + & this sidecar diagnoses the prefix/suffix split so an operator can + judge whether the elided segment was benign. + + Conservative: only fires when literal ``[...]`` appears in the span + AND the same string is absent from source (so a Wikipedia article + that genuinely contains ``[...]`` won't false-positive). + """ + if "[...]" not in nspan or "[...]" in norm_base_ctx: + return None + idx = nspan.index("[...]") + prefix = nspan[:idx].strip() + suffix = nspan[idx + len("[...]"):].strip() + prefix_in = bool(prefix) and prefix in norm_base_ctx + suffix_in = bool(suffix) and suffix in norm_base_ctx + return { + "diagnosis": "synthetic_elision_inside_quote", + "elision_marker": "[...]", + "prefix_chars": len(prefix), + "suffix_chars": len(suffix), + "prefix_in_source": prefix_in, + "suffix_in_source": suffix_in, + } + + def _try_interior_elision( nspan: str, norm_base_ctx: str ) -> dict[str, Any] | None: @@ -125,6 +162,12 @@ def _classify_span(span: str, norm_base_ctx: str, norm_raw_ctx: str) -> dict[str this — likely a verifier or canonicalization bug). - ``verbatim_in_raw_only``: matches raw wikitext but not the base form (wikitext-strip edge case). + - ``synthetic_elision_inside_quote``: model wrote a literal + ``"..."`` span containing ``[...]`` between fragments. The + ``[...]`` marker isn't in source — the model itself signaled it + skipped content while claiming verbatim citation. Quote-integrity + failure. Sidecar reports prefix/suffix presence so operator can + judge whether the elided middle is benign. - ``interior_elision``: source has ``A + (aside) + B``, model wrote ``A + B``. Content faithful to the main thread; the parenthetical aside got dropped for prose flow. Distinct from trailing_artifact @@ -157,6 +200,14 @@ def _classify_span(span: str, norm_base_ctx: str, norm_raw_ctx: str) -> dict[str # is source-minus-one-parenthetical, that's a more specific finding # than "tail doesn't match." Falls through if no prefix+`(...)`+suffix # pattern matches in source. + # Synthetic-elision probe runs FIRST: if the model wrote a literal + # `[...]` between two fragments, that's a more specific finding + # than "the whole span doesn't substring-match." Falls through if + # no `[...]` marker is in the span. + synthetic = _try_synthetic_elision(nspan, norm_base_ctx) + if synthetic is not None: + return synthetic + elision = _try_interior_elision(nspan, norm_base_ctx) if elision is not None: return elision diff --git a/aborist/qa/query.py b/aborist/qa/query.py index b02a766..6ed063d 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -270,6 +270,63 @@ class _Hit: score: float shard_path: str chunk_idx: int + source_role: str = "unclassified" + + +# Heuristic role classification + per-role budget multiplier. Lets the +# primary answer page (e.g. `Jurassic Park (film)` for a JP film query) +# claim a wider context slice than peripheral pages (`Jurassic Park (film +# score)`, `Jurassic Park video games`). The running `char_budget` check +# still bounds total context to `max_context_chars`; weights just shift +# how the budget gets divided. +SOURCE_ROLE_BUDGET_WEIGHTS = { + "primary_answer_source": 2.0, + "secondary_context_source": 1.0, + "noisy_background_source": 0.5, + "sequel_background_source": 0.5, + "background_source": 1.0, + "unclassified": 1.0, +} + +# Title patterns that demote a source's role. Lower-cased substring match. +_NOISY_TITLE_MARKERS = ( + "score", "music", "soundtrack", "video game", "merchandise", + "discography", +) +_SEQUEL_TITLE_MARKERS = ( + " iii", " ii)", " ii ", " iv", " v ", " v)", "lost world", "sequel", + " 2)", " 3)", " 4)", +) +_SECONDARY_TITLE_MARKERS = ( + "list of", "characters", "franchise", "history of", "people", + "timeline of", +) + + +def _classify_source_role(title: str | None, qtokens_stem: set[str]) -> str: + """Tag a source by its likely role for an N-token query. + + Order matters: noisy/sequel/secondary markers fire first because they + catch peripheral pages whose titles otherwise overlap query tokens + fully (e.g. `Jurassic Park (film score)` shares 3 stems with + `{dinosaur, jurassic, park, film}` but is not the primary answer + source for a dinosaurs question). Primary requires the strongest + title coverage (N-1 of N stems present). + """ + if not title: + return "unclassified" + t = title.lower() + if any(k in t for k in _NOISY_TITLE_MARKERS): + return "noisy_background_source" + if any(k in t for k in _SEQUEL_TITLE_MARKERS): + return "sequel_background_source" + if any(k in t for k in _SECONDARY_TITLE_MARKERS): + return "secondary_context_source" + title_tokens = _title_query_tokens(t.replace("_", " ")) + title_stems = {_stem_token_for_match(tok) for tok in title_tokens} + if qtokens_stem and len(title_stems & qtokens_stem) >= max(1, len(qtokens_stem) - 1): + return "primary_answer_source" + return "background_source" def _search_titles(conn, qtokens: list[str], limit: int) -> list[tuple]: @@ -753,13 +810,20 @@ def query( context_parts: list[str] = [] per_source_cap = max(1, max_context_chars // max(1, top_k)) char_budget = max_context_chars + qtokens_stem = {_stem_token_for_match(t) for t in _title_query_tokens(question)} for h in hits[:top_k]: text = _load_doc_text(h.shard_path, h.document_root) if not text: continue - # Cap each source first; then respect any remaining budget. - if len(text) > per_source_cap: - text = text[:per_source_cap] + # Tag by role & apply role-weighted cap. Primary answer source + # gets 2× the baseline cap, noisy/sequel get 0.5×, secondary & + # background get 1×. Total context still bounded by + # `char_budget` (running cap to `max_context_chars`). + h.source_role = _classify_source_role(h.title, qtokens_stem) + weight = SOURCE_ROLE_BUDGET_WEIGHTS.get(h.source_role, 1.0) + hit_cap = max(1, int(per_source_cap * weight)) + if len(text) > hit_cap: + text = text[:hit_cap] if len(text) > char_budget: text = text[:char_budget] if not text: @@ -1008,6 +1072,7 @@ def query( "score": h.score, "chunk_idx": h.chunk_idx, "shard": Path(h.shard_path).name, + "source_role": h.source_role, } for h in chosen ], diff --git a/tests/test_inspect.py b/tests/test_inspect.py index 3d18072..a503a9a 100644 --- a/tests/test_inspect.py +++ b/tests/test_inspect.py @@ -112,6 +112,38 @@ def test_classify_interior_elision_falls_through_when_suffix_doesnt_match(): assert out["diagnosis"] != "interior_elision" +def test_classify_synthetic_elision_caught(): + """Fox 2026-04-30 (Brachiosaurus / Jurassic Park): the model wrote + a `"..."` quote with literal `[...]` between fragments, signaling + self-elision while claiming verbatim citation. Distinct from + `interior_elision` (model dropped a `(...)` aside source carries). + Sidecar reports prefix/suffix presence in source.""" + base = ( + "The film centers on the fictional Isla Nublar, in Costa Rica, where " + "billionaire philanthropist John Hammond has created an amusement park " + "of cloned dinosaurs. Universal Studios acquired the rights." + ) + span = ( + "The film centers on the fictional Isla Nublar [...] Universal Studios " + "acquired the rights." + ) + out = _classify_span(span, _norm(base), _norm(base)) + assert out["diagnosis"] == "synthetic_elision_inside_quote" + assert out["elision_marker"] == "[...]" + assert out["prefix_in_source"] is True + assert out["suffix_in_source"] is True + + +def test_classify_synthetic_elision_does_not_fire_when_source_has_brackets(): + """If `[...]` literally appears in source (e.g. a citation + formatting), the substring check would have passed earlier — sidecar + falls through to its other diagnoses.""" + base = "Some prose with [...] literal brackets in source." + span = "Some prose with [...] literal brackets in source." + out = _classify_span(span, _norm(base), _norm(base)) + assert out["diagnosis"] == "verbatim_in_base" + + def test_classify_paraphrase_high_token_coverage(): """Tokens all present, sequence different — model rewrote the source.""" base = ( diff --git a/tests/test_query.py b/tests/test_query.py index 9cb863d..33eac33 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -333,6 +333,47 @@ def test_query_strict_fidelity_does_not_fall_back(tmp_path): assert len(captured) == 1 +def test_classify_source_role_separates_primary_from_noisy(): + """Direct unit test on the role classifier. JP film-score should be + noisy_background; JP (film) should be primary; JP franchise should + be secondary; The Lost World should be sequel; off-topic background. + Catches the case where peripheral pages with strong title overlap + used to share the primary slot.""" + from aborist.qa.query import _classify_source_role + + qstems = {"dinosaur", "jurassic", "park", "film"} + assert _classify_source_role("Jurassic Park (film)", qstems) == "primary_answer_source" + assert _classify_source_role("Jurassic Park (film score)", qstems) == "noisy_background_source" + assert _classify_source_role("Jurassic Park video games", qstems) == "noisy_background_source" + assert _classify_source_role("Jurassic Park (franchise)", qstems) == "secondary_context_source" + assert _classify_source_role("List of Jurassic Park characters", qstems) == "secondary_context_source" + assert _classify_source_role("The Lost World: Jurassic Park", qstems) == "sequel_background_source" + # Off-topic title (no shared stems): falls through to background. + assert _classify_source_role("Anarchism", qstems) == "background_source" + + +def test_query_role_weighted_budget_persists_role_on_sources(tmp_path): + """Each source in the providence record's merkle_proof.sources gains + a `source_role` field — verifies the role made it into the audit + trail so an inspector can see which slot a source occupied.""" + main_db = tmp_path / "corpus.db" + qa_db = tmp_path / "qa.db" + conn = connect(main_db) + try: + ingest_source(conn, FakeSource(DOCS)) + finally: + conn.close() + result = query( + question="What is anarcho-capitalism?", + qa_db=qa_db, + chat_client=StubClient(answer="x"), + model_id="m", + single_db=main_db, + top_k=3, + ) + assert all("source_role" in s for s in result["sources"]) + + def test_query_no_sources_when_empty_corpus(tmp_path): main_db = tmp_path / "empty.db" qa_db = tmp_path / "qa.db"