arborist/aborist/qa/inspect.py

"""Sidecar diagnostic for `providence_cache` records.

Read-only inspection tool. Pulls the same context the verifier saw,
runs the same canonicalization (norm-v1 + lowercase + wikitext strip),
and classifies each ``unverified_quote`` against the corpus so an
operator can tell paraphrase from invention from formatting wart.

By design this writes nothing — no audit events, no providence_cache
mutations, no v9.8 field changes. The verifier's binary output stays
authoritative; this tool is a debugging lens. See the memory rule
``feedback_verifier_no_diagnostics``: soft signals stay in sidecar
verbs, never feed back into the hard chain.
"""

from __future__ import annotations

import json
import os
import sqlite3
import unicodedata
from pathlib import Path
from typing import Any

from aborist.compress import unpack_chunk
from aborist.store import connect

try:
    from aborist.wikitext import to_base as _wikitext_to_base
except ImportError:  # pragma: no cover
    _wikitext_to_base = None


def _normalize(s: str) -> str:
    """Same as aborist.qa.verify._normalize — kept local to avoid an
    import cycle and to make the diagnostic self-contained."""
    s = unicodedata.normalize("NFC", s)
    s = " ".join(s.split())
    return s.lower()


_INTERIOR_ELISION_MIN_PREFIX = 12
_INTERIOR_ELISION_MIN_SUFFIX = 20
_INTERIOR_ELISION_MAX_ASIDE = 250


def _try_synthetic_elision(
    nspan: str, norm_base_ctx: str
) -> dict[str, Any] | None:
    """Probe whether the model wrote ``[...]`` inside a quoted span.

    Distinct from ``interior_elision`` (model dropped a real ``(...)``
    aside that source carries) — synthetic elision means the model
    INSERTED a literal ``[...]`` ellipsis marker between fragments of
    its quote, signaling that it itself skipped content. The substring
    test fails because ``[...]`` isn't in source.

    Per the corrected verifier rule (fox 2026-04-30): ``[...]`` inside
    ``"..."`` is a quote-integrity failure. The verifier rejects (binary)
    & this sidecar diagnoses the prefix/suffix split so an operator can
    judge whether the elided segment was benign.

    Conservative: only fires when literal ``[...]`` appears in the span
    AND the same string is absent from source (so a Wikipedia article
    that genuinely contains ``[...]`` won't false-positive).
    """
    if "[...]" not in nspan or "[...]" in norm_base_ctx:
        return None
    idx = nspan.index("[...]")
    prefix = nspan[:idx].strip()
    suffix = nspan[idx + len("[...]"):].strip()
    prefix_in = bool(prefix) and prefix in norm_base_ctx
    suffix_in = bool(suffix) and suffix in norm_base_ctx
    return {
        "diagnosis": "synthetic_elision_inside_quote",
        "elision_marker": "[...]",
        "prefix_chars": len(prefix),
        "suffix_chars": len(suffix),
        "prefix_in_source": prefix_in,
        "suffix_in_source": suffix_in,
    }


def _try_interior_elision(
    nspan: str, norm_base_ctx: str
) -> dict[str, Any] | None:
    """Probe whether the span is source minus a single ``(...)`` aside.

    Pattern: source has ``A + (X) + B``; model wrote ``A + B``. Common on
    Wikipedia leads where parentheticals interrupt the prose flow
    (e.g. ``"Clark Joseph Kent (middle name is also Jerome ...) is a
    fictional character..."`` → model elides the parenthetical for
    readability).

    Walk every ``(`` in base. For each open paren at position P:
    longest k where ``base[:P]`` ends with ``nspan[:k]`` is the model's
    prefix; the span tail (≥ ``_INTERIOR_ELISION_MIN_SUFFIX`` chars)
    must then match the source after the close paren. First paren that
    satisfies both checks wins. Returns ``None`` if no paren in base
    matches the pattern — caller falls through to ``trailing_artifact``
    / paraphrase detection.

    Conservative: BOTH a meaningful prefix AND a meaningful suffix must
    match. A model that drops the aside AND rewords the rest will fail
    the suffix check & fall through to paraphrase, where it belongs.

    Distinct from ``trailing_artifact`` (model APPENDED a parenthetical
    that's not in source) and from ``paraphrase`` (token-coverage but
    different sequence). Interior_elision is "model dropped one aside
    that source carries" — content faithful, formatting condensed.
    """
    if len(nspan) < _INTERIOR_ELISION_MIN_PREFIX + _INTERIOR_ELISION_MIN_SUFFIX:
        return None

    pos = 0
    while True:
        paren_open = norm_base_ctx.find("(", pos)
        if paren_open < 0:
            return None
        pos = paren_open + 1
        # base[:paren_open] must end with span[:k] for some k. Trim a
        # trailing space so `Foo (aside)` matches when span has no
        # trailing space after `Foo`.
        b_before = norm_base_ctx[:paren_open].rstrip(" ")
        if len(b_before) < _INTERIOR_ELISION_MIN_PREFIX:
            continue
        max_k = min(len(b_before), len(nspan) - _INTERIOR_ELISION_MIN_SUFFIX)
        if max_k < _INTERIOR_ELISION_MIN_PREFIX:
            continue
        # Longest k ≤ max_k where b_before.endswith(nspan[:k]).
        # Linear sweep from longest down; first hit wins.
        found_k = 0
        for k in range(max_k, _INTERIOR_ELISION_MIN_PREFIX - 1, -1):
            if b_before.endswith(nspan[:k]):
                found_k = k
                break
        if found_k == 0:
            continue
        paren_close = norm_base_ctx.find(")", paren_open + 1)
        if paren_close < 0 or paren_close - paren_open > _INTERIOR_ELISION_MAX_ASIDE:
            continue
        aside = norm_base_ctx[paren_open + 1 : paren_close]
        span_tail = nspan[found_k:].lstrip()
        if len(span_tail) < _INTERIOR_ELISION_MIN_SUFFIX:
            continue
        src_after = norm_base_ctx[paren_close + 1 : paren_close + 1 + len(span_tail) + 16]
        src_after = src_after.lstrip()
        if not src_after.startswith(span_tail):
            continue
        return {
            "diagnosis": "interior_elision",
            "matched_prefix_chars": found_k,
            "matched_suffix_chars": len(span_tail),
            "dropped_aside": aside.strip(),
        }


def _classify_span(span: str, norm_base_ctx: str, norm_raw_ctx: str) -> dict[str, Any]:
    """Classify one unverified span against the canonical context.

    Returns one of:

    - ``verbatim_in_base``: substring match (verifier should have caught
      this — likely a verifier or canonicalization bug).
    - ``verbatim_in_raw_only``: matches raw wikitext but not the base
      form (wikitext-strip edge case).
    - ``synthetic_elision_inside_quote``: model wrote a literal
      ``"..."`` span containing ``[...]`` between fragments. The
      ``[...]`` marker isn't in source — the model itself signaled it
      skipped content while claiming verbatim citation. Quote-integrity
      failure. Sidecar reports prefix/suffix presence so operator can
      judge whether the elided middle is benign.
    - ``interior_elision``: source has ``A + (aside) + B``, model wrote
      ``A + B``. Content faithful to the main thread; the parenthetical
      aside got dropped for prose flow. Distinct from trailing_artifact
      (which is model-APPENDED text) and paraphrase (different sequence).
    - ``trailing_artifact``: a prefix of length >= 60 chars matches; the
      tail (often a model-added citation like ``(Source: ...)``) doesn't.
    - ``paraphrase``: query tokens (>4 chars) all present in base context
      but not in this sequence — model rewrote source content.
    - ``partial_paraphrase``: some query tokens present, others missing.
    - ``no_overlap``: most query tokens missing — likely full invention.
    """
    nspan = _normalize(span)
    if nspan in norm_base_ctx:
        return {"diagnosis": "verbatim_in_base"}
    if nspan in norm_raw_ctx:
        return {"diagnosis": "verbatim_in_raw_only"}

    # Trailing-artifact probe: sweep prefix lengths from longest down,
    # find the longest prefix that DOES match. If >= 60 chars matched,
    # the tail is the artifact.
    longest_prefix_match = 0
    for plen in (len(nspan) - 5, 200, 150, 120, 100, 80, 60):
        if plen <= 0 or plen > len(nspan):
            continue
        if nspan[:plen] in norm_base_ctx:
            longest_prefix_match = plen
            break

    # Interior_elision probe runs BEFORE trailing_artifact: if the span
    # is source-minus-one-parenthetical, that's a more specific finding
    # than "tail doesn't match." Falls through if no prefix+`(...)`+suffix
    # pattern matches in source.
    # Synthetic-elision probe runs FIRST: if the model wrote a literal
    # `[...]` between two fragments, that's a more specific finding
    # than "the whole span doesn't substring-match." Falls through if
    # no `[...]` marker is in the span.
    synthetic = _try_synthetic_elision(nspan, norm_base_ctx)
    if synthetic is not None:
        synthetic["repair"] = _repair_for_synthetic_elision(span, synthetic)
        return synthetic

    elision = _try_interior_elision(nspan, norm_base_ctx)
    if elision is not None:
        elision["repair"] = _repair_for_interior_elision(span, elision, norm_base_ctx)
        return elision

    if longest_prefix_match >= 60:
        tail = span[longest_prefix_match:]
        out = {
            "diagnosis": "trailing_artifact",
            "matched_prefix_chars": longest_prefix_match,
            "trailing_artifact": tail.strip(),
        }
        out["repair"] = _repair_for_trailing_artifact(span, out)
        return out

    # Token coverage: how many >4-char tokens from the span appear at all
    # in the base context.
    tokens = [t for t in nspan.split() if len(t) > 4]
    if not tokens:
        return {
            "diagnosis": "no_overlap",
            "tokens_checked": 0,
            "repair": {"action": "remove_claim", "reason": "no_topical_overlap"},
        }
    present = [t for t in tokens if t in norm_base_ctx]
    coverage = len(present) / len(tokens)
    counts = {t: norm_base_ctx.count(t) for t in present}

    if coverage >= 0.85:
        return {
            "diagnosis": "paraphrase",
            "token_coverage": round(coverage, 2),
            "token_counts": counts,
            "repair": {
                "action": "downgrade_to_paraphrase",
                "reason": "high_token_coverage_no_verbatim_match",
            },
        }
    if coverage >= 0.4:
        return {
            "diagnosis": "partial_paraphrase",
            "token_coverage": round(coverage, 2),
            "token_counts": counts,
            "missing_tokens": [t for t in tokens if t not in norm_base_ctx],
            "repair": {
                "action": "split_or_remove",
                "reason": "partial_token_coverage",
            },
        }
    return {
        "diagnosis": "no_overlap",
        "token_coverage": round(coverage, 2),
        "tokens_checked": len(tokens),
        "tokens_present": len(present),
        "repair": {
            "action": "remove_claim",
            "reason": "low_token_coverage_likely_invention",
        },
    }


# --------------------------------------------------------------- repair


def _repair_for_synthetic_elision(span: str, diag: dict[str, Any]) -> dict[str, Any]:
    """Repair plan for ``"prefix [...] suffix"``: split into two quotes
    if both halves landed in source, otherwise downgrade to paraphrase
    or remove. The actual repair is up to the caller — sidecar emits
    the suggestion only.
    """
    if diag.get("prefix_in_source") and diag.get("suffix_in_source"):
        prefix = span.split("[...]", 1)[0].strip()
        suffix = span.split("[...]", 1)[1].strip() if "[...]" in span else ""
        return {
            "action": "split_into_two_quotes",
            "reason": "both_halves_verbatim_in_source",
            "quotes": [prefix, suffix],
        }
    if diag.get("prefix_in_source") or diag.get("suffix_in_source"):
        return {
            "action": "trim_to_verified_half",
            "reason": "only_one_half_verbatim_in_source",
        }
    return {
        "action": "remove_claim",
        "reason": "neither_half_verbatim_in_source",
    }


def _repair_for_interior_elision(
    span: str, diag: dict[str, Any], norm_base_ctx: str
) -> dict[str, Any]:
    """Repair plan for ``A + (aside) + B`` where source carries the aside.
    Suggested fix: include the parenthetical so the quote becomes
    verbatim against source. Returns the suggested verbatim form when
    we can locate it; otherwise the abstract action.
    """
    aside = diag.get("dropped_aside", "").strip()
    if aside:
        # The verbatim form is A + " (" + aside + ") " + B. We can give
        # the operator a starting point even if exact whitespace varies.
        return {
            "action": "include_aside_for_verbatim",
            "reason": "model_dropped_parenthetical_source_carries",
            "aside_to_restore": aside,
        }
    return {"action": "include_aside_for_verbatim"}


def _repair_for_trailing_artifact(span: str, diag: dict[str, Any]) -> dict[str, Any]:
    """Repair plan for ``<verbatim_prefix> + <model-appended tail>``:
    drop the tail. Returns the trimmed span the operator should use
    instead.
    """
    matched = diag.get("matched_prefix_chars", 0)
    if matched > 0:
        return {
            "action": "trim_trailing_artifact",
            "reason": "model_appended_text_not_in_source",
            "kept_prefix": span[:matched].rstrip(),
        }
    return {"action": "trim_trailing_artifact"}


# ---------------------------------------------------------------- deflection signal


# Stopword set for question/answer token overlap. Mirrors the title-search
# stopwords in aborist.qa.query._TITLE_STOPWORDS but kept local so this
# sidecar never reaches into the verifier's machinery for soft computations.
_DEFLECTION_STOPWORDS = frozenset(
    """
    the a an is are was were be been being of to in on at for with by from
    as about into through during and or but not no nor so yet too very also just
    what who where when why how which this that these those such i you he she
    it we they me him her us them do does did have has had can could should
    would will may might
    tell show describe explain summarize say give list find make please
    name names called known does did can could would all there
    """.split()
)


def _stem_for_deflection(t: str) -> str:
    """Mirror of aborist.qa.query._stem_token_for_match: strip trailing
    `'s` (possessive) then trailing `s` on tokens >4 chars (skip
    `ss`-enders). Keeps 4-char words like 'mars' intact while
    collapsing 'mars's' → 'mars' and 'rivers' → 'river'."""
    if t.endswith("'s") and len(t) > 3:
        t = t[:-2]
    if len(t) > 4 and t.endswith("s") and not t.endswith("ss"):
        t = t[:-1]
    return t


def _content_tokens_for_deflection(text: str) -> set[str]:
    """Lower-cased content tokens (≥3 chars, non-stopword) for soft
    question/answer overlap analysis.

    Lower minimum length than the verifier's ``_content_tokens`` (≥4)
    so that 3-char proper-noun stems ("amd", "bsd", "fox") still
    register on the deflection signal — the cost of false-positives
    is just a soft sidecar advisory, not a verifier promotion."""
    return set(_content_tokens_in_order(text))


def _content_tokens_in_order(text: str) -> list[str]:
    """Same content-token extraction as ``_content_tokens_for_deflection``
    but preserves source order. Used by the subject-anchor heuristic
    where the LAST content token is treated as the question's primary
    subject ('who is a benevolent dictator for life for mars?' → 'mars')."""
    import re
    out: list[str] = []
    for raw in re.findall(r"[A-Za-z][A-Za-z'\-]+", text):
        t = _stem_for_deflection(raw.lower())
        if len(t) < 3 or t in _DEFLECTION_STOPWORDS:
            continue
        out.append(t)
    return out


# Question-shape leaders that *structurally* don't expect the answer
# to echo the question's subject — date/count/cause shapes where the
# answer is a year, a number, or a cause-explanation that may or may
# not re-mention the topic. Bench evidence (2026-05-01, 65-question
# sweep, n=3): subject-anchor heuristic produced false-positive
# deflections on 4 of 7 deflecting questions, all of these shapes:
# 'when did the soviet union dissolve?' (answer '1991'),
# 'what year did the berlin wall fall?' ('1989'),
# 'how many wives did henry the eighth have?' ('six'),
# 'what year does our cold fusion breakthrough happen?' ('1989').
# For these shapes, suppress the subject-anchor check entirely —
# fall back to overlap-ratio and accept that some short numeric
# answers will read as zero-overlap deflection (a known noisy
# signal we won't pretend is more reliable than it is).
_DEFLECTION_NUMERIC_OR_CAUSE_LEADERS = (
    "when ",
    "what year ",
    "what date ",
    "what time ",
    "how many ",
    "how much ",
    "how long ",
    "how old ",
    "why ",
)


def _question_is_numeric_or_cause_shape(question_text: str) -> bool:
    """Returns True for shapes whose answer doesn't structurally
    echo the question's subject (date/count/cause)."""
    q = (question_text or "").strip().lower()
    return any(q.startswith(lead) for lead in _DEFLECTION_NUMERIC_OR_CAUSE_LEADERS)


def diagnose_title_relevance(
    claim_text: str,
    cited_source_titles: list[str],
) -> dict[str, Any]:
    """Soft signal: do the cited chunk's source titles overlap the
    claim's content tokens?

    Empirically observed 2026-05-01 on 'explain spin glass modeling,
    tensors?' — verifier returned STRICT 1/1 on a claim 'Spin glass
    modeling involves... mathematical tools such as tensors' cited
    to a chunk from the *Quantum chromodynamics* article. The claim
    has zero stemmed-token overlap with the source title 'Quantum
    chromodynamics'. Token-coverage check inside the chunk passed
    accidentally on shared physics vocabulary (spin/model/etc.), but
    the article isn't about spin glasses.

    This sidecar surfaces the mismatch without modifying the binary
    verifier output. ``kind`` of ``TITLE_MISMATCH`` means the cited
    source's topic is structurally unrelated to the claim's subject;
    the claim is likely a retrieval-driven hallucination grounded
    against an incidentally-overlapping chunk.

    Returns:
        {
            "kind": "title_match" | "title_mismatch" | "no_claim_tokens" | "no_titles",
            "claim_tokens": [...],
            "title_tokens": [...],   # union across all cited titles
            "overlap": [...],
        }

    Sidecar-only — never enters the binary verifier output. Title
    underscores in titles ('Quantum_chromodynamics' / 'Lois_Lane')
    are normalized to spaces before tokenization."""
    claim_set = set(_content_tokens_in_order(claim_text or ""))
    if not claim_set:
        return {
            "kind": "no_claim_tokens",
            "claim_tokens": [],
            "title_tokens": [],
            "overlap": [],
        }
    if not cited_source_titles:
        return {
            "kind": "no_titles",
            "claim_tokens": sorted(claim_set),
            "title_tokens": [],
            "overlap": [],
        }
    title_set: set[str] = set()
    for t in cited_source_titles:
        title_set |= set(_content_tokens_in_order((t or "").replace("_", " ")))
    overlap = claim_set & title_set
    return {
        "kind": "title_match" if overlap else "title_mismatch",
        "claim_tokens": sorted(claim_set),
        "title_tokens": sorted(title_set),
        "overlap": sorted(overlap),
    }


# Cue tokens that flag a question as carrying poetic / metaphorical
# framing. Empirically motivated by the 2026-05-02 emergent log entry
# 'How can a swallowtail butterfly, gracefully fluttering amidst the
# rockiest terrain, remain undeterred by the upbraiding winds...' —
# the model traded the metaphor for literal Macleay's-Swallowtail
# taxonomic facts, the warrant passed (the literal anchor IS in cited
# spans), but the user's metaphorical question was never answered.
# Sidecar only — flags candidates for human review; the verifier's
# binary output stays authoritative.
#
# Cues are DERIVED from the system English wordlist
# (/usr/share/dict/words), not hand-curated. Same Phase-2 lesson the
# concept_relations layer learned: hand-typed exception lists scale
# poorly. The wordlist already encodes "is this a real English word";
# adverbiality / participle-ness / superlativity comes from the
# morphological-stem-test against the wordlist. Pure derivation.
#
# Cue families:
#   - Closed-class prepositions that almost always carry metaphorical
#     setting (amidst, despite, beneath, ...). Tiny fixed list — these
#     are a closed grammatical class, not a vocabulary tail.
#   - -ly adverbs whose stem (or stem+e, or stem-i+y) is in the
#     wordlist (gracefully → graceful, truly → true, happily → happy).
#     The stem test filters -ly NOUNS like butterfly (stem 'butterf'
#     not in wordlist) without a hand-curated noun list.
#   - -ing present-participle adjectives whose stem (or stem+e, or
#     consonant-de-doubled stem) is a verb in the wordlist
#     (fluttering → flutter, running → run, upbraiding → upbraid).
#   - -est superlatives whose stem (or stem+e, or stem-i+y) is an
#     adjective in the wordlist (rockiest → rocky, safest → safe,
#     longest → long).

_METAPHOR_PREPOSITION_CUES = frozenset({
    "amidst", "amid", "despite", "against", "beneath", "amongst",
    "throughout", "alongside", "betwixt", "atop",
})

# Default wordlist sources. The standard Unix `/usr/share/dict/words`
# (and its Debian split into american-english + british-english) is
# the morphological substrate. Both spelling regions get unioned so a
# British speaker's metaphor cues (`colouredly`, `realisingly`) land
# alongside American (`coloredly`, `realizingly`).
#
# Operators can supplement with corpus / domain-specific vocabulary
# via the ``ABORIST_METAPHOR_DICTS`` environment variable (colon-
# separated list of paths, one word per line) or the
# ``register_metaphor_dictionary(path)`` helper below. A forest with
# its own jargon (aviation, medical, legal, dynastic) registers the
# domain wordlist once and the suffix tests pick up domain-specific
# adverbs / participles automatically. Same Phase-2 derivation rule
# applies — `aerodynamically` → stem `aerodynamic` lookup → if in the
# union, classified as adverbial.
_DEFAULT_DICT_PATHS = (
    Path("/usr/share/dict/words"),                # OS default symlink
    Path("/usr/share/dict/american-english"),     # Debian split
    Path("/usr/share/dict/british-english"),      # Debian split
)

_english_wordlist_cache: frozenset[str] | None = None
_extra_dict_paths: list[Path] = []


def register_metaphor_dictionary(path: str | Path) -> None:
    """Register a supplemental wordlist (one word per line, lowercase
    or mixed-case, blank lines OK). Words union into the default set
    on next ``_english_wordlist()`` call. Use for corpus / domain
    vocabulary so the metaphor sidecar's morphological tests can pick
    up domain-specific stems.

    Programmatic equivalent of the ``ABORIST_METAPHOR_DICTS`` env var.
    Calling this invalidates the cache so subsequent lookups re-build
    the union. Idempotent — re-registering the same path is a no-op.
    """
    global _english_wordlist_cache
    p = Path(path)
    if p not in _extra_dict_paths:
        _extra_dict_paths.append(p)
        _english_wordlist_cache = None


def _load_dict(path: Path) -> set[str]:
    """Read ``path`` if it exists; return lowercase token set."""
    if not path.exists():
        return set()
    try:
        text = path.read_text(encoding="utf-8", errors="replace")
    except OSError:
        return set()
    return {w.strip().lower() for w in text.splitlines() if w.strip()}


def _english_wordlist() -> frozenset[str]:
    """Union of the default Unix wordlists + supplemental dictionaries.

    Sources (in order, all unioned):
      1. ``/usr/share/dict/words`` (OS default)
      2. ``/usr/share/dict/american-english`` (Debian split)
      3. ``/usr/share/dict/british-english`` (Debian split)
      4. Paths in ``ABORIST_METAPHOR_DICTS`` (colon-separated env var)
      5. Paths registered via ``register_metaphor_dictionary()``

    Cached on first call. Graceful degradation: missing paths skip
    silently; empty union returns an empty frozenset and the suffix
    tests all return False (sidecar returns ``no_signal`` quietly).
    """
    global _english_wordlist_cache
    if _english_wordlist_cache is not None:
        return _english_wordlist_cache
    union: set[str] = set()
    for p in _DEFAULT_DICT_PATHS:
        union |= _load_dict(p)
    env_paths = os.environ.get("ABORIST_METAPHOR_DICTS", "")
    if env_paths:
        for raw in env_paths.split(":"):
            raw = raw.strip()
            if raw:
                union |= _load_dict(Path(raw))
    for p in _extra_dict_paths:
        union |= _load_dict(p)
    _english_wordlist_cache = frozenset(union)
    return _english_wordlist_cache


def _is_adverbial_ly(word: str) -> bool:
    """True if ``word`` is an -ly adverb whose stem is a real word."""
    if len(word) < 5 or not word.endswith("ly"):
        return False
    dictionary = _english_wordlist()
    if not dictionary:
        return False
    stem = word[:-2]
    if len(stem) < 3:
        return False
    if stem in dictionary:                            # gracefully → graceful
        return True
    if stem + "e" in dictionary:                      # truly → true
        return True
    if stem.endswith("i") and stem[:-1] + "y" in dictionary:  # happily → happy
        return True
    return False


def _is_present_participle(word: str) -> bool:
    """True if ``word`` is an -ing form of a verb in the wordlist."""
    if len(word) < 6 or not word.endswith("ing"):
        return False
    dictionary = _english_wordlist()
    if not dictionary:
        return False
    stem = word[:-3]
    if len(stem) < 3:
        return False
    if stem in dictionary:                            # fluttering → flutter
        return True
    if stem + "e" in dictionary:                      # making → make
        return True
    if len(stem) >= 3 and stem[-1] == stem[-2] and stem[:-1] in dictionary:
        return True                                   # running → run
    return False


def _is_superlative_est(word: str) -> bool:
    """True if ``word`` is an -est superlative of an adjective in the wordlist."""
    if len(word) < 6 or not word.endswith("est"):
        return False
    dictionary = _english_wordlist()
    if not dictionary:
        return False
    stem = word[:-3]
    if len(stem) < 3:
        return False
    if stem in dictionary:                            # longest → long
        return True
    if stem + "e" in dictionary:                      # safest → safe
        return True
    if stem.endswith("i") and stem[:-1] + "y" in dictionary:  # rockiest → rocky
        return True
    return False


def _extract_metaphor_cues(text: str) -> list[str]:
    """Extract poetic / metaphorical cue tokens from ``text``.

    Returns the cue tokens in lowercase, sorted unique. Three suffix
    classes (-ly / -ing / -est) plus a tiny fixed prepositional set.
    Suffix tests derive from the system wordlist via stem-existence
    rules — no hand-curated noun-exception lists.
    """
    tokens = _content_tokens_in_order(text or "")
    cues: set[str] = set()
    for t in tokens:
        if t in _METAPHOR_PREPOSITION_CUES:
            cues.add(t)
        elif _is_adverbial_ly(t):
            cues.add(t)
        elif _is_superlative_est(t):
            cues.add(t)
        elif _is_present_participle(t):
            cues.add(t)
    return sorted(cues)


def diagnose_metaphor_deflection(
    question_text: str, answer_text: str
) -> dict[str, Any]:
    """Soft signal: did the question carry metaphorical framing the
    answer ignored?

    Empirically observed 2026-05-02 on 'How can a swallowtail butterfly,
    gracefully fluttering amidst the rockiest terrain, remain
    undeterred by the upbraiding winds that seem to challenge its
    delicate flight?' — model traded the metaphor for literal Macleay's
    Swallowtail taxonomic facts. Warrant passed (the literal noun
    anchor IS in cited spans), DEFLECTION_DETECTED didn't fire (the
    last content token did echo in the answer), but the user's
    metaphorical question was never answered. Honest gap; structural
    catch requires NLI-grade semantics.

    Detection rule (purely lexical):

      1. Extract metaphor cue tokens from the question (adverbs,
         poetic -ing adjectives, superlatives, prepositional cues —
         see ``_extract_metaphor_cues``).
      2. Extract content tokens from the answer.
      3. ``metaphor_deflection`` when:
           question has >= 3 cue tokens, AND
           answer's content tokens overlap zero of them.

      Otherwise: ``no_signal`` (either too few cues to decide, or
      answer engaged with at least one cue token).

    Returns:
        {
            "kind": "metaphor_deflection" | "no_signal",
            "cue_tokens": [...],          # cues found in question
            "answer_overlap": [...],      # cues that appeared in answer
            "answer_overlap_count": int,
            "cue_count": int,
        }

    Sidecar only — never enters the binary verifier output. The
    threshold (3 cues + 0 overlap) is deliberately conservative; the
    smell triggers only when the question is *strongly* poetic and
    the answer is *purely* literal.
    """
    cues = _extract_metaphor_cues(question_text)
    answer_tokens = set(_content_tokens_in_order(answer_text or ""))
    overlap = sorted(t for t in cues if t in answer_tokens)
    if len(cues) >= 3 and not overlap:
        kind = "metaphor_deflection"
    else:
        kind = "no_signal"
    return {
        "kind": kind,
        "cue_tokens": cues,
        "answer_overlap": overlap,
        "answer_overlap_count": len(overlap),
        "cue_count": len(cues),
    }


def diagnose_deflection(question_text: str, answer_text: str) -> dict[str, Any]:
    """Soft signal: did the answer change topic?

    Empirically observed 2026-04-30 on 'who is a benevolent dictator
    for life for mars?' — JSON mode grounded its claims (STRICT) but
    the answer mentioned only Guido van Rossum and Python, never Mars.
    Verifier did its job (claims are grounded); the user's question
    wasn't answered. A real failure mode for adversarial-premise
    questions where the model deflects rather than refuses.

    Detection rule:

    1. **Numeric/cause-shape suppression**: questions starting with
       ``when``/``what year``/``how many``/``how much``/``why``/etc.
       have answers that don't structurally echo the subject (a year,
       a count, a cause-explanation). For these, skip the subject-
       anchor check and use overlap-ratio only — bench finding
       2026-05-01: subject-anchor produced 4 false-positive
       deflections out of 7 across these shapes.
    2. **Subject anchor (default)**: the LAST content token in the
       question (after stopword strip) is a strong heuristic for the
       question's primary subject ('mars' in the BDFL case, 'beatles'
       in 'who are the members of the beatles?'). If the subject
       anchor doesn't appear in the answer → ``deflection``.
    3. Otherwise, fall back to overlap-ratio: 1.0 → ``on_topic``,
       partial → ``partial_overlap``, 0 → ``deflection``.

    Subject-anchor catches BDFL-Mars (3/4 generic vocab overlap with
    missing subject) but mis-fires on numeric/cause shapes — the
    leader-suppression closes that gap.

    Returns:
        {
            "kind": "deflection" | "partial_overlap" | "on_topic" | "no_question_tokens",
            "question_tokens": [...],
            "answer_tokens": [...],
            "overlap": [...],
            "overlap_ratio": float,  # |overlap| / |question_tokens|
            "subject_anchor": str | None,
            "subject_in_answer": bool,
            "shape_suppressed": bool,  # True if subject-anchor was
                                       # skipped due to numeric/cause shape
        }

    Sidecar-only — never enters the binary verifier output."""
    q_in_order = _content_tokens_in_order(question_text or "")
    qtok = set(q_in_order)
    atok = _content_tokens_for_deflection(answer_text or "")
    if not qtok:
        return {
            "kind": "no_question_tokens",
            "question_tokens": [],
            "answer_tokens": sorted(atok),
            "overlap": [],
            "overlap_ratio": 0.0,
            "subject_anchor": None,
            "subject_in_answer": False,
            "shape_suppressed": False,
        }
    shape_suppressed = _question_is_numeric_or_cause_shape(question_text)
    subject_anchor = q_in_order[-1] if q_in_order else None
    subject_in_answer = subject_anchor in atok if subject_anchor else False
    overlap = qtok & atok
    if not shape_suppressed and subject_anchor and not subject_in_answer:
        # Subject token missing dominates — even if 3/4 generic vocab
        # overlapped, the actual topic isn't in the answer.
        kind = "deflection"
    elif not overlap:
        kind = "deflection"
    elif len(overlap) < len(qtok):
        kind = "partial_overlap"
    else:
        kind = "on_topic"
    return {
        "kind": kind,
        "question_tokens": sorted(qtok),
        "answer_tokens": sorted(atok),
        "overlap": sorted(overlap),
        "overlap_ratio": len(overlap) / len(qtok),
        "subject_anchor": subject_anchor,
        "subject_in_answer": subject_in_answer,
        "shape_suppressed": shape_suppressed,
    }


def inspect_cache_key(
    cache_key: str,
    *,
    qa_db: Path,
    shards_dir: Path | None,
    single_db: Path | None = None,
) -> dict[str, Any]:
    """Pull the cache record + sources + classify each unverified span.

    Returns a dict containing:
        - record: the providence_cache row (subset)
        - sources: per-source size + chunk count
        - context: char counts for raw and base forms
        - unverified: per-span diagnosis (list)
    """
    conn = connect(qa_db)
    try:
        row = conn.execute(
            "SELECT cache_key, source_root, document_uri, question_text, "
            " answer_text, audit_mode, n_quotes, n_verified, "
            " unverified_quotes, verifier_method, merkle_proof, "
            " falsification_state, created_at "
            "FROM providence_cache WHERE cache_key = ?",
            (cache_key,),
        ).fetchone()
    finally:
        conn.close()
    if row is None:
        return {"status": "not_found", "cache_key": cache_key}

    proof = json.loads(row["merkle_proof"]) if row["merkle_proof"] else {}
    sources = proof.get("sources") or []
    unverified_list = (
        json.loads(row["unverified_quotes"]) if row["unverified_quotes"] else []
    )

    parts_raw: list[str] = []
    parts_base: list[str] = []
    src_summary: list[dict[str, Any]] = []
    for s in sources:
        shard_path = (
            shards_dir / s["shard"] if (shards_dir and s.get("shard"))
            else single_db
        )
        if shard_path is None:
            continue
        sc = sqlite3.connect(str(shard_path))
        sc.row_factory = sqlite3.Row
        try:
            rows = sc.execute(
                "SELECT idx, content FROM chunks "
                "WHERE document_root = ? AND tier = 'hot' ORDER BY idx",
                (s["document_root"],),
            ).fetchall()
        finally:
            sc.close()
        text = "\n\n".join(unpack_chunk(r["content"]) or "" for r in rows)
        parts_raw.append(text)
        if _wikitext_to_base is not None:
            parts_base.append(_wikitext_to_base(text))
        else:
            parts_base.append(text)
        src_summary.append(
            {
                "document_uri": s.get("document_uri"),
                "title": s.get("title"),
                "shard": s.get("shard"),
                "chunk_count": len(rows),
                "raw_chars": len(text),
            }
        )

    full_raw = "\n\n".join(parts_raw)
    full_base = "\n\n".join(parts_base)
    n_raw = _normalize(full_raw)
    n_base = _normalize(full_base)

    diagnoses: list[dict[str, Any]] = []
    for q in unverified_list:
        d = _classify_span(q, norm_base_ctx=n_base, norm_raw_ctx=n_raw)
        diagnoses.append({"span": q, **d})

    # Soft deflection signal — see diagnose_deflection. Computed even
    # for STRICT records: a STRICT record where the answer never
    # mentions the question's subject is the Mars-BDFL pattern (model
    # deflected to a related grounded fact rather than refusing or
    # answering). Sidecar-only, never feeds back into providence.
    deflection = diagnose_deflection(row["question_text"], row["answer_text"])

    return {
        "status": "ok",
        "record": {
            "cache_key": row["cache_key"],
            "question_text": row["question_text"],
            "audit_mode": row["audit_mode"],
            "n_quotes": row["n_quotes"],
            "n_verified": row["n_verified"],
            "verifier_method": row["verifier_method"],
            "falsification_state": row["falsification_state"],
            "created_at": row["created_at"],
        },
        "sources": src_summary,
        "context": {
            "raw_chars": len(full_raw),
            "base_chars": len(full_base),
            "wikitext_strip_active": _wikitext_to_base is not None,
        },
        "unverified": diagnoses,
        "deflection": deflection,
    }