arborist/aborist/qa/verify.py

"""Post-LLM faithfulness check: did the answer ground its claims in context?

Three layered strategies, tried in order. The first one that finds evidence
classifies the answer. `verifier_method` on the result records which path
fired so the audit chain stays diagnostic.

    1. quote     model wrapped claims in double quotes per system prompt.
                 Strongest signal — explicit, verbatim, model-asserted.
    2. span      no quotes, but bullet/sentence-level lines from the answer
                 appear verbatim in context. Catches models that quote
                 inline without "..." marks.
    3. entity    no quotes and no span match, but multi-word proper-noun
                 phrases from the answer appear verbatim in context.
                 Catches the Wikipedia-infobox-to-prose case: the model
                 paraphrases structure so spans diverge, but every named
                 entity is intact and grounded.

Each strategy classifies into v9.8's audit-mode trichotomy (RAG-adapted
vocabulary; substrate calls UNGROUNDED "VISUAL"):

    STRICT      every evidence unit (>=1) verifies verbatim against context
    HYBRID      some verify, others do not  (mixed source / emergent)
    UNGROUNDED  no evidence, or none verify (purely emergent)

`unverified_quotes` (kept under that name for schema continuity) collects
spans the model produced that don't appear in any source — the
corpus-growth signal mined by `aborist emergent`.

Hard rule (CLAUDE.md "soft hash vs hard hash"): every check is a lexical
substring test under norm-v1 + lowercase canonicalization. No embeddings,
no semantic similarity, no fuzzy alignment. The contract is "this token
sequence either is or isn't in the context."

Wikitext context is run through ``aborist.wikitext.to_base`` before the
substring test. The corpus stores raw wikitext (so the link graph is
recoverable from any page), but the LLM produces clean prose. Without
the strip, every wikilink-carrying source paragraph compares as
"different surface form" and the verifier wrongly reports UNGROUNDED on
genuine source-grounded quotes. With the strip, paraphrases of *markup*
(``[[Cloud]]`` vs ``Cloud``) verify, while paraphrases of *prose* still
flag honestly. mwparserfromhell is an optional dep; if absent, the
strip is a no-op and verification falls back to today's behavior.
"""

from __future__ import annotations

import re
import unicodedata

try:
    from aborist.wikitext import to_base as _wikitext_to_base
except ImportError:  # pragma: no cover
    _wikitext_to_base = None


# Locate every double-quote character (ASCII or curly). Sequential
# pairing in extract_quotes() turns these into intentional (open, close)
# pairs: 1st & 2nd char, 3rd & 4th, etc. Pure-regex pairing fails on
# adjacent quote pairs like `"title" prose "quote"` — the regex captures
# `prose` as a "quoted span" because every `"` looks like both an opener
# and a closer to it.
_QUOTE_CHAR_RE = re.compile(r'["“”]')

# Bullet markers at line start: -, *, +, •, 1., 2), etc.
_BULLET_RE = re.compile(r'^\s*(?:[-*+•]|\d+[.)])\s+')

# Sentence boundary: punctuation + whitespace + capital letter.
_SENT_RE = re.compile(r'(?<=[.!?])\s+(?=[A-Z])')

# Multi-word capitalized phrase. Two or more whitespace-separated tokens,
# each starting with a capital letter (allowing initials like "A.", hyphens
# like "Carrie-Anne", and trailing lowercase like "Smith"). Matches "Keanu
# Reeves", "Thomas A. Anderson", "Carrie-Anne Moss", "Agent Smith". Skips
# single capitalized words to avoid sentence-starter false positives.
_PROPER_NOUN_RE = re.compile(
    # Token gap is non-newline whitespace so a phrase never crosses a
    # paragraph break. Caught a real case where "Joe Pantoliano\n\nThe
    # sources" matched as one phrase via \s+.
    r"\b[A-Z][A-Za-z'’\-]*(?:[ \t]+(?:[A-Z]\.|[A-Z][A-Za-z'’\-]+))+\b"
)

MIN_QUOTE_CHARS = 8
MIN_SPAN_CHARS = 12

# Entity-path policies. Entity-existence in source is circumstantial, not
# claim-level proof — the model could correctly name entities while making
# claims around them that came from training. These four policies span the
# trade-off: max-trust to no-trust.
#
#   strict     all entities verify → STRICT (legacy behavior; overclaims)
#   hybrid     any entity verifies → HYBRID (honest cap; safe default)
#   drop       skip entity path entirely → UNGROUNDED (most conservative)
#   proximity  STRICT only if N verified entities cluster within W chars
#              of each other in context (e.g. an infobox cast list).
#              Otherwise demotes to HYBRID/UNGROUNDED based on partial match.
ENTITY_POLICIES = ("strict", "hybrid", "drop", "proximity")
DEFAULT_ENTITY_POLICY = "proximity"

# Proximity tuning. N entities within W chars of each other.
DEFAULT_PROXIMITY_N = 3
DEFAULT_PROXIMITY_WINDOW = 300

# Framing phrases that aren't real claims even when they're in source. We
# strip these from the answer before span/entity extraction so they don't
# become noise in unverified_quotes.
_FRAMING_PREFIXES = (
    "based on the provided sources",
    "based on the provided source",
    "based on the source",
    "based on the document",
    "the sources do not",
    "the source does not",
    "the document does not",
    "according to the sources",
    "according to the source",
)


def _normalize(s: str) -> str:
    """norm-v1 + lowercase. Same canonicalization as chunk leaf hashing,
    plus case-folding so the verifier doesn't fail on capitalization drift
    between source prose and the model's quoted span."""
    s = unicodedata.normalize("NFC", s)
    s = " ".join(s.split())
    return s.lower()


def _is_framing(span: str) -> bool:
    n = _normalize(span)
    return any(n.startswith(p) for p in _FRAMING_PREFIXES)


def extract_quotes(answer_text: str) -> list[str]:
    """Pull double-quoted spans of length >= MIN_QUOTE_CHARS from `answer_text`.

    Sequential pairing: locate every double-quote character, then pair
    them as (1st, 2nd), (3rd, 4th), .... Each pair brackets one quoted
    span; text between consecutive pairs is the model's own framing
    prose (not captured). This is the correct model for adjacent quote
    pairs like `"title" prose "quote"` — naive regex matching paired
    the close of "title" with the open of "quote" and captured `prose`
    as a phantom quote, dragging classifications down to HYBRID
    incorrectly.
    """
    positions = [m.start() for m in _QUOTE_CHAR_RE.finditer(answer_text)]
    quotes: list[str] = []
    for i in range(0, len(positions) - 1, 2):
        span = answer_text[positions[i] + 1:positions[i + 1]]
        if len(span) >= MIN_QUOTE_CHARS:
            quotes.append(span)
    return quotes


def extract_claim_spans(answer_text: str) -> list[str]:
    """Strip bullet markers, split into sentences, drop framing prefixes.

    Returns each non-empty span of length >= MIN_SPAN_CHARS. These are the
    "claim units" the model wrote — each one we'll substring-test against
    context.
    """
    spans: list[str] = []
    for line in answer_text.splitlines():
        stripped = _BULLET_RE.sub("", line).strip()
        if not stripped:
            continue
        for sent in _SENT_RE.split(stripped):
            sent = sent.strip().rstrip(".:,;")
            if len(sent) < MIN_SPAN_CHARS:
                continue
            if _is_framing(sent):
                continue
            spans.append(sent)
    return spans


def extract_proper_nouns(answer_text: str) -> list[str]:
    """Pull multi-word capitalized phrases. Deduplicated, order preserved.

    Multi-word only — single capitalized words at sentence start are too
    noisy ("Based", "Now", "However"). Multi-word phrases like
    "Keanu Reeves" or "Thomas A. Anderson" are reliable proper-noun
    candidates and substring-test cleanly against source prose or
    structured wikitext.
    """
    seen: set[str] = set()
    out: list[str] = []
    for m in _PROPER_NOUN_RE.findall(answer_text):
        if m in seen:
            continue
        seen.add(m)
        out.append(m)
    return out


def _classify(verified: list[str], unverified: list[str]) -> str:
    if verified and not unverified:
        return "STRICT"
    if verified:
        return "HYBRID"
    return "UNGROUNDED"


def _has_entity_cluster(
    verified_entities: list[str],
    norm_ctx: str,
    n: int,
    window: int,
) -> bool:
    """Do at least `n` distinct verified entities appear within `window`
    chars of each other in `norm_ctx`? Crude proxy for "the source has a
    section about these entities" (cast list, infobox, roster) vs
    "the source incidentally mentions them in scattered prose."
    """
    if len(verified_entities) < n:
        return False
    positions: list[int] = []
    for e in verified_entities:
        idx = norm_ctx.find(_normalize(e))
        if idx >= 0:
            positions.append(idx)
    if len(positions) < n:
        return False
    positions.sort()
    for i in range(len(positions) - n + 1):
        if positions[i + n - 1] - positions[i] <= window:
            return True
    return False


def _check_each(items: list[str], norm_ctx: str) -> tuple[list[str], list[str]]:
    verified: list[str] = []
    unverified: list[str] = []
    for it in items:
        if _normalize(it) in norm_ctx:
            verified.append(it)
        else:
            unverified.append(it)
    return verified, unverified


def verify_quotes(
    answer_text: str,
    context: str,
    *,
    entity_policy: str = DEFAULT_ENTITY_POLICY,
    proximity_n: int = DEFAULT_PROXIMITY_N,
    proximity_window: int = DEFAULT_PROXIMITY_WINDOW,
) -> dict:
    """Classify an answer's grounding against its retrieved context.

    Tries quote → span → entity verification in sequence. The first
    strategy that finds evidence classifies the answer; later strategies
    don't run.

    `entity_policy` controls how the entity path classifies — see
    ENTITY_POLICIES. The quote and span paths are unaffected; they are
    explicit-claim evidence and always classify per the trichotomy.

    Returns:
        {
          "n_quotes":          int,    # evidence units extracted (any path)
          "n_verified":        int,    # of those, how many appear verbatim
          "audit_mode":        str,    # STRICT | HYBRID | UNGROUNDED
          "unverified_quotes": [str],  # spans we couldn't ground in context
          "verifier_method":   str,    # 'quote' | 'span' | 'entity' | 'none'
        }
    """
    if entity_policy not in ENTITY_POLICIES:
        raise ValueError(
            f"entity_policy must be one of {ENTITY_POLICIES}, got {entity_policy!r}"
        )

    # Wikitext markup → plain prose. Identity if mwparserfromhell isn't
    # installed (extras: pip install 'aborist[wikitext]').
    if _wikitext_to_base is not None:
        context = _wikitext_to_base(context)

    norm_ctx = _normalize(context)

    # Strategy 1: explicit double-quoted spans.
    quotes = extract_quotes(answer_text)
    if quotes:
        verified, unverified = _check_each(quotes, norm_ctx)
        return {
            "n_quotes": len(quotes),
            "n_verified": len(verified),
            "audit_mode": _classify(verified, unverified),
            "unverified_quotes": unverified,
            "verifier_method": "quote",
        }

    # Strategy 2: bullet/sentence spans verbatim in context.
    spans = extract_claim_spans(answer_text)
    if spans:
        verified, unverified = _check_each(spans, norm_ctx)
        if verified:
            return {
                "n_quotes": len(spans),
                "n_verified": len(verified),
                "audit_mode": _classify(verified, unverified),
                "unverified_quotes": unverified,
                "verifier_method": "span",
            }

    # Strategy 3: multi-word proper nouns (entity grounding) — gated by
    # `entity_policy`. Entity-existence is weaker proof than quote or span;
    # the operator chooses how much weight to give it.
    if entity_policy == "drop":
        # Skip entity path entirely. Falls through to UNGROUNDED/none.
        pass
    else:
        entities = extract_proper_nouns(answer_text)
        if entities:
            verified, unverified = _check_each(entities, norm_ctx)
            if verified:
                if entity_policy == "strict":
                    # Legacy behavior: all match → STRICT. Overclaims.
                    mode = _classify(verified, unverified)
                elif entity_policy == "hybrid":
                    # Cap at HYBRID. Honest middle: evidence exists, but
                    # entity-existence ≠ claim-existence.
                    mode = "HYBRID"
                else:  # proximity
                    cluster = _has_entity_cluster(
                        verified, norm_ctx, proximity_n, proximity_window
                    )
                    if cluster and not unverified:
                        mode = "STRICT"
                    elif cluster:
                        mode = "HYBRID"
                    elif verified:
                        mode = "HYBRID"
                    else:
                        mode = "UNGROUNDED"
                return {
                    "n_quotes": len(entities),
                    "n_verified": len(verified),
                    "audit_mode": mode,
                    "unverified_quotes": unverified,
                    "verifier_method": "entity",
                }

    # Nothing extracted, or nothing verified. Truly emergent.
    return {
        "n_quotes": 0,
        "n_verified": 0,
        "audit_mode": "UNGROUNDED",
        "unverified_quotes": [],
        "verifier_method": "none",
    }