diff --git a/Makefile b/Makefile index 06297f4..4c64ff2 100644 --- a/Makefile +++ b/Makefile @@ -155,17 +155,23 @@ burn-kindergarten: bootstrap ## bust providence rows < SECONDS old [SECONDS=3600 # Multi-source RAG query against the shard cluster. # Usage: make query Q="What is anarcho-capitalism?" QUERY_TOP_K ?= 8 -query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N]; sources picked across shards +# G0 / CTI — pointer-mode is the testing default. Library DEFAULT_POLICY +# stays "quote" so Python callers aren't surprised; the Makefile +# harness ships pointer-mode-on so `make query` exercises the new path +# end-to-end. Override with ANSWER_MODE=quote to test the legacy path, +# ANSWER_MODE= (empty) to use whatever DEFAULT_QUERY_POLICY says. +ANSWER_MODE ?= claim_lattice_pointer +query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N ANSWER_MODE=quote|claim_lattice_pointer]; pointer mode by default @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ - echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N]"; exit 2; \ + echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N ANSWER_MODE=quote|claim_lattice_pointer]"; exit 2; \ fi - $(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) "$(Q)" + $(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) "$(Q)" -query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1] +query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=...] @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ - echo "usage: make query-dry Q=\"your question\" [JSON=1 BURN=1]"; exit 2; \ + echo "usage: make query-dry Q=\"your question\" [JSON=1 BURN=1 ANSWER_MODE=quote|claim_lattice_pointer]"; exit 2; \ fi - $(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) "$(Q)" + $(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) "$(Q)" verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample $(ABORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N) diff --git a/aborist/cli.py b/aborist/cli.py index 797691a..b913438 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -288,6 +288,12 @@ def _cmd_ask(args: argparse.Namespace) -> int: if args.global_shards_dir else connect(args.db) ) + # Per-call policy override for --answer-mode. Other knobs flow from + # DEFAULT_POLICY. + from aborist.qa.runner import DEFAULT_POLICY as _DEFAULT_ASK_POLICY + call_policy = dict(_DEFAULT_ASK_POLICY) + if getattr(args, "answer_mode", None): + call_policy["answer_mode"] = args.answer_mode try: result = ask( conn, @@ -297,6 +303,7 @@ def _cmd_ask(args: argparse.Namespace) -> int: model_id=model, revision=revision, quantization=quantization, + policy=call_policy, ) finally: conn.close() @@ -343,12 +350,15 @@ def _cmd_query(args: argparse.Namespace) -> int: ) single_db = None if shards_dir else args.db - # Apply per-call policy overrides (question_dedup, repair) on top of - # the default. fidelity is a function-level kwarg, not in the policy. + # Apply per-call policy overrides (question_dedup, repair, answer_mode) + # on top of the default. fidelity is a function-level kwarg, not in + # the policy. from aborist.qa.query import DEFAULT_QUERY_POLICY call_policy = dict(DEFAULT_QUERY_POLICY) if getattr(args, "question_dedup", None): call_policy["question_dedup"] = args.question_dedup + if getattr(args, "answer_mode", None): + call_policy["answer_mode"] = args.answer_mode if getattr(args, "repair", False): # Mechanical-only repair when --repair is set; --repair-reprompts # adds the optional re-prompt tier on top. Both default off so @@ -2810,6 +2820,14 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="use StubClient — no network call", ) + ask_cmd.add_argument( + "--answer-mode", dest="answer_mode", default=None, + choices=["quote", "claim_lattice_pointer"], + help=( + "answer schema. See `query --answer-mode` for full semantics. " + "Default 'quote'; 'claim_lattice_pointer' enables quote-by-pointer." + ), + ) ask_cmd.set_defaults(func=_cmd_ask) query_cmd = sub.add_parser( @@ -2899,6 +2917,19 @@ def build_parser() -> argparse.ArgumentParser: "using only verbatim citations. Requires --repair." ), ) + query_cmd.add_argument( + "--answer-mode", dest="answer_mode", default=None, + choices=["quote", "claim_lattice_pointer"], + help=( + "answer schema. 'quote' (default): model writes prose with " + "verbatim quote spans inline. 'claim_lattice_pointer' (G0 " + "/ CTI quote-by-pointer): runtime builds a labeled evidence " + "map (E1, E2, …); model writes pointer-line prose ('Claim. " + "[E12]'); renderer interpolates literal spans. Synthetic-" + "elision-by-construction-impossible. No repair loop " + "(one-shot discipline)." + ), + ) query_cmd.set_defaults(func=_cmd_query) inspect_cmd = sub.add_parser( diff --git a/aborist/qa/dag.py b/aborist/qa/dag.py index 0c8ced2..1dd980a 100644 --- a/aborist/qa/dag.py +++ b/aborist/qa/dag.py @@ -106,6 +106,12 @@ def build_run_dag( n_verified: int, claim_statuses: list[dict] | None = None, lookup_path: str | None = None, + evidence_map_root: str | None = None, + answer_mode: str | None = None, + violations: list[dict] | None = None, + raw_answer_text: str | None = None, + parsed_lattice: list | None = None, + rendered_text: str | None = None, ) -> dict: """Return ``{"root": , "nodes": [, ], ...}``. @@ -113,6 +119,29 @@ def build_run_dag( deterministic — same inputs always produce the same root, byte-for- byte across machines (as long as the Merkle conventions stay pinned; they do, via ``aborist.merkle``). + + Two DAG shapes: + + - **Quote mode (default).** 7 stages — + ``question / retrieval / context / prompt / answer / verify / + final_label``. Triggered when ``evidence_map_root`` is None. + Backward-compatible with all run_dag_root values written by code + that pre-dates G0. + + - **Claim-lattice-pointer mode (G0 / CTI).** 9 stages — + ``question / retrieval / evidence_map / prompt / raw_answer / + parsed_claim_lattice / verify / render / final_label``. Triggered + when ``evidence_map_root`` is non-None. Splits the single + ``answer`` node into three: the model's raw output, the parsed + claim-lattice, and the rendered prose with literal spans + interpolated. ``context`` drops out (the context IS the evidence + map). All three of ``raw_answer_text`` / ``parsed_lattice`` / + ``rendered_text`` should be supplied; missing args fall back to + ``answer_text`` for the raw_answer & render hashes and ``[]`` for + the parsed_lattice hash. + + ``answer_mode`` & ``violations`` fold into the verify & final_label + payloads when provided. """ sources_summary = [ { @@ -139,23 +168,55 @@ def build_run_dag( "claim_statuses": claim_statuses or [], "failure_stage": failure_stage, } + if violations is not None: + verify_payload["violations"] = violations verify_hash = _sha256_hex(_canonical_json(verify_payload)) final_label_payload = { "audit_mode": audit_mode, "verifier_method": verifier_method, "lookup_path": lookup_path, } + if answer_mode is not None: + final_label_payload["answer_mode"] = answer_mode final_label_hash = _sha256_hex(_canonical_json(final_label_payload)) - nodes = [ - {"stage": "question", "hash": question_hash}, - {"stage": "retrieval", "hash": retrieval_hash}, - {"stage": "context", "hash": context_root}, - {"stage": "prompt", "hash": conversation_hash}, - {"stage": "answer", "hash": answer_hash}, - {"stage": "verify", "hash": verify_hash}, - {"stage": "final_label", "hash": final_label_hash}, - ] + if evidence_map_root is None: + # Quote-mode 7-stage shape — backward-compatible. + nodes = [ + {"stage": "question", "hash": question_hash}, + {"stage": "retrieval", "hash": retrieval_hash}, + {"stage": "context", "hash": context_root}, + {"stage": "prompt", "hash": conversation_hash}, + {"stage": "answer", "hash": answer_hash}, + {"stage": "verify", "hash": verify_hash}, + {"stage": "final_label", "hash": final_label_hash}, + ] + else: + # Pointer-mode 9-stage shape (CTI). ``context`` drops out; + # ``answer`` splits into raw_answer / parsed_claim_lattice / + # render so each provenance step gets its own commitment. + raw_text = raw_answer_text if raw_answer_text is not None else answer_text + rendered = rendered_text if rendered_text is not None else answer_text + raw_answer_hash = _sha256_hex(raw_text) + # Parsed lattice = list of {claim_text, evidence_ids[]} dicts in + # input order; canonical-json so reordering claims changes the + # hash. Pointer ids are run-dependent — we prefer the + # content-addressed evidence_ids here for run-stable provenance. + parsed_lattice_hash = _sha256_hex( + _canonical_json(parsed_lattice or []) + ) + rendered_hash = _sha256_hex(rendered) + nodes = [ + {"stage": "question", "hash": question_hash}, + {"stage": "retrieval", "hash": retrieval_hash}, + {"stage": "evidence_map", "hash": evidence_map_root}, + {"stage": "prompt", "hash": conversation_hash}, + {"stage": "raw_answer", "hash": raw_answer_hash}, + {"stage": "parsed_claim_lattice", "hash": parsed_lattice_hash}, + {"stage": "verify", "hash": verify_hash}, + {"stage": "render", "hash": rendered_hash}, + {"stage": "final_label", "hash": final_label_hash}, + ] leaves = [bytes.fromhex(n["hash"]) for n in nodes] root_hex = MerkleTree.build(leaves).root.hex() return {"root": root_hex, "nodes": nodes} diff --git a/aborist/qa/evidence.py b/aborist/qa/evidence.py new file mode 100644 index 0000000..99a3376 --- /dev/null +++ b/aborist/qa/evidence.py @@ -0,0 +1,318 @@ +"""Evidence map for claim-lattice-pointer (quote-by-pointer) answer mode. + +Builds a deterministic table of evidence objects from already-retrieved +chunks. The model references each object by a short ``pointer_id`` +("E1", "E2", ...) which the runtime maps back to a content-addressed +``evidence_id`` for the cache, run-DAG, and audit chain. + +Design pinned by G0 ticket (2026-04-29) and the CTI / Clause Lattice +Intelligence reframe: + + Models should not generate verbatim quotes. + They should point to evidence IDs extracted by deterministic code. + +This kills the synthetic-elision class by construction — the model +never types the quote string, so it can't drop characters from one. + +Two-layer id scheme: + +- ``pointer_id`` short numeric tag the model sees in the prompt and + writes back in pointer-line answers. ``E`` + the + 1-based position of the evidence object in the map + (``E1``, ``E2``, …, ``E37``). One BPE token per id + in standard tokenizers. Stays in the model's + in-distribution citation style. +- ``evidence_id`` content-addressed handle. ``E`` + first 8 hex of + ``sha256(chunk_root:offset_start:offset_end)``. + Same chunk + same offsets in two runs → same id, + forever. The cache_key, run-DAG, and audit chain + all use this form so provenance is run-stable. + +The verifier (``verify_claim_lattice``) maps each pointer_id the model +writes back to its content-addressed evidence_id before persistence. +The model's literal output is run-dependent (run #1's ``E1`` and run +#2's ``E1`` likely point at different chunks); the content-addressed +layer is what stays stable. + +For the first cut every chunk produces exactly one evidence object +covering ``offset_start=0 .. offset_end=len(span)``. Sub-chunk +extraction (paragraph-level, sentence-level) is a future refinement; +the schema already accepts arbitrary offsets so adding finer-grained +splits later doesn't break the contract. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import asdict, dataclass + +from aborist.merkle import MerkleTree + + +def _sha256_hex(s: str) -> str: + return hashlib.sha256(s.encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class EvidenceObject: + """One pinned span the model may reference by ``pointer_id``. + + All fields are deterministic from inputs; same chunk + offsets yields + the same object byte-for-byte. + + - ``pointer_id`` prompt-facing id ("E1", "E2", …) — what the + model sees & writes back. Position-derived; + run-dependent on purpose. + - ``evidence_id`` content-addressed handle ("E" + 8 hex of + sha256(chunk_root:start:end)) — what the + cache, run-DAG, and audit chain use. + Run-stable. + - ``source_root`` document_root the chunk belongs to + - ``document_uri`` human-readable URI (for the renderer) + - ``title`` doc title (for the renderer & prompt) + - ``chunk_idx`` chunk index within the document + - ``chunk_root`` leaf hash of the chunk + - ``offset_start`` byte offset within the chunk (0 for whole-chunk) + - ``offset_end`` end offset (exclusive) + - ``source_role`` role classification (primary_answer_source / ...) + - ``text_hash`` sha256 of the span (for tamper detection) + - ``span`` the literal text (what the renderer interpolates) + """ + + pointer_id: str + evidence_id: str + source_root: str + document_uri: str + title: str | None + chunk_idx: int + chunk_root: str + offset_start: int + offset_end: int + source_role: str + text_hash: str + span: str + + def to_dict(self) -> dict: + return asdict(self) + + +def _evidence_id_for(chunk_root: str, offset_start: int, offset_end: int) -> str: + """Content-addressed handle. Same chunk + offsets = same ID, forever.""" + h = _sha256_hex(f"{chunk_root}:{offset_start}:{offset_end}") + return f"E{h[:8]}" + + +def build_evidence_map( + chunks: list[dict], +) -> list[EvidenceObject]: + """Build the evidence table from retrieved chunks. + + ``chunks`` is a list of dicts with keys: + source_root, document_uri, title (optional), chunk_idx, + chunk_root, span, source_role (optional, default 'unclassified') + + Returns a list of ``EvidenceObject``s, one per chunk, in input order. + The 1-based position drives ``pointer_id`` (E1, E2, …); the chunk's + content drives ``evidence_id`` (sha256-derived). For the first cut + each chunk = one whole-span evidence object (offset 0 .. len(span)). + Sub-chunk splitting is a future refinement. + """ + out: list[EvidenceObject] = [] + for i, c in enumerate(chunks): + span = c["span"] + offset_start = 0 + offset_end = len(span) + chunk_root = c["chunk_root"] + evidence_id = _evidence_id_for(chunk_root, offset_start, offset_end) + out.append( + EvidenceObject( + pointer_id=f"E{i + 1}", + evidence_id=evidence_id, + source_root=c["source_root"], + document_uri=c["document_uri"], + title=c.get("title"), + chunk_idx=c["chunk_idx"], + chunk_root=chunk_root, + offset_start=offset_start, + offset_end=offset_end, + source_role=c.get("source_role", "unclassified"), + text_hash=_sha256_hex(span), + span=span, + ) + ) + return out + + +def evidence_map_root(evidence: list[EvidenceObject]) -> str: + """Merkle root over the sorted evidence_id leaves. + + Sorting makes the root order-independent — two retrieval runs that + return the same chunks in different orders produce the same root. + Use as the ``evidence_map`` stage hash in the run-DAG. + """ + if not evidence: + return "00" * 32 + leaves_hex = sorted(_sha256_hex(e.evidence_id) for e in evidence) + if len(leaves_hex) == 1: + return leaves_hex[0] + leaves = [bytes.fromhex(h) for h in leaves_hex] + return MerkleTree.build(leaves).root.hex() + + +def render_evidence_block(e: EvidenceObject) -> str: + """Format one evidence object for the LLM prompt. + + Header carries the prompt-facing pointer_id, title (or URI tail), + and source role so the model has everything it needs to cite + without typing the span: + + === E1 (Jurassic_Park_(film) | primary_answer_source) === + + + The runtime maps E1 back to the content-addressed evidence_id + before persistence; the model never sees the hex form. + """ + label = (e.title or e.document_uri.rsplit("/", 1)[-1]) or "untitled" + return f"=== {e.pointer_id} ({label} | {e.source_role}) ===\n{e.span}" + + +def render_evidence_map(evidence: list[EvidenceObject]) -> str: + """Concatenated evidence blocks, ready to drop into the prompt.""" + return "\n\n".join(render_evidence_block(e) for e in evidence) + + +def evidence_map_by_pointer_id( + evidence: list[EvidenceObject], +) -> dict[str, EvidenceObject]: + """Index by prompt-facing pointer_id (E1, E2, …).""" + return {e.pointer_id: e for e in evidence} + + +def evidence_map_by_evidence_id( + evidence: list[EvidenceObject], +) -> dict[str, EvidenceObject]: + """Index by content-addressed evidence_id (E1f8e4c2a, …).""" + return {e.evidence_id: e for e in evidence} + + +def render_claim_lattice( + claims: list[dict], + by_id: dict[str, EvidenceObject], + *, + window: int = 200, +) -> str: + """Convert structured claims to human-readable prose with literal spans. + + Each claim becomes one bullet line followed by inlined evidence + excerpts. The model's claim text is rendered verbatim; each cited + pointer_id is followed by a **spotlight excerpt** of the literal + source span — a window of ``window`` chars centered on the first + content token from the claim that appears in the span. Falls back + to the leading window when no claim token matches. + + Why spotlight over leading-N truncation: when the cited evidence is + a whole article and the model lazy-anchors every claim at the same + pointer, the leading-N strategy displayed the same article-intro + sentence under every claim. The spotlight finds the part of the + span the claim is *about* — "Brachiosaurus appears in the film" + + a 15 KB article span gets a window centered on the first + "brachiosaurus" mention, not the production-history opener. Same + cited evidence id, but the displayed text actually supports + different claims differently. + + ``by_id`` is the pointer_id → EvidenceObject index — what + ``evidence_map_by_pointer_id`` returns. Determinism: same + (claims, by_id, window) → same prose, byte-for-byte. Unknown ids + render as ``[: ?]`` so violations are visible at a glance. + """ + lines: list[str] = [] + for c in claims: + text = (c.get("text") or "").strip() + if not text: + continue + lines.append(f"- {text}") + for eid in c.get("pointer_ids") or c.get("evidence_ids") or []: + obj = by_id.get(eid) + if obj is None: + lines.append(f' [{eid}: ?]') + continue + excerpt = _spotlight_excerpt(text, obj.span, window=window) + lines.append(f' [{eid}: "{excerpt}"]') + return "\n".join(lines) + + +# Stopwords for spotlight content-token extraction. Smaller set than +# verify._ENGLISH_STOPWORDS — we only need to filter the words the +# model is most likely to share between claim text & every span. A +# handful of high-frequency 4+ char fillers is enough; the spotlight +# falls back to the leading window when no token matches anyway, so +# false negatives degrade gracefully. +_SPOTLIGHT_STOPWORDS = frozenset({ + "from", "with", "into", "onto", "upon", "this", "that", "these", + "those", "have", "been", "being", "their", "there", "they", + "them", "your", "what", "when", "where", "while", "which", + "would", "could", "should", "might", "shall", "first", "last", + "also", "very", "much", "many", "more", "most", "less", "some", + "such", "than", "then", "back", "next", "after", "before", + "between", "through", "across", "above", "below", "during", + "without", "within", "until", "since", "about", "around", + "along", "among", "appear", "appears", "appeared", "feature", + "features", "include", "includes", "shown", "shows", +}) + +_TOKEN_PUNCT_STRIP_R = ".,;:!?\"'()[]{}—-" + + +def _content_tokens(text: str) -> list[str]: + """Lowercase content tokens from ``text``, sorted by length desc. + + Filters: drop ``< 4`` chars (function words), drop a small stopword + set, dedup. Sorted longest-first so the spotlight matches the most + specific topical token before generic ones — for "Brachiosaurus + appears in the film", that's ``brachiosaurus`` ahead of ``film``. + """ + seen: set[str] = set() + out: list[str] = [] + for raw in text.lower().split(): + t = raw.strip(_TOKEN_PUNCT_STRIP_R) + if len(t) < 4 or t in _SPOTLIGHT_STOPWORDS or t in seen: + continue + seen.add(t) + out.append(t) + out.sort(key=len, reverse=True) + return out + + +def _spotlight_excerpt(claim_text: str, span: str, *, window: int) -> str: + """Return a ``window``-char excerpt of ``span``, centered on the + first match of any ``_content_tokens(claim_text)`` token. + + If ``span`` already fits in ``window``, returns it unchanged. + If no claim token matches anywhere in ``span``, falls back to the + leading ``window`` chars with a trailing ``"..."``. + + Determinism: same inputs → same output. Match search is + case-insensitive but the returned excerpt is sliced from the + original ``span`` (case preserved). + """ + if len(span) <= window: + return span + span_lower = span.lower() + match_idx = -1 + for tok in _content_tokens(claim_text): + idx = span_lower.find(tok) + if idx >= 0: + match_idx = idx + break + if match_idx < 0: + return span[: window - 3] + "..." + # Center the window on the match, clamped to span boundaries. + half = window // 2 + start = max(0, match_idx - half) + end = min(len(span), start + window) + # Re-anchor so the window is full-width when the match sits near + # either boundary (start near 0, or end near len). + start = max(0, end - window) + prefix = "..." if start > 0 else "" + suffix = "..." if end < len(span) else "" + return f"{prefix}{span[start:end]}{suffix}" diff --git a/aborist/qa/parse_claims.py b/aborist/qa/parse_claims.py new file mode 100644 index 0000000..3a0a05a --- /dev/null +++ b/aborist/qa/parse_claims.py @@ -0,0 +1,142 @@ +"""Pointer-line parser for ``answer_mode="claim_lattice_pointer"``. + +The CTI / Clause Lattice Intelligence runtime accepts weak natural-language +pointer clauses from the model and compiles them into structured claim +nodes. Hermes does NOT produce JSON, does NOT produce a formal graph — +Hermes only writes: + + Claim text here. [E12] + Another claim. [E13,E14] + +The runtime owns the structure. The model owns the prose. This stays +inside the model's prose-generation distribution (citation-style is +heavily represented in training; random hex tokens are not), so one- +shot SCHEMA_INVALID failures from JSON drift disappear. + +Two-layer id discipline: + + pointer_id E1, E2, E3, ... short numeric tag the model writes + evidence_id E1f8e4c2a, ... content-addressed sha256, run-stable + +The model's prompt only ever shows pointer ids. The runtime maps each +pointer id back to its content-addressed evidence_id and stores the +content-addressed form in the cache, run-DAG, and audit chain so +provenance is run-stable. + +Parser status taxonomy: + + PARSED line had at least one ``[E\\d+...]`` tag and + surviving claim text after the strip + NO_EVIDENCE_POINTER line had non-empty prose but no pointer tag — + model produced a claim without citing + EMPTY line was bullet/whitespace only (skipped) +""" + +from __future__ import annotations + +import re +from dataclasses import asdict, dataclass + + +# Match a bracketed pointer-id list. Examples: +# [E12] +# [E12,E13] +# [E12, E13 ,E14] +# +# The outer brackets are required so the parser refuses fuzzy alternatives +# like (E12) or {E12}. The inner content is a comma-separated list of +# E\d+ tokens with optional whitespace. Pointer ids carry one capture +# group across the WHOLE bracket so the strip is clean. +_BRACKET_RE = re.compile(r"\[\s*(E\d+(?:\s*,\s*E\d+)*)\s*\]") + +# Pointer id within a captured bracket payload. +_POINTER_RE = re.compile(r"E\d+") + +# Bullet markers (subset of verify._BULLET_RE — kept narrower here so +# this module has no cross-import to verify.py): -, *, +, •, "1.", "2)". +_BULLET_RE = re.compile(r"^\s*(?:[-*+•]|\d+[.)])\s+") + +# Trailing connector punctuation often left attached when the parser +# strips the trailing bracket. We keep terminal sentence punctuation +# (".", "!", "?", ":") because that's part of the claim text. +_TRAILING_STRIP = " ,;-—\t" + + +@dataclass(frozen=True) +class ParsedClaim: + """One line of model output, parsed. + + - ``claim_text`` line with bullet marker stripped and pointer + tags removed; the natural-language claim + surviving for verification & rendering. + - ``pointer_ids`` short ids the model wrote, in the order they + appeared. Decimal-suffixed E-prefixed + (``E1``, ``E12``). Verifier maps each back + to the content-addressed evidence_id. + - ``parse_status`` PARSED / NO_EVIDENCE_POINTER. Empty/whitespace + lines are filtered by the parser and never + surface as claims. + - ``raw_line`` the line as the model wrote it, pre-strip. + Useful for the audit trail and for the + render stage hash. + """ + + claim_text: str + pointer_ids: list[str] + parse_status: str + raw_line: str + + def to_dict(self) -> dict: + return asdict(self) + + +def parse_pointer_claims(text: str) -> list[ParsedClaim]: + """Walk ``text`` line by line; emit one ``ParsedClaim`` per non-empty + line. + + Skips empty / whitespace-only lines. Lines with at least one + ``[E\\d+...]`` bracket get ``parse_status="PARSED"``; lines with prose + but no bracket get ``parse_status="NO_EVIDENCE_POINTER"``. The + verifier owns the policy on what to do with a NO_EVIDENCE_POINTER + claim — typically it counts toward the denominator and downgrades + the verdict to HYBRID/UNGROUNDED. + + Pointer-id ordering is preserved as the model wrote them; duplicates + are NOT deduped here so the verifier can flag repeated cites. + """ + out: list[ParsedClaim] = [] + for raw_line in text.splitlines(): + # Skip lines that are pure whitespace or only a bullet glyph. + stripped_for_emptiness = _BULLET_RE.sub("", raw_line).strip() + if not stripped_for_emptiness: + continue + + # Pull every bracket-list match; collect all pointer ids inside. + bracket_matches = list(_BRACKET_RE.finditer(raw_line)) + if not bracket_matches: + out.append(ParsedClaim( + claim_text=stripped_for_emptiness, + pointer_ids=[], + parse_status="NO_EVIDENCE_POINTER", + raw_line=raw_line, + )) + continue + + pointer_ids: list[str] = [] + for m in bracket_matches: + pointer_ids.extend(_POINTER_RE.findall(m.group(1))) + + # Claim text = line minus every bracket match, then bullet- and + # trailing-punct-stripped. Preserves prose word order even when + # the model interleaves tags mid-sentence. + claim_text = _BRACKET_RE.sub("", raw_line) + claim_text = _BULLET_RE.sub("", claim_text).strip() + claim_text = claim_text.rstrip(_TRAILING_STRIP) + + out.append(ParsedClaim( + claim_text=claim_text, + pointer_ids=pointer_ids, + parse_status="PARSED", + raw_line=raw_line, + )) + return out diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 33eef5f..fb82035 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -65,8 +65,18 @@ from aborist.qa.keys import ( question_hash, ) from aborist.qa.dag import build_run_dag +from aborist.qa.evidence import ( + build_evidence_map, + evidence_map_root, + render_evidence_map, +) from aborist.qa.repair import mechanical_repair, reprompt_repair -from aborist.qa.verify import verify_quotes +from aborist.qa.verify import ( + ANSWER_MODES, + DEFAULT_ANSWER_MODE, + verify_claim_lattice, + verify_quotes, +) try: from aborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION @@ -274,6 +284,59 @@ DEFAULT_QUERY_POLICY = { # re-prompt (mechanical only). 1 = at most one extra LLM call # asking the model to rewrite around the failed quotes. "repair_max_reprompts": 0, + # G0 / CTI — claim-lattice-pointer answer mode. See + # aborist/qa/runner.py:DEFAULT_POLICY for full semantics. Default + # "quote" preserves existing behavior; "claim_lattice_pointer" + # instructs the runtime to build an evidence map, show short + # pointer ids (E1, E2, …) to the model, and accept pointer-line + # output ("Claim. [E12]") that the verifier maps back to + # content-addressed evidence_ids for the cache & run-DAG. + "answer_mode": DEFAULT_ANSWER_MODE, + "claim_lattice_system_prompt": ( + "You will see numbered EVIDENCE blocks tagged E1, E2, E3, etc. " + "Answer the question using natural-language pointer-lines: one " + "claim per line, each followed by [E#] or [E#,E#,...] " + "referencing the EVIDENCE blocks above.\n\n" + "WORKED EXAMPLE\n" + "--------------\n" + "EVIDENCE:\n\n" + "=== E1 (Apple_Inc | primary_answer_source) ===\n" + "Apple Inc. was founded by Steve Jobs, Steve Wozniak, and " + "Ronald Wayne in April 1976.\n\n" + "=== E2 (Steve_Wozniak | secondary_context_source) ===\n" + "Steve Wozniak co-founded Apple Computer Company alongside " + "Steve Jobs in 1976 and designed the Apple I.\n\n" + "QUESTION: who founded Apple?\n\n" + "ANSWER:\n" + "Steve Jobs co-founded Apple. [E1]\n" + "Steve Wozniak co-founded Apple. [E1,E2]\n" + "Ronald Wayne co-founded Apple. [E1]\n\n" + "END OF EXAMPLE\n\n" + "RULES:\n" + "1. Do not type quotation marks anywhere. Do not quote source " + "text manually. The runtime will display literal evidence " + "spans next to each claim — your job is to reference evidence " + "by pointer id, not to reproduce its text.\n" + "2. Pointer ids must be from the EVIDENCE blocks. Do not " + "invent ids.\n" + "3. Every claim line must end with [E#] or [E#,E#,...].\n" + "4. If no evidence supports a claim, omit that claim. If no " + "evidence supports any claim, output nothing.\n" + "5. No JSON. No markdown tables. No headings. No preamble. " + "Plain prose lines only, each ending with a bracket tag." + ), + "claim_lattice_grounding_reminder": ( + "REMINDER: pointer-line format only — `Claim. [E1]` per line. " + "No quotation marks anywhere. No JSON. Pointer ids must be " + "from the EVIDENCE blocks. Now answer the question on the next " + "message." + ), + "claim_lattice_allowed_source_roles": [ + "primary_answer_source", + "secondary_context_source", + "background_source", + "unclassified", + ], } @@ -455,6 +518,31 @@ def _body_count_with_stem(body: str, t: str) -> int: return 0 +def _chunk_query_relevance( + span: str, qtokens_stem: set[str] +) -> tuple[int, int]: + """Rank one chunk by query-token overlap. Returns + ``(distinct_present, total_mentions)``. + + Stem-aware: same ``_body_count_with_stem`` we use elsewhere, so + ``"supermans"`` matches ``"superman"`` in the chunk text. Soft + signal — the score never enters the proof path; it only orders + chunks within a source so the most-relevant chunk gets the lowest + pointer id (and the model's lazy-anchor habit lands on a useful + chunk by accident). + + Sort callers should walk ``(-distinct, -total, chunk_idx_asc)`` + to break ties stably toward document order. + """ + if not qtokens_stem: + return (0, 0) + body = span.lower() + counts = {t: _body_count_with_stem(body, t) for t in qtokens_stem} + distinct_present = sum(1 for n in counts.values() if n > 0) + total = sum(counts.values()) + return (distinct_present, total) + + def _body_density_passes( conn, document_root: str, qtokens: set[str], min_mentions: int = 3 ) -> bool: @@ -715,6 +803,40 @@ def _load_doc_text(shard_path: str, document_root: str) -> str | None: return "\n\n".join(unpack_chunk(r["content"]) or "" for r in rows) +def _load_doc_chunks( + shard_path: str, document_root: str +) -> list[tuple[int, str, str]] | None: + """Per-chunk hot rows of a document. + + Returns ``[(chunk_idx, leaf_hash, span), ...]`` in chunk order, or + ``None`` if the document is cold or absent. Cold individual chunks + are skipped (the WHERE clause filters NULL content). Used by the + claim-lattice-pointer evidence-map builder to emit one + ``EvidenceObject`` per chunk so the model can cite the specific + paragraph that supports a claim instead of lazy-anchoring the + whole article. + """ + conn = connect(shard_path) + try: + rows = conn.execute( + "SELECT idx, leaf_hash, content FROM chunks " + "WHERE document_root = ? AND content IS NOT NULL " + "ORDER BY idx ASC", + (document_root,), + ).fetchall() + finally: + conn.close() + if not rows: + return None + out: list[tuple[int, str, str]] = [] + for r in rows: + span = unpack_chunk(r["content"]) or "" + if not span: + continue + out.append((r["idx"], r["leaf_hash"], span)) + return out or None + + def _context_root(source_roots: list[str]) -> str: """Merkle root over sorted source document_roots — the v9.8 'source' dim for multi-source answers. Sorting makes the root deterministic regardless @@ -773,6 +895,11 @@ def query( raise ValueError( f"fidelity must be one of {FIDELITY_MODES}, got {fidelity!r}" ) + answer_mode = policy.get("answer_mode", DEFAULT_ANSWER_MODE) + if answer_mode not in ANSWER_MODES: + raise ValueError( + f"policy['answer_mode'] must be one of {ANSWER_MODES}, got {answer_mode!r}" + ) t_start = time.monotonic() # 1. Search. @@ -871,25 +998,95 @@ def query( context = _wikitext_to_base(context) context_ms = _ms_since(t_phase) - # 3. Build messages + hashes. - # System sets the policy; a user-turn reminder restates the verbatim-quote - # rule one message before the sources arrive (Hermes follows recent - # user-turn instructions more reliably than system-turn rules under long - # context). The sources + question land last so they are the most-recent - # tokens before generation begins. - messages = [ - {"role": "system", "content": policy["system_prompt"]}, - ] - if policy.get("grounding_reminder"): - messages.append( - {"role": "user", "content": policy["grounding_reminder"]} - ) - messages.append( - { - "role": "user", - "content": f"Sources:\n\n{context}\n\n---\n\nQuestion: {question}", + # 3. Build messages + hashes. Branch on answer_mode: + # "quote" (default) raw sources block + verbatim-quote rules. + # "claim_lattice" one evidence object per chosen source, each + # labeled with a content-addressed evidence_id; + # model emits JSON referencing IDs. + evidence_map = [] + if answer_mode == "claim_lattice_pointer": + # G0.1 — per-chunk evidence granularity. Each retrieved source + # contributes ONE evidence object per chunk (up to the + # role-weighted per_source_cap budget) instead of one + # whole-doc span. + # + # G0.3 — query-relevance chunk ordering. Within each source, + # chunks are ranked by (distinct_query_tokens_present, + # total_mentions, chunk_idx_asc) so the chunk that most + # textually supports the question gets the lowest pointer id + # and lands at the top of the per-source evidence stack. Without + # this re-rank, Hermes-3-8B lazy-anchors on the first few + # chunks regardless of relevance — burying the actual answer + # paragraph behind irrelevant article-header text. Soft signal + # only (token overlap; no embeddings) so it stays out of the + # proof path; the verifier still runs the same hard checks + # against whatever the model picked. + qtokens_stem_for_chunks = { + _stem_token_for_match(t.lower()) + for t in _title_query_tokens(question) } - ) + chunks_for_map: list[dict] = [] + for h in chosen: + doc_chunks = _load_doc_chunks(h.shard_path, h.document_root) + if not doc_chunks: + continue + weight = SOURCE_ROLE_BUDGET_WEIGHTS.get(h.source_role, 1.0) + hit_cap = max(1, int(per_source_cap * weight)) + # Score each chunk by query-token overlap. Tuple sort: + # distinct present DESC, total mentions DESC, doc order ASC. + scored = [] + for chunk_idx, leaf_hash, span in doc_chunks: + distinct, total = _chunk_query_relevance( + span, qtokens_stem_for_chunks + ) + scored.append( + (chunk_idx, leaf_hash, span, distinct, total) + ) + scored.sort(key=lambda r: (-r[3], -r[4], r[0])) + # Greedy: fill the per-source budget with relevance-ranked + # chunks. If a single chunk exceeds what's left, truncate + # that one chunk and stop. Total context across the source + # stays bounded by hit_cap, same shape as the prose path. + spent = 0 + for chunk_idx, leaf_hash, span, _d, _t in scored: + if spent >= hit_cap: + break + if policy.get("base_version") and _wikitext_to_base is not None: + span = _wikitext_to_base(span) + remaining = hit_cap - spent + if len(span) > remaining: + span = span[:remaining] + if not span: + break + chunks_for_map.append({ + "source_root": h.document_root, + "document_uri": h.document_uri, + "title": h.title, + "chunk_idx": chunk_idx, + "chunk_root": leaf_hash, + "span": span, + "source_role": h.source_role, + }) + spent += len(span) + evidence_map = build_evidence_map(chunks_for_map) + sys_prompt = policy["claim_lattice_system_prompt"] + grounding_reminder = policy.get("claim_lattice_grounding_reminder") + rendered_evidence = render_evidence_map(evidence_map) + + def _user_payload(q: str) -> str: + return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}" + else: + sys_prompt = policy["system_prompt"] + grounding_reminder = policy.get("grounding_reminder") + + def _user_payload(q: str) -> str: + return f"Sources:\n\n{context}\n\n---\n\nQuestion: {q}" + + messages = [{"role": "system", "content": sys_prompt}] + if grounding_reminder: + messages.append({"role": "user", "content": grounding_reminder}) + messages.append({"role": "user", "content": _user_payload(question)}) + context_root = _context_root([h.document_root for h in chosen]) mhash = model_profile_hash(model_id, revision, quantization) @@ -904,12 +1101,7 @@ def query( def _ckey_for_mode(mode: str) -> str: canon_q = canonical_question(question, mode=mode) canon_msgs = list(messages[:-1]) + [ - { - "role": "user", - "content": ( - f"Sources:\n\n{context}\n\n---\n\nQuestion: {canon_q}" - ), - } + {"role": "user", "content": _user_payload(canon_q)}, ] policy_variant = dict(policy, question_dedup=mode) return cache_key( @@ -937,12 +1129,7 @@ def query( qhash = question_hash(question, mode=primary_dedup) canonical_q_primary = canonical_question(question, mode=primary_dedup) canonical_messages_primary = list(messages[:-1]) + [ - { - "role": "user", - "content": ( - f"Sources:\n\n{context}\n\n---\n\nQuestion: {canonical_q_primary}" - ), - } + {"role": "user", "content": _user_payload(canonical_q_primary)}, ] chash = conversation_hash(canonical_messages_primary) primary_ckey = _ckey_for_mode(primary_dedup) @@ -1051,7 +1238,7 @@ def query( # 5. Cache miss — call LLM. t_phase = time.monotonic() - answer_text = chat_client.chat_completion( + raw_answer = chat_client.chat_completion( messages, model=model_id, temperature=policy["temperature"], @@ -1060,82 +1247,100 @@ def query( ) llm_ms = _ms_since(t_phase) - # 5b. Faithfulness check — pull quoted spans, verify each verbatim - # against the assembled context, classify into v9.8 audit_mode. - verdict = verify_quotes( - answer_text, - context, - entity_policy=policy.get("entity_policy", "hybrid"), - proximity_n=policy.get("entity_proximity_n", 3), - proximity_window=policy.get("entity_proximity_window", 300), - ) - - # 5c. Optional hybrid repair loop. Mechanical first (deterministic - # string substitutions for synthetic_elision / trailing_artifact / - # no_overlap), then up to `repair_max_reprompts` LLM re-prompts - # for the cases mechanical declined (paraphrase, interior_elision - # needing semantic judgment). Both gated by `policy["repair_enabled"]`. - # Persist the post-repair answer + the change log; cache_key - # inputs are unchanged. + # 5b. Faithfulness check. Branch on answer_mode: + # "quote" substring-verify quoted spans against context. + # Optional repair loop (mechanical + reprompt). + # "claim_lattice" deterministic checks on the JSON output: + # evidence_id resolution, source_role allowlist, + # manual-quote prohibition. NO repair loop — + # one-shot benchmark discipline. repair_changes: list[dict] = [] pre_repair_verdict: dict | None = None - def _verify(text: str) -> dict: - return verify_quotes( - text, + if answer_mode == "claim_lattice_pointer": + verdict = verify_claim_lattice( + raw_answer, + evidence_map, + allowed_source_roles=tuple( + policy.get( + "claim_lattice_allowed_source_roles", + [ + "primary_answer_source", + "secondary_context_source", + "background_source", + "unclassified", + ], + ) + ), + ) + rendered = verdict["rendered_text"] + answer_text = rendered if rendered else raw_answer + else: + answer_text = raw_answer + verdict = verify_quotes( + answer_text, context, entity_policy=policy.get("entity_policy", "hybrid"), proximity_n=policy.get("entity_proximity_n", 3), proximity_window=policy.get("entity_proximity_window", 300), ) - if ( - policy.get("repair_enabled") - and verdict["audit_mode"] != "STRICT" - and verdict.get("unverified_quotes") - ): - # Tier 1: mechanical (deterministic, no extra LLM call). - repair_result = mechanical_repair( - answer_text, verdict["unverified_quotes"], context - ) - if repair_result["changes"]: - new_verdict = _verify(repair_result["repaired_text"]) - if new_verdict["n_verified"] >= verdict["n_verified"]: - pre_repair_verdict = verdict - answer_text = repair_result["repaired_text"] - verdict = new_verdict - repair_changes = list(repair_result["changes"]) - - # Tier 2: re-prompt feedback (one extra LLM call max). - max_reprompts = int(policy.get("repair_max_reprompts", 0)) - for _ in range(max_reprompts): - if ( - verdict["audit_mode"] == "STRICT" - or not verdict.get("unverified_quotes") - ): - break - new_text = reprompt_repair( - chat_client=chat_client, - model_id=model_id, - original_messages=messages, - original_answer=answer_text, - failed_quotes=verdict["unverified_quotes"], - policy=policy, + def _verify(text: str) -> dict: + return verify_quotes( + text, + context, + entity_policy=policy.get("entity_policy", "hybrid"), + proximity_n=policy.get("entity_proximity_n", 3), + proximity_window=policy.get("entity_proximity_window", 300), ) - if not new_text: - break - new_verdict = _verify(new_text) - if new_verdict["n_verified"] > verdict["n_verified"]: - if pre_repair_verdict is None: + + if ( + policy.get("repair_enabled") + and verdict["audit_mode"] != "STRICT" + and verdict.get("unverified_quotes") + ): + # Tier 1: mechanical (deterministic, no extra LLM call). + repair_result = mechanical_repair( + answer_text, verdict["unverified_quotes"], context + ) + if repair_result["changes"]: + new_verdict = _verify(repair_result["repaired_text"]) + if new_verdict["n_verified"] >= verdict["n_verified"]: pre_repair_verdict = verdict - answer_text = new_text - verdict = new_verdict - repair_changes.append({ - "action": "reprompt_rewrite", - "diagnosis": "model_feedback_loop", - }) - else: - break + answer_text = repair_result["repaired_text"] + verdict = new_verdict + repair_changes = list(repair_result["changes"]) + + # Tier 2: re-prompt feedback (one extra LLM call max). + max_reprompts = int(policy.get("repair_max_reprompts", 0)) + for _ in range(max_reprompts): + if ( + verdict["audit_mode"] == "STRICT" + or not verdict.get("unverified_quotes") + ): + break + new_text = reprompt_repair( + chat_client=chat_client, + model_id=model_id, + original_messages=messages, + original_answer=answer_text, + failed_quotes=verdict["unverified_quotes"], + policy=policy, + ) + if not new_text: + break + new_verdict = _verify(new_text) + if new_verdict["n_verified"] > verdict["n_verified"]: + if pre_repair_verdict is None: + pre_repair_verdict = verdict + answer_text = new_text + verdict = new_verdict + repair_changes.append({ + "action": "reprompt_rewrite", + "diagnosis": "model_feedback_loop", + }) + else: + break unverified_blob = ( json.dumps(verdict["unverified_quotes"], separators=(",", ":")) @@ -1162,12 +1367,23 @@ def query( } proof_blob = json.dumps(proof_obj, separators=(",", ":")) - # Per-run Merkle-DAG: question / retrieval / context / prompt / - # answer / verify / final_label nodes hashed in order. The root - # is the run's computation-provenance fingerprint, distinct from - # cache_key (which fingerprints the inputs that determined the - # answer; the DAG fingerprints the answer itself + the path - # that produced it). + # Per-run Merkle-DAG. Quote mode keeps the original 7-stage + # shape (question / retrieval / context / prompt / answer / + # verify / final_label). Pointer mode swaps to the 9-stage CTI + # shape — context drops out, answer splits into raw_answer / + # parsed_claim_lattice / render — and threads violations + + # answer_mode into the verify/final_label payloads. + ev_root = evidence_map_root(evidence_map) if evidence_map else None + parsed_lattice = None + if answer_mode == "claim_lattice_pointer": + evidence_id_pairs = verdict.get("evidence_id_pairs") or [] + parsed_lattice = [ + { + "claim_text": cs.get("text", ""), + "evidence_ids": evidence_id_pairs[i] if i < len(evidence_id_pairs) else [], + } + for i, cs in enumerate(verdict.get("claim_statuses") or []) + ] run_dag = build_run_dag( question_hash=qhash, sources=proof_obj["sources"], @@ -1180,6 +1396,12 @@ def query( n_verified=verdict["n_verified"], claim_statuses=verdict.get("claim_statuses", []), lookup_path="miss", + evidence_map_root=ev_root, + answer_mode=answer_mode if answer_mode != "quote" else None, + violations=verdict.get("violations"), + raw_answer_text=raw_answer if answer_mode == "claim_lattice_pointer" else None, + parsed_lattice=parsed_lattice, + rendered_text=answer_text if answer_mode == "claim_lattice_pointer" else None, ) run_dag_blob = json.dumps(run_dag, separators=(",", ":")) diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index adac39c..211f81a 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -35,8 +35,18 @@ from aborist.qa.keys import ( question_hash, ) from aborist.qa.dag import build_run_dag +from aborist.qa.evidence import ( + build_evidence_map, + evidence_map_root, + render_evidence_map, +) from aborist.qa.repair import mechanical_repair, reprompt_repair -from aborist.qa.verify import verify_quotes +from aborist.qa.verify import ( + ANSWER_MODES, + DEFAULT_ANSWER_MODE, + verify_claim_lattice, + verify_quotes, +) from aborist.store import append_audit, transaction try: @@ -83,6 +93,69 @@ DEFAULT_POLICY = { # answers under raw-wikitext policy stay distinct on lookup. Set # via the wikitext extras; no-op if mwparserfromhell isn't installed. "base_version": _WIKITEXT_BASE_VERSION, + # G0 / CTI — claim-lattice-pointer answer mode. "quote" (default): + # existing behavior, model writes prose with verbatim quotes inline. + # "claim_lattice_pointer": runtime builds an evidence map and shows + # the model short pointer ids (E1, E2, …); model writes natural + # prose with bracket pointer tags ("Claim. [E12]") instead of + # quoting source text. Renderer interpolates literal spans at + # display time. Synthetic-elision-by-construction-impossible: the + # model never types the quote string. Two-layer id discipline keeps + # the cache & run-DAG keyed on content-addressed evidence_ids. + # Folds into governance_policy_hash so two modes write under + # different cache_keys and never alias. No iterative repair in + # pointer mode (one-shot benchmark discipline). + "answer_mode": DEFAULT_ANSWER_MODE, + "claim_lattice_system_prompt": ( + "You will see numbered EVIDENCE blocks tagged E1, E2, E3, etc. " + "Answer the question using natural-language pointer-lines: one " + "claim per line, each followed by [E#] or [E#,E#,...] " + "referencing the EVIDENCE blocks above.\n\n" + "WORKED EXAMPLE\n" + "--------------\n" + "EVIDENCE:\n\n" + "=== E1 (Apple_Inc | primary_answer_source) ===\n" + "Apple Inc. was founded by Steve Jobs, Steve Wozniak, and " + "Ronald Wayne in April 1976.\n\n" + "=== E2 (Steve_Wozniak | secondary_context_source) ===\n" + "Steve Wozniak co-founded Apple Computer Company alongside " + "Steve Jobs in 1976 and designed the Apple I.\n\n" + "QUESTION: who founded Apple?\n\n" + "ANSWER:\n" + "Steve Jobs co-founded Apple. [E1]\n" + "Steve Wozniak co-founded Apple. [E1,E2]\n" + "Ronald Wayne co-founded Apple. [E1]\n\n" + "END OF EXAMPLE\n\n" + "RULES:\n" + "1. Do not type quotation marks anywhere. Do not quote source " + "text manually. The runtime will display literal evidence " + "spans next to each claim — your job is to reference evidence " + "by pointer id, not to reproduce its text.\n" + "2. Pointer ids must be from the EVIDENCE blocks. Do not " + "invent ids.\n" + "3. Every claim line must end with [E#] or [E#,E#,...].\n" + "4. If no evidence supports a claim, omit that claim. If no " + "evidence supports any claim, output nothing.\n" + "5. No JSON. No markdown tables. No headings. No preamble. " + "Plain prose lines only, each ending with a bracket tag." + ), + "claim_lattice_grounding_reminder": ( + "REMINDER: pointer-line format only — `Claim. [E1]` per line. " + "No quotation marks anywhere. No JSON. Pointer ids must be " + "from the EVIDENCE blocks. Now answer the question on the next " + "message." + ), + # Allowed source roles for claim-lattice verification. Roles outside + # this set get classified SOURCE_ROLE_BLOCKED and downgrade the + # verdict. Mirrors aborist.qa.verify.DEFAULT_ALLOWED_SOURCE_ROLES; + # noisy_background_source / sequel_background_source are excluded by + # default. Folds into governance_policy_hash on change. + "claim_lattice_allowed_source_roles": [ + "primary_answer_source", + "secondary_context_source", + "background_source", + "unclassified", + ], } @@ -139,7 +212,14 @@ def ask( if any(r["content"] is None for r in chunk_rows): return {"status": "source_cold", "msg": "rehydrate before asking"} - document_text = "\n\n".join(unpack_chunk(r["content"]) for r in chunk_rows) + answer_mode = policy.get("answer_mode", DEFAULT_ANSWER_MODE) + if answer_mode not in ANSWER_MODES: + raise ValueError( + f"policy['answer_mode'] must be one of {ANSWER_MODES}, got {answer_mode!r}" + ) + + chunk_texts = [unpack_chunk(r["content"]) for r in chunk_rows] + document_text = "\n\n".join(chunk_texts) # Wikitext → prose before the LLM sees it. The model can then quote # verbatim against the prose form; the verifier compares like-against- @@ -147,26 +227,49 @@ def ask( # policy["base_version"] so this is part of governance_policy_hash. if policy.get("base_version") and _wikitext_to_base is not None: document_text = _wikitext_to_base(document_text) + chunk_texts = [_wikitext_to_base(t) for t in chunk_texts] - # System sets the policy; a user-turn reminder restates the verbatim-quote - # rule one message before the document arrives. Document + question land - # last as the most-recent tokens before generation. - messages = [ - {"role": "system", "content": policy["system_prompt"]}, - ] - if policy.get("grounding_reminder"): - messages.append( - {"role": "user", "content": policy["grounding_reminder"]} - ) - messages.append( - { - "role": "user", - "content": ( - f"Document:\n\n{document_text}\n\n---\n\n" - f"Question: {question}" - ), - } - ) + evidence_map = [] + if answer_mode == "claim_lattice_pointer": + # Quote-by-pointer: one evidence object per chunk. The model sees + # the literal spans labeled with content-addressed IDs and is + # instructed to reference IDs, not type quote text. Synthetic + # elision is impossible by construction — the model never produces + # the quote string. + chunks_for_map = [ + { + "source_root": document_root, + "document_uri": doc["document_uri"], + "title": None, + "chunk_idx": r["idx"], + "chunk_root": r["leaf_hash"], + "span": chunk_texts[i], + "source_role": "primary_answer_source", + } + for i, r in enumerate(chunk_rows) + ] + evidence_map = build_evidence_map(chunks_for_map) + sys_prompt = policy["claim_lattice_system_prompt"] + grounding_reminder = policy.get("claim_lattice_grounding_reminder") + rendered_evidence = render_evidence_map(evidence_map) + + def _user_payload(q: str) -> str: + return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}" + else: + sys_prompt = policy["system_prompt"] + grounding_reminder = policy.get("grounding_reminder") + + def _user_payload(q: str) -> str: + return f"Document:\n\n{document_text}\n\n---\n\nQuestion: {q}" + + # System sets the policy; a user-turn reminder restates the rule one + # message before the payload arrives. Payload (document or evidence + # map + question) lands last as the most-recent tokens before + # generation. + messages = [{"role": "system", "content": sys_prompt}] + if grounding_reminder: + messages.append({"role": "user", "content": grounding_reminder}) + messages.append({"role": "user", "content": _user_payload(question)}) mhash = model_profile_hash(model_id, revision, quantization) @@ -177,13 +280,7 @@ def ask( def _ckey_for_mode(mode: str) -> str: canon_q = canonical_question(question, mode=mode) canon_msgs = list(messages[:-1]) + [ - { - "role": "user", - "content": ( - f"Document:\n\n{document_text}\n\n---\n\n" - f"Question: {canon_q}" - ), - } + {"role": "user", "content": _user_payload(canon_q)}, ] policy_variant = dict(policy, question_dedup=mode) return cache_key( @@ -210,13 +307,7 @@ def ask( qhash = question_hash(question, mode=primary_dedup) canonical_q_primary = canonical_question(question, mode=primary_dedup) canonical_messages_primary = list(messages[:-1]) + [ - { - "role": "user", - "content": ( - f"Document:\n\n{document_text}\n\n---\n\n" - f"Question: {canonical_q_primary}" - ), - } + {"role": "user", "content": _user_payload(canonical_q_primary)}, ] chash = conversation_hash(canonical_messages_primary) primary_ckey = _ckey_for_mode(primary_dedup) @@ -278,7 +369,7 @@ def ask( } t_llm = time.monotonic() - answer_text = client.chat_completion( + raw_answer = client.chat_completion( messages, model=model_id, temperature=policy["temperature"], @@ -287,72 +378,95 @@ def ask( ) llm_ms = _ms_since(t_llm) - verdict = verify_quotes( - answer_text, - document_text, - entity_policy=policy.get("entity_policy", "hybrid"), - proximity_n=policy.get("entity_proximity_n", 3), - proximity_window=policy.get("entity_proximity_window", 300), - ) - - # Optional hybrid repair loop — see aborist/qa/query.py for shape. repair_changes: list[dict] = [] pre_repair_verdict: dict | None = None - def _verify(text: str) -> dict: - return verify_quotes( - text, + if answer_mode == "claim_lattice_pointer": + verdict = verify_claim_lattice( + raw_answer, + evidence_map, + allowed_source_roles=tuple( + policy.get( + "claim_lattice_allowed_source_roles", + [ + "primary_answer_source", + "secondary_context_source", + "background_source", + "unclassified", + ], + ) + ), + ) + # Rendered prose (literal spans interpolated) is the user-facing + # answer text — never the model's raw pointer-line output. If + # rendering produced nothing (no valid claims), persist the raw + # output so an operator can see what the model actually said. + rendered = verdict["rendered_text"] + answer_text = rendered if rendered else raw_answer + else: + answer_text = raw_answer + verdict = verify_quotes( + answer_text, document_text, entity_policy=policy.get("entity_policy", "hybrid"), proximity_n=policy.get("entity_proximity_n", 3), proximity_window=policy.get("entity_proximity_window", 300), ) - if ( - policy.get("repair_enabled") - and verdict["audit_mode"] != "STRICT" - and verdict.get("unverified_quotes") - ): - repair_result = mechanical_repair( - answer_text, verdict["unverified_quotes"], document_text - ) - if repair_result["changes"]: - new_verdict = _verify(repair_result["repaired_text"]) - if new_verdict["n_verified"] >= verdict["n_verified"]: - pre_repair_verdict = verdict - answer_text = repair_result["repaired_text"] - verdict = new_verdict - repair_changes = list(repair_result["changes"]) - - max_reprompts = int(policy.get("repair_max_reprompts", 0)) - for _ in range(max_reprompts): - if ( - verdict["audit_mode"] == "STRICT" - or not verdict.get("unverified_quotes") - ): - break - new_text = reprompt_repair( - chat_client=client, - model_id=model_id, - original_messages=messages, - original_answer=answer_text, - failed_quotes=verdict["unverified_quotes"], - policy=policy, + def _verify(text: str) -> dict: + return verify_quotes( + text, + document_text, + entity_policy=policy.get("entity_policy", "hybrid"), + proximity_n=policy.get("entity_proximity_n", 3), + proximity_window=policy.get("entity_proximity_window", 300), ) - if not new_text: - break - new_verdict = _verify(new_text) - if new_verdict["n_verified"] > verdict["n_verified"]: - if pre_repair_verdict is None: + + if ( + policy.get("repair_enabled") + and verdict["audit_mode"] != "STRICT" + and verdict.get("unverified_quotes") + ): + repair_result = mechanical_repair( + answer_text, verdict["unverified_quotes"], document_text + ) + if repair_result["changes"]: + new_verdict = _verify(repair_result["repaired_text"]) + if new_verdict["n_verified"] >= verdict["n_verified"]: pre_repair_verdict = verdict - answer_text = new_text - verdict = new_verdict - repair_changes.append({ - "action": "reprompt_rewrite", - "diagnosis": "model_feedback_loop", - }) - else: - break + answer_text = repair_result["repaired_text"] + verdict = new_verdict + repair_changes = list(repair_result["changes"]) + + max_reprompts = int(policy.get("repair_max_reprompts", 0)) + for _ in range(max_reprompts): + if ( + verdict["audit_mode"] == "STRICT" + or not verdict.get("unverified_quotes") + ): + break + new_text = reprompt_repair( + chat_client=client, + model_id=model_id, + original_messages=messages, + original_answer=answer_text, + failed_quotes=verdict["unverified_quotes"], + policy=policy, + ) + if not new_text: + break + new_verdict = _verify(new_text) + if new_verdict["n_verified"] > verdict["n_verified"]: + if pre_repair_verdict is None: + pre_repair_verdict = verdict + answer_text = new_text + verdict = new_verdict + repair_changes.append({ + "action": "reprompt_rewrite", + "diagnosis": "model_feedback_loop", + }) + else: + break unverified_blob = ( json.dumps(verdict["unverified_quotes"], separators=(",", ":")) @@ -368,9 +482,25 @@ def ask( } proof_blob = json.dumps(proof_obj, separators=(",", ":")) - # Per-run Merkle-DAG (see aborist/qa/dag.py). Single-doc shape: - # the only "source" is document_root; that's the retrieval/context - # node identity. + # Per-run Merkle-DAG (see aborist/qa/dag.py). Single-doc shape: the + # only "source" is document_root. Pointer mode swaps the 7-stage + # quote shape for the 9-stage CTI shape — context drops out and + # answer splits into raw_answer / parsed_claim_lattice / render. + ev_root = evidence_map_root(evidence_map) if evidence_map else None + parsed_lattice = None + if answer_mode == "claim_lattice_pointer": + # Per-claim list of {claim_text, content-addressed evidence_ids} + # for the parsed_claim_lattice node hash. Pointer ids are + # run-dependent; evidence_ids are content-addressed → the run- + # DAG hashes the run-stable form. + evidence_id_pairs = verdict.get("evidence_id_pairs") or [] + parsed_lattice = [ + { + "claim_text": cs.get("text", ""), + "evidence_ids": evidence_id_pairs[i] if i < len(evidence_id_pairs) else [], + } + for i, cs in enumerate(verdict.get("claim_statuses") or []) + ] run_dag = build_run_dag( question_hash=qhash, sources=[{ @@ -388,6 +518,12 @@ def ask( n_verified=verdict["n_verified"], claim_statuses=verdict.get("claim_statuses", []), lookup_path="miss", + evidence_map_root=ev_root, + answer_mode=answer_mode if answer_mode != "quote" else None, + violations=verdict.get("violations"), + raw_answer_text=raw_answer if answer_mode == "claim_lattice_pointer" else None, + parsed_lattice=parsed_lattice, + rendered_text=answer_text if answer_mode == "claim_lattice_pointer" else None, ) run_dag_blob = json.dumps(run_dag, separators=(",", ":")) diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index cecc0d9..11638cc 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -641,3 +641,285 @@ def verify_quotes( "verifier_method": "none", "claim_statuses": [], } + + +# --------------------------------------------------------------------------- +# Claim-lattice-pointer verifier — G0 / CTI Clause Lattice Intelligence. +# +# Companion to ``verify_quotes`` for ``policy["answer_mode"] == +# "claim_lattice_pointer"``. The model emits weak natural-language +# pointer clauses (``Claim text. [E12]``); the runtime parses them +# into structured claim nodes and runs deterministic checks against +# the runtime-built evidence map. +# +# Why pointer-line, not JSON: small instruction-tuned models follow +# citation-style prose far more reliably than free-form JSON. JSON +# discipline failures generate spurious one-shot SCHEMA_INVALID +# verdicts even when the model knew the right answer; pointer-line +# stays inside the model's prose-generation distribution. +# +# Why decimal pointer ids (E1, E2, …) not hex (E1f8e4c2a): +# - One BPE token per id in standard tokenizers (8-hex tokenizes to +# 4–6 tokens of out-of-distribution noise that nudges the model +# toward DSL/code mode). +# - Citation style (footnotes, references) is heavily represented in +# training; random hex is not. +# - The runtime maps pointer ids back to content-addressed evidence +# ids (``evidence_map_by_pointer_id``) for cache, run-DAG, audit. +# +# Hard-soft boundary: only deterministic checks. No entailment, no +# completeness, no predicate compatibility. Those stay sidecar. +# --------------------------------------------------------------------------- + +ANSWER_MODES = ("quote", "claim_lattice_pointer") +DEFAULT_ANSWER_MODE = "quote" + +# Default allowed source roles for claim_lattice_pointer mode. Roles +# outside this set get classified as SOURCE_ROLE_BLOCKED. Mirrors the +# role classifications in aborist/qa/query.py:_classify_source_role; +# "noisy_background_source" and "sequel_background_source" are +# deliberately excluded by default. +DEFAULT_ALLOWED_SOURCE_ROLES = ( + "primary_answer_source", + "secondary_context_source", + "background_source", + "unclassified", +) + + +def _has_manual_quote(text: str) -> bool: + """Strict no-quote rule: ANY double-quote character in claim text + is a violation. + + The premise of claim-lattice-pointer mode is that models do not type + quote text — period. Even a 3-char quoted span (``"hi"``) is a + model-asserted verbatim citation that the runtime didn't authorize. + Catching every quote keeps the discipline honest: the model is + forbidden, not just length-discouraged. + + Covers ASCII (``"``) and curly quotes (``“`` / ``”``) — same set + ``_QUOTE_CHAR_RE`` recognizes for the legacy quote verifier. + """ + return any(ch in text for ch in ('"', '“', '”')) + + +def verify_claim_lattice( + answer_text: str, + evidence_map, + *, + allowed_source_roles: tuple[str, ...] = DEFAULT_ALLOWED_SOURCE_ROLES, +) -> dict: + """Deterministic verifier for ``answer_mode="claim_lattice_pointer"``. + + The model wrote pointer-line prose (``Claim text. [E12]``); the + parser pulled (claim_text, [pointer_ids]) pairs from each non-empty + line. This verifier maps each pointer id back to its + content-addressed evidence object and runs five hard checks: + + 1. Parser succeeded — ``parse_status == "PARSED"`` (line had a + bracket tag). NO_EVIDENCE_POINTER claims (prose without tag) + count toward the denominator and downgrade the verdict. + 2. Pointer id resolves to an entry in the runtime-built evidence + map. No model-invented ids. + 3. Resolved entry's ``source_role`` is in ``allowed_source_roles``. + 4. Claim text contains no double-quote characters anywhere (strict + no-quote rule — see ``_has_manual_quote``). + 5. Claim text non-empty after tag strip. + + Returns a verdict in the same shape as ``verify_quotes`` + extras: + + n_quotes total claim-pointer pairs (denominator) + n_verified pairs where pointer resolved AND + source_role allowed AND no manual quote + AND claim text non-empty + audit_mode STRICT / HYBRID / UNGROUNDED + unverified_quotes claim texts that didn't reach + EVIDENCE_LINKED — kept under that name + for schema continuity with verify_quotes + verifier_method "claim_lattice" + claim_statuses per-claim {text, evidence_ids, + pointer_ids, status, reasons[]}; status ∈ + {EVIDENCE_LINKED, EVIDENCE_LINKED_PARTIAL, + UNKNOWN_EVIDENCE_ID, + SOURCE_ROLE_BLOCKED, + MANUAL_QUOTE_VIOLATION, + NO_EVIDENCE_POINTER, SCHEMA_INVALID} + violations structured violation records for the + run-DAG / sidecar + rendered_text human-readable prose with literal spans + interpolated; what the runner persists + as ``answer_text`` + evidence_id_pairs per-claim list of resolved + content-addressed evidence_ids (run-stable + form). Used to thread the parsed lattice + into the run-DAG. + """ + from aborist.qa.evidence import ( + evidence_map_by_pointer_id as _by_pointer, + render_claim_lattice as _render, + ) + from aborist.qa.parse_claims import parse_pointer_claims + + by_pointer = _by_pointer(evidence_map) + violations: list[dict] = [] + claim_statuses: list[dict] = [] + unverified: list[str] = [] + + raw_claims = parse_pointer_claims(answer_text or "") + + n_pairs = 0 + n_pairs_verified = 0 + # Renderer claims: pointer-id form so the human display still shows + # the short tags the model used. + valid_claims: list[dict] = [] + # Evidence-id pairs: content-addressed form for the run-DAG & + # cache. Per-claim list so the parsed_claim_lattice node hashes the + # run-stable handle, not the run-dependent pointer-id. + evidence_id_pairs: list[list[str]] = [] + + for idx, c in enumerate(raw_claims): + claim_text = c.claim_text + pointer_ids = c.pointer_ids + parse_status = c.parse_status + + if parse_status == "NO_EVIDENCE_POINTER": + violations.append({ + "kind": "NO_EVIDENCE_POINTER", + "claim_idx": idx, + "claim_text": claim_text, + }) + claim_statuses.append({ + "claim_idx": idx, + "text": claim_text, + "pointer_ids": [], + "evidence_ids": [], + "status": "NO_EVIDENCE_POINTER", + "reasons": ["no [E\\d+] tag on line"], + }) + unverified.append(claim_text) + evidence_id_pairs.append([]) + # NO_EVIDENCE_POINTER counts as one denominator pair so the + # verdict reflects the failure rate. + n_pairs += 1 + continue + + if not claim_text: + violations.append({ + "kind": "SCHEMA_INVALID", + "claim_idx": idx, + "reason": "tag with no claim text", + }) + claim_statuses.append({ + "claim_idx": idx, + "text": "", + "pointer_ids": pointer_ids, + "evidence_ids": [], + "status": "SCHEMA_INVALID", + "reasons": ["tag with no claim text"], + }) + n_pairs += len(pointer_ids) + evidence_id_pairs.append([]) + continue + + # Strict no-quote rule: any double-quote in claim text is a + # MANUAL_QUOTE_VIOLATION. Block every pointer link on this claim. + manual_quote = _has_manual_quote(claim_text) + + per_id_results: list[dict] = [] + resolved_evidence_ids: list[str] = [] + for pid in pointer_ids: + n_pairs += 1 + obj = by_pointer.get(pid) + if obj is None: + violations.append({ + "kind": "UNKNOWN_EVIDENCE_ID", + "claim_idx": idx, + "pointer_id": pid, + }) + per_id_results.append({ + "pid": pid, "ok": False, "kind": "UNKNOWN_EVIDENCE_ID", + }) + continue + if obj.source_role not in allowed_source_roles: + violations.append({ + "kind": "SOURCE_ROLE_BLOCKED", + "claim_idx": idx, + "pointer_id": pid, + "evidence_id": obj.evidence_id, + "source_role": obj.source_role, + }) + per_id_results.append({ + "pid": pid, "ok": False, "kind": "SOURCE_ROLE_BLOCKED", + }) + continue + if manual_quote: + per_id_results.append({ + "pid": pid, "ok": False, "kind": "MANUAL_QUOTE_VIOLATION", + }) + continue + per_id_results.append({"pid": pid, "ok": True, "evidence_id": obj.evidence_id}) + resolved_evidence_ids.append(obj.evidence_id) + n_pairs_verified += 1 + + if manual_quote: + violations.append({ + "kind": "MANUAL_QUOTE_VIOLATION", + "claim_idx": idx, + "claim_text": claim_text, + }) + + ok_pids = [r["pid"] for r in per_id_results if r["ok"]] + bad_kinds = sorted({r["kind"] for r in per_id_results if not r["ok"]}) + + if manual_quote: + status = "MANUAL_QUOTE_VIOLATION" + elif ok_pids and not bad_kinds: + status = "EVIDENCE_LINKED" + elif ok_pids: + status = "EVIDENCE_LINKED_PARTIAL" + elif "UNKNOWN_EVIDENCE_ID" in bad_kinds: + status = "UNKNOWN_EVIDENCE_ID" + elif "SOURCE_ROLE_BLOCKED" in bad_kinds: + status = "SOURCE_ROLE_BLOCKED" + else: + status = "SCHEMA_INVALID" + + claim_statuses.append({ + "claim_idx": idx, + "text": claim_text, + "pointer_ids": pointer_ids, + "evidence_ids": resolved_evidence_ids, + "status": status, + "reasons": bad_kinds, + }) + evidence_id_pairs.append(resolved_evidence_ids) + if status in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"): + valid_claims.append({"text": claim_text, "pointer_ids": ok_pids}) + if status != "EVIDENCE_LINKED": + unverified.append(claim_text) + + rendered_text = _render(valid_claims, by_pointer) if valid_claims else "" + + # Aggregate. STRICT requires ≥1 verified pair AND zero violations of + # any kind (schema, unknown pointer, blocked role, manual quote, + # missing pointer). HYBRID = some pairs verified, some failed. + # UNGROUNDED = no verified pairs (no parseable claims, or every + # claim failed at least one check). + if n_pairs_verified > 0 and not violations: + audit_mode = "STRICT" + elif n_pairs_verified > 0: + audit_mode = "HYBRID" + else: + audit_mode = "UNGROUNDED" + + return { + "n_quotes": n_pairs, + "n_verified": n_pairs_verified, + "audit_mode": audit_mode, + "unverified_quotes": unverified, + "verifier_method": "claim_lattice", + "claim_statuses": claim_statuses, + "violations": violations, + "rendered_text": rendered_text, + "evidence_id_pairs": evidence_id_pairs, + } diff --git a/aborist/store.py b/aborist/store.py index 890acfb..79ef882 100644 --- a/aborist/store.py +++ b/aborist/store.py @@ -184,10 +184,13 @@ CREATE TABLE IF NOT EXISTS providence_cache ( -- Which verifier strategy classified this record. 'quote' = model -- followed the format and wrapped claims in double quotes. 'span' = -- bullet/sentence-level substring match in context. 'entity' = no - -- spans matched but multi-word proper nouns did. 'none' = no + -- spans matched but multi-word proper nouns did. 'paraphrase' = + -- token-coverage match (soft signal). 'claim_lattice' = quote-by- + -- pointer mode (model emitted JSON; verifier checked evidence_id + -- resolution + source_role + manual-quote prohibition). 'none' = no -- evidence at all (truly emergent). verifier_method TEXT NOT NULL DEFAULT 'none' - CHECK (verifier_method IN ('quote','span','entity','paraphrase','none')), + CHECK (verifier_method IN ('quote','span','entity','paraphrase','claim_lattice','none')), -- Per-run Merkle-DAG. run_dag_root = MerkleTree over ordered stage -- hashes (question / retrieval / context / prompt / answer / verify / -- final_label). run_dag_blob carries the full {root, nodes} JSON so @@ -404,6 +407,13 @@ def _migrate_audit_mode(conn: sqlite3.Connection) -> None: if ddl_row and "'paraphrase'" not in (ddl_row[0] or ""): _rebuild_providence_cache_paraphrase(conn) + # claim_lattice verifier_method addition (G0 — quote-by-pointer mode). + ddl_row = conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='providence_cache'" + ).fetchone() + if ddl_row and "'claim_lattice'" not in (ddl_row[0] or ""): + _rebuild_providence_cache_claim_lattice(conn) + conn.execute( "CREATE INDEX IF NOT EXISTS idx_providence_audit " "ON providence_cache(audit_mode)" @@ -533,6 +543,86 @@ def _rebuild_providence_cache_paraphrase(conn: sqlite3.Connection) -> None: raise +def _rebuild_providence_cache_claim_lattice(conn: sqlite3.Connection) -> None: + """One-time table rebuild: expand verifier_method CHECK to include + 'claim_lattice' (quote-by-pointer answer mode added in G0). + + Same pattern as the paraphrase / ungrounded rebuilds: SQLite cannot + modify a CHECK constraint in place, so we create a new table with + the expanded CHECK, copy the data verbatim (claim_lattice is + additive — no value translation needed), drop the old, rename the + new. Caller probes existing CHECK from sqlite_master and only + invokes this when 'claim_lattice' is missing. + + Unlike the older rebuilds this one preserves ``run_dag_root`` & + ``run_dag_blob`` columns in the copy. By the time a shard reaches + this migration those columns may already be populated; dropping + them would erase per-run computation provenance. + """ + new_create = """ + CREATE TABLE providence_cache_new ( + cache_key TEXT PRIMARY KEY, + source_root TEXT NOT NULL, + document_uri TEXT NOT NULL, + question_hash TEXT NOT NULL, + question_text TEXT NOT NULL, + answer_text TEXT NOT NULL, + merkle_proof TEXT NOT NULL, + model_profile_hash TEXT NOT NULL, + conversation_hash TEXT NOT NULL, + governance_policy_hash TEXT NOT NULL, + schema_version TEXT NOT NULL, + canonicalization_version TEXT NOT NULL, + chunking_version TEXT NOT NULL, + falsification_state TEXT NOT NULL DEFAULT 'live' + CHECK (falsification_state IN ('live','failed','stale','quarantined')), + chain TEXT NOT NULL DEFAULT 'private' + CHECK (chain IN ('private','public')), + audit_event_hash TEXT, + created_at INTEGER NOT NULL, + last_hit_at INTEGER, + hit_count INTEGER NOT NULL DEFAULT 0, + audit_mode TEXT NOT NULL DEFAULT 'UNGROUNDED' + CHECK (audit_mode IN ('STRICT','HYBRID','UNGROUNDED')), + n_quotes INTEGER NOT NULL DEFAULT 0, + n_verified INTEGER NOT NULL DEFAULT 0, + unverified_quotes TEXT, + verifier_method TEXT NOT NULL DEFAULT 'none' + CHECK (verifier_method IN ('quote','span','entity','paraphrase','claim_lattice','none')), + run_dag_root TEXT, + run_dag_blob TEXT + ) + """ + conn.execute("BEGIN IMMEDIATE") + try: + conn.execute(new_create) + conn.execute( + "INSERT INTO providence_cache_new " + "(cache_key, source_root, document_uri, question_hash, question_text, " + " answer_text, merkle_proof, model_profile_hash, conversation_hash, " + " governance_policy_hash, schema_version, canonicalization_version, " + " chunking_version, falsification_state, chain, audit_event_hash, " + " created_at, last_hit_at, hit_count, audit_mode, n_quotes, " + " n_verified, unverified_quotes, verifier_method, " + " run_dag_root, run_dag_blob) " + "SELECT " + " cache_key, source_root, document_uri, question_hash, question_text, " + " answer_text, merkle_proof, model_profile_hash, conversation_hash, " + " governance_policy_hash, schema_version, canonicalization_version, " + " chunking_version, falsification_state, chain, audit_event_hash, " + " created_at, last_hit_at, hit_count, audit_mode, n_quotes, " + " n_verified, unverified_quotes, verifier_method, " + " run_dag_root, run_dag_blob " + "FROM providence_cache" + ) + conn.execute("DROP TABLE providence_cache") + conn.execute("ALTER TABLE providence_cache_new RENAME TO providence_cache") + conn.execute("COMMIT") + except Exception: + conn.execute("ROLLBACK") + raise + + def _rebuild_providence_cache_ungrounded(conn: sqlite3.Connection) -> None: """One-time table rebuild: rename audit_mode value VISUAL → UNGROUNDED. diff --git a/tests/test_claim_lattice.py b/tests/test_claim_lattice.py new file mode 100644 index 0000000..4b319ef --- /dev/null +++ b/tests/test_claim_lattice.py @@ -0,0 +1,922 @@ +"""G0 / CTI — claim-lattice-pointer mode (quote-by-pointer). + +Covers the pointer-line protocol Hermes emits, plus the runtime +two-layer id mapping (prompt-facing E1/E2/…, content-addressed +E########). Tests the deterministic verifier + 9-stage run-DAG + +schema migration round-trip. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from aborist.qa.dag import build_run_dag, verify_run_dag +from aborist.qa.evidence import ( + build_evidence_map, + evidence_map_by_evidence_id, + evidence_map_by_pointer_id, + evidence_map_root, + render_claim_lattice, + render_evidence_map, +) +from aborist.qa.keys import governance_policy_hash +from aborist.qa.parse_claims import parse_pointer_claims +from aborist.qa.runner import DEFAULT_POLICY +from aborist.qa.query import DEFAULT_QUERY_POLICY +from aborist.qa.verify import ( + ANSWER_MODES, + DEFAULT_ANSWER_MODE, + verify_claim_lattice, +) + + +# ---------- evidence map ---------------------------------------------------- + + +def _sample_chunks() -> list[dict]: + """Three chunks across two sources, two source roles.""" + return [ + { + "source_root": "a" * 64, + "document_uri": "https://example.org/jp", + "title": "Jurassic Park (film)", + "chunk_idx": 0, + "chunk_root": "11" * 32, + "span": "Tyrannosaurus rex appears in the climactic scene.", + "source_role": "primary_answer_source", + }, + { + "source_root": "a" * 64, + "document_uri": "https://example.org/jp", + "title": "Jurassic Park (film)", + "chunk_idx": 1, + "chunk_root": "22" * 32, + "span": "Velociraptors are shown attacking workers.", + "source_role": "primary_answer_source", + }, + { + "source_root": "b" * 64, + "document_uri": "https://example.org/jp-game", + "title": "Jurassic Park (video game)", + "chunk_idx": 0, + "chunk_root": "33" * 32, + "span": "The Sega Genesis adaptation was released in 1993.", + "source_role": "noisy_background_source", + }, + ] + + +def test_pointer_id_is_sequential(): + em = build_evidence_map(_sample_chunks()) + assert [e.pointer_id for e in em] == ["E1", "E2", "E3"] + + +def test_evidence_id_is_content_addressed_and_stable(): + chunks = _sample_chunks() + map1 = build_evidence_map(chunks) + map2 = build_evidence_map(chunks) + # Content-addressed evidence_ids stay stable across runs. + assert [e.evidence_id for e in map1] == [e.evidence_id for e in map2] + assert all(e.evidence_id.startswith("E") and len(e.evidence_id) == 9 + for e in map1) + # Pointer ids are deterministic too (position-based). + assert [e.pointer_id for e in map1] == [e.pointer_id for e in map2] + + +def test_pointer_id_is_run_dependent_when_chunks_reorder(): + chunks = _sample_chunks() + map1 = build_evidence_map(chunks) + map2 = build_evidence_map(list(reversed(chunks))) + # Chunk B's content-addressed evidence_id stays stable... + assert {e.chunk_root: e.evidence_id for e in map1} == \ + {e.chunk_root: e.evidence_id for e in map2} + # ...but its pointer_id moves with position. + assert {e.chunk_root: e.pointer_id for e in map1} != \ + {e.chunk_root: e.pointer_id for e in map2} + + +def test_evidence_map_root_order_independent(): + chunks = _sample_chunks() + r1 = evidence_map_root(build_evidence_map(chunks)) + r2 = evidence_map_root(build_evidence_map(list(reversed(chunks)))) + assert r1 == r2 + assert r1 != "00" * 32 + + +def test_render_evidence_map_uses_pointer_ids(): + em = build_evidence_map(_sample_chunks()) + text = render_evidence_map(em) + for e in em: + assert f"=== {e.pointer_id} " in text + # Hex evidence_id must NOT leak into the prompt. + assert e.evidence_id not in text + assert e.span in text + + +# ---------- parser --------------------------------------------------------- + + +def test_parse_pointer_claims_basic(): + out = parse_pointer_claims( + "Trex appears in the film. [E1]\n" + "Velociraptors stalk workers. [E2,E3]\n" + ) + assert len(out) == 2 + assert out[0].claim_text == "Trex appears in the film." + assert out[0].pointer_ids == ["E1"] + assert out[0].parse_status == "PARSED" + assert out[1].pointer_ids == ["E2", "E3"] + + +def test_parse_handles_bullet_markers(): + out = parse_pointer_claims("- Trex appears. [E1]\n") + assert out[0].claim_text == "Trex appears." + + +def test_parse_handles_whitespace_inside_brackets(): + out = parse_pointer_claims("Trex appears. [E1, E2 ,E3]\n") + assert out[0].pointer_ids == ["E1", "E2", "E3"] + + +def test_parse_skips_blank_lines(): + out = parse_pointer_claims("\n \nTrex appears. [E1]\n\n") + assert len(out) == 1 + + +def test_parse_no_evidence_pointer_status(): + out = parse_pointer_claims("Trex appears in the film.\n") + assert out[0].parse_status == "NO_EVIDENCE_POINTER" + assert out[0].pointer_ids == [] + assert out[0].claim_text == "Trex appears in the film." + + +def test_parse_inline_tag_preserves_prose(): + out = parse_pointer_claims("Trex [E1] appears in the film.\n") + # Tag stripped; prose around it stays. + assert "Trex" in out[0].claim_text + assert "appears in the film" in out[0].claim_text + assert out[0].pointer_ids == ["E1"] + + +# ---------- verifier: happy path ------------------------------------------ + + +def test_verify_strict_when_all_resolve(): + em = build_evidence_map(_sample_chunks()) + answer = ( + "Trex appears in the film. [E1]\n" + "Velociraptors are shown. [E2]\n" + ) + v = verify_claim_lattice(answer, em) + assert v["audit_mode"] == "STRICT" + assert v["verifier_method"] == "claim_lattice" + assert v["n_quotes"] == 2 + assert v["n_verified"] == 2 + assert v["violations"] == [] + # Evidence_id pairs use the content-addressed form for run-DAG. + pairs = v["evidence_id_pairs"] + assert pairs[0] == [em[0].evidence_id] + assert pairs[1] == [em[1].evidence_id] + + +def test_verify_renders_with_literal_spans_from_runtime(): + em = build_evidence_map(_sample_chunks()) + answer = "Trex is in the film. [E1]\n" + v = verify_claim_lattice(answer, em) + # Renderer pulls literal span from runtime, not from model output. + assert em[0].span in v["rendered_text"] + assert "[E1:" in v["rendered_text"] + + +def test_verify_passes_multi_pointer_claims(): + em = build_evidence_map(_sample_chunks()) + answer = "Trex and raptors appear. [E1,E2]\n" + v = verify_claim_lattice(answer, em) + assert v["audit_mode"] == "STRICT" + assert v["n_quotes"] == 2 + assert v["n_verified"] == 2 + + +# ---------- verifier: failure modes --------------------------------------- + + +def test_unknown_pointer_id_is_violation(): + em = build_evidence_map(_sample_chunks()) + answer = "Imaginary claim. [E99]\n" + v = verify_claim_lattice(answer, em) + assert v["audit_mode"] == "UNGROUNDED" + assert v["n_verified"] == 0 + assert any(viol["kind"] == "UNKNOWN_EVIDENCE_ID" for viol in v["violations"]) + assert v["claim_statuses"][0]["status"] == "UNKNOWN_EVIDENCE_ID" + + +def test_source_role_blocked_violation(): + em = build_evidence_map(_sample_chunks()) + # E3 → noisy_background_source, disallowed by default. + answer = "Sega adaptation exists. [E3]\n" + v = verify_claim_lattice(answer, em) + assert v["audit_mode"] == "UNGROUNDED" + assert any(viol["kind"] == "SOURCE_ROLE_BLOCKED" for viol in v["violations"]) + assert v["claim_statuses"][0]["status"] == "SOURCE_ROLE_BLOCKED" + + +def test_strict_no_quote_rule_any_double_quote_violates(): + """Even a 4-char quoted span triggers — model is forbidden to type + quote characters at all in pointer mode.""" + em = build_evidence_map(_sample_chunks()) + answer = 'The "T-rex" appears. [E1]\n' + v = verify_claim_lattice(answer, em) + assert v["audit_mode"] == "UNGROUNDED" + assert any(viol["kind"] == "MANUAL_QUOTE_VIOLATION" + for viol in v["violations"]) + assert v["claim_statuses"][0]["status"] == "MANUAL_QUOTE_VIOLATION" + + +def test_curly_quotes_also_violate(): + em = build_evidence_map(_sample_chunks()) + answer = "The “T-rex” appears. [E1]\n" + v = verify_claim_lattice(answer, em) + assert v["audit_mode"] == "UNGROUNDED" + assert any(viol["kind"] == "MANUAL_QUOTE_VIOLATION" + for viol in v["violations"]) + + +def test_no_evidence_pointer_downgrades(): + em = build_evidence_map(_sample_chunks()) + answer = ( + "Trex appears in the film.\n" # no pointer + "Velociraptors are shown. [E2]\n" + ) + v = verify_claim_lattice(answer, em) + # Mixed: one verified, one NO_EVIDENCE_POINTER → HYBRID. + assert v["audit_mode"] == "HYBRID" + assert any(viol["kind"] == "NO_EVIDENCE_POINTER" for viol in v["violations"]) + statuses = {c["status"] for c in v["claim_statuses"]} + assert "NO_EVIDENCE_POINTER" in statuses + assert "EVIDENCE_LINKED" in statuses + + +def test_partial_resolution_yields_hybrid(): + em = build_evidence_map(_sample_chunks()) + answer = ( + "Trex appears. [E1]\n" + "Made-up. [E99]\n" + ) + v = verify_claim_lattice(answer, em) + assert v["audit_mode"] == "HYBRID" + assert v["n_verified"] == 1 + assert v["n_quotes"] == 2 + + +def test_partial_pointer_within_claim_yields_partial(): + em = build_evidence_map(_sample_chunks()) + answer = "Trex appears. [E1,E99]\n" + v = verify_claim_lattice(answer, em) + assert v["audit_mode"] == "HYBRID" + assert v["claim_statuses"][0]["status"] == "EVIDENCE_LINKED_PARTIAL" + # Renderer keeps only resolved ids. + assert "E1:" in v["rendered_text"] + assert "E99" not in v["rendered_text"] + + +def test_empty_response_yields_ungrounded(): + em = build_evidence_map(_sample_chunks()) + v = verify_claim_lattice("", em) + assert v["audit_mode"] == "UNGROUNDED" + assert v["n_verified"] == 0 + assert v["claim_statuses"] == [] + assert v["evidence_id_pairs"] == [] + + +def test_non_pointer_response_yields_ungrounded(): + em = build_evidence_map(_sample_chunks()) + # Model wrote prose without any pointer tags. + v = verify_claim_lattice( + "I don't know based on the provided sources.", em + ) + assert v["audit_mode"] == "UNGROUNDED" + assert v["n_verified"] == 0 + # Each non-empty line without a tag becomes a NO_EVIDENCE_POINTER claim. + assert v["claim_statuses"][0]["status"] == "NO_EVIDENCE_POINTER" + + +def test_orphan_tag_with_no_text_yields_schema_invalid(): + em = build_evidence_map(_sample_chunks()) + answer = "[E1]\n" + v = verify_claim_lattice(answer, em) + assert v["audit_mode"] == "UNGROUNDED" + assert v["claim_statuses"][0]["status"] == "SCHEMA_INVALID" + + +# ---------- policy & cache aliasing --------------------------------------- + + +def test_default_answer_mode_is_quote(): + assert DEFAULT_POLICY["answer_mode"] == "quote" + assert DEFAULT_QUERY_POLICY["answer_mode"] == "quote" + assert DEFAULT_ANSWER_MODE == "quote" + assert "claim_lattice_pointer" in ANSWER_MODES + assert "quote" in ANSWER_MODES + + +def test_governance_hash_differs_between_modes(): + p_quote = dict(DEFAULT_POLICY, answer_mode="quote") + p_lattice = dict(DEFAULT_POLICY, answer_mode="claim_lattice_pointer") + assert governance_policy_hash(p_quote) != governance_policy_hash(p_lattice) + + +# ---------- run-DAG ------------------------------------------------------- + + +def test_run_dag_quote_mode_is_seven_stages(): + common = dict( + question_hash="11" * 32, + sources=[{"document_root": "aa" * 32, "source_role": "primary_answer_source"}], + context_root="bb" * 32, + conversation_hash="cc" * 32, + answer_text="hello", + audit_mode="STRICT", + verifier_method="quote", + n_quotes=1, + n_verified=1, + claim_statuses=[], + lookup_path="miss", + ) + dag = build_run_dag(**common) + stages = [n["stage"] for n in dag["nodes"]] + assert stages == [ + "question", "retrieval", "context", "prompt", + "answer", "verify", "final_label", + ] + assert verify_run_dag(dag) is True + + +def test_run_dag_pointer_mode_is_nine_stages(): + common = dict( + question_hash="11" * 32, + sources=[{"document_root": "aa" * 32, "source_role": "primary_answer_source"}], + context_root="bb" * 32, + conversation_hash="cc" * 32, + answer_text="rendered prose", + audit_mode="STRICT", + verifier_method="claim_lattice", + n_quotes=1, + n_verified=1, + claim_statuses=[], + lookup_path="miss", + evidence_map_root="dd" * 32, + answer_mode="claim_lattice_pointer", + violations=[], + raw_answer_text="Claim. [E1]", + parsed_lattice=[{"claim_text": "Claim.", "evidence_ids": ["E12345678"]}], + rendered_text="rendered prose", + ) + dag = build_run_dag(**common) + stages = [n["stage"] for n in dag["nodes"]] + assert stages == [ + "question", "retrieval", "evidence_map", "prompt", + "raw_answer", "parsed_claim_lattice", "verify", "render", + "final_label", + ] + # No 'context' or single 'answer' nodes in pointer mode. + assert "context" not in stages + assert "answer" not in stages + assert verify_run_dag(dag) is True + + +def test_run_dag_root_changes_when_evidence_map_changes(): + base = dict( + question_hash="11" * 32, + sources=[{"document_root": "aa" * 32, "source_role": "primary_answer_source"}], + context_root="bb" * 32, + conversation_hash="cc" * 32, + answer_text="hello", + audit_mode="STRICT", + verifier_method="claim_lattice", + n_quotes=1, + n_verified=1, + claim_statuses=[], + lookup_path="miss", + answer_mode="claim_lattice_pointer", + violations=[], + raw_answer_text="raw", + parsed_lattice=[], + rendered_text="hello", + ) + a = build_run_dag(**base, evidence_map_root="dd" * 32) + b = build_run_dag(**base, evidence_map_root="ee" * 32) + assert a["root"] != b["root"] + + +def test_run_dag_root_changes_when_parsed_lattice_changes(): + base = dict( + question_hash="11" * 32, + sources=[{"document_root": "aa" * 32, "source_role": "primary_answer_source"}], + context_root="bb" * 32, + conversation_hash="cc" * 32, + answer_text="hello", + audit_mode="STRICT", + verifier_method="claim_lattice", + n_quotes=1, + n_verified=1, + claim_statuses=[], + lookup_path="miss", + evidence_map_root="dd" * 32, + answer_mode="claim_lattice_pointer", + violations=[], + raw_answer_text="raw", + rendered_text="hello", + ) + a = build_run_dag(**base, parsed_lattice=[{"claim_text": "x", "evidence_ids": ["E12345678"]}]) + b = build_run_dag(**base, parsed_lattice=[{"claim_text": "y", "evidence_ids": ["E12345678"]}]) + assert a["root"] != b["root"] + + +# ---------- schema / migration -------------------------------------------- + + +def test_verifier_method_check_accepts_claim_lattice(tmp_path: Path): + """Fresh shards include 'claim_lattice' in the verifier_method CHECK.""" + from aborist.store import connect + + db = tmp_path / "fresh.db" + conn = connect(db) + try: + ddl = conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='providence_cache'" + ).fetchone()[0] + assert "'claim_lattice'" in ddl + finally: + conn.close() + + +def test_legacy_shard_migrates_in_place(tmp_path: Path): + """An old shard with the pre-claim_lattice CHECK gets rebuilt on open + AND keeps run_dag_root / run_dag_blob populated.""" + from aborist.store import connect + + db = tmp_path / "legacy.db" + c = sqlite3.connect(db) + c.executescript( + """ + PRAGMA journal_mode=WAL; + CREATE TABLE providence_cache ( + cache_key TEXT PRIMARY KEY, + source_root TEXT NOT NULL, + document_uri TEXT NOT NULL, + question_hash TEXT NOT NULL, + question_text TEXT NOT NULL, + answer_text TEXT NOT NULL, + merkle_proof TEXT NOT NULL, + model_profile_hash TEXT NOT NULL, + conversation_hash TEXT NOT NULL, + governance_policy_hash TEXT NOT NULL, + schema_version TEXT NOT NULL, + canonicalization_version TEXT NOT NULL, + chunking_version TEXT NOT NULL, + falsification_state TEXT NOT NULL DEFAULT 'live' + CHECK (falsification_state IN ('live','failed','stale','quarantined')), + chain TEXT NOT NULL DEFAULT 'private' + CHECK (chain IN ('private','public')), + audit_event_hash TEXT, + created_at INTEGER NOT NULL, + last_hit_at INTEGER, + hit_count INTEGER NOT NULL DEFAULT 0, + audit_mode TEXT NOT NULL DEFAULT 'UNGROUNDED' + CHECK (audit_mode IN ('STRICT','HYBRID','UNGROUNDED')), + n_quotes INTEGER NOT NULL DEFAULT 0, + n_verified INTEGER NOT NULL DEFAULT 0, + unverified_quotes TEXT, + verifier_method TEXT NOT NULL DEFAULT 'none' + CHECK (verifier_method IN ('quote','span','entity','paraphrase','none')), + run_dag_root TEXT, + run_dag_blob TEXT + ); + INSERT INTO providence_cache + (cache_key, source_root, document_uri, question_hash, question_text, + answer_text, merkle_proof, model_profile_hash, conversation_hash, + governance_policy_hash, schema_version, canonicalization_version, + chunking_version, audit_event_hash, created_at, + run_dag_root, run_dag_blob) + VALUES + ('ck1','sr1','u','qh','q','a','{}','mh','ch','gh','v','c','c1','eh', + 1, 'rdr', '{"root":"rdr","nodes":[]}'); + """ + ) + c.commit() + c.close() + + conn = connect(db) + try: + ddl = conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='providence_cache'" + ).fetchone()[0] + assert "'claim_lattice'" in ddl + row = conn.execute( + "SELECT run_dag_root, run_dag_blob FROM providence_cache WHERE cache_key=?", + ("ck1",), + ).fetchone() + assert row[0] == "rdr" + assert row[1] == '{"root":"rdr","nodes":[]}' + finally: + conn.close() + + +# ---------- soft-signal isolation ----------------------------------------- + + +def test_no_entailment_field_in_verify_output(): + """Hard verifier must not assert semantic claims.""" + em = build_evidence_map(_sample_chunks()) + answer = "Trex appears. [E1]\n" + v = verify_claim_lattice(answer, em) + forbidden = {"entailment", "entailed", "completeness", + "predicate_compatibility", "scope_match"} + assert not (set(v) & forbidden) + for c in v["claim_statuses"]: + assert not (set(c) & forbidden) + + +# ---------- evidence_map index helpers ------------------------------------ + + +def test_evidence_map_by_pointer_id_index(): + em = build_evidence_map(_sample_chunks()) + by_pid = evidence_map_by_pointer_id(em) + assert set(by_pid) == {"E1", "E2", "E3"} + assert by_pid["E1"] is em[0] + + +def test_evidence_map_by_evidence_id_index(): + em = build_evidence_map(_sample_chunks()) + by_eid = evidence_map_by_evidence_id(em) + assert set(by_eid) == {e.evidence_id for e in em} + # Ids are 9 chars: "E" + 8 hex. + assert all(len(eid) == 9 for eid in by_eid) + + +def test_render_claim_lattice_with_pointer_ids(): + em = build_evidence_map(_sample_chunks()) + by_pid = evidence_map_by_pointer_id(em) + text = render_claim_lattice( + [ + {"text": "trex appears", "pointer_ids": ["E1"]}, + {"text": "raptors attack", "pointer_ids": ["E2"]}, + ], + by_pid, + ) + assert "Tyrannosaurus rex appears" in text + assert "Velociraptors are shown" in text + assert "[E1:" in text + assert "[E2:" in text + + +def test_render_claim_lattice_marks_unknown(): + em = build_evidence_map(_sample_chunks()) + by_pid = evidence_map_by_pointer_id(em) + text = render_claim_lattice( + [{"text": "fake", "pointer_ids": ["E99"]}], + by_pid, + ) + assert "E99: ?" in text + + +# ---------- spotlight excerpt (G0.2) -------------------------------------- + + +def _long_chunk_with_buried_term() -> list[dict]: + """Single chunk much longer than the spotlight window with the + target term ('Brachiosaurus') buried mid-span.""" + intro = "Jurassic Park is a 1993 American science fiction thriller film. " * 30 + middle = ( + "Among the dinosaurs depicted on screen, Brachiosaurus is shown " + "lifting its long neck to eat from tall trees in an early scene." + ) + tail = "Production took place across multiple studios. " * 30 + long_span = intro + middle + tail + return [{ + "source_root": "a" * 64, + "document_uri": "https://example.org/jp", + "title": "Jurassic Park (film)", + "chunk_idx": 0, + "chunk_root": "11" * 32, + "span": long_span, + "source_role": "primary_answer_source", + }] + + +def test_spotlight_finds_buried_topic_token(): + em = build_evidence_map(_long_chunk_with_buried_term()) + by_pid = evidence_map_by_pointer_id(em) + text = render_claim_lattice( + [{"text": "Brachiosaurus appears in the film.", + "pointer_ids": ["E1"]}], + by_pid, + ) + # The displayed excerpt must contain the actual brachiosaurus + # mention from mid-span, not the leading article-intro sentence. + assert "Brachiosaurus is shown lifting" in text + + +def test_spotlight_falls_back_when_no_token_matches(): + em = build_evidence_map(_long_chunk_with_buried_term()) + by_pid = evidence_map_by_pointer_id(em) + # Claim contains no content tokens that appear in the span. + text = render_claim_lattice( + [{"text": "Quokka spotted yesterday morning.", + "pointer_ids": ["E1"]}], + by_pid, + ) + # Falls back to leading window with trailing ellipsis. + assert text.rstrip().endswith('..."]') + # And the leading content of the span is shown. + assert "Jurassic Park is a 1993" in text + + +def test_spotlight_no_truncation_for_short_spans(): + em = build_evidence_map(_sample_chunks()) + by_pid = evidence_map_by_pointer_id(em) + text = render_claim_lattice( + [{"text": "trex", "pointer_ids": ["E1"]}], + by_pid, + ) + # Sample span is <100 chars, fits in window; rendered intact. + assert em[0].span in text + + +def test_spotlight_window_size_overridable(): + em = build_evidence_map(_long_chunk_with_buried_term()) + by_pid = evidence_map_by_pointer_id(em) + narrow = render_claim_lattice( + [{"text": "Brachiosaurus appears in the film.", + "pointer_ids": ["E1"]}], + by_pid, + window=80, + ) + wide = render_claim_lattice( + [{"text": "Brachiosaurus appears in the film.", + "pointer_ids": ["E1"]}], + by_pid, + window=400, + ) + # Both windows hit the buried mention. + assert "Brachiosaurus" in narrow + assert "Brachiosaurus" in wide + # Wider window carries more surrounding prose. + assert len(wide) > len(narrow) + + +def test_spotlight_picks_longest_token_for_specificity(): + """Claim has both a generic ('film') and a specific ('Brachiosaurus') + content token. Spotlight should pick the specific one — sorted by + length desc — so the displayed excerpt anchors on what the claim + is actually *about*.""" + em = build_evidence_map(_long_chunk_with_buried_term()) + by_pid = evidence_map_by_pointer_id(em) + text = render_claim_lattice( + [{"text": "Brachiosaurus appears in the film.", + "pointer_ids": ["E1"]}], + by_pid, + ) + # Window is centered on Brachiosaurus, not on the first occurrence + # of "film" (which is in the intro at offset 0). + assert "Brachiosaurus is shown" in text + + +def test_spotlight_deterministic(): + em = build_evidence_map(_long_chunk_with_buried_term()) + by_pid = evidence_map_by_pointer_id(em) + args = ( + [{"text": "Brachiosaurus appears in the film.", + "pointer_ids": ["E1"]}], + by_pid, + ) + a = render_claim_lattice(*args) + b = render_claim_lattice(*args) + assert a == b + + +# ---------- G0.1: per-chunk evidence granularity -------------------------- + + +def test_per_chunk_evidence_map_query_path(tmp_path): + """A single source with three chunks should produce three evidence + objects in the prompt-facing map — not one whole-doc entry.""" + from typing import Iterator + from aborist.document import Document + from aborist.ingest import ingest_source + from aborist.qa.client import StubClient + from aborist.qa.query import DEFAULT_QUERY_POLICY, query + from aborist.source import Source + from aborist.store import connect + + class FakeSource(Source): + source_type = "test" + def __init__(self, docs): self.docs = docs + def iter_documents(self) -> Iterator[Document]: + yield from self.docs + + # Long enough to chunk into multiple pieces under tok-512-v1. + txt = ( + "Paragraph one introduces Brachiosaurus as a long-necked sauropod. " + * 80 + + "Paragraph two introduces Velociraptor as a small carnivorous theropod. " + * 80 + + "Paragraph three introduces Tyrannosaurus rex as the apex predator. " + * 80 + ) + shard = tmp_path / "shard.db" + qa_db = tmp_path / "qa.db" + src_conn = connect(shard) + try: + ingest_source(src_conn, FakeSource([ + Document(uri="t://chunked", content=txt, source_type="test", title="Chunked") + ])) + finally: + src_conn.close() + + # Stub answer that hits all three pointer ids — proves the + # evidence map exposed >1 chunk to the model. + stub = StubClient(answer=( + "Brachiosaurus is a sauropod. [E1]\n" + "Velociraptor is a theropod. [E2]\n" + "T-rex is apex. [E3]\n" + )) + policy = dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice_pointer") + result = query( + question="dinosaur paragraphs", + qa_db=qa_db, + chat_client=stub, + model_id="stub", + single_db=shard, + top_k=4, + over_fetch=8, + max_context_chars=60000, + policy=policy, + ) + # All three pointers resolved → STRICT. + assert result["audit_mode"] == "STRICT" + assert result["n_quotes"] == 3 + assert result["n_verified"] == 3 + + +def test_chunk_query_relevance_scores_by_overlap(): + from aborist.qa.query import _chunk_query_relevance + + qstem = {"brachiosaurus", "film"} + # Many distinct + many mentions wins. + a = _chunk_query_relevance( + "Brachiosaurus appears in the film. The film features Brachiosaurus.", + qstem, + ) + # Only one of the two query tokens present. + b = _chunk_query_relevance( + "The film opens with credits and a logo.", qstem, + ) + # Neither token present. + c = _chunk_query_relevance( + "Production took place across multiple studios.", qstem, + ) + assert a > b > c + assert a[0] == 2 and a[1] >= 3 + assert b[0] == 1 + assert c == (0, 0) + + +def test_chunk_query_relevance_stem_aware(): + from aborist.qa.query import _chunk_query_relevance + + # Query has the plural "dinosaurs"; chunk has the singular "dinosaur". + # The stem strip in _body_count_with_stem should still match. + qstem = {"dinosaur"} # already stemmed + score = _chunk_query_relevance( + "Each dinosaur was modeled with animatronics.", qstem + ) + assert score[0] == 1 + + +def test_query_orders_chunks_by_relevance_g03(tmp_path): + """End-to-end: a source whose query-relevant chunk is in the MIDDLE + of the document — G0.3 must promote it to E1 so a model that + lazy-anchors on early pointers still cites the relevant chunk.""" + from typing import Iterator + from aborist.document import Document + from aborist.ingest import ingest_source + from aborist.qa.client import StubClient + from aborist.qa.evidence import build_evidence_map + from aborist.qa.query import ( + DEFAULT_QUERY_POLICY, _load_doc_chunks, _chunk_query_relevance, + ) + from aborist.qa.query import _stem_token_for_match, _title_query_tokens + from aborist.source import Source + from aborist.store import connect + + class FakeSource(Source): + source_type = "test" + def __init__(self, docs): self.docs = docs + def iter_documents(self) -> Iterator[Document]: + yield from self.docs + + # Three chunks worth of content; the middle one mentions the + # rare term. tok-512-v1 chunks ~500 tokens (~2000 chars) each. + irrelevant = "Production notes and scheduling details. " * 80 + relevant = "The Brachiosaurus appears in a key scene. " * 80 + more_irrelevant = "Cast and crew biographies follow. " * 80 + txt = irrelevant + relevant + more_irrelevant + + shard = tmp_path / "shard.db" + qa_db = tmp_path / "qa.db" + src_conn = connect(shard) + try: + ingest_source(src_conn, FakeSource([ + Document(uri="t://reorder", content=txt, source_type="test", title="r") + ])) + doc_root = src_conn.execute( + "SELECT document_root FROM documents WHERE document_uri='t://reorder'" + ).fetchone()["document_root"] + finally: + src_conn.close() + + chunks = _load_doc_chunks(str(shard), doc_root) + assert chunks is not None + assert len(chunks) >= 3 + + # Mirror what query() does: rank chunks by query-relevance. + qstem = { + _stem_token_for_match(t.lower()) + for t in _title_query_tokens("Brachiosaurus appearance") + } + scored = [] + for idx, leaf, span in chunks: + d, t = _chunk_query_relevance(span, qstem) + scored.append((idx, leaf, span, d, t)) + scored.sort(key=lambda r: (-r[3], -r[4], r[0])) + + # The first chunk after sorting MUST be the one containing + # Brachiosaurus — even though it's not chunk_idx=0 in doc order. + assert "Brachiosaurus" in scored[0][2] + # Document-order first chunk should NOT be the top-ranked one + # (otherwise the test fixture isn't exercising the reorder). + assert scored[0][0] != 0 + + +def test_per_chunk_evidence_map_uses_distinct_chunk_roots(tmp_path): + """Chunk-level evidence_ids must be content-addressed per chunk — + different chunks of the same source produce different evidence_ids + even though they share source_root.""" + from typing import Iterator + from aborist.document import Document + from aborist.ingest import ingest_source + from aborist.qa.query import _load_doc_chunks + from aborist.source import Source + from aborist.store import connect + + class FakeSource(Source): + source_type = "test" + def __init__(self, docs): self.docs = docs + def iter_documents(self) -> Iterator[Document]: + yield from self.docs + + txt = "para alpha. " * 200 + "para beta. " * 200 + "para gamma. " * 200 + shard = tmp_path / "shard.db" + conn = connect(shard) + try: + ingest_source(conn, FakeSource([ + Document(uri="t://multi", content=txt, source_type="test", title="m") + ])) + doc_root = conn.execute( + "SELECT document_root FROM documents WHERE document_uri='t://multi'" + ).fetchone()["document_root"] + finally: + conn.close() + + chunks = _load_doc_chunks(str(shard), doc_root) + assert chunks is not None + assert len(chunks) > 1, "test fixture must produce multiple chunks" + + # Build a chunks_for_map dict like query() does, then verify each + # chunk gets a distinct content-addressed evidence_id. + cfm = [ + { + "source_root": doc_root, + "document_uri": "t://multi", + "title": "m", + "chunk_idx": idx, + "chunk_root": leaf_hash, + "span": span, + "source_role": "primary_answer_source", + } + for idx, leaf_hash, span in chunks + ] + em = build_evidence_map(cfm) + eids = [e.evidence_id for e in em] + pids = [e.pointer_id for e in em] + assert len(set(eids)) == len(eids), "evidence_ids must be unique per chunk" + # Pointer ids are sequential. + assert pids == [f"E{i + 1}" for i in range(len(em))] + # All share the same source_root but different chunk_roots. + assert len({e.source_root for e in em}) == 1 + assert len({e.chunk_root for e in em}) == len(em)