diff --git a/aborist/qa/client.py b/aborist/qa/client.py index 12cd769..9c8d3d6 100644 --- a/aborist/qa/client.py +++ b/aborist/qa/client.py @@ -23,8 +23,17 @@ class ChatClient(Protocol): temperature: float = 0.1, max_tokens: int = 512, top_p: float = 1.0, + extra_body: dict | None = None, ) -> str: - """Return the assistant's text response.""" + """Return the assistant's text response. + + ``extra_body`` is forwarded as additional fields in the JSON + request payload — used for vLLM-specific knobs like + ``guided_json`` (constrain output to a JSON Schema at sampling + time, eliminating SCHEMA_INVALID failures from prompt drift). + Endpoints that don't recognize the field ignore it; the client + passes it through opaque-ly. + """ ... @@ -44,6 +53,10 @@ class StubClient: return self._answer(messages, **kwargs) return self._answer + # StubClient ignores extra_body — the offline path doesn't go through + # any inference engine that would honor grammar guidance. Tests that + # want to assert extra_body was passed should inspect `self.calls`. + class OpenAICompatibleClient: """OpenAI-compatible chat completion over HTTP. @@ -70,6 +83,7 @@ class OpenAICompatibleClient: temperature: float = 0.1, max_tokens: int = 512, top_p: float = 1.0, + extra_body: dict | None = None, ) -> str: import httpx @@ -83,6 +97,12 @@ class OpenAICompatibleClient: "max_tokens": max_tokens, "top_p": top_p, } + # extra_body merges into the payload root — vLLM accepts knobs + # like {"guided_json": {...schema...}} or {"guided_grammar": "..."}. + # Endpoints that don't recognize a key silently drop it. + if extra_body: + for k, v in extra_body.items(): + payload[k] = v url = f"{self.base_url}/chat/completions" with httpx.Client(timeout=self.timeout) as client: resp = client.post(url, headers=headers, json=payload) diff --git a/aborist/qa/evidence.py b/aborist/qa/evidence.py index 99a3376..bb70421 100644 --- a/aborist/qa/evidence.py +++ b/aborist/qa/evidence.py @@ -181,6 +181,30 @@ def render_evidence_map(evidence: list[EvidenceObject]) -> str: return "\n\n".join(render_evidence_block(e) for e in evidence) +def render_evidence_block_for_json(e: EvidenceObject) -> str: + """Format one evidence object for the JSON-mode LLM prompt. + + Header carries the content-addressed ``evidence_id`` (the long hex + handle the model will cite in its JSON output) instead of the + prompt-facing pointer_id used by claim_lattice_pointer mode:: + + === E1f8e4c2a (Jurassic_Park_(film) | primary_answer_source) === + + + JSON mode pairs naturally with grammar-constrained inference where + the model can be schema-pinned to emit valid evidence_id strings. + The longer handle is out-of-distribution prose for small models — + use claim_lattice_pointer for those. + """ + label = (e.title or e.document_uri.rsplit("/", 1)[-1]) or "untitled" + return f"=== {e.evidence_id} ({label} | {e.source_role}) ===\n{e.span}" + + +def render_evidence_map_for_json(evidence: list[EvidenceObject]) -> str: + """Concatenated evidence blocks for JSON mode.""" + return "\n\n".join(render_evidence_block_for_json(e) for e in evidence) + + def evidence_map_by_pointer_id( evidence: list[EvidenceObject], ) -> dict[str, EvidenceObject]: diff --git a/aborist/qa/query.py b/aborist/qa/query.py index dac2974..367d9e4 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -69,10 +69,13 @@ from aborist.qa.evidence import ( build_evidence_map, evidence_map_root, render_evidence_map, + render_evidence_map_for_json, ) from aborist.qa.repair import mechanical_repair, reprompt_repair from aborist.qa.verify import ( ANSWER_MODES, + CLAIM_LATTICE_JSON_SCHEMA, + verify_claim_lattice_json, DEFAULT_ANSWER_MODE, verify_claim_lattice, verify_quotes, @@ -297,8 +300,8 @@ DEFAULT_QUERY_POLICY = { "Answer using natural-language pointer-lines: one claim per " "line, followed by a bracket tag with the pointer IDs that " "directly support that claim.\n\n" - "WORKED EXAMPLE\n" - "--------------\n" + "WORKED EXAMPLE 1 — narrow factoid\n" + "---------------------------------\n" "EVIDENCE:\n\n" "=== E1 (Apple_Inc | primary_answer_source) ===\n" "Apple Inc. was founded by Steve Jobs, Steve Wozniak, and " @@ -311,7 +314,22 @@ DEFAULT_QUERY_POLICY = { "Steve Jobs co-founded Apple. [E1]\n" "Steve Wozniak co-founded Apple. [E1,E2]\n" "Ronald Wayne co-founded Apple. [E1]\n\n" - "END OF EXAMPLE\n\n" + "WORKED EXAMPLE 2 — broad descriptive\n" + "------------------------------------\n" + "EVIDENCE:\n\n" + "=== E1 (Mars | primary_answer_source) ===\n" + "Mars is the fourth planet from the Sun. It has two moons, " + "Phobos and Deimos. Mars has a thin atmosphere of carbon " + "dioxide. Average surface temperature is around -60 " + "degrees Celsius.\n\n" + "QUESTION: tell me about Mars\n\n" + "ANSWER:\n" + "Mars is the fourth planet from the Sun. [E1]\n" + "Mars has two moons, Phobos and Deimos. [E1]\n" + "Mars has a thin atmosphere of carbon dioxide. [E1]\n" + "The average surface temperature on Mars is around -60 " + "degrees Celsius. [E1]\n\n" + "END OF EXAMPLES\n\n" "RULES (each rule says what TO do):\n" "1. Reference evidence by pointer ID. The runtime displays " "the literal source span beside each claim — referencing is " @@ -326,12 +344,28 @@ DEFAULT_QUERY_POLICY = { "answer is the right answer when only short evidence " "exists.\n" "6. Write each claim as one plain-prose sentence on its own " - "line." + "line.\n" + "7. Your answer goes only in the ANSWER position. Never " + "begin a line with the word EVIDENCE or with a pointer-id " + "prefix like E1: or E2: — those tags belong only above the " + "QUESTION, never in your output.\n" + "8. Do not include the double-quote character anywhere in " + "your answer. The runtime interpolates the literal source " + "span at display time, including its punctuation. Paraphrase " + "any phrase that the source has wrapped in quote marks " + "instead of copying the marks themselves.\n" + "9. Each claim line cites at most two pointers. Lines with " + "three or more pointers are rejected. If more than two " + "evidence blocks support a claim, pick the two that most " + "directly contain the claim's key terms; or split the " + "claim into two lines." ), "claim_lattice_grounding_reminder": ( "REMINDER: format = pointer-line — `Claim text. [E1]` per " "line, plain prose with bracket tags. Pointer IDs come from " - "the EVIDENCE blocks above. Cite 1 or 2 pointers per claim. " + "the EVIDENCE blocks above. At most two pointers per claim. " + "No `EVIDENCE:` header in your answer, no `E#:` line " + "prefix, no double-quote characters. " "Now answer the question on the next message." ), "claim_lattice_allowed_source_roles": [ @@ -340,6 +374,52 @@ DEFAULT_QUERY_POLICY = { "background_source", "unclassified", ], + "claim_lattice_max_pointers_per_claim": 2, + # Cap on evidence blocks (chunks) per retrieved source. Default 2. + # Without this cap, a long Wikipedia article alone can split into + # ~20 chunks, each becoming a separate E* — the model then sees + # E1-E26 for what is really 5 sources and writes a mega-claim + # citing all of them. With the cap, 5 sources × 2 = 10 evidence + # blocks max. Chunks within each source are still relevance-ranked + # (distinct_query_tokens DESC, total_mentions DESC, chunk_idx ASC) + # so the cap keeps the most-relevant 2 chunks. Folds into + # governance_policy_hash; changing the cap invalidates prior + # cached records. + "claim_lattice_max_chunks_per_source": 2, + "claim_lattice_min_citation_coverage": 0.30, + # JSON variant — `answer_mode="claim_lattice"`. Pairs with grammar- + # constrained inference (vLLM guided_json, Claude/GPT-4 native + # JSON, Qwen 3.6 reasoner). Lenient pre-parser keeps the path + # survivable on inference paths without grammar guidance. Toggling + # `claim_lattice_use_guided_json=False` disables the extra_body + # pass for endpoints that 400 on unknown fields. + "claim_lattice_json_system_prompt": ( + "You will see numbered EVIDENCE blocks tagged with content-" + "addressed evidence IDs (long hex strings starting with 'E'). " + "Answer as a single JSON object using EXACTLY this schema, " + "with NO prose, NO markdown fences, NO preamble:\n\n" + ' {"claims":[{"text":"","evidence_ids":["E........"]}]}\n\n' + "RULES:\n" + "1. Output a single JSON object. No code fences. No commentary.\n" + "2. `text` is plain prose with NO double-quote characters " + "anywhere — the JSON string-quotes are not the same as quoted " + "spans inside the text. If you need to mention a name " + "containing punctuation, use the source's own form without " + "wrapping it in quotes.\n" + "3. `evidence_ids` MUST be IDs from the EVIDENCE blocks above. " + "Do not invent IDs. At most two IDs per claim.\n" + "4. Each claim must reference at least one evidence_id.\n" + "5. If no evidence supports a claim, omit the claim." + ), + "claim_lattice_json_grounding_reminder": ( + "REMINDER: emit a single JSON object with the exact schema " + '`{"claims":[{"text":"...","evidence_ids":["E........"]}]}`. ' + "No code fences, no prose preamble. Each claim references at " + "most two evidence_ids from the blocks above. The text field " + "must contain no double-quote characters. " + "Now answer the question on the next message." + ), + "claim_lattice_use_guided_json": True, } @@ -1132,9 +1212,17 @@ def query( # chunks. If a single chunk exceeds what's left, truncate # that one chunk and stop. Total context across the source # stays bounded by hit_cap, same shape as the prose path. + # Per-source chunk cap also bounds chunk COUNT (in addition + # to char budget) so an encyclopedic article doesn't inflate + # the evidence catalog into E1-E26 territory and induce + # mega-claim failures. + max_chunks = max(1, int(policy.get( + "claim_lattice_max_chunks_per_source", 2 + ))) spent = 0 + chunks_used = 0 for chunk_idx, leaf_hash, span, _d, _t in scored: - if spent >= hit_cap: + if spent >= hit_cap or chunks_used >= max_chunks: break if policy.get("base_version") and _wikitext_to_base is not None: span = _wikitext_to_base(span) @@ -1153,11 +1241,77 @@ def query( "source_role": h.source_role, }) spent += len(span) + chunks_used += 1 evidence_map = build_evidence_map(chunks_for_map) sys_prompt = policy["claim_lattice_system_prompt"] grounding_reminder = policy.get("claim_lattice_grounding_reminder") rendered_evidence = render_evidence_map(evidence_map) + def _user_payload(q: str) -> str: + return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}" + elif answer_mode == "claim_lattice": + # JSON variant — same evidence-map construction as the pointer + # path, but blocks are labeled with content-addressed + # ``evidence_id`` (long hex) since the model emits IDs in JSON. + # The lenient pre-parser in verify_claim_lattice_json keeps the + # path survivable on inference paths without grammar guidance; + # vLLM ``guided_json`` (passed via extra_body below) eliminates + # SCHEMA_INVALID failures at sampling time when available. + qtokens_stem_for_chunks = { + _stem_token_for_match(t.lower()) + for t in _title_query_tokens(question) + } + chunks_for_map: list[dict] = [] + for h in chosen: + doc_chunks = _load_doc_chunks(h.shard_path, h.document_root) + if not doc_chunks: + continue + weight = SOURCE_ROLE_BUDGET_WEIGHTS.get(h.source_role, 1.0) + hit_cap = max(1, int(per_source_cap * weight)) + scored = [] + for chunk_idx, leaf_hash, span in doc_chunks: + distinct, total = _chunk_query_relevance( + span, qtokens_stem_for_chunks + ) + scored.append((chunk_idx, leaf_hash, span, distinct, total)) + scored.sort(key=lambda r: (-r[3], -r[4], r[0])) + max_chunks = max(1, int(policy.get( + "claim_lattice_max_chunks_per_source", 2 + ))) + spent = 0 + chunks_used = 0 + for chunk_idx, leaf_hash, span, _d, _t in scored: + if spent >= hit_cap or chunks_used >= max_chunks: + break + if policy.get("base_version") and _wikitext_to_base is not None: + span = _wikitext_to_base(span) + remaining = hit_cap - spent + if len(span) > remaining: + span = span[:remaining] + if not span: + break + chunks_for_map.append({ + "source_root": h.document_root, + "document_uri": h.document_uri, + "title": h.title, + "chunk_idx": chunk_idx, + "chunk_root": leaf_hash, + "span": span, + "source_role": h.source_role, + }) + spent += len(span) + chunks_used += 1 + evidence_map = build_evidence_map(chunks_for_map) + sys_prompt = policy.get( + "claim_lattice_json_system_prompt", + policy["claim_lattice_system_prompt"], + ) + grounding_reminder = policy.get( + "claim_lattice_json_grounding_reminder", + policy.get("claim_lattice_grounding_reminder"), + ) + rendered_evidence = render_evidence_map_for_json(evidence_map) + def _user_payload(q: str) -> str: return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}" else: @@ -1321,7 +1475,15 @@ def query( }, } - # 5. Cache miss — call LLM. + # 5. Cache miss — call LLM. JSON mode passes guided_json via + # extra_body so vLLM constrains output to the schema at sampling + # time. Non-vLLM endpoints silently drop the field; the lenient + # pre-parser in the verifier handles whatever drift remains. + extra_body: dict | None = None + if answer_mode == "claim_lattice" and policy.get( + "claim_lattice_use_guided_json", True + ): + extra_body = {"guided_json": CLAIM_LATTICE_JSON_SCHEMA} t_phase = time.monotonic() raw_answer = chat_client.chat_completion( messages, @@ -1329,6 +1491,7 @@ def query( temperature=policy["temperature"], max_tokens=policy["max_tokens"], top_p=policy.get("top_p", 1.0), + extra_body=extra_body, ) llm_ms = _ms_since(t_phase) @@ -1357,6 +1520,36 @@ def query( ], ) ), + max_pointers_per_claim=int(policy.get( + "claim_lattice_max_pointers_per_claim", 2 + )), + min_citation_coverage=float(policy.get( + "claim_lattice_min_citation_coverage", 0.30 + )), + ) + rendered = verdict["rendered_text"] + answer_text = rendered if rendered else raw_answer + elif answer_mode == "claim_lattice": + verdict = verify_claim_lattice_json( + raw_answer, + evidence_map, + allowed_source_roles=tuple( + policy.get( + "claim_lattice_allowed_source_roles", + [ + "primary_answer_source", + "secondary_context_source", + "background_source", + "unclassified", + ], + ) + ), + max_evidence_per_claim=int(policy.get( + "claim_lattice_max_pointers_per_claim", 2 + )), + min_citation_coverage=float(policy.get( + "claim_lattice_min_citation_coverage", 0.30 + )), ) rendered = verdict["rendered_text"] answer_text = rendered if rendered else raw_answer @@ -1460,7 +1653,8 @@ def query( # answer_mode into the verify/final_label payloads. ev_root = evidence_map_root(evidence_map) if evidence_map else None parsed_lattice = None - if answer_mode == "claim_lattice_pointer": + is_lattice_mode = answer_mode in ("claim_lattice_pointer", "claim_lattice") + if is_lattice_mode: evidence_id_pairs = verdict.get("evidence_id_pairs") or [] parsed_lattice = [ { @@ -1484,9 +1678,9 @@ def query( evidence_map_root=ev_root, answer_mode=answer_mode if answer_mode != "quote" else None, violations=verdict.get("violations"), - raw_answer_text=raw_answer if answer_mode == "claim_lattice_pointer" else None, + raw_answer_text=raw_answer if is_lattice_mode else None, parsed_lattice=parsed_lattice, - rendered_text=answer_text if answer_mode == "claim_lattice_pointer" else None, + rendered_text=answer_text if is_lattice_mode else None, ) run_dag_blob = json.dumps(run_dag, separators=(",", ":")) diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index b9ce982..dc9ee03 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -671,9 +671,78 @@ def verify_quotes( # completeness, no predicate compatibility. Those stay sidecar. # --------------------------------------------------------------------------- -ANSWER_MODES = ("quote", "claim_lattice_pointer") +ANSWER_MODES = ("quote", "claim_lattice_pointer", "claim_lattice") DEFAULT_ANSWER_MODE = "quote" +# JSON-mode pre-parser. 8B and small-context models drift on JSON +# discipline (markdown fences, prose preamble, smart quotes, trailing +# commas). Larger reasoning models (Qwen 3.6 reasoner, Claude, GPT-4) +# emit valid JSON natively; the pre-parser is the defensive belt that +# keeps the JSON path survivable across the inference-quality spectrum. +# Lenient on syntax, strict on semantics: parsed JSON still has to +# pass the schema check & the same hard verifier rules as pointer mode. +_JSON_FENCE_RE = re.compile(r"^\s*```(?:json)?\s*\n(.*?)\n\s*```\s*$", re.DOTALL) +_TRAILING_COMMA_RE = re.compile(r",(\s*[}\]])") + + +def _lenient_json_parse(raw: str) -> tuple[object, list[str]]: + """Parse ``raw`` as JSON, defensively peeling common model drift. + + Returns ``(parsed_obj, fixups_applied)`` — fixups list is empty + when strict parse succeeded, otherwise names what we had to peel + (``"fence"``, ``"prose_trim"``, ``"curly_quotes"``, ``"trailing_comma"``). + Raises ``json.JSONDecodeError`` if the lenient pass also fails. + + The fixups list lands in the verify payload so an agent can + observe model drift across runs & decide whether the inference + path is JSON-clean enough to keep using. + """ + import json as _json + fixups: list[str] = [] + try: + return _json.loads(raw), fixups + except _json.JSONDecodeError: + pass + + text = raw + + # 1. Strip markdown fence wrappers (```json\n...\n``` or ```\n...\n```). + m = _JSON_FENCE_RE.match(text) + if m: + text = m.group(1) + fixups.append("fence") + + # 2. Trim leading prose to first `{` or `[`; trailing prose past last + # matching `}`/`]`. Preserves the JSON object even when the model + # writes "Here is the JSON: {...}\n\nLet me know if you need more." + first_brace = min( + (text.find(c) for c in "{[" if text.find(c) >= 0), + default=-1, + ) + last_brace = max(text.rfind("}"), text.rfind("]")) + if first_brace > 0 or (last_brace >= 0 and last_brace < len(text) - 1): + if first_brace >= 0 and last_brace >= first_brace: + text = text[first_brace : last_brace + 1] + fixups.append("prose_trim") + + # 3. Normalize curly quotes — model-emitted “…” / ‘…’ become "…" / '…'. + if any(c in text for c in "“”‘’"): + text = ( + text.replace("“", '"').replace("”", '"') + .replace("‘", "'").replace("’", "'") + ) + fixups.append("curly_quotes") + + # 4. Fix trailing commas before `}` or `]`. Conservative: only + # comma immediately followed by whitespace + close bracket. + if "," in text: + new_text = _TRAILING_COMMA_RE.sub(r"\1", text) + if new_text != text: + text = new_text + fixups.append("trailing_comma") + + return _json.loads(text), fixups + # Default allowed source roles for claim_lattice_pointer mode. Roles # outside this set get classified as SOURCE_ROLE_BLOCKED. Mirrors the # role classifications in aborist/qa/query.py:_classify_source_role; @@ -703,18 +772,34 @@ def _has_manual_quote(text: str) -> bool: return any(ch in text for ch in ('"', '“', '”')) +DEFAULT_MIN_CITATION_COVERAGE = 0.30 + + def _claim_textually_overlaps_evidence( - claim_text: str, evidence_span: str + claim_text: str, + evidence_span: str, + *, + min_coverage: float = DEFAULT_MIN_CITATION_COVERAGE, ) -> bool: - """Return True if at least one content token from ``claim_text`` - appears in ``evidence_span`` (case-insensitive substring). + """Return True if claim's content-token coverage in ``evidence_span`` + meets ``min_coverage`` (case-insensitive substring match). Hard 6th check on a (claim, pointer) pair. Catches the lazy-anchor failure where the model cites an evidence pointer whose text has - zero overlap with the claim's actual subject — e.g. claim - "Brachiosaurus" cited to a "1993 in film" overview paragraph that - never mentions the dinosaur. Lexical only, no NER, no embeddings; - stays inside the soft/hard boundary. + insufficient overlap with the claim's actual subject — e.g. claim + "Yale University in New Haven and the University of Connecticut..." + cited to a highway-data span containing only the token + ``connecticut`` (1/10 = 10% coverage; below the 30% default + threshold → CITATION_MISMATCH). + + Pre-2026-04-30 this function required only ≥1 shared content token, + which let through lazy-anchored claims whose only overlap was a + common topical word. Coverage-based threshold scales with claim + length: short claims (1-3 content tokens) need 1 match (same as + the old behavior), longer claims need a proportional share. + + Lexical only, no NER, no embeddings; stays inside the soft/hard + boundary. A pure-stopword claim (no content tokens after the spotlight token extractor's filter) returns True vacuously — there's nothing @@ -727,7 +812,19 @@ def _claim_textually_overlaps_evidence( if not tokens: return True span_lower = evidence_span.lower() - return any(t in span_lower for t in tokens) + matched = sum(1 for t in tokens if t in span_lower) + coverage = matched / len(tokens) + # Floor: a single shared content token always counts when the claim + # is itself short (≤3 content tokens) so single-fact narrow claims + # like "Steve Jobs co-founded Apple" don't fail on a coverage + # technicality. The threshold bites on prose-shaped multi-token + # claims where 1/10 token overlap is the lazy-anchor signature. + if matched >= 1 and len(tokens) <= 3: + return True + return coverage >= min_coverage + + +DEFAULT_MAX_POINTERS_PER_CLAIM = 2 def verify_claim_lattice( @@ -735,13 +832,15 @@ def verify_claim_lattice( evidence_map, *, allowed_source_roles: tuple[str, ...] = DEFAULT_ALLOWED_SOURCE_ROLES, + max_pointers_per_claim: int = DEFAULT_MAX_POINTERS_PER_CLAIM, + min_citation_coverage: float = DEFAULT_MIN_CITATION_COVERAGE, ) -> dict: """Deterministic verifier for ``answer_mode="claim_lattice_pointer"``. The model wrote pointer-line prose (``Claim text. [E12]``); the parser pulled (claim_text, [pointer_ids]) pairs from each non-empty line. This verifier maps each pointer id back to its - content-addressed evidence object and runs six hard checks: + content-addressed evidence object and runs seven hard checks: 1. Parser succeeded — ``parse_status == "PARSED"`` (line had a bracket tag). NO_EVIDENCE_POINTER claims (prose without tag) @@ -757,6 +856,10 @@ def verify_claim_lattice( ``_claim_textually_overlaps_evidence``). Catches the magnet- chunk lazy-anchor where the model cites an evidence pointer whose text contains zero claim-content tokens. + 7. Pointer count per claim does not exceed ``max_pointers_per_claim`` + (default 2 — matches the prompt's "1 or 2 pointers per claim" + rule). Catches the encyclopedic-mega-claim failure where the + model produces one giant claim line citing every pointer. Returns a verdict in the same shape as ``verify_quotes`` + extras: @@ -854,6 +957,29 @@ def verify_claim_lattice( evidence_id_pairs.append([]) continue + # Pointer-count cap (Rule 9). Catches the encyclopedic-mega- + # claim where the model produces one line citing every + # pointer at once. Counts every pointer toward the denominator + # so the failure is loud in n_quotes. + if len(pointer_ids) > max_pointers_per_claim: + violations.append({ + "kind": "SCHEMA_INVALID", + "claim_idx": idx, + "reason": f"too many pointers ({len(pointer_ids)} > {max_pointers_per_claim})", + }) + claim_statuses.append({ + "claim_idx": idx, + "text": claim_text, + "pointer_ids": pointer_ids, + "evidence_ids": [], + "status": "SCHEMA_INVALID", + "reasons": [f"too many pointers ({len(pointer_ids)} > {max_pointers_per_claim})"], + }) + n_pairs += len(pointer_ids) + evidence_id_pairs.append([]) + unverified.append(claim_text) + continue + # Strict no-quote rule: any double-quote in claim text is a # MANUAL_QUOTE_VIOLATION. Block every pointer link on this claim. manual_quote = _has_manual_quote(claim_text) @@ -890,7 +1016,9 @@ def verify_claim_lattice( "pid": pid, "ok": False, "kind": "MANUAL_QUOTE_VIOLATION", }) continue - if not _claim_textually_overlaps_evidence(claim_text, obj.span): + if not _claim_textually_overlaps_evidence( + claim_text, obj.span, min_coverage=min_citation_coverage + ): # Cited evidence span has zero textual overlap with any # content token from the claim. Strongest lazy-anchor # signal promoted to a hard fail — the model cited a @@ -999,3 +1127,258 @@ def verify_claim_lattice( "pointer_id_distribution": pointer_distribution, "lazy_anchor_ratio": lazy_anchor_ratio, } + + +# --------------------------------------------------------------------------- +# JSON variant — `answer_mode="claim_lattice"`. Same lattice semantics as +# the pointer variant, but the model emits a structured JSON object +# {"claims":[{"text":str,"evidence_ids":[str,...]}]} with content- +# addressed evidence_ids directly. Pairs naturally with grammar- +# constrained inference (vLLM guided_json, Claude/GPT-4 native JSON +# mode, Qwen 3.6 reasoner) where schema-conformance is generation-time- +# enforced. The lenient pre-parser above keeps the path survivable on +# inference paths without grammar guidance. +# --------------------------------------------------------------------------- + + +CLAIM_LATTICE_JSON_SCHEMA = { + "type": "object", + "properties": { + "claims": { + "type": "array", + "items": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "evidence_ids": { + "type": "array", + "items": {"type": "string"}, + }, + }, + "required": ["text", "evidence_ids"], + "additionalProperties": False, + }, + }, + }, + "required": ["claims"], + "additionalProperties": False, +} + + +def verify_claim_lattice_json( + answer_json_text: str, + evidence_map, + *, + allowed_source_roles: tuple[str, ...] = DEFAULT_ALLOWED_SOURCE_ROLES, + max_evidence_per_claim: int = DEFAULT_MAX_POINTERS_PER_CLAIM, + min_citation_coverage: float = DEFAULT_MIN_CITATION_COVERAGE, +) -> dict: + """Deterministic verifier for ``answer_mode="claim_lattice"`` (JSON). + + Parses the model's JSON output (lenient pre-parser handles markdown + fences / preamble / curly quotes / trailing commas), validates the + schema, then runs the same hard checks as ``verify_claim_lattice`` + but on content-addressed ``evidence_id``s directly: + + 1. JSON parses (lenient). Failure → SCHEMA_INVALID, UNGROUNDED. + 2. Top-level is ``{"claims": [...]}``. + 3. Each claim is ``{"text": str, "evidence_ids": [str, ...]}``. + 4. Each evidence_id resolves in the runtime-built evidence map + (no model-invented IDs). + 5. Resolved entry's ``source_role`` is in ``allowed_source_roles``. + 6. Claim text contains no double-quote characters anywhere. + 7. Claim text non-empty. + 8. Claim's content tokens textually overlap the cited evidence span. + 9. ``len(evidence_ids) <= max_evidence_per_claim``. + + Returns a verdict in the same shape as ``verify_claim_lattice`` plus + a ``json_fixups`` field naming any drift the lenient parser had to + peel (``"fence"`` / ``"prose_trim"`` / ``"curly_quotes"`` / + ``"trailing_comma"``). Empty list = strict JSON parse on first try. + """ + from aborist.qa.evidence import ( + evidence_map_by_evidence_id as _by_eid, + render_claim_lattice as _render, + ) + + by_eid = _by_eid(evidence_map) + violations: list[dict] = [] + claim_statuses: list[dict] = [] + unverified: list[str] = [] + json_fixups: list[str] = [] + + parsed = None + try: + parsed, json_fixups = _lenient_json_parse(answer_json_text or "") + except Exception as exc: + violations.append({ + "kind": "SCHEMA_INVALID", + "reason": f"json parse: {str(exc)[:200]}", + }) + if parsed is not None and not isinstance(parsed, dict): + violations.append({ + "kind": "SCHEMA_INVALID", + "reason": f"top-level not object (got {type(parsed).__name__})", + }) + parsed = None + raw_claims = (parsed or {}).get("claims") if parsed is not None else None + if parsed is not None and not isinstance(raw_claims, list): + violations.append({ + "kind": "SCHEMA_INVALID", + "reason": "missing or non-list 'claims'", + }) + raw_claims = None + + n_pairs = 0 + n_pairs_verified = 0 + valid_claims: list[dict] = [] + evidence_id_pairs: list[list[str]] = [] + + for idx, c in enumerate(raw_claims or []): + if not isinstance(c, dict): + violations.append({ + "kind": "SCHEMA_INVALID", + "claim_idx": idx, + "reason": f"claim not object (got {type(c).__name__})", + }) + claim_statuses.append({ + "text": "", "evidence_ids": [], + "status": "SCHEMA_INVALID", "reasons": ["not_object"], + }) + continue + claim_text = c.get("text") or "" + eids = c.get("evidence_ids") or [] + if not isinstance(claim_text, str) or not isinstance(eids, list): + violations.append({ + "kind": "SCHEMA_INVALID", "claim_idx": idx, + "reason": "claim shape: text=str, evidence_ids=list[str]", + }) + claim_statuses.append({ + "text": str(claim_text)[:200], "evidence_ids": [], + "status": "SCHEMA_INVALID", "reasons": ["bad_field_types"], + }) + continue + + # Manual-quote prohibition (same rule as pointer mode). + if _has_manual_quote(claim_text): + violations.append({ + "kind": "MANUAL_QUOTE_VIOLATION", "claim_idx": idx, + "claim_text": claim_text[:200], + }) + unverified.append(claim_text) + claim_statuses.append({ + "text": claim_text, "evidence_ids": eids, + "status": "MANUAL_QUOTE_VIOLATION", + "reasons": ["double_quote_in_text"], + }) + n_pairs += max(1, len(eids)) + continue + + if not claim_text.strip(): + violations.append({ + "kind": "SCHEMA_INVALID", "claim_idx": idx, + "reason": "empty claim text", + }) + claim_statuses.append({ + "text": "", "evidence_ids": eids, + "status": "SCHEMA_INVALID", "reasons": ["empty_text"], + }) + continue + + if len(eids) > max_evidence_per_claim: + violations.append({ + "kind": "TOO_MANY_EVIDENCE_IDS", "claim_idx": idx, + "claim_text": claim_text[:200], + "n_ids": len(eids), "max": max_evidence_per_claim, + }) + + # Per-id resolution + checks. + per_id_results = [] + verified_ids = [] + for eid in eids: + if not isinstance(eid, str): + per_id_results.append({"eid": str(eid), "ok": False, "kind": "SCHEMA_INVALID"}) + continue + obj = by_eid.get(eid) + if obj is None: + per_id_results.append({"eid": eid, "ok": False, "kind": "UNKNOWN_EVIDENCE_ID"}) + violations.append({ + "kind": "UNKNOWN_EVIDENCE_ID", + "claim_idx": idx, "evidence_id": eid, + }) + continue + if obj.source_role not in allowed_source_roles: + per_id_results.append({"eid": eid, "ok": False, "kind": "SOURCE_ROLE_BLOCKED"}) + violations.append({ + "kind": "SOURCE_ROLE_BLOCKED", + "claim_idx": idx, "evidence_id": eid, + "source_role": obj.source_role, + }) + continue + if not _claim_textually_overlaps_evidence( + claim_text, obj.span, min_coverage=min_citation_coverage + ): + per_id_results.append({"eid": eid, "ok": False, "kind": "CITATION_MISMATCH"}) + violations.append({ + "kind": "CITATION_MISMATCH", + "claim_idx": idx, "evidence_id": eid, + "claim_text": claim_text[:200], + }) + continue + per_id_results.append({"eid": eid, "ok": True}) + verified_ids.append(eid) + + n_pairs += max(1, len(eids)) + n_pairs_verified += len(verified_ids) + + if not eids: + claim_statuses.append({ + "text": claim_text, "evidence_ids": [], + "status": "NO_EVIDENCE_POINTER", + "reasons": ["no_evidence_ids"], + }) + unverified.append(claim_text) + n_pairs += 1 + continue + + if len(verified_ids) == len(eids): + status = "EVIDENCE_LINKED" + elif verified_ids: + status = "EVIDENCE_LINKED_PARTIAL" + else: + # Pick the worst per-id reason for the claim status. + kinds = [r["kind"] for r in per_id_results if not r["ok"]] + status = kinds[0] if kinds else "UNKNOWN_EVIDENCE_ID" + unverified.append(claim_text) + + claim_statuses.append({ + "text": claim_text, + "evidence_ids": eids, + "status": status, + "reasons": [r["kind"] for r in per_id_results if not r["ok"]], + }) + if verified_ids: + valid_claims.append({"text": claim_text, "evidence_ids": verified_ids}) + evidence_id_pairs.append(list(verified_ids)) + + rendered_text = _render(valid_claims, by_eid) if valid_claims else "" + + if n_pairs_verified > 0 and not violations: + audit_mode = "STRICT" + elif n_pairs_verified > 0: + audit_mode = "HYBRID" + else: + audit_mode = "UNGROUNDED" + + return { + "n_quotes": n_pairs, + "n_verified": n_pairs_verified, + "audit_mode": audit_mode, + "unverified_quotes": unverified, + "verifier_method": "claim_lattice_json", + "claim_statuses": claim_statuses, + "violations": violations, + "rendered_text": rendered_text, + "evidence_id_pairs": evidence_id_pairs, + "json_fixups": json_fixups, + } diff --git a/tests/test_verify_json.py b/tests/test_verify_json.py new file mode 100644 index 0000000..66264e4 --- /dev/null +++ b/tests/test_verify_json.py @@ -0,0 +1,186 @@ +"""JSON-mode claim-lattice verifier (`answer_mode="claim_lattice"`). + +Pairs with grammar-constrained inference (vLLM guided_json, Claude/GPT-4 +native JSON, Qwen 3.6 reasoner). The lenient pre-parser keeps the path +survivable on inference paths where the model emits non-strict JSON +(markdown fences, prose preamble, curly quotes, trailing commas). +""" + +from __future__ import annotations + +import json + +import pytest + +from aborist.qa.evidence import EvidenceObject +from aborist.qa.verify import _lenient_json_parse, verify_claim_lattice_json + + +# ---------------------------------------------------------------- lenient parser + + +def test_lenient_strict_passes_through(): + obj, fixups = _lenient_json_parse('{"a": 1}') + assert obj == {"a": 1} + assert fixups == [] + + +def test_lenient_strips_markdown_fence(): + raw = '```json\n{"claims": []}\n```' + obj, fixups = _lenient_json_parse(raw) + assert obj == {"claims": []} + assert "fence" in fixups + + +def test_lenient_strips_unlabeled_fence(): + raw = '```\n{"claims": []}\n```' + obj, fixups = _lenient_json_parse(raw) + assert obj == {"claims": []} + assert "fence" in fixups + + +def test_lenient_trims_preamble_and_suffix(): + raw = 'Here is the JSON:\n{"claims": []}\nLet me know if you need more.' + obj, fixups = _lenient_json_parse(raw) + assert obj == {"claims": []} + assert "prose_trim" in fixups + + +def test_lenient_normalizes_curly_quotes(): + raw = '{“claims”: []}' + obj, fixups = _lenient_json_parse(raw) + assert obj == {"claims": []} + assert "curly_quotes" in fixups + + +def test_lenient_fixes_trailing_comma(): + raw = '{"claims": [],}' + obj, fixups = _lenient_json_parse(raw) + assert obj == {"claims": []} + assert "trailing_comma" in fixups + + +def test_lenient_combines_multiple_fixups(): + raw = '```json\nHere:\n{“claims”: [],}\n```' + obj, fixups = _lenient_json_parse(raw) + assert obj == {"claims": []} + assert "fence" in fixups + # At least one of the inner fixups also fired. + assert any(f in fixups for f in ("curly_quotes", "trailing_comma", "prose_trim")) + + +def test_lenient_raises_on_truly_broken(): + with pytest.raises(json.JSONDecodeError): + _lenient_json_parse("not json at all { ] [") + + +# ---------------------------------------------------------------- JSON verifier + + +def _ev(eid: str, span: str, role: str = "primary_answer_source") -> EvidenceObject: + """Stub evidence object with deterministic eid for the test.""" + return EvidenceObject( + evidence_id=eid, + source_root="00" * 32, + document_uri="test://doc", + title="Test Doc", + chunk_idx=0, + chunk_root="11" * 32, + offset_start=0, + offset_end=len(span), + source_role=role, + text_hash="22" * 32, + span=span, + pointer_id=None, + ) + + +def test_verify_json_strict_when_all_claims_resolve(): + evidence = [ + _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore."), + _ev("E2c9d7b3f", "Velociraptor is featured prominently throughout Jurassic Park."), + ] + answer = json.dumps({ + "claims": [ + {"text": "Brachiosaurus appears in the film", "evidence_ids": ["E1f8e4c2a"]}, + {"text": "Velociraptor is featured", "evidence_ids": ["E2c9d7b3f"]}, + ] + }) + v = verify_claim_lattice_json(answer, evidence) + assert v["audit_mode"] == "STRICT" + assert v["verifier_method"] == "claim_lattice_json" + assert v["n_verified"] == 2 + assert v["violations"] == [] + + +def test_verify_json_hybrid_when_some_unknown_evidence_id(): + evidence = [ + _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore."), + ] + answer = json.dumps({ + "claims": [ + {"text": "Brachiosaurus appears in the film", "evidence_ids": ["E1f8e4c2a"]}, + {"text": "Made-up claim", "evidence_ids": ["EFAKEFAKE"]}, + ] + }) + v = verify_claim_lattice_json(answer, evidence) + assert v["audit_mode"] == "HYBRID" + assert any(vio["kind"] == "UNKNOWN_EVIDENCE_ID" for vio in v["violations"]) + + +def test_verify_json_ungrounded_on_schema_invalid(): + """Lenient parser fails too → SCHEMA_INVALID → UNGROUNDED.""" + v = verify_claim_lattice_json("not json {[", []) + assert v["audit_mode"] == "UNGROUNDED" + assert any(vio["kind"] == "SCHEMA_INVALID" for vio in v["violations"]) + + +def test_verify_json_recovers_from_markdown_fence(): + """JSON-fenced output still parses & verifies; fence fixup logged.""" + evidence = [ + _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore."), + ] + answer = ( + '```json\n' + '{"claims": [{"text": "Brachiosaurus appears in the film", ' + '"evidence_ids": ["E1f8e4c2a"]}]}\n' + '```' + ) + v = verify_claim_lattice_json(answer, evidence) + assert v["audit_mode"] == "STRICT" + assert "fence" in v["json_fixups"] + + +def test_verify_json_manual_quote_violation(): + """Strict no-double-quote rule — even valid JSON with double quotes + inside a claim's text field fails MANUAL_QUOTE_VIOLATION.""" + evidence = [ + _ev("E1f8e4c2a", "Brachiosaurus appears in the Jurassic Park film as a herbivore."), + ] + answer = json.dumps({ + "claims": [ + { + "text": 'Brachiosaurus is "a herbivore" appears in the film', + "evidence_ids": ["E1f8e4c2a"], + } + ] + }) + v = verify_claim_lattice_json(answer, evidence) + assert any(vio["kind"] == "MANUAL_QUOTE_VIOLATION" for vio in v["violations"]) + assert v["audit_mode"] == "UNGROUNDED" + + +def test_verify_json_blocks_disallowed_source_role(): + """Evidence resolved but source_role outside the allowlist fails + SOURCE_ROLE_BLOCKED.""" + evidence = [ + _ev("E1f8e4c2a", "Brachiosaurus content here.", role="noisy_background_source"), + ] + answer = json.dumps({ + "claims": [ + {"text": "Brachiosaurus appears", "evidence_ids": ["E1f8e4c2a"]}, + ] + }) + v = verify_claim_lattice_json(answer, evidence) + assert any(vio["kind"] == "SOURCE_ROLE_BLOCKED" for vio in v["violations"]) + assert v["audit_mode"] == "UNGROUNDED"