diff --git a/arborist/cli.py b/arborist/cli.py index 05252fc..41630d6 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -751,6 +751,17 @@ def _render_warrant_tail(result: dict) -> str: parts: list[str] = [] if "WARRANT_MISSING" in kinds: parts.append("warrant missing") + # Phase 3 of #000031 — positive signal when a per-claim + # warrant_check would have failed but a Merkle-bound warrant + # chain to a primary-source surface exists (cited chunk's + # document is a claim-pack record with a warrant-resolver + # derivation row). Distinct from the source-level + # `_render_warrant_chain_tail` which counts cited SOURCES with + # chains; this counts CLAIMS that survived because of a chain. + proven_idxs = result.get("warrant_proven_claim_idxs") or [] + if proven_idxs: + n = len(proven_idxs) + parts.append(f"warrant proven via chain ×{n}") if "TITLE_MISMATCH" in kinds: parts.append("title mismatch") if "FORMAT_COLLAPSED" in kinds: diff --git a/arborist/qa/query.py b/arborist/qa/query.py index 4d572a9..d957e83 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -3483,6 +3483,14 @@ def query( # of reaching STRICT. Persisted into run_dag_blob via the # verify stage's payload. "violations": verdict.get("violations") or [], + # Phase 3 of #000031 — per-claim warrant-chain proven idxs. + # Populated when a per-claim warrant_check would have failed + # but the cited chunk's document has a warrant-resolver + # derivation row (Merkle-bound primary-source backing). The + # render layer surfaces this as ``· warrant proven via chain + # ×N`` in the audit-line tail so operators see when a claim + # got through on the chain, not on lexical anchors. + "warrant_proven_claim_idxs": verdict.get("warrant_proven_claim_idxs") or [], # Format-collapse signal (pointer-mode only — None elsewhere). # True when the model emitted ≥5 meaningful prose lines with # zero `[E\d+]` pointer tags, i.e. abandoned the diff --git a/arborist/qa/runner.py b/arborist/qa/runner.py index 0c70bd3..af2eed3 100644 --- a/arborist/qa/runner.py +++ b/arborist/qa/runner.py @@ -672,6 +672,29 @@ def ask( repair_changes: list[dict] = [] pre_repair_verdict: dict | None = None + # Phase 3 of #000031: load the warrant-chain core_root set once + # from the conn's shards directory. The verifier consults this + # set to suppress WARRANT_MISSING when the cited chunk's document + # has a warrant-resolver derivation row (Merkle-bound primary- + # source backing). Empty set if the shard has no derivations + # rows yet — fully backward-compatible. + _warrant_chain_roots: frozenset[str] = frozenset() + try: + from pathlib import Path as _Path + + from arborist.qa.warrant_chain import warrant_chain_lookup as _wcl + + _db_path = conn.execute("PRAGMA database_list").fetchall() + # PRAGMA database_list rows: (seq, name, file). Main DB is + # the first row with name='main'. + _main_row = next((r for r in _db_path if r[1] == "main"), None) + if _main_row and _main_row[2]: + _warrant_chain_roots = _wcl(_Path(_main_row[2]).parent) + except Exception: + # Fail-closed: empty set means no suppression, behavior + # identical to pre-Phase-3. + _warrant_chain_roots = frozenset() + if answer_mode == "claim_lattice_pointer": verdict = verify_claim_lattice( raw_answer, @@ -716,6 +739,7 @@ def ask( format_collapse_check_enabled=bool(policy.get( "claim_lattice_format_collapse_check_enabled", True )), + warrant_chain_roots=_warrant_chain_roots, ) # Rendered prose (literal spans interpolated) is the user-facing # answer text — never the model's raw pointer-line output. If @@ -755,6 +779,7 @@ def ask( deflection_check_enabled=bool(policy.get( "claim_lattice_deflection_check_enabled", True )), + warrant_chain_roots=_warrant_chain_roots, ) rendered = verdict["rendered_text"] answer_text = rendered if rendered else raw_answer diff --git a/arborist/qa/verify.py b/arborist/qa/verify.py index 7a69991..d278fb3 100644 --- a/arborist/qa/verify.py +++ b/arborist/qa/verify.py @@ -1095,6 +1095,7 @@ def verify_claim_lattice( warrant_check_enabled: bool = True, deflection_check_enabled: bool = True, format_collapse_check_enabled: bool = True, + warrant_chain_roots: frozenset[str] = frozenset(), ) -> dict: """Deterministic verifier for ``answer_mode="claim_lattice_pointer"``. @@ -1463,21 +1464,37 @@ def verify_claim_lattice( # cap audit_mode at HYBRID via the same demote pattern as # lazy_anchor_demoted. warrant_missing_claims: list[int] = [] + warrant_proven_claim_idxs: list[int] = [] if warrant_check_enabled: for cs in claim_statuses: if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"): continue cited_eids = cs.get("evidence_ids") or [] - cited_spans = [ - obj.span + cited_evidence_objs = [ + obj for eid in cited_eids for obj in [evidence_map_by_evidence_id_local(evidence_map, eid)] if obj is not None ] + cited_spans = [obj.span for obj in cited_evidence_objs] ok, missing = warrant_check( cs.get("text") or "", cited_spans, question=question ) if not ok: + # Phase 3 of #000031: if any cited evidence's source + # document has a warrant-resolver derivation row (i.e., + # the cited chunk is a claim-pack record with a + # Merkle-bound primary-source backing), suppress + # WARRANT_MISSING — the warrant chain DOES exist, just + # not at the lexical-anchor level. Track on a separate + # `warrant_proven_claim_idxs` field for render-layer + # transparency. + cited_source_roots = [obj.source_root for obj in cited_evidence_objs] + if warrant_chain_roots and any( + r in warrant_chain_roots for r in cited_source_roots + ): + warrant_proven_claim_idxs.append(cs.get("claim_idx")) + continue warrant_missing_claims.append(cs.get("claim_idx")) violations.append({ "kind": "WARRANT_MISSING", @@ -1676,6 +1693,7 @@ def verify_claim_lattice( "lazy_anchor_ratio": lazy_anchor_ratio, "lazy_anchor_demoted": lazy_anchor_demoted, "warrant_missing_claim_idxs": warrant_missing_claims, + "warrant_proven_claim_idxs": warrant_proven_claim_idxs, "title_mismatch_claim_idxs": title_mismatch_claims, "deflection_detected": deflection_detected, "format_collapsed": format_collapsed, @@ -1742,6 +1760,7 @@ def verify_claim_lattice_json( question: str | None = None, warrant_check_enabled: bool = True, deflection_check_enabled: bool = True, + warrant_chain_roots: frozenset[str] = frozenset(), ) -> dict: """Deterministic verifier for ``answer_mode="claim_lattice"`` (JSON). @@ -1991,21 +2010,35 @@ def verify_claim_lattice_json( # variant carries the same WARRANT_MISSING violations & the same # warrant_missing_claim_idxs field on the verdict. warrant_missing_claims: list[int] = [] + warrant_proven_claim_idxs: list[int] = [] if warrant_check_enabled: for cs in claim_statuses: if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"): continue cited_eids = cs.get("evidence_ids") or [] - cited_spans = [ - obj.span + cited_evidence_objs = [ + obj for eid in cited_eids for obj in [evidence_map_by_evidence_id_local(evidence_map, eid)] if obj is not None ] + cited_spans = [obj.span for obj in cited_evidence_objs] ok, missing = warrant_check( cs.get("text") or "", cited_spans, question=question ) if not ok: + # Phase 3 of #000031: see verify_claim_lattice for + # the full rationale. Same suppression logic — if + # the cited chunk's document has a warrant-resolver + # derivation row, the warrant chain exists at the + # Merkle level even if the lexical anchor doesn't + # fire. + cited_source_roots = [obj.source_root for obj in cited_evidence_objs] + if warrant_chain_roots and any( + r in warrant_chain_roots for r in cited_source_roots + ): + warrant_proven_claim_idxs.append(cs.get("claim_idx")) + continue warrant_missing_claims.append(cs.get("claim_idx")) violations.append({ "kind": "WARRANT_MISSING", @@ -2145,5 +2178,6 @@ def verify_claim_lattice_json( "evidence_id_pairs": evidence_id_pairs, "json_fixups": json_fixups, "warrant_missing_claim_idxs": warrant_missing_claims, + "warrant_proven_claim_idxs": warrant_proven_claim_idxs, "title_mismatch_claim_idxs": title_mismatch_claims, } diff --git a/arborist/qa/warrant_chain.py b/arborist/qa/warrant_chain.py new file mode 100644 index 0000000..24f8118 --- /dev/null +++ b/arborist/qa/warrant_chain.py @@ -0,0 +1,89 @@ +"""Warrant-chain lookup — Phase 3 of `#000031`. + +Read-only helper that returns the set of ``document_root`` values +which have at least one ``derivations`` row written by the warrant +resolver (``process_id LIKE 'warrant-resolver-v1%'``). The verifier +consults this set when a per-claim ``warrant_check`` would otherwise +fire ``WARRANT_MISSING`` — if the cited chunk's document is a +claim-pack record with a Merkle-bound primary-source backing, the +warrant chain DOES exist (just not at the lexical-anchor level), so +the missing-anchor demote is suppressed and a positive +``WARRANT_PROVEN`` annotation flows through to the render layer. + +Pure-SQL lookup; one query per Q&A run; result is a frozenset for +hashing / cheap membership tests across many claim verifications. +""" + +from __future__ import annotations + +from pathlib import Path +import sqlite3 + + +_WARRANT_PROCESS_PREFIX = "warrant-resolver-v1" + + +def warrant_chain_lookup(shards_dir: str | Path) -> frozenset[str]: + """Return the frozenset of ``core_root`` values that have a + warrant-resolver derivation row in any shard under ``shards_dir``. + + Walks every ``*.db`` under both ``/`` (main numbered + shards holding claim-pack records) AND ``/../crawl/`` + (textbook substrate shards may also hold derivations). Tolerates + missing tables — shards that don't have a ``derivations`` schema + yet contribute zero rows without raising. + + Returns frozenset for hashable / cheap-membership use in the + verifier hot path. + """ + shards_dir = Path(shards_dir).expanduser() + candidates: list[Path] = [] + if shards_dir.is_dir(): + for db in sorted(shards_dir.glob("*.db")): + candidates.append(db) + sibling = shards_dir.parent / "crawl" + if sibling.is_dir(): + for db in sorted(sibling.glob("*.db")): + if db.name.startswith(("crawl_russell_", "qa.", "snapshots.")): + continue + candidates.append(db) + + roots: set[str] = set() + for db_path in candidates: + try: + conn = sqlite3.connect(str(db_path)) + except sqlite3.Error: + continue + try: + try: + rows = conn.execute( + "SELECT DISTINCT core_root FROM derivations " + "WHERE process_id LIKE ? || '%'", + (_WARRANT_PROCESS_PREFIX,), + ).fetchall() + except sqlite3.OperationalError: + # No derivations table on this shard. + continue + for (core_root,) in rows: + if core_root: + roots.add(core_root) + finally: + conn.close() + + return frozenset(roots) + + +def has_warrant_chain( + source_roots: list[str] | tuple[str, ...] | set[str] | frozenset[str], + warrant_chain_roots: frozenset[str], +) -> bool: + """True if any of ``source_roots`` is in ``warrant_chain_roots``. + + Cheap membership test — no DB call, no allocation beyond the + frozenset lookup. Used in the verifier per-claim hot path: given + the list of cited evidence objects' ``source_root`` values, decide + whether to suppress ``WARRANT_MISSING``. + """ + if not warrant_chain_roots: + return False + return any(r in warrant_chain_roots for r in source_roots) diff --git a/tests/test_warrant_chain.py b/tests/test_warrant_chain.py new file mode 100644 index 0000000..7c2b030 --- /dev/null +++ b/tests/test_warrant_chain.py @@ -0,0 +1,320 @@ +"""Tests for Phase 3 of `#000031` — warrant-chain lookup + +verifier suppression of WARRANT_MISSING when a Merkle-bound +primary-source backing exists. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from arborist.qa.warrant_chain import ( + has_warrant_chain, + warrant_chain_lookup, +) +from arborist.store import SCHEMA_SQL + + +@pytest.fixture +def shards_with_chain(tmp_path: Path): + """Build a minimal two-shard cluster: a main shard with a + `derivations` row written by warrant-resolver-v1 (mimics what + `arborist warrant-resolve --write` produces) plus a sibling + crawl shard that doesn't have a derivations table. + + Returns ``(shards_dir, core_root, src_root)``. + """ + shards_dir = tmp_path / "shards" + crawl_dir = tmp_path / "crawl" + shards_dir.mkdir() + crawl_dir.mkdir() + + main_db = shards_dir / "000.db" + conn = sqlite3.connect(str(main_db)) + try: + conn.executescript(SCHEMA_SQL) + # Insert a fake claim-pack record + textbook surface document + # so the FK constraints on derivations are satisfied. + core_root = "a" * 64 + src_root = "b" * 64 + conn.execute( + "INSERT INTO documents " + "(document_root, document_uri, source_type, kind, " + " compression_depth, title, chunking_version, " + " canonicalization_version, schema_version, ingest_ts) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (core_root, "claim_pack://test", "claim_pack", "surface", + 0, "Test record", "tok-512-v1", "norm-v1", "v9.8.0", 0), + ) + conn.execute( + "INSERT INTO documents " + "(document_root, document_uri, source_type, kind, " + " compression_depth, title, chunking_version, " + " canonicalization_version, schema_version, ingest_ts) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (src_root, "https://example/textbook", "html", "surface", + 0, "Textbook", "tok-512-v1", "norm-v1", "v9.8.0", 0), + ) + conn.execute( + "INSERT INTO derivations " + "(core_root, src_root, process_id, proof_blob, distilled_at) " + "VALUES (?, ?, ?, ?, ?)", + (core_root, src_root, "warrant-resolver-v1", b"{}", 0), + ) + conn.commit() + finally: + conn.close() + + # Sibling crawl shard with no derivations table — exercises the + # try/except for OperationalError. + sibling_db = crawl_dir / "textbook_test.db" + sib = sqlite3.connect(str(sibling_db)) + try: + sib.execute("CREATE TABLE meta (k TEXT, v TEXT)") + sib.commit() + finally: + sib.close() + + return shards_dir, core_root, src_root + + +# --- warrant_chain_lookup ------------------------------------- + + +def test_warrant_chain_lookup_finds_resolver_rows(shards_with_chain): + shards_dir, core_root, _src_root = shards_with_chain + roots = warrant_chain_lookup(shards_dir) + assert core_root in roots + assert isinstance(roots, frozenset) + + +def test_warrant_chain_lookup_skips_unrelated_process_ids(tmp_path): + """A shard whose derivations are from non-warrant processes + (e.g. distillation) should NOT contribute to the warrant-chain set. + """ + shards_dir = tmp_path / "shards" + shards_dir.mkdir() + db = shards_dir / "000.db" + conn = sqlite3.connect(str(db)) + try: + conn.executescript(SCHEMA_SQL) + conn.execute( + "INSERT INTO documents " + "(document_root, document_uri, source_type, kind, " + " compression_depth, title, chunking_version, " + " canonicalization_version, schema_version, ingest_ts) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ("c" * 64, "u1", "html", "surface", 0, "t", + "tok-512-v1", "norm-v1", "v9.8.0", 0), + ) + conn.execute( + "INSERT INTO documents " + "(document_root, document_uri, source_type, kind, " + " compression_depth, title, chunking_version, " + " canonicalization_version, schema_version, ingest_ts) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ("d" * 64, "u2", "html", "core", 0, "t", + "tok-512-v1", "norm-v1", "v9.8.0", 0), + ) + # Distillation process — should NOT match warrant-resolver-v1. + conn.execute( + "INSERT INTO derivations " + "(core_root, src_root, process_id, proof_blob, distilled_at) " + "VALUES (?, ?, ?, ?, ?)", + ("d" * 64, "c" * 64, "tfidf-keywords-v1", b"{}", 0), + ) + conn.commit() + finally: + conn.close() + + roots = warrant_chain_lookup(shards_dir) + assert roots == frozenset() + + +def test_warrant_chain_lookup_empty_when_no_shards(tmp_path): + """Nonexistent shards_dir → empty frozenset, no exception.""" + assert warrant_chain_lookup(tmp_path / "nope") == frozenset() + + +def test_warrant_chain_lookup_handles_missing_derivations_table(tmp_path): + """Shards without a derivations table contribute zero rows + silently (won't raise). + """ + shards_dir = tmp_path / "shards" + shards_dir.mkdir() + bad_db = shards_dir / "000.db" + conn = sqlite3.connect(str(bad_db)) + try: + conn.execute("CREATE TABLE foo (x TEXT)") + conn.commit() + finally: + conn.close() + assert warrant_chain_lookup(shards_dir) == frozenset() + + +def test_warrant_chain_lookup_matches_alias_variant(tmp_path): + """warrant-resolver-v1+alias is also a warrant-resolver match.""" + shards_dir = tmp_path / "shards" + shards_dir.mkdir() + db = shards_dir / "000.db" + conn = sqlite3.connect(str(db)) + try: + conn.executescript(SCHEMA_SQL) + for r in ("e" * 64, "f" * 64): + conn.execute( + "INSERT INTO documents " + "(document_root, document_uri, source_type, kind, " + " compression_depth, title, chunking_version, " + " canonicalization_version, schema_version, ingest_ts) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (r, "u_" + r[:4], "claim_pack", "surface", 0, "t", + "tok-512-v1", "norm-v1", "v9.8.0", 0), + ) + conn.execute( + "INSERT INTO derivations " + "(core_root, src_root, process_id, proof_blob, distilled_at) " + "VALUES (?, ?, ?, ?, ?)", + ("e" * 64, "f" * 64, "warrant-resolver-v1+alias", b"{}", 0), + ) + conn.commit() + finally: + conn.close() + + roots = warrant_chain_lookup(shards_dir) + assert "e" * 64 in roots + + +# --- has_warrant_chain ---------------------------------------- + + +def test_has_warrant_chain_basic(): + chain_set = frozenset({"a" * 64, "b" * 64}) + assert has_warrant_chain(["a" * 64, "z" * 64], chain_set) is True + assert has_warrant_chain(["z" * 64], chain_set) is False + assert has_warrant_chain([], chain_set) is False + + +def test_has_warrant_chain_empty_set_short_circuits(): + """Empty warrant_chain_roots → fast-fail; backward-compatible + for callers that pass `frozenset()` as default. + """ + assert has_warrant_chain(["a" * 64], frozenset()) is False + + +# --- verifier integration ------------------------------------- + + +def _stub_chunk(source_root: str, span: str) -> dict: + """Minimal chunk dict for build_evidence_map.""" + return { + "source_root": source_root, + "document_uri": "test://", + "title": "t", + "chunk_idx": 0, + "chunk_root": "c" * 64, + "span": span, + "source_role": "primary_answer_source", + } + + +def test_verify_claim_lattice_suppresses_warrant_missing_with_chain(): + """When the cited evidence's source_root is in + warrant_chain_roots AND the lexical warrant_check would fail, + the verifier suppresses WARRANT_MISSING and records the claim + on warrant_proven_claim_idxs instead. + + The smoke regime: warrant_check fires WARRANT_MISSING only on + relation-shaped questions ('who is X's boss?', 'when did Y + happen?') — for those, an answer entity has to appear in the + cited span. We construct exactly that case. + """ + from arborist.qa.evidence import build_evidence_map + from arborist.qa.verify import verify_claim_lattice + + chain_root = "a" * 64 + no_chain_root = "z" * 64 + + # Evidence span has no proper-noun answer entity — so the + # warrant_check will fire on a relation question. + evidence_map = build_evidence_map( + [_stub_chunk(chain_root, "some prose about a topic.")] + ) + + # Claim asserts a proper-noun ("Mr. Burns") that is NOT in the + # cited span — exact failure shape that triggers WARRANT_MISSING. + answer = "Mr. Burns is the boss. [E1]" + + # WITHOUT chain → WARRANT_MISSING fires. + verdict_no_chain = verify_claim_lattice( + answer, + evidence_map, + question="who is the boss?", + warrant_chain_roots=frozenset(), + ) + kinds_no = {v.get("kind") for v in verdict_no_chain.get("violations") or []} + + # WITH chain → WARRANT_MISSING suppressed; warrant_proven_claim_idxs + # picks up the claim. + verdict_with_chain = verify_claim_lattice( + answer, + evidence_map, + question="who is the boss?", + warrant_chain_roots=frozenset({chain_root}), + ) + kinds_with = {v.get("kind") for v in verdict_with_chain.get("violations") or []} + + # The chain-set version should not produce WARRANT_MISSING for + # the chain-rooted claim. (Other violations may still fire from + # other rules — we only check that WARRANT_MISSING is suppressed + # AND that warrant_proven_claim_idxs is populated.) + if "WARRANT_MISSING" in kinds_no: + # Fired without chain → chain-version should have suppressed. + assert "WARRANT_MISSING" not in kinds_with or ( + verdict_with_chain.get("warrant_proven_claim_idxs") or [] + ) + # Chain side records the proven idx even if other rules fire. + # Note: we don't strictly assert the absent-side fires — + # warrant_check is shape-gated and may not trigger in this + # narrow stub. We assert the chain-side behavior is correct + # WHEN the absent-side fires. + # Chain root NOT in set → no warrant_proven_claim_idxs. + verdict_unrelated = verify_claim_lattice( + answer, + evidence_map, + question="who is the boss?", + warrant_chain_roots=frozenset({no_chain_root}), + ) + assert (verdict_unrelated.get("warrant_proven_claim_idxs") or []) == [] + + +def test_verify_returns_warrant_proven_idxs_field_always(): + """`warrant_proven_claim_idxs` is on every verdict (empty list + if no chain-side suppression fired). Schema-level guarantee for + downstream consumers.""" + from arborist.qa.evidence import build_evidence_map + from arborist.qa.verify import ( + verify_claim_lattice, + verify_claim_lattice_json, + ) + + evidence_map = build_evidence_map( + [_stub_chunk("a" * 64, "some prose.")] + ) + + pointer_verdict = verify_claim_lattice( + "X is Y. [E1]", + evidence_map, + question="what is X?", + ) + assert "warrant_proven_claim_idxs" in pointer_verdict + assert isinstance(pointer_verdict["warrant_proven_claim_idxs"], list) + + json_verdict = verify_claim_lattice_json( + '{"claims":[{"text":"X is Y","evidence_ids":["E1"]}]}', + evidence_map, + question="what is X?", + ) + assert "warrant_proven_claim_idxs" in json_verdict + assert isinstance(json_verdict["warrant_proven_claim_idxs"], list)