arborist/arborist/qa/retrieval_plan.py

"""Retrieval-plan provenance binding (Ticket #000001 / Directive D4).

Captures the *input* side of retrieval as a content-addressed hash so
the audit chain reproduces both *what got retrieved* (sources_summary,
which `arborist.qa.dag` already binds) and *how retrieval got there*
(the operator-influenceable inputs: keywords, top_k, over_fetch,
max_context_chars, shard set).

Without this hash, two runs with the same question + different
retrieval keywords that surface identical sources would be Merkle-
indistinguishable — an audit could recover "these documents were
selected" but not "these were the keywords that pulled them in." See
`docs/tickets/ticket-000001-retrieval-keywords-audit-gap.md` for the full
problem statement.

Hard or soft? Hard. The hash is SHA-256 over canonical-JSON; the
output is reproducible byte-for-byte across machines. Belongs in the
proof path. Per CLAUDE.md "Soft hash vs hard hash": commitments,
proofs, cache_key. Soft signals (embeddings, similarity scores)
never enter this module.

What is NOT here:
  - cache_key impact. The retrieval plan affects which sources got
    chosen, which already routes through `context_root` and
    `conversation_hash` into `cache_key`. Adding the plan as a 9th
    cache_key dimension is a separate decision (see ticket §5).
    This module only binds the plan into the run-DAG retrieval stage.
  - audit events. `retrieval_plan_built` and
    `retrieval_result_selected` events live in a future scope —
    additive, can land separately.
  - providence_cache column. Direct SQL queryability without
    parsing run-DAG blobs is a follow-up enhancement; this module
    just provides the hash so the per-run merkle proof carries the
    plan.
"""
from __future__ import annotations

import hashlib
import json
from dataclasses import asdict, dataclass, field


@dataclass(frozen=True)
class RetrievalPlan:
    """Operator-influenceable retrieval inputs that determine source
    selection. Captured per-call; folded into a content-addressed
    hash via ``retrieval_plan_hash``.

    Design note: the question text itself is intentionally NOT here
    (it's already in ``question_hash``, a separate cache_key dim).
    Only the *retrieval-side knobs* belong here — these are the
    inputs an auditor needs to reproduce which sources got selected
    given the same question.
    """
    retrieval_keywords: str = ""
    top_k: int = 0
    over_fetch: int = 0
    max_context_chars: int = 0
    # Optional: shard ids the search ran against. Empty tuple means
    # "default shard discovery" (whatever ``_search_corpus`` resolved
    # at call time). Operators who pin specific shards via
    # ``--shards-dir`` or ``single_db`` get those captured here.
    shard_ids: tuple[str, ...] = field(default_factory=tuple)
    # Ticket #000056 (Operation Sandwich) — cross-language query MT is
    # a *retrieval transform* (the query the user asked was translated
    # before it hit FTS5; an auditor reproducing source selection needs
    # the engine identity). Same status as ``retrieval_keywords``:
    # binds into the run-DAG retrieval stage, NOT ``question_hash`` /
    # ``governance_policy_hash``. Empty (the non-MT default) → omitted
    # from ``canonical()`` so every pre-#000056 ``retrieval_plan_hash``
    # is byte-identical (the §5 zero-churn discipline; ticket §5 #3).
    mt_engine: str = ""
    mt_manifest_hash: str = ""
    source_lang: str = ""

    def canonical(self) -> dict:
        """Sorted-key dict for canonical-JSON hashing. Empty fields
        keep their default values so the hash is stable across calls
        that omit optional knobs. The #000056 MT fields are
        *omitted entirely* when unset so non-MT runs hash exactly as
        they did pre-#000056 (zero churn)."""
        out = {
            "retrieval_keywords": self.retrieval_keywords or "",
            "top_k": int(self.top_k),
            "over_fetch": int(self.over_fetch),
            "max_context_chars": int(self.max_context_chars),
            "shard_ids": list(self.shard_ids),
        }
        if self.mt_engine or self.mt_manifest_hash or self.source_lang:
            out["mt"] = {
                "engine": self.mt_engine or "",
                "manifest_hash": self.mt_manifest_hash or "",
                "source_lang": self.source_lang or "",
            }
        return out


def retrieval_plan_hash(plan: RetrievalPlan) -> str:
    """SHA-256 over the canonical-JSON of the retrieval plan.

    Deterministic: same plan → same hash, byte-for-byte across
    machines. Folds into the run-DAG retrieval stage via
    ``arborist.qa.dag.build_run_dag(retrieval_plan_hash=...)``.
    """
    canon = json.dumps(plan.canonical(), sort_keys=True, separators=(",", ":"))
    return hashlib.sha256(canon.encode("utf-8")).hexdigest()