arborist/aborist/merkle.py
russell@unturf.com 856b3116d7
phase 0 explore: aborist core + sources + distill + evict
A content-addressed, Merkle-committed document store implementing the
runtime spec from Merkle Providence Reverse RAG (April 2026 whitepaper)
and Merkle-AGI v9.8 admissibility ledger. Ports proxy.unturf.com Go
merkle conventions to Python: non-commutative HashCombine with 0x03
prefix, explicit IsLeft per sibling, self-duplicate odd elements.

What's in:

- merkle.py — proof generation/verification, JSON serialization
- store.py — v9.8 SQLite schema: 8-dim providence_cache key,
  falsification_state, append-only audit chain, surface/core kind,
  hot/warm/cold tier, derivations, edges
- ingest.py — Source -> normalize -> chunk -> merkle -> upsert,
  idempotent on document_root collision
- search/ — SearchBackend ABC with explicit AuditMode (STRICT/HYBRID/
  VISUAL), FTS5 backend returning VISUAL hits
- sources/ — wikipedia.py (streaming bz2/MySQL extended-INSERT parser
  for 2003-era cur dumps); html_page.py (selectolax + httpx, robots.txt
  honored automatically)
- distill/ — Distiller ABC + first-sentence-v1 stub. Runner generates
  per-contributing-chunk Merkle proofs binding cores back to source
  document_root.
- evict.py — hot->cold demote (NULLs content, drops FTS row, retains
  leaf_hash). rehydrate() refetches via source pipeline; matching root
  restores content, mismatching root marks providence stale and writes
  rehydrate_drift event. Cores never evict.
- cli.py — ingest / search / verify / stats / distill / evict /
  rehydrate
- 31 tests covering merkle round-trip, ingest+audit, chunker version
  binding, html parse, distillation proof verification, evict+
  rehydrate including drift detection.

Smoke: 503 Wikipedia 2003-05-16 + 3 fox-owned HTML pages ingested,
478 cores produced (24 surface->core merkle dedups), 7 chunks evicted
to cold and round-tripped via rehydrate, 987 audit events chained 0
breaks.
2026-04-27 07:53:18 -04:00

160 lines
4.7 KiB
Python

"""Merkle tree with non-commutative HashCombine.
Python port of ~/git/proxy.unturf.com/pkg/verified/merkle.go conventions:
- Domain separation via single-byte prefixes (leaf=0x00, node=0x03).
- HashCombine is non-commutative; sibling order matters always.
- Odd layers self-duplicate the trailing element (NOT zero-pad).
- Proof carries explicit IsLeft flag per sibling (NOT lexical sort).
- Empty tree root is ZeroHash (32 zero bytes).
"""
from __future__ import annotations
import hashlib
from dataclasses import dataclass, field
from typing import Iterable
LEAF_PREFIX = b"\x00"
NODE_PREFIX = b"\x03"
ZERO_HASH = b"\x00" * 32
HASH_LEN = 32
def _sha256(*parts: bytes) -> bytes:
h = hashlib.sha256()
for p in parts:
h.update(p)
return h.digest()
def hash_leaf(content: bytes) -> bytes:
"""Hash a leaf with domain prefix 0x00."""
return _sha256(LEAF_PREFIX, content)
def hash_combine(left: bytes, right: bytes) -> bytes:
"""Non-commutative interior combine with domain prefix 0x03."""
if len(left) != HASH_LEN or len(right) != HASH_LEN:
raise ValueError("hash inputs must be 32 bytes")
return _sha256(NODE_PREFIX, left, right)
@dataclass(frozen=True)
class ProofNode:
"""One sibling step in a Merkle inclusion proof.
is_left=True means the sibling sits to the LEFT of the running hash,
so verification order is: HashCombine(sibling, current).
"""
hash: bytes
is_left: bool
@dataclass(frozen=True)
class MerkleProof:
leaf: bytes
leaf_index: int
siblings: tuple[ProofNode, ...]
root: bytes
@dataclass
class MerkleTree:
"""Layered tree. layers[0] = leaves, layers[-1] = [root]."""
layers: list[list[bytes]] = field(default_factory=list)
@property
def root(self) -> bytes:
if not self.layers or not self.layers[-1]:
return ZERO_HASH
return self.layers[-1][0]
@property
def leaves(self) -> list[bytes]:
return self.layers[0] if self.layers else []
@classmethod
def build(cls, leaves: Iterable[bytes]) -> MerkleTree:
leaves = list(leaves)
if not leaves:
return cls(layers=[[]])
layers: list[list[bytes]] = [list(leaves)]
current = list(leaves)
while len(current) > 1:
nxt: list[bytes] = []
i = 0
while i < len(current):
left = current[i]
right = current[i + 1] if i + 1 < len(current) else current[i]
nxt.append(hash_combine(left, right))
i += 2
layers.append(nxt)
current = nxt
return cls(layers=layers)
def proof(self, leaf_index: int) -> MerkleProof:
if not self.layers or not self.layers[0]:
raise IndexError("empty tree has no proofs")
if leaf_index < 0 or leaf_index >= len(self.layers[0]):
raise IndexError(f"leaf_index {leaf_index} out of range")
siblings: list[ProofNode] = []
idx = leaf_index
# Walk up every layer except the root layer.
for layer in self.layers[:-1]:
if idx % 2 == 0:
sibling_idx = idx + 1
is_left = False # sibling is to our right
else:
sibling_idx = idx - 1
is_left = True # sibling is to our left
if sibling_idx >= len(layer):
# odd-element rule: self-duplicate
sibling_idx = idx
siblings.append(ProofNode(hash=layer[sibling_idx], is_left=is_left))
idx //= 2
return MerkleProof(
leaf=self.layers[0][leaf_index],
leaf_index=leaf_index,
siblings=tuple(siblings),
root=self.root,
)
def verify_proof(proof: MerkleProof) -> bool:
"""Recompute root from leaf + sibling path. Returns True iff matches."""
current = proof.leaf
for node in proof.siblings:
if node.is_left:
current = hash_combine(node.hash, current)
else:
current = hash_combine(current, node.hash)
return current == proof.root
def proof_to_dict(proof: MerkleProof) -> dict:
"""JSON-serializable form for storage in providence_cache.merkle_proof."""
return {
"leaf": proof.leaf.hex(),
"leaf_index": proof.leaf_index,
"siblings": [
{"hash": s.hash.hex(), "is_left": s.is_left} for s in proof.siblings
],
"root": proof.root.hex(),
}
def proof_from_dict(d: dict) -> MerkleProof:
return MerkleProof(
leaf=bytes.fromhex(d["leaf"]),
leaf_index=int(d["leaf_index"]),
siblings=tuple(
ProofNode(hash=bytes.fromhex(s["hash"]), is_left=bool(s["is_left"]))
for s in d["siblings"]
),
root=bytes.fromhex(d["root"]),
)