russell@unturf.com
856b3116d7
A content-addressed, Merkle-committed document store implementing the runtime spec from Merkle Providence Reverse RAG (April 2026 whitepaper) and Merkle-AGI v9.8 admissibility ledger. Ports proxy.unturf.com Go merkle conventions to Python: non-commutative HashCombine with 0x03 prefix, explicit IsLeft per sibling, self-duplicate odd elements. What's in: - merkle.py — proof generation/verification, JSON serialization - store.py — v9.8 SQLite schema: 8-dim providence_cache key, falsification_state, append-only audit chain, surface/core kind, hot/warm/cold tier, derivations, edges - ingest.py — Source -> normalize -> chunk -> merkle -> upsert, idempotent on document_root collision - search/ — SearchBackend ABC with explicit AuditMode (STRICT/HYBRID/ VISUAL), FTS5 backend returning VISUAL hits - sources/ — wikipedia.py (streaming bz2/MySQL extended-INSERT parser for 2003-era cur dumps); html_page.py (selectolax + httpx, robots.txt honored automatically) - distill/ — Distiller ABC + first-sentence-v1 stub. Runner generates per-contributing-chunk Merkle proofs binding cores back to source document_root. - evict.py — hot->cold demote (NULLs content, drops FTS row, retains leaf_hash). rehydrate() refetches via source pipeline; matching root restores content, mismatching root marks providence stale and writes rehydrate_drift event. Cores never evict. - cli.py — ingest / search / verify / stats / distill / evict / rehydrate - 31 tests covering merkle round-trip, ingest+audit, chunker version binding, html parse, distillation proof verification, evict+ rehydrate including drift detection. Smoke: 503 Wikipedia 2003-05-16 + 3 fox-owned HTML pages ingested, 478 cores produced (24 surface->core merkle dedups), 7 chunks evicted to cold and round-tripped via rehydrate, 987 audit events chained 0 breaks.
101 lines
3.2 KiB
Python
101 lines
3.2 KiB
Python
"""Document and Chunk dataclasses + canonical chunkers.
|
|
|
|
Every Document carries a URI (identification, backtrack, cross-link) and content.
|
|
Chunkers split content into byte-determined chunks before Merkle hashing. The
|
|
chunker's name is committed in chunking_version — changing chunker invalidates
|
|
all prior cache records under v9.8 admissibility.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import unicodedata
|
|
from dataclasses import dataclass, field
|
|
from typing import Protocol
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Edge:
|
|
"""A cross-link from this document to another."""
|
|
|
|
edge_type: str # wikilink | citation | derived_from | ...
|
|
dst_uri: str # always present
|
|
dst_root: str | None = None # filled in later if/when target is ingested
|
|
anchor: str | None = None # optional fragment / chunk index
|
|
|
|
|
|
@dataclass
|
|
class Document:
|
|
"""An ingestable document: URI + content + outbound edges."""
|
|
|
|
uri: str
|
|
content: str # normalized text
|
|
source_type: str # wikipedia_xml | html | git | ...
|
|
title: str | None = None
|
|
edges: list[Edge] = field(default_factory=list)
|
|
extra: dict = field(default_factory=dict) # source-specific metadata
|
|
|
|
|
|
def canonicalize(text: str) -> str:
|
|
"""Stable text normalization. Bumping this requires CANONICALIZATION_VERSION bump."""
|
|
# NFC unicode, normalize whitespace runs to single spaces, strip ends.
|
|
text = unicodedata.normalize("NFC", text)
|
|
text = re.sub(r"[\r\n\t\f\v]+", "\n", text)
|
|
text = re.sub(r"[ ]{2,}", " ", text)
|
|
return text.strip()
|
|
|
|
|
|
class Chunker(Protocol):
|
|
"""A chunker splits canonicalized text into ordered chunks."""
|
|
|
|
name: str
|
|
|
|
def split(self, text: str) -> list[str]: ...
|
|
|
|
|
|
class TokenChunker:
|
|
"""512-token chunker (whitespace-tokenized, byte-deterministic).
|
|
|
|
"Token" here means whitespace-separated unit, NOT a model BPE token. This
|
|
avoids tokenizer-version drift in the chunking_version.
|
|
"""
|
|
|
|
name = "tok-512-v1"
|
|
|
|
def __init__(self, tokens_per_chunk: int = 512):
|
|
self.tokens_per_chunk = tokens_per_chunk
|
|
|
|
def split(self, text: str) -> list[str]:
|
|
if not text:
|
|
return []
|
|
tokens = text.split()
|
|
if not tokens:
|
|
return []
|
|
chunks: list[str] = []
|
|
for start in range(0, len(tokens), self.tokens_per_chunk):
|
|
chunks.append(" ".join(tokens[start : start + self.tokens_per_chunk]))
|
|
return chunks
|
|
|
|
|
|
class SentenceChunker:
|
|
"""Sentence-aligned chunker (better for short docs like 2003 Wikipedia)."""
|
|
|
|
name = "sent-v1"
|
|
|
|
_split_re = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9])")
|
|
|
|
def split(self, text: str) -> list[str]:
|
|
if not text:
|
|
return []
|
|
# Naive but deterministic.
|
|
sentences = [s.strip() for s in self._split_re.split(text) if s.strip()]
|
|
return sentences or ([text] if text else [])
|
|
|
|
|
|
def get_chunker(name: str | None = None) -> Chunker:
|
|
"""Lookup chunker by name. Default = TokenChunker."""
|
|
if name is None or name == TokenChunker.name:
|
|
return TokenChunker()
|
|
if name == SentenceChunker.name:
|
|
return SentenceChunker()
|
|
raise ValueError(f"unknown chunker: {name}")
|