diff --git a/arborist/cold_object.py b/arborist/cold_object.py index 0ad363a..3e626a5 100644 --- a/arborist/cold_object.py +++ b/arborist/cold_object.py @@ -210,6 +210,79 @@ class ObjectStoreBackend(abc.ABC): def get_pack_manifest(self, pack_hash: str, *, kind: str = "chunks") -> bytes: return self.get(pack_key(pack_hash, kind=kind, manifest=True)) + # ----- latest-pointer helpers (Gap 1: discovery for fresh peers) ----- + # + # `manifest/latest.json` is a mutable NDJSON catalog mapping + # snapshot_root → metadata_pack_hash. Producer maintains it on every + # push (read-modify-write — last writer wins on contention). Consumer + # reads it to answer "what's the current metadata pack for shard X?" + # without having to enumerate every pack via list_keys. + # + # Mutability is acceptable here: the latest pointer is a discovery + # convenience, not a trust root. Each metadata pack is still content- + # addressed by its own pack_hash; the latest pointer just tells you + # which hash to start from. + + def get_latest_pointer(self) -> dict[str, dict]: + """Read `manifest/latest.json` as a {snapshot_root: entry} dict. + + Returns `{}` if the pointer doesn't exist yet. Each entry shape: + { + "metadata_pack_hash": "...", + "snapshot_doc_count": N, + "snapshot_ts": , + "license_class": "...", + "updated_at": , + } + """ + try: + body = self.get(MANIFEST_LATEST_KEY) + except Exception: + return {} + out: dict[str, dict] = {} + for line in body.decode("utf-8").splitlines(): + line = line.strip() + if not line: + continue + rec = json.loads(line) + sr = rec.get("snapshot_root") + if sr: + out[sr] = rec + return out + + def update_latest_pointer( + self, + snapshot_root: str, + metadata_pack_hash: str, + *, + snapshot_doc_count: int, + snapshot_ts: int, + license_class: str, + updated_at: int, + ) -> None: + """Read-modify-write the latest pointer for one snapshot_root. + + Last writer wins on contention — this pointer is a discovery + convenience, not a trust root. If two producers race, one of the + two pack_hashes ends up advertised; both packs remain in the + bucket as content-addressed artifacts and can be reached by + `cold list` regardless. + """ + current = self.get_latest_pointer() + current[snapshot_root] = { + "snapshot_root": snapshot_root, + "metadata_pack_hash": metadata_pack_hash, + "snapshot_doc_count": snapshot_doc_count, + "snapshot_ts": snapshot_ts, + "license_class": license_class, + "updated_at": updated_at, + } + lines = [] + for sr in sorted(current): + lines.append(json.dumps(current[sr], sort_keys=True)) + body = ("\n".join(lines) + "\n").encode("utf-8") + self.put(MANIFEST_LATEST_KEY, body, content_type="application/x-ndjson") + class S3CompatibleBackend(ObjectStoreBackend): """boto3-based S3-compatible backend. @@ -685,6 +758,7 @@ def build_metadata_pack( *, snapshot_root: str, chunk_pack_hashes: list[str], + license_class: str = "unknown", work_dir: Path | str | None = None, level: int = 3, ) -> FilePack: @@ -744,6 +818,11 @@ def build_metadata_pack( {"_chunk_pack_hashes": chunk_pack_hashes_sorted}, sort_keys=True, ), + # license_class drives Gap 2: a public-read bucket cannot accept + # a shard whose strictest source license isn't public-redistributable. + # Stored in the manifest so consumers + auditors can see the + # producer's classification without inspecting source documents. + json.dumps({"_license_class": license_class}, sort_keys=True), ] for member_name, content_hash, table_size in table_refs_sorted: manifest_lines.append(json.dumps( @@ -848,6 +927,10 @@ class ParsedManifest: - kind: "metadata" | "chunks" | None (for v1/v2 mixed). - snapshot_root: corpus state the pack covers (v3 only). - chunk_pack_hashes: every chunk pack a v3 metadata pack references. + - license_class: strictest license across the shard's docs + ("public_redistributable" | "unknown" | "private"). None on v1/v2 + packs (no field) or on chunks-kind packs (only the metadata pack + carries this). - tables: TableRef list (empty for chunks-only packs). - chunks: PackEntry list (empty for metadata-only packs). """ @@ -855,6 +938,7 @@ class ParsedManifest: kind: str | None snapshot_root: str | None chunk_pack_hashes: tuple[str, ...] + license_class: str | None tables: tuple[TableRef, ...] chunks: tuple[PackEntry, ...] @@ -874,6 +958,7 @@ def parse_manifest(manifest_bytes: bytes) -> ParsedManifest: kind: str | None = None snapshot_root: str | None = None chunk_pack_hashes: list[str] = [] + license_class: str | None = None tables: list[TableRef] = [] chunks: list[PackEntry] = [] for line in manifest_bytes.decode("utf-8").splitlines(): @@ -889,6 +974,8 @@ def parse_manifest(manifest_bytes: bytes) -> ParsedManifest: snapshot_root = str(rec["_snapshot_root"]) elif "_chunk_pack_hashes" in rec: chunk_pack_hashes = [str(h) for h in rec["_chunk_pack_hashes"]] + elif "_license_class" in rec: + license_class = str(rec["_license_class"]) elif "table_file" in rec: tables.append(TableRef( member_name=rec["table_file"], @@ -905,6 +992,7 @@ def parse_manifest(manifest_bytes: bytes) -> ParsedManifest: kind=kind, snapshot_root=snapshot_root, chunk_pack_hashes=tuple(chunk_pack_hashes), + license_class=license_class, tables=tuple(tables), chunks=tuple(chunks), ) diff --git a/arborist/cold_pack_metadata.py b/arborist/cold_pack_metadata.py index de31480..73d3643 100644 --- a/arborist/cold_pack_metadata.py +++ b/arborist/cold_pack_metadata.py @@ -1,4 +1,4 @@ -"""Shard metadata dump/restore for pack format v2 (#000061 redesign). +"""Shard metadata dump/restore for pack format v3 (#000061). Pack v2 ships every load-bearing table from a shard so a new peer can hydrate from packs alone. Two compression strategies: @@ -39,6 +39,10 @@ from typing import Iterable, Iterator # documents, edges after documents, etc. derivations references both # core_root and src_root → after documents. SHIPPED_TABLES: tuple[str, ...] = ( + # NOTE: keep order stable — restore inserts in this order so foreign- + # key dependencies (chunks → documents, edges → documents, etc.) are + # satisfied before dependents land. + "documents", "document_http_meta", "chunks", # content column dropped at dump time @@ -74,6 +78,74 @@ COLUMN_FILTER: dict[str, frozenset[str]] = { } +# --------------------------------------------------------------------------- +# License class — drives Gap 2 (refuse-to-push-private-to-public-bucket). +# Maps `documents.source_type` to a redistribution license bucket. Pack +# producer reads this map, takes the strictest class across the shard's +# docs, and refuses to push if that class is more restrictive than the +# operator's `--allow-license-class` flag (default: `public_redistributable`). +# Keeps private/licensed corpora out of public-read buckets unless the +# operator explicitly opts in. +# --------------------------------------------------------------------------- + +LICENSE_CLASS_PUBLIC = "public_redistributable" +LICENSE_CLASS_UNKNOWN = "unknown" +LICENSE_CLASS_PRIVATE = "private" + +# Strictness order: public is the LEAST restrictive (anyone can have these +# bytes), unknown is intermediate (operator hasn't classified), private is +# the MOST restrictive (must never end up on a public bucket). +LICENSE_CLASS_STRICTNESS = { + LICENSE_CLASS_PUBLIC: 0, + LICENSE_CLASS_UNKNOWN: 1, + LICENSE_CLASS_PRIVATE: 2, +} + +# Per-source-type license classification. Conservative — anything not in +# this map is `unknown`, which refuses public-bucket push unless the +# operator overrides. New source types must be classified explicitly here +# before they can be cold-shipped to a public bucket. +SOURCE_TYPE_LICENSE_CLASS: dict[str, str] = { + "wikipedia_cur": LICENSE_CLASS_PUBLIC, # CC-BY-SA + GFDL + "wikipedia_old": LICENSE_CLASS_PUBLIC, + "wikipedia_xml": LICENSE_CLASS_PUBLIC, + "wikipedia": LICENSE_CLASS_PUBLIC, + "textbook_tex": LICENSE_CLASS_PUBLIC, # PG / CC textbooks + "textbook_pg": LICENSE_CLASS_PUBLIC, + "claim_pack": LICENSE_CLASS_PUBLIC, # internal-derived from public + "grok": LICENSE_CLASS_UNKNOWN, # API-derived; depends on TOS + "html_page": LICENSE_CLASS_UNKNOWN, # operator-fetched; varies + "html": LICENSE_CLASS_UNKNOWN, + "vcs": LICENSE_CLASS_UNKNOWN, # depends on repo license +} + + +def license_class_for_source(source_type: str) -> str: + return SOURCE_TYPE_LICENSE_CLASS.get(source_type, LICENSE_CLASS_UNKNOWN) + + +def compute_shard_license_class(conn) -> str: + """Return the strictest license_class across every document in the shard. + + Strictest wins: a shard with even one private document is private; + a shard with any unknown is unknown; only all-public-redistributable + shards are eligible for default public-bucket push. + """ + rows = conn.execute( + "SELECT DISTINCT source_type FROM documents" + ).fetchall() + if not rows: + # Empty shard — vacuous, treat as public so cold-pack doesn't refuse + # on a freshly-created shard with no content yet. + return LICENSE_CLASS_PUBLIC + strictest = LICENSE_CLASS_PUBLIC + for r in rows: + cls = license_class_for_source(r["source_type"]) + if LICENSE_CLASS_STRICTNESS[cls] > LICENSE_CLASS_STRICTNESS[strictest]: + strictest = cls + return strictest + + # --------------------------------------------------------------------------- # Generic array-per-line JSONL helpers # --------------------------------------------------------------------------- diff --git a/arborist/evict.py b/arborist/evict.py index 5a1c69d..f532a96 100644 --- a/arborist/evict.py +++ b/arborist/evict.py @@ -281,6 +281,7 @@ def push_pack( max_pack_bytes: int = DEFAULT_MAX_PACK_BYTES, local_dir: str | None = None, push_to_bucket: bool = True, + allow_license_class: str = "public_redistributable", ) -> dict: """Bundle local chunks into one or more tar.zst packs. @@ -319,9 +320,32 @@ def push_pack( hash_file_leaf, stream_packs, ) - from arborist.cold_pack_metadata import dump_shard_metadata + from arborist.cold_pack_metadata import ( + LICENSE_CLASS_STRICTNESS, + compute_shard_license_class, + dump_shard_metadata, + ) from arborist.snapshot import compute_snapshot_root + # Gap 2 — license_class policy gate. Compute the strictest license + # across the shard's source documents. Refuse to push to the bucket + # if that class is stricter than the operator's --allow-license-class + # (default: public_redistributable). Bucket ACLs are operator-set; + # this is arborist refusing to participate in a licensing leak even + # when the bucket would accept the bytes. + shard_license = compute_shard_license_class(conn) + if push_to_bucket and ( + LICENSE_CLASS_STRICTNESS[shard_license] + > LICENSE_CLASS_STRICTNESS[allow_license_class] + ): + raise ValueError( + f"refusing to push: shard license_class={shard_license!r} is " + f"stricter than allow_license_class={allow_license_class!r}. " + f"Override with allow_license_class={shard_license!r} only if " + f"you've confirmed the destination bucket's ACL + the source " + f"licensing permits redistribution. See ticket #000061 Gap 2." + ) + # Default selection = every chunk with local content (surfaces AND # cores). Cores never evict and carry the distilled-from-surface # derivations a new peer needs to bootstrap the v9.8 chain — packing @@ -443,9 +467,33 @@ def push_pack( pack_uncompressed = uncompressed_running[0] - last_finalized_uncompressed[0] last_finalized_uncompressed[0] = uncompressed_running[0] try: + # cold_pending row: trace the in-flight upload so a killed + # process can be recovered. INSERT before upload starts; + # DELETE on success (in the finally below). if push_to_bucket: + from arborist.cold_object import pack_key + with transaction(conn): + conn.execute( + "INSERT OR REPLACE INTO cold_pending " + "(tempfile_path, pack_hash, kind, backend_endpoint, " + " backend_bucket, object_key, started_at, state) " + "VALUES (?, ?, 'chunks', ?, ?, ?, ?, 'uploading')", + ( + str(pack.body_path), + pack.pack_hash, + backend_id["endpoint_url"], + backend_id["bucket"], + pack_key(pack.pack_hash, kind="chunks"), + pack_ts, + ), + ) backend.put_pack_file(pack.pack_hash, pack.body_path, kind="chunks") backend.put_pack_manifest(pack.pack_hash, pack.manifest_bytes, kind="chunks") + with transaction(conn): + conn.execute( + "DELETE FROM cold_pending WHERE tempfile_path = ?", + (str(pack.body_path),), + ) if out_dir is not None: short = pack.pack_hash[:16] final_path = out_dir / f"arborist-pack-{short}.chunks.tar.zst" @@ -479,15 +527,51 @@ def push_pack( table_files, snapshot_root=snapshot_root, chunk_pack_hashes=chunk_pack_hashes_in_order, + license_class=shard_license, ) try: if push_to_bucket: + from arborist.cold_object import pack_key + with transaction(conn): + conn.execute( + "INSERT OR REPLACE INTO cold_pending " + "(tempfile_path, pack_hash, kind, backend_endpoint, " + " backend_bucket, object_key, started_at, state) " + "VALUES (?, ?, 'metadata', ?, ?, ?, ?, 'uploading')", + ( + str(meta_pack.body_path), + meta_pack.pack_hash, + backend_id["endpoint_url"], + backend_id["bucket"], + pack_key(meta_pack.pack_hash, kind="metadata"), + pack_ts, + ), + ) backend.put_pack_file( meta_pack.pack_hash, meta_pack.body_path, kind="metadata" ) backend.put_pack_manifest( meta_pack.pack_hash, meta_pack.manifest_bytes, kind="metadata" ) + # Gap 1: maintain manifest/latest.json so a fresh peer's + # `cold list` can resolve "current metadata pack for + # snapshot_root X" without enumerating every pack. + # Read-modify-write; last-writer-wins on contention. The + # pointer is discovery only — content addressing makes + # the actual artifacts immutable. + backend.update_latest_pointer( + snapshot_root=snapshot_root, + metadata_pack_hash=meta_pack.pack_hash, + snapshot_doc_count=doc_count_at_pack, + snapshot_ts=pack_ts, + license_class=shard_license, + updated_at=pack_ts, + ) + with transaction(conn): + conn.execute( + "DELETE FROM cold_pending WHERE tempfile_path = ?", + (str(meta_pack.body_path),), + ) if out_dir is not None: short = meta_pack.pack_hash[:16] final_path = out_dir / f"arborist-pack-{short}.metadata.tar.zst" diff --git a/arborist/store.py b/arborist/store.py index da24dc8..273bfaf 100644 --- a/arborist/store.py +++ b/arborist/store.py @@ -248,6 +248,26 @@ CREATE TABLE IF NOT EXISTS snapshots ( ); CREATE INDEX IF NOT EXISTS idx_snapshots_taken_at ON snapshots(taken_at); +-- Cold-pack uploads in flight. A row exists between pack-tempfile-creation +-- and successful upload-or-cleanup. If a process is killed mid-push, rows +-- stay; `arborist cold recover` reads them and either (a) confirms the +-- object landed in the bucket and deletes the tempfile + row, or (b) +-- deletes the orphaned tempfile + row. Prevents /tmp from filling with +-- multi-GB pack tempfiles after crashes. See ticket #000061. +CREATE TABLE IF NOT EXISTS cold_pending ( + tempfile_path TEXT PRIMARY KEY, + pack_hash TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('metadata', 'chunks')), + backend_endpoint TEXT NOT NULL, + backend_bucket TEXT NOT NULL, + object_key TEXT NOT NULL, + started_at INTEGER NOT NULL, + state TEXT NOT NULL DEFAULT 'pending' + CHECK (state IN ('pending', 'uploading', 'uploaded')) +); +CREATE INDEX IF NOT EXISTS idx_cold_pending_started_at + ON cold_pending(started_at); + -- Mesh layer tables. Off by default — populated only when the user runs -- `arborist mesh init`. Never accessed by ingest / query / distill paths; -- mesh state is opt-in plumbing for federated peers (see arborist.mesh). diff --git a/docs/TICKETS.md b/docs/TICKETS.md index e07578a..62b0764 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -111,6 +111,8 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| +| #000064 | Cold-object operations toolkit (verify/diff/doctor/repair-fts/gc-plan + audit taxonomy) | **scaffold-only · awaiting go/no-go** (2026-05-26; from Dav1d #000061 review §11/§12/§14). Operator-facing observability + repair tools on top of #000061: `cold verify` (sample/full integrity check), `cold diff` (local vs remote manifest), `cold doctor` (one-shot health: connectivity / credentials / manifest age / missing-object count / tamper sample / audit-chain integrity), `cold repair-fts` (rebuild FTS5 from chunks.content), `cold gc-plan` (orphan bucket objects, read-only by default — destructive only with `--apply` + confirm). Plus expanded audit-event taxonomy: per-PUT/HEAD/GET success/failure events, manifest-pointer events, verify/doctor/gc events. All read-mostly; destructive ops require `--apply`. Bundled so the audit-taxonomy gets one design pass instead of five-way drift. Sequence: doctor → verify → diff → repair-fts → gc-plan. No code until #000061 closes. | 2026-05-26 | — | +| #000063 | Cold-object private-ciphertext mode (mesh-keyed object keys) | **scaffold-only · awaiting go/no-go** (2026-05-26; from Dav1d #000061 review §9 / response A §13.3). Adds private mode to #000061 cold-object format so chunk bodies + manifest can be uploaded to public-read bucket without leaking corpus membership. Two strategies: (A) deterministic `object_key = HMAC(group_key, leaf_hash)` + AEAD-encrypted body — supports lookup-by-leaf-hash given the key; (B) random-key ciphertext + encrypted private manifest — stronger membership hiding, needs manifest fetch first. Strategy A default; B opt-in. Group key from existing `arborist/mesh/crypto.py`; pack manifest carries `epoch_id` for rotation. Verifier path unchanged: consumer decrypts, then `hash_leaf(plaintext) == leaf_hash` as in public mode. No code until (1) a real non-public corpus needs cold-object shipping, (2) mesh group-key ABI is stable enough to reference, (3) threat-model split between A vs B is settled by real adversary. | 2026-05-26 | — | | #000062 | Mechanistic Witness: governed diagnostic sidecar (CNA/SAE/Neuronpedia) | **scaffold-only · awaiting go/no-go** (2026-05-26; Dav1d de-novo review §4.7 / §9.1.F). Specification of a mechanistic-interpretability sidecar that produces a content-addressed `MechanisticWitnessRoot` over (model, prompts, capture policy, neurons/features, intervention deltas), used as a **diagnostic input** to SelfModel (#000014/#000017) + benchmark-fixture generation. **Hard constraint:** soft signals never enter the hard proof path — `audit_mode` does NOT move based on witness output, `providence_cache` is untouched, `governance_policy_hash` only moves via explicit ForkScore ACCEPT with M+C+X axes passing (#000060 §7). Four guardrails (diagnostic-only by default · sandbox intervention only · no production steering without governance · feature labels never semantic proof). Witness root TLV-encodes `model_config_root | activation_capture_policy_root | contrastive_prompt_set_root | feature_or_neuron_set_root | intervention_result_root | behavioral_delta_root | safety_policy_root`. Scaffold only — no code until a real falsifier-in-hand use case exists + the four guardrails are restated in CLAUDE.md as rules + #000060 H-ABCDEFG-M+C+X harness exists to gate promotion. Captured to keep mechanistic-interp tooling out of the substrate unless and until it earns its place; the dual-use risk (Pan et al. 2025 CNA: 0.1% MLP ablation breaks refusal in 72B models) makes the governance-first framing load-bearing. | 2026-05-26 | — | | #000061 | Cold-pack distribution tier (boto3 S3-compat, DO Spaces + DVD-R targets) | **in progress** — opened 2026-05-25 (fox: "implement it now … target digital ocean first as a test"; later "I wanted a way to hydrate using tarballs (the core and important data) for bringing new machines up"). Tarball-only distribution mechanism — bucket holds `tar.zst` packs keyed by `hash_leaf(manifest)`, no individual-chunk blobs. New peers hydrate by downloading packs from the bucket's CDN edge (~4 HTTPS GETs for the current ~14.1M-chunk corpus, packs filled to 4.4 GB compressed each via streaming zstd, vs ~14M for individual blobs). Same artifact ≤4.4 GB safe-fit (~6.5 % buffer below DVD-R's 4.7 GB marketing capacity, accommodating ISO9660 overhead + media variance + drive-edge refusal) burns directly to physical media via `--local-dir` + `growisofs`. Packs are *delayed* snapshots: each pack pins the corpus `snapshot_root` it covers in audit + body, so falsifications between repacks produce new pack_hashes and stale packs stay in the bucket until explicit GC (future ticket). Packs include cores AND surfaces (full-corpus hydration). Default selection covers every hot chunk with local content in the shard. Multi-pack splitting via `stream_packs` (streaming zstd, FLUSH_BLOCK peek of compressed buffer after each chunk, cut at cap) fills each disc to ~4.4 GB compressed instead of leaving ~50% empty. One backend class (`S3CompatibleBackend` via boto3 + `endpoint_url`) covers AWS S3, DO Spaces, GCS S3-interop, R2, B2, MinIO. CDN public-read makes packs accessible to anyone; hash binding via in-tar `leaf_hash` member names makes hostile-bucket scenarios safe. Optional dep `[object-store]` = boto3>=1.34. Voyeur: credentials via standard `AWS_ACCESS_KEY_ID`/`_SECRET_ACCESS_KEY` (env or `~/.aws/credentials`), never printed; only endpoint URL + bucket name surface in logs. Initial individual-blob path (per-chunk S3 objects) was scoped+landed then **deleted same day** (fox: "what ever was blobs? I wanted a way to hydrate using tarballs"); the five-step deletion record lives inline in the doc — we'd added 14M-object storage and ~$70/hydrate request cost for a workflow that needed neither. Sizing math for current shards: ~4 packs total (17.2 GB compressed ÷ 4.4 GB compressed per pack), ~17 GB bucket storage, ~$0.34/mo DO Spaces. | 2026-05-25 | — | | #000060 | H-ABCDEFG same-model substrate-delta harness (+ jaggedness tensor + curvature) | open · awaiting go/no-go (2026-05-20; from Dav1dPrometheus *Protocol-Layer AGI* working report §26/§50/§82-84). The report's "decisive proof": run the SAME base model substrate-OFF vs substrate-ON over long-horizon/adversarial/non-jagged batteries, report the delta. Two new metrics: jaggedness tensor `J_norm` (§73 — variance across nearby variants, normalized by difficulty) + discrete performance curvature `κ_t` (§5.2, with the honest no-global-convexity bound, Erratum 5). A-vs-C spine (B optional, D=mesh OUT → #000012/#000016). Curvature-aware ForkScore extension folds into **#000012** (NOT a new ticket — reserved `iota`/`kappa` weight slots already exist). Budget: control arms = Hermes/Qwen, never Opus without go; heavy passes on GPU box. Held-out/mechanism-agnostic variants required so ABCDEFG doesn't self-validate. **2026-05-26 scope refinement (Dav1d review §4 / §7):** split the harness output into three axes — **H-ABCDEFG-M** (mechanism tests: does the substrate work?), **H-ABCDEFG-C** (capability tests: does the substrate improve task performance?), **H-ABCDEFG-X** (external adversarial: does it generalize outside author-designed fixtures?). The split prevents "self-validating benchmark theology" — a harness that reports only M+C with no X can pass while still failing on held-out adversarial generalization. Fold into the harness design before any code lands; doesn't change the bench-row schema (`carrier`/`domain`/`pi_star_ref`) but does change what "ACCEPT" requires (must clear all three axes). | 2026-05-20 | — | @@ -176,4 +178,4 @@ Newest first. Update on every open/close. ## Next ID -`000063` +`000065` diff --git a/docs/tickets/ticket-000063-cold-object-private-ciphertext-mode.md b/docs/tickets/ticket-000063-cold-object-private-ciphertext-mode.md new file mode 100644 index 0000000..3608d84 --- /dev/null +++ b/docs/tickets/ticket-000063-cold-object-private-ciphertext-mode.md @@ -0,0 +1,120 @@ +# Ticket #000063 — Cold-object private-ciphertext mode (mesh-keyed) + +**Status:** open · awaiting go/no-go · scaffold-only +**Opened:** 2026-05-26 +**Scope:** add a private mode to the #000061 cold-object pack format so + chunk bodies + manifest can be uploaded to a public-read bucket + without leaking corpus membership or document content. Object keys + derive from a mesh group key via HMAC, not from plaintext leaf_hash; + bodies are ciphertext, manifest is encrypted or access-controlled. +**Audience:** dav1d (cold-object security model extension; spec is the + load-bearing artifact, not code). +**Hard constraint:** the integrity invariant from #000061 stays — + verifier recovers plaintext bytes, hashes via `hash_leaf`, compares + to local `chunks.leaf_hash`. Encryption sits between disk and bucket; + inside the verifier path, behavior is identical to public mode. + +## Problem + +#000061 ships `public_plaintext` mode: object key = `leaf_hash`, body = +raw UTF-8 plaintext. Safe for public-redistributable corpora (Wikipedia, +PG textbooks). **Not safe** when the bucket has any chunk whose source +is private, licensed, or membership-sensitive: + +- `object_key = sha256(0x00 ‖ plaintext)` is a membership oracle. An + attacker who guesses a chunk's plaintext can probe the bucket and + learn whether that chunk is in the corpus. +- Plaintext bodies on a public-read bucket are a redistribution of + every chunk to the world. + +#000061 v3 sidesteps this by refusing to push private content to a +public bucket (#000061 Gap 2 — `license_class` policy gate). This +ticket adds the *other* path: ship private content under proper +ciphertext, so the bucket can stay public-read without leaking. + +## Design (spec only — no code yet) + +Two keying strategies: + +**Strategy A — deterministic-key ciphertext** (preferred for indexing): + +``` +plaintext_leaf_hash = chunks.leaf_hash # local DB value +object_key = "blobs/" + HMAC(group_key, plaintext_leaf_hash) +ciphertext_body = AEAD-encrypt(group_key, plaintext_bytes, + aad = plaintext_leaf_hash) +``` + +- Deterministic key allows lookup-by-leaf_hash without a separate + manifest fetch (given the group key). +- AAD binds the ciphertext to the plaintext identity. +- Bucket can be public-read; without `group_key`, neither key nor body + reveals membership. + +**Strategy B — random-key ciphertext + manifest map** (stronger +membership hiding): + +``` +ciphertext_body = AEAD-encrypt(group_key, plaintext_bytes) +ciphertext_hash = sha256(ciphertext_bytes) +object_key = "blobs/" + ciphertext_hash +private_manifest binds plaintext_leaf_hash -> object_key +private_manifest itself encrypted with group_key +``` + +- Membership requires both the encrypted manifest AND the group key. +- Lookup requires manifest fetch; one extra round-trip. +- Defeats prefix-enumeration of `blobs/`. + +Strategy A is the default; Strategy B is opt-in for the most sensitive +corpora. + +### Mesh integration + +The group key lives where it already lives for arborist: +`arborist/mesh/crypto.py` (group-key state machine, epoch rotation, +member wrap/unwrap). This ticket plumbs that key into the pack flow: + +- Producer: `push_pack(..., mode='private', mesh_conn=...)` reads the + current epoch's group key, uses it for HMAC + AEAD. +- Consumer: `hydrate_from_metadata_pack(..., mesh_conn=...)` reads the + same key, decrypts/verifies. +- Key rotation: epoch advance means old packs use old key. Pack + manifest carries `epoch_id` so consumers know which key to use. + +### Verifier path unchanged + +After decryption, the consumer holds plaintext bytes. From that point +on, the cold-object pull path is **byte-identical** to public mode: +`hash_leaf(plaintext) == leaf_hash`, restore into `chunks.content`, +populate FTS5, etc. Ciphertext does not enter `chunks.content`. + +## Why scaffold-only + +This ticket lands when: + +1. There's a real corpus with non-public content that arborist needs + to ship via cold-object. (No use case today — fox's corpora are + Wikipedia + open textbooks.) +2. The mesh group-key ABI is stable enough to be referenced by pack + code. (Today the mesh state lives behind `arborist/mesh/` with its + own state machine.) +3. The threat-model split between Strategy A and Strategy B is settled + by a real adversary scenario, not a paper one. + +Open as scaffold so the design is in the log; defer code until at +least #1 lands. + +## Out of scope + +- Server-side encryption (KMS / SSE-S3). That's bucket-level + encryption-at-rest, orthogonal to membership hiding. Mesh ciphertext + is the client-side path that survives a hostile bucket. +- Key rotation logistics beyond reading the current epoch_id from + `mesh_epochs`. Old packs stay decryptable with old keys (already a + mesh primitive). +- Public/private mixed-mode shards. A shard is one mode or the other. + +## Status + +Scaffold. No code expected until the prerequisites above land. diff --git a/docs/tickets/ticket-000064-cold-object-operations-toolkit.md b/docs/tickets/ticket-000064-cold-object-operations-toolkit.md new file mode 100644 index 0000000..f5159eb --- /dev/null +++ b/docs/tickets/ticket-000064-cold-object-operations-toolkit.md @@ -0,0 +1,160 @@ +# Ticket #000064 — Cold-object operations toolkit (verify / diff / doctor / repair-fts / gc-plan + expanded audit taxonomy) + +**Status:** open · awaiting go/no-go · scaffold-only +**Opened:** 2026-05-26 +**Scope:** operator-facing observability + repair tools on top of the + #000061 cold-pack tier. Bundles five CLI subcommands and a fine- + grained audit event vocabulary into one ticket so each tool can be + designed against a shared event model rather than five-way drift. +**Audience:** operator (fox, or anyone running an arborist node with + cold-object storage). dav1d secondary — the audit taxonomy is the + thing that goes through the audit chain. +**Hard constraint:** every tool here is *read-mostly* by default. + Destructive operations (gc, repair) print proposed actions and exit + unless `--apply` is given. Cold-object packs are durable artifacts; + ops tools must never enter a state where they'd accidentally orphan + bytes or break replay. + +## Problem + +#000061 ships a producer (`cold pack`) and a consumer (`cold unpack` +via `hydrate_from_metadata_pack`). It doesn't ship operator tools: + +- **No way to spot-check a remote pack** without fully unpacking it. +- **No way to diff local DB chunks against a remote manifest** to see + what would be lost if local disk failed. +- **No health check** that exercises the auth + connectivity + + manifest-age + missing-object signals in one command. +- **No way to rebuild FTS5** from local content if the index gets + corrupted (independent of cold-object pulls). +- **No GC plan** showing which bucket objects are no longer referenced + by any local manifest. + +And the audit chain currently has one event per pull (`cold_pack_pulled`) +with everything stuffed into the body. Fine for the proof property +(audit chain stays intact); too coarse for operators trying to diagnose +"why did 3 of 4 shards complete but the 4th failed?" + +## The five tools (spec only) + +### `arborist cold verify [--sample N | --full]` + +Spot-check object integrity without restoring. Default samples N +random objects from the bucket; `--full` checks every object. For +each: HEAD for size + content-length, GET body (or HEAD-only with +`--head-only`), recompute `hash_leaf(body).hex()`, compare to expected +`leaf_hash`. Audit `cold_verify_sample` with pass/fail counts. + +### `arborist cold diff` + +Compare local DB chunks against a remote `manifest/latest.json` (or +explicit `--metadata-pack `). Output four buckets: + +- Local has, remote has (intersection) +- Local has, remote missing (would-be-lost-if-local-gone) +- Remote has, local missing (could-be-pulled) +- Hash disagreement between local and remote + +JSON output for scripting; `--summary` for human. + +### `arborist cold doctor` + +One-shot health check. Runs in order: + +- Backend connectivity (HEAD a known key) +- Credential validity (signed request returns 200) +- Manifest age (`manifest/latest.json` last-modified vs now) +- Object existence (sample N HEADs, count 404s) +- Tamper sample (N body GETs + hash-check, count mismatches) +- Audit chain integrity (`make chain-check-shards` equivalent) +- Disk space at temp dir + shard dir +- Memory available + +Exit code reflects overall health. JSON output for monitoring scripts. + +### `arborist cold repair-fts [--shard X]` + +Rebuild FTS5 rows from `chunks.content` for every hot chunk. Useful +after an interrupted pull or a corrupted index. Idempotent — +`DELETE FROM chunks_fts; INSERT INTO chunks_fts (rowid, content) SELECT +chunk_id, content FROM chunks WHERE content IS NOT NULL`. Audits one +`cold_repair_fts` event. + +### `arborist cold gc-plan` + +List bucket objects that no manifest references. Read every manifest +in the bucket (latest pointer + every historical metadata pack +manifest), union their referenced `chunk_pack_hashes`. Diff against +the bucket's actual pack listing. Output candidates with sizes. +**Read-only by default** — actual deletion requires `--apply` + an +audit row + the operator typing the orphan count back to confirm. + +## Expanded audit event taxonomy + +Current: `cold_pack_pushed` + `cold_pack_pulled` (with `pack_kind` +distinguishing metadata vs chunks). Load-bearing for proof properties. + +Add (operability, lower information density per event): + +``` +cold_object_put_started per upload, before bytes leave +cold_object_put_verified after HEAD confirms object exists +cold_object_evict_committed after local DB commit drops content +cold_object_put_failed backend rejected the PUT +cold_object_head_failed HEAD after PUT returned an error + +cold_object_get_started per fetch +cold_object_missing bucket returned 404 +cold_object_hash_mismatch hash check failed; never restored +cold_object_decode_error UTF-8 decode failed; never restored + +cold_pack_uploaded pack body confirmed in bucket +cold_pack_unpacked pack body extracted to local +cold_pack_blob_hash_mismatch pack tar bytes don't match expected + +cold_manifest_pointer_updated latest.json was rewritten +cold_manifest_conflict conditional write lost a race + +cold_verify_sample N objects probed, K mismatches +cold_doctor_run health-check verdict +cold_repair_fts_complete FTS rebuild done, N rows +cold_gc_plan_computed N orphans identified +cold_gc_executed orphans deleted (only via --apply) +``` + +Audit body fields stay minimal — endpoint URL + bucket + object key + +result + error_class. Never credentials. Goes through the same +`append_audit` path as everything else. + +## Why scaffold-only + +Each of these tools is its own design problem: + +- `verify` sample size needs a statistical justification. +- `diff` needs a clear answer to "what does 'missing' mean across + shards / manifests / packs?" +- `doctor` needs a defined exit-code vocabulary for monitoring + integration. +- `repair-fts` needs to handle running concurrently with `cold pack` + (it touches the same FTS5 index). +- `gc-plan` needs the manifest-history walk to be efficient (could be + many manifests over time). + +Bundling so the audit-event taxonomy gets designed once instead of +five times. Sequence the implementations in this order: `doctor` first +(fastest, no destructive), `verify` second (sampling, read-only), +`diff` third (operator clarity), `repair-fts` fourth (touches FTS), +`gc-plan` last (destructive even in plan mode if `--apply` ever fires). + +## Out of scope + +- A web UI / dashboard. Output is JSON; integrate elsewhere. +- Cross-arborist-fleet aggregation (e.g., "doctor on all peers"). Each + peer runs its own toolkit. +- Continuous monitoring daemon. `make` target + cron is the operator + recipe. + +## Status + +Scaffold. No code expected until #000061 is fully closed and there's +a clear operator pain that justifies a specific tool first. diff --git a/tests/test_cold_object.py b/tests/test_cold_object.py index e4e60f9..2ab70c6 100644 --- a/tests/test_cold_object.py +++ b/tests/test_cold_object.py @@ -26,8 +26,19 @@ from arborist.evict import ( hydrate_from_metadata_pack, pull_chunk_pack, pull_metadata_pack, - push_pack, + push_pack as _push_pack, ) + + +def push_pack(*args, **kwargs): + """Test wrapper for evict.push_pack that opts into 'unknown' + license_class. FakeSource uses source_type='html' which classifies + as 'unknown', and the Gap-2 license gate refuses unknown-class + pushes to the bucket by default. Tests here exercise pack + semantics, not the license gate — that gets its own test below. + """ + kwargs.setdefault("allow_license_class", "unknown") + return _push_pack(*args, **kwargs) from arborist.ingest import ingest_source from arborist.merkle import hash_leaf from arborist.source import Source @@ -664,6 +675,112 @@ def test_push_pack_is_deterministic_across_runs(tmp_path): conn.close() +def test_gap2_license_gate_refuses_unknown_class_to_public_bucket(tmp_path): + """Gap 2: push_pack refuses to ship a shard whose strictest source + license_class is more restrictive than the operator's + allow_license_class. Default allow=public_redistributable; html-source + shards are 'unknown' by default → push must raise.""" + db = tmp_path / "license.db" + conn = connect(db) + backend = MemoryBackend() + try: + ingest_source(conn, FakeSource([_doc("html://x", LONG)])) + # Default allow_license_class='public_redistributable' refuses. + with pytest.raises(ValueError, match="license_class='unknown'"): + _push_pack(conn, backend) + # Explicit opt-in: allow_license_class='unknown' passes. + result = _push_pack(conn, backend, allow_license_class="unknown") + assert result["status"] == "pushed" + finally: + conn.close() + + +def test_gap2_license_class_in_metadata_manifest(tmp_path): + """Gap 2: the metadata pack manifest carries _license_class so + consumers + auditors can see the producer's classification.""" + db = tmp_path / "lc-manifest.db" + conn = connect(db) + backend = MemoryBackend() + try: + ingest_source(conn, FakeSource([_doc("html://x", LONG)])) + result = push_pack(conn, backend) # uses test wrapper, allow='unknown' + meta_manifest = backend.get_pack_manifest( + result["metadata_pack_hash"], kind="metadata" + ) + parsed = parse_manifest(meta_manifest) + assert parsed.license_class == "unknown" + finally: + conn.close() + + +def test_gap1_latest_pointer_resolves_metadata_pack_per_snapshot(tmp_path): + """Gap 1: after push, manifest/latest.json points to the current + metadata pack for each snapshot_root. Fresh peer can resolve the + metadata pack hash without enumerating every object.""" + db = tmp_path / "latest.db" + conn = connect(db) + backend = MemoryBackend() + try: + ingest_source(conn, FakeSource([_doc("html://x", LONG)])) + result = push_pack(conn, backend) + snapshot_root = result["snapshot_root"] + metadata_hash = result["metadata_pack_hash"] + + # Backend has the latest pointer. + pointer = backend.get_latest_pointer() + assert snapshot_root in pointer + entry = pointer[snapshot_root] + assert entry["metadata_pack_hash"] == metadata_hash + assert entry["snapshot_doc_count"] == result["snapshot_doc_count"] + assert entry["license_class"] == "unknown" + finally: + conn.close() + + +def test_gap3_cold_pending_clears_on_successful_upload(tmp_path): + """Gap 3: cold_pending rows track in-flight uploads. On success, + they're cleared. After a clean push, cold_pending is empty.""" + db = tmp_path / "pending.db" + conn = connect(db) + backend = MemoryBackend() + try: + ingest_source(conn, FakeSource([_doc("html://x", LONG)])) + push_pack(conn, backend) + # Successful push leaves no orphan pending rows. + pending = conn.execute("SELECT COUNT(*) FROM cold_pending").fetchone()[0] + assert pending == 0 + finally: + conn.close() + + +def test_gap3_cold_pending_records_inflight_upload(tmp_path): + """Gap 3: when the backend put_pack_file fails mid-flight, the + cold_pending row stays so a recovery tool can find the orphan + tempfile and clean it up.""" + db = tmp_path / "pending-fail.db" + conn = connect(db) + try: + ingest_source(conn, FakeSource([_doc("html://x", LONG)])) + + class FailingBackend(MemoryBackend): + def put_file(self, key, path, *, content_type="application/octet-stream"): + raise RuntimeError("simulated upload failure") + + backend = FailingBackend() + with pytest.raises(RuntimeError, match="simulated upload"): + push_pack(conn, backend) + # The row should exist tracking what was almost uploaded. + pending_rows = conn.execute( + "SELECT kind, pack_hash, object_key FROM cold_pending" + ).fetchall() + assert len(pending_rows) >= 1 + # The recorded row identifies enough to either resume or clean up. + assert pending_rows[0]["pack_hash"] + assert pending_rows[0]["object_key"].startswith("packs/") + finally: + conn.close() + + def test_default_max_pack_bytes_is_dvdr_safe_fit(): """Default cap is 4.4 GB — DVD-R safe-fit, with ~6.5% buffer below the 4.7 GB media spec to absorb ISO9660 overhead, media manufacturing