arborist/scripts/upload_fts_sidecars.py

"""Upload slim FTS5 sidecars to the bucket + publish manifest-fts.json.

Reads slim FTS5 sidecars from ~/.arborist/sidecar-fts/00*.idx.db, uploads
them to s3://${ARBORIST_COLD_BUCKET}/clones/sidecars-fts/<n>.idx.db, and
publishes a new manifest at clones/manifest-fts.json that references both:
  - the new fts_sidecar_url (slim FTS5)
  - the existing shard URL from manifest-sidecar.json (chunk-content fallback)

Idempotent: skips uploads whose remote size matches local. Existing
manifest-sidecar.json is NOT modified.

Credentials via boto3 standard discovery (env vars / ~/.aws/credentials).
Operation Voyeur: credentials never appear in argv, logs, or this file.

Usage:
    .venv/bin/python3 scripts/upload_fts_sidecars.py [--dry-run]
"""
from __future__ import annotations

import argparse
import json
import os
import sys
import urllib.request
from pathlib import Path

SIDECAR_DIR = Path.home() / ".arborist" / "sidecar-fts"
PREFIX = "clones/sidecars-fts"
NEW_MANIFEST_KEY = "clones/manifest-fts.json"
EXISTING_MANIFEST_URL = (
    "https://nyc3.digitaloceanspaces.com/arborist/clones/manifest-sidecar.json"
)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--dry-run", action="store_true",
                    help="print what would happen; upload nothing")
    args = ap.parse_args()

    bucket = os.environ.get("ARBORIST_COLD_BUCKET")
    endpoint = os.environ.get("ARBORIST_COLD_ENDPOINT_URL")
    if not bucket or not endpoint:
        print("ERROR: set ARBORIST_COLD_BUCKET + ARBORIST_COLD_ENDPOINT_URL",
              file=sys.stderr)
        return 2

    sidecars = sorted(SIDECAR_DIR.glob("[0-9][0-9][0-9].idx.db"))
    if not sidecars:
        print(f"ERROR: no sidecars at {SIDECAR_DIR}/[0-9][0-9][0-9].idx.db",
              file=sys.stderr)
        return 2

    print(f"bucket:   s3://{bucket}/", file=sys.stderr)
    print(f"endpoint: {endpoint}", file=sys.stderr)
    print(f"prefix:   {PREFIX}/", file=sys.stderr)
    print(f"sidecars: {len(sidecars)} files ({sum(p.stat().st_size for p in sidecars)/1e9:.2f} GB)",
          file=sys.stderr)
    print(file=sys.stderr)

    # Load the existing manifest's shard URLs so the new manifest reuses
    # them for the chunk-content fallback. We don't touch the existing
    # manifest itself — just borrow the URLs.
    with urllib.request.urlopen(EXISTING_MANIFEST_URL, timeout=30) as resp:
        existing = json.loads(resp.read().decode())
    # Map by trailing 3-digit shard index parsed from label
    # ("genesis-shard-000" → "000", "genesis-shard-002-64k" → "002").
    # Drops non-genesis shards (virtback, etc.) so the basename collision
    # virtback/000.db vs full-bench/000.db doesn't matter.
    import re as _re
    _GENESIS_IDX = _re.compile(r"genesis-shard-(\d{3})")
    by_idx = {}
    for sh in existing.get("shards", []):
        m = _GENESIS_IDX.search(sh.get("label", ""))
        if m:
            by_idx[m.group(1)] = sh

    try:
        import boto3
        from boto3.s3.transfer import TransferConfig
    except ImportError:
        print("ERROR: boto3 not installed. run: make bootstrap-object-store",
              file=sys.stderr)
        return 2

    s3 = boto3.client("s3", endpoint_url=endpoint, region_name=os.environ.get(
        "ARBORIST_COLD_REGION", "us-east-1"))
    # 64 MB parts; parallel transfer.
    cfg = TransferConfig(
        multipart_threshold=64 * 1024 * 1024,
        multipart_chunksize=64 * 1024 * 1024,
        max_concurrency=4,
    )

    new_shards = []
    for sp in sidecars:
        n = sp.stem.split(".")[0]   # 000.idx → 000
        key = f"{PREFIX}/{n}.idx.db"
        local_size = sp.stat().st_size

        # Idempotency: skip if remote size matches.
        try:
            head = s3.head_object(Bucket=bucket, Key=key)
            remote_size = head["ContentLength"]
            if remote_size == local_size:
                print(f"  skip {key}  (already {local_size/1e9:.2f} GB)",
                      file=sys.stderr)
            else:
                print(f"  remote size mismatch on {key} "
                      f"(local {local_size}, remote {remote_size}) — re-uploading",
                      file=sys.stderr)
                if not args.dry_run:
                    s3.upload_file(str(sp), bucket, key, Config=cfg)
                else:
                    print(f"  [dry-run] would re-upload {key}",
                          file=sys.stderr)
        except s3.exceptions.ClientError as e:
            code = e.response.get("Error", {}).get("Code")
            if code in ("404", "NoSuchKey", "NotFound"):
                if args.dry_run:
                    print(f"  [dry-run] would upload {key} ({local_size/1e9:.2f} GB)",
                          file=sys.stderr)
                else:
                    print(f"  upload {key} ({local_size/1e9:.2f} GB) ...",
                          file=sys.stderr)
                    s3.upload_file(str(sp), bucket, key, Config=cfg)
                    s3.put_object_acl(Bucket=bucket, Key=key, ACL="public-read")
                    print(f"  done", file=sys.stderr)
            else:
                raise

        # Ensure ACL is public-read (idempotent for ACL-aware backends).
        if not args.dry_run:
            try:
                s3.put_object_acl(Bucket=bucket, Key=key, ACL="public-read")
            except Exception:
                pass  # endpoint may not honor ACLs

        # Build new manifest entry — reuse existing shard url + blob_base.
        ex = by_idx.get(n)
        if ex is None:
            print(f"  WARNING: no existing genesis-shard-{n} in live manifest; "
                  f"the new manifest will lack a content-fallback URL "
                  f"(slim FTS5 retrieval still works; blob fetch falls back "
                  f"to bucket blobs/<hash> which 403s today)",
                  file=sys.stderr)
            new_shard = {
                "url": None,
                "shard_idx": int(n),
                "label": f"genesis-shard-{n}",
                "fts_sidecar_url": f"{endpoint}/{bucket}/{key}",
            }
        else:
            new_shard = dict(ex)
            new_shard["fts_sidecar_url"] = f"{endpoint}/{bucket}/{key}"
            new_shard.pop("sidecar_url", None)  # this manifest is FTS5-only
        new_shards.append(new_shard)

    new_manifest = {
        "shards": new_shards,
        "blob_base": existing.get("blob_base"),
        "snapshot_root": existing.get("snapshot_root"),
        "snapshot_ts": existing.get("snapshot_ts"),
        "format": "fts5-slim-v1",
    }
    manifest_bytes = json.dumps(new_manifest, indent=2).encode()
    print(file=sys.stderr)
    print(f"new manifest: {NEW_MANIFEST_KEY} ({len(manifest_bytes)} bytes)",
          file=sys.stderr)
    print(f"  shards: {len(new_shards)}", file=sys.stderr)
    for sh in new_shards:
        print(f"    fts_sidecar={sh['fts_sidecar_url'].rsplit('/',1)[-1]:14s}  "
              f"shard_url={sh.get('url','(none)').rsplit('/',1)[-1]}",
              file=sys.stderr)

    if args.dry_run:
        print(file=sys.stderr)
        print("[dry-run] manifest content:", file=sys.stderr)
        print(manifest_bytes.decode(), file=sys.stderr)
        return 0

    s3.put_object(
        Bucket=bucket, Key=NEW_MANIFEST_KEY,
        Body=manifest_bytes,
        ContentType="application/json",
        ACL="public-read",
    )
    print(f"published: {endpoint}/{bucket}/{NEW_MANIFEST_KEY}",
          file=sys.stderr)
    return 0


if __name__ == "__main__":
    sys.exit(main())