engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

make cloud-*: default SHARD_URL to the virtback Spaces shard

Browse source 
Drops the "SHARD_URL required" friction. Default points at the
russell.ballestrini.net web-crawl shard now live on DO Spaces
(public-read, ~2.5 MB) — the smallest end-to-end real-data target
for proving bucket-direct queries work against a real S3-compatible
bucket without any local arborist data.

Override SHARD_URL on the command line to point at a different bucket
shard:

    make cloud-search Q='"virt-back"'                # uses default
    make cloud-snapshot-root SHARD_URL=https://other.bucket/.../000.db

Recipe also switched to single-quoted $(Q) so FTS5 phrase quotes
(`'"phrase here"'`) survive shell parsing.
This commit is contained in:
russell@unturf.com 2026-05-30 09:16:24 -04:00
parent b438fd6559
commit 96557535b2
No known key found for this signature in database
1 changed files with 18 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

27
Makefile
View file

@ -1537,30 +1537,39 @@ wallet-ask: bootstrap ## query the wallet server with your own question [Q="..."
# ---------------------------------------------------------------------------
# Cloud (bucket-direct) — no server, no local DB. apsw + HTTP-range VFS.
# ---------------------------------------------------------------------------
# `make cloud-search Q="..." SHARD_URL=https://bucket.../clones/.../000.db`
# `make cloud-search Q="..."` — defaults SHARD_URL to the russell.ballestrini.net
# crawl shard on DO Spaces; override with `SHARD_URL=https://other.bucket/path/000.db`
# Open the bucket-resident .db in place via SQLite HTTP-range VFS, run FTS5,
# print hits. No arborist server in the picture. Requires `arborist[bucket]`
# (apsw). The `cloud-demo` target proves the whole stack on a tiny corpus
# served from an in-process HTTP server with Range support.
# Default bucket = the russell.ballestrini.net web-crawl shard on DO Spaces
# (public-read; ~2.5 MB; the "who wrote virt-back?" demo target). Override
# SHARD_URL on the command line or in your environment to point cloud-*
# commands at any S3-compatible bucket that returns 206 Partial Content
# on Range requests.
SHARD_URL ?= https://nyc3.digitaloceanspaces.com/arborist/clones/virtback/000.db
BLOB_BASE ?=
bootstrap-bucket: bootstrap ## install apsw (bucket-direct extra)
$(PIP) install 'apsw>=3.45'
cloud-search: bootstrap ## FTS5 search a bucket-resident shard [Q="..." SHARD_URL=https://...]
@test -n "$(Q)" || { echo 'usage: make cloud-search Q="..." SHARD_URL=https://.../000.db [BLOB_BASE=https://.../blobs] [LIMIT=N] [CACHE_MB=N]'; exit 2; }
@test -n "$(SHARD_URL)" || { echo 'SHARD_URL required'; exit 2; }
@$(ARBORIST) cloud search "$(Q)" --shard-url "$(SHARD_URL)" \
cloud-search: bootstrap ## FTS5 search the bucket-resident shard [Q="..." SHARD_URL=https://... LIMIT=N CACHE_MB=N]
@test -n "$(Q)" || { echo 'usage: make cloud-search Q="your query" [SHARD_URL=...] [BLOB_BASE=...] [LIMIT=N] [CACHE_MB=N]'; exit 2; }
@echo "# shard: $(SHARD_URL)" >&2
@$(ARBORIST) cloud search '$(Q)' --shard-url "$(SHARD_URL)" \
$(if $(BLOB_BASE),--blob-base "$(BLOB_BASE)") \
--limit $(or $(LIMIT),8) \
--cache-mb $(or $(CACHE_MB),32)
cloud-snapshot-root: bootstrap ## compute snapshot_root of a bucket-resident shard [SHARD_URL=https://...]
@test -n "$(SHARD_URL)" || { echo 'SHARD_URL required'; exit 2; }
cloud-snapshot-root: bootstrap ## compute snapshot_root of the bucket-resident shard [SHARD_URL=https://...]
@echo "# shard: $(SHARD_URL)" >&2
@$(ARBORIST) cloud snapshot-root --shard-url "$(SHARD_URL)" --cache-mb $(or $(CACHE_MB),32)
cloud-fetch-chunk: bootstrap ## fetch + hash-verify one chunk from a bucket [LEAF_HASH=hex BLOB_BASE=https://...]
@test -n "$(LEAF_HASH)" || { echo 'LEAF_HASH required'; exit 2; }
@test -n "$(BLOB_BASE)" || { echo 'BLOB_BASE required'; exit 2; }
@test -n "$(LEAF_HASH)" || { echo 'usage: make cloud-fetch-chunk LEAF_HASH=hex BLOB_BASE=https://.../blobs'; exit 2; }
@test -n "$(BLOB_BASE)" || { echo 'BLOB_BASE required (per-chunk blobs/<hash> base URL)'; exit 2; }
@$(ARBORIST) cloud fetch-chunk "$(LEAF_HASH)" --blob-base "$(BLOB_BASE)"
CLOUD_DEMO_PORT ?= 18785