russell@unturf.com
7d2bf4e7fa
Each ingest shard owns its own SQLite file. N processes write in
true parallel (no WAL writer-lock contention). Cross-shard reads
attach all shards as UNION ALL views over the standard tables, so
existing query code (`SELECT * FROM documents`) works unchanged.
CLI:
aborist ingest --shards-dir DIR --shard 0/4 # write to DIR/000.db
aborist --shards-dir DIR stats # read across all shards
aborist --shards-dir DIR analyze # ditto
aborist --shards-dir DIR verify -n 20 # Merkle round-trips
# on a sample drawn
# from all shards
Makefile:
SHARDS_DIR=~/.aborist/shards
ingest-cur-attached SHARDS=4 ingest-old-attached SHARDS=4
stats-shards cross-shard stats
Implementation lives in store.connect_query():
- opens a fresh in-memory connection
- ATTACH DATABASE per shard file under aliases sh000, sh001, ...
- CREATE TEMP VIEW <table> AS UNION ALL SELECT * FROM <alias>.<table>
for the eight shardable tables (documents, chunks, merkle_nodes,
edges, derivations, providence_cache, audit_events, falsifications)
- exposes a _shards temp table for tools that want the shard list
Benchmark (20k cur, 4 cores):
Phase 1a serial 55.3 s
Phase 1b shared-WAL 43.1 s
Phase 2 attached-forever 40.1 s (no WAL contention)
The 7% gap vs Phase 1b is bz2-decompression duplication: each shard
decodes the full dump independently. Architectural wins are bigger
than that number — write isolation, per-shard crash recovery,
audit chain stays per-shard checkable, shards can migrate to other
disks, new shards are pure append (never rewrite existing files).
Merge cost remains: zero. `ls $SHARDS_DIR/*.db` is the registry.
135 lines
5.1 KiB
Makefile
135 lines
5.1 KiB
Makefile
# aborist — Makefile entry points
|
|
# Every workflow lives behind a `make` target. Bare python commands are not
|
|
# the user interface.
|
|
|
|
# Tools and config
|
|
PYTHON ?= python3
|
|
VENV ?= .venv
|
|
PIP := $(VENV)/bin/pip
|
|
PY := $(VENV)/bin/python
|
|
ABORIST := $(VENV)/bin/aborist
|
|
|
|
# Data + DB
|
|
DATA_DIR ?= data
|
|
WP_BASE_URL ?= https://dumps.wikimedia.org/archive/2003/2003-05-16/en
|
|
WP_CUR := $(DATA_DIR)/20030516_cur_tablesql.bz2
|
|
WP_OLD_1 := $(DATA_DIR)/old_tablesqlbz2.1
|
|
WP_OLD_2 := $(DATA_DIR)/old_tablesqlbz2.2
|
|
WP_OLD := $(DATA_DIR)/20030516_old_tablesql.bz2
|
|
# Back-compat alias (older callers used WP_DUMP for the cur snapshot).
|
|
WP_DUMP := $(WP_CUR)
|
|
DB ?= $(HOME)/.aborist/aborist.db
|
|
|
|
# Smoke-test caps so make all stays fast
|
|
INGEST_LIMIT ?= 500
|
|
VERIFY_N ?= 10
|
|
SEARCH_Q ?= computer
|
|
|
|
.PHONY: all bootstrap fetch fetch-cur fetch-old ingest ingest-cur ingest-old \
|
|
verify search stats test clean clean-db clean-data help
|
|
|
|
all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats
|
|
|
|
help: ## show this help
|
|
@awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " %-16s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
|
|
|
|
$(VENV)/bin/activate: pyproject.toml
|
|
$(PYTHON) -m venv $(VENV)
|
|
$(PIP) install --upgrade pip wheel
|
|
$(PIP) install -e '.[dev]'
|
|
@touch $(VENV)/bin/activate
|
|
|
|
bootstrap: $(VENV)/bin/activate ## create venv and install editable package
|
|
|
|
$(DATA_DIR):
|
|
mkdir -p $(DATA_DIR)
|
|
|
|
$(WP_CUR): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
|
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
|
|
|
$(WP_OLD_1): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
|
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
|
|
|
$(WP_OLD_2): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
|
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
|
|
|
# The two old_tablesqlbz2.{1,2} parts are split halves of a single bzip2
|
|
# stream (.1 is exactly 640 MiB). Concatenate to get a working bz2 file.
|
|
$(WP_OLD): $(WP_OLD_1) $(WP_OLD_2)
|
|
@echo ">> concatenating old dump parts"
|
|
cat $(WP_OLD_1) $(WP_OLD_2) > $@
|
|
|
|
fetch-cur: $(WP_CUR) ## download cur table dump (~82 MB)
|
|
|
|
fetch-old: $(WP_OLD) ## download old (revision history) parts and concatenate (~893 MB)
|
|
|
|
fetch: fetch-cur fetch-old ## download all 3 files (cur + old.1 + old.2 + concat)
|
|
|
|
ingest-cur: bootstrap fetch-cur ## ingest INGEST_LIMIT cur articles
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) --limit $(INGEST_LIMIT)
|
|
|
|
ingest-old: bootstrap fetch-old ## ingest INGEST_LIMIT old (history) revisions
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) --limit $(INGEST_LIMIT)
|
|
|
|
# Phase 1b: parallel shards into ONE shared SQLite. Workers parallelize
|
|
# parser CPU; writes serialize at the WAL writer-lock. ~1.3x wall on 4 cores.
|
|
SHARDS ?= 4
|
|
ingest-cur-parallel: bootstrap fetch-cur ## ingest cur with SHARDS=N processes -> one shared DB
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) \
|
|
--shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
ingest-old-parallel: bootstrap fetch-old ## ingest old history with SHARDS=N processes -> one shared DB
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) \
|
|
--shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
# Phase 2: attach-forever sharding. Each shard owns its own SQLite file —
|
|
# no WAL contention. Reads via `aborist --shards-dir <dir> <cmd>` attach
|
|
# all shards as UNION views. "Merge cost" = 0.
|
|
SHARDS_DIR ?= $(HOME)/.aborist/shards
|
|
ingest-cur-attached: bootstrap fetch-cur ## sharded ingest, no WAL contention (Phase 2)
|
|
@mkdir -p $(SHARDS_DIR)
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) ingest --source wikipedia_cur --path $(WP_CUR) \
|
|
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
ingest-old-attached: bootstrap fetch-old ## sharded ingest of old history (Phase 2)
|
|
@mkdir -p $(SHARDS_DIR)
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) ingest --source wikipedia_old --path $(WP_OLD) \
|
|
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
stats-shards: bootstrap ## cross-shard stats via UNION views over $(SHARDS_DIR)
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) stats
|
|
|
|
ingest: ingest-cur ## default ingest = cur (use ingest-old or *-parallel for full)
|
|
|
|
verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents
|
|
$(ABORIST) --db $(DB) verify -n $(VERIFY_N)
|
|
|
|
search: bootstrap ## keyword search; override SEARCH_Q (or pass Q=...)
|
|
$(ABORIST) --db $(DB) search '$(if $(Q),$(Q),$(SEARCH_Q))'
|
|
|
|
stats: bootstrap ## counts: documents, chunks, edges, audit chain
|
|
$(ABORIST) --db $(DB) stats
|
|
|
|
test: bootstrap ## run pytest suite
|
|
$(VENV)/bin/pytest -q
|
|
|
|
clean: ## remove venv + caches (keeps fetched data and db)
|
|
rm -rf $(VENV) .pytest_cache **/__pycache__ aborist.egg-info
|
|
find . -type d -name __pycache__ -prune -exec rm -rf {} +
|
|
|
|
clean-db: ## drop the aborist db (keeps fetched data and venv)
|
|
rm -f $(DB) $(DB)-journal $(DB)-wal $(DB)-shm
|
|
|
|
clean-data: ## remove fetched dumps
|
|
rm -rf $(DATA_DIR)
|