russell@unturf.com
649aeec79a
aborist/progress.py — stdlib-only rate-limited stderr reporter.
Periodic lines (default every 2s) showing:
elapsed | seen | inserted | docs/s now | docs/s avg | percent | ETA
Wired into ingest_source via a `progress` parameter; CLI default-on
with --quiet to suppress and --total-estimate N to enable percent/ETA.
Demo on a 5k-doc ingest with --progress-interval 0.8:
[ 1s] | 200 seen | 200 new | 193 now | ( 193 avg) docs/s | 4.0% | ETA 24s
[ 7s] | 3,200 seen | 3,200 new | 458 now | ( 424 avg) docs/s | 64.0% | ETA 3s
[ 12s] | 5,001 seen | 5,000 new | 392 now | ( 413 avg) docs/s | 100.0% | ETA 0s
bench/run.sh + `make bench` — reproducible 5000-doc workload through
three configs:
serial single process, single SQLite
parallel-shared N shards, one shared SQLite (WAL serialized)
attached N shards, per-shard SQLite (true parallel writes)
Output is a one-shot table plus CSV at /tmp/aborist-bench/results.csv
so the ratchet is visible as we keep optimizing. Override via
BENCH_DOCS=N and SHARDS=N.
Latest baseline (this commit, on this machine):
config wall_s docs docs/s
serial 11.74 5000 425.7
parallel-shared 10.39 5000 481.1
attached 7.99 5000 625.5
53 tests still passing.
160 lines
6.3 KiB
Makefile
160 lines
6.3 KiB
Makefile
# aborist — Makefile entry points
|
|
# Every workflow lives behind a `make` target. Bare python commands are not
|
|
# the user interface.
|
|
|
|
# Tools and config
|
|
PYTHON ?= python3
|
|
VENV ?= .venv
|
|
PIP := $(VENV)/bin/pip
|
|
PY := $(VENV)/bin/python
|
|
ABORIST := $(VENV)/bin/aborist
|
|
|
|
# Data + DB
|
|
DATA_DIR ?= data
|
|
WP_BASE_URL ?= https://dumps.wikimedia.org/archive/2003/2003-05-16/en
|
|
WP_CUR := $(DATA_DIR)/20030516_cur_tablesql.bz2
|
|
WP_OLD_1 := $(DATA_DIR)/old_tablesqlbz2.1
|
|
WP_OLD_2 := $(DATA_DIR)/old_tablesqlbz2.2
|
|
WP_OLD := $(DATA_DIR)/20030516_old_tablesql.bz2
|
|
# Back-compat alias (older callers used WP_DUMP for the cur snapshot).
|
|
WP_DUMP := $(WP_CUR)
|
|
DB ?= $(HOME)/.aborist/aborist.db
|
|
|
|
# Smoke-test caps so make all stays fast
|
|
INGEST_LIMIT ?= 500
|
|
VERIFY_N ?= 10
|
|
SEARCH_Q ?= computer
|
|
|
|
.PHONY: all bootstrap fetch fetch-cur fetch-old ingest ingest-cur ingest-old \
|
|
verify search stats test clean clean-db clean-data help
|
|
|
|
all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats
|
|
|
|
help: ## show this help
|
|
@awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " %-16s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
|
|
|
|
$(VENV)/bin/activate: pyproject.toml
|
|
$(PYTHON) -m venv $(VENV)
|
|
$(PIP) install --upgrade pip wheel
|
|
$(PIP) install -e '.[dev]'
|
|
@touch $(VENV)/bin/activate
|
|
|
|
bootstrap: $(VENV)/bin/activate ## create venv and install editable package
|
|
|
|
$(DATA_DIR):
|
|
mkdir -p $(DATA_DIR)
|
|
|
|
$(WP_CUR): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
|
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
|
|
|
$(WP_OLD_1): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
|
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
|
|
|
$(WP_OLD_2): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
|
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
|
|
|
# The two old_tablesqlbz2.{1,2} parts are split halves of a single bzip2
|
|
# stream (.1 is exactly 640 MiB). Concatenate to get a working bz2 file.
|
|
$(WP_OLD): $(WP_OLD_1) $(WP_OLD_2)
|
|
@echo ">> concatenating old dump parts"
|
|
cat $(WP_OLD_1) $(WP_OLD_2) > $@
|
|
|
|
fetch-cur: $(WP_CUR) ## download cur table dump (~82 MB)
|
|
|
|
fetch-old: $(WP_OLD) ## download old (revision history) parts and concatenate (~893 MB)
|
|
|
|
fetch: fetch-cur fetch-old ## download all 3 files (cur + old.1 + old.2 + concat)
|
|
|
|
ingest-cur: bootstrap fetch-cur ## ingest INGEST_LIMIT cur articles
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) --limit $(INGEST_LIMIT)
|
|
|
|
ingest-old: bootstrap fetch-old ## ingest INGEST_LIMIT old (history) revisions
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) --limit $(INGEST_LIMIT)
|
|
|
|
# Phase 1b: parallel shards into ONE shared SQLite. Workers parallelize
|
|
# parser CPU; writes serialize at the WAL writer-lock. ~1.3x wall on 4 cores.
|
|
SHARDS ?= 4
|
|
ingest-cur-parallel: bootstrap fetch-cur ## ingest cur with SHARDS=N processes -> one shared DB
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) \
|
|
--shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
ingest-old-parallel: bootstrap fetch-old ## ingest old history with SHARDS=N processes -> one shared DB
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) \
|
|
--shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
# Phase 2: attach-forever sharding. Each shard owns its own SQLite file —
|
|
# no WAL contention. Reads via `aborist --shards-dir <dir> <cmd>` attach
|
|
# all shards as UNION views. "Merge cost" = 0.
|
|
SHARDS_DIR ?= $(HOME)/.aborist/shards
|
|
ingest-cur-attached: bootstrap fetch-cur ## sharded ingest, no WAL contention (Phase 2)
|
|
@mkdir -p $(SHARDS_DIR)
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) ingest --source wikipedia_cur --path $(WP_CUR) \
|
|
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
ingest-old-attached: bootstrap fetch-old ## sharded ingest of old history (Phase 2)
|
|
@mkdir -p $(SHARDS_DIR)
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) ingest --source wikipedia_old --path $(WP_OLD) \
|
|
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
stats-shards: bootstrap ## cross-shard stats via UNION views over $(SHARDS_DIR)
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) stats
|
|
|
|
verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N)
|
|
|
|
analyze-shards: bootstrap ## cross-shard compression spectrum + audit integrity
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) analyze
|
|
|
|
# Sequential per-shard distill (one process iterates all shards).
|
|
distill-shards: bootstrap ## distill every shard in $(SHARDS_DIR), surface -> depth=1 cores
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) distill --process first-sentence-v1 --kind surface
|
|
|
|
# Parallel per-shard distill: one process per shard. No DB contention
|
|
# because each shard is its own file.
|
|
distill-shards-parallel: bootstrap ## one distill process per shard (parallel)
|
|
@for shard in $(SHARDS_DIR)/*.db; do \
|
|
$(ABORIST) --db $$shard distill --process first-sentence-v1 --kind surface & \
|
|
done; wait
|
|
|
|
ingest: ingest-cur ## default ingest = cur (use ingest-old or *-parallel for full)
|
|
|
|
verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents
|
|
$(ABORIST) --db $(DB) verify -n $(VERIFY_N)
|
|
|
|
search: bootstrap ## keyword search; override SEARCH_Q (or pass Q=...)
|
|
$(ABORIST) --db $(DB) search '$(if $(Q),$(Q),$(SEARCH_Q))'
|
|
|
|
stats: bootstrap ## counts: documents, chunks, edges, audit chain
|
|
$(ABORIST) --db $(DB) stats
|
|
|
|
test: bootstrap ## run pytest suite
|
|
$(VENV)/bin/pytest -q
|
|
|
|
# Reproducible micro-benchmark over a fixed slice of cur. Lets you compare
|
|
# ETL throughput across configs and catches regressions on optimization
|
|
# work. Override BENCH_DOCS=N (default 5000).
|
|
BENCH_DOCS ?= 5000
|
|
BENCH_DIR := /tmp/aborist-bench
|
|
bench: bootstrap fetch-cur ## benchmark serial vs parallel-shared vs attached at $(BENCH_DOCS) docs
|
|
@bash bench/run.sh $(BENCH_DOCS)
|
|
|
|
clean: ## remove venv + caches (keeps fetched data and db)
|
|
rm -rf $(VENV) .pytest_cache **/__pycache__ aborist.egg-info
|
|
find . -type d -name __pycache__ -prune -exec rm -rf {} +
|
|
|
|
clean-db: ## drop the aborist db (keeps fetched data and venv)
|
|
rm -f $(DB) $(DB)-journal $(DB)-wal $(DB)-shm
|
|
|
|
clean-data: ## remove fetched dumps
|
|
rm -rf $(DATA_DIR)
|