# aborist — Makefile entry points
# Every workflow lives behind a `make` target. Bare python commands are not
# the user interface.

# Tools and config
PYTHON       ?= python3
VENV         ?= .venv
PIP          := $(VENV)/bin/pip
PY           := $(VENV)/bin/python
ABORIST      := $(VENV)/bin/aborist

# Data + DB
DATA_DIR     ?= data
WP_BASE_URL  ?= https://dumps.wikimedia.org/archive/2003/2003-05-16/en
WP_CUR       := $(DATA_DIR)/20030516_cur_tablesql.bz2
WP_OLD_1     := $(DATA_DIR)/old_tablesqlbz2.1
WP_OLD_2     := $(DATA_DIR)/old_tablesqlbz2.2
WP_OLD       := $(DATA_DIR)/20030516_old_tablesql.bz2
# Back-compat alias (older callers used WP_DUMP for the cur snapshot).
WP_DUMP      := $(WP_CUR)
DB           ?= $(HOME)/.aborist/aborist.db

# Smoke-test caps so make all stays fast
INGEST_LIMIT ?= 500
VERIFY_N     ?= 10
SEARCH_Q     ?= computer

.PHONY: all bootstrap fetch fetch-cur fetch-old ingest ingest-cur ingest-old \
        verify search stats test clean clean-db clean-data help

all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats

help: ## show this help
	@awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf "  %-16s %s\n", $$1, $$2}' $(MAKEFILE_LIST)

$(VENV)/bin/activate: pyproject.toml
	$(PYTHON) -m venv $(VENV)
	$(PIP) install --upgrade pip wheel
	$(PIP) install -e '.[dev]'
	@touch $(VENV)/bin/activate

bootstrap: $(VENV)/bin/activate ## create venv and install editable package

$(DATA_DIR):
	mkdir -p $(DATA_DIR)

$(WP_CUR): | $(DATA_DIR)
	@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
	curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"

$(WP_OLD_1): | $(DATA_DIR)
	@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
	curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"

$(WP_OLD_2): | $(DATA_DIR)
	@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
	curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"

# The two old_tablesqlbz2.{1,2} parts are split halves of a single bzip2
# stream (.1 is exactly 640 MiB). Concatenate to get a working bz2 file.
$(WP_OLD): $(WP_OLD_1) $(WP_OLD_2)
	@echo ">> concatenating old dump parts"
	cat $(WP_OLD_1) $(WP_OLD_2) > $@

fetch-cur: $(WP_CUR) ## download cur table dump (~82 MB)

fetch-old: $(WP_OLD) ## download old (revision history) parts and concatenate (~893 MB)

fetch: fetch-cur fetch-old ## download all 3 files (cur + old.1 + old.2 + concat)

ingest-cur: bootstrap fetch-cur ## ingest INGEST_LIMIT cur articles
	$(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) --limit $(INGEST_LIMIT)

ingest-old: bootstrap fetch-old ## ingest INGEST_LIMIT old (history) revisions
	$(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) --limit $(INGEST_LIMIT)

# Phase 1b: parallel shards into ONE shared SQLite. Workers parallelize
# parser CPU; writes serialize at the WAL writer-lock. ~1.3x wall on 4 cores.
SHARDS ?= 4
ingest-cur-parallel: bootstrap fetch-cur ## ingest cur with SHARDS=N processes -> one shared DB
	@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
	  $(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) \
	    --shard $$i/$(SHARDS) & \
	done; wait

ingest-old-parallel: bootstrap fetch-old ## ingest old history with SHARDS=N processes -> one shared DB
	@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
	  $(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) \
	    --shard $$i/$(SHARDS) & \
	done; wait

# Phase 2: attach-forever sharding. Each shard owns its own SQLite file —
# no WAL contention. Reads via `aborist --shards-dir <dir> <cmd>` attach
# all shards as UNION views. "Merge cost" = 0.
SHARDS_DIR ?= $(HOME)/.aborist/shards
ingest-cur-attached: bootstrap fetch-cur ## sharded ingest, no WAL contention (Phase 2)
	@mkdir -p $(SHARDS_DIR)
	@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
	  $(ABORIST) ingest --source wikipedia_cur --path $(WP_CUR) \
	    --shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
	done; wait

ingest-old-attached: bootstrap fetch-old ## sharded ingest of old history (Phase 2)
	@mkdir -p $(SHARDS_DIR)
	@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
	  $(ABORIST) ingest --source wikipedia_old --path $(WP_OLD) \
	    --shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
	done; wait

stats-shards: bootstrap ## cross-shard stats via UNION views over $(SHARDS_DIR)
	$(ABORIST) --shards-dir $(SHARDS_DIR) stats

verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample
	$(ABORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N)

analyze-shards: bootstrap ## cross-shard compression spectrum + audit integrity
	$(ABORIST) --shards-dir $(SHARDS_DIR) analyze

# Sequential per-shard distill (one process iterates all shards).
distill-shards: bootstrap ## distill every shard in $(SHARDS_DIR), surface -> depth=1 cores
	$(ABORIST) --shards-dir $(SHARDS_DIR) distill --process first-sentence-v1 --kind surface

# Parallel per-shard distill: one process per shard. No DB contention
# because each shard is its own file.
distill-shards-parallel: bootstrap ## one distill process per shard (parallel)
	@for shard in $(SHARDS_DIR)/*.db; do \
	  $(ABORIST) --db $$shard distill --process first-sentence-v1 --kind surface & \
	done; wait

ingest: ingest-cur ## default ingest = cur (use ingest-old or *-parallel for full)

verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents
	$(ABORIST) --db $(DB) verify -n $(VERIFY_N)

search: bootstrap ## keyword search; override SEARCH_Q (or pass Q=...)
	$(ABORIST) --db $(DB) search '$(if $(Q),$(Q),$(SEARCH_Q))'

stats: bootstrap ## counts: documents, chunks, edges, audit chain
	$(ABORIST) --db $(DB) stats

test: bootstrap ## run pytest suite
	$(VENV)/bin/pytest -q

# Reproducible micro-benchmark over a fixed slice of cur. Lets you compare
# ETL throughput across configs and catches regressions on optimization
# work. Override BENCH_DOCS=N (default 5000).
BENCH_DOCS ?= 5000
BENCH_DIR  := /tmp/aborist-bench
bench: bootstrap fetch-cur ## benchmark serial vs parallel-shared vs attached at $(BENCH_DOCS) docs
	@bash bench/run.sh $(BENCH_DOCS)

clean: ## remove venv + caches (keeps fetched data and db)
	rm -rf $(VENV) .pytest_cache **/__pycache__ aborist.egg-info
	find . -type d -name __pycache__ -prune -exec rm -rf {} +

clean-db: ## drop the aborist db (keeps fetched data and venv)
	rm -f $(DB) $(DB)-journal $(DB)-wal $(DB)-shm

clean-data: ## remove fetched dumps
	rm -rf $(DATA_DIR)