russell@unturf.com
a056acfb5b
The 2003-05-16 archive ships three files:
20030516_cur_tablesql.bz2 82 MB current snapshot (single revision/page)
old_tablesqlbz2.1 640 MiB \
old_tablesqlbz2.2 252 MiB / split halves of old (full revision
history). Concatenate before bzcat.
Generalize the parser:
WikipediaSqlDump(table='cur'|'old') — shared statement parser, single
column-position contract for the
first 4 fields (id/ns/title/text)
WikipediaCurDump — back-compat wrapper, table='cur'
WikipediaOldDump — new, table='old'; old has no is_redirect, every
revision is real
Old rows surface old_id and old_timestamp via Document.extra so a
downstream pass can sort revisions chronologically before re-ingesting
through the supersedes-edge path.
Makefile gains:
fetch-cur / fetch-old / fetch (both)
ingest-cur / ingest-old / ingest (cur default)
WP_OLD target concatenates the two split parts
CLI ingest --source now accepts wikipedia_cur or wikipedia_old.
Smoke (real dump): 5 revisions of "AtlasShrugged/Companies" yielded
correctly with old_id=2..10, timestamps from January 2002.
99 lines
3.5 KiB
Makefile
99 lines
3.5 KiB
Makefile
# aborist — Makefile entry points
|
|
# Every workflow lives behind a `make` target. Bare python commands are not
|
|
# the user interface.
|
|
|
|
# Tools and config
|
|
PYTHON ?= python3
|
|
VENV ?= .venv
|
|
PIP := $(VENV)/bin/pip
|
|
PY := $(VENV)/bin/python
|
|
ABORIST := $(VENV)/bin/aborist
|
|
|
|
# Data + DB
|
|
DATA_DIR ?= data
|
|
WP_BASE_URL ?= https://dumps.wikimedia.org/archive/2003/2003-05-16/en
|
|
WP_CUR := $(DATA_DIR)/20030516_cur_tablesql.bz2
|
|
WP_OLD_1 := $(DATA_DIR)/old_tablesqlbz2.1
|
|
WP_OLD_2 := $(DATA_DIR)/old_tablesqlbz2.2
|
|
WP_OLD := $(DATA_DIR)/20030516_old_tablesql.bz2
|
|
# Back-compat alias (older callers used WP_DUMP for the cur snapshot).
|
|
WP_DUMP := $(WP_CUR)
|
|
DB ?= $(HOME)/.aborist/aborist.db
|
|
|
|
# Smoke-test caps so make all stays fast
|
|
INGEST_LIMIT ?= 500
|
|
VERIFY_N ?= 10
|
|
SEARCH_Q ?= computer
|
|
|
|
.PHONY: all bootstrap fetch fetch-cur fetch-old ingest ingest-cur ingest-old \
|
|
verify search stats test clean clean-db clean-data help
|
|
|
|
all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats
|
|
|
|
help: ## show this help
|
|
@awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " %-16s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
|
|
|
|
$(VENV)/bin/activate: pyproject.toml
|
|
$(PYTHON) -m venv $(VENV)
|
|
$(PIP) install --upgrade pip wheel
|
|
$(PIP) install -e '.[dev]'
|
|
@touch $(VENV)/bin/activate
|
|
|
|
bootstrap: $(VENV)/bin/activate ## create venv and install editable package
|
|
|
|
$(DATA_DIR):
|
|
mkdir -p $(DATA_DIR)
|
|
|
|
$(WP_CUR): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
|
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
|
|
|
$(WP_OLD_1): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
|
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
|
|
|
$(WP_OLD_2): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
|
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
|
|
|
# The two old_tablesqlbz2.{1,2} parts are split halves of a single bzip2
|
|
# stream (.1 is exactly 640 MiB). Concatenate to get a working bz2 file.
|
|
$(WP_OLD): $(WP_OLD_1) $(WP_OLD_2)
|
|
@echo ">> concatenating old dump parts"
|
|
cat $(WP_OLD_1) $(WP_OLD_2) > $@
|
|
|
|
fetch-cur: $(WP_CUR) ## download cur table dump (~82 MB)
|
|
|
|
fetch-old: $(WP_OLD) ## download old (revision history) parts and concatenate (~893 MB)
|
|
|
|
fetch: fetch-cur fetch-old ## download all 3 files (cur + old.1 + old.2 + concat)
|
|
|
|
ingest-cur: bootstrap fetch-cur ## ingest INGEST_LIMIT cur articles
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) --limit $(INGEST_LIMIT)
|
|
|
|
ingest-old: bootstrap fetch-old ## ingest INGEST_LIMIT old (history) revisions
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) --limit $(INGEST_LIMIT)
|
|
|
|
ingest: ingest-cur ## default ingest = cur (use ingest-old or both for full)
|
|
|
|
verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents
|
|
$(ABORIST) --db $(DB) verify -n $(VERIFY_N)
|
|
|
|
search: bootstrap ## keyword search; override SEARCH_Q (or pass Q=...)
|
|
$(ABORIST) --db $(DB) search '$(if $(Q),$(Q),$(SEARCH_Q))'
|
|
|
|
stats: bootstrap ## counts: documents, chunks, edges, audit chain
|
|
$(ABORIST) --db $(DB) stats
|
|
|
|
test: bootstrap ## run pytest suite
|
|
$(VENV)/bin/pytest -q
|
|
|
|
clean: ## remove venv + caches (keeps fetched data and db)
|
|
rm -rf $(VENV) .pytest_cache **/__pycache__ aborist.egg-info
|
|
find . -type d -name __pycache__ -prune -exec rm -rf {} +
|
|
|
|
clean-db: ## drop the aborist db (keeps fetched data and venv)
|
|
rm -f $(DB) $(DB)-journal $(DB)-wal $(DB)-shm
|
|
|
|
clean-data: ## remove fetched dumps
|
|
rm -rf $(DATA_DIR)
|