russell@unturf.com
22a8071936
Adds aborist/wikitext.py with to_base() — deterministic wikitext → prose conversion via mwparserfromhell. Drops <ref>...</ref>, [[File:...]], [[Image:...]], [[Category:...]] entirely; resolves piped wikilinks to display text; collapses templates, formatting, and HTML markup. extract_wikilinks() preserves the link graph for the definition-cloud artifact (recoverable from any page on demand without re-parsing wikitext at query time). Wires to_base() into verify_quotes(). Without the strip, the verifier compared the model's clean prose against [[Cloud Strife]], [[Shinra Electric Power Company|Shinra]], etc. and falsely flagged real source quotes as VISUAL. Concrete case from a make-query run: 'Cloud Strife, an unsociable mercenary who claims to be a former 1st Class member of Shinra's SOLDIER unit;' is verbatim in the Final_Fantasy_VII article wikitext (modulo markup). With the strip that quote now verifies; the genuine model hallucinations in the same answer still flag honestly. Side-effect: 43% smaller context size on average so less LLM token waste. BASE_VERSION = 'wikitext-base-v1' is the algorithm pin. Bump when the strip rules change. Soft-imported in verify.py so environments without mwparserfromhell installed degrade gracefully (no strip, same behavior as before this commit). Build: - pyproject.toml: new [wikitext] extras (mwparserfromhell>=0.6), pulled in by [dev] - Makefile: chain-check / chain-check-shards targets — fast audit-chain integrity probe, counts dangling prev_event_hash references; 0 = chain intact Tests: - tests/test_wikitext.py: 31 tests (rules, idempotence, real-corpus fixture) - tests/test_verify.py: 2 regression tests pinning the FF7 flip (Cloud Strife quote: VISUAL → STRICT after strip; genuine hallucination: stays VISUAL) - tests/fixtures/ff7_characters_chunk0.wikitext: real chunk from a shard, used to validate the strip on actual Wikipedia content
327 lines
14 KiB
Makefile
327 lines
14 KiB
Makefile
# aborist — Makefile entry points
|
|
# Every workflow lives behind a `make` target. Bare python commands are not
|
|
# the user interface.
|
|
|
|
# Tools and config
|
|
PYTHON ?= python3
|
|
VENV ?= .venv
|
|
PIP := $(VENV)/bin/pip
|
|
PY := $(VENV)/bin/python
|
|
ABORIST := $(VENV)/bin/aborist
|
|
|
|
# Data + DB
|
|
DATA_DIR ?= data
|
|
WP_BASE_URL ?= https://dumps.wikimedia.org/archive/2003/2003-05-16/en
|
|
WP_CUR := $(DATA_DIR)/20030516_cur_tablesql.bz2
|
|
WP_OLD_1 := $(DATA_DIR)/old_tablesqlbz2.1
|
|
WP_OLD_2 := $(DATA_DIR)/old_tablesqlbz2.2
|
|
WP_OLD := $(DATA_DIR)/20030516_old_tablesql.bz2
|
|
# Back-compat alias (older callers used WP_DUMP for the cur snapshot).
|
|
WP_DUMP := $(WP_CUR)
|
|
DB ?= $(HOME)/.aborist/aborist.db
|
|
|
|
# Smoke-test caps so make all stays fast
|
|
INGEST_LIMIT ?= 500
|
|
VERIFY_N ?= 10
|
|
SEARCH_Q ?= computer
|
|
|
|
.PHONY: all bootstrap fetch fetch-cur fetch-old fetch-xml fetch-abstract \
|
|
ingest ingest-cur ingest-old ingest-xml ingest-xml-history \
|
|
ingest-xml-attached ingest-abstract \
|
|
ingest-grok ingest-grok-media \
|
|
ingest-self ingest-git ingest-hg \
|
|
verify search stats test docs chain-check chain-check-shards \
|
|
clean clean-db clean-data help
|
|
|
|
all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats
|
|
|
|
help: ## show this help
|
|
@awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " %-16s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
|
|
|
|
$(VENV)/bin/activate: pyproject.toml
|
|
$(PYTHON) -m venv $(VENV)
|
|
$(PIP) install --upgrade pip wheel
|
|
$(PIP) install -e '.[dev]'
|
|
@touch $(VENV)/bin/activate
|
|
|
|
bootstrap: $(VENV)/bin/activate ## create venv and install editable package
|
|
|
|
$(DATA_DIR):
|
|
mkdir -p $(DATA_DIR)
|
|
|
|
$(WP_CUR): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
|
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
|
|
|
$(WP_OLD_1): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
|
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
|
|
|
$(WP_OLD_2): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
|
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
|
|
|
# The two old_tablesqlbz2.{1,2} parts are split halves of a single bzip2
|
|
# stream (.1 is exactly 640 MiB). Concatenate to get a working bz2 file.
|
|
$(WP_OLD): $(WP_OLD_1) $(WP_OLD_2)
|
|
@echo ">> concatenating old dump parts"
|
|
cat $(WP_OLD_1) $(WP_OLD_2) > $@
|
|
|
|
fetch-cur: $(WP_CUR) ## download cur table dump (~82 MB)
|
|
|
|
fetch-old: $(WP_OLD) ## download old (revision history) parts and concatenate (~893 MB)
|
|
|
|
fetch: fetch-cur fetch-old ## download all 3 files (cur + old.1 + old.2 + concat)
|
|
|
|
ingest-cur: bootstrap fetch-cur ## ingest INGEST_LIMIT cur articles
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) --limit $(INGEST_LIMIT)
|
|
|
|
ingest-old: bootstrap fetch-old ## ingest INGEST_LIMIT old (history) revisions
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) --limit $(INGEST_LIMIT)
|
|
|
|
# Phase 1b: parallel shards into ONE shared SQLite. Workers parallelize
|
|
# parser CPU; writes serialize at the WAL writer-lock. ~1.3x wall on 4 cores.
|
|
SHARDS ?= 4
|
|
ingest-cur-parallel: bootstrap fetch-cur ## ingest cur with SHARDS=N processes -> one shared DB
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) \
|
|
--shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
ingest-old-parallel: bootstrap fetch-old ## ingest old history with SHARDS=N processes -> one shared DB
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) \
|
|
--shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
# Phase 2: attach-forever sharding. Each shard owns its own SQLite file —
|
|
# no WAL contention. Reads via `aborist --shards-dir <dir> <cmd>` attach
|
|
# all shards as UNION views. "Merge cost" = 0.
|
|
SHARDS_DIR ?= $(HOME)/.aborist/shards
|
|
ingest-cur-attached: bootstrap fetch-cur ## sharded ingest, no WAL contention (Phase 2)
|
|
@mkdir -p $(SHARDS_DIR)
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) ingest --source wikipedia_cur --path $(WP_CUR) \
|
|
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
ingest-old-attached: bootstrap fetch-old ## sharded ingest of old history (Phase 2)
|
|
@mkdir -p $(SHARDS_DIR)
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) ingest --source wikipedia_old --path $(WP_OLD) \
|
|
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
stats-shards: bootstrap ## cross-shard stats via UNION views over $(SHARDS_DIR)
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) stats
|
|
|
|
ACTIVITY_LIMIT ?= 10
|
|
activity: bootstrap ## recent Q&A + freshly cached docs (agent timeline)
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) activity --limit $(ACTIVITY_LIMIT)
|
|
|
|
falsify: bootstrap ## mark a cached answer wrong: make falsify KEY=hex REASON='why'
|
|
@if [ -z "$(KEY)" ]; then echo "usage: make falsify KEY=<cache_key> REASON='why'" >&2; exit 2; fi
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) providence --falsify $(KEY) --reason "$(REASON)"
|
|
|
|
# Multi-source RAG query against the shard cluster.
|
|
# Usage: make query Q="What is anarcho-capitalism?"
|
|
QUERY_TOP_K ?= 8
|
|
query: bootstrap ## ask the corpus a question; sources are picked across shards
|
|
@if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \
|
|
echo "usage: make query Q=\"your question\""; exit 2; \
|
|
fi
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) "$(Q)"
|
|
|
|
query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run)
|
|
@if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \
|
|
echo "usage: make query-dry Q=\"your question\""; exit 2; \
|
|
fi
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run "$(Q)"
|
|
|
|
verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N)
|
|
|
|
analyze-shards: bootstrap ## cross-shard compression spectrum + audit integrity
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) analyze
|
|
|
|
# Audit-chain integrity probe: counts dangling prev_event_hash references.
|
|
# Faster than `analyze` and trivially scriptable. 0 = chain intact.
|
|
define CHAIN_CHECK_SQL
|
|
SELECT COUNT(*) AS chain_breaks FROM audit_events a1
|
|
LEFT JOIN audit_events a2 ON a2.event_hash = a1.prev_event_hash
|
|
WHERE a1.prev_event_hash IS NOT NULL AND a2.event_hash IS NULL
|
|
endef
|
|
export CHAIN_CHECK_SQL
|
|
|
|
chain-check: ## audit-chain break count for $(DB) (0 = intact)
|
|
@printf '%s ' "$(DB)"; sqlite3 $(DB) "$$CHAIN_CHECK_SQL"
|
|
|
|
chain-check-shards: ## audit-chain break count for every *.db in $(SHARDS_DIR)
|
|
@for db in $(SHARDS_DIR)/*.db; do \
|
|
printf '%s ' "$$db"; sqlite3 "$$db" "$$CHAIN_CHECK_SQL"; \
|
|
done
|
|
|
|
# Sequential per-shard distill (one process iterates all shards).
|
|
distill-shards: bootstrap ## distill every shard in $(SHARDS_DIR), surface -> depth=1 cores
|
|
$(ABORIST) --shards-dir $(SHARDS_DIR) distill --process first-sentence-v1 --kind surface
|
|
|
|
# Parallel per-shard distill: one process per shard. No DB contention
|
|
# because each shard is its own file.
|
|
distill-shards-parallel: bootstrap ## one distill process per shard (parallel)
|
|
@for shard in $(SHARDS_DIR)/*.db; do \
|
|
$(ABORIST) --db $$shard distill --process first-sentence-v1 --kind surface & \
|
|
done; wait
|
|
|
|
# TF-IDF cores serve as enriched titles for retrieval — distinctive
|
|
# low-frequency terms that surface in body text get promoted into
|
|
# something queryable without a real title match.
|
|
distill-shards-tfidf-parallel: bootstrap ## TF-IDF cores per shard, in parallel
|
|
@for shard in $(SHARDS_DIR)/*.db; do \
|
|
$(ABORIST) --db $$shard distill --process tfidf-keywords-v1 --kind surface & \
|
|
done; wait
|
|
|
|
ingest: ingest-cur ## default ingest = cur (use ingest-old or *-parallel for full)
|
|
|
|
# Grok account-export ETL.
|
|
# Point GROK_EXPORT at the directory xAI delivered (the one containing
|
|
# `ttl/30d/export_data/<user-id>/prod-grok-backend.json`). The source class
|
|
# auto-walks down to find the JSON.
|
|
GROK_EXPORT ?= $(HOME)/Downloads/ab8ef1f0-0d08-4f87-89c2-d4509e18115b
|
|
ingest-grok: bootstrap ## ingest Grok conversations into $(DB) (single-DB mode)
|
|
$(ABORIST) --db $(DB) ingest --source grok_export --path $(GROK_EXPORT)
|
|
|
|
ingest-grok-media: bootstrap ## ingest Grok media-generation prompts into $(DB)
|
|
$(ABORIST) --db $(DB) ingest --source grok_media --path $(GROK_EXPORT)
|
|
|
|
# Grok lives in its own shard inside the attach-forever cluster so cross-
|
|
# shard queries (`make query`) see it alongside the Wikipedia 2003 corpus.
|
|
# Single shard (rank 0/1) — Grok conversations are private and small.
|
|
GROK_SHARD := $(SHARDS_DIR)/grok.db
|
|
ingest-grok-attached: bootstrap ## ingest Grok conversations into $(GROK_SHARD)
|
|
@mkdir -p $(SHARDS_DIR)
|
|
$(ABORIST) --db $(GROK_SHARD) ingest --source grok_export --path $(GROK_EXPORT) --resume
|
|
|
|
ingest-grok-media-attached: bootstrap ## ingest Grok media prompts into $(GROK_SHARD)
|
|
@mkdir -p $(SHARDS_DIR)
|
|
$(ABORIST) --db $(GROK_SHARD) ingest --source grok_media --path $(GROK_EXPORT) --resume
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Phase IV (2006+) Wikipedia XML dumps. Drop-in for any dated snapshot in
|
|
# the dumps.wikimedia.org/archive tree by overriding WP_XML_YEAR/MONTH/DATE
|
|
# (and WP_XML_LANG for non-English wikis).
|
|
#
|
|
# Defaults point at enwiki 20101011 (2010-11 archive), the largest snapshot
|
|
# in the archive — 6.2 GB compressed, ~3.4M articles.
|
|
# Other useful snapshots:
|
|
# make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-07 WP_XML_DATE=20061104
|
|
# make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-12 WP_XML_DATE=20061130
|
|
# ----------------------------------------------------------------------------
|
|
WP_XML_LANG ?= en
|
|
WP_XML_YEAR ?= 2010
|
|
WP_XML_MONTH ?= 2010-11
|
|
WP_XML_DATE ?= 20101011
|
|
WP_XML_BASE_URL ?= https://dumps.wikimedia.org/archive/$(WP_XML_YEAR)/$(WP_XML_MONTH)/$(WP_XML_LANG)wiki/$(WP_XML_DATE)
|
|
WP_XML_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-pages-articles.xml.bz2
|
|
WP_XML := $(DATA_DIR)/$(WP_XML_FILE)
|
|
WP_ABSTRACT_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-abstract.xml
|
|
WP_ABSTRACT := $(DATA_DIR)/$(WP_ABSTRACT_FILE)
|
|
|
|
$(WP_XML): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_XML_BASE_URL)/$(WP_XML_FILE)"
|
|
curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_XML_FILE)"
|
|
|
|
$(WP_ABSTRACT): | $(DATA_DIR)
|
|
@echo ">> fetching $(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)"
|
|
curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)"
|
|
|
|
fetch-xml: $(WP_XML) ## download Phase IV XML cur dump (default: enwiki 20101011, 6.2 GB)
|
|
|
|
fetch-abstract: $(WP_ABSTRACT) ## download Phase IV abstract.xml (default: enwiki 20101011, ~3 GB)
|
|
|
|
ingest-xml: bootstrap fetch-xml ## ingest INGEST_LIMIT pages from $(WP_XML)
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_xml --path $(WP_XML) --limit $(INGEST_LIMIT)
|
|
|
|
ingest-xml-history: bootstrap ## ingest every revision (multi-revision mode); set WP_XML to a pages-meta-history file
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_xml_history --path $(WP_XML) --limit $(INGEST_LIMIT)
|
|
|
|
# Sharded XML ingest into the attach-forever cluster — same pattern as
|
|
# ingest-cur-attached. One process per shard, one SQLite file per shard,
|
|
# zero WAL contention.
|
|
ingest-xml-attached: bootstrap fetch-xml ## sharded XML ingest, one process per shard
|
|
@mkdir -p $(SHARDS_DIR)
|
|
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
|
$(ABORIST) ingest --source wikipedia_xml --path $(WP_XML) \
|
|
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
|
done; wait
|
|
|
|
ingest-abstract: bootstrap fetch-abstract ## ingest INGEST_LIMIT abstract docs from $(WP_ABSTRACT)
|
|
$(ABORIST) --db $(DB) ingest --source wikipedia_abstract --path $(WP_ABSTRACT) --limit $(INGEST_LIMIT)
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Self-ingest: aborist consults its own source code as a queryable corpus.
|
|
# Re-running picks up new commits — same path + new content gets a fresh
|
|
# document_root chained to the prior version via a `supersedes` edge, so the
|
|
# audit chain grows as the repo grows. Lands in a dedicated shard file so it
|
|
# doesn't compete with the wikipedia/grok shards' WAL writer lock.
|
|
#
|
|
# Override SELF_REPO to ingest a different repo's tree.
|
|
# ----------------------------------------------------------------------------
|
|
SELF_REPO ?= $(CURDIR)
|
|
SELF_SHARD := $(SHARDS_DIR)/aborist-self.db
|
|
|
|
ingest-self: bootstrap ## ingest this repo's HEAD into a dedicated shard
|
|
@mkdir -p $(SHARDS_DIR)
|
|
$(ABORIST) --db $(SELF_SHARD) ingest --source git_repo --path $(SELF_REPO)
|
|
|
|
# Generic git-repo ingest: aim it at any local clone via GIT_REPO=...
|
|
GIT_REPO ?= $(CURDIR)
|
|
GIT_SHARD := $(SHARDS_DIR)/$(notdir $(GIT_REPO))-git.db
|
|
ingest-git: bootstrap ## ingest GIT_REPO=<path> into its own shard
|
|
@mkdir -p $(SHARDS_DIR)
|
|
$(ABORIST) --db $(GIT_SHARD) ingest --source git_repo --path $(GIT_REPO)
|
|
|
|
# Generic hg-repo ingest. HG_REPO=<path>.
|
|
HG_REPO ?=
|
|
HG_SHARD := $(SHARDS_DIR)/$(notdir $(HG_REPO))-hg.db
|
|
ingest-hg: bootstrap ## ingest HG_REPO=<path> (mercurial) into its own shard
|
|
@if [ -z "$(HG_REPO)" ]; then echo "usage: make ingest-hg HG_REPO=/path/to/repo" >&2; exit 2; fi
|
|
@mkdir -p $(SHARDS_DIR)
|
|
$(ABORIST) --db $(HG_SHARD) ingest --source hg_repo --path $(HG_REPO)
|
|
|
|
verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents
|
|
$(ABORIST) --db $(DB) verify -n $(VERIFY_N)
|
|
|
|
search: bootstrap ## keyword search; override SEARCH_Q (or pass Q=...)
|
|
$(ABORIST) --db $(DB) search '$(if $(Q),$(Q),$(SEARCH_Q))'
|
|
|
|
stats: bootstrap ## counts: documents, chunks, edges, audit chain
|
|
$(ABORIST) --db $(DB) stats
|
|
|
|
test: bootstrap ## run pytest suite
|
|
$(VENV)/bin/pytest -q
|
|
|
|
DOT_SRCS := $(wildcard docs/diagrams/*.dot)
|
|
DOT_PNGS := $(DOT_SRCS:.dot=.png)
|
|
|
|
docs/diagrams/%.png: docs/diagrams/%.dot
|
|
dot -Tpng $< -o $@
|
|
|
|
docs: $(DOT_PNGS) ## render docs/diagrams/*.dot -> .png via graphviz
|
|
|
|
# Reproducible micro-benchmark over a fixed slice of cur. Lets you compare
|
|
# ETL throughput across configs and catches regressions on optimization
|
|
# work. Override BENCH_DOCS=N (default 5000).
|
|
BENCH_DOCS ?= 5000
|
|
BENCH_DIR := /tmp/aborist-bench
|
|
bench: bootstrap fetch-cur ## benchmark serial vs parallel-shared vs attached at $(BENCH_DOCS) docs
|
|
@bash bench/run.sh $(BENCH_DOCS)
|
|
|
|
clean: ## remove venv + caches (keeps fetched data and db)
|
|
rm -rf $(VENV) .pytest_cache **/__pycache__ aborist.egg-info
|
|
find . -type d -name __pycache__ -prune -exec rm -rf {} +
|
|
|
|
clean-db: ## drop the aborist db (keeps fetched data and venv)
|
|
rm -f $(DB) $(DB)-journal $(DB)-wal $(DB)-shm
|
|
|
|
clean-data: ## remove fetched dumps
|
|
rm -rf $(DATA_DIR)
|