russell@unturf.com
1ef93f4d2f
New tests/test_qa_quality_live.py with 9 fixtures gated by
ABORIST_LIVE_TESTS=1 (the `make test-live` target sets it). Each test
runs ONE live query against Hermes + the configured shard set and
asserts:
* audit_mode is at least HYBRID (or UNGROUNDED for the honest-
refusal case)
* key entity tokens appear in the rendered answer
(e.g. "Torvalds" for the linux-kernel question, all four turtle
names + "Splinter" for the TMNT multi-part)
* for the Mars PM no-such-thing case, either UNGROUNDED OR a
refutation phrase ("no prime minister", "does not have", etc.)
— bare affirmative claims of a Mars PM fail the test as
hallucination
Why functional tests matter alongside unit + bench:
- unit tests (test_qa, test_query, test_claim_lattice) validate
plumbing with StubClient — can't tell you "did the model
actually answer correctly?"
- bench (`make bench-qa`) measures aggregate strict-rate /
grounded counts across many questions but doesn't assert
specific content
- these live fixtures sit between: each test is a named gate
around one known-good answer. When a future change improves
things, the bench number climbs AND every fixture passes (or
gets stricter assertions). When something regresses, the bench
number falls AND specific fixtures fail by name, telling you
where the regression landed. "Benchmax" rationale: the bench
is the scoreboard, the fixtures are the gates that translate
quality drift into named test failures.
Default `make test` is unaffected — 460 passed + 10 skipped (was 1).
427 lines
20 KiB
Makefile
427 lines
20 KiB
Makefile
# aborist — Makefile entry points
|
||
# Every workflow lives behind a `make` target. Bare python commands are not
|
||
# the user interface.
|
||
|
||
# Tools and config
|
||
PYTHON ?= python3
|
||
VENV ?= .venv
|
||
PIP := $(VENV)/bin/pip
|
||
PY := $(VENV)/bin/python
|
||
ABORIST := $(VENV)/bin/aborist
|
||
|
||
# Data + DB
|
||
DATA_DIR ?= data
|
||
WP_BASE_URL ?= https://dumps.wikimedia.org/archive/2003/2003-05-16/en
|
||
WP_CUR := $(DATA_DIR)/20030516_cur_tablesql.bz2
|
||
WP_OLD_1 := $(DATA_DIR)/old_tablesqlbz2.1
|
||
WP_OLD_2 := $(DATA_DIR)/old_tablesqlbz2.2
|
||
WP_OLD := $(DATA_DIR)/20030516_old_tablesql.bz2
|
||
# Back-compat alias (older callers used WP_DUMP for the cur snapshot).
|
||
WP_DUMP := $(WP_CUR)
|
||
DB ?= $(HOME)/.aborist/aborist.db
|
||
|
||
# Smoke-test caps so make all stays fast
|
||
INGEST_LIMIT ?= 500
|
||
VERIFY_N ?= 10
|
||
SEARCH_Q ?= computer
|
||
|
||
.PHONY: all bootstrap fetch fetch-cur fetch-old fetch-xml fetch-abstract \
|
||
ingest ingest-cur ingest-old ingest-xml ingest-xml-history \
|
||
ingest-xml-attached ingest-abstract \
|
||
ingest-grok ingest-grok-media \
|
||
ingest-self ingest-git ingest-hg \
|
||
verify search stats test test-live docs chain-check chain-check-shards \
|
||
falsify burn burn-kindergarten inspect bootstrap-crawler test-crawler crawl-ingest \
|
||
recrawl-check bench-qa clean clean-db clean-data help
|
||
|
||
all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats
|
||
|
||
help: ## show this help
|
||
@awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " %-16s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
|
||
|
||
$(VENV)/bin/activate: pyproject.toml
|
||
$(PYTHON) -m venv $(VENV)
|
||
$(PIP) install --upgrade pip wheel
|
||
$(PIP) install -e '.[dev]'
|
||
@touch $(VENV)/bin/activate
|
||
|
||
bootstrap: $(VENV)/bin/activate ## create venv and install editable package
|
||
|
||
$(DATA_DIR):
|
||
mkdir -p $(DATA_DIR)
|
||
|
||
$(WP_CUR): | $(DATA_DIR)
|
||
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
||
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
||
|
||
$(WP_OLD_1): | $(DATA_DIR)
|
||
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
||
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
||
|
||
$(WP_OLD_2): | $(DATA_DIR)
|
||
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
||
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
||
|
||
# The two old_tablesqlbz2.{1,2} parts are split halves of a single bzip2
|
||
# stream (.1 is exactly 640 MiB). Concatenate to get a working bz2 file.
|
||
$(WP_OLD): $(WP_OLD_1) $(WP_OLD_2)
|
||
@echo ">> concatenating old dump parts"
|
||
cat $(WP_OLD_1) $(WP_OLD_2) > $@
|
||
|
||
fetch-cur: $(WP_CUR) ## download cur table dump (~82 MB)
|
||
|
||
fetch-old: $(WP_OLD) ## download old (revision history) parts and concatenate (~893 MB)
|
||
|
||
fetch: fetch-cur fetch-old ## download all 3 files (cur + old.1 + old.2 + concat)
|
||
|
||
ingest-cur: bootstrap fetch-cur ## ingest INGEST_LIMIT cur articles
|
||
$(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) --limit $(INGEST_LIMIT)
|
||
|
||
ingest-old: bootstrap fetch-old ## ingest INGEST_LIMIT old (history) revisions
|
||
$(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) --limit $(INGEST_LIMIT)
|
||
|
||
# Phase 1b: parallel shards into ONE shared SQLite. Workers parallelize
|
||
# parser CPU; writes serialize at the WAL writer-lock. ~1.3x wall on 4 cores.
|
||
SHARDS ?= 4
|
||
ingest-cur-parallel: bootstrap fetch-cur ## ingest cur with SHARDS=N processes -> one shared DB
|
||
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
||
$(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) \
|
||
--shard $$i/$(SHARDS) & \
|
||
done; wait
|
||
|
||
ingest-old-parallel: bootstrap fetch-old ## ingest old history with SHARDS=N processes -> one shared DB
|
||
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
||
$(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) \
|
||
--shard $$i/$(SHARDS) & \
|
||
done; wait
|
||
|
||
# Phase 2: attach-forever sharding. Each shard owns its own SQLite file —
|
||
# no WAL contention. Reads via `aborist --shards-dir <dir> <cmd>` attach
|
||
# all shards as UNION views. "Merge cost" = 0.
|
||
SHARDS_DIR ?= $(HOME)/.aborist/shards
|
||
ingest-cur-attached: bootstrap fetch-cur ## sharded ingest, no WAL contention (Phase 2)
|
||
@mkdir -p $(SHARDS_DIR)
|
||
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
||
$(ABORIST) ingest --source wikipedia_cur --path $(WP_CUR) \
|
||
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
||
done; wait
|
||
|
||
ingest-old-attached: bootstrap fetch-old ## sharded ingest of old history (Phase 2)
|
||
@mkdir -p $(SHARDS_DIR)
|
||
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
||
$(ABORIST) ingest --source wikipedia_old --path $(WP_OLD) \
|
||
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
||
done; wait
|
||
|
||
stats-shards: bootstrap ## cross-shard stats via UNION views over $(SHARDS_DIR)
|
||
$(ABORIST) --shards-dir $(SHARDS_DIR) stats
|
||
|
||
ACTIVITY_LIMIT ?= 10
|
||
activity: bootstrap ## recent Q&A + freshly cached docs (agent timeline)
|
||
$(ABORIST) --shards-dir $(SHARDS_DIR) activity --limit $(ACTIVITY_LIMIT)
|
||
|
||
inspect: bootstrap ## sidecar diagnose unverified spans for a cache_key: make inspect KEY=hex [JSON=1]
|
||
@if [ -z "$(KEY)" ]; then echo "usage: make inspect KEY=<cache_key> [JSON=1]" >&2; exit 2; fi
|
||
$(ABORIST) --shards-dir $(SHARDS_DIR) inspect --cache-key $(KEY) $(if $(JSON),--json,)
|
||
|
||
falsify: bootstrap ## mark a cached answer wrong: make falsify KEY=hex REASON='why'
|
||
@if [ -z "$(KEY)" ]; then echo "usage: make falsify KEY=<cache_key> REASON='why'" >&2; exit 2; fi
|
||
$(ABORIST) --shards-dir $(SHARDS_DIR) providence --falsify $(KEY) --reason "$(REASON)"
|
||
|
||
burn: bootstrap ## delete a leaf with no children. providence: KEY=<cache_key>; document/core: KIND=document|core ROOT=<hex>. REASON='why' [FORCE=1]
|
||
@kind="$${KIND:-providence}"; \
|
||
if [ "$$kind" = "providence" ]; then \
|
||
if [ -z "$(KEY)" ]; then echo "usage: make burn KEY=<cache_key> REASON='why' [FORCE=1]" >&2; exit 2; fi; \
|
||
$(ABORIST) --shards-dir $(SHARDS_DIR) burn --kind providence --cache-key $(KEY) --reason "$(REASON)" $(if $(FORCE),--force,); \
|
||
elif [ "$$kind" = "document" ] || [ "$$kind" = "core" ]; then \
|
||
if [ -z "$(ROOT)" ]; then echo "usage: make burn KIND=$$kind ROOT=<document_root> REASON='why' [FORCE=1]" >&2; exit 2; fi; \
|
||
$(ABORIST) --shards-dir $(SHARDS_DIR) burn --kind $$kind --root $(ROOT) --reason "$(REASON)" $(if $(FORCE),--force,); \
|
||
else \
|
||
echo "unknown KIND: $$kind (expected: providence|document|core)" >&2; exit 2; \
|
||
fi
|
||
|
||
# Mass-burn providence_cache rows younger than the kindergarten window.
|
||
# Mirrors mesh sync's kindergarten so what's still un-broadcast is what's
|
||
# safe to bust without confusing peers. Useful while iterating on
|
||
# retrieval/verifier tunings — wipe recent test runs in one shot.
|
||
KG_SECONDS ?= 3600
|
||
burn-kindergarten: bootstrap ## bust providence rows < SECONDS old [SECONDS=3600 FORCE=1 DRY_RUN=1 REASON='why']
|
||
$(ABORIST) --shards-dir $(SHARDS_DIR) burn-kindergarten \
|
||
--kindergarten-seconds $(KG_SECONDS) \
|
||
$(if $(REASON),--reason "$(REASON)",) \
|
||
$(if $(FORCE),--force,) \
|
||
$(if $(DRY_RUN),--dry-run,)
|
||
|
||
# Multi-source RAG query against the shard cluster.
|
||
# Usage: make query Q="What is anarcho-capitalism?"
|
||
QUERY_TOP_K ?= 8
|
||
# G0 / CTI — pointer-mode is the testing default. Library DEFAULT_POLICY
|
||
# stays "quote" so Python callers aren't surprised; the Makefile
|
||
# harness ships pointer-mode-on so `make query` exercises the new path
|
||
# end-to-end. Override with ANSWER_MODE=quote to test the legacy path,
|
||
# ANSWER_MODE= (empty) to use whatever DEFAULT_QUERY_POLICY says.
|
||
ANSWER_MODE ?= claim_lattice_pointer
|
||
query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N ANSWER_MODE=quote|claim_lattice_pointer]; pointer mode by default
|
||
@if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \
|
||
echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N ANSWER_MODE=quote|claim_lattice_pointer]"; exit 2; \
|
||
fi
|
||
$(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) "$(Q)"
|
||
|
||
query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=...]
|
||
@if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \
|
||
echo "usage: make query-dry Q=\"your question\" [JSON=1 BURN=1 ANSWER_MODE=quote|claim_lattice_pointer]"; exit 2; \
|
||
fi
|
||
$(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) "$(Q)"
|
||
|
||
BENCH_QA_QUESTIONS ?= bench/qa_questions.txt
|
||
BENCH_QA_OUT ?= bench/qa_results
|
||
BENCH_QA_MODES ?= quote,claim_lattice_pointer,claim_lattice
|
||
BENCH_QA_LIMIT ?= 0
|
||
BENCH_QA_N ?= 3
|
||
bench-qa: bootstrap ## QA-quality sweep: questions × modes × N samples [BENCH_QA_N=3 BENCH_QA_LIMIT=N BENCH_QA_MODES=...]
|
||
PYTHONUNBUFFERED=1 $(PY) bench/qa_sweep.py \
|
||
--questions $(BENCH_QA_QUESTIONS) \
|
||
--shards-dir $(SHARDS_DIR) \
|
||
--out-dir $(BENCH_QA_OUT) \
|
||
--top-k $(QUERY_TOP_K) \
|
||
--modes $(BENCH_QA_MODES) \
|
||
--limit $(BENCH_QA_LIMIT) \
|
||
--n $(BENCH_QA_N)
|
||
|
||
test-live: bootstrap ## live QA quality tests against Hermes (gated; ~1 min)
|
||
ABORIST_LIVE_TESTS=1 ABORIST_LIVE_SHARDS_DIR=$(SHARDS_DIR) \
|
||
.venv/bin/pytest tests/test_qa_quality_live.py -v
|
||
|
||
verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample
|
||
$(ABORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N)
|
||
|
||
analyze-shards: bootstrap ## cross-shard compression spectrum + audit integrity
|
||
$(ABORIST) --shards-dir $(SHARDS_DIR) analyze
|
||
|
||
# Audit-chain integrity probe: counts dangling prev_event_hash references.
|
||
# Faster than `analyze` and trivially scriptable. 0 = chain intact.
|
||
define CHAIN_CHECK_SQL
|
||
SELECT COUNT(*) AS chain_breaks FROM audit_events a1
|
||
LEFT JOIN audit_events a2 ON a2.event_hash = a1.prev_event_hash
|
||
WHERE a1.prev_event_hash IS NOT NULL AND a2.event_hash IS NULL
|
||
endef
|
||
export CHAIN_CHECK_SQL
|
||
|
||
chain-check: ## audit-chain break count for $(DB) (0 = intact)
|
||
@printf '%s ' "$(DB)"; sqlite3 $(DB) "$$CHAIN_CHECK_SQL"
|
||
|
||
chain-check-shards: ## audit-chain break count for every *.db in $(SHARDS_DIR)
|
||
@for db in $(SHARDS_DIR)/*.db; do \
|
||
printf '%s ' "$$db"; sqlite3 "$$db" "$$CHAIN_CHECK_SQL"; \
|
||
done
|
||
|
||
# Sequential per-shard distill (one process iterates all shards).
|
||
distill-shards: bootstrap ## distill every shard in $(SHARDS_DIR), surface -> depth=1 cores
|
||
$(ABORIST) --shards-dir $(SHARDS_DIR) distill --process first-sentence-v1 --kind surface
|
||
|
||
# Parallel per-shard distill: one process per shard. No DB contention
|
||
# because each shard is its own file.
|
||
distill-shards-parallel: bootstrap ## one distill process per shard (parallel)
|
||
@for shard in $(SHARDS_DIR)/*.db; do \
|
||
$(ABORIST) --db $$shard distill --process first-sentence-v1 --kind surface & \
|
||
done; wait
|
||
|
||
# TF-IDF cores serve as enriched titles for retrieval — distinctive
|
||
# low-frequency terms that surface in body text get promoted into
|
||
# something queryable without a real title match.
|
||
distill-shards-tfidf-parallel: bootstrap ## TF-IDF cores per shard, in parallel
|
||
@for shard in $(SHARDS_DIR)/*.db; do \
|
||
$(ABORIST) --db $$shard distill --process tfidf-keywords-v1 --kind surface & \
|
||
done; wait
|
||
|
||
ingest: ingest-cur ## default ingest = cur (use ingest-old or *-parallel for full)
|
||
|
||
# Grok account-export ETL.
|
||
# Point GROK_EXPORT at the directory xAI delivered (the one containing
|
||
# `ttl/30d/export_data/<user-id>/prod-grok-backend.json`). The source class
|
||
# auto-walks down to find the JSON.
|
||
GROK_EXPORT ?= $(HOME)/Downloads/ab8ef1f0-0d08-4f87-89c2-d4509e18115b
|
||
ingest-grok: bootstrap ## ingest Grok conversations into $(DB) (single-DB mode)
|
||
$(ABORIST) --db $(DB) ingest --source grok_export --path $(GROK_EXPORT)
|
||
|
||
ingest-grok-media: bootstrap ## ingest Grok media-generation prompts into $(DB)
|
||
$(ABORIST) --db $(DB) ingest --source grok_media --path $(GROK_EXPORT)
|
||
|
||
# Grok lives in its own shard inside the attach-forever cluster so cross-
|
||
# shard queries (`make query`) see it alongside the Wikipedia 2003 corpus.
|
||
# Single shard (rank 0/1) — Grok conversations are private and small.
|
||
GROK_SHARD := $(SHARDS_DIR)/grok.db
|
||
ingest-grok-attached: bootstrap ## ingest Grok conversations into $(GROK_SHARD)
|
||
@mkdir -p $(SHARDS_DIR)
|
||
$(ABORIST) --db $(GROK_SHARD) ingest --source grok_export --path $(GROK_EXPORT) --resume
|
||
|
||
ingest-grok-media-attached: bootstrap ## ingest Grok media prompts into $(GROK_SHARD)
|
||
@mkdir -p $(SHARDS_DIR)
|
||
$(ABORIST) --db $(GROK_SHARD) ingest --source grok_media --path $(GROK_EXPORT) --resume
|
||
|
||
# ----------------------------------------------------------------------------
|
||
# Phase IV (2006+) Wikipedia XML dumps. Drop-in for any dated snapshot in
|
||
# the dumps.wikimedia.org/archive tree by overriding WP_XML_YEAR/MONTH/DATE
|
||
# (and WP_XML_LANG for non-English wikis).
|
||
#
|
||
# Defaults point at enwiki 20101011 (2010-11 archive), the largest snapshot
|
||
# in the archive — 6.2 GB compressed, ~3.4M articles.
|
||
# Other useful snapshots:
|
||
# make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-07 WP_XML_DATE=20061104
|
||
# make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-12 WP_XML_DATE=20061130
|
||
# ----------------------------------------------------------------------------
|
||
WP_XML_LANG ?= en
|
||
WP_XML_YEAR ?= 2010
|
||
WP_XML_MONTH ?= 2010-11
|
||
WP_XML_DATE ?= 20101011
|
||
WP_XML_BASE_URL ?= https://dumps.wikimedia.org/archive/$(WP_XML_YEAR)/$(WP_XML_MONTH)/$(WP_XML_LANG)wiki/$(WP_XML_DATE)
|
||
WP_XML_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-pages-articles.xml.bz2
|
||
WP_XML := $(DATA_DIR)/$(WP_XML_FILE)
|
||
WP_ABSTRACT_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-abstract.xml
|
||
WP_ABSTRACT := $(DATA_DIR)/$(WP_ABSTRACT_FILE)
|
||
|
||
$(WP_XML): | $(DATA_DIR)
|
||
@echo ">> fetching $(WP_XML_BASE_URL)/$(WP_XML_FILE)"
|
||
curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_XML_FILE)"
|
||
|
||
$(WP_ABSTRACT): | $(DATA_DIR)
|
||
@echo ">> fetching $(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)"
|
||
curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)"
|
||
|
||
fetch-xml: $(WP_XML) ## download Phase IV XML cur dump (default: enwiki 20101011, 6.2 GB)
|
||
|
||
fetch-abstract: $(WP_ABSTRACT) ## download Phase IV abstract.xml (default: enwiki 20101011, ~3 GB)
|
||
|
||
ingest-xml: bootstrap fetch-xml ## ingest INGEST_LIMIT pages from $(WP_XML)
|
||
$(ABORIST) --db $(DB) ingest --source wikipedia_xml --path $(WP_XML) --limit $(INGEST_LIMIT)
|
||
|
||
ingest-xml-history: bootstrap ## ingest every revision (multi-revision mode); set WP_XML to a pages-meta-history file
|
||
$(ABORIST) --db $(DB) ingest --source wikipedia_xml_history --path $(WP_XML) --limit $(INGEST_LIMIT)
|
||
|
||
# Sharded XML ingest into the attach-forever cluster — same pattern as
|
||
# ingest-cur-attached. One process per shard, one SQLite file per shard,
|
||
# zero WAL contention.
|
||
ingest-xml-attached: bootstrap fetch-xml ## sharded XML ingest, one process per shard
|
||
@mkdir -p $(SHARDS_DIR)
|
||
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
||
$(ABORIST) ingest --source wikipedia_xml --path $(WP_XML) \
|
||
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
||
done; wait
|
||
|
||
ingest-abstract: bootstrap fetch-abstract ## ingest INGEST_LIMIT abstract docs from $(WP_ABSTRACT)
|
||
$(ABORIST) --db $(DB) ingest --source wikipedia_abstract --path $(WP_ABSTRACT) --limit $(INGEST_LIMIT)
|
||
|
||
# ----------------------------------------------------------------------------
|
||
# Self-ingest: aborist consults its own source code as a queryable corpus.
|
||
# Re-running picks up new commits — same path + new content gets a fresh
|
||
# document_root chained to the prior version via a `supersedes` edge, so the
|
||
# audit chain grows as the repo grows. Lands in a dedicated shard file so it
|
||
# doesn't compete with the wikipedia/grok shards' WAL writer lock.
|
||
#
|
||
# Override SELF_REPO to ingest a different repo's tree.
|
||
# ----------------------------------------------------------------------------
|
||
SELF_REPO ?= $(CURDIR)
|
||
SELF_SHARD := $(SHARDS_DIR)/aborist-self.db
|
||
|
||
ingest-self: bootstrap ## ingest this repo's HEAD into a dedicated shard
|
||
@mkdir -p $(SHARDS_DIR)
|
||
$(ABORIST) --db $(SELF_SHARD) ingest --source git_repo --path $(SELF_REPO)
|
||
|
||
# Generic git-repo ingest: aim it at any local clone via GIT_REPO=...
|
||
GIT_REPO ?= $(CURDIR)
|
||
GIT_SHARD := $(SHARDS_DIR)/$(notdir $(GIT_REPO))-git.db
|
||
ingest-git: bootstrap ## ingest GIT_REPO=<path> into its own shard
|
||
@mkdir -p $(SHARDS_DIR)
|
||
$(ABORIST) --db $(GIT_SHARD) ingest --source git_repo --path $(GIT_REPO)
|
||
|
||
# Generic hg-repo ingest. HG_REPO=<path>.
|
||
HG_REPO ?=
|
||
HG_SHARD := $(SHARDS_DIR)/$(notdir $(HG_REPO))-hg.db
|
||
ingest-hg: bootstrap ## ingest HG_REPO=<path> (mercurial) into its own shard
|
||
@if [ -z "$(HG_REPO)" ]; then echo "usage: make ingest-hg HG_REPO=/path/to/repo" >&2; exit 2; fi
|
||
@mkdir -p $(SHARDS_DIR)
|
||
$(ABORIST) --db $(HG_SHARD) ingest --source hg_repo --path $(HG_REPO)
|
||
|
||
verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents
|
||
$(ABORIST) --db $(DB) verify -n $(VERIFY_N)
|
||
|
||
search: bootstrap ## keyword search; override SEARCH_Q (or pass Q=...)
|
||
$(ABORIST) --db $(DB) search '$(if $(Q),$(Q),$(SEARCH_Q))'
|
||
|
||
stats: bootstrap ## counts: documents, chunks, edges, audit chain
|
||
$(ABORIST) --db $(DB) stats
|
||
|
||
test: bootstrap ## run pytest suite (excludes opt-in crawler tests)
|
||
$(VENV)/bin/pytest -q --ignore=tests/crawler
|
||
|
||
# Crawler tests are off-by-default — they hit the network in many cases
|
||
# and require the heavy [crawler] extras (aiohttp, bs4, lxml, etc.).
|
||
# Bootstrap installs the extras into the existing venv idempotently.
|
||
bootstrap-crawler: bootstrap ## install [crawler] extras into the venv
|
||
$(PIP) install -e '.[crawler]'
|
||
|
||
test-crawler: bootstrap-crawler ## run only the lifted crawler tests
|
||
$(VENV)/bin/pytest -q tests/crawler
|
||
|
||
# Bridge target: BFS a seed URL, ingest discovered pages into a shard,
|
||
# capture ETag/Last-Modified for each so recrawl-check can do conditional
|
||
# HEAD requests later. URL is required; DEPTH and MAX have safe defaults.
|
||
CRAWL_DEPTH ?= 2
|
||
CRAWL_MAX ?= 0
|
||
# Per-domain shard so `make query` (which reads $(SHARDS_DIR)) sees the
|
||
# crawled content. Shard filename derived from the seed URL's hostname:
|
||
# https://russell.ballestrini.net -> $(SHARDS_DIR)/crawl_russell_ballestrini_net.db
|
||
# Override with CRAWL_SHARD=... when you want a custom path.
|
||
crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] into $(SHARDS_DIR)/crawl_<domain>.db
|
||
@if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1 CRAWL_SHARD=path]" >&2; exit 2; fi
|
||
@mkdir -p $(SHARDS_DIR)
|
||
@shard="$(CRAWL_SHARD)"; \
|
||
if [ -z "$$shard" ]; then \
|
||
domain=$$(echo "$(URL)" | sed -E 's,^https?://([^/]+).*$$,\1,' | tr '.' '_'); \
|
||
shard="$(SHARDS_DIR)/crawl_$${domain}.db"; \
|
||
fi; \
|
||
echo " shard: $$shard" >&2; \
|
||
$(ABORIST) --db "$$shard" crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest
|
||
|
||
# Fast freshness probe: conditional HEAD per doc, classify fresh/stale/gone.
|
||
# Send only If-None-Match + If-Modified-Since headers — server returns 304
|
||
# with no body when content unchanged.
|
||
RECRAWL_LIMIT ?= 100
|
||
# By default check every shard under $(SHARDS_DIR). Pass CRAWL_SHARD=path
|
||
# to scope to one. DOMAIN= filters to URLs containing the substring.
|
||
recrawl-check: bootstrap-crawler ## conditional HEAD per ingested doc [DOMAIN=x.com LIMIT=100 CRAWL_SHARD=path]
|
||
@if [ -n "$(CRAWL_SHARD)" ]; then \
|
||
$(ABORIST) --db $(CRAWL_SHARD) crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \
|
||
else \
|
||
for db in $(SHARDS_DIR)/*.db; do \
|
||
case "$$(basename $$db)" in qa.db|snapshots.db) continue;; esac; \
|
||
echo " shard: $$db" >&2; \
|
||
$(ABORIST) --db "$$db" crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \
|
||
done; \
|
||
fi
|
||
|
||
DOT_SRCS := $(wildcard docs/diagrams/*.dot)
|
||
DOT_PNGS := $(DOT_SRCS:.dot=.png)
|
||
|
||
docs/diagrams/%.png: docs/diagrams/%.dot
|
||
dot -Tpng $< -o $@
|
||
|
||
docs: $(DOT_PNGS) ## render docs/diagrams/*.dot -> .png via graphviz
|
||
|
||
# Reproducible micro-benchmark over a fixed slice of cur. Lets you compare
|
||
# ETL throughput across configs and catches regressions on optimization
|
||
# work. Override BENCH_DOCS=N (default 5000).
|
||
BENCH_DOCS ?= 5000
|
||
BENCH_DIR := /tmp/aborist-bench
|
||
bench: bootstrap fetch-cur ## benchmark serial vs parallel-shared vs attached at $(BENCH_DOCS) docs
|
||
@bash bench/run.sh $(BENCH_DOCS)
|
||
|
||
clean: ## remove venv + caches (keeps fetched data and db)
|
||
rm -rf $(VENV) .pytest_cache **/__pycache__ aborist.egg-info
|
||
find . -type d -name __pycache__ -prune -exec rm -rf {} +
|
||
|
||
clean-db: ## drop the aborist db (keeps fetched data and venv)
|
||
rm -f $(DB) $(DB)-journal $(DB)-wal $(DB)-shm
|
||
|
||
clean-data: ## remove fetched dumps
|
||
rm -rf $(DATA_DIR)
|