engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add Phase IV Wikipedia XML + abstract sources

Browse source 
Drops in for any dated snapshot in dumps.wikimedia.org/archive — pages-articles.xml.bz2
and pages-meta-current.xml.bz2 (cur snapshot, single-revision-per-page) plus
pages-meta-history.xml.bz2 (multi_revision=True). abstract.xml feed yields
pre-distilled summaries at ~1/100th the chunk volume. Streams .bz2/.gz directly
via iterparse with bounded memory; same shard / resume contract as the SQL
source. Falls back to title-prefix namespace filtering when older export
schemas omit the per-page <ns> element (e.g. enwiki 20101011).

Makefile defaults target enwiki 20101011 (6.2 GB). Override with WP_XML_YEAR /
WP_XML_MONTH / WP_XML_DATE / WP_XML_LANG to fetch any other archived snapshot.
This commit is contained in:
russell@unturf.com 2026-04-27 17:24:28 -04:00
parent 5cf76dbc8f
commit f90b7c69f0
No known key found for this signature in database
5 changed files with 661 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

56
Makefile
View file

@ -25,7 +25,9 @@ INGEST_LIMIT ?= 500
VERIFY_N ?= 10
SEARCH_Q ?= computer
.PHONY: all bootstrap fetch fetch-cur fetch-old ingest ingest-cur ingest-old \
.PHONY: all bootstrap fetch fetch-cur fetch-old fetch-xml fetch-abstract \
ingest ingest-cur ingest-old ingest-xml ingest-xml-history \
ingest-xml-attached ingest-abstract \
ingest-grok ingest-grok-media \
verify search stats test clean clean-db clean-data help
@ -184,6 +186,58 @@ ingest-grok-media-attached: bootstrap ## ingest Grok media prompts into $(GROK_S
@mkdir -p $(SHARDS_DIR)
$(ABORIST) --db $(GROK_SHARD) ingest --source grok_media --path $(GROK_EXPORT) --resume
# ----------------------------------------------------------------------------
# Phase IV (2006+) Wikipedia XML dumps. Drop-in for any dated snapshot in
# the dumps.wikimedia.org/archive tree by overriding WP_XML_YEAR/MONTH/DATE
# (and WP_XML_LANG for non-English wikis).
#
# Defaults point at enwiki 20101011 (2010-11 archive), the largest snapshot
# in the archive — 6.2 GB compressed, ~3.4M articles.
# Other useful snapshots:
# make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-07 WP_XML_DATE=20061104
# make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-12 WP_XML_DATE=20061130
# ----------------------------------------------------------------------------
WP_XML_LANG ?= en
WP_XML_YEAR ?= 2010
WP_XML_MONTH ?= 2010-11
WP_XML_DATE ?= 20101011
WP_XML_BASE_URL ?= https://dumps.wikimedia.org/archive/$(WP_XML_YEAR)/$(WP_XML_MONTH)/$(WP_XML_LANG)wiki/$(WP_XML_DATE)
WP_XML_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-pages-articles.xml.bz2
WP_XML := $(DATA_DIR)/$(WP_XML_FILE)
WP_ABSTRACT_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-abstract.xml
WP_ABSTRACT := $(DATA_DIR)/$(WP_ABSTRACT_FILE)
$(WP_XML): | $(DATA_DIR)
@echo ">> fetching $(WP_XML_BASE_URL)/$(WP_XML_FILE)"
curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_XML_FILE)"
$(WP_ABSTRACT): | $(DATA_DIR)
@echo ">> fetching $(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)"
curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)"
fetch-xml: $(WP_XML) ## download Phase IV XML cur dump (default: enwiki 20101011, 6.2 GB)
fetch-abstract: $(WP_ABSTRACT) ## download Phase IV abstract.xml (default: enwiki 20101011, ~3 GB)
ingest-xml: bootstrap fetch-xml ## ingest INGEST_LIMIT pages from $(WP_XML)
$(ABORIST) --db $(DB) ingest --source wikipedia_xml --path $(WP_XML) --limit $(INGEST_LIMIT)
ingest-xml-history: bootstrap ## ingest every revision (multi-revision mode); set WP_XML to a pages-meta-history file
$(ABORIST) --db $(DB) ingest --source wikipedia_xml_history --path $(WP_XML) --limit $(INGEST_LIMIT)
# Sharded XML ingest into the attach-forever cluster — same pattern as
# ingest-cur-attached. One process per shard, one SQLite file per shard,
# zero WAL contention.
ingest-xml-attached: bootstrap fetch-xml ## sharded XML ingest, one process per shard
@mkdir -p $(SHARDS_DIR)
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
$(ABORIST) ingest --source wikipedia_xml --path $(WP_XML) \
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
done; wait
ingest-abstract: bootstrap fetch-abstract ## ingest INGEST_LIMIT abstract docs from $(WP_ABSTRACT)
$(ABORIST) --db $(DB) ingest --source wikipedia_abstract --path $(WP_ABSTRACT) --limit $(INGEST_LIMIT)
verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents
$(ABORIST) --db $(DB) verify -n $(VERIFY_N)