engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

prepare full Wikipedia 2003-05-16 ingest: cur + old (revisions)

Browse source 
The 2003-05-16 archive ships three files:
  20030516_cur_tablesql.bz2   82 MB  current snapshot (single revision/page)
  old_tablesqlbz2.1          640 MiB \
  old_tablesqlbz2.2          252 MiB / split halves of old (full revision
                                       history). Concatenate before bzcat.

Generalize the parser:
  WikipediaSqlDump(table='cur'|'old')  — shared statement parser, single
                                         column-position contract for the
                                         first 4 fields (id/ns/title/text)
  WikipediaCurDump  — back-compat wrapper, table='cur'
  WikipediaOldDump  — new, table='old'; old has no is_redirect, every
                      revision is real

Old rows surface old_id and old_timestamp via Document.extra so a
downstream pass can sort revisions chronologically before re-ingesting
through the supersedes-edge path.

Makefile gains:
  fetch-cur / fetch-old / fetch (both)
  ingest-cur / ingest-old / ingest (cur default)
  WP_OLD target concatenates the two split parts
CLI ingest --source now accepts wikipedia_cur or wikipedia_old.

Smoke (real dump): 5 revisions of "AtlasShrugged/Companies" yielded
correctly with old_id=2..10, timestamps from January 2002.
This commit is contained in:
russell@unturf.com 2026-04-27 08:10:42 -04:00
parent 57cf1b183b
commit a056acfb5b
No known key found for this signature in database
5 changed files with 221 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

53
Makefile
View file

@ -11,8 +11,13 @@ ABORIST := $(VENV)/bin/aborist
# Data + DB
DATA_DIR ?= data
WP_DUMP_URL ?= https://dumps.wikimedia.org/archive/2003/2003-05-16/en/20030516_cur_tablesql.bz2
WP_DUMP := $(DATA_DIR)/20030516_cur_tablesql.bz2
WP_BASE_URL ?= https://dumps.wikimedia.org/archive/2003/2003-05-16/en
WP_CUR := $(DATA_DIR)/20030516_cur_tablesql.bz2
WP_OLD_1 := $(DATA_DIR)/old_tablesqlbz2.1
WP_OLD_2 := $(DATA_DIR)/old_tablesqlbz2.2
WP_OLD := $(DATA_DIR)/20030516_old_tablesql.bz2
# Back-compat alias (older callers used WP_DUMP for the cur snapshot).
WP_DUMP := $(WP_CUR)
DB ?= $(HOME)/.aborist/aborist.db
# Smoke-test caps so make all stays fast
@ -20,12 +25,13 @@ INGEST_LIMIT ?= 500
VERIFY_N ?= 10
SEARCH_Q ?= computer
.PHONY: all bootstrap fetch ingest verify search stats test clean clean-db clean-data help
.PHONY: all bootstrap fetch fetch-cur fetch-old ingest ingest-cur ingest-old \
verify search stats test clean clean-db clean-data help
all: bootstrap fetch ingest verify stats ## bootstrap → fetch → ingest → verify → stats
all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats
help: ## show this help
@awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " %-14s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " %-16s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
$(VENV)/bin/activate: pyproject.toml
$(PYTHON) -m venv $(VENV)
@ -35,17 +41,40 @@ $(VENV)/bin/activate: pyproject.toml
bootstrap: $(VENV)/bin/activate ## create venv and install editable package
$(WP_DUMP): | $(DATA_DIR)
@echo ">> fetching $(WP_DUMP_URL)"
curl -fL --retry 3 -o $@ "$(WP_DUMP_URL)"
$(DATA_DIR):
mkdir -p $(DATA_DIR)
fetch: $(WP_DUMP) ## download Wikipedia 2003-05-16 cur dump (cached)
$(WP_CUR): | $(DATA_DIR)
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
ingest: bootstrap fetch ## ingest INGEST_LIMIT articles into $(DB)
$(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_DUMP) --limit $(INGEST_LIMIT)
$(WP_OLD_1): | $(DATA_DIR)
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
$(WP_OLD_2): | $(DATA_DIR)
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
# The two old_tablesqlbz2.{1,2} parts are split halves of a single bzip2
# stream (.1 is exactly 640 MiB). Concatenate to get a working bz2 file.
$(WP_OLD): $(WP_OLD_1) $(WP_OLD_2)
@echo ">> concatenating old dump parts"
cat $(WP_OLD_1) $(WP_OLD_2) > $@
fetch-cur: $(WP_CUR) ## download cur table dump (~82 MB)
fetch-old: $(WP_OLD) ## download old (revision history) parts and concatenate (~893 MB)
fetch: fetch-cur fetch-old ## download all 3 files (cur + old.1 + old.2 + concat)
ingest-cur: bootstrap fetch-cur ## ingest INGEST_LIMIT cur articles
$(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) --limit $(INGEST_LIMIT)
ingest-old: bootstrap fetch-old ## ingest INGEST_LIMIT old (history) revisions
$(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) --limit $(INGEST_LIMIT)
ingest: ingest-cur ## default ingest = cur (use ingest-old or both for full)
verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents
$(ABORIST) --db $(DB) verify -n $(VERIFY_N)

11
aborist/cli.py
View file

@ -15,11 +15,14 @@ from aborist.store import DEFAULT_DB_PATH, connect, stats
def _cmd_ingest(args: argparse.Namespace) -> int:
if args.source == "wikipedia_cur":
if args.source in ("wikipedia_cur", "wikipedia_old"):
if not args.path:
print("--path is required for wikipedia_cur", file=sys.stderr)
print(f"--path is required for {args.source}", file=sys.stderr)
return 2
src = WikipediaCurDump(path=args.path)
from aborist.sources import WikipediaSqlDump
table = "cur" if args.source == "wikipedia_cur" else "old"
src = WikipediaSqlDump(path=args.path, table=table)
elif args.source == "html":
try:
from aborist.sources import HtmlPageSource
@ -283,7 +286,7 @@ def build_parser() -> argparse.ArgumentParser:
ingest.add_argument(
"--source",
required=True,
choices=["wikipedia_cur", "html"],
choices=["wikipedia_cur", "wikipedia_old", "html"],
help="source type",
)
ingest.add_argument("--path", help="path to dump file (file-backed sources)")

16
aborist/sources/__init__.py
View file

@ -1,8 +1,20 @@
"""Source implementations. Add a new corpus = add a new module here."""
from aborist.sources.wikipedia import WikipediaCurDump
from aborist.sources.wikipedia import (
WikipediaCurDump,
WikipediaOldDump,
WikipediaSqlDump,
)
__all__ = ["WikipediaCurDump"]
__all__ = ["WikipediaCurDump", "WikipediaOldDump", "WikipediaSqlDump"]
# HtmlPageSource has optional deps (selectolax). Surface import errors only
# when callers actually request it.
try:
from aborist.sources.html_page import HtmlPageSource # noqa: F401
__all__.append("HtmlPageSource")
except ImportError: # pragma: no cover
pass
# HtmlPageSource has optional deps (httpx + selectolax). Surface import errors
# only when callers actually request it.

76
aborist/sources/wikipedia.py
View file

@ -156,20 +156,30 @@ def _extract_wikilinks(text: str, base_uri: str) -> list[Edge]:
return edges
class WikipediaCurDump(Source):
"""Iterates a MediaWiki 'cur' table SQL dump (bz2-compressed)."""
class WikipediaSqlDump(Source):
"""Iterates a MediaWiki SQL table dump (cur or old), bz2 or plain.
source_type = "wikipedia_cur"
Both `cur` (current snapshot) and `old` (revision history) tables share
the first four column positions: id, namespace, title, text. The `cur`
table has `cur_is_redirect` at position 10 (we skip redirects); `old`
has no redirect flag (every revision is real).
"""
def __init__(
self,
path: str | Path,
*,
table: str = "cur",
namespace: int = NAMESPACE_MAIN,
base_uri: str = "https://en.wikipedia.org/wiki/",
):
if table not in ("cur", "old"):
raise ValueError("table must be 'cur' or 'old'")
self.path = Path(path)
self.table = table
self.namespace = namespace
self.base_uri = base_uri
self.source_type = f"wikipedia_{table}"
def iter_documents(self) -> Iterator[Document]:
opener = bz2.open if str(self.path).endswith(".bz2") else open
@ -183,7 +193,10 @@ class WikipediaCurDump(Source):
if vidx < 0:
continue
table_clause = head[:vidx].lower()
if "cur" not in table_clause:
# Whitespace-bounded match: "INSERT INTO cur" must not also
# accept "INSERT INTO cursor" or similar.
token = f" {self.table} "
if token not in (table_clause + " "):
continue
payload = head[vidx + len("VALUES"):]
payload = payload.rstrip().rstrip(";").rstrip()
@ -203,21 +216,58 @@ class WikipediaCurDump(Source):
text = row[3] or ""
if not title or not text:
return
# is_redirect lives at position 10 in the standard 2003 schema; be defensive.
is_redirect = False
if len(row) > 10 and row[10] is not None:
try:
is_redirect = bool(int(row[10]))
except (ValueError, TypeError):
is_redirect = False
if is_redirect:
return
# Cur-only: skip rows flagged as redirects (col 10 in the 2003 schema).
if self.table == "cur":
is_redirect = False
if len(row) > 10 and row[10] is not None:
try:
is_redirect = bool(int(row[10]))
except (ValueError, TypeError):
is_redirect = False
if is_redirect:
return
edges = _extract_wikilinks(text, self.base_uri)
uri = self.base_uri + title.replace(" ", "_")
extra: dict = {}
# `old` rows carry timestamp at position 7 — surface it for chronological
# ingest order downstream.
if self.table == "old" and len(row) > 7 and row[7]:
extra["old_timestamp"] = row[7]
extra["old_id"] = row[0]
yield Document(
uri=uri,
content=text,
source_type=self.source_type,
title=title,
edges=edges,
extra=extra,
)
# Backward-compatible thin wrapper.
class WikipediaCurDump(WikipediaSqlDump):
"""Iterates a MediaWiki 'cur' table SQL dump."""
def __init__(
self,
path: str | Path,
namespace: int = NAMESPACE_MAIN,
base_uri: str = "https://en.wikipedia.org/wiki/",
):
super().__init__(
path, table="cur", namespace=namespace, base_uri=base_uri
)
class WikipediaOldDump(WikipediaSqlDump):
"""Iterates a MediaWiki 'old' (revision history) table SQL dump."""
def __init__(
self,
path: str | Path,
namespace: int = NAMESPACE_MAIN,
base_uri: str = "https://en.wikipedia.org/wiki/",
):
super().__init__(
path, table="old", namespace=namespace, base_uri=base_uri
)

96
tests/test_wikipedia_old.py Normal file
View file

@ -0,0 +1,96 @@
"""WikipediaSqlDump on the 'old' (revision history) table."""
from __future__ import annotations
import bz2
from pathlib import Path
import pytest
from aborist.sources import WikipediaOldDump, WikipediaSqlDump
# Minimal fabricated 'old' table dump matching the 2003-05-16 schema.
# Columns: old_id, old_namespace, old_title, old_text, old_comment,
# old_user, old_user_text, old_timestamp, old_minor_edit, old_flags,
# inverse_timestamp
_SAMPLE_SQL = """\
-- MySQL dump
DROP TABLE IF EXISTS old;
CREATE TABLE old (
old_id int(8) unsigned NOT NULL auto_increment
);
INSERT INTO old VALUES (1, 0, 'Anarchism', 'first revision text [[liberty]] [[freedom]]', 'init', 0, 'alice', '20020101000000', 0, '', '79979898'),(2, 0, 'Anarchism', 'second revision text with [[autonomy]]', 'edit', 0, 'bob', '20020201000000', 0, '', '79979897'),(3, 1, 'Talk:Anarchism', 'talk page (skipped: namespace 1)', 'tk', 0, 'carol', '20020301000000', 0, '', '79979896');
"""
def test_old_dump_yields_revisions(tmp_path):
p = tmp_path / "fake_old.sql.bz2"
with bz2.open(p, "wt", encoding="utf-8") as f:
f.write(_SAMPLE_SQL)
src = WikipediaOldDump(path=p)
docs = list(src.iter_documents())
# Talk:Anarchism (namespace 1) is filtered; main namespace yields 2 revisions.
assert len(docs) == 2
# Revision 1
assert docs[0].title == "Anarchism"
assert docs[0].source_type == "wikipedia_old"
assert "first revision" in docs[0].content
assert docs[0].extra["old_timestamp"] == "20020101000000"
assert docs[0].extra["old_id"] == "1"
assert any(e.dst_uri.endswith("liberty") for e in docs[0].edges)
# Revision 2 — same title, different content, different timestamp.
assert docs[1].title == "Anarchism"
assert "second revision" in docs[1].content
assert docs[1].extra["old_timestamp"] == "20020201000000"
assert docs[1].extra["old_id"] == "2"
def test_cur_and_old_share_uri_namespace(tmp_path):
"""Both tables produce the same URI for the same article title."""
cur_sql = (
"INSERT INTO cur VALUES "
"(7, 0, 'Anarchism', 'cur revision text', '', 0, 'sys', '20030516000000', "
"'', 0, 0, 0, 0, 0, '', '');\n"
)
old_sql = (
"INSERT INTO old VALUES "
"(11, 0, 'Anarchism', 'old revision text', '', 0, 'sys', "
"'20020101000000', 0, '', '');\n"
)
cur_path = tmp_path / "c.sql.bz2"
old_path = tmp_path / "o.sql.bz2"
with bz2.open(cur_path, "wt", encoding="utf-8") as f:
f.write(cur_sql)
with bz2.open(old_path, "wt", encoding="utf-8") as f:
f.write(old_sql)
cur_doc = next(WikipediaSqlDump(cur_path, table="cur").iter_documents())
old_doc = next(WikipediaSqlDump(old_path, table="old").iter_documents())
assert cur_doc.uri == old_doc.uri
assert cur_doc.source_type == "wikipedia_cur"
assert old_doc.source_type == "wikipedia_old"
# But content differs -> different document_root.
assert cur_doc.content != old_doc.content
def test_real_dump_smoke():
"""If the real concatenated old dump exists locally, parse the first few rows."""
real = Path("data/20030516_old_tablesql.bz2")
if not real.exists():
pytest.skip("real old dump not fetched; run 'make fetch-old'")
src = WikipediaOldDump(path=real)
docs: list = []
for d in src.iter_documents():
docs.append(d)
if len(docs) >= 3:
break
assert len(docs) == 3
for d in docs:
assert d.title
assert d.content
assert d.extra.get("old_id")
assert d.extra.get("old_timestamp")