diff --git a/Makefile b/Makefile index 938be27..2f055aa 100644 --- a/Makefile +++ b/Makefile @@ -11,8 +11,13 @@ ABORIST := $(VENV)/bin/aborist # Data + DB DATA_DIR ?= data -WP_DUMP_URL ?= https://dumps.wikimedia.org/archive/2003/2003-05-16/en/20030516_cur_tablesql.bz2 -WP_DUMP := $(DATA_DIR)/20030516_cur_tablesql.bz2 +WP_BASE_URL ?= https://dumps.wikimedia.org/archive/2003/2003-05-16/en +WP_CUR := $(DATA_DIR)/20030516_cur_tablesql.bz2 +WP_OLD_1 := $(DATA_DIR)/old_tablesqlbz2.1 +WP_OLD_2 := $(DATA_DIR)/old_tablesqlbz2.2 +WP_OLD := $(DATA_DIR)/20030516_old_tablesql.bz2 +# Back-compat alias (older callers used WP_DUMP for the cur snapshot). +WP_DUMP := $(WP_CUR) DB ?= $(HOME)/.aborist/aborist.db # Smoke-test caps so make all stays fast @@ -20,12 +25,13 @@ INGEST_LIMIT ?= 500 VERIFY_N ?= 10 SEARCH_Q ?= computer -.PHONY: all bootstrap fetch ingest verify search stats test clean clean-db clean-data help +.PHONY: all bootstrap fetch fetch-cur fetch-old ingest ingest-cur ingest-old \ + verify search stats test clean clean-db clean-data help -all: bootstrap fetch ingest verify stats ## bootstrap → fetch → ingest → verify → stats +all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats help: ## show this help - @awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " %-14s %s\n", $$1, $$2}' $(MAKEFILE_LIST) + @awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " %-16s %s\n", $$1, $$2}' $(MAKEFILE_LIST) $(VENV)/bin/activate: pyproject.toml $(PYTHON) -m venv $(VENV) @@ -35,17 +41,40 @@ $(VENV)/bin/activate: pyproject.toml bootstrap: $(VENV)/bin/activate ## create venv and install editable package -$(WP_DUMP): | $(DATA_DIR) - @echo ">> fetching $(WP_DUMP_URL)" - curl -fL --retry 3 -o $@ "$(WP_DUMP_URL)" - $(DATA_DIR): mkdir -p $(DATA_DIR) -fetch: $(WP_DUMP) ## download Wikipedia 2003-05-16 cur dump (cached) +$(WP_CUR): | $(DATA_DIR) + @echo ">> fetching $(WP_BASE_URL)/$$(basename $@)" + curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)" -ingest: bootstrap fetch ## ingest INGEST_LIMIT articles into $(DB) - $(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_DUMP) --limit $(INGEST_LIMIT) +$(WP_OLD_1): | $(DATA_DIR) + @echo ">> fetching $(WP_BASE_URL)/$$(basename $@)" + curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)" + +$(WP_OLD_2): | $(DATA_DIR) + @echo ">> fetching $(WP_BASE_URL)/$$(basename $@)" + curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)" + +# The two old_tablesqlbz2.{1,2} parts are split halves of a single bzip2 +# stream (.1 is exactly 640 MiB). Concatenate to get a working bz2 file. +$(WP_OLD): $(WP_OLD_1) $(WP_OLD_2) + @echo ">> concatenating old dump parts" + cat $(WP_OLD_1) $(WP_OLD_2) > $@ + +fetch-cur: $(WP_CUR) ## download cur table dump (~82 MB) + +fetch-old: $(WP_OLD) ## download old (revision history) parts and concatenate (~893 MB) + +fetch: fetch-cur fetch-old ## download all 3 files (cur + old.1 + old.2 + concat) + +ingest-cur: bootstrap fetch-cur ## ingest INGEST_LIMIT cur articles + $(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) --limit $(INGEST_LIMIT) + +ingest-old: bootstrap fetch-old ## ingest INGEST_LIMIT old (history) revisions + $(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) --limit $(INGEST_LIMIT) + +ingest: ingest-cur ## default ingest = cur (use ingest-old or both for full) verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents $(ABORIST) --db $(DB) verify -n $(VERIFY_N) diff --git a/aborist/cli.py b/aborist/cli.py index 611021e..835a41a 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -15,11 +15,14 @@ from aborist.store import DEFAULT_DB_PATH, connect, stats def _cmd_ingest(args: argparse.Namespace) -> int: - if args.source == "wikipedia_cur": + if args.source in ("wikipedia_cur", "wikipedia_old"): if not args.path: - print("--path is required for wikipedia_cur", file=sys.stderr) + print(f"--path is required for {args.source}", file=sys.stderr) return 2 - src = WikipediaCurDump(path=args.path) + from aborist.sources import WikipediaSqlDump + + table = "cur" if args.source == "wikipedia_cur" else "old" + src = WikipediaSqlDump(path=args.path, table=table) elif args.source == "html": try: from aborist.sources import HtmlPageSource @@ -283,7 +286,7 @@ def build_parser() -> argparse.ArgumentParser: ingest.add_argument( "--source", required=True, - choices=["wikipedia_cur", "html"], + choices=["wikipedia_cur", "wikipedia_old", "html"], help="source type", ) ingest.add_argument("--path", help="path to dump file (file-backed sources)") diff --git a/aborist/sources/__init__.py b/aborist/sources/__init__.py index 0fa884e..e12bddc 100644 --- a/aborist/sources/__init__.py +++ b/aborist/sources/__init__.py @@ -1,8 +1,20 @@ """Source implementations. Add a new corpus = add a new module here.""" -from aborist.sources.wikipedia import WikipediaCurDump +from aborist.sources.wikipedia import ( + WikipediaCurDump, + WikipediaOldDump, + WikipediaSqlDump, +) -__all__ = ["WikipediaCurDump"] +__all__ = ["WikipediaCurDump", "WikipediaOldDump", "WikipediaSqlDump"] + +# HtmlPageSource has optional deps (selectolax). Surface import errors only +# when callers actually request it. +try: + from aborist.sources.html_page import HtmlPageSource # noqa: F401 + __all__.append("HtmlPageSource") +except ImportError: # pragma: no cover + pass # HtmlPageSource has optional deps (httpx + selectolax). Surface import errors # only when callers actually request it. diff --git a/aborist/sources/wikipedia.py b/aborist/sources/wikipedia.py index 9aee18b..ad2cc82 100644 --- a/aborist/sources/wikipedia.py +++ b/aborist/sources/wikipedia.py @@ -156,20 +156,30 @@ def _extract_wikilinks(text: str, base_uri: str) -> list[Edge]: return edges -class WikipediaCurDump(Source): - """Iterates a MediaWiki 'cur' table SQL dump (bz2-compressed).""" +class WikipediaSqlDump(Source): + """Iterates a MediaWiki SQL table dump (cur or old), bz2 or plain. - source_type = "wikipedia_cur" + Both `cur` (current snapshot) and `old` (revision history) tables share + the first four column positions: id, namespace, title, text. The `cur` + table has `cur_is_redirect` at position 10 (we skip redirects); `old` + has no redirect flag (every revision is real). + """ def __init__( self, path: str | Path, + *, + table: str = "cur", namespace: int = NAMESPACE_MAIN, base_uri: str = "https://en.wikipedia.org/wiki/", ): + if table not in ("cur", "old"): + raise ValueError("table must be 'cur' or 'old'") self.path = Path(path) + self.table = table self.namespace = namespace self.base_uri = base_uri + self.source_type = f"wikipedia_{table}" def iter_documents(self) -> Iterator[Document]: opener = bz2.open if str(self.path).endswith(".bz2") else open @@ -183,7 +193,10 @@ class WikipediaCurDump(Source): if vidx < 0: continue table_clause = head[:vidx].lower() - if "cur" not in table_clause: + # Whitespace-bounded match: "INSERT INTO cur" must not also + # accept "INSERT INTO cursor" or similar. + token = f" {self.table} " + if token not in (table_clause + " "): continue payload = head[vidx + len("VALUES"):] payload = payload.rstrip().rstrip(";").rstrip() @@ -203,21 +216,58 @@ class WikipediaCurDump(Source): text = row[3] or "" if not title or not text: return - # is_redirect lives at position 10 in the standard 2003 schema; be defensive. - is_redirect = False - if len(row) > 10 and row[10] is not None: - try: - is_redirect = bool(int(row[10])) - except (ValueError, TypeError): - is_redirect = False - if is_redirect: - return + # Cur-only: skip rows flagged as redirects (col 10 in the 2003 schema). + if self.table == "cur": + is_redirect = False + if len(row) > 10 and row[10] is not None: + try: + is_redirect = bool(int(row[10])) + except (ValueError, TypeError): + is_redirect = False + if is_redirect: + return edges = _extract_wikilinks(text, self.base_uri) uri = self.base_uri + title.replace(" ", "_") + extra: dict = {} + # `old` rows carry timestamp at position 7 — surface it for chronological + # ingest order downstream. + if self.table == "old" and len(row) > 7 and row[7]: + extra["old_timestamp"] = row[7] + extra["old_id"] = row[0] yield Document( uri=uri, content=text, source_type=self.source_type, title=title, edges=edges, + extra=extra, + ) + + +# Backward-compatible thin wrapper. +class WikipediaCurDump(WikipediaSqlDump): + """Iterates a MediaWiki 'cur' table SQL dump.""" + + def __init__( + self, + path: str | Path, + namespace: int = NAMESPACE_MAIN, + base_uri: str = "https://en.wikipedia.org/wiki/", + ): + super().__init__( + path, table="cur", namespace=namespace, base_uri=base_uri + ) + + +class WikipediaOldDump(WikipediaSqlDump): + """Iterates a MediaWiki 'old' (revision history) table SQL dump.""" + + def __init__( + self, + path: str | Path, + namespace: int = NAMESPACE_MAIN, + base_uri: str = "https://en.wikipedia.org/wiki/", + ): + super().__init__( + path, table="old", namespace=namespace, base_uri=base_uri ) diff --git a/tests/test_wikipedia_old.py b/tests/test_wikipedia_old.py new file mode 100644 index 0000000..5ca39aa --- /dev/null +++ b/tests/test_wikipedia_old.py @@ -0,0 +1,96 @@ +"""WikipediaSqlDump on the 'old' (revision history) table.""" + +from __future__ import annotations + +import bz2 +from pathlib import Path + +import pytest + +from aborist.sources import WikipediaOldDump, WikipediaSqlDump + + +# Minimal fabricated 'old' table dump matching the 2003-05-16 schema. +# Columns: old_id, old_namespace, old_title, old_text, old_comment, +# old_user, old_user_text, old_timestamp, old_minor_edit, old_flags, +# inverse_timestamp +_SAMPLE_SQL = """\ +-- MySQL dump +DROP TABLE IF EXISTS old; +CREATE TABLE old ( + old_id int(8) unsigned NOT NULL auto_increment +); +INSERT INTO old VALUES (1, 0, 'Anarchism', 'first revision text [[liberty]] [[freedom]]', 'init', 0, 'alice', '20020101000000', 0, '', '79979898'),(2, 0, 'Anarchism', 'second revision text with [[autonomy]]', 'edit', 0, 'bob', '20020201000000', 0, '', '79979897'),(3, 1, 'Talk:Anarchism', 'talk page (skipped: namespace 1)', 'tk', 0, 'carol', '20020301000000', 0, '', '79979896'); +""" + + +def test_old_dump_yields_revisions(tmp_path): + p = tmp_path / "fake_old.sql.bz2" + with bz2.open(p, "wt", encoding="utf-8") as f: + f.write(_SAMPLE_SQL) + + src = WikipediaOldDump(path=p) + docs = list(src.iter_documents()) + # Talk:Anarchism (namespace 1) is filtered; main namespace yields 2 revisions. + assert len(docs) == 2 + + # Revision 1 + assert docs[0].title == "Anarchism" + assert docs[0].source_type == "wikipedia_old" + assert "first revision" in docs[0].content + assert docs[0].extra["old_timestamp"] == "20020101000000" + assert docs[0].extra["old_id"] == "1" + assert any(e.dst_uri.endswith("liberty") for e in docs[0].edges) + + # Revision 2 — same title, different content, different timestamp. + assert docs[1].title == "Anarchism" + assert "second revision" in docs[1].content + assert docs[1].extra["old_timestamp"] == "20020201000000" + assert docs[1].extra["old_id"] == "2" + + +def test_cur_and_old_share_uri_namespace(tmp_path): + """Both tables produce the same URI for the same article title.""" + cur_sql = ( + "INSERT INTO cur VALUES " + "(7, 0, 'Anarchism', 'cur revision text', '', 0, 'sys', '20030516000000', " + "'', 0, 0, 0, 0, 0, '', '');\n" + ) + old_sql = ( + "INSERT INTO old VALUES " + "(11, 0, 'Anarchism', 'old revision text', '', 0, 'sys', " + "'20020101000000', 0, '', '');\n" + ) + cur_path = tmp_path / "c.sql.bz2" + old_path = tmp_path / "o.sql.bz2" + with bz2.open(cur_path, "wt", encoding="utf-8") as f: + f.write(cur_sql) + with bz2.open(old_path, "wt", encoding="utf-8") as f: + f.write(old_sql) + + cur_doc = next(WikipediaSqlDump(cur_path, table="cur").iter_documents()) + old_doc = next(WikipediaSqlDump(old_path, table="old").iter_documents()) + assert cur_doc.uri == old_doc.uri + assert cur_doc.source_type == "wikipedia_cur" + assert old_doc.source_type == "wikipedia_old" + # But content differs -> different document_root. + assert cur_doc.content != old_doc.content + + +def test_real_dump_smoke(): + """If the real concatenated old dump exists locally, parse the first few rows.""" + real = Path("data/20030516_old_tablesql.bz2") + if not real.exists(): + pytest.skip("real old dump not fetched; run 'make fetch-old'") + src = WikipediaOldDump(path=real) + docs: list = [] + for d in src.iter_documents(): + docs.append(d) + if len(docs) >= 3: + break + assert len(docs) == 3 + for d in docs: + assert d.title + assert d.content + assert d.extra.get("old_id") + assert d.extra.get("old_timestamp")