From f90b7c69f06235dfca232198fb904e839dc4ba99 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 27 Apr 2026 17:24:28 -0400 Subject: [PATCH] add Phase IV Wikipedia XML + abstract sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops in for any dated snapshot in dumps.wikimedia.org/archive — pages-articles.xml.bz2 and pages-meta-current.xml.bz2 (cur snapshot, single-revision-per-page) plus pages-meta-history.xml.bz2 (multi_revision=True). abstract.xml feed yields pre-distilled summaries at ~1/100th the chunk volume. Streams .bz2/.gz directly via iterparse with bounded memory; same shard / resume contract as the SQL source. Falls back to title-prefix namespace filtering when older export schemas omit the per-page element (e.g. enwiki 20101011). Makefile defaults target enwiki 20101011 (6.2 GB). Override with WP_XML_YEAR / WP_XML_MONTH / WP_XML_DATE / WP_XML_LANG to fetch any other archived snapshot. --- Makefile | 56 +++++- aborist/cli.py | 21 ++ aborist/sources/__init__.py | 3 + aborist/sources/wikipedia_xml.py | 320 +++++++++++++++++++++++++++++++ tests/test_wikipedia_xml.py | 262 +++++++++++++++++++++++++ 5 files changed, 661 insertions(+), 1 deletion(-) create mode 100644 aborist/sources/wikipedia_xml.py create mode 100644 tests/test_wikipedia_xml.py diff --git a/Makefile b/Makefile index 9b9e30c..432770b 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,9 @@ INGEST_LIMIT ?= 500 VERIFY_N ?= 10 SEARCH_Q ?= computer -.PHONY: all bootstrap fetch fetch-cur fetch-old ingest ingest-cur ingest-old \ +.PHONY: all bootstrap fetch fetch-cur fetch-old fetch-xml fetch-abstract \ + ingest ingest-cur ingest-old ingest-xml ingest-xml-history \ + ingest-xml-attached ingest-abstract \ ingest-grok ingest-grok-media \ verify search stats test clean clean-db clean-data help @@ -184,6 +186,58 @@ ingest-grok-media-attached: bootstrap ## ingest Grok media prompts into $(GROK_S @mkdir -p $(SHARDS_DIR) $(ABORIST) --db $(GROK_SHARD) ingest --source grok_media --path $(GROK_EXPORT) --resume +# ---------------------------------------------------------------------------- +# Phase IV (2006+) Wikipedia XML dumps. Drop-in for any dated snapshot in +# the dumps.wikimedia.org/archive tree by overriding WP_XML_YEAR/MONTH/DATE +# (and WP_XML_LANG for non-English wikis). +# +# Defaults point at enwiki 20101011 (2010-11 archive), the largest snapshot +# in the archive — 6.2 GB compressed, ~3.4M articles. +# Other useful snapshots: +# make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-07 WP_XML_DATE=20061104 +# make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-12 WP_XML_DATE=20061130 +# ---------------------------------------------------------------------------- +WP_XML_LANG ?= en +WP_XML_YEAR ?= 2010 +WP_XML_MONTH ?= 2010-11 +WP_XML_DATE ?= 20101011 +WP_XML_BASE_URL ?= https://dumps.wikimedia.org/archive/$(WP_XML_YEAR)/$(WP_XML_MONTH)/$(WP_XML_LANG)wiki/$(WP_XML_DATE) +WP_XML_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-pages-articles.xml.bz2 +WP_XML := $(DATA_DIR)/$(WP_XML_FILE) +WP_ABSTRACT_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-abstract.xml +WP_ABSTRACT := $(DATA_DIR)/$(WP_ABSTRACT_FILE) + +$(WP_XML): | $(DATA_DIR) + @echo ">> fetching $(WP_XML_BASE_URL)/$(WP_XML_FILE)" + curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_XML_FILE)" + +$(WP_ABSTRACT): | $(DATA_DIR) + @echo ">> fetching $(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)" + curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)" + +fetch-xml: $(WP_XML) ## download Phase IV XML cur dump (default: enwiki 20101011, 6.2 GB) + +fetch-abstract: $(WP_ABSTRACT) ## download Phase IV abstract.xml (default: enwiki 20101011, ~3 GB) + +ingest-xml: bootstrap fetch-xml ## ingest INGEST_LIMIT pages from $(WP_XML) + $(ABORIST) --db $(DB) ingest --source wikipedia_xml --path $(WP_XML) --limit $(INGEST_LIMIT) + +ingest-xml-history: bootstrap ## ingest every revision (multi-revision mode); set WP_XML to a pages-meta-history file + $(ABORIST) --db $(DB) ingest --source wikipedia_xml_history --path $(WP_XML) --limit $(INGEST_LIMIT) + +# Sharded XML ingest into the attach-forever cluster — same pattern as +# ingest-cur-attached. One process per shard, one SQLite file per shard, +# zero WAL contention. +ingest-xml-attached: bootstrap fetch-xml ## sharded XML ingest, one process per shard + @mkdir -p $(SHARDS_DIR) + @for i in $$(seq 0 $$(($(SHARDS) - 1))); do \ + $(ABORIST) ingest --source wikipedia_xml --path $(WP_XML) \ + --shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \ + done; wait + +ingest-abstract: bootstrap fetch-abstract ## ingest INGEST_LIMIT abstract docs from $(WP_ABSTRACT) + $(ABORIST) --db $(DB) ingest --source wikipedia_abstract --path $(WP_ABSTRACT) --limit $(INGEST_LIMIT) + verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents $(ABORIST) --db $(DB) verify -n $(VERIFY_N) diff --git a/aborist/cli.py b/aborist/cli.py index 232a9e5..750c102 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -57,6 +57,24 @@ def _cmd_ingest(args: argparse.Namespace) -> int: cls = GrokExportSource if args.source == "grok_export" else GrokMediaPostsSource src = cls(path=args.path) + elif args.source in ("wikipedia_xml", "wikipedia_xml_history", "wikipedia_abstract"): + if not args.path: + print(f"--path is required for {args.source}", file=sys.stderr) + return 2 + from aborist.sources import WikipediaAbstractDump, WikipediaXmlDump + + if args.source == "wikipedia_abstract": + src = WikipediaAbstractDump(path=args.path) + else: + shard = None + if args.shard: + rank_str, total_str = args.shard.split("/", 1) + shard = (int(rank_str), int(total_str)) + src = WikipediaXmlDump( + path=args.path, + shard=shard, + multi_revision=(args.source == "wikipedia_xml_history"), + ) else: print(f"unknown source: {args.source}", file=sys.stderr) return 2 @@ -881,6 +899,9 @@ def build_parser() -> argparse.ArgumentParser: choices=[ "wikipedia_cur", "wikipedia_old", + "wikipedia_xml", + "wikipedia_xml_history", + "wikipedia_abstract", "html", "grok_export", "grok_media", diff --git a/aborist/sources/__init__.py b/aborist/sources/__init__.py index 0d30a5a..e91da94 100644 --- a/aborist/sources/__init__.py +++ b/aborist/sources/__init__.py @@ -6,13 +6,16 @@ from aborist.sources.wikipedia import ( WikipediaOldDump, WikipediaSqlDump, ) +from aborist.sources.wikipedia_xml import WikipediaAbstractDump, WikipediaXmlDump __all__ = [ "GrokExportSource", "GrokMediaPostsSource", + "WikipediaAbstractDump", "WikipediaCurDump", "WikipediaOldDump", "WikipediaSqlDump", + "WikipediaXmlDump", ] # HtmlPageSource has optional deps (selectolax). Surface import errors only diff --git a/aborist/sources/wikipedia_xml.py b/aborist/sources/wikipedia_xml.py new file mode 100644 index 0000000..32d576b --- /dev/null +++ b/aborist/sources/wikipedia_xml.py @@ -0,0 +1,320 @@ +"""MediaWiki XML 'pages-articles' / 'pages-meta-current' / 'abstract' source. + +Phase IV (2006+) Wikipedia dumps switched from MySQL extended INSERT +syntax to a custom XML schema. These sources stream that XML and yield +Documents the same way the WikipediaSqlDump source yields rows from +the 2003 cur table — same URI scheme, same edge extraction, same +shard / resume contract — so downstream ingest, sharding, and resume +logic Just Works. + +Two source classes: + +* `WikipediaXmlDump` — pages-articles.xml.bz2 (cur snapshot, main NS) or + pages-meta-current.xml.bz2 (cur snapshot, all NS) or + pages-meta-history.xml.bz2 (full history; pass multi_revision=True). + One Document per page (or per revision in history mode). + +* `WikipediaAbstractDump` — abstract.xml (uncompressed). Each `` + has , <url>, <abstract>: short pre-distilled summaries of every + article. Useful as cheap retrieval targets for FTS5 queries. +""" + +from __future__ import annotations + +import bz2 +import gzip +import re +from pathlib import Path +from typing import IO, Iterator +from xml.etree.ElementTree import Element, iterparse + +from aborist.document import Document, Edge +from aborist.source import Source +from aborist.sources.wikipedia import _extract_wikilinks + + +# MediaWiki export schema declares an xmlns; ElementTree returns tag names +# as `{http://www.mediawiki.org/xml/export-0.10/}page`. Strip that prefix +# off so the rest of the parser doesn't have to care which schema version +# any given dump uses. +_NS_RE = re.compile(r"^\{[^}]+\}") + +# Body that begins with `#REDIRECT [[Target]]` is a redirect — skip. +# Real dumps also use the explicit <redirect title="..."/> element, but +# older or imported pages sometimes lack that and only carry the body +# directive; we belt-and-suspenders both. +_REDIRECT_RE = re.compile(r"\s*#REDIRECT\b", re.IGNORECASE) + +NAMESPACE_MAIN = 0 + + +# Title-prefix-encoded namespaces. Only used as a fallback filter when the +# export schema doesn't declare <ns> per page. Excludes Wiktionary-style +# language prefixes (those don't appear in enwiki main namespace anyway). +_NON_MAIN_NS_PREFIXES = frozenset({ + "Talk", "User", "User talk", "Wikipedia", "Wikipedia talk", + "File", "File talk", "MediaWiki", "MediaWiki talk", + "Template", "Template talk", "Help", "Help talk", + "Category", "Category talk", "Portal", "Portal talk", + "Book", "Book talk", "Draft", "Draft talk", + "Module", "Module talk", "Image", "Image talk", +}) + + +def _strip_ns(tag: str) -> str: + return _NS_RE.sub("", tag) + + +def _open_streaming(path: Path) -> IO[bytes]: + """Open a file as a binary byte stream, transparently decompressing.""" + p = str(path) + if p.endswith(".bz2"): + return bz2.open(p, "rb") + if p.endswith(".gz"): + return gzip.open(p, "rb") + return open(p, "rb") + + +class WikipediaXmlDump(Source): + """Iterates a MediaWiki XML dump. + + Phase IV format: `<mediawiki>` root containing one `<siteinfo>` plus + many `<page>` children. Each `<page>` has `<title>`, `<ns>`, `<id>`, + optional `<redirect>`, and one or more `<revision>` children. + + For pages-articles or pages-meta-current dumps, each page has exactly + one `<revision>` (the current one). For pages-meta-history dumps each + page has many. By default we emit the LAST revision per page (matches + cur semantics); pass `multi_revision=True` to emit every revision + (matches old semantics, with `supersedes` chaining at the URI level + via aborist's existing prior-document detection). + """ + + source_type = "wikipedia_xml" + + def __init__( + self, + path: str | Path, + *, + namespace: int = NAMESPACE_MAIN, + base_uri: str = "https://en.wikipedia.org/wiki/", + shard: tuple[int, int] | None = None, + start_id: int = 0, + multi_revision: bool = False, + ): + self.path = Path(path) + self.namespace = namespace + self.base_uri = base_uri + if shard is not None: + rank, total = shard + if not (0 <= rank < total) or total < 1: + raise ValueError( + f"invalid shard {shard}: need 0 <= rank < total, total >= 1" + ) + self.shard = shard + # Resume support — same contract as WikipediaSqlDump. + self.start_id = start_id + self.last_id: int = start_id + self.multi_revision = multi_revision + + def iter_documents(self) -> Iterator[Document]: + rank, total = (0, 1) if self.shard is None else self.shard + idx = 0 + with _open_streaming(self.path) as f: + context = iter(iterparse(f, events=("start", "end"))) + try: + _, root = next(context) + except StopIteration: + return + for event, elem in context: + if event != "end": + continue + tag = _strip_ns(elem.tag) + if tag != "page": + # Free siteinfo/etc immediately — no useful data downstream. + if tag in ("siteinfo", "namespaces", "namespace"): + elem.clear() + continue + # Stride filter applied at the page index. Every shard sees + # the full XML stream but only emits its share — matches the + # SQL source's behavior so ingest counters line up. + if idx % total != rank: + idx += 1 + elem.clear() + root.clear() + continue + idx += 1 + yield from self._page_to_docs(elem) + elem.clear() + root.clear() + + def _page_to_docs(self, page: Element) -> Iterator[Document]: + title: str | None = None + ns_text: str | None = None + page_id: int = 0 + is_redirect = False + revisions: list[Element] = [] + for child in page: + ctag = _strip_ns(child.tag) + if ctag == "title": + title = (child.text or "").strip() or None + elif ctag == "ns": + ns_text = (child.text or "").strip() + elif ctag == "id": + try: + page_id = int((child.text or "0").strip()) + except ValueError: + pass + elif ctag == "redirect": + # Self-closing element with title attr. Body may also start + # with #REDIRECT — both flag the same condition. + is_redirect = True + elif ctag == "revision": + revisions.append(child) + + try: + ns_int = int(ns_text) if ns_text is not None else None + except ValueError: + ns_int = None + # Older export schemas (pre-0.4) had no <ns> child element under + # <page>; namespace was encoded into the title prefix only. The + # `pages-articles.xml.bz2` dump is documented to contain just the + # main namespace plus redirects, so trust the file when <ns> is + # absent. For files that include sister namespaces (e.g. + # `pages-meta-current.xml.bz2`), filter on the title prefix as + # a safety belt — any title starting `Talk:` / `User:` / + # `Wikipedia:` / `File:` / `Template:` / `Help:` / `Category:` / + # `Portal:` / `Book:` / `Draft:` / `MediaWiki:` is non-main. + if ns_int is None: + if title and ":" in title: + head = title.split(":", 1)[0] + if head in _NON_MAIN_NS_PREFIXES: + return + elif ns_int != self.namespace: + return + if is_redirect: + return + if not title or not revisions: + return + + # Resume fast-forward. + if self.start_id and page_id <= self.start_id: + return + if page_id > self.last_id: + self.last_id = page_id + + revs = revisions if self.multi_revision else revisions[-1:] + uri = self.base_uri + title.replace(" ", "_") + for rev in revs: + text = "" + timestamp = "" + rev_id = 0 + for child in rev: + ctag = _strip_ns(child.tag) + if ctag == "text": + text = child.text or "" + elif ctag == "timestamp": + timestamp = (child.text or "").strip() + elif ctag == "id": + try: + rev_id = int((child.text or "0").strip()) + except ValueError: + pass + if not text: + continue + if _REDIRECT_RE.match(text): + continue + edges = _extract_wikilinks(text, self.base_uri) + extra: dict = {"page_id": str(page_id)} + if timestamp: + extra["timestamp"] = timestamp + if rev_id: + extra["rev_id"] = str(rev_id) + yield Document( + uri=uri, + content=text, + source_type=self.source_type, + title=title, + edges=edges, + extra=extra, + ) + + +class WikipediaAbstractDump(Source): + """Iterates a MediaWiki `abstract.xml` dump. + + Format: + <feed> + <doc> + <title>Wikipedia: Anarchism + http://en.wikipedia.org/wiki/Anarchism + Anarchism is a political philosophy... + ... + + ... + + + Each already contains a one-paragraph summary — pre-distilled + by the dump generator. Ingesting these alongside (or instead of) full + pages-articles gives FTS5 a high-signal corpus at ~1/100th the bytes. + """ + + source_type = "wikipedia_abstract" + + def __init__(self, path: str | Path): + self.path = Path(path) + # Resume contract: abstract docs have no stable numeric id, so + # start_id / last_id are no-ops here. Kept for interface symmetry. + self.start_id = 0 + self.last_id = 0 + + def iter_documents(self) -> Iterator[Document]: + with _open_streaming(self.path) as f: + context = iter(iterparse(f, events=("start", "end"))) + try: + _, root = next(context) + except StopIteration: + return + for event, elem in context: + if event != "end": + continue + tag = _strip_ns(elem.tag) + if tag != "doc": + continue + title = "" + url = "" + abstract = "" + links: list[str] = [] + for child in elem: + ctag = _strip_ns(child.tag) + if ctag == "title": + title = (child.text or "").strip() + elif ctag == "url": + url = (child.text or "").strip() + elif ctag == "abstract": + abstract = (child.text or "").strip() + elif ctag == "links": + for sublink in child: + for sub in sublink: + if _strip_ns(sub.tag) == "link": + href = (sub.text or "").strip() + if href: + links.append(href) + elem.clear() + root.clear() + if not url or not abstract: + continue + # Abstract feed prepends "Wikipedia: " to titles. Strip it. + clean_title = title + if clean_title.startswith("Wikipedia: "): + clean_title = clean_title[len("Wikipedia: "):] + edges = [ + Edge(edge_type="abstract_link", dst_uri=u) for u in links + ] + yield Document( + uri=url, + content=abstract, + source_type=self.source_type, + title=clean_title or None, + edges=edges, + ) diff --git a/tests/test_wikipedia_xml.py b/tests/test_wikipedia_xml.py new file mode 100644 index 0000000..346afc4 --- /dev/null +++ b/tests/test_wikipedia_xml.py @@ -0,0 +1,262 @@ +"""Tests for the MediaWiki XML and abstract dump sources. + +Synthetic XML fixtures match the Phase IV (2006+) export schema: + pages-articles.xml: <ns/><id/><revision/></page>... + abstract.xml: <feed><doc><title/><url/><abstract/></doc>... +No real dumps are downloaded by the test suite. +""" + +from __future__ import annotations + +import bz2 +from pathlib import Path + +from aborist.ingest import ingest_source +from aborist.sources.wikipedia_xml import ( + WikipediaAbstractDump, + WikipediaXmlDump, + _strip_ns, +) +from aborist.store import connect + + +PAGES_ARTICLES_XML = """<?xml version="1.0" encoding="UTF-8"?> +<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.10/" version="0.10" xml:lang="en"> + <siteinfo> + <sitename>Wikipedia</sitename> + <dbname>enwiki</dbname> + <base>http://en.wikipedia.org/wiki/Main_Page</base> + <generator>MediaWiki 1.36</generator> + </siteinfo> + <page> + <title>Anarchism + 0 + 12 + + 1234567 + 2010-11-08T13:00:00Z + '''Anarchism''' is a [[political philosophy]] that advocates [[stateless society|stateless societies]]. + + + + Foo Redirect + 0 + 13 + + + 1234568 + 2010-11-08T13:00:01Z + #REDIRECT [[Foo Target]] + + + + Talk:Anarchism + 1 + 14 + + 1234569 + 2010-11-08T13:00:02Z + Discussion about [[anarchism]]. + + + + Capitalism + 0 + 15 + + 2000001 + 2010-11-08T13:00:03Z + First version of capitalism article. + + + 2000002 + 2010-11-08T13:00:04Z + Second version of capitalism article. References [[markets]]. + + + + Body Redirect Without Marker + 0 + 16 + + 3000001 + 2010-11-08T13:00:05Z + #redirect [[Some Other Page]] + + +""" + + +ABSTRACT_XML = """ + + + Wikipedia: Anarchism + http://en.wikipedia.org/wiki/Anarchism + Anarchism is a political philosophy that advocates self-governed societies. + + + Origins + http://en.wikipedia.org/wiki/Anarchism#Origins + + + Schools + http://en.wikipedia.org/wiki/Anarchism#Schools + + + + + Wikipedia: Capitalism + http://en.wikipedia.org/wiki/Capitalism + Capitalism is an economic system based on private ownership. + + + Wikipedia: Empty + http://en.wikipedia.org/wiki/Empty + + +""" + + +def _write(path: Path, text: str) -> Path: + path.write_text(text, encoding="utf-8") + return path + + +def _write_bz2(path: Path, text: str) -> Path: + with bz2.open(path, "wt", encoding="utf-8") as f: + f.write(text) + return path + + +def test_strip_ns_removes_xmlns_braces(): + assert _strip_ns("{http://www.mediawiki.org/xml/export-0.10/}page") == "page" + assert _strip_ns("page") == "page" + + +def test_xml_dump_yields_main_namespace_pages_only(tmp_path): + src = WikipediaXmlDump(_write(tmp_path / "wp.xml", PAGES_ARTICLES_XML)) + docs = list(src.iter_documents()) + titles = [d.title for d in docs] + # Anarchism in, Capitalism in (last revision), Talk:Anarchism out (ns=1), + # Foo Redirect out (), Body Redirect Without Marker out (#REDIRECT body). + assert titles == ["Anarchism", "Capitalism"] + # URI shape matches the SQL source: spaces -> underscores, base prefix. + uris = [d.uri for d in docs] + assert uris == [ + "https://en.wikipedia.org/wiki/Anarchism", + "https://en.wikipedia.org/wiki/Capitalism", + ] + + +def test_xml_dump_extracts_wikilinks_as_edges(tmp_path): + src = WikipediaXmlDump(_write(tmp_path / "wp.xml", PAGES_ARTICLES_XML)) + docs = list(src.iter_documents()) + anarchism = next(d for d in docs if d.title == "Anarchism") + edge_uris = {e.dst_uri for e in anarchism.edges} + assert "https://en.wikipedia.org/wiki/political_philosophy" in edge_uris + # [[stateless society|stateless societies]] -> dst is target before pipe. + assert "https://en.wikipedia.org/wiki/stateless_society" in edge_uris + assert all(e.edge_type == "wikilink" for e in anarchism.edges) + + +def test_xml_dump_default_takes_last_revision_only(tmp_path): + src = WikipediaXmlDump(_write(tmp_path / "wp.xml", PAGES_ARTICLES_XML)) + docs = list(src.iter_documents()) + capitalism = next(d for d in docs if d.title == "Capitalism") + assert "Second version" in capitalism.content + assert "First version" not in capitalism.content + + +def test_xml_dump_multi_revision_yields_every_revision(tmp_path): + src = WikipediaXmlDump( + _write(tmp_path / "wp.xml", PAGES_ARTICLES_XML), + multi_revision=True, + ) + docs = list(src.iter_documents()) + capitalism_docs = [d for d in docs if d.title == "Capitalism"] + assert len(capitalism_docs) == 2 + contents = [d.content for d in capitalism_docs] + assert any("First version" in c for c in contents) + assert any("Second version" in c for c in contents) + + +def test_xml_dump_handles_bz2_streaming(tmp_path): + src = WikipediaXmlDump(_write_bz2(tmp_path / "wp.xml.bz2", PAGES_ARTICLES_XML)) + docs = list(src.iter_documents()) + assert {d.title for d in docs} == {"Anarchism", "Capitalism"} + + +def test_xml_dump_shard_strides_pages(tmp_path): + src_a = WikipediaXmlDump( + _write(tmp_path / "wp.xml", PAGES_ARTICLES_XML), shard=(0, 2) + ) + src_b = WikipediaXmlDump(tmp_path / "wp.xml", shard=(1, 2)) + a_titles = {d.title for d in src_a.iter_documents()} + b_titles = {d.title for d in src_b.iter_documents()} + # Stride is at the page index level (every page seen by every shard, + # filtered post-namespace). Together they must equal the full set. + assert a_titles | b_titles == {"Anarchism", "Capitalism"} + assert not (a_titles & b_titles) + + +def test_xml_dump_resume_skips_pages_at_or_below_start_id(tmp_path): + # First run: bring last_id to whatever it sees. + src = WikipediaXmlDump(_write(tmp_path / "wp.xml", PAGES_ARTICLES_XML)) + list(src.iter_documents()) + high_water = src.last_id + assert high_water > 0 + # Second run with start_id at high_water = no docs. + src2 = WikipediaXmlDump(tmp_path / "wp.xml", start_id=high_water) + assert list(src2.iter_documents()) == [] + + +def test_xml_dump_round_trip_into_sqlite(tmp_path): + src = WikipediaXmlDump(_write(tmp_path / "wp.xml", PAGES_ARTICLES_XML)) + db_path = tmp_path / "aborist.db" + conn = connect(db_path) + try: + stats = ingest_source(conn, src) + finally: + conn.close() + assert stats.inserted == 2 + # Re-ingest is idempotent. + conn = connect(db_path) + try: + again = ingest_source(conn, WikipediaXmlDump(tmp_path / "wp.xml")) + finally: + conn.close() + assert again.inserted == 0 + assert again.skipped_duplicate == 2 + + +def test_abstract_dump_yields_one_doc_per_nonempty_abstract(tmp_path): + src = WikipediaAbstractDump(_write(tmp_path / "abstract.xml", ABSTRACT_XML)) + docs = list(src.iter_documents()) + assert len(docs) == 2 # "Empty" doc has no abstract -> skipped + titles = [d.title for d in docs] + assert titles == ["Anarchism", "Capitalism"] + # Wikipedia: prefix stripped from titles. + assert all(not (t or "").startswith("Wikipedia:") for t in titles) + # URL preserved verbatim. + assert docs[0].uri == "http://en.wikipedia.org/wiki/Anarchism" + + +def test_abstract_dump_extracts_sublinks_as_edges(tmp_path): + src = WikipediaAbstractDump(_write(tmp_path / "abstract.xml", ABSTRACT_XML)) + docs = list(src.iter_documents()) + anarchism = next(d for d in docs if d.title == "Anarchism") + edge_uris = {e.dst_uri for e in anarchism.edges} + assert "http://en.wikipedia.org/wiki/Anarchism#Origins" in edge_uris + assert "http://en.wikipedia.org/wiki/Anarchism#Schools" in edge_uris + assert all(e.edge_type == "abstract_link" for e in anarchism.edges) + + +def test_abstract_dump_round_trip_into_sqlite(tmp_path): + src = WikipediaAbstractDump(_write(tmp_path / "abstract.xml", ABSTRACT_XML)) + db_path = tmp_path / "aborist.db" + conn = connect(db_path) + try: + stats = ingest_source(conn, src) + finally: + conn.close() + assert stats.inserted == 2