add Phase IV Wikipedia XML + abstract sources
Drops in for any dated snapshot in dumps.wikimedia.org/archive — pages-articles.xml.bz2 and pages-meta-current.xml.bz2 (cur snapshot, single-revision-per-page) plus pages-meta-history.xml.bz2 (multi_revision=True). abstract.xml feed yields pre-distilled summaries at ~1/100th the chunk volume. Streams .bz2/.gz directly via iterparse with bounded memory; same shard / resume contract as the SQL source. Falls back to title-prefix namespace filtering when older export schemas omit the per-page <ns> element (e.g. enwiki 20101011). Makefile defaults target enwiki 20101011 (6.2 GB). Override with WP_XML_YEAR / WP_XML_MONTH / WP_XML_DATE / WP_XML_LANG to fetch any other archived snapshot.
This commit is contained in:
parent
5cf76dbc8f
commit
f90b7c69f0
5 changed files with 661 additions and 1 deletions
56
Makefile
56
Makefile
|
|
@ -25,7 +25,9 @@ INGEST_LIMIT ?= 500
|
|||
VERIFY_N ?= 10
|
||||
SEARCH_Q ?= computer
|
||||
|
||||
.PHONY: all bootstrap fetch fetch-cur fetch-old ingest ingest-cur ingest-old \
|
||||
.PHONY: all bootstrap fetch fetch-cur fetch-old fetch-xml fetch-abstract \
|
||||
ingest ingest-cur ingest-old ingest-xml ingest-xml-history \
|
||||
ingest-xml-attached ingest-abstract \
|
||||
ingest-grok ingest-grok-media \
|
||||
verify search stats test clean clean-db clean-data help
|
||||
|
||||
|
|
@ -184,6 +186,58 @@ ingest-grok-media-attached: bootstrap ## ingest Grok media prompts into $(GROK_S
|
|||
@mkdir -p $(SHARDS_DIR)
|
||||
$(ABORIST) --db $(GROK_SHARD) ingest --source grok_media --path $(GROK_EXPORT) --resume
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Phase IV (2006+) Wikipedia XML dumps. Drop-in for any dated snapshot in
|
||||
# the dumps.wikimedia.org/archive tree by overriding WP_XML_YEAR/MONTH/DATE
|
||||
# (and WP_XML_LANG for non-English wikis).
|
||||
#
|
||||
# Defaults point at enwiki 20101011 (2010-11 archive), the largest snapshot
|
||||
# in the archive — 6.2 GB compressed, ~3.4M articles.
|
||||
# Other useful snapshots:
|
||||
# make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-07 WP_XML_DATE=20061104
|
||||
# make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-12 WP_XML_DATE=20061130
|
||||
# ----------------------------------------------------------------------------
|
||||
WP_XML_LANG ?= en
|
||||
WP_XML_YEAR ?= 2010
|
||||
WP_XML_MONTH ?= 2010-11
|
||||
WP_XML_DATE ?= 20101011
|
||||
WP_XML_BASE_URL ?= https://dumps.wikimedia.org/archive/$(WP_XML_YEAR)/$(WP_XML_MONTH)/$(WP_XML_LANG)wiki/$(WP_XML_DATE)
|
||||
WP_XML_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-pages-articles.xml.bz2
|
||||
WP_XML := $(DATA_DIR)/$(WP_XML_FILE)
|
||||
WP_ABSTRACT_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-abstract.xml
|
||||
WP_ABSTRACT := $(DATA_DIR)/$(WP_ABSTRACT_FILE)
|
||||
|
||||
$(WP_XML): | $(DATA_DIR)
|
||||
@echo ">> fetching $(WP_XML_BASE_URL)/$(WP_XML_FILE)"
|
||||
curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_XML_FILE)"
|
||||
|
||||
$(WP_ABSTRACT): | $(DATA_DIR)
|
||||
@echo ">> fetching $(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)"
|
||||
curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)"
|
||||
|
||||
fetch-xml: $(WP_XML) ## download Phase IV XML cur dump (default: enwiki 20101011, 6.2 GB)
|
||||
|
||||
fetch-abstract: $(WP_ABSTRACT) ## download Phase IV abstract.xml (default: enwiki 20101011, ~3 GB)
|
||||
|
||||
ingest-xml: bootstrap fetch-xml ## ingest INGEST_LIMIT pages from $(WP_XML)
|
||||
$(ABORIST) --db $(DB) ingest --source wikipedia_xml --path $(WP_XML) --limit $(INGEST_LIMIT)
|
||||
|
||||
ingest-xml-history: bootstrap ## ingest every revision (multi-revision mode); set WP_XML to a pages-meta-history file
|
||||
$(ABORIST) --db $(DB) ingest --source wikipedia_xml_history --path $(WP_XML) --limit $(INGEST_LIMIT)
|
||||
|
||||
# Sharded XML ingest into the attach-forever cluster — same pattern as
|
||||
# ingest-cur-attached. One process per shard, one SQLite file per shard,
|
||||
# zero WAL contention.
|
||||
ingest-xml-attached: bootstrap fetch-xml ## sharded XML ingest, one process per shard
|
||||
@mkdir -p $(SHARDS_DIR)
|
||||
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
||||
$(ABORIST) ingest --source wikipedia_xml --path $(WP_XML) \
|
||||
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
||||
done; wait
|
||||
|
||||
ingest-abstract: bootstrap fetch-abstract ## ingest INGEST_LIMIT abstract docs from $(WP_ABSTRACT)
|
||||
$(ABORIST) --db $(DB) ingest --source wikipedia_abstract --path $(WP_ABSTRACT) --limit $(INGEST_LIMIT)
|
||||
|
||||
verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents
|
||||
$(ABORIST) --db $(DB) verify -n $(VERIFY_N)
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,24 @@ def _cmd_ingest(args: argparse.Namespace) -> int:
|
|||
|
||||
cls = GrokExportSource if args.source == "grok_export" else GrokMediaPostsSource
|
||||
src = cls(path=args.path)
|
||||
elif args.source in ("wikipedia_xml", "wikipedia_xml_history", "wikipedia_abstract"):
|
||||
if not args.path:
|
||||
print(f"--path is required for {args.source}", file=sys.stderr)
|
||||
return 2
|
||||
from aborist.sources import WikipediaAbstractDump, WikipediaXmlDump
|
||||
|
||||
if args.source == "wikipedia_abstract":
|
||||
src = WikipediaAbstractDump(path=args.path)
|
||||
else:
|
||||
shard = None
|
||||
if args.shard:
|
||||
rank_str, total_str = args.shard.split("/", 1)
|
||||
shard = (int(rank_str), int(total_str))
|
||||
src = WikipediaXmlDump(
|
||||
path=args.path,
|
||||
shard=shard,
|
||||
multi_revision=(args.source == "wikipedia_xml_history"),
|
||||
)
|
||||
else:
|
||||
print(f"unknown source: {args.source}", file=sys.stderr)
|
||||
return 2
|
||||
|
|
@ -881,6 +899,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
choices=[
|
||||
"wikipedia_cur",
|
||||
"wikipedia_old",
|
||||
"wikipedia_xml",
|
||||
"wikipedia_xml_history",
|
||||
"wikipedia_abstract",
|
||||
"html",
|
||||
"grok_export",
|
||||
"grok_media",
|
||||
|
|
|
|||
|
|
@ -6,13 +6,16 @@ from aborist.sources.wikipedia import (
|
|||
WikipediaOldDump,
|
||||
WikipediaSqlDump,
|
||||
)
|
||||
from aborist.sources.wikipedia_xml import WikipediaAbstractDump, WikipediaXmlDump
|
||||
|
||||
__all__ = [
|
||||
"GrokExportSource",
|
||||
"GrokMediaPostsSource",
|
||||
"WikipediaAbstractDump",
|
||||
"WikipediaCurDump",
|
||||
"WikipediaOldDump",
|
||||
"WikipediaSqlDump",
|
||||
"WikipediaXmlDump",
|
||||
]
|
||||
|
||||
# HtmlPageSource has optional deps (selectolax). Surface import errors only
|
||||
|
|
|
|||
320
aborist/sources/wikipedia_xml.py
Normal file
320
aborist/sources/wikipedia_xml.py
Normal file
|
|
@ -0,0 +1,320 @@
|
|||
"""MediaWiki XML 'pages-articles' / 'pages-meta-current' / 'abstract' source.
|
||||
|
||||
Phase IV (2006+) Wikipedia dumps switched from MySQL extended INSERT
|
||||
syntax to a custom XML schema. These sources stream that XML and yield
|
||||
Documents the same way the WikipediaSqlDump source yields rows from
|
||||
the 2003 cur table — same URI scheme, same edge extraction, same
|
||||
shard / resume contract — so downstream ingest, sharding, and resume
|
||||
logic Just Works.
|
||||
|
||||
Two source classes:
|
||||
|
||||
* `WikipediaXmlDump` — pages-articles.xml.bz2 (cur snapshot, main NS) or
|
||||
pages-meta-current.xml.bz2 (cur snapshot, all NS) or
|
||||
pages-meta-history.xml.bz2 (full history; pass multi_revision=True).
|
||||
One Document per page (or per revision in history mode).
|
||||
|
||||
* `WikipediaAbstractDump` — abstract.xml (uncompressed). Each `<doc>`
|
||||
has <title>, <url>, <abstract>: short pre-distilled summaries of every
|
||||
article. Useful as cheap retrieval targets for FTS5 queries.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import bz2
|
||||
import gzip
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import IO, Iterator
|
||||
from xml.etree.ElementTree import Element, iterparse
|
||||
|
||||
from aborist.document import Document, Edge
|
||||
from aborist.source import Source
|
||||
from aborist.sources.wikipedia import _extract_wikilinks
|
||||
|
||||
|
||||
# MediaWiki export schema declares an xmlns; ElementTree returns tag names
|
||||
# as `{http://www.mediawiki.org/xml/export-0.10/}page`. Strip that prefix
|
||||
# off so the rest of the parser doesn't have to care which schema version
|
||||
# any given dump uses.
|
||||
_NS_RE = re.compile(r"^\{[^}]+\}")
|
||||
|
||||
# Body that begins with `#REDIRECT [[Target]]` is a redirect — skip.
|
||||
# Real dumps also use the explicit <redirect title="..."/> element, but
|
||||
# older or imported pages sometimes lack that and only carry the body
|
||||
# directive; we belt-and-suspenders both.
|
||||
_REDIRECT_RE = re.compile(r"\s*#REDIRECT\b", re.IGNORECASE)
|
||||
|
||||
NAMESPACE_MAIN = 0
|
||||
|
||||
|
||||
# Title-prefix-encoded namespaces. Only used as a fallback filter when the
|
||||
# export schema doesn't declare <ns> per page. Excludes Wiktionary-style
|
||||
# language prefixes (those don't appear in enwiki main namespace anyway).
|
||||
_NON_MAIN_NS_PREFIXES = frozenset({
|
||||
"Talk", "User", "User talk", "Wikipedia", "Wikipedia talk",
|
||||
"File", "File talk", "MediaWiki", "MediaWiki talk",
|
||||
"Template", "Template talk", "Help", "Help talk",
|
||||
"Category", "Category talk", "Portal", "Portal talk",
|
||||
"Book", "Book talk", "Draft", "Draft talk",
|
||||
"Module", "Module talk", "Image", "Image talk",
|
||||
})
|
||||
|
||||
|
||||
def _strip_ns(tag: str) -> str:
|
||||
return _NS_RE.sub("", tag)
|
||||
|
||||
|
||||
def _open_streaming(path: Path) -> IO[bytes]:
|
||||
"""Open a file as a binary byte stream, transparently decompressing."""
|
||||
p = str(path)
|
||||
if p.endswith(".bz2"):
|
||||
return bz2.open(p, "rb")
|
||||
if p.endswith(".gz"):
|
||||
return gzip.open(p, "rb")
|
||||
return open(p, "rb")
|
||||
|
||||
|
||||
class WikipediaXmlDump(Source):
|
||||
"""Iterates a MediaWiki XML dump.
|
||||
|
||||
Phase IV format: `<mediawiki>` root containing one `<siteinfo>` plus
|
||||
many `<page>` children. Each `<page>` has `<title>`, `<ns>`, `<id>`,
|
||||
optional `<redirect>`, and one or more `<revision>` children.
|
||||
|
||||
For pages-articles or pages-meta-current dumps, each page has exactly
|
||||
one `<revision>` (the current one). For pages-meta-history dumps each
|
||||
page has many. By default we emit the LAST revision per page (matches
|
||||
cur semantics); pass `multi_revision=True` to emit every revision
|
||||
(matches old semantics, with `supersedes` chaining at the URI level
|
||||
via aborist's existing prior-document detection).
|
||||
"""
|
||||
|
||||
source_type = "wikipedia_xml"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: str | Path,
|
||||
*,
|
||||
namespace: int = NAMESPACE_MAIN,
|
||||
base_uri: str = "https://en.wikipedia.org/wiki/",
|
||||
shard: tuple[int, int] | None = None,
|
||||
start_id: int = 0,
|
||||
multi_revision: bool = False,
|
||||
):
|
||||
self.path = Path(path)
|
||||
self.namespace = namespace
|
||||
self.base_uri = base_uri
|
||||
if shard is not None:
|
||||
rank, total = shard
|
||||
if not (0 <= rank < total) or total < 1:
|
||||
raise ValueError(
|
||||
f"invalid shard {shard}: need 0 <= rank < total, total >= 1"
|
||||
)
|
||||
self.shard = shard
|
||||
# Resume support — same contract as WikipediaSqlDump.
|
||||
self.start_id = start_id
|
||||
self.last_id: int = start_id
|
||||
self.multi_revision = multi_revision
|
||||
|
||||
def iter_documents(self) -> Iterator[Document]:
|
||||
rank, total = (0, 1) if self.shard is None else self.shard
|
||||
idx = 0
|
||||
with _open_streaming(self.path) as f:
|
||||
context = iter(iterparse(f, events=("start", "end")))
|
||||
try:
|
||||
_, root = next(context)
|
||||
except StopIteration:
|
||||
return
|
||||
for event, elem in context:
|
||||
if event != "end":
|
||||
continue
|
||||
tag = _strip_ns(elem.tag)
|
||||
if tag != "page":
|
||||
# Free siteinfo/etc immediately — no useful data downstream.
|
||||
if tag in ("siteinfo", "namespaces", "namespace"):
|
||||
elem.clear()
|
||||
continue
|
||||
# Stride filter applied at the page index. Every shard sees
|
||||
# the full XML stream but only emits its share — matches the
|
||||
# SQL source's behavior so ingest counters line up.
|
||||
if idx % total != rank:
|
||||
idx += 1
|
||||
elem.clear()
|
||||
root.clear()
|
||||
continue
|
||||
idx += 1
|
||||
yield from self._page_to_docs(elem)
|
||||
elem.clear()
|
||||
root.clear()
|
||||
|
||||
def _page_to_docs(self, page: Element) -> Iterator[Document]:
|
||||
title: str | None = None
|
||||
ns_text: str | None = None
|
||||
page_id: int = 0
|
||||
is_redirect = False
|
||||
revisions: list[Element] = []
|
||||
for child in page:
|
||||
ctag = _strip_ns(child.tag)
|
||||
if ctag == "title":
|
||||
title = (child.text or "").strip() or None
|
||||
elif ctag == "ns":
|
||||
ns_text = (child.text or "").strip()
|
||||
elif ctag == "id":
|
||||
try:
|
||||
page_id = int((child.text or "0").strip())
|
||||
except ValueError:
|
||||
pass
|
||||
elif ctag == "redirect":
|
||||
# Self-closing element with title attr. Body may also start
|
||||
# with #REDIRECT — both flag the same condition.
|
||||
is_redirect = True
|
||||
elif ctag == "revision":
|
||||
revisions.append(child)
|
||||
|
||||
try:
|
||||
ns_int = int(ns_text) if ns_text is not None else None
|
||||
except ValueError:
|
||||
ns_int = None
|
||||
# Older export schemas (pre-0.4) had no <ns> child element under
|
||||
# <page>; namespace was encoded into the title prefix only. The
|
||||
# `pages-articles.xml.bz2` dump is documented to contain just the
|
||||
# main namespace plus redirects, so trust the file when <ns> is
|
||||
# absent. For files that include sister namespaces (e.g.
|
||||
# `pages-meta-current.xml.bz2`), filter on the title prefix as
|
||||
# a safety belt — any title starting `Talk:` / `User:` /
|
||||
# `Wikipedia:` / `File:` / `Template:` / `Help:` / `Category:` /
|
||||
# `Portal:` / `Book:` / `Draft:` / `MediaWiki:` is non-main.
|
||||
if ns_int is None:
|
||||
if title and ":" in title:
|
||||
head = title.split(":", 1)[0]
|
||||
if head in _NON_MAIN_NS_PREFIXES:
|
||||
return
|
||||
elif ns_int != self.namespace:
|
||||
return
|
||||
if is_redirect:
|
||||
return
|
||||
if not title or not revisions:
|
||||
return
|
||||
|
||||
# Resume fast-forward.
|
||||
if self.start_id and page_id <= self.start_id:
|
||||
return
|
||||
if page_id > self.last_id:
|
||||
self.last_id = page_id
|
||||
|
||||
revs = revisions if self.multi_revision else revisions[-1:]
|
||||
uri = self.base_uri + title.replace(" ", "_")
|
||||
for rev in revs:
|
||||
text = ""
|
||||
timestamp = ""
|
||||
rev_id = 0
|
||||
for child in rev:
|
||||
ctag = _strip_ns(child.tag)
|
||||
if ctag == "text":
|
||||
text = child.text or ""
|
||||
elif ctag == "timestamp":
|
||||
timestamp = (child.text or "").strip()
|
||||
elif ctag == "id":
|
||||
try:
|
||||
rev_id = int((child.text or "0").strip())
|
||||
except ValueError:
|
||||
pass
|
||||
if not text:
|
||||
continue
|
||||
if _REDIRECT_RE.match(text):
|
||||
continue
|
||||
edges = _extract_wikilinks(text, self.base_uri)
|
||||
extra: dict = {"page_id": str(page_id)}
|
||||
if timestamp:
|
||||
extra["timestamp"] = timestamp
|
||||
if rev_id:
|
||||
extra["rev_id"] = str(rev_id)
|
||||
yield Document(
|
||||
uri=uri,
|
||||
content=text,
|
||||
source_type=self.source_type,
|
||||
title=title,
|
||||
edges=edges,
|
||||
extra=extra,
|
||||
)
|
||||
|
||||
|
||||
class WikipediaAbstractDump(Source):
|
||||
"""Iterates a MediaWiki `abstract.xml` dump.
|
||||
|
||||
Format:
|
||||
<feed>
|
||||
<doc>
|
||||
<title>Wikipedia: Anarchism</title>
|
||||
<url>http://en.wikipedia.org/wiki/Anarchism</url>
|
||||
<abstract>Anarchism is a political philosophy...</abstract>
|
||||
<links>...</links>
|
||||
</doc>
|
||||
...
|
||||
</feed>
|
||||
|
||||
Each <doc> already contains a one-paragraph summary — pre-distilled
|
||||
by the dump generator. Ingesting these alongside (or instead of) full
|
||||
pages-articles gives FTS5 a high-signal corpus at ~1/100th the bytes.
|
||||
"""
|
||||
|
||||
source_type = "wikipedia_abstract"
|
||||
|
||||
def __init__(self, path: str | Path):
|
||||
self.path = Path(path)
|
||||
# Resume contract: abstract docs have no stable numeric id, so
|
||||
# start_id / last_id are no-ops here. Kept for interface symmetry.
|
||||
self.start_id = 0
|
||||
self.last_id = 0
|
||||
|
||||
def iter_documents(self) -> Iterator[Document]:
|
||||
with _open_streaming(self.path) as f:
|
||||
context = iter(iterparse(f, events=("start", "end")))
|
||||
try:
|
||||
_, root = next(context)
|
||||
except StopIteration:
|
||||
return
|
||||
for event, elem in context:
|
||||
if event != "end":
|
||||
continue
|
||||
tag = _strip_ns(elem.tag)
|
||||
if tag != "doc":
|
||||
continue
|
||||
title = ""
|
||||
url = ""
|
||||
abstract = ""
|
||||
links: list[str] = []
|
||||
for child in elem:
|
||||
ctag = _strip_ns(child.tag)
|
||||
if ctag == "title":
|
||||
title = (child.text or "").strip()
|
||||
elif ctag == "url":
|
||||
url = (child.text or "").strip()
|
||||
elif ctag == "abstract":
|
||||
abstract = (child.text or "").strip()
|
||||
elif ctag == "links":
|
||||
for sublink in child:
|
||||
for sub in sublink:
|
||||
if _strip_ns(sub.tag) == "link":
|
||||
href = (sub.text or "").strip()
|
||||
if href:
|
||||
links.append(href)
|
||||
elem.clear()
|
||||
root.clear()
|
||||
if not url or not abstract:
|
||||
continue
|
||||
# Abstract feed prepends "Wikipedia: " to titles. Strip it.
|
||||
clean_title = title
|
||||
if clean_title.startswith("Wikipedia: "):
|
||||
clean_title = clean_title[len("Wikipedia: "):]
|
||||
edges = [
|
||||
Edge(edge_type="abstract_link", dst_uri=u) for u in links
|
||||
]
|
||||
yield Document(
|
||||
uri=url,
|
||||
content=abstract,
|
||||
source_type=self.source_type,
|
||||
title=clean_title or None,
|
||||
edges=edges,
|
||||
)
|
||||
262
tests/test_wikipedia_xml.py
Normal file
262
tests/test_wikipedia_xml.py
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
"""Tests for the MediaWiki XML and abstract dump sources.
|
||||
|
||||
Synthetic XML fixtures match the Phase IV (2006+) export schema:
|
||||
pages-articles.xml: <mediawiki><page><title/><ns/><id/><revision/></page>...
|
||||
abstract.xml: <feed><doc><title/><url/><abstract/></doc>...
|
||||
No real dumps are downloaded by the test suite.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import bz2
|
||||
from pathlib import Path
|
||||
|
||||
from aborist.ingest import ingest_source
|
||||
from aborist.sources.wikipedia_xml import (
|
||||
WikipediaAbstractDump,
|
||||
WikipediaXmlDump,
|
||||
_strip_ns,
|
||||
)
|
||||
from aborist.store import connect
|
||||
|
||||
|
||||
PAGES_ARTICLES_XML = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.10/" version="0.10" xml:lang="en">
|
||||
<siteinfo>
|
||||
<sitename>Wikipedia</sitename>
|
||||
<dbname>enwiki</dbname>
|
||||
<base>http://en.wikipedia.org/wiki/Main_Page</base>
|
||||
<generator>MediaWiki 1.36</generator>
|
||||
</siteinfo>
|
||||
<page>
|
||||
<title>Anarchism</title>
|
||||
<ns>0</ns>
|
||||
<id>12</id>
|
||||
<revision>
|
||||
<id>1234567</id>
|
||||
<timestamp>2010-11-08T13:00:00Z</timestamp>
|
||||
<text xml:space="preserve">'''Anarchism''' is a [[political philosophy]] that advocates [[stateless society|stateless societies]].</text>
|
||||
</revision>
|
||||
</page>
|
||||
<page>
|
||||
<title>Foo Redirect</title>
|
||||
<ns>0</ns>
|
||||
<id>13</id>
|
||||
<redirect title="Foo Target"/>
|
||||
<revision>
|
||||
<id>1234568</id>
|
||||
<timestamp>2010-11-08T13:00:01Z</timestamp>
|
||||
<text xml:space="preserve">#REDIRECT [[Foo Target]]</text>
|
||||
</revision>
|
||||
</page>
|
||||
<page>
|
||||
<title>Talk:Anarchism</title>
|
||||
<ns>1</ns>
|
||||
<id>14</id>
|
||||
<revision>
|
||||
<id>1234569</id>
|
||||
<timestamp>2010-11-08T13:00:02Z</timestamp>
|
||||
<text xml:space="preserve">Discussion about [[anarchism]].</text>
|
||||
</revision>
|
||||
</page>
|
||||
<page>
|
||||
<title>Capitalism</title>
|
||||
<ns>0</ns>
|
||||
<id>15</id>
|
||||
<revision>
|
||||
<id>2000001</id>
|
||||
<timestamp>2010-11-08T13:00:03Z</timestamp>
|
||||
<text xml:space="preserve">First version of capitalism article.</text>
|
||||
</revision>
|
||||
<revision>
|
||||
<id>2000002</id>
|
||||
<timestamp>2010-11-08T13:00:04Z</timestamp>
|
||||
<text xml:space="preserve">Second version of capitalism article. References [[markets]].</text>
|
||||
</revision>
|
||||
</page>
|
||||
<page>
|
||||
<title>Body Redirect Without Marker</title>
|
||||
<ns>0</ns>
|
||||
<id>16</id>
|
||||
<revision>
|
||||
<id>3000001</id>
|
||||
<timestamp>2010-11-08T13:00:05Z</timestamp>
|
||||
<text xml:space="preserve"> #redirect [[Some Other Page]]</text>
|
||||
</revision>
|
||||
</page>
|
||||
</mediawiki>"""
|
||||
|
||||
|
||||
ABSTRACT_XML = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed>
|
||||
<doc>
|
||||
<title>Wikipedia: Anarchism</title>
|
||||
<url>http://en.wikipedia.org/wiki/Anarchism</url>
|
||||
<abstract>Anarchism is a political philosophy that advocates self-governed societies.</abstract>
|
||||
<links>
|
||||
<sublink linktype="nav">
|
||||
<anchor>Origins</anchor>
|
||||
<link>http://en.wikipedia.org/wiki/Anarchism#Origins</link>
|
||||
</sublink>
|
||||
<sublink linktype="nav">
|
||||
<anchor>Schools</anchor>
|
||||
<link>http://en.wikipedia.org/wiki/Anarchism#Schools</link>
|
||||
</sublink>
|
||||
</links>
|
||||
</doc>
|
||||
<doc>
|
||||
<title>Wikipedia: Capitalism</title>
|
||||
<url>http://en.wikipedia.org/wiki/Capitalism</url>
|
||||
<abstract>Capitalism is an economic system based on private ownership.</abstract>
|
||||
</doc>
|
||||
<doc>
|
||||
<title>Wikipedia: Empty</title>
|
||||
<url>http://en.wikipedia.org/wiki/Empty</url>
|
||||
<abstract></abstract>
|
||||
</doc>
|
||||
</feed>"""
|
||||
|
||||
|
||||
def _write(path: Path, text: str) -> Path:
|
||||
path.write_text(text, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def _write_bz2(path: Path, text: str) -> Path:
|
||||
with bz2.open(path, "wt", encoding="utf-8") as f:
|
||||
f.write(text)
|
||||
return path
|
||||
|
||||
|
||||
def test_strip_ns_removes_xmlns_braces():
|
||||
assert _strip_ns("{http://www.mediawiki.org/xml/export-0.10/}page") == "page"
|
||||
assert _strip_ns("page") == "page"
|
||||
|
||||
|
||||
def test_xml_dump_yields_main_namespace_pages_only(tmp_path):
|
||||
src = WikipediaXmlDump(_write(tmp_path / "wp.xml", PAGES_ARTICLES_XML))
|
||||
docs = list(src.iter_documents())
|
||||
titles = [d.title for d in docs]
|
||||
# Anarchism in, Capitalism in (last revision), Talk:Anarchism out (ns=1),
|
||||
# Foo Redirect out (<redirect/>), Body Redirect Without Marker out (#REDIRECT body).
|
||||
assert titles == ["Anarchism", "Capitalism"]
|
||||
# URI shape matches the SQL source: spaces -> underscores, base prefix.
|
||||
uris = [d.uri for d in docs]
|
||||
assert uris == [
|
||||
"https://en.wikipedia.org/wiki/Anarchism",
|
||||
"https://en.wikipedia.org/wiki/Capitalism",
|
||||
]
|
||||
|
||||
|
||||
def test_xml_dump_extracts_wikilinks_as_edges(tmp_path):
|
||||
src = WikipediaXmlDump(_write(tmp_path / "wp.xml", PAGES_ARTICLES_XML))
|
||||
docs = list(src.iter_documents())
|
||||
anarchism = next(d for d in docs if d.title == "Anarchism")
|
||||
edge_uris = {e.dst_uri for e in anarchism.edges}
|
||||
assert "https://en.wikipedia.org/wiki/political_philosophy" in edge_uris
|
||||
# [[stateless society|stateless societies]] -> dst is target before pipe.
|
||||
assert "https://en.wikipedia.org/wiki/stateless_society" in edge_uris
|
||||
assert all(e.edge_type == "wikilink" for e in anarchism.edges)
|
||||
|
||||
|
||||
def test_xml_dump_default_takes_last_revision_only(tmp_path):
|
||||
src = WikipediaXmlDump(_write(tmp_path / "wp.xml", PAGES_ARTICLES_XML))
|
||||
docs = list(src.iter_documents())
|
||||
capitalism = next(d for d in docs if d.title == "Capitalism")
|
||||
assert "Second version" in capitalism.content
|
||||
assert "First version" not in capitalism.content
|
||||
|
||||
|
||||
def test_xml_dump_multi_revision_yields_every_revision(tmp_path):
|
||||
src = WikipediaXmlDump(
|
||||
_write(tmp_path / "wp.xml", PAGES_ARTICLES_XML),
|
||||
multi_revision=True,
|
||||
)
|
||||
docs = list(src.iter_documents())
|
||||
capitalism_docs = [d for d in docs if d.title == "Capitalism"]
|
||||
assert len(capitalism_docs) == 2
|
||||
contents = [d.content for d in capitalism_docs]
|
||||
assert any("First version" in c for c in contents)
|
||||
assert any("Second version" in c for c in contents)
|
||||
|
||||
|
||||
def test_xml_dump_handles_bz2_streaming(tmp_path):
|
||||
src = WikipediaXmlDump(_write_bz2(tmp_path / "wp.xml.bz2", PAGES_ARTICLES_XML))
|
||||
docs = list(src.iter_documents())
|
||||
assert {d.title for d in docs} == {"Anarchism", "Capitalism"}
|
||||
|
||||
|
||||
def test_xml_dump_shard_strides_pages(tmp_path):
|
||||
src_a = WikipediaXmlDump(
|
||||
_write(tmp_path / "wp.xml", PAGES_ARTICLES_XML), shard=(0, 2)
|
||||
)
|
||||
src_b = WikipediaXmlDump(tmp_path / "wp.xml", shard=(1, 2))
|
||||
a_titles = {d.title for d in src_a.iter_documents()}
|
||||
b_titles = {d.title for d in src_b.iter_documents()}
|
||||
# Stride is at the page index level (every page seen by every shard,
|
||||
# filtered post-namespace). Together they must equal the full set.
|
||||
assert a_titles | b_titles == {"Anarchism", "Capitalism"}
|
||||
assert not (a_titles & b_titles)
|
||||
|
||||
|
||||
def test_xml_dump_resume_skips_pages_at_or_below_start_id(tmp_path):
|
||||
# First run: bring last_id to whatever it sees.
|
||||
src = WikipediaXmlDump(_write(tmp_path / "wp.xml", PAGES_ARTICLES_XML))
|
||||
list(src.iter_documents())
|
||||
high_water = src.last_id
|
||||
assert high_water > 0
|
||||
# Second run with start_id at high_water = no docs.
|
||||
src2 = WikipediaXmlDump(tmp_path / "wp.xml", start_id=high_water)
|
||||
assert list(src2.iter_documents()) == []
|
||||
|
||||
|
||||
def test_xml_dump_round_trip_into_sqlite(tmp_path):
|
||||
src = WikipediaXmlDump(_write(tmp_path / "wp.xml", PAGES_ARTICLES_XML))
|
||||
db_path = tmp_path / "aborist.db"
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
stats = ingest_source(conn, src)
|
||||
finally:
|
||||
conn.close()
|
||||
assert stats.inserted == 2
|
||||
# Re-ingest is idempotent.
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
again = ingest_source(conn, WikipediaXmlDump(tmp_path / "wp.xml"))
|
||||
finally:
|
||||
conn.close()
|
||||
assert again.inserted == 0
|
||||
assert again.skipped_duplicate == 2
|
||||
|
||||
|
||||
def test_abstract_dump_yields_one_doc_per_nonempty_abstract(tmp_path):
|
||||
src = WikipediaAbstractDump(_write(tmp_path / "abstract.xml", ABSTRACT_XML))
|
||||
docs = list(src.iter_documents())
|
||||
assert len(docs) == 2 # "Empty" doc has no abstract -> skipped
|
||||
titles = [d.title for d in docs]
|
||||
assert titles == ["Anarchism", "Capitalism"]
|
||||
# Wikipedia: prefix stripped from titles.
|
||||
assert all(not (t or "").startswith("Wikipedia:") for t in titles)
|
||||
# URL preserved verbatim.
|
||||
assert docs[0].uri == "http://en.wikipedia.org/wiki/Anarchism"
|
||||
|
||||
|
||||
def test_abstract_dump_extracts_sublinks_as_edges(tmp_path):
|
||||
src = WikipediaAbstractDump(_write(tmp_path / "abstract.xml", ABSTRACT_XML))
|
||||
docs = list(src.iter_documents())
|
||||
anarchism = next(d for d in docs if d.title == "Anarchism")
|
||||
edge_uris = {e.dst_uri for e in anarchism.edges}
|
||||
assert "http://en.wikipedia.org/wiki/Anarchism#Origins" in edge_uris
|
||||
assert "http://en.wikipedia.org/wiki/Anarchism#Schools" in edge_uris
|
||||
assert all(e.edge_type == "abstract_link" for e in anarchism.edges)
|
||||
|
||||
|
||||
def test_abstract_dump_round_trip_into_sqlite(tmp_path):
|
||||
src = WikipediaAbstractDump(_write(tmp_path / "abstract.xml", ABSTRACT_XML))
|
||||
db_path = tmp_path / "aborist.db"
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
stats = ingest_source(conn, src)
|
||||
finally:
|
||||
conn.close()
|
||||
assert stats.inserted == 2
|
||||
Loading…
Add table
Add a link
Reference in a new issue