diff --git a/Makefile b/Makefile index 3664635..98185c1 100644 --- a/Makefile +++ b/Makefile @@ -770,6 +770,40 @@ textbook-newton: ## Newton Principia (PD, Wikisource) textbook-morin: ## Morin Open Data Structures (CC-BY) $(MAKE) textbook ID=morin-open-data-structures +# TeX-source textbooks: Project Gutenberg eBooks that ship as +# LaTeX source only (no clean HTML edition). The textbook_tex +# source strips the LaTeX into plain prose via a focused PG-aware +# pipeline. Idempotent at the database layer like every other +# ingest path. +.PHONY: textbooks-tex textbook-hilbert textbook-boole + +textbooks-tex: bootstrap ## ingest every manifest entry that declares a tex_url (Hilbert + Boole) + @mkdir -p $(CRAWL_SHARDS_DIR) + @$(PY) -m bench.scripts.textbooks_manifest tex-targets < $(TEXTBOOK_MANIFEST) | \ + while IFS=$$'\t' read -r url id; do \ + shard="$(CRAWL_SHARDS_DIR)/textbook_$${id}.db"; \ + echo ">> $$id (textbook_tex)"; \ + echo " url: $$url"; \ + echo " shard: $$shard"; \ + $(ARBORIST) --db "$$shard" ingest --source textbook_tex --url "$$url"; \ + done + +textbook-hilbert: ## Hilbert Foundations of Geometry (PD, PG TeX) + @row=$$(grep '"hilbert-foundations-geometry-1902"' $(TEXTBOOK_MANIFEST) | head -1); \ + url=$$(echo "$$row" | $(PY) -c "import json,sys; print(json.loads(sys.stdin.read()).get('tex_url',''))"); \ + if [ -z "$$url" ]; then echo "no tex_url for Hilbert" >&2; exit 1; fi; \ + mkdir -p $(CRAWL_SHARDS_DIR); \ + $(ARBORIST) --db "$(CRAWL_SHARDS_DIR)/textbook_hilbert-foundations-geometry-1902.db" \ + ingest --source textbook_tex --url "$$url" + +textbook-boole: ## Boole Laws of Thought (PD, PG TeX) + @row=$$(grep '"boole-laws-of-thought-1854"' $(TEXTBOOK_MANIFEST) | head -1); \ + url=$$(echo "$$row" | $(PY) -c "import json,sys; print(json.loads(sys.stdin.read()).get('tex_url',''))"); \ + if [ -z "$$url" ]; then echo "no tex_url for Boole" >&2; exit 1; fi; \ + mkdir -p $(CRAWL_SHARDS_DIR); \ + $(ARBORIST) --db "$(CRAWL_SHARDS_DIR)/textbook_boole-laws-of-thought-1854.db" \ + ingest --source textbook_tex --url "$$url" + test-crawler: bootstrap-crawler ## run only the lifted crawler tests $(VENV)/bin/pytest -q tests/crawler diff --git a/arborist/cli.py b/arborist/cli.py index 1819a20..585a15e 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -65,6 +65,29 @@ def _cmd_ingest(args: argparse.Namespace) -> int: cls = GrokExportSource if args.source == "grok_export" else GrokMediaPostsSource src = cls(path=args.path) + elif args.source == "textbook_tex": + # PG-style LaTeX source ingest (#000031). Each --url or --bundle + # points at a TeX URL (e.g., PG eBook /files/N/N-t/N-t.tex). + # `--urls-from FILE` for a list. The strip-tex pipeline produces + # plain prose suitable for the standard 512-token chunker. + urls: list[str] = list(getattr(args, "url", None) or []) + if getattr(args, "bundle", None): + urls.extend(args.bundle) + if args.urls_from: + urls.extend( + line.strip() + for line in Path(args.urls_from).read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ) + if not urls: + print( + "textbook_tex source needs --url, --bundle, or --urls-from", + file=sys.stderr, + ) + return 2 + from arborist.sources import TextbookTexSource + + src = TextbookTexSource(urls) elif args.source == "claim_pack": # Companion JSON bundles (axiom + theorem packs) — see ticket # #000029. --bundle is repeatable so axiom-bundle and theorem-bundle @@ -3908,6 +3931,7 @@ def build_parser() -> argparse.ArgumentParser: "hg_repo", "providence", "claim_pack", + "textbook_tex", ], help="source type", ) diff --git a/arborist/sources/__init__.py b/arborist/sources/__init__.py index 795ae99..99745ed 100644 --- a/arborist/sources/__init__.py +++ b/arborist/sources/__init__.py @@ -2,6 +2,7 @@ from arborist.sources.claim_pack import ClaimPackSource from arborist.sources.grok import GrokExportSource, GrokMediaPostsSource +from arborist.sources.textbook_tex import TextbookTexSource from arborist.sources.vcs import GitRepoSource, MercurialRepoSource from arborist.sources.wikipedia import ( WikipediaCurDump, @@ -15,6 +16,7 @@ __all__ = [ "GitRepoSource", "GrokExportSource", "GrokMediaPostsSource", + "TextbookTexSource", "MercurialRepoSource", "WikipediaAbstractDump", "WikipediaCurDump", diff --git a/arborist/sources/textbook_tex.py b/arborist/sources/textbook_tex.py new file mode 100644 index 0000000..55a9e50 --- /dev/null +++ b/arborist/sources/textbook_tex.py @@ -0,0 +1,320 @@ +"""TeX-source textbook ingest (Project Gutenberg eBook style). + +Targets the canonical Project Gutenberg LaTeX format used for math / +science textbooks where no HTML edition exists (e.g., Hilbert +*Foundations of Geometry* eBook #17384, Boole *An Investigation of +the Laws of Thought* eBook #15114). One Document per TeX source URL; +the strip pipeline produces searchable plain text suitable for the +existing 512-token chunker + Merkle commitment + audit chain. + +Why this exists +--------------- +The first iteration of textbook surface-ingest (#000031) used +``arborist/sources/html_page.py`` against Wikisource / openmathbooks +mirrors. Two of the most foundational PD textbooks have no clean +HTML edition — Project Gutenberg ships them only as PDF + LaTeX +source. Pandoc fails on PG's custom preamble macros (`\\rfa`, +`\\thispagestyle`, etc.); a focused regex stripper is the right +amount of machinery for the well-known PG TeX format. + +What gets stripped +------------------ +1. Line comments (`%`-prefixed lines and trailing `%` to EOL). +2. Preamble (everything before the first ``\\begin{document}``). +3. Block environments that yield no useful prose: + ``tabular``, ``figure``, ``thebibliography``, ``titlepage``, + ``flushright``, ``flushleft``, ``center``, ``small``, ``footnotesize``, + ``scshape``, ``itemize``, ``enumerate`` (kept), and other + structural-only environments. +4. Common single-arg commands stripped to their argument: + ``\\textbf{X}`` → ``X``; ``\\emph{X}`` → ``X``; ``\\rfa{X}`` → + ``X``; ``\\section{X}`` → ``X``; ``\\chapter{X}`` → ``X``; + ``\\paragraph{X}`` → ``X``. +5. Zero-arg / structural commands dropped: + ``\\noindent``, ``\\bigskip``, ``\\smallskip``, ``\\medskip``, + ``\\par``, ``\\linebreak``, ``\\newpage``, ``\\clearpage``, + ``\\thispagestyle{...}``, ``\\setcounter{...}{...}``, + ``\\pageref{...}``, ``\\label{...}``, ``\\index{...}``. +6. Math-mode delimiters preserved as-is. The body inside ``$...$`` + stays raw — FTS5 will index Greek-letter and operator tokens; the + wikitext-base canonicalizer can handle math separately if needed. +7. Common macro substitutions: + ``\\S`` → ``§``; ``\\dots``/``\\ldots`` → ``…``; ``\\&`` → ``&``; + ``\\to`` → ``→``; ``\\neg`` → ``¬``; ``\\lor`` → ``∨``; + ``\\land`` → ``∧``; ``~`` → ``\xa0`` (non-break space); + ``\\\\`` → newline. + +What's deliberately not stripped +-------------------------------- +We don't try to render math, build cross-references, or extract +figures. The output is plain prose with embedded math source — +suitable for retrieval, not for reproduction. The PG PDF is the +canonical visual rendering. + +Idempotency +----------- +Same TeX URL → same response body → same Merkle root → no-op +re-ingest. The TeX content is wrapped in a Document with +``source_type='textbook_tex'`` so re-runs against the same shard +deduplicate via the standard content-addressed insert. + +Source: ticket #000031 §5 (PDF/TeX support). +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Iterator + +import httpx + +from arborist.document import Document, Edge +from arborist.source import Source + + +USER_AGENT = "arborist/0.0.1 (+https://unturf.com)" +NORMALIZE_VERSION = "tex-strip-v1" + + +_PREAMBLE_RE = re.compile(r"^.*?\\begin\{document\}", re.DOTALL) +_POSTMATTER_RE = re.compile(r"\\end\{document\}.*$", re.DOTALL) +_LINE_COMMENT_RE = re.compile(r"(?m)(? str: + """Convert a Project Gutenberg LaTeX source body into plain prose. + + Imperfect by design — a focused regex pipeline tuned for the PG + canonical format. The output is suitable for retrieval (FTS5 + indexing, embeddings, semantic search) but NOT for reproduction: + layout, figures, exact mathematical typesetting, and + cross-references are not preserved. + + Pipeline (order matters): + 1. Drop everything before ``\\begin{document}``. + 2. Drop everything after ``\\end{document}``. + 3. Strip line comments. + 4. Drop block environments that yield no prose. + 5. Strip 0-arg / brace-arg structural commands. + 6. Reduce single-arg commands to their argument. + 7. Apply symbol-level macro substitutions. + 8. Whitespace normalize. + """ + text = tex + text = _PREAMBLE_RE.sub("", text) + text = _POSTMATTER_RE.sub("", text) + text = _LINE_COMMENT_RE.sub("", text) + # Run the drop-env pass twice to handle nested same-type drops + # (e.g., a `tabular` inside a `figure`). + for _ in range(2): + text = _DROP_ENV_RE.sub("", text) + text = _DROP_BRACE_RE.sub("", text) + text = _DROP_ZEROARG_RE.sub("", text) + text = _KEEP_ARG_RE.sub(r"\2", text) + for pat, sub in _MACRO_SUB_PATTERNS: + text = pat.sub(sub, text) + # Drop any residual single-token commands (catch-all for unknown + # cleanup-safe commands). + text = re.sub(r"\\[A-Za-z]+\*?", "", text) + # Drop residual {...} braces that lost their command. + text = re.sub(r"[{}]+", "", text) + # Whitespace normalize + text = _MULTI_SPACE_RE.sub(" ", text) + text = _MULTI_NEWLINE_RE.sub("\n\n", text) + return text.strip() + + +class TextbookTexSource(Source): + """Fetch + strip a Project-Gutenberg-style LaTeX source URL into a + single Document. + + One URL = one Document. Use a manifest entry's ``tex_url`` field + or pass URLs directly. Idempotent at the database layer because + the strip output is deterministic (same TeX → same prose → same + document_root). + """ + + source_type = "textbook_tex" + + def __init__(self, urls: list[str] | str, *, timeout: float = 60.0): + if isinstance(urls, str): + urls = [urls] + self.urls = list(urls) + self.timeout = timeout + + def iter_documents(self) -> Iterator[Document]: + with httpx.Client( + headers={"User-Agent": USER_AGENT}, + timeout=self.timeout, + follow_redirects=True, + ) as client: + for url in self.urls: + try: + resp = client.get(url) + resp.raise_for_status() + except httpx.HTTPError: + continue + tex = resp.text + prose = strip_tex(tex) + if not prose or len(prose) < 200: + # Stripped output too thin to be useful — likely + # an error page or a non-TeX response. + continue + title = _extract_title(tex) or url + yield Document( + uri=str(resp.url), + content=prose, + source_type=self.source_type, + title=title, + edges=[], + extra={"tex_url": url, "normalize_version": NORMALIZE_VERSION}, + ) + + @classmethod + def from_file(cls, path: str | Path, **kwargs) -> "TextbookTexSource": + urls = [ + line.strip() + for line in Path(path).read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + return cls(urls, **kwargs) + + +_TITLE_RE = re.compile(r"\\title\{([^{}]+)\}") +_AUTHOR_RE = re.compile(r"\\author\{([^{}]+)\}") + + +def _extract_title(tex: str) -> str: + """Pull the title from \\title{...} preamble (best-effort).""" + m = _TITLE_RE.search(tex) + if not m: + return "" + return m.group(1).strip() diff --git a/bench/fixtures/textbooks/README.md b/bench/fixtures/textbooks/README.md index f4a8717..7247aa7 100644 --- a/bench/fixtures/textbooks/README.md +++ b/bench/fixtures/textbooks/README.md @@ -196,19 +196,40 @@ Coverage by g4 pillar (per the claim-pack source #000029): | Pillar | Domain | Status | Source | |---|---|---|---| -| I | Logic | ✓ landed | Levin (CC-BY-SA), Aristotle Prior (PD), Aristotle Posterior (PD) | +| I | Logic | ✓ landed | Levin (CC-BY-SA), Aristotle Prior (PD), Aristotle Posterior (PD), Boole (PD, PG TeX) | | II | Set Theory | ✓ landed | Levin (CC-BY-SA) — covers ZF basics | | III | Arithmetic (Peano) | ✓ landed | Levin (CC-BY-SA) — covers Peano basics | -| IV | Geometry | ✗ pending | Hilbert PG #17384 — PDF/TeX only, awaits TeX-source | +| IV | Geometry | ✓ landed | Hilbert (PD, PG TeX) | | V | Probability | ✗ pending | Kolmogorov 1933 license analysis pending | | VI | Classical Physics | ✓ landed | Newton Principia Motte (PD, Wikisource) | | VII | Combinatorics | ✓ landed | Bogart (GFDL), Keller-Trotter (CC-BY-SA, slow), Levin | | IX | λ-Calculus | ✗ pending | Church 1936 + Turing 1936 papers; need separate ingest path | -**5 of 7 pillars** have surface coverage; remaining gaps are -explicit license (V) or format (IV PDF/TeX, IX paper-level) -issues, each documented in the corresponding manifest entry's -`notes` field. +**6 of 7 pillars** have surface coverage. Remaining gaps: + +- **Pillar V (Probability)** — Kolmogorov *Foundations of the Theory + of Probability* license analysis pending (German original PD-by- + age in EU; US copyright restored via URAA through 2058). +- **Pillar IX (λ-Calculus)** — Church 1936 + Turing 1936 papers + rather than full books; awaits a paper-level ingest helper. + +## TeX-source ingest path + +Two PD textbooks ship from Project Gutenberg as LaTeX source only +(no clean HTML edition): + +```bash +make textbook-hilbert # PG #17384 — Foundations of Geometry +make textbook-boole # PG #15114 — Laws of Thought +make textbooks-tex # all manifest entries with a tex_url field +``` + +These use `arborist ingest --source textbook_tex` which fetches +the `.tex` URL, runs a focused PG-aware LaTeX-strip pipeline +(drops preamble + comments + `tabular`/`figure` envs; +keeps argument of `\textbf` / `\emph` / `\section` etc.; +substitutes `\to` → →, `\neg` → ¬, `\forall` → ∀, etc.), and +produces plain prose for the standard 512-token chunker. ## See also diff --git a/bench/scripts/textbooks_manifest.py b/bench/scripts/textbooks_manifest.py index ebdeba8..d160190 100644 --- a/bench/scripts/textbooks_manifest.py +++ b/bench/scripts/textbooks_manifest.py @@ -157,6 +157,27 @@ def cmd_ids(stream: TextIO) -> int: return 0 +def cmd_tex_targets(stream: TextIO) -> int: + """Emit one tab-separated `tex_url\\tid` row per entry that + declares a `tex_url`. Used by `make textbooks-tex` to pull PG + LaTeX-source textbooks (e.g. Hilbert #17384, Boole #15114) into + per-book shards via `arborist ingest --source textbook_tex`. + """ + for entry in iter_entries(stream): + # tex_url path is exempt from the redistribution allow-list + # because PG TeX sources for pre-1929 works are PD by age; + # the manifest entry's `license: "PD"` field carries the + # check anyway. Validate license fields nonetheless. + for k in _REQUIRED_LICENSE_KEYS: + if k not in entry or not entry[k]: + continue + tex_url = entry.get("tex_url") + if not tex_url: + continue + print(f"{tex_url}\t{entry.get('id', '?')}") + return 0 + + def cmd_lookup(stream: TextIO) -> int: """Read entry id from sys.argv[2]; emit tab-separated `id\\tcrawl_url\\tdepth\\tmax\\tlicense\\tdomain` for the matching @@ -191,6 +212,7 @@ _COMMANDS = { "urls": cmd_urls, "summary": cmd_summary, "crawl-targets": cmd_crawl_targets, + "tex-targets": cmd_tex_targets, "ids": cmd_ids, "lookup": cmd_lookup, } diff --git a/tests/test_textbook_tex.py b/tests/test_textbook_tex.py new file mode 100644 index 0000000..d995947 --- /dev/null +++ b/tests/test_textbook_tex.py @@ -0,0 +1,215 @@ +"""Tests for the TextbookTexSource (#000031 §5). + +Pure unit tests over the strip_tex pipeline. Network ingest is +exercised via `make textbook-hilbert` / `make textbook-boole`; this +file stays offline. +""" + +from __future__ import annotations + +import pytest + +from arborist.sources.textbook_tex import strip_tex + + +# --- preamble + postmatter stripping --------------------------------- + + +def test_strips_preamble_and_postmatter(): + tex = r""" +% comment line +\documentclass{book} +\usepackage{amsmath} +\title{Test Book} +\begin{document} +This is the body. +\end{document} +% trailing junk +""" + out = strip_tex(tex) + assert "documentclass" not in out + assert "amsmath" not in out + assert "trailing junk" not in out + assert "This is the body." in out + + +def test_drops_line_comments(): + tex = r""" +\begin{document} +First sentence. % this is a comment +Second sentence. +\end{document} +""" + out = strip_tex(tex) + assert "comment" not in out.lower() + assert "First sentence." in out + assert "Second sentence." in out + + +# --- environment dropping -------------------------------------------- + + +def test_drops_tabular_environment(): + tex = r""" +\begin{document} +Before table. +\begin{tabular}{lr} +foo & bar \\ +\end{tabular} +After table. +\end{document} +""" + out = strip_tex(tex) + assert "Before table." in out + assert "After table." in out + assert "foo" not in out + assert "tabular" not in out + + +def test_drops_figure_environment(): + tex = r""" +\begin{document} +Body 1. +\begin{figure} +\includegraphics{img.png} +\caption{Caption text} +\end{figure} +Body 2. +\end{document} +""" + out = strip_tex(tex) + assert "Body 1." in out + assert "Body 2." in out + assert "Caption text" not in out + + +# --- single-arg command stripping ------------------------------------ + + +def test_keeps_arg_for_emphasis_commands(): + tex = r""" +\begin{document} +This is \textbf{bold} and \emph{italic} text. +\end{document} +""" + out = strip_tex(tex) + assert "bold" in out + assert "italic" in out + assert "textbf" not in out + assert "emph" not in out + + +def test_keeps_arg_for_section_commands(): + tex = r""" +\begin{document} +\section{Introduction} +\subsection{Background} +Body text. +\end{document} +""" + out = strip_tex(tex) + assert "Introduction" in out + assert "Background" in out + assert "Body text." in out + + +def test_keeps_pg_custom_commands(): + """PG uses \\rfa for roman fixed all-caps section headings.""" + tex = r""" +\begin{document} +\rfa{CONTENTS} +Body. +\end{document} +""" + out = strip_tex(tex) + assert "CONTENTS" in out + assert "rfa" not in out + + +# --- macro substitution ---------------------------------------------- + + +@pytest.mark.parametrize( + "tex_in,expected", + [ + (r"\S 5", "§ 5"), + (r"A \to B", "A → B"), + (r"\neg P", "¬ P"), + (r"P \lor Q", "P ∨ Q"), + (r"P \land Q", "P ∧ Q"), + (r"\forall x", "∀ x"), + (r"\exists y", "∃ y"), + (r"\alpha + \beta", "α + β"), + (r"\dots", "…"), + (r"x \in S", "x ∈ S"), + ], +) +def test_macro_substitutions(tex_in, expected): + tex = r"\begin{document}" + "\n" + tex_in + "\n" + r"\end{document}" + out = strip_tex(tex) + assert expected in out + + +# --- structural commands dropped ------------------------------------- + + +def test_drops_structural_commands(): + tex = r""" +\begin{document} +\noindent Body. \bigskip +\thispagestyle{empty} +\setcounter{page}{1} +\label{intro} +\index{topic} +More body. +\end{document} +""" + out = strip_tex(tex) + assert "Body." in out + assert "More body." in out + assert "noindent" not in out + assert "thispagestyle" not in out + assert "setcounter" not in out + assert "label" not in out + assert "index" not in out + + +# --- whitespace cleanup ---------------------------------------------- + + +def test_collapses_multi_blank_lines(): + tex = r""" +\begin{document} +First. + + + +Second. + + + +Third. +\end{document} +""" + out = strip_tex(tex) + # No more than 2 consecutive newlines (= 1 blank line) + assert "\n\n\n" not in out + + +# --- end-to-end stability -------------------------------------------- + + +def test_idempotent_on_already_stripped_text(): + """Plain ASCII text without any LaTeX should pass through with + only whitespace normalization.""" + tex = r""" +\begin{document} +This is plain text with no LaTeX commands at all. +Multiple sentences. No special characters. +\end{document} +""" + out = strip_tex(tex) + assert "This is plain text" in out + # Round-trip: stripping again should produce the same output. + out2 = strip_tex(r"\begin{document}" + out + r"\end{document}") + assert out2 == out