From 4a199c9dadbfe2d1bbec91b1d722df4d172f992a Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 9 May 2026 14:58:22 -0400 Subject: [PATCH] textbooks: PD/open-licensed math+logic+CS surface-ingest pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A make-target scaffold for pulling public-domain and copyleft- redistributable textbooks into a dedicated arborist shard via the existing HtmlPageSource ingest pipeline. No new ingest machinery — the existing path (robots.txt → noise-strip → 512-token chunk → Merkle root → audit-event) IS the consistent process. Manifest format =============== bench/fixtures/textbooks/manifest-v1.jsonl — one JSON record per textbook with explicit license tokens (PD / CC-BY / CC-BY-SA / GFDL / OSI-approved). Fail-closed validation in bench/scripts/textbooks_manifest.py refuses to emit URLs from entries with missing or disallowed license tokens, so license discipline surfaces at the URL-emit step rather than getting silently buried in a fetch run. Allow-list excludes CC-BY-NC (incompatible with arborist's AGPLv3 distribution profile) and CC-BY-ND (no-derivatives prevents chunking). Wilf's generatingfunctionology stays out because its educational-use license forbids rehosting; citable but not redistributable. Initial entries (7 textbooks, 8 seed URLs) ========================================== - Bogart, Combinatorics Through Guided Discovery — GFDL-1.3 - Keller & Trotter, Applied Combinatorics — CC-BY-SA-4.0 - Levin, Discrete Mathematics: An Open Introduction (3rd ed) — CC-BY-SA-4.0 (4th ed switched to NC; pin 3rd) - Hilbert, Foundations of Geometry (Townsend 1902) — PD via PG - Boole, An Investigation of the Laws of Thought (1854) — PD - Aristotle, Prior Analytics (Jenkinson) — PD via Wikisource - Morin, Open Data Structures — CC-BY-2.5 Covers pillars I (logic) / II (set theory) / III (arithmetic) / IV (geometry) / VII (combinatorics) on the claim-pack (#000029) layout, plus a CS anchor for downstream domain expansion. Each entry's pillar_targets field lists which claim-pack pillars its records are candidate citations for. Make targets ============ - textbooks-summary — license + URL counts per entry - textbooks-urls — flat URL list to stdout - fetch-textbooks — ingest all manifested URLs into $(TEXTBOOK_DB) (default ~/.arborist/textbooks.db) - textbooks-stats — documents / chunks / edges - textbooks-verify — sample Merkle proof verification To grow coverage of one textbook: append more chapter URLs to its `urls` array. For deep-BFS of a textbook home: use the existing `make crawl-ingest URL= DEPTH=N` instead. Smoke-tested end-to-end against Wikisource Prior Analytics: 1 doc / 1 chunk / 12 outbound edges / Merkle proof passes. Shows the pipeline works; populating each book to depth needs either more URLs in the manifest or the crawler. Out of scope ============ - PDF processing. The Internet Archive hosts PD textbooks (MacMahon's Combinatory Analysis 1915, Whitworth's Choice and Chance 1867, the Motte 1729 Principia) as scanned PDFs; a TextbookPdfSource with pdftotext / pypdf extraction is a separate ticket — current scope is HTML-shaped sources. - Mendelson + Enderton (proprietary; await #000031 §2.1 decision). - Wilf generatingfunctionology (license forbids redistribution). Test suite stays at 1554 passed / 28 skipped — no source-code changes to arborist itself; the textbook layer is pure tooling on top of the existing pipeline. --- Makefile | 46 ++++++- bench/fixtures/textbooks/README.md | 131 ++++++++++++++++++++ bench/fixtures/textbooks/manifest-v1.jsonl | 8 ++ bench/scripts/textbooks_manifest.py | 132 +++++++++++++++++++++ 4 files changed, 316 insertions(+), 1 deletion(-) create mode 100644 bench/fixtures/textbooks/README.md create mode 100644 bench/fixtures/textbooks/manifest-v1.jsonl create mode 100644 bench/scripts/textbooks_manifest.py diff --git a/Makefile b/Makefile index 740df76..fed71ff 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,8 @@ SEARCH_Q ?= computer verify search stats test test-live docs docs-api docs-api-clean \ chain-check chain-check-shards \ falsify burn burn-kindergarten inspect bootstrap-crawler test-crawler crawl-ingest \ - recrawl-check bench-qa bootstrap-math clean clean-db clean-data help + recrawl-check bench-qa bootstrap-math clean clean-db clean-data help \ + textbooks-summary textbooks-urls fetch-textbooks textbooks-stats textbooks-verify all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats @@ -623,6 +624,49 @@ bootstrap-crawler: bootstrap ## install [crawler] extras into the venv bootstrap-math: bootstrap ## install [math] extras (sympy) into the venv $(PIP) install -e '.[math]' +# --------------------------------------------------------------------------- +# Public-domain + open-licensed textbooks (#000031 / surface-ingest scope). +# +# Manifest at $(TEXTBOOK_MANIFEST) lists math / logic / CS textbooks with +# explicit license tokens (PD / CC-BY-SA / GFDL / etc.). The fetch path +# reuses `arborist ingest --source html`, which already does the +# consistent "robots.txt → noise-strip → 512-token chunk → Merkle root → +# audit-event" pipeline that every other surface uses. No separate +# downloader / Merkle process — the existing ingest IS the consistent +# process. +# +# To grow coverage of a textbook, append more chapter URLs to its `urls` +# array in the manifest. For deep-BFS of a textbook home, use +# `make crawl-ingest URL= DEPTH=N` (the existing crawler) instead. +# --------------------------------------------------------------------------- + +TEXTBOOK_MANIFEST ?= bench/fixtures/textbooks/manifest-v1.jsonl +TEXTBOOK_DB ?= $(HOME)/.arborist/textbooks.db +TEXTBOOK_URLS_TMP ?= /tmp/arborist-textbook-urls.txt + +textbooks-summary: bootstrap ## list manifest entries with license + URL counts + $(PY) -m bench.scripts.textbooks_manifest summary < $(TEXTBOOK_MANIFEST) + +textbooks-urls: bootstrap ## emit one textbook URL per line to stdout + @$(PY) -m bench.scripts.textbooks_manifest urls < $(TEXTBOOK_MANIFEST) + +# License-validated fetch + ingest of every URL in $(TEXTBOOK_MANIFEST). +# The manifest helper refuses to emit URLs from entries with missing or +# disallowed license tokens — license-discipline is fail-closed at the +# URL-emit step, so the fetch never sees a non-redistributable URL. +fetch-textbooks: bootstrap ## fetch + ingest manifested textbooks → $(TEXTBOOK_DB) + @$(PY) -m bench.scripts.textbooks_manifest urls < $(TEXTBOOK_MANIFEST) > $(TEXTBOOK_URLS_TMP) + @count=$$(wc -l < $(TEXTBOOK_URLS_TMP)); \ + echo ">> ingesting $$count textbook URLs into $(TEXTBOOK_DB)" + $(ARBORIST) --db $(TEXTBOOK_DB) ingest --source html --urls-from $(TEXTBOOK_URLS_TMP) + @rm -f $(TEXTBOOK_URLS_TMP) + +textbooks-stats: bootstrap ## stats of the textbook shard + $(ARBORIST) --db $(TEXTBOOK_DB) stats + +textbooks-verify: bootstrap ## sample-Merkle-verify the textbook shard + $(ARBORIST) --db $(TEXTBOOK_DB) verify + test-crawler: bootstrap-crawler ## run only the lifted crawler tests $(VENV)/bin/pytest -q tests/crawler diff --git a/bench/fixtures/textbooks/README.md b/bench/fixtures/textbooks/README.md new file mode 100644 index 0000000..8657456 --- /dev/null +++ b/bench/fixtures/textbooks/README.md @@ -0,0 +1,131 @@ +# Textbook surface-ingest manifest + +Public-domain and copyleft-redistributable math / logic / computer-science +textbooks fetched into a dedicated arborist shard via the existing +`HtmlPageSource` ingest pipeline. + +## Files + +- `manifest-v1.jsonl` — one JSON record per textbook. First line is a + `_meta` block; remaining lines are entries. + +## Make targets + +```bash +make textbooks-summary # license + URL counts per entry +make textbooks-urls # one URL per line on stdout +make fetch-textbooks # ingest all manifested URLs into $(TEXTBOOK_DB) +make textbooks-stats # documents / chunks / edges in the textbook shard +make textbooks-verify # sample Merkle proof verification +``` + +`$(TEXTBOOK_DB)` defaults to `~/.arborist/textbooks.db`. Override with +`TEXTBOOK_DB=...` if you want to write to a different location. + +## License discipline (fail-closed) + +The helper `bench/scripts/textbooks_manifest.py` refuses to emit URLs +from entries with missing or disallowed license tokens. The allow-list: + +| Token | Meaning | +|---|---| +| `PD` | Public domain by age or explicit dedication | +| `CC0-1.0` | Public domain dedication | +| `CC-BY-2.5` / `CC-BY-3.0` / `CC-BY-4.0` | Creative Commons Attribution | +| `CC-BY-SA-2.5` / `CC-BY-SA-3.0` / `CC-BY-SA-4.0` | Attribution-ShareAlike | +| `GFDL-1.2` / `GFDL-1.3` | GNU Free Documentation License | +| `AGPL-3.0-only` / `Apache-2.0` / `MIT` | OSI-approved redistributable | + +**Excluded:** any `CC-BY-NC-*` (non-commercial clauses are incompatible +with arborist's AGPLv3 distribution profile), any `CC-BY-ND-*` +(no-derivatives prevents chunking), and any proprietary / educational- +use-only license. These belong on a separate "private research only" +manifest if needed; do not mix them in. + +## Schema + +Required fields per entry: + +| Field | Type | Notes | +|---|---|---| +| `id` | string | Hyphen-lowercase slug, must be unique | +| `title` | string | Human-readable book title | +| `author` | string | Authors / translators | +| `year` | string | Publication year (any reasonable string) | +| `license` | string | Token from the allow-list above | +| `license_url` | string | Canonical URL of the license text | +| `domain` | string | Domain tag (`logic`, `combinatorics`, …) | +| `urls` | array | Seed URLs ingested via HtmlPageSource | + +Optional but recommended: + +| Field | Type | Notes | +|---|---|---| +| `home_url` | string | Top-level entry point of the text | +| `pillar_targets` | array | Roman-numeral pillar IDs the text supports (e.g., `["I", "VII"]`) | +| `notes` | string | Free-text caveats / version pinning | + +## How the consistent process works + +1. `make textbooks-urls` invokes + `bench/scripts/textbooks_manifest.py urls`, which validates every + entry's license + emits its seed URLs (deduped). +2. `make fetch-textbooks` pipes that URL list into + `arborist ingest --source html --urls-from`. +3. `arborist/sources/html_page.py` (`HtmlPageSource`) handles each URL: + - Honors `robots.txt` (default: `respect_robots=True`). + - Strips noise (script / style / nav / footer / header / aside). + - Extracts main body text + outbound `` links as `Edge`s. + - Records loss reports for stripped-noise spans (#000022). +4. The standard ingest path takes over: 512-token chunker + per-doc + Merkle root + per-chunk inclusion proofs + audit-event with + chained sha256. +5. Idempotent re-ingest: same URL + same body → same `document_root` + → no-op insert. Same URL + different body → new doc + `supersedes` + edge (lossless history). + +This is the same pipeline every other surface uses; no new ingest +machinery was added for textbooks. + +## Adding a textbook + +1. Confirm the license. If `PD` / `CC0` / `CC-BY` / `CC-BY-SA` / `GFDL` + / OSI-approved → proceed. If `CC-BY-NC` / `ND` / proprietary → + stop; do not add to this manifest. +2. Append a JSON record to `manifest-v1.jsonl` with the schema above. +3. Run `make textbooks-summary` — confirm your entry appears. +4. Run `make fetch-textbooks` — ingest into `$(TEXTBOOK_DB)`. +5. `make textbooks-verify` — confirm Merkle proofs pass. + +## Going deeper on one textbook + +The manifest's `urls` field is a small seed list per entry — usually +just a TOC or representative chapter. To pull a full book: + +- **Append more chapter URLs to the manifest entry** (cheapest path). +- **OR** use `make crawl-ingest URL= DEPTH=N` (the existing + crawler) with the textbook's home URL. The crawler does BFS, + honors robots.txt, dedups by URL, and writes to a separate + `crawl_.db` shard. + +## Out of scope + +- **PDF processing.** The Internet Archive hosts PD textbooks as PDFs + (MacMahon's *Combinatory Analysis* 1915, Whitworth's *Choice and + Chance* 1867, the Motte 1729 *Principia*). Adding a `TextbookPdfSource` + with `pdftotext` or `pypdf` extraction is a separate ticket; the + current scope is HTML-shaped sources. +- **Mendelson + Enderton** (proprietary logic textbooks cited by the + claim-pack pillar I records). Out by license discipline; awaits a + separate decision per #000031 §2.1. +- **Wilf's *generatingfunctionology*.** The free PDF from UPenn + forbids commercial use and rehosting. Citable but not ingestible + here under our redistribution profile. + +## See also + +- Ticket #000031 — surface-ingest cited textbooks (warrant promotion + for claim-pack records). +- Ticket #000033 — claim-pack pillar VII (combinatorics) which cites + several entries on this manifest as primary sources. +- `arborist/sources/html_page.py` — the consistent-process source. diff --git a/bench/fixtures/textbooks/manifest-v1.jsonl b/bench/fixtures/textbooks/manifest-v1.jsonl new file mode 100644 index 0000000..bbfdddb --- /dev/null +++ b/bench/fixtures/textbooks/manifest-v1.jsonl @@ -0,0 +1,8 @@ +{"_meta":{"version":"v1","created":"2026-05-09","notes":"Manifest of public-domain or copyleft-redistributable math / logic / computer-science textbooks for surface-ingest into arborist (#000031). License discipline: PD = public domain by age; CC-BY-SA = redistributable with attribution + share-alike; GFDL = redistributable under GNU Free Documentation License. Excludes CC-BY-NC (non-commercial clauses incompatible with arborist's AGPLv3 distribution profile) and excludes proprietary texts. Each entry's `urls` field is the seed URL list ingested via HtmlPageSource. To grow coverage of a textbook, add more chapter URLs to its `urls` array; the existing crawler (`make crawl-ingest URL=`) is the alternative for full BFS."}} +{"id":"bogart-ctgd-2017","title":"Combinatorics Through Guided Discovery","author":"Kenneth P. Bogart","year":"2017 (estate-released open)","license":"GFDL-1.3","license_url":"https://www.gnu.org/licenses/fdl-1.3.html","domain":"combinatorics","pillar_targets":["VII"],"urls":["https://bogart.openmathbooks.org/ctgd/index.html"],"home_url":"https://bogart.openmathbooks.org/","notes":"Pedagogical combinatorics text. Author's estate explicitly opened under GFDL. Excellent fit for claim-pack pillar VII source citations."} +{"id":"keller-trotter-applied-comb-2017","title":"Applied Combinatorics","author":"Mitchel T. Keller, William T. Trotter","year":"2017+","license":"CC-BY-SA-4.0","license_url":"https://creativecommons.org/licenses/by-sa/4.0/","domain":"combinatorics","pillar_targets":["VII"],"urls":["https://www.appliedcombinatorics.org/appcomb/sec_pre.html"],"home_url":"https://www.appliedcombinatorics.org/appcomb/","notes":"AIM-approved open textbook from Georgia Tech. Covers permutations / combinations / inclusion-exclusion / generating functions / recurrence / Polya / graphs / discrete optimization."} +{"id":"levin-discrete-math-3rd","title":"Discrete Mathematics: An Open Introduction (3rd edition)","author":"Oscar Levin","year":"2019","license":"CC-BY-SA-4.0","license_url":"https://creativecommons.org/licenses/by-sa/4.0/","domain":"discrete-mathematics","pillar_targets":["I","II","III","VII"],"urls":["https://discrete.openmathbooks.org/dmoi3/frontmatter.html"],"home_url":"https://discrete.openmathbooks.org/dmoi3/","notes":"Discrete-math text covering logic, set theory, proof techniques, basic number theory, functions, relations, elementary combinatorics. 3rd ed is CC-BY-SA-4.0; the 4th edition switched to CC-BY-NC-SA which is incompatible with our distribution profile — pin to the 3rd."} +{"id":"hilbert-foundations-geometry-1902","title":"The Foundations of Geometry","author":"David Hilbert (E. J. Townsend, transl.)","year":"1899/1902 (Townsend translation)","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"geometry","pillar_targets":["IV"],"urls":["https://www.gutenberg.org/cache/epub/17384/pg17384-images.html"],"home_url":"https://www.gutenberg.org/ebooks/17384","notes":"Project Gutenberg eBook #17384. Hilbert's axiomatization of Euclidean geometry — foundational source cited by claim-pack pillar IV."} +{"id":"boole-laws-of-thought-1854","title":"An Investigation of the Laws of Thought","author":"George Boole","year":"1854","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"logic","pillar_targets":["I"],"urls":["https://www.gutenberg.org/cache/epub/15114/pg15114-images.html"],"home_url":"https://www.gutenberg.org/ebooks/15114","notes":"Project Gutenberg eBook #15114. Boole's foundational treatise establishing what became propositional logic / Boolean algebra. PD by age."} +{"id":"aristotle-prior-analytics-jenkinson","title":"Prior Analytics","author":"Aristotle (A. J. Jenkinson, transl.)","year":"~350 BCE; Jenkinson translation 1928","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"logic","pillar_targets":["I"],"urls":["https://en.wikisource.org/wiki/Prior_Analytics"],"home_url":"https://en.wikisource.org/wiki/Prior_Analytics","notes":"Wikisource HTML. Aristotle's foundational treatise on syllogistic logic. Modus ponens / modus tollens / law of excluded middle all trace back here."} +{"id":"morin-open-data-structures","title":"Open Data Structures","author":"Pat Morin","year":"2013+","license":"CC-BY-2.5","license_url":"https://creativecommons.org/licenses/by/2.5/","domain":"computer-science","pillar_targets":[],"urls":["https://opendatastructures.org/ods-cpp.html","https://opendatastructures.org/ods-python.html"],"home_url":"https://opendatastructures.org/","notes":"Comprehensive open data-structures text. CC-BY-2.5 — fully redistributable. CS algorithms / complexity / asymptotic analysis foundation."} diff --git a/bench/scripts/textbooks_manifest.py b/bench/scripts/textbooks_manifest.py new file mode 100644 index 0000000..5b8fed2 --- /dev/null +++ b/bench/scripts/textbooks_manifest.py @@ -0,0 +1,132 @@ +"""Read the textbook manifest and emit URLs / license / metadata helpers. + +Usage: + python -m bench.scripts.textbooks_manifest urls < manifest.jsonl + → one URL per line on stdout + + python -m bench.scripts.textbooks_manifest summary < manifest.jsonl + → human-readable license summary + +The manifest format is one JSON record per line. The first record may be a +``_meta`` block; it is ignored by all subcommands. See +``bench/fixtures/textbooks/manifest-v1.jsonl`` for the canonical schema. + +Hard discipline: every entry MUST carry ``license`` and ``license_url``. +This script refuses to emit URLs from entries missing those fields, so a +fetcher run is never the place a license-discipline bug becomes visible — +it surfaces here. +""" + +from __future__ import annotations + +import json +import sys +from typing import Iterator, TextIO + + +_REQUIRED_LICENSE_KEYS = ("license", "license_url") +# Allow-list of license tokens we redistribute under arborist's AGPLv3 +# distribution profile. CC-BY-NC, CC-BY-ND, and proprietary licenses are +# explicitly rejected — they don't fit the AGPLv3 downstream-distribution +# contract or arborist's "every patch ships" mission. +_ALLOWED_LICENSES = frozenset({ + "PD", + "CC0-1.0", + "CC-BY-2.5", "CC-BY-3.0", "CC-BY-4.0", + "CC-BY-SA-2.5", "CC-BY-SA-3.0", "CC-BY-SA-4.0", + "GFDL-1.2", "GFDL-1.3", + "AGPL-3.0-only", + "Apache-2.0", + "MIT", +}) + + +def iter_entries(stream: TextIO) -> Iterator[dict]: + """Yield non-meta entries from a JSONL manifest.""" + for line in stream: + line = line.strip() + if not line: + continue + obj = json.loads(line) + if "_meta" in obj: + continue + yield obj + + +def _validate(entry: dict) -> None: + """Raise ValueError if an entry is missing required license fields or + declares a license outside the redistribution allow-list.""" + for k in _REQUIRED_LICENSE_KEYS: + if k not in entry or not entry[k]: + raise ValueError( + f"manifest entry {entry.get('id', '?')!r} missing required " + f"field {k!r}" + ) + if entry["license"] not in _ALLOWED_LICENSES: + raise ValueError( + f"manifest entry {entry.get('id', '?')!r} declares license " + f"{entry['license']!r} outside arborist redistribution allow-list " + f"({sorted(_ALLOWED_LICENSES)})" + ) + + +def cmd_urls(stream: TextIO) -> int: + seen: set[str] = set() + for entry in iter_entries(stream): + _validate(entry) + for url in entry.get("urls", []): + if url and url not in seen: + seen.add(url) + print(url) + return 0 + + +def cmd_summary(stream: TextIO) -> int: + rows = [] + by_license: dict[str, int] = {} + by_domain: dict[str, int] = {} + total_urls = 0 + for entry in iter_entries(stream): + _validate(entry) + rows.append(entry) + by_license[entry["license"]] = by_license.get(entry["license"], 0) + 1 + d = entry.get("domain", "?") + by_domain[d] = by_domain.get(d, 0) + 1 + total_urls += len(entry.get("urls", [])) + + print(f"# Textbook manifest summary") + print(f"entries: {len(rows)}") + print(f"seed URLs: {total_urls}") + print() + print("## by license") + for k, n in sorted(by_license.items()): + print(f" {k:20s} {n}") + print() + print("## by domain") + for k, n in sorted(by_domain.items()): + print(f" {k:20s} {n}") + print() + print("## entries") + for e in rows: + urls_n = len(e.get("urls", [])) + title = e.get("title", "?") + license_ = e.get("license", "?") + print(f" {e.get('id'):42s} {license_:18s} {urls_n} URL(s) {title}") + return 0 + + +_COMMANDS = {"urls": cmd_urls, "summary": cmd_summary} + + +def main(argv: list[str]) -> int: + if len(argv) < 2 or argv[1] not in _COMMANDS: + print( + f"usage: {argv[0]} {{{'|'.join(sorted(_COMMANDS))}}} < manifest.jsonl", + file=sys.stderr, + ) + return 2 + return _COMMANDS[argv[1]](sys.stdin) + + +if __name__ == "__main__": + sys.exit(main(sys.argv))