textbooks: PD/open-licensed math+logic+CS surface-ingest pipeline
A make-target scaffold for pulling public-domain and copyleft- redistributable textbooks into a dedicated arborist shard via the existing HtmlPageSource ingest pipeline. No new ingest machinery — the existing path (robots.txt → noise-strip → 512-token chunk → Merkle root → audit-event) IS the consistent process. Manifest format =============== bench/fixtures/textbooks/manifest-v1.jsonl — one JSON record per textbook with explicit license tokens (PD / CC-BY / CC-BY-SA / GFDL / OSI-approved). Fail-closed validation in bench/scripts/textbooks_manifest.py refuses to emit URLs from entries with missing or disallowed license tokens, so license discipline surfaces at the URL-emit step rather than getting silently buried in a fetch run. Allow-list excludes CC-BY-NC (incompatible with arborist's AGPLv3 distribution profile) and CC-BY-ND (no-derivatives prevents chunking). Wilf's generatingfunctionology stays out because its educational-use license forbids rehosting; citable but not redistributable. Initial entries (7 textbooks, 8 seed URLs) ========================================== - Bogart, Combinatorics Through Guided Discovery — GFDL-1.3 - Keller & Trotter, Applied Combinatorics — CC-BY-SA-4.0 - Levin, Discrete Mathematics: An Open Introduction (3rd ed) — CC-BY-SA-4.0 (4th ed switched to NC; pin 3rd) - Hilbert, Foundations of Geometry (Townsend 1902) — PD via PG - Boole, An Investigation of the Laws of Thought (1854) — PD - Aristotle, Prior Analytics (Jenkinson) — PD via Wikisource - Morin, Open Data Structures — CC-BY-2.5 Covers pillars I (logic) / II (set theory) / III (arithmetic) / IV (geometry) / VII (combinatorics) on the claim-pack (#000029) layout, plus a CS anchor for downstream domain expansion. Each entry's pillar_targets field lists which claim-pack pillars its records are candidate citations for. Make targets ============ - textbooks-summary — license + URL counts per entry - textbooks-urls — flat URL list to stdout - fetch-textbooks — ingest all manifested URLs into $(TEXTBOOK_DB) (default ~/.arborist/textbooks.db) - textbooks-stats — documents / chunks / edges - textbooks-verify — sample Merkle proof verification To grow coverage of one textbook: append more chapter URLs to its `urls` array. For deep-BFS of a textbook home: use the existing `make crawl-ingest URL=<base> DEPTH=N` instead. Smoke-tested end-to-end against Wikisource Prior Analytics: 1 doc / 1 chunk / 12 outbound edges / Merkle proof passes. Shows the pipeline works; populating each book to depth needs either more URLs in the manifest or the crawler. Out of scope ============ - PDF processing. The Internet Archive hosts PD textbooks (MacMahon's Combinatory Analysis 1915, Whitworth's Choice and Chance 1867, the Motte 1729 Principia) as scanned PDFs; a TextbookPdfSource with pdftotext / pypdf extraction is a separate ticket — current scope is HTML-shaped sources. - Mendelson + Enderton (proprietary; await #000031 §2.1 decision). - Wilf generatingfunctionology (license forbids redistribution). Test suite stays at 1554 passed / 28 skipped — no source-code changes to arborist itself; the textbook layer is pure tooling on top of the existing pipeline.
This commit is contained in:
parent
951002c372
commit
4a199c9dad
4 changed files with 316 additions and 1 deletions
132
bench/scripts/textbooks_manifest.py
Normal file
132
bench/scripts/textbooks_manifest.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
"""Read the textbook manifest and emit URLs / license / metadata helpers.
|
||||
|
||||
Usage:
|
||||
python -m bench.scripts.textbooks_manifest urls < manifest.jsonl
|
||||
→ one URL per line on stdout
|
||||
|
||||
python -m bench.scripts.textbooks_manifest summary < manifest.jsonl
|
||||
→ human-readable license summary
|
||||
|
||||
The manifest format is one JSON record per line. The first record may be a
|
||||
``_meta`` block; it is ignored by all subcommands. See
|
||||
``bench/fixtures/textbooks/manifest-v1.jsonl`` for the canonical schema.
|
||||
|
||||
Hard discipline: every entry MUST carry ``license`` and ``license_url``.
|
||||
This script refuses to emit URLs from entries missing those fields, so a
|
||||
fetcher run is never the place a license-discipline bug becomes visible —
|
||||
it surfaces here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from typing import Iterator, TextIO
|
||||
|
||||
|
||||
_REQUIRED_LICENSE_KEYS = ("license", "license_url")
|
||||
# Allow-list of license tokens we redistribute under arborist's AGPLv3
|
||||
# distribution profile. CC-BY-NC, CC-BY-ND, and proprietary licenses are
|
||||
# explicitly rejected — they don't fit the AGPLv3 downstream-distribution
|
||||
# contract or arborist's "every patch ships" mission.
|
||||
_ALLOWED_LICENSES = frozenset({
|
||||
"PD",
|
||||
"CC0-1.0",
|
||||
"CC-BY-2.5", "CC-BY-3.0", "CC-BY-4.0",
|
||||
"CC-BY-SA-2.5", "CC-BY-SA-3.0", "CC-BY-SA-4.0",
|
||||
"GFDL-1.2", "GFDL-1.3",
|
||||
"AGPL-3.0-only",
|
||||
"Apache-2.0",
|
||||
"MIT",
|
||||
})
|
||||
|
||||
|
||||
def iter_entries(stream: TextIO) -> Iterator[dict]:
|
||||
"""Yield non-meta entries from a JSONL manifest."""
|
||||
for line in stream:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
obj = json.loads(line)
|
||||
if "_meta" in obj:
|
||||
continue
|
||||
yield obj
|
||||
|
||||
|
||||
def _validate(entry: dict) -> None:
|
||||
"""Raise ValueError if an entry is missing required license fields or
|
||||
declares a license outside the redistribution allow-list."""
|
||||
for k in _REQUIRED_LICENSE_KEYS:
|
||||
if k not in entry or not entry[k]:
|
||||
raise ValueError(
|
||||
f"manifest entry {entry.get('id', '?')!r} missing required "
|
||||
f"field {k!r}"
|
||||
)
|
||||
if entry["license"] not in _ALLOWED_LICENSES:
|
||||
raise ValueError(
|
||||
f"manifest entry {entry.get('id', '?')!r} declares license "
|
||||
f"{entry['license']!r} outside arborist redistribution allow-list "
|
||||
f"({sorted(_ALLOWED_LICENSES)})"
|
||||
)
|
||||
|
||||
|
||||
def cmd_urls(stream: TextIO) -> int:
|
||||
seen: set[str] = set()
|
||||
for entry in iter_entries(stream):
|
||||
_validate(entry)
|
||||
for url in entry.get("urls", []):
|
||||
if url and url not in seen:
|
||||
seen.add(url)
|
||||
print(url)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_summary(stream: TextIO) -> int:
|
||||
rows = []
|
||||
by_license: dict[str, int] = {}
|
||||
by_domain: dict[str, int] = {}
|
||||
total_urls = 0
|
||||
for entry in iter_entries(stream):
|
||||
_validate(entry)
|
||||
rows.append(entry)
|
||||
by_license[entry["license"]] = by_license.get(entry["license"], 0) + 1
|
||||
d = entry.get("domain", "?")
|
||||
by_domain[d] = by_domain.get(d, 0) + 1
|
||||
total_urls += len(entry.get("urls", []))
|
||||
|
||||
print(f"# Textbook manifest summary")
|
||||
print(f"entries: {len(rows)}")
|
||||
print(f"seed URLs: {total_urls}")
|
||||
print()
|
||||
print("## by license")
|
||||
for k, n in sorted(by_license.items()):
|
||||
print(f" {k:20s} {n}")
|
||||
print()
|
||||
print("## by domain")
|
||||
for k, n in sorted(by_domain.items()):
|
||||
print(f" {k:20s} {n}")
|
||||
print()
|
||||
print("## entries")
|
||||
for e in rows:
|
||||
urls_n = len(e.get("urls", []))
|
||||
title = e.get("title", "?")
|
||||
license_ = e.get("license", "?")
|
||||
print(f" {e.get('id'):42s} {license_:18s} {urls_n} URL(s) {title}")
|
||||
return 0
|
||||
|
||||
|
||||
_COMMANDS = {"urls": cmd_urls, "summary": cmd_summary}
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) < 2 or argv[1] not in _COMMANDS:
|
||||
print(
|
||||
f"usage: {argv[0]} {{{'|'.join(sorted(_COMMANDS))}}} < manifest.jsonl",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
return _COMMANDS[argv[1]](sys.stdin)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
Loading…
Add table
Add a link
Reference in a new issue