From 57dafbdc3dc3d428f93f34f9ca4abd3fcab1bc3d Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 9 May 2026 15:59:31 -0400 Subject: [PATCH] textbooks: per-book make targets + idempotency docs + vital-books checklist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three concrete improvements to the textbook ingest layer: Per-book make targets ===================== - `make textbook ID=` — generic, looks up one entry from the manifest and runs the right ingest path (BFS crawl if `crawl_url` is set, shallow URL ingest otherwise). - `make textbook-list` — list ingestable textbook ids. - Convenience targets per active manifest entry: textbook-bogart, textbook-keller-trotter, textbook-levin, textbook-aristotle-prior, textbook-aristotle-posterior, textbook-newton, textbook-morin Each per-book target writes to a dedicated shard at `$(CRAWL_SHARDS_DIR)/textbook_.db` — one shard per textbook id rather than one per host (which would conflate multiple Wikisource books — Aristotle Prior, Posterior, Newton — into one shard). The manifest helper script gains `ids` and `lookup ` subcommands so the Makefile can introspect manifest entries without grep gymnastics. Idempotency contract documented ================================ Every ingest path is content-addressed. Same content → same `document_root` → no-op insert. Re-running `make textbook-*` or `make crawl-textbooks` on the same network state is safe at the database layer — no duplicates, no schema drift, no audit-event proliferation. HTTP refetch happens every run; pair with `make recrawl-check` for a HEAD-only freshness probe. README rewritten to make this contract explicit, with shard- layout table + crawl-delay caveats consolidated. Vital-books checklist ===================== The README now carries a g4-pillar coverage table: Pillar I Logic ✓ Levin + Aristotle Prior + Posterior Pillar II Set Theory ✓ Levin Pillar III Arithmetic ✓ Levin Pillar IV Geometry ✗ Hilbert PG #17384 (PDF/TeX only) Pillar V Probability ✗ Kolmogorov license analysis pending Pillar VI Phys. Newton ✓ Newton Principia Motte Pillar VII Combin. ✓ Bogart + Keller-Trotter + Levin Pillar IX λ-Calculus ✗ Church + Turing 1936 papers pending 5 of 7 pillars now have surface coverage. The remaining gaps are explicit license (V) or format (IV PDF/TeX, IX paper-level) issues, each documented in the corresponding manifest entry's notes field. Test suite stays at 1554 passed / 28 skipped — no source-code changes to arborist itself; the textbook layer is pure tooling on top of the existing pipeline. --- Makefile | 69 +++++++++++++++++++- bench/fixtures/textbooks/README.md | 97 +++++++++++++++++++++++------ bench/scripts/textbooks_manifest.py | 57 ++++++++++++++++- 3 files changed, 200 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index 1bd51e2..3664635 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ SEARCH_Q ?= computer falsify burn burn-kindergarten inspect bootstrap-crawler test-crawler crawl-ingest \ recrawl-check bench-qa bootstrap-math clean clean-db clean-data help \ textbooks-summary textbooks-urls fetch-textbooks textbooks-stats textbooks-verify \ - crawl-textbooks crawl-textbooks-stats + crawl-textbooks crawl-textbooks-stats textbook textbook-list all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats @@ -703,6 +703,73 @@ crawl-textbooks-stats: bootstrap ## docs-per-shard summary across textbook crawl printf " %-60s docs=%-5s chunks=%-5s size=%s\n" "$$(basename $$shard)" "$$docs" "$$chunks" "$$size"; \ done +# Per-book idempotent ingest: `make textbook ID=` looks up one +# entry from the manifest and runs the right ingest path (HTML +# urls vs BFS crawl). Re-running on a shard already ingested is a +# no-op at the document level — same content_root → no insert. The +# crawl path re-fetches HTTP every run; pair with `make recrawl-check` +# for a HEAD-only freshness probe before re-ingest. +textbook: bootstrap-crawler ## ingest one textbook by id: make textbook ID=bogart-ctgd-2017 + @if [ -z "$(ID)" ]; then \ + echo "usage: make textbook ID= (use 'make textbook-list' for available ids)" >&2; \ + exit 2; \ + fi + @row=$$($(PY) -m bench.scripts.textbooks_manifest lookup $(ID) < $(TEXTBOOK_MANIFEST)) || exit 1; \ + id=$$(echo "$$row" | cut -f1); \ + url=$$(echo "$$row" | cut -f2); \ + depth=$$(echo "$$row" | cut -f3); \ + max=$$(echo "$$row" | cut -f4); \ + license=$$(echo "$$row" | cut -f5); \ + domain=$$(echo "$$row" | cut -f6); \ + mkdir -p $(CRAWL_SHARDS_DIR); \ + shard="$(CRAWL_SHARDS_DIR)/textbook_$${id}.db"; \ + echo ">> $$id ($$license, $$domain)"; \ + echo " crawl: $$url depth=$$depth max=$$max"; \ + echo " shard: $$shard"; \ + if [ -n "$$url" ]; then \ + $(ARBORIST) --db "$$shard" crawl --seed-url "$$url" --depth $$depth --max-pages $$max --ingest > /tmp/textbook-$${id}.log 2>&1 && \ + docs=$$(.venv/bin/python -c "import sqlite3; print(sqlite3.connect(\"$$shard\").execute(\"SELECT COUNT(*) FROM documents\").fetchone()[0])"); \ + chunks=$$(.venv/bin/python -c "import sqlite3; print(sqlite3.connect(\"$$shard\").execute(\"SELECT COUNT(*) FROM chunks\").fetchone()[0])"); \ + echo " landed: $$docs docs / $$chunks chunks"; \ + else \ + echo " no crawl_url for $$id; using shallow ingest path"; \ + echo "$$id" | $(PY) -c 'import sys, json, os; sys.path.insert(0, "."); ids = sys.stdin.read().split(); manifest = "$(TEXTBOOK_MANIFEST)"; [print(u) for line in open(manifest) for e in [json.loads(line)] if "_meta" not in e and e.get("id") in ids for u in e.get("urls", [])]' > /tmp/textbook-$${id}-urls.txt; \ + $(ARBORIST) --db "$$shard" ingest --source html --urls-from /tmp/textbook-$${id}-urls.txt; \ + rm -f /tmp/textbook-$${id}-urls.txt; \ + fi + +textbook-list: bootstrap ## list ingestable textbook ids (entries with urls or crawl_url) + @$(PY) -m bench.scripts.textbooks_manifest ids < $(TEXTBOOK_MANIFEST) + +# Per-book convenience targets — one per active manifest id. Re-running +# any of these is idempotent at the database layer (content-addressed +# inserts) but does refetch HTTP. Use `make recrawl-check` first if you +# want a freshness probe before paying network cost. +.PHONY: textbook-bogart textbook-keller-trotter textbook-levin \ + textbook-aristotle-prior textbook-aristotle-posterior \ + textbook-newton textbook-morin + +textbook-bogart: ## Bogart Combinatorics Through Guided Discovery (GFDL) + $(MAKE) textbook ID=bogart-ctgd-2017 + +textbook-keller-trotter: ## Keller-Trotter Applied Combinatorics (CC-BY-SA, slow ~25min crawl-delay) + $(MAKE) textbook ID=keller-trotter-applied-comb-2017 + +textbook-levin: ## Levin Discrete Mathematics (CC-BY-SA) + $(MAKE) textbook ID=levin-discrete-math-3rd + +textbook-aristotle-prior: ## Aristotle Prior Analytics (PD, Wikisource) + $(MAKE) textbook ID=aristotle-prior-analytics-jenkinson + +textbook-aristotle-posterior: ## Aristotle Posterior Analytics (PD, Wikisource) + $(MAKE) textbook ID=aristotle-posterior-analytics + +textbook-newton: ## Newton Principia (PD, Wikisource) + $(MAKE) textbook ID=newton-principia-motte + +textbook-morin: ## Morin Open Data Structures (CC-BY) + $(MAKE) textbook ID=morin-open-data-structures + test-crawler: bootstrap-crawler ## run only the lifted crawler tests $(VENV)/bin/pytest -q tests/crawler diff --git a/bench/fixtures/textbooks/README.md b/bench/fixtures/textbooks/README.md index 736ec04..f4a8717 100644 --- a/bench/fixtures/textbooks/README.md +++ b/bench/fixtures/textbooks/README.md @@ -11,34 +11,73 @@ textbooks fetched into a dedicated arborist shard via the existing ## Make targets -Two ingest paths — pick based on how much depth you need: +Three ingest paths, idempotent at the database layer: ```bash -# Shallow: explicit URL list per entry → single textbook shard +# Manifest inspection make textbooks-summary # license + URL counts per entry make textbooks-urls # one URL per line on stdout -make fetch-textbooks # ingest manifest `urls` into $(TEXTBOOK_DB) -make textbooks-stats # documents / chunks / edges -make textbooks-verify # sample Merkle proof verification +make textbook-list # ingestable textbook ids -# Deep: BFS crawler on each entry's `crawl_url` → one shard per host -make crawl-textbooks # crawl every entry with a `crawl_url` -make crawl-textbooks-stats # docs/chunks/size summary across crawl shards +# All-in-one: shallow URL ingest into a single shard +make fetch-textbooks # → $(TEXTBOOK_DB) (default ~/.arborist/textbooks.db) +make textbooks-stats +make textbooks-verify + +# All-in-one: deep BFS crawl, one shard per textbook id +make crawl-textbooks # → $(CRAWL_SHARDS_DIR)/textbook_.db each +make crawl-textbooks-stats + +# Per-book: refresh / ingest one textbook +make textbook ID= # generic; works on any manifest entry id +make textbook-bogart # convenience target +make textbook-keller-trotter # convenience target (slow ~25min) +make textbook-levin # convenience target +make textbook-aristotle-prior # convenience target +make textbook-aristotle-posterior # convenience target +make textbook-newton # convenience target +make textbook-morin # convenience target (Open Data Structures) ``` -The shallow path is fast but limited to the URLs you list — fine for -spot-citing or smoke tests. The deep path uses the existing crawler -(`make crawl-ingest`) per textbook home, honors robots.txt + crawl -delays, and yields full-book coverage. +## Idempotency -`$(TEXTBOOK_DB)` defaults to `~/.arborist/textbooks.db`. Crawl shards -land at `$(SHARDS_DIR)/crawl_.db`. Override either via -environment variable. +**Database-layer:** every ingest path is content-addressed. Same +content → same `document_root` → no-op insert. Re-running any +target on the same data is safe — no duplicates, no schema +changes, no audit-event proliferation. -**Crawl-Delay caveat:** `appliedcombinatorics.org`'s `robots.txt` -declares `Crawl-Delay: 20` — full BFS of Keller-Trotter takes ~25 -minutes. Background it (`&`) and let it run; the others all complete -in seconds-to-minutes. +**Network-layer:** `make textbook-*` and `make crawl-textbooks` +do refetch HTTP every run. To probe freshness without paying +network cost, use the existing `make recrawl-check` target — it +sends conditional HEAD requests (`If-None-Match`, +`If-Modified-Since`) and classifies each ingested doc as +fresh / stale / gone. Pair with selective re-ingest based on +that report. + +The shallow `fetch-textbooks` path is fast but limited to the URLs +you list in the manifest's `urls` field. The deep `crawl-textbooks` +path uses the existing crawler per textbook home, honors robots.txt ++ crawl delays, and yields full-book coverage. The per-book targets +(`textbook-*`) call into the deep path with one entry's parameters. + +## Shard layout + +| Target | Shard path | +|---|---| +| `fetch-textbooks` | `$(TEXTBOOK_DB)` (single) — default `~/.arborist/textbooks.db` | +| `crawl-textbooks` / `textbook ID=...` | `$(CRAWL_SHARDS_DIR)/textbook_.db` (one per textbook) — default `~/.arborist/crawl/` | + +Crawl shards live in `~/.arborist/crawl/` to keep them out of the +main `~/.arborist/shards/` directory — SQLite caps simultaneous +attached databases at 10, so co-mingling 4 main shards + N crawl +shards trips the limit on cross-shard queries. + +## Crawl-Delay caveat + +`appliedcombinatorics.org`'s `robots.txt` declares +`Crawl-Delay: 20` — full BFS of Keller-Trotter takes ~25 minutes. +Background it (`make textbook-keller-trotter &`) and let it run; +the others all complete in seconds-to-minutes. ## License discipline (fail-closed) @@ -151,6 +190,26 @@ just a TOC or representative chapter. To pull a full book: forbids commercial use and rehosting. Citable but not ingestible here under our redistribution profile. +## Vital-books checklist + +Coverage by g4 pillar (per the claim-pack source #000029): + +| Pillar | Domain | Status | Source | +|---|---|---|---| +| I | Logic | ✓ landed | Levin (CC-BY-SA), Aristotle Prior (PD), Aristotle Posterior (PD) | +| II | Set Theory | ✓ landed | Levin (CC-BY-SA) — covers ZF basics | +| III | Arithmetic (Peano) | ✓ landed | Levin (CC-BY-SA) — covers Peano basics | +| IV | Geometry | ✗ pending | Hilbert PG #17384 — PDF/TeX only, awaits TeX-source | +| V | Probability | ✗ pending | Kolmogorov 1933 license analysis pending | +| VI | Classical Physics | ✓ landed | Newton Principia Motte (PD, Wikisource) | +| VII | Combinatorics | ✓ landed | Bogart (GFDL), Keller-Trotter (CC-BY-SA, slow), Levin | +| IX | λ-Calculus | ✗ pending | Church 1936 + Turing 1936 papers; need separate ingest path | + +**5 of 7 pillars** have surface coverage; remaining gaps are +explicit license (V) or format (IV PDF/TeX, IX paper-level) +issues, each documented in the corresponding manifest entry's +`notes` field. + ## See also - Ticket #000031 — surface-ingest cited textbooks (warrant promotion diff --git a/bench/scripts/textbooks_manifest.py b/bench/scripts/textbooks_manifest.py index cb7b174..ebdeba8 100644 --- a/bench/scripts/textbooks_manifest.py +++ b/bench/scripts/textbooks_manifest.py @@ -127,9 +127,9 @@ def cmd_summary(stream: TextIO) -> int: def cmd_crawl_targets(stream: TextIO) -> int: - """Emit one tab-separated `crawl_url\\tdepth\\tmax` triple per entry + """Emit one tab-separated `crawl_url\\tdepth\\tmax\\tid` row per entry that declares a `crawl_url`. Used by `make crawl-textbooks` to drive - the BFS crawler, one shard per host. + the BFS crawler, one shard per textbook id. """ for entry in iter_entries(stream): _validate(entry) @@ -142,7 +142,58 @@ def cmd_crawl_targets(stream: TextIO) -> int: return 0 -_COMMANDS = {"urls": cmd_urls, "summary": cmd_summary, "crawl-targets": cmd_crawl_targets} +def cmd_ids(stream: TextIO) -> int: + """Emit one entry id per line for entries that have either `urls` or + `crawl_url`. License-fail placeholders (no fetchable URLs) are + excluded — they have nothing to ingest. + """ + for entry in iter_entries(stream): + try: + _validate(entry) + except ValueError: + continue + if entry.get("urls") or entry.get("crawl_url"): + print(entry.get("id", "")) + return 0 + + +def cmd_lookup(stream: TextIO) -> int: + """Read entry id from sys.argv[2]; emit tab-separated + `id\\tcrawl_url\\tdepth\\tmax\\tlicense\\tdomain` for the matching + entry. Used by per-book make targets to look up one textbook's + crawl parameters from the manifest without grep wizardry. + """ + target_id = sys.argv[2] if len(sys.argv) > 2 else "" + if not target_id: + print("usage: lookup ", file=sys.stderr) + return 2 + for entry in iter_entries(stream): + if entry.get("id") != target_id: + continue + try: + _validate(entry) + except ValueError as exc: + print(f"# license-fail placeholder: {exc}", file=sys.stderr) + return 3 + crawl_url = entry.get("crawl_url", "") + depth = entry.get("crawl_depth", 2) + max_pages = entry.get("crawl_max", 80) + print( + f"{entry.get('id')}\t{crawl_url}\t{depth}\t{max_pages}\t" + f"{entry.get('license')}\t{entry.get('domain')}" + ) + return 0 + print(f"# id {target_id!r} not found in manifest", file=sys.stderr) + return 4 + + +_COMMANDS = { + "urls": cmd_urls, + "summary": cmd_summary, + "crawl-targets": cmd_crawl_targets, + "ids": cmd_ids, + "lookup": cmd_lookup, +} def main(argv: list[str]) -> int: