From f5c2c03d6a0e535c8b4d7c3a7b1a33f5653d5b35 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 9 May 2026 15:47:42 -0400 Subject: [PATCH] textbooks: extend manifest for #000031 PD scope (Newton, Posterior Analytics, +placeholders) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the #000031 implementation track — surface-ingest cited textbooks for warrant promotion of claim-pack records. Five new manifest entries spanning the PD-scope additions plus license-fail placeholders for citation traceability: newton-principia-motte (PD) Newton's Mathematical Principles of Natural Philosophy (Motte 1729 / Motte-Cajori 1846 English translations) via Wikisource. Crawled @ depth=2: 60 docs, 289 chunks, 10K edges. FTS5 spot-checks: "law of motion" → top hit on the 1729 Motte title page; "gravitation" → Newton-titled chapters in the Wikisource Physics category. Cited by claim-pack pillar VI. aristotle-posterior-analytics (PD) Mure / Owen translations via Wikisource. Crawled @ depth=1: 20 docs, 71 chunks. FTS5: "syllogism" → Posterior Analytics Bouchier + Owen translations as top hits. Companion to Prior Analytics already in the manifest; together cover pillar I logic axioms. kolmogorov-foundations-probability-1933 (placeholder) German original PD-by-age in EU; US copyright restored via URAA through life+70 (Kolmogorov d.1987 → US PD 2058). Morrison 1956 English translation: Chelsea Publishing copyright. Entry kept with empty urls field for citation traceability; license analysis pending. lukasiewicz-aristotle-syllogistic-1951 (proprietary placeholder) Oxford University Press, copyright through life+70 (Łukasiewicz d.1956 → PD 2027 EU, longer US via URAA). NOT redistributable. Kept as a license-fail placeholder; the 1921 Polish primary papers are themselves PD by age but lack accessible HTML editions. License-validator drive-by ========================== The textbooks_manifest.py validator now treats placeholder entries (urls + crawl_url both empty) as exempt from the allow-list license check. License discipline stays fail-closed at the URL-emit step — placeholders without URLs raise no exception under any license token, so we can document yellow-light proprietary citations (Mendelson, Enderton, Łukasiewicz) without hacks. Coverage summary post-commit ============================ g4 pillar | textbook coverage ─────────────┼───────────────────────────────────────────────── I Logic | Levin (CC-BY-SA), Aristotle Prior + Posterior (PD) II Set Theory| Levin (CC-BY-SA) III Arithm. | Levin (CC-BY-SA) IV Geometry | (Hilbert PG #17384 placeholder — PDF/TeX only) V Probability| (Kolmogorov 1933 license analysis pending) VI Phys. | Newton Principia Motte (PD) VII Combin. | Bogart (GFDL), Keller-Trotter (CC-BY-SA), Levin Out-of-scope follow-ups: PDF/TeX source support (#000031 §5), Mendelson + Enderton license decision (#000031 §2.1), chunk-resolution + derivations.proof_blob warrant promotion (separate ticket). --- bench/fixtures/textbooks/manifest-v1.jsonl | 4 ++++ bench/scripts/textbooks_manifest.py | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/bench/fixtures/textbooks/manifest-v1.jsonl b/bench/fixtures/textbooks/manifest-v1.jsonl index b595dd6..05c416b 100644 --- a/bench/fixtures/textbooks/manifest-v1.jsonl +++ b/bench/fixtures/textbooks/manifest-v1.jsonl @@ -6,3 +6,7 @@ {"id":"boole-laws-of-thought-1854","title":"An Investigation of the Laws of Thought","author":"George Boole","year":"1854","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"logic","pillar_targets":["I"],"urls":[],"home_url":"https://www.gutenberg.org/ebooks/15114","tex_url":"https://www.gutenberg.org/files/15114/15114-t/15114-t.tex","pdf_url":"https://www.gutenberg.org/files/15114/15114-pdf.pdf","notes":"Project Gutenberg eBook #15114 — PDF/TeX only, no HTML format. URLs intentionally empty until PDF / TeX source support lands. Boole's foundational treatise establishing propositional logic / Boolean algebra."} {"id":"aristotle-prior-analytics-jenkinson","title":"Prior Analytics","author":"Aristotle (A. J. Jenkinson, transl.)","year":"~350 BCE; Jenkinson translation 1928","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"logic","pillar_targets":["I"],"urls":["https://en.wikisource.org/wiki/Prior_Analytics"],"home_url":"https://en.wikisource.org/wiki/Prior_Analytics","crawl_url":"https://en.wikisource.org/wiki/Prior_Analytics","crawl_depth":1,"crawl_max":20,"notes":"Wikisource HTML. The /wiki/Prior_Analytics page is a disambiguation/landing — only ~75% complete as raw text; the deeper transcription lives at /wiki/Index:O._F._Owen%27s_Organon_of_Aristotle_Vol._1_(1853).djvu and its /Page: subpages, which the crawler can follow at depth=2+."} {"id":"morin-open-data-structures","title":"Open Data Structures","author":"Pat Morin","year":"2013+","license":"CC-BY-2.5","license_url":"https://creativecommons.org/licenses/by/2.5/ca/","domain":"computer-science","pillar_targets":[],"urls":["https://opendatastructures.org/ods-cpp/","https://opendatastructures.org/ods-python/","https://opendatastructures.org/ods-java/"],"home_url":"https://opendatastructures.org/","crawl_url":"https://opendatastructures.org/","crawl_depth":3,"crawl_max":120,"notes":"Comprehensive open data-structures text. CC-BY-2.5 (Canadian port) — fully redistributable. Three language editions (C++, Python, Java) of the same content. ODS crawl @ depth=3 yields ~64 chapters / ~80KB across all three editions."} +{"id":"newton-principia-motte","title":"The Mathematical Principles of Natural Philosophy","author":"Isaac Newton (Andrew Motte transl., 1729 / Motte-Cajori 1846)","year":"1687/1729/1846","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"classical-physics","pillar_targets":["VI"],"urls":["https://en.wikisource.org/wiki/The_Mathematical_Principles_of_Natural_Philosophy"],"home_url":"https://en.wikisource.org/wiki/The_Mathematical_Principles_of_Natural_Philosophy","crawl_url":"https://en.wikisource.org/wiki/The_Mathematical_Principles_of_Natural_Philosophy","crawl_depth":2,"crawl_max":60,"notes":"Wikisource HTML, English Motte 1729 / Motte-Cajori 1846 translations. Newton's foundational treatise — Laws of Motion + universal gravitation. Cited by claim-pack pillar VI records."} +{"id":"aristotle-posterior-analytics","title":"Posterior Analytics","author":"Aristotle (G. R. G. Mure, transl.)","year":"~350 BCE; Mure translation 1928","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"logic","pillar_targets":["I"],"urls":["https://en.wikisource.org/wiki/Posterior_Analytics"],"home_url":"https://en.wikisource.org/wiki/Posterior_Analytics","crawl_url":"https://en.wikisource.org/wiki/Posterior_Analytics","crawl_depth":1,"crawl_max":20,"notes":"Wikisource HTML. Aristotle's treatise on demonstrative knowledge, companion to Prior Analytics. The traditional Mure / Owen translations are PD by age."} +{"id":"kolmogorov-foundations-probability-1933","title":"Foundations of the Theory of Probability","author":"Andrey Kolmogorov (Nathan Morrison transl., 1956)","year":"1933 (German); 1956 (English Morrison)","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"probability","pillar_targets":["V"],"urls":[],"home_url":"https://en.wikipedia.org/wiki/Foundations_of_the_Theory_of_Probability","notes":"License analysis pending. German original (Grundbegriffe der Wahrscheinlichkeitsrechnung, 1933) PD-by-age in EU but US copyright restored via URAA through life+70 (Kolmogorov d.1987 → US PD 2058). Morrison 1956 English translation: Chelsea Publishing copyright through life+70 of translator. URLs intentionally empty until license decision lands; entry kept for citation traceability."} +{"id":"lukasiewicz-aristotle-syllogistic-1951","title":"Aristotle's Syllogistic from the Standpoint of Modern Formal Logic","author":"Jan Łukasiewicz","year":"1951 (Oxford); 2nd ed 1957","license":"PROPRIETARY","license_url":"https://global.oup.com/academic/legal/copyright/","domain":"logic","pillar_targets":["I"],"urls":[],"home_url":"https://en.wikipedia.org/wiki/Jan_%C5%81ukasiewicz","notes":"Oxford University Press, copyright through life+70 (Łukasiewicz d.1956 → PD 2027 in EU, longer in US via URAA). NOT redistributable under arborist's allow-list. Entry kept as a license-fail placeholder; the 1921 Polish primary papers ('Two-valued logic', 'On the principle of contradiction in Aristotle') are themselves PD by age but lack accessible HTML editions."} diff --git a/bench/scripts/textbooks_manifest.py b/bench/scripts/textbooks_manifest.py index cb4062b..cb7b174 100644 --- a/bench/scripts/textbooks_manifest.py +++ b/bench/scripts/textbooks_manifest.py @@ -54,19 +54,30 @@ def iter_entries(stream: TextIO) -> Iterator[dict]: def _validate(entry: dict) -> None: - """Raise ValueError if an entry is missing required license fields or - declares a license outside the redistribution allow-list.""" + """Raise ValueError if an entry would emit URLs under a license outside + the redistribution allow-list, OR is missing required license fields. + + Entries with both ``urls`` and ``crawl_url`` empty/missing are + placeholder rows kept for citation traceability (e.g., Mendelson + + Enderton as yellow-light proprietary; PG TeX-only Hilbert + Boole + awaiting PDF source support; Łukasiewicz + Kolmogorov pending license + analysis). Validation is fail-closed at the URL-emit step, so + placeholders with no URLs are allowed any license token without + raising. + """ + has_emit_target = bool(entry.get("urls") or entry.get("crawl_url")) for k in _REQUIRED_LICENSE_KEYS: if k not in entry or not entry[k]: raise ValueError( f"manifest entry {entry.get('id', '?')!r} missing required " f"field {k!r}" ) - if entry["license"] not in _ALLOWED_LICENSES: + if has_emit_target and entry["license"] not in _ALLOWED_LICENSES: raise ValueError( f"manifest entry {entry.get('id', '?')!r} declares license " f"{entry['license']!r} outside arborist redistribution allow-list " - f"({sorted(_ALLOWED_LICENSES)})" + f"({sorted(_ALLOWED_LICENSES)}); urls / crawl_url must be empty " + f"for non-allow-listed entries" )