diff --git a/bench/fixtures/textbooks/manifest-v1.jsonl b/bench/fixtures/textbooks/manifest-v1.jsonl index b595dd6..05c416b 100644 --- a/bench/fixtures/textbooks/manifest-v1.jsonl +++ b/bench/fixtures/textbooks/manifest-v1.jsonl @@ -6,3 +6,7 @@ {"id":"boole-laws-of-thought-1854","title":"An Investigation of the Laws of Thought","author":"George Boole","year":"1854","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"logic","pillar_targets":["I"],"urls":[],"home_url":"https://www.gutenberg.org/ebooks/15114","tex_url":"https://www.gutenberg.org/files/15114/15114-t/15114-t.tex","pdf_url":"https://www.gutenberg.org/files/15114/15114-pdf.pdf","notes":"Project Gutenberg eBook #15114 — PDF/TeX only, no HTML format. URLs intentionally empty until PDF / TeX source support lands. Boole's foundational treatise establishing propositional logic / Boolean algebra."} {"id":"aristotle-prior-analytics-jenkinson","title":"Prior Analytics","author":"Aristotle (A. J. Jenkinson, transl.)","year":"~350 BCE; Jenkinson translation 1928","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"logic","pillar_targets":["I"],"urls":["https://en.wikisource.org/wiki/Prior_Analytics"],"home_url":"https://en.wikisource.org/wiki/Prior_Analytics","crawl_url":"https://en.wikisource.org/wiki/Prior_Analytics","crawl_depth":1,"crawl_max":20,"notes":"Wikisource HTML. The /wiki/Prior_Analytics page is a disambiguation/landing — only ~75% complete as raw text; the deeper transcription lives at /wiki/Index:O._F._Owen%27s_Organon_of_Aristotle_Vol._1_(1853).djvu and its /Page: subpages, which the crawler can follow at depth=2+."} {"id":"morin-open-data-structures","title":"Open Data Structures","author":"Pat Morin","year":"2013+","license":"CC-BY-2.5","license_url":"https://creativecommons.org/licenses/by/2.5/ca/","domain":"computer-science","pillar_targets":[],"urls":["https://opendatastructures.org/ods-cpp/","https://opendatastructures.org/ods-python/","https://opendatastructures.org/ods-java/"],"home_url":"https://opendatastructures.org/","crawl_url":"https://opendatastructures.org/","crawl_depth":3,"crawl_max":120,"notes":"Comprehensive open data-structures text. CC-BY-2.5 (Canadian port) — fully redistributable. Three language editions (C++, Python, Java) of the same content. ODS crawl @ depth=3 yields ~64 chapters / ~80KB across all three editions."} +{"id":"newton-principia-motte","title":"The Mathematical Principles of Natural Philosophy","author":"Isaac Newton (Andrew Motte transl., 1729 / Motte-Cajori 1846)","year":"1687/1729/1846","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"classical-physics","pillar_targets":["VI"],"urls":["https://en.wikisource.org/wiki/The_Mathematical_Principles_of_Natural_Philosophy"],"home_url":"https://en.wikisource.org/wiki/The_Mathematical_Principles_of_Natural_Philosophy","crawl_url":"https://en.wikisource.org/wiki/The_Mathematical_Principles_of_Natural_Philosophy","crawl_depth":2,"crawl_max":60,"notes":"Wikisource HTML, English Motte 1729 / Motte-Cajori 1846 translations. Newton's foundational treatise — Laws of Motion + universal gravitation. Cited by claim-pack pillar VI records."} +{"id":"aristotle-posterior-analytics","title":"Posterior Analytics","author":"Aristotle (G. R. G. Mure, transl.)","year":"~350 BCE; Mure translation 1928","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"logic","pillar_targets":["I"],"urls":["https://en.wikisource.org/wiki/Posterior_Analytics"],"home_url":"https://en.wikisource.org/wiki/Posterior_Analytics","crawl_url":"https://en.wikisource.org/wiki/Posterior_Analytics","crawl_depth":1,"crawl_max":20,"notes":"Wikisource HTML. Aristotle's treatise on demonstrative knowledge, companion to Prior Analytics. The traditional Mure / Owen translations are PD by age."} +{"id":"kolmogorov-foundations-probability-1933","title":"Foundations of the Theory of Probability","author":"Andrey Kolmogorov (Nathan Morrison transl., 1956)","year":"1933 (German); 1956 (English Morrison)","license":"PD","license_url":"https://en.wikipedia.org/wiki/Public_domain","domain":"probability","pillar_targets":["V"],"urls":[],"home_url":"https://en.wikipedia.org/wiki/Foundations_of_the_Theory_of_Probability","notes":"License analysis pending. German original (Grundbegriffe der Wahrscheinlichkeitsrechnung, 1933) PD-by-age in EU but US copyright restored via URAA through life+70 (Kolmogorov d.1987 → US PD 2058). Morrison 1956 English translation: Chelsea Publishing copyright through life+70 of translator. URLs intentionally empty until license decision lands; entry kept for citation traceability."} +{"id":"lukasiewicz-aristotle-syllogistic-1951","title":"Aristotle's Syllogistic from the Standpoint of Modern Formal Logic","author":"Jan Łukasiewicz","year":"1951 (Oxford); 2nd ed 1957","license":"PROPRIETARY","license_url":"https://global.oup.com/academic/legal/copyright/","domain":"logic","pillar_targets":["I"],"urls":[],"home_url":"https://en.wikipedia.org/wiki/Jan_%C5%81ukasiewicz","notes":"Oxford University Press, copyright through life+70 (Łukasiewicz d.1956 → PD 2027 in EU, longer in US via URAA). NOT redistributable under arborist's allow-list. Entry kept as a license-fail placeholder; the 1921 Polish primary papers ('Two-valued logic', 'On the principle of contradiction in Aristotle') are themselves PD by age but lack accessible HTML editions."} diff --git a/bench/scripts/textbooks_manifest.py b/bench/scripts/textbooks_manifest.py index cb4062b..cb7b174 100644 --- a/bench/scripts/textbooks_manifest.py +++ b/bench/scripts/textbooks_manifest.py @@ -54,19 +54,30 @@ def iter_entries(stream: TextIO) -> Iterator[dict]: def _validate(entry: dict) -> None: - """Raise ValueError if an entry is missing required license fields or - declares a license outside the redistribution allow-list.""" + """Raise ValueError if an entry would emit URLs under a license outside + the redistribution allow-list, OR is missing required license fields. + + Entries with both ``urls`` and ``crawl_url`` empty/missing are + placeholder rows kept for citation traceability (e.g., Mendelson + + Enderton as yellow-light proprietary; PG TeX-only Hilbert + Boole + awaiting PDF source support; Łukasiewicz + Kolmogorov pending license + analysis). Validation is fail-closed at the URL-emit step, so + placeholders with no URLs are allowed any license token without + raising. + """ + has_emit_target = bool(entry.get("urls") or entry.get("crawl_url")) for k in _REQUIRED_LICENSE_KEYS: if k not in entry or not entry[k]: raise ValueError( f"manifest entry {entry.get('id', '?')!r} missing required " f"field {k!r}" ) - if entry["license"] not in _ALLOWED_LICENSES: + if has_emit_target and entry["license"] not in _ALLOWED_LICENSES: raise ValueError( f"manifest entry {entry.get('id', '?')!r} declares license " f"{entry['license']!r} outside arborist redistribution allow-list " - f"({sorted(_ALLOWED_LICENSES)})" + f"({sorted(_ALLOWED_LICENSES)}); urls / crawl_url must be empty " + f"for non-allow-listed entries" )