"""Generate docs/_source/api/makefile.rst from Makefile ## annotations. Convention: every documented target has the form target-name: deps ## one-line description Targets are grouped into workflow phases (manual mapping below) rather than alphabetical prefix, because phase tells a new reader the order they'd actually run things. """ from pathlib import Path import re TARGET_RE = re.compile(r"^([a-zA-Z0-9_-]+):.*?##\s*(.*)$") # Workflow phases. Order = the order a new operator usually runs them. # Targets not listed here go into UNCATEGORIZED so we notice on review. PHASES: list[tuple[str, str, list[str]]] = [ ( "Setup", "One-time installation. Creates the venv and (optionally) the " "crawler's heavy extras. Re-running is a no-op when up to date.", ["bootstrap", "bootstrap-crawler", "all"], ), ( "Fetch", "Download corpus snapshots into ``data/``. Idempotent — ``curl`` " "skips files already present.", ["fetch", "fetch-cur", "fetch-old", "fetch-xml", "fetch-abstract"], ), ( "Ingest", "Parse a source into Merkle-committed shards. ``-attached`` " "variants are the canonical sharded path (one SQLite per shard, " "no WAL contention). Single-DB variants are for experiments.", [ "ingest", "ingest-cur", "ingest-cur-attached", "ingest-old", "ingest-old-attached", "ingest-xml", "ingest-xml-history", "ingest-xml-attached", "ingest-abstract", "ingest-grok-attached", "ingest-grok-media-attached", "ingest-self", "ingest-self-providence", "ingest-git", "ingest-hg", "crawl-ingest", "recrawl-check", ], ), ( "Distill", "Surface → core distillation. Cores are Merkle-bound back to " "their source chunks via inclusion proofs.", [ "distill-shards-parallel", "distill-shards-tfidf-parallel", "backfill-concepts", ], ), ( "Query", "Ask the corpus a question. ``query-dry`` skips the LLM call " "and returns the assembled context — useful for prompt iteration.", ["query", "query-dry", "search"], ), ( "Verify and inspect", "Round-trip Merkle proofs, audit chain integrity, sidecar " "diagnostics on cached answers.", [ "verify", "verify-shards", "chain-check", "chain-check-shards", "analyze-shards", "stats", "stats-shards", "activity", "inspect", ], ), ( "Operations on cached records", "Mark a record falsified (audit-preserving) or burn it from " "the database (refuses if it has children unless ``FORCE=1``).", ["falsify", "burn", "burn-kindergarten"], ), ( "Tests and benches", "Default test suite excludes opt-in crawler tests. ``bench-qa`` " "runs the full QA-quality sweep; ``bench-qa-smoke`` is the 5-question " "fast loop.", [ "test", "test-crawler", "test-live", "bench", "bench-qa", "bench-qa-smoke", "bench-emergent", "bench-emergent-pending", ], ), ( "Docs", "Render diagrams (graphviz) and build the Sphinx API reference. " "RTD rebuilds on push; these targets are for local previews.", ["docs", "docs-api", "docs-api-clean"], ), ( "Clean", "Reversible by re-running the matching ``bootstrap`` / ``fetch`` " "/ ``ingest`` target. ``clean-data`` deletes the largest payload " "(downloaded dumps).", ["clean", "clean-db", "clean-data", "help"], ), ] def parse_makefile(makefile_path: Path) -> dict[str, str]: """Return {target: description} from a Makefile.""" targets = {} for line in makefile_path.read_text(encoding="utf-8").splitlines(): m = TARGET_RE.match(line) if m: targets[m.group(1)] = m.group(2).strip() return targets def _escape_rst(text: str) -> str: # Bare `*` in description text (e.g. ``*-parallel``, ``*.db``, # ``π*``) trips the docutils inline-emphasis scanner. Escape every # asterisk so it renders literally. return text.replace("*", r"\*") def generate_rst(all_targets: dict[str, str], output_path: Path) -> None: """Write a single RST page grouping every target by workflow phase.""" lines = [ "Makefile reference", "==================", "", "Every arborist workflow lives behind a ``make`` target. This page is", "auto-generated from the project ``Makefile``'s ``## description``", "annotations at Sphinx build time, so it stays in sync with the source.", "", "Run ``make help`` locally for a flat alphabetized listing.", "", ".. note::", "", " Targets are grouped below by **workflow phase**, in the order a", " new operator typically runs them. The first row of each table is", " the most common entry point for that phase.", "", ] used: set[str] = set() for phase_title, blurb, target_names in PHASES: # Filter to targets that actually exist in the parsed Makefile. present = [(t, all_targets[t]) for t in target_names if t in all_targets] if not present: continue used.update(t for t, _ in present) lines.append(phase_title) lines.append("-" * len(phase_title)) lines.append("") lines.append(blurb) lines.append("") lines.append(".. list-table::") lines.append(" :widths: 30 70") lines.append(" :header-rows: 1") lines.append("") lines.append(" * - Target") lines.append(" - Description") for name, desc in present: lines.append(f" * - ``make {name}``") lines.append(f" - {_escape_rst(desc)}") lines.append("") # Surface anything we forgot to categorize so it shows up in review. leftover = [(t, d) for t, d in all_targets.items() if t not in used] if leftover: lines.append("Uncategorized") lines.append("-------------") lines.append("") lines.append( "Targets not yet placed in a workflow phase. If you see one here, " "add it to ``docs/_source/_ext/makefile_targets.py`` ``PHASES``." ) lines.append("") lines.append(".. list-table::") lines.append(" :widths: 30 70") lines.append(" :header-rows: 1") lines.append("") lines.append(" * - Target") lines.append(" - Description") for name, desc in sorted(leftover): lines.append(f" * - ``make {name}``") lines.append(f" - {_escape_rst(desc)}") lines.append("") output_path.write_text("\n".join(lines), encoding="utf-8") def setup(app): """Sphinx hook: regenerate makefile.rst at the start of every build.""" project_root = Path(app.srcdir).parent.parent makefile = project_root / "Makefile" output = Path(app.srcdir) / "api" / "makefile.rst" if makefile.exists(): targets = parse_makefile(makefile) generate_rst(targets, output) return {"version": "1.1", "parallel_read_safe": True}