arborist/docs/_source/_ext/makefile_targets.py

"""Generate docs/_source/api/makefile.rst from Makefile ## annotations.

Convention: every documented target has the form
    target-name: deps ## one-line description

Targets are grouped into workflow phases (manual mapping below) rather
than alphabetical prefix, because phase tells a new reader the order
they'd actually run things.
"""

from pathlib import Path
import re

TARGET_RE = re.compile(r"^([a-zA-Z0-9_-]+):.*?##\s*(.*)$")

# Workflow phases. Order = the order a new operator usually runs them.
# Targets not listed here go into UNCATEGORIZED so we notice on review.
PHASES: list[tuple[str, str, list[str]]] = [
    (
        "Setup",
        "One-time installation. Creates the venv and (optionally) the "
        "crawler's heavy extras. Re-running is a no-op when up to date.",
        ["bootstrap", "bootstrap-crawler", "all"],
    ),
    (
        "Fetch",
        "Download corpus snapshots into ``data/``. Idempotent — ``curl`` "
        "skips files already present.",
        ["fetch", "fetch-cur", "fetch-old", "fetch-xml", "fetch-abstract"],
    ),
    (
        "Ingest",
        "Parse a source into Merkle-committed shards. ``-attached`` "
        "variants are the canonical sharded path (one SQLite per shard, "
        "no WAL contention). Single-DB variants are for experiments.",
        [
            "ingest",
            "ingest-cur",
            "ingest-cur-attached",
            "ingest-old",
            "ingest-old-attached",
            "ingest-xml",
            "ingest-xml-history",
            "ingest-xml-attached",
            "ingest-abstract",
            "ingest-grok-attached",
            "ingest-grok-media-attached",
            "ingest-self",
            "ingest-self-providence",
            "ingest-git",
            "ingest-hg",
            "crawl-ingest",
            "recrawl-check",
        ],
    ),
    (
        "Distill",
        "Surface → core distillation. Cores are Merkle-bound back to "
        "their source chunks via inclusion proofs.",
        [
            "distill-shards-parallel",
            "distill-shards-tfidf-parallel",
            "backfill-concepts",
        ],
    ),
    (
        "Query",
        "Ask the corpus a question. ``query-dry`` skips the LLM call "
        "and returns the assembled context — useful for prompt iteration.",
        ["query", "query-dry", "search"],
    ),
    (
        "Verify and inspect",
        "Round-trip Merkle proofs, audit chain integrity, sidecar "
        "diagnostics on cached answers.",
        [
            "verify",
            "verify-shards",
            "chain-check",
            "chain-check-shards",
            "analyze-shards",
            "stats",
            "stats-shards",
            "activity",
            "inspect",
        ],
    ),
    (
        "Operations on cached records",
        "Mark a record falsified (audit-preserving) or burn it from "
        "the database (refuses if it has children unless ``FORCE=1``).",
        ["falsify", "burn", "burn-kindergarten"],
    ),
    (
        "Tests and benches",
        "Default test suite excludes opt-in crawler tests. ``bench-qa`` "
        "runs the full QA-quality sweep; ``bench-qa-smoke`` is the 5-question "
        "fast loop.",
        [
            "test",
            "test-crawler",
            "test-live",
            "bench",
            "bench-qa",
            "bench-qa-smoke",
            "bench-emergent",
            "bench-emergent-pending",
        ],
    ),
    (
        "Docs",
        "Render diagrams (graphviz) and build the Sphinx API reference. "
        "RTD rebuilds on push; these targets are for local previews.",
        ["docs", "docs-api", "docs-api-clean"],
    ),
    (
        "Clean",
        "Reversible by re-running the matching ``bootstrap`` / ``fetch`` "
        "/ ``ingest`` target. ``clean-data`` deletes the largest payload "
        "(downloaded dumps).",
        ["clean", "clean-db", "clean-data", "help"],
    ),
]


def parse_makefile(makefile_path: Path) -> dict[str, str]:
    """Return {target: description} from a Makefile."""
    targets = {}
    for line in makefile_path.read_text(encoding="utf-8").splitlines():
        m = TARGET_RE.match(line)
        if m:
            targets[m.group(1)] = m.group(2).strip()
    return targets


def generate_rst(all_targets: dict[str, str], output_path: Path) -> None:
    """Write a single RST page grouping every target by workflow phase."""
    lines = [
        "Makefile reference",
        "==================",
        "",
        "Every arborist workflow lives behind a ``make`` target. This page is",
        "auto-generated from the project ``Makefile``'s ``## description``",
        "annotations at Sphinx build time, so it stays in sync with the source.",
        "",
        "Run ``make help`` locally for a flat alphabetized listing.",
        "",
        ".. note::",
        "",
        "   Targets are grouped below by **workflow phase**, in the order a",
        "   new operator typically runs them. The first row of each table is",
        "   the most common entry point for that phase.",
        "",
    ]

    used: set[str] = set()
    for phase_title, blurb, target_names in PHASES:
        # Filter to targets that actually exist in the parsed Makefile.
        present = [(t, all_targets[t]) for t in target_names if t in all_targets]
        if not present:
            continue
        used.update(t for t, _ in present)

        lines.append(phase_title)
        lines.append("-" * len(phase_title))
        lines.append("")
        lines.append(blurb)
        lines.append("")
        lines.append(".. list-table::")
        lines.append("   :widths: 30 70")
        lines.append("   :header-rows: 1")
        lines.append("")
        lines.append("   * - Target")
        lines.append("     - Description")
        for name, desc in present:
            lines.append(f"   * - ``make {name}``")
            lines.append(f"     - {desc}")
        lines.append("")

    # Surface anything we forgot to categorize so it shows up in review.
    leftover = [(t, d) for t, d in all_targets.items() if t not in used]
    if leftover:
        lines.append("Uncategorized")
        lines.append("-------------")
        lines.append("")
        lines.append(
            "Targets not yet placed in a workflow phase. If you see one here, "
            "add it to ``docs/_source/_ext/makefile_targets.py`` ``PHASES``."
        )
        lines.append("")
        lines.append(".. list-table::")
        lines.append("   :widths: 30 70")
        lines.append("   :header-rows: 1")
        lines.append("")
        lines.append("   * - Target")
        lines.append("     - Description")
        for name, desc in sorted(leftover):
            lines.append(f"   * - ``make {name}``")
            lines.append(f"     - {desc}")
        lines.append("")

    output_path.write_text("\n".join(lines), encoding="utf-8")


def setup(app):
    """Sphinx hook: regenerate makefile.rst at the start of every build."""
    project_root = Path(app.srcdir).parent.parent
    makefile = project_root / "Makefile"
    output = Path(app.srcdir) / "api" / "makefile.rst"
    if makefile.exists():
        targets = parse_makefile(makefile)
        generate_rst(targets, output)
    return {"version": "1.1", "parallel_read_safe": True}