arborist/aborist/sources/vcs.py

"""Version-control-system sources: git and Mercurial.

Each yields one Document per text file at HEAD/tip, with a stable URI
that does NOT include the commit hash — so re-ingesting the same repo
after new commits produces *new* documents for changed files which
aborist's prior-document detection auto-chains via `supersedes` edges.
That gives "the Merkle tree grows over time" semantics for free: every
new commit appends to the audit chain, every changed file gets a new
content-addressed Document, and the supersedes edges connect them.

Binary files are skipped (best-effort UTF-8 decode; fall back rejects
the file). Files larger than `max_bytes` are skipped to avoid pulling
generated artifacts (build outputs, vendored libraries) into the
content store.

Both sources subprocess the underlying CLI rather than importing a
client library — keeps the dependency surface minimal and works with
whatever git/hg the user has installed.
"""

from __future__ import annotations

import shutil
import subprocess
from pathlib import Path
from typing import Iterator

from aborist.document import Document
from aborist.source import Source


# Skip files >5 MB by default. Source code, prose, configs all fit
# comfortably; this filters out lockfiles, generated assets, and
# accidentally-committed binaries.
_DEFAULT_MAX_BYTES = 5 * 1024 * 1024


def _run(cmd: list[str], cwd: Path) -> str:
    """Run a subprocess and return decoded stdout. Raise on non-zero."""
    result = subprocess.run(
        cmd,
        cwd=cwd,
        capture_output=True,
        check=True,
    )
    return result.stdout.decode("utf-8", errors="replace")


def _run_bytes(cmd: list[str], cwd: Path) -> bytes:
    """Run a subprocess and return raw stdout bytes."""
    result = subprocess.run(
        cmd,
        cwd=cwd,
        capture_output=True,
        check=True,
    )
    return result.stdout


def _try_decode(raw: bytes) -> str | None:
    """Best-effort UTF-8 decode; return None for binary content."""
    if b"\x00" in raw[:4096]:
        return None  # NUL bytes -> almost certainly binary
    try:
        return raw.decode("utf-8")
    except UnicodeDecodeError:
        return None


class GitRepoSource(Source):
    """Yields one Document per text file at the given git ref.

    URI shape: `git://<repo-basename>/file/<relative-path>`.
    Re-ingesting after new commits auto-chains via supersedes edges —
    the same file path with new content gets a fresh document_root
    plus an edge `(new_root, old_root, edge_type='supersedes')`.

    `extra` carries the commit hash, author timestamp, and short
    summary of HEAD at ingest time (informational; not Merkle-bound).
    """

    source_type = "git_repo"

    def __init__(
        self,
        repo_path: str | Path,
        *,
        ref: str = "HEAD",
        repo_name: str | None = None,
        max_bytes: int = _DEFAULT_MAX_BYTES,
    ):
        self.repo_path = Path(repo_path).resolve()
        if not (self.repo_path / ".git").exists():
            raise FileNotFoundError(f"not a git repo: {self.repo_path}")
        if shutil.which("git") is None:
            raise RuntimeError("git executable not found in PATH")
        self.ref = ref
        self.repo_name = repo_name or self.repo_path.name
        self.max_bytes = max_bytes

    def _commit_meta(self) -> dict[str, str]:
        try:
            line = _run(
                ["git", "log", "-1", "--format=%H%x09%at%x09%s", self.ref],
                self.repo_path,
            ).strip()
            commit_hash, ts, subject = line.split("\t", 2)
        except Exception:
            return {}
        return {
            "commit_hash": commit_hash,
            "commit_ts": ts,
            "commit_subject": subject,
        }

    def iter_documents(self) -> Iterator[Document]:
        meta = self._commit_meta()
        listing = _run(
            ["git", "ls-tree", "-r", "--name-only", self.ref],
            self.repo_path,
        )
        for relpath in listing.splitlines():
            relpath = relpath.strip()
            if not relpath:
                continue
            try:
                raw = _run_bytes(
                    ["git", "show", f"{self.ref}:{relpath}"], self.repo_path
                )
            except subprocess.CalledProcessError:
                # Submodule entry, broken ref, etc.
                continue
            if len(raw) > self.max_bytes:
                continue
            text = _try_decode(raw)
            if text is None or not text.strip():
                continue
            yield Document(
                uri=f"git://{self.repo_name}/file/{relpath}",
                content=text,
                source_type=self.source_type,
                title=relpath,
                extra={**meta, "path": relpath, "size_bytes": str(len(raw))},
            )


class MercurialRepoSource(Source):
    """Yields one Document per text file at the given hg revision.

    Mirror of GitRepoSource. URI shape:
    `hg://<repo-basename>/file/<relative-path>`. Same supersedes-on-rerun
    semantics.
    """

    source_type = "hg_repo"

    def __init__(
        self,
        repo_path: str | Path,
        *,
        rev: str = "tip",
        repo_name: str | None = None,
        max_bytes: int = _DEFAULT_MAX_BYTES,
    ):
        self.repo_path = Path(repo_path).resolve()
        if not (self.repo_path / ".hg").exists():
            raise FileNotFoundError(f"not a mercurial repo: {self.repo_path}")
        if shutil.which("hg") is None:
            raise RuntimeError("hg executable not found in PATH")
        self.rev = rev
        self.repo_name = repo_name or self.repo_path.name
        self.max_bytes = max_bytes

    def _changeset_meta(self) -> dict[str, str]:
        try:
            # template emits: full-hash<TAB>unix-time<TAB>summary
            line = _run(
                [
                    "hg",
                    "log",
                    "-r",
                    self.rev,
                    "--template",
                    "{node}\t{date|hgdate}\t{desc|firstline}",
                ],
                self.repo_path,
            ).strip()
            parts = line.split("\t", 2)
            if len(parts) != 3:
                return {}
            changeset_hash, hgdate, subject = parts
            # hgdate is "<unix> <tzoffset>"; keep just the unix part.
            ts = hgdate.split()[0] if hgdate else ""
        except Exception:
            return {}
        return {
            "changeset_hash": changeset_hash,
            "commit_ts": ts,
            "commit_subject": subject,
        }

    def iter_documents(self) -> Iterator[Document]:
        meta = self._changeset_meta()
        listing = _run(
            ["hg", "manifest", "-r", self.rev], self.repo_path
        )
        for relpath in listing.splitlines():
            relpath = relpath.strip()
            if not relpath:
                continue
            try:
                raw = _run_bytes(
                    ["hg", "cat", "-r", self.rev, relpath], self.repo_path
                )
            except subprocess.CalledProcessError:
                continue
            if len(raw) > self.max_bytes:
                continue
            text = _try_decode(raw)
            if text is None or not text.strip():
                continue
            yield Document(
                uri=f"hg://{self.repo_name}/file/{relpath}",
                content=text,
                source_type=self.source_type,
                title=relpath,
                extra={**meta, "path": relpath, "size_bytes": str(len(raw))},
            )