From 082143e15821f2e1673326037e18344044c709cb Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 27 Apr 2026 18:17:39 -0400 Subject: [PATCH] add git and mercurial repo sources for self-play MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitRepoSource walks the working tree at HEAD via `git ls-tree` + `git show`, yielding one Document per text file. URI shape `git:///file/` intentionally omits the commit hash — re-ingest after new commits produces fresh document_roots that aborist's prior-doc detection chains via `supersedes` edges, so the audit trail and Merkle tree grow as the repo grows. Binaries skipped via NUL-byte + UTF-8 decode probes; >5 MB files filtered out by default. Commit hash + timestamp + subject ride along in `extra` (informational only; not part of the Merkle commitment). MercurialRepoSource mirrors via `hg manifest` + `hg cat`. Same supersedes semantics, same shape. Makefile adds: make ingest-self (this repo -> aborist-self.db) make ingest-git GIT_REPO=/path/to/repo (arbitrary git clone) make ingest-hg HG_REPO=/path/to/repo (mercurial) Each lands in its own shard file alongside existing shards/grok.db, keeping per-shard write paths independent of the wikipedia 4-way ingest's WAL writer lock. --- Makefile | 32 +++++ README.md | 12 ++ aborist/cli.py | 10 ++ aborist/sources/__init__.py | 3 + aborist/sources/vcs.py | 228 ++++++++++++++++++++++++++++++++++++ tests/test_vcs_source.py | 185 +++++++++++++++++++++++++++++ 6 files changed, 470 insertions(+) create mode 100644 aborist/sources/vcs.py create mode 100644 tests/test_vcs_source.py diff --git a/Makefile b/Makefile index 432770b..f212ca1 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ SEARCH_Q ?= computer ingest ingest-cur ingest-old ingest-xml ingest-xml-history \ ingest-xml-attached ingest-abstract \ ingest-grok ingest-grok-media \ + ingest-self ingest-git ingest-hg \ verify search stats test clean clean-db clean-data help all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats @@ -238,6 +239,37 @@ ingest-xml-attached: bootstrap fetch-xml ## sharded XML ingest, one process per ingest-abstract: bootstrap fetch-abstract ## ingest INGEST_LIMIT abstract docs from $(WP_ABSTRACT) $(ABORIST) --db $(DB) ingest --source wikipedia_abstract --path $(WP_ABSTRACT) --limit $(INGEST_LIMIT) +# ---------------------------------------------------------------------------- +# Self-ingest: aborist consults its own source code as a queryable corpus. +# Re-running picks up new commits — same path + new content gets a fresh +# document_root chained to the prior version via a `supersedes` edge, so the +# audit chain grows as the repo grows. Lands in a dedicated shard file so it +# doesn't compete with the wikipedia/grok shards' WAL writer lock. +# +# Override SELF_REPO to ingest a different repo's tree. +# ---------------------------------------------------------------------------- +SELF_REPO ?= $(CURDIR) +SELF_SHARD := $(SHARDS_DIR)/aborist-self.db + +ingest-self: bootstrap ## ingest this repo's HEAD into a dedicated shard + @mkdir -p $(SHARDS_DIR) + $(ABORIST) --db $(SELF_SHARD) ingest --source git_repo --path $(SELF_REPO) + +# Generic git-repo ingest: aim it at any local clone via GIT_REPO=... +GIT_REPO ?= $(CURDIR) +GIT_SHARD := $(SHARDS_DIR)/$(notdir $(GIT_REPO))-git.db +ingest-git: bootstrap ## ingest GIT_REPO= into its own shard + @mkdir -p $(SHARDS_DIR) + $(ABORIST) --db $(GIT_SHARD) ingest --source git_repo --path $(GIT_REPO) + +# Generic hg-repo ingest. HG_REPO=. +HG_REPO ?= +HG_SHARD := $(SHARDS_DIR)/$(notdir $(HG_REPO))-hg.db +ingest-hg: bootstrap ## ingest HG_REPO= (mercurial) into its own shard + @if [ -z "$(HG_REPO)" ]; then echo "usage: make ingest-hg HG_REPO=/path/to/repo" >&2; exit 2; fi + @mkdir -p $(SHARDS_DIR) + $(ABORIST) --db $(HG_SHARD) ingest --source hg_repo --path $(HG_REPO) + verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents $(ABORIST) --db $(DB) verify -n $(VERIFY_N) diff --git a/README.md b/README.md index 4d1bb87..b9233ce 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,18 @@ Both walk the export tree, find `prod-grok-backend.json`, and yield one Document `--resume` is the default for these targets. Re-run any time to pick up new exports. +## Data: git and Mercurial repos (self-play) + +Aborist can consult itself. Point a source at any local clone and every text file at HEAD becomes a queryable Document; re-ingesting after new commits chains old → new via `supersedes` edges, so the audit trail grows alongside the repo: + +``` +make ingest-self # this aborist tree, into ~/.aborist/shards/aborist-self.db +make ingest-git GIT_REPO=/path/to/repo # any other git clone +make ingest-hg HG_REPO=/path/to/repo # mercurial flavor +``` + +URI shape: `git:///file/` (no commit hash — that's what enables the supersedes chain on re-ingest). Binary files are skipped (NUL-byte heuristic + UTF-8 decode probe). `extra` carries the current commit hash, timestamp, and subject for informational purposes; the cryptographic identity is the content-derived `document_root` as for every other Document. + ## Data: OpenAI / ChatGPT export (planned) Not implemented yet. The shape will be one new `Source` subclass at `aborist/sources/openai.py` plus a Makefile target. The OpenAI ChatGPT data export is a `.zip` containing `conversations.json` with the `mapping`/`messages` tree shape. Adding it follows the same pattern as `aborist/sources/grok.py` — see that file as the template. diff --git a/aborist/cli.py b/aborist/cli.py index 750c102..18c64d2 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -75,6 +75,14 @@ def _cmd_ingest(args: argparse.Namespace) -> int: shard=shard, multi_revision=(args.source == "wikipedia_xml_history"), ) + elif args.source in ("git_repo", "hg_repo"): + if not args.path: + print(f"--path is required for {args.source}", file=sys.stderr) + return 2 + from aborist.sources import GitRepoSource, MercurialRepoSource + + cls = GitRepoSource if args.source == "git_repo" else MercurialRepoSource + src = cls(repo_path=args.path) else: print(f"unknown source: {args.source}", file=sys.stderr) return 2 @@ -905,6 +913,8 @@ def build_parser() -> argparse.ArgumentParser: "html", "grok_export", "grok_media", + "git_repo", + "hg_repo", ], help="source type", ) diff --git a/aborist/sources/__init__.py b/aborist/sources/__init__.py index e91da94..069e553 100644 --- a/aborist/sources/__init__.py +++ b/aborist/sources/__init__.py @@ -1,6 +1,7 @@ """Source implementations. Add a new corpus = add a new module here.""" from aborist.sources.grok import GrokExportSource, GrokMediaPostsSource +from aborist.sources.vcs import GitRepoSource, MercurialRepoSource from aborist.sources.wikipedia import ( WikipediaCurDump, WikipediaOldDump, @@ -9,8 +10,10 @@ from aborist.sources.wikipedia import ( from aborist.sources.wikipedia_xml import WikipediaAbstractDump, WikipediaXmlDump __all__ = [ + "GitRepoSource", "GrokExportSource", "GrokMediaPostsSource", + "MercurialRepoSource", "WikipediaAbstractDump", "WikipediaCurDump", "WikipediaOldDump", diff --git a/aborist/sources/vcs.py b/aborist/sources/vcs.py new file mode 100644 index 0000000..079ac0e --- /dev/null +++ b/aborist/sources/vcs.py @@ -0,0 +1,228 @@ +"""Version-control-system sources: git and Mercurial. + +Each yields one Document per text file at HEAD/tip, with a stable URI +that does NOT include the commit hash — so re-ingesting the same repo +after new commits produces *new* documents for changed files which +aborist's prior-document detection auto-chains via `supersedes` edges. +That gives "the Merkle tree grows over time" semantics for free: every +new commit appends to the audit chain, every changed file gets a new +content-addressed Document, and the supersedes edges connect them. + +Binary files are skipped (best-effort UTF-8 decode; fall back rejects +the file). Files larger than `max_bytes` are skipped to avoid pulling +generated artifacts (build outputs, vendored libraries) into the +content store. + +Both sources subprocess the underlying CLI rather than importing a +client library — keeps the dependency surface minimal and works with +whatever git/hg the user has installed. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path +from typing import Iterator + +from aborist.document import Document +from aborist.source import Source + + +# Skip files >5 MB by default. Source code, prose, configs all fit +# comfortably; this filters out lockfiles, generated assets, and +# accidentally-committed binaries. +_DEFAULT_MAX_BYTES = 5 * 1024 * 1024 + + +def _run(cmd: list[str], cwd: Path) -> str: + """Run a subprocess and return decoded stdout. Raise on non-zero.""" + result = subprocess.run( + cmd, + cwd=cwd, + capture_output=True, + check=True, + ) + return result.stdout.decode("utf-8", errors="replace") + + +def _run_bytes(cmd: list[str], cwd: Path) -> bytes: + """Run a subprocess and return raw stdout bytes.""" + result = subprocess.run( + cmd, + cwd=cwd, + capture_output=True, + check=True, + ) + return result.stdout + + +def _try_decode(raw: bytes) -> str | None: + """Best-effort UTF-8 decode; return None for binary content.""" + if b"\x00" in raw[:4096]: + return None # NUL bytes -> almost certainly binary + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + return None + + +class GitRepoSource(Source): + """Yields one Document per text file at the given git ref. + + URI shape: `git:///file/`. + Re-ingesting after new commits auto-chains via supersedes edges — + the same file path with new content gets a fresh document_root + plus an edge `(new_root, old_root, edge_type='supersedes')`. + + `extra` carries the commit hash, author timestamp, and short + summary of HEAD at ingest time (informational; not Merkle-bound). + """ + + source_type = "git_repo" + + def __init__( + self, + repo_path: str | Path, + *, + ref: str = "HEAD", + repo_name: str | None = None, + max_bytes: int = _DEFAULT_MAX_BYTES, + ): + self.repo_path = Path(repo_path).resolve() + if not (self.repo_path / ".git").exists(): + raise FileNotFoundError(f"not a git repo: {self.repo_path}") + if shutil.which("git") is None: + raise RuntimeError("git executable not found in PATH") + self.ref = ref + self.repo_name = repo_name or self.repo_path.name + self.max_bytes = max_bytes + + def _commit_meta(self) -> dict[str, str]: + try: + line = _run( + ["git", "log", "-1", "--format=%H%x09%at%x09%s", self.ref], + self.repo_path, + ).strip() + commit_hash, ts, subject = line.split("\t", 2) + except Exception: + return {} + return { + "commit_hash": commit_hash, + "commit_ts": ts, + "commit_subject": subject, + } + + def iter_documents(self) -> Iterator[Document]: + meta = self._commit_meta() + listing = _run( + ["git", "ls-tree", "-r", "--name-only", self.ref], + self.repo_path, + ) + for relpath in listing.splitlines(): + relpath = relpath.strip() + if not relpath: + continue + try: + raw = _run_bytes( + ["git", "show", f"{self.ref}:{relpath}"], self.repo_path + ) + except subprocess.CalledProcessError: + # Submodule entry, broken ref, etc. + continue + if len(raw) > self.max_bytes: + continue + text = _try_decode(raw) + if text is None or not text.strip(): + continue + yield Document( + uri=f"git://{self.repo_name}/file/{relpath}", + content=text, + source_type=self.source_type, + title=relpath, + extra={**meta, "path": relpath, "size_bytes": str(len(raw))}, + ) + + +class MercurialRepoSource(Source): + """Yields one Document per text file at the given hg revision. + + Mirror of GitRepoSource. URI shape: + `hg:///file/`. Same supersedes-on-rerun + semantics. + """ + + source_type = "hg_repo" + + def __init__( + self, + repo_path: str | Path, + *, + rev: str = "tip", + repo_name: str | None = None, + max_bytes: int = _DEFAULT_MAX_BYTES, + ): + self.repo_path = Path(repo_path).resolve() + if not (self.repo_path / ".hg").exists(): + raise FileNotFoundError(f"not a mercurial repo: {self.repo_path}") + if shutil.which("hg") is None: + raise RuntimeError("hg executable not found in PATH") + self.rev = rev + self.repo_name = repo_name or self.repo_path.name + self.max_bytes = max_bytes + + def _changeset_meta(self) -> dict[str, str]: + try: + # template emits: full-hashunix-timesummary + line = _run( + [ + "hg", + "log", + "-r", + self.rev, + "--template", + "{node}\t{date|hgdate}\t{desc|firstline}", + ], + self.repo_path, + ).strip() + parts = line.split("\t", 2) + if len(parts) != 3: + return {} + changeset_hash, hgdate, subject = parts + # hgdate is " "; keep just the unix part. + ts = hgdate.split()[0] if hgdate else "" + except Exception: + return {} + return { + "changeset_hash": changeset_hash, + "commit_ts": ts, + "commit_subject": subject, + } + + def iter_documents(self) -> Iterator[Document]: + meta = self._changeset_meta() + listing = _run( + ["hg", "manifest", "-r", self.rev], self.repo_path + ) + for relpath in listing.splitlines(): + relpath = relpath.strip() + if not relpath: + continue + try: + raw = _run_bytes( + ["hg", "cat", "-r", self.rev, relpath], self.repo_path + ) + except subprocess.CalledProcessError: + continue + if len(raw) > self.max_bytes: + continue + text = _try_decode(raw) + if text is None or not text.strip(): + continue + yield Document( + uri=f"hg://{self.repo_name}/file/{relpath}", + content=text, + source_type=self.source_type, + title=relpath, + extra={**meta, "path": relpath, "size_bytes": str(len(raw))}, + ) diff --git a/tests/test_vcs_source.py b/tests/test_vcs_source.py new file mode 100644 index 0000000..466a08f --- /dev/null +++ b/tests/test_vcs_source.py @@ -0,0 +1,185 @@ +"""Tests for the git and Mercurial repo sources. + +Each test builds a tiny synthetic repo via the actual `git` / `hg` CLIs in +a tmp_path, ingests it, and asserts the round-trip and supersedes-chain +behavior. If the underlying VCS isn't on PATH, the test is skipped. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +from aborist.ingest import ingest_source +from aborist.sources.vcs import GitRepoSource, MercurialRepoSource +from aborist.store import connect + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _git_available() -> bool: + return shutil.which("git") is not None + + +def _hg_available() -> bool: + return shutil.which("hg") is not None + + +def _mk_git_repo(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + env = { + **os.environ, + # Detached author so commits are reproducible and tests don't + # accidentally pick up the host's git identity. + "GIT_AUTHOR_NAME": "test", + "GIT_AUTHOR_EMAIL": "t@example.invalid", + "GIT_COMMITTER_NAME": "test", + "GIT_COMMITTER_EMAIL": "t@example.invalid", + "GIT_AUTHOR_DATE": "2026-01-01T00:00:00Z", + "GIT_COMMITTER_DATE": "2026-01-01T00:00:00Z", + } + subprocess.run(["git", "init", "-q", "-b", "main"], cwd=path, check=True) + subprocess.run(["git", "config", "user.email", "t@example.invalid"], cwd=path, check=True) + subprocess.run(["git", "config", "user.name", "test"], cwd=path, check=True) + (path / "README.md").write_text("# Demo\n\nHello world.\n") + (path / "src.py").write_text("def hello():\n return 'hi'\n") + (path / "logo.bin").write_bytes(b"\x00\x01\x02\x03" * 64) # binary + subprocess.run(["git", "add", "-A"], cwd=path, check=True) + subprocess.run( + ["git", "commit", "-q", "-m", "initial"], + cwd=path, + check=True, + env=env, + ) + + +def _commit_file(path: Path, name: str, body: str, msg: str) -> None: + env = { + **os.environ, + "GIT_AUTHOR_NAME": "test", + "GIT_AUTHOR_EMAIL": "t@example.invalid", + "GIT_COMMITTER_NAME": "test", + "GIT_COMMITTER_EMAIL": "t@example.invalid", + "GIT_AUTHOR_DATE": "2026-02-01T00:00:00Z", + "GIT_COMMITTER_DATE": "2026-02-01T00:00:00Z", + } + (path / name).write_text(body) + subprocess.run(["git", "add", name], cwd=path, check=True) + subprocess.run( + ["git", "commit", "-q", "-m", msg], cwd=path, check=True, env=env + ) + + +def _mk_hg_repo(path: Path) -> None: + path.mkdir(parents=True, exist_ok=True) + subprocess.run(["hg", "init", "-q"], cwd=path, check=True) + (path / ".hg" / "hgrc").write_text( + "[ui]\nusername = test \n" + ) + (path / "README.md").write_text("# Demo\n\nHello hg.\n") + (path / "src.py").write_text("def world():\n return 'world'\n") + subprocess.run(["hg", "add"], cwd=path, check=True) + subprocess.run( + ["hg", "commit", "-m", "initial", "-d", "1735689600 0"], + cwd=path, + check=True, + ) + + +# --------------------------------------------------------------------------- +# GitRepoSource +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _git_available(), reason="git not on PATH") +def test_git_source_yields_text_files_and_skips_binary(tmp_path): + repo = tmp_path / "demo-repo" + _mk_git_repo(repo) + src = GitRepoSource(repo) + docs = list(src.iter_documents()) + titles = sorted(d.title for d in docs) + # Binary file (logo.bin) should not appear. + assert titles == ["README.md", "src.py"] + readme = next(d for d in docs if d.title == "README.md") + assert readme.uri == "git://demo-repo/file/README.md" + assert "Hello world" in readme.content + assert readme.source_type == "git_repo" + # Commit metadata is captured (informational; not Merkle-bound). + assert "commit_hash" in readme.extra + assert len(readme.extra["commit_hash"]) == 40 + + +@pytest.mark.skipif(not _git_available(), reason="git not on PATH") +def test_git_source_round_trip_and_supersedes_chain(tmp_path): + repo = tmp_path / "demo-repo" + _mk_git_repo(repo) + db = tmp_path / "aborist.db" + + # First ingest: 2 docs (README.md, src.py). + conn = connect(db) + try: + first = ingest_source(conn, GitRepoSource(repo)) + finally: + conn.close() + assert first.inserted == 2 + + # Modify src.py and re-ingest. Same URI, new content -> new document_root + # plus an automatic `supersedes` edge to the prior root. + _commit_file(repo, "src.py", "def hello():\n return 'updated'\n", "edit src") + conn = connect(db) + try: + second = ingest_source(conn, GitRepoSource(repo)) + finally: + conn.close() + # README.md is unchanged so its content_root collides -> skipped. + # src.py changed -> 1 new doc inserted with a supersedes edge. + assert second.inserted == 1 + assert second.skipped_duplicate >= 1 + + conn = connect(db) + try: + edges = conn.execute( + "SELECT src_root, dst_root FROM edges WHERE edge_type='supersedes'" + ).fetchall() + finally: + conn.close() + assert len(edges) == 1 + + +@pytest.mark.skipif(not _git_available(), reason="git not on PATH") +def test_git_source_rejects_non_git_path(tmp_path): + with pytest.raises(FileNotFoundError): + GitRepoSource(tmp_path) # tmp_path has no .git + + +# --------------------------------------------------------------------------- +# MercurialRepoSource +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _hg_available(), reason="hg not on PATH") +def test_hg_source_yields_text_files(tmp_path): + repo = tmp_path / "demo-hg" + _mk_hg_repo(repo) + src = MercurialRepoSource(repo) + docs = list(src.iter_documents()) + titles = sorted(d.title for d in docs) + assert titles == ["README.md", "src.py"] + readme = next(d for d in docs if d.title == "README.md") + assert readme.uri == "hg://demo-hg/file/README.md" + assert "Hello hg" in readme.content + assert readme.source_type == "hg_repo" + assert "changeset_hash" in readme.extra + + +@pytest.mark.skipif(not _hg_available(), reason="hg not on PATH") +def test_hg_source_rejects_non_hg_path(tmp_path): + with pytest.raises(FileNotFoundError): + MercurialRepoSource(tmp_path)