engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add git and mercurial repo sources for self-play

Browse source 
GitRepoSource walks the working tree at HEAD via `git ls-tree` + `git show`,
yielding one Document per text file. URI shape `git://<repo>/file/<path>`
intentionally omits the commit hash — re-ingest after new commits produces
fresh document_roots that aborist's prior-doc detection chains via
`supersedes` edges, so the audit trail and Merkle tree grow as the repo
grows. Binaries skipped via NUL-byte + UTF-8 decode probes; >5 MB files
filtered out by default. Commit hash + timestamp + subject ride along in
`extra` (informational only; not part of the Merkle commitment).

MercurialRepoSource mirrors via `hg manifest` + `hg cat`. Same supersedes
semantics, same shape.

Makefile adds:
  make ingest-self                          (this repo -> aborist-self.db)
  make ingest-git GIT_REPO=/path/to/repo    (arbitrary git clone)
  make ingest-hg  HG_REPO=/path/to/repo     (mercurial)

Each lands in its own shard file alongside existing shards/grok.db,
keeping per-shard write paths independent of the wikipedia 4-way ingest's
WAL writer lock.
This commit is contained in:
russell@unturf.com 2026-04-27 18:17:39 -04:00
parent 5c2eced7db
commit 082143e158
No known key found for this signature in database
6 changed files with 470 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

32
Makefile
View file

@ -29,6 +29,7 @@ SEARCH_Q ?= computer
ingest ingest-cur ingest-old ingest-xml ingest-xml-history \
ingest-xml-attached ingest-abstract \
ingest-grok ingest-grok-media \
ingest-self ingest-git ingest-hg \
verify search stats test clean clean-db clean-data help
all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats
@ -238,6 +239,37 @@ ingest-xml-attached: bootstrap fetch-xml ## sharded XML ingest, one process per
ingest-abstract: bootstrap fetch-abstract ## ingest INGEST_LIMIT abstract docs from $(WP_ABSTRACT)
$(ABORIST) --db $(DB) ingest --source wikipedia_abstract --path $(WP_ABSTRACT) --limit $(INGEST_LIMIT)
# ----------------------------------------------------------------------------
# Self-ingest: aborist consults its own source code as a queryable corpus.
# Re-running picks up new commits — same path + new content gets a fresh
# document_root chained to the prior version via a `supersedes` edge, so the
# audit chain grows as the repo grows. Lands in a dedicated shard file so it
# doesn't compete with the wikipedia/grok shards' WAL writer lock.
#
# Override SELF_REPO to ingest a different repo's tree.
# ----------------------------------------------------------------------------
SELF_REPO ?= $(CURDIR)
SELF_SHARD := $(SHARDS_DIR)/aborist-self.db
ingest-self: bootstrap ## ingest this repo's HEAD into a dedicated shard
@mkdir -p $(SHARDS_DIR)
$(ABORIST) --db $(SELF_SHARD) ingest --source git_repo --path $(SELF_REPO)
# Generic git-repo ingest: aim it at any local clone via GIT_REPO=...
GIT_REPO ?= $(CURDIR)
GIT_SHARD := $(SHARDS_DIR)/$(notdir $(GIT_REPO))-git.db
ingest-git: bootstrap ## ingest GIT_REPO=<path> into its own shard
@mkdir -p $(SHARDS_DIR)
$(ABORIST) --db $(GIT_SHARD) ingest --source git_repo --path $(GIT_REPO)
# Generic hg-repo ingest. HG_REPO=<path>.
HG_REPO ?=
HG_SHARD := $(SHARDS_DIR)/$(notdir $(HG_REPO))-hg.db
ingest-hg: bootstrap ## ingest HG_REPO=<path> (mercurial) into its own shard
@if [ -z "$(HG_REPO)" ]; then echo "usage: make ingest-hg HG_REPO=/path/to/repo" >&2; exit 2; fi
@mkdir -p $(SHARDS_DIR)
$(ABORIST) --db $(HG_SHARD) ingest --source hg_repo --path $(HG_REPO)
verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents
$(ABORIST) --db $(DB) verify -n $(VERIFY_N)

12
README.md
View file

@ -179,6 +179,18 @@ Both walk the export tree, find `prod-grok-backend.json`, and yield one Document
`--resume` is the default for these targets. Re-run any time to pick up new exports.
## Data: git and Mercurial repos (self-play)
Aborist can consult itself. Point a source at any local clone and every text file at HEAD becomes a queryable Document; re-ingesting after new commits chains old → new via `supersedes` edges, so the audit trail grows alongside the repo:
```
make ingest-self # this aborist tree, into ~/.aborist/shards/aborist-self.db
make ingest-git GIT_REPO=/path/to/repo # any other git clone
make ingest-hg HG_REPO=/path/to/repo # mercurial flavor
```
URI shape: `git://<repo-name>/file/<relative-path>` (no commit hash — that's what enables the supersedes chain on re-ingest). Binary files are skipped (NUL-byte heuristic + UTF-8 decode probe). `extra` carries the current commit hash, timestamp, and subject for informational purposes; the cryptographic identity is the content-derived `document_root` as for every other Document.
## Data: OpenAI / ChatGPT export (planned)
Not implemented yet. The shape will be one new `Source` subclass at `aborist/sources/openai.py` plus a Makefile target. The OpenAI ChatGPT data export is a `.zip` containing `conversations.json` with the `mapping`/`messages` tree shape. Adding it follows the same pattern as `aborist/sources/grok.py` — see that file as the template.

10
aborist/cli.py
View file

@ -75,6 +75,14 @@ def _cmd_ingest(args: argparse.Namespace) -> int:
shard=shard,
multi_revision=(args.source == "wikipedia_xml_history"),
)
elif args.source in ("git_repo", "hg_repo"):
if not args.path:
print(f"--path is required for {args.source}", file=sys.stderr)
return 2
from aborist.sources import GitRepoSource, MercurialRepoSource
cls = GitRepoSource if args.source == "git_repo" else MercurialRepoSource
src = cls(repo_path=args.path)
else:
print(f"unknown source: {args.source}", file=sys.stderr)
return 2
@ -905,6 +913,8 @@ def build_parser() -> argparse.ArgumentParser:
"html",
"grok_export",
"grok_media",
"git_repo",
"hg_repo",
],
help="source type",
)

3
aborist/sources/__init__.py
View file

@ -1,6 +1,7 @@
"""Source implementations. Add a new corpus = add a new module here."""
from aborist.sources.grok import GrokExportSource, GrokMediaPostsSource
from aborist.sources.vcs import GitRepoSource, MercurialRepoSource
from aborist.sources.wikipedia import (
WikipediaCurDump,
WikipediaOldDump,
@ -9,8 +10,10 @@ from aborist.sources.wikipedia import (
from aborist.sources.wikipedia_xml import WikipediaAbstractDump, WikipediaXmlDump
__all__ = [
"GitRepoSource",
"GrokExportSource",
"GrokMediaPostsSource",
"MercurialRepoSource",
"WikipediaAbstractDump",
"WikipediaCurDump",
"WikipediaOldDump",

228
aborist/sources/vcs.py Normal file
View file

@ -0,0 +1,228 @@
"""Version-control-system sources: git and Mercurial.
Each yields one Document per text file at HEAD/tip, with a stable URI
that does NOT include the commit hash — so re-ingesting the same repo
after new commits produces *new* documents for changed files which
aborist's prior-document detection auto-chains via `supersedes` edges.
That gives "the Merkle tree grows over time" semantics for free: every
new commit appends to the audit chain, every changed file gets a new
content-addressed Document, and the supersedes edges connect them.
Binary files are skipped (best-effort UTF-8 decode; fall back rejects
the file). Files larger than `max_bytes` are skipped to avoid pulling
generated artifacts (build outputs, vendored libraries) into the
content store.
Both sources subprocess the underlying CLI rather than importing a
client library — keeps the dependency surface minimal and works with
whatever git/hg the user has installed.
"""
from __future__ import annotations
import shutil
import subprocess
from pathlib import Path
from typing import Iterator
from aborist.document import Document
from aborist.source import Source
# Skip files >5 MB by default. Source code, prose, configs all fit
# comfortably; this filters out lockfiles, generated assets, and
# accidentally-committed binaries.
_DEFAULT_MAX_BYTES = 5 * 1024 * 1024
def _run(cmd: list[str], cwd: Path) -> str:
"""Run a subprocess and return decoded stdout. Raise on non-zero."""
result = subprocess.run(
cmd,
cwd=cwd,
capture_output=True,
check=True,
)
return result.stdout.decode("utf-8", errors="replace")
def _run_bytes(cmd: list[str], cwd: Path) -> bytes:
"""Run a subprocess and return raw stdout bytes."""
result = subprocess.run(
cmd,
cwd=cwd,
capture_output=True,
check=True,
)
return result.stdout
def _try_decode(raw: bytes) -> str | None:
"""Best-effort UTF-8 decode; return None for binary content."""
if b"\x00" in raw[:4096]:
return None # NUL bytes -> almost certainly binary
try:
return raw.decode("utf-8")
except UnicodeDecodeError:
return None
class GitRepoSource(Source):
"""Yields one Document per text file at the given git ref.
URI shape: `git://<repo-basename>/file/<relative-path>`.
Re-ingesting after new commits auto-chains via supersedes edges —
the same file path with new content gets a fresh document_root
plus an edge `(new_root, old_root, edge_type='supersedes')`.
`extra` carries the commit hash, author timestamp, and short
summary of HEAD at ingest time (informational; not Merkle-bound).
"""
source_type = "git_repo"
def __init__(
self,
repo_path: str | Path,
*,
ref: str = "HEAD",
repo_name: str | None = None,
max_bytes: int = _DEFAULT_MAX_BYTES,
):
self.repo_path = Path(repo_path).resolve()
if not (self.repo_path / ".git").exists():
raise FileNotFoundError(f"not a git repo: {self.repo_path}")
if shutil.which("git") is None:
raise RuntimeError("git executable not found in PATH")
self.ref = ref
self.repo_name = repo_name or self.repo_path.name
self.max_bytes = max_bytes
def _commit_meta(self) -> dict[str, str]:
try:
line = _run(
["git", "log", "-1", "--format=%H%x09%at%x09%s", self.ref],
self.repo_path,
).strip()
commit_hash, ts, subject = line.split("\t", 2)
except Exception:
return {}
return {
"commit_hash": commit_hash,
"commit_ts": ts,
"commit_subject": subject,
}
def iter_documents(self) -> Iterator[Document]:
meta = self._commit_meta()
listing = _run(
["git", "ls-tree", "-r", "--name-only", self.ref],
self.repo_path,
)
for relpath in listing.splitlines():
relpath = relpath.strip()
if not relpath:
continue
try:
raw = _run_bytes(
["git", "show", f"{self.ref}:{relpath}"], self.repo_path
)
except subprocess.CalledProcessError:
# Submodule entry, broken ref, etc.
continue
if len(raw) > self.max_bytes:
continue
text = _try_decode(raw)
if text is None or not text.strip():
continue
yield Document(
uri=f"git://{self.repo_name}/file/{relpath}",
content=text,
source_type=self.source_type,
title=relpath,
extra={**meta, "path": relpath, "size_bytes": str(len(raw))},
)
class MercurialRepoSource(Source):
"""Yields one Document per text file at the given hg revision.
Mirror of GitRepoSource. URI shape:
`hg://<repo-basename>/file/<relative-path>`. Same supersedes-on-rerun
semantics.
"""
source_type = "hg_repo"
def __init__(
self,
repo_path: str | Path,
*,
rev: str = "tip",
repo_name: str | None = None,
max_bytes: int = _DEFAULT_MAX_BYTES,
):
self.repo_path = Path(repo_path).resolve()
if not (self.repo_path / ".hg").exists():
raise FileNotFoundError(f"not a mercurial repo: {self.repo_path}")
if shutil.which("hg") is None:
raise RuntimeError("hg executable not found in PATH")
self.rev = rev
self.repo_name = repo_name or self.repo_path.name
self.max_bytes = max_bytes
def _changeset_meta(self) -> dict[str, str]:
try:
# template emits: full-hash<TAB>unix-time<TAB>summary
line = _run(
[
"hg",
"log",
"-r",
self.rev,
"--template",
"{node}\t{date|hgdate}\t{desc|firstline}",
],
self.repo_path,
).strip()
parts = line.split("\t", 2)
if len(parts) != 3:
return {}
changeset_hash, hgdate, subject = parts
# hgdate is "<unix> <tzoffset>"; keep just the unix part.
ts = hgdate.split()[0] if hgdate else ""
except Exception:
return {}
return {
"changeset_hash": changeset_hash,
"commit_ts": ts,
"commit_subject": subject,
}
def iter_documents(self) -> Iterator[Document]:
meta = self._changeset_meta()
listing = _run(
["hg", "manifest", "-r", self.rev], self.repo_path
)
for relpath in listing.splitlines():
relpath = relpath.strip()
if not relpath:
continue
try:
raw = _run_bytes(
["hg", "cat", "-r", self.rev, relpath], self.repo_path
)
except subprocess.CalledProcessError:
continue
if len(raw) > self.max_bytes:
continue
text = _try_decode(raw)
if text is None or not text.strip():
continue
yield Document(
uri=f"hg://{self.repo_name}/file/{relpath}",
content=text,
source_type=self.source_type,
title=relpath,
extra={**meta, "path": relpath, "size_bytes": str(len(raw))},
)

185
tests/test_vcs_source.py Normal file
View file

@ -0,0 +1,185 @@
"""Tests for the git and Mercurial repo sources.
Each test builds a tiny synthetic repo via the actual `git` / `hg` CLIs in
a tmp_path, ingests it, and asserts the round-trip and supersedes-chain
behavior. If the underlying VCS isn't on PATH, the test is skipped.
"""
from __future__ import annotations
import os
import shutil
import subprocess
from pathlib import Path
import pytest
from aborist.ingest import ingest_source
from aborist.sources.vcs import GitRepoSource, MercurialRepoSource
from aborist.store import connect
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _git_available() -> bool:
return shutil.which("git") is not None
def _hg_available() -> bool:
return shutil.which("hg") is not None
def _mk_git_repo(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True)
env = {
**os.environ,
# Detached author so commits are reproducible and tests don't
# accidentally pick up the host's git identity.
"GIT_AUTHOR_NAME": "test",
"GIT_AUTHOR_EMAIL": "t@example.invalid",
"GIT_COMMITTER_NAME": "test",
"GIT_COMMITTER_EMAIL": "t@example.invalid",
"GIT_AUTHOR_DATE": "2026-01-01T00:00:00Z",
"GIT_COMMITTER_DATE": "2026-01-01T00:00:00Z",
}
subprocess.run(["git", "init", "-q", "-b", "main"], cwd=path, check=True)
subprocess.run(["git", "config", "user.email", "t@example.invalid"], cwd=path, check=True)
subprocess.run(["git", "config", "user.name", "test"], cwd=path, check=True)
(path / "README.md").write_text("# Demo\n\nHello world.\n")
(path / "src.py").write_text("def hello():\n return 'hi'\n")
(path / "logo.bin").write_bytes(b"\x00\x01\x02\x03" * 64) # binary
subprocess.run(["git", "add", "-A"], cwd=path, check=True)
subprocess.run(
["git", "commit", "-q", "-m", "initial"],
cwd=path,
check=True,
env=env,
)
def _commit_file(path: Path, name: str, body: str, msg: str) -> None:
env = {
**os.environ,
"GIT_AUTHOR_NAME": "test",
"GIT_AUTHOR_EMAIL": "t@example.invalid",
"GIT_COMMITTER_NAME": "test",
"GIT_COMMITTER_EMAIL": "t@example.invalid",
"GIT_AUTHOR_DATE": "2026-02-01T00:00:00Z",
"GIT_COMMITTER_DATE": "2026-02-01T00:00:00Z",
}
(path / name).write_text(body)
subprocess.run(["git", "add", name], cwd=path, check=True)
subprocess.run(
["git", "commit", "-q", "-m", msg], cwd=path, check=True, env=env
)
def _mk_hg_repo(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True)
subprocess.run(["hg", "init", "-q"], cwd=path, check=True)
(path / ".hg" / "hgrc").write_text(
"[ui]\nusername = test <t@example.invalid>\n"
)
(path / "README.md").write_text("# Demo\n\nHello hg.\n")
(path / "src.py").write_text("def world():\n return 'world'\n")
subprocess.run(["hg", "add"], cwd=path, check=True)
subprocess.run(
["hg", "commit", "-m", "initial", "-d", "1735689600 0"],
cwd=path,
check=True,
)
# ---------------------------------------------------------------------------
# GitRepoSource
# ---------------------------------------------------------------------------
@pytest.mark.skipif(not _git_available(), reason="git not on PATH")
def test_git_source_yields_text_files_and_skips_binary(tmp_path):
repo = tmp_path / "demo-repo"
_mk_git_repo(repo)
src = GitRepoSource(repo)
docs = list(src.iter_documents())
titles = sorted(d.title for d in docs)
# Binary file (logo.bin) should not appear.
assert titles == ["README.md", "src.py"]
readme = next(d for d in docs if d.title == "README.md")
assert readme.uri == "git://demo-repo/file/README.md"
assert "Hello world" in readme.content
assert readme.source_type == "git_repo"
# Commit metadata is captured (informational; not Merkle-bound).
assert "commit_hash" in readme.extra
assert len(readme.extra["commit_hash"]) == 40
@pytest.mark.skipif(not _git_available(), reason="git not on PATH")
def test_git_source_round_trip_and_supersedes_chain(tmp_path):
repo = tmp_path / "demo-repo"
_mk_git_repo(repo)
db = tmp_path / "aborist.db"
# First ingest: 2 docs (README.md, src.py).
conn = connect(db)
try:
first = ingest_source(conn, GitRepoSource(repo))
finally:
conn.close()
assert first.inserted == 2
# Modify src.py and re-ingest. Same URI, new content -> new document_root
# plus an automatic `supersedes` edge to the prior root.
_commit_file(repo, "src.py", "def hello():\n return 'updated'\n", "edit src")
conn = connect(db)
try:
second = ingest_source(conn, GitRepoSource(repo))
finally:
conn.close()
# README.md is unchanged so its content_root collides -> skipped.
# src.py changed -> 1 new doc inserted with a supersedes edge.
assert second.inserted == 1
assert second.skipped_duplicate >= 1
conn = connect(db)
try:
edges = conn.execute(
"SELECT src_root, dst_root FROM edges WHERE edge_type='supersedes'"
).fetchall()
finally:
conn.close()
assert len(edges) == 1
@pytest.mark.skipif(not _git_available(), reason="git not on PATH")
def test_git_source_rejects_non_git_path(tmp_path):
with pytest.raises(FileNotFoundError):
GitRepoSource(tmp_path) # tmp_path has no .git
# ---------------------------------------------------------------------------
# MercurialRepoSource
# ---------------------------------------------------------------------------
@pytest.mark.skipif(not _hg_available(), reason="hg not on PATH")
def test_hg_source_yields_text_files(tmp_path):
repo = tmp_path / "demo-hg"
_mk_hg_repo(repo)
src = MercurialRepoSource(repo)
docs = list(src.iter_documents())
titles = sorted(d.title for d in docs)
assert titles == ["README.md", "src.py"]
readme = next(d for d in docs if d.title == "README.md")
assert readme.uri == "hg://demo-hg/file/README.md"
assert "Hello hg" in readme.content
assert readme.source_type == "hg_repo"
assert "changeset_hash" in readme.extra
@pytest.mark.skipif(not _hg_available(), reason="hg not on PATH")
def test_hg_source_rejects_non_hg_path(tmp_path):
with pytest.raises(FileNotFoundError):
MercurialRepoSource(tmp_path)