engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add git and mercurial repo sources for self-play

Browse source 
GitRepoSource walks the working tree at HEAD via `git ls-tree` + `git show`,
yielding one Document per text file. URI shape `git://<repo>/file/<path>`
intentionally omits the commit hash — re-ingest after new commits produces
fresh document_roots that aborist's prior-doc detection chains via
`supersedes` edges, so the audit trail and Merkle tree grow as the repo
grows. Binaries skipped via NUL-byte + UTF-8 decode probes; >5 MB files
filtered out by default. Commit hash + timestamp + subject ride along in
`extra` (informational only; not part of the Merkle commitment).

MercurialRepoSource mirrors via `hg manifest` + `hg cat`. Same supersedes
semantics, same shape.

Makefile adds:
  make ingest-self                          (this repo -> aborist-self.db)
  make ingest-git GIT_REPO=/path/to/repo    (arbitrary git clone)
  make ingest-hg  HG_REPO=/path/to/repo     (mercurial)

Each lands in its own shard file alongside existing shards/grok.db,
keeping per-shard write paths independent of the wikipedia 4-way ingest's
WAL writer lock.
This commit is contained in:
russell@unturf.com 2026-04-27 18:17:39 -04:00
parent 5c2eced7db
commit 082143e158
No known key found for this signature in database
6 changed files with 470 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

32
Makefile
View file

@ -29,6 +29,7 @@ SEARCH_Q ?= computer
ingest ingest-cur ingest-old ingest-xml ingest-xml-history \
ingest-xml-attached ingest-abstract \
ingest-grok ingest-grok-media \
ingest-self ingest-git ingest-hg \
verify search stats test clean clean-db clean-data help
all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats
@ -238,6 +239,37 @@ ingest-xml-attached: bootstrap fetch-xml ## sharded XML ingest, one process per
ingest-abstract: bootstrap fetch-abstract ## ingest INGEST_LIMIT abstract docs from $(WP_ABSTRACT)
$(ABORIST) --db $(DB) ingest --source wikipedia_abstract --path $(WP_ABSTRACT) --limit $(INGEST_LIMIT)
# ----------------------------------------------------------------------------
# Self-ingest: aborist consults its own source code as a queryable corpus.
# Re-running picks up new commits — same path + new content gets a fresh
# document_root chained to the prior version via a `supersedes` edge, so the
# audit chain grows as the repo grows. Lands in a dedicated shard file so it
# doesn't compete with the wikipedia/grok shards' WAL writer lock.
#
# Override SELF_REPO to ingest a different repo's tree.
# ----------------------------------------------------------------------------
SELF_REPO ?= $(CURDIR)
SELF_SHARD := $(SHARDS_DIR)/aborist-self.db
ingest-self: bootstrap ## ingest this repo's HEAD into a dedicated shard
@mkdir -p $(SHARDS_DIR)
$(ABORIST) --db $(SELF_SHARD) ingest --source git_repo --path $(SELF_REPO)
# Generic git-repo ingest: aim it at any local clone via GIT_REPO=...
GIT_REPO ?= $(CURDIR)
GIT_SHARD := $(SHARDS_DIR)/$(notdir $(GIT_REPO))-git.db
ingest-git: bootstrap ## ingest GIT_REPO=<path> into its own shard
@mkdir -p $(SHARDS_DIR)
$(ABORIST) --db $(GIT_SHARD) ingest --source git_repo --path $(GIT_REPO)
# Generic hg-repo ingest. HG_REPO=<path>.
HG_REPO ?=
HG_SHARD := $(SHARDS_DIR)/$(notdir $(HG_REPO))-hg.db
ingest-hg: bootstrap ## ingest HG_REPO=<path> (mercurial) into its own shard
@if [ -z "$(HG_REPO)" ]; then echo "usage: make ingest-hg HG_REPO=/path/to/repo" >&2; exit 2; fi
@mkdir -p $(SHARDS_DIR)
$(ABORIST) --db $(HG_SHARD) ingest --source hg_repo --path $(HG_REPO)
verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents
$(ABORIST) --db $(DB) verify -n $(VERIFY_N)