russell@unturf.com
082143e158
GitRepoSource walks the working tree at HEAD via `git ls-tree` + `git show`, yielding one Document per text file. URI shape `git://<repo>/file/<path>` intentionally omits the commit hash — re-ingest after new commits produces fresh document_roots that aborist's prior-doc detection chains via `supersedes` edges, so the audit trail and Merkle tree grow as the repo grows. Binaries skipped via NUL-byte + UTF-8 decode probes; >5 MB files filtered out by default. Commit hash + timestamp + subject ride along in `extra` (informational only; not part of the Merkle commitment). MercurialRepoSource mirrors via `hg manifest` + `hg cat`. Same supersedes semantics, same shape. Makefile adds: make ingest-self (this repo -> aborist-self.db) make ingest-git GIT_REPO=/path/to/repo (arbitrary git clone) make ingest-hg HG_REPO=/path/to/repo (mercurial) Each lands in its own shard file alongside existing shards/grok.db, keeping per-shard write paths independent of the wikipedia 4-way ingest's WAL writer lock.
38 lines
1.2 KiB
Python
38 lines
1.2 KiB
Python
"""Source implementations. Add a new corpus = add a new module here."""
|
|
|
|
from aborist.sources.grok import GrokExportSource, GrokMediaPostsSource
|
|
from aborist.sources.vcs import GitRepoSource, MercurialRepoSource
|
|
from aborist.sources.wikipedia import (
|
|
WikipediaCurDump,
|
|
WikipediaOldDump,
|
|
WikipediaSqlDump,
|
|
)
|
|
from aborist.sources.wikipedia_xml import WikipediaAbstractDump, WikipediaXmlDump
|
|
|
|
__all__ = [
|
|
"GitRepoSource",
|
|
"GrokExportSource",
|
|
"GrokMediaPostsSource",
|
|
"MercurialRepoSource",
|
|
"WikipediaAbstractDump",
|
|
"WikipediaCurDump",
|
|
"WikipediaOldDump",
|
|
"WikipediaSqlDump",
|
|
"WikipediaXmlDump",
|
|
]
|
|
|
|
# HtmlPageSource has optional deps (selectolax). Surface import errors only
|
|
# when callers actually request it.
|
|
try:
|
|
from aborist.sources.html_page import HtmlPageSource # noqa: F401
|
|
__all__.append("HtmlPageSource")
|
|
except ImportError: # pragma: no cover
|
|
pass
|
|
|
|
# HtmlPageSource has optional deps (httpx + selectolax). Surface import errors
|
|
# only when callers actually request it.
|
|
try:
|
|
from aborist.sources.html_page import HtmlPageSource # noqa: F401
|
|
__all__.append("HtmlPageSource")
|
|
except ImportError: # pragma: no cover
|
|
pass
|