Add VCS repo cloning support (git/hg/svn/fossil)

Smart source detection: like Hydra for RSS feeds, auto-detect
repository URLs and clone instead of HTTP crawling.

- repo.py: detect_vcs(), clone_repo(), walk_files(), symlink_to_vault()
- Symlink storage: files in repo_vault/ symlink to content-addressed vault/
- Database: repo_uri, repo_path, commit_hash, vcs_type columns
- Auto-detect GitHub, GitLab, Bitbucket, sr.ht, hg.mozilla.org, etc.
This commit is contained in:
Russell Ballestrini 2026-01-02 14:37:53 -05:00
parent 55146915e5
commit 0696e92ae3
4 changed files with 718 additions and 1 deletions

View file

@ -110,6 +110,11 @@ class MediaSource(Base):
searchable_text = Column(Text)
crawl_job_id = Column(Integer, ForeignKey('crawl_jobs.id'))
discovered_at = Column(Text, nullable=False)
# VCS repo metadata (for code files from git/hg/svn clones)
repo_uri = Column(Text) # Clone URL
repo_path = Column(Text) # File path within repo
commit_hash = Column(Text) # Commit/revision when indexed
vcs_type = Column(Text) # git, hg, svn, fossil
media = relationship('Media', back_populates='sources')
crawl_job = relationship('CrawlJob', back_populates='media_sources')
@ -120,6 +125,7 @@ class MediaSource(Base):
Index('idx_sources_job', 'crawl_job_id'),
Index('idx_sources_media_uri', 'media_uri'),
Index('idx_sources_page_uri', 'page_uri'),
Index('idx_sources_repo', 'repo_uri'),
)
@ -485,6 +491,12 @@ class Database:
detail_content: str = "",
searchable_text: str = "",
score: int = SCORE_THUMBNAIL,
# VCS repo metadata
repo_uri: str = "",
repo_path: str = "",
commit_hash: str = "",
vcs_type: str = "",
keywords: list = None,
) -> None:
"""Create a new media record and add source context."""
now = datetime.now(timezone.utc).isoformat()
@ -521,7 +533,11 @@ class Database:
detail_content=detail_content,
searchable_text=searchable_text,
crawl_job_id=crawl_job_id,
discovered_at=now
discovered_at=now,
repo_uri=repo_uri or None,
repo_path=repo_path or None,
commit_hash=commit_hash or None,
vcs_type=vcs_type or None,
).on_conflict_do_nothing()
await session.execute(source_stmt)
await session.commit()