Add VCS repo cloning support (git/hg/svn/fossil)
Smart source detection: like Hydra for RSS feeds, auto-detect repository URLs and clone instead of HTTP crawling. - repo.py: detect_vcs(), clone_repo(), walk_files(), symlink_to_vault() - Symlink storage: files in repo_vault/ symlink to content-addressed vault/ - Database: repo_uri, repo_path, commit_hash, vcs_type columns - Auto-detect GitHub, GitLab, Bitbucket, sr.ht, hg.mozilla.org, etc.
This commit is contained in:
parent
55146915e5
commit
0696e92ae3
4 changed files with 718 additions and 1 deletions
18
database.py
18
database.py
|
|
@ -110,6 +110,11 @@ class MediaSource(Base):
|
|||
searchable_text = Column(Text)
|
||||
crawl_job_id = Column(Integer, ForeignKey('crawl_jobs.id'))
|
||||
discovered_at = Column(Text, nullable=False)
|
||||
# VCS repo metadata (for code files from git/hg/svn clones)
|
||||
repo_uri = Column(Text) # Clone URL
|
||||
repo_path = Column(Text) # File path within repo
|
||||
commit_hash = Column(Text) # Commit/revision when indexed
|
||||
vcs_type = Column(Text) # git, hg, svn, fossil
|
||||
|
||||
media = relationship('Media', back_populates='sources')
|
||||
crawl_job = relationship('CrawlJob', back_populates='media_sources')
|
||||
|
|
@ -120,6 +125,7 @@ class MediaSource(Base):
|
|||
Index('idx_sources_job', 'crawl_job_id'),
|
||||
Index('idx_sources_media_uri', 'media_uri'),
|
||||
Index('idx_sources_page_uri', 'page_uri'),
|
||||
Index('idx_sources_repo', 'repo_uri'),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -485,6 +491,12 @@ class Database:
|
|||
detail_content: str = "",
|
||||
searchable_text: str = "",
|
||||
score: int = SCORE_THUMBNAIL,
|
||||
# VCS repo metadata
|
||||
repo_uri: str = "",
|
||||
repo_path: str = "",
|
||||
commit_hash: str = "",
|
||||
vcs_type: str = "",
|
||||
keywords: list = None,
|
||||
) -> None:
|
||||
"""Create a new media record and add source context."""
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
|
|
@ -521,7 +533,11 @@ class Database:
|
|||
detail_content=detail_content,
|
||||
searchable_text=searchable_text,
|
||||
crawl_job_id=crawl_job_id,
|
||||
discovered_at=now
|
||||
discovered_at=now,
|
||||
repo_uri=repo_uri or None,
|
||||
repo_path=repo_path or None,
|
||||
commit_hash=commit_hash or None,
|
||||
vcs_type=vcs_type or None,
|
||||
).on_conflict_do_nothing()
|
||||
await session.execute(source_stmt)
|
||||
await session.commit()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue