engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

git: use a fetch_sync based creation of remote repos.

Browse source 
- skips fragile dulwich code that has problems with large repos
- is in sync with logic that does remote fetch
- is faster and more efficient
This commit is contained in:
Marcin Kuzminski 2018-09-21 13:35:45 +02:00
parent c5f41094fe
commit 5a4a98b40e
9 changed files with 97 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

4
rhodecode/api/tests/test_create_pull_request.py
View file

@ -66,6 +66,7 @@ class TestCreatePullRequestApi(object):
expected_message = "Created new pull request `{title}`".format(
title=data['title'])
result = response.json
assert result['error'] == None
assert result['result']['msg'] == expected_message
pull_request_id = result['result']['pull_request_id']
pull_request = PullRequestModel().get(pull_request_id)
@ -88,6 +89,7 @@ class TestCreatePullRequestApi(object):
expected_message = "Created new pull request `{title}`".format(
title=data['title'])
result = response.json
assert result['error'] == None
assert result['result']['msg'] == expected_message
pull_request_id = result['result']['pull_request_id']
pull_request = PullRequestModel().get(pull_request_id)
@ -127,6 +129,7 @@ class TestCreatePullRequestApi(object):
expected_message = "Created new pull request `{title}`".format(
title=data['title'])
result = response.json
assert result['error'] == None
assert result['result']['msg'] == expected_message
pull_request_id = result['result']['pull_request_id']
pull_request = PullRequestModel().get(pull_request_id)
@ -170,6 +173,7 @@ class TestCreatePullRequestApi(object):
expected_message = "Created new pull request `{title}`".format(
title=data['title'])
result = response.json
assert result['error'] == None
assert result['result']['msg'] == expected_message
pull_request_id = result['result']['pull_request_id']
pull_request = PullRequestModel().get(pull_request_id)

2
rhodecode/lib/celerylib/tasks.py
View file

@ -123,7 +123,7 @@ def create_repo(form_data, cur_user):
'enable_downloads', defs.get('repo_enable_downloads'))
try:
repo = RepoModel()._create_repo(
RepoModel()._create_repo(
repo_name=repo_name_full,
repo_type=repo_type,
description=description,

74
rhodecode/lib/vcs/backends/git/repository.py
View file

@ -58,13 +58,13 @@ class GitRepository(BaseRepository):
contact = BaseRepository.DEFAULT_CONTACT
def __init__(self, repo_path, config=None, create=False, src_url=None,
update_after_clone=False, with_wire=None, bare=False):
do_workspace_checkout=False, with_wire=None, bare=False):
self.path = safe_str(os.path.abspath(repo_path))
self.config = config if config else self.get_default_config()
self.with_wire = with_wire
self._init_repo(create, src_url, update_after_clone, bare)
self._init_repo(create, src_url, do_workspace_checkout, bare)
# caches
self._commit_ids = {}
@ -145,24 +145,36 @@ class GitRepository(BaseRepository):
pass
return False
def _init_repo(self, create, src_url=None, update_after_clone=False,
def _init_repo(self, create, src_url=None, do_workspace_checkout=False,
bare=False):
if create and os.path.exists(self.path):
raise RepositoryError(
"Cannot create repository at %s, location already exist"
% self.path)
if bare and do_workspace_checkout:
raise RepositoryError("Cannot update a bare repository")
try:
if create and src_url:
if src_url:
# check URL before any actions
GitRepository.check_url(src_url, self.config)
self.clone(src_url, update_after_clone, bare)
elif create:
if create:
os.makedirs(self.path, mode=0755)
if bare:
self._remote.init_bare()
else:
self._remote.init()
if src_url and bare:
# bare repository only allows a fetch and checkout is not allowed
self.fetch(src_url, commit_ids=None)
elif src_url:
self.pull(src_url, commit_ids=None,
update_after=do_workspace_checkout)
else:
if not self._remote.assert_correct_path():
raise RepositoryError(
@ -630,49 +642,27 @@ class GitRepository(BaseRepository):
"""
return GitInMemoryCommit(self)
def clone(self, url, update_after_clone=True, bare=False):
def pull(self, url, commit_ids=None, update_after=False):
"""
Tries to clone commits from external location.
Pull changes from external location. Pull is different in GIT
that fetch since it's doing a checkout
:param update_after_clone: If set to ``False``, git won't checkout
working directory
:param bare: If set to ``True``, repository would be cloned into
*bare* git repository (no working directory at all).
:param commit_ids: Optional. Can be set to a list of commit ids
which shall be pulled from the other repository.
"""
# init_bare and init expect empty dir created to proceed
if not os.path.exists(self.path):
os.mkdir(self.path)
if bare:
self._remote.init_bare()
else:
self._remote.init()
deferred = '^{}'
valid_refs = ('refs/heads', 'refs/tags', 'HEAD')
return self._remote.clone(
url, deferred, valid_refs, update_after_clone)
def pull(self, url, commit_ids=None):
"""
Tries to pull changes from external location. We use fetch here since
pull in get does merges and we want to be compatible with hg backend so
pull == fetch in this case
"""
self.fetch(url, commit_ids=commit_ids)
refs = None
if commit_ids is not None:
remote_refs = self._remote.get_remote_refs(url)
refs = [ref for ref in remote_refs if remote_refs[ref] in commit_ids]
self._remote.pull(url, refs=refs, update_after=update_after)
self._remote.invalidate_vcs_cache()
def fetch(self, url, commit_ids=None):
"""
Tries to fetch changes from external location.
Fetch all git objects from external location.
"""
refs = None
if commit_ids is not None:
remote_refs = self._remote.get_remote_refs(url)
refs = [
ref for ref in remote_refs if remote_refs[ref] in commit_ids]
self._remote.fetch(url, refs=refs)
self._remote.sync_fetch(url, refs=commit_ids)
self._remote.invalidate_vcs_cache()
def push(self, url):
refs = None

21
rhodecode/lib/vcs/backends/hg/repository.py
View file

@ -57,7 +57,7 @@ class MercurialRepository(BaseRepository):
DEFAULT_BRANCH_NAME = 'default'
def __init__(self, repo_path, config=None, create=False, src_url=None,
update_after_clone=False, with_wire=None):
do_workspace_checkout=False, with_wire=None, bare=False):
"""
Raises RepositoryError if repository could not be find at the given
``repo_path``.
@ -67,8 +67,9 @@ class MercurialRepository(BaseRepository):
:param create=False: if set to True, would try to create repository if
it does not exist rather than raising exception
:param src_url=None: would try to clone repository from given location
:param update_after_clone=False: sets update of working copy after
:param do_workspace_checkout=False: sets update of working copy after
making a clone
:param bare: not used, compatible with other VCS
"""
self.path = safe_str(os.path.abspath(repo_path))
@ -79,7 +80,7 @@ class MercurialRepository(BaseRepository):
default=[('extensions', 'largefiles', '1')])
self.with_wire = with_wire
self._init_repo(create, src_url, update_after_clone)
self._init_repo(create, src_url, do_workspace_checkout)
# caches
self._commit_ids = {}
@ -328,7 +329,7 @@ class MercurialRepository(BaseRepository):
def is_valid_repository(path):
return os.path.isdir(os.path.join(path, '.hg'))
def _init_repo(self, create, src_url=None, update_after_clone=False):
def _init_repo(self, create, src_url=None, do_workspace_checkout=False):
"""
Function will check for mercurial repository in given path. If there
is no repository in that path it will raise an exception unless
@ -337,7 +338,7 @@ class MercurialRepository(BaseRepository):
If `src_url` is given, would try to clone repository from the
location at given clone_point. Additionally it'll make update to
working copy accordingly to `update_after_clone` flag.
working copy accordingly to `do_workspace_checkout` flag.
"""
if create and os.path.exists(self.path):
raise RepositoryError(
@ -348,7 +349,7 @@ class MercurialRepository(BaseRepository):
url = str(self._get_url(src_url))
MercurialRepository.check_url(url, self.config)
self._remote.clone(url, self.path, update_after_clone)
self._remote.clone(url, self.path, do_workspace_checkout)
# Don't try to create if we've already cloned repo
create = False
@ -538,7 +539,7 @@ class MercurialRepository(BaseRepository):
def pull(self, url, commit_ids=None):
"""
Tries to pull changes from external location.
Pull changes from external location.
:param commit_ids: Optional. Can be set to a list of commit ids
which shall be pulled from the other repository.
@ -547,6 +548,12 @@ class MercurialRepository(BaseRepository):
self._remote.pull(url, commit_ids=commit_ids)
self._remote.invalidate_vcs_cache()
def fetch(self, url, commit_ids=None):
"""
Backward compatibility with GIT fetch==pull
"""
return self.pull(url, commit_ids=commit_ids)
def push(self, url):
url = self._get_url(url)
self._remote.sync_push(url)

2
rhodecode/lib/vcs/backends/svn/repository.py
View file

@ -68,7 +68,7 @@ class SubversionRepository(base.BaseRepository):
contact = base.BaseRepository.DEFAULT_CONTACT
description = base.BaseRepository.DEFAULT_DESCRIPTION
def __init__(self, repo_path, config=None, create=False, src_url=None,
def __init__(self, repo_path, config=None, create=False, src_url=None, bare=False,
**kwargs):
self.path = safe_str(os.path.abspath(repo_path))
self.config = config if config else self.get_default_config()

2
rhodecode/model/scm.py
View file

@ -399,7 +399,7 @@ class ScmModel(BaseModel):
repo_name = dbrepo.repo_name
try:
# TODO: we need to make sure those operations call proper hooks !
repo.pull(remote_uri)
repo.fetch(remote_uri)
self.mark_for_invalidation(repo_name)
except Exception:

18
rhodecode/tests/plugin.py
View file

@ -581,7 +581,7 @@ class Backend(object):
def create_repo(
self, commits=None, number_of_commits=0, heads=None,
name_suffix=u'', **kwargs):
name_suffix=u'', bare=False, **kwargs):
"""
Create a repository and record it for later cleanup.
@ -591,16 +591,17 @@ class Backend(object):
commits will be added to the new repository.
:param heads: Optional. Can be set to a sequence of of commit
names which shall be pulled in from the master repository.
:param name_suffix: adds special suffix to generated repo name
:param bare: set a repo as bare (no checkout)
"""
self.repo_name = self._next_repo_name() + name_suffix
repo = self._fixture.create_repo(
self.repo_name, repo_type=self.alias, **kwargs)
self.repo_name, repo_type=self.alias, bare=bare, **kwargs)
self._cleanup_repos.append(repo.repo_name)
commits = commits or [
{'message': 'Commit %s of %s' % (x, self.repo_name)}
for x in xrange(number_of_commits)]
for x in range(number_of_commits)]
self._add_commits_to_repo(repo.scm_instance(), commits)
if heads:
self.pull_heads(repo, heads)
@ -773,14 +774,15 @@ class VcsBackend(object):
"""
return get_backend(self.alias)
def create_repo(self, commits=None, number_of_commits=0, _clone_repo=None):
def create_repo(self, commits=None, number_of_commits=0, _clone_repo=None,
bare=False):
repo_name = self._next_repo_name()
self._repo_path = get_new_dir(repo_name)
repo_class = get_backend(self.alias)
src_url = None
if _clone_repo:
src_url = _clone_repo.path
repo = repo_class(self._repo_path, create=True, src_url=src_url)
repo = repo_class(self._repo_path, create=True, src_url=src_url, bare=bare)
self._cleanup_repos.append(repo)
commits = commits or [
@ -1158,13 +1160,13 @@ class UserUtility(object):
return repo_group
def create_repo(self, owner=TEST_USER_ADMIN_LOGIN, parent=None,
auto_cleanup=True, repo_type='hg'):
auto_cleanup=True, repo_type='hg', bare=False):
repo_name = "{prefix}_repository_{count}".format(
prefix=self._test_name,
count=len(self.repos_ids))
repository = self.fixture.create_repo(
repo_name, cur_user=owner, repo_group=parent, repo_type=repo_type)
repo_name, cur_user=owner, repo_group=parent, repo_type=repo_type, bare=bare)
if auto_cleanup:
self.repos_ids.append(repository.repo_id)
return repository

65
rhodecode/tests/vcs/test_git.py
View file

@ -94,7 +94,7 @@ class TestGitRepository:
repo = GitRepository(TEST_GIT_REPO)
repo_clone = GitRepository(
TEST_GIT_REPO_CLONE,
src_url=TEST_GIT_REPO, create=True, update_after_clone=True)
src_url=TEST_GIT_REPO, create=True, do_workspace_checkout=True)
assert len(repo.commit_ids) == len(repo_clone.commit_ids)
# Checking hashes of commits should be enough
for commit in repo.get_commits():
@ -111,7 +111,7 @@ class TestGitRepository:
clone_path = TEST_GIT_REPO_CLONE + '_with_update'
repo_clone = GitRepository(
clone_path,
create=True, src_url=TEST_GIT_REPO, update_after_clone=True)
create=True, src_url=TEST_GIT_REPO, do_workspace_checkout=True)
assert len(repo.commit_ids) == len(repo_clone.commit_ids)
# check if current workdir was updated
@ -123,7 +123,7 @@ class TestGitRepository:
clone_path = TEST_GIT_REPO_CLONE + '_without_update'
repo_clone = GitRepository(
clone_path,
create=True, src_url=TEST_GIT_REPO, update_after_clone=False)
create=True, src_url=TEST_GIT_REPO, do_workspace_checkout=False)
assert len(repo.commit_ids) == len(repo_clone.commit_ids)
# check if current workdir was *NOT* updated
fpath = os.path.join(clone_path, 'MANIFEST.in')
@ -153,7 +153,7 @@ class TestGitRepository:
# Note: This is a git specific part of the API, it's only implemented
# by the git backend.
source_repo = vcsbackend_git.repo
target_repo = vcsbackend_git.create_repo()
target_repo = vcsbackend_git.create_repo(bare=True)
target_repo.fetch(source_repo.path)
# Note: Get a fresh instance, avoids caching trouble
target_repo = vcsbackend_git.backend(target_repo.path)
@ -162,32 +162,31 @@ class TestGitRepository:
def test_commit_ids(self):
# there are 112 commits (by now)
# so we can assume they would be available from now on
subset = set([
'c1214f7e79e02fc37156ff215cd71275450cffc3',
'38b5fe81f109cb111f549bfe9bb6b267e10bc557',
'fa6600f6848800641328adbf7811fd2372c02ab2',
'102607b09cdd60e2793929c4f90478be29f85a17',
'49d3fd156b6f7db46313fac355dca1a0b94a0017',
'2d1028c054665b962fa3d307adfc923ddd528038',
'd7e0d30fbcae12c90680eb095a4f5f02505ce501',
'ff7ca51e58c505fec0dd2491de52c622bb7a806b',
'dd80b0f6cf5052f17cc738c2951c4f2070200d7f',
'8430a588b43b5d6da365400117c89400326e7992',
'd955cd312c17b02143c04fa1099a352b04368118',
'f67b87e5c629c2ee0ba58f85197e423ff28d735b',
'add63e382e4aabc9e1afdc4bdc24506c269b7618',
'f298fe1189f1b69779a4423f40b48edf92a703fc',
'bd9b619eb41994cac43d67cf4ccc8399c1125808',
'6e125e7c890379446e98980d8ed60fba87d0f6d1',
'd4a54db9f745dfeba6933bf5b1e79e15d0af20bd',
'0b05e4ed56c802098dfc813cbe779b2f49e92500',
'191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e',
'45223f8f114c64bf4d6f853e3c35a369a6305520',
'ca1eb7957a54bce53b12d1a51b13452f95bc7c7e',
'f5ea29fc42ef67a2a5a7aecff10e1566699acd68',
'27d48942240f5b91dfda77accd2caac94708cc7d',
'622f0eb0bafd619d2560c26f80f09e3b0b0d78af',
'e686b958768ee96af8029fe19c6050b1a8dd3b2b'])
subset = {'c1214f7e79e02fc37156ff215cd71275450cffc3',
'38b5fe81f109cb111f549bfe9bb6b267e10bc557',
'fa6600f6848800641328adbf7811fd2372c02ab2',
'102607b09cdd60e2793929c4f90478be29f85a17',
'49d3fd156b6f7db46313fac355dca1a0b94a0017',
'2d1028c054665b962fa3d307adfc923ddd528038',
'd7e0d30fbcae12c90680eb095a4f5f02505ce501',
'ff7ca51e58c505fec0dd2491de52c622bb7a806b',
'dd80b0f6cf5052f17cc738c2951c4f2070200d7f',
'8430a588b43b5d6da365400117c89400326e7992',
'd955cd312c17b02143c04fa1099a352b04368118',
'f67b87e5c629c2ee0ba58f85197e423ff28d735b',
'add63e382e4aabc9e1afdc4bdc24506c269b7618',
'f298fe1189f1b69779a4423f40b48edf92a703fc',
'bd9b619eb41994cac43d67cf4ccc8399c1125808',
'6e125e7c890379446e98980d8ed60fba87d0f6d1',
'd4a54db9f745dfeba6933bf5b1e79e15d0af20bd',
'0b05e4ed56c802098dfc813cbe779b2f49e92500',
'191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e',
'45223f8f114c64bf4d6f853e3c35a369a6305520',
'ca1eb7957a54bce53b12d1a51b13452f95bc7c7e',
'f5ea29fc42ef67a2a5a7aecff10e1566699acd68',
'27d48942240f5b91dfda77accd2caac94708cc7d',
'622f0eb0bafd619d2560c26f80f09e3b0b0d78af',
'e686b958768ee96af8029fe19c6050b1a8dd3b2b'}
assert subset.issubset(set(self.repo.commit_ids))
def test_slicing(self):
@ -281,12 +280,12 @@ TODO: To be written...
new_branch = 'new_branch'
assert repo_clone._current_branch() == 'master'
assert set(repo_clone.branches) == set(('master',))
assert set(repo_clone.branches) == {'master'}
repo_clone._checkout(new_branch, create=True)
# Branches is a lazy property so we need to recrete the Repo object.
repo_clone = GitRepository(repo_clone.path)
assert set(repo_clone.branches) == set(('master', new_branch))
assert set(repo_clone.branches) == {'master', new_branch}
assert repo_clone._current_branch() == new_branch
def test_checkout(self):
@ -1171,7 +1170,7 @@ class TestGitRegression(BackendTestMixin):
assert paths == expected_paths
class TestDiscoverGitVersion:
class TestDiscoverGitVersion(object):
def test_returns_git_version(self, baseapp):
version = discover_git_version()

4
rhodecode/tests/vcs/test_hg.py
View file

@ -122,7 +122,7 @@ class TestMercurialRepository:
repo = MercurialRepository(TEST_HG_REPO)
repo_clone = MercurialRepository(
TEST_HG_REPO_CLONE + '_w_update',
src_url=TEST_HG_REPO, update_after_clone=True)
src_url=TEST_HG_REPO, do_workspace_checkout=True)
assert len(repo.commit_ids) == len(repo_clone.commit_ids)
# check if current workdir was updated
@ -133,7 +133,7 @@ class TestMercurialRepository:
repo = MercurialRepository(TEST_HG_REPO)
repo_clone = MercurialRepository(
TEST_HG_REPO_CLONE + '_wo_update',
src_url=TEST_HG_REPO, update_after_clone=False)
src_url=TEST_HG_REPO, do_workspace_checkout=False)
assert len(repo.commit_ids) == len(repo_clone.commit_ids)
assert not os.path.isfile(
os.path.join(TEST_HG_REPO_CLONE + '_wo_update', 'MANIFEST.in'))