engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(remap and rescan): added more relient remap and removal option, and also split the logic to either add or cleanup

Browse source
This commit is contained in:
RhodeCode Admin 2024-11-13 13:39:21 +01:00
parent 2b623ea7f2
commit ef109d8f05
16 changed files with 375 additions and 208 deletions
Download patch file Download diff file Expand all files Collapse all files

44
rhodecode/api/tests/test_cleanup_repos.py Normal file
View file

@ -0,0 +1,44 @@
# Copyright (C) 2010-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import mock
import pytest
from rhodecode.model.scm import ScmModel
from rhodecode.api.tests.utils import (
build_data, api_call, assert_ok, assert_error, crash)
@pytest.mark.usefixtures("testuser_api", "app")
class TestCleanupRepos(object):
def test_api_cleanup_repos(self):
id_, params = build_data(self.apikey, 'cleanup_repos')
response = api_call(self.app, params)
expected = {'removed': [], 'errors': []}
assert_ok(id_, expected, given=response.body)
def test_api_cleanup_repos_error(self):
id_, params = build_data(self.apikey, 'cleanup_repos', )
with mock.patch('rhodecode.lib.utils.repo2db_cleanup', side_effect=crash):
response = api_call(self.app, params)
expected = 'Error occurred during repo storage cleanup action'
assert_error(id_, expected, given=response.body)

10
rhodecode/api/tests/test_rescan_repos.py
View file

@ -19,7 +19,6 @@
import mock
import pytest
from rhodecode.model.scm import ScmModel
from rhodecode.api.tests.utils import (
build_data, api_call, assert_ok, assert_error, crash)
@ -30,13 +29,14 @@ class TestRescanRepos(object):
id_, params = build_data(self.apikey, 'rescan_repos')
response = api_call(self.app, params)
expected = {'added': [], 'removed': []}
expected = {'added': [], 'errors': []}
assert_ok(id_, expected, given=response.body)
@mock.patch.object(ScmModel, 'repo_scan', crash)
def test_api_rescann_error(self):
def test_api_rescan_repos_error(self):
id_, params = build_data(self.apikey, 'rescan_repos', )
response = api_call(self.app, params)
with mock.patch('rhodecode.lib.utils.repo2db_mapper', side_effect=crash):
response = api_call(self.app, params)
expected = 'Error occurred during rescan repositories action'
assert_error(id_, expected, given=response.body)

58
rhodecode/api/views/server_api.py
View file

@ -18,14 +18,13 @@
import logging
import itertools
import base64
from rhodecode.api import (
jsonrpc_method, JSONRPCError, JSONRPCForbidden, find_methods)
from rhodecode.api.utils import (
Optional, OAttr, has_superadmin_permission, get_user_or_error)
from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path
from rhodecode.lib.utils import get_rhodecode_repo_store_path
from rhodecode.lib import system_info
from rhodecode.lib import user_sessions
from rhodecode.lib import exc_tracking
@ -33,9 +32,7 @@ from rhodecode.lib.ext_json import json
from rhodecode.lib.utils2 import safe_int
from rhodecode.model.db import UserIpMap
from rhodecode.model.scm import ScmModel
from rhodecode.apps.file_store import utils as store_utils
from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
FileOverSizeException
log = logging.getLogger(__name__)
@ -158,13 +155,10 @@ def get_ip(request, apiuser, userid=Optional(OAttr('apiuser'))):
@jsonrpc_method()
def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
def rescan_repos(request, apiuser):
"""
Triggers a rescan of the specified repositories.
* If the ``remove_obsolete`` option is set, it also deletes repositories
that are found in the database but not on the file system, so called
"clean zombies".
It returns list of added repositories, and errors during scan.
This command can only be run using an |authtoken| with admin rights to
the specified repository.
@ -173,9 +167,6 @@ def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
:param apiuser: This is filled automatically from the |authtoken|.
:type apiuser: AuthUser
:param remove_obsolete: Deletes repositories from the database that
are not found on the filesystem.
:type remove_obsolete: Optional(``True`` | ``False``)
Example output:
@ -184,7 +175,7 @@ def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
id : <id_given_in_input>
result : {
'added': [<added repository name>,...]
'removed': [<removed repository name>,...]
'errors': [<error_list>,...]
}
error : null
@ -199,28 +190,24 @@ def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
}
"""
from rhodecode.lib.utils import repo2db_mapper # re-import for testing patches
if not has_superadmin_permission(apiuser):
raise JSONRPCForbidden()
try:
rm_obsolete = Optional.extract(remove_obsolete)
added, removed = repo2db_mapper(ScmModel().repo_scan(),
remove_obsolete=rm_obsolete, force_hooks_rebuild=True)
return {'added': added, 'removed': removed}
added, errors = repo2db_mapper(ScmModel().repo_scan(), force_hooks_rebuild=True)
return {'added': added, 'errors': errors}
except Exception:
log.exception('Failed to run repo rescann')
log.exception('Failed to run repo rescan')
raise JSONRPCError(
'Error occurred during rescan repositories action'
)
@jsonrpc_method()
def cleanup_repos(request, apiuser, remove_obsolete=Optional(False)):
def cleanup_repos(request, apiuser):
"""
Triggers a rescan of the specified repositories.
* If the ``remove_obsolete`` option is set, it also deletes repositories
that are found in the database but not on the file system, so called
"clean zombies".
Triggers a cleanup of non-existing repositories or repository groups in filesystem.
This command can only be run using an |authtoken| with admin rights to
the specified repository.
@ -229,9 +216,6 @@ def cleanup_repos(request, apiuser, remove_obsolete=Optional(False)):
:param apiuser: This is filled automatically from the |authtoken|.
:type apiuser: AuthUser
:param remove_obsolete: Deletes repositories from the database that
are not found on the filesystem.
:type remove_obsolete: Optional(``True`` | ``False``)
Example output:
@ -239,8 +223,8 @@ def cleanup_repos(request, apiuser, remove_obsolete=Optional(False)):
id : <id_given_in_input>
result : {
'added': [<added repository name>,...]
'removed': [<removed repository name>,...]
'removed': [<removed repository name or repository group name>,...]
'errors': [<error list of failures to remove>,...]
}
error : null
@ -251,22 +235,22 @@ def cleanup_repos(request, apiuser, remove_obsolete=Optional(False)):
id : <id_given_in_input>
result : null
error : {
'Error occurred during rescan repositories action'
'Error occurred during repo storage cleanup action'
}
"""
from rhodecode.lib.utils import repo2db_cleanup # re-import for testing patches
if not has_superadmin_permission(apiuser):
raise JSONRPCForbidden()
try:
rm_obsolete = Optional.extract(remove_obsolete)
added, removed = repo2db_mapper(ScmModel().repo_scan(),
remove_obsolete=rm_obsolete, force_hooks_rebuild=True)
return {'added': added, 'removed': removed}
removed, errors = repo2db_cleanup()
return {'removed': removed, 'errors': errors}
except Exception:
log.exception('Failed to run repo rescann')
log.exception('Failed to run repo storage cleanup')
raise JSONRPCError(
'Error occurred during rescan repositories action'
'Error occurred during repo storage cleanup action'
)

17
rhodecode/apps/admin/__init__.py
View file

@ -361,12 +361,21 @@ def admin_routes(config):
renderer='rhodecode:templates/admin/settings/settings.mako')
config.add_route(
name='admin_settings_mapping_update',
pattern='/settings/mapping/update')
name='admin_settings_mapping_create',
pattern='/settings/mapping/create')
config.add_view(
AdminSettingsView,
attr='settings_mapping_update',
route_name='admin_settings_mapping_update', request_method='POST',
attr='settings_mapping_create',
route_name='admin_settings_mapping_create', request_method='POST',
renderer='rhodecode:templates/admin/settings/settings.mako')
config.add_route(
name='admin_settings_mapping_cleanup',
pattern='/settings/mapping/cleanup')
config.add_view(
AdminSettingsView,
attr='settings_mapping_cleanup',
route_name='admin_settings_mapping_cleanup', request_method='POST',
renderer='rhodecode:templates/admin/settings/settings.mako')
config.add_route(

8
rhodecode/apps/admin/tests/test_admin_repos.py
View file

@ -110,9 +110,11 @@ class TestAdminRepos(object):
repo_type=backend.alias,
repo_description=description,
csrf_token=csrf_token))
self.assert_repository_is_created_correctly(
repo_name, description, backend)
try:
self.assert_repository_is_created_correctly(repo_name, description, backend)
finally:
RepoModel().delete(numeric_repo)
Session().commit()
@pytest.mark.parametrize("suffix", ['', '_ąćę'], ids=['', 'non-ascii'])
def test_create_in_group(

35
rhodecode/apps/admin/views/settings.py
View file

@ -38,7 +38,7 @@ from rhodecode.lib.auth import (
LoginRequired, HasPermissionAllDecorator, CSRFRequired)
from rhodecode.lib.celerylib import tasks, run_task
from rhodecode.lib.str_utils import safe_str
from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path
from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path, repo2db_cleanup
from rhodecode.lib.utils2 import str2bool, AttributeDict
from rhodecode.lib.index import searcher_from_config
@ -233,13 +233,12 @@ class AdminSettingsView(BaseAppView):
@LoginRequired()
@HasPermissionAllDecorator('hg.admin')
@CSRFRequired()
def settings_mapping_update(self):
def settings_mapping_create(self):
_ = self.request.translate
c = self.load_default_context()
c.active = 'mapping'
rm_obsolete = self.request.POST.get('destroy', False)
invalidate_cache = self.request.POST.get('invalidate', False)
log.debug('rescanning repo location with destroy obsolete=%s', rm_obsolete)
log.debug('rescanning repo location')
if invalidate_cache:
log.debug('invalidating all repositories cache')
@ -247,16 +246,34 @@ class AdminSettingsView(BaseAppView):
ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
filesystem_repos = ScmModel().repo_scan()
added, removed = repo2db_mapper(filesystem_repos, rm_obsolete, force_hooks_rebuild=True)
added, errors = repo2db_mapper(filesystem_repos, force_hooks_rebuild=True)
PermissionModel().trigger_permission_flush()
def _repr(rm_repo):
return ', '.join(map(safe_str, rm_repo)) or '-'
h.flash(_('Repositories successfully '
'rescanned added: %s ; removed: %s') %
(_repr(added), _repr(removed)),
category='success')
if errors:
h.flash(_('Errors during scan: {}').format(_repr(errors), ), category='error')
h.flash(_('Repositories successfully scanned: Added: {}').format(_repr(added)), category='success')
raise HTTPFound(h.route_path('admin_settings_mapping'))
@LoginRequired()
@HasPermissionAllDecorator('hg.admin')
@CSRFRequired()
def settings_mapping_cleanup(self):
_ = self.request.translate
c = self.load_default_context()
c.active = 'mapping'
log.debug('rescanning repo location')
removed, errors = repo2db_cleanup()
PermissionModel().trigger_permission_flush()
def _repr(rm_repo):
return ', '.join(map(safe_str, rm_repo)) or '-'
h.flash(_('Repositories successfully scanned: Errors: {}, Added: {}').format(errors, _repr(removed)), category='success')
raise HTTPFound(h.route_path('admin_settings_mapping'))
@LoginRequired()

171
rhodecode/lib/utils.py
View file

@ -582,23 +582,19 @@ def map_groups(path):
return group
def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild=False):
def repo2db_mapper(initial_repo_list, force_hooks_rebuild=False):
"""
maps all repos given in initial_repo_list, non existing repositories
are created, if remove_obsolete is True it also checks for db entries
that are not in initial_repo_list and removes them.
:param initial_repo_list: list of repositories found by scanning methods
:param remove_obsolete: check for obsolete entries in database
maps all repos given in initial_repo_list, non-existing repositories
are created
"""
from rhodecode.model.repo import RepoModel
from rhodecode.model.repo_group import RepoGroupModel
from rhodecode.model.settings import SettingsModel
sa = meta.Session()
repo_model = RepoModel()
user = User.get_first_super_admin()
added = []
errors = []
# creation defaults
defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
@ -616,9 +612,7 @@ def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild
if not db_repo:
log.info('repository `%s` not found in the database, creating now', name)
added.append(name)
desc = (repo.description
if repo.description != 'unknown'
else '%s repository' % name)
desc = repo.description if repo.description != 'unknown' else f'{name} repository'
db_repo = repo_model._create_repo(
repo_name=name,
@ -633,76 +627,115 @@ def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild
state=Repository.STATE_CREATED
)
sa.commit()
try:
config = db_repo._config
config.set('extensions', 'largefiles', '')
scm_repo = db_repo.scm_instance(config=config)
except Exception:
log.error(traceback.format_exc())
errors.append(f'getting vcs instance for {name} failed')
continue
try:
db_repo.update_commit_cache(recursive=False)
except Exception:
log.error(traceback.format_exc())
errors.append(f'update_commit_cache for {name} failed')
continue
try:
scm_repo.install_hooks(force=force_hooks_rebuild)
except Exception:
log.error(traceback.format_exc())
errors.append(f'install_hooks for {name} failed')
continue
try:
# we added that repo just now, and make sure we updated server info
if db_repo.repo_type == 'git':
git_repo = db_repo.scm_instance()
# update repository server-info
log.debug('Running update server info')
git_repo._update_server_info(force=True)
scm_repo._update_server_info(force=True)
except Exception:
log.error(traceback.format_exc())
errors.append(f'update_server_info for {name} failed')
continue
db_repo.update_commit_cache(recursive=False)
return added, errors
config = db_repo._config
config.set('extensions', 'largefiles', '')
repo = db_repo.scm_instance(config=config)
repo.install_hooks(force=force_hooks_rebuild)
def repo2db_cleanup(skip_repos=None, skip_groups=None):
from rhodecode.model.repo import RepoModel
from rhodecode.model.repo_group import RepoGroupModel
sa = meta.Session()
removed = []
if remove_obsolete:
# remove from database those repositories that are not in the filesystem
for repo in sa.query(Repository).all():
if repo.repo_name not in list(initial_repo_list.keys()):
log.debug("Removing non-existing repository found in db `%s`",
repo.repo_name)
try:
RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
sa.commit()
removed.append(repo.repo_name)
except Exception:
# don't hold further removals on error
log.error(traceback.format_exc())
sa.rollback()
errors = []
def splitter(full_repo_name):
_parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
gr_name = None
if len(_parts) == 2:
gr_name = _parts[0]
return gr_name
initial_repo_group_list = [splitter(x) for x in
list(initial_repo_list.keys()) if splitter(x)]
all_repos = Repository.execute(
Repository.select(Repository)\
.order_by(Repository.repo_name)
).scalars()
# remove from database those repository groups that are not in the
# filesystem due to parent child relationships we need to delete them
# in a specific order of most nested first
all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
def nested_sort(gr):
return len(gr.split('/'))
for group_name in sorted(all_groups, key=nested_sort, reverse=True):
if group_name not in initial_repo_group_list:
repo_group = RepoGroup.get_by_group_name(group_name)
if (repo_group.children.all() or
not RepoGroupModel().check_exist_filesystem(
group_name=group_name, exc_on_failure=False)):
continue
# remove from database those repositories that are not in the filesystem
for db_repo in all_repos:
db_repo_name = db_repo.repo_name
if skip_repos and db_repo_name in skip_repos:
log.debug('Skipping repo `%s`', db_repo_name)
continue
try:
instance = db_repo.scm_instance()
except Exception:
instance = None
log.info(
'Removing non-existing repository group found in db `%s`',
group_name)
try:
RepoGroupModel(sa).delete(group_name, fs_remove=False)
sa.commit()
removed.append(group_name)
except Exception:
# don't hold further removals on error
log.exception(
'Unable to remove repository group `%s`',
group_name)
sa.rollback()
raise
if not instance:
log.debug("Removing non-existing repository found in db `%s`", db_repo_name)
try:
RepoModel(sa).delete(db_repo, forks='detach', fs_remove=False, call_events=False)
sa.commit()
removed.append(db_repo_name)
except Exception:
# don't hold further removals on error
log.error(traceback.format_exc())
sa.rollback()
errors.append(db_repo_name)
# remove from database those repository groups that are not in the
# filesystem due to parent child relationships we need to delete them
# in a specific order of most nested first
all_groups = RepoGroup.execute(
RepoGroup.select(RepoGroup.group_name)\
.order_by(RepoGroup.group_name)
).scalars().all()
def nested_sort(gr):
return len(gr.split('/'))
for group_name in sorted(all_groups, key=nested_sort, reverse=True):
if skip_groups and group_name in skip_groups:
log.debug('Skipping repo group `%s`', group_name)
continue
repo_group = RepoGroup.get_by_group_name(group_name)
if repo_group.children.all() or not RepoGroupModel().check_exist_filesystem(group_name=group_name, exc_on_failure=False):
continue
log.info('Removing non-existing repository group found in db `%s`', group_name)
try:
RepoGroupModel(sa).delete(group_name, fs_remove=False, call_events=False)
sa.commit()
removed.append(group_name)
except Exception:
# don't hold further removals on error
log.exception('Unable to remove repository group `%s`',group_name)
sa.rollback()
errors.append(group_name)
return removed, errors
return added, removed
def deep_reload_package(package_name):
"""
@ -829,7 +862,7 @@ def create_test_repositories(test_path, config):
raise ImportError('Failed to import rc_testdata, '
'please make sure this package is installed from requirements_test.txt')
from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
from rhodecode.bootstrap import HG_REPO, GIT_REPO, SVN_REPO
log.debug('making test vcs repositories at %s', test_path)

7
rhodecode/lib/vcs/backends/__init__.py
View file

@ -58,9 +58,10 @@ def get_vcs_instance(repo_path, *args, **kwargs) -> typing.VCSRepo | None:
raise VCSError(f"Given path {repo_path} is not a directory")
except VCSError:
log.exception(
'Perhaps this repository is in db and not in '
'filesystem run rescan repositories with '
'"destroy old data" option from admin panel')
'Perhaps this repository is in db and not in filesystem.'
'Run cleanup filesystem option from admin settings under Remap and rescan'
)
return None
return backend(repo_path=repo_path, *args, **kwargs)

79
rhodecode/model/repo.py
View file

@ -745,7 +745,7 @@ class RepoModel(BaseModel):
log.error(traceback.format_exc())
raise
def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None):
def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None, call_events=True):
"""
Delete given repository, forks parameter defines what do do with
attached forks. Throws AttachedForksError if deleted repo has attached
@ -760,47 +760,54 @@ class RepoModel(BaseModel):
if not cur_user:
cur_user = getattr(get_current_rhodecode_user(), 'username', None)
repo = self._get_repo(repo)
if repo:
if forks == 'detach':
for r in repo.forks:
r.fork = None
self.sa.add(r)
elif forks == 'delete':
for r in repo.forks:
self.delete(r, forks='delete')
elif [f for f in repo.forks]:
raise AttachedForksError()
if not repo:
return False
# check for pull requests
pr_sources = repo.pull_requests_source
pr_targets = repo.pull_requests_target
if pull_requests != 'delete' and (pr_sources or pr_targets):
raise AttachedPullRequestsError()
if forks == 'detach':
for r in repo.forks:
r.fork = None
self.sa.add(r)
elif forks == 'delete':
for r in repo.forks:
self.delete(r, forks='delete')
elif [f for f in repo.forks]:
raise AttachedForksError()
artifacts_objs = repo.artifacts
if artifacts == 'delete':
for a in artifacts_objs:
self.sa.delete(a)
elif [a for a in artifacts_objs]:
raise AttachedArtifactsError()
# check for pull requests
pr_sources = repo.pull_requests_source
pr_targets = repo.pull_requests_target
if pull_requests != 'delete' and (pr_sources or pr_targets):
raise AttachedPullRequestsError()
old_repo_dict = repo.get_dict()
artifacts_objs = repo.artifacts
if artifacts == 'delete':
for a in artifacts_objs:
self.sa.delete(a)
elif [a for a in artifacts_objs]:
raise AttachedArtifactsError()
old_repo_dict = repo.get_dict()
if call_events:
events.trigger(events.RepoPreDeleteEvent(repo))
try:
self.sa.delete(repo)
if fs_remove:
self._delete_filesystem_repo(repo)
else:
log.debug('skipping removal from filesystem')
old_repo_dict.update({
'deleted_by': cur_user,
'deleted_on': time.time(),
})
try:
self.sa.delete(repo)
if fs_remove:
self._delete_filesystem_repo(repo)
else:
log.debug('skipping removal from filesystem')
old_repo_dict.update({
'deleted_by': cur_user,
'deleted_on': time.time(),
})
if call_events:
hooks_base.delete_repository(**old_repo_dict)
events.trigger(events.RepoDeleteEvent(repo))
except Exception:
log.error(traceback.format_exc())
raise
except Exception:
log.error(traceback.format_exc())
raise
return True
def grant_user_permission(self, repo, user, perm):
"""

13
rhodecode/model/repo_group.py
View file

@ -156,7 +156,7 @@ class RepoGroupModel(BaseModel):
def check_exist_filesystem(self, group_name, exc_on_failure=True):
create_path = os.path.join(self.repos_path, group_name)
log.debug('creating new group in %s', create_path)
log.debug('checking FS presence for repo group in %s', create_path)
if os.path.isdir(create_path):
if exc_on_failure:
@ -573,10 +573,11 @@ class RepoGroupModel(BaseModel):
log.error(traceback.format_exc())
raise
def delete(self, repo_group, force_delete=False, fs_remove=True):
def delete(self, repo_group, force_delete=False, fs_remove=True, call_events=True):
repo_group = self._get_repo_group(repo_group)
if not repo_group:
return False
repo_group_name = repo_group.group_name
try:
self.sa.delete(repo_group)
if fs_remove:
@ -585,13 +586,15 @@ class RepoGroupModel(BaseModel):
log.debug('skipping removal from filesystem')
# Trigger delete event.
events.trigger(events.RepoGroupDeleteEvent(repo_group))
return True
if call_events:
events.trigger(events.RepoGroupDeleteEvent(repo_group))
except Exception:
log.error('Error removing repo_group %s', repo_group)
log.error('Error removing repo_group %s', repo_group_name)
raise
return True
def grant_user_permission(self, repo_group, user, perm):
"""
Grant permission for user on given repository group, or update

2
rhodecode/subscribers.py
View file

@ -121,7 +121,7 @@ def scan_repositories_if_enabled(event):
from rhodecode.lib.utils import repo2db_mapper
scm = ScmModel()
repositories = scm.repo_scan(scm.repos_path)
repo2db_mapper(repositories, remove_obsolete=False)
repo2db_mapper(repositories)
def write_metadata_if_needed(event):

38
rhodecode/templates/admin/settings/settings_mapping.mako
View file

@ -1,21 +1,16 @@
${h.secure_form(h.route_path('admin_settings_mapping_update'), request=request)}
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">${_('Import New Groups or Repositories')}</h3>
<h3 class="panel-title">${_('Import new repository groups and repositories')}</h3>
</div>
<div class="panel-body">
${h.secure_form(h.route_path('admin_settings_mapping_create'), request=request)}
<p>
${_('This function will scann all data under the current storage path location at')} <code>${c.storage_path}</code>
${_('This function will scan all data under the current storage path location at')} <code>${c.storage_path}</code><br/>
${_('Each folder will be imported as a new repository group, and each repository found will be also imported to root level or corresponding repository group')}
</p>
<div class="checkbox">
${h.checkbox('destroy',True)}
<label for="destroy">${_('Destroy old data')}</label>
</div>
<span class="help-block">${_('In case a repository or a group was deleted from the filesystem and it still exists in the database, check this option to remove obsolete data from the database.')}</span>
<div class="checkbox">
${h.checkbox('invalidate',True)}
<label for="invalidate"> ${_('Invalidate cache for all repositories')}</label>
@ -23,11 +18,28 @@ ${h.secure_form(h.route_path('admin_settings_mapping_update'), request=request)}
<span class="help-block">${_('Each cache data for repositories will be cleaned with this option selected. Use this to reload data and clear cache keys.')}</span>
<div class="buttons">
${h.submit('rescan',_('Rescan Filesystem'),class_="btn")}
${h.submit('rescan',_('Scan filesystem'),class_="btn")}
</div>
${h.end_form()}
</div>
</div>
${h.end_form()}
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">${_('Cleanup removed Repository Groups or Repositories')}</h3>
</div>
<div class="panel-body">
${h.secure_form(h.route_path('admin_settings_mapping_cleanup'), request=request)}
<p>
${_('This function will scan all data under the current storage path location at')} <code>${c.storage_path}</code>
${_('Then it will remove all repository groups and repositories that are no longer present in the filesystem.')}
</p>
<div class="buttons">
${h.submit('rescan',_('Cleanup filesystem'),class_="btn btn-danger")}
</div>
${h.end_form()}
</div>
</div>

2
rhodecode/tests/fixtures/fixture_utils.py vendored
View file

@ -1558,6 +1558,7 @@ def stub_integration_settings():
@pytest.fixture()
def repo_integration_stub(request, repo_stub, StubIntegrationType, stub_integration_settings):
repo_id = repo_stub.repo_id
integration = IntegrationModel().create(
StubIntegrationType,
settings=stub_integration_settings,
@ -1571,6 +1572,7 @@ def repo_integration_stub(request, repo_stub, StubIntegrationType, stub_integrat
@request.addfinalizer
def cleanup():
IntegrationModel().delete(integration)
RepoModel().delete(repo_id)
return integration

12
rhodecode/tests/integrations/test_integration.py
View file

@ -20,23 +20,21 @@ import time
import pytest
from rhodecode import events
from rhodecode.model.repo import RepoModel
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.model.db import Session, Integration
from rhodecode.model.integration import IntegrationModel
class TestDeleteScopesDeletesIntegrations(object):
def test_delete_repo_with_integration_deletes_integration(
self, repo_integration_stub):
Session().delete(repo_integration_stub.repo)
def test_delete_repo_with_integration_deletes_integration(self, repo_integration_stub):
RepoModel().delete(repo_integration_stub.repo)
Session().commit()
Session().expire_all()
integration = Integration.get(repo_integration_stub.integration_id)
assert integration is None
def test_delete_repo_group_with_integration_deletes_integration(
self, repogroup_integration_stub):
def test_delete_repo_group_with_integration_deletes_integration(self, repogroup_integration_stub):
Session().delete(repogroup_integration_stub.repo_group)
Session().commit()
@ -52,7 +50,7 @@ def counter():
global count
val = count
count += 1
return '{}_{}'.format(val, time.time())
return f'{val}_{time.time()}'
@pytest.fixture()

84
rhodecode/tests/lib/test_utils.py
View file

@ -18,20 +18,25 @@
import multiprocessing
import os
import shutil
import mock
import py
import pytest
import rhodecode
from rhodecode.lib import caching_query
from rhodecode.lib import utils
from rhodecode.lib.str_utils import safe_bytes
from rhodecode.model import settings
from rhodecode.model import db
from rhodecode.model import meta
from rhodecode.model.meta import Session
from rhodecode.model.repo import RepoModel
from rhodecode.model.repo_group import RepoGroupModel
from rhodecode.model.scm import ScmModel
from rhodecode.model.settings import UiSetting, SettingsModel
from rhodecode.tests.fixtures.fixture_pyramid import rhodecode_factory
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode_tools.lib.hash_utils import md5_safe
from rhodecode.lib.ext_json import json
@ -230,7 +235,7 @@ def platform_encodes_filenames():
return path_with_latin1 != read_path
def test_repo2db_mapper_groups(repo_groups):
def test_repo2db_cleaner_removes_zombie_groups(repo_groups):
session = meta.Session()
zombie_group, parent_group, child_group = repo_groups
zombie_path = os.path.join(
@ -238,10 +243,9 @@ def test_repo2db_mapper_groups(repo_groups):
os.rmdir(zombie_path)
# Avoid removing test repos when calling repo2db_mapper
repo_list = {
repo.repo_name: 'test' for repo in session.query(db.Repository).all()
}
utils.repo2db_mapper(repo_list, remove_obsolete=True)
repo_list = [repo.repo_name for repo in session.query(db.Repository).all()]
utils.repo2db_cleanup(skip_repos=repo_list)
groups_in_db = session.query(db.RepoGroup).all()
assert child_group in groups_in_db
@ -249,20 +253,68 @@ def test_repo2db_mapper_groups(repo_groups):
assert zombie_path not in groups_in_db
def test_repo2db_mapper_enables_largefiles(backend):
@pytest.mark.backends("hg", "git", "svn")
def test_repo2db_cleaner_removes_zombie_repos(backend):
repo = backend.create_repo()
repo_list = {repo.repo_name: 'test'}
with mock.patch('rhodecode.model.db.Repository.scm_instance') as scm_mock:
utils.repo2db_mapper(repo_list, remove_obsolete=False)
_, kwargs = scm_mock.call_args
assert kwargs['config'].get('extensions', 'largefiles') == ''
zombie_path = repo.repo_full_path
shutil.rmtree(zombie_path)
removed, errors = utils.repo2db_cleanup()
assert len(removed) == 1
assert not errors
def test_repo2db_mapper_adds_new_repos(request, backend):
repo = backend.create_repo()
cleanup_repos = []
cleanup_groups = []
for num in range(5):
copy_repo_name = f'{repo.repo_name}-{num}'
copy_repo_path = f'{repo.repo_full_path}-{num}'
shutil.copytree(repo.repo_full_path, copy_repo_path)
cleanup_repos.append(copy_repo_name)
for gr_num in range(5):
gr_name = f'my_gr_{gr_num}'
dest_gr = os.path.join(os.path.dirname(repo.repo_full_path), gr_name)
os.makedirs(dest_gr, exist_ok=True)
copy_repo_name = f'{gr_name}/{repo.repo_name}-{gr_num}'
copy_repo_path = f'{dest_gr}/{repo.repo_name}-{gr_num}'
shutil.copytree(repo.repo_full_path, copy_repo_path)
cleanup_repos.append(copy_repo_name)
cleanup_groups.append(gr_name)
repo_list = ScmModel().repo_scan()
added, errors = utils.repo2db_mapper(repo_list)
Session().commit()
assert not errors
assert len(added) == 10
@request.addfinalizer
def cleanup():
for _repo in cleanup_repos:
del_result = RepoModel().delete(_repo, call_events=False)
Session().commit()
assert del_result is True
for _repo_group in cleanup_groups:
del_result = RepoGroupModel().delete(_repo_group, force_delete=True, call_events=False)
Session().commit()
assert del_result is True
@pytest.mark.backends("git", "svn")
def test_repo2db_mapper_installs_hooks_for_repos_in_db(backend):
repo = backend.create_repo()
repo_list = {repo.repo_name: 'test'}
utils.repo2db_mapper(repo_list, remove_obsolete=False)
added, errors = utils.repo2db_mapper(repo_list)
assert not errors
assert repo.scm_instance().get_hooks_info() == {'pre_version': rhodecode.__version__, 'post_version': rhodecode.__version__}
@pytest.mark.backends("git", "svn")
@ -271,7 +323,9 @@ def test_repo2db_mapper_installs_hooks_for_newly_added_repos(backend):
RepoModel().delete(repo, fs_remove=False)
meta.Session().commit()
repo_list = {repo.repo_name: repo.scm_instance()}
utils.repo2db_mapper(repo_list, remove_obsolete=False)
added, errors = utils.repo2db_mapper(repo_list)
assert not errors
assert len(added) == 1
class TestPasswordChanged(object):
@ -453,7 +507,7 @@ class TestGetEnabledHooks(object):
def test_obfuscate_url_pw():
from rhodecode.lib.utils2 import obfuscate_url_pw
engine = u'/home/repos/malmö'
engine = '/home/repos/malmö'
assert obfuscate_url_pw(engine)

3
rhodecode/tests/routes.py
View file

@ -182,7 +182,8 @@ def get_url_defs():
"admin_settings_vcs_svn_pattern_delete": ADMIN_PREFIX
+ "/settings/vcs/svn_pattern_delete",
"admin_settings_mapping": ADMIN_PREFIX + "/settings/mapping",
"admin_settings_mapping_update": ADMIN_PREFIX + "/settings/mapping/update",
"admin_settings_mapping_create": ADMIN_PREFIX + "/settings/mapping/create",
"admin_settings_mapping_cleanup": ADMIN_PREFIX + "/settings/mapping/cleanup",
"admin_settings_visual": ADMIN_PREFIX + "/settings/visual",
"admin_settings_visual_update": ADMIN_PREFIX + "/settings/visual/update",
"admin_settings_issuetracker": ADMIN_PREFIX + "/settings/issue-tracker",