engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(remap and rescan): added more relient remap and removal option, and also split the logic to either add or cleanup

Browse source
This commit is contained in:
RhodeCode Admin 2024-11-13 13:39:21 +01:00
parent 2b623ea7f2
commit ef109d8f05
16 changed files with 375 additions and 208 deletions
Download patch file Download diff file Expand all files Collapse all files

44
rhodecode/api/tests/test_cleanup_repos.py Normal file
View file

@ -0,0 +1,44 @@
# Copyright (C) 2010-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import mock
import pytest
from rhodecode.model.scm import ScmModel
from rhodecode.api.tests.utils import (
build_data, api_call, assert_ok, assert_error, crash)
@pytest.mark.usefixtures("testuser_api", "app")
class TestCleanupRepos(object):
def test_api_cleanup_repos(self):
id_, params = build_data(self.apikey, 'cleanup_repos')
response = api_call(self.app, params)
expected = {'removed': [], 'errors': []}
assert_ok(id_, expected, given=response.body)
def test_api_cleanup_repos_error(self):
id_, params = build_data(self.apikey, 'cleanup_repos', )
with mock.patch('rhodecode.lib.utils.repo2db_cleanup', side_effect=crash):
response = api_call(self.app, params)
expected = 'Error occurred during repo storage cleanup action'
assert_error(id_, expected, given=response.body)

10
rhodecode/api/tests/test_rescan_repos.py
View file

@ -19,7 +19,6 @@
import mock
import pytest
from rhodecode.model.scm import ScmModel
from rhodecode.api.tests.utils import (
build_data, api_call, assert_ok, assert_error, crash)
@ -30,13 +29,14 @@ class TestRescanRepos(object):
id_, params = build_data(self.apikey, 'rescan_repos')
response = api_call(self.app, params)
expected = {'added': [], 'removed': []}
expected = {'added': [], 'errors': []}
assert_ok(id_, expected, given=response.body)
@mock.patch.object(ScmModel, 'repo_scan', crash)
def test_api_rescann_error(self):
def test_api_rescan_repos_error(self):
id_, params = build_data(self.apikey, 'rescan_repos', )
response = api_call(self.app, params)
with mock.patch('rhodecode.lib.utils.repo2db_mapper', side_effect=crash):
response = api_call(self.app, params)
expected = 'Error occurred during rescan repositories action'
assert_error(id_, expected, given=response.body)

58
rhodecode/api/views/server_api.py
View file

@ -18,14 +18,13 @@
import logging
import itertools
import base64
from rhodecode.api import (
jsonrpc_method, JSONRPCError, JSONRPCForbidden, find_methods)
from rhodecode.api.utils import (
Optional, OAttr, has_superadmin_permission, get_user_or_error)
from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path
from rhodecode.lib.utils import get_rhodecode_repo_store_path
from rhodecode.lib import system_info
from rhodecode.lib import user_sessions
from rhodecode.lib import exc_tracking
@ -33,9 +32,7 @@ from rhodecode.lib.ext_json import json
from rhodecode.lib.utils2 import safe_int
from rhodecode.model.db import UserIpMap
from rhodecode.model.scm import ScmModel
from rhodecode.apps.file_store import utils as store_utils
from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
FileOverSizeException
log = logging.getLogger(__name__)
@ -158,13 +155,10 @@ def get_ip(request, apiuser, userid=Optional(OAttr('apiuser'))):
@jsonrpc_method()
def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
def rescan_repos(request, apiuser):
"""
Triggers a rescan of the specified repositories.
* If the ``remove_obsolete`` option is set, it also deletes repositories
that are found in the database but not on the file system, so called
"clean zombies".
It returns list of added repositories, and errors during scan.
This command can only be run using an |authtoken| with admin rights to
the specified repository.
@ -173,9 +167,6 @@ def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
:param apiuser: This is filled automatically from the |authtoken|.
:type apiuser: AuthUser
:param remove_obsolete: Deletes repositories from the database that
are not found on the filesystem.
:type remove_obsolete: Optional(``True`` | ``False``)
Example output:
@ -184,7 +175,7 @@ def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
id : <id_given_in_input>
result : {
'added': [<added repository name>,...]
'removed': [<removed repository name>,...]
'errors': [<error_list>,...]
}
error : null
@ -199,28 +190,24 @@ def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
}
"""
from rhodecode.lib.utils import repo2db_mapper # re-import for testing patches
if not has_superadmin_permission(apiuser):
raise JSONRPCForbidden()
try:
rm_obsolete = Optional.extract(remove_obsolete)
added, removed = repo2db_mapper(ScmModel().repo_scan(),
remove_obsolete=rm_obsolete, force_hooks_rebuild=True)
return {'added': added, 'removed': removed}
added, errors = repo2db_mapper(ScmModel().repo_scan(), force_hooks_rebuild=True)
return {'added': added, 'errors': errors}
except Exception:
log.exception('Failed to run repo rescann')
log.exception('Failed to run repo rescan')
raise JSONRPCError(
'Error occurred during rescan repositories action'
)
@jsonrpc_method()
def cleanup_repos(request, apiuser, remove_obsolete=Optional(False)):
def cleanup_repos(request, apiuser):
"""
Triggers a rescan of the specified repositories.
* If the ``remove_obsolete`` option is set, it also deletes repositories
that are found in the database but not on the file system, so called
"clean zombies".
Triggers a cleanup of non-existing repositories or repository groups in filesystem.
This command can only be run using an |authtoken| with admin rights to
the specified repository.
@ -229,9 +216,6 @@ def cleanup_repos(request, apiuser, remove_obsolete=Optional(False)):
:param apiuser: This is filled automatically from the |authtoken|.
:type apiuser: AuthUser
:param remove_obsolete: Deletes repositories from the database that
are not found on the filesystem.
:type remove_obsolete: Optional(``True`` | ``False``)
Example output:
@ -239,8 +223,8 @@ def cleanup_repos(request, apiuser, remove_obsolete=Optional(False)):
id : <id_given_in_input>
result : {
'added': [<added repository name>,...]
'removed': [<removed repository name>,...]
'removed': [<removed repository name or repository group name>,...]
'errors': [<error list of failures to remove>,...]
}
error : null
@ -251,22 +235,22 @@ def cleanup_repos(request, apiuser, remove_obsolete=Optional(False)):
id : <id_given_in_input>
result : null
error : {
'Error occurred during rescan repositories action'
'Error occurred during repo storage cleanup action'
}
"""
from rhodecode.lib.utils import repo2db_cleanup # re-import for testing patches
if not has_superadmin_permission(apiuser):
raise JSONRPCForbidden()
try:
rm_obsolete = Optional.extract(remove_obsolete)
added, removed = repo2db_mapper(ScmModel().repo_scan(),
remove_obsolete=rm_obsolete, force_hooks_rebuild=True)
return {'added': added, 'removed': removed}
removed, errors = repo2db_cleanup()
return {'removed': removed, 'errors': errors}
except Exception:
log.exception('Failed to run repo rescann')
log.exception('Failed to run repo storage cleanup')
raise JSONRPCError(
'Error occurred during rescan repositories action'
'Error occurred during repo storage cleanup action'
)