-
${_('Import New Groups or Repositories')}
+ ${_('Import new repository groups and repositories')}
-
+ ${h.secure_form(h.route_path('admin_settings_mapping_create'), request=request)}
- ${_('This function will scann all data under the current storage path location at')} ${c.storage_path}
+ ${_('This function will scan all data under the current storage path location at')} ${c.storage_path}
+ ${_('Each folder will be imported as a new repository group, and each repository found will be also imported to root level or corresponding repository group')}
-
- ${h.checkbox('destroy',True)}
- ${_('Destroy old data')}
-
-
${_('In case a repository or a group was deleted from the filesystem and it still exists in the database, check this option to remove obsolete data from the database.')}
-
${h.checkbox('invalidate',True)}
${_('Invalidate cache for all repositories')}
@@ -23,11 +18,28 @@ ${h.secure_form(h.route_path('admin_settings_mapping_update'), request=request)}
${_('Each cache data for repositories will be cleaned with this option selected. Use this to reload data and clear cache keys.')}
- ${h.submit('rescan',_('Rescan Filesystem'),class_="btn")}
+ ${h.submit('rescan',_('Scan filesystem'),class_="btn")}
-
+ ${h.end_form()}
-${h.end_form()}
+
+
+
${_('Cleanup removed Repository Groups or Repositories')}
+
+
+ ${h.secure_form(h.route_path('admin_settings_mapping_cleanup'), request=request)}
+
+ ${_('This function will scan all data under the current storage path location at')} ${c.storage_path}
+ ${_('Then it will remove all repository groups and repositories that are no longer present in the filesystem.')}
+
+
+
+ ${h.submit('rescan',_('Cleanup filesystem'),class_="btn btn-danger")}
+
+ ${h.end_form()}
+
+
+
diff --git a/rhodecode/tests/fixtures/fixture_utils.py b/rhodecode/tests/fixtures/fixture_utils.py
index 5caedd3f..8b64efc8 100644
--- a/rhodecode/tests/fixtures/fixture_utils.py
+++ b/rhodecode/tests/fixtures/fixture_utils.py
@@ -1558,6 +1558,7 @@ def stub_integration_settings():
@pytest.fixture()
def repo_integration_stub(request, repo_stub, StubIntegrationType, stub_integration_settings):
+ repo_id = repo_stub.repo_id
integration = IntegrationModel().create(
StubIntegrationType,
settings=stub_integration_settings,
@@ -1571,6 +1572,7 @@ def repo_integration_stub(request, repo_stub, StubIntegrationType, stub_integrat
@request.addfinalizer
def cleanup():
IntegrationModel().delete(integration)
+ RepoModel().delete(repo_id)
return integration
diff --git a/rhodecode/tests/integrations/test_integration.py b/rhodecode/tests/integrations/test_integration.py
index 70dc62e3..43adb423 100644
--- a/rhodecode/tests/integrations/test_integration.py
+++ b/rhodecode/tests/integrations/test_integration.py
@@ -20,23 +20,21 @@ import time
import pytest
from rhodecode import events
+from rhodecode.model.repo import RepoModel
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.model.db import Session, Integration
from rhodecode.model.integration import IntegrationModel
class TestDeleteScopesDeletesIntegrations(object):
- def test_delete_repo_with_integration_deletes_integration(
- self, repo_integration_stub):
-
- Session().delete(repo_integration_stub.repo)
+ def test_delete_repo_with_integration_deletes_integration(self, repo_integration_stub):
+ RepoModel().delete(repo_integration_stub.repo)
Session().commit()
Session().expire_all()
integration = Integration.get(repo_integration_stub.integration_id)
assert integration is None
- def test_delete_repo_group_with_integration_deletes_integration(
- self, repogroup_integration_stub):
+ def test_delete_repo_group_with_integration_deletes_integration(self, repogroup_integration_stub):
Session().delete(repogroup_integration_stub.repo_group)
Session().commit()
@@ -52,7 +50,7 @@ def counter():
global count
val = count
count += 1
- return '{}_{}'.format(val, time.time())
+ return f'{val}_{time.time()}'
@pytest.fixture()
diff --git a/rhodecode/tests/lib/test_utils.py b/rhodecode/tests/lib/test_utils.py
index e46e38a2..0a1b5851 100644
--- a/rhodecode/tests/lib/test_utils.py
+++ b/rhodecode/tests/lib/test_utils.py
@@ -18,20 +18,25 @@
import multiprocessing
import os
+import shutil
import mock
import py
import pytest
+import rhodecode
from rhodecode.lib import caching_query
from rhodecode.lib import utils
from rhodecode.lib.str_utils import safe_bytes
from rhodecode.model import settings
from rhodecode.model import db
from rhodecode.model import meta
+from rhodecode.model.meta import Session
from rhodecode.model.repo import RepoModel
from rhodecode.model.repo_group import RepoGroupModel
+from rhodecode.model.scm import ScmModel
from rhodecode.model.settings import UiSetting, SettingsModel
+from rhodecode.tests.fixtures.fixture_pyramid import rhodecode_factory
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode_tools.lib.hash_utils import md5_safe
from rhodecode.lib.ext_json import json
@@ -230,7 +235,7 @@ def platform_encodes_filenames():
return path_with_latin1 != read_path
-def test_repo2db_mapper_groups(repo_groups):
+def test_repo2db_cleaner_removes_zombie_groups(repo_groups):
session = meta.Session()
zombie_group, parent_group, child_group = repo_groups
zombie_path = os.path.join(
@@ -238,10 +243,9 @@ def test_repo2db_mapper_groups(repo_groups):
os.rmdir(zombie_path)
# Avoid removing test repos when calling repo2db_mapper
- repo_list = {
- repo.repo_name: 'test' for repo in session.query(db.Repository).all()
- }
- utils.repo2db_mapper(repo_list, remove_obsolete=True)
+ repo_list = [repo.repo_name for repo in session.query(db.Repository).all()]
+
+ utils.repo2db_cleanup(skip_repos=repo_list)
groups_in_db = session.query(db.RepoGroup).all()
assert child_group in groups_in_db
@@ -249,20 +253,68 @@ def test_repo2db_mapper_groups(repo_groups):
assert zombie_path not in groups_in_db
-def test_repo2db_mapper_enables_largefiles(backend):
+
+@pytest.mark.backends("hg", "git", "svn")
+def test_repo2db_cleaner_removes_zombie_repos(backend):
repo = backend.create_repo()
- repo_list = {repo.repo_name: 'test'}
- with mock.patch('rhodecode.model.db.Repository.scm_instance') as scm_mock:
- utils.repo2db_mapper(repo_list, remove_obsolete=False)
- _, kwargs = scm_mock.call_args
- assert kwargs['config'].get('extensions', 'largefiles') == ''
+ zombie_path = repo.repo_full_path
+ shutil.rmtree(zombie_path)
+
+ removed, errors = utils.repo2db_cleanup()
+ assert len(removed) == 1
+ assert not errors
+
+
+def test_repo2db_mapper_adds_new_repos(request, backend):
+ repo = backend.create_repo()
+ cleanup_repos = []
+ cleanup_groups = []
+ for num in range(5):
+ copy_repo_name = f'{repo.repo_name}-{num}'
+ copy_repo_path = f'{repo.repo_full_path}-{num}'
+
+ shutil.copytree(repo.repo_full_path, copy_repo_path)
+ cleanup_repos.append(copy_repo_name)
+
+ for gr_num in range(5):
+ gr_name = f'my_gr_{gr_num}'
+ dest_gr = os.path.join(os.path.dirname(repo.repo_full_path), gr_name)
+ os.makedirs(dest_gr, exist_ok=True)
+
+ copy_repo_name = f'{gr_name}/{repo.repo_name}-{gr_num}'
+ copy_repo_path = f'{dest_gr}/{repo.repo_name}-{gr_num}'
+
+ shutil.copytree(repo.repo_full_path, copy_repo_path)
+ cleanup_repos.append(copy_repo_name)
+ cleanup_groups.append(gr_name)
+
+ repo_list = ScmModel().repo_scan()
+
+ added, errors = utils.repo2db_mapper(repo_list)
+ Session().commit()
+ assert not errors
+
+ assert len(added) == 10
+
+ @request.addfinalizer
+ def cleanup():
+ for _repo in cleanup_repos:
+ del_result = RepoModel().delete(_repo, call_events=False)
+ Session().commit()
+ assert del_result is True
+
+ for _repo_group in cleanup_groups:
+ del_result = RepoGroupModel().delete(_repo_group, force_delete=True, call_events=False)
+ Session().commit()
+ assert del_result is True
-@pytest.mark.backends("git", "svn")
def test_repo2db_mapper_installs_hooks_for_repos_in_db(backend):
repo = backend.create_repo()
repo_list = {repo.repo_name: 'test'}
- utils.repo2db_mapper(repo_list, remove_obsolete=False)
+ added, errors = utils.repo2db_mapper(repo_list)
+ assert not errors
+ assert repo.scm_instance().get_hooks_info() == {'pre_version': rhodecode.__version__, 'post_version': rhodecode.__version__}
@pytest.mark.backends("git", "svn")
@@ -271,7 +323,9 @@ def test_repo2db_mapper_installs_hooks_for_newly_added_repos(backend):
RepoModel().delete(repo, fs_remove=False)
meta.Session().commit()
repo_list = {repo.repo_name: repo.scm_instance()}
- utils.repo2db_mapper(repo_list, remove_obsolete=False)
+ added, errors = utils.repo2db_mapper(repo_list)
+ assert not errors
+ assert len(added) == 1
class TestPasswordChanged(object):
@@ -453,7 +507,7 @@ class TestGetEnabledHooks(object):
def test_obfuscate_url_pw():
from rhodecode.lib.utils2 import obfuscate_url_pw
- engine = u'/home/repos/malmö'
+ engine = '/home/repos/malmö'
assert obfuscate_url_pw(engine)
diff --git a/rhodecode/tests/routes.py b/rhodecode/tests/routes.py
index fb6c30b7..696df030 100644
--- a/rhodecode/tests/routes.py
+++ b/rhodecode/tests/routes.py
@@ -182,7 +182,8 @@ def get_url_defs():
"admin_settings_vcs_svn_pattern_delete": ADMIN_PREFIX
+ "/settings/vcs/svn_pattern_delete",
"admin_settings_mapping": ADMIN_PREFIX + "/settings/mapping",
- "admin_settings_mapping_update": ADMIN_PREFIX + "/settings/mapping/update",
+ "admin_settings_mapping_create": ADMIN_PREFIX + "/settings/mapping/create",
+ "admin_settings_mapping_cleanup": ADMIN_PREFIX + "/settings/mapping/cleanup",
"admin_settings_visual": ADMIN_PREFIX + "/settings/visual",
"admin_settings_visual_update": ADMIN_PREFIX + "/settings/visual/update",
"admin_settings_issuetracker": ADMIN_PREFIX + "/settings/issue-tracker",