tests: fixed test suite for celery adoption

conftest.py

@@ -27,8 +27,11 @@ from rhodecode.tests.conftest_common import ( # noqa
 
 
 pytest_plugins = [
-    "rhodecode.tests.fixture_mods.fixture_pyramid",
-    "rhodecode.tests.fixture_mods.fixture_utils",
+    "rhodecode.tests.fixtures.fixture_pyramid",
+    "rhodecode.tests.fixtures.fixture_utils",
+    "rhodecode.tests.fixtures.function_scoped_baseapp",
+    "rhodecode.tests.fixtures.module_scoped_baseapp",
+    "rhodecode.tests.fixtures.rcextensions_fixtures",
 ]
 
 

pyproject.toml

@@ -65,8 +65,7 @@ dependencies = {file = ["requirements.txt"]}
 optional-dependencies.tests = {file = ["requirements_test.txt"]}
 
 [tool.ruff]
-
-select = [
+lint.select = [
     # Pyflakes
     "F",
     # Pycodestyle
@@ -75,16 +74,13 @@ select = [
     # isort
     "I001"
 ]
-
-ignore = [
+lint.ignore = [
     "E501",  # line too long, handled by black
 ]
-
 # Same as Black.
 line-length = 120
 
-[tool.ruff.isort]
-
+[tool.ruff.lint.isort]
 known-first-party = ["rhodecode"]
 
 [tool.ruff.format]

pytest.ini

@@ -4,8 +4,10 @@ norecursedirs = rhodecode/public rhodecode/templates tests/scripts
 cache_dir = /tmp/.pytest_cache
 
 pyramid_config = rhodecode/tests/rhodecode.ini
-vcsserver_protocol = http
-vcsserver_config_http = rhodecode/tests/vcsserver_http.ini
+
+vcsserver_config = rhodecode/tests/vcsserver_http.ini
+rhodecode_config = rhodecode/tests/rhodecode.ini
+celery_config = rhodecode/tests/rhodecode.ini
 
 addopts =
   --pdbcls=IPython.terminal.debugger:TerminalPdb

rhodecode/api/tests/test_add_field_to_repo.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2010-2023 RhodeCode GmbH
+# Copyright (C) 2010-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3

rhodecode/api/tests/test_create_gist.py

@@ -24,7 +24,7 @@ from rhodecode.model.db import Gist
 from rhodecode.model.gist import GistModel
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_error, assert_ok, crash)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 
 
 @pytest.mark.usefixtures("testuser_api", "app")

rhodecode/api/tests/test_create_repo.py

@@ -27,7 +27,7 @@ from rhodecode.model.user import UserModel
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_ok, assert_error, crash)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.str_utils import safe_str
 

rhodecode/api/tests/test_create_repo_group.py

@@ -26,7 +26,7 @@ from rhodecode.model.user import UserModel
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_ok, assert_error, crash)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 
 
 fixture = Fixture()

rhodecode/api/tests/test_create_user.py

@@ -26,7 +26,7 @@ from rhodecode.tests import (
     TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_EMAIL)
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_ok, assert_error, jsonify, crash)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.model.db import RepoGroup
 
 

rhodecode/api/tests/test_create_user_group.py

@@ -25,7 +25,7 @@ from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_error, assert_ok, crash, jsonify)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 
 
 @pytest.mark.usefixtures("testuser_api", "app")

rhodecode/api/tests/test_fork_repo.py

@@ -28,7 +28,7 @@ from rhodecode.model.user import UserModel
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_error, assert_ok, crash)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 
 
 fixture = Fixture()

rhodecode/api/tests/test_update_repo.py

@@ -25,8 +25,8 @@ from rhodecode.model.scm import ScmModel
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_LOGIN
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_error, assert_ok, crash, jsonify)
-from rhodecode.tests.fixture import Fixture
-from rhodecode.tests.fixture_mods.fixture_utils import plain_http_host_only_stub
+from rhodecode.tests.fixtures.rc_fixture import Fixture
+from rhodecode.tests.fixtures.fixture_utils import plain_http_host_only_stub
 
 fixture = Fixture()
 

rhodecode/apps/admin/tests/test_admin_audit_logs.py

@@ -26,7 +26,7 @@ import pytest
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.tests import *
 from rhodecode.tests.routes import route_path
-from rhodecode.tests.fixture import FIXTURES
+from rhodecode.tests.fixtures.rc_fixture import FIXTURES
 from rhodecode.model.db import UserLog
 from rhodecode.model.meta import Session
 

rhodecode/apps/admin/tests/test_admin_main_views.py

@@ -20,7 +20,7 @@
 import pytest
 
 from rhodecode.tests import TestController
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/admin/tests/test_admin_repos.py

@@ -37,7 +37,7 @@ from rhodecode.model.user import UserModel
 from rhodecode.tests import (
     login_user_session, assert_session_flash, TEST_USER_ADMIN_LOGIN,
     TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
-from rhodecode.tests.fixture import Fixture, error_function
+from rhodecode.tests.fixtures.rc_fixture import Fixture, error_function
 from rhodecode.tests.utils import repo_on_filesystem
 from rhodecode.tests.routes import route_path
 

rhodecode/apps/admin/tests/test_admin_repository_groups.py

@@ -27,7 +27,7 @@ from rhodecode.model.meta import Session
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.tests import (
     assert_session_flash, TEST_USER_REGULAR_LOGIN, TESTS_TMP_PATH)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 

rhodecode/apps/admin/tests/test_admin_user_groups.py

@@ -24,7 +24,7 @@ from rhodecode.model.meta import Session
 
 from rhodecode.tests import (
     TestController, assert_session_flash)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/admin/tests/test_admin_users.py

@@ -28,7 +28,7 @@ from rhodecode.model.user import UserModel
 
 from rhodecode.tests import (
     TestController, TEST_USER_REGULAR_LOGIN, assert_session_flash)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py

@@ -22,7 +22,7 @@ import pytest
 from rhodecode.model.db import User, UserSshKeys
 
 from rhodecode.tests import TestController, assert_session_flash
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/home/tests/test_get_goto_switched_data.py

@@ -27,7 +27,7 @@ from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.db import Session, Repository, RepoGroup
 
 from rhodecode.tests import TestController, TEST_USER_ADMIN_LOGIN
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/home/tests/test_get_repo_list_data.py

@@ -22,7 +22,7 @@ from rhodecode.model.db import Repository
 from rhodecode.lib.ext_json import json
 
 from rhodecode.tests import TestController
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/home/tests/test_get_user_data.py

@@ -20,7 +20,7 @@ import pytest
 from rhodecode.lib.ext_json import json
 
 from rhodecode.tests import TestController
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/home/tests/test_get_user_group_data.py

@@ -40,7 +40,7 @@ import pytest
 from rhodecode.lib.ext_json import json
 
 from rhodecode.tests import TestController
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/home/tests/test_home.py

@@ -24,7 +24,7 @@ from rhodecode.model.db import Repository, RepoGroup, User
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import SettingsModel
 from rhodecode.tests import TestController
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 

rhodecode/apps/login/tests/test_2fa.py

@@ -3,7 +3,7 @@ import mock
 
 from rhodecode.lib.type_utils import AttributeDict
 from rhodecode.model.meta import Session
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 from rhodecode.model.settings import SettingsModel
 

rhodecode/apps/login/tests/test_login.py

@@ -31,7 +31,7 @@ from rhodecode.model.meta import Session
 from rhodecode.tests import (
     assert_session_flash, HG_REPO, TEST_USER_ADMIN_LOGIN,
     no_newline_id_generator)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/login/tests/test_password_reset.py

@@ -22,7 +22,7 @@ from rhodecode.lib import helpers as h
 from rhodecode.tests import (
     TestController, clear_cache_regions,
     TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.utils import AssertResponse
 from rhodecode.tests.routes import route_path
 

rhodecode/apps/my_account/tests/test_my_account_auth_tokens.py

@@ -22,7 +22,7 @@ from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.model.db import User
 from rhodecode.tests import (
     TestController, assert_session_flash)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 

rhodecode/apps/my_account/tests/test_my_account_emails.py

@@ -23,7 +23,7 @@ from rhodecode.model.db import User, UserEmailMap
 from rhodecode.tests import (
     TestController, TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_EMAIL,
     assert_session_flash, TEST_USER_REGULAR_PASS)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 

rhodecode/apps/my_account/tests/test_my_account_notifications.py

@@ -21,7 +21,7 @@ import pytest
 from rhodecode.tests import (
     TestController, TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
     TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 from rhodecode.model.db import Notification, User

rhodecode/apps/my_account/tests/test_my_account_password.py

@@ -24,7 +24,7 @@ from rhodecode.lib.auth import check_password
 from rhodecode.model.meta import Session
 from rhodecode.model.user import UserModel
 from rhodecode.tests import assert_session_flash, TestController
-from rhodecode.tests.fixture import Fixture, error_function
+from rhodecode.tests.fixtures.rc_fixture import Fixture, error_function
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/my_account/tests/test_my_account_profile.py

@@ -20,7 +20,7 @@
 from rhodecode.tests import (
     TestController, TEST_USER_ADMIN_LOGIN,
     TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/my_account/tests/test_my_account_simple_views.py

@@ -19,7 +19,7 @@
 
 from rhodecode.model.db import User, Repository, UserFollowing
 from rhodecode.tests import TestController, TEST_USER_ADMIN_LOGIN
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py

@@ -21,7 +21,7 @@
 from rhodecode.model.db import User, UserSshKeys
 
 from rhodecode.tests import TestController, assert_session_flash
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/repository/tests/test_repo_compare_on_single_file.py

@@ -22,7 +22,7 @@ import pytest
 from rhodecode.apps.repository.tests.test_repo_compare import ComparePage
 from rhodecode.lib.vcs import nodes
 from rhodecode.lib.vcs.backends.base import EmptyCommit
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.utils import commit_change
 from rhodecode.tests.routes import route_path
 
@@ -166,14 +166,15 @@ class TestSideBySideDiff(object):
         response.mustcontain('Collapse 2 commits')
         response.mustcontain('123 file changed')
 
-        response.mustcontain(
-            'r%s:%s...r%s:%s' % (
-                commit1.idx, commit1.short_id, commit2.idx, commit2.short_id))
+        response.mustcontain(f'r{commit1.idx}:{commit1.short_id}...r{commit2.idx}:{commit2.short_id}')
 
         response.mustcontain(f_path)
 
-    @pytest.mark.xfail(reason='GIT does not handle empty commit compare correct (missing 1 commit)')
+    #@pytest.mark.xfail(reason='GIT does not handle empty commit compare correct (missing 1 commit)')
     def test_diff_side_by_side_from_0_commit_with_file_filter(self, app, backend, backend_stub):
+        if backend.alias == 'git':
+            pytest.skip('GIT does not handle empty commit compare correct (missing 1 commit)')
+
         f_path = b'test_sidebyside_file.py'
         commit1_content = b'content-25d7e49c18b159446c\n'
         commit2_content = b'content-603d6c72c46d953420\n'
@@ -200,9 +201,7 @@ class TestSideBySideDiff(object):
         response.mustcontain('Collapse 2 commits')
         response.mustcontain('1 file changed')
 
-        response.mustcontain(
-            'r%s:%s...r%s:%s' % (
-                commit1.idx, commit1.short_id, commit2.idx, commit2.short_id))
+        response.mustcontain(f'r{commit1.idx}:{commit1.short_id}...r{commit2.idx}:{commit2.short_id}')
 
         response.mustcontain(f_path)
 

rhodecode/apps/repository/tests/test_repo_files.py

@@ -33,7 +33,7 @@ from rhodecode.lib.vcs.conf import settings
 from rhodecode.model.db import Session, Repository
 
 from rhodecode.tests import assert_session_flash
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 

rhodecode/apps/repository/tests/test_repo_forks.py

@@ -21,7 +21,7 @@ import pytest
 
 from rhodecode.tests import TestController, assert_session_flash, HG_FORK, GIT_FORK
 
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.lib import helpers as h
 
 from rhodecode.model.db import Repository

rhodecode/apps/repository/tests/test_repo_maintainance.py

@@ -21,7 +21,7 @@ import pytest
 
 from rhodecode.model.db import Repository, UserRepoToPerm, Permission, User
 
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/repository/tests/test_repo_pullrequests.py

@@ -15,6 +15,9 @@
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
+import logging
+import os
+
 import mock
 import pytest
 
@@ -41,7 +44,7 @@ from rhodecode.tests import (
     TEST_USER_ADMIN_LOGIN,
     TEST_USER_REGULAR_LOGIN,
 )
-from rhodecode.tests.fixture_mods.fixture_utils import PRTestUtility
+from rhodecode.tests.fixtures.fixture_utils import PRTestUtility
 from rhodecode.tests.routes import route_path
 
 
@@ -1050,7 +1053,6 @@ class TestPullrequestsView(object):
         )
         assert len(notifications.all()) == 2
 
-    @pytest.mark.xfail(reason="unable to fix this test after python3 migration")
     def test_create_pull_request_stores_ancestor_commit_id(self, backend, csrf_token):
         commits = [
             {
@@ -1125,20 +1127,38 @@ class TestPullrequestsView(object):
         response.mustcontain(no=["content_of_ancestor-child"])
         response.mustcontain("content_of_change")
 
-    def test_merge_pull_request_enabled(self, pr_util, csrf_token):
-        # Clear any previous calls to rcextensions
-        rhodecode.EXTENSIONS.calls.clear()
+    def test_merge_pull_request_enabled(self, pr_util, csrf_token, rcextensions_modification):
 
         pull_request = pr_util.create_pull_request(approved=True, mergeable=True)
         pull_request_id = pull_request.pull_request_id
-        repo_name = (pull_request.target_repo.scm_instance().name,)
+        repo_name = pull_request.target_repo.scm_instance().name
 
         url = route_path(
             "pullrequest_merge",
-            repo_name=str(repo_name[0]),
+            repo_name=repo_name,
             pull_request_id=pull_request_id,
         )
-        response = self.app.post(url, params={"csrf_token": csrf_token}).follow()
+
+        rcstack_location = os.path.dirname(self.app._pyramid_registry.settings['__file__'])
+        rc_ext_location = os.path.join(rcstack_location, 'rcextension-output.txt')
+
+
+        mods = [
+            ('_push_hook',
+             f"""
+             import os
+             action = kwargs['action']
+             commit_ids = kwargs['commit_ids']
+             with open('{rc_ext_location}', 'w') as f:
+                 f.write('test-execution'+os.linesep)
+                 f.write(f'{{action}}'+os.linesep)
+                 f.write(f'{{commit_ids}}'+os.linesep)
+             return HookResponse(0, 'HOOK_TEST')
+            """)
+        ]
+        # Add the hook
+        with rcextensions_modification(rcstack_location, mods, create_if_missing=True, force_create=True):
+            response = self.app.post(url, params={"csrf_token": csrf_token}).follow()
 
         pull_request = PullRequest.get(pull_request_id)
 
@@ -1162,12 +1182,39 @@ class TestPullrequestsView(object):
         assert actions[-1].action == "user.push"
         assert actions[-1].action_data["commit_ids"] == pr_commit_ids
 
-        # Check post_push rcextension was really executed
-        push_calls = rhodecode.EXTENSIONS.calls["_push_hook"]
-        assert len(push_calls) == 1
-        unused_last_call_args, last_call_kwargs = push_calls[0]
-        assert last_call_kwargs["action"] == "push"
-        assert last_call_kwargs["commit_ids"] == pr_commit_ids
+        with open(rc_ext_location) as f:
+            f_data = f.read()
+            assert 'test-execution' in f_data
+            for commit_id in pr_commit_ids:
+                assert f'{commit_id}' in f_data
+
+    def test_merge_pull_request_forbidden_by_pre_push_hook(self, pr_util, csrf_token, rcextensions_modification, caplog):
+        caplog.set_level(logging.WARNING, logger="rhodecode.model.pull_request")
+
+        pull_request = pr_util.create_pull_request(approved=True, mergeable=True)
+        pull_request_id = pull_request.pull_request_id
+        repo_name = pull_request.target_repo.scm_instance().name
+
+        url = route_path(
+            "pullrequest_merge",
+            repo_name=repo_name,
+            pull_request_id=pull_request_id,
+        )
+
+        rcstack_location = os.path.dirname(self.app._pyramid_registry.settings['__file__'])
+
+        mods = [
+            ('_pre_push_hook',
+             f"""
+             return HookResponse(1, 'HOOK_TEST_FORBIDDEN')
+            """)
+        ]
+        # Add the hook
+        with rcextensions_modification(rcstack_location, mods, create_if_missing=True, force_create=True):
+            self.app.post(url, params={"csrf_token": csrf_token})
+
+        assert 'Merge failed, not updating the pull request.' in [r[2] for r in caplog.record_tuples]
+
 
     def test_merge_pull_request_disabled(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request(mergeable=False)
@@ -1523,7 +1570,6 @@ class TestPullrequestsView(object):
 
         assert pull_request.revisions == [commit_ids["change-rebased"]]
 
-
     def test_remove_pull_request_branch(self, backend_git, csrf_token):
         branch_name = "development"
         commits = [

rhodecode/apps/repository/tests/test_repo_settings.py

@@ -26,7 +26,7 @@ from rhodecode.model.db import Repository, UserRepoToPerm, Permission, User
 from rhodecode.model.meta import Session
 from rhodecode.tests import (
     TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_LOGIN, assert_session_flash)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/repository/tests/test_repo_settings_advanced.py

@@ -24,7 +24,7 @@ from rhodecode.model.db import Repository
 from rhodecode.model.repo import RepoModel
 from rhodecode.tests import (
     HG_REPO, GIT_REPO, assert_session_flash, no_newline_id_generator)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.utils import repo_on_filesystem
 from rhodecode.tests.routes import route_path
 

rhodecode/apps/repository/tests/test_repo_summary.py

@@ -31,7 +31,7 @@ from rhodecode.model.meta import Session
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.scm import ScmModel
 from rhodecode.tests import assert_session_flash
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.utils import AssertResponse, repo_on_filesystem
 from rhodecode.tests.routes import route_path
 

rhodecode/apps/repository/tests/test_repo_vcs_settings.py

@@ -30,7 +30,7 @@ from rhodecode.model.user import UserModel
 from rhodecode.tests import (
     login_user_session, logout_user_session,
     TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.utils import AssertResponse
 from rhodecode.tests.routes import route_path
 

rhodecode/apps/repository/tests/test_vcs_settings.py

@@ -32,16 +32,13 @@ class TestAdminRepoVcsSettings(object):
     @pytest.mark.parametrize('setting_name, setting_backends', [
         ('hg_use_rebase_for_merging', ['hg']),
     ])
-    def test_labs_settings_visible_if_enabled(
-            self, setting_name, setting_backends, backend):
+    def test_labs_settings_visible_if_enabled(self, setting_name, setting_backends, backend):
         if backend.alias not in setting_backends:
             pytest.skip('Setting not available for backend {}'.format(backend))
 
-        vcs_settings_url = route_path(
-            'edit_repo_vcs', repo_name=backend.repo.repo_name)
+        vcs_settings_url = route_path('edit_repo_vcs', repo_name=backend.repo.repo_name)
 
-        with mock.patch.dict(
-                rhodecode.CONFIG, {'labs_settings_active': 'true'}):
+        with mock.patch.dict(rhodecode.CONFIG, {'labs_settings_active': 'true'}):
             response = self.app.get(vcs_settings_url)
 
         assertr = response.assert_response()

rhodecode/apps/ssh_support/lib/backends/base.py

@@ -20,7 +20,7 @@ import os
 import sys
 import logging
 
-from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
+from rhodecode.lib.hook_daemon.utils import prepare_callback_daemon
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.api_utils import call_service_api
@@ -162,9 +162,7 @@ class SshVcsServer(object):
         extras = {}
         extras.update(tunnel_extras)
 
-        callback_daemon, extras = prepare_callback_daemon(
-            extras, protocol=self.hooks_protocol,
-            host=vcs_settings.HOOKS_HOST)
+        callback_daemon, extras = prepare_callback_daemon(extras, protocol=self.hooks_protocol)
 
         with callback_daemon:
             try:

rhodecode/apps/ssh_support/tests/test_server_git.py

@@ -33,19 +33,24 @@ class GitServerCreator(object):
         'app:main': {
             'ssh.executable.git': git_path,
             'vcs.hooks.protocol.v2': 'celery',
+            'app.service_api.host': 'http://localhost',
+            'app.service_api.token': 'secret4',
+            'rhodecode.api.url': '/_admin/api',
         }
     }
     repo_name = 'test_git'
     repo_mode = 'receive-pack'
     user = plain_dummy_user()
 
-    def __init__(self):
-        pass
+    def __init__(self, service_api_url, ini_file):
+        self.service_api_url = service_api_url
+        self.ini_file = ini_file
 
     def create(self, **kwargs):
+        self.config_data['app:main']['app.service_api.host'] = self.service_api_url
         parameters = {
             'store': self.root,
-            'ini_path': '',
+            'ini_path': self.ini_file,
             'user': self.user,
             'repo_name': self.repo_name,
             'repo_mode': self.repo_mode,
@@ -60,12 +65,30 @@ class GitServerCreator(object):
         return server
 
 
-@pytest.fixture()
-def git_server(app):
-    return GitServerCreator()
+@pytest.fixture(scope='module')
+def git_server(request, module_app, rhodecode_factory, available_port_factory):
+    ini_file = module_app._pyramid_settings['__file__']
+    vcsserver_host = module_app._pyramid_settings['vcs.server']
+
+    store_dir = os.path.dirname(ini_file)
+
+    # start rhodecode for service API
+    rc = rhodecode_factory(
+        request,
+        store_dir=store_dir,
+        port=available_port_factory(),
+        overrides=(
+            {'handler_console': {'level': 'DEBUG'}},
+            {'app:main': {'vcs.server': vcsserver_host}},
+            {'app:main': {'repo_store.path': store_dir}}
+        ))
+
+    service_api_url = f'http://{rc.bind_addr}'
+
+    return GitServerCreator(service_api_url, ini_file)
 
 
-class TestGitServer(object):
+class TestGitServer:
 
     def test_command(self, git_server):
         server = git_server.create()
@@ -102,14 +125,14 @@ class TestGitServer(object):
         assert result is value
 
     def test_run_returns_executes_command(self, git_server):
-        server = git_server.create()
         from rhodecode.apps.ssh_support.lib.backends.git import GitTunnelWrapper
+        server = git_server.create()
 
         os.environ['SSH_CLIENT'] = '127.0.0.1'
         with mock.patch.object(GitTunnelWrapper, 'create_hooks_env') as _patch:
             _patch.return_value = 0
             with mock.patch.object(GitTunnelWrapper, 'command', return_value='date'):
-                exit_code = server.run()
+                exit_code = server.run(tunnel_extras={'config': server.ini_path})
 
         assert exit_code == (0, False)
 
@@ -135,7 +158,7 @@ class TestGitServer(object):
             'action': action,
             'ip': '10.10.10.10',
             'locked_by': [None, None],
-            'config': '',
+            'config': git_server.ini_file,
             'repo_store': store,
             'server_url': None,
             'hooks': ['push', 'pull'],

rhodecode/apps/ssh_support/tests/test_server_hg.py

@@ -17,6 +17,7 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
+
 import mock
 import pytest
 
@@ -32,22 +33,27 @@ class MercurialServerCreator(object):
         'app:main': {
             'ssh.executable.hg': hg_path,
             'vcs.hooks.protocol.v2': 'celery',
+            'app.service_api.host': 'http://localhost',
+            'app.service_api.token': 'secret4',
+            'rhodecode.api.url': '/_admin/api',
         }
     }
     repo_name = 'test_hg'
     user = plain_dummy_user()
 
-    def __init__(self):
-        pass
+    def __init__(self, service_api_url, ini_file):
+        self.service_api_url = service_api_url
+        self.ini_file = ini_file
 
     def create(self, **kwargs):
+        self.config_data['app:main']['app.service_api.host'] = self.service_api_url
         parameters = {
             'store': self.root,
-            'ini_path': '',
+            'ini_path': self.ini_file,
             'user': self.user,
             'repo_name': self.repo_name,
             'user_permissions': {
-                'test_hg': 'repository.admin'
+                self.repo_name: 'repository.admin'
             },
             'settings': self.config_data['app:main'],
             'env': plain_dummy_env()
@@ -57,12 +63,30 @@ class MercurialServerCreator(object):
         return server
 
 
-@pytest.fixture()
-def hg_server(app):
-    return MercurialServerCreator()
+@pytest.fixture(scope='module')
+def hg_server(request, module_app, rhodecode_factory, available_port_factory):
+    ini_file = module_app._pyramid_settings['__file__']
+    vcsserver_host = module_app._pyramid_settings['vcs.server']
+
+    store_dir = os.path.dirname(ini_file)
+
+    # start rhodecode for service API
+    rc = rhodecode_factory(
+        request,
+        store_dir=store_dir,
+        port=available_port_factory(),
+        overrides=(
+            {'handler_console': {'level': 'DEBUG'}},
+            {'app:main': {'vcs.server': vcsserver_host}},
+            {'app:main': {'repo_store.path': store_dir}}
+        ))
+
+    service_api_url = f'http://{rc.bind_addr}'
+
+    return MercurialServerCreator(service_api_url, ini_file)
 
 
-class TestMercurialServer(object):
+class TestMercurialServer:
 
     def test_command(self, hg_server, tmpdir):
         server = hg_server.create()
@@ -107,7 +131,7 @@ class TestMercurialServer(object):
         with mock.patch.object(MercurialTunnelWrapper, 'create_hooks_env') as _patch:
             _patch.return_value = 0
             with mock.patch.object(MercurialTunnelWrapper, 'command', return_value='date'):
-                exit_code = server.run()
+                exit_code = server.run(tunnel_extras={'config': server.ini_path})
 
         assert exit_code == (0, False)
 

rhodecode/apps/ssh_support/tests/test_server_svn.py

@@ -15,7 +15,9 @@
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
+
 import os
+
 import mock
 import pytest
 
@@ -26,39 +28,62 @@ from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dum
 class SubversionServerCreator(object):
     root = '/tmp/repo/path/'
     svn_path = '/usr/local/bin/svnserve'
+
     config_data = {
         'app:main': {
             'ssh.executable.svn': svn_path,
             'vcs.hooks.protocol.v2': 'celery',
+            'app.service_api.host': 'http://localhost',
+            'app.service_api.token': 'secret4',
+            'rhodecode.api.url': '/_admin/api',
         }
     }
     repo_name = 'test-svn'
     user = plain_dummy_user()
 
-    def __init__(self):
-        pass
+    def __init__(self, service_api_url, ini_file):
+        self.service_api_url = service_api_url
+        self.ini_file = ini_file
 
     def create(self, **kwargs):
+        self.config_data['app:main']['app.service_api.host'] = self.service_api_url
         parameters = {
             'store': self.root,
-            'repo_name': self.repo_name,
-            'ini_path': '',
+            'ini_path': self.ini_file,
             'user': self.user,
+            'repo_name': self.repo_name,
             'user_permissions': {
                 self.repo_name: 'repository.admin'
             },
             'settings': self.config_data['app:main'],
             'env': plain_dummy_env()
         }
-
         parameters.update(kwargs)
         server = SubversionServer(**parameters)
         return server
 
 
-@pytest.fixture()
-def svn_server(app):
-    return SubversionServerCreator()
+@pytest.fixture(scope='module')
+def svn_server(request, module_app, rhodecode_factory, available_port_factory):
+    ini_file = module_app._pyramid_settings['__file__']
+    vcsserver_host = module_app._pyramid_settings['vcs.server']
+
+    store_dir = os.path.dirname(ini_file)
+
+    # start rhodecode for service API
+    rc = rhodecode_factory(
+        request,
+        store_dir=store_dir,
+        port=available_port_factory(),
+        overrides=(
+            {'handler_console': {'level': 'DEBUG'}},
+            {'app:main': {'vcs.server': vcsserver_host}},
+            {'app:main': {'repo_store.path': store_dir}}
+        ))
+
+    service_api_url = f'http://{rc.bind_addr}'
+
+    return SubversionServerCreator(service_api_url, ini_file)
 
 
 class TestSubversionServer(object):
@@ -168,8 +193,9 @@ class TestSubversionServer(object):
             assert repo_name == expected_match
 
     def test_run_returns_executes_command(self, svn_server):
-        server = svn_server.create()
         from rhodecode.apps.ssh_support.lib.backends.svn import SubversionTunnelWrapper
+
+        server = svn_server.create()
         os.environ['SSH_CLIENT'] = '127.0.0.1'
         with mock.patch.object(
                 SubversionTunnelWrapper, 'get_first_client_response',
@@ -184,20 +210,18 @@ class TestSubversionServer(object):
                             SubversionTunnelWrapper, 'command',
                             return_value=['date']):
 
-                        exit_code = server.run()
+                        exit_code = server.run(tunnel_extras={'config': server.ini_path})
         # SVN has this differently configured, and we get in our mock env
         # None as return code
         assert exit_code == (None, False)
 
     def test_run_returns_executes_command_that_cannot_extract_repo_name(self, svn_server):
-        server = svn_server.create()
         from rhodecode.apps.ssh_support.lib.backends.svn import SubversionTunnelWrapper
-        with mock.patch.object(
-                SubversionTunnelWrapper, 'command',
-                return_value=['date']):
-            with mock.patch.object(
-                    SubversionTunnelWrapper, 'get_first_client_response',
+
+        server = svn_server.create()
+        with mock.patch.object(SubversionTunnelWrapper, 'command', return_value=['date']):
+            with mock.patch.object(SubversionTunnelWrapper, 'get_first_client_response',
                     return_value=None):
-                    exit_code = server.run()
+                    exit_code = server.run(tunnel_extras={'config': server.ini_path})
 
         assert exit_code == (1, False)

rhodecode/apps/user_group/tests/test_user_groups.py

@@ -22,7 +22,7 @@ from rhodecode.tests import (
     TestController, assert_session_flash, TEST_USER_ADMIN_LOGIN)
 from rhodecode.model.db import UserGroup
 from rhodecode.model.meta import Session
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/user_group_profile/tests/test_user_group.py

@@ -18,7 +18,7 @@
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.tests import (
     TestController, TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.routes import route_path
 
 fixture = Fixture()

rhodecode/apps/user_profile/tests/test_users.py

@@ -22,7 +22,7 @@ from rhodecode.model.db import User
 from rhodecode.tests import (
     TestController, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
     TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
-from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.fixtures.rc_fixture import Fixture
 from rhodecode.tests.utils import AssertResponse
 from rhodecode.tests.routes import route_path
 

rhodecode/config/environment.py

@@ -30,7 +30,7 @@ from rhodecode.lib.vcs import connect_vcs
 log = logging.getLogger(__name__)
 
 
-def propagate_rhodecode_config(global_config, settings, config):
+def propagate_rhodecode_config(global_config, settings, config, full=True):
     # Store the settings to make them available to other modules.
     settings_merged = global_config.copy()
     settings_merged.update(settings)
@@ -40,7 +40,7 @@ def propagate_rhodecode_config(global_config, settings, config):
     rhodecode.PYRAMID_SETTINGS = settings_merged
     rhodecode.CONFIG = settings_merged
 
-    if 'default_user_id' not in rhodecode.CONFIG:
+    if full and 'default_user_id' not in rhodecode.CONFIG:
         rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
     log.debug('set rhodecode.CONFIG data')
 
@@ -93,6 +93,7 @@ def load_pyramid_environment(global_config, settings):
     # first run, to store data...
     propagate_rhodecode_config(global_config, settings, {})
 
+
     if vcs_server_enabled:
         connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
     else:

rhodecode/config/middleware.py

@@ -101,6 +101,9 @@ def make_pyramid_app(global_config, **settings):
     patches.inspect_getargspec()
     patches.repoze_sendmail_lf_fix()
 
+    # first init, so load_pyramid_enviroment, can access some critical data, like __file__
+    propagate_rhodecode_config(global_config, {}, {}, full=False)
+
     load_pyramid_environment(global_config, settings)
 
     # Static file view comes first

rhodecode/config/rcextensions/__init__.py

@@ -17,7 +17,7 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
-rcextensions module, please edit `hooks.py` to over write hooks logic
+rcextensions module, please edit `hooks.py` to over-write hooks logic
 """
 
 from .hooks import (

rhodecode/config/rcextensions/examples/validate_pushed_files_name_and_size.py

@@ -85,7 +85,7 @@ def _pre_push_hook(*args, **kwargs):
 
         # check files names
         if forbidden_files:
-            reason = 'File {} is forbidden to be pushed'.format(file_name)
+            reason = f'File {file_name} is forbidden to be pushed'
             for forbidden_pattern in forbid_files:
                 # here we can also filter for operation, e.g if check for only ADDED files
                 # if operation == 'A':

rhodecode/config/rcextensions/helpers/extra_fields.py

@@ -1,4 +1,3 @@
-
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
@@ -55,7 +54,7 @@ def run(*args, **kwargs):
     return fields
 
 
-class _Undefined(object):
+class _Undefined:
     pass
 
 
@@ -67,7 +66,7 @@ def get_field(extra_fields_data, key, default=_Undefined(), convert_type=True):
 
     if key not in extra_fields_data:
         if isinstance(default, _Undefined):
-            raise ValueError('key {} not present in extra_fields'.format(key))
+            raise ValueError(f'key {key} not present in extra_fields')
         return default
 
     # NOTE(dan): from metadata we get field_label, field_value, field_desc, field_type

rhodecode/config/rcextensions/helpers/extract_post_commits.py

@@ -1,4 +1,3 @@
-
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify

rhodecode/config/rcextensions/helpers/extract_pre_commits.py

@@ -1,4 +1,3 @@
-
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
@@ -52,7 +51,7 @@ def get_git_commits(repo, refs):
         cmd = [
             'log',
             '--pretty=format:{"commit_id": "%H",  "author": "%aN <%aE>",  "date": "%ad",  "message": "%s"}',
-            '{}...{}'.format(old_rev, new_rev)
+            f'{old_rev}...{new_rev}'
         ]
 
         stdout, stderr = repo.run_git_command(cmd, extra_env=git_env)
@@ -80,12 +79,12 @@ def run(*args, **kwargs):
 
     if vcs_type == 'git':
         for rev_data in kwargs['commit_ids']:
-            new_environ = dict((k, v) for k, v in rev_data['git_env'])
+            new_environ = {k: v for k, v in rev_data['git_env']}
         commits = get_git_commits(vcs_repo, kwargs['commit_ids'])
 
     if vcs_type == 'hg':
         for rev_data in kwargs['commit_ids']:
-            new_environ = dict((k, v) for k, v in rev_data['hg_env'])
+            new_environ = {k: v for k, v in rev_data['hg_env']}
         commits = get_hg_commits(vcs_repo, kwargs['commit_ids'])
 
     return commits

rhodecode/config/rcextensions/helpers/extract_pre_files.py

@@ -1,4 +1,3 @@
-
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
@@ -133,12 +132,12 @@ def run(*args, **kwargs):
 
     if vcs_type == 'git':
         for rev_data in kwargs['commit_ids']:
-            new_environ = dict((k, v) for k, v in rev_data['git_env'])
+            new_environ = {k: v for k, v in rev_data['git_env']}
         files = get_git_files(repo, vcs_repo, kwargs['commit_ids'])
 
     if vcs_type == 'hg':
         for rev_data in kwargs['commit_ids']:
-            new_environ = dict((k, v) for k, v in rev_data['hg_env'])
+            new_environ = {k: v for k, v in rev_data['hg_env']}
         files = get_hg_files(repo, vcs_repo, kwargs['commit_ids'])
 
     if vcs_type == 'svn':

rhodecode/config/rcextensions/helpers/http_call.py

@@ -1,4 +1,3 @@
-
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify

rhodecode/config/rcextensions/utils.py

@@ -28,7 +28,7 @@ import urllib.error
 log = logging.getLogger('rhodecode.' + __name__)
 
 
-class HookResponse(object):
+class HookResponse:
     def __init__(self, status, output):
         self.status = status
         self.output = output
@@ -44,6 +44,11 @@ class HookResponse(object):
     def __bool__(self):
         return self.status == 0
 
+    def to_json(self):
+        return {'status': self.status, 'output': self.output}
+
+    def __repr__(self):
+        return self.to_json().__repr__()
 
 class DotDict(dict):
 
@@ -91,8 +96,8 @@ class DotDict(dict):
     def __repr__(self):
         keys = list(self.keys())
         keys.sort()
-        args = ', '.join(['%s=%r' % (key, self[key]) for key in keys])
-        return '%s(%s)' % (self.__class__.__name__, args)
+        args = ', '.join(['{}={!r}'.format(key, self[key]) for key in keys])
+        return '{}({})'.format(self.__class__.__name__, args)
 
     @staticmethod
     def fromDict(d):
@@ -110,7 +115,7 @@ def serialize(x):
 
 def unserialize(x):
     if isinstance(x, dict):
-        return dict((k, unserialize(v)) for k, v in x.items())
+        return {k: unserialize(v) for k, v in x.items()}
     elif isinstance(x, (list, tuple)):
         return type(x)(unserialize(v) for v in x)
     else:
@@ -161,7 +166,8 @@ def str2bool(_str) -> bool:
     string into boolean
 
     :param _str: string value to translate into boolean
-    :returns: bool from given string
+    :rtype: boolean
+    :returns: boolean from given string
     """
     if _str is None:
         return False

rhodecode/config/routing_links.py

@@ -49,22 +49,22 @@ link_config = [
     {
         "name": "enterprise_docs",
         "target": "https://rhodecode.com/r1/enterprise/docs/",
-        "external_target": "https://docs.rhodecode.com/RhodeCode-Enterprise/",
+        "external_target": "https://docs.rhodecode.com/4.x/rce/index.html",
     },
     {
         "name": "enterprise_log_file_locations",
         "target": "https://rhodecode.com/r1/enterprise/docs/admin-system-overview/",
-        "external_target": "https://docs.rhodecode.com/RhodeCode-Enterprise/admin/system-overview.html#log-files",
+        "external_target": "https://docs.rhodecode.com/4.x/rce/admin/system-overview.html#log-files",
     },
     {
         "name": "enterprise_issue_tracker_settings",
         "target": "https://rhodecode.com/r1/enterprise/docs/issue-trackers-overview/",
-        "external_target": "https://docs.rhodecode.com/RhodeCode-Enterprise/issue-trackers/issue-trackers.html",
+        "external_target": "https://docs.rhodecode.com/4.x/rce/issue-trackers/issue-trackers.html",
     },
     {
         "name": "enterprise_svn_setup",
         "target": "https://rhodecode.com/r1/enterprise/docs/svn-setup/",
-        "external_target": "https://docs.rhodecode.com/RhodeCode-Enterprise/admin/svn-http.html",
+        "external_target": "https://docs.rhodecode.com/4.x/rce/admin/svn-http.html",
     },
     {
         "name": "enterprise_license_convert_from_old",

rhodecode/config/utils.py

@@ -19,6 +19,8 @@
 import os
 import platform
 
+from rhodecode.lib.type_utils import str2bool
+
 DEFAULT_USER = 'default'
 
 
@@ -48,28 +50,23 @@ def initialize_database(config):
     engine = engine_from_config(config, 'sqlalchemy.db1.')
     init_model(engine, encryption_key=get_encryption_key(config))
 
+def initialize_test_environment(settings):
+    skip_test_env = str2bool(os.environ.get('RC_NO_TEST_ENV'))
+    if skip_test_env:
+        return
 
-def initialize_test_environment(settings, test_env=None):
-    if test_env is None:
-        test_env = not int(os.environ.get('RC_NO_TMP_PATH', 0))
+    repo_store_path = os.environ.get('RC_TEST_ENV_REPO_STORE') or settings['repo_store.path']
 
     from rhodecode.lib.utils import (
         create_test_directory, create_test_database, create_test_repositories,
         create_test_index)
-    from rhodecode.tests import TESTS_TMP_PATH
-    from rhodecode.lib.vcs.backends.hg import largefiles_store
-    from rhodecode.lib.vcs.backends.git import lfs_store
 
+    create_test_directory(repo_store_path)
+
+    create_test_database(repo_store_path, settings)
     # test repos
-    if test_env:
-        create_test_directory(TESTS_TMP_PATH)
-        # large object stores
-        create_test_directory(largefiles_store(TESTS_TMP_PATH))
-        create_test_directory(lfs_store(TESTS_TMP_PATH))
-
-        create_test_database(TESTS_TMP_PATH, settings)
-        create_test_repositories(TESTS_TMP_PATH, settings)
-        create_test_index(TESTS_TMP_PATH, settings)
+    create_test_repositories(repo_store_path, settings)
+    create_test_index(repo_store_path, settings)
 
 
 def get_vcs_server_protocol(config):

rhodecode/lib/exceptions.py

@@ -20,8 +20,7 @@
 Set of custom exceptions used in RhodeCode
 """
 
-from webob.exc import HTTPClientError
-from pyramid.httpexceptions import HTTPBadGateway
+from pyramid.httpexceptions import HTTPBadGateway, HTTPClientError
 
 
 class LdapUsernameError(Exception):
@@ -102,12 +101,7 @@ class HTTPRequirementError(HTTPClientError):
         self.args = (message, )
 
 
-class ClientNotSupportedError(HTTPRequirementError):
-    title = explanation = 'Client Not Supported'
-    reason = None
-
-
-class HTTPLockedRC(HTTPClientError):
+class HTTPLockedRepo(HTTPClientError):
     """
     Special Exception For locked Repos in RhodeCode, the return code can
     be overwritten by _code keyword argument passed into constructors
@@ -131,14 +125,13 @@ class HTTPBranchProtected(HTTPClientError):
     Special Exception For Indicating that branch is protected in RhodeCode, the
     return code can be overwritten by _code keyword argument passed into constructors
     """
-    code = 403
     title = explanation = 'Branch Protected'
     reason = None
 
-    def __init__(self, message, *args, **kwargs):
-        self.title = self.explanation = message
-        super().__init__(*args, **kwargs)
-        self.args = (message, )
+
+class ClientNotSupported(HTTPRequirementError):
+    title = explanation = 'Client Not Supported'
+    reason = None
 
 
 class IMCCommitError(Exception):

rhodecode/lib/hook_daemon/base.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2010-2023 RhodeCode GmbH
+# Copyright (C) 2010-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
@@ -16,13 +16,14 @@
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-import os
-import time
 import logging
+import traceback
 
-from rhodecode.lib.config_utils import get_app_config_lightweight
+from rhodecode.model import meta
+from rhodecode.lib import hooks_base
+from rhodecode.lib.utils2 import AttributeDict
+from rhodecode.lib.exceptions import HTTPLockedRepo, HTTPBranchProtected
 
-from rhodecode.lib.svn_txn_utils import get_txn_id_from_store
 
 log = logging.getLogger(__name__)
 
@@ -42,53 +43,82 @@ class BaseHooksCallbackDaemon:
         log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
 
 
-class HooksModuleCallbackDaemon(BaseHooksCallbackDaemon):
+class Hooks(object):
+    """
+    Exposes the hooks module for calling them using the local HooksModuleCallbackDaemon
+    """
+    def __init__(self, request=None, log_prefix=''):
+        self.log_prefix = log_prefix
+        self.request = request
 
-    def __init__(self, module):
-        super().__init__()
-        self.hooks_module = module
+    def repo_size(self, extras):
+        log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.repo_size, extras)
 
-    def __repr__(self):
-        return f'HooksModuleCallbackDaemon(hooks_module={self.hooks_module})'
+    def pre_pull(self, extras):
+        log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.pre_pull, extras)
 
+    def post_pull(self, extras):
+        log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.post_pull, extras)
 
-def prepare_callback_daemon(extras, protocol, host, txn_id=None):
+    def pre_push(self, extras):
+        log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.pre_push, extras)
 
-    match protocol:
-        case 'http':
-            from rhodecode.lib.hook_daemon.http_hooks_deamon import HttpHooksCallbackDaemon
-            port = 0
-            if txn_id:
-                # read txn-id to re-use the PORT for callback daemon
-                repo_path = os.path.join(extras['repo_store'], extras['repository'])
-                txn_details = get_txn_id_from_store(repo_path, txn_id)
-                port = txn_details.get('port', 0)
+    def post_push(self, extras):
+        log.debug("%sCalled post_push of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.post_push, extras)
 
-            callback_daemon = HttpHooksCallbackDaemon(
-                txn_id=txn_id, host=host, port=port)
-        case 'celery':
-            from rhodecode.lib.hook_daemon.celery_hooks_deamon import CeleryHooksCallbackDaemon
+    def _call_hook(self, hook, extras):
+        extras = AttributeDict(extras)
+        _server_url = extras['server_url']
 
-            config = get_app_config_lightweight(extras['config'])
-            task_queue = config.get('celery.broker_url')
-            task_backend = config.get('celery.result_backend')
+        extras.request = self.request
+        try:
+            result = hook(extras)
+            if result is None:
+                raise Exception(f'Failed to obtain hook result from func: {hook}')
+        except HTTPBranchProtected as error:
+            # Those special cases don't need error reporting. It's a case of
+            # locked repo or protected branch
+            result = AttributeDict({
+                'status': error.code,
+                'output': error.explanation
+            })
+        except HTTPLockedRepo as error:
+            # Those special cases don't need error reporting. It's a case of
+            # locked repo or protected branch
+            result = AttributeDict({
+                'status': error.code,
+                'output': error.explanation
+            })
+        except Exception as error:
+            # locked needs different handling since we need to also
+            # handle PULL operations
+            log.exception('%sException when handling hook %s', self.log_prefix, hook)
+            exc_tb = traceback.format_exc()
+            error_args = error.args
+            return {
+                'status': 128,
+                'output': '',
+                'exception': type(error).__name__,
+                'exception_traceback': exc_tb,
+                'exception_args': error_args,
+            }
+        finally:
+            meta.Session.remove()
 
-            callback_daemon = CeleryHooksCallbackDaemon(task_queue, task_backend)
-        case 'local':
-            from rhodecode.lib.hook_daemon.hook_module import Hooks
-            callback_daemon = HooksModuleCallbackDaemon(Hooks.__module__)
-        case _:
-            log.error('Unsupported callback daemon protocol "%s"', protocol)
-            raise Exception('Unsupported callback daemon protocol.')
+        log.debug('%sGot hook call response %s', self.log_prefix, result)
+        return {
+            'status': result.status,
+            'output': result.output,
+        }
 
-    extras['hooks_uri'] = getattr(callback_daemon, 'hooks_uri', '')
-    extras['task_queue'] = getattr(callback_daemon, 'task_queue', '')
-    extras['task_backend'] = getattr(callback_daemon, 'task_backend', '')
-    extras['hooks_protocol'] = protocol
-    extras['time'] = time.time()
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
 
-    # register txn_id
-    extras['txn_id'] = txn_id
-    log.debug('Prepared a callback daemon: %s',
-              callback_daemon.__class__.__name__)
-    return callback_daemon, extras

rhodecode/lib/hook_daemon/celery_hooks_deamon.py

@@ -22,14 +22,16 @@ from rhodecode.lib.hook_daemon.base import BaseHooksCallbackDaemon
 class CeleryHooksCallbackDaemon(BaseHooksCallbackDaemon):
     """
     Context manger for achieving a compatibility with celery backend
+    It is calling a call to vcsserver, where it uses HooksCeleryClient to actually call a task from
+
+    f'rhodecode.lib.celerylib.tasks.{method}'
+
     """
 
-    def __init__(self, task_queue, task_backend):
-        self.task_queue = task_queue
-        self.task_backend = task_backend
+    def __init__(self, broker_url, result_backend):
+        super().__init__()
+        self.broker_url = broker_url
+        self.result_backend = result_backend
 
     def __repr__(self):
-        return f'CeleryHooksCallbackDaemon(task_queue={self.task_queue}, task_backend={self.task_backend})'
-
-    def __repr__(self):
-        return f'CeleryHooksCallbackDaemon(task_queue={self.task_queue}, task_backend={self.task_backend})'
+        return f'CeleryHooksCallbackDaemon(broker_url={self.broker_url}, result_backend={self.result_backend})'

rhodecode/lib/hook_daemon/hook_module.py

@@ -17,88 +17,18 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
-import traceback
 
-from rhodecode.model import meta
-
-from rhodecode.lib import hooks_base
-from rhodecode.lib.exceptions import HTTPLockedRC, HTTPBranchProtected
-from rhodecode.lib.utils2 import AttributeDict
+from rhodecode.lib.hook_daemon.base import BaseHooksCallbackDaemon
 
 log = logging.getLogger(__name__)
 
 
-class Hooks(object):
-    """
-    Exposes the hooks for remote callbacks
-    """
-    def __init__(self, request=None, log_prefix=''):
-        self.log_prefix = log_prefix
-        self.request = request
+class HooksModuleCallbackDaemon(BaseHooksCallbackDaemon):
 
-    def repo_size(self, extras):
-        log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
-        return self._call_hook(hooks_base.repo_size, extras)
+    def __init__(self, module):
+        super().__init__()
+        self.hooks_module = module
 
-    def pre_pull(self, extras):
-        log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
-        return self._call_hook(hooks_base.pre_pull, extras)
+    def __repr__(self):
+        return f'HooksModuleCallbackDaemon(hooks_module={self.hooks_module})'
 
-    def post_pull(self, extras):
-        log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
-        return self._call_hook(hooks_base.post_pull, extras)
-
-    def pre_push(self, extras):
-        log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
-        return self._call_hook(hooks_base.pre_push, extras)
-
-    def post_push(self, extras):
-        log.debug("%sCalled post_push of %s object", self.log_prefix, self)
-        return self._call_hook(hooks_base.post_push, extras)
-
-    def _call_hook(self, hook, extras):
-        extras = AttributeDict(extras)
-        _server_url = extras['server_url']
-
-        extras.request = self.request
-
-        try:
-            result = hook(extras)
-            if result is None:
-                raise Exception(f'Failed to obtain hook result from func: {hook}')
-        except HTTPBranchProtected as error:
-            # Those special cases don't need error reporting. It's a case of
-            # locked repo or protected branch
-            result = AttributeDict({
-                'status': error.code,
-                'output': error.explanation
-            })
-        except (HTTPLockedRC, Exception) as error:
-            # locked needs different handling since we need to also
-            # handle PULL operations
-            exc_tb = ''
-            if not isinstance(error, HTTPLockedRC):
-                exc_tb = traceback.format_exc()
-                log.exception('%sException when handling hook %s', self.log_prefix, hook)
-            error_args = error.args
-            return {
-                'status': 128,
-                'output': '',
-                'exception': type(error).__name__,
-                'exception_traceback': exc_tb,
-                'exception_args': error_args,
-            }
-        finally:
-            meta.Session.remove()
-
-        log.debug('%sGot hook call response %s', self.log_prefix, result)
-        return {
-            'status': result.status,
-            'output': result.output,
-        }
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        pass

rhodecode/lib/hook_daemon/http_hooks_deamon.py

@@ -1,287 +0,0 @@
-# Copyright (C) 2010-2023 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-import os
-import logging
-import traceback
-import threading
-import socket
-import msgpack
-import gevent
-
-from http.server import BaseHTTPRequestHandler
-from socketserver import TCPServer
-
-from rhodecode.model import meta
-from rhodecode.lib.ext_json import json
-from rhodecode.lib import rc_cache
-from rhodecode.lib.svn_txn_utils import get_txn_id_data_key
-from rhodecode.lib.hook_daemon.hook_module import Hooks
-
-log = logging.getLogger(__name__)
-
-
-class HooksHttpHandler(BaseHTTPRequestHandler):
-
-    JSON_HOOKS_PROTO = 'json.v1'
-    MSGPACK_HOOKS_PROTO = 'msgpack.v1'
-    # starting with RhodeCode 5.0.0 MsgPack is the default, prior it used json
-    DEFAULT_HOOKS_PROTO = MSGPACK_HOOKS_PROTO
-
-    @classmethod
-    def serialize_data(cls, data, proto=DEFAULT_HOOKS_PROTO):
-        if proto == cls.MSGPACK_HOOKS_PROTO:
-            return msgpack.packb(data)
-        return json.dumps(data)
-
-    @classmethod
-    def deserialize_data(cls, data, proto=DEFAULT_HOOKS_PROTO):
-        if proto == cls.MSGPACK_HOOKS_PROTO:
-            return msgpack.unpackb(data)
-        return json.loads(data)
-
-    def do_POST(self):
-        hooks_proto, method, extras = self._read_request()
-        log.debug('Handling HooksHttpHandler %s with %s proto', method, hooks_proto)
-
-        txn_id = getattr(self.server, 'txn_id', None)
-        if txn_id:
-            log.debug('Computing TXN_ID based on `%s`:`%s`',
-                      extras['repository'], extras['txn_id'])
-            computed_txn_id = rc_cache.utils.compute_key_from_params(
-                extras['repository'], extras['txn_id'])
-            if txn_id != computed_txn_id:
-                raise Exception(
-                    'TXN ID fail: expected {} got {} instead'.format(
-                        txn_id, computed_txn_id))
-
-        request = getattr(self.server, 'request', None)
-        try:
-            hooks = Hooks(request=request, log_prefix='HOOKS: {} '.format(self.server.server_address))
-            result = self._call_hook_method(hooks, method, extras)
-
-        except Exception as e:
-            exc_tb = traceback.format_exc()
-            result = {
-                'exception': e.__class__.__name__,
-                'exception_traceback': exc_tb,
-                'exception_args': e.args
-            }
-        self._write_response(hooks_proto, result)
-
-    def _read_request(self):
-        length = int(self.headers['Content-Length'])
-        # respect sent headers, fallback to OLD proto for compatability
-        hooks_proto = self.headers.get('rc-hooks-protocol') or self.JSON_HOOKS_PROTO
-        if hooks_proto == self.MSGPACK_HOOKS_PROTO:
-            # support for new vcsserver msgpack based protocol hooks
-            body = self.rfile.read(length)
-            data = self.deserialize_data(body)
-        else:
-            body = self.rfile.read(length)
-            data = self.deserialize_data(body)
-
-        return hooks_proto, data['method'], data['extras']
-
-    def _write_response(self, hooks_proto, result):
-        self.send_response(200)
-        if hooks_proto == self.MSGPACK_HOOKS_PROTO:
-            self.send_header("Content-type", "application/msgpack")
-            self.end_headers()
-            data = self.serialize_data(result)
-            self.wfile.write(data)
-        else:
-            self.send_header("Content-type", "text/json")
-            self.end_headers()
-            data = self.serialize_data(result)
-            self.wfile.write(data)
-
-    def _call_hook_method(self, hooks, method, extras):
-        try:
-            result = getattr(hooks, method)(extras)
-        finally:
-            meta.Session.remove()
-        return result
-
-    def log_message(self, format, *args):
-        """
-        This is an overridden method of BaseHTTPRequestHandler which logs using
-        a logging library instead of writing directly to stderr.
-        """
-
-        message = format % args
-
-        log.debug(
-            "HOOKS: client=%s - - [%s] %s", self.client_address,
-            self.log_date_time_string(), message)
-
-
-class ThreadedHookCallbackDaemon(object):
-
-    _callback_thread = None
-    _daemon = None
-    _done = False
-    use_gevent = False
-
-    def __init__(self, txn_id=None, host=None, port=None):
-        self._prepare(txn_id=txn_id, host=host, port=port)
-        if self.use_gevent:
-            self._run_func = self._run_gevent
-            self._stop_func = self._stop_gevent
-        else:
-            self._run_func = self._run
-            self._stop_func = self._stop
-
-    def __enter__(self):
-        log.debug('Running `%s` callback daemon', self.__class__.__name__)
-        self._run_func()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
-        self._stop_func()
-
-    def _prepare(self, txn_id=None, host=None, port=None):
-        raise NotImplementedError()
-
-    def _run(self):
-        raise NotImplementedError()
-
-    def _stop(self):
-        raise NotImplementedError()
-
-    def _run_gevent(self):
-        raise NotImplementedError()
-
-    def _stop_gevent(self):
-        raise NotImplementedError()
-
-
-class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
-    """
-    Context manager which will run a callback daemon in a background thread.
-    """
-
-    hooks_uri = None
-
-    # From Python docs: Polling reduces our responsiveness to a shutdown
-    # request and wastes cpu at all other times.
-    POLL_INTERVAL = 0.01
-
-    use_gevent = False
-
-    def __repr__(self):
-        return f'HttpHooksCallbackDaemon(hooks_uri={self.hooks_uri})'
-
-    @property
-    def _hook_prefix(self):
-        return f'HOOKS: {self.hooks_uri} '
-
-    def get_hostname(self):
-        return socket.gethostname() or '127.0.0.1'
-
-    def get_available_port(self, min_port=20000, max_port=65535):
-        from rhodecode.lib.utils2 import get_available_port as _get_port
-        return _get_port(min_port, max_port)
-
-    def _prepare(self, txn_id=None, host=None, port=None):
-        from pyramid.threadlocal import get_current_request
-
-        if not host or host == "*":
-            host = self.get_hostname()
-        if not port:
-            port = self.get_available_port()
-
-        server_address = (host, port)
-        self.hooks_uri = f'{host}:{port}'
-        self.txn_id = txn_id
-        self._done = False
-
-        log.debug(
-            "%s Preparing HTTP callback daemon registering hook object: %s",
-            self._hook_prefix, HooksHttpHandler)
-
-        self._daemon = TCPServer(server_address, HooksHttpHandler)
-        # inject transaction_id for later verification
-        self._daemon.txn_id = self.txn_id
-
-        # pass the WEB app request into daemon
-        self._daemon.request = get_current_request()
-
-    def _run(self):
-        log.debug("Running thread-based loop of callback daemon in background")
-        callback_thread = threading.Thread(
-            target=self._daemon.serve_forever,
-            kwargs={'poll_interval': self.POLL_INTERVAL})
-        callback_thread.daemon = True
-        callback_thread.start()
-        self._callback_thread = callback_thread
-
-    def _run_gevent(self):
-        log.debug("Running gevent-based loop of callback daemon in background")
-        # create a new greenlet for the daemon's serve_forever method
-        callback_greenlet = gevent.spawn(
-            self._daemon.serve_forever,
-            poll_interval=self.POLL_INTERVAL)
-
-        # store reference to greenlet
-        self._callback_greenlet = callback_greenlet
-
-        # switch to this greenlet
-        gevent.sleep(0.01)
-
-    def _stop(self):
-        log.debug("Waiting for background thread to finish.")
-        self._daemon.shutdown()
-        self._callback_thread.join()
-        self._daemon = None
-        self._callback_thread = None
-        if self.txn_id:
-            #TODO: figure out the repo_path...
-            repo_path = ''
-            txn_id_file = get_txn_id_data_key(repo_path, self.txn_id)
-            log.debug('Cleaning up TXN ID %s', txn_id_file)
-            if os.path.isfile(txn_id_file):
-                os.remove(txn_id_file)
-
-        log.debug("Background thread done.")
-
-    def _stop_gevent(self):
-        log.debug("Waiting for background greenlet to finish.")
-
-        # if greenlet exists and is running
-        if self._callback_greenlet and not self._callback_greenlet.dead:
-            # shutdown daemon if it exists
-            if self._daemon:
-                self._daemon.shutdown()
-
-            # kill the greenlet
-            self._callback_greenlet.kill()
-
-        self._daemon = None
-        self._callback_greenlet = None
-
-        if self.txn_id:
-            #TODO: figure out the repo_path...
-            repo_path = ''
-            txn_id_file = get_txn_id_data_key(repo_path, self.txn_id)
-            log.debug('Cleaning up TXN ID %s', txn_id_file)
-            if os.path.isfile(txn_id_file):
-                os.remove(txn_id_file)
-
-        log.debug("Background greenlet done.")

rhodecode/lib/hook_daemon/utils.py

@@ -0,0 +1,61 @@
+# Copyright (C) 2010-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import time
+import logging
+
+from rhodecode.lib.config_utils import get_app_config_lightweight
+
+from rhodecode.lib.hook_daemon.base import Hooks
+from rhodecode.lib.hook_daemon.hook_module import HooksModuleCallbackDaemon
+from rhodecode.lib.hook_daemon.celery_hooks_deamon import CeleryHooksCallbackDaemon
+from rhodecode.lib.type_utils import str2bool
+
+log = logging.getLogger(__name__)
+
+
+
+def prepare_callback_daemon(extras, protocol: str, txn_id=None):
+    hooks_config = {}
+    match protocol:
+        case 'celery':
+            config = get_app_config_lightweight(extras['config'])
+
+            broker_url = config.get('celery.broker_url')
+            result_backend = config.get('celery.result_backend')
+
+            hooks_config = {
+                'broker_url': broker_url,
+                'result_backend': result_backend,
+            }
+
+            callback_daemon = CeleryHooksCallbackDaemon(broker_url, result_backend)
+        case 'local':
+            callback_daemon = HooksModuleCallbackDaemon(Hooks.__module__)
+        case _:
+            log.error('Unsupported callback daemon protocol "%s"', protocol)
+            raise Exception('Unsupported callback daemon protocol.')
+
+    extras['hooks_config'] = hooks_config
+    extras['hooks_protocol'] = protocol
+    extras['time'] = time.time()
+
+    # register txn_id
+    extras['txn_id'] = txn_id
+    log.debug('Prepared a callback daemon: %s', callback_daemon.__class__.__name__)
+    return callback_daemon, extras

rhodecode/lib/hooks_base.py

@@ -30,14 +30,14 @@ from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.utils2 import safe_str, user_agent_normalizer
 from rhodecode.lib.exceptions import (
-    HTTPLockedRC, HTTPBranchProtected, UserCreationError, ClientNotSupportedError)
+    HTTPLockedRepo, HTTPBranchProtected, UserCreationError, ClientNotSupported)
 from rhodecode.model.db import Repository, User
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
-class HookResponse(object):
+class HookResponse:
     def __init__(self, status, output):
         self.status = status
         self.output = output
@@ -56,6 +56,8 @@ class HookResponse(object):
     def to_json(self):
         return {'status': self.status, 'output': self.output}
 
+    def __repr__(self):
+        return self.to_json().__repr__()
 
 def is_shadow_repo(extras):
     """
@@ -73,8 +75,69 @@ def check_vcs_client(extras):
     except ModuleNotFoundError:
         is_vcs_client_whitelisted = lambda *x: True
     backend = extras.get('scm')
-    if not is_vcs_client_whitelisted(extras.get('user_agent'), backend):
-        raise ClientNotSupportedError(f"Your {backend} client is forbidden")
+    user_agent = extras.get('user_agent')
+    if not is_vcs_client_whitelisted(user_agent, backend):
+        raise ClientNotSupported(f"Your {backend} client (version={user_agent}) is forbidden by security rules")
+
+
+def check_locked_repo(extras, check_same_user=True):
+    user = User.get_by_username(extras.username)
+    output = ''
+    if extras.locked_by[0] and (not check_same_user or user.user_id != extras.locked_by[0]):
+
+        locked_by = User.get(extras.locked_by[0]).username
+        reason = extras.locked_by[2]
+        # this exception is interpreted in git/hg middlewares and based
+        # on that proper return code is server to client
+        _http_ret = HTTPLockedRepo(_locked_by_explanation(extras.repository, locked_by, reason))
+        if str(_http_ret.code).startswith('2'):
+            # 2xx Codes don't raise exceptions
+            output = _http_ret.title
+        else:
+            raise _http_ret
+
+    return output
+
+
+def check_branch_protected(extras):
+    if extras.commit_ids and extras.check_branch_perms:
+        user = User.get_by_username(extras.username)
+        auth_user = user.AuthUser()
+        repo = Repository.get_by_repo_name(extras.repository)
+        if not repo:
+            raise ValueError(f'Repo for {extras.repository} not found')
+        affected_branches = []
+        if repo.repo_type == 'hg':
+            for entry in extras.commit_ids:
+                if entry['type'] == 'branch':
+                    is_forced = bool(entry['multiple_heads'])
+                    affected_branches.append([entry['name'], is_forced])
+        elif repo.repo_type == 'git':
+            for entry in extras.commit_ids:
+                if entry['type'] == 'heads':
+                    is_forced = bool(entry['pruned_sha'])
+                    affected_branches.append([entry['name'], is_forced])
+
+        for branch_name, is_forced in affected_branches:
+
+            rule, branch_perm = auth_user.get_rule_and_branch_permission(extras.repository, branch_name)
+            if not branch_perm:
+                # no branch permission found for this branch, just keep checking
+                continue
+
+            if branch_perm == 'branch.push_force':
+                continue
+            elif branch_perm == 'branch.push' and is_forced is False:
+                continue
+            elif branch_perm == 'branch.push' and is_forced is True:
+                halt_message = f'Branch `{branch_name}` changes rejected by rule {rule}. ' \
+                               f'FORCE PUSH FORBIDDEN.'
+            else:
+                halt_message = f'Branch `{branch_name}` changes rejected by rule {rule}.'
+
+            if halt_message:
+                _http_ret = HTTPBranchProtected(halt_message)
+                raise _http_ret
 
 def _get_scm_size(alias, root_path):
 
@@ -109,116 +172,30 @@ def repo_size(extras):
     repo = Repository.get_by_repo_name(extras.repository)
     vcs_part = f'.{repo.repo_type}'
     size_vcs, size_root, size_total = _get_scm_size(vcs_part, repo.repo_full_path)
-    msg = (f'RhodeCode: `{repo.repo_name}` size summary {vcs_part}:{size_vcs} repo:{size_root} total:{size_total}\n')
+    msg = f'RhodeCode: `{repo.repo_name}` size summary {vcs_part}:{size_vcs} repo:{size_root} total:{size_total}\n'
     return HookResponse(0, msg)
 
 
-def pre_push(extras):
-    """
-    Hook executed before pushing code.
-
-    It bans pushing when the repository is locked.
-    """
-
-    check_vcs_client(extras)
-    user = User.get_by_username(extras.username)
-    output = ''
-    if extras.locked_by[0] and user.user_id != int(extras.locked_by[0]):
-        locked_by = User.get(extras.locked_by[0]).username
-        reason = extras.locked_by[2]
-        # this exception is interpreted in git/hg middlewares and based
-        # on that proper return code is server to client
-        _http_ret = HTTPLockedRC(
-            _locked_by_explanation(extras.repository, locked_by, reason))
-        if str(_http_ret.code).startswith('2'):
-            # 2xx Codes don't raise exceptions
-            output = _http_ret.title
-        else:
-            raise _http_ret
-
-    hook_response = ''
-    if not is_shadow_repo(extras):
-
-        if extras.commit_ids and extras.check_branch_perms:
-            auth_user = user.AuthUser()
-            repo = Repository.get_by_repo_name(extras.repository)
-            if not repo:
-                raise ValueError(f'Repo for {extras.repository} not found')
-            affected_branches = []
-            if repo.repo_type == 'hg':
-                for entry in extras.commit_ids:
-                    if entry['type'] == 'branch':
-                        is_forced = bool(entry['multiple_heads'])
-                        affected_branches.append([entry['name'], is_forced])
-            elif repo.repo_type == 'git':
-                for entry in extras.commit_ids:
-                    if entry['type'] == 'heads':
-                        is_forced = bool(entry['pruned_sha'])
-                        affected_branches.append([entry['name'], is_forced])
-
-            for branch_name, is_forced in affected_branches:
-
-                rule, branch_perm = auth_user.get_rule_and_branch_permission(
-                    extras.repository, branch_name)
-                if not branch_perm:
-                    # no branch permission found for this branch, just keep checking
-                    continue
-
-                if branch_perm == 'branch.push_force':
-                    continue
-                elif branch_perm == 'branch.push' and is_forced is False:
-                    continue
-                elif branch_perm == 'branch.push' and is_forced is True:
-                    halt_message = f'Branch `{branch_name}` changes rejected by rule {rule}. ' \
-                                   f'FORCE PUSH FORBIDDEN.'
-                else:
-                    halt_message = f'Branch `{branch_name}` changes rejected by rule {rule}.'
-
-                if halt_message:
-                    _http_ret = HTTPBranchProtected(halt_message)
-                    raise _http_ret
-
-        # Propagate to external components. This is done after checking the
-        # lock, for consistent behavior.
-        hook_response = pre_push_extension(
-            repo_store_path=Repository.base_path(), **extras)
-        events.trigger(events.RepoPrePushEvent(
-            repo_name=extras.repository, extras=extras))
-
-    return HookResponse(0, output) + hook_response
-
-
 def pre_pull(extras):
     """
     Hook executed before pulling the code.
 
     It bans pulling when the repository is locked.
+    It bans pulling when incorrect client is used.
     """
-
-    check_vcs_client(extras)
     output = ''
-    if extras.locked_by[0]:
-        locked_by = User.get(extras.locked_by[0]).username
-        reason = extras.locked_by[2]
-        # this exception is interpreted in git/hg middlewares and based
-        # on that proper return code is server to client
-        _http_ret = HTTPLockedRC(
-            _locked_by_explanation(extras.repository, locked_by, reason))
-        if str(_http_ret.code).startswith('2'):
-            # 2xx Codes don't raise exceptions
-            output = _http_ret.title
-        else:
-            raise _http_ret
+    check_vcs_client(extras)
+
+    # locking repo can, but not have to stop the operation it can also just produce output
+    output += check_locked_repo(extras, check_same_user=False)
 
     # Propagate to external components. This is done after checking the
     # lock, for consistent behavior.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'pre_pull'
-        hook_response = pre_pull_extension(
-            repo_store_path=Repository.base_path(), **extras)
-        events.trigger(events.RepoPrePullEvent(
-            repo_name=extras.repository, extras=extras))
+        hook_response = pre_pull_extension(repo_store_path=Repository.base_path(), **extras)
+        events.trigger(events.RepoPrePullEvent(repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
@@ -239,6 +216,7 @@ def post_pull(extras):
         statsd.incr('rhodecode_pull_total', tags=[
             f'user-agent:{user_agent_normalizer(extras.user_agent)}',
         ])
+
     output = ''
     # make lock is a tri state False, True, None. We only make lock on True
     if extras.make_lock is True and not is_shadow_repo(extras):
@@ -246,18 +224,9 @@ def post_pull(extras):
         Repository.lock(Repository.get_by_repo_name(extras.repository),
                         user.user_id,
                         lock_reason=Repository.LOCK_PULL)
-        msg = 'Made lock on repo `{}`'.format(extras.repository)
+        msg = f'Made lock on repo `{extras.repository}`'
         output += msg
 
-    if extras.locked_by[0]:
-        locked_by = User.get(extras.locked_by[0]).username
-        reason = extras.locked_by[2]
-        _http_ret = HTTPLockedRC(
-            _locked_by_explanation(extras.repository, locked_by, reason))
-        if str(_http_ret.code).startswith('2'):
-            # 2xx Codes don't raise exceptions
-            output += _http_ret.title
-
     # Propagate to external components.
     hook_response = ''
     if not is_shadow_repo(extras):
@@ -270,6 +239,33 @@ def post_pull(extras):
     return HookResponse(0, output) + hook_response
 
 
+def pre_push(extras):
+    """
+    Hook executed before pushing code.
+
+    It bans pushing when the repository is locked.
+    It banks pushing when incorrect client is used.
+    It also checks for Branch protection
+    """
+    output = ''
+    check_vcs_client(extras)
+
+    # locking repo can, but not have to stop the operation it can also just produce output
+    output += check_locked_repo(extras)
+
+    hook_response = ''
+    if not is_shadow_repo(extras):
+
+        check_branch_protected(extras)
+
+        # Propagate to external components. This is done after checking the
+        # lock, for consistent behavior.
+        hook_response = pre_push_extension(repo_store_path=Repository.base_path(), **extras)
+        events.trigger(events.RepoPrePushEvent(repo_name=extras.repository, extras=extras))
+
+    return HookResponse(0, output) + hook_response
+
+
 def post_push(extras):
     """Hook executed after user pushes to the repository."""
     commit_ids = extras.commit_ids
@@ -292,22 +288,13 @@ def post_push(extras):
 
     # Propagate to external components.
     output = ''
+
     # make lock is a tri state False, True, None. We only release lock on False
     if extras.make_lock is False and not is_shadow_repo(extras):
         Repository.unlock(Repository.get_by_repo_name(extras.repository))
         msg = f'Released lock on repo `{extras.repository}`\n'
         output += msg
 
-    if extras.locked_by[0]:
-        locked_by = User.get(extras.locked_by[0]).username
-        reason = extras.locked_by[2]
-        _http_ret = HTTPLockedRC(
-            _locked_by_explanation(extras.repository, locked_by, reason))
-        # TODO: johbo: if not?
-        if str(_http_ret.code).startswith('2'):
-            # 2xx Codes don't raise exceptions
-            output += _http_ret.title
-
     if extras.new_refs:
         tmpl = '{}/{}/pull-request/new?{{ref_type}}={{ref_name}}'.format(
             safe_str(extras.server_url), safe_str(extras.repository))
@@ -322,11 +309,8 @@ def post_push(extras):
 
     hook_response = ''
     if not is_shadow_repo(extras):
-        hook_response = post_push_extension(
-            repo_store_path=Repository.base_path(),
-            **extras)
-        events.trigger(events.RepoPushEvent(
-            repo_name=extras.repository, pushed_commit_ids=commit_ids, extras=extras))
+        hook_response = post_push_extension(repo_store_path=Repository.base_path(), **extras)
+        events.trigger(events.RepoPushEvent(repo_name=extras.repository, pushed_commit_ids=commit_ids, extras=extras))
 
     output += 'RhodeCode: push completed\n'
     return HookResponse(0, output) + hook_response
@@ -380,12 +364,20 @@ class ExtensionCallback(object):
         # with older rcextensions that require api_key present
         if self._hook_name in ['CREATE_USER_HOOK', 'DELETE_USER_HOOK']:
             kwargs_to_pass['api_key'] = '_DEPRECATED_'
-        return callback(**kwargs_to_pass)
+        result = callback(**kwargs_to_pass)
+        log.debug('got rcextensions result: %s', result)
+        return result
 
     def is_active(self):
         return hasattr(rhodecode.EXTENSIONS, self._hook_name)
 
     def _get_callback(self):
+        if rhodecode.is_test:
+            log.debug('In test mode, reloading rcextensions...')
+            # NOTE: for test re-load rcextensions always so we can dynamically change them for testing purposes
+            from rhodecode.lib.utils import load_rcextensions
+            load_rcextensions(root_path=os.path.dirname(rhodecode.CONFIG['__file__']))
+            return getattr(rhodecode.EXTENSIONS, self._hook_name, None)
         return getattr(rhodecode.EXTENSIONS, self._hook_name, None)
 
 

rhodecode/lib/middleware/simplegit.py

@@ -40,16 +40,6 @@ GIT_PROTO_PAT = re.compile(
 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
 
 
-def default_lfs_store():
-    """
-    Default lfs store location, it's consistent with Mercurials large file
-    store which is in .cache/largefiles
-    """
-    from rhodecode.lib.vcs.backends.git import lfs_store
-    user_home = os.path.expanduser("~")
-    return lfs_store(user_home)
-
-
 class SimpleGit(simplevcs.SimpleVCS):
 
     SCM = 'git'
@@ -151,6 +141,6 @@ class SimpleGit(simplevcs.SimpleVCS):
 
         extras['git_lfs_enabled'] = utils2.str2bool(
             config.get('vcs_git_lfs', 'enabled'))
-        extras['git_lfs_store_path'] = custom_store or default_lfs_store()
+        extras['git_lfs_store_path'] = custom_store
         extras['git_lfs_http_scheme'] = scheme
         return extras

rhodecode/lib/middleware/simplevcs.py

@@ -1,5 +1,3 @@
-
-
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
@@ -32,8 +30,7 @@ from functools import wraps
 import time
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
-from pyramid.httpexceptions import (
-    HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
+from pyramid.httpexceptions import HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
@@ -41,10 +38,9 @@ from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
 from rhodecode.lib import rc_cache
 from rhodecode.lib.svn_txn_utils import store_txn_id_data
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
-from rhodecode.lib.base import (
-    BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
-from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
-from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
+from rhodecode.lib.base import BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context
+from rhodecode.lib.exceptions import UserCreationError, NotAllowedToCreateUserError
+from rhodecode.lib.hook_daemon.utils import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.lib.str_utils import safe_bytes, safe_int
@@ -78,17 +74,18 @@ def initialize_generator(factory):
         try:
             init = next(gen)
         except StopIteration:
-            raise ValueError('Generator must yield at least one element.')
+            raise ValueError("Generator must yield at least one element.")
         if init != "__init__":
             raise ValueError('First yielded element must be "__init__".')
         return gen
+
     return wrapper
 
 
 class SimpleVCS(object):
     """Common functionality for SCM HTTP handlers."""
 
-    SCM = 'unknown'
+    SCM = "unknown"
 
     acl_repo_name = None
     url_repo_name = None
@@ -100,11 +97,11 @@ class SimpleVCS(object):
     # we use this regex which will match only on URLs pointing to shadow
     # repositories.
     shadow_repo_re = re.compile(
-        '(?P<groups>(?:{slug_pat}/)*)'   # repo groups
-        '(?P<target>{slug_pat})/'        # target repo
-        'pull-request/(?P<pr_id>\\d+)/'  # pull request
-        'repository$'                    # shadow repo
-        .format(slug_pat=SLUG_RE.pattern))
+        "(?P<groups>(?:{slug_pat}/)*)"  # repo groups
+        "(?P<target>{slug_pat})/"  # target repo
+        "pull-request/(?P<pr_id>\\d+)/"  # pull request
+        "repository$".format(slug_pat=SLUG_RE.pattern)  # shadow repo
+    )
 
     def __init__(self, config, registry):
         self.registry = registry
@@ -113,15 +110,14 @@ class SimpleVCS(object):
         self.repo_vcs_config = base.Config()
 
         rc_settings = SettingsModel().get_all_settings(cache=True, from_request=False)
-        realm = rc_settings.get('rhodecode_realm') or 'RhodeCode AUTH'
+        realm = rc_settings.get("rhodecode_realm") or "RhodeCode AUTH"
 
         # authenticate this VCS request using authfunc
-        auth_ret_code_detection = \
-            str2bool(self.config.get('auth_ret_code_detection', False))
+        auth_ret_code_detection = str2bool(self.config.get("auth_ret_code_detection", False))
         self.authenticate = BasicAuth(
-            '', authenticate, registry, config.get('auth_ret_code'),
-            auth_ret_code_detection, rc_realm=realm)
-        self.ip_addr = '0.0.0.0'
+            "", authenticate, registry, config.get("auth_ret_code"), auth_ret_code_detection, rc_realm=realm
+        )
+        self.ip_addr = "0.0.0.0"
 
     @LazyProperty
     def global_vcs_config(self):
@@ -132,10 +128,10 @@ class SimpleVCS(object):
 
     @property
     def base_path(self):
-        settings_path = self.config.get('repo_store.path')
+        settings_path = self.config.get("repo_store.path")
 
         if not settings_path:
-            raise ValueError('FATAL: repo_store.path is empty')
+            raise ValueError("FATAL: repo_store.path is empty")
         return settings_path
 
     def set_repo_names(self, environ):
@@ -164,17 +160,16 @@ class SimpleVCS(object):
             match_dict = match.groupdict()
 
             # Build acl repo name from regex match.
-            acl_repo_name = safe_str('{groups}{target}'.format(
-                groups=match_dict['groups'] or '',
-                target=match_dict['target']))
+            acl_repo_name = safe_str(
+                "{groups}{target}".format(groups=match_dict["groups"] or "", target=match_dict["target"])
+            )
 
             # Retrieve pull request instance by ID from regex match.
-            pull_request = PullRequest.get(match_dict['pr_id'])
+            pull_request = PullRequest.get(match_dict["pr_id"])
 
             # Only proceed if we got a pull request and if acl repo name from
             # URL equals the target repo name of the pull request.
             if pull_request and (acl_repo_name == pull_request.target_repo.repo_name):
-
                 # Get file system path to shadow repository.
                 workspace_id = PullRequestModel()._workspace_id(pull_request)
                 vcs_repo_name = pull_request.target_repo.get_shadow_repository_path(workspace_id)
@@ -184,21 +179,23 @@ class SimpleVCS(object):
                 self.acl_repo_name = acl_repo_name
                 self.is_shadow_repo = True
 
-        log.debug('Setting all VCS repository names: %s', {
-            'acl_repo_name': self.acl_repo_name,
-            'url_repo_name': self.url_repo_name,
-            'vcs_repo_name': self.vcs_repo_name,
-        })
+        log.debug(
+            "Setting all VCS repository names: %s",
+            {
+                "acl_repo_name": self.acl_repo_name,
+                "url_repo_name": self.url_repo_name,
+                "vcs_repo_name": self.vcs_repo_name,
+            },
+        )
 
     @property
     def scm_app(self):
-        custom_implementation = self.config['vcs.scm_app_implementation']
-        if custom_implementation == 'http':
-            log.debug('Using HTTP implementation of scm app.')
+        custom_implementation = self.config["vcs.scm_app_implementation"]
+        if custom_implementation == "http":
+            log.debug("Using HTTP implementation of scm app.")
             scm_app_impl = scm_app_http
         else:
-            log.debug('Using custom implementation of scm_app: "{}"'.format(
-                custom_implementation))
+            log.debug('Using custom implementation of scm_app: "{}"'.format(custom_implementation))
             scm_app_impl = importlib.import_module(custom_implementation)
         return scm_app_impl
 
@@ -208,17 +205,18 @@ class SimpleVCS(object):
         with a repository_name for support of _<ID> non changeable urls
         """
 
-        data = repo_name.split('/')
+        data = repo_name.split("/")
         if len(data) >= 2:
             from rhodecode.model.repo import RepoModel
+
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 data[1] = by_id_match.repo_name
 
         # Because PEP-3333-WSGI uses bytes-tunneled-in-latin-1 as PATH_INFO
         # and we use this data
-        maybe_new_path = '/'.join(data)
-        return safe_bytes(maybe_new_path).decode('latin1')
+        maybe_new_path = "/".join(data)
+        return safe_bytes(maybe_new_path).decode("latin1")
 
     def _invalidate_cache(self, repo_name):
         """
@@ -231,21 +229,18 @@ class SimpleVCS(object):
     def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
         db_repo = Repository.get_by_repo_name(repo_name)
         if not db_repo:
-            log.debug('Repository `%s` not found inside the database.',
-                      repo_name)
+            log.debug("Repository `%s` not found inside the database.", repo_name)
             return False
 
         if db_repo.repo_type != scm_type:
             log.warning(
-                'Repository `%s` have incorrect scm_type, expected %s got %s',
-                repo_name, db_repo.repo_type, scm_type)
+                "Repository `%s` have incorrect scm_type, expected %s got %s", repo_name, db_repo.repo_type, scm_type
+            )
             return False
 
         config = db_repo._config
-        config.set('extensions', 'largefiles', '')
-        return is_valid_repo(
-            repo_name, base_path,
-            explicit_scm=scm_type, expect_scm=scm_type, config=config)
+        config.set("extensions", "largefiles", "")
+        return is_valid_repo(repo_name, base_path, explicit_scm=scm_type, expect_scm=scm_type, config=config)
 
     def valid_and_active_user(self, user):
         """
@@ -267,8 +262,9 @@ class SimpleVCS(object):
     def is_shadow_repo_dir(self):
         return os.path.isdir(self.vcs_repo_name)
 
-    def _check_permission(self, action, user, auth_user, repo_name, ip_addr=None,
-                          plugin_id='', plugin_cache_active=False, cache_ttl=0):
+    def _check_permission(
+        self, action, user, auth_user, repo_name, ip_addr=None, plugin_id="", plugin_cache_active=False, cache_ttl=0
+    ):
         """
         Checks permissions using action (push/pull) user and repository
         name. If plugin_cache and ttl is set it will use the plugin which
@@ -280,71 +276,67 @@ class SimpleVCS(object):
         :param repo_name: repository name
         """
 
-        log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
-                  plugin_id, plugin_cache_active, cache_ttl)
+        log.debug("AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)", plugin_id, plugin_cache_active, cache_ttl)
 
         user_id = user.user_id
-        cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}'
-        region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
+        cache_namespace_uid = f"cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}"
+        region = rc_cache.get_or_create_region("cache_perms", cache_namespace_uid)
 
-        @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
-                                               expiration_time=cache_ttl,
-                                               condition=plugin_cache_active)
-        def compute_perm_vcs(
-                cache_name, plugin_id, action, user_id, repo_name, ip_addr):
-
-            log.debug('auth: calculating permission access now for vcs operation: %s', action)
+        @region.conditional_cache_on_arguments(
+            namespace=cache_namespace_uid, expiration_time=cache_ttl, condition=plugin_cache_active
+        )
+        def compute_perm_vcs(cache_name, plugin_id, action, user_id, repo_name, ip_addr):
+            log.debug("auth: calculating permission access now for vcs operation: %s", action)
             # check IP
             inherit = user.inherit_default_permissions
-            ip_allowed = AuthUser.check_ip_allowed(
-                user_id, ip_addr, inherit_from_default=inherit)
+            ip_allowed = AuthUser.check_ip_allowed(user_id, ip_addr, inherit_from_default=inherit)
             if ip_allowed:
-                log.info('Access for IP:%s allowed', ip_addr)
+                log.info("Access for IP:%s allowed", ip_addr)
             else:
                 return False
 
-            if action == 'push':
-                perms = ('repository.write', 'repository.admin')
+            if action == "push":
+                perms = ("repository.write", "repository.admin")
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             else:
                 # any other action need at least read permission
-                perms = (
-                    'repository.read', 'repository.write', 'repository.admin')
+                perms = ("repository.read", "repository.write", "repository.admin")
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             return True
 
         start = time.time()
-        log.debug('Running plugin `%s` permissions check', plugin_id)
+        log.debug("Running plugin `%s` permissions check", plugin_id)
 
         # for environ based auth, password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
-        perm_result = compute_perm_vcs(
-            'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
+        perm_result = compute_perm_vcs("vcs_permissions", plugin_id, action, user.user_id, repo_name, ip_addr)
 
         auth_time = time.time() - start
-        log.debug('Permissions for plugin `%s` completed in %.4fs, '
-                  'expiration time of fetched cache %.1fs.',
-                  plugin_id, auth_time, cache_ttl)
+        log.debug(
+            "Permissions for plugin `%s` completed in %.4fs, " "expiration time of fetched cache %.1fs.",
+            plugin_id,
+            auth_time,
+            cache_ttl,
+        )
 
         return perm_result
 
     def _get_http_scheme(self, environ):
         try:
-            return environ['wsgi.url_scheme']
+            return environ["wsgi.url_scheme"]
         except Exception:
-            log.exception('Failed to read http scheme')
-            return 'http'
+            log.exception("Failed to read http scheme")
+            return "http"
 
     def _get_default_cache_ttl(self):
         # take AUTH_CACHE_TTL from the `rhodecode` auth plugin
-        plugin = loadplugin('egg:rhodecode-enterprise-ce#rhodecode')
+        plugin = loadplugin("egg:rhodecode-enterprise-ce#rhodecode")
         plugin_settings = plugin.get_settings()
-        plugin_cache_active, cache_ttl = plugin.get_ttl_cache(
-            plugin_settings) or (False, 0)
+        plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings) or (False, 0)
         return plugin_cache_active, cache_ttl
 
     def __call__(self, environ, start_response):
@@ -359,17 +351,17 @@ class SimpleVCS(object):
 
     def _handle_request(self, environ, start_response):
         if not self.url_repo_name:
-            log.warning('Repository name is empty: %s', self.url_repo_name)
+            log.warning("Repository name is empty: %s", self.url_repo_name)
             # failed to get repo name, we fail now
             return HTTPNotFound()(environ, start_response)
-        log.debug('Extracted repo name is %s', self.url_repo_name)
+        log.debug("Extracted repo name is %s", self.url_repo_name)
 
         ip_addr = get_ip_addr(environ)
         user_agent = get_user_agent(environ)
         username = None
 
         # skip passing error to error controller
-        environ['pylons.status_code_redirect'] = True
+        environ["pylons.status_code_redirect"] = True
 
         # ======================================================================
         # GET ACTION PULL or PUSH
@@ -380,17 +372,15 @@ class SimpleVCS(object):
         # Check if this is a request to a shadow repository of a pull request.
         # In this case only pull action is allowed.
         # ======================================================================
-        if self.is_shadow_repo and action != 'pull':
-            reason = 'Only pull action is allowed for shadow repositories.'
-            log.debug('User not allowed to proceed, %s', reason)
+        if self.is_shadow_repo and action != "pull":
+            reason = "Only pull action is allowed for shadow repositories."
+            log.debug("User not allowed to proceed, %s", reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         # Check if the shadow repo actually exists, in case someone refers
         # to it, and it has been deleted because of successful merge.
         if self.is_shadow_repo and not self.is_shadow_repo_dir:
-            log.debug(
-                'Shadow repo detected, and shadow repo dir `%s` is missing',
-                self.is_shadow_repo_dir)
+            log.debug("Shadow repo detected, and shadow repo dir `%s` is missing", self.is_shadow_repo_dir)
             return HTTPNotFound()(environ, start_response)
 
         # ======================================================================
@@ -398,7 +388,7 @@ class SimpleVCS(object):
         # ======================================================================
         detect_force_push = False
         check_branch_perms = False
-        if action in ['pull', 'push']:
+        if action in ["pull", "push"]:
             user_obj = anonymous_user = User.get_default_user()
             auth_user = user_obj.AuthUser()
             username = anonymous_user.username
@@ -406,8 +396,12 @@ class SimpleVCS(object):
                 plugin_cache_active, cache_ttl = self._get_default_cache_ttl()
                 # ONLY check permissions if the user is activated
                 anonymous_perm = self._check_permission(
-                    action, anonymous_user, auth_user, self.acl_repo_name, ip_addr,
-                    plugin_id='anonymous_access',
+                    action,
+                    anonymous_user,
+                    auth_user,
+                    self.acl_repo_name,
+                    ip_addr,
+                    plugin_id="anonymous_access",
                     plugin_cache_active=plugin_cache_active,
                     cache_ttl=cache_ttl,
                 )
@@ -416,12 +410,13 @@ class SimpleVCS(object):
 
             if not anonymous_user.active or not anonymous_perm:
                 if not anonymous_user.active:
-                    log.debug('Anonymous access is disabled, running '
-                              'authentication')
+                    log.debug("Anonymous access is disabled, running " "authentication")
 
                 if not anonymous_perm:
-                    log.debug('Not enough credentials to access repo: `%s` '
-                              'repository as anonymous user', self.acl_repo_name)
+                    log.debug(
+                        "Not enough credentials to access repo: `%s` " "repository as anonymous user",
+                        self.acl_repo_name,
+                    )
 
                 username = None
                 # ==============================================================
@@ -430,19 +425,18 @@ class SimpleVCS(object):
                 # ==============================================================
 
                 # try to auth based on environ, container auth methods
-                log.debug('Running PRE-AUTH for container|headers based authentication')
+                log.debug("Running PRE-AUTH for container|headers based authentication")
 
                 # headers auth, by just reading special headers and bypass the auth with user/passwd
                 pre_auth = authenticate(
-                    '', '', environ, VCS_TYPE, registry=self.registry,
-                    acl_repo_name=self.acl_repo_name)
+                    "", "", environ, VCS_TYPE, registry=self.registry, acl_repo_name=self.acl_repo_name
+                )
 
-                if pre_auth and pre_auth.get('username'):
-                    username = pre_auth['username']
-                log.debug('PRE-AUTH got `%s` as username', username)
+                if pre_auth and pre_auth.get("username"):
+                    username = pre_auth["username"]
+                log.debug("PRE-AUTH got `%s` as username", username)
                 if pre_auth:
-                    log.debug('PRE-AUTH successful from %s',
-                              pre_auth.get('auth_data', {}).get('_plugin'))
+                    log.debug("PRE-AUTH successful from %s", pre_auth.get("auth_data", {}).get("_plugin"))
 
                 # If not authenticated by the container, running basic auth
                 # before inject the calling repo_name for special scope checks
@@ -463,16 +457,16 @@ class SimpleVCS(object):
                         return HTTPNotAcceptable(reason)(environ, start_response)
 
                     if isinstance(auth_result, dict):
-                        AUTH_TYPE.update(environ, 'basic')
-                        REMOTE_USER.update(environ, auth_result['username'])
-                        username = auth_result['username']
-                        plugin = auth_result.get('auth_data', {}).get('_plugin')
-                        log.info(
-                            'MAIN-AUTH successful for user `%s` from %s plugin',
-                            username, plugin)
+                        AUTH_TYPE.update(environ, "basic")
+                        REMOTE_USER.update(environ, auth_result["username"])
+                        username = auth_result["username"]
+                        plugin = auth_result.get("auth_data", {}).get("_plugin")
+                        log.info("MAIN-AUTH successful for user `%s` from %s plugin", username, plugin)
 
-                        plugin_cache_active, cache_ttl = auth_result.get(
-                            'auth_data', {}).get('_ttl_cache') or (False, 0)
+                        plugin_cache_active, cache_ttl = auth_result.get("auth_data", {}).get("_ttl_cache") or (
+                            False,
+                            0,
+                        )
                     else:
                         return auth_result.wsgi_application(environ, start_response)
 
@@ -488,21 +482,24 @@ class SimpleVCS(object):
                 # check user attributes for password change flag
                 user_obj = user
                 auth_user = user_obj.AuthUser()
-                if user_obj and user_obj.username != User.DEFAULT_USER and \
-                        user_obj.user_data.get('force_password_change'):
-                    reason = 'password change required'
-                    log.debug('User not allowed to authenticate, %s', reason)
+                if (
+                    user_obj
+                    and user_obj.username != User.DEFAULT_USER
+                    and user_obj.user_data.get("force_password_change")
+                ):
+                    reason = "password change required"
+                    log.debug("User not allowed to authenticate, %s", reason)
                     return HTTPNotAcceptable(reason)(environ, start_response)
 
                 # check permissions for this repository
                 perm = self._check_permission(
-                    action, user, auth_user, self.acl_repo_name, ip_addr,
-                    plugin, plugin_cache_active, cache_ttl)
+                    action, user, auth_user, self.acl_repo_name, ip_addr, plugin, plugin_cache_active, cache_ttl
+                )
                 if not perm:
                     return HTTPForbidden()(environ, start_response)
-                environ['rc_auth_user_id'] = str(user_id)
+                environ["rc_auth_user_id"] = str(user_id)
 
-            if action == 'push':
+            if action == "push":
                 perms = auth_user.get_branch_permissions(self.acl_repo_name)
                 if perms:
                     check_branch_perms = True
@@ -510,41 +507,48 @@ class SimpleVCS(object):
 
         # extras are injected into UI object and later available
         # in hooks executed by RhodeCode
-        check_locking = _should_check_locking(environ.get('QUERY_STRING'))
+        check_locking = _should_check_locking(environ.get("QUERY_STRING"))
 
         extras = vcs_operation_context(
-            environ, repo_name=self.acl_repo_name, username=username,
-            action=action, scm=self.SCM, check_locking=check_locking,
-            is_shadow_repo=self.is_shadow_repo, check_branch_perms=check_branch_perms,
-            detect_force_push=detect_force_push
+            environ,
+            repo_name=self.acl_repo_name,
+            username=username,
+            action=action,
+            scm=self.SCM,
+            check_locking=check_locking,
+            is_shadow_repo=self.is_shadow_repo,
+            check_branch_perms=check_branch_perms,
+            detect_force_push=detect_force_push,
         )
 
         # ======================================================================
         # REQUEST HANDLING
         # ======================================================================
-        repo_path = os.path.join(
-            safe_str(self.base_path), safe_str(self.vcs_repo_name))
-        log.debug('Repository path is %s', repo_path)
+        repo_path = os.path.join(safe_str(self.base_path), safe_str(self.vcs_repo_name))
+        log.debug("Repository path is %s", repo_path)
 
         fix_PATH()
 
         log.info(
             '%s action on %s repo "%s" by "%s" from %s %s',
-            action, self.SCM, safe_str(self.url_repo_name),
-            safe_str(username), ip_addr, user_agent)
+            action,
+            self.SCM,
+            safe_str(self.url_repo_name),
+            safe_str(username),
+            ip_addr,
+            user_agent,
+        )
 
-        return self._generate_vcs_response(
-            environ, start_response, repo_path, extras, action)
+        return self._generate_vcs_response(environ, start_response, repo_path, extras, action)
 
     def _get_txn_id(self, environ):
-
-        for k in ['RAW_URI', 'HTTP_DESTINATION']:
+        for k in ["RAW_URI", "HTTP_DESTINATION"]:
             url = environ.get(k)
             if not url:
                 continue
 
             # regex to search for svn-txn-id
-            pattern = r'/!svn/txr/([^/]+)/'
+            pattern = r"/!svn/txr/([^/]+)/"
 
             # Search for the pattern in the URL
             match = re.search(pattern, url)
@@ -555,8 +559,7 @@ class SimpleVCS(object):
                 return txn_id
 
     @initialize_generator
-    def _generate_vcs_response(
-            self, environ, start_response, repo_path, extras, action):
+    def _generate_vcs_response(self, environ, start_response, repo_path, extras, action):
         """
         Returns a generator for the response content.
 
@@ -565,24 +568,20 @@ class SimpleVCS(object):
         also handles the locking exceptions which will be triggered when
         the first chunk is produced by the underlying WSGI application.
         """
-        svn_txn_id = ''
-        if action == 'push':
+        svn_txn_id = ""
+        if action == "push":
             svn_txn_id = self._get_txn_id(environ)
 
-        callback_daemon, extras = self._prepare_callback_daemon(
-            extras, environ, action, txn_id=svn_txn_id)
+        callback_daemon, extras = self._prepare_callback_daemon(extras, environ, action, txn_id=svn_txn_id)
 
         if svn_txn_id:
-
-            port = safe_int(extras['hooks_uri'].split(':')[-1])
             txn_id_data = extras.copy()
-            txn_id_data.update({'port': port})
-            txn_id_data.update({'req_method': environ['REQUEST_METHOD']})
+            txn_id_data.update({"req_method": environ["REQUEST_METHOD"]})
 
             full_repo_path = repo_path
             store_txn_id_data(full_repo_path, svn_txn_id, txn_id_data)
 
-        log.debug('HOOKS extras is %s', extras)
+        log.debug("HOOKS extras is %s", extras)
 
         http_scheme = self._get_http_scheme(environ)
 
@@ -609,7 +608,7 @@ class SimpleVCS(object):
 
             try:
                 # invalidate cache on push
-                if action == 'push':
+                if action == "push":
                     self._invalidate_cache(self.url_repo_name)
             finally:
                 meta.Session.remove()
@@ -632,12 +631,12 @@ class SimpleVCS(object):
         """Return the WSGI app that will finally handle the request."""
         raise NotImplementedError()
 
-    def _create_config(self, extras, repo_name, scheme='http'):
+    def _create_config(self, extras, repo_name, scheme="http"):
         """Create a safe config representation."""
         raise NotImplementedError()
 
     def _should_use_callback_daemon(self, extras, environ, action):
-        if extras.get('is_shadow_repo'):
+        if extras.get("is_shadow_repo"):
             # we don't want to execute hooks, and callback daemon for shadow repos
             return False
         return True
@@ -647,11 +646,9 @@ class SimpleVCS(object):
 
         if not self._should_use_callback_daemon(extras, environ, action):
             # disable callback daemon for actions that don't require it
-            protocol = 'local'
+            protocol = "local"
 
-        return prepare_callback_daemon(
-            extras, protocol=protocol,
-            host=vcs_settings.HOOKS_HOST, txn_id=txn_id)
+        return prepare_callback_daemon(extras, protocol=protocol, txn_id=txn_id)
 
 
 def _should_check_locking(query_string):
@@ -659,4 +656,4 @@ def _should_check_locking(query_string):
     # server see all operation on commit; bookmarks, phases and
     # obsolescence marker in different transaction, we don't want to check
     # locking on those
-    return query_string not in ['cmd=listkeys']
+    return query_string not in ["cmd=listkeys"]

rhodecode/lib/utils.py

@@ -21,6 +21,7 @@ Utilities library for RhodeCode
 """
 
 import datetime
+import importlib
 
 import decorator
 import logging
@@ -42,8 +43,9 @@ from webhelpers2.text import collapse, strip_tags, convert_accented_entities, co
 
 from mako import exceptions
 
+import rhodecode
 from rhodecode import ConfigGet
-from rhodecode.lib.exceptions import HTTPBranchProtected, HTTPLockedRC
+from rhodecode.lib.exceptions import HTTPBranchProtected, HTTPLockedRepo, ClientNotSupported
 from rhodecode.lib.hash_utils import sha256_safe, md5, sha1
 from rhodecode.lib.type_utils import AttributeDict
 from rhodecode.lib.str_utils import safe_bytes, safe_str
@@ -86,6 +88,7 @@ def adopt_for_celery(func):
     @wraps(func)
     def wrapper(extras):
         extras = AttributeDict(extras)
+
         try:
             # HooksResponse implements to_json method which must be used there.
             return func(extras).to_json()
@@ -100,7 +103,18 @@ def adopt_for_celery(func):
                 'exception_args': error_args,
                 'exception_traceback': '',
             }
-        except HTTPLockedRC as error:
+        except ClientNotSupported as error:
+            # Those special cases don't need error reporting. It's a case of
+            # locked repo or protected branch
+            error_args = error.args
+            return {
+                'status': error.code,
+                'output': error.explanation,
+                'exception': type(error).__name__,
+                'exception_args': error_args,
+                'exception_traceback': '',
+            }
+        except HTTPLockedRepo as error:
             # Those special cases don't need error reporting. It's a case of
             # locked repo or protected branch
             error_args = error.args
@@ -117,7 +131,7 @@ def adopt_for_celery(func):
                 'output': '',
                 'exception': type(e).__name__,
                 'exception_args': e.args,
-                'exception_traceback': '',
+                'exception_traceback': traceback.format_exc(),
             }
     return wrapper
 
@@ -411,6 +425,10 @@ def prepare_config_data(clear_session=True, repo=None):
         ('web', 'push_ssl', 'false'),
     ]
     for setting in ui_settings:
+        # skip certain deprecated keys that might be still in DB
+        if f"{setting.section}_{setting.key}" in ['extensions_hgsubversion']:
+            continue
+
         # Todo: remove this section once transition to *.ini files will be completed
         if setting.section in ('largefiles', 'vcs_git_lfs'):
             if setting.key != 'enabled':
@@ -686,22 +704,41 @@ def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild
 
     return added, removed
 
+def deep_reload_package(package_name):
+    """
+    Deeply reload a package by removing it and its submodules from sys.modules,
+    then re-importing it.
+    """
+    # Remove the package and its submodules from sys.modules
+    to_reload = [name for name in sys.modules if name == package_name or name.startswith(package_name + ".")]
+    for module_name in to_reload:
+        del sys.modules[module_name]
+        log.debug(f"Removed module from cache: {module_name}")
+
+    # Re-import the package
+    package = importlib.import_module(package_name)
+    log.debug(f"Re-imported package: {package_name}")
+
+    return package
 
 def load_rcextensions(root_path):
     import rhodecode
     from rhodecode.config import conf
 
     path = os.path.join(root_path)
-    sys.path.append(path)
+    deep_reload = path in sys.path
+    sys.path.insert(0, path)
 
     try:
-        rcextensions = __import__('rcextensions')
+        rcextensions = __import__('rcextensions', fromlist=[''])
     except ImportError:
         if os.path.isdir(os.path.join(path, 'rcextensions')):
             log.warning('Unable to load rcextensions from %s', path)
         rcextensions = None
 
     if rcextensions:
+        if deep_reload:
+            rcextensions = deep_reload_package('rcextensions')
         log.info('Loaded rcextensions from %s...', rcextensions)
         rhodecode.EXTENSIONS = rcextensions
 
@@ -741,6 +778,7 @@ def create_test_index(repo_location, config):
     except ImportError:
         raise ImportError('Failed to import rc_testdata, '
                           'please make sure this package is installed from requirements_test.txt')
+
     rc_testdata.extract_search_index(
         'vcs_search_index', os.path.dirname(config['search.location']))
 
@@ -785,22 +823,15 @@ def create_test_repositories(test_path, config):
     Creates test repositories in the temporary directory. Repositories are
     extracted from archives within the rc_testdata package.
     """
-    import rc_testdata
+    try:
+        import rc_testdata
+    except ImportError:
+        raise ImportError('Failed to import rc_testdata, '
+                          'please make sure this package is installed from requirements_test.txt')
+
     from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
 
-    log.debug('making test vcs repositories')
-
-    idx_path = config['search.location']
-    data_path = config['cache_dir']
-
-    # clean index and data
-    if idx_path and os.path.exists(idx_path):
-        log.debug('remove %s', idx_path)
-        shutil.rmtree(idx_path)
-
-    if data_path and os.path.exists(data_path):
-        log.debug('remove %s', data_path)
-        shutil.rmtree(data_path)
+    log.debug('making test vcs repositories at %s', test_path)
 
     rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
     rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))

rhodecode/lib/vcs/__init__.py

@@ -140,7 +140,7 @@ class CurlSession(object):
         try:
             curl.perform()
         except pycurl.error as exc:
-            log.error('Failed to call endpoint url: {} using pycurl'.format(url))
+            log.error('Failed to call endpoint url: %s using pycurl', url)
             raise
 
         status_code = curl.getinfo(pycurl.HTTP_CODE)

rhodecode/lib/vcs/backends/git/__init__.py

@@ -45,10 +45,3 @@ def discover_git_version(raise_on_exc=False):
         if raise_on_exc:
             raise
         return ''
-
-
-def lfs_store(base_location):
-    """
-    Return a lfs store relative to base_location
-    """
-    return os.path.join(base_location, '.cache', 'lfs_store')

rhodecode/lib/vcs/backends/hg/__init__.py

@@ -45,10 +45,3 @@ def discover_hg_version(raise_on_exc=False):
         if raise_on_exc:
             raise
         return ''
-
-
-def largefiles_store(base_location):
-    """
-    Return a largefile store relative to base_location
-    """
-    return os.path.join(base_location, '.cache', 'largefiles')

rhodecode/lib/vcs/client_http.py

@@ -216,7 +216,7 @@ class RemoteRepo(object):
         self._cache_region, self._cache_namespace = \
             remote_maker.init_cache_region(cache_repo_id)
 
-        with_wire = with_wire or {}
+        with_wire = with_wire or {"cache": False}
 
         repo_state_uid = with_wire.get('repo_state_uid') or 'state'
 

rhodecode/model/comment.py

@@ -373,6 +373,7 @@ class CommentsModel(BaseModel):
 
         Session().add(comment)
         Session().flush()
+
         kwargs = {
             'user': user,
             'renderer_type': renderer,
@@ -387,8 +388,7 @@ class CommentsModel(BaseModel):
         }
 
         if commit_obj:
-            recipients = ChangesetComment.get_users(
-                revision=commit_obj.raw_id)
+            recipients = ChangesetComment.get_users(revision=commit_obj.raw_id)
             # add commit author if it's in RhodeCode system
             cs_author = User.get_from_cs_author(commit_obj.author)
             if not cs_author:
@@ -397,16 +397,13 @@ class CommentsModel(BaseModel):
             recipients += [cs_author]
 
             commit_comment_url = self.get_url(comment, request=request)
-            commit_comment_reply_url = self.get_url(
-                comment, request=request,
-                anchor=f'comment-{comment.comment_id}/?/ReplyToComment')
+            commit_comment_reply_url = self.get_url(comment, request=request, anchor=f'comment-{comment.comment_id}/?/ReplyToComment')
 
             target_repo_url = h.link_to(
                 repo.repo_name,
                 h.route_url('repo_summary', repo_name=repo.repo_name))
 
-            commit_url = h.route_url('repo_commit', repo_name=repo.repo_name,
-                                     commit_id=commit_id)
+            commit_url = h.route_url('repo_commit', repo_name=repo.repo_name, commit_id=commit_id)
 
             # commit specifics
             kwargs.update({
@@ -489,7 +486,6 @@ class CommentsModel(BaseModel):
 
         if not is_draft:
             comment_data = comment.get_api_data()
-
             self._log_audit_action(
                 action, {'data': comment_data}, auth_user, comment)
 

rhodecode/model/pull_request.py

@@ -38,7 +38,7 @@ from rhodecode.translation import lazy_ugettext
 from rhodecode.lib import helpers as h, hooks_utils, diffs
 from rhodecode.lib import audit_logger
 from collections import OrderedDict
-from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
+from rhodecode.lib.hook_daemon.utils import prepare_callback_daemon
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.markup_renderer import (
     DEFAULT_COMMENTS_RENDERER, RstTemplateRenderer)
@@ -980,9 +980,7 @@ class PullRequestModel(BaseModel):
         target_ref = self._refresh_reference(
             pull_request.target_ref_parts, target_vcs)
 
-        callback_daemon, extras = prepare_callback_daemon(
-            extras, protocol=vcs_settings.HOOKS_PROTOCOL,
-            host=vcs_settings.HOOKS_HOST)
+        callback_daemon, extras = prepare_callback_daemon(extras, protocol=vcs_settings.HOOKS_PROTOCOL)
 
         with callback_daemon:
             # TODO: johbo: Implement a clean way to run a config_override

rhodecode/model/settings.py

@@ -862,27 +862,3 @@ class VcsSettingsModel(object):
                 raise ValueError(
                     f'The given data does not contain {data_key} key')
         return data_keys
-
-    def create_largeobjects_dirs_if_needed(self, repo_store_path):
-        """
-        This is subscribed to the `pyramid.events.ApplicationCreated` event. It
-        does a repository scan if enabled in the settings.
-        """
-
-        from rhodecode.lib.vcs.backends.hg import largefiles_store
-        from rhodecode.lib.vcs.backends.git import lfs_store
-
-        paths = [
-            largefiles_store(repo_store_path),
-            lfs_store(repo_store_path)]
-
-        for path in paths:
-            if os.path.isdir(path):
-                continue
-            if os.path.isfile(path):
-                continue
-            # not a file nor dir, we try to create it
-            try:
-                os.makedirs(path)
-            except Exception:
-                log.warning('Failed to create largefiles dir:%s', path)

rhodecode/tests/__init__.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2010-2023 RhodeCode GmbH
+# Copyright (C) 2010-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
@@ -38,7 +37,7 @@ from rhodecode.lib.hash_utils import sha1_safe
 log = logging.getLogger(__name__)
 
 __all__ = [
-    'get_new_dir', 'TestController',
+    'get_new_dir', 'TestController', 'console_printer',
     'clear_cache_regions',
     'assert_session_flash', 'login_user', 'no_newline_id_generator',
     'TESTS_TMP_PATH', 'HG_REPO', 'GIT_REPO', 'SVN_REPO',
@@ -244,3 +243,11 @@ def no_newline_id_generator(test_name):
 
     return test_name or 'test-with-empty-name'
 
+def console_printer(*msg):
+    print_func = print
+    try:
+        from rich import print as print_func
+    except ImportError:
+        pass
+
+    print_func(*msg)

rhodecode/tests/auth_external_test.py

@@ -1,5 +1,4 @@
-
-# Copyright (C) 2010-2023 RhodeCode GmbH
+# Copyright (C) 2010-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
@@ -90,7 +89,7 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
                 'firstname': firstname,
                 'lastname': lastname,
                 'groups': [],
-                'email': '%s@rhodecode.com' % username,
+                'email': f'{username}@rhodecode.com',
                 'admin': admin,
                 'active': active,
                 "active_from_extern": None,

rhodecode/tests/config/test_routing_links.py

@@ -20,14 +20,14 @@
 import pytest
 import requests
 from rhodecode.config import routing_links
-
+from rhodecode.tests import console_printer
 
 def check_connection():
     try:
         response = requests.get('https://rhodecode.com')
         return response.status_code == 200
     except Exception as e:
-        print(e)
+        console_printer(e)
 
     return False
 

rhodecode/tests/conftest.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2010-2023 RhodeCode GmbH
+# Copyright (C) 2010-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
@@ -16,23 +16,10 @@
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-"""
-py.test config for test suite for making push/pull operations.
-
-.. important::
-
-   You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
-   to redirect things to stderr instead of stdout.
-"""
-
-import pytest
+import pytest # noqa
 import logging
-
-from rhodecode.authentication import AuthenticationPluginRegistry
-from rhodecode.model.db import Permission, User
-from rhodecode.model.meta import Session
-from rhodecode.model.settings import SettingsModel
-from rhodecode.model.user import UserModel
+import collections
+import rhodecode
 
 
 log = logging.getLogger(__name__)
@@ -40,99 +27,3 @@ log = logging.getLogger(__name__)
 # Docker image running httpbin...
 HTTPBIN_DOMAIN = 'http://httpbin'
 HTTPBIN_POST = HTTPBIN_DOMAIN + '/post'
-
-
-@pytest.fixture()
-def enable_auth_plugins(request, baseapp, csrf_token):
-    """
-    Return a factory object that when called, allows to control which
-    authentication plugins are enabled.
-    """
-
-    class AuthPluginManager(object):
-
-        def cleanup(self):
-            self._enable_plugins(['egg:rhodecode-enterprise-ce#rhodecode'])
-
-        def enable(self, plugins_list, override=None):
-            return self._enable_plugins(plugins_list, override)
-
-        def _enable_plugins(self, plugins_list, override=None):
-            override = override or {}
-            params = {
-                'auth_plugins': ','.join(plugins_list),
-            }
-
-            # helper translate some names to others, to fix settings code
-            name_map = {
-                'token': 'authtoken'
-            }
-            log.debug('enable_auth_plugins: enabling following auth-plugins: %s', plugins_list)
-
-            for module in plugins_list:
-                plugin_name = module.partition('#')[-1]
-                if plugin_name in name_map:
-                    plugin_name = name_map[plugin_name]
-                enabled_plugin = f'auth_{plugin_name}_enabled'
-                cache_ttl = f'auth_{plugin_name}_cache_ttl'
-
-                # default params that are needed for each plugin,
-                # `enabled` and `cache_ttl`
-                params.update({
-                    enabled_plugin: True,
-                    cache_ttl: 0
-                })
-                if override.get:
-                    params.update(override.get(module, {}))
-
-                validated_params = params
-
-                for k, v in validated_params.items():
-                    setting = SettingsModel().create_or_update_setting(k, v)
-                    Session().add(setting)
-                Session().commit()
-
-            AuthenticationPluginRegistry.invalidate_auth_plugins_cache(hard=True)
-
-            enabled_plugins = SettingsModel().get_auth_plugins()
-            assert plugins_list == enabled_plugins
-
-    enabler = AuthPluginManager()
-    request.addfinalizer(enabler.cleanup)
-
-    return enabler
-
-
-@pytest.fixture()
-def test_user_factory(request, baseapp):
-
-    def user_factory(username='test_user', password='qweqwe', first_name='John', last_name='Testing', **kwargs):
-        usr = UserModel().create_or_update(
-            username=username,
-            password=password,
-            email=f'{username}@rhodecode.org',
-            firstname=first_name, lastname=last_name)
-        Session().commit()
-
-        for k, v in kwargs.items():
-            setattr(usr, k, v)
-            Session().add(usr)
-
-        new_usr = User.get_by_username(username)
-        new_usr_id = new_usr.user_id
-        assert new_usr == usr
-
-        @request.addfinalizer
-        def cleanup():
-            if User.get(new_usr_id) is None:
-                return
-
-            perm = Permission.query().all()
-            for p in perm:
-                UserModel().revoke_perm(usr, p)
-
-            UserModel().delete(new_usr_id)
-            Session().commit()
-        return usr
-
-    return user_factory

rhodecode/tests/conftest_common.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2010-2023 RhodeCode GmbH
+# Copyright (C) 2010-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
@@ -98,16 +98,16 @@ def pytest_addoption(parser):
         'pyramid_config',
         "Set up a Pyramid environment with the specified config file.")
 
+    parser.addini('rhodecode_config', 'rhodecode config ini for tests')
+    parser.addini('celery_config', 'celery config ini for tests')
+    parser.addini('vcsserver_config', 'vcsserver config ini for tests')
+
     vcsgroup = parser.getgroup('vcs')
+
     vcsgroup.addoption(
         '--without-vcsserver', dest='with_vcsserver', action='store_false',
         help="Do not start the VCSServer in a background process.")
-    vcsgroup.addoption(
-        '--with-vcsserver-http', dest='vcsserver_config_http',
-        help="Start the HTTP VCSServer with the specified config file.")
-    vcsgroup.addoption(
-        '--vcsserver-protocol', dest='vcsserver_protocol',
-        help="Start the VCSServer with HTTP protocol support.")
+
     vcsgroup.addoption(
         '--vcsserver-config-override', action='store', type=_parse_json,
         default=None, dest='vcsserver_config_override', help=(
@@ -122,12 +122,6 @@ def pytest_addoption(parser):
             "Allows to set the port of the vcsserver. Useful when testing "
             "against an already running server and random ports cause "
             "trouble."))
-    parser.addini(
-        'vcsserver_config_http',
-        "Start the HTTP VCSServer with the specified config file.")
-    parser.addini(
-        'vcsserver_protocol',
-        "Start the VCSServer with HTTP protocol support.")
 
 
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)

rhodecode/tests/database/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/

rhodecode/tests/database/conftest.py

@@ -1,4 +1,3 @@
-
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
@@ -17,7 +16,7 @@
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-from subprocess import Popen, PIPE
+import subprocess
 import os
 import sys
 import tempfile
@@ -26,87 +25,71 @@ import pytest
 from sqlalchemy.engine import url
 
 from rhodecode.lib.str_utils import safe_str, safe_bytes
-from rhodecode.tests.fixture import TestINI
+from rhodecode.tests.fixtures.rc_fixture import TestINI
 
 
 def _get_dbs_from_metafunc(metafunc):
-    dbs_mark = metafunc.definition.get_closest_marker('dbs')
+    dbs_mark = metafunc.definition.get_closest_marker("dbs")
 
     if dbs_mark:
         # Supported backends by this test function, created from pytest.mark.dbs
         backends = dbs_mark.args
     else:
-        backends = metafunc.config.getoption('--dbs')
+        backends = metafunc.config.getoption("--dbs")
     return backends
 
 
 def pytest_generate_tests(metafunc):
     # Support test generation based on --dbs parameter
-    if 'db_backend' in metafunc.fixturenames:
-        requested_backends = set(metafunc.config.getoption('--dbs'))
+    if "db_backend" in metafunc.fixturenames:
+        requested_backends = set(metafunc.config.getoption("--dbs"))
         backends = _get_dbs_from_metafunc(metafunc)
         backends = requested_backends.intersection(backends)
         # TODO: johbo: Disabling a backend did not work out with
         # parametrization, find better way to achieve this.
         if not backends:
             metafunc.function._skip = True
-        metafunc.parametrize('db_backend_name', backends)
+        metafunc.parametrize("db_backend_name", backends)
 
 
 def pytest_collection_modifyitems(session, config, items):
-    remaining = [
-        i for i in items if not getattr(i.obj, '_skip', False)]
+    remaining = [i for i in items if not getattr(i.obj, "_skip", False)]
     items[:] = remaining
 
 
 @pytest.fixture()
-def db_backend(
-        request, db_backend_name, ini_config, tmpdir_factory):
+def db_backend(request, db_backend_name, ini_config, tmpdir_factory):
     basetemp = tmpdir_factory.getbasetemp().strpath
     klass = _get_backend(db_backend_name)
 
-    option_name = '--{}-connection-string'.format(db_backend_name)
+    option_name = "--{}-connection-string".format(db_backend_name)
     connection_string = request.config.getoption(option_name) or None
 
-    return klass(
-        config_file=ini_config, basetemp=basetemp,
-        connection_string=connection_string)
+    return klass(config_file=ini_config, basetemp=basetemp, connection_string=connection_string)
 
 
 def _get_backend(backend_type):
-    return {
-        'sqlite': SQLiteDBBackend,
-        'postgres': PostgresDBBackend,
-        'mysql': MySQLDBBackend,
-        '': EmptyDBBackend
-    }[backend_type]
+    return {"sqlite": SQLiteDBBackend, "postgres": PostgresDBBackend, "mysql": MySQLDBBackend, "": EmptyDBBackend}[
+        backend_type
+    ]
 
 
 class DBBackend(object):
     _store = os.path.dirname(os.path.abspath(__file__))
     _type = None
-    _base_ini_config = [{'app:main': {'vcs.start_server': 'false',
-                                      'startup.import_repos': 'false'}}]
-    _db_url = [{'app:main': {'sqlalchemy.db1.url': ''}}]
-    _base_db_name = 'rhodecode_test_db_backend'
-    std_env = {'RC_TEST': '0'}
-
-    def __init__(
-            self, config_file, db_name=None, basetemp=None,
-            connection_string=None):
-
-        from rhodecode.lib.vcs.backends.hg import largefiles_store
-        from rhodecode.lib.vcs.backends.git import lfs_store
+    _base_ini_config = [{"app:main": {"vcs.start_server": "false", "startup.import_repos": "false"}}]
+    _db_url = [{"app:main": {"sqlalchemy.db1.url": ""}}]
+    _base_db_name = "rhodecode_test_db_backend"
+    std_env = {"RC_TEST": "0"}
 
+    def __init__(self, config_file, db_name=None, basetemp=None, connection_string=None):
         self.fixture_store = os.path.join(self._store, self._type)
         self.db_name = db_name or self._base_db_name
         self._base_ini_file = config_file
-        self.stderr = ''
-        self.stdout = ''
+        self.stderr = ""
+        self.stdout = ""
         self._basetemp = basetemp or tempfile.gettempdir()
-        self._repos_location = os.path.join(self._basetemp, 'rc_test_repos')
-        self._repos_hg_largefiles_store = largefiles_store(self._basetemp)
-        self._repos_git_lfs_store = lfs_store(self._basetemp)
+        self._repos_location = os.path.join(self._basetemp, "rc_test_repos")
         self.connection_string = connection_string
 
     @property
@@ -118,8 +101,7 @@ class DBBackend(object):
         if not new_connection_string:
             new_connection_string = self.get_default_connection_string()
         else:
-            new_connection_string = new_connection_string.format(
-                db_name=self.db_name)
+            new_connection_string = new_connection_string.format(db_name=self.db_name)
         url_parts = url.make_url(new_connection_string)
         self._connection_string = new_connection_string
         self.user = url_parts.username
@@ -127,73 +109,67 @@ class DBBackend(object):
         self.host = url_parts.host
 
     def get_default_connection_string(self):
-        raise NotImplementedError('default connection_string is required.')
+        raise NotImplementedError("default connection_string is required.")
 
     def execute(self, cmd, env=None, *args):
         """
         Runs command on the system with given ``args``.
         """
 
-        command = cmd + ' ' + ' '.join(args)
-        sys.stdout.write(f'CMD: {command}')
+        command = cmd + " " + " ".join(args)
+        sys.stdout.write(f"CMD: {command}")
 
         # Tell Python to use UTF-8 encoding out stdout
         _env = os.environ.copy()
-        _env['PYTHONIOENCODING'] = 'UTF-8'
+        _env["PYTHONIOENCODING"] = "UTF-8"
         _env.update(self.std_env)
         if env:
             _env.update(env)
 
-        self.p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, env=_env)
+        self.p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=_env)
         self.stdout, self.stderr = self.p.communicate()
         stdout_str = safe_str(self.stdout)
-        sys.stdout.write(f'COMMAND:{command}\n')
+        sys.stdout.write(f"COMMAND:{command}\n")
         sys.stdout.write(stdout_str)
         return self.stdout, self.stderr
 
     def assert_returncode_success(self):
         from rich import print as pprint
+
         if not self.p.returncode == 0:
             pprint(safe_str(self.stderr))
-            raise AssertionError(f'non 0 retcode:{self.p.returncode}')
+            raise AssertionError(f"non 0 retcode:{self.p.returncode}")
 
     def assert_correct_output(self, stdout, version):
-        assert b'UPGRADE FOR STEP %b COMPLETED' % safe_bytes(version) in stdout
+        assert b"UPGRADE FOR STEP %b COMPLETED" % safe_bytes(version) in stdout
 
     def setup_rhodecode_db(self, ini_params=None, env=None):
         if not ini_params:
             ini_params = self._base_ini_config
 
         ini_params.extend(self._db_url)
-        with TestINI(self._base_ini_file, ini_params,
-                     self._type, destroy=True) as _ini_file:
-
+        with TestINI(self._base_ini_file, ini_params, self._type, destroy=True) as _ini_file:
             if not os.path.isdir(self._repos_location):
                 os.makedirs(self._repos_location)
-            if not os.path.isdir(self._repos_hg_largefiles_store):
-                os.makedirs(self._repos_hg_largefiles_store)
-            if not os.path.isdir(self._repos_git_lfs_store):
-                os.makedirs(self._repos_git_lfs_store)
 
             return self.execute(
                 "rc-setup-app {0} --user=marcink "
                 "--email=marcin@rhodeocode.com --password={1} "
-                "--repos={2} --force-yes".format(
-                    _ini_file, 'qweqwe', self._repos_location), env=env)
+                "--repos={2} --force-yes".format(_ini_file, "qweqwe", self._repos_location),
+                env=env,
+            )
 
     def upgrade_database(self, ini_params=None):
         if not ini_params:
             ini_params = self._base_ini_config
         ini_params.extend(self._db_url)
 
-        test_ini = TestINI(
-            self._base_ini_file, ini_params, self._type, destroy=True)
+        test_ini = TestINI(self._base_ini_file, ini_params, self._type, destroy=True)
         with test_ini as ini_file:
             if not os.path.isdir(self._repos_location):
                 os.makedirs(self._repos_location)
 
-            return self.execute(
-                "rc-upgrade-db {0} --force-yes".format(ini_file))
+            return self.execute("rc-upgrade-db {0} --force-yes".format(ini_file))
 
     def setup_db(self):
         raise NotImplementedError
@@ -206,7 +182,7 @@ class DBBackend(object):
 
 
 class EmptyDBBackend(DBBackend):
-    _type = ''
+    _type = ""
 
     def setup_db(self):
         pass
@@ -222,21 +198,20 @@ class EmptyDBBackend(DBBackend):
 
 
 class SQLiteDBBackend(DBBackend):
-    _type = 'sqlite'
+    _type = "sqlite"
 
     def get_default_connection_string(self):
-        return 'sqlite:///{}/{}.sqlite'.format(self._basetemp, self.db_name)
+        return "sqlite:///{}/{}.sqlite".format(self._basetemp, self.db_name)
 
     def setup_db(self):
         # dump schema for tests
         # cp -v $TEST_DB_NAME
-        self._db_url = [{'app:main': {
-            'sqlalchemy.db1.url': self.connection_string}}]
+        self._db_url = [{"app:main": {"sqlalchemy.db1.url": self.connection_string}}]
 
     def import_dump(self, dumpname):
         dump = os.path.join(self.fixture_store, dumpname)
-        target = os.path.join(self._basetemp, '{0.db_name}.sqlite'.format(self))
-        return self.execute(f'cp -v {dump} {target}')
+        target = os.path.join(self._basetemp, "{0.db_name}.sqlite".format(self))
+        return self.execute(f"cp -v {dump} {target}")
 
     def teardown_db(self):
         target_db = os.path.join(self._basetemp, self.db_name)
@@ -244,39 +219,39 @@ class SQLiteDBBackend(DBBackend):
 
 
 class MySQLDBBackend(DBBackend):
-    _type = 'mysql'
+    _type = "mysql"
 
     def get_default_connection_string(self):
-        return 'mysql://root:qweqwe@127.0.0.1/{}'.format(self.db_name)
+        return "mysql://root:qweqwe@127.0.0.1/{}".format(self.db_name)
 
     def setup_db(self):
         # dump schema for tests
         # mysqldump -uroot -pqweqwe $TEST_DB_NAME
-        self._db_url = [{'app:main': {
-            'sqlalchemy.db1.url': self.connection_string}}]
-        return self.execute("mysql -v -u{} -p{} -e 'create database '{}';'".format(
-            self.user, self.password, self.db_name))
+        self._db_url = [{"app:main": {"sqlalchemy.db1.url": self.connection_string}}]
+        return self.execute(
+            "mysql -v -u{} -p{} -e 'create database '{}';'".format(self.user, self.password, self.db_name)
+        )
 
     def import_dump(self, dumpname):
         dump = os.path.join(self.fixture_store, dumpname)
-        return self.execute("mysql -u{} -p{} {} < {}".format(
-            self.user, self.password, self.db_name, dump))
+        return self.execute("mysql -u{} -p{} {} < {}".format(self.user, self.password, self.db_name, dump))
 
     def teardown_db(self):
-        return self.execute("mysql -v -u{} -p{} -e 'drop database '{}';'".format(
-            self.user, self.password, self.db_name))
+        return self.execute(
+            "mysql -v -u{} -p{} -e 'drop database '{}';'".format(self.user, self.password, self.db_name)
+        )
 
 
 class PostgresDBBackend(DBBackend):
-    _type = 'postgres'
+    _type = "postgres"
 
     def get_default_connection_string(self):
-        return 'postgresql://postgres:qweqwe@localhost/{}'.format(self.db_name)
+        return "postgresql://postgres:qweqwe@localhost/{}".format(self.db_name)
 
     def setup_db(self):
         # dump schema for tests
         # pg_dump -U postgres -h localhost $TEST_DB_NAME
-        self._db_url = [{'app:main': {'sqlalchemy.db1.url': self.connection_string}}]
+        self._db_url = [{"app:main": {"sqlalchemy.db1.url": self.connection_string}}]
         cmd = f"PGPASSWORD={self.password} psql -U {self.user} -h localhost -c 'create database '{self.db_name}';'"
         return self.execute(cmd)
 

rhodecode/tests/database/test_creation.py

@@ -1,4 +1,3 @@
-
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify

rhodecode/tests/database/test_migration.py

@@ -1,4 +1,3 @@
-
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
@@ -21,33 +20,42 @@ import pytest
 
 
 @pytest.mark.dbs("postgres")
-@pytest.mark.parametrize("dumpname", [
-    '1.4.4.sql',
-    '1.5.0.sql',
-    '1.6.0.sql',
-    '1.6.0_no_repo_name_index.sql',
-])
+@pytest.mark.parametrize(
+    "dumpname",
+    [
+        "1.4.4.sql",
+        "1.5.0.sql",
+        "1.6.0.sql",
+        "1.6.0_no_repo_name_index.sql",
+    ],
+)
 def test_migrate_postgres_db(db_backend, dumpname):
     _run_migration_test(db_backend, dumpname)
 
 
 @pytest.mark.dbs("sqlite")
-@pytest.mark.parametrize("dumpname", [
-    'rhodecode.1.4.4.sqlite',
-    'rhodecode.1.4.4_with_groups.sqlite',
-    'rhodecode.1.4.4_with_ldap_active.sqlite',
-])
+@pytest.mark.parametrize(
+    "dumpname",
+    [
+        "rhodecode.1.4.4.sqlite",
+        "rhodecode.1.4.4_with_groups.sqlite",
+        "rhodecode.1.4.4_with_ldap_active.sqlite",
+    ],
+)
 def test_migrate_sqlite_db(db_backend, dumpname):
     _run_migration_test(db_backend, dumpname)
 
 
 @pytest.mark.dbs("mysql")
-@pytest.mark.parametrize("dumpname", [
-    '1.4.4.sql',
-    '1.5.0.sql',
-    '1.6.0.sql',
-    '1.6.0_no_repo_name_index.sql',
-])
+@pytest.mark.parametrize(
+    "dumpname",
+    [
+        "1.4.4.sql",
+        "1.5.0.sql",
+        "1.6.0.sql",
+        "1.6.0_no_repo_name_index.sql",
+    ],
+)
 def test_migrate_mysql_db(db_backend, dumpname):
     _run_migration_test(db_backend, dumpname)
 
@@ -60,5 +68,5 @@ def _run_migration_test(db_backend, dumpname):
     db_backend.import_dump(dumpname)
     stdout, stderr = db_backend.upgrade_database()
 
-    db_backend.assert_correct_output(stdout+stderr, version='16')
+    db_backend.assert_correct_output(stdout + stderr, version="16")
     db_backend.assert_returncode_success()

rhodecode/tests/fixture_mods/fixture_pyramid.py

@@ -1,226 +0,0 @@
-
-# Copyright (C) 2010-2023 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-import pytest
-
-from rhodecode.lib.config_utils import get_app_config
-from rhodecode.tests.fixture import TestINI
-from rhodecode.tests import TESTS_TMP_PATH
-from rhodecode.tests.server_utils import RcVCSServer
-
-
-@pytest.fixture(scope='session')
-def vcsserver(request, vcsserver_port, vcsserver_factory):
-    """
-    Session scope VCSServer.
-
-    Tests which need the VCSServer have to rely on this fixture in order
-    to ensure it will be running.
-
-    For specific needs, the fixture vcsserver_factory can be used. It allows to
-    adjust the configuration file for the test run.
-
-    Command line args:
-
-    --without-vcsserver: Allows to switch this fixture off. You have to
-    manually start the server.
-
-    --vcsserver-port: Will expect the VCSServer to listen on this port.
-    """
-
-    if not request.config.getoption('with_vcsserver'):
-        return None
-
-    return vcsserver_factory(
-        request, vcsserver_port=vcsserver_port)
-
-
-@pytest.fixture(scope='session')
-def vcsserver_factory(tmpdir_factory):
-    """
-    Use this if you need a running vcsserver with a special configuration.
-    """
-
-    def factory(request, overrides=(), vcsserver_port=None,
-                log_file=None, workers='3'):
-
-        if vcsserver_port is None:
-            vcsserver_port = get_available_port()
-
-        overrides = list(overrides)
-        overrides.append({'server:main': {'port': vcsserver_port}})
-
-        option_name = 'vcsserver_config_http'
-        override_option_name = 'vcsserver_config_override'
-        config_file = get_config(
-            request.config, option_name=option_name,
-            override_option_name=override_option_name, overrides=overrides,
-            basetemp=tmpdir_factory.getbasetemp().strpath,
-            prefix='test_vcs_')
-
-        server = RcVCSServer(config_file, log_file, workers)
-        server.start()
-
-        @request.addfinalizer
-        def cleanup():
-            server.shutdown()
-
-        server.wait_until_ready()
-        return server
-
-    return factory
-
-
-def _use_log_level(config):
-    level = config.getoption('test_loglevel') or 'critical'
-    return level.upper()
-
-
-@pytest.fixture(scope='session')
-def ini_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
-    option_name = 'pyramid_config'
-    log_level = _use_log_level(request.config)
-
-    overrides = [
-        {'server:main': {'port': rcserver_port}},
-        {'app:main': {
-            'cache_dir': '%(here)s/rc-tests/rc_data',
-            'vcs.server': f'localhost:{vcsserver_port}',
-            # johbo: We will always start the VCSServer on our own based on the
-            # fixtures of the test cases. For the test run it must always be
-            # off in the INI file.
-            'vcs.start_server': 'false',
-
-            'vcs.server.protocol': 'http',
-            'vcs.scm_app_implementation': 'http',
-            'vcs.svn.proxy.enabled': 'true',
-            'vcs.hooks.protocol.v2': 'celery',
-            'vcs.hooks.host': '*',
-            'repo_store.path': TESTS_TMP_PATH,
-            'app.service_api.token': 'service_secret_token',
-        }},
-
-        {'handler_console': {
-            'class': 'StreamHandler',
-            'args': '(sys.stderr,)',
-            'level': log_level,
-        }},
-
-    ]
-
-    filename = get_config(
-        request.config, option_name=option_name,
-        override_option_name='{}_override'.format(option_name),
-        overrides=overrides,
-        basetemp=tmpdir_factory.getbasetemp().strpath,
-        prefix='test_rce_')
-    return filename
-
-
-@pytest.fixture(scope='session')
-def ini_settings(ini_config):
-    ini_path = ini_config
-    return get_app_config(ini_path)
-
-
-def get_available_port(min_port=40000, max_port=55555):
-    from rhodecode.lib.utils2 import get_available_port as _get_port
-    return _get_port(min_port, max_port)
-
-
-@pytest.fixture(scope='session')
-def rcserver_port(request):
-    port = get_available_port()
-    print(f'Using rhodecode port {port}')
-    return port
-
-
-@pytest.fixture(scope='session')
-def vcsserver_port(request):
-    port = request.config.getoption('--vcsserver-port')
-    if port is None:
-        port = get_available_port()
-        print(f'Using vcsserver port {port}')
-    return port
-
-
-@pytest.fixture(scope='session')
-def available_port_factory() -> get_available_port:
-    """
-    Returns a callable which returns free port numbers.
-    """
-    return get_available_port
-
-
-@pytest.fixture()
-def available_port(available_port_factory):
-    """
-    Gives you one free port for the current test.
-
-    Uses "available_port_factory" to retrieve the port.
-    """
-    return available_port_factory()
-
-
-@pytest.fixture(scope='session')
-def testini_factory(tmpdir_factory, ini_config):
-    """
-    Factory to create an INI file based on TestINI.
-
-    It will make sure to place the INI file in the correct directory.
-    """
-    basetemp = tmpdir_factory.getbasetemp().strpath
-    return TestIniFactory(basetemp, ini_config)
-
-
-class TestIniFactory(object):
-
-    def __init__(self, basetemp, template_ini):
-        self._basetemp = basetemp
-        self._template_ini = template_ini
-
-    def __call__(self, ini_params, new_file_prefix='test'):
-        ini_file = TestINI(
-            self._template_ini, ini_params=ini_params,
-            new_file_prefix=new_file_prefix, dir=self._basetemp)
-        result = ini_file.create()
-        return result
-
-
-def get_config(
-        config, option_name, override_option_name, overrides=None,
-        basetemp=None, prefix='test'):
-    """
-    Find a configuration file and apply overrides for the given `prefix`.
-    """
-    config_file = (
-        config.getoption(option_name) or config.getini(option_name))
-    if not config_file:
-        pytest.exit(
-            "Configuration error, could not extract {}.".format(option_name))
-
-    overrides = overrides or []
-    config_override = config.getoption(override_option_name)
-    if config_override:
-        overrides.append(config_override)
-    temp_ini_file = TestINI(
-        config_file, ini_params=overrides, new_file_prefix=prefix,
-        dir=basetemp)
-
-    return temp_ini_file.create()