engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tests: fixed test suite for celery adoption

Browse source
This commit is contained in:
RhodeCode Admin 2024-11-08 16:17:57 +01:00
parent 840634c16f
commit 1d5d2c0155
209 changed files with 5248 additions and 5275 deletions
Download patch file Download diff file Expand all files Collapse all files

3
rhodecode/api/tests/test_add_field_to_repo.py
View file

@ -1,5 +1,4 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
# Copyright (C) 2010-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3

2
rhodecode/api/tests/test_create_gist.py
View file

@ -24,7 +24,7 @@ from rhodecode.model.db import Gist
from rhodecode.model.gist import GistModel
from rhodecode.api.tests.utils import (
build_data, api_call, assert_error, assert_ok, crash)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
@pytest.mark.usefixtures("testuser_api", "app")

2
rhodecode/api/tests/test_create_repo.py
View file

@ -27,7 +27,7 @@ from rhodecode.model.user import UserModel
from rhodecode.tests import TEST_USER_ADMIN_LOGIN
from rhodecode.api.tests.utils import (
build_data, api_call, assert_ok, assert_error, crash)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.lib.ext_json import json
from rhodecode.lib.str_utils import safe_str

2
rhodecode/api/tests/test_create_repo_group.py
View file

@ -26,7 +26,7 @@ from rhodecode.model.user import UserModel
from rhodecode.tests import TEST_USER_ADMIN_LOGIN
from rhodecode.api.tests.utils import (
build_data, api_call, assert_ok, assert_error, crash)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
fixture = Fixture()

2
rhodecode/api/tests/test_create_user.py
View file

@ -26,7 +26,7 @@ from rhodecode.tests import (
TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_EMAIL)
from rhodecode.api.tests.utils import (
build_data, api_call, assert_ok, assert_error, jsonify, crash)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.model.db import RepoGroup

2
rhodecode/api/tests/test_create_user_group.py
View file

@ -25,7 +25,7 @@ from rhodecode.model.user import UserModel
from rhodecode.model.user_group import UserGroupModel
from rhodecode.api.tests.utils import (
build_data, api_call, assert_error, assert_ok, crash, jsonify)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
@pytest.mark.usefixtures("testuser_api", "app")

2
rhodecode/api/tests/test_fork_repo.py
View file

@ -28,7 +28,7 @@ from rhodecode.model.user import UserModel
from rhodecode.tests import TEST_USER_ADMIN_LOGIN
from rhodecode.api.tests.utils import (
build_data, api_call, assert_error, assert_ok, crash)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
fixture = Fixture()

4
rhodecode/api/tests/test_update_repo.py
View file

@ -25,8 +25,8 @@ from rhodecode.model.scm import ScmModel
from rhodecode.tests import TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_LOGIN
from rhodecode.api.tests.utils import (
build_data, api_call, assert_error, assert_ok, crash, jsonify)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixture_mods.fixture_utils import plain_http_host_only_stub
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.fixtures.fixture_utils import plain_http_host_only_stub
fixture = Fixture()

2
rhodecode/apps/admin/tests/test_admin_audit_logs.py
View file

@ -26,7 +26,7 @@ import pytest
from rhodecode.lib.str_utils import safe_str
from rhodecode.tests import *
from rhodecode.tests.routes import route_path
from rhodecode.tests.fixture import FIXTURES
from rhodecode.tests.fixtures.rc_fixture import FIXTURES
from rhodecode.model.db import UserLog
from rhodecode.model.meta import Session

2
rhodecode/apps/admin/tests/test_admin_main_views.py
View file

@ -20,7 +20,7 @@
import pytest
from rhodecode.tests import TestController
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/admin/tests/test_admin_repos.py
View file

@ -37,7 +37,7 @@ from rhodecode.model.user import UserModel
from rhodecode.tests import (
login_user_session, assert_session_flash, TEST_USER_ADMIN_LOGIN,
TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
from rhodecode.tests.fixture import Fixture, error_function
from rhodecode.tests.fixtures.rc_fixture import Fixture, error_function
from rhodecode.tests.utils import repo_on_filesystem
from rhodecode.tests.routes import route_path

2
rhodecode/apps/admin/tests/test_admin_repository_groups.py
View file

@ -27,7 +27,7 @@ from rhodecode.model.meta import Session
from rhodecode.model.repo_group import RepoGroupModel
from rhodecode.tests import (
assert_session_flash, TEST_USER_REGULAR_LOGIN, TESTS_TMP_PATH)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path

2
rhodecode/apps/admin/tests/test_admin_user_groups.py
View file

@ -24,7 +24,7 @@ from rhodecode.model.meta import Session
from rhodecode.tests import (
TestController, assert_session_flash)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/admin/tests/test_admin_users.py
View file

@ -28,7 +28,7 @@ from rhodecode.model.user import UserModel
from rhodecode.tests import (
TestController, TEST_USER_REGULAR_LOGIN, assert_session_flash)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py
View file

@ -22,7 +22,7 @@ import pytest
from rhodecode.model.db import User, UserSshKeys
from rhodecode.tests import TestController, assert_session_flash
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/home/tests/test_get_goto_switched_data.py
View file

@ -27,7 +27,7 @@ from rhodecode.model.repo_group import RepoGroupModel
from rhodecode.model.db import Session, Repository, RepoGroup
from rhodecode.tests import TestController, TEST_USER_ADMIN_LOGIN
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/home/tests/test_get_repo_list_data.py
View file

@ -22,7 +22,7 @@ from rhodecode.model.db import Repository
from rhodecode.lib.ext_json import json
from rhodecode.tests import TestController
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/home/tests/test_get_user_data.py
View file

@ -20,7 +20,7 @@ import pytest
from rhodecode.lib.ext_json import json
from rhodecode.tests import TestController
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/home/tests/test_get_user_group_data.py
View file

@ -40,7 +40,7 @@ import pytest
from rhodecode.lib.ext_json import json
from rhodecode.tests import TestController
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/home/tests/test_home.py
View file

@ -24,7 +24,7 @@ from rhodecode.model.db import Repository, RepoGroup, User
from rhodecode.model.meta import Session
from rhodecode.model.settings import SettingsModel
from rhodecode.tests import TestController
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path

2
rhodecode/apps/login/tests/test_2fa.py
View file

@ -3,7 +3,7 @@ import mock
from rhodecode.lib.type_utils import AttributeDict
from rhodecode.model.meta import Session
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
from rhodecode.model.settings import SettingsModel

2
rhodecode/apps/login/tests/test_login.py
View file

@ -31,7 +31,7 @@ from rhodecode.model.meta import Session
from rhodecode.tests import (
assert_session_flash, HG_REPO, TEST_USER_ADMIN_LOGIN,
no_newline_id_generator)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/login/tests/test_password_reset.py
View file

@ -22,7 +22,7 @@ from rhodecode.lib import helpers as h
from rhodecode.tests import (
TestController, clear_cache_regions,
TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.utils import AssertResponse
from rhodecode.tests.routes import route_path

2
rhodecode/apps/my_account/tests/test_my_account_auth_tokens.py
View file

@ -22,7 +22,7 @@ from rhodecode.apps._base import ADMIN_PREFIX
from rhodecode.model.db import User
from rhodecode.tests import (
TestController, assert_session_flash)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path

2
rhodecode/apps/my_account/tests/test_my_account_emails.py
View file

@ -23,7 +23,7 @@ from rhodecode.model.db import User, UserEmailMap
from rhodecode.tests import (
TestController, TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_EMAIL,
assert_session_flash, TEST_USER_REGULAR_PASS)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path

2
rhodecode/apps/my_account/tests/test_my_account_notifications.py
View file

@ -21,7 +21,7 @@ import pytest
from rhodecode.tests import (
TestController, TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
from rhodecode.model.db import Notification, User

2
rhodecode/apps/my_account/tests/test_my_account_password.py
View file

@ -24,7 +24,7 @@ from rhodecode.lib.auth import check_password
from rhodecode.model.meta import Session
from rhodecode.model.user import UserModel
from rhodecode.tests import assert_session_flash, TestController
from rhodecode.tests.fixture import Fixture, error_function
from rhodecode.tests.fixtures.rc_fixture import Fixture, error_function
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/my_account/tests/test_my_account_profile.py
View file

@ -20,7 +20,7 @@
from rhodecode.tests import (
TestController, TEST_USER_ADMIN_LOGIN,
TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/my_account/tests/test_my_account_simple_views.py
View file

@ -19,7 +19,7 @@
from rhodecode.model.db import User, Repository, UserFollowing
from rhodecode.tests import TestController, TEST_USER_ADMIN_LOGIN
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py
View file

@ -21,7 +21,7 @@
from rhodecode.model.db import User, UserSshKeys
from rhodecode.tests import TestController, assert_session_flash
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

15
rhodecode/apps/repository/tests/test_repo_compare_on_single_file.py
View file

@ -22,7 +22,7 @@ import pytest
from rhodecode.apps.repository.tests.test_repo_compare import ComparePage
from rhodecode.lib.vcs import nodes
from rhodecode.lib.vcs.backends.base import EmptyCommit
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.utils import commit_change
from rhodecode.tests.routes import route_path
@ -166,14 +166,15 @@ class TestSideBySideDiff(object):
response.mustcontain('Collapse 2 commits')
response.mustcontain('123 file changed')
response.mustcontain(
'r%s:%s...r%s:%s' % (
commit1.idx, commit1.short_id, commit2.idx, commit2.short_id))
response.mustcontain(f'r{commit1.idx}:{commit1.short_id}...r{commit2.idx}:{commit2.short_id}')
response.mustcontain(f_path)
@pytest.mark.xfail(reason='GIT does not handle empty commit compare correct (missing 1 commit)')
#@pytest.mark.xfail(reason='GIT does not handle empty commit compare correct (missing 1 commit)')
def test_diff_side_by_side_from_0_commit_with_file_filter(self, app, backend, backend_stub):
if backend.alias == 'git':
pytest.skip('GIT does not handle empty commit compare correct (missing 1 commit)')
f_path = b'test_sidebyside_file.py'
commit1_content = b'content-25d7e49c18b159446c\n'
commit2_content = b'content-603d6c72c46d953420\n'
@ -200,9 +201,7 @@ class TestSideBySideDiff(object):
response.mustcontain('Collapse 2 commits')
response.mustcontain('1 file changed')
response.mustcontain(
'r%s:%s...r%s:%s' % (
commit1.idx, commit1.short_id, commit2.idx, commit2.short_id))
response.mustcontain(f'r{commit1.idx}:{commit1.short_id}...r{commit2.idx}:{commit2.short_id}')
response.mustcontain(f_path)

2
rhodecode/apps/repository/tests/test_repo_files.py
View file

@ -33,7 +33,7 @@ from rhodecode.lib.vcs.conf import settings
from rhodecode.model.db import Session, Repository
from rhodecode.tests import assert_session_flash
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path

2
rhodecode/apps/repository/tests/test_repo_forks.py
View file

@ -21,7 +21,7 @@ import pytest
from rhodecode.tests import TestController, assert_session_flash, HG_FORK, GIT_FORK
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.lib import helpers as h
from rhodecode.model.db import Repository

2
rhodecode/apps/repository/tests/test_repo_maintainance.py
View file

@ -21,7 +21,7 @@ import pytest
from rhodecode.model.db import Repository, UserRepoToPerm, Permission, User
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

76
rhodecode/apps/repository/tests/test_repo_pullrequests.py
View file

@ -15,6 +15,9 @@
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
import os
import mock
import pytest
@ -41,7 +44,7 @@ from rhodecode.tests import (
TEST_USER_ADMIN_LOGIN,
TEST_USER_REGULAR_LOGIN,
)
from rhodecode.tests.fixture_mods.fixture_utils import PRTestUtility
from rhodecode.tests.fixtures.fixture_utils import PRTestUtility
from rhodecode.tests.routes import route_path
@ -1050,7 +1053,6 @@ class TestPullrequestsView(object):
)
assert len(notifications.all()) == 2
@pytest.mark.xfail(reason="unable to fix this test after python3 migration")
def test_create_pull_request_stores_ancestor_commit_id(self, backend, csrf_token):
commits = [
{
@ -1125,20 +1127,38 @@ class TestPullrequestsView(object):
response.mustcontain(no=["content_of_ancestor-child"])
response.mustcontain("content_of_change")
def test_merge_pull_request_enabled(self, pr_util, csrf_token):
# Clear any previous calls to rcextensions
rhodecode.EXTENSIONS.calls.clear()
def test_merge_pull_request_enabled(self, pr_util, csrf_token, rcextensions_modification):
pull_request = pr_util.create_pull_request(approved=True, mergeable=True)
pull_request_id = pull_request.pull_request_id
repo_name = (pull_request.target_repo.scm_instance().name,)
repo_name = pull_request.target_repo.scm_instance().name
url = route_path(
"pullrequest_merge",
repo_name=str(repo_name[0]),
repo_name=repo_name,
pull_request_id=pull_request_id,
)
response = self.app.post(url, params={"csrf_token": csrf_token}).follow()
rcstack_location = os.path.dirname(self.app._pyramid_registry.settings['__file__'])
rc_ext_location = os.path.join(rcstack_location, 'rcextension-output.txt')
mods = [
('_push_hook',
f"""
import os
action = kwargs['action']
commit_ids = kwargs['commit_ids']
with open('{rc_ext_location}', 'w') as f:
f.write('test-execution'+os.linesep)
f.write(f'{{action}}'+os.linesep)
f.write(f'{{commit_ids}}'+os.linesep)
return HookResponse(0, 'HOOK_TEST')
""")
]
# Add the hook
with rcextensions_modification(rcstack_location, mods, create_if_missing=True, force_create=True):
response = self.app.post(url, params={"csrf_token": csrf_token}).follow()
pull_request = PullRequest.get(pull_request_id)
@ -1162,12 +1182,39 @@ class TestPullrequestsView(object):
assert actions[-1].action == "user.push"
assert actions[-1].action_data["commit_ids"] == pr_commit_ids
# Check post_push rcextension was really executed
push_calls = rhodecode.EXTENSIONS.calls["_push_hook"]
assert len(push_calls) == 1
unused_last_call_args, last_call_kwargs = push_calls[0]
assert last_call_kwargs["action"] == "push"
assert last_call_kwargs["commit_ids"] == pr_commit_ids
with open(rc_ext_location) as f:
f_data = f.read()
assert 'test-execution' in f_data
for commit_id in pr_commit_ids:
assert f'{commit_id}' in f_data
def test_merge_pull_request_forbidden_by_pre_push_hook(self, pr_util, csrf_token, rcextensions_modification, caplog):
caplog.set_level(logging.WARNING, logger="rhodecode.model.pull_request")
pull_request = pr_util.create_pull_request(approved=True, mergeable=True)
pull_request_id = pull_request.pull_request_id
repo_name = pull_request.target_repo.scm_instance().name
url = route_path(
"pullrequest_merge",
repo_name=repo_name,
pull_request_id=pull_request_id,
)
rcstack_location = os.path.dirname(self.app._pyramid_registry.settings['__file__'])
mods = [
('_pre_push_hook',
f"""
return HookResponse(1, 'HOOK_TEST_FORBIDDEN')
""")
]
# Add the hook
with rcextensions_modification(rcstack_location, mods, create_if_missing=True, force_create=True):
self.app.post(url, params={"csrf_token": csrf_token})
assert 'Merge failed, not updating the pull request.' in [r[2] for r in caplog.record_tuples]
def test_merge_pull_request_disabled(self, pr_util, csrf_token):
pull_request = pr_util.create_pull_request(mergeable=False)
@ -1523,7 +1570,6 @@ class TestPullrequestsView(object):
assert pull_request.revisions == [commit_ids["change-rebased"]]
def test_remove_pull_request_branch(self, backend_git, csrf_token):
branch_name = "development"
commits = [

2
rhodecode/apps/repository/tests/test_repo_settings.py
View file

@ -26,7 +26,7 @@ from rhodecode.model.db import Repository, UserRepoToPerm, Permission, User
from rhodecode.model.meta import Session
from rhodecode.tests import (
TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_LOGIN, assert_session_flash)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/repository/tests/test_repo_settings_advanced.py
View file

@ -24,7 +24,7 @@ from rhodecode.model.db import Repository
from rhodecode.model.repo import RepoModel
from rhodecode.tests import (
HG_REPO, GIT_REPO, assert_session_flash, no_newline_id_generator)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.utils import repo_on_filesystem
from rhodecode.tests.routes import route_path

2
rhodecode/apps/repository/tests/test_repo_summary.py
View file

@ -31,7 +31,7 @@ from rhodecode.model.meta import Session
from rhodecode.model.repo import RepoModel
from rhodecode.model.scm import ScmModel
from rhodecode.tests import assert_session_flash
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.utils import AssertResponse, repo_on_filesystem
from rhodecode.tests.routes import route_path

2
rhodecode/apps/repository/tests/test_repo_vcs_settings.py
View file

@ -30,7 +30,7 @@ from rhodecode.model.user import UserModel
from rhodecode.tests import (
login_user_session, logout_user_session,
TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.utils import AssertResponse
from rhodecode.tests.routes import route_path

9
rhodecode/apps/repository/tests/test_vcs_settings.py
View file

@ -32,16 +32,13 @@ class TestAdminRepoVcsSettings(object):
@pytest.mark.parametrize('setting_name, setting_backends', [
('hg_use_rebase_for_merging', ['hg']),
])
def test_labs_settings_visible_if_enabled(
self, setting_name, setting_backends, backend):
def test_labs_settings_visible_if_enabled(self, setting_name, setting_backends, backend):
if backend.alias not in setting_backends:
pytest.skip('Setting not available for backend {}'.format(backend))
vcs_settings_url = route_path(
'edit_repo_vcs', repo_name=backend.repo.repo_name)
vcs_settings_url = route_path('edit_repo_vcs', repo_name=backend.repo.repo_name)
with mock.patch.dict(
rhodecode.CONFIG, {'labs_settings_active': 'true'}):
with mock.patch.dict(rhodecode.CONFIG, {'labs_settings_active': 'true'}):
response = self.app.get(vcs_settings_url)
assertr = response.assert_response()

6
rhodecode/apps/ssh_support/lib/backends/base.py
View file

@ -20,7 +20,7 @@ import os
import sys
import logging
from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
from rhodecode.lib.hook_daemon.utils import prepare_callback_daemon
from rhodecode.lib.ext_json import sjson as json
from rhodecode.lib.vcs.conf import settings as vcs_settings
from rhodecode.lib.api_utils import call_service_api
@ -162,9 +162,7 @@ class SshVcsServer(object):
extras = {}
extras.update(tunnel_extras)
callback_daemon, extras = prepare_callback_daemon(
extras, protocol=self.hooks_protocol,
host=vcs_settings.HOOKS_HOST)
callback_daemon, extras = prepare_callback_daemon(extras, protocol=self.hooks_protocol)
with callback_daemon:
try:

43
rhodecode/apps/ssh_support/tests/test_server_git.py
View file

@ -33,19 +33,24 @@ class GitServerCreator(object):
'app:main': {
'ssh.executable.git': git_path,
'vcs.hooks.protocol.v2': 'celery',
'app.service_api.host': 'http://localhost',
'app.service_api.token': 'secret4',
'rhodecode.api.url': '/_admin/api',
}
}
repo_name = 'test_git'
repo_mode = 'receive-pack'
user = plain_dummy_user()
def __init__(self):
pass
def __init__(self, service_api_url, ini_file):
self.service_api_url = service_api_url
self.ini_file = ini_file
def create(self, **kwargs):
self.config_data['app:main']['app.service_api.host'] = self.service_api_url
parameters = {
'store': self.root,
'ini_path': '',
'ini_path': self.ini_file,
'user': self.user,
'repo_name': self.repo_name,
'repo_mode': self.repo_mode,
@ -60,12 +65,30 @@ class GitServerCreator(object):
return server
@pytest.fixture()
def git_server(app):
return GitServerCreator()
@pytest.fixture(scope='module')
def git_server(request, module_app, rhodecode_factory, available_port_factory):
ini_file = module_app._pyramid_settings['__file__']
vcsserver_host = module_app._pyramid_settings['vcs.server']
store_dir = os.path.dirname(ini_file)
# start rhodecode for service API
rc = rhodecode_factory(
request,
store_dir=store_dir,
port=available_port_factory(),
overrides=(
{'handler_console': {'level': 'DEBUG'}},
{'app:main': {'vcs.server': vcsserver_host}},
{'app:main': {'repo_store.path': store_dir}}
))
service_api_url = f'http://{rc.bind_addr}'
return GitServerCreator(service_api_url, ini_file)
class TestGitServer(object):
class TestGitServer:
def test_command(self, git_server):
server = git_server.create()
@ -102,14 +125,14 @@ class TestGitServer(object):
assert result is value
def test_run_returns_executes_command(self, git_server):
server = git_server.create()
from rhodecode.apps.ssh_support.lib.backends.git import GitTunnelWrapper
server = git_server.create()
os.environ['SSH_CLIENT'] = '127.0.0.1'
with mock.patch.object(GitTunnelWrapper, 'create_hooks_env') as _patch:
_patch.return_value = 0
with mock.patch.object(GitTunnelWrapper, 'command', return_value='date'):
exit_code = server.run()
exit_code = server.run(tunnel_extras={'config': server.ini_path})
assert exit_code == (0, False)
@ -135,7 +158,7 @@ class TestGitServer(object):
'action': action,
'ip': '10.10.10.10',
'locked_by': [None, None],
'config': '',
'config': git_server.ini_file,
'repo_store': store,
'server_url': None,
'hooks': ['push', 'pull'],

42
rhodecode/apps/ssh_support/tests/test_server_hg.py
View file

@ -17,6 +17,7 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import mock
import pytest
@ -32,22 +33,27 @@ class MercurialServerCreator(object):
'app:main': {
'ssh.executable.hg': hg_path,
'vcs.hooks.protocol.v2': 'celery',
'app.service_api.host': 'http://localhost',
'app.service_api.token': 'secret4',
'rhodecode.api.url': '/_admin/api',
}
}
repo_name = 'test_hg'
user = plain_dummy_user()
def __init__(self):
pass
def __init__(self, service_api_url, ini_file):
self.service_api_url = service_api_url
self.ini_file = ini_file
def create(self, **kwargs):
self.config_data['app:main']['app.service_api.host'] = self.service_api_url
parameters = {
'store': self.root,
'ini_path': '',
'ini_path': self.ini_file,
'user': self.user,
'repo_name': self.repo_name,
'user_permissions': {
'test_hg': 'repository.admin'
self.repo_name: 'repository.admin'
},
'settings': self.config_data['app:main'],
'env': plain_dummy_env()
@ -57,12 +63,30 @@ class MercurialServerCreator(object):
return server
@pytest.fixture()
def hg_server(app):
return MercurialServerCreator()
@pytest.fixture(scope='module')
def hg_server(request, module_app, rhodecode_factory, available_port_factory):
ini_file = module_app._pyramid_settings['__file__']
vcsserver_host = module_app._pyramid_settings['vcs.server']
store_dir = os.path.dirname(ini_file)
# start rhodecode for service API
rc = rhodecode_factory(
request,
store_dir=store_dir,
port=available_port_factory(),
overrides=(
{'handler_console': {'level': 'DEBUG'}},
{'app:main': {'vcs.server': vcsserver_host}},
{'app:main': {'repo_store.path': store_dir}}
))
service_api_url = f'http://{rc.bind_addr}'
return MercurialServerCreator(service_api_url, ini_file)
class TestMercurialServer(object):
class TestMercurialServer:
def test_command(self, hg_server, tmpdir):
server = hg_server.create()
@ -107,7 +131,7 @@ class TestMercurialServer(object):
with mock.patch.object(MercurialTunnelWrapper, 'create_hooks_env') as _patch:
_patch.return_value = 0
with mock.patch.object(MercurialTunnelWrapper, 'command', return_value='date'):
exit_code = server.run()
exit_code = server.run(tunnel_extras={'config': server.ini_path})
assert exit_code == (0, False)

58
rhodecode/apps/ssh_support/tests/test_server_svn.py
View file

@ -15,7 +15,9 @@
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import mock
import pytest
@ -26,39 +28,62 @@ from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dum
class SubversionServerCreator(object):
root = '/tmp/repo/path/'
svn_path = '/usr/local/bin/svnserve'
config_data = {
'app:main': {
'ssh.executable.svn': svn_path,
'vcs.hooks.protocol.v2': 'celery',
'app.service_api.host': 'http://localhost',
'app.service_api.token': 'secret4',
'rhodecode.api.url': '/_admin/api',
}
}
repo_name = 'test-svn'
user = plain_dummy_user()
def __init__(self):
pass
def __init__(self, service_api_url, ini_file):
self.service_api_url = service_api_url
self.ini_file = ini_file
def create(self, **kwargs):
self.config_data['app:main']['app.service_api.host'] = self.service_api_url
parameters = {
'store': self.root,
'repo_name': self.repo_name,
'ini_path': '',
'ini_path': self.ini_file,
'user': self.user,
'repo_name': self.repo_name,
'user_permissions': {
self.repo_name: 'repository.admin'
},
'settings': self.config_data['app:main'],
'env': plain_dummy_env()
}
parameters.update(kwargs)
server = SubversionServer(**parameters)
return server
@pytest.fixture()
def svn_server(app):
return SubversionServerCreator()
@pytest.fixture(scope='module')
def svn_server(request, module_app, rhodecode_factory, available_port_factory):
ini_file = module_app._pyramid_settings['__file__']
vcsserver_host = module_app._pyramid_settings['vcs.server']
store_dir = os.path.dirname(ini_file)
# start rhodecode for service API
rc = rhodecode_factory(
request,
store_dir=store_dir,
port=available_port_factory(),
overrides=(
{'handler_console': {'level': 'DEBUG'}},
{'app:main': {'vcs.server': vcsserver_host}},
{'app:main': {'repo_store.path': store_dir}}
))
service_api_url = f'http://{rc.bind_addr}'
return SubversionServerCreator(service_api_url, ini_file)
class TestSubversionServer(object):
@ -168,8 +193,9 @@ class TestSubversionServer(object):
assert repo_name == expected_match
def test_run_returns_executes_command(self, svn_server):
server = svn_server.create()
from rhodecode.apps.ssh_support.lib.backends.svn import SubversionTunnelWrapper
server = svn_server.create()
os.environ['SSH_CLIENT'] = '127.0.0.1'
with mock.patch.object(
SubversionTunnelWrapper, 'get_first_client_response',
@ -184,20 +210,18 @@ class TestSubversionServer(object):
SubversionTunnelWrapper, 'command',
return_value=['date']):
exit_code = server.run()
exit_code = server.run(tunnel_extras={'config': server.ini_path})
# SVN has this differently configured, and we get in our mock env
# None as return code
assert exit_code == (None, False)
def test_run_returns_executes_command_that_cannot_extract_repo_name(self, svn_server):
server = svn_server.create()
from rhodecode.apps.ssh_support.lib.backends.svn import SubversionTunnelWrapper
with mock.patch.object(
SubversionTunnelWrapper, 'command',
return_value=['date']):
with mock.patch.object(
SubversionTunnelWrapper, 'get_first_client_response',
server = svn_server.create()
with mock.patch.object(SubversionTunnelWrapper, 'command', return_value=['date']):
with mock.patch.object(SubversionTunnelWrapper, 'get_first_client_response',
return_value=None):
exit_code = server.run()
exit_code = server.run(tunnel_extras={'config': server.ini_path})
assert exit_code == (1, False)

2
rhodecode/apps/user_group/tests/test_user_groups.py
View file

@ -22,7 +22,7 @@ from rhodecode.tests import (
TestController, assert_session_flash, TEST_USER_ADMIN_LOGIN)
from rhodecode.model.db import UserGroup
from rhodecode.model.meta import Session
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/user_group_profile/tests/test_user_group.py
View file

@ -18,7 +18,7 @@
from rhodecode.model.user_group import UserGroupModel
from rhodecode.tests import (
TestController, TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.routes import route_path
fixture = Fixture()

2
rhodecode/apps/user_profile/tests/test_users.py
View file

@ -22,7 +22,7 @@ from rhodecode.model.db import User
from rhodecode.tests import (
TestController, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.fixtures.rc_fixture import Fixture
from rhodecode.tests.utils import AssertResponse
from rhodecode.tests.routes import route_path

5
rhodecode/config/environment.py
View file

@ -30,7 +30,7 @@ from rhodecode.lib.vcs import connect_vcs
log = logging.getLogger(__name__)
def propagate_rhodecode_config(global_config, settings, config):
def propagate_rhodecode_config(global_config, settings, config, full=True):
# Store the settings to make them available to other modules.
settings_merged = global_config.copy()
settings_merged.update(settings)
@ -40,7 +40,7 @@ def propagate_rhodecode_config(global_config, settings, config):
rhodecode.PYRAMID_SETTINGS = settings_merged
rhodecode.CONFIG = settings_merged
if 'default_user_id' not in rhodecode.CONFIG:
if full and 'default_user_id' not in rhodecode.CONFIG:
rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
log.debug('set rhodecode.CONFIG data')
@ -93,6 +93,7 @@ def load_pyramid_environment(global_config, settings):
# first run, to store data...
propagate_rhodecode_config(global_config, settings, {})
if vcs_server_enabled:
connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
else:

3
rhodecode/config/middleware.py
View file

@ -101,6 +101,9 @@ def make_pyramid_app(global_config, **settings):
patches.inspect_getargspec()
patches.repoze_sendmail_lf_fix()
# first init, so load_pyramid_enviroment, can access some critical data, like __file__
propagate_rhodecode_config(global_config, {}, {}, full=False)
load_pyramid_environment(global_config, settings)
# Static file view comes first

2
rhodecode/config/rcextensions/__init__.py
View file

@ -17,7 +17,7 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
"""
rcextensions module, please edit `hooks.py` to over write hooks logic
rcextensions module, please edit `hooks.py` to over-write hooks logic
"""
from .hooks import (

2
rhodecode/config/rcextensions/examples/validate_pushed_files_name_and_size.py
View file

@ -85,7 +85,7 @@ def _pre_push_hook(*args, **kwargs):
# check files names
if forbidden_files:
reason = 'File {} is forbidden to be pushed'.format(file_name)
reason = f'File {file_name} is forbidden to be pushed'
for forbidden_pattern in forbid_files:
# here we can also filter for operation, e.g if check for only ADDED files
# if operation == 'A':

5
rhodecode/config/rcextensions/helpers/extra_fields.py
View file

@ -1,4 +1,3 @@
# Copyright (C) 2016-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
@ -55,7 +54,7 @@ def run(*args, **kwargs):
return fields
class _Undefined(object):
class _Undefined:
pass
@ -67,7 +66,7 @@ def get_field(extra_fields_data, key, default=_Undefined(), convert_type=True):
if key not in extra_fields_data:
if isinstance(default, _Undefined):
raise ValueError('key {} not present in extra_fields'.format(key))
raise ValueError(f'key {key} not present in extra_fields')
return default
# NOTE(dan): from metadata we get field_label, field_value, field_desc, field_type

1
rhodecode/config/rcextensions/helpers/extract_post_commits.py
View file

@ -1,4 +1,3 @@
# Copyright (C) 2016-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify

7
rhodecode/config/rcextensions/helpers/extract_pre_commits.py
View file

@ -1,4 +1,3 @@
# Copyright (C) 2016-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
@ -52,7 +51,7 @@ def get_git_commits(repo, refs):
cmd = [
'log',
'--pretty=format:{"commit_id": "%H", "author": "%aN <%aE>", "date": "%ad", "message": "%s"}',
'{}...{}'.format(old_rev, new_rev)
f'{old_rev}...{new_rev}'
]
stdout, stderr = repo.run_git_command(cmd, extra_env=git_env)
@ -80,12 +79,12 @@ def run(*args, **kwargs):
if vcs_type == 'git':
for rev_data in kwargs['commit_ids']:
new_environ = dict((k, v) for k, v in rev_data['git_env'])
new_environ = {k: v for k, v in rev_data['git_env']}
commits = get_git_commits(vcs_repo, kwargs['commit_ids'])
if vcs_type == 'hg':
for rev_data in kwargs['commit_ids']:
new_environ = dict((k, v) for k, v in rev_data['hg_env'])
new_environ = {k: v for k, v in rev_data['hg_env']}
commits = get_hg_commits(vcs_repo, kwargs['commit_ids'])
return commits

5
rhodecode/config/rcextensions/helpers/extract_pre_files.py
View file

@ -1,4 +1,3 @@
# Copyright (C) 2016-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
@ -133,12 +132,12 @@ def run(*args, **kwargs):
if vcs_type == 'git':
for rev_data in kwargs['commit_ids']:
new_environ = dict((k, v) for k, v in rev_data['git_env'])
new_environ = {k: v for k, v in rev_data['git_env']}
files = get_git_files(repo, vcs_repo, kwargs['commit_ids'])
if vcs_type == 'hg':
for rev_data in kwargs['commit_ids']:
new_environ = dict((k, v) for k, v in rev_data['hg_env'])
new_environ = {k: v for k, v in rev_data['hg_env']}
files = get_hg_files(repo, vcs_repo, kwargs['commit_ids'])
if vcs_type == 'svn':

1
rhodecode/config/rcextensions/helpers/http_call.py
View file

@ -1,4 +1,3 @@
# Copyright (C) 2016-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify

16
rhodecode/config/rcextensions/utils.py
View file

@ -28,7 +28,7 @@ import urllib.error
log = logging.getLogger('rhodecode.' + __name__)
class HookResponse(object):
class HookResponse:
def __init__(self, status, output):
self.status = status
self.output = output
@ -44,6 +44,11 @@ class HookResponse(object):
def __bool__(self):
return self.status == 0
def to_json(self):
return {'status': self.status, 'output': self.output}
def __repr__(self):
return self.to_json().__repr__()
class DotDict(dict):
@ -91,8 +96,8 @@ class DotDict(dict):
def __repr__(self):
keys = list(self.keys())
keys.sort()
args = ', '.join(['%s=%r' % (key, self[key]) for key in keys])
return '%s(%s)' % (self.__class__.__name__, args)
args = ', '.join(['{}={!r}'.format(key, self[key]) for key in keys])
return '{}({})'.format(self.__class__.__name__, args)
@staticmethod
def fromDict(d):
@ -110,7 +115,7 @@ def serialize(x):
def unserialize(x):
if isinstance(x, dict):
return dict((k, unserialize(v)) for k, v in x.items())
return {k: unserialize(v) for k, v in x.items()}
elif isinstance(x, (list, tuple)):
return type(x)(unserialize(v) for v in x)
else:
@ -161,7 +166,8 @@ def str2bool(_str) -> bool:
string into boolean
:param _str: string value to translate into boolean
:returns: bool from given string
:rtype: boolean
:returns: boolean from given string
"""
if _str is None:
return False

8
rhodecode/config/routing_links.py
View file

@ -49,22 +49,22 @@ link_config = [
{
"name": "enterprise_docs",
"target": "https://rhodecode.com/r1/enterprise/docs/",
"external_target": "https://docs.rhodecode.com/RhodeCode-Enterprise/",
"external_target": "https://docs.rhodecode.com/4.x/rce/index.html",
},
{
"name": "enterprise_log_file_locations",
"target": "https://rhodecode.com/r1/enterprise/docs/admin-system-overview/",
"external_target": "https://docs.rhodecode.com/RhodeCode-Enterprise/admin/system-overview.html#log-files",
"external_target": "https://docs.rhodecode.com/4.x/rce/admin/system-overview.html#log-files",
},
{
"name": "enterprise_issue_tracker_settings",
"target": "https://rhodecode.com/r1/enterprise/docs/issue-trackers-overview/",
"external_target": "https://docs.rhodecode.com/RhodeCode-Enterprise/issue-trackers/issue-trackers.html",
"external_target": "https://docs.rhodecode.com/4.x/rce/issue-trackers/issue-trackers.html",
},
{
"name": "enterprise_svn_setup",
"target": "https://rhodecode.com/r1/enterprise/docs/svn-setup/",
"external_target": "https://docs.rhodecode.com/RhodeCode-Enterprise/admin/svn-http.html",
"external_target": "https://docs.rhodecode.com/4.x/rce/admin/svn-http.html",
},
{
"name": "enterprise_license_convert_from_old",

27
rhodecode/config/utils.py
View file

@ -19,6 +19,8 @@
import os
import platform
from rhodecode.lib.type_utils import str2bool
DEFAULT_USER = 'default'
@ -48,28 +50,23 @@ def initialize_database(config):
engine = engine_from_config(config, 'sqlalchemy.db1.')
init_model(engine, encryption_key=get_encryption_key(config))
def initialize_test_environment(settings):
skip_test_env = str2bool(os.environ.get('RC_NO_TEST_ENV'))
if skip_test_env:
return
def initialize_test_environment(settings, test_env=None):
if test_env is None:
test_env = not int(os.environ.get('RC_NO_TMP_PATH', 0))
repo_store_path = os.environ.get('RC_TEST_ENV_REPO_STORE') or settings['repo_store.path']
from rhodecode.lib.utils import (
create_test_directory, create_test_database, create_test_repositories,
create_test_index)
from rhodecode.tests import TESTS_TMP_PATH
from rhodecode.lib.vcs.backends.hg import largefiles_store
from rhodecode.lib.vcs.backends.git import lfs_store
create_test_directory(repo_store_path)
create_test_database(repo_store_path, settings)
# test repos
if test_env:
create_test_directory(TESTS_TMP_PATH)
# large object stores
create_test_directory(largefiles_store(TESTS_TMP_PATH))
create_test_directory(lfs_store(TESTS_TMP_PATH))
create_test_database(TESTS_TMP_PATH, settings)
create_test_repositories(TESTS_TMP_PATH, settings)
create_test_index(TESTS_TMP_PATH, settings)
create_test_repositories(repo_store_path, settings)
create_test_index(repo_store_path, settings)
def get_vcs_server_protocol(config):

19
rhodecode/lib/exceptions.py
View file

@ -20,8 +20,7 @@
Set of custom exceptions used in RhodeCode
"""
from webob.exc import HTTPClientError
from pyramid.httpexceptions import HTTPBadGateway
from pyramid.httpexceptions import HTTPBadGateway, HTTPClientError
class LdapUsernameError(Exception):
@ -102,12 +101,7 @@ class HTTPRequirementError(HTTPClientError):
self.args = (message, )
class ClientNotSupportedError(HTTPRequirementError):
title = explanation = 'Client Not Supported'
reason = None
class HTTPLockedRC(HTTPClientError):
class HTTPLockedRepo(HTTPClientError):
"""
Special Exception For locked Repos in RhodeCode, the return code can
be overwritten by _code keyword argument passed into constructors
@ -131,14 +125,13 @@ class HTTPBranchProtected(HTTPClientError):
Special Exception For Indicating that branch is protected in RhodeCode, the
return code can be overwritten by _code keyword argument passed into constructors
"""
code = 403
title = explanation = 'Branch Protected'
reason = None
def __init__(self, message, *args, **kwargs):
self.title = self.explanation = message
super().__init__(*args, **kwargs)
self.args = (message, )
class ClientNotSupported(HTTPRequirementError):
title = explanation = 'Client Not Supported'
reason = None
class IMCCommitError(Exception):

120
rhodecode/lib/hook_daemon/base.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
# Copyright (C) 2010-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
@ -16,13 +16,14 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import time
import logging
import traceback
from rhodecode.lib.config_utils import get_app_config_lightweight
from rhodecode.model import meta
from rhodecode.lib import hooks_base
from rhodecode.lib.utils2 import AttributeDict
from rhodecode.lib.exceptions import HTTPLockedRepo, HTTPBranchProtected
from rhodecode.lib.svn_txn_utils import get_txn_id_from_store
log = logging.getLogger(__name__)
@ -42,53 +43,82 @@ class BaseHooksCallbackDaemon:
log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
class HooksModuleCallbackDaemon(BaseHooksCallbackDaemon):
class Hooks(object):
"""
Exposes the hooks module for calling them using the local HooksModuleCallbackDaemon
"""
def __init__(self, request=None, log_prefix=''):
self.log_prefix = log_prefix
self.request = request
def __init__(self, module):
super().__init__()
self.hooks_module = module
def repo_size(self, extras):
log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.repo_size, extras)
def __repr__(self):
return f'HooksModuleCallbackDaemon(hooks_module={self.hooks_module})'
def pre_pull(self, extras):
log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.pre_pull, extras)
def post_pull(self, extras):
log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.post_pull, extras)
def prepare_callback_daemon(extras, protocol, host, txn_id=None):
def pre_push(self, extras):
log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.pre_push, extras)
match protocol:
case 'http':
from rhodecode.lib.hook_daemon.http_hooks_deamon import HttpHooksCallbackDaemon
port = 0
if txn_id:
# read txn-id to re-use the PORT for callback daemon
repo_path = os.path.join(extras['repo_store'], extras['repository'])
txn_details = get_txn_id_from_store(repo_path, txn_id)
port = txn_details.get('port', 0)
def post_push(self, extras):
log.debug("%sCalled post_push of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.post_push, extras)
callback_daemon = HttpHooksCallbackDaemon(
txn_id=txn_id, host=host, port=port)
case 'celery':
from rhodecode.lib.hook_daemon.celery_hooks_deamon import CeleryHooksCallbackDaemon
def _call_hook(self, hook, extras):
extras = AttributeDict(extras)
_server_url = extras['server_url']
config = get_app_config_lightweight(extras['config'])
task_queue = config.get('celery.broker_url')
task_backend = config.get('celery.result_backend')
extras.request = self.request
try:
result = hook(extras)
if result is None:
raise Exception(f'Failed to obtain hook result from func: {hook}')
except HTTPBranchProtected as error:
# Those special cases don't need error reporting. It's a case of
# locked repo or protected branch
result = AttributeDict({
'status': error.code,
'output': error.explanation
})
except HTTPLockedRepo as error:
# Those special cases don't need error reporting. It's a case of
# locked repo or protected branch
result = AttributeDict({
'status': error.code,
'output': error.explanation
})
except Exception as error:
# locked needs different handling since we need to also
# handle PULL operations
log.exception('%sException when handling hook %s', self.log_prefix, hook)
exc_tb = traceback.format_exc()
error_args = error.args
return {
'status': 128,
'output': '',
'exception': type(error).__name__,
'exception_traceback': exc_tb,
'exception_args': error_args,
}
finally:
meta.Session.remove()
callback_daemon = CeleryHooksCallbackDaemon(task_queue, task_backend)
case 'local':
from rhodecode.lib.hook_daemon.hook_module import Hooks
callback_daemon = HooksModuleCallbackDaemon(Hooks.__module__)
case _:
log.error('Unsupported callback daemon protocol "%s"', protocol)
raise Exception('Unsupported callback daemon protocol.')
log.debug('%sGot hook call response %s', self.log_prefix, result)
return {
'status': result.status,
'output': result.output,
}
extras['hooks_uri'] = getattr(callback_daemon, 'hooks_uri', '')
extras['task_queue'] = getattr(callback_daemon, 'task_queue', '')
extras['task_backend'] = getattr(callback_daemon, 'task_backend', '')
extras['hooks_protocol'] = protocol
extras['time'] = time.time()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
# register txn_id
extras['txn_id'] = txn_id
log.debug('Prepared a callback daemon: %s',
callback_daemon.__class__.__name__)
return callback_daemon, extras

16
rhodecode/lib/hook_daemon/celery_hooks_deamon.py
View file

@ -22,14 +22,16 @@ from rhodecode.lib.hook_daemon.base import BaseHooksCallbackDaemon
class CeleryHooksCallbackDaemon(BaseHooksCallbackDaemon):
"""
Context manger for achieving a compatibility with celery backend
It is calling a call to vcsserver, where it uses HooksCeleryClient to actually call a task from
f'rhodecode.lib.celerylib.tasks.{method}'
"""
def __init__(self, task_queue, task_backend):
self.task_queue = task_queue
self.task_backend = task_backend
def __init__(self, broker_url, result_backend):
super().__init__()
self.broker_url = broker_url
self.result_backend = result_backend
def __repr__(self):
return f'CeleryHooksCallbackDaemon(task_queue={self.task_queue}, task_backend={self.task_backend})'
def __repr__(self):
return f'CeleryHooksCallbackDaemon(task_queue={self.task_queue}, task_backend={self.task_backend})'
return f'CeleryHooksCallbackDaemon(broker_url={self.broker_url}, result_backend={self.result_backend})'

84
rhodecode/lib/hook_daemon/hook_module.py
View file

@ -17,88 +17,18 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
import traceback
from rhodecode.model import meta
from rhodecode.lib import hooks_base
from rhodecode.lib.exceptions import HTTPLockedRC, HTTPBranchProtected
from rhodecode.lib.utils2 import AttributeDict
from rhodecode.lib.hook_daemon.base import BaseHooksCallbackDaemon
log = logging.getLogger(__name__)
class Hooks(object):
"""
Exposes the hooks for remote callbacks
"""
def __init__(self, request=None, log_prefix=''):
self.log_prefix = log_prefix
self.request = request
class HooksModuleCallbackDaemon(BaseHooksCallbackDaemon):
def repo_size(self, extras):
log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.repo_size, extras)
def __init__(self, module):
super().__init__()
self.hooks_module = module
def pre_pull(self, extras):
log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.pre_pull, extras)
def __repr__(self):
return f'HooksModuleCallbackDaemon(hooks_module={self.hooks_module})'
def post_pull(self, extras):
log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.post_pull, extras)
def pre_push(self, extras):
log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.pre_push, extras)
def post_push(self, extras):
log.debug("%sCalled post_push of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.post_push, extras)
def _call_hook(self, hook, extras):
extras = AttributeDict(extras)
_server_url = extras['server_url']
extras.request = self.request
try:
result = hook(extras)
if result is None:
raise Exception(f'Failed to obtain hook result from func: {hook}')
except HTTPBranchProtected as error:
# Those special cases don't need error reporting. It's a case of
# locked repo or protected branch
result = AttributeDict({
'status': error.code,
'output': error.explanation
})
except (HTTPLockedRC, Exception) as error:
# locked needs different handling since we need to also
# handle PULL operations
exc_tb = ''
if not isinstance(error, HTTPLockedRC):
exc_tb = traceback.format_exc()
log.exception('%sException when handling hook %s', self.log_prefix, hook)
error_args = error.args
return {
'status': 128,
'output': '',
'exception': type(error).__name__,
'exception_traceback': exc_tb,
'exception_args': error_args,
}
finally:
meta.Session.remove()
log.debug('%sGot hook call response %s', self.log_prefix, result)
return {
'status': result.status,
'output': result.output,
}
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass

287
rhodecode/lib/hook_daemon/http_hooks_deamon.py
View file

@ -1,287 +0,0 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import logging
import traceback
import threading
import socket
import msgpack
import gevent
from http.server import BaseHTTPRequestHandler
from socketserver import TCPServer
from rhodecode.model import meta
from rhodecode.lib.ext_json import json
from rhodecode.lib import rc_cache
from rhodecode.lib.svn_txn_utils import get_txn_id_data_key
from rhodecode.lib.hook_daemon.hook_module import Hooks
log = logging.getLogger(__name__)
class HooksHttpHandler(BaseHTTPRequestHandler):
JSON_HOOKS_PROTO = 'json.v1'
MSGPACK_HOOKS_PROTO = 'msgpack.v1'
# starting with RhodeCode 5.0.0 MsgPack is the default, prior it used json
DEFAULT_HOOKS_PROTO = MSGPACK_HOOKS_PROTO
@classmethod
def serialize_data(cls, data, proto=DEFAULT_HOOKS_PROTO):
if proto == cls.MSGPACK_HOOKS_PROTO:
return msgpack.packb(data)
return json.dumps(data)
@classmethod
def deserialize_data(cls, data, proto=DEFAULT_HOOKS_PROTO):
if proto == cls.MSGPACK_HOOKS_PROTO:
return msgpack.unpackb(data)
return json.loads(data)
def do_POST(self):
hooks_proto, method, extras = self._read_request()
log.debug('Handling HooksHttpHandler %s with %s proto', method, hooks_proto)
txn_id = getattr(self.server, 'txn_id', None)
if txn_id:
log.debug('Computing TXN_ID based on `%s`:`%s`',
extras['repository'], extras['txn_id'])
computed_txn_id = rc_cache.utils.compute_key_from_params(
extras['repository'], extras['txn_id'])
if txn_id != computed_txn_id:
raise Exception(
'TXN ID fail: expected {} got {} instead'.format(
txn_id, computed_txn_id))
request = getattr(self.server, 'request', None)
try:
hooks = Hooks(request=request, log_prefix='HOOKS: {} '.format(self.server.server_address))
result = self._call_hook_method(hooks, method, extras)
except Exception as e:
exc_tb = traceback.format_exc()
result = {
'exception': e.__class__.__name__,
'exception_traceback': exc_tb,
'exception_args': e.args
}
self._write_response(hooks_proto, result)
def _read_request(self):
length = int(self.headers['Content-Length'])
# respect sent headers, fallback to OLD proto for compatability
hooks_proto = self.headers.get('rc-hooks-protocol') or self.JSON_HOOKS_PROTO
if hooks_proto == self.MSGPACK_HOOKS_PROTO:
# support for new vcsserver msgpack based protocol hooks
body = self.rfile.read(length)
data = self.deserialize_data(body)
else:
body = self.rfile.read(length)
data = self.deserialize_data(body)
return hooks_proto, data['method'], data['extras']
def _write_response(self, hooks_proto, result):
self.send_response(200)
if hooks_proto == self.MSGPACK_HOOKS_PROTO:
self.send_header("Content-type", "application/msgpack")
self.end_headers()
data = self.serialize_data(result)
self.wfile.write(data)
else:
self.send_header("Content-type", "text/json")
self.end_headers()
data = self.serialize_data(result)
self.wfile.write(data)
def _call_hook_method(self, hooks, method, extras):
try:
result = getattr(hooks, method)(extras)
finally:
meta.Session.remove()
return result
def log_message(self, format, *args):
"""
This is an overridden method of BaseHTTPRequestHandler which logs using
a logging library instead of writing directly to stderr.
"""
message = format % args
log.debug(
"HOOKS: client=%s - - [%s] %s", self.client_address,
self.log_date_time_string(), message)
class ThreadedHookCallbackDaemon(object):
_callback_thread = None
_daemon = None
_done = False
use_gevent = False
def __init__(self, txn_id=None, host=None, port=None):
self._prepare(txn_id=txn_id, host=host, port=port)
if self.use_gevent:
self._run_func = self._run_gevent
self._stop_func = self._stop_gevent
else:
self._run_func = self._run
self._stop_func = self._stop
def __enter__(self):
log.debug('Running `%s` callback daemon', self.__class__.__name__)
self._run_func()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
self._stop_func()
def _prepare(self, txn_id=None, host=None, port=None):
raise NotImplementedError()
def _run(self):
raise NotImplementedError()
def _stop(self):
raise NotImplementedError()
def _run_gevent(self):
raise NotImplementedError()
def _stop_gevent(self):
raise NotImplementedError()
class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
"""
Context manager which will run a callback daemon in a background thread.
"""
hooks_uri = None
# From Python docs: Polling reduces our responsiveness to a shutdown
# request and wastes cpu at all other times.
POLL_INTERVAL = 0.01
use_gevent = False
def __repr__(self):
return f'HttpHooksCallbackDaemon(hooks_uri={self.hooks_uri})'
@property
def _hook_prefix(self):
return f'HOOKS: {self.hooks_uri} '
def get_hostname(self):
return socket.gethostname() or '127.0.0.1'
def get_available_port(self, min_port=20000, max_port=65535):
from rhodecode.lib.utils2 import get_available_port as _get_port
return _get_port(min_port, max_port)
def _prepare(self, txn_id=None, host=None, port=None):
from pyramid.threadlocal import get_current_request
if not host or host == "*":
host = self.get_hostname()
if not port:
port = self.get_available_port()
server_address = (host, port)
self.hooks_uri = f'{host}:{port}'
self.txn_id = txn_id
self._done = False
log.debug(
"%s Preparing HTTP callback daemon registering hook object: %s",
self._hook_prefix, HooksHttpHandler)
self._daemon = TCPServer(server_address, HooksHttpHandler)
# inject transaction_id for later verification
self._daemon.txn_id = self.txn_id
# pass the WEB app request into daemon
self._daemon.request = get_current_request()
def _run(self):
log.debug("Running thread-based loop of callback daemon in background")
callback_thread = threading.Thread(
target=self._daemon.serve_forever,
kwargs={'poll_interval': self.POLL_INTERVAL})
callback_thread.daemon = True
callback_thread.start()
self._callback_thread = callback_thread
def _run_gevent(self):
log.debug("Running gevent-based loop of callback daemon in background")
# create a new greenlet for the daemon's serve_forever method
callback_greenlet = gevent.spawn(
self._daemon.serve_forever,
poll_interval=self.POLL_INTERVAL)
# store reference to greenlet
self._callback_greenlet = callback_greenlet
# switch to this greenlet
gevent.sleep(0.01)
def _stop(self):
log.debug("Waiting for background thread to finish.")
self._daemon.shutdown()
self._callback_thread.join()
self._daemon = None
self._callback_thread = None
if self.txn_id:
#TODO: figure out the repo_path...
repo_path = ''
txn_id_file = get_txn_id_data_key(repo_path, self.txn_id)
log.debug('Cleaning up TXN ID %s', txn_id_file)
if os.path.isfile(txn_id_file):
os.remove(txn_id_file)
log.debug("Background thread done.")
def _stop_gevent(self):
log.debug("Waiting for background greenlet to finish.")
# if greenlet exists and is running
if self._callback_greenlet and not self._callback_greenlet.dead:
# shutdown daemon if it exists
if self._daemon:
self._daemon.shutdown()
# kill the greenlet
self._callback_greenlet.kill()
self._daemon = None
self._callback_greenlet = None
if self.txn_id:
#TODO: figure out the repo_path...
repo_path = ''
txn_id_file = get_txn_id_data_key(repo_path, self.txn_id)
log.debug('Cleaning up TXN ID %s', txn_id_file)
if os.path.isfile(txn_id_file):
os.remove(txn_id_file)
log.debug("Background greenlet done.")

61
rhodecode/lib/hook_daemon/utils.py Normal file
View file

@ -0,0 +1,61 @@
# Copyright (C) 2010-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import time
import logging
from rhodecode.lib.config_utils import get_app_config_lightweight
from rhodecode.lib.hook_daemon.base import Hooks
from rhodecode.lib.hook_daemon.hook_module import HooksModuleCallbackDaemon
from rhodecode.lib.hook_daemon.celery_hooks_deamon import CeleryHooksCallbackDaemon
from rhodecode.lib.type_utils import str2bool
log = logging.getLogger(__name__)
def prepare_callback_daemon(extras, protocol: str, txn_id=None):
hooks_config = {}
match protocol:
case 'celery':
config = get_app_config_lightweight(extras['config'])
broker_url = config.get('celery.broker_url')
result_backend = config.get('celery.result_backend')
hooks_config = {
'broker_url': broker_url,
'result_backend': result_backend,
}
callback_daemon = CeleryHooksCallbackDaemon(broker_url, result_backend)
case 'local':
callback_daemon = HooksModuleCallbackDaemon(Hooks.__module__)
case _:
log.error('Unsupported callback daemon protocol "%s"', protocol)
raise Exception('Unsupported callback daemon protocol.')
extras['hooks_config'] = hooks_config
extras['hooks_protocol'] = protocol
extras['time'] = time.time()
# register txn_id
extras['txn_id'] = txn_id
log.debug('Prepared a callback daemon: %s', callback_daemon.__class__.__name__)
return callback_daemon, extras

240
rhodecode/lib/hooks_base.py
View file

@ -30,14 +30,14 @@ from rhodecode.lib import helpers as h
from rhodecode.lib import audit_logger
from rhodecode.lib.utils2 import safe_str, user_agent_normalizer
from rhodecode.lib.exceptions import (
HTTPLockedRC, HTTPBranchProtected, UserCreationError, ClientNotSupportedError)
HTTPLockedRepo, HTTPBranchProtected, UserCreationError, ClientNotSupported)
from rhodecode.model.db import Repository, User
from rhodecode.lib.statsd_client import StatsdClient
log = logging.getLogger(__name__)
class HookResponse(object):
class HookResponse:
def __init__(self, status, output):
self.status = status
self.output = output
@ -56,6 +56,8 @@ class HookResponse(object):
def to_json(self):
return {'status': self.status, 'output': self.output}
def __repr__(self):
return self.to_json().__repr__()
def is_shadow_repo(extras):
"""
@ -73,8 +75,69 @@ def check_vcs_client(extras):
except ModuleNotFoundError:
is_vcs_client_whitelisted = lambda *x: True
backend = extras.get('scm')
if not is_vcs_client_whitelisted(extras.get('user_agent'), backend):
raise ClientNotSupportedError(f"Your {backend} client is forbidden")
user_agent = extras.get('user_agent')
if not is_vcs_client_whitelisted(user_agent, backend):
raise ClientNotSupported(f"Your {backend} client (version={user_agent}) is forbidden by security rules")
def check_locked_repo(extras, check_same_user=True):
user = User.get_by_username(extras.username)
output = ''
if extras.locked_by[0] and (not check_same_user or user.user_id != extras.locked_by[0]):
locked_by = User.get(extras.locked_by[0]).username
reason = extras.locked_by[2]
# this exception is interpreted in git/hg middlewares and based
# on that proper return code is server to client
_http_ret = HTTPLockedRepo(_locked_by_explanation(extras.repository, locked_by, reason))
if str(_http_ret.code).startswith('2'):
# 2xx Codes don't raise exceptions
output = _http_ret.title
else:
raise _http_ret
return output
def check_branch_protected(extras):
if extras.commit_ids and extras.check_branch_perms:
user = User.get_by_username(extras.username)
auth_user = user.AuthUser()
repo = Repository.get_by_repo_name(extras.repository)
if not repo:
raise ValueError(f'Repo for {extras.repository} not found')
affected_branches = []
if repo.repo_type == 'hg':
for entry in extras.commit_ids:
if entry['type'] == 'branch':
is_forced = bool(entry['multiple_heads'])
affected_branches.append([entry['name'], is_forced])
elif repo.repo_type == 'git':
for entry in extras.commit_ids:
if entry['type'] == 'heads':
is_forced = bool(entry['pruned_sha'])
affected_branches.append([entry['name'], is_forced])
for branch_name, is_forced in affected_branches:
rule, branch_perm = auth_user.get_rule_and_branch_permission(extras.repository, branch_name)
if not branch_perm:
# no branch permission found for this branch, just keep checking
continue
if branch_perm == 'branch.push_force':
continue
elif branch_perm == 'branch.push' and is_forced is False:
continue
elif branch_perm == 'branch.push' and is_forced is True:
halt_message = f'Branch `{branch_name}` changes rejected by rule {rule}. ' \
f'FORCE PUSH FORBIDDEN.'
else:
halt_message = f'Branch `{branch_name}` changes rejected by rule {rule}.'
if halt_message:
_http_ret = HTTPBranchProtected(halt_message)
raise _http_ret
def _get_scm_size(alias, root_path):
@ -109,116 +172,30 @@ def repo_size(extras):
repo = Repository.get_by_repo_name(extras.repository)
vcs_part = f'.{repo.repo_type}'
size_vcs, size_root, size_total = _get_scm_size(vcs_part, repo.repo_full_path)
msg = (f'RhodeCode: `{repo.repo_name}` size summary {vcs_part}:{size_vcs} repo:{size_root} total:{size_total}\n')
msg = f'RhodeCode: `{repo.repo_name}` size summary {vcs_part}:{size_vcs} repo:{size_root} total:{size_total}\n'
return HookResponse(0, msg)
def pre_push(extras):
"""
Hook executed before pushing code.
It bans pushing when the repository is locked.
"""
check_vcs_client(extras)
user = User.get_by_username(extras.username)
output = ''
if extras.locked_by[0] and user.user_id != int(extras.locked_by[0]):
locked_by = User.get(extras.locked_by[0]).username
reason = extras.locked_by[2]
# this exception is interpreted in git/hg middlewares and based
# on that proper return code is server to client
_http_ret = HTTPLockedRC(
_locked_by_explanation(extras.repository, locked_by, reason))
if str(_http_ret.code).startswith('2'):
# 2xx Codes don't raise exceptions
output = _http_ret.title
else:
raise _http_ret
hook_response = ''
if not is_shadow_repo(extras):
if extras.commit_ids and extras.check_branch_perms:
auth_user = user.AuthUser()
repo = Repository.get_by_repo_name(extras.repository)
if not repo:
raise ValueError(f'Repo for {extras.repository} not found')
affected_branches = []
if repo.repo_type == 'hg':
for entry in extras.commit_ids:
if entry['type'] == 'branch':
is_forced = bool(entry['multiple_heads'])
affected_branches.append([entry['name'], is_forced])
elif repo.repo_type == 'git':
for entry in extras.commit_ids:
if entry['type'] == 'heads':
is_forced = bool(entry['pruned_sha'])
affected_branches.append([entry['name'], is_forced])
for branch_name, is_forced in affected_branches:
rule, branch_perm = auth_user.get_rule_and_branch_permission(
extras.repository, branch_name)
if not branch_perm:
# no branch permission found for this branch, just keep checking
continue
if branch_perm == 'branch.push_force':
continue
elif branch_perm == 'branch.push' and is_forced is False:
continue
elif branch_perm == 'branch.push' and is_forced is True:
halt_message = f'Branch `{branch_name}` changes rejected by rule {rule}. ' \
f'FORCE PUSH FORBIDDEN.'
else:
halt_message = f'Branch `{branch_name}` changes rejected by rule {rule}.'
if halt_message:
_http_ret = HTTPBranchProtected(halt_message)
raise _http_ret
# Propagate to external components. This is done after checking the
# lock, for consistent behavior.
hook_response = pre_push_extension(
repo_store_path=Repository.base_path(), **extras)
events.trigger(events.RepoPrePushEvent(
repo_name=extras.repository, extras=extras))
return HookResponse(0, output) + hook_response
def pre_pull(extras):
"""
Hook executed before pulling the code.
It bans pulling when the repository is locked.
It bans pulling when incorrect client is used.
"""
check_vcs_client(extras)
output = ''
if extras.locked_by[0]:
locked_by = User.get(extras.locked_by[0]).username
reason = extras.locked_by[2]
# this exception is interpreted in git/hg middlewares and based
# on that proper return code is server to client
_http_ret = HTTPLockedRC(
_locked_by_explanation(extras.repository, locked_by, reason))
if str(_http_ret.code).startswith('2'):
# 2xx Codes don't raise exceptions
output = _http_ret.title
else:
raise _http_ret
check_vcs_client(extras)
# locking repo can, but not have to stop the operation it can also just produce output
output += check_locked_repo(extras, check_same_user=False)
# Propagate to external components. This is done after checking the
# lock, for consistent behavior.
hook_response = ''
if not is_shadow_repo(extras):
extras.hook_type = extras.hook_type or 'pre_pull'
hook_response = pre_pull_extension(
repo_store_path=Repository.base_path(), **extras)
events.trigger(events.RepoPrePullEvent(
repo_name=extras.repository, extras=extras))
hook_response = pre_pull_extension(repo_store_path=Repository.base_path(), **extras)
events.trigger(events.RepoPrePullEvent(repo_name=extras.repository, extras=extras))
return HookResponse(0, output) + hook_response
@ -239,6 +216,7 @@ def post_pull(extras):
statsd.incr('rhodecode_pull_total', tags=[
f'user-agent:{user_agent_normalizer(extras.user_agent)}',
])
output = ''
# make lock is a tri state False, True, None. We only make lock on True
if extras.make_lock is True and not is_shadow_repo(extras):
@ -246,18 +224,9 @@ def post_pull(extras):
Repository.lock(Repository.get_by_repo_name(extras.repository),
user.user_id,
lock_reason=Repository.LOCK_PULL)
msg = 'Made lock on repo `{}`'.format(extras.repository)
msg = f'Made lock on repo `{extras.repository}`'
output += msg
if extras.locked_by[0]:
locked_by = User.get(extras.locked_by[0]).username
reason = extras.locked_by[2]
_http_ret = HTTPLockedRC(
_locked_by_explanation(extras.repository, locked_by, reason))
if str(_http_ret.code).startswith('2'):
# 2xx Codes don't raise exceptions
output += _http_ret.title
# Propagate to external components.
hook_response = ''
if not is_shadow_repo(extras):
@ -270,6 +239,33 @@ def post_pull(extras):
return HookResponse(0, output) + hook_response
def pre_push(extras):
"""
Hook executed before pushing code.
It bans pushing when the repository is locked.
It banks pushing when incorrect client is used.
It also checks for Branch protection
"""
output = ''
check_vcs_client(extras)
# locking repo can, but not have to stop the operation it can also just produce output
output += check_locked_repo(extras)
hook_response = ''
if not is_shadow_repo(extras):
check_branch_protected(extras)
# Propagate to external components. This is done after checking the
# lock, for consistent behavior.
hook_response = pre_push_extension(repo_store_path=Repository.base_path(), **extras)
events.trigger(events.RepoPrePushEvent(repo_name=extras.repository, extras=extras))
return HookResponse(0, output) + hook_response
def post_push(extras):
"""Hook executed after user pushes to the repository."""
commit_ids = extras.commit_ids
@ -292,22 +288,13 @@ def post_push(extras):
# Propagate to external components.
output = ''
# make lock is a tri state False, True, None. We only release lock on False
if extras.make_lock is False and not is_shadow_repo(extras):
Repository.unlock(Repository.get_by_repo_name(extras.repository))
msg = f'Released lock on repo `{extras.repository}`\n'
output += msg
if extras.locked_by[0]:
locked_by = User.get(extras.locked_by[0]).username
reason = extras.locked_by[2]
_http_ret = HTTPLockedRC(
_locked_by_explanation(extras.repository, locked_by, reason))
# TODO: johbo: if not?
if str(_http_ret.code).startswith('2'):
# 2xx Codes don't raise exceptions
output += _http_ret.title
if extras.new_refs:
tmpl = '{}/{}/pull-request/new?{{ref_type}}={{ref_name}}'.format(
safe_str(extras.server_url), safe_str(extras.repository))
@ -322,11 +309,8 @@ def post_push(extras):
hook_response = ''
if not is_shadow_repo(extras):
hook_response = post_push_extension(
repo_store_path=Repository.base_path(),
**extras)
events.trigger(events.RepoPushEvent(
repo_name=extras.repository, pushed_commit_ids=commit_ids, extras=extras))
hook_response = post_push_extension(repo_store_path=Repository.base_path(), **extras)
events.trigger(events.RepoPushEvent(repo_name=extras.repository, pushed_commit_ids=commit_ids, extras=extras))
output += 'RhodeCode: push completed\n'
return HookResponse(0, output) + hook_response
@ -380,12 +364,20 @@ class ExtensionCallback(object):
# with older rcextensions that require api_key present
if self._hook_name in ['CREATE_USER_HOOK', 'DELETE_USER_HOOK']:
kwargs_to_pass['api_key'] = '_DEPRECATED_'
return callback(**kwargs_to_pass)
result = callback(**kwargs_to_pass)
log.debug('got rcextensions result: %s', result)
return result
def is_active(self):
return hasattr(rhodecode.EXTENSIONS, self._hook_name)
def _get_callback(self):
if rhodecode.is_test:
log.debug('In test mode, reloading rcextensions...')
# NOTE: for test re-load rcextensions always so we can dynamically change them for testing purposes
from rhodecode.lib.utils import load_rcextensions
load_rcextensions(root_path=os.path.dirname(rhodecode.CONFIG['__file__']))
return getattr(rhodecode.EXTENSIONS, self._hook_name, None)
return getattr(rhodecode.EXTENSIONS, self._hook_name, None)

12
rhodecode/lib/middleware/simplegit.py
View file

@ -40,16 +40,6 @@ GIT_PROTO_PAT = re.compile(
GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
def default_lfs_store():
"""
Default lfs store location, it's consistent with Mercurials large file
store which is in .cache/largefiles
"""
from rhodecode.lib.vcs.backends.git import lfs_store
user_home = os.path.expanduser("~")
return lfs_store(user_home)
class SimpleGit(simplevcs.SimpleVCS):
SCM = 'git'
@ -151,6 +141,6 @@ class SimpleGit(simplevcs.SimpleVCS):
extras['git_lfs_enabled'] = utils2.str2bool(
config.get('vcs_git_lfs', 'enabled'))
extras['git_lfs_store_path'] = custom_store or default_lfs_store()
extras['git_lfs_store_path'] = custom_store
extras['git_lfs_http_scheme'] = scheme
return extras

309
rhodecode/lib/middleware/simplevcs.py
View file

@ -1,5 +1,3 @@
# Copyright (C) 2014-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
@ -32,8 +30,7 @@ from functools import wraps
import time
from paste.httpheaders import REMOTE_USER, AUTH_TYPE
from pyramid.httpexceptions import (
HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
from pyramid.httpexceptions import HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError
from zope.cachedescriptors.property import Lazy as LazyProperty
import rhodecode
@ -41,10 +38,9 @@ from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
from rhodecode.lib import rc_cache
from rhodecode.lib.svn_txn_utils import store_txn_id_data
from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
from rhodecode.lib.base import (
BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
from rhodecode.lib.base import BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context
from rhodecode.lib.exceptions import UserCreationError, NotAllowedToCreateUserError
from rhodecode.lib.hook_daemon.utils import prepare_callback_daemon
from rhodecode.lib.middleware import appenlight
from rhodecode.lib.middleware.utils import scm_app_http
from rhodecode.lib.str_utils import safe_bytes, safe_int
@ -78,17 +74,18 @@ def initialize_generator(factory):
try:
init = next(gen)
except StopIteration:
raise ValueError('Generator must yield at least one element.')
raise ValueError("Generator must yield at least one element.")
if init != "__init__":
raise ValueError('First yielded element must be "__init__".')
return gen
return wrapper
class SimpleVCS(object):
"""Common functionality for SCM HTTP handlers."""
SCM = 'unknown'
SCM = "unknown"
acl_repo_name = None
url_repo_name = None
@ -100,11 +97,11 @@ class SimpleVCS(object):
# we use this regex which will match only on URLs pointing to shadow
# repositories.
shadow_repo_re = re.compile(
'(?P<groups>(?:{slug_pat}/)*)' # repo groups
'(?P<target>{slug_pat})/' # target repo
'pull-request/(?P<pr_id>\\d+)/' # pull request
'repository$' # shadow repo
.format(slug_pat=SLUG_RE.pattern))
"(?P<groups>(?:{slug_pat}/)*)" # repo groups
"(?P<target>{slug_pat})/" # target repo
"pull-request/(?P<pr_id>\\d+)/" # pull request
"repository$".format(slug_pat=SLUG_RE.pattern) # shadow repo
)
def __init__(self, config, registry):
self.registry = registry
@ -113,15 +110,14 @@ class SimpleVCS(object):
self.repo_vcs_config = base.Config()
rc_settings = SettingsModel().get_all_settings(cache=True, from_request=False)
realm = rc_settings.get('rhodecode_realm') or 'RhodeCode AUTH'
realm = rc_settings.get("rhodecode_realm") or "RhodeCode AUTH"
# authenticate this VCS request using authfunc
auth_ret_code_detection = \
str2bool(self.config.get('auth_ret_code_detection', False))
auth_ret_code_detection = str2bool(self.config.get("auth_ret_code_detection", False))
self.authenticate = BasicAuth(
'', authenticate, registry, config.get('auth_ret_code'),
auth_ret_code_detection, rc_realm=realm)
self.ip_addr = '0.0.0.0'
"", authenticate, registry, config.get("auth_ret_code"), auth_ret_code_detection, rc_realm=realm
)
self.ip_addr = "0.0.0.0"
@LazyProperty
def global_vcs_config(self):
@ -132,10 +128,10 @@ class SimpleVCS(object):
@property
def base_path(self):
settings_path = self.config.get('repo_store.path')
settings_path = self.config.get("repo_store.path")
if not settings_path:
raise ValueError('FATAL: repo_store.path is empty')
raise ValueError("FATAL: repo_store.path is empty")
return settings_path
def set_repo_names(self, environ):
@ -164,17 +160,16 @@ class SimpleVCS(object):
match_dict = match.groupdict()
# Build acl repo name from regex match.
acl_repo_name = safe_str('{groups}{target}'.format(
groups=match_dict['groups'] or '',
target=match_dict['target']))
acl_repo_name = safe_str(
"{groups}{target}".format(groups=match_dict["groups"] or "", target=match_dict["target"])
)
# Retrieve pull request instance by ID from regex match.
pull_request = PullRequest.get(match_dict['pr_id'])
pull_request = PullRequest.get(match_dict["pr_id"])
# Only proceed if we got a pull request and if acl repo name from
# URL equals the target repo name of the pull request.
if pull_request and (acl_repo_name == pull_request.target_repo.repo_name):
# Get file system path to shadow repository.
workspace_id = PullRequestModel()._workspace_id(pull_request)
vcs_repo_name = pull_request.target_repo.get_shadow_repository_path(workspace_id)
@ -184,21 +179,23 @@ class SimpleVCS(object):
self.acl_repo_name = acl_repo_name
self.is_shadow_repo = True
log.debug('Setting all VCS repository names: %s', {
'acl_repo_name': self.acl_repo_name,
'url_repo_name': self.url_repo_name,
'vcs_repo_name': self.vcs_repo_name,
})
log.debug(
"Setting all VCS repository names: %s",
{
"acl_repo_name": self.acl_repo_name,
"url_repo_name": self.url_repo_name,
"vcs_repo_name": self.vcs_repo_name,
},
)
@property
def scm_app(self):
custom_implementation = self.config['vcs.scm_app_implementation']
if custom_implementation == 'http':
log.debug('Using HTTP implementation of scm app.')
custom_implementation = self.config["vcs.scm_app_implementation"]
if custom_implementation == "http":
log.debug("Using HTTP implementation of scm app.")
scm_app_impl = scm_app_http
else:
log.debug('Using custom implementation of scm_app: "{}"'.format(
custom_implementation))
log.debug('Using custom implementation of scm_app: "{}"'.format(custom_implementation))
scm_app_impl = importlib.import_module(custom_implementation)
return scm_app_impl
@ -208,17 +205,18 @@ class SimpleVCS(object):
with a repository_name for support of _<ID> non changeable urls
"""
data = repo_name.split('/')
data = repo_name.split("/")
if len(data) >= 2:
from rhodecode.model.repo import RepoModel
by_id_match = RepoModel().get_repo_by_id(repo_name)
if by_id_match:
data[1] = by_id_match.repo_name
# Because PEP-3333-WSGI uses bytes-tunneled-in-latin-1 as PATH_INFO
# and we use this data
maybe_new_path = '/'.join(data)
return safe_bytes(maybe_new_path).decode('latin1')
maybe_new_path = "/".join(data)
return safe_bytes(maybe_new_path).decode("latin1")
def _invalidate_cache(self, repo_name):
"""
@ -231,21 +229,18 @@ class SimpleVCS(object):
def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
db_repo = Repository.get_by_repo_name(repo_name)
if not db_repo:
log.debug('Repository `%s` not found inside the database.',
repo_name)
log.debug("Repository `%s` not found inside the database.", repo_name)
return False
if db_repo.repo_type != scm_type:
log.warning(
'Repository `%s` have incorrect scm_type, expected %s got %s',
repo_name, db_repo.repo_type, scm_type)
"Repository `%s` have incorrect scm_type, expected %s got %s", repo_name, db_repo.repo_type, scm_type
)
return False
config = db_repo._config
config.set('extensions', 'largefiles', '')
return is_valid_repo(
repo_name, base_path,
explicit_scm=scm_type, expect_scm=scm_type, config=config)
config.set("extensions", "largefiles", "")
return is_valid_repo(repo_name, base_path, explicit_scm=scm_type, expect_scm=scm_type, config=config)
def valid_and_active_user(self, user):
"""
@ -267,8 +262,9 @@ class SimpleVCS(object):
def is_shadow_repo_dir(self):
return os.path.isdir(self.vcs_repo_name)
def _check_permission(self, action, user, auth_user, repo_name, ip_addr=None,
plugin_id='', plugin_cache_active=False, cache_ttl=0):
def _check_permission(
self, action, user, auth_user, repo_name, ip_addr=None, plugin_id="", plugin_cache_active=False, cache_ttl=0
):
"""
Checks permissions using action (push/pull) user and repository
name. If plugin_cache and ttl is set it will use the plugin which
@ -280,71 +276,67 @@ class SimpleVCS(object):
:param repo_name: repository name
"""
log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
plugin_id, plugin_cache_active, cache_ttl)
log.debug("AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)", plugin_id, plugin_cache_active, cache_ttl)
user_id = user.user_id
cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}'
region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
cache_namespace_uid = f"cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}"
region = rc_cache.get_or_create_region("cache_perms", cache_namespace_uid)
@region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
expiration_time=cache_ttl,
condition=plugin_cache_active)
def compute_perm_vcs(
cache_name, plugin_id, action, user_id, repo_name, ip_addr):
log.debug('auth: calculating permission access now for vcs operation: %s', action)
@region.conditional_cache_on_arguments(
namespace=cache_namespace_uid, expiration_time=cache_ttl, condition=plugin_cache_active
)
def compute_perm_vcs(cache_name, plugin_id, action, user_id, repo_name, ip_addr):
log.debug("auth: calculating permission access now for vcs operation: %s", action)
# check IP
inherit = user.inherit_default_permissions
ip_allowed = AuthUser.check_ip_allowed(
user_id, ip_addr, inherit_from_default=inherit)
ip_allowed = AuthUser.check_ip_allowed(user_id, ip_addr, inherit_from_default=inherit)
if ip_allowed:
log.info('Access for IP:%s allowed', ip_addr)
log.info("Access for IP:%s allowed", ip_addr)
else:
return False
if action == 'push':
perms = ('repository.write', 'repository.admin')
if action == "push":
perms = ("repository.write", "repository.admin")
if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
return False
else:
# any other action need at least read permission
perms = (
'repository.read', 'repository.write', 'repository.admin')
perms = ("repository.read", "repository.write", "repository.admin")
if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
return False
return True
start = time.time()
log.debug('Running plugin `%s` permissions check', plugin_id)
log.debug("Running plugin `%s` permissions check", plugin_id)
# for environ based auth, password can be empty, but then the validation is
# on the server that fills in the env data needed for authentication
perm_result = compute_perm_vcs(
'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
perm_result = compute_perm_vcs("vcs_permissions", plugin_id, action, user.user_id, repo_name, ip_addr)
auth_time = time.time() - start
log.debug('Permissions for plugin `%s` completed in %.4fs, '
'expiration time of fetched cache %.1fs.',
plugin_id, auth_time, cache_ttl)
log.debug(
"Permissions for plugin `%s` completed in %.4fs, " "expiration time of fetched cache %.1fs.",
plugin_id,
auth_time,
cache_ttl,
)
return perm_result
def _get_http_scheme(self, environ):
try:
return environ['wsgi.url_scheme']
return environ["wsgi.url_scheme"]
except Exception:
log.exception('Failed to read http scheme')
return 'http'
log.exception("Failed to read http scheme")
return "http"
def _get_default_cache_ttl(self):
# take AUTH_CACHE_TTL from the `rhodecode` auth plugin
plugin = loadplugin('egg:rhodecode-enterprise-ce#rhodecode')
plugin = loadplugin("egg:rhodecode-enterprise-ce#rhodecode")
plugin_settings = plugin.get_settings()
plugin_cache_active, cache_ttl = plugin.get_ttl_cache(
plugin_settings) or (False, 0)
plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings) or (False, 0)
return plugin_cache_active, cache_ttl
def __call__(self, environ, start_response):
@ -359,17 +351,17 @@ class SimpleVCS(object):
def _handle_request(self, environ, start_response):
if not self.url_repo_name:
log.warning('Repository name is empty: %s', self.url_repo_name)
log.warning("Repository name is empty: %s", self.url_repo_name)
# failed to get repo name, we fail now
return HTTPNotFound()(environ, start_response)
log.debug('Extracted repo name is %s', self.url_repo_name)
log.debug("Extracted repo name is %s", self.url_repo_name)
ip_addr = get_ip_addr(environ)
user_agent = get_user_agent(environ)
username = None
# skip passing error to error controller
environ['pylons.status_code_redirect'] = True
environ["pylons.status_code_redirect"] = True
# ======================================================================
# GET ACTION PULL or PUSH
@ -380,17 +372,15 @@ class SimpleVCS(object):
# Check if this is a request to a shadow repository of a pull request.
# In this case only pull action is allowed.
# ======================================================================
if self.is_shadow_repo and action != 'pull':
reason = 'Only pull action is allowed for shadow repositories.'
log.debug('User not allowed to proceed, %s', reason)
if self.is_shadow_repo and action != "pull":
reason = "Only pull action is allowed for shadow repositories."
log.debug("User not allowed to proceed, %s", reason)
return HTTPNotAcceptable(reason)(environ, start_response)
# Check if the shadow repo actually exists, in case someone refers
# to it, and it has been deleted because of successful merge.
if self.is_shadow_repo and not self.is_shadow_repo_dir:
log.debug(
'Shadow repo detected, and shadow repo dir `%s` is missing',
self.is_shadow_repo_dir)
log.debug("Shadow repo detected, and shadow repo dir `%s` is missing", self.is_shadow_repo_dir)
return HTTPNotFound()(environ, start_response)
# ======================================================================
@ -398,7 +388,7 @@ class SimpleVCS(object):
# ======================================================================
detect_force_push = False
check_branch_perms = False
if action in ['pull', 'push']:
if action in ["pull", "push"]:
user_obj = anonymous_user = User.get_default_user()
auth_user = user_obj.AuthUser()
username = anonymous_user.username
@ -406,8 +396,12 @@ class SimpleVCS(object):
plugin_cache_active, cache_ttl = self._get_default_cache_ttl()
# ONLY check permissions if the user is activated
anonymous_perm = self._check_permission(
action, anonymous_user, auth_user, self.acl_repo_name, ip_addr,
plugin_id='anonymous_access',
action,
anonymous_user,
auth_user,
self.acl_repo_name,
ip_addr,
plugin_id="anonymous_access",
plugin_cache_active=plugin_cache_active,
cache_ttl=cache_ttl,
)
@ -416,12 +410,13 @@ class SimpleVCS(object):
if not anonymous_user.active or not anonymous_perm:
if not anonymous_user.active:
log.debug('Anonymous access is disabled, running '
'authentication')
log.debug("Anonymous access is disabled, running " "authentication")
if not anonymous_perm:
log.debug('Not enough credentials to access repo: `%s` '
'repository as anonymous user', self.acl_repo_name)
log.debug(
"Not enough credentials to access repo: `%s` " "repository as anonymous user",
self.acl_repo_name,
)
username = None
# ==============================================================
@ -430,19 +425,18 @@ class SimpleVCS(object):
# ==============================================================
# try to auth based on environ, container auth methods
log.debug('Running PRE-AUTH for container|headers based authentication')
log.debug("Running PRE-AUTH for container|headers based authentication")
# headers auth, by just reading special headers and bypass the auth with user/passwd
pre_auth = authenticate(
'', '', environ, VCS_TYPE, registry=self.registry,
acl_repo_name=self.acl_repo_name)
"", "", environ, VCS_TYPE, registry=self.registry, acl_repo_name=self.acl_repo_name
)
if pre_auth and pre_auth.get('username'):
username = pre_auth['username']
log.debug('PRE-AUTH got `%s` as username', username)
if pre_auth and pre_auth.get("username"):
username = pre_auth["username"]
log.debug("PRE-AUTH got `%s` as username", username)
if pre_auth:
log.debug('PRE-AUTH successful from %s',
pre_auth.get('auth_data', {}).get('_plugin'))
log.debug("PRE-AUTH successful from %s", pre_auth.get("auth_data", {}).get("_plugin"))
# If not authenticated by the container, running basic auth
# before inject the calling repo_name for special scope checks
@ -463,16 +457,16 @@ class SimpleVCS(object):
return HTTPNotAcceptable(reason)(environ, start_response)
if isinstance(auth_result, dict):
AUTH_TYPE.update(environ, 'basic')
REMOTE_USER.update(environ, auth_result['username'])
username = auth_result['username']
plugin = auth_result.get('auth_data', {}).get('_plugin')
log.info(
'MAIN-AUTH successful for user `%s` from %s plugin',
username, plugin)
AUTH_TYPE.update(environ, "basic")
REMOTE_USER.update(environ, auth_result["username"])
username = auth_result["username"]
plugin = auth_result.get("auth_data", {}).get("_plugin")
log.info("MAIN-AUTH successful for user `%s` from %s plugin", username, plugin)
plugin_cache_active, cache_ttl = auth_result.get(
'auth_data', {}).get('_ttl_cache') or (False, 0)
plugin_cache_active, cache_ttl = auth_result.get("auth_data", {}).get("_ttl_cache") or (
False,
0,
)
else:
return auth_result.wsgi_application(environ, start_response)
@ -488,21 +482,24 @@ class SimpleVCS(object):
# check user attributes for password change flag
user_obj = user
auth_user = user_obj.AuthUser()
if user_obj and user_obj.username != User.DEFAULT_USER and \
user_obj.user_data.get('force_password_change'):
reason = 'password change required'
log.debug('User not allowed to authenticate, %s', reason)
if (
user_obj
and user_obj.username != User.DEFAULT_USER
and user_obj.user_data.get("force_password_change")
):
reason = "password change required"
log.debug("User not allowed to authenticate, %s", reason)
return HTTPNotAcceptable(reason)(environ, start_response)
# check permissions for this repository
perm = self._check_permission(
action, user, auth_user, self.acl_repo_name, ip_addr,
plugin, plugin_cache_active, cache_ttl)
action, user, auth_user, self.acl_repo_name, ip_addr, plugin, plugin_cache_active, cache_ttl
)
if not perm:
return HTTPForbidden()(environ, start_response)
environ['rc_auth_user_id'] = str(user_id)
environ["rc_auth_user_id"] = str(user_id)
if action == 'push':
if action == "push":
perms = auth_user.get_branch_permissions(self.acl_repo_name)
if perms:
check_branch_perms = True
@ -510,41 +507,48 @@ class SimpleVCS(object):
# extras are injected into UI object and later available
# in hooks executed by RhodeCode
check_locking = _should_check_locking(environ.get('QUERY_STRING'))
check_locking = _should_check_locking(environ.get("QUERY_STRING"))
extras = vcs_operation_context(
environ, repo_name=self.acl_repo_name, username=username,
action=action, scm=self.SCM, check_locking=check_locking,
is_shadow_repo=self.is_shadow_repo, check_branch_perms=check_branch_perms,
detect_force_push=detect_force_push
environ,
repo_name=self.acl_repo_name,
username=username,
action=action,
scm=self.SCM,
check_locking=check_locking,
is_shadow_repo=self.is_shadow_repo,
check_branch_perms=check_branch_perms,
detect_force_push=detect_force_push,
)
# ======================================================================
# REQUEST HANDLING
# ======================================================================
repo_path = os.path.join(
safe_str(self.base_path), safe_str(self.vcs_repo_name))
log.debug('Repository path is %s', repo_path)
repo_path = os.path.join(safe_str(self.base_path), safe_str(self.vcs_repo_name))
log.debug("Repository path is %s", repo_path)
fix_PATH()
log.info(
'%s action on %s repo "%s" by "%s" from %s %s',
action, self.SCM, safe_str(self.url_repo_name),
safe_str(username), ip_addr, user_agent)
action,
self.SCM,
safe_str(self.url_repo_name),
safe_str(username),
ip_addr,
user_agent,
)
return self._generate_vcs_response(
environ, start_response, repo_path, extras, action)
return self._generate_vcs_response(environ, start_response, repo_path, extras, action)
def _get_txn_id(self, environ):
for k in ['RAW_URI', 'HTTP_DESTINATION']:
for k in ["RAW_URI", "HTTP_DESTINATION"]:
url = environ.get(k)
if not url:
continue
# regex to search for svn-txn-id
pattern = r'/!svn/txr/([^/]+)/'
pattern = r"/!svn/txr/([^/]+)/"
# Search for the pattern in the URL
match = re.search(pattern, url)
@ -555,8 +559,7 @@ class SimpleVCS(object):
return txn_id
@initialize_generator
def _generate_vcs_response(
self, environ, start_response, repo_path, extras, action):
def _generate_vcs_response(self, environ, start_response, repo_path, extras, action):
"""
Returns a generator for the response content.
@ -565,24 +568,20 @@ class SimpleVCS(object):
also handles the locking exceptions which will be triggered when
the first chunk is produced by the underlying WSGI application.
"""
svn_txn_id = ''
if action == 'push':
svn_txn_id = ""
if action == "push":
svn_txn_id = self._get_txn_id(environ)
callback_daemon, extras = self._prepare_callback_daemon(
extras, environ, action, txn_id=svn_txn_id)
callback_daemon, extras = self._prepare_callback_daemon(extras, environ, action, txn_id=svn_txn_id)
if svn_txn_id:
port = safe_int(extras['hooks_uri'].split(':')[-1])
txn_id_data = extras.copy()
txn_id_data.update({'port': port})
txn_id_data.update({'req_method': environ['REQUEST_METHOD']})
txn_id_data.update({"req_method": environ["REQUEST_METHOD"]})
full_repo_path = repo_path
store_txn_id_data(full_repo_path, svn_txn_id, txn_id_data)
log.debug('HOOKS extras is %s', extras)
log.debug("HOOKS extras is %s", extras)
http_scheme = self._get_http_scheme(environ)
@ -609,7 +608,7 @@ class SimpleVCS(object):
try:
# invalidate cache on push
if action == 'push':
if action == "push":
self._invalidate_cache(self.url_repo_name)
finally:
meta.Session.remove()
@ -632,12 +631,12 @@ class SimpleVCS(object):
"""Return the WSGI app that will finally handle the request."""
raise NotImplementedError()
def _create_config(self, extras, repo_name, scheme='http'):
def _create_config(self, extras, repo_name, scheme="http"):
"""Create a safe config representation."""
raise NotImplementedError()
def _should_use_callback_daemon(self, extras, environ, action):
if extras.get('is_shadow_repo'):
if extras.get("is_shadow_repo"):
# we don't want to execute hooks, and callback daemon for shadow repos
return False
return True
@ -647,11 +646,9 @@ class SimpleVCS(object):
if not self._should_use_callback_daemon(extras, environ, action):
# disable callback daemon for actions that don't require it
protocol = 'local'
protocol = "local"
return prepare_callback_daemon(
extras, protocol=protocol,
host=vcs_settings.HOOKS_HOST, txn_id=txn_id)
return prepare_callback_daemon(extras, protocol=protocol, txn_id=txn_id)
def _should_check_locking(query_string):
@ -659,4 +656,4 @@ def _should_check_locking(query_string):
# server see all operation on commit; bookmarks, phases and
# obsolescence marker in different transaction, we don't want to check
# locking on those
return query_string not in ['cmd=listkeys']
return query_string not in ["cmd=listkeys"]

69
rhodecode/lib/utils.py
View file

@ -21,6 +21,7 @@ Utilities library for RhodeCode
"""
import datetime
import importlib
import decorator
import logging
@ -42,8 +43,9 @@ from webhelpers2.text import collapse, strip_tags, convert_accented_entities, co
from mako import exceptions
import rhodecode
from rhodecode import ConfigGet
from rhodecode.lib.exceptions import HTTPBranchProtected, HTTPLockedRC
from rhodecode.lib.exceptions import HTTPBranchProtected, HTTPLockedRepo, ClientNotSupported
from rhodecode.lib.hash_utils import sha256_safe, md5, sha1
from rhodecode.lib.type_utils import AttributeDict
from rhodecode.lib.str_utils import safe_bytes, safe_str
@ -86,6 +88,7 @@ def adopt_for_celery(func):
@wraps(func)
def wrapper(extras):
extras = AttributeDict(extras)
try:
# HooksResponse implements to_json method which must be used there.
return func(extras).to_json()
@ -100,7 +103,18 @@ def adopt_for_celery(func):
'exception_args': error_args,
'exception_traceback': '',
}
except HTTPLockedRC as error:
except ClientNotSupported as error:
# Those special cases don't need error reporting. It's a case of
# locked repo or protected branch
error_args = error.args
return {
'status': error.code,
'output': error.explanation,
'exception': type(error).__name__,
'exception_args': error_args,
'exception_traceback': '',
}
except HTTPLockedRepo as error:
# Those special cases don't need error reporting. It's a case of
# locked repo or protected branch
error_args = error.args
@ -117,7 +131,7 @@ def adopt_for_celery(func):
'output': '',
'exception': type(e).__name__,
'exception_args': e.args,
'exception_traceback': '',
'exception_traceback': traceback.format_exc(),
}
return wrapper
@ -411,6 +425,10 @@ def prepare_config_data(clear_session=True, repo=None):
('web', 'push_ssl', 'false'),
]
for setting in ui_settings:
# skip certain deprecated keys that might be still in DB
if f"{setting.section}_{setting.key}" in ['extensions_hgsubversion']:
continue
# Todo: remove this section once transition to *.ini files will be completed
if setting.section in ('largefiles', 'vcs_git_lfs'):
if setting.key != 'enabled':
@ -686,22 +704,41 @@ def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild
return added, removed
def deep_reload_package(package_name):
"""
Deeply reload a package by removing it and its submodules from sys.modules,
then re-importing it.
"""
# Remove the package and its submodules from sys.modules
to_reload = [name for name in sys.modules if name == package_name or name.startswith(package_name + ".")]
for module_name in to_reload:
del sys.modules[module_name]
log.debug(f"Removed module from cache: {module_name}")
# Re-import the package
package = importlib.import_module(package_name)
log.debug(f"Re-imported package: {package_name}")
return package
def load_rcextensions(root_path):
import rhodecode
from rhodecode.config import conf
path = os.path.join(root_path)
sys.path.append(path)
deep_reload = path in sys.path
sys.path.insert(0, path)
try:
rcextensions = __import__('rcextensions')
rcextensions = __import__('rcextensions', fromlist=[''])
except ImportError:
if os.path.isdir(os.path.join(path, 'rcextensions')):
log.warning('Unable to load rcextensions from %s', path)
rcextensions = None
if rcextensions:
if deep_reload:
rcextensions = deep_reload_package('rcextensions')
log.info('Loaded rcextensions from %s...', rcextensions)
rhodecode.EXTENSIONS = rcextensions
@ -741,6 +778,7 @@ def create_test_index(repo_location, config):
except ImportError:
raise ImportError('Failed to import rc_testdata, '
'please make sure this package is installed from requirements_test.txt')
rc_testdata.extract_search_index(
'vcs_search_index', os.path.dirname(config['search.location']))
@ -785,22 +823,15 @@ def create_test_repositories(test_path, config):
Creates test repositories in the temporary directory. Repositories are
extracted from archives within the rc_testdata package.
"""
import rc_testdata
try:
import rc_testdata
except ImportError:
raise ImportError('Failed to import rc_testdata, '
'please make sure this package is installed from requirements_test.txt')
from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
log.debug('making test vcs repositories')
idx_path = config['search.location']
data_path = config['cache_dir']
# clean index and data
if idx_path and os.path.exists(idx_path):
log.debug('remove %s', idx_path)
shutil.rmtree(idx_path)
if data_path and os.path.exists(data_path):
log.debug('remove %s', data_path)
shutil.rmtree(data_path)
log.debug('making test vcs repositories at %s', test_path)
rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))

2
rhodecode/lib/vcs/__init__.py
View file

@ -140,7 +140,7 @@ class CurlSession(object):
try:
curl.perform()
except pycurl.error as exc:
log.error('Failed to call endpoint url: {} using pycurl'.format(url))
log.error('Failed to call endpoint url: %s using pycurl', url)
raise
status_code = curl.getinfo(pycurl.HTTP_CODE)

7
rhodecode/lib/vcs/backends/git/__init__.py
View file

@ -45,10 +45,3 @@ def discover_git_version(raise_on_exc=False):
if raise_on_exc:
raise
return ''
def lfs_store(base_location):
"""
Return a lfs store relative to base_location
"""
return os.path.join(base_location, '.cache', 'lfs_store')

7
rhodecode/lib/vcs/backends/hg/__init__.py
View file

@ -45,10 +45,3 @@ def discover_hg_version(raise_on_exc=False):
if raise_on_exc:
raise
return ''
def largefiles_store(base_location):
"""
Return a largefile store relative to base_location
"""
return os.path.join(base_location, '.cache', 'largefiles')

2
rhodecode/lib/vcs/client_http.py
View file

@ -216,7 +216,7 @@ class RemoteRepo(object):
self._cache_region, self._cache_namespace = \
remote_maker.init_cache_region(cache_repo_id)
with_wire = with_wire or {}
with_wire = with_wire or {"cache": False}
repo_state_uid = with_wire.get('repo_state_uid') or 'state'

12
rhodecode/model/comment.py
View file

@ -373,6 +373,7 @@ class CommentsModel(BaseModel):
Session().add(comment)
Session().flush()
kwargs = {
'user': user,
'renderer_type': renderer,
@ -387,8 +388,7 @@ class CommentsModel(BaseModel):
}
if commit_obj:
recipients = ChangesetComment.get_users(
revision=commit_obj.raw_id)
recipients = ChangesetComment.get_users(revision=commit_obj.raw_id)
# add commit author if it's in RhodeCode system
cs_author = User.get_from_cs_author(commit_obj.author)
if not cs_author:
@ -397,16 +397,13 @@ class CommentsModel(BaseModel):
recipients += [cs_author]
commit_comment_url = self.get_url(comment, request=request)
commit_comment_reply_url = self.get_url(
comment, request=request,
anchor=f'comment-{comment.comment_id}/?/ReplyToComment')
commit_comment_reply_url = self.get_url(comment, request=request, anchor=f'comment-{comment.comment_id}/?/ReplyToComment')
target_repo_url = h.link_to(
repo.repo_name,
h.route_url('repo_summary', repo_name=repo.repo_name))
commit_url = h.route_url('repo_commit', repo_name=repo.repo_name,
commit_id=commit_id)
commit_url = h.route_url('repo_commit', repo_name=repo.repo_name, commit_id=commit_id)
# commit specifics
kwargs.update({
@ -489,7 +486,6 @@ class CommentsModel(BaseModel):
if not is_draft:
comment_data = comment.get_api_data()
self._log_audit_action(
action, {'data': comment_data}, auth_user, comment)

6
rhodecode/model/pull_request.py
View file

@ -38,7 +38,7 @@ from rhodecode.translation import lazy_ugettext
from rhodecode.lib import helpers as h, hooks_utils, diffs
from rhodecode.lib import audit_logger
from collections import OrderedDict
from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
from rhodecode.lib.hook_daemon.utils import prepare_callback_daemon
from rhodecode.lib.ext_json import sjson as json
from rhodecode.lib.markup_renderer import (
DEFAULT_COMMENTS_RENDERER, RstTemplateRenderer)
@ -980,9 +980,7 @@ class PullRequestModel(BaseModel):
target_ref = self._refresh_reference(
pull_request.target_ref_parts, target_vcs)
callback_daemon, extras = prepare_callback_daemon(
extras, protocol=vcs_settings.HOOKS_PROTOCOL,
host=vcs_settings.HOOKS_HOST)
callback_daemon, extras = prepare_callback_daemon(extras, protocol=vcs_settings.HOOKS_PROTOCOL)
with callback_daemon:
# TODO: johbo: Implement a clean way to run a config_override

24
rhodecode/model/settings.py
View file

@ -862,27 +862,3 @@ class VcsSettingsModel(object):
raise ValueError(
f'The given data does not contain {data_key} key')
return data_keys
def create_largeobjects_dirs_if_needed(self, repo_store_path):
"""
This is subscribed to the `pyramid.events.ApplicationCreated` event. It
does a repository scan if enabled in the settings.
"""
from rhodecode.lib.vcs.backends.hg import largefiles_store
from rhodecode.lib.vcs.backends.git import lfs_store
paths = [
largefiles_store(repo_store_path),
lfs_store(repo_store_path)]
for path in paths:
if os.path.isdir(path):
continue
if os.path.isfile(path):
continue
# not a file nor dir, we try to create it
try:
os.makedirs(path)
except Exception:
log.warning('Failed to create largefiles dir:%s', path)

13
rhodecode/tests/__init__.py
View file

@ -1,5 +1,4 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
# Copyright (C) 2010-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
@ -38,7 +37,7 @@ from rhodecode.lib.hash_utils import sha1_safe
log = logging.getLogger(__name__)
__all__ = [
'get_new_dir', 'TestController',
'get_new_dir', 'TestController', 'console_printer',
'clear_cache_regions',
'assert_session_flash', 'login_user', 'no_newline_id_generator',
'TESTS_TMP_PATH', 'HG_REPO', 'GIT_REPO', 'SVN_REPO',
@ -244,3 +243,11 @@ def no_newline_id_generator(test_name):
return test_name or 'test-with-empty-name'
def console_printer(*msg):
print_func = print
try:
from rich import print as print_func
except ImportError:
pass
print_func(*msg)

5
rhodecode/tests/auth_external_test.py
View file

@ -1,5 +1,4 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
# Copyright (C) 2010-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
@ -90,7 +89,7 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
'firstname': firstname,
'lastname': lastname,
'groups': [],
'email': '%s@rhodecode.com' % username,
'email': f'{username}@rhodecode.com',
'admin': admin,
'active': active,
"active_from_extern": None,

4
rhodecode/tests/config/test_routing_links.py
View file

@ -20,14 +20,14 @@
import pytest
import requests
from rhodecode.config import routing_links
from rhodecode.tests import console_printer
def check_connection():
try:
response = requests.get('https://rhodecode.com')
return response.status_code == 200
except Exception as e:
print(e)
console_printer(e)
return False

117
rhodecode/tests/conftest.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
# Copyright (C) 2010-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
@ -16,23 +16,10 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
"""
py.test config for test suite for making push/pull operations.
.. important::
You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
to redirect things to stderr instead of stdout.
"""
import pytest
import pytest # noqa
import logging
from rhodecode.authentication import AuthenticationPluginRegistry
from rhodecode.model.db import Permission, User
from rhodecode.model.meta import Session
from rhodecode.model.settings import SettingsModel
from rhodecode.model.user import UserModel
import collections
import rhodecode
log = logging.getLogger(__name__)
@ -40,99 +27,3 @@ log = logging.getLogger(__name__)
# Docker image running httpbin...
HTTPBIN_DOMAIN = 'http://httpbin'
HTTPBIN_POST = HTTPBIN_DOMAIN + '/post'
@pytest.fixture()
def enable_auth_plugins(request, baseapp, csrf_token):
"""
Return a factory object that when called, allows to control which
authentication plugins are enabled.
"""
class AuthPluginManager(object):
def cleanup(self):
self._enable_plugins(['egg:rhodecode-enterprise-ce#rhodecode'])
def enable(self, plugins_list, override=None):
return self._enable_plugins(plugins_list, override)
def _enable_plugins(self, plugins_list, override=None):
override = override or {}
params = {
'auth_plugins': ','.join(plugins_list),
}
# helper translate some names to others, to fix settings code
name_map = {
'token': 'authtoken'
}
log.debug('enable_auth_plugins: enabling following auth-plugins: %s', plugins_list)
for module in plugins_list:
plugin_name = module.partition('#')[-1]
if plugin_name in name_map:
plugin_name = name_map[plugin_name]
enabled_plugin = f'auth_{plugin_name}_enabled'
cache_ttl = f'auth_{plugin_name}_cache_ttl'
# default params that are needed for each plugin,
# `enabled` and `cache_ttl`
params.update({
enabled_plugin: True,
cache_ttl: 0
})
if override.get:
params.update(override.get(module, {}))
validated_params = params
for k, v in validated_params.items():
setting = SettingsModel().create_or_update_setting(k, v)
Session().add(setting)
Session().commit()
AuthenticationPluginRegistry.invalidate_auth_plugins_cache(hard=True)
enabled_plugins = SettingsModel().get_auth_plugins()
assert plugins_list == enabled_plugins
enabler = AuthPluginManager()
request.addfinalizer(enabler.cleanup)
return enabler
@pytest.fixture()
def test_user_factory(request, baseapp):
def user_factory(username='test_user', password='qweqwe', first_name='John', last_name='Testing', **kwargs):
usr = UserModel().create_or_update(
username=username,
password=password,
email=f'{username}@rhodecode.org',
firstname=first_name, lastname=last_name)
Session().commit()
for k, v in kwargs.items():
setattr(usr, k, v)
Session().add(usr)
new_usr = User.get_by_username(username)
new_usr_id = new_usr.user_id
assert new_usr == usr
@request.addfinalizer
def cleanup():
if User.get(new_usr_id) is None:
return
perm = Permission.query().all()
for p in perm:
UserModel().revoke_perm(usr, p)
UserModel().delete(new_usr_id)
Session().commit()
return usr
return user_factory

20
rhodecode/tests/conftest_common.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
# Copyright (C) 2010-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
@ -98,16 +98,16 @@ def pytest_addoption(parser):
'pyramid_config',
"Set up a Pyramid environment with the specified config file.")
parser.addini('rhodecode_config', 'rhodecode config ini for tests')
parser.addini('celery_config', 'celery config ini for tests')
parser.addini('vcsserver_config', 'vcsserver config ini for tests')
vcsgroup = parser.getgroup('vcs')
vcsgroup.addoption(
'--without-vcsserver', dest='with_vcsserver', action='store_false',
help="Do not start the VCSServer in a background process.")
vcsgroup.addoption(
'--with-vcsserver-http', dest='vcsserver_config_http',
help="Start the HTTP VCSServer with the specified config file.")
vcsgroup.addoption(
'--vcsserver-protocol', dest='vcsserver_protocol',
help="Start the VCSServer with HTTP protocol support.")
vcsgroup.addoption(
'--vcsserver-config-override', action='store', type=_parse_json,
default=None, dest='vcsserver_config_override', help=(
@ -122,12 +122,6 @@ def pytest_addoption(parser):
"Allows to set the port of the vcsserver. Useful when testing "
"against an already running server and random ports cause "
"trouble."))
parser.addini(
'vcsserver_config_http',
"Start the HTTP VCSServer with the specified config file.")
parser.addini(
'vcsserver_protocol',
"Start the VCSServer with HTTP protocol support.")
@pytest.hookimpl(tryfirst=True, hookwrapper=True)

17
rhodecode/tests/database/__init__.py Normal file
View file

@ -0,0 +1,17 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/

139
rhodecode/tests/database/conftest.py
View file

@ -1,4 +1,3 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
@ -17,7 +16,7 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
from subprocess import Popen, PIPE
import subprocess
import os
import sys
import tempfile
@ -26,87 +25,71 @@ import pytest
from sqlalchemy.engine import url
from rhodecode.lib.str_utils import safe_str, safe_bytes
from rhodecode.tests.fixture import TestINI
from rhodecode.tests.fixtures.rc_fixture import TestINI
def _get_dbs_from_metafunc(metafunc):
dbs_mark = metafunc.definition.get_closest_marker('dbs')
dbs_mark = metafunc.definition.get_closest_marker("dbs")
if dbs_mark:
# Supported backends by this test function, created from pytest.mark.dbs
backends = dbs_mark.args
else:
backends = metafunc.config.getoption('--dbs')
backends = metafunc.config.getoption("--dbs")
return backends
def pytest_generate_tests(metafunc):
# Support test generation based on --dbs parameter
if 'db_backend' in metafunc.fixturenames:
requested_backends = set(metafunc.config.getoption('--dbs'))
if "db_backend" in metafunc.fixturenames:
requested_backends = set(metafunc.config.getoption("--dbs"))
backends = _get_dbs_from_metafunc(metafunc)
backends = requested_backends.intersection(backends)
# TODO: johbo: Disabling a backend did not work out with
# parametrization, find better way to achieve this.
if not backends:
metafunc.function._skip = True
metafunc.parametrize('db_backend_name', backends)
metafunc.parametrize("db_backend_name", backends)
def pytest_collection_modifyitems(session, config, items):
remaining = [
i for i in items if not getattr(i.obj, '_skip', False)]
remaining = [i for i in items if not getattr(i.obj, "_skip", False)]
items[:] = remaining
@pytest.fixture()
def db_backend(
request, db_backend_name, ini_config, tmpdir_factory):
def db_backend(request, db_backend_name, ini_config, tmpdir_factory):
basetemp = tmpdir_factory.getbasetemp().strpath
klass = _get_backend(db_backend_name)
option_name = '--{}-connection-string'.format(db_backend_name)
option_name = "--{}-connection-string".format(db_backend_name)
connection_string = request.config.getoption(option_name) or None
return klass(
config_file=ini_config, basetemp=basetemp,
connection_string=connection_string)
return klass(config_file=ini_config, basetemp=basetemp, connection_string=connection_string)
def _get_backend(backend_type):
return {
'sqlite': SQLiteDBBackend,
'postgres': PostgresDBBackend,
'mysql': MySQLDBBackend,
'': EmptyDBBackend
}[backend_type]
return {"sqlite": SQLiteDBBackend, "postgres": PostgresDBBackend, "mysql": MySQLDBBackend, "": EmptyDBBackend}[
backend_type
]
class DBBackend(object):
_store = os.path.dirname(os.path.abspath(__file__))
_type = None
_base_ini_config = [{'app:main': {'vcs.start_server': 'false',
'startup.import_repos': 'false'}}]
_db_url = [{'app:main': {'sqlalchemy.db1.url': ''}}]
_base_db_name = 'rhodecode_test_db_backend'
std_env = {'RC_TEST': '0'}
def __init__(
self, config_file, db_name=None, basetemp=None,
connection_string=None):
from rhodecode.lib.vcs.backends.hg import largefiles_store
from rhodecode.lib.vcs.backends.git import lfs_store
_base_ini_config = [{"app:main": {"vcs.start_server": "false", "startup.import_repos": "false"}}]
_db_url = [{"app:main": {"sqlalchemy.db1.url": ""}}]
_base_db_name = "rhodecode_test_db_backend"
std_env = {"RC_TEST": "0"}
def __init__(self, config_file, db_name=None, basetemp=None, connection_string=None):
self.fixture_store = os.path.join(self._store, self._type)
self.db_name = db_name or self._base_db_name
self._base_ini_file = config_file
self.stderr = ''
self.stdout = ''
self.stderr = ""
self.stdout = ""
self._basetemp = basetemp or tempfile.gettempdir()
self._repos_location = os.path.join(self._basetemp, 'rc_test_repos')
self._repos_hg_largefiles_store = largefiles_store(self._basetemp)
self._repos_git_lfs_store = lfs_store(self._basetemp)
self._repos_location = os.path.join(self._basetemp, "rc_test_repos")
self.connection_string = connection_string
@property
@ -118,8 +101,7 @@ class DBBackend(object):
if not new_connection_string:
new_connection_string = self.get_default_connection_string()
else:
new_connection_string = new_connection_string.format(
db_name=self.db_name)
new_connection_string = new_connection_string.format(db_name=self.db_name)
url_parts = url.make_url(new_connection_string)
self._connection_string = new_connection_string
self.user = url_parts.username
@ -127,73 +109,67 @@ class DBBackend(object):
self.host = url_parts.host
def get_default_connection_string(self):
raise NotImplementedError('default connection_string is required.')
raise NotImplementedError("default connection_string is required.")
def execute(self, cmd, env=None, *args):
"""
Runs command on the system with given ``args``.
"""
command = cmd + ' ' + ' '.join(args)
sys.stdout.write(f'CMD: {command}')
command = cmd + " " + " ".join(args)
sys.stdout.write(f"CMD: {command}")
# Tell Python to use UTF-8 encoding out stdout
_env = os.environ.copy()
_env['PYTHONIOENCODING'] = 'UTF-8'
_env["PYTHONIOENCODING"] = "UTF-8"
_env.update(self.std_env)
if env:
_env.update(env)
self.p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, env=_env)
self.p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=_env)
self.stdout, self.stderr = self.p.communicate()
stdout_str = safe_str(self.stdout)
sys.stdout.write(f'COMMAND:{command}\n')
sys.stdout.write(f"COMMAND:{command}\n")
sys.stdout.write(stdout_str)
return self.stdout, self.stderr
def assert_returncode_success(self):
from rich import print as pprint
if not self.p.returncode == 0:
pprint(safe_str(self.stderr))
raise AssertionError(f'non 0 retcode:{self.p.returncode}')
raise AssertionError(f"non 0 retcode:{self.p.returncode}")
def assert_correct_output(self, stdout, version):
assert b'UPGRADE FOR STEP %b COMPLETED' % safe_bytes(version) in stdout
assert b"UPGRADE FOR STEP %b COMPLETED" % safe_bytes(version) in stdout
def setup_rhodecode_db(self, ini_params=None, env=None):
if not ini_params:
ini_params = self._base_ini_config
ini_params.extend(self._db_url)
with TestINI(self._base_ini_file, ini_params,
self._type, destroy=True) as _ini_file:
with TestINI(self._base_ini_file, ini_params, self._type, destroy=True) as _ini_file:
if not os.path.isdir(self._repos_location):
os.makedirs(self._repos_location)
if not os.path.isdir(self._repos_hg_largefiles_store):
os.makedirs(self._repos_hg_largefiles_store)
if not os.path.isdir(self._repos_git_lfs_store):
os.makedirs(self._repos_git_lfs_store)
return self.execute(
"rc-setup-app {0} --user=marcink "
"--email=marcin@rhodeocode.com --password={1} "
"--repos={2} --force-yes".format(
_ini_file, 'qweqwe', self._repos_location), env=env)
"--repos={2} --force-yes".format(_ini_file, "qweqwe", self._repos_location),
env=env,
)
def upgrade_database(self, ini_params=None):
if not ini_params:
ini_params = self._base_ini_config
ini_params.extend(self._db_url)
test_ini = TestINI(
self._base_ini_file, ini_params, self._type, destroy=True)
test_ini = TestINI(self._base_ini_file, ini_params, self._type, destroy=True)
with test_ini as ini_file:
if not os.path.isdir(self._repos_location):
os.makedirs(self._repos_location)
return self.execute(
"rc-upgrade-db {0} --force-yes".format(ini_file))
return self.execute("rc-upgrade-db {0} --force-yes".format(ini_file))
def setup_db(self):
raise NotImplementedError
@ -206,7 +182,7 @@ class DBBackend(object):
class EmptyDBBackend(DBBackend):
_type = ''
_type = ""
def setup_db(self):
pass
@ -222,21 +198,20 @@ class EmptyDBBackend(DBBackend):
class SQLiteDBBackend(DBBackend):
_type = 'sqlite'
_type = "sqlite"
def get_default_connection_string(self):
return 'sqlite:///{}/{}.sqlite'.format(self._basetemp, self.db_name)
return "sqlite:///{}/{}.sqlite".format(self._basetemp, self.db_name)
def setup_db(self):
# dump schema for tests
# cp -v $TEST_DB_NAME
self._db_url = [{'app:main': {
'sqlalchemy.db1.url': self.connection_string}}]
self._db_url = [{"app:main": {"sqlalchemy.db1.url": self.connection_string}}]
def import_dump(self, dumpname):
dump = os.path.join(self.fixture_store, dumpname)
target = os.path.join(self._basetemp, '{0.db_name}.sqlite'.format(self))
return self.execute(f'cp -v {dump} {target}')
target = os.path.join(self._basetemp, "{0.db_name}.sqlite".format(self))
return self.execute(f"cp -v {dump} {target}")
def teardown_db(self):
target_db = os.path.join(self._basetemp, self.db_name)
@ -244,39 +219,39 @@ class SQLiteDBBackend(DBBackend):
class MySQLDBBackend(DBBackend):
_type = 'mysql'
_type = "mysql"
def get_default_connection_string(self):
return 'mysql://root:qweqwe@127.0.0.1/{}'.format(self.db_name)
return "mysql://root:qweqwe@127.0.0.1/{}".format(self.db_name)
def setup_db(self):
# dump schema for tests
# mysqldump -uroot -pqweqwe $TEST_DB_NAME
self._db_url = [{'app:main': {
'sqlalchemy.db1.url': self.connection_string}}]
return self.execute("mysql -v -u{} -p{} -e 'create database '{}';'".format(
self.user, self.password, self.db_name))
self._db_url = [{"app:main": {"sqlalchemy.db1.url": self.connection_string}}]
return self.execute(
"mysql -v -u{} -p{} -e 'create database '{}';'".format(self.user, self.password, self.db_name)
)
def import_dump(self, dumpname):
dump = os.path.join(self.fixture_store, dumpname)
return self.execute("mysql -u{} -p{} {} < {}".format(
self.user, self.password, self.db_name, dump))
return self.execute("mysql -u{} -p{} {} < {}".format(self.user, self.password, self.db_name, dump))
def teardown_db(self):
return self.execute("mysql -v -u{} -p{} -e 'drop database '{}';'".format(
self.user, self.password, self.db_name))
return self.execute(
"mysql -v -u{} -p{} -e 'drop database '{}';'".format(self.user, self.password, self.db_name)
)
class PostgresDBBackend(DBBackend):
_type = 'postgres'
_type = "postgres"
def get_default_connection_string(self):
return 'postgresql://postgres:qweqwe@localhost/{}'.format(self.db_name)
return "postgresql://postgres:qweqwe@localhost/{}".format(self.db_name)
def setup_db(self):
# dump schema for tests
# pg_dump -U postgres -h localhost $TEST_DB_NAME
self._db_url = [{'app:main': {'sqlalchemy.db1.url': self.connection_string}}]
self._db_url = [{"app:main": {"sqlalchemy.db1.url": self.connection_string}}]
cmd = f"PGPASSWORD={self.password} psql -U {self.user} -h localhost -c 'create database '{self.db_name}';'"
return self.execute(cmd)

1
rhodecode/tests/database/test_creation.py
View file

@ -1,4 +1,3 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify

46
rhodecode/tests/database/test_migration.py
View file

@ -1,4 +1,3 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
@ -21,33 +20,42 @@ import pytest
@pytest.mark.dbs("postgres")
@pytest.mark.parametrize("dumpname", [
'1.4.4.sql',
'1.5.0.sql',
'1.6.0.sql',
'1.6.0_no_repo_name_index.sql',
])
@pytest.mark.parametrize(
"dumpname",
[
"1.4.4.sql",
"1.5.0.sql",
"1.6.0.sql",
"1.6.0_no_repo_name_index.sql",
],
)
def test_migrate_postgres_db(db_backend, dumpname):
_run_migration_test(db_backend, dumpname)
@pytest.mark.dbs("sqlite")
@pytest.mark.parametrize("dumpname", [
'rhodecode.1.4.4.sqlite',
'rhodecode.1.4.4_with_groups.sqlite',
'rhodecode.1.4.4_with_ldap_active.sqlite',
])
@pytest.mark.parametrize(
"dumpname",
[
"rhodecode.1.4.4.sqlite",
"rhodecode.1.4.4_with_groups.sqlite",
"rhodecode.1.4.4_with_ldap_active.sqlite",
],
)
def test_migrate_sqlite_db(db_backend, dumpname):
_run_migration_test(db_backend, dumpname)
@pytest.mark.dbs("mysql")
@pytest.mark.parametrize("dumpname", [
'1.4.4.sql',
'1.5.0.sql',
'1.6.0.sql',
'1.6.0_no_repo_name_index.sql',
])
@pytest.mark.parametrize(
"dumpname",
[
"1.4.4.sql",
"1.5.0.sql",
"1.6.0.sql",
"1.6.0_no_repo_name_index.sql",
],
)
def test_migrate_mysql_db(db_backend, dumpname):
_run_migration_test(db_backend, dumpname)
@ -60,5 +68,5 @@ def _run_migration_test(db_backend, dumpname):
db_backend.import_dump(dumpname)
stdout, stderr = db_backend.upgrade_database()
db_backend.assert_correct_output(stdout+stderr, version='16')
db_backend.assert_correct_output(stdout + stderr, version="16")
db_backend.assert_returncode_success()

226
rhodecode/tests/fixture_mods/fixture_pyramid.py
View file

@ -1,226 +0,0 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import pytest
from rhodecode.lib.config_utils import get_app_config
from rhodecode.tests.fixture import TestINI
from rhodecode.tests import TESTS_TMP_PATH
from rhodecode.tests.server_utils import RcVCSServer
@pytest.fixture(scope='session')
def vcsserver(request, vcsserver_port, vcsserver_factory):
"""
Session scope VCSServer.
Tests which need the VCSServer have to rely on this fixture in order
to ensure it will be running.
For specific needs, the fixture vcsserver_factory can be used. It allows to
adjust the configuration file for the test run.
Command line args:
--without-vcsserver: Allows to switch this fixture off. You have to
manually start the server.
--vcsserver-port: Will expect the VCSServer to listen on this port.
"""
if not request.config.getoption('with_vcsserver'):
return None
return vcsserver_factory(
request, vcsserver_port=vcsserver_port)
@pytest.fixture(scope='session')
def vcsserver_factory(tmpdir_factory):
"""
Use this if you need a running vcsserver with a special configuration.
"""
def factory(request, overrides=(), vcsserver_port=None,
log_file=None, workers='3'):
if vcsserver_port is None:
vcsserver_port = get_available_port()
overrides = list(overrides)
overrides.append({'server:main': {'port': vcsserver_port}})
option_name = 'vcsserver_config_http'
override_option_name = 'vcsserver_config_override'
config_file = get_config(
request.config, option_name=option_name,
override_option_name=override_option_name, overrides=overrides,
basetemp=tmpdir_factory.getbasetemp().strpath,
prefix='test_vcs_')
server = RcVCSServer(config_file, log_file, workers)
server.start()
@request.addfinalizer
def cleanup():
server.shutdown()
server.wait_until_ready()
return server
return factory
def _use_log_level(config):
level = config.getoption('test_loglevel') or 'critical'
return level.upper()
@pytest.fixture(scope='session')
def ini_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
option_name = 'pyramid_config'
log_level = _use_log_level(request.config)
overrides = [
{'server:main': {'port': rcserver_port}},
{'app:main': {
'cache_dir': '%(here)s/rc-tests/rc_data',
'vcs.server': f'localhost:{vcsserver_port}',
# johbo: We will always start the VCSServer on our own based on the
# fixtures of the test cases. For the test run it must always be
# off in the INI file.
'vcs.start_server': 'false',
'vcs.server.protocol': 'http',
'vcs.scm_app_implementation': 'http',
'vcs.svn.proxy.enabled': 'true',
'vcs.hooks.protocol.v2': 'celery',
'vcs.hooks.host': '*',
'repo_store.path': TESTS_TMP_PATH,
'app.service_api.token': 'service_secret_token',
}},
{'handler_console': {
'class': 'StreamHandler',
'args': '(sys.stderr,)',
'level': log_level,
}},
]
filename = get_config(
request.config, option_name=option_name,
override_option_name='{}_override'.format(option_name),
overrides=overrides,
basetemp=tmpdir_factory.getbasetemp().strpath,
prefix='test_rce_')
return filename
@pytest.fixture(scope='session')
def ini_settings(ini_config):
ini_path = ini_config
return get_app_config(ini_path)
def get_available_port(min_port=40000, max_port=55555):
from rhodecode.lib.utils2 import get_available_port as _get_port
return _get_port(min_port, max_port)
@pytest.fixture(scope='session')
def rcserver_port(request):
port = get_available_port()
print(f'Using rhodecode port {port}')
return port
@pytest.fixture(scope='session')
def vcsserver_port(request):
port = request.config.getoption('--vcsserver-port')
if port is None:
port = get_available_port()
print(f'Using vcsserver port {port}')
return port
@pytest.fixture(scope='session')
def available_port_factory() -> get_available_port:
"""
Returns a callable which returns free port numbers.
"""
return get_available_port
@pytest.fixture()
def available_port(available_port_factory):
"""
Gives you one free port for the current test.
Uses "available_port_factory" to retrieve the port.
"""
return available_port_factory()
@pytest.fixture(scope='session')
def testini_factory(tmpdir_factory, ini_config):
"""
Factory to create an INI file based on TestINI.
It will make sure to place the INI file in the correct directory.
"""
basetemp = tmpdir_factory.getbasetemp().strpath
return TestIniFactory(basetemp, ini_config)
class TestIniFactory(object):
def __init__(self, basetemp, template_ini):
self._basetemp = basetemp
self._template_ini = template_ini
def __call__(self, ini_params, new_file_prefix='test'):
ini_file = TestINI(
self._template_ini, ini_params=ini_params,
new_file_prefix=new_file_prefix, dir=self._basetemp)
result = ini_file.create()
return result
def get_config(
config, option_name, override_option_name, overrides=None,
basetemp=None, prefix='test'):
"""
Find a configuration file and apply overrides for the given `prefix`.
"""
config_file = (
config.getoption(option_name) or config.getini(option_name))
if not config_file:
pytest.exit(
"Configuration error, could not extract {}.".format(option_name))
overrides = overrides or []
config_override = config.getoption(override_option_name)
if config_override:
overrides.append(config_override)
temp_ini_file = TestINI(
config_file, ini_params=overrides, new_file_prefix=prefix,
dir=basetemp)
return temp_ini_file.create()

0
rhodecode/tests/fixture_mods/__init__.py → rhodecode/tests/fixtures/__init__.py vendored
View file

0
rhodecode/tests/fixtures/diff_with_diff_data.diff → rhodecode/tests/fixtures/diff_fixtures/diff_with_diff_data.diff vendored
View file

0
rhodecode/tests/fixtures/git_diff_binary_and_normal.diff → rhodecode/tests/fixtures/diff_fixtures/git_diff_binary_and_normal.diff vendored
View file

0
rhodecode/tests/fixtures/git_diff_binary_special_files.diff → rhodecode/tests/fixtures/diff_fixtures/git_diff_binary_special_files.diff vendored
View file

0
rhodecode/tests/fixtures/git_diff_binary_special_files_2.diff → rhodecode/tests/fixtures/diff_fixtures/git_diff_binary_special_files_2.diff vendored
View file

0
rhodecode/tests/fixtures/git_diff_chmod.diff → rhodecode/tests/fixtures/diff_fixtures/git_diff_chmod.diff vendored
View file

0
rhodecode/tests/fixtures/git_diff_js_chars.diff → rhodecode/tests/fixtures/diff_fixtures/git_diff_js_chars.diff vendored
View file

0
rhodecode/tests/fixtures/git_diff_mod_single_binary_file.diff → rhodecode/tests/fixtures/diff_fixtures/git_diff_mod_single_binary_file.diff vendored
View file

0
rhodecode/tests/fixtures/git_diff_rename_file.diff → rhodecode/tests/fixtures/diff_fixtures/git_diff_rename_file.diff vendored
View file

0
rhodecode/tests/fixtures/git_diff_rename_file_with_spaces.diff → rhodecode/tests/fixtures/diff_fixtures/git_diff_rename_file_with_spaces.diff vendored
View file

0
rhodecode/tests/fixtures/git_node_history_response.json → rhodecode/tests/fixtures/diff_fixtures/git_node_history_response.json vendored
View file

0
rhodecode/tests/fixtures/hg_diff_add_single_binary_file.diff → rhodecode/tests/fixtures/diff_fixtures/hg_diff_add_single_binary_file.diff vendored
View file

0
rhodecode/tests/fixtures/hg_diff_binary_and_normal.diff → rhodecode/tests/fixtures/diff_fixtures/hg_diff_binary_and_normal.diff vendored
View file

0
rhodecode/tests/fixtures/hg_diff_chmod.diff → rhodecode/tests/fixtures/diff_fixtures/hg_diff_chmod.diff vendored
View file

Some files were not shown because too many files have changed in this diff Show more