engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ssh-wrapper-speedup): major rewrite of code to address imports problem with ssh-wrapper-v2

Browse source 
- use bootstrapped settings rather than config
- use more code split to make sure we don't import heavy code
This commit is contained in:
RhodeCode Admin 2024-02-28 09:30:58 +01:00
parent d57482be62
commit fc0ee0e99b
33 changed files with 764 additions and 562 deletions
Download patch file Download diff file Expand all files Collapse all files

63
rhodecode/apps/ssh_support/lib/backends/__init__.py
View file

@ -20,12 +20,11 @@ import os
import re
import logging
import datetime
import configparser
from sqlalchemy import Table
from rhodecode.lib.utils import call_service_api
from rhodecode.lib.api_utils import call_service_api
from rhodecode.lib.utils2 import AttributeDict
from rhodecode.model.scm import ScmModel
from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
from .hg import MercurialServer
from .git import GitServer
@ -39,7 +38,7 @@ class SshWrapper(object):
svn_cmd_pat = re.compile(r'^svnserve -t')
def __init__(self, command, connection_info, mode,
user, user_id, key_id: int, shell, ini_path: str, env):
user, user_id, key_id: int, shell, ini_path: str, settings, env):
self.command = command
self.connection_info = connection_info
self.mode = mode
@ -49,15 +48,9 @@ class SshWrapper(object):
self.shell = shell
self.ini_path = ini_path
self.env = env
self.config = self.parse_config(ini_path)
self.settings = settings
self.server_impl = None
def parse_config(self, config_path):
parser = configparser.ConfigParser()
parser.read(config_path)
return parser
def update_key_access_time(self, key_id):
from rhodecode.model.meta import raw_query_executor, Base
@ -162,6 +155,9 @@ class SshWrapper(object):
return vcs_type, repo_name, mode
def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
# TODO: remove this once we have .ini defined access path...
from rhodecode.model.scm import ScmModel
store = ScmModel().repos_path
check_branch_perms = False
@ -186,7 +182,7 @@ class SshWrapper(object):
server = MercurialServer(
store=store, ini_path=self.ini_path,
repo_name=repo, user=user,
user_permissions=permissions, config=self.config, env=self.env)
user_permissions=permissions, settings=self.settings, env=self.env)
self.server_impl = server
return server.run(tunnel_extras=extras)
@ -194,7 +190,7 @@ class SshWrapper(object):
server = GitServer(
store=store, ini_path=self.ini_path,
repo_name=repo, repo_mode=mode, user=user,
user_permissions=permissions, config=self.config, env=self.env)
user_permissions=permissions, settings=self.settings, env=self.env)
self.server_impl = server
return server.run(tunnel_extras=extras)
@ -202,7 +198,7 @@ class SshWrapper(object):
server = SubversionServer(
store=store, ini_path=self.ini_path,
repo_name=None, user=user,
user_permissions=permissions, config=self.config, env=self.env)
user_permissions=permissions, settings=self.settings, env=self.env)
self.server_impl = server
return server.run(tunnel_extras=extras)
@ -269,6 +265,35 @@ class SshWrapperStandalone(SshWrapper):
New version of SshWrapper designed to be depended only on service API
"""
repos_path = None
service_api_host: str
service_api_token: str
api_url: str
def __init__(self, command, connection_info, mode,
user, user_id, key_id: int, shell, ini_path: str, settings, env):
# validate our settings for making a standalone calls
try:
self.service_api_host = settings['app.service_api.host']
self.service_api_token = settings['app.service_api.token']
except KeyError:
raise ImproperlyConfiguredError(
"app.service_api.host or app.service_api.token are missing. "
"Please ensure that app.service_api.host and app.service_api.token are "
"defined inside of .ini configuration file."
)
try:
self.api_url = settings['rhodecode.api.url']
except KeyError:
raise ImproperlyConfiguredError(
"rhodecode.api.url is missing. "
"Please ensure that rhodecode.api.url is "
"defined inside of .ini configuration file."
)
super(SshWrapperStandalone, self).__init__(
command, connection_info, mode, user, user_id, key_id, shell, ini_path, settings, env)
@staticmethod
def parse_user_related_data(user_data):
@ -301,7 +326,7 @@ class SshWrapperStandalone(SshWrapper):
exit_code = 1
elif scm_detected:
data = call_service_api(self.ini_path, {
data = call_service_api(self.service_api_host, self.service_api_token, self.api_url, {
"method": "service_get_data_for_ssh_wrapper",
"args": {"user_id": user_id, "repo_name": scm_repo, "key_id": self.key_id}
})
@ -339,7 +364,7 @@ class SshWrapperStandalone(SshWrapper):
if repo_name.startswith('_'):
org_repo_name = repo_name
log.debug('translating UID repo %s', org_repo_name)
by_id_match = call_service_api(self.ini_path, {
by_id_match = call_service_api(self.service_api_host, self.service_api_token, self.api_url, {
'method': 'service_get_repo_name_by_id',
"args": {"repo_id": repo_name}
})
@ -375,17 +400,17 @@ class SshWrapperStandalone(SshWrapper):
server = MercurialServer(
store=store, ini_path=self.ini_path,
repo_name=repo, user=user,
user_permissions=permissions, config=self.config, env=self.env)
user_permissions=permissions, settings=self.settings, env=self.env)
case 'git':
server = GitServer(
store=store, ini_path=self.ini_path,
repo_name=repo, repo_mode=mode, user=user,
user_permissions=permissions, config=self.config, env=self.env)
user_permissions=permissions, settings=self.settings, env=self.env)
case 'svn':
server = SubversionServer(
store=store, ini_path=self.ini_path,
repo_name=None, user=user,
user_permissions=permissions, config=self.config, env=self.env)
user_permissions=permissions, settings=self.settings, env=self.env)
case _:
raise Exception(f'Unrecognised VCS: {vcs}')
self.server_impl = server

23
rhodecode/apps/ssh_support/lib/backends/base.py
View file

@ -20,27 +20,27 @@ import os
import sys
import logging
from rhodecode.lib.hooks_daemon import prepare_callback_daemon
from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
from rhodecode.lib.ext_json import sjson as json
from rhodecode.lib.vcs.conf import settings as vcs_settings
from rhodecode.lib.utils import call_service_api
from rhodecode.model.scm import ScmModel
from rhodecode.lib.api_utils import call_service_api
log = logging.getLogger(__name__)
class VcsServer(object):
class SSHVcsServer(object):
repo_user_agent = None # set in child classes
_path = None # set executable path for hg/git/svn binary
backend = None # set in child classes
tunnel = None # subprocess handling tunnel
settings = None # parsed settings module
write_perms = ['repository.admin', 'repository.write']
read_perms = ['repository.read', 'repository.admin', 'repository.write']
def __init__(self, user, user_permissions, config, env):
def __init__(self, user, user_permissions, settings, env):
self.user = user
self.user_permissions = user_permissions
self.config = config
self.settings = settings
self.env = env
self.stdin = sys.stdin
@ -59,9 +59,14 @@ class VcsServer(object):
# Todo: Leave only "celery" case after transition.
match self.hooks_protocol:
case 'http':
from rhodecode.model.scm import ScmModel
ScmModel().mark_for_invalidation(repo_name)
case 'celery':
call_service_api(self.ini_path, {
service_api_host = self.settings['app.service_api.host']
service_api_token = self.settings['app.service_api.token']
api_url = self.settings['rhodecode.api.url']
call_service_api(service_api_host, service_api_token, api_url, {
"method": "service_mark_for_invalidation",
"args": {"repo_name": repo_name}
})
@ -118,7 +123,7 @@ class VcsServer(object):
'server_url': None,
'user_agent': f'{self.repo_user_agent}/ssh-user-agent',
'hooks': ['push', 'pull'],
'hooks_module': 'rhodecode.lib.hooks_daemon',
'hooks_module': 'rhodecode.lib.hook_daemon.hook_module',
'is_shadow_repo': False,
'detect_force_push': False,
'check_branch_perms': False,
@ -156,7 +161,7 @@ class VcsServer(object):
return exit_code, action == "push"
def run(self, tunnel_extras=None):
self.hooks_protocol = self.config.get('app:main', 'vcs.hooks.protocol')
self.hooks_protocol = self.settings['vcs.hooks.protocol']
tunnel_extras = tunnel_extras or {}
extras = {}
extras.update(tunnel_extras)

12
rhodecode/apps/ssh_support/lib/backends/git.py
View file

@ -21,7 +21,7 @@ import logging
import subprocess
from vcsserver import hooks
from .base import VcsServer
from .base import SSHVcsServer
log = logging.getLogger(__name__)
@ -70,19 +70,17 @@ class GitTunnelWrapper(object):
return result
class GitServer(VcsServer):
class GitServer(SSHVcsServer):
backend = 'git'
repo_user_agent = 'git'
def __init__(self, store, ini_path, repo_name, repo_mode,
user, user_permissions, config, env):
super().\
__init__(user, user_permissions, config, env)
def __init__(self, store, ini_path, repo_name, repo_mode, user, user_permissions, settings, env):
super().__init__(user, user_permissions, settings, env)
self.store = store
self.ini_path = ini_path
self.repo_name = repo_name
self._path = self.git_path = config.get('app:main', 'ssh.executable.git')
self._path = self.git_path = settings['ssh.executable.git']
self.repo_mode = repo_mode
self.tunnel = GitTunnelWrapper(server=self)

85
rhodecode/apps/ssh_support/lib/backends/hg.py
View file

@ -22,10 +22,10 @@ import logging
import tempfile
import textwrap
import collections
from .base import VcsServer
from rhodecode.lib.utils import call_service_api
from rhodecode.model.db import RhodeCodeUi
from rhodecode.model.settings import VcsSettingsModel
from .base import SSHVcsServer
from rhodecode.lib.api_utils import call_service_api
log = logging.getLogger(__name__)
@ -57,7 +57,7 @@ class MercurialTunnelWrapper(object):
# cleanup custom hgrc file
if os.path.isfile(hgrc_custom):
with open(hgrc_custom, 'wb') as f:
f.write('')
f.write(b'')
log.debug('Cleanup custom hgrc file under %s', hgrc_custom)
# write temp
@ -94,62 +94,67 @@ class MercurialTunnelWrapper(object):
self.remove_configs()
class MercurialServer(VcsServer):
class MercurialServer(SSHVcsServer):
backend = 'hg'
repo_user_agent = 'mercurial'
cli_flags = ['phases', 'largefiles', 'extensions', 'experimental', 'hooks']
def __init__(self, store, ini_path, repo_name, user, user_permissions, config, env):
super().__init__(user, user_permissions, config, env)
def __init__(self, store, ini_path, repo_name, user, user_permissions, settings, env):
super().__init__(user, user_permissions, settings, env)
self.store = store
self.ini_path = ini_path
self.repo_name = repo_name
self._path = self.hg_path = config.get('app:main', 'ssh.executable.hg')
self._path = self.hg_path = settings['ssh.executable.hg']
self.tunnel = MercurialTunnelWrapper(server=self)
def config_to_hgrc(self, repo_name):
# Todo: once transition is done only call to service api should exist
if self.hooks_protocol == 'celery':
data = call_service_api(self.ini_path, {
service_api_host = self.settings['app.service_api.host']
service_api_token = self.settings['app.service_api.token']
api_url = self.settings['rhodecode.api.url']
data = call_service_api(service_api_host, service_api_token, api_url, {
"method": "service_config_to_hgrc",
"args": {"cli_flags": self.cli_flags, "repo_name": repo_name}
})
return data['flags']
else:
from rhodecode.model.db import RhodeCodeUi
from rhodecode.model.settings import VcsSettingsModel
ui_sections = collections.defaultdict(list)
ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
ui_sections = collections.defaultdict(list)
ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
# write default hooks
default_hooks = [
('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
# write default hooks
default_hooks = [
('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
]
('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
]
for k, v in default_hooks:
ui_sections['hooks'].append((k, v))
for k, v in default_hooks:
ui_sections['hooks'].append((k, v))
for entry in ui:
if not entry.active:
continue
sec = entry.section
key = entry.key
if sec in self.cli_flags:
# we want only custom hooks, so we skip builtins
if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
for entry in ui:
if not entry.active:
continue
sec = entry.section
key = entry.key
ui_sections[sec].append([key, entry.value])
if sec in self.cli_flags:
# we want only custom hooks, so we skip builtins
if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
continue
flags = []
for _sec, key_val in ui_sections.items():
flags.append(' ')
flags.append(f'[{_sec}]')
for key, val in key_val:
flags.append(f'{key}= {val}')
return flags
ui_sections[sec].append([key, entry.value])
flags = []
for _sec, key_val in ui_sections.items():
flags.append(' ')
flags.append(f'[{_sec}]')
for key, val in key_val:
flags.append(f'{key}= {val}')
return flags

12
rhodecode/apps/ssh_support/lib/backends/svn.py
View file

@ -25,7 +25,7 @@ import tempfile
from subprocess import Popen, PIPE
import urllib.parse
from .base import VcsServer
from .base import SSHVcsServer
log = logging.getLogger(__name__)
@ -218,20 +218,18 @@ class SubversionTunnelWrapper(object):
return self.return_code
class SubversionServer(VcsServer):
class SubversionServer(SSHVcsServer):
backend = 'svn'
repo_user_agent = 'svn'
def __init__(self, store, ini_path, repo_name,
user, user_permissions, config, env):
super()\
.__init__(user, user_permissions, config, env)
def __init__(self, store, ini_path, repo_name, user, user_permissions, settings, env):
super().__init__(user, user_permissions, settings, env)
self.store = store
self.ini_path = ini_path
# NOTE(dan): repo_name at this point is empty,
# this is set later in .run() based from parsed input stream
self.repo_name = repo_name
self._path = self.svn_path = config.get('app:main', 'ssh.executable.svn')
self._path = self.svn_path = settings['ssh.executable.svn']
self.tunnel = SubversionTunnelWrapper(server=self)

4
rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py
View file

@ -31,7 +31,6 @@ from .utils import setup_custom_logging
log = logging.getLogger(__name__)
@click.command()
@click.argument('ini_path', type=click.Path(exists=True))
@click.option(
@ -55,11 +54,12 @@ def main(ini_path, mode, user, user_id, key_id, shell, debug):
connection_info = os.environ.get('SSH_CONNECTION', '')
time_start = time.time()
with bootstrap(ini_path, env={'RC_CMD_SSH_WRAPPER': '1'}) as env:
settings = env['registry'].settings
statsd = StatsdClient.statsd
try:
ssh_wrapper = SshWrapper(
command, connection_info, mode,
user, user_id, key_id, shell, ini_path, env)
user, user_id, key_id, shell, ini_path, settings, env)
except Exception:
log.exception('Failed to execute SshWrapper')
sys.exit(-5)

32
rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
View file

@ -16,6 +16,14 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
"""
WARNING: be really carefully with changing ANY imports in this file
# This script is to mean as really fast executable, doing some imports here that would yield an import chain change
# can affect execution times...
# This can be easily debugged using such command::
# time PYTHONPROFILEIMPORTTIME=1 rc-ssh-wrapper-v2 --debug --mode=test .dev/dev.ini
"""
import os
import sys
import time
@ -23,9 +31,12 @@ import logging
import click
from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
from rhodecode.lib.statsd_client import StatsdClient
from .backends import SshWrapperStandalone
from rhodecode.lib.config_utils import get_app_config_lightweight
from .utils import setup_custom_logging
from .backends import SshWrapperStandalone
log = logging.getLogger(__name__)
@ -42,6 +53,8 @@ log = logging.getLogger(__name__)
@click.option('--shell', '-s', is_flag=True, help='Allow Shell')
@click.option('--debug', is_flag=True, help='Enabled detailed output logging')
def main(ini_path, mode, user, user_id, key_id, shell, debug):
time_start = time.time()
setup_custom_logging(ini_path, debug)
command = os.environ.get('SSH_ORIGINAL_COMMAND', '')
@ -50,21 +63,30 @@ def main(ini_path, mode, user, user_id, key_id, shell, debug):
'Unable to fetch SSH_ORIGINAL_COMMAND from environment.'
'Please make sure this is set and available during execution '
'of this script.')
connection_info = os.environ.get('SSH_CONNECTION', '')
time_start = time.time()
env = {'RC_CMD_SSH_WRAPPER': '1'}
# initialize settings and get defaults
settings = get_app_config_lightweight(ini_path)
settings = sanitize_settings_and_apply_defaults({'__file__': ini_path}, settings)
# init and bootstrap StatsdClient
StatsdClient.setup(settings)
statsd = StatsdClient.statsd
try:
connection_info = os.environ.get('SSH_CONNECTION', '')
env = {'RC_CMD_SSH_WRAPPER': '1'}
ssh_wrapper = SshWrapperStandalone(
command, connection_info, mode,
user, user_id, key_id, shell, ini_path, env)
user, user_id, key_id, shell, ini_path, settings, env)
except Exception:
log.exception('Failed to execute SshWrapper')
sys.exit(-5)
return_code = ssh_wrapper.wrap()
operation_took = time.time() - time_start
if statsd:
operation_took_ms = round(1000.0 * operation_took)
statsd.timing("rhodecode_ssh_wrapper_timing.histogram", operation_took_ms,
use_decimals=False)
sys.exit(return_code)

2
rhodecode/apps/ssh_support/lib/utils.py
View file

@ -17,11 +17,11 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
from pyramid.paster import setup_logging
def setup_custom_logging(ini_path, debug):
if debug:
from pyramid.paster import setup_logging # Lazy import
# enabled rhodecode.ini controlled logging setup
setup_logging(ini_path)
else:

7
rhodecode/apps/ssh_support/tests/conftest.py
View file

@ -52,7 +52,10 @@ def dummy_env():
def plain_dummy_user():
return AttributeDict(username='test_user')
return AttributeDict(
user_id=1,
username='test_user'
)
@pytest.fixture()
@ -65,4 +68,4 @@ def ssh_wrapper(app, dummy_conf_file, dummy_env):
conn_info = '127.0.0.1 22 10.0.0.1 443'
return SshWrapper(
'random command', conn_info, 'auto', 'admin', '1', key_id='1',
shell=False, ini_path=dummy_conf_file, env=dummy_env)
shell=False, ini_path=dummy_conf_file, settings={}, env=dummy_env)

10
rhodecode/apps/ssh_support/tests/test_server_git.py
View file

@ -25,6 +25,7 @@ from rhodecode.apps.ssh_support.lib.backends.git import GitServer
from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dummy_user
from rhodecode.lib.ext_json import json
class GitServerCreator(object):
root = '/tmp/repo/path/'
git_path = '/usr/local/bin/git'
@ -39,10 +40,7 @@ class GitServerCreator(object):
user = plain_dummy_user()
def __init__(self):
def config_get(part, key):
return self.config_data.get(part, {}).get(key)
self.config_mock = mock.Mock()
self.config_mock.get = mock.Mock(side_effect=config_get)
pass
def create(self, **kwargs):
parameters = {
@ -54,7 +52,7 @@ class GitServerCreator(object):
'user_permissions': {
self.repo_name: 'repository.admin'
},
'config': self.config_mock,
'settings': self.config_data['app:main'],
'env': plain_dummy_env()
}
parameters.update(kwargs)
@ -142,7 +140,7 @@ class TestGitServer(object):
'server_url': None,
'hooks': ['push', 'pull'],
'is_shadow_repo': False,
'hooks_module': 'rhodecode.lib.hooks_daemon',
'hooks_module': 'rhodecode.lib.hook_daemon.hook_module',
'check_branch_perms': False,
'detect_force_push': False,
'user_agent': u'git/ssh-user-agent',

7
rhodecode/apps/ssh_support/tests/test_server_hg.py
View file

@ -38,10 +38,7 @@ class MercurialServerCreator(object):
user = plain_dummy_user()
def __init__(self):
def config_get(part, key):
return self.config_data.get(part, {}).get(key)
self.config_mock = mock.Mock()
self.config_mock.get = mock.Mock(side_effect=config_get)
pass
def create(self, **kwargs):
parameters = {
@ -52,7 +49,7 @@ class MercurialServerCreator(object):
'user_permissions': {
'test_hg': 'repository.admin'
},
'config': self.config_mock,
'settings': self.config_data['app:main'],
'env': plain_dummy_env()
}
parameters.update(kwargs)

7
rhodecode/apps/ssh_support/tests/test_server_svn.py
View file

@ -36,10 +36,7 @@ class SubversionServerCreator(object):
user = plain_dummy_user()
def __init__(self):
def config_get(part, key):
return self.config_data.get(part, {}).get(key)
self.config_mock = mock.Mock()
self.config_mock.get = mock.Mock(side_effect=config_get)
pass
def create(self, **kwargs):
parameters = {
@ -50,7 +47,7 @@ class SubversionServerCreator(object):
'user_permissions': {
self.repo_name: 'repository.admin'
},
'config': self.config_mock,
'settings': self.config_data['app:main'],
'env': plain_dummy_env()
}

4
rhodecode/apps/ssh_support/tests/test_ssh_wrapper.py
View file

@ -28,10 +28,6 @@ class TestSSHWrapper(object):
permissions={}, branch_permissions={})
assert str(exc_info.value) == 'Unrecognised VCS: microsoft-tfs'
def test_parse_config(self, ssh_wrapper):
config = ssh_wrapper.parse_config(ssh_wrapper.ini_path)
assert config
def test_get_connection_info(self, ssh_wrapper):
conn_info = ssh_wrapper.get_connection_info()
assert {'client_ip': '127.0.0.1',

198
rhodecode/config/config_maker.py Normal file
View file

@ -0,0 +1,198 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import tempfile
import logging
from pyramid.settings import asbool
from rhodecode.config.settings_maker import SettingsMaker
from rhodecode.config import utils as config_utils
log = logging.getLogger(__name__)
def sanitize_settings_and_apply_defaults(global_config, settings):
"""
Applies settings defaults and does all type conversion.
We would move all settings parsing and preparation into this place, so that
we have only one place left which deals with this part. The remaining parts
of the application would start to rely fully on well-prepared settings.
This piece would later be split up per topic to avoid a big fat monster
function.
"""
jn = os.path.join
global_settings_maker = SettingsMaker(global_config)
global_settings_maker.make_setting('debug', default=False, parser='bool')
debug_enabled = asbool(global_config.get('debug'))
settings_maker = SettingsMaker(settings)
settings_maker.make_setting(
'logging.autoconfigure',
default=False,
parser='bool')
logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
# Default includes, possible to change as a user
pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
log.debug(
"Using the following pyramid.includes: %s",
pyramid_includes)
settings_maker.make_setting('rhodecode.edition', 'Community Edition')
settings_maker.make_setting('rhodecode.edition_id', 'CE')
if 'mako.default_filters' not in settings:
# set custom default filters if we don't have it defined
settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
settings['mako.default_filters'] = 'h_filter'
if 'mako.directories' not in settings:
mako_directories = settings.setdefault('mako.directories', [
# Base templates of the original application
'rhodecode:templates',
])
log.debug(
"Using the following Mako template directories: %s",
mako_directories)
# NOTE(marcink): fix redis requirement for schema of connection since 3.X
if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
raw_url = settings['beaker.session.url']
if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
settings['beaker.session.url'] = 'redis://' + raw_url
settings_maker.make_setting('__file__', global_config.get('__file__'))
# TODO: johbo: Re-think this, usually the call to config.include
# should allow to pass in a prefix.
settings_maker.make_setting('rhodecode.api.url', '/_admin/api')
# Sanitize generic settings.
settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
settings_maker.make_setting('is_test', False, parser='bool')
settings_maker.make_setting('gzip_responses', False, parser='bool')
# statsd
settings_maker.make_setting('statsd.enabled', False, parser='bool')
settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
settings_maker.make_setting('statsd.statsd_prefix', '')
settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
settings_maker.make_setting('vcs.svn.compatible_version', '')
settings_maker.make_setting('vcs.hooks.protocol', 'http')
settings_maker.make_setting('vcs.hooks.host', '*')
settings_maker.make_setting('vcs.scm_app_implementation', 'http')
settings_maker.make_setting('vcs.server', '')
settings_maker.make_setting('vcs.server.protocol', 'http')
settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
# Support legacy values of vcs.scm_app_implementation. Legacy
# configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
# disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
scm_app_impl = settings['vcs.scm_app_implementation']
if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
settings['vcs.scm_app_implementation'] = 'http'
settings_maker.make_setting('appenlight', False, parser='bool')
temp_store = tempfile.gettempdir()
tmp_cache_dir = jn(temp_store, 'rc_cache')
# save default, cache dir, and use it for all backends later.
default_cache_dir = settings_maker.make_setting(
'cache_dir',
default=tmp_cache_dir, default_when_empty=True,
parser='dir:ensured')
# exception store cache
settings_maker.make_setting(
'exception_tracker.store_path',
default=jn(default_cache_dir, 'exc_store'), default_when_empty=True,
parser='dir:ensured'
)
settings_maker.make_setting(
'celerybeat-schedule.path',
default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
parser='file:ensured'
)
settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
# sessions, ensure file since no-value is memory
settings_maker.make_setting('beaker.session.type', 'file')
settings_maker.make_setting('beaker.session.data_dir', jn(default_cache_dir, 'session_data'))
# cache_general
settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db'))
# cache_perms
settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db'))
# cache_repo
settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db'))
# cache_license
settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db'))
# cache_repo_longterm memory, 96H
settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
# sql_cache_short
settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
# archive_cache
settings_maker.make_setting('archive_cache.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
settings_maker.make_setting('archive_cache.cache_size_gb', 10, parser='float')
settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int')
settings_maker.env_expand()
# configure instance id
config_utils.set_instance_id(settings)
return settings

177
rhodecode/config/middleware.py
View file

@ -19,7 +19,7 @@
import os
import sys
import collections
import tempfile
import time
import logging.config
@ -32,14 +32,13 @@ from pyramid.httpexceptions import (
HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
from pyramid.renderers import render_to_response
from rhodecode import api
from rhodecode.model import meta
from rhodecode.config import patches
from rhodecode.config import utils as config_utils
from rhodecode.config.settings_maker import SettingsMaker
from rhodecode.config.environment import load_pyramid_environment
import rhodecode.events
from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
from rhodecode.lib.middleware.vcs import VCSMiddleware
from rhodecode.lib.request import Request
from rhodecode.lib.vcs import VCSCommunicationError
@ -465,173 +464,3 @@ def wrap_app_in_wsgi_middlewares(pyramid_app, config):
log.debug('Request processing finalized: %.4fs', total)
return pyramid_app_with_cleanup
def sanitize_settings_and_apply_defaults(global_config, settings):
"""
Applies settings defaults and does all type conversion.
We would move all settings parsing and preparation into this place, so that
we have only one place left which deals with this part. The remaining parts
of the application would start to rely fully on well prepared settings.
This piece would later be split up per topic to avoid a big fat monster
function.
"""
jn = os.path.join
global_settings_maker = SettingsMaker(global_config)
global_settings_maker.make_setting('debug', default=False, parser='bool')
debug_enabled = asbool(global_config.get('debug'))
settings_maker = SettingsMaker(settings)
settings_maker.make_setting(
'logging.autoconfigure',
default=False,
parser='bool')
logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
# Default includes, possible to change as a user
pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
log.debug(
"Using the following pyramid.includes: %s",
pyramid_includes)
settings_maker.make_setting('rhodecode.edition', 'Community Edition')
settings_maker.make_setting('rhodecode.edition_id', 'CE')
if 'mako.default_filters' not in settings:
# set custom default filters if we don't have it defined
settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
settings['mako.default_filters'] = 'h_filter'
if 'mako.directories' not in settings:
mako_directories = settings.setdefault('mako.directories', [
# Base templates of the original application
'rhodecode:templates',
])
log.debug(
"Using the following Mako template directories: %s",
mako_directories)
# NOTE(marcink): fix redis requirement for schema of connection since 3.X
if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
raw_url = settings['beaker.session.url']
if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
settings['beaker.session.url'] = 'redis://' + raw_url
settings_maker.make_setting('__file__', global_config.get('__file__'))
# TODO: johbo: Re-think this, usually the call to config.include
# should allow to pass in a prefix.
settings_maker.make_setting('rhodecode.api.url', api.DEFAULT_URL)
# Sanitize generic settings.
settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
settings_maker.make_setting('is_test', False, parser='bool')
settings_maker.make_setting('gzip_responses', False, parser='bool')
# statsd
settings_maker.make_setting('statsd.enabled', False, parser='bool')
settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
settings_maker.make_setting('statsd.statsd_prefix', '')
settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
settings_maker.make_setting('vcs.svn.compatible_version', '')
settings_maker.make_setting('vcs.hooks.protocol', 'http')
settings_maker.make_setting('vcs.hooks.host', '*')
settings_maker.make_setting('vcs.scm_app_implementation', 'http')
settings_maker.make_setting('vcs.server', '')
settings_maker.make_setting('vcs.server.protocol', 'http')
settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
# Support legacy values of vcs.scm_app_implementation. Legacy
# configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
# disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
scm_app_impl = settings['vcs.scm_app_implementation']
if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
settings['vcs.scm_app_implementation'] = 'http'
settings_maker.make_setting('appenlight', False, parser='bool')
temp_store = tempfile.gettempdir()
tmp_cache_dir = jn(temp_store, 'rc_cache')
# save default, cache dir, and use it for all backends later.
default_cache_dir = settings_maker.make_setting(
'cache_dir',
default=tmp_cache_dir, default_when_empty=True,
parser='dir:ensured')
# exception store cache
settings_maker.make_setting(
'exception_tracker.store_path',
default=jn(default_cache_dir, 'exc_store'), default_when_empty=True,
parser='dir:ensured'
)
settings_maker.make_setting(
'celerybeat-schedule.path',
default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
parser='file:ensured'
)
settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
# sessions, ensure file since no-value is memory
settings_maker.make_setting('beaker.session.type', 'file')
settings_maker.make_setting('beaker.session.data_dir', jn(default_cache_dir, 'session_data'))
# cache_general
settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db'))
# cache_perms
settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db'))
# cache_repo
settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db'))
# cache_license
settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db'))
# cache_repo_longterm memory, 96H
settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
# sql_cache_short
settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
# archive_cache
settings_maker.make_setting('archive_cache.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
settings_maker.make_setting('archive_cache.cache_size_gb', 10, parser='float')
settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int')
settings_maker.env_expand()
# configure instance id
config_utils.set_instance_id(settings)
return settings

3
rhodecode/config/utils.py
View file

@ -19,8 +19,6 @@
import os
import platform
from rhodecode.model import init_model
def configure_vcs(config):
"""
@ -44,6 +42,7 @@ def configure_vcs(config):
def initialize_database(config):
from rhodecode.lib.utils2 import engine_from_config, get_encryption_key
from rhodecode.model import init_model
engine = engine_from_config(config, 'sqlalchemy.db1.')
init_model(engine, encryption_key=get_encryption_key(config))

38
rhodecode/lib/api_utils.py Normal file
View file

@ -0,0 +1,38 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import urllib.parse
from rhodecode.lib.vcs import CurlSession
from rhodecode.lib.ext_json import json
def call_service_api(service_api_host, service_api_token, api_url, payload):
payload.update({
'id': 'service',
'auth_token': service_api_token
})
service_api_url = urllib.parse.urljoin(service_api_host, api_url)
response = CurlSession().post(service_api_url, json.dumps(payload))
if response.status_code != 200:
raise Exception(f"Service API at {service_api_url} responded with error: {response.status_code}")
return json.loads(response.content)['result']

2
rhodecode/lib/base.py
View file

@ -567,7 +567,7 @@ def add_events_routes(config):
def bootstrap_config(request, registry_name='RcTestRegistry'):
from rhodecode.config.middleware import sanitize_settings_and_apply_defaults
from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
import pyramid.testing
registry = pyramid.testing.Registry(registry_name)

40
rhodecode/lib/config_utils.py Normal file
View file

@ -0,0 +1,40 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
def get_config(ini_path, **kwargs):
import configparser
parser = configparser.ConfigParser(**kwargs)
parser.read(ini_path)
return parser
def get_app_config_lightweight(ini_path):
parser = get_config(ini_path)
parser.set('app:main', 'here', os.getcwd())
parser.set('app:main', '__file__', ini_path)
return dict(parser.items('app:main'))
def get_app_config(ini_path):
"""
This loads the app context and provides a heavy type iniliaziation of config
"""
from paste.deploy.loadwsgi import appconfig
return appconfig(f'config:{ini_path}', relative_to=os.getcwd())

17
rhodecode/lib/hook_daemon/__init__.py Normal file
View file

@ -0,0 +1,17 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/

115
rhodecode/lib/hook_daemon/base.py Normal file
View file

@ -0,0 +1,115 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import time
import logging
import tempfile
from rhodecode.lib.config_utils import get_config
from rhodecode.lib.ext_json import json
log = logging.getLogger(__name__)
class BaseHooksCallbackDaemon:
"""
Basic context manager for actions that don't require some extra
"""
def __init__(self):
pass
def __enter__(self):
log.debug('Running `%s` callback daemon', self.__class__.__name__)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
class HooksModuleCallbackDaemon(BaseHooksCallbackDaemon):
def __init__(self, module):
super().__init__()
self.hooks_module = module
def get_txn_id_data_path(txn_id):
import rhodecode
root = rhodecode.CONFIG.get('cache_dir') or tempfile.gettempdir()
final_dir = os.path.join(root, 'svn_txn_id')
if not os.path.isdir(final_dir):
os.makedirs(final_dir)
return os.path.join(final_dir, 'rc_txn_id_{}'.format(txn_id))
def store_txn_id_data(txn_id, data_dict):
if not txn_id:
log.warning('Cannot store txn_id because it is empty')
return
path = get_txn_id_data_path(txn_id)
try:
with open(path, 'wb') as f:
f.write(json.dumps(data_dict))
except Exception:
log.exception('Failed to write txn_id metadata')
def get_txn_id_from_store(txn_id):
"""
Reads txn_id from store and if present returns the data for callback manager
"""
path = get_txn_id_data_path(txn_id)
try:
with open(path, 'rb') as f:
return json.loads(f.read())
except Exception:
return {}
def prepare_callback_daemon(extras, protocol, host, txn_id=None):
txn_details = get_txn_id_from_store(txn_id)
port = txn_details.get('port', 0)
match protocol:
case 'http':
from rhodecode.lib.hook_daemon.http_hooks_deamon import HttpHooksCallbackDaemon
callback_daemon = HttpHooksCallbackDaemon(
txn_id=txn_id, host=host, port=port)
case 'celery':
from rhodecode.lib.hook_daemon.celery_hooks_deamon import CeleryHooksCallbackDaemon
callback_daemon = CeleryHooksCallbackDaemon(get_config(extras['config']))
case 'local':
from rhodecode.lib.hook_daemon.hook_module import Hooks
callback_daemon = HooksModuleCallbackDaemon(Hooks.__module__)
case _:
log.error('Unsupported callback daemon protocol "%s"', protocol)
raise Exception('Unsupported callback daemon protocol.')
extras['hooks_uri'] = getattr(callback_daemon, 'hooks_uri', '')
extras['task_queue'] = getattr(callback_daemon, 'task_queue', '')
extras['task_backend'] = getattr(callback_daemon, 'task_backend', '')
extras['hooks_protocol'] = protocol
extras['time'] = time.time()
# register txn_id
extras['txn_id'] = txn_id
log.debug('Prepared a callback daemon: %s',
callback_daemon.__class__.__name__)
return callback_daemon, extras

30
rhodecode/lib/hook_daemon/celery_hooks_deamon.py Normal file
View file

@ -0,0 +1,30 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
from rhodecode.lib.hook_daemon.base import BaseHooksCallbackDaemon
class CeleryHooksCallbackDaemon(BaseHooksCallbackDaemon):
"""
Context manger for achieving a compatibility with celery backend
"""
def __init__(self, config):
# TODO: replace this with settings bootstrapped...
self.task_queue = config.get('app:main', 'celery.broker_url')
self.task_backend = config.get('app:main', 'celery.result_backend')

104
rhodecode/lib/hook_daemon/hook_module.py Normal file
View file

@ -0,0 +1,104 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
import traceback
from rhodecode.model import meta
from rhodecode.lib import hooks_base
from rhodecode.lib.exceptions import HTTPLockedRC, HTTPBranchProtected
from rhodecode.lib.utils2 import AttributeDict
log = logging.getLogger(__name__)
class Hooks(object):
"""
Exposes the hooks for remote callbacks
"""
def __init__(self, request=None, log_prefix=''):
self.log_prefix = log_prefix
self.request = request
def repo_size(self, extras):
log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.repo_size, extras)
def pre_pull(self, extras):
log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.pre_pull, extras)
def post_pull(self, extras):
log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.post_pull, extras)
def pre_push(self, extras):
log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.pre_push, extras)
def post_push(self, extras):
log.debug("%sCalled post_push of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.post_push, extras)
def _call_hook(self, hook, extras):
extras = AttributeDict(extras)
_server_url = extras['server_url']
extras.request = self.request
try:
result = hook(extras)
if result is None:
raise Exception(f'Failed to obtain hook result from func: {hook}')
except HTTPBranchProtected as handled_error:
# Those special cases don't need error reporting. It's a case of
# locked repo or protected branch
result = AttributeDict({
'status': handled_error.code,
'output': handled_error.explanation
})
except (HTTPLockedRC, Exception) as error:
# locked needs different handling since we need to also
# handle PULL operations
exc_tb = ''
if not isinstance(error, HTTPLockedRC):
exc_tb = traceback.format_exc()
log.exception('%sException when handling hook %s', self.log_prefix, hook)
error_args = error.args
return {
'status': 128,
'output': '',
'exception': type(error).__name__,
'exception_traceback': exc_tb,
'exception_args': error_args,
}
finally:
meta.Session.remove()
log.debug('%sGot hook call response %s', self.log_prefix, result)
return {
'status': result.status,
'output': result.output,
}
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass

177
rhodecode/lib/hooks_daemon.py → rhodecode/lib/hook_daemon/http_hooks_deamon.py
View file

@ -17,9 +17,7 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import time
import logging
import tempfile
import traceback
import threading
import socket
@ -29,14 +27,11 @@ import gevent
from http.server import BaseHTTPRequestHandler
from socketserver import TCPServer
import rhodecode
from rhodecode.lib.exceptions import HTTPLockedRC, HTTPBranchProtected
from rhodecode.model import meta
from rhodecode.lib import hooks_base
from rhodecode.lib.utils2 import AttributeDict
from rhodecode.lib.pyramid_utils import get_config
from rhodecode.lib.ext_json import json
from rhodecode.lib import rc_cache
from rhodecode.lib.hook_daemon.base import get_txn_id_data_path
from rhodecode.lib.hook_daemon.hook_module import Hooks
log = logging.getLogger(__name__)
@ -126,7 +121,7 @@ class HooksHttpHandler(BaseHTTPRequestHandler):
def log_message(self, format, *args):
"""
This is an overridden method of BaseHTTPRequestHandler which logs using
logging library instead of writing directly to stderr.
a logging library instead of writing directly to stderr.
"""
message = format % args
@ -136,31 +131,6 @@ class HooksHttpHandler(BaseHTTPRequestHandler):
self.log_date_time_string(), message)
class BaseHooksCallbackDaemon:
"""
Basic context manager for actions that don't require some extra
"""
def __init__(self):
self.hooks_module = Hooks.__module__
def __enter__(self):
log.debug('Running `%s` callback daemon', self.__class__.__name__)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
class CeleryHooksCallbackDaemon(BaseHooksCallbackDaemon):
"""
Context manger for achieving a compatibility with celery backend
"""
def __init__(self, config):
self.task_queue = config.get('app:main', 'celery.broker_url')
self.task_backend = config.get('app:main', 'celery.result_backend')
class ThreadedHookCallbackDaemon(object):
_callback_thread = None
@ -308,144 +278,3 @@ class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
os.remove(txn_id_file)
log.debug("Background greenlet done.")
def get_txn_id_data_path(txn_id):
import rhodecode
root = rhodecode.CONFIG.get('cache_dir') or tempfile.gettempdir()
final_dir = os.path.join(root, 'svn_txn_id')
if not os.path.isdir(final_dir):
os.makedirs(final_dir)
return os.path.join(final_dir, 'rc_txn_id_{}'.format(txn_id))
def store_txn_id_data(txn_id, data_dict):
if not txn_id:
log.warning('Cannot store txn_id because it is empty')
return
path = get_txn_id_data_path(txn_id)
try:
with open(path, 'wb') as f:
f.write(json.dumps(data_dict))
except Exception:
log.exception('Failed to write txn_id metadata')
def get_txn_id_from_store(txn_id):
"""
Reads txn_id from store and if present returns the data for callback manager
"""
path = get_txn_id_data_path(txn_id)
try:
with open(path, 'rb') as f:
return json.loads(f.read())
except Exception:
return {}
def prepare_callback_daemon(extras, protocol, host, txn_id=None):
txn_details = get_txn_id_from_store(txn_id)
port = txn_details.get('port', 0)
match protocol:
case 'http':
callback_daemon = HttpHooksCallbackDaemon(
txn_id=txn_id, host=host, port=port)
case 'celery':
callback_daemon = CeleryHooksCallbackDaemon(get_config(extras['config']))
case 'local':
callback_daemon = BaseHooksCallbackDaemon()
case _:
log.error('Unsupported callback daemon protocol "%s"', protocol)
raise Exception('Unsupported callback daemon protocol.')
extras['hooks_uri'] = getattr(callback_daemon, 'hooks_uri', '')
extras['task_queue'] = getattr(callback_daemon, 'task_queue', '')
extras['task_backend'] = getattr(callback_daemon, 'task_backend', '')
extras['hooks_protocol'] = protocol
extras['time'] = time.time()
# register txn_id
extras['txn_id'] = txn_id
log.debug('Prepared a callback daemon: %s',
callback_daemon.__class__.__name__)
return callback_daemon, extras
class Hooks(object):
"""
Exposes the hooks for remote call backs
"""
def __init__(self, request=None, log_prefix=''):
self.log_prefix = log_prefix
self.request = request
def repo_size(self, extras):
log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.repo_size, extras)
def pre_pull(self, extras):
log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.pre_pull, extras)
def post_pull(self, extras):
log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.post_pull, extras)
def pre_push(self, extras):
log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.pre_push, extras)
def post_push(self, extras):
log.debug("%sCalled post_push of %s object", self.log_prefix, self)
return self._call_hook(hooks_base.post_push, extras)
def _call_hook(self, hook, extras):
extras = AttributeDict(extras)
server_url = extras['server_url']
extras.request = self.request
try:
result = hook(extras)
if result is None:
raise Exception(
'Failed to obtain hook result from func: {}'.format(hook))
except HTTPBranchProtected as handled_error:
# Those special cases doesn't need error reporting. It's a case of
# locked repo or protected branch
result = AttributeDict({
'status': handled_error.code,
'output': handled_error.explanation
})
except (HTTPLockedRC, Exception) as error:
# locked needs different handling since we need to also
# handle PULL operations
exc_tb = ''
if not isinstance(error, HTTPLockedRC):
exc_tb = traceback.format_exc()
log.exception('%sException when handling hook %s', self.log_prefix, hook)
error_args = error.args
return {
'status': 128,
'output': '',
'exception': type(error).__name__,
'exception_traceback': exc_tb,
'exception_args': error_args,
}
finally:
meta.Session.remove()
log.debug('%sGot hook call response %s', self.log_prefix, result)
return {
'status': result.status,
'output': result.output,
}
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass

3
rhodecode/lib/middleware/simplesvn.py
View file

@ -34,8 +34,7 @@ from rhodecode.lib.utils import is_valid_repo
from rhodecode.lib.str_utils import safe_str, safe_int, safe_bytes
from rhodecode.lib.type_utils import str2bool
from rhodecode.lib.ext_json import json
from rhodecode.lib.hooks_daemon import store_txn_id_data
from rhodecode.lib.hook_daemon.base import store_txn_id_data
log = logging.getLogger(__name__)

2
rhodecode/lib/middleware/simplevcs.py
View file

@ -45,7 +45,7 @@ from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
from rhodecode.lib.base import (
BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
from rhodecode.lib.hooks_daemon import prepare_callback_daemon
from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
from rhodecode.lib.middleware import appenlight
from rhodecode.lib.middleware.utils import scm_app_http
from rhodecode.lib.str_utils import safe_bytes

16
rhodecode/lib/pyramid_utils.py
View file

@ -20,24 +20,14 @@
import os
import configparser
from rhodecode.lib.config_utils import get_config
from pyramid.paster import bootstrap as pyramid_bootstrap, setup_logging # pragma: no cover
from rhodecode.lib.request import Request
def get_config(ini_path, **kwargs):
parser = configparser.ConfigParser(**kwargs)
parser.read(ini_path)
return parser
def get_app_config(ini_path):
from paste.deploy.loadwsgi import appconfig
return appconfig(f'config:{ini_path}', relative_to=os.getcwd())
def bootstrap(config_uri, options=None, env=None):
from rhodecode.lib.utils2 import AttributeDict
from rhodecode.lib.request import Request
if env:
os.environ.update(env)

3
rhodecode/lib/rc_commands/setup_rc.py
View file

@ -20,7 +20,8 @@ import logging
import click
import pyramid.paster
from rhodecode.lib.pyramid_utils import bootstrap, get_app_config
from rhodecode.lib.pyramid_utils import bootstrap
from rhodecode.lib.config_utils import get_app_config
from rhodecode.lib.db_manage import DbManage
from rhodecode.lib.utils2 import get_encryption_key
from rhodecode.model.db import Session

37
rhodecode/lib/utils.py
View file

@ -32,11 +32,9 @@ import socket
import tempfile
import traceback
import tarfile
import urllib.parse
import warnings
from functools import wraps
from os.path import join as jn
from configparser import NoOptionError
import paste
import pkg_resources
@ -55,9 +53,6 @@ from rhodecode.model import meta
from rhodecode.model.db import (
Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
from rhodecode.model.meta import Session
from rhodecode.lib.pyramid_utils import get_config
from rhodecode.lib.vcs import CurlSession
from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
log = logging.getLogger(__name__)
@ -827,33 +822,3 @@ def send_test_email(recipients, email_body='TEST EMAIL'):
email_body = email_body_plaintext = email_body
subject = f'SUBJECT FROM: {socket.gethostname()}'
tasks.send_email(recipients, subject, email_body_plaintext, email_body)
def call_service_api(ini_path, payload):
config = get_config(ini_path)
try:
host = config.get('app:main', 'app.service_api.host')
except NoOptionError:
raise ImproperlyConfiguredError(
"app.service_api.host is missing. "
"Please ensure that app.service_api.host and app.service_api.token are "
"defined inside of .ini configuration file."
)
try:
api_url = config.get('app:main', 'rhodecode.api.url')
except NoOptionError:
from rhodecode import api
log.debug('Cannot find rhodecode.api.url, setting API URL TO Default value')
api_url = api.DEFAULT_URL
payload.update({
'id': 'service',
'auth_token': config.get('app:main', 'app.service_api.token')
})
response = CurlSession().post(urllib.parse.urljoin(host, api_url), json.dumps(payload))
if response.status_code != 200:
raise Exception("Service API responded with error")
return json.loads(response.content)['result']

2
rhodecode/model/pull_request.py
View file

@ -38,7 +38,7 @@ from rhodecode.translation import lazy_ugettext
from rhodecode.lib import helpers as h, hooks_utils, diffs
from rhodecode.lib import audit_logger
from collections import OrderedDict
from rhodecode.lib.hooks_daemon import prepare_callback_daemon
from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
from rhodecode.lib.ext_json import sjson as json
from rhodecode.lib.markup_renderer import (
DEFAULT_COMMENTS_RENDERER, RstTemplateRenderer)

2
rhodecode/tests/fixture_mods/fixture_pyramid.py
View file

@ -19,7 +19,7 @@
import pytest
from rhodecode.lib.pyramid_utils import get_app_config
from rhodecode.lib.config_utils import get_app_config
from rhodecode.tests.fixture import TestINI
from rhodecode.tests.server_utils import RcVCSServer

2
rhodecode/tests/fixture_mods/fixture_utils.py
View file

@ -174,7 +174,7 @@ def http_environ():
@pytest.fixture(scope='session')
def baseapp(ini_config, vcsserver, http_environ_session):
from rhodecode.lib.pyramid_utils import get_app_config
from rhodecode.lib.config_utils import get_app_config
from rhodecode.config.middleware import make_pyramid_app
log.info("Using the RhodeCode configuration:{}".format(ini_config))

90
rhodecode/tests/lib/test_hooks_daemon.py
View file

@ -25,17 +25,20 @@ import msgpack
import pytest
import tempfile
from rhodecode.lib import hooks_daemon
from rhodecode.lib.hook_daemon import http_hooks_deamon
from rhodecode.lib.hook_daemon import celery_hooks_deamon
from rhodecode.lib.hook_daemon import hook_module
from rhodecode.lib.hook_daemon import base as hook_base
from rhodecode.lib.str_utils import safe_bytes
from rhodecode.tests.utils import assert_message_in_log
from rhodecode.lib.ext_json import json
test_proto = hooks_daemon.HooksHttpHandler.MSGPACK_HOOKS_PROTO
test_proto = http_hooks_deamon.HooksHttpHandler.MSGPACK_HOOKS_PROTO
class TestHooks(object):
def test_hooks_can_be_used_as_a_context_processor(self):
hooks = hooks_daemon.Hooks()
hooks = hook_module.Hooks()
with hooks as return_value:
pass
assert hooks == return_value
@ -52,10 +55,10 @@ class TestHooksHttpHandler(object):
}
request = self._generate_post_request(data)
hooks_patcher = mock.patch.object(
hooks_daemon.Hooks, data['method'], create=True, return_value=1)
hook_module.Hooks, data['method'], create=True, return_value=1)
with hooks_patcher as hooks_mock:
handler = hooks_daemon.HooksHttpHandler
handler = http_hooks_deamon.HooksHttpHandler
handler.DEFAULT_HOOKS_PROTO = test_proto
handler.wbufsize = 10240
MockServer(handler, request)
@ -73,21 +76,21 @@ class TestHooksHttpHandler(object):
# patching our _read to return test method and proto used
read_patcher = mock.patch.object(
hooks_daemon.HooksHttpHandler, '_read_request',
http_hooks_deamon.HooksHttpHandler, '_read_request',
return_value=(test_proto, rpc_method, extras))
# patch Hooks instance to return hook_result data on 'test' call
hooks_patcher = mock.patch.object(
hooks_daemon.Hooks, rpc_method, create=True,
hook_module.Hooks, rpc_method, create=True,
return_value=hook_result)
with read_patcher, hooks_patcher:
handler = hooks_daemon.HooksHttpHandler
handler = http_hooks_deamon.HooksHttpHandler
handler.DEFAULT_HOOKS_PROTO = test_proto
handler.wbufsize = 10240
server = MockServer(handler, request)
expected_result = hooks_daemon.HooksHttpHandler.serialize_data(hook_result)
expected_result = http_hooks_deamon.HooksHttpHandler.serialize_data(hook_result)
server.request.output_stream.seek(0)
assert server.request.output_stream.readlines()[-1] == expected_result
@ -97,15 +100,15 @@ class TestHooksHttpHandler(object):
rpc_method = 'test'
read_patcher = mock.patch.object(
hooks_daemon.HooksHttpHandler, '_read_request',
http_hooks_deamon.HooksHttpHandler, '_read_request',
return_value=(test_proto, rpc_method, {}))
hooks_patcher = mock.patch.object(
hooks_daemon.Hooks, rpc_method, create=True,
hook_module.Hooks, rpc_method, create=True,
side_effect=Exception('Test exception'))
with read_patcher, hooks_patcher:
handler = hooks_daemon.HooksHttpHandler
handler = http_hooks_deamon.HooksHttpHandler
handler.DEFAULT_HOOKS_PROTO = test_proto
handler.wbufsize = 10240
server = MockServer(handler, request)
@ -113,7 +116,7 @@ class TestHooksHttpHandler(object):
server.request.output_stream.seek(0)
data = server.request.output_stream.readlines()
msgpack_data = b''.join(data[5:])
org_exc = hooks_daemon.HooksHttpHandler.deserialize_data(msgpack_data)
org_exc = http_hooks_deamon.HooksHttpHandler.deserialize_data(msgpack_data)
expected_result = {
'exception': 'Exception',
'exception_traceback': org_exc['exception_traceback'],
@ -123,8 +126,7 @@ class TestHooksHttpHandler(object):
def test_log_message_writes_to_debug_log(self, caplog):
ip_port = ('0.0.0.0', 8888)
handler = hooks_daemon.HooksHttpHandler(
MockRequest('POST /'), ip_port, mock.Mock())
handler = http_hooks_deamon.HooksHttpHandler(MockRequest('POST /'), ip_port, mock.Mock())
fake_date = '1/Nov/2015 00:00:00'
date_patcher = mock.patch.object(
handler, 'log_date_time_string', return_value=fake_date)
@ -136,10 +138,10 @@ class TestHooksHttpHandler(object):
assert_message_in_log(
caplog.records, expected_message,
levelno=logging.DEBUG, module='hooks_daemon')
levelno=logging.DEBUG, module='http_hooks_deamon')
def _generate_post_request(self, data, proto=test_proto):
if proto == hooks_daemon.HooksHttpHandler.MSGPACK_HOOKS_PROTO:
if proto == http_hooks_deamon.HooksHttpHandler.MSGPACK_HOOKS_PROTO:
payload = msgpack.packb(data)
else:
payload = json.dumps(data)
@ -151,18 +153,18 @@ class TestHooksHttpHandler(object):
class ThreadedHookCallbackDaemon(object):
def test_constructor_calls_prepare(self):
prepare_daemon_patcher = mock.patch.object(
hooks_daemon.ThreadedHookCallbackDaemon, '_prepare')
http_hooks_deamon.ThreadedHookCallbackDaemon, '_prepare')
with prepare_daemon_patcher as prepare_daemon_mock:
hooks_daemon.ThreadedHookCallbackDaemon()
http_hooks_deamon.ThreadedHookCallbackDaemon()
prepare_daemon_mock.assert_called_once_with()
def test_run_is_called_on_context_start(self):
patchers = mock.patch.multiple(
hooks_daemon.ThreadedHookCallbackDaemon,
http_hooks_deamon.ThreadedHookCallbackDaemon,
_run=mock.DEFAULT, _prepare=mock.DEFAULT, __exit__=mock.DEFAULT)
with patchers as mocks:
daemon = hooks_daemon.ThreadedHookCallbackDaemon()
daemon = http_hooks_deamon.ThreadedHookCallbackDaemon()
with daemon as daemon_context:
pass
mocks['_run'].assert_called_once_with()
@ -170,11 +172,11 @@ class ThreadedHookCallbackDaemon(object):
def test_stop_is_called_on_context_exit(self):
patchers = mock.patch.multiple(
hooks_daemon.ThreadedHookCallbackDaemon,
http_hooks_deamon.ThreadedHookCallbackDaemon,
_run=mock.DEFAULT, _prepare=mock.DEFAULT, _stop=mock.DEFAULT)
with patchers as mocks:
daemon = hooks_daemon.ThreadedHookCallbackDaemon()
daemon = http_hooks_deamon.ThreadedHookCallbackDaemon()
with daemon as daemon_context:
assert mocks['_stop'].call_count == 0
@ -185,46 +187,47 @@ class ThreadedHookCallbackDaemon(object):
class TestHttpHooksCallbackDaemon(object):
def test_hooks_callback_generates_new_port(self, caplog):
with caplog.at_level(logging.DEBUG):
daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
assert daemon._daemon.server_address == ('127.0.0.1', 8881)
with caplog.at_level(logging.DEBUG):
daemon = hooks_daemon.HttpHooksCallbackDaemon(host=None, port=None)
daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host=None, port=None)
assert daemon._daemon.server_address[1] in range(0, 66000)
assert daemon._daemon.server_address[0] != '127.0.0.1'
def test_prepare_inits_daemon_variable(self, tcp_server, caplog):
with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
assert daemon._daemon == tcp_server
_, port = tcp_server.server_address
msg = f"HOOKS: 127.0.0.1:{port} Preparing HTTP callback daemon registering " \
f"hook object: <class 'rhodecode.lib.hooks_daemon.HooksHttpHandler'>"
f"hook object: <class 'rhodecode.lib.hook_daemon.http_hooks_deamon.HooksHttpHandler'>"
assert_message_in_log(
caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon')
caplog.records, msg, levelno=logging.DEBUG, module='http_hooks_deamon')
def test_prepare_inits_hooks_uri_and_logs_it(
self, tcp_server, caplog):
with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
_, port = tcp_server.server_address
expected_uri = '{}:{}'.format('127.0.0.1', port)
assert daemon.hooks_uri == expected_uri
msg = f"HOOKS: 127.0.0.1:{port} Preparing HTTP callback daemon registering " \
f"hook object: <class 'rhodecode.lib.hooks_daemon.HooksHttpHandler'>"
f"hook object: <class 'rhodecode.lib.hook_daemon.http_hooks_deamon.HooksHttpHandler'>"
assert_message_in_log(
caplog.records, msg,
levelno=logging.DEBUG, module='hooks_daemon')
levelno=logging.DEBUG, module='http_hooks_deamon')
def test_run_creates_a_thread(self, tcp_server):
thread = mock.Mock()
with self._tcp_patcher(tcp_server):
daemon = hooks_daemon.HttpHooksCallbackDaemon()
daemon = http_hooks_deamon.HttpHooksCallbackDaemon()
with self._thread_patcher(thread) as thread_mock:
daemon._run()
@ -238,7 +241,7 @@ class TestHttpHooksCallbackDaemon(object):
def test_run_logs(self, tcp_server, caplog):
with self._tcp_patcher(tcp_server):
daemon = hooks_daemon.HttpHooksCallbackDaemon()
daemon = http_hooks_deamon.HttpHooksCallbackDaemon()
with self._thread_patcher(mock.Mock()), caplog.at_level(logging.DEBUG):
daemon._run()
@ -246,13 +249,13 @@ class TestHttpHooksCallbackDaemon(object):
assert_message_in_log(
caplog.records,
'Running thread-based loop of callback daemon in background',
levelno=logging.DEBUG, module='hooks_daemon')
levelno=logging.DEBUG, module='http_hooks_deamon')
def test_stop_cleans_up_the_connection(self, tcp_server, caplog):
thread = mock.Mock()
with self._tcp_patcher(tcp_server):
daemon = hooks_daemon.HttpHooksCallbackDaemon()
daemon = http_hooks_deamon.HttpHooksCallbackDaemon()
with self._thread_patcher(thread), caplog.at_level(logging.DEBUG):
with daemon:
@ -266,18 +269,19 @@ class TestHttpHooksCallbackDaemon(object):
assert_message_in_log(
caplog.records, 'Waiting for background thread to finish.',
levelno=logging.DEBUG, module='hooks_daemon')
levelno=logging.DEBUG, module='http_hooks_deamon')
def _tcp_patcher(self, tcp_server):
return mock.patch.object(
hooks_daemon, 'TCPServer', return_value=tcp_server)
http_hooks_deamon, 'TCPServer', return_value=tcp_server)
def _thread_patcher(self, thread):
return mock.patch.object(
hooks_daemon.threading, 'Thread', return_value=thread)
http_hooks_deamon.threading, 'Thread', return_value=thread)
class TestPrepareHooksDaemon(object):
@pytest.mark.parametrize('protocol', ('celery',))
def test_returns_celery_hooks_callback_daemon_when_celery_protocol_specified(
self, protocol):
@ -286,12 +290,12 @@ class TestPrepareHooksDaemon(object):
"celery.result_backend = redis://redis/0")
temp_file.flush()
expected_extras = {'config': temp_file.name}
callback, extras = hooks_daemon.prepare_callback_daemon(
callback, extras = hook_base.prepare_callback_daemon(
expected_extras, protocol=protocol, host='')
assert isinstance(callback, hooks_daemon.CeleryHooksCallbackDaemon)
assert isinstance(callback, celery_hooks_deamon.CeleryHooksCallbackDaemon)
@pytest.mark.parametrize('protocol, expected_class', (
('http', hooks_daemon.HttpHooksCallbackDaemon),
('http', http_hooks_deamon.HttpHooksCallbackDaemon),
))
def test_returns_real_hooks_callback_daemon_when_protocol_is_specified(
self, protocol, expected_class):
@ -302,7 +306,7 @@ class TestPrepareHooksDaemon(object):
'task_backend': '',
'task_queue': ''
}
callback, extras = hooks_daemon.prepare_callback_daemon(
callback, extras = hook_base.prepare_callback_daemon(
expected_extras.copy(), protocol=protocol, host='127.0.0.1',
txn_id='txnid2')
assert isinstance(callback, expected_class)
@ -321,7 +325,7 @@ class TestPrepareHooksDaemon(object):
'hooks_protocol': protocol.lower()
}
with pytest.raises(Exception):
callback, extras = hooks_daemon.prepare_callback_daemon(
callback, extras = hook_base.prepare_callback_daemon(
expected_extras.copy(),
protocol=protocol, host='127.0.0.1')