engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ssh-wrapper-speedup): major rewrite of code to address imports problem with ssh-wrapper-v2

Browse source 
- use bootstrapped settings rather than config
- use more code split to make sure we don't import heavy code
This commit is contained in:
RhodeCode Admin 2024-02-28 09:30:58 +01:00
parent d57482be62
commit fc0ee0e99b
33 changed files with 764 additions and 562 deletions
Download patch file Download diff file Expand all files Collapse all files

63
rhodecode/apps/ssh_support/lib/backends/__init__.py
View file

@ -20,12 +20,11 @@ import os
import re
import logging
import datetime
import configparser
from sqlalchemy import Table
from rhodecode.lib.utils import call_service_api
from rhodecode.lib.api_utils import call_service_api
from rhodecode.lib.utils2 import AttributeDict
from rhodecode.model.scm import ScmModel
from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
from .hg import MercurialServer
from .git import GitServer
@ -39,7 +38,7 @@ class SshWrapper(object):
svn_cmd_pat = re.compile(r'^svnserve -t')
def __init__(self, command, connection_info, mode,
user, user_id, key_id: int, shell, ini_path: str, env):
user, user_id, key_id: int, shell, ini_path: str, settings, env):
self.command = command
self.connection_info = connection_info
self.mode = mode
@ -49,15 +48,9 @@ class SshWrapper(object):
self.shell = shell
self.ini_path = ini_path
self.env = env
self.config = self.parse_config(ini_path)
self.settings = settings
self.server_impl = None
def parse_config(self, config_path):
parser = configparser.ConfigParser()
parser.read(config_path)
return parser
def update_key_access_time(self, key_id):
from rhodecode.model.meta import raw_query_executor, Base
@ -162,6 +155,9 @@ class SshWrapper(object):
return vcs_type, repo_name, mode
def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
# TODO: remove this once we have .ini defined access path...
from rhodecode.model.scm import ScmModel
store = ScmModel().repos_path
check_branch_perms = False
@ -186,7 +182,7 @@ class SshWrapper(object):
server = MercurialServer(
store=store, ini_path=self.ini_path,
repo_name=repo, user=user,
user_permissions=permissions, config=self.config, env=self.env)
user_permissions=permissions, settings=self.settings, env=self.env)
self.server_impl = server
return server.run(tunnel_extras=extras)
@ -194,7 +190,7 @@ class SshWrapper(object):
server = GitServer(
store=store, ini_path=self.ini_path,
repo_name=repo, repo_mode=mode, user=user,
user_permissions=permissions, config=self.config, env=self.env)
user_permissions=permissions, settings=self.settings, env=self.env)
self.server_impl = server
return server.run(tunnel_extras=extras)
@ -202,7 +198,7 @@ class SshWrapper(object):
server = SubversionServer(
store=store, ini_path=self.ini_path,
repo_name=None, user=user,
user_permissions=permissions, config=self.config, env=self.env)
user_permissions=permissions, settings=self.settings, env=self.env)
self.server_impl = server
return server.run(tunnel_extras=extras)
@ -269,6 +265,35 @@ class SshWrapperStandalone(SshWrapper):
New version of SshWrapper designed to be depended only on service API
"""
repos_path = None
service_api_host: str
service_api_token: str
api_url: str
def __init__(self, command, connection_info, mode,
user, user_id, key_id: int, shell, ini_path: str, settings, env):
# validate our settings for making a standalone calls
try:
self.service_api_host = settings['app.service_api.host']
self.service_api_token = settings['app.service_api.token']
except KeyError:
raise ImproperlyConfiguredError(
"app.service_api.host or app.service_api.token are missing. "
"Please ensure that app.service_api.host and app.service_api.token are "
"defined inside of .ini configuration file."
)
try:
self.api_url = settings['rhodecode.api.url']
except KeyError:
raise ImproperlyConfiguredError(
"rhodecode.api.url is missing. "
"Please ensure that rhodecode.api.url is "
"defined inside of .ini configuration file."
)
super(SshWrapperStandalone, self).__init__(
command, connection_info, mode, user, user_id, key_id, shell, ini_path, settings, env)
@staticmethod
def parse_user_related_data(user_data):
@ -301,7 +326,7 @@ class SshWrapperStandalone(SshWrapper):
exit_code = 1
elif scm_detected:
data = call_service_api(self.ini_path, {
data = call_service_api(self.service_api_host, self.service_api_token, self.api_url, {
"method": "service_get_data_for_ssh_wrapper",
"args": {"user_id": user_id, "repo_name": scm_repo, "key_id": self.key_id}
})
@ -339,7 +364,7 @@ class SshWrapperStandalone(SshWrapper):
if repo_name.startswith('_'):
org_repo_name = repo_name
log.debug('translating UID repo %s', org_repo_name)
by_id_match = call_service_api(self.ini_path, {
by_id_match = call_service_api(self.service_api_host, self.service_api_token, self.api_url, {
'method': 'service_get_repo_name_by_id',
"args": {"repo_id": repo_name}
})
@ -375,17 +400,17 @@ class SshWrapperStandalone(SshWrapper):
server = MercurialServer(
store=store, ini_path=self.ini_path,
repo_name=repo, user=user,
user_permissions=permissions, config=self.config, env=self.env)
user_permissions=permissions, settings=self.settings, env=self.env)
case 'git':
server = GitServer(
store=store, ini_path=self.ini_path,
repo_name=repo, repo_mode=mode, user=user,
user_permissions=permissions, config=self.config, env=self.env)
user_permissions=permissions, settings=self.settings, env=self.env)
case 'svn':
server = SubversionServer(
store=store, ini_path=self.ini_path,
repo_name=None, user=user,
user_permissions=permissions, config=self.config, env=self.env)
user_permissions=permissions, settings=self.settings, env=self.env)
case _:
raise Exception(f'Unrecognised VCS: {vcs}')
self.server_impl = server

23
rhodecode/apps/ssh_support/lib/backends/base.py
View file

@ -20,27 +20,27 @@ import os
import sys
import logging
from rhodecode.lib.hooks_daemon import prepare_callback_daemon
from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
from rhodecode.lib.ext_json import sjson as json
from rhodecode.lib.vcs.conf import settings as vcs_settings
from rhodecode.lib.utils import call_service_api
from rhodecode.model.scm import ScmModel
from rhodecode.lib.api_utils import call_service_api
log = logging.getLogger(__name__)
class VcsServer(object):
class SSHVcsServer(object):
repo_user_agent = None # set in child classes
_path = None # set executable path for hg/git/svn binary
backend = None # set in child classes
tunnel = None # subprocess handling tunnel
settings = None # parsed settings module
write_perms = ['repository.admin', 'repository.write']
read_perms = ['repository.read', 'repository.admin', 'repository.write']
def __init__(self, user, user_permissions, config, env):
def __init__(self, user, user_permissions, settings, env):
self.user = user
self.user_permissions = user_permissions
self.config = config
self.settings = settings
self.env = env
self.stdin = sys.stdin
@ -59,9 +59,14 @@ class VcsServer(object):
# Todo: Leave only "celery" case after transition.
match self.hooks_protocol:
case 'http':
from rhodecode.model.scm import ScmModel
ScmModel().mark_for_invalidation(repo_name)
case 'celery':
call_service_api(self.ini_path, {
service_api_host = self.settings['app.service_api.host']
service_api_token = self.settings['app.service_api.token']
api_url = self.settings['rhodecode.api.url']
call_service_api(service_api_host, service_api_token, api_url, {
"method": "service_mark_for_invalidation",
"args": {"repo_name": repo_name}
})
@ -118,7 +123,7 @@ class VcsServer(object):
'server_url': None,
'user_agent': f'{self.repo_user_agent}/ssh-user-agent',
'hooks': ['push', 'pull'],
'hooks_module': 'rhodecode.lib.hooks_daemon',
'hooks_module': 'rhodecode.lib.hook_daemon.hook_module',
'is_shadow_repo': False,
'detect_force_push': False,
'check_branch_perms': False,
@ -156,7 +161,7 @@ class VcsServer(object):
return exit_code, action == "push"
def run(self, tunnel_extras=None):
self.hooks_protocol = self.config.get('app:main', 'vcs.hooks.protocol')
self.hooks_protocol = self.settings['vcs.hooks.protocol']
tunnel_extras = tunnel_extras or {}
extras = {}
extras.update(tunnel_extras)

12
rhodecode/apps/ssh_support/lib/backends/git.py
View file

@ -21,7 +21,7 @@ import logging
import subprocess
from vcsserver import hooks
from .base import VcsServer
from .base import SSHVcsServer
log = logging.getLogger(__name__)
@ -70,19 +70,17 @@ class GitTunnelWrapper(object):
return result
class GitServer(VcsServer):
class GitServer(SSHVcsServer):
backend = 'git'
repo_user_agent = 'git'
def __init__(self, store, ini_path, repo_name, repo_mode,
user, user_permissions, config, env):
super().\
__init__(user, user_permissions, config, env)
def __init__(self, store, ini_path, repo_name, repo_mode, user, user_permissions, settings, env):
super().__init__(user, user_permissions, settings, env)
self.store = store
self.ini_path = ini_path
self.repo_name = repo_name
self._path = self.git_path = config.get('app:main', 'ssh.executable.git')
self._path = self.git_path = settings['ssh.executable.git']
self.repo_mode = repo_mode
self.tunnel = GitTunnelWrapper(server=self)

85
rhodecode/apps/ssh_support/lib/backends/hg.py
View file

@ -22,10 +22,10 @@ import logging
import tempfile
import textwrap
import collections
from .base import VcsServer
from rhodecode.lib.utils import call_service_api
from rhodecode.model.db import RhodeCodeUi
from rhodecode.model.settings import VcsSettingsModel
from .base import SSHVcsServer
from rhodecode.lib.api_utils import call_service_api
log = logging.getLogger(__name__)
@ -57,7 +57,7 @@ class MercurialTunnelWrapper(object):
# cleanup custom hgrc file
if os.path.isfile(hgrc_custom):
with open(hgrc_custom, 'wb') as f:
f.write('')
f.write(b'')
log.debug('Cleanup custom hgrc file under %s', hgrc_custom)
# write temp
@ -94,62 +94,67 @@ class MercurialTunnelWrapper(object):
self.remove_configs()
class MercurialServer(VcsServer):
class MercurialServer(SSHVcsServer):
backend = 'hg'
repo_user_agent = 'mercurial'
cli_flags = ['phases', 'largefiles', 'extensions', 'experimental', 'hooks']
def __init__(self, store, ini_path, repo_name, user, user_permissions, config, env):
super().__init__(user, user_permissions, config, env)
def __init__(self, store, ini_path, repo_name, user, user_permissions, settings, env):
super().__init__(user, user_permissions, settings, env)
self.store = store
self.ini_path = ini_path
self.repo_name = repo_name
self._path = self.hg_path = config.get('app:main', 'ssh.executable.hg')
self._path = self.hg_path = settings['ssh.executable.hg']
self.tunnel = MercurialTunnelWrapper(server=self)
def config_to_hgrc(self, repo_name):
# Todo: once transition is done only call to service api should exist
if self.hooks_protocol == 'celery':
data = call_service_api(self.ini_path, {
service_api_host = self.settings['app.service_api.host']
service_api_token = self.settings['app.service_api.token']
api_url = self.settings['rhodecode.api.url']
data = call_service_api(service_api_host, service_api_token, api_url, {
"method": "service_config_to_hgrc",
"args": {"cli_flags": self.cli_flags, "repo_name": repo_name}
})
return data['flags']
else:
from rhodecode.model.db import RhodeCodeUi
from rhodecode.model.settings import VcsSettingsModel
ui_sections = collections.defaultdict(list)
ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
ui_sections = collections.defaultdict(list)
ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
# write default hooks
default_hooks = [
('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
# write default hooks
default_hooks = [
('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
]
('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
]
for k, v in default_hooks:
ui_sections['hooks'].append((k, v))
for k, v in default_hooks:
ui_sections['hooks'].append((k, v))
for entry in ui:
if not entry.active:
continue
sec = entry.section
key = entry.key
if sec in self.cli_flags:
# we want only custom hooks, so we skip builtins
if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
for entry in ui:
if not entry.active:
continue
sec = entry.section
key = entry.key
ui_sections[sec].append([key, entry.value])
if sec in self.cli_flags:
# we want only custom hooks, so we skip builtins
if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
continue
flags = []
for _sec, key_val in ui_sections.items():
flags.append(' ')
flags.append(f'[{_sec}]')
for key, val in key_val:
flags.append(f'{key}= {val}')
return flags
ui_sections[sec].append([key, entry.value])
flags = []
for _sec, key_val in ui_sections.items():
flags.append(' ')
flags.append(f'[{_sec}]')
for key, val in key_val:
flags.append(f'{key}= {val}')
return flags

12
rhodecode/apps/ssh_support/lib/backends/svn.py
View file

@ -25,7 +25,7 @@ import tempfile
from subprocess import Popen, PIPE
import urllib.parse
from .base import VcsServer
from .base import SSHVcsServer
log = logging.getLogger(__name__)
@ -218,20 +218,18 @@ class SubversionTunnelWrapper(object):
return self.return_code
class SubversionServer(VcsServer):
class SubversionServer(SSHVcsServer):
backend = 'svn'
repo_user_agent = 'svn'
def __init__(self, store, ini_path, repo_name,
user, user_permissions, config, env):
super()\
.__init__(user, user_permissions, config, env)
def __init__(self, store, ini_path, repo_name, user, user_permissions, settings, env):
super().__init__(user, user_permissions, settings, env)
self.store = store
self.ini_path = ini_path
# NOTE(dan): repo_name at this point is empty,
# this is set later in .run() based from parsed input stream
self.repo_name = repo_name
self._path = self.svn_path = config.get('app:main', 'ssh.executable.svn')
self._path = self.svn_path = settings['ssh.executable.svn']
self.tunnel = SubversionTunnelWrapper(server=self)

4
rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py
View file

@ -31,7 +31,6 @@ from .utils import setup_custom_logging
log = logging.getLogger(__name__)
@click.command()
@click.argument('ini_path', type=click.Path(exists=True))
@click.option(
@ -55,11 +54,12 @@ def main(ini_path, mode, user, user_id, key_id, shell, debug):
connection_info = os.environ.get('SSH_CONNECTION', '')
time_start = time.time()
with bootstrap(ini_path, env={'RC_CMD_SSH_WRAPPER': '1'}) as env:
settings = env['registry'].settings
statsd = StatsdClient.statsd
try:
ssh_wrapper = SshWrapper(
command, connection_info, mode,
user, user_id, key_id, shell, ini_path, env)
user, user_id, key_id, shell, ini_path, settings, env)
except Exception:
log.exception('Failed to execute SshWrapper')
sys.exit(-5)

32
rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
View file

@ -16,6 +16,14 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
"""
WARNING: be really carefully with changing ANY imports in this file
# This script is to mean as really fast executable, doing some imports here that would yield an import chain change
# can affect execution times...
# This can be easily debugged using such command::
# time PYTHONPROFILEIMPORTTIME=1 rc-ssh-wrapper-v2 --debug --mode=test .dev/dev.ini
"""
import os
import sys
import time
@ -23,9 +31,12 @@ import logging
import click
from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
from rhodecode.lib.statsd_client import StatsdClient
from .backends import SshWrapperStandalone
from rhodecode.lib.config_utils import get_app_config_lightweight
from .utils import setup_custom_logging
from .backends import SshWrapperStandalone
log = logging.getLogger(__name__)
@ -42,6 +53,8 @@ log = logging.getLogger(__name__)
@click.option('--shell', '-s', is_flag=True, help='Allow Shell')
@click.option('--debug', is_flag=True, help='Enabled detailed output logging')
def main(ini_path, mode, user, user_id, key_id, shell, debug):
time_start = time.time()
setup_custom_logging(ini_path, debug)
command = os.environ.get('SSH_ORIGINAL_COMMAND', '')
@ -50,21 +63,30 @@ def main(ini_path, mode, user, user_id, key_id, shell, debug):
'Unable to fetch SSH_ORIGINAL_COMMAND from environment.'
'Please make sure this is set and available during execution '
'of this script.')
connection_info = os.environ.get('SSH_CONNECTION', '')
time_start = time.time()
env = {'RC_CMD_SSH_WRAPPER': '1'}
# initialize settings and get defaults
settings = get_app_config_lightweight(ini_path)
settings = sanitize_settings_and_apply_defaults({'__file__': ini_path}, settings)
# init and bootstrap StatsdClient
StatsdClient.setup(settings)
statsd = StatsdClient.statsd
try:
connection_info = os.environ.get('SSH_CONNECTION', '')
env = {'RC_CMD_SSH_WRAPPER': '1'}
ssh_wrapper = SshWrapperStandalone(
command, connection_info, mode,
user, user_id, key_id, shell, ini_path, env)
user, user_id, key_id, shell, ini_path, settings, env)
except Exception:
log.exception('Failed to execute SshWrapper')
sys.exit(-5)
return_code = ssh_wrapper.wrap()
operation_took = time.time() - time_start
if statsd:
operation_took_ms = round(1000.0 * operation_took)
statsd.timing("rhodecode_ssh_wrapper_timing.histogram", operation_took_ms,
use_decimals=False)
sys.exit(return_code)

2
rhodecode/apps/ssh_support/lib/utils.py
View file

@ -17,11 +17,11 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
from pyramid.paster import setup_logging
def setup_custom_logging(ini_path, debug):
if debug:
from pyramid.paster import setup_logging # Lazy import
# enabled rhodecode.ini controlled logging setup
setup_logging(ini_path)
else:

7
rhodecode/apps/ssh_support/tests/conftest.py
View file

@ -52,7 +52,10 @@ def dummy_env():
def plain_dummy_user():
return AttributeDict(username='test_user')
return AttributeDict(
user_id=1,
username='test_user'
)
@pytest.fixture()
@ -65,4 +68,4 @@ def ssh_wrapper(app, dummy_conf_file, dummy_env):
conn_info = '127.0.0.1 22 10.0.0.1 443'
return SshWrapper(
'random command', conn_info, 'auto', 'admin', '1', key_id='1',
shell=False, ini_path=dummy_conf_file, env=dummy_env)
shell=False, ini_path=dummy_conf_file, settings={}, env=dummy_env)

10
rhodecode/apps/ssh_support/tests/test_server_git.py
View file

@ -25,6 +25,7 @@ from rhodecode.apps.ssh_support.lib.backends.git import GitServer
from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dummy_user
from rhodecode.lib.ext_json import json
class GitServerCreator(object):
root = '/tmp/repo/path/'
git_path = '/usr/local/bin/git'
@ -39,10 +40,7 @@ class GitServerCreator(object):
user = plain_dummy_user()
def __init__(self):
def config_get(part, key):
return self.config_data.get(part, {}).get(key)
self.config_mock = mock.Mock()
self.config_mock.get = mock.Mock(side_effect=config_get)
pass
def create(self, **kwargs):
parameters = {
@ -54,7 +52,7 @@ class GitServerCreator(object):
'user_permissions': {
self.repo_name: 'repository.admin'
},
'config': self.config_mock,
'settings': self.config_data['app:main'],
'env': plain_dummy_env()
}
parameters.update(kwargs)
@ -142,7 +140,7 @@ class TestGitServer(object):
'server_url': None,
'hooks': ['push', 'pull'],
'is_shadow_repo': False,
'hooks_module': 'rhodecode.lib.hooks_daemon',
'hooks_module': 'rhodecode.lib.hook_daemon.hook_module',
'check_branch_perms': False,
'detect_force_push': False,
'user_agent': u'git/ssh-user-agent',

7
rhodecode/apps/ssh_support/tests/test_server_hg.py
View file

@ -38,10 +38,7 @@ class MercurialServerCreator(object):
user = plain_dummy_user()
def __init__(self):
def config_get(part, key):
return self.config_data.get(part, {}).get(key)
self.config_mock = mock.Mock()
self.config_mock.get = mock.Mock(side_effect=config_get)
pass
def create(self, **kwargs):
parameters = {
@ -52,7 +49,7 @@ class MercurialServerCreator(object):
'user_permissions': {
'test_hg': 'repository.admin'
},
'config': self.config_mock,
'settings': self.config_data['app:main'],
'env': plain_dummy_env()
}
parameters.update(kwargs)

7
rhodecode/apps/ssh_support/tests/test_server_svn.py
View file

@ -36,10 +36,7 @@ class SubversionServerCreator(object):
user = plain_dummy_user()
def __init__(self):
def config_get(part, key):
return self.config_data.get(part, {}).get(key)
self.config_mock = mock.Mock()
self.config_mock.get = mock.Mock(side_effect=config_get)
pass
def create(self, **kwargs):
parameters = {
@ -50,7 +47,7 @@ class SubversionServerCreator(object):
'user_permissions': {
self.repo_name: 'repository.admin'
},
'config': self.config_mock,
'settings': self.config_data['app:main'],
'env': plain_dummy_env()
}

4
rhodecode/apps/ssh_support/tests/test_ssh_wrapper.py
View file

@ -28,10 +28,6 @@ class TestSSHWrapper(object):
permissions={}, branch_permissions={})
assert str(exc_info.value) == 'Unrecognised VCS: microsoft-tfs'
def test_parse_config(self, ssh_wrapper):
config = ssh_wrapper.parse_config(ssh_wrapper.ini_path)
assert config
def test_get_connection_info(self, ssh_wrapper):
conn_info = ssh_wrapper.get_connection_info()
assert {'client_ip': '127.0.0.1',