fix(ssh): Added alternative SshWrapper and changes needed to support it + service api. Fixes: RCCE-6

This commit is contained in:
Serhii Ilin 2024-02-19 13:49:51 +02:00
parent 17631cb9fd
commit 16c169bec2
15 changed files with 505 additions and 47 deletions

View file

@ -153,6 +153,12 @@ startup.import_repos = false
; SSH calls. Set this for events to receive proper url for SSH calls.
app.base_url = http://rhodecode.local
; Host at which the Service API is running.
app.service_api.host = http://rhodecode.local:10020
; Secret for Service API authentication.
app.service_api.token =
; Unique application ID. Should be a random unique string for security.
app_instance_uuid = rc-production

View file

@ -104,6 +104,12 @@ startup.import_repos = false
; SSH calls. Set this for events to receive proper url for SSH calls.
app.base_url = http://rhodecode.local
; Host at which the Service API is running.
app.service_api.host= http://rhodecode.local:10020
; Secret for Service API authentication.
app.service_api.token =
; Unique application ID. Should be a random unique string for security.
app_instance_uuid = rc-production

View file

@ -46,6 +46,7 @@ log = logging.getLogger(__name__)
DEFAULT_RENDERER = 'jsonrpc_renderer'
DEFAULT_URL = '/_admin/apiv2'
SERVICE_API_IDENTIFIER = 'service_'
def find_methods(jsonrpc_methods, pattern):
@ -54,7 +55,9 @@ def find_methods(jsonrpc_methods, pattern):
pattern = [pattern]
for single_pattern in pattern:
for method_name, method in jsonrpc_methods.items():
for method_name, method in filter(
lambda x: not x[0].startswith(SERVICE_API_IDENTIFIER), jsonrpc_methods.items()
):
if fnmatch.fnmatch(method_name, single_pattern):
matches[method_name] = method
return matches
@ -190,43 +193,48 @@ def request_view(request):
# check if we can find this session using api_key, get_by_auth_token
# search not expired tokens only
try:
api_user = User.get_by_auth_token(request.rpc_api_key)
if not request.rpc_method.startswith(SERVICE_API_IDENTIFIER):
api_user = User.get_by_auth_token(request.rpc_api_key)
if api_user is None:
return jsonrpc_error(
request, retid=request.rpc_id, message='Invalid API KEY')
if api_user is None:
return jsonrpc_error(
request, retid=request.rpc_id, message='Invalid API KEY')
if not api_user.active:
return jsonrpc_error(
request, retid=request.rpc_id,
message='Request from this user not allowed')
if not api_user.active:
return jsonrpc_error(
request, retid=request.rpc_id,
message='Request from this user not allowed')
# check if we are allowed to use this IP
auth_u = AuthUser(
api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr)
if not auth_u.ip_allowed:
return jsonrpc_error(
request, retid=request.rpc_id,
message='Request from IP:{} not allowed'.format(
request.rpc_ip_addr))
# check if we are allowed to use this IP
auth_u = AuthUser(
api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr)
if not auth_u.ip_allowed:
return jsonrpc_error(
request, retid=request.rpc_id,
message='Request from IP:{} not allowed'.format(
request.rpc_ip_addr))
else:
log.info('Access for IP:%s allowed', request.rpc_ip_addr)
# register our auth-user
request.rpc_user = auth_u
request.environ['rc_auth_user_id'] = str(auth_u.user_id)
# now check if token is valid for API
auth_token = request.rpc_api_key
token_match = api_user.authenticate_by_token(
auth_token, roles=[UserApiKeys.ROLE_API])
invalid_token = not token_match
log.debug('Checking if API KEY is valid with proper role')
if invalid_token:
return jsonrpc_error(
request, retid=request.rpc_id,
message='API KEY invalid or, has bad role for an API call')
else:
log.info('Access for IP:%s allowed', request.rpc_ip_addr)
# register our auth-user
request.rpc_user = auth_u
request.environ['rc_auth_user_id'] = str(auth_u.user_id)
# now check if token is valid for API
auth_token = request.rpc_api_key
token_match = api_user.authenticate_by_token(
auth_token, roles=[UserApiKeys.ROLE_API])
invalid_token = not token_match
log.debug('Checking if API KEY is valid with proper role')
if invalid_token:
return jsonrpc_error(
request, retid=request.rpc_id,
message='API KEY invalid or, has bad role for an API call')
auth_u = 'service'
if request.rpc_api_key != request.registry.settings['app.service_api.token']:
raise Exception("Provided service secret is not recognized!")
except Exception:
log.exception('Error on API AUTH')
@ -290,7 +298,8 @@ def request_view(request):
})
# register some common functions for usage
attach_context_attributes(TemplateArgs(), request, request.rpc_user.user_id)
rpc_user = request.rpc_user.user_id if hasattr(request, 'rpc_user') else None
attach_context_attributes(TemplateArgs(), request, rpc_user)
statsd = request.registry.statsd

View file

@ -0,0 +1,55 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import pytest
from rhodecode.api.tests.utils import (
build_data, api_call)
@pytest.mark.usefixtures("app")
class TestServiceApi:
def test_service_api_with_wrong_secret(self):
id, payload = build_data("wrong_api_key", 'service_get_repo_name_by_id')
response = api_call(self.app, payload)
assert 'Invalid API KEY' == response.json['error']
def test_service_api_with_legit_secret(self):
id, payload = build_data(self.app.app.config.get_settings()['app.service_api.token'],
'service_get_repo_name_by_id', repo_id='1')
response = api_call(self.app, payload)
assert not response.json['error']
def test_service_api_not_a_part_of_public_api_suggestions(self):
id, payload = build_data("secret", 'some_random_guess_method')
response = api_call(self.app, payload)
assert 'service_' not in response.json['error']
def test_service_get_data_for_ssh_wrapper_output(self):
id, payload = build_data(
self.app.app.config.get_settings()['app.service_api.token'],
'service_get_data_for_ssh_wrapper',
user_id=1,
repo_name='vcs_test_git')
response = api_call(self.app, payload)
assert ['branch_permissions', 'repo_permissions', 'repos_path', 'user_id', 'username']\
== list(response.json['result'].keys())

View file

@ -0,0 +1,125 @@
# Copyright (C) 2011-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
import datetime
from collections import defaultdict
from sqlalchemy import Table
from rhodecode.api import jsonrpc_method, SERVICE_API_IDENTIFIER
log = logging.getLogger(__name__)
@jsonrpc_method()
def service_get_data_for_ssh_wrapper(request, apiuser, user_id, repo_name, key_id=None):
from rhodecode.model.db import User
from rhodecode.model.scm import ScmModel
from rhodecode.model.meta import raw_query_executor, Base
if key_id:
table = Table('user_ssh_keys', Base.metadata, autoload=False)
atime = datetime.datetime.utcnow()
stmt = (
table.update()
.where(table.c.ssh_key_id == key_id)
.values(accessed_on=atime)
)
res_count = None
with raw_query_executor() as session:
result = session.execute(stmt)
if result.rowcount:
res_count = result.rowcount
if res_count:
log.debug(f'Update key id:{key_id} access time')
db_user = User.get(user_id)
if not db_user:
return None
auth_user = db_user.AuthUser()
return {
'user_id': db_user.user_id,
'username': db_user.username,
'repo_permissions': auth_user.permissions['repositories'],
"branch_permissions": auth_user.get_branch_permissions(repo_name),
"repos_path": ScmModel().repos_path
}
@jsonrpc_method()
def service_get_repo_name_by_id(request, apiuser, repo_id):
from rhodecode.model.repo import RepoModel
by_id_match = RepoModel().get_repo_by_id(repo_id)
if by_id_match:
repo_name = by_id_match.repo_name
return {
'repo_name': repo_name
}
return None
@jsonrpc_method()
def service_mark_for_invalidation(request, apiuser, repo_name):
from rhodecode.model.scm import ScmModel
ScmModel().mark_for_invalidation(repo_name)
return {'msg': "Applied"}
@jsonrpc_method()
def service_config_to_hgrc(request, apiuser, cli_flags, repo_name):
from rhodecode.model.db import RhodeCodeUi
from rhodecode.model.settings import VcsSettingsModel
ui_sections = defaultdict(list)
ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
default_hooks = [
('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
]
for k, v in default_hooks:
ui_sections['hooks'].append((k, v))
for entry in ui:
if not entry.active:
continue
sec = entry.section
key = entry.key
if sec in cli_flags:
# we want only custom hooks, so we skip builtins
if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
continue
ui_sections[sec].append([key, entry.value])
flags = []
for _sec, key_val in ui_sections.items():
flags.append(' ')
flags.append(f'[{_sec}]')
for key, val in key_val:
flags.append(f'{key}= {val}')
return {'flags': flags}

View file

@ -23,6 +23,7 @@ import datetime
import configparser
from sqlalchemy import Table
from rhodecode.lib.utils import call_service_api
from rhodecode.lib.utils2 import AttributeDict
from rhodecode.model.scm import ScmModel
@ -261,3 +262,131 @@ class SshWrapper(object):
exit_code = -1
return exit_code
class SshWrapperStandalone(SshWrapper):
"""
New version of SshWrapper designed to be depended only on service API
"""
repos_path = None
@staticmethod
def parse_user_related_data(user_data):
user = AttributeDict()
user.user_id = user_data['user_id']
user.username = user_data['username']
user.repo_permissions = user_data['repo_permissions']
user.branch_permissions = user_data['branch_permissions']
return user
def wrap(self):
mode = self.mode
username = self.username
user_id = self.user_id
shell = self.shell
scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
log.debug(
'Mode: `%s` User: `name:%s : id:%s` Shell: `%s` SSH Command: `\"%s\"` '
'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
mode, username, user_id, shell, self.command,
scm_detected, scm_mode, scm_repo)
log.debug('SSH Connection info %s', self.get_connection_info())
if shell and self.command is None:
log.info('Dropping to shell, no command given and shell is allowed')
os.execl('/bin/bash', '-l')
exit_code = 1
elif scm_detected:
data = call_service_api(self.ini_path, {
"method": "service_get_data_for_ssh_wrapper",
"args": {"user_id": user_id, "repo_name": scm_repo, "key_id": self.key_id}
})
user = self.parse_user_related_data(data)
if not user:
log.warning('User with id %s not found', user_id)
exit_code = -1
return exit_code
self.repos_path = data['repos_path']
permissions = user.repo_permissions
repo_branch_permissions = user.branch_permissions
try:
exit_code, is_updated = self.serve(
scm_detected, scm_repo, scm_mode, user, permissions,
repo_branch_permissions)
except Exception:
log.exception('Error occurred during execution of SshWrapper')
exit_code = -1
elif self.command is None and shell is False:
log.error('No Command given.')
exit_code = -1
else:
log.error('Unhandled Command: "%s" Aborting.', self.command)
exit_code = -1
return exit_code
def maybe_translate_repo_uid(self, repo_name):
_org_name = repo_name
if _org_name.startswith('_'):
_org_name = _org_name.split('/', 1)[0]
if repo_name.startswith('_'):
org_repo_name = repo_name
log.debug('translating UID repo %s', org_repo_name)
by_id_match = call_service_api(self.ini_path, {
'method': 'service_get_repo_name_by_id',
"args": {"repo_id": repo_name}
})
if by_id_match:
repo_name = by_id_match['repo_name']
log.debug('translation of UID repo %s got `%s`', org_repo_name, repo_name)
return repo_name, _org_name
def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
store = self.repos_path
check_branch_perms = False
detect_force_push = False
if branch_permissions:
check_branch_perms = True
detect_force_push = True
log.debug(
'VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s',
vcs, mode, repo, check_branch_perms)
# detect if we have to check branch permissions
extras = {
'detect_force_push': detect_force_push,
'check_branch_perms': check_branch_perms,
'config': self.ini_path
}
match vcs:
case 'hg':
server = MercurialServer(
store=store, ini_path=self.ini_path,
repo_name=repo, user=user,
user_permissions=permissions, config=self.config, env=self.env)
case 'git':
server = GitServer(
store=store, ini_path=self.ini_path,
repo_name=repo, repo_mode=mode, user=user,
user_permissions=permissions, config=self.config, env=self.env)
case 'svn':
server = SubversionServer(
store=store, ini_path=self.ini_path,
repo_name=None, user=user,
user_permissions=permissions, config=self.config, env=self.env)
case _:
raise Exception(f'Unrecognised VCS: {vcs}')
self.server_impl = server
return server.run(tunnel_extras=extras)

View file

@ -23,6 +23,7 @@ import logging
from rhodecode.lib.hooks_daemon import prepare_callback_daemon
from rhodecode.lib.ext_json import sjson as json
from rhodecode.lib.vcs.conf import settings as vcs_settings
from rhodecode.lib.utils import call_service_api
from rhodecode.model.scm import ScmModel
log = logging.getLogger(__name__)
@ -47,6 +48,7 @@ class VcsServer(object):
self.repo_mode = None
self.store = ''
self.ini_path = ''
self.hooks_protocol = None
def _invalidate_cache(self, repo_name):
"""
@ -54,7 +56,15 @@ class VcsServer(object):
:param repo_name: full repo name, also a cache key
"""
ScmModel().mark_for_invalidation(repo_name)
# Todo: Leave only "celery" case after transition.
match self.hooks_protocol:
case 'http':
ScmModel().mark_for_invalidation(repo_name)
case 'celery':
call_service_api(self.ini_path, {
"method": "service_mark_for_invalidation",
"args": {"repo_name": repo_name}
})
def has_write_perm(self):
permission = self.user_permissions.get(self.repo_name)
@ -65,30 +75,31 @@ class VcsServer(object):
def _check_permissions(self, action):
permission = self.user_permissions.get(self.repo_name)
user_info = f'{self.user["user_id"]}:{self.user["username"]}'
log.debug('permission for %s on %s are: %s',
self.user, self.repo_name, permission)
user_info, self.repo_name, permission)
if not permission:
log.error('user `%s` permissions to repo:%s are empty. Forbidding access.',
self.user, self.repo_name)
user_info, self.repo_name)
return -2
if action == 'pull':
if permission in self.read_perms:
log.info(
'READ Permissions for User "%s" detected to repo "%s"!',
self.user, self.repo_name)
user_info, self.repo_name)
return 0
else:
if permission in self.write_perms:
log.info(
'WRITE, or Higher Permissions for User "%s" detected to repo "%s"!',
self.user, self.repo_name)
user_info, self.repo_name)
return 0
log.error('Cannot properly fetch or verify user `%s` permissions. '
'Permissions: %s, vcs action: %s',
self.user, permission, action)
user_info, permission, action)
return -2
def update_environment(self, action, extras=None):
@ -134,9 +145,10 @@ class VcsServer(object):
if exit_code:
return exit_code, False
req = self.env['request']
server_url = req.host_url + req.script_name
extras['server_url'] = server_url
req = self.env.get('request')
if req:
server_url = req.host_url + req.script_name
extras['server_url'] = server_url
log.debug('Using %s binaries from path %s', self.backend, self._path)
exit_code = self.tunnel.run(extras)
@ -144,12 +156,13 @@ class VcsServer(object):
return exit_code, action == "push"
def run(self, tunnel_extras=None):
self.hooks_protocol = self.config.get('app:main', 'vcs.hooks.protocol')
tunnel_extras = tunnel_extras or {}
extras = {}
extras.update(tunnel_extras)
callback_daemon, extras = prepare_callback_daemon(
extras, protocol=vcs_settings.HOOKS_PROTOCOL,
extras, protocol=self.hooks_protocol,
host=vcs_settings.HOOKS_HOST)
with callback_daemon:

View file

@ -23,6 +23,7 @@ import tempfile
import textwrap
import collections
from .base import VcsServer
from rhodecode.lib.utils import call_service_api
from rhodecode.model.db import RhodeCodeUi
from rhodecode.model.settings import VcsSettingsModel
@ -108,6 +109,14 @@ class MercurialServer(VcsServer):
self.tunnel = MercurialTunnelWrapper(server=self)
def config_to_hgrc(self, repo_name):
# Todo: once transition is done only call to service api should exist
if self.hooks_protocol == 'celery':
data = call_service_api(self.ini_path, {
"method": "service_config_to_hgrc",
"args": {"cli_flags": self.cli_flags, "repo_name": repo_name}
})
return data['flags']
ui_sections = collections.defaultdict(list)
ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)

View file

@ -0,0 +1,72 @@
# Copyright (C) 2016-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import sys
import time
import logging
import click
from pyramid.paster import setup_logging
from rhodecode.lib.statsd_client import StatsdClient
from .backends import SshWrapperStandalone
from .ssh_wrapper_v1 import setup_custom_logging
log = logging.getLogger(__name__)
@click.command()
@click.argument('ini_path', type=click.Path(exists=True))
@click.option(
'--mode', '-m', required=False, default='auto',
type=click.Choice(['auto', 'vcs', 'git', 'hg', 'svn', 'test']),
help='mode of operation')
@click.option('--user', help='Username for which the command will be executed')
@click.option('--user-id', help='User ID for which the command will be executed')
@click.option('--key-id', help='ID of the key from the database')
@click.option('--shell', '-s', is_flag=True, help='Allow Shell')
@click.option('--debug', is_flag=True, help='Enabled detailed output logging')
def main(ini_path, mode, user, user_id, key_id, shell, debug):
setup_custom_logging(ini_path, debug)
command = os.environ.get('SSH_ORIGINAL_COMMAND', '')
if not command and mode not in ['test']:
raise ValueError(
'Unable to fetch SSH_ORIGINAL_COMMAND from environment.'
'Please make sure this is set and available during execution '
'of this script.')
connection_info = os.environ.get('SSH_CONNECTION', '')
time_start = time.time()
env = {'RC_CMD_SSH_WRAPPER': '1'}
statsd = StatsdClient.statsd
try:
ssh_wrapper = SshWrapperStandalone(
command, connection_info, mode,
user, user_id, key_id, shell, ini_path, env)
except Exception:
log.exception('Failed to execute SshWrapper')
sys.exit(-5)
return_code = ssh_wrapper.wrap()
operation_took = time.time() - time_start
if statsd:
operation_took_ms = round(1000.0 * operation_took)
statsd.timing("rhodecode_ssh_wrapper_timing.histogram", operation_took_ms,
use_decimals=False)
sys.exit(return_code)

View file

@ -20,7 +20,7 @@ import os
import pytest
import configparser
from rhodecode.apps.ssh_support.lib.ssh_wrapper import SshWrapper
from rhodecode.apps.ssh_support.lib.ssh_wrapper_v1 import SshWrapper
from rhodecode.lib.utils2 import AttributeDict

View file

@ -34,6 +34,7 @@ import tarfile
import warnings
from functools import wraps
from os.path import join as jn
from configparser import NoOptionError
import paste
import pkg_resources
@ -52,6 +53,9 @@ from rhodecode.model import meta
from rhodecode.model.db import (
Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
from rhodecode.model.meta import Session
from rhodecode.lib.pyramid_utils import get_config
from rhodecode.lib.vcs import CurlSession
from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
log = logging.getLogger(__name__)
@ -821,3 +825,27 @@ def send_test_email(recipients, email_body='TEST EMAIL'):
email_body = email_body_plaintext = email_body
subject = f'SUBJECT FROM: {socket.gethostname()}'
tasks.send_email(recipients, subject, email_body_plaintext, email_body)
def call_service_api(ini_path, payload):
config = get_config(ini_path)
try:
host = config.get('app:main', 'app.service_api.host')
except NoOptionError:
raise ImproperlyConfiguredError(
"app.service_api.host is missing. "
"Please ensure that app.service_api.host and app.service_api.token are "
"defined inside of .ini configuration file."
)
api_url = config.get('app:main', 'rhodecode.api.url')
payload.update({
'id': 'service',
'auth_token': config.get('app:main', 'app.service_api.token')
})
response = CurlSession().post(f'{host}{api_url}', json.dumps(payload))
if response.status_code != 200:
raise Exception("Service API responded with error")
return json.loads(response.content)['result']

View file

@ -146,6 +146,10 @@ class CommandError(VCSError):
pass
class ImproperlyConfiguredError(Exception):
pass
class UnhandledException(VCSError):
"""
Signals that something unexpected went wrong.

View file

@ -110,6 +110,7 @@ def ini_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
'vcs.scm_app_implementation': 'http',
'vcs.hooks.protocol': 'http',
'vcs.hooks.host': '*',
'app.service_api.token': 'service_secret_token',
}},
{'handler_console': {

View file

@ -196,7 +196,8 @@ setup(
'rc-upgrade-db=rhodecode.lib.rc_commands.upgrade_db:main',
'rc-ishell=rhodecode.lib.rc_commands.ishell:main',
'rc-add-artifact=rhodecode.lib.rc_commands.add_artifact:main',
'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper:main',
'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper_v1:main',
'rc-ssh-wrapper-v2=rhodecode.apps.ssh_support.lib.ssh_wrapper_v2:main',
],
'beaker.backends': [
'memorylru_base=rhodecode.lib.memory_lru_dict:MemoryLRUNamespaceManagerBase',