From d79877ff230d8128ecef9925f6cdc2596bf1a0b2 Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Mon, 6 Jan 2025 12:09:22 +0100 Subject: [PATCH] ssh-wrapper: added extra logging in case of failed user fetch --- .pre-commit-config.yaml | 10 + rhodecode/api/views/service_api.py | 53 ++-- .../apps/ssh_support/lib/backends/__init__.py | 274 +++++++++++------- rhodecode/lib/api_utils.py | 15 +- 4 files changed, 208 insertions(+), 144 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..adc0d8e6 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.8.6 + hooks: + # Run the linter. + - id: ruff + args: [ "--ignore=F401", "--ignore=I001", "--ignore=E402", "--ignore=E501", "--ignore=F841" ] + # Run the formatter. + - id: ruff-format diff --git a/rhodecode/api/views/service_api.py b/rhodecode/api/views/service_api.py index f86f27a5..5eeb62fe 100644 --- a/rhodecode/api/views/service_api.py +++ b/rhodecode/api/views/service_api.py @@ -34,13 +34,9 @@ def service_get_data_for_ssh_wrapper(request, apiuser, user_id, repo_name, key_i from rhodecode.model.meta import raw_query_executor, Base if key_id: - table = Table('user_ssh_keys', Base.metadata, autoload=False) - atime = datetime.datetime.utcnow() - stmt = ( - table.update() - .where(table.c.ssh_key_id == key_id) - .values(accessed_on=atime) - ) + table = Table("user_ssh_keys", Base.metadata, autoload=False) + atime = datetime.datetime.now(datetime.UTC) + stmt = table.update().where(table.c.ssh_key_id == key_id).values(accessed_on=atime) res_count = None with raw_query_executor() as session: @@ -49,38 +45,38 @@ def service_get_data_for_ssh_wrapper(request, apiuser, user_id, repo_name, key_i res_count = result.rowcount if res_count: - log.debug(f'Update key id:{key_id} access time') + log.debug(f"Update key id:{key_id} access time") db_user = User.get(user_id) if not db_user: - return None + return {} auth_user = db_user.AuthUser() return { - 'user_id': db_user.user_id, - 'username': db_user.username, - 'repo_permissions': auth_user.permissions['repositories'], + "user_id": db_user.user_id, + "username": db_user.username, + "repo_permissions": auth_user.permissions["repositories"], "branch_permissions": auth_user.get_branch_permissions(repo_name), - "repos_path": ScmModel().repos_path + "repos_path": ScmModel().repos_path, } @jsonrpc_method() def service_get_repo_name_by_id(request, apiuser, repo_id): from rhodecode.model.repo import RepoModel + by_id_match = RepoModel().get_repo_by_id(repo_id) if by_id_match: repo_name = by_id_match.repo_name - return { - 'repo_name': repo_name - } + return {"repo_name": repo_name} return None @jsonrpc_method() def service_mark_for_invalidation(request, apiuser, repo_name): from rhodecode.model.scm import ScmModel + ScmModel().mark_for_invalidation(repo_name) - return {'msg': "Applied"} + return {"msg": "Applied"} @jsonrpc_method() @@ -92,16 +88,15 @@ def service_config_to_hgrc(request, apiuser, cli_flags, repo_name): ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None) default_hooks = [ - ('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'), - ('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'), - ('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'), - - ('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'), - ('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'), + ("pretxnchangegroup.ssh_auth", "python:vcsserver.hooks.pre_push_ssh_auth"), + ("pretxnchangegroup.ssh", "python:vcsserver.hooks.pre_push_ssh"), + ("changegroup.ssh", "python:vcsserver.hooks.post_push_ssh"), + ("preoutgoing.ssh", "python:vcsserver.hooks.pre_pull_ssh"), + ("outgoing.ssh", "python:vcsserver.hooks.post_pull_ssh"), ] for k, v in default_hooks: - ui_sections['hooks'].append((k, v)) + ui_sections["hooks"].append((k, v)) for entry in ui: if not entry.active: @@ -111,15 +106,15 @@ def service_config_to_hgrc(request, apiuser, cli_flags, repo_name): if sec in cli_flags: # we want only custom hooks, so we skip builtins - if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN: + if sec == "hooks" and key in RhodeCodeUi.HOOKS_BUILTIN: continue ui_sections[sec].append([key, entry.value]) flags = [] for _sec, key_val in ui_sections.items(): - flags.append(' ') - flags.append(f'[{_sec}]') + flags.append(" ") + flags.append(f"[{_sec}]") for key, val in key_val: - flags.append(f'{key}= {val}') - return {'flags': flags} + flags.append(f"{key}= {val}") + return {"flags": flags} diff --git a/rhodecode/apps/ssh_support/lib/backends/__init__.py b/rhodecode/apps/ssh_support/lib/backends/__init__.py index 0394baa0..fd5837f7 100644 --- a/rhodecode/apps/ssh_support/lib/backends/__init__.py +++ b/rhodecode/apps/ssh_support/lib/backends/__init__.py @@ -28,16 +28,16 @@ from rhodecode.lib.utils2 import AttributeDict from .hg import MercurialServer from .git import GitServer from .svn import SubversionServer + log = logging.getLogger(__name__) class SshWrapper(object): - hg_cmd_pat = re.compile(r'^hg\s+\-R\s+(\S+)\s+serve\s+\-\-stdio$') - git_cmd_pat = re.compile(r'^git-(receive-pack|upload-pack)\s\'[/]?(\S+?)(|\.git)\'$') - svn_cmd_pat = re.compile(r'^svnserve -t') + hg_cmd_pat = re.compile(r"^hg\s+\-R\s+(\S+)\s+serve\s+\-\-stdio$") + git_cmd_pat = re.compile(r"^git-(receive-pack|upload-pack)\s\'[/]?(\S+?)(|\.git)\'$") + svn_cmd_pat = re.compile(r"^svnserve -t") - def __init__(self, command, connection_info, mode, - user, user_id, key_id: int, shell, ini_path: str, settings, env): + def __init__(self, command, connection_info, mode, user, user_id, key_id: int, shell, ini_path: str, settings, env): self.command = command self.connection_info = connection_info self.mode = mode @@ -53,14 +53,12 @@ class SshWrapper(object): def update_key_access_time(self, key_id): from rhodecode.model.meta import raw_query_executor, Base - table = Table('user_ssh_keys', Base.metadata, autoload=False) + table = Table("user_ssh_keys", Base.metadata, autoload=False) atime = datetime.datetime.utcnow() stmt = ( - table.update() - .where(table.c.ssh_key_id == key_id) - .values(accessed_on=atime) + table.update().where(table.c.ssh_key_id == key_id).values(accessed_on=atime) # no MySQL Support for .returning :(( - #.returning(table.c.accessed_on, table.c.ssh_key_fingerprint) + # .returning(table.c.accessed_on, table.c.ssh_key_fingerprint) ) res_count = None @@ -70,12 +68,13 @@ class SshWrapper(object): res_count = result.rowcount if res_count: - log.debug('Update key id:`%s` access time', key_id) + log.debug("Update key id:`%s` access time", key_id) def get_user(self, user_id): user = AttributeDict() # lazy load db imports from rhodecode.model.db import User + dbuser = User.get(user_id) if not dbuser: return None @@ -99,54 +98,55 @@ class SshWrapper(object): server_port=None, ) - info = self.connection_info.split(' ') + info = self.connection_info.split(" ") if len(info) == 4: - conn['client_ip'] = info[0] - conn['client_port'] = info[1] - conn['server_ip'] = info[2] - conn['server_port'] = info[3] + conn["client_ip"] = info[0] + conn["client_port"] = info[1] + conn["server_ip"] = info[2] + conn["server_port"] = info[3] return conn def maybe_translate_repo_uid(self, repo_name): _org_name = repo_name - if _org_name.startswith('_'): + if _org_name.startswith("_"): # remove format of _ID/subrepo - _org_name = _org_name.split('/', 1)[0] + _org_name = _org_name.split("/", 1)[0] - if repo_name.startswith('_'): + if repo_name.startswith("_"): from rhodecode.model.repo import RepoModel + org_repo_name = repo_name - log.debug('translating UID repo %s', org_repo_name) + log.debug("translating UID repo %s", org_repo_name) by_id_match = RepoModel().get_repo_by_id(repo_name) if by_id_match: repo_name = by_id_match.repo_name - log.debug('translation of UID repo %s got `%s`', org_repo_name, repo_name) + log.debug("translation of UID repo %s got `%s`", org_repo_name, repo_name) return repo_name, _org_name def get_repo_details(self, mode): - vcs_type = mode if mode in ['svn', 'hg', 'git'] else None + vcs_type = mode if mode in ["svn", "hg", "git"] else None repo_name = None hg_match = self.hg_cmd_pat.match(self.command) if hg_match is not None: - vcs_type = 'hg' - repo_id = hg_match.group(1).strip('/') + vcs_type = "hg" + repo_id = hg_match.group(1).strip("/") repo_name, org_name = self.maybe_translate_repo_uid(repo_id) return vcs_type, repo_name, mode git_match = self.git_cmd_pat.match(self.command) if git_match is not None: mode = git_match.group(1) - vcs_type = 'git' - repo_id = git_match.group(2).strip('/') + vcs_type = "git" + repo_id = git_match.group(2).strip("/") repo_name, org_name = self.maybe_translate_repo_uid(repo_id) return vcs_type, repo_name, mode svn_match = self.svn_cmd_pat.match(self.command) if svn_match is not None: - vcs_type = 'svn' + vcs_type = "svn" # Repo name should be extracted from the input stream, we're unable to # extract it at this point in execution return vcs_type, repo_name, mode @@ -167,42 +167,62 @@ class SshWrapper(object): detect_force_push = True log.debug( - 'VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s', - vcs, mode, repo, check_branch_perms) + "VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s", + vcs, + mode, + repo, + check_branch_perms, + ) # detect if we have to check branch permissions extras = { - 'detect_force_push': detect_force_push, - 'check_branch_perms': check_branch_perms, - 'config': self.ini_path + "detect_force_push": detect_force_push, + "check_branch_perms": check_branch_perms, + "config": self.ini_path, } - if vcs == 'hg': + if vcs == "hg": server = MercurialServer( - store=store, ini_path=self.ini_path, - repo_name=repo, user=user, - user_permissions=permissions, settings=self.settings, env=self.env) + store=store, + ini_path=self.ini_path, + repo_name=repo, + user=user, + user_permissions=permissions, + settings=self.settings, + env=self.env, + ) self.server_impl = server return server.run(tunnel_extras=extras) - elif vcs == 'git': + elif vcs == "git": server = GitServer( - store=store, ini_path=self.ini_path, - repo_name=repo, repo_mode=mode, user=user, - user_permissions=permissions, settings=self.settings, env=self.env) + store=store, + ini_path=self.ini_path, + repo_name=repo, + repo_mode=mode, + user=user, + user_permissions=permissions, + settings=self.settings, + env=self.env, + ) self.server_impl = server return server.run(tunnel_extras=extras) - elif vcs == 'svn': + elif vcs == "svn": server = SubversionServer( - store=store, ini_path=self.ini_path, - repo_name=None, user=user, - user_permissions=permissions, settings=self.settings, env=self.env) + store=store, + ini_path=self.ini_path, + repo_name=None, + user=user, + user_permissions=permissions, + settings=self.settings, + env=self.env, + ) self.server_impl = server return server.run(tunnel_extras=extras) else: - raise Exception(f'Unrecognised VCS: {vcs}') + raise Exception(f"Unrecognised VCS: {vcs}") def wrap(self): mode = self.mode @@ -214,42 +234,49 @@ class SshWrapper(object): scm_detected, scm_repo, scm_mode = self.get_repo_details(mode) log.debug( - 'Mode: `%s` User: `name:%s : id:%s` Shell: `%s` SSH Command: `\"%s\"` ' - 'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`', - mode, username, user_id, shell, self.command, - scm_detected, scm_mode, scm_repo) + 'Mode: `%s` User: `name:%s : id:%s` Shell: `%s` SSH Command: `"%s"` ' + "SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`", + mode, + username, + user_id, + shell, + self.command, + scm_detected, + scm_mode, + scm_repo, + ) - log.debug('SSH Connection info %s', self.get_connection_info()) + log.debug("SSH Connection info %s", self.get_connection_info()) # update last access time for this key if key_id: self.update_key_access_time(key_id) if shell and self.command is None: - log.info('Dropping to shell, no command given and shell is allowed') - os.execl('/bin/bash', '-l') + log.info("Dropping to shell, no command given and shell is allowed") + os.execl("/bin/bash", "-l") exit_code = 1 elif scm_detected: user = self.get_user(user_id) if not user: - log.warning('User with id %s not found', user_id) + log.warning("User with id %s not found", user_id) exit_code = -1 return exit_code auth_user = user.auth_user - permissions = auth_user.permissions['repositories'] + permissions = auth_user.permissions["repositories"] repo_branch_permissions = auth_user.get_branch_permissions(scm_repo) try: exit_code, is_updated = self.serve( - scm_detected, scm_repo, scm_mode, user, permissions, - repo_branch_permissions) + scm_detected, scm_repo, scm_mode, user, permissions, repo_branch_permissions + ) except Exception: - log.exception('Error occurred during execution of SshWrapper') + log.exception("Error occurred during execution of SshWrapper") exit_code = -1 elif self.command is None and shell is False: - log.error('No Command given.') + log.error("No Command given.") exit_code = -1 else: @@ -263,15 +290,16 @@ class SshWrapperStandalone(SshWrapper): """ New version of SshWrapper designed to be depended only on service API """ + repos_path = None @staticmethod def parse_user_related_data(user_data): user = AttributeDict() - user.user_id = user_data['user_id'] - user.username = user_data['username'] - user.repo_permissions = user_data['repo_permissions'] - user.branch_permissions = user_data['branch_permissions'] + user.user_id = user_data["user_id"] + user.username = user_data["username"] + user.repo_permissions = user_data["repo_permissions"] + user.branch_permissions = user_data["branch_permissions"] return user def wrap(self): @@ -283,41 +311,54 @@ class SshWrapperStandalone(SshWrapper): scm_detected, scm_repo, scm_mode = self.get_repo_details(mode) log.debug( - 'Mode: `%s` User: `name:%s : id:%s` Shell: `%s` SSH Command: `\"%s\"` ' - 'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`', - mode, username, user_id, shell, self.command, - scm_detected, scm_mode, scm_repo) + 'Mode: `%s` User: `name:%s : id:%s` Shell: `%s` SSH Command: `"%s"` ' + "SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`", + mode, + username, + user_id, + shell, + self.command, + scm_detected, + scm_mode, + scm_repo, + ) - log.debug('SSH Connection info %s', self.get_connection_info()) + log.debug("SSH Connection info %s", self.get_connection_info()) if shell and self.command is None: - log.info('Dropping to shell, no command given and shell is allowed') - os.execl('/bin/bash', '-l') + log.info("Dropping to shell, no command given and shell is allowed") + os.execl("/bin/bash", "-l") exit_code = 1 elif scm_detected: - data = call_service_api(self.settings, { - "method": "service_get_data_for_ssh_wrapper", - "args": {"user_id": user_id, "repo_name": scm_repo, "key_id": self.key_id} - }) + data = call_service_api( + self.settings, + { + "method": "service_get_data_for_ssh_wrapper", + "args": {"user_id": user_id, "repo_name": scm_repo, "key_id": self.key_id}, + }, + ) + if not data: + log.error("Error occurred during execution of SshWrapper service API call") + user = self.parse_user_related_data(data) if not user: - log.warning('User with id %s not found', user_id) + log.warning("User with id %s not found", user_id) exit_code = -1 return exit_code - self.repos_path = data['repos_path'] + self.repos_path = data["repos_path"] permissions = user.repo_permissions repo_branch_permissions = user.branch_permissions try: exit_code, is_updated = self.serve( - scm_detected, scm_repo, scm_mode, user, permissions, - repo_branch_permissions) + scm_detected, scm_repo, scm_mode, user, permissions, repo_branch_permissions + ) except Exception: - log.exception('Error occurred during execution of SshWrapper') + log.exception("Error occurred during execution of SshWrapper") exit_code = -1 elif self.command is None and shell is False: - log.error('No Command given.') + log.error("No Command given.") exit_code = -1 else: @@ -328,19 +369,18 @@ class SshWrapperStandalone(SshWrapper): def maybe_translate_repo_uid(self, repo_name): _org_name = repo_name - if _org_name.startswith('_'): - _org_name = _org_name.split('/', 1)[0] + if _org_name.startswith("_"): + _org_name = _org_name.split("/", 1)[0] - if repo_name.startswith('_'): + if repo_name.startswith("_"): org_repo_name = repo_name - log.debug('translating UID repo %s', org_repo_name) - by_id_match = call_service_api(self.settings, { - 'method': 'service_get_repo_name_by_id', - "args": {"repo_id": repo_name} - }) + log.debug("translating UID repo %s", org_repo_name) + by_id_match = call_service_api( + self.settings, {"method": "service_get_repo_name_by_id", "args": {"repo_id": repo_name}} + ) if by_id_match: - repo_name = by_id_match['repo_name'] - log.debug('translation of UID repo %s got `%s`', org_repo_name, repo_name) + repo_name = by_id_match["repo_name"] + log.debug("translation of UID repo %s got `%s`", org_repo_name, repo_name) return repo_name, _org_name @@ -355,33 +395,53 @@ class SshWrapperStandalone(SshWrapper): detect_force_push = True log.debug( - 'VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s', - vcs, mode, repo, check_branch_perms) + "VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s", + vcs, + mode, + repo, + check_branch_perms, + ) # detect if we have to check branch permissions extras = { - 'detect_force_push': detect_force_push, - 'check_branch_perms': check_branch_perms, - 'config': self.ini_path + "detect_force_push": detect_force_push, + "check_branch_perms": check_branch_perms, + "config": self.ini_path, } match vcs: - case 'hg': + case "hg": server = MercurialServer( - store=store, ini_path=self.ini_path, - repo_name=repo, user=user, - user_permissions=permissions, settings=self.settings, env=self.env) - case 'git': + store=store, + ini_path=self.ini_path, + repo_name=repo, + user=user, + user_permissions=permissions, + settings=self.settings, + env=self.env, + ) + case "git": server = GitServer( - store=store, ini_path=self.ini_path, - repo_name=repo, repo_mode=mode, user=user, - user_permissions=permissions, settings=self.settings, env=self.env) - case 'svn': + store=store, + ini_path=self.ini_path, + repo_name=repo, + repo_mode=mode, + user=user, + user_permissions=permissions, + settings=self.settings, + env=self.env, + ) + case "svn": server = SubversionServer( - store=store, ini_path=self.ini_path, - repo_name=None, user=user, - user_permissions=permissions, settings=self.settings, env=self.env) + store=store, + ini_path=self.ini_path, + repo_name=None, + user=user, + user_permissions=permissions, + settings=self.settings, + env=self.env, + ) case _: - raise Exception(f'Unrecognised VCS: {vcs}') + raise Exception(f"Unrecognised VCS: {vcs}") self.server_impl = server return server.run(tunnel_extras=extras) diff --git a/rhodecode/lib/api_utils.py b/rhodecode/lib/api_utils.py index 92a06d7e..7969fbec 100644 --- a/rhodecode/lib/api_utils.py +++ b/rhodecode/lib/api_utils.py @@ -25,23 +25,22 @@ from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError def call_service_api(settings, payload): try: - api_host = settings['app.service_api.host'] - api_token = settings['app.service_api.token'] - api_url = settings['rhodecode.api.url'] + api_host = settings["app.service_api.host"] + api_token = settings["app.service_api.token"] + api_url = settings["rhodecode.api.url"] except KeyError as exc: raise ImproperlyConfiguredError( f"{str(exc)} is missing. " "Please ensure that app.service_api.host, app.service_api.token and rhodecode.api.url are " "defined inside of .ini configuration file." ) - payload.update({ - 'id': 'service', - 'auth_token': api_token - }) + payload.update({"id": "service", "auth_token": api_token}) service_api_url = urllib.parse.urljoin(api_host, api_url) response = CurlSession().post(service_api_url, json.dumps(payload)) if response.status_code != 200: raise Exception(f"Service API at {service_api_url} responded with error: {response.status_code}") - return json.loads(response.content)['result'] + result = json.loads(response.content)["result"] + + return result