From 5ef8f4f6d63350db85d7a33a62a0038d26da65d8 Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Fri, 6 Aug 2021 11:30:14 +0000 Subject: [PATCH 1/8] release: Finish preparation for 4.26.0 --- .release.cfg | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.release.cfg b/.release.cfg index 546c013b..eeaa3359 100644 --- a/.release.cfg +++ b/.release.cfg @@ -5,19 +5,25 @@ done = false done = true [task:rc_tools_pinned] +done = true [task:fixes_on_stable] +done = true [task:pip2nix_generated] +done = true [task:changelog_updated] +done = true [task:generate_api_docs] +done = true [task:updated_translation] +done = true [release] -state = in_progress +state = prepared version = 4.26.0 [task:generate_js_routes] From a079a8e8742da544fb0a9d3e89f9dae9374e51be Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Fri, 6 Aug 2021 11:30:15 +0000 Subject: [PATCH 2/8] Added tag v4.26.0 for changeset b4ba10dcb4ab From 25e528c10ffb31f5055b94ad31024f06b42ed9bd Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Tue, 12 Oct 2021 13:39:46 +0200 Subject: [PATCH 3/8] docs: added admin section docs about problematic PR deletion --- docs/admin/repo_admin/repo-admin-tasks.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/admin/repo_admin/repo-admin-tasks.rst b/docs/admin/repo_admin/repo-admin-tasks.rst index 92e8cae0..e4061c03 100644 --- a/docs/admin/repo_admin/repo-admin-tasks.rst +++ b/docs/admin/repo_admin/repo-admin-tasks.rst @@ -101,4 +101,18 @@ the permissions onto *all* repositories and/or repository groups. In [3]: permission_name = 'group.write' In [4]: for repo_group in RepoGroup.get_all(): ...: RepoGroupModel().grant_user_permission(repo_group, user, permission_name) - ...: Session().commit() \ No newline at end of file + ...: Session().commit() + + +Delete a problematic pull request +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + :dedent: 1 + + In [1]: from rhodecode.model.pull_request import PullRequestModel + In [2]: pullrequest_id = 123 + In [3]: pr = PullRequest.get(pullrequest_id) + In [4]: super_admin = User.get_first_super_admin() + In [5]: PullRequestModel().delete(pr, super_admin) + In [6]: Session().commit() From 98cc67d8891ef21aff1acd94dad1140c2e36ca4d Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Tue, 12 Oct 2021 13:40:31 +0200 Subject: [PATCH 4/8] caches: added debug and timings --- rhodecode/lib/rc_cache/backends.py | 13 +++++++++++-- rhodecode/lib/rc_cache/utils.py | 6 +++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/rhodecode/lib/rc_cache/backends.py b/rhodecode/lib/rc_cache/backends.py index 10365c99..76663061 100644 --- a/rhodecode/lib/rc_cache/backends.py +++ b/rhodecode/lib/rc_cache/backends.py @@ -300,7 +300,6 @@ class BaseRedisBackend(redis_backend.RedisBackend): def get_mutex(self, key): if self.distributed_lock: lock_key = redis_backend.u('_lock_{0}').format(key) - log.debug('Trying to acquire Redis lock for key %s', lock_key) return get_mutex_lock(self.client, lock_key, self._lock_timeout, auto_renewal=self._lock_auto_renewal) else: @@ -333,12 +332,22 @@ def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False): strict=True, ) + def __repr__(self): + return "{}:{}".format(self.__class__.__name__, lock_key) + + def __str__(self): + return "{}:{}".format(self.__class__.__name__, lock_key) + def __init__(self): self.lock = self.get_lock() + self.lock_key = lock_key def acquire(self, wait=True): + log.debug('Trying to acquire Redis lock for key %s', self.lock_key) try: - return self.lock.acquire(wait) + acquired = self.lock.acquire(wait) + log.debug('Got lock for key %s, %s', self.lock_key, acquired) + return acquired except redis_lock.AlreadyAcquired: return False except redis_lock.AlreadyStarted: diff --git a/rhodecode/lib/rc_cache/utils.py b/rhodecode/lib/rc_cache/utils.py index 5d84dc8d..1fca5fe8 100644 --- a/rhodecode/lib/rc_cache/utils.py +++ b/rhodecode/lib/rc_cache/utils.py @@ -122,7 +122,11 @@ class RhodeCodeCacheRegion(CacheRegion): if not condition: log.debug('Calling un-cached func:%s', user_func.func_name) - return user_func(*arg, **kw) + start = time.time() + result = user_func(*arg, **kw) + total = time.time() - start + log.debug('un-cached func:%s took %.4fs', user_func.func_name, total) + return result key = key_generator(*arg, **kw) From 448f66785cf80331bf2dcf18c48789e6572012fb Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Tue, 12 Oct 2021 13:40:59 +0200 Subject: [PATCH 5/8] sys-info: added helpers for faster loading of sys-info data --- rhodecode/lib/system_info.py | 56 ++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py index 3d821bc6..a297e3d2 100644 --- a/rhodecode/lib/system_info.py +++ b/rhodecode/lib/system_info.py @@ -24,6 +24,8 @@ import sys import time import platform import collections +from functools import wraps + import pkg_resources import logging import resource @@ -51,6 +53,26 @@ STATE_WARN = 'warning' STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK} +registered_helpers = {} + + +def register_sysinfo(func): + """ + @register_helper + def db_check(): + pass + + db_check == registered_helpers['db_check'] + """ + global registered_helpers + registered_helpers[func.__name__] = func + + @wraps(func) + def _wrapper(*args, **kwargs): + return func(*args, **kwargs) + return _wrapper + + # HELPERS def percentage(part, whole): whole = float(whole) @@ -136,12 +158,14 @@ class SysInfo(object): # SysInfo functions +@register_sysinfo def python_info(): value = dict(version=' '.join(platform._sys_version()), executable=sys.executable) return SysInfoRes(value=value) +@register_sysinfo def py_modules(): mods = dict([(p.project_name, {'version': p.version, 'location': p.location}) for p in pkg_resources.working_set]) @@ -150,6 +174,7 @@ def py_modules(): return SysInfoRes(value=value) +@register_sysinfo def platform_type(): from rhodecode.lib.utils import safe_unicode, generate_platform_uuid @@ -160,6 +185,7 @@ def platform_type(): return SysInfoRes(value=value) +@register_sysinfo def locale_info(): import locale @@ -175,6 +201,7 @@ def locale_info(): return SysInfoRes(value=value, human_value=human_value) +@register_sysinfo def ulimit_info(): data = collections.OrderedDict([ ('cpu time (seconds)', get_resource(resource.RLIMIT_CPU)), @@ -198,6 +225,7 @@ def ulimit_info(): return SysInfoRes(value=value) +@register_sysinfo def uptime(): from rhodecode.lib.helpers import age, time_to_datetime from rhodecode.translation import TranslationString @@ -223,6 +251,7 @@ def uptime(): return SysInfoRes(value=value, human_value=human_value) +@register_sysinfo def memory(): from rhodecode.lib.helpers import format_byte_size_binary value = dict(available=0, used=0, used_real=0, cached=0, percent=0, @@ -262,6 +291,7 @@ def memory(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def machine_load(): value = {'1_min': _NA, '5_min': _NA, '15_min': _NA, 'text': ''} state = STATE_OK_DEFAULT @@ -284,6 +314,7 @@ def machine_load(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def cpu(): value = {'cpu': 0, 'cpu_count': 0, 'cpu_usage': []} state = STATE_OK_DEFAULT @@ -302,6 +333,7 @@ def cpu(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def storage(): from rhodecode.lib.helpers import format_byte_size_binary from rhodecode.model.settings import VcsSettingsModel @@ -337,6 +369,7 @@ def storage(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def storage_inodes(): from rhodecode.model.settings import VcsSettingsModel path = VcsSettingsModel().get_repos_location() @@ -371,6 +404,7 @@ def storage_inodes(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def storage_archives(): import rhodecode from rhodecode.lib.utils import safe_str @@ -414,6 +448,7 @@ def storage_archives(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def storage_gist(): from rhodecode.model.gist import GIST_STORE_LOC from rhodecode.model.settings import VcsSettingsModel @@ -457,6 +492,7 @@ def storage_gist(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def storage_temp(): import tempfile from rhodecode.lib.helpers import format_byte_size_binary @@ -485,6 +521,7 @@ def storage_temp(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def search_info(): import rhodecode from rhodecode.lib.index import searcher_from_config @@ -508,6 +545,7 @@ def search_info(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def git_info(): from rhodecode.lib.vcs.backends import git state = STATE_OK_DEFAULT @@ -521,6 +559,7 @@ def git_info(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def hg_info(): from rhodecode.lib.vcs.backends import hg state = STATE_OK_DEFAULT @@ -533,6 +572,7 @@ def hg_info(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def svn_info(): from rhodecode.lib.vcs.backends import svn state = STATE_OK_DEFAULT @@ -545,6 +585,7 @@ def svn_info(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def vcs_backends(): import rhodecode value = rhodecode.CONFIG.get('vcs.backends') @@ -552,6 +593,7 @@ def vcs_backends(): return SysInfoRes(value=value, human_value=human_value) +@register_sysinfo def vcs_server(): import rhodecode from rhodecode.lib.vcs.backends import get_vcsserver_service_data @@ -595,6 +637,7 @@ def vcs_server(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def vcs_server_config(): from rhodecode.lib.vcs.backends import get_vcsserver_service_data state = STATE_OK_DEFAULT @@ -612,6 +655,7 @@ def vcs_server_config(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def rhodecode_app_info(): import rhodecode edition = rhodecode.CONFIG.get('rhodecode.edition') @@ -628,6 +672,7 @@ def rhodecode_app_info(): return SysInfoRes(value=value, human_value=human_value) +@register_sysinfo def rhodecode_config(): import rhodecode path = rhodecode.CONFIG.get('__file__') @@ -683,6 +728,7 @@ def rhodecode_config(): 'path': path, 'cert_path': cert_path}) +@register_sysinfo def database_info(): import rhodecode from sqlalchemy.engine import url as engine_url @@ -727,6 +773,7 @@ def database_info(): return SysInfoRes(value=db_info, state=state, human_value=human_value) +@register_sysinfo def server_info(environ): import rhodecode from rhodecode.lib.base import get_server_ip_addr, get_server_port @@ -741,6 +788,7 @@ def server_info(environ): return SysInfoRes(value=value) +@register_sysinfo def usage_info(): from rhodecode.model.db import User, Repository value = { @@ -795,3 +843,11 @@ def get_system_info(environ): 'hg': SysInfo(hg_info)(), 'svn': SysInfo(svn_info)(), } + + +def load_system_info(key): + """ + get_sys_info('vcs_server') + get_sys_info('database') + """ + return SysInfo(registered_helpers[key])() From f5cf1a1f867ac5ff63a67ca90193db2c13878d3c Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Tue, 12 Oct 2021 14:09:11 +0200 Subject: [PATCH 6/8] celerylib: fixed broken tasks for auto-update --- rhodecode/lib/celerylib/__init__.py | 3 +++ rhodecode/lib/celerylib/tasks.py | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/rhodecode/lib/celerylib/__init__.py b/rhodecode/lib/celerylib/__init__.py index 6a70f170..e56b35eb 100644 --- a/rhodecode/lib/celerylib/__init__.py +++ b/rhodecode/lib/celerylib/__init__.py @@ -43,6 +43,9 @@ class ResultWrapper(object): def run_task(task, *args, **kwargs): log.debug('Got task `%s` for execution', task) + if task is None: + raise ValueError('Got non-existing task for execution') + if rhodecode.CELERY_ENABLED: celery_is_up = False try: diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py index 0c16c4b6..b847c306 100644 --- a/rhodecode/lib/celerylib/tasks.py +++ b/rhodecode/lib/celerylib/tasks.py @@ -386,9 +386,7 @@ def beat_check(*args, **kwargs): return time.time() -@async_task(ignore_result=True) -def sync_last_update(*args, **kwargs): - +def sync_last_update_for_objects(*args, **kwargs): skip_repos = kwargs.get('skip_repos') if not skip_repos: repos = Repository.query() \ @@ -405,3 +403,8 @@ def sync_last_update(*args, **kwargs): for root_gr in repo_groups: for repo_gr in reversed(root_gr.recursive_groups()): repo_gr.update_commit_cache() + + +@async_task(ignore_result=True) +def sync_last_update(*args, **kwargs): + sync_last_update_for_objects(*args, **kwargs) From 720dbed814f7a77fb0b131644ac3997538d46914 Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Tue, 12 Oct 2021 14:29:26 +0200 Subject: [PATCH 7/8] ops: added healthcheck view --- rhodecode/apps/ops/__init__.py | 8 ++++++++ rhodecode/apps/ops/views.py | 20 ++++++++++++++++++++ rhodecode/lib/middleware/vcs.py | 3 ++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/rhodecode/apps/ops/__init__.py b/rhodecode/apps/ops/__init__.py index 16e42d60..7279e061 100644 --- a/rhodecode/apps/ops/__init__.py +++ b/rhodecode/apps/ops/__init__.py @@ -51,6 +51,14 @@ def admin_routes(config): route_name='ops_redirect_test', request_method='GET', renderer='json_ext') + config.add_route( + name='ops_healthcheck', + pattern='/status') + config.add_view( + OpsView, + attr='ops_healthcheck', + route_name='ops_healthcheck', request_method='GET', + renderer='json_ext') def includeme(config): config.include(admin_routes, route_prefix=ADMIN_PREFIX + '/ops') diff --git a/rhodecode/apps/ops/views.py b/rhodecode/apps/ops/views.py index 56eae33d..20c12d23 100644 --- a/rhodecode/apps/ops/views.py +++ b/rhodecode/apps/ops/views.py @@ -72,3 +72,23 @@ class OpsView(BaseAppView): """ redirect_to = self.request.GET.get('to') or h.route_path('home') raise HTTPFound(redirect_to) + + def ops_healthcheck(self): + from rhodecode.lib.system_info import load_system_info + + vcsserver_info = load_system_info('vcs_server') + if vcsserver_info: + vcsserver_info = vcsserver_info['human_value'] + + db_info = load_system_info('database_info') + if db_info: + db_info = db_info['human_value'] + + health_spec = { + 'caller_ip': self.request.user.ip_addr, + 'vcsserver': vcsserver_info, + 'db': db_info, + } + + return {'healthcheck': health_spec} + diff --git a/rhodecode/lib/middleware/vcs.py b/rhodecode/lib/middleware/vcs.py index 9b8a673b..d4b57b29 100644 --- a/rhodecode/lib/middleware/vcs.py +++ b/rhodecode/lib/middleware/vcs.py @@ -166,8 +166,9 @@ def detect_vcs_request(environ, backends): # static files no detection '_static', - # skip ops ping + # skip ops ping, status '_admin/ops/ping', + '_admin/ops/status', # full channelstream connect should be VCS skipped '_admin/channelstream/connect', From d1bfd0ade0e1009f3e59992983d6c7bfd5211672 Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Tue, 12 Oct 2021 14:47:16 +0200 Subject: [PATCH 8/8] healthcheck: Added authentication because we expose DB information --- rhodecode/apps/ops/views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rhodecode/apps/ops/views.py b/rhodecode/apps/ops/views.py index 20c12d23..5cacf350 100644 --- a/rhodecode/apps/ops/views.py +++ b/rhodecode/apps/ops/views.py @@ -26,6 +26,8 @@ from pyramid.httpexceptions import HTTPFound from rhodecode.apps._base import BaseAppView from rhodecode.lib import helpers as h +from rhodecode.lib.auth import LoginRequired +from rhodecode.model.db import UserApiKeys log = logging.getLogger(__name__) @@ -73,6 +75,7 @@ class OpsView(BaseAppView): redirect_to = self.request.GET.get('to') or h.route_path('home') raise HTTPFound(redirect_to) + @LoginRequired(auth_token_access=[UserApiKeys.ROLE_HTTP]) def ops_healthcheck(self): from rhodecode.lib.system_info import load_system_info