From 4b5baaabb1e3dbb535d586a06ef6106f30f4ce32 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Thu, 15 Feb 2018 17:34:09 +0100 Subject: [PATCH] caches: use individual namespaces per user to prevent beaker caching problems. - especially for mysql in case large number of data in caches there could be critical errors storing cache, and thus preventing users from authentication. This is caused by the fact that we used single namespace for ALL users. It means it grew as number of users grew reaching mysql single column limit. This changes the behaviour and now we use namespace per-user it means that each user-id will have it's own cache namespace fragmenting maximum column data to a single user cache. Which we should never reach. --- rhodecode/authentication/base.py | 17 ++++++++++++----- rhodecode/authentication/views.py | 17 +++++++++++------ rhodecode/lib/middleware/simplevcs.py | 3 ++- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/rhodecode/authentication/base.py b/rhodecode/authentication/base.py index 5683a2d8..bd96300f 100644 --- a/rhodecode/authentication/base.py +++ b/rhodecode/authentication/base.py @@ -578,14 +578,20 @@ def get_authn_registry(registry=None): return authn_registry -def get_auth_cache_manager(custom_ttl=None): +def get_auth_cache_manager(custom_ttl=None, suffix=None): + cache_name = 'rhodecode.authentication' + if suffix: + cache_name = 'rhodecode.authentication.{}'.format(suffix) return caches.get_cache_manager( - 'auth_plugins', 'rhodecode.authentication', custom_ttl) + 'auth_plugins', cache_name, custom_ttl) -def get_perms_cache_manager(custom_ttl=None): +def get_perms_cache_manager(custom_ttl=None, suffix=None): + cache_name = 'rhodecode.permissions' + if suffix: + cache_name = 'rhodecode.permissions.{}'.format(suffix) return caches.get_cache_manager( - 'auth_plugins', 'rhodecode.permissions', custom_ttl) + 'auth_plugins', cache_name, custom_ttl) def authenticate(username, password, environ=None, auth_type=None, @@ -646,7 +652,8 @@ def authenticate(username, password, environ=None, auth_type=None, plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings) # get instance of cache manager configured for a namespace - cache_manager = get_auth_cache_manager(custom_ttl=cache_ttl) + cache_manager = get_auth_cache_manager( + custom_ttl=cache_ttl, suffix=user.user_id) log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)', plugin.get_id(), plugin_cache_active, cache_ttl) diff --git a/rhodecode/authentication/views.py b/rhodecode/authentication/views.py index b721bf41..620af0be 100644 --- a/rhodecode/authentication/views.py +++ b/rhodecode/authentication/views.py @@ -32,6 +32,7 @@ from rhodecode.authentication.base import ( from rhodecode.lib import helpers as h from rhodecode.lib.auth import ( LoginRequired, HasPermissionAllDecorator, CSRFRequired) +from rhodecode.lib.caches import clear_cache_manager from rhodecode.model.forms import AuthSettingsForm from rhodecode.model.meta import Session from rhodecode.model.settings import SettingsModel @@ -101,6 +102,16 @@ class AuthnPluginViewBase(BaseAppView): self.plugin.create_or_update_setting(name, value) Session().commit() + # cleanup cache managers in case of change for plugin + # TODO(marcink): because we can register multiple namespaces + # we should at some point figure out how to retrieve ALL namespace + # cache managers and clear them... + cache_manager = get_auth_cache_manager() + clear_cache_manager(cache_manager) + + cache_manager = get_perms_cache_manager() + clear_cache_manager(cache_manager) + # Display success message and redirect. h.flash(_('Auth settings updated successfully.'), category='success') redirect_to = self.request.resource_path( @@ -161,12 +172,6 @@ class AuthSettingsView(BaseAppView): Session().add(setting) Session().commit() - cache_manager = get_auth_cache_manager() - cache_manager.clear() - - cache_manager = get_perms_cache_manager() - cache_manager.clear() - h.flash(_('Auth settings updated successfully.'), category='success') except formencode.Invalid as errors: e = errors.error_dict or {} diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py index ef04537e..f9cd9077 100644 --- a/rhodecode/lib/middleware/simplevcs.py +++ b/rhodecode/lib/middleware/simplevcs.py @@ -288,7 +288,8 @@ class SimpleVCS(object): """ # get instance of cache manager configured for a namespace - cache_manager = get_perms_cache_manager(custom_ttl=cache_ttl) + cache_manager = get_perms_cache_manager( + custom_ttl=cache_ttl, suffix=user.user_id) log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)', plugin_id, plugin_cache_active, cache_ttl)