engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

caches: rewrite of auth/permission caches to dogpile.

Browse source
This commit is contained in:
Marcin Kuzminski 2018-07-05 20:14:33 +02:00
parent c8cd3e9740
commit ddc445856e
20 changed files with 565 additions and 231 deletions
Download patch file Download diff file Expand all files Collapse all files

17
configs/development.ini
View file

@ -320,12 +320,7 @@ cache_dir = %(here)s/data
beaker.cache.data_dir = %(here)s/data/cache/beaker_data
beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
# used for caching user permissions
beaker.cache.short_term.type = file
beaker.cache.short_term.expire = 0
beaker.cache.short_term.key_length = 256
beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
beaker.cache.long_term.type = memory
beaker.cache.long_term.expire = 36000
@ -335,16 +330,6 @@ beaker.cache.sql_cache_short.type = memory
beaker.cache.sql_cache_short.expire = 10
beaker.cache.sql_cache_short.key_length = 256
## default is memory cache, configure only if required
## using multi-node or multi-worker setup
#beaker.cache.auth_plugins.type = ext:database
#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
#beaker.cache.auth_plugins.sa.pool_recycle = 3600
#beaker.cache.auth_plugins.sa.pool_size = 10
#beaker.cache.auth_plugins.sa.max_overflow = 0
beaker.cache.repo_cache_long.type = memorylru_base
beaker.cache.repo_cache_long.max_items = 4096
beaker.cache.repo_cache_long.expire = 2592000

17
configs/production.ini
View file

@ -295,12 +295,7 @@ cache_dir = %(here)s/data
beaker.cache.data_dir = %(here)s/data/cache/beaker_data
beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
# used for caching user permissions
beaker.cache.short_term.type = file
beaker.cache.short_term.expire = 0
beaker.cache.short_term.key_length = 256
beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
beaker.cache.long_term.type = memory
beaker.cache.long_term.expire = 36000
@ -310,16 +305,6 @@ beaker.cache.sql_cache_short.type = memory
beaker.cache.sql_cache_short.expire = 10
beaker.cache.sql_cache_short.key_length = 256
## default is memory cache, configure only if required
## using multi-node or multi-worker setup
#beaker.cache.auth_plugins.type = ext:database
#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
#beaker.cache.auth_plugins.sa.pool_recycle = 3600
#beaker.cache.auth_plugins.sa.pool_size = 10
#beaker.cache.auth_plugins.sa.max_overflow = 0
beaker.cache.repo_cache_long.type = memorylru_base
beaker.cache.repo_cache_long.max_items = 4096
beaker.cache.repo_cache_long.expire = 2592000

10
rhodecode/apps/admin/__init__.py
View file

@ -356,6 +356,16 @@ def admin_routes(config):
name='edit_user_audit_logs',
pattern='/users/{user_id:\d+}/edit/audit', user_route=True)
# user caches
config.add_route(
name='edit_user_caches',
pattern='/users/{user_id:\d+}/edit/caches',
user_route=True)
config.add_route(
name='edit_user_caches_update',
pattern='/users/{user_id:\d+}/edit/caches/update',
user_route=True)
# user-groups admin
config.add_route(
name='user_groups',

56
rhodecode/apps/admin/views/users.py
View file

@ -34,7 +34,7 @@ from rhodecode.authentication.plugins import auth_rhodecode
from rhodecode.events import trigger
from rhodecode.model.db import true
from rhodecode.lib import audit_logger
from rhodecode.lib import audit_logger, rc_cache
from rhodecode.lib.exceptions import (
UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException,
UserOwnsUserGroupsException, DefaultUserException)
@ -1198,3 +1198,57 @@ class UsersView(UserAppView):
perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr)
return perm_user.permissions
def _get_user_cache_keys(self, cache_namespace_uid, keys):
user_keys = []
for k in sorted(keys):
if k.startswith(cache_namespace_uid):
user_keys.append(k)
return user_keys
@LoginRequired()
@HasPermissionAllDecorator('hg.admin')
@view_config(
route_name='edit_user_caches', request_method='GET',
renderer='rhodecode:templates/admin/users/user_edit.mako')
def user_caches(self):
_ = self.request.translate
c = self.load_default_context()
c.user = self.db_user
c.active = 'caches'
c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
c.backend = c.region.backend
c.user_keys = self._get_user_cache_keys(
cache_namespace_uid, c.region.backend.list_keys())
return self._get_template_context(c)
@LoginRequired()
@HasPermissionAllDecorator('hg.admin')
@CSRFRequired()
@view_config(
route_name='edit_user_caches_update', request_method='POST')
def user_caches_update(self):
_ = self.request.translate
c = self.load_default_context()
c.user = self.db_user
c.active = 'caches'
c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
c.user_keys = self._get_user_cache_keys(
cache_namespace_uid, c.region.backend.list_keys())
for k in c.user_keys:
c.region.delete(k)
h.flash(_("Deleted {} cache keys").format(len(c.user_keys)), category='success')
return HTTPFound(h.route_path(
'edit_user_caches', user_id=c.user.user_id))

68
rhodecode/authentication/base.py
View file

@ -35,7 +35,7 @@ from pyramid.threadlocal import get_current_registry
from rhodecode.authentication.interface import IAuthnPluginRegistry
from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
from rhodecode.lib import caches
from rhodecode.lib import caches, rc_cache
from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
from rhodecode.lib.utils2 import safe_int, safe_str
from rhodecode.lib.exceptions import LdapConnectionError
@ -633,22 +633,6 @@ def get_authn_registry(registry=None):
return authn_registry
def get_auth_cache_manager(custom_ttl=None, suffix=None):
cache_name = 'rhodecode.authentication'
if suffix:
cache_name = 'rhodecode.authentication.{}'.format(suffix)
return caches.get_cache_manager(
'auth_plugins', cache_name, custom_ttl)
def get_perms_cache_manager(custom_ttl=None, suffix=None):
cache_name = 'rhodecode.permissions'
if suffix:
cache_name = 'rhodecode.permissions.{}'.format(suffix)
return caches.get_cache_manager(
'auth_plugins', cache_name, custom_ttl)
def authenticate(username, password, environ=None, auth_type=None,
skip_missing=False, registry=None, acl_repo_name=None):
"""
@ -707,45 +691,35 @@ def authenticate(username, password, environ=None, auth_type=None,
plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings)
# get instance of cache manager configured for a namespace
cache_manager = get_auth_cache_manager(
custom_ttl=cache_ttl, suffix=user.user_id if user else '')
log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)',
plugin.get_id(), plugin_cache_active, cache_ttl)
# for environ based password can be empty, but then the validation is
# on the server that fills in the env data needed for authentication
user_id = user.user_id
cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
_password_hash = caches.compute_key_from_params(
plugin.name, username, (password or ''))
@region.cache_on_arguments(namespace=cache_namespace_uid,
expiration_time=cache_ttl,
should_cache_fn=lambda v: plugin_cache_active)
def compute_auth(
cache_name, plugin_name, username, password):
# _authenticate is a wrapper for .auth() method of plugin.
# it checks if .auth() sends proper data.
# For RhodeCodeExternalAuthPlugin it also maps users to
# Database and maps the attributes returned from .auth()
# to RhodeCode database. If this function returns data
# then auth is correct.
start = time.time()
log.debug('Running plugin `%s` _authenticate method', plugin.get_id())
def auth_func():
"""
This function is used internally in Cache of Beaker to calculate
Results
"""
log.debug('auth: calculating password access now...')
# _authenticate is a wrapper for .auth() method of plugin.
# it checks if .auth() sends proper data.
# For RhodeCodeExternalAuthPlugin it also maps users to
# Database and maps the attributes returned from .auth()
# to RhodeCode database. If this function returns data
# then auth is correct.
log.debug('Running plugin `%s` _authenticate method '
'using username and password', plugin.get_id())
return plugin._authenticate(
user, username, password, plugin_settings,
environ=environ or {})
if plugin_cache_active:
log.debug('Trying to fetch cached auth by pwd hash `...%s`',
_password_hash[:6])
plugin_user = cache_manager.get(
_password_hash, createfunc=auth_func)
else:
plugin_user = auth_func()
start = time.time()
# for environ based auth, password can be empty, but then the validation is
# on the server that fills in the env data needed for authentication
plugin_user = compute_auth('auth', plugin.name, username, (password or ''))
auth_time = time.time() - start
log.debug('Authentication for plugin `%s` completed in %.3fs, '

13
rhodecode/authentication/views.py
View file

@ -27,8 +27,7 @@ from pyramid.renderers import render
from pyramid.response import Response
from rhodecode.apps._base import BaseAppView
from rhodecode.authentication.base import (
get_auth_cache_manager, get_perms_cache_manager, get_authn_registry)
from rhodecode.authentication.base import get_authn_registry
from rhodecode.lib import helpers as h
from rhodecode.lib.auth import (
LoginRequired, HasPermissionAllDecorator, CSRFRequired)
@ -102,16 +101,6 @@ class AuthnPluginViewBase(BaseAppView):
self.plugin.create_or_update_setting(name, value)
Session().commit()
# cleanup cache managers in case of change for plugin
# TODO(marcink): because we can register multiple namespaces
# we should at some point figure out how to retrieve ALL namespace
# cache managers and clear them...
cache_manager = get_auth_cache_manager()
clear_cache_manager(cache_manager)
cache_manager = get_perms_cache_manager()
clear_cache_manager(cache_manager)
# Display success message and redirect.
h.flash(_('Auth settings updated successfully.'), category='success')
redirect_to = self.request.resource_path(

15
rhodecode/config/middleware.py
View file

@ -22,6 +22,7 @@ import os
import logging
import traceback
import collections
import tempfile
from paste.gzipper import make_gzip_middleware
from pyramid.wsgi import wsgiapp
@ -220,6 +221,7 @@ def includeme(config):
config.include('pyramid_mako')
config.include('pyramid_beaker')
config.include('rhodecode.lib.caches')
config.include('rhodecode.lib.rc_cache')
config.include('rhodecode.authentication')
config.include('rhodecode.integrations')
@ -382,6 +384,7 @@ def sanitize_settings_and_apply_defaults(settings):
# Call split out functions that sanitize settings for each topic.
_sanitize_appenlight_settings(settings)
_sanitize_vcs_settings(settings)
_sanitize_cache_settings(settings)
# configure instance id
config_utils.set_instance_id(settings)
@ -421,6 +424,18 @@ def _sanitize_vcs_settings(settings):
settings['vcs.scm_app_implementation'] = 'http'
def _sanitize_cache_settings(settings):
_string_setting(settings, 'cache_dir',
os.path.join(tempfile.gettempdir(), 'rc_cache'))
_string_setting(settings, 'rc_cache.cache_perms.backend',
'dogpile.cache.rc.file_namespace')
_int_setting(settings, 'rc_cache.cache_perms.expiration_time',
60)
_string_setting(settings, 'rc_cache.cache_perms.arguments.filename',
os.path.join(tempfile.gettempdir(), 'rc_cache_1'))
def _int_setting(settings, name, default):
settings[name] = int(settings.get(name, default))

68
rhodecode/lib/auth.py
View file

@ -48,7 +48,7 @@ from rhodecode.model.user import UserModel
from rhodecode.model.db import (
User, Repository, Permission, UserToPerm, UserGroupToPerm, UserGroupMember,
UserIpMap, UserApiKeys, RepoGroup, UserGroup)
from rhodecode.lib import caches
from rhodecode.lib import rc_cache
from rhodecode.lib.utils2 import safe_unicode, aslist, safe_str, md5, safe_int, sha1
from rhodecode.lib.utils import (
get_repo_slug, get_repo_group_slug, get_user_group_slug)
@ -976,21 +976,18 @@ class AuthUser(object):
obj = Repository.get_by_repo_name(scope['repo_name'])
if obj:
scope['repo_id'] = obj.repo_id
_scope = {
'repo_id': -1,
'user_group_id': -1,
'repo_group_id': -1,
}
_scope.update(scope)
cache_key = "_".join(map(safe_str, reduce(lambda a, b: a+b,
_scope.items())))
if cache_key not in self._permissions_scoped_cache:
# store in cache to mimic how the @LazyProperty works,
# the difference here is that we use the unique key calculated
# from params and values
res = self.get_perms(user=self, cache=False, scope=_scope)
self._permissions_scoped_cache[cache_key] = res
return self._permissions_scoped_cache[cache_key]
_scope = collections.OrderedDict()
_scope['repo_id'] = -1
_scope['user_group_id'] = -1
_scope['repo_group_id'] = -1
for k in sorted(scope.keys()):
_scope[k] = scope[k]
# store in cache to mimic how the @LazyProperty works,
# the difference here is that we use the unique key calculated
# from params and values
return self.get_perms(user=self, cache=False, scope=_scope)
def get_instance(self):
return User.get(self.user_id)
@ -1072,16 +1069,27 @@ class AuthUser(object):
user_inherit_default_permissions = user.inherit_default_permissions
cache_seconds = safe_int(
rhodecode.CONFIG.get('beaker.cache.short_term.expire'))
rhodecode.CONFIG.get('rc_cache.cache_perms.expiration_time'))
cache_on = cache or cache_seconds > 0
log.debug(
'Computing PERMISSION tree for user %s scope `%s` '
'with caching: %s[%ss]' % (user, scope, cache_on, cache_seconds))
'with caching: %s[TTL: %ss]' % (user, scope, cache_on, cache_seconds or 0))
cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
@region.cache_on_arguments(namespace=cache_namespace_uid,
should_cache_fn=lambda v: cache_on)
def compute_perm_tree(cache_name,
user_id, scope, user_is_admin,user_inherit_default_permissions,
explicit, algo, calculate_super_admin):
return _cached_perms_data(
user_id, scope, user_is_admin, user_inherit_default_permissions,
explicit, algo, calculate_super_admin)
start = time.time()
compute = caches.conditional_cache(
'short_term', 'cache_desc.{}'.format(user_id),
condition=cache_on, func=_cached_perms_data)
result = compute(user_id, scope, user_is_admin,
result = compute_perm_tree('permissions', user_id, scope, user_is_admin,
user_inherit_default_permissions, explicit, algo,
calculate_super_admin)
@ -1091,6 +1099,7 @@ class AuthUser(object):
total = time.time() - start
log.debug('PERMISSION tree for user %s computed in %.3fs: %s' % (
user, total, result_repr))
return result
@property
@ -1153,10 +1162,7 @@ class AuthUser(object):
return [x.repo_id for x in
RepoList(qry, perm_set=perm_def)]
compute = caches.conditional_cache(
'short_term', 'repo_acl_ids.{}'.format(self.user_id),
condition=cache, func=_cached_repo_acl)
return compute(self.user_id, perms, name_filter)
return _cached_repo_acl(self.user_id, perms, name_filter)
def repo_group_acl_ids(self, perms=None, name_filter=None, cache=False):
"""
@ -1179,10 +1185,7 @@ class AuthUser(object):
return [x.group_id for x in
RepoGroupList(qry, perm_set=perm_def)]
compute = caches.conditional_cache(
'short_term', 'repo_group_acl_ids.{}'.format(self.user_id),
condition=cache, func=_cached_repo_group_acl)
return compute(self.user_id, perms, name_filter)
return _cached_repo_group_acl(self.user_id, perms, name_filter)
def user_group_acl_ids(self, perms=None, name_filter=None, cache=False):
"""
@ -1205,10 +1208,7 @@ class AuthUser(object):
return [x.users_group_id for x in
UserGroupList(qry, perm_set=perm_def)]
compute = caches.conditional_cache(
'short_term', 'user_group_acl_ids.{}'.format(self.user_id),
condition=cache, func=_cached_user_group_acl)
return compute(self.user_id, perms, name_filter)
return _cached_user_group_acl(self.user_id, perms, name_filter)
@property
def ip_allowed(self):

1
rhodecode/lib/base.py
View file

@ -505,6 +505,7 @@ def bootstrap_config(request):
config.include('pyramid_mako')
config.include('pyramid_beaker')
config.include('rhodecode.lib.caches')
config.include('rhodecode.lib.rc_cache')
add_events_routes(config)

4
rhodecode/lib/caches.py
View file

@ -96,7 +96,7 @@ def get_cache_manager(region_name, cache_name, custom_ttl=None):
Creates a Beaker cache manager. Such instance can be used like that::
_namespace = caches.get_repo_namespace_key(caches.XXX, repo_name)
cache_manager = caches.get_cache_manager('repo_cache_long', _namespace)
cache_manager = caches.get_cache_manager('some_namespace_name', _namespace)
_cache_key = caches.compute_key_from_params(repo_name, commit.raw_id)
def heavy_compute():
...
@ -121,7 +121,7 @@ def get_cache_manager(region_name, cache_name, custom_ttl=None):
def clear_cache_manager(cache_manager):
"""
namespace = 'foobar'
cache_manager = get_cache_manager('repo_cache_long', namespace)
cache_manager = get_cache_manager('some_namespace_name', namespace)
clear_cache_manager(cache_manager)
"""

48
rhodecode/lib/middleware/simplevcs.py
View file

@ -39,9 +39,8 @@ from pyramid.httpexceptions import (
from zope.cachedescriptors.property import Lazy as LazyProperty
import rhodecode
from rhodecode.authentication.base import (
authenticate, get_perms_cache_manager, VCS_TYPE, loadplugin)
from rhodecode.lib import caches
from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
from rhodecode.lib import caches, rc_cache
from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
from rhodecode.lib.base import (
BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
@ -311,36 +310,24 @@ class SimpleVCS(object):
:param repo_name: repository name
"""
# get instance of cache manager configured for a namespace
cache_manager = get_perms_cache_manager(
custom_ttl=cache_ttl, suffix=user.user_id)
log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
plugin_id, plugin_cache_active, cache_ttl)
# for environ based password can be empty, but then the validation is
# on the server that fills in the env data needed for authentication
_perm_calc_hash = caches.compute_key_from_params(
plugin_id, action, user.user_id, repo_name, ip_addr)
user_id = user.user_id
cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
# _authenticate is a wrapper for .auth() method of plugin.
# it checks if .auth() sends proper data.
# For RhodeCodeExternalAuthPlugin it also maps users to
# Database and maps the attributes returned from .auth()
# to RhodeCode database. If this function returns data
# then auth is correct.
start = time.time()
log.debug('Running plugin `%s` permissions check', plugin_id)
@region.cache_on_arguments(namespace=cache_namespace_uid,
expiration_time=cache_ttl,
should_cache_fn=lambda v: plugin_cache_active)
def compute_perm_vcs(
cache_name, plugin_id, action, user_id, repo_name, ip_addr):
def perm_func():
"""
This function is used internally in Cache of Beaker to calculate
Results
"""
log.debug('auth: calculating permission access now...')
# check IP
inherit = user.inherit_default_permissions
ip_allowed = AuthUser.check_ip_allowed(
user.user_id, ip_addr, inherit_from_default=inherit)
user_id, ip_addr, inherit_from_default=inherit)
if ip_allowed:
log.info('Access for IP:%s allowed', ip_addr)
else:
@ -360,12 +347,13 @@ class SimpleVCS(object):
return True
if plugin_cache_active:
log.debug('Trying to fetch cached perms by %s', _perm_calc_hash[:6])
perm_result = cache_manager.get(
_perm_calc_hash, createfunc=perm_func)
else:
perm_result = perm_func()
start = time.time()
log.debug('Running plugin `%s` permissions check', plugin_id)
# for environ based auth, password can be empty, but then the validation is
# on the server that fills in the env data needed for authentication
perm_result = compute_perm_vcs(
'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
auth_time = time.time() - start
log.debug('Permissions for plugin `%s` completed in %.3fs, '

66
rhodecode/lib/rc_cache/__init__.py Normal file
View file

@ -0,0 +1,66 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2015-2018 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
from dogpile.cache import register_backend
from dogpile.cache import make_region
register_backend(
"dogpile.cache.rc.memory_lru", "rhodecode.lib.rc_cache.backends",
"LRUMemoryBackend")
register_backend(
"dogpile.cache.rc.file_namespace", "rhodecode.lib.rc_cache.backends",
"FileNamespaceBackend")
register_backend(
"dogpile.cache.rc.redis", "rhodecode.lib.rc_cache.backends",
"RedisPickleBackend")
from . import region_meta
from .utils import get_default_cache_settings, key_generator, get_or_create_region
def configure_dogpile_cache(settings):
cache_dir = settings.get('cache_dir')
if cache_dir:
region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
# inspect available namespaces
avail_regions = set()
for key in rc_cache_data.keys():
namespace_name = key.split('.', 1)[0]
avail_regions.add(namespace_name)
# register them into namespace
for region_name in avail_regions:
new_region = make_region(
name=region_name,
function_key_generator=key_generator
)
new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
region_meta.dogpile_cache_regions[region_name] = new_region
def includeme(config):
configure_dogpile_cache(config.registry.settings)

109
rhodecode/lib/rc_cache/backends.py Normal file
View file

@ -0,0 +1,109 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2015-2018 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
from dogpile.cache.backends import memory as memory_backend
from dogpile.cache.backends import file as file_backend
from dogpile.cache.backends import redis as redis_backend
from dogpile.cache.backends.file import NO_VALUE, compat
from rhodecode.lib.memory_lru_debug import LRUDict
_default_max_size = 1024
class LRUMemoryBackend(memory_backend.MemoryBackend):
def __init__(self, arguments):
max_size = arguments.pop('max_size', _default_max_size)
arguments['cache_dict'] = LRUDict(max_size)
super(LRUMemoryBackend, self).__init__(arguments)
class Serializer(object):
def _dumps(self, value):
return compat.pickle.dumps(value)
def _loads(self, value):
return compat.pickle.loads(value)
class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
def __init__(self, arguments):
super(FileNamespaceBackend, self).__init__(arguments)
def list_keys(self):
with self._dbm_file(True) as dbm:
return dbm.keys()
def get_store(self):
return self.filename
def get(self, key):
with self._dbm_file(False) as dbm:
if hasattr(dbm, 'get'):
value = dbm.get(key, NO_VALUE)
else:
# gdbm objects lack a .get method
try:
value = dbm[key]
except KeyError:
value = NO_VALUE
if value is not NO_VALUE:
value = self._loads(value)
return value
def set(self, key, value):
with self._dbm_file(True) as dbm:
dbm[key] = self._dumps(value)
def set_multi(self, mapping):
with self._dbm_file(True) as dbm:
for key, value in mapping.items():
dbm[key] = self._dumps(value)
class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
def list_keys(self):
return self.client.keys()
def get_store(self):
return self.client.connection_pool
def set(self, key, value):
if self.redis_expiration_time:
self.client.setex(key, self.redis_expiration_time,
self._dumps(value))
else:
self.client.set(key, self._dumps(value))
def set_multi(self, mapping):
mapping = dict(
(k, self._dumps(v))
for k, v in mapping.items()
)
if not self.redis_expiration_time:
self.client.mset(mapping)
else:
pipe = self.client.pipeline()
for key, value in mapping.items():
pipe.setex(key, self.redis_expiration_time, value)
pipe.execute()

28
rhodecode/lib/rc_cache/region_meta.py Normal file
View file

@ -0,0 +1,28 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2015-2018 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import tempfile
dogpile_config_defaults = {
'cache_dir': os.path.join(tempfile.gettempdir(), 'rc_cache')
}
# GLOBAL TO STORE ALL REGISTERED REGIONS
dogpile_cache_regions = {}

99
rhodecode/lib/rc_cache/utils.py Normal file
View file

@ -0,0 +1,99 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2015-2018 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import logging
from dogpile.cache import make_region
from rhodecode.lib.utils import safe_str, sha1
from . import region_meta
log = logging.getLogger(__name__)
def get_default_cache_settings(settings, prefixes=None):
prefixes = prefixes or []
cache_settings = {}
for key in settings.keys():
for prefix in prefixes:
if key.startswith(prefix):
name = key.split(prefix)[1].strip()
val = settings[key]
if isinstance(val, basestring):
val = val.strip()
cache_settings[name] = val
return cache_settings
def compute_key_from_params(*args):
"""
Helper to compute key from given params to be used in cache manager
"""
return sha1("_".join(map(safe_str, args)))
def key_generator(namespace, fn):
fname = fn.__name__
def generate_key(*args):
namespace_pref = namespace or 'default'
arg_key = compute_key_from_params(*args)
final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
return final_key
return generate_key
def get_or_create_region(region_name, region_namespace=None):
from rhodecode.lib.rc_cache.backends import FileNamespaceBackend
region_obj = region_meta.dogpile_cache_regions.get(region_name)
if not region_obj:
raise EnvironmentError(
'Region `{}` not in configured: {}.'.format(
region_name, region_meta.dogpile_cache_regions.keys()))
region_uid_name = '{}:{}'.format(region_name, region_namespace)
if isinstance(region_obj.actual_backend, FileNamespaceBackend):
region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
if region_exist:
log.debug('Using already configured region: %s', region_namespace)
return region_exist
cache_dir = region_meta.dogpile_config_defaults['cache_dir']
expiration_time = region_obj.expiration_time
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
new_region = make_region(
name=region_uid_name, function_key_generator=key_generator
)
namespace_filename = os.path.join(
cache_dir, "{}.cache.dbm".format(region_namespace))
# special type that allows 1db per namespace
new_region.configure(
backend='dogpile.cache.rc.file_namespace',
expiration_time=expiration_time,
arguments={"filename": namespace_filename}
)
# create and save in region caches
log.debug('configuring new region: %s',region_uid_name)
region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
return region_obj

2
rhodecode/public/js/rhodecode/routes.js
View file

@ -119,6 +119,8 @@ function registerRCRoutes() {
pyroutes.register('edit_user_groups_management', '/_admin/users/%(user_id)s/edit/groups_management', ['user_id']);
pyroutes.register('edit_user_groups_management_updates', '/_admin/users/%(user_id)s/edit/edit_user_groups_management/updates', ['user_id']);
pyroutes.register('edit_user_audit_logs', '/_admin/users/%(user_id)s/edit/audit', ['user_id']);
pyroutes.register('edit_user_caches', '/_admin/users/%(user_id)s/edit/caches', ['user_id']);
pyroutes.register('edit_user_caches_update', '/_admin/users/%(user_id)s/edit/caches/update', ['user_id']);
pyroutes.register('user_groups', '/_admin/user_groups', []);
pyroutes.register('user_groups_data', '/_admin/user_groups_data', []);
pyroutes.register('user_groups_new', '/_admin/user_groups/new', []);

1
rhodecode/templates/admin/users/user_edit.mako
View file

@ -45,6 +45,7 @@
<li class="${'active' if c.active=='ips' else ''}"><a href="${h.route_path('edit_user_ips', user_id=c.user.user_id)}">${_('Ip Whitelist')}</a></li>
<li class="${'active' if c.active=='groups' else ''}"><a href="${h.route_path('edit_user_groups_management', user_id=c.user.user_id)}">${_('User Groups Management')}</a></li>
<li class="${'active' if c.active=='audit' else ''}"><a href="${h.route_path('edit_user_audit_logs', user_id=c.user.user_id)}">${_('Audit logs')}</a></li>
<li class="${'active' if c.active=='caches' else ''}"><a href="${h.route_path('edit_user_caches', user_id=c.user.user_id)}">${_('Caches')}</a></li>
</ul>
</div>

29
rhodecode/templates/admin/users/user_edit_caches.mako Normal file
View file

@ -0,0 +1,29 @@
<%namespace name="base" file="/base/base.mako"/>
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">${_('Caches')}</h3>
</div>
<div class="panel-body">
<pre>
region: ${c.region.name}
backend: ${c.region.actual_backend.__class__}
store: ${c.region.actual_backend.get_store()}
% for k in c.user_keys:
- ${k}
% endfor
</pre>
${h.secure_form(h.route_path('edit_user_caches_update', user_id=c.user.user_id), request=request)}
<div class="form">
<div class="fields">
${h.submit('reset_cache_%s' % c.user.user_id, _('Invalidate user cache'),class_="btn btn-small",onclick="return confirm('"+_('Confirm to invalidate user cache')+"');")}
</div>
</div>
${h.end_form()}
</div>
</div>

119
rhodecode/tests/lib/test_caches.py
View file

@ -22,36 +22,24 @@ import time
import pytest
from rhodecode.lib import caches
from rhodecode.lib.memory_lru_debug import MemoryLRUNamespaceManagerBase
from rhodecode.lib import rc_cache
class TestCacheManager(object):
@pytest.mark.usefixtures( 'app')
class TestCaches(object):
@pytest.mark.parametrize('repo_name', [
('',),
(u'',),
(u'ac',),
('ac', ),
(u'ęćc',),
('ąac',),
def test_cache_decorator_init_not_configured(self):
with pytest.raises(EnvironmentError):
rc_cache.get_or_create_region('dontexist')
@pytest.mark.parametrize('region_name', [
'cache_perms', u'cache_perms',
])
def test_cache_manager_init(self, repo_name):
cache_manager_instance = caches.get_cache_manager(
repo_name, 'my_cache')
assert cache_manager_instance
def test_cache_manager_init_undefined_namespace(self):
cache_manager_instance = caches.get_cache_manager(
'repo_cache_long_undefined', 'my_cache')
assert cache_manager_instance
def_config = caches.DEFAULT_CACHE_MANAGER_CONFIG.copy()
def_config.pop('type')
assert cache_manager_instance.nsargs == def_config
assert isinstance(
cache_manager_instance.namespace, MemoryLRUNamespaceManagerBase)
def test_cache_decorator_init(self, region_name):
namespace = region_name
cache_region = rc_cache.get_or_create_region(
region_name, region_namespace=namespace)
assert cache_region
@pytest.mark.parametrize('example_input', [
('',),
@ -62,45 +50,60 @@ class TestCacheManager(object):
(u'/ac', ),
])
def test_cache_manager_create_key(self, example_input):
key = caches.compute_key_from_params(*example_input)
key = rc_cache.utils.compute_key_from_params(*example_input)
assert key
def test_store_and_invalidate_value_from_manager(self):
cache_manger_instance = caches.get_cache_manager(
'repo_cache_long', 'my_cache_store')
@pytest.mark.parametrize('example_namespace', [
'namespace', None
])
@pytest.mark.parametrize('example_input', [
('',),
(u'/ac',),
(u'/ac', 1, 2, object()),
(u'/ęćc', 1, 2, object()),
('/ąac',),
(u'/ac', ),
])
def test_cache_keygen(self, example_input, example_namespace):
def func_wrapped():
return 1
func = rc_cache.utils.key_generator(example_namespace, func_wrapped)
key = func(*example_input)
assert key
def compute():
def test_store_value_in_cache(self):
cache_region = rc_cache.get_or_create_region('cache_perms')
# make sure we empty the cache now
for key in cache_region.backend.list_keys():
cache_region.delete(key)
assert cache_region.backend.list_keys() == []
@cache_region.cache_on_arguments(expiration_time=5)
def compute(key):
return time.time()
added_keys = []
for i in xrange(3):
_cache_key = caches.compute_key_from_params('foo_bar', 'p%s' % i)
added_keys.append(_cache_key)
for x in xrange(10):
cache_manger_instance.get(_cache_key, createfunc=compute)
for x in range(10):
compute(x)
for key in added_keys:
assert cache_manger_instance[key]
assert len(set(cache_region.backend.list_keys())) == 10
caches.clear_cache_manager(cache_manger_instance)
def test_store_and_get_value_from_region(self):
cache_region = rc_cache.get_or_create_region('cache_perms')
# make sure we empty the cache now
for key in cache_region.backend.list_keys():
cache_region.delete(key)
assert cache_region.backend.list_keys() == []
for key in added_keys:
assert key not in cache_manger_instance
assert len(cache_manger_instance.namespace.keys()) == 0
@cache_region.cache_on_arguments(expiration_time=5)
def compute(key):
return time.time()
def test_store_and_get_value_from_manager(self):
cache_manger_instance = caches.get_cache_manager(
'repo_cache_long', 'my_cache_store')
result = set()
for x in range(10):
ret = compute('x')
result.add(ret)
_cache_key = caches.compute_key_from_params('foo_bar', 'multicall')
def compute():
return time.time()
result = set()
for x in xrange(10):
ret = cache_manger_instance.get(_cache_key, createfunc=compute)
result.add(ret)
# once computed we have only one value after executing it 10x
assert len(result) == 1
# once computed we have only one value (the same from cache)
# after executing it 10x
assert len(result) == 1

26
rhodecode/tests/rhodecode.ini
View file

@ -292,11 +292,7 @@ cache_dir = %(here)s/data
beaker.cache.data_dir = %(here)s/rc/data/cache/beaker_data
beaker.cache.lock_dir = %(here)s/rc/data/cache/beaker_lock
beaker.cache.regions = short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
beaker.cache.short_term.type = file
beaker.cache.short_term.expire = 0
beaker.cache.short_term.key_length = 256
beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
beaker.cache.long_term.type = memory
beaker.cache.long_term.expire = 36000
@ -306,16 +302,6 @@ beaker.cache.sql_cache_short.type = memory
beaker.cache.sql_cache_short.expire = 1
beaker.cache.sql_cache_short.key_length = 256
## default is memory cache, configure only if required
## using multi-node or multi-worker setup
#beaker.cache.auth_plugins.type = memory
#beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
#beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
#beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
#beaker.cache.auth_plugins.sa.pool_recycle = 3600
#beaker.cache.auth_plugins.sa.pool_size = 10
#beaker.cache.auth_plugins.sa.max_overflow = 0
beaker.cache.repo_cache_long.type = memorylru_base
beaker.cache.repo_cache_long.max_items = 4096
beaker.cache.repo_cache_long.expire = 2592000
@ -327,6 +313,16 @@ beaker.cache.repo_cache_long.expire = 2592000
#beaker.cache.repo_cache_long.expire = 1209600
#beaker.cache.repo_cache_long.key_length = 256
#####################################
### DOGPILE CACHE ####
#####################################
## permission tree cache settings
rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
rc_cache.cache_perms.expiration_time = 0
rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1
####################################
### BEAKER SESSION ####
####################################