engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

auth: use cache_ttl from a plugin to also cache permissions.

Browse source 
- this gives a 30% speed increase in operations like svn commit
- generally uses the same mechanismy like auth cache to cache permissions for faster
access to vcs commands
This commit is contained in:
Marcin Kuzminski 2017-10-07 10:02:39 +02:00
parent 34175fd787
commit 7781eabe94
6 changed files with 143 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

47
rhodecode/authentication/base.py
View file

@ -37,7 +37,7 @@ from rhodecode.authentication.interface import IAuthnPluginRegistry
from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
from rhodecode.lib import caches
from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
from rhodecode.lib.utils2 import md5_safe, safe_int
from rhodecode.lib.utils2 import safe_int
from rhodecode.lib.utils2 import safe_str
from rhodecode.model.db import User
from rhodecode.model.meta import Session
@ -423,11 +423,14 @@ class RhodeCodeAuthPluginBase(object):
"""
auth = self.auth(userobj, username, passwd, settings, **kwargs)
if auth:
auth['_plugin'] = self.name
auth['_ttl_cache'] = self.get_ttl_cache(settings)
# check if hash should be migrated ?
new_hash = auth.get('_hash_migrate')
if new_hash:
self._migrate_hash_to_bcrypt(username, passwd, new_hash)
return self._validate_auth_return(auth)
return auth
def _migrate_hash_to_bcrypt(self, username, password, new_hash):
@ -450,6 +453,19 @@ class RhodeCodeAuthPluginBase(object):
raise Exception('Missing %s attribute from returned data' % k)
return ret
def get_ttl_cache(self, settings=None):
plugin_settings = settings or self.get_settings()
cache_ttl = 0
if isinstance(self.AUTH_CACHE_TTL, (int, long)):
# plugin cache set inside is more important than the settings value
cache_ttl = self.AUTH_CACHE_TTL
elif plugin_settings.get('cache_ttl'):
cache_ttl = safe_int(plugin_settings.get('cache_ttl'), 0)
plugin_cache_active = bool(cache_ttl and cache_ttl > 0)
return plugin_cache_active, cache_ttl
class RhodeCodeExternalAuthPlugin(RhodeCodeAuthPluginBase):
@ -578,6 +594,11 @@ def get_auth_cache_manager(custom_ttl=None):
'auth_plugins', 'rhodecode.authentication', custom_ttl)
def get_perms_cache_manager(custom_ttl=None):
return caches.get_cache_manager(
'auth_plugins', 'rhodecode.permissions', custom_ttl)
def authenticate(username, password, environ=None, auth_type=None,
skip_missing=False, registry=None, acl_repo_name=None):
"""
@ -633,25 +654,19 @@ def authenticate(username, password, environ=None, auth_type=None,
log.info('Authenticating user `%s` using %s plugin',
display_user, plugin.get_id())
_cache_ttl = 0
if isinstance(plugin.AUTH_CACHE_TTL, (int, long)):
# plugin cache set inside is more important than the settings value
_cache_ttl = plugin.AUTH_CACHE_TTL
elif plugin_settings.get('cache_ttl'):
_cache_ttl = safe_int(plugin_settings.get('cache_ttl'), 0)
plugin_cache_active = bool(_cache_ttl and _cache_ttl > 0)
plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings)
# get instance of cache manager configured for a namespace
cache_manager = get_auth_cache_manager(custom_ttl=_cache_ttl)
cache_manager = get_auth_cache_manager(custom_ttl=cache_ttl)
log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)',
plugin.get_id(), plugin_cache_active, _cache_ttl)
plugin.get_id(), plugin_cache_active, cache_ttl)
# for environ based password can be empty, but then the validation is
# on the server that fills in the env data needed for authentication
_password_hash = md5_safe(plugin.name + username + (password or ''))
_password_hash = caches.compute_key_from_params(
plugin.name, username, (password or ''))
# _authenticate is a wrapper for .auth() method of plugin.
# it checks if .auth() sends proper data.
@ -667,11 +682,13 @@ def authenticate(username, password, environ=None, auth_type=None,
This function is used internally in Cache of Beaker to calculate
Results
"""
log.debug('auth: calculating password access now...')
return plugin._authenticate(
user, username, password, plugin_settings,
environ=environ or {})
if plugin_cache_active:
log.debug('Trying to fetch cached auth by %s', _password_hash[:6])
plugin_user = cache_manager.get(
_password_hash, createfunc=auth_func)
else:
@ -680,7 +697,7 @@ def authenticate(username, password, environ=None, auth_type=None,
auth_time = time.time() - start
log.debug('Authentication for plugin `%s` completed in %.3fs, '
'expiration time of fetched cache %.1fs.',
plugin.get_id(), auth_time, _cache_ttl)
plugin.get_id(), auth_time, cache_ttl)
log.debug('PLUGIN USER DATA: %s', plugin_user)
@ -690,6 +707,8 @@ def authenticate(username, password, environ=None, auth_type=None,
# we failed to Auth because .auth() method didn't return proper user
log.debug("User `%s` failed to authenticate against %s",
display_user, plugin.get_id())
# case when we failed to authenticate against all defined plugins
return None

9
rhodecode/authentication/schema.py
View file

@ -40,11 +40,10 @@ class AuthnPluginSettingsSchemaBase(colander.MappingSchema):
cache_ttl = colander.SchemaNode(
colander.Int(),
default=0,
description=_('Amount of seconds to cache the authentication response'
'call for this plugin. \n'
'Useful for long calls like LDAP to improve the '
'performance of the authentication system '
'(0 means disabled).'),
description=_('Amount of seconds to cache the authentication and '
'permissions check response call for this plugin. \n'
'Useful for expensive calls like LDAP to improve the '
'performance of the system (0 means disabled).'),
missing=0,
title=_('Auth Cache TTL'),
validator=colander.Range(min=0, max=None),

7
rhodecode/lib/base.py
View file

@ -277,10 +277,11 @@ class BasicAuth(AuthBasicAuthenticator):
_parts = auth.split(':', 1)
if len(_parts) == 2:
username, password = _parts
if self.authfunc(
auth_data = self.authfunc(
username, password, environ, VCS_TYPE,
registry=self.registry, acl_repo_name=self.acl_repo_name):
return username
registry=self.registry, acl_repo_name=self.acl_repo_name)
if auth_data:
return {'username': username, 'auth_data': auth_data}
if username and password:
# we mark that we actually executed authentication once, at
# that point we can use the alternative auth code

125
rhodecode/lib/middleware/simplevcs.py
View file

@ -24,17 +24,20 @@ It's implemented with basic auth function
"""
import os
import re
import logging
import importlib
import re
from functools import wraps
import time
from paste.httpheaders import REMOTE_USER, AUTH_TYPE
from webob.exc import (
HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
import rhodecode
from rhodecode.authentication.base import authenticate, VCS_TYPE
from rhodecode.authentication.base import (
authenticate, get_perms_cache_manager, VCS_TYPE)
from rhodecode.lib import caches
from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
from rhodecode.lib.base import (
BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
@ -44,8 +47,7 @@ from rhodecode.lib.exceptions import (
from rhodecode.lib.hooks_daemon import prepare_callback_daemon
from rhodecode.lib.middleware import appenlight
from rhodecode.lib.middleware.utils import scm_app_http
from rhodecode.lib.utils import (
is_valid_repo, get_rhodecode_base_path, SLUG_RE)
from rhodecode.lib.utils import is_valid_repo, SLUG_RE
from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
from rhodecode.lib.vcs.conf import settings as vcs_settings
from rhodecode.lib.vcs.backends import base
@ -242,39 +244,80 @@ class SimpleVCS(object):
def is_shadow_repo_dir(self):
return os.path.isdir(self.vcs_repo_name)
def _check_permission(self, action, user, repo_name, ip_addr=None):
def _check_permission(self, action, user, repo_name, ip_addr=None,
plugin_id='', plugin_cache_active=False, cache_ttl=0):
"""
Checks permissions using action (push/pull) user and repository
name
name. If plugin_cache and ttl is set it will use the plugin which
authenticated the user to store the cached permissions result for N
amount of seconds as in cache_ttl
:param action: push or pull action
:param user: user instance
:param repo_name: repository name
"""
# check IP
inherit = user.inherit_default_permissions
ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr,
inherit_from_default=inherit)
if ip_allowed:
log.info('Access for IP:%s allowed', ip_addr)
else:
return False
if action == 'push':
if not HasPermissionAnyMiddleware('repository.write',
'repository.admin')(user,
repo_name):
# get instance of cache manager configured for a namespace
cache_manager = get_perms_cache_manager(custom_ttl=cache_ttl)
log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
plugin_id, plugin_cache_active, cache_ttl)
# for environ based password can be empty, but then the validation is
# on the server that fills in the env data needed for authentication
_perm_calc_hash = caches.compute_key_from_params(
plugin_id, action, user.user_id, repo_name, ip_addr)
# _authenticate is a wrapper for .auth() method of plugin.
# it checks if .auth() sends proper data.
# For RhodeCodeExternalAuthPlugin it also maps users to
# Database and maps the attributes returned from .auth()
# to RhodeCode database. If this function returns data
# then auth is correct.
start = time.time()
log.debug('Running plugin `%s` permissions check', plugin_id)
def perm_func():
"""
This function is used internally in Cache of Beaker to calculate
Results
"""
log.debug('auth: calculating permission access now...')
# check IP
inherit = user.inherit_default_permissions
ip_allowed = AuthUser.check_ip_allowed(
user.user_id, ip_addr, inherit_from_default=inherit)
if ip_allowed:
log.info('Access for IP:%s allowed', ip_addr)
else:
return False
else:
# any other action need at least read permission
if not HasPermissionAnyMiddleware('repository.read',
'repository.write',
'repository.admin')(user,
repo_name):
return False
if action == 'push':
perms = ('repository.write', 'repository.admin')
if not HasPermissionAnyMiddleware(*perms)(user, repo_name):
return False
return True
else:
# any other action need at least read permission
perms = (
'repository.read', 'repository.write', 'repository.admin')
if not HasPermissionAnyMiddleware(*perms)(user, repo_name):
return False
return True
if plugin_cache_active:
log.debug('Trying to fetch cached perms by %s', _perm_calc_hash[:6])
perm_result = cache_manager.get(
_perm_calc_hash, createfunc=perm_func)
else:
perm_result = perm_func()
auth_time = time.time() - start
log.debug('Permissions for plugin `%s` completed in %.3fs, '
'expiration time of fetched cache %.1fs.',
plugin_id, auth_time, cache_ttl)
return perm_result
def _check_ssl(self, environ, start_response):
"""
@ -376,26 +419,41 @@ class SimpleVCS(object):
if pre_auth and pre_auth.get('username'):
username = pre_auth['username']
log.debug('PRE-AUTH got %s as username', username)
if pre_auth:
log.debug('PRE-AUTH successful from %s',
pre_auth.get('auth_data', {}).get('_plugin'))
# If not authenticated by the container, running basic auth
# before inject the calling repo_name for special scope checks
self.authenticate.acl_repo_name = self.acl_repo_name
plugin_cache_active, cache_ttl = False, 0
plugin = None
if not username:
self.authenticate.realm = self.authenticate.get_rc_realm()
try:
result = self.authenticate(environ)
auth_result = self.authenticate(environ)
except (UserCreationError, NotAllowedToCreateUserError) as e:
log.error(e)
reason = safe_str(e)
return HTTPNotAcceptable(reason)(environ, start_response)
if isinstance(result, str):
if isinstance(auth_result, dict):
AUTH_TYPE.update(environ, 'basic')
REMOTE_USER.update(environ, result)
username = result
REMOTE_USER.update(environ, auth_result['username'])
username = auth_result['username']
plugin = auth_result.get('auth_data', {}).get('_plugin')
log.info(
'MAIN-AUTH successful for user `%s` from %s plugin',
username, plugin)
plugin_cache_active, cache_ttl = auth_result.get(
'auth_data', {}).get('_ttl_cache') or (False, 0)
else:
return result.wsgi_application(environ, start_response)
return auth_result.wsgi_application(
environ, start_response)
# ==============================================================
# CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
@ -417,12 +475,13 @@ class SimpleVCS(object):
# check permissions for this repository
perm = self._check_permission(
action, user, self.acl_repo_name, ip_addr)
action, user, self.acl_repo_name, ip_addr,
plugin, plugin_cache_active, cache_ttl)
if not perm:
return HTTPForbidden()(environ, start_response)
# extras are injected into UI object and later available
# in hooks executed by rhodecode
# in hooks executed by RhodeCode
check_locking = _should_check_locking(environ.get('QUERY_STRING'))
extras = vcs_operation_context(
environ, repo_name=self.acl_repo_name, username=username,

2
rhodecode/model/db.py
View file

@ -918,7 +918,7 @@ class User(Base, BaseModel):
"""Update user lastactivity"""
self.last_activity = datetime.datetime.now()
Session().add(self)
log.debug('updated user %s lastactivity', self.username)
log.debug('updated user `%s` last activity', self.username)
def update_password(self, new_password):
from rhodecode.lib.auth import get_crypt_password

13
rhodecode/tests/lib/auth_modules/test_auth_modules.py
View file

@ -27,8 +27,13 @@ from rhodecode.authentication.plugins.auth_ldap import RhodeCodeAuthPlugin
from rhodecode.model import db
class TestAuthPlugin(RhodeCodeAuthPluginBase):
def name(self):
return 'stub_auth'
def test_authenticate_returns_from_auth(stub_auth_data):
plugin = RhodeCodeAuthPluginBase('stub_id')
plugin = TestAuthPlugin('stub_id')
with mock.patch.object(plugin, 'auth') as auth_mock:
auth_mock.return_value = stub_auth_data
result = plugin._authenticate(mock.Mock(), 'test', 'password', {})
@ -37,7 +42,7 @@ def test_authenticate_returns_from_auth(stub_auth_data):
def test_authenticate_returns_empty_auth_data():
auth_data = {}
plugin = RhodeCodeAuthPluginBase('stub_id')
plugin = TestAuthPlugin('stub_id')
with mock.patch.object(plugin, 'auth') as auth_mock:
auth_mock.return_value = auth_data
result = plugin._authenticate(mock.Mock(), 'test', 'password', {})
@ -46,7 +51,7 @@ def test_authenticate_returns_empty_auth_data():
def test_authenticate_skips_hash_migration_if_mismatch(stub_auth_data):
stub_auth_data['_hash_migrate'] = 'new-hash'
plugin = RhodeCodeAuthPluginBase('stub_id')
plugin = TestAuthPlugin('stub_id')
with mock.patch.object(plugin, 'auth') as auth_mock:
auth_mock.return_value = stub_auth_data
result = plugin._authenticate(mock.Mock(), 'test', 'password', {})
@ -60,7 +65,7 @@ def test_authenticate_migrates_to_new_hash(stub_auth_data):
new_password = b'new-password'
new_hash = _RhodeCodeCryptoBCrypt().hash_create(new_password)
stub_auth_data['_hash_migrate'] = new_hash
plugin = RhodeCodeAuthPluginBase('stub_id')
plugin = TestAuthPlugin('stub_id')
with mock.patch.object(plugin, 'auth') as auth_mock:
auth_mock.return_value = stub_auth_data
result = plugin._authenticate(