engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

vcs: reduce sql queries used during pull/push operations.

Browse source
This commit is contained in:
Marcin Kuzminski 2017-09-28 21:45:19 +02:00
parent 84564f6aba
commit f445addfdd
8 changed files with 76 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

13
rhodecode/authentication/base.py
View file

@ -31,6 +31,7 @@ import warnings
import functools
from pyramid.threadlocal import get_current_registry
from zope.cachedescriptors.property import Lazy as LazyProperty
from rhodecode.authentication.interface import IAuthnPluginRegistry
from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
@ -168,6 +169,11 @@ class RhodeCodeAuthPluginBase(object):
db_type = '{}.encrypted'.format(db_type)
return db_type
@LazyProperty
def plugin_settings(self):
settings = SettingsModel().get_all_settings()
return settings
def is_enabled(self):
"""
Returns true if this plugin is enabled. An enabled plugin can be
@ -206,9 +212,10 @@ class RhodeCodeAuthPluginBase(object):
"""
Returns a plugin setting by name.
"""
full_name = self._get_setting_full_name(name)
db_setting = SettingsModel().get_setting_by_name(full_name)
return db_setting.app_settings_value if db_setting else default
full_name = 'rhodecode_{}'.format(self._get_setting_full_name(name))
plugin_settings = self.plugin_settings
return plugin_settings.get(full_name) or default
def create_or_update_setting(self, name, value):
"""

2
rhodecode/authentication/registry.py
View file

@ -68,7 +68,7 @@ class AuthenticationPluginRegistry(object):
plugins = []
# Add all enabled and active plugins to the list. We iterate over the
# auth_plugins setting from DB beacuse it also represents the ordering.
# auth_plugins setting from DB because it also represents the ordering.
enabled_plugins = SettingsModel().get_auth_plugins()
for plugin_id in enabled_plugins:
plugin = self.get_plugin(plugin_id)

5
rhodecode/lib/base.py
View file

@ -45,7 +45,7 @@ from rhodecode.lib.utils import (
get_repo_slug, set_rhodecode_config, password_changed,
get_enabled_hook_classes)
from rhodecode.lib.utils2 import (
str2bool, safe_unicode, AttributeDict, safe_int, md5, aslist)
str2bool, safe_unicode, AttributeDict, safe_int, md5, aslist, safe_str)
from rhodecode.model import meta
from rhodecode.model.db import Repository, User, ChangesetComment
from rhodecode.model.notification import NotificationModel
@ -252,6 +252,9 @@ class BasicAuth(AuthBasicAuthenticator):
log.exception('Failed to fetch response for code %s' % http_code)
return HTTPForbidden
def get_rc_realm(self):
return safe_str(self.registry.rhodecode_settings.get('rhodecode_realm'))
def build_authentication(self):
head = WWW_AUTHENTICATE.tuples('Basic realm="%s"' % self.realm)
if self._rc_auth_http_code and not self.initial_call:

12
rhodecode/lib/middleware/simplevcs.py
View file

@ -45,7 +45,7 @@ from rhodecode.lib.hooks_daemon import prepare_callback_daemon
from rhodecode.lib.middleware import appenlight
from rhodecode.lib.middleware.utils import scm_app_http
from rhodecode.lib.utils import (
is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path, SLUG_RE)
is_valid_repo, get_rhodecode_base_path, SLUG_RE)
from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
from rhodecode.lib.vcs.conf import settings as vcs_settings
from rhodecode.lib.vcs.backends import base
@ -53,7 +53,7 @@ from rhodecode.model import meta
from rhodecode.model.db import User, Repository, PullRequest
from rhodecode.model.scm import ScmModel
from rhodecode.model.pull_request import PullRequestModel
from rhodecode.model.settings import SettingsModel
log = logging.getLogger(__name__)
@ -107,9 +107,9 @@ class SimpleVCS(object):
self.config = config
# re-populated by specialized middleware
self.repo_vcs_config = base.Config()
# base path of repo locations
self.basepath = get_rhodecode_base_path()
self.rhodecode_settings = SettingsModel().get_all_settings(cache=True)
self.basepath = rhodecode.CONFIG['base_path']
registry.rhodecode_settings = self.rhodecode_settings
# authenticate this VCS request using authfunc
auth_ret_code_detection = \
str2bool(self.config.get('auth_ret_code_detection', False))
@ -381,7 +381,7 @@ class SimpleVCS(object):
# before inject the calling repo_name for special scope checks
self.authenticate.acl_repo_name = self.acl_repo_name
if not username:
self.authenticate.realm = get_rhodecode_realm()
self.authenticate.realm = self.authenticate.get_rc_realm()
try:
result = self.authenticate(environ)

16
rhodecode/tests/lib/middleware/test_simplegit.py
View file

@ -73,10 +73,10 @@ def get_environ(url, request_method):
('/info/lfs/info/lfs/objects/batch', 'pull', 'POST'),
])
def test_get_action(url, expected_action, request_method, pylonsapp):
def test_get_action(url, expected_action, request_method, pylonsapp, request_stub):
app = simplegit.SimpleGit(application=None,
config={'auth_ret_code': '', 'base_path': ''},
registry=None)
registry=request_stub.registry)
assert expected_action == app._get_action(get_environ(url, request_method))
@ -102,19 +102,19 @@ def test_get_action(url, expected_action, request_method, pylonsapp):
('/info/lfs/info/lfs/objects/batch', 'info/lfs', 'POST'),
])
def test_get_repository_name(url, expected_repo_name, request_method, pylonsapp):
def test_get_repository_name(url, expected_repo_name, request_method, pylonsapp, request_stub):
app = simplegit.SimpleGit(application=None,
config={'auth_ret_code': '', 'base_path': ''},
registry=None)
registry=request_stub.registry)
assert expected_repo_name == app._get_repository_name(
get_environ(url, request_method))
def test_get_config(pylonsapp, user_util):
def test_get_config(user_util, pylonsapp, request_stub):
repo = user_util.create_repo(repo_type='git')
app = simplegit.SimpleGit(application=None,
config={'auth_ret_code': '', 'base_path': ''},
registry=None)
registry=request_stub.registry)
extras = {'foo': 'FOO', 'bar': 'BAR'}
# We copy the extras as the method below will change the contents.
@ -130,13 +130,13 @@ def test_get_config(pylonsapp, user_util):
assert git_config == expected_config
def test_create_wsgi_app_uses_scm_app_from_simplevcs(pylonsapp):
def test_create_wsgi_app_uses_scm_app_from_simplevcs(pylonsapp, request_stub):
config = {
'auth_ret_code': '',
'base_path': '',
'vcs.scm_app_implementation':
'rhodecode.tests.lib.middleware.mock_scm_app',
}
app = simplegit.SimpleGit(application=None, config=config, registry=None)
app = simplegit.SimpleGit(application=None, config=config, registry=request_stub.registry)
wsgi_app = app._create_wsgi_app('/tmp/test', 'test_repo', {})
assert wsgi_app is mock_scm_app.mock_git_wsgi

17
rhodecode/tests/lib/middleware/test_simplehg.py
View file

@ -53,10 +53,10 @@ def get_environ(url):
# Edge case: not cmd argument
('/foo/bar?key=tip', 'pull'),
])
def test_get_action(url, expected_action):
def test_get_action(url, expected_action, request_stub):
app = simplehg.SimpleHg(application=None,
config={'auth_ret_code': '', 'base_path': ''},
registry=None)
registry=request_stub.registry)
assert expected_action == app._get_action(get_environ(url))
@ -71,18 +71,18 @@ def test_get_action(url, expected_action):
('/foo/bar/?cmd=pushkey&key=tip', 'foo/bar'),
('/foo/bar/baz/?cmd=listkeys&key=tip', 'foo/bar/baz'),
])
def test_get_repository_name(url, expected_repo_name):
def test_get_repository_name(url, expected_repo_name, request_stub):
app = simplehg.SimpleHg(application=None,
config={'auth_ret_code': '', 'base_path': ''},
registry=None)
registry=request_stub.registry)
assert expected_repo_name == app._get_repository_name(get_environ(url))
def test_get_config(pylonsapp, user_util):
def test_get_config(user_util, pylonsapp, request_stub):
repo = user_util.create_repo(repo_type='git')
app = simplehg.SimpleHg(application=None,
config={'auth_ret_code': '', 'base_path': ''},
registry=None)
registry=request_stub.registry)
extras = [('foo', 'FOO', 'bar', 'BAR')]
hg_config = app._create_config(extras, repo_name=repo.repo_name)
@ -116,13 +116,14 @@ def test_get_config(pylonsapp, user_util):
assert entry in hg_config
def test_create_wsgi_app_uses_scm_app_from_simplevcs():
def test_create_wsgi_app_uses_scm_app_from_simplevcs(request_stub):
config = {
'auth_ret_code': '',
'base_path': '',
'vcs.scm_app_implementation':
'rhodecode.tests.lib.middleware.mock_scm_app',
}
app = simplehg.SimpleHg(application=None, config=config, registry=None)
app = simplehg.SimpleHg(
application=None, config=config, registry=request_stub.registry)
wsgi_app = app._create_wsgi_app('/tmp/test', 'test_repo', {})
assert wsgi_app is mock_scm_app.mock_hg_wsgi

4
rhodecode/tests/lib/middleware/test_simplesvn.py
View file

@ -29,12 +29,12 @@ from rhodecode.lib.middleware.simplesvn import SimpleSvn, SimpleSvnApp
class TestSimpleSvn(object):
@pytest.fixture(autouse=True)
def simple_svn(self, pylonsapp):
def simple_svn(self, pylonsapp, request_stub):
self.app = SimpleSvn(
application='None',
config={'auth_ret_code': '',
'base_path': rhodecode.CONFIG['base_path']},
registry=None)
registry=request_stub.registry)
def test_get_config(self):
extras = {'foo': 'FOO', 'bar': 'BAR'}

59
rhodecode/tests/lib/middleware/test_simplevcs.py
View file

@ -23,6 +23,7 @@ import base64
import mock
import pytest
from rhodecode.lib.utils2 import AttributeDict
from rhodecode.tests.utils import CustomTestApp
from rhodecode.lib.caching_query import FromCache
@ -73,12 +74,12 @@ class StubVCSController(simplevcs.SimpleVCS):
@pytest.fixture
def vcscontroller(pylonsapp, config_stub):
def vcscontroller(pylonsapp, config_stub, request_stub):
config_stub.testing_securitypolicy()
config_stub.include('rhodecode.authentication')
#set_anonymous_access(True)
controller = StubVCSController(pylonsapp, pylonsapp.config, None)
controller = StubVCSController(
pylonsapp, pylonsapp.config, request_stub.registry)
app = HttpsFixup(controller, pylonsapp.config)
app = CustomTestApp(app)
@ -134,7 +135,8 @@ class StubFailVCSController(simplevcs.SimpleVCS):
@pytest.fixture(scope='module')
def fail_controller(pylonsapp):
controller = StubFailVCSController(pylonsapp, pylonsapp.config, None)
controller = StubFailVCSController(
pylonsapp, pylonsapp.config, pylonsapp.config)
controller = HttpsFixup(controller, pylonsapp.config)
controller = CustomTestApp(controller)
return controller
@ -150,16 +152,18 @@ def test_provides_traceback_for_appenlight(fail_controller):
assert 'appenlight.__traceback' in response.request.environ
def test_provides_utils_scm_app_as_scm_app_by_default(pylonsapp):
controller = StubVCSController(pylonsapp, pylonsapp.config, None)
def test_provides_utils_scm_app_as_scm_app_by_default(pylonsapp, request_stub):
controller = StubVCSController(
pylonsapp, pylonsapp.config, request_stub.registry)
assert controller.scm_app is scm_app_http
def test_allows_to_override_scm_app_via_config(pylonsapp):
def test_allows_to_override_scm_app_via_config(pylonsapp, request_stub):
config = pylonsapp.config.copy()
config['vcs.scm_app_implementation'] = (
'rhodecode.tests.lib.middleware.mock_scm_app')
controller = StubVCSController(pylonsapp, config, None)
controller = StubVCSController(
pylonsapp, config, request_stub.registry)
assert controller.scm_app is mock_scm_app
@ -214,12 +218,14 @@ class TestShadowRepoRegularExpression(object):
@pytest.mark.backends('git', 'hg')
class TestShadowRepoExposure(object):
def test_pull_on_shadow_repo_propagates_to_wsgi_app(self, pylonsapp):
def test_pull_on_shadow_repo_propagates_to_wsgi_app(
self, pylonsapp, request_stub):
"""
Check that a pull action to a shadow repo is propagated to the
underlying wsgi app.
"""
controller = StubVCSController(pylonsapp, pylonsapp.config, None)
controller = StubVCSController(
pylonsapp, pylonsapp.config, request_stub.registry)
controller._check_ssl = mock.Mock()
controller.is_shadow_repo = True
controller._action = 'pull'
@ -238,12 +244,13 @@ class TestShadowRepoExposure(object):
# Assert that we got the response from the wsgi app.
assert response_body == controller.stub_response_body
def test_pull_on_shadow_repo_that_is_missing(self, pylonsapp):
def test_pull_on_shadow_repo_that_is_missing(self, pylonsapp, request_stub):
"""
Check that a pull action to a shadow repo is propagated to the
underlying wsgi app.
"""
controller = StubVCSController(pylonsapp, pylonsapp.config, None)
controller = StubVCSController(
pylonsapp, pylonsapp.config, request_stub.registry)
controller._check_ssl = mock.Mock()
controller.is_shadow_repo = True
controller._action = 'pull'
@ -262,11 +269,12 @@ class TestShadowRepoExposure(object):
# Assert that we got the response from the wsgi app.
assert '404 Not Found' in response_body
def test_push_on_shadow_repo_raises(self, pylonsapp):
def test_push_on_shadow_repo_raises(self, pylonsapp, request_stub):
"""
Check that a push action to a shadow repo is aborted.
"""
controller = StubVCSController(pylonsapp, pylonsapp.config, None)
controller = StubVCSController(
pylonsapp, pylonsapp.config, request_stub.registry)
controller._check_ssl = mock.Mock()
controller.is_shadow_repo = True
controller._action = 'push'
@ -285,13 +293,14 @@ class TestShadowRepoExposure(object):
# Assert that a 406 error is returned.
assert '406 Not Acceptable' in response_body
def test_set_repo_names_no_shadow(self, pylonsapp):
def test_set_repo_names_no_shadow(self, pylonsapp, request_stub):
"""
Check that the set_repo_names method sets all names to the one returned
by the _get_repository_name method on a request to a non shadow repo.
"""
environ_stub = {}
controller = StubVCSController(pylonsapp, pylonsapp.config, None)
controller = StubVCSController(
pylonsapp, pylonsapp.config, request_stub.registry)
controller._name = 'RepoGroup/MyRepo'
controller.set_repo_names(environ_stub)
assert not controller.is_shadow_repo
@ -300,7 +309,8 @@ class TestShadowRepoExposure(object):
controller.vcs_repo_name ==
controller._get_repository_name(environ_stub))
def test_set_repo_names_with_shadow(self, pylonsapp, pr_util, config_stub):
def test_set_repo_names_with_shadow(
self, pylonsapp, pr_util, config_stub, request_stub):
"""
Check that the set_repo_names method sets correct names on a request
to a shadow repo.
@ -313,7 +323,8 @@ class TestShadowRepoExposure(object):
pr_id=pull_request.pull_request_id,
pr_segment=TestShadowRepoRegularExpression.pr_segment,
shadow_segment=TestShadowRepoRegularExpression.shadow_segment)
controller = StubVCSController(pylonsapp, pylonsapp.config, None)
controller = StubVCSController(
pylonsapp, pylonsapp.config, request_stub.registry)
controller._name = shadow_url
controller.set_repo_names({})
@ -329,7 +340,7 @@ class TestShadowRepoExposure(object):
assert controller.is_shadow_repo
def test_set_repo_names_with_shadow_but_missing_pr(
self, pylonsapp, pr_util, config_stub):
self, pylonsapp, pr_util, config_stub, request_stub):
"""
Checks that the set_repo_names method enforces matching target repos
and pull request IDs.
@ -340,7 +351,8 @@ class TestShadowRepoExposure(object):
pr_id=999999999,
pr_segment=TestShadowRepoRegularExpression.pr_segment,
shadow_segment=TestShadowRepoRegularExpression.shadow_segment)
controller = StubVCSController(pylonsapp, pylonsapp.config, None)
controller = StubVCSController(
pylonsapp, pylonsapp.config, request_stub.registry)
controller._name = shadow_url
controller.set_repo_names({})
@ -416,7 +428,8 @@ class TestGenerateVcsResponse(object):
'vcs.hooks.protocol': 'http',
'vcs.hooks.direct_calls': False,
}
controller = StubVCSController(None, settings, None)
registry = AttributeDict()
controller = StubVCSController(None, settings, registry)
controller._invalidate_cache = mock.Mock()
controller.stub_response_body = response_body
self.start_response = mock.Mock()
@ -465,11 +478,11 @@ class TestInitializeGenerator(object):
class TestPrepareHooksDaemon(object):
def test_calls_imported_prepare_callback_daemon(self, app_settings):
def test_calls_imported_prepare_callback_daemon(self, app_settings, request_stub):
expected_extras = {'extra1': 'value1'}
daemon = DummyHooksCallbackDaemon()
controller = StubVCSController(None, app_settings, None)
controller = StubVCSController(None, app_settings, request_stub.registry)
prepare_patcher = mock.patch.object(
simplevcs, 'prepare_callback_daemon',
return_value=(daemon, expected_extras))