engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

core: removed pyro4 from Enterprise code. Fixes #5198

Browse source
This commit is contained in:
Marcin Kuzminski 2017-02-07 15:58:49 +01:00
parent 18f667c234
commit 3d3f2db34a
38 changed files with 47 additions and 854 deletions
Download patch file Download diff file Expand all files Collapse all files

5
configs/development.ini
View file

@ -539,18 +539,15 @@ vcs.server = localhost:9900
## Web server connectivity protocol, responsible for web based VCS operatations
## Available protocols are:
## `pyro4` - use pyro4 server
## `http` - use http-rpc backend (default)
vcs.server.protocol = http
## Push/Pull operations protocol, available options are:
## `pyro4` - use pyro4 server
## `http` - use http-rpc backend (default)
##
vcs.scm_app_implementation = http
## Push/Pull operations hooks protocol, available options are:
## `pyro4` - use pyro4 server
## `http` - use http-rpc backend (default)
vcs.hooks.protocol = http
@ -666,7 +663,7 @@ formatter = color_formatter_sql
################
[formatter_generic]
class = rhodecode.lib.logging_formatter.Pyro4AwareFormatter
class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %Y-%m-%d %H:%M:%S

5
configs/production.ini
View file

@ -508,18 +508,15 @@ vcs.server = localhost:9900
## Web server connectivity protocol, responsible for web based VCS operatations
## Available protocols are:
## `pyro4` - use pyro4 server
## `http` - use http-rpc backend (default)
vcs.server.protocol = http
## Push/Pull operations protocol, available options are:
## `pyro4` - use pyro4 server
## `http` - use http-rpc backend (default)
##
vcs.scm_app_implementation = http
## Push/Pull operations hooks protocol, available options are:
## `pyro4` - use pyro4 server
## `http` - use http-rpc backend (default)
vcs.hooks.protocol = http
@ -635,7 +632,7 @@ formatter = generic
################
[formatter_generic]
class = rhodecode.lib.logging_formatter.Pyro4AwareFormatter
class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %Y-%m-%d %H:%M:%S

15
docs-internal/vcs/http-transition.rst
View file

@ -1,15 +0,0 @@
.. _vcs-http:
========================================
Transition to HTTP based communication
========================================
We are in the process of replacing the Pyro4 based communication with an HTTP
based implementation. Currently both backends are supported and can be
activated via various settings in the configuration.
To run the system in full HTTP based mode, use the following settings::
vcs.hooks.protocol = http
vcs.scm_app_implementation = rhodecode.lib.middleware.utils.scm_app_http
vcs.server.protocol = http

1
docs-internal/vcs/index.rst
View file

@ -17,7 +17,6 @@ implemented in :mod:`rhodecode.lib.middleware`.
.. toctree::
:maxdepth: 2
http-transition
middleware
vcsserver
subversion

8
docs/admin/enable-debug.rst
View file

@ -47,7 +47,7 @@ the ``debug`` level.
### LOGGING CONFIGURATION ####
################################
[loggers]
keys = root, routes, rhodecode, sqlalchemy, beaker, pyro4, templates
keys = root, routes, rhodecode, sqlalchemy, beaker, templates
[handlers]
keys = console, console_sql, file, file_rotating
@ -75,12 +75,6 @@ the ``debug`` level.
qualname = beaker.container
propagate = 1
[logger_pyro4]
level = DEBUG
handlers =
qualname = Pyro4
propagate = 1
[logger_templates]
level = INFO
handlers =

8
docs/admin/vcs-server.rst
View file

@ -232,7 +232,7 @@ For a more detailed explanation of the logger levers, see :ref:`debug-mode`.
### LOGGING CONFIGURATION ####
################################
[loggers]
keys = root, vcsserver, pyro4, beaker
keys = root, vcsserver, beaker
[handlers]
keys = console
@ -259,12 +259,6 @@ For a more detailed explanation of the logger levers, see :ref:`debug-mode`.
qualname = beaker
propagate = 1
[logger_pyro4]
level = DEBUG
handlers =
qualname = Pyro4
propagate = 1
##############
## HANDLERS ##

28
pkgs/python-packages.nix
View file

@ -197,19 +197,6 @@
license = [ pkgs.lib.licenses.bsdOriginal ];
};
};
Pyro4 = super.buildPythonPackage {
name = "Pyro4-4.41";
buildInputs = with self; [];
doCheck = false;
propagatedBuildInputs = with self; [serpent];
src = fetchurl {
url = "https://pypi.python.org/packages/56/2b/89b566b4bf3e7f8ba790db2d1223852f8cb454c52cab7693dd41f608ca2a/Pyro4-4.41.tar.gz";
md5 = "ed69e9bfafa9c06c049a87cb0c4c2b6c";
};
meta = {
license = [ pkgs.lib.licenses.mit ];
};
};
Routes = super.buildPythonPackage {
name = "Routes-1.13";
buildInputs = with self; [];
@ -1501,7 +1488,7 @@
name = "rhodecode-enterprise-ce-4.7.0";
buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage cssselect lxml configobj];
doCheck = true;
propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments pygments-markdown-lexer Pylons Pyro4 Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery channelstream colander decorator deform docutils gevent gunicorn infrae.cache ipython iso8601 kombu msgpack-python packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson subprocess32 waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt];
propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments pygments-markdown-lexer Pylons Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery channelstream colander decorator deform docutils gevent gunicorn infrae.cache ipython iso8601 kombu msgpack-python packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson subprocess32 waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt];
src = ./.;
meta = {
license = [ { fullName = "Affero GNU General Public License v3 or later (AGPLv3+)"; } { fullName = "AGPLv3, and Commercial License"; } ];
@ -1520,19 +1507,6 @@
license = [ { fullName = "AGPLv3 and Proprietary"; } ];
};
};
serpent = super.buildPythonPackage {
name = "serpent-1.15";
buildInputs = with self; [];
doCheck = false;
propagatedBuildInputs = with self; [];
src = fetchurl {
url = "https://pypi.python.org/packages/7b/38/b2b27673a882ff2ea5871bb3e3e6b496ebbaafd1612e51990ffb158b9254/serpent-1.15.tar.gz";
md5 = "e27b1aad5c218e16442f52abb7c7053a";
};
meta = {
license = [ pkgs.lib.licenses.mit ];
};
};
setproctitle = super.buildPythonPackage {
name = "setproctitle-1.1.8";
buildInputs = with self; [];

1
pytest.ini
View file

@ -2,7 +2,6 @@
testpaths = ./rhodecode
pylons_config = rhodecode/tests/rhodecode.ini
vcsserver_protocol = http
vcsserver_config_pyro4 = rhodecode/tests/vcsserver_pyro4.ini
vcsserver_config_http = rhodecode/tests/vcsserver_http.ini
norecursedirs = tests/scripts
addopts = -k "not _BaseTest"

4
requirements.txt
View file

@ -123,9 +123,5 @@ https://code.rhodecode.com/rhodecode-tools-ce/archive/v0.11.0.tar.gz?md5=e5fd0a8
## appenlight
appenlight-client==0.6.14
# Pyro/Deprecated TODO(Marcink): remove in 4.7 release.
Pyro4==4.41
serpent==1.15
## test related requirements
-r requirements_test.txt

1
rhodecode/config/environment.py
View file

@ -184,7 +184,6 @@ def load_pyramid_environment(global_config, settings):
protocol=utils.get_vcs_server_protocol(settings),
log_level=settings['vcs.server.log_level'])
utils.configure_pyro4(settings)
utils.configure_vcs(settings)
if vcs_server_enabled:
connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))

10
rhodecode/config/licenses.json
View file

@ -40,10 +40,7 @@
},
"python2.7-Pylons-1.0.1-patch1": {
"BSD 4-clause \"Original\" or \"Old\" License": "http://spdx.org/licenses/BSD-4-Clause"
},
"python2.7-Pyro4-4.35": {
"MIT License": "http://spdx.org/licenses/MIT"
},
},
"python2.7-Routes-1.13": {
"BSD 4-clause \"Original\" or \"Old\" License": "http://spdx.org/licenses/BSD-4-Clause"
},
@ -214,10 +211,7 @@
},
"python2.7-requests-2.9.1": {
"Apache License 2.0": "http://spdx.org/licenses/Apache-2.0"
},
"python2.7-serpent-1.12": {
"MIT License": "http://spdx.org/licenses/MIT"
},
},
"python2.7-setuptools-19.4": {
"Python Software Foundation License version 2": "http://spdx.org/licenses/Python-2.0",
"Zope Public License 2.0": "http://spdx.org/licenses/ZPL-2.0"

18
rhodecode/config/utils.py
View file

@ -20,29 +20,11 @@
import os
import shlex
import Pyro4
import platform
from rhodecode.model import init_model
def configure_pyro4(config):
"""
Configure Pyro4 based on `config`.
This will mainly set the different configuration parameters of the Pyro4
library based on the settings in our INI files. The Pyro4 documentation
lists more details about the specific settings and their meaning.
"""
Pyro4.config.COMMTIMEOUT = float(config['vcs.connection_timeout'])
Pyro4.config.SERIALIZER = 'pickle'
Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')
# Note: We need server configuration in the WSGI processes
# because we provide a callback server in certain vcs operations.
Pyro4.config.SERVERTYPE = "multiplex"
Pyro4.config.POLLTIMEOUT = 0.01
def configure_vcs(config):
"""

1
rhodecode/lib/celerylib/__init__.py
View file

@ -226,7 +226,6 @@ def vcsconnection(func):
for alias in rhodecode.BACKENDS.keys():
if alias not in backends:
del rhodecode.BACKENDS[alias]
utils.configure_pyro4(settings)
utils.configure_vcs(settings)
connect_vcs(
settings['vcs.server'],

44
rhodecode/lib/hooks_daemon.py
View file

@ -26,7 +26,6 @@ from BaseHTTPServer import BaseHTTPRequestHandler
from SocketServer import TCPServer
from routes.util import URLGenerator
import Pyro4
import pylons
import rhodecode
@ -73,7 +72,7 @@ class HooksHttpHandler(BaseHTTPRequestHandler):
def log_message(self, format, *args):
"""
This is an overriden method of BaseHTTPRequestHandler which logs using
This is an overridden method of BaseHTTPRequestHandler which logs using
logging library instead of writing directly to stderr.
"""
@ -123,38 +122,6 @@ class ThreadedHookCallbackDaemon(object):
raise NotImplementedError()
class Pyro4HooksCallbackDaemon(ThreadedHookCallbackDaemon):
"""
Context manager which will run a callback daemon in a background thread.
"""
hooks_uri = None
def _prepare(self):
log.debug("Preparing callback daemon and registering hook object")
self._daemon = Pyro4.Daemon()
hooks_interface = Hooks()
self.hooks_uri = str(self._daemon.register(hooks_interface))
log.debug("Hooks uri is: %s", self.hooks_uri)
def _run(self):
log.debug("Running event loop of callback daemon in background thread")
callback_thread = threading.Thread(
target=self._daemon.requestLoop,
kwargs={'loopCondition': lambda: not self._done})
callback_thread.daemon = True
callback_thread.start()
self._callback_thread = callback_thread
def _stop(self):
log.debug("Waiting for background thread to finish.")
self._done = True
self._callback_thread.join()
self._daemon.close()
self._daemon = None
self._callback_thread = None
class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
"""
Context manager which will run a callback daemon in a background thread.
@ -202,9 +169,7 @@ def prepare_callback_daemon(extras, protocol, use_direct_calls):
callback_daemon = DummyHooksCallbackDaemon()
extras['hooks_module'] = callback_daemon.hooks_module
else:
if protocol == 'pyro4':
callback_daemon = Pyro4HooksCallbackDaemon()
elif protocol == 'http':
if protocol == 'http':
callback_daemon = HttpHooksCallbackDaemon()
else:
log.error('Unsupported callback daemon protocol "%s"', protocol)
@ -221,27 +186,22 @@ class Hooks(object):
Exposes the hooks for remote call backs
"""
@Pyro4.callback
def repo_size(self, extras):
log.debug("Called repo_size of Hooks object")
return self._call_hook(hooks_base.repo_size, extras)
@Pyro4.callback
def pre_pull(self, extras):
log.debug("Called pre_pull of Hooks object")
return self._call_hook(hooks_base.pre_pull, extras)
@Pyro4.callback
def post_pull(self, extras):
log.debug("Called post_pull of Hooks object")
return self._call_hook(hooks_base.post_pull, extras)
@Pyro4.callback
def pre_push(self, extras):
log.debug("Called pre_push of Hooks object")
return self._call_hook(hooks_base.pre_push, extras)
@Pyro4.callback
def post_push(self, extras):
log.debug("Called post_push of Hooks object")
return self._call_hook(hooks_base.post_push, extras)

15
rhodecode/lib/logging_formatter.py
View file

@ -67,9 +67,9 @@ def format_sql(sql):
return sql
class Pyro4AwareFormatter(logging.Formatter):
class ExceptionAwareFormatter(logging.Formatter):
"""
Extended logging formatter which prints out Pyro4 remote tracebacks.
Extended logging formatter which prints out remote tracebacks.
"""
def formatException(self, ei):
@ -86,10 +86,12 @@ class Pyro4AwareFormatter(logging.Formatter):
try:
if ex_type is not None and ex_value is None and ex_tb is None:
# possible old (3.x) call syntax where caller is only providing exception object
# possible old (3.x) call syntax where caller is only
# providing exception object
if type(ex_type) is not type:
raise TypeError(
"invalid argument: ex_type should be an exception type, or just supply no arguments at all")
"invalid argument: ex_type should be an exception "
"type, or just supply no arguments at all")
if ex_type is None and ex_tb is None:
ex_type, ex_value, ex_tb = sys.exc_info()
@ -105,7 +107,7 @@ class Pyro4AwareFormatter(logging.Formatter):
return local_tb
class ColorFormatter(Pyro4AwareFormatter):
class ColorFormatter(ExceptionAwareFormatter):
def format(self, record):
"""
@ -134,3 +136,6 @@ class ColorFormatterSql(logging.Formatter):
colored_record = ''.join([start, def_record, end])
return colored_record
# marcink: needs to stay with this name for backward .ini compatability
Pyro4AwareFormatter = ExceptionAwareFormatter

7
rhodecode/lib/middleware/simplevcs.py
View file

@ -42,7 +42,7 @@ from rhodecode.lib.exceptions import (
NotAllowedToCreateUserError)
from rhodecode.lib.hooks_daemon import prepare_callback_daemon
from rhodecode.lib.middleware import appenlight
from rhodecode.lib.middleware.utils import scm_app, scm_app_http
from rhodecode.lib.middleware.utils import scm_app_http
from rhodecode.lib.utils import (
is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path, SLUG_RE)
from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
@ -177,9 +177,6 @@ class SimpleVCS(object):
if custom_implementation == 'http':
log.info('Using HTTP implementation of scm app.')
scm_app_impl = scm_app_http
elif custom_implementation == 'pyro4':
log.info('Using Pyro implementation of scm app.')
scm_app_impl = scm_app
else:
log.info('Using custom implementation of scm_app: "{}"'.format(
custom_implementation))
@ -512,7 +509,7 @@ class SimpleVCS(object):
raise NotImplementedError()
def _create_config(self, extras, repo_name):
"""Create a Pyro safe config representation."""
"""Create a safe config representation."""
raise NotImplementedError()
def _prepare_callback_daemon(self, extras):

4
rhodecode/lib/middleware/utils/scm_app.py
View file

@ -42,7 +42,7 @@ def create_git_wsgi_app(repo_path, repo_name, config):
"""
factory = GIT_REMOTE_WSGI
if not factory:
log.error('Pyro server has not been initialized yet')
log.error('VCSServer has not been initialized yet')
return wsgi_app_caller_client.RemoteAppCaller(
factory, repo_path, repo_name, config)
@ -57,7 +57,7 @@ def create_hg_wsgi_app(repo_path, repo_name, config):
"""
factory = HG_REMOTE_WSGI
if not factory:
log.error('Pyro server has not been initialized yet')
log.error('VCSServer has not been initialized yet')
return wsgi_app_caller_client.RemoteAppCaller(
factory, repo_path, repo_name, config)

12
rhodecode/lib/middleware/utils/wsgi_app_caller_client.py
View file

@ -24,8 +24,6 @@ Utility to call a WSGI app wrapped in a WSGIAppCaller object.
import logging
from Pyro4.errors import ConnectionClosedError
log = logging.getLogger(__name__)
@ -83,14 +81,8 @@ class RemoteAppCaller(object):
input_data = environ['wsgi.input'].read()
clean_environ = _get_clean_environ(environ)
try:
data, status, headers = self._remote_wsgi.handle(
clean_environ, input_data, *self._args, **self._kwargs)
except ConnectionClosedError:
log.debug('Remote Pyro Server ConnectionClosedError')
self._remote_wsgi._pyroReconnect(tries=15)
data, status, headers = self._remote_wsgi.handle(
clean_environ, input_data, *self._args, **self._kwargs)
data, status, headers = self._remote_wsgi.handle(
clean_environ, input_data, *self._args, **self._kwargs)
log.debug("Got result from proxy, returning to WSGI container")
start_response(status, headers)

2
rhodecode/lib/middleware/vcs.py
View file

@ -216,8 +216,6 @@ class VCSMiddleware(object):
vcs_handler.SCM):
return HTTPNotFound()(environ, start_response)
# TODO: johbo: Needed for the Pyro4 backend and Mercurial only.
# Remove once we fully switched to the HTTP backend.
environ['REPO_NAME'] = vcs_handler.url_repo_name
# register repo config back to the handler

73
rhodecode/lib/vcs/__init__.py
View file

@ -40,8 +40,6 @@ import time
import urlparse
from cStringIO import StringIO
import Pyro4
from Pyro4.errors import CommunicationError
from rhodecode.lib.vcs.conf import settings
from rhodecode.lib.vcs.backends import get_vcs_instance, get_backend
@ -72,36 +70,6 @@ def get_version():
return '.'.join((str(each) for each in VERSION[:3]))
def connect_pyro4(server_and_port):
from rhodecode.lib.vcs import connection, client
from rhodecode.lib.middleware.utils import scm_app
git_remote = client.RequestScopeProxyFactory(
settings.pyro_remote(settings.PYRO_GIT, server_and_port))
hg_remote = client.RequestScopeProxyFactory(
settings.pyro_remote(settings.PYRO_HG, server_and_port))
svn_remote = client.RequestScopeProxyFactory(
settings.pyro_remote(settings.PYRO_SVN, server_and_port))
connection.Git = client.RepoMaker(proxy_factory=git_remote)
connection.Hg = client.RepoMaker(proxy_factory=hg_remote)
connection.Svn = client.RepoMaker(proxy_factory=svn_remote)
scm_app.GIT_REMOTE_WSGI = Pyro4.Proxy(
settings.pyro_remote(
settings.PYRO_GIT_REMOTE_WSGI, server_and_port))
scm_app.HG_REMOTE_WSGI = Pyro4.Proxy(
settings.pyro_remote(
settings.PYRO_HG_REMOTE_WSGI, server_and_port))
@atexit.register
def free_connection_resources():
connection.Git = None
connection.Hg = None
connection.Svn = None
connection.Service = None
def connect_http(server_and_port):
from rhodecode.lib.vcs import connection, client_http
from rhodecode.lib.middleware.utils import scm_app
@ -135,11 +103,9 @@ def connect_vcs(server_and_port, protocol):
Initializes the connection to the vcs server.
:param server_and_port: str, e.g. "localhost:9900"
:param protocol: str, "pyro4" or "http"
:param protocol: str or "http"
"""
if protocol == 'pyro4':
connect_pyro4(server_and_port)
elif protocol == 'http':
if protocol == 'http':
connect_http(server_and_port)
else:
raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
@ -154,31 +120,10 @@ def start_vcs_server(server_and_port, protocol, log_level=None):
log.info('Starting VCSServer as a sub process with %s protocol', protocol)
if protocol == 'http':
return _start_http_vcs_server(server_and_port, log_level)
elif protocol == 'pyro4':
return _start_pyro4_vcs_server(server_and_port, log_level)
else:
raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
def _start_pyro4_vcs_server(server_and_port, log_level=None):
_try_to_shutdown_running_server(server_and_port, protocol='pyro4')
host, port = server_and_port.rsplit(":", 1)
host = host.strip('[]')
args = [
'vcsserver', '--port', port, '--host', host, '--locale', 'en_US.UTF-8',
'--threadpool', '32']
if log_level:
args += ['--log-level', log_level]
proc = subprocess32.Popen(args)
def cleanup_server_process():
proc.kill()
atexit.register(cleanup_server_process)
server = create_vcsserver_proxy(server_and_port, protocol='pyro4')
_wait_until_vcs_server_is_reachable(server)
def _start_http_vcs_server(server_and_port, log_level=None):
# TODO: mikhail: shutdown if an http server already runs
@ -202,7 +147,7 @@ def _wait_until_vcs_server_is_reachable(server, timeout=40):
try:
server.ping()
return
except (VCSCommunicationError, CommunicationError, pycurl.error):
except (VCSCommunicationError, pycurl.error):
log.debug('VCSServer not started yet, retry to connect.')
time.sleep(0.5)
raise Exception(
@ -214,7 +159,7 @@ def _try_to_shutdown_running_server(server_and_port, protocol):
server = create_vcsserver_proxy(server_and_port, protocol)
try:
server.shutdown()
except (CommunicationError, pycurl.error):
except pycurl.error:
return
# TODO: Not sure why this is important, but without it the following start
@ -224,20 +169,12 @@ def _try_to_shutdown_running_server(server_and_port, protocol):
def create_vcsserver_proxy(server_and_port, protocol):
if protocol == 'pyro4':
return _create_vcsserver_proxy_pyro4(server_and_port)
elif protocol == 'http':
if protocol == 'http':
return _create_vcsserver_proxy_http(server_and_port)
else:
raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
def _create_vcsserver_proxy_pyro4(server_and_port):
server = Pyro4.Proxy(
settings.pyro_remote(settings.PYRO_VCSSERVER, server_and_port))
return server
def _create_vcsserver_proxy_http(server_and_port):
from rhodecode.lib.vcs import client_http

210
rhodecode/lib/vcs/client.py
View file

@ -1,210 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2017 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
"""
Provides the implementation of various client utilities to reach the vcsserver.
"""
import copy
import logging
import uuid
import weakref
import Pyro4
from pyramid.threadlocal import get_current_request
from Pyro4.errors import CommunicationError, ConnectionClosedError, DaemonError
from rhodecode.lib.vcs import exceptions
from rhodecode.lib.vcs.conf import settings
log = logging.getLogger(__name__)
class RepoMaker(object):
def __init__(self, proxy_factory):
self._proxy_factory = proxy_factory
def __call__(self, path, config, with_wire=None):
log.debug('RepoMaker call on %s', path)
return RemoteRepo(
path, config, remote_proxy=self._proxy_factory(),
with_wire=with_wire)
def __getattr__(self, name):
remote_proxy = self._proxy_factory()
func = _get_proxy_method(remote_proxy, name)
return _wrap_remote_call(remote_proxy, func)
class RequestScopeProxyFactory(object):
"""
This factory returns pyro proxy instances based on a per request scope.
It returns the same instance if called from within the same request and
different instances if called from different requests.
"""
def __init__(self, remote_uri):
self._remote_uri = remote_uri
self._proxy_pool = []
self._borrowed_proxies = {}
def __call__(self, request=None):
"""
Wrapper around `getProxy`.
"""
request = request or get_current_request()
return self.getProxy(request)
def getProxy(self, request):
"""
Call this to get the pyro proxy instance for the request.
"""
# If called without a request context we return new proxy instances
# on every call. This allows to run e.g. invoke tasks.
if request is None:
log.info('Creating pyro proxy without request context for '
'remote_uri=%s', self._remote_uri)
return Pyro4.Proxy(self._remote_uri)
# If there is an already borrowed proxy for the request context we
# return that instance instead of creating a new one.
if request in self._borrowed_proxies:
return self._borrowed_proxies[request]
# Get proxy from pool or create new instance.
try:
proxy = self._proxy_pool.pop()
except IndexError:
log.info('Creating pyro proxy for remote_uri=%s', self._remote_uri)
proxy = Pyro4.Proxy(self._remote_uri)
# Mark proxy as borrowed for the request context and add a callback
# that returns it when the request processing is finished.
self._borrowed_proxies[request] = proxy
request.add_finished_callback(self._returnProxy)
return proxy
def _returnProxy(self, request):
"""
Callback that gets called by pyramid when the request is finished.
It puts the proxy back into the pool.
"""
if request in self._borrowed_proxies:
proxy = self._borrowed_proxies.pop(request)
self._proxy_pool.append(proxy)
else:
log.warn('Return proxy for remote_uri=%s but no proxy borrowed '
'for this request.', self._remote_uri)
class RemoteRepo(object):
def __init__(self, path, config, remote_proxy, with_wire=None):
self._wire = {
"path": path,
"config": config,
"context": self._create_vcs_cache_context(),
}
if with_wire:
self._wire.update(with_wire)
self._remote_proxy = remote_proxy
self.refs = RefsWrapper(self)
def __getattr__(self, name):
log.debug('Calling %s@%s', self._remote_proxy, name)
# TODO: oliver: This is currently necessary pre-call since the
# config object is being changed for hooking scenarios
wire = copy.deepcopy(self._wire)
wire["config"] = wire["config"].serialize()
try:
func = _get_proxy_method(self._remote_proxy, name)
except DaemonError as e:
if e.message == 'unknown object':
raise exceptions.VCSBackendNotSupportedError
else:
raise
return _wrap_remote_call(self._remote_proxy, func, wire)
def __getitem__(self, key):
return self.revision(key)
def _create_vcs_cache_context(self):
"""
Creates a unique string which is passed to the VCSServer on every
remote call. It is used as cache key in the VCSServer.
"""
return str(uuid.uuid4())
def invalidate_vcs_cache(self):
"""
This is a no-op method for the pyro4 backend but we want to have the
same API for client.RemoteRepo and client_http.RemoteRepo classes.
"""
def _get_proxy_method(proxy, name):
try:
return getattr(proxy, name)
except CommunicationError:
raise exceptions.PyroVCSCommunicationError(
'Unable to connect to remote pyro server %s' % proxy)
def _wrap_remote_call(proxy, func, *args):
all_args = list(args)
@exceptions.map_vcs_exceptions
def caller(*args, **kwargs):
all_args.extend(args)
try:
return func(*all_args, **kwargs)
except ConnectionClosedError:
log.debug('Connection to VCSServer closed, trying to reconnect.')
proxy._pyroReconnect(tries=settings.PYRO_RECONNECT_TRIES)
return func(*all_args, **kwargs)
return caller
class RefsWrapper(object):
def __init__(self, repo):
self._repo = weakref.proxy(repo)
def __setitem__(self, key, value):
self._repo._assign_ref(key, value)
class FunctionWrapper(object):
def __init__(self, func, wire):
self._func = func
self._wire = wire
@exceptions.map_vcs_exceptions
def __call__(self, *args, **kwargs):
return self._func(self._wire, *args, **kwargs)

7
rhodecode/lib/vcs/client_http.py
View file

@ -20,13 +20,6 @@
"""
Client for the VCSServer implemented based on HTTP.
Status
------
This client implementation shall eventually replace the Pyro4 based
implementation.
"""
import copy

21
rhodecode/lib/vcs/conf/settings.py
View file

@ -52,27 +52,6 @@ ARCHIVE_SPECS = {
HOOKS_PROTOCOL = None
HOOKS_DIRECT_CALLS = False
PYRO_PORT = 9900
PYRO_GIT = 'git_remote'
PYRO_HG = 'hg_remote'
PYRO_SVN = 'svn_remote'
PYRO_VCSSERVER = 'vcs_server'
PYRO_GIT_REMOTE_WSGI = 'git_remote_wsgi'
PYRO_HG_REMOTE_WSGI = 'hg_remote_wsgi'
PYRO_RECONNECT_TRIES = 15
"""
How many retries to reconnect will be performed if the connection was lost.
Each try takes 2s. Doing 15 means that we will give it up to 30s for a
connection to be re-established.
"""
def pyro_remote(object_id, server_and_port):
return "PYRO:%s@%s" % (object_id, server_and_port)
def available_aliases():
"""

4
rhodecode/lib/vcs/exceptions.py
View file

@ -30,10 +30,6 @@ class VCSCommunicationError(Exception):
pass
class PyroVCSCommunicationError(VCSCommunicationError):
pass
class HttpVCSCommunicationError(VCSCommunicationError):
pass

2
rhodecode/tests/lib/middleware/test_simplevcs.py
View file

@ -326,8 +326,6 @@ class TestShadowRepoExposure(object):
@pytest.mark.usefixtures('db')
@mock.patch.multiple(
'Pyro4.config', SERVERTYPE='multiplex', POLLTIMEOUT=0.01)
class TestGenerateVcsResponse:
def test_ensures_that_start_response_is_called_early_enough(self):

4
rhodecode/tests/lib/middleware/test_vcs_unavailable.py
View file

@ -38,12 +38,10 @@ def test_vcs_unavailable_returns_vcs_error_page(app, backend, app_settings):
# Depending on the used VCSServer protocol we have to patch a different
# RemoteRepo class to raise an exception. For the test it doesn't matter
# if http or pyro4 is used, it just requires the exception to be raised.
# if http is used, it just requires the exception to be raised.
vcs_protocol = app_settings['vcs.server.protocol']
if vcs_protocol == 'http':
from rhodecode.lib.vcs.client_http import RemoteRepo
elif vcs_protocol == 'pyro4':
from rhodecode.lib.vcs.client import RemoteRepo
else:
pytest.fail('Unknown VCS server protocol: "{}"'.format(vcs_protocol))

47
rhodecode/tests/lib/middleware/utils/test_scm_app_http.py
View file

@ -18,8 +18,6 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import mock
import Pyro4
import pytest
from rhodecode.tests.utils import CustomTestApp
@ -83,48 +81,3 @@ def test_create_app_per_request_with_data(
app = vcs_http_app(vcsserver_http_echo_app)
response = app.post('/', params=data)
assert response.status_code == 200
@pytest.fixture(scope='module')
def vcsserver_pyro_echo_app(request, vcsserver_factory):
"""
A running VCSServer with the EchoApp activated via Pyro4.
"""
vcsserver = vcsserver_factory(
request=request,
use_http=False,
overrides=[{'DEFAULT': {'dev.use_echo_app': 'true'}}])
return vcsserver
def vcs_pyro4_app(vcsserver_pyro_echo_app):
"""
Pyro4 based Vcs proxy wrapped in WebTest
"""
stub_config = {
'git_update_server_info': 'stub',
}
server_and_port = vcsserver_pyro_echo_app.server_and_port
GIT_REMOTE_WSGI = Pyro4.Proxy(
settings.pyro_remote(
settings.PYRO_GIT_REMOTE_WSGI, server_and_port))
with mock.patch('rhodecode.lib.middleware.utils.scm_app.GIT_REMOTE_WSGI',
GIT_REMOTE_WSGI):
pyro4_app = scm_app.create_git_wsgi_app(
'stub_path', 'stub_name', stub_config)
app = CustomTestApp(pyro4_app)
return app
def test_pyro4_no_data(repeat, pylonsapp, vcsserver_pyro_echo_app):
for x in xrange(repeat / 10):
app = vcs_pyro4_app(vcsserver_pyro_echo_app)
response = app.post('/')
assert response.status_code == 200
def test_pyro4_with_data(repeat, pylonsapp, vcsserver_pyro_echo_app, data):
for x in xrange(repeat / 10):
app = vcs_pyro4_app(vcsserver_pyro_echo_app)
response = app.post('/', params=data)
assert response.status_code == 200

73
rhodecode/tests/lib/test_hooks_daemon.py
View file

@ -171,71 +171,6 @@ class ThreadedHookCallbackDaemon(object):
assert daemon_context == daemon
class TestPyro4HooksCallbackDaemon(object):
def test_prepare_inits_pyro4_and_registers_hooks(self, caplog):
pyro4_daemon = mock.Mock()
with self._pyro4_patcher(pyro4_daemon), caplog.at_level(logging.DEBUG):
daemon = hooks_daemon.Pyro4HooksCallbackDaemon()
assert daemon._daemon == pyro4_daemon
assert pyro4_daemon.register.call_count == 1
args, kwargs = pyro4_daemon.register.call_args
assert len(args) == 1
assert isinstance(args[0], hooks_daemon.Hooks)
assert_message_in_log(
caplog.records,
'Preparing callback daemon and registering hook object',
levelno=logging.DEBUG, module='hooks_daemon')
def test_run_creates_a_thread(self):
thread = mock.Mock()
pyro4_daemon = mock.Mock()
with self._pyro4_patcher(pyro4_daemon):
daemon = hooks_daemon.Pyro4HooksCallbackDaemon()
with self._thread_patcher(thread) as thread_mock:
daemon._run()
assert thread_mock.call_count == 1
args, kwargs = thread_mock.call_args
assert args == ()
assert kwargs['target'] == pyro4_daemon.requestLoop
assert kwargs['kwargs']['loopCondition']() is True
def test_stop_cleans_up_the_connection(self, caplog):
thread = mock.Mock()
pyro4_daemon = mock.Mock()
with self._pyro4_patcher(pyro4_daemon):
daemon = hooks_daemon.Pyro4HooksCallbackDaemon()
with self._thread_patcher(thread), caplog.at_level(logging.DEBUG):
with daemon:
assert daemon._daemon == pyro4_daemon
assert daemon._callback_thread == thread
assert daemon._daemon is None
assert daemon._callback_thread is None
pyro4_daemon.close.assert_called_with()
thread.join.assert_called_once_with()
assert_message_in_log(
caplog.records, 'Waiting for background thread to finish.',
levelno=logging.DEBUG, module='hooks_daemon')
def _pyro4_patcher(self, daemon):
return mock.patch.object(
hooks_daemon.Pyro4, 'Daemon', return_value=daemon)
def _thread_patcher(self, thread):
return mock.patch.object(
hooks_daemon.threading, 'Thread', return_value=thread)
class TestHttpHooksCallbackDaemon(object):
def test_prepare_inits_daemon_variable(self, tcp_server, caplog):
with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
@ -318,7 +253,7 @@ class TestHttpHooksCallbackDaemon(object):
class TestPrepareHooksDaemon(object):
@pytest.mark.parametrize('protocol', ('http', 'pyro4'))
@pytest.mark.parametrize('protocol', ('http',))
def test_returns_dummy_hooks_callback_daemon_when_using_direct_calls(
self, protocol):
expected_extras = {'extra1': 'value1'}
@ -329,8 +264,7 @@ class TestPrepareHooksDaemon(object):
assert extras == expected_extras
@pytest.mark.parametrize('protocol, expected_class', (
('pyro4', hooks_daemon.Pyro4HooksCallbackDaemon),
('http', hooks_daemon.HttpHooksCallbackDaemon)
('http', hooks_daemon.HttpHooksCallbackDaemon),
))
def test_returns_real_hooks_callback_daemon_when_protocol_is_specified(
self, protocol, expected_class):
@ -343,12 +277,9 @@ class TestPrepareHooksDaemon(object):
assert isinstance(callback, expected_class)
hooks_uri = extras.pop('hooks_uri')
assert extras == expected_extras
if protocol.lower() == 'pyro4':
assert hooks_uri.startswith('PYRO')
@pytest.mark.parametrize('protocol', (
'invalid',
'Pyro4',
'Http',
'HTTP',
))

1
rhodecode/tests/models/test_pullrequest.py
View file

@ -375,7 +375,6 @@ class TestPullRequestModel:
class TestIntegrationMerge(object):
@pytest.mark.parametrize('extra_config', (
{'vcs.hooks.protocol': 'http', 'vcs.hooks.direct_calls': False},
{'vcs.hooks.protocol': 'Pyro4', 'vcs.hooks.direct_calls': False},
))
def test_merge_triggers_push_hooks(
self, pr_util, user_admin, capture_rcextensions, merge_extras,

26
rhodecode/tests/plugin.py
View file

@ -321,27 +321,6 @@ def tests_tmp_path(request):
return TESTS_TMP_PATH
@pytest.fixture(scope='session', autouse=True)
def patch_pyro_request_scope_proxy_factory(request):
"""
Patch the pyro proxy factory to always use the same dummy request object
when under test. This will return the same pyro proxy on every call.
"""
dummy_request = pyramid.testing.DummyRequest()
def mocked_call(self, request=None):
return self.getProxy(request=dummy_request)
patcher = mock.patch(
'rhodecode.lib.vcs.client.RequestScopeProxyFactory.__call__',
new=mocked_call)
patcher.start()
@request.addfinalizer
def undo_patching():
patcher.stop()
@pytest.fixture
def test_repo_group(request):
"""
@ -1350,7 +1329,7 @@ def pytest_runtest_makereport(item, call):
"""
Adding the remote traceback if the exception has this information.
Pyro4 attaches this information as the attribute `_vcs_server_traceback`
VCSServer attaches this information as the attribute `_vcs_server_traceback`
to the exception instance.
"""
outcome = yield
@ -1411,7 +1390,8 @@ def collect_appenlight_stats(request, testrun):
})
server_and_port = pylonsapp.config['vcs.server']
server = create_vcsserver_proxy(server_and_port)
protocol = pylonsapp.config['vcs.server.protocol']
server = create_vcsserver_proxy(server_and_port, protocol)
with server:
vcs_pid = server.get_pid()
server.run_gc()

65
rhodecode/tests/pylons_plugin.py
View file

@ -35,7 +35,6 @@ from beaker.session import SessionObject
from paste.deploy import loadapp
from pylons.i18n.translation import _get_translator
from pylons.util import ContextObj
from Pyro4.errors import CommunicationError
from routes.util import URLGenerator
from rhodecode.lib import vcs
@ -70,15 +69,12 @@ def pytest_addoption(parser):
vcsgroup.addoption(
'--without-vcsserver', dest='with_vcsserver', action='store_false',
help="Do not start the VCSServer in a background process.")
vcsgroup.addoption(
'--with-vcsserver', dest='vcsserver_config_pyro4',
help="Start the VCSServer with the specified config file.")
vcsgroup.addoption(
'--with-vcsserver-http', dest='vcsserver_config_http',
help="Start the HTTP VCSServer with the specified config file.")
vcsgroup.addoption(
'--vcsserver-protocol', dest='vcsserver_protocol',
help="Start the VCSServer with HTTP / Pyro4 protocol support.")
help="Start the VCSServer with HTTP protocol support.")
vcsgroup.addoption(
'--vcsserver-config-override', action='store', type=_parse_json,
default=None, dest='vcsserver_config_override', help=(
@ -93,15 +89,12 @@ def pytest_addoption(parser):
"Allows to set the port of the vcsserver. Useful when testing "
"against an already running server and random ports cause "
"trouble."))
parser.addini(
'vcsserver_config_pyro4',
"Start the VCSServer with the specified config file.")
parser.addini(
'vcsserver_config_http',
"Start the HTTP VCSServer with the specified config file.")
parser.addini(
'vcsserver_protocol',
"Start the VCSServer with HTTP / Pyro4 protocol support.")
"Start the VCSServer with HTTP protocol support.")
@pytest.fixture(scope='session')
@ -153,8 +146,7 @@ def vcsserver_factory(tmpdir_factory):
'beaker.cache.repo_object.type': 'nocache'}}
overrides.append(platform_override)
option_name = (
'vcsserver_config_http' if use_http else 'vcsserver_config_pyro4')
option_name = 'vcsserver_config_http' if use_http else ''
override_option_name = 'vcsserver_config_override'
config_file = get_config(
request.config, option_name=option_name,
@ -162,8 +154,8 @@ def vcsserver_factory(tmpdir_factory):
basetemp=tmpdir_factory.getbasetemp().strpath,
prefix='test_vcs_')
print "Using the VCSServer configuration", config_file
ServerClass = HttpVCSServer if use_http else Pyro4VCSServer
print("Using the VCSServer configuration:{}".format(config_file))
ServerClass = HttpVCSServer if use_http else None
server = ServerClass(config_file)
server.start()
@ -213,39 +205,6 @@ class VCSServer(object):
self.process.kill()
class Pyro4VCSServer(VCSServer):
def __init__(self, config_file):
"""
:param config_file: The config file to start the server with
"""
config_data = configobj.ConfigObj(config_file)
self._config = config_data['DEFAULT']
args = ['vcsserver', '--config', config_file]
self._args = args
def wait_until_ready(self, timeout=30):
remote_server = vcs.create_vcsserver_proxy(
self.server_and_port, 'pyro4')
start = time.time()
with remote_server:
while time.time() - start < timeout:
try:
remote_server.ping()
break
except CommunicationError:
time.sleep(0.2)
else:
pytest.exit(
"Starting the VCSServer failed or took more than {} "
"seconds.".format(timeout))
@property
def server_and_port(self):
return '{host}:{port}'.format(**self._config)
class HttpVCSServer(VCSServer):
"""
Represents a running VCSServer instance.
@ -316,14 +275,6 @@ def pylons_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
'vcs.hooks.protocol': 'http',
}
})
else:
overrides.append({
'app:main': {
'vcs.server.protocol': 'pyro4',
'vcs.scm_app_implementation': 'pyro4',
'vcs.hooks.protocol': 'pyro4',
}
})
filename = get_config(
request.config, option_name=option_name,
@ -337,7 +288,7 @@ def pylons_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
@pytest.fixture(scope='session')
def rcserver_port(request):
port = get_available_port()
print 'Using rcserver port %s' % (port, )
print('Using rcserver port {}'.format(port))
return port
@ -346,7 +297,7 @@ def vcsserver_port(request):
port = request.config.getoption('--vcsserver-port')
if port is None:
port = get_available_port()
print 'Using vcsserver port %s' % (port, )
print('Using vcsserver port {}'.format(port))
return port
@ -383,7 +334,7 @@ def available_port(available_port_factory):
@pytest.fixture(scope='session')
def pylonsapp(pylons_config, vcsserver, http_environ_session):
print "Using the RhodeCode configuration", pylons_config
print("Using the RhodeCode configuration:{}".format(pylons_config))
logging.config.fileConfig(
pylons_config, disable_existing_loggers=False)
app = _setup_pylons_environment(pylons_config, http_environ_session)

13
rhodecode/tests/rhodecode.ini
View file

@ -575,18 +575,15 @@ vcs.server = localhost:9901
## Web server connectivity protocol, responsible for web based VCS operatations
## Available protocols are:
## `pyro4` - using pyro4 server
## `http` - using http-rpc backend
vcs.server.protocol = http
## Push/Pull operations protocol, available options are:
## `pyro4` - using pyro4 server
## `rhodecode.lib.middleware.utils.scm_app_http` - Http based, recommended
## `vcsserver.scm_app` - internal app (EE only)
vcs.scm_app_implementation = http
## Push/Pull operations hooks protocol, available options are:
## `pyro4` - using pyro4 server
## `http` - using http-rpc backend
vcs.hooks.protocol = http
@ -628,7 +625,7 @@ svn.proxy.location_root = /
### LOGGING CONFIGURATION ####
################################
[loggers]
keys = root, routes, rhodecode, sqlalchemy, beaker, pyro4, templates
keys = root, routes, rhodecode, sqlalchemy, beaker, templates
[handlers]
keys = console, console_sql
@ -656,12 +653,6 @@ handlers =
qualname = beaker.container
propagate = 1
[logger_pyro4]
level = DEBUG
handlers =
qualname = Pyro4
propagate = 1
[logger_templates]
level = INFO
handlers =
@ -701,7 +692,7 @@ formatter = generic
################
[formatter_generic]
class = rhodecode.lib.logging_formatter.Pyro4AwareFormatter
class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %Y-%m-%d %H:%M:%S

4
rhodecode/tests/vcs/test_client_http.py
View file

@ -94,7 +94,3 @@ def test_connect_passes_in_the_same_session(
session_factory.return_value = stub_session
vcs.connect_http('server_and_port')
assert connection.Hg._session_factory() == stub_session
assert connection.Svn._session_factory() == stub_session
assert connection.Git._session_factory() == stub_session

5
rhodecode/tests/vcs/test_hg_vcsserver_cache_invalidation.py
View file

@ -123,9 +123,6 @@ class TestMercurialRemoteRepoInvalidation(object):
"""
from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
if pylonsapp.config['vcs.server.protocol'] == 'pyro4':
pytest.skip('Test is intended for the HTTP protocol only.')
pull_request = pr_util.create_pull_request()
target_vcs = pull_request.target_repo.scm_instance()
source_vcs = pull_request.source_repo.scm_instance()
@ -163,8 +160,6 @@ class TestMercurialRemoteRepoInvalidation(object):
Without correct cache invalidation this leads to an error when
retrieving the pulled commits afterwards.
"""
if pylonsapp.config['vcs.server.protocol'] == 'pyro4':
pytest.skip('Test is intended for the HTTP protocol only.')
pull_request = pr_util.create_pull_request()
target_vcs = pull_request.target_repo.scm_instance()

69
rhodecode/tests/vcs/test_vcsclient.py
View file

@ -1,69 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2010-2017 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import mock
import Pyro4
import pytest
from Pyro4.errors import TimeoutError, ConnectionClosedError
from rhodecode.lib.vcs import client, create_vcsserver_proxy
from rhodecode.lib.vcs.conf import settings
@pytest.fixture
def short_timeout(request):
old_timeout = Pyro4.config.COMMTIMEOUT
Pyro4.config.COMMTIMEOUT = 0.5
@request.addfinalizer
def cleanup():
Pyro4.config.COMMTIMEOUT = old_timeout
return Pyro4.config.COMMTIMEOUT
@pytest.mark.timeout(20)
def test_vcs_connections_have_a_timeout_set(pylonsapp, short_timeout):
server_and_port = pylonsapp.config['vcs.server']
proxy_objects = []
with pytest.raises(TimeoutError):
# TODO: johbo: Find a better way to set this number
for number in xrange(100):
server = create_vcsserver_proxy(server_and_port, protocol='pyro4')
server.ping()
proxy_objects.append(server)
def test_vcs_remote_calls_are_bound_by_timeout(pylonsapp, short_timeout):
server_and_port = pylonsapp.config['vcs.server']
with pytest.raises(TimeoutError):
server = create_vcsserver_proxy(server_and_port, protocol='pyro4')
server.sleep(short_timeout + 1.0)
def test_wrap_remote_call_triggers_reconnect():
# First call fails, the second is successful
func = mock.Mock(side_effect=[ConnectionClosedError, None])
proxy_mock = mock.Mock()
wrapped_func = client._wrap_remote_call(proxy_mock, func)
wrapped_func()
proxy_mock._pyroReconnect.assert_called_with(
tries=settings.PYRO_RECONNECT_TRIES)

8
rhodecode/tests/vcsserver_http.ini
View file

@ -30,7 +30,7 @@ port = 9900
### LOGGING CONFIGURATION ####
################################
[loggers]
keys = root, vcsserver, pyro4, beaker
keys = root, vcsserver, beaker
[handlers]
keys = console
@ -57,12 +57,6 @@ handlers =
qualname = beaker
propagate = 1
[logger_pyro4]
level = DEBUG
handlers =
qualname = Pyro4
propagate = 1
##############
## HANDLERS ##

79
rhodecode/tests/vcsserver_pyro4.ini
View file

@ -1,79 +0,0 @@
################################################################################
# RhodeCode VCSServer - configuration #
# #
################################################################################
[DEFAULT]
host = 127.0.0.1
port = 9900
locale = en_US.UTF-8
# number of worker threads, this should be set based on a formula threadpool=N*6
# where N is number of RhodeCode Enterprise workers, eg. running 2 instances
# 8 gunicorn workers each would be 2 * 8 * 6 = 96, threadpool_size = 96
threadpool_size = 96
timeout = 0
# cache regions, please don't change
beaker.cache.regions = repo_object
beaker.cache.repo_object.type = memorylru
beaker.cache.repo_object.max_items = 100
# cache auto-expires after N seconds
beaker.cache.repo_object.expire = 300
beaker.cache.repo_object.enabled = true
################################
### LOGGING CONFIGURATION ####
################################
[loggers]
keys = root, vcsserver, pyro4, beaker
[handlers]
keys = console
[formatters]
keys = generic
#############
## LOGGERS ##
#############
[logger_root]
level = NOTSET
handlers = console
[logger_vcsserver]
level = DEBUG
handlers =
qualname = vcsserver
propagate = 1
[logger_beaker]
level = DEBUG
handlers =
qualname = beaker
propagate = 1
[logger_pyro4]
level = DEBUG
handlers =
qualname = Pyro4
propagate = 1
##############
## HANDLERS ##
##############
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = INFO
formatter = generic
################
## FORMATTERS ##
################
[formatter_generic]
format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %Y-%m-%d %H:%M:%S

1
setup.py
View file

@ -84,7 +84,6 @@ install_requirements = [
'Pygments',
'pygments-markdown-lexer',
'Pylons',
'Pyro4',
'Routes',
'SQLAlchemy',
'Tempita',