svn: enable hooks and integration framework execution.

- This change will allow SVN to execute rhodecode integration framework
via executing the builting hooks.
- uses txn_id for svn to provide metadata required via Apache
- Interception of SVN transaction is required to use temporary stores for metadata
- hooks are now part of vcsserver where the install/control logic is stored
This commit is contained in:
Marcin Kuzminski 2018-04-06 12:44:01 +02:00
parent 3ba4262026
commit bdf5f0afb6
18 changed files with 226 additions and 513 deletions

View file

@ -16,9 +16,6 @@ recursive-include configs *
# translations
recursive-include rhodecode/i18n *
# hook templates
recursive-include rhodecode/config/hook_templates *
# non-python core stuff
recursive-include rhodecode *.cfg
recursive-include rhodecode *.json

View file

@ -42,7 +42,8 @@ class TestGetRepoChangeset(object):
if details == 'full':
assert result['refs']['bookmarks'] == getattr(
commit, 'bookmarks', [])
assert result['refs']['branches'] == [commit.branch]
branches = [commit.branch] if commit.branch else []
assert result['refs']['branches'] == branches
assert result['refs']['tags'] == commit.tags
@pytest.mark.parametrize("details", ['basic', 'extended', 'full'])

View file

@ -1,46 +0,0 @@
#!_ENV_
import os
import sys
try:
from vcsserver import hooks
except ImportError:
if os.environ.get('RC_DEBUG_GIT_HOOK'):
import traceback
print traceback.format_exc()
hooks = None
RC_HOOK_VER = '_TMPL_'
def main():
if hooks is None:
# exit with success if we cannot import vcsserver.hooks !!
# this allows simply push to this repo even without rhodecode
sys.exit(0)
if os.environ.get('RC_SKIP_HOOKS'):
sys.exit(0)
repo_path = os.getcwd()
push_data = sys.stdin.readlines()
os.environ['RC_HOOK_VER'] = RC_HOOK_VER
# os.environ is modified here by a subprocess call that
# runs git and later git executes this hook.
# Environ gets some additional info from rhodecode system
# like IP or username from basic-auth
try:
result = hooks.git_post_receive(repo_path, push_data, os.environ)
sys.exit(result)
except Exception as error:
# TODO: johbo: Improve handling of this special case
if not getattr(error, '_vcs_kind', None) == 'repo_locked':
raise
print 'ERROR:', error
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main()

View file

@ -1,46 +0,0 @@
#!_ENV_
import os
import sys
try:
from vcsserver import hooks
except ImportError:
if os.environ.get('RC_DEBUG_GIT_HOOK'):
import traceback
print traceback.format_exc()
hooks = None
RC_HOOK_VER = '_TMPL_'
def main():
if hooks is None:
# exit with success if we cannot import vcsserver.hooks !!
# this allows simply push to this repo even without rhodecode
sys.exit(0)
if os.environ.get('RC_SKIP_HOOKS'):
sys.exit(0)
repo_path = os.getcwd()
push_data = sys.stdin.readlines()
os.environ['RC_HOOK_VER'] = RC_HOOK_VER
# os.environ is modified here by a subprocess call that
# runs git and later git executes this hook.
# Environ gets some additional info from rhodecode system
# like IP or username from basic-auth
try:
result = hooks.git_pre_receive(repo_path, push_data, os.environ)
sys.exit(result)
except Exception as error:
# TODO: johbo: Improve handling of this special case
if not getattr(error, '_vcs_kind', None) == 'repo_locked':
raise
print 'ERROR:', error
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main()

View file

@ -1,14 +0,0 @@
#!_ENV_
import sys
RC_HOOK_VER = '_TMPL_'
def main():
sys.exit(0)
if __name__ == '__main__':
main()

View file

@ -1,19 +0,0 @@
#!_ENV_
import os
import sys
RC_HOOK_VER = '_TMPL_'
def main():
if os.environ.get('SSH_READ_ONLY') == '1':
sys.stderr.write('Only read-only access is allowed')
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main()

View file

@ -18,10 +18,13 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import json
import os
import time
import logging
import tempfile
import traceback
import threading
from BaseHTTPServer import BaseHTTPRequestHandler
from SocketServer import TCPServer
@ -30,14 +33,28 @@ from rhodecode.model import meta
from rhodecode.lib.base import bootstrap_request, bootstrap_config
from rhodecode.lib import hooks_base
from rhodecode.lib.utils2 import AttributeDict
from rhodecode.lib.ext_json import json
log = logging.getLogger(__name__)
class HooksHttpHandler(BaseHTTPRequestHandler):
def do_POST(self):
method, extras = self._read_request()
txn_id = getattr(self.server, 'txn_id', None)
if txn_id:
from rhodecode.lib.caches import compute_key_from_params
log.debug('Computing TXN_ID based on `%s`:`%s`',
extras['repository'], extras['txn_id'])
computed_txn_id = compute_key_from_params(
extras['repository'], extras['txn_id'])
if txn_id != computed_txn_id:
raise Exception(
'TXN ID fail: expected {} got {} instead'.format(
txn_id, computed_txn_id))
try:
result = self._call_hook(method, extras)
except Exception as e:
@ -77,13 +94,14 @@ class HooksHttpHandler(BaseHTTPRequestHandler):
message = format % args
# TODO: mikhail: add different log levels support
log.debug(
"%s - - [%s] %s", self.client_address[0],
self.log_date_time_string(), message)
class DummyHooksCallbackDaemon(object):
hooks_uri = ''
def __init__(self):
self.hooks_module = Hooks.__module__
@ -101,8 +119,8 @@ class ThreadedHookCallbackDaemon(object):
_daemon = None
_done = False
def __init__(self):
self._prepare()
def __init__(self, txn_id=None, port=None):
self._prepare(txn_id=txn_id, port=port)
def __enter__(self):
self._run()
@ -112,7 +130,7 @@ class ThreadedHookCallbackDaemon(object):
log.debug('Callback daemon exiting now...')
self._stop()
def _prepare(self):
def _prepare(self, txn_id=None, port=None):
raise NotImplementedError()
def _run(self):
@ -135,15 +153,18 @@ class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
# request and wastes cpu at all other times.
POLL_INTERVAL = 0.01
def _prepare(self):
log.debug("Preparing HTTP callback daemon and registering hook object")
def _prepare(self, txn_id=None, port=None):
self._done = False
self._daemon = TCPServer((self.IP_ADDRESS, 0), HooksHttpHandler)
self._daemon = TCPServer((self.IP_ADDRESS, port or 0), HooksHttpHandler)
_, port = self._daemon.server_address
self.hooks_uri = '{}:{}'.format(self.IP_ADDRESS, port)
self.txn_id = txn_id
# inject transaction_id for later verification
self._daemon.txn_id = self.txn_id
log.debug("Hooks uri is: %s", self.hooks_uri)
log.debug(
"Preparing HTTP callback daemon at `%s` and registering hook object",
self.hooks_uri)
def _run(self):
log.debug("Running event loop of callback daemon in background thread")
@ -160,26 +181,67 @@ class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
self._callback_thread.join()
self._daemon = None
self._callback_thread = None
if self.txn_id:
txn_id_file = get_txn_id_data_path(self.txn_id)
log.debug('Cleaning up TXN ID %s', txn_id_file)
if os.path.isfile(txn_id_file):
os.remove(txn_id_file)
log.debug("Background thread done.")
def prepare_callback_daemon(extras, protocol, use_direct_calls):
callback_daemon = None
def get_txn_id_data_path(txn_id):
root = tempfile.gettempdir()
return os.path.join(root, 'rc_txn_id_{}'.format(txn_id))
def store_txn_id_data(txn_id, data_dict):
if not txn_id:
log.warning('Cannot store txn_id because it is empty')
return
path = get_txn_id_data_path(txn_id)
try:
with open(path, 'wb') as f:
f.write(json.dumps(data_dict))
except Exception:
log.exception('Failed to write txn_id metadata')
def get_txn_id_from_store(txn_id):
"""
Reads txn_id from store and if present returns the data for callback manager
"""
path = get_txn_id_data_path(txn_id)
try:
with open(path, 'rb') as f:
return json.loads(f.read())
except Exception:
return {}
def prepare_callback_daemon(extras, protocol, use_direct_calls, txn_id=None):
txn_details = get_txn_id_from_store(txn_id)
port = txn_details.get('port', 0)
if use_direct_calls:
callback_daemon = DummyHooksCallbackDaemon()
extras['hooks_module'] = callback_daemon.hooks_module
else:
if protocol == 'http':
callback_daemon = HttpHooksCallbackDaemon()
callback_daemon = HttpHooksCallbackDaemon(txn_id=txn_id, port=port)
else:
log.error('Unsupported callback daemon protocol "%s"', protocol)
raise Exception('Unsupported callback daemon protocol.')
extras['hooks_uri'] = callback_daemon.hooks_uri
extras['hooks_protocol'] = protocol
extras['hooks_uri'] = callback_daemon.hooks_uri
extras['hooks_protocol'] = protocol
extras['time'] = time.time()
log.debug('Prepared a callback daemon: %s', callback_daemon)
# register txn_id
extras['txn_id'] = txn_id
log.debug('Prepared a callback daemon: %s at url `%s`',
callback_daemon.__class__.__name__, callback_daemon.hooks_uri)
return callback_daemon, extras

View file

@ -18,17 +18,21 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import base64
import logging
import urllib
from urlparse import urljoin
import requests
from webob.exc import HTTPNotAcceptable
from rhodecode.lib import caches
from rhodecode.lib.middleware import simplevcs
from rhodecode.lib.utils import is_valid_repo
from rhodecode.lib.utils2 import str2bool
from rhodecode.lib.utils2 import str2bool, safe_int
from rhodecode.lib.ext_json import json
from rhodecode.lib.hooks_daemon import store_txn_id_data
log = logging.getLogger(__name__)
@ -39,7 +43,6 @@ class SimpleSvnApp(object):
'transfer-encoding', 'content-length']
rc_extras = {}
def __init__(self, config):
self.config = config
@ -52,9 +55,19 @@ class SimpleSvnApp(object):
# length, then we should transfer the payload in one request.
if environ['REQUEST_METHOD'] == 'MKCOL' or 'CONTENT_LENGTH' in environ:
data = data.read()
if data.startswith('(create-txn-with-props'):
# store on-the-fly our rc_extra using svn revision properties
# those can be read later on in hooks executed so we have a way
# to pass in the data into svn hooks
rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
rc_data_len = len(rc_data)
# header defines data lenght, and serialized data
skel = ' rc-scm-extras {} {}'.format(rc_data_len, rc_data)
data = data[:-2] + skel + '))'
log.debug('Calling: %s method via `%s`', environ['REQUEST_METHOD'],
self._get_url(environ['PATH_INFO']))
response = requests.request(
environ['REQUEST_METHOD'], self._get_url(environ['PATH_INFO']),
data=data, headers=request_headers)
@ -70,6 +83,14 @@ class SimpleSvnApp(object):
log.debug('got response code: %s', response.status_code)
response_headers = self._get_response_headers(response.headers)
if response.headers.get('SVN-Txn-name'):
svn_tx_id = response.headers.get('SVN-Txn-name')
txn_id = caches.compute_key_from_params(
self.config['repository'], svn_tx_id)
port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1])
store_txn_id_data(txn_id, {'port': port})
start_response(
'{} {}'.format(response.status_code, response.reason),
response_headers)
@ -156,6 +177,14 @@ class SimpleSvn(simplevcs.SimpleVCS):
if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS
else 'push')
def _should_use_callback_daemon(self, extras, environ, action):
# only MERGE command triggers hooks, so we don't want to start
# hooks server too many times. POST however starts the svn transaction
# so we also need to run the init of callback daemon of POST
if environ['REQUEST_METHOD'] in ['MERGE', 'POST']:
return True
return False
def _create_wsgi_app(self, repo_path, repo_name, config):
if self._is_svn_enabled():
return SimpleSvnApp(config)

View file

@ -28,10 +28,12 @@ import re
import logging
import importlib
from functools import wraps
from StringIO import StringIO
from lxml import etree
import time
from paste.httpheaders import REMOTE_USER, AUTH_TYPE
# TODO(marcink): check if we should use webob.exc here ?
from pyramid.httpexceptions import (
HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
from zope.cachedescriptors.property import Lazy as LazyProperty
@ -43,9 +45,7 @@ from rhodecode.lib import caches
from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
from rhodecode.lib.base import (
BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
from rhodecode.lib.exceptions import (
HTTPLockedRC, HTTPRequirementError, UserCreationError,
NotAllowedToCreateUserError)
from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
from rhodecode.lib.hooks_daemon import prepare_callback_daemon
from rhodecode.lib.middleware import appenlight
from rhodecode.lib.middleware.utils import scm_app_http
@ -53,6 +53,7 @@ from rhodecode.lib.utils import is_valid_repo, SLUG_RE
from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
from rhodecode.lib.vcs.conf import settings as vcs_settings
from rhodecode.lib.vcs.backends import base
from rhodecode.model import meta
from rhodecode.model.db import User, Repository, PullRequest
from rhodecode.model.scm import ScmModel
@ -62,6 +63,28 @@ from rhodecode.model.settings import SettingsModel, VcsSettingsModel
log = logging.getLogger(__name__)
def extract_svn_txn_id(acl_repo_name, data):
"""
Helper method for extraction of svn txn_id from submited XML data during
POST operations
"""
try:
root = etree.fromstring(data)
pat = re.compile(r'/txn/(?P<txn_id>.*)')
for el in root:
if el.tag == '{DAV:}source':
for sub_el in el:
if sub_el.tag == '{DAV:}href':
match = pat.search(sub_el.text)
if match:
svn_tx_id = match.groupdict()['txn_id']
txn_id = caches.compute_key_from_params(
acl_repo_name, svn_tx_id)
return txn_id
except Exception:
log.exception('Failed to extract txn_id')
def initialize_generator(factory):
"""
Initializes the returned generator by draining its first element.
@ -565,48 +588,42 @@ class SimpleVCS(object):
also handles the locking exceptions which will be triggered when
the first chunk is produced by the underlying WSGI application.
"""
callback_daemon, extras = self._prepare_callback_daemon(extras)
config = self._create_config(extras, self.acl_repo_name)
txn_id = ''
if 'CONTENT_LENGTH' in environ and environ['REQUEST_METHOD'] == 'MERGE':
# case for SVN, we want to re-use the callback daemon port
# so we use the txn_id, for this we peek the body, and still save
# it as wsgi.input
data = environ['wsgi.input'].read()
environ['wsgi.input'] = StringIO(data)
txn_id = extract_svn_txn_id(self.acl_repo_name, data)
callback_daemon, extras = self._prepare_callback_daemon(
extras, environ, action, txn_id=txn_id)
log.debug('HOOKS extras is %s', extras)
config = self._create_config(extras, self.acl_repo_name)
app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
app.rc_extras = extras
with callback_daemon:
app.rc_extras = extras
try:
with callback_daemon:
try:
response = app(environ, start_response)
finally:
# This statement works together with the decorator
# "initialize_generator" above. The decorator ensures that
# we hit the first yield statement before the generator is
# returned back to the WSGI server. This is needed to
# ensure that the call to "app" above triggers the
# needed callback to "start_response" before the
# generator is actually used.
yield "__init__"
for chunk in response:
yield chunk
except Exception as exc:
# TODO: martinb: Exceptions are only raised in case of the Pyro4
# backend. Refactor this except block after dropping Pyro4 support.
# TODO: johbo: Improve "translating" back the exception.
if getattr(exc, '_vcs_kind', None) == 'repo_locked':
exc = HTTPLockedRC(*exc.args)
_code = rhodecode.CONFIG.get('lock_ret_code')
log.debug('Repository LOCKED ret code %s!', (_code,))
elif getattr(exc, '_vcs_kind', None) == 'requirement':
log.debug(
'Repository requires features unknown to this Mercurial')
exc = HTTPRequirementError(*exc.args)
else:
raise
for chunk in exc(environ, start_response):
yield chunk
finally:
# invalidate cache on push
try:
response = app(environ, start_response)
finally:
# This statement works together with the decorator
# "initialize_generator" above. The decorator ensures that
# we hit the first yield statement before the generator is
# returned back to the WSGI server. This is needed to
# ensure that the call to "app" above triggers the
# needed callback to "start_response" before the
# generator is actually used.
yield "__init__"
# iter content
for chunk in response:
yield chunk
try:
# invalidate cache on push
if action == 'push':
self._invalidate_cache(self.url_repo_name)
finally:
@ -634,10 +651,18 @@ class SimpleVCS(object):
"""Create a safe config representation."""
raise NotImplementedError()
def _prepare_callback_daemon(self, extras):
def _should_use_callback_daemon(self, extras, environ, action):
return True
def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
direct_calls = vcs_settings.HOOKS_DIRECT_CALLS
if not self._should_use_callback_daemon(extras, environ, action):
# disable callback daemon for actions that don't require it
direct_calls = True
return prepare_callback_daemon(
extras, protocol=vcs_settings.HOOKS_PROTOCOL,
use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
use_direct_calls=direct_calls, txn_id=txn_id)
def _should_check_locking(query_string):

View file

@ -514,7 +514,6 @@ def repo2db_mapper(initial_repo_list, remove_obsolete=False):
:param remove_obsolete: check for obsolete entries in database
"""
from rhodecode.model.repo import RepoModel
from rhodecode.model.scm import ScmModel
from rhodecode.model.repo_group import RepoGroupModel
from rhodecode.model.settings import SettingsModel
@ -566,9 +565,8 @@ def repo2db_mapper(initial_repo_list, remove_obsolete=False):
config = db_repo._config
config.set('extensions', 'largefiles', '')
ScmModel().install_hooks(
db_repo.scm_instance(config=config),
repo_type=db_repo.repo_type)
repo = db_repo.scm_instance(config=config)
repo.install_hooks()
removed = []
if remove_obsolete:

View file

@ -175,6 +175,7 @@ class BaseRepository(object):
EMPTY_COMMIT_ID = '0' * 40
path = None
_remote = None
def __init__(self, repo_path, config=None, create=False, **kwargs):
"""
@ -648,6 +649,9 @@ class BaseRepository(object):
"""
return None
def install_hooks(self, force=False):
return self._remote.install_hooks(force)
class BaseCommit(object):
"""
@ -743,7 +747,7 @@ class BaseCommit(object):
def _get_refs(self):
return {
'branches': [self.branch],
'branches': [self.branch] if self.branch else [],
'bookmarks': getattr(self, 'bookmarks', []),
'tags': self.tags
}

View file

@ -858,7 +858,7 @@ class RepoModel(BaseModel):
repo = backend(
repo_path, config=config, create=True, src_url=clone_uri)
ScmModel().install_hooks(repo, repo_type=repo_type)
repo.install_hooks()
log.debug('Created repo %s with %s backend',
safe_unicode(repo_name), safe_unicode(repo_type))

View file

@ -807,116 +807,6 @@ class ScmModel(BaseModel):
return choices, hist_l
def install_git_hook(self, repo, force_create=False):
"""
Creates a rhodecode hook inside a git repository
:param repo: Instance of VCS repo
:param force_create: Create even if same name hook exists
"""
loc = os.path.join(repo.path, 'hooks')
if not repo.bare:
loc = os.path.join(repo.path, '.git', 'hooks')
if not os.path.isdir(loc):
os.makedirs(loc, mode=0777)
tmpl_post = pkg_resources.resource_string(
'rhodecode', '/'.join(
('config', 'hook_templates', 'git_post_receive.py.tmpl')))
tmpl_pre = pkg_resources.resource_string(
'rhodecode', '/'.join(
('config', 'hook_templates', 'git_pre_receive.py.tmpl')))
for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:
_hook_file = os.path.join(loc, '%s-receive' % h_type)
log.debug('Installing git hook in repo %s', repo)
_rhodecode_hook = _check_rhodecode_hook(_hook_file)
if _rhodecode_hook or force_create:
log.debug('writing %s hook file !', h_type)
try:
with open(_hook_file, 'wb') as f:
tmpl = tmpl.replace('_TMPL_', rhodecode.__version__)
tmpl = tmpl.replace('_ENV_', sys.executable)
f.write(tmpl)
os.chmod(_hook_file, 0755)
except IOError:
log.exception('error writing hook file %s', _hook_file)
else:
log.debug('skipping writing hook file')
def install_svn_hooks(self, repo, force_create=False):
"""
Creates rhodecode hooks inside a svn repository
:param repo: Instance of VCS repo
:param force_create: Create even if same name hook exists
"""
hooks_path = os.path.join(repo.path, 'hooks')
if not os.path.isdir(hooks_path):
os.makedirs(hooks_path)
post_commit_tmpl = pkg_resources.resource_string(
'rhodecode', '/'.join(
('config', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
pre_commit_template = pkg_resources.resource_string(
'rhodecode', '/'.join(
('config', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
templates = {
'post-commit': post_commit_tmpl,
'pre-commit': pre_commit_template
}
for filename in templates:
_hook_file = os.path.join(hooks_path, filename)
_rhodecode_hook = _check_rhodecode_hook(_hook_file)
if _rhodecode_hook or force_create:
log.debug('writing %s hook file !', filename)
template = templates[filename]
try:
with open(_hook_file, 'wb') as f:
template = template.replace(
'_TMPL_', rhodecode.__version__)
template = template.replace('_ENV_', sys.executable)
f.write(template)
os.chmod(_hook_file, 0755)
except IOError:
log.exception('error writing hook file %s', filename)
else:
log.debug('skipping writing hook file')
def install_hooks(self, repo, repo_type):
if repo_type == 'git':
self.install_git_hook(repo)
elif repo_type == 'svn':
self.install_svn_hooks(repo)
def get_server_info(self, environ=None):
server_info = get_system_info(environ)
return server_info
def _check_rhodecode_hook(hook_path):
"""
Check if the hook was created by RhodeCode
"""
if not os.path.exists(hook_path):
return True
log.debug('hook exists, checking if it is from rhodecode')
hook_content = _read_hook(hook_path)
matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content)
if matches:
try:
version = matches.groups()[0]
log.debug('got %s, it is rhodecode', version)
return True
except Exception:
log.exception("Exception while reading the hook version.")
return False
def _read_hook(hook_path):
with open(hook_path, 'rb') as f:
content = f.read()
return content

View file

@ -377,34 +377,6 @@ class TestGenerateVcsResponse(object):
list(result)
assert self.was_cache_invalidated()
@mock.patch('rhodecode.lib.middleware.simplevcs.HTTPLockedRC')
def test_handles_locking_exception(self, http_locked_rc):
result = self.call_controller_with_response_body(
self.raise_result_iter(vcs_kind='repo_locked'))
assert not http_locked_rc.called
# Consume the result
list(result)
assert http_locked_rc.called
@mock.patch('rhodecode.lib.middleware.simplevcs.HTTPRequirementError')
def test_handles_requirement_exception(self, http_requirement):
result = self.call_controller_with_response_body(
self.raise_result_iter(vcs_kind='requirement'))
assert not http_requirement.called
# Consume the result
list(result)
assert http_requirement.called
@mock.patch('rhodecode.lib.middleware.simplevcs.HTTPLockedRC')
def test_handles_locking_exception_in_app_call(self, http_locked_rc):
app_factory_patcher = mock.patch.object(
StubVCSController, '_create_wsgi_app')
with app_factory_patcher as app_factory:
app_factory().side_effect = self.vcs_exception()
result = self.call_controller_with_response_body(['a'])
list(result)
assert http_locked_rc.called
def test_raises_unknown_exceptions(self):
result = self.call_controller_with_response_body(
self.raise_result_iter(vcs_kind='unknown'))
@ -412,7 +384,7 @@ class TestGenerateVcsResponse(object):
list(result)
def test_prepare_callback_daemon_is_called(self):
def side_effect(extras):
def side_effect(extras, environ, action, txn_id=None):
return DummyHooksCallbackDaemon(), extras
prepare_patcher = mock.patch.object(
@ -489,10 +461,11 @@ class TestPrepareHooksDaemon(object):
return_value=(daemon, expected_extras))
with prepare_patcher as prepare_mock:
callback_daemon, extras = controller._prepare_callback_daemon(
expected_extras.copy())
expected_extras.copy(), {}, 'push')
prepare_mock.assert_called_once_with(
expected_extras,
protocol=app_settings['vcs.hooks.protocol'],
txn_id=None,
use_direct_calls=app_settings['vcs.hooks.direct_calls'])
assert callback_daemon == daemon

View file

@ -179,10 +179,12 @@ class TestHttpHooksCallbackDaemon(object):
daemon = hooks_daemon.HttpHooksCallbackDaemon()
assert daemon._daemon == tcp_server
_, port = tcp_server.server_address
expected_uri = '{}:{}'.format(daemon.IP_ADDRESS, port)
msg = 'Preparing HTTP callback daemon at `{}` and ' \
'registering hook object'.format(expected_uri)
assert_message_in_log(
caplog.records,
'Preparing HTTP callback daemon and registering hook object',
levelno=logging.DEBUG, module='hooks_daemon')
caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon')
def test_prepare_inits_hooks_uri_and_logs_it(
self, tcp_server, caplog):
@ -193,8 +195,10 @@ class TestHttpHooksCallbackDaemon(object):
expected_uri = '{}:{}'.format(daemon.IP_ADDRESS, port)
assert daemon.hooks_uri == expected_uri
msg = 'Preparing HTTP callback daemon at `{}` and ' \
'registering hook object'.format(expected_uri)
assert_message_in_log(
caplog.records, 'Hooks uri is: {}'.format(expected_uri),
caplog.records, msg,
levelno=logging.DEBUG, module='hooks_daemon')
def test_run_creates_a_thread(self, tcp_server):
@ -263,7 +267,8 @@ class TestPrepareHooksDaemon(object):
expected_extras.copy(), protocol=protocol, use_direct_calls=True)
assert isinstance(callback, hooks_daemon.DummyHooksCallbackDaemon)
expected_extras['hooks_module'] = 'rhodecode.lib.hooks_daemon'
assert extras == expected_extras
expected_extras['time'] = extras['time']
assert 'extra1' in extras
@pytest.mark.parametrize('protocol, expected_class', (
('http', hooks_daemon.HttpHooksCallbackDaemon),
@ -272,12 +277,15 @@ class TestPrepareHooksDaemon(object):
self, protocol, expected_class):
expected_extras = {
'extra1': 'value1',
'txn_id': 'txnid2',
'hooks_protocol': protocol.lower()
}
callback, extras = hooks_daemon.prepare_callback_daemon(
expected_extras.copy(), protocol=protocol, use_direct_calls=False)
expected_extras.copy(), protocol=protocol, use_direct_calls=False,
txn_id='txnid2')
assert isinstance(callback, expected_class)
hooks_uri = extras.pop('hooks_uri')
extras.pop('hooks_uri')
expected_extras['time'] = extras['time']
assert extras == expected_extras
@pytest.mark.parametrize('protocol', (

View file

@ -245,22 +245,16 @@ def test_repo2db_mapper_enables_largefiles(backend):
repo = backend.create_repo()
repo_list = {repo.repo_name: 'test'}
with mock.patch('rhodecode.model.db.Repository.scm_instance') as scm_mock:
with mock.patch.multiple('rhodecode.model.scm.ScmModel',
install_git_hook=mock.DEFAULT,
install_svn_hooks=mock.DEFAULT):
utils.repo2db_mapper(repo_list, remove_obsolete=False)
_, kwargs = scm_mock.call_args
assert kwargs['config'].get('extensions', 'largefiles') == ''
utils.repo2db_mapper(repo_list, remove_obsolete=False)
_, kwargs = scm_mock.call_args
assert kwargs['config'].get('extensions', 'largefiles') == ''
@pytest.mark.backends("git", "svn")
def test_repo2db_mapper_installs_hooks_for_repos_in_db(backend):
repo = backend.create_repo()
repo_list = {repo.repo_name: 'test'}
with mock.patch.object(ScmModel, 'install_hooks') as install_hooks_mock:
utils.repo2db_mapper(repo_list, remove_obsolete=False)
install_hooks_mock.assert_called_once_with(
repo.scm_instance(), repo_type=backend.alias)
utils.repo2db_mapper(repo_list, remove_obsolete=False)
@pytest.mark.backends("git", "svn")
@ -269,11 +263,7 @@ def test_repo2db_mapper_installs_hooks_for_newly_added_repos(backend):
RepoModel().delete(repo, fs_remove=False)
meta.Session().commit()
repo_list = {repo.repo_name: repo.scm_instance()}
with mock.patch.object(ScmModel, 'install_hooks') as install_hooks_mock:
utils.repo2db_mapper(repo_list, remove_obsolete=False)
assert install_hooks_mock.call_count == 1
install_hooks_args, _ = install_hooks_mock.call_args
assert install_hooks_args[0].name == repo.repo_name
utils.repo2db_mapper(repo_list, remove_obsolete=False)
class TestPasswordChanged(object):

View file

@ -18,10 +18,11 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import tempfile
import os
import mock
import pytest
import tempfile
from rhodecode.lib.exceptions import AttachedForksError
from rhodecode.lib.utils import make_db_config
@ -119,22 +120,22 @@ class TestRepoModel(object):
@pytest.mark.backends("git", "svn")
def test_create_filesystem_repo_installs_hooks(self, tmpdir, backend):
hook_methods = {
'git': 'install_git_hook',
'svn': 'install_svn_hooks'
}
repo = backend.create_repo()
repo_name = repo.repo_name
model = RepoModel()
repo_location = tempfile.mkdtemp()
model.repos_path = repo_location
method = hook_methods[backend.alias]
with mock.patch.object(ScmModel, method) as hooks_mock:
model._create_filesystem_repo(
repo_name, backend.alias, repo_group='', clone_uri=None)
assert hooks_mock.call_count == 1
hook_args, hook_kwargs = hooks_mock.call_args
assert hook_args[0].name == repo_name
repo = model._create_filesystem_repo(
repo_name, backend.alias, repo_group='', clone_uri=None)
hooks = {
'svn': ('pre-commit', 'post-commit'),
'git': ('pre-receive', 'post-receive'),
}
for hook in hooks[backend.alias]:
with open(os.path.join(repo.path, 'hooks', hook)) as f:
data = f.read()
assert 'RC_HOOK_VER' in data
@pytest.mark.parametrize("use_global_config, repo_name_passed", [
(True, False),

View file

@ -194,143 +194,3 @@ def test_get_non_unicode_reference(backend):
u'tag:Ad\xc4\xb1n\xc4\xb1']
assert choices == valid_choices
class TestInstallSvnHooks(object):
HOOK_FILES = ('pre-commit', 'post-commit')
def test_new_hooks_are_created(self, backend_svn):
model = scm.ScmModel()
repo = backend_svn.create_repo()
vcs_repo = repo.scm_instance()
model.install_svn_hooks(vcs_repo)
hooks_path = os.path.join(vcs_repo.path, 'hooks')
assert os.path.isdir(hooks_path)
for file_name in self.HOOK_FILES:
file_path = os.path.join(hooks_path, file_name)
self._check_hook_file_mode(file_path)
self._check_hook_file_content(file_path)
def test_rc_hooks_are_replaced(self, backend_svn):
model = scm.ScmModel()
repo = backend_svn.create_repo()
vcs_repo = repo.scm_instance()
hooks_path = os.path.join(vcs_repo.path, 'hooks')
file_paths = [os.path.join(hooks_path, f) for f in self.HOOK_FILES]
for file_path in file_paths:
self._create_fake_hook(
file_path, content="RC_HOOK_VER = 'abcde'\n")
model.install_svn_hooks(vcs_repo)
for file_path in file_paths:
self._check_hook_file_content(file_path)
def test_non_rc_hooks_are_not_replaced_without_force_create(
self, backend_svn):
model = scm.ScmModel()
repo = backend_svn.create_repo()
vcs_repo = repo.scm_instance()
hooks_path = os.path.join(vcs_repo.path, 'hooks')
file_paths = [os.path.join(hooks_path, f) for f in self.HOOK_FILES]
non_rc_content = "exit 0\n"
for file_path in file_paths:
self._create_fake_hook(file_path, content=non_rc_content)
model.install_svn_hooks(vcs_repo)
for file_path in file_paths:
with open(file_path, 'rt') as hook_file:
content = hook_file.read()
assert content == non_rc_content
def test_non_rc_hooks_are_replaced_with_force_create(self, backend_svn):
model = scm.ScmModel()
repo = backend_svn.create_repo()
vcs_repo = repo.scm_instance()
hooks_path = os.path.join(vcs_repo.path, 'hooks')
file_paths = [os.path.join(hooks_path, f) for f in self.HOOK_FILES]
non_rc_content = "exit 0\n"
for file_path in file_paths:
self._create_fake_hook(file_path, content=non_rc_content)
model.install_svn_hooks(vcs_repo, force_create=True)
for file_path in file_paths:
self._check_hook_file_content(file_path)
def _check_hook_file_mode(self, file_path):
assert os.path.exists(file_path)
stat_info = os.stat(file_path)
file_mode = stat.S_IMODE(stat_info.st_mode)
expected_mode = int('755', 8)
assert expected_mode == file_mode
def _check_hook_file_content(self, file_path):
with open(file_path, 'rt') as hook_file:
content = hook_file.read()
expected_env = '#!{}'.format(sys.executable)
expected_rc_version = "\nRC_HOOK_VER = '{}'\n".format(
rhodecode.__version__)
assert content.strip().startswith(expected_env)
assert expected_rc_version in content
def _create_fake_hook(self, file_path, content):
with open(file_path, 'w') as hook_file:
hook_file.write(content)
class TestCheckRhodecodeHook(object):
@patch('os.path.exists', Mock(return_value=False))
def test_returns_true_when_no_hook_found(self):
result = scm._check_rhodecode_hook('/tmp/fake_hook_file.py')
assert result
@pytest.mark.parametrize("file_content, expected_result", [
("RC_HOOK_VER = '3.3.3'\n", True),
("RC_HOOK = '3.3.3'\n", False),
], ids=no_newline_id_generator)
@patch('os.path.exists', Mock(return_value=True))
def test_signatures(self, file_content, expected_result):
hook_content_patcher = patch.object(
scm, '_read_hook', return_value=file_content)
with hook_content_patcher:
result = scm._check_rhodecode_hook('/tmp/fake_hook_file.py')
assert result is expected_result
class TestInstallHooks(object):
def test_hooks_are_installed_for_git_repo(self, backend_git):
repo = backend_git.create_repo()
model = scm.ScmModel()
scm_repo = repo.scm_instance()
with patch.object(model, 'install_git_hook') as hooks_mock:
model.install_hooks(scm_repo, repo_type='git')
hooks_mock.assert_called_once_with(scm_repo)
def test_hooks_are_installed_for_svn_repo(self, backend_svn):
repo = backend_svn.create_repo()
scm_repo = repo.scm_instance()
model = scm.ScmModel()
with patch.object(scm.ScmModel, 'install_svn_hooks') as hooks_mock:
model.install_hooks(scm_repo, repo_type='svn')
hooks_mock.assert_called_once_with(scm_repo)
@pytest.mark.parametrize('hook_method', [
'install_svn_hooks',
'install_git_hook'])
def test_mercurial_doesnt_trigger_hooks(self, backend_hg, hook_method):
repo = backend_hg.create_repo()
scm_repo = repo.scm_instance()
model = scm.ScmModel()
with patch.object(scm.ScmModel, hook_method) as hooks_mock:
model.install_hooks(scm_repo, repo_type='hg')
assert hooks_mock.call_count == 0