engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

webhooks: added variables into the call URL. Fixes #4211

Browse source
This commit is contained in:
Marcin Kuzminski 2016-10-06 23:20:55 +02:00
parent 0a0255a92d
commit 39879f0dff
5 changed files with 290 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

155
rhodecode/integrations/types/webhook.py
View file

@ -19,13 +19,15 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
from __future__ import unicode_literals
import string
from collections import OrderedDict
import deform
import logging
import requests
import colander
from celery.task import task
from mako.template import Template
from requests.packages.urllib3.util.retry import Retry
from rhodecode import events
from rhodecode.translation import _
@ -33,12 +35,127 @@ from rhodecode.integrations.types.base import IntegrationTypeBase
log = logging.getLogger(__name__)
# updating this required to update the `base_vars` passed in url calling func
WEBHOOK_URL_VARS = [
'repo_name',
'repo_type',
'repo_id',
'repo_url',
# special attrs below that we handle, using multi-call
'branch',
'commit_id',
# pr events vars
'pull_request_id',
'pull_request_url',
]
URL_VARS = ', '.join('${' + x + '}' for x in WEBHOOK_URL_VARS)
class WebhookHandler(object):
def __init__(self, template_url, secret_token):
self.template_url = template_url
self.secret_token = secret_token
def get_base_parsed_template(self, data):
"""
initially parses the passed in template with some common variables
available on ALL calls
"""
# note: make sure to update the `WEBHOOK_URL_VARS` if this changes
common_vars = {
'repo_name': data['repo']['repo_name'],
'repo_type': data['repo']['repo_type'],
'repo_id': data['repo']['repo_id'],
'repo_url': data['repo']['url'],
}
return string.Template(
self.template_url).safe_substitute(**common_vars)
def repo_push_event_handler(self, event, data):
url = self.get_base_parsed_template(data)
url_cals = []
branch_data = OrderedDict()
for obj in data['push']['branches']:
branch_data[obj['name']] = obj
branches_commits = OrderedDict()
for commit in data['push']['commits']:
if commit['branch'] not in branches_commits:
branch_commits = {'branch': branch_data[commit['branch']],
'commits': []}
branches_commits[commit['branch']] = branch_commits
branch_commits = branches_commits[commit['branch']]
branch_commits['commits'].append(commit)
if '${branch}' in url:
# call it multiple times, for each branch if used in variables
for branch, commit_ids in branches_commits.items():
branch_url = string.Template(url).safe_substitute(branch=branch)
# call further down for each commit if used
if '${commit_id}' in branch_url:
for commit_data in commit_ids['commits']:
commit_id = commit_data['raw_id']
commit_url = string.Template(branch_url).safe_substitute(
commit_id=commit_id)
# register per-commit call
log.debug(
'register webhook call(%s) to url %s', event, commit_url)
url_cals.append((commit_url, self.secret_token, data))
else:
# register per-branch call
log.debug(
'register webhook call(%s) to url %s', event, branch_url)
url_cals.append((branch_url, self.secret_token, data))
else:
log.debug(
'register webhook call(%s) to url %s', event, url)
url_cals.append((url, self.secret_token, data))
return url_cals
def repo_create_event_handler(self, event, data):
url = self.get_base_parsed_template(data)
log.debug(
'register webhook call(%s) to url %s', event, url)
return [(url, self.secret_token, data)]
def pull_request_event_handler(self, event, data):
url = self.get_base_parsed_template(data)
log.debug(
'register webhook call(%s) to url %s', event, url)
url = string.Template(url).safe_substitute(
pull_request_id=data['pullrequest']['pull_request_id'],
pull_request_url=data['pullrequest']['url'])
return [(url, self.secret_token, data)]
def __call__(self, event, data):
if isinstance(event, events.RepoPushEvent):
return self.repo_push_event_handler(event, data)
elif isinstance(event, events.RepoCreateEvent):
return self.repo_create_event_handler(event, data)
elif isinstance(event, events.PullRequestEvent):
return self.pull_request_event_handler(event, data)
else:
raise ValueError('event type not supported: %s' % events)
class WebhookSettingsSchema(colander.Schema):
url = colander.SchemaNode(
colander.String(),
title=_('Webhook URL'),
description=_('URL of the webhook to receive POST event.'),
description=
_('URL of the webhook to receive POST event. Following variables '
'are allowed to be used: {vars}. Some of the variables would '
'trigger multiple calls, like ${{branch}} or ${{commit_id}}. '
'Webhook will be called as many times as unique objects in '
'data in such cases.').format(vars=URL_VARS),
missing=colander.required,
required=True,
validator=colander.url,
@ -58,8 +175,6 @@ class WebhookSettingsSchema(colander.Schema):
)
class WebhookIntegrationType(IntegrationTypeBase):
key = 'webhook'
display_name = _('Webhook')
@ -104,14 +219,30 @@ class WebhookIntegrationType(IntegrationTypeBase):
return
data = event.as_dict()
post_to_webhook(data, self.settings)
template_url = self.settings['url']
handler = WebhookHandler(template_url, self.settings['secret_token'])
url_calls = handler(event, data)
log.debug('webhook: calling following urls: %s',
[x[0] for x in url_calls])
post_to_webhook(url_calls)
@task(ignore_result=True)
def post_to_webhook(data, settings):
log.debug('sending event:%s to webhook %s', data['name'], settings['url'])
resp = requests.post(settings['url'], json={
'token': settings['secret_token'],
'event': data
})
resp.raise_for_status() # raise exception on a failed request
def post_to_webhook(url_calls):
max_retries = 3
for url, token, data in url_calls:
# retry max N times
retries = Retry(
total=max_retries,
backoff_factor=0.15,
status_forcelist=[500, 502, 503, 504])
req_session = requests.Session()
req_session.mount(
'http://', requests.adapters.HTTPAdapter(max_retries=retries))
resp = req_session.post(url, json={
'token': token,
'event': data
})
resp.raise_for_status() # raise exception on a failed request

52
rhodecode/tests/integrations/conftest.py Normal file
View file

@ -0,0 +1,52 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2010-2016 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import pytest
from rhodecode import events
@pytest.fixture
def repo_push_event(backend, user_regular):
commits = [
{'message': 'ancestor commit fixes #15'},
{'message': 'quick fixes'},
{'message': 'change that fixes #41, #2'},
{'message': 'this is because 5b23c3532 broke stuff'},
{'message': 'last commit'},
]
commit_ids = backend.create_master_repo(commits).values()
repo = backend.create_repo()
scm_extras = {
'ip': '127.0.0.1',
'username': user_regular.username,
'action': '',
'repository': repo.repo_name,
'scm': repo.scm_instance().alias,
'config': '',
'server_url': 'http://example.com',
'make_lock': None,
'locked_by': [None],
'commit_ids': commit_ids,
}
return events.RepoPushEvent(repo_name=repo.repo_name,
pushed_commit_ids=commit_ids,
extras=scm_extras)

31
rhodecode/tests/integrations/test_slack.py
View file

@ -19,41 +19,12 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import pytest
import requests
from mock import Mock, patch
from mock import patch
from rhodecode import events
from rhodecode.model.db import Session, Integration
from rhodecode.integrations.types.slack import SlackIntegrationType
@pytest.fixture
def repo_push_event(backend, user_regular):
commits = [
{'message': 'ancestor commit fixes #15'},
{'message': 'quick fixes'},
{'message': 'change that fixes #41, #2'},
{'message': 'this is because 5b23c3532 broke stuff'},
{'message': 'last commit'},
]
commit_ids = backend.create_master_repo(commits).values()
repo = backend.create_repo()
scm_extras = {
'ip': '127.0.0.1',
'username': user_regular.username,
'action': '',
'repository': repo.repo_name,
'scm': repo.scm_instance().alias,
'config': '',
'server_url': 'http://example.com',
'make_lock': None,
'locked_by': [None],
'commit_ids': commit_ids,
}
return events.RepoPushEvent(repo_name=repo.repo_name,
pushed_commit_ids=commit_ids,
extras=scm_extras)
@pytest.fixture
def slack_settings():

93
rhodecode/tests/integrations/test_webhook.py Normal file
View file

@ -0,0 +1,93 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2010-2016 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import pytest
from rhodecode import events
from rhodecode.lib.utils2 import AttributeDict
from rhodecode.integrations.types.webhook import WebhookHandler
@pytest.fixture
def base_data():
return {
'repo': {
'repo_name': 'foo',
'repo_type': 'hg',
'repo_id': '12',
'url': 'http://repo.url/foo'
}
}
def test_webhook_parse_url_invalid_event():
template_url = 'http://server.com/${repo_name}/build'
handler = WebhookHandler(template_url, 'secret_token')
with pytest.raises(ValueError) as err:
handler(events.RepoDeleteEvent(''), {})
assert str(err.value).startswith('event type not supported')
@pytest.mark.parametrize('template,expected_urls', [
('http://server.com/${repo_name}/build', ['http://server.com/foo/build']),
('http://server.com/${repo_name}/${repo_type}', ['http://server.com/foo/hg']),
('http://${server}.com/${repo_name}/${repo_id}', ['http://${server}.com/foo/12']),
('http://server.com/${branch}/build', ['http://server.com/${branch}/build']),
])
def test_webook_parse_url_for_create_event(base_data, template, expected_urls):
handler = WebhookHandler(template, 'secret_token')
urls = handler(events.RepoCreateEvent(''), base_data)
assert urls == [(url, 'secret_token', base_data) for url in expected_urls]
@pytest.mark.parametrize('template,expected_urls', [
('http://server.com/${repo_name}/${pull_request_id}', ['http://server.com/foo/999']),
('http://server.com/${repo_name}/${pull_request_url}', ['http://server.com/foo/http://pr-url.com']),
])
def test_webook_parse_url_for_pull_request_event(base_data, template, expected_urls):
base_data['pullrequest'] = {
'pull_request_id': 999,
'url': 'http://pr-url.com',
}
handler = WebhookHandler(template, 'secret_token')
urls = handler(events.PullRequestCreateEvent(
AttributeDict({'target_repo': 'foo'})), base_data)
assert urls == [(url, 'secret_token', base_data) for url in expected_urls]
@pytest.mark.parametrize('template,expected_urls', [
('http://server.com/${branch}/build', ['http://server.com/stable/build',
'http://server.com/dev/build']),
('http://server.com/${branch}/${commit_id}', ['http://server.com/stable/stable-xxx',
'http://server.com/stable/stable-yyy',
'http://server.com/dev/dev-xxx',
'http://server.com/dev/dev-yyy']),
])
def test_webook_parse_url_for_push_event(pylonsapp, repo_push_event, base_data, template, expected_urls):
base_data['push'] = {
'branches': [{'name': 'stable'}, {'name': 'dev'}],
'commits': [{'branch': 'stable', 'raw_id': 'stable-xxx'},
{'branch': 'stable', 'raw_id': 'stable-yyy'},
{'branch': 'dev', 'raw_id': 'dev-xxx'},
{'branch': 'dev', 'raw_id': 'dev-yyy'}]
}
handler = WebhookHandler(template, 'secret_token')
urls = handler(repo_push_event, base_data)
assert urls == [(url, 'secret_token', base_data) for url in expected_urls]

1
rhodecode/translation.py
View file

@ -21,6 +21,7 @@ from pyramid.i18n import TranslationStringFactory
# Create a translation string factory for the 'rhodecode' domain.
_ = TranslationStringFactory('rhodecode')
class LazyString(object):
def __init__(self, *args, **kw):
self.args = args