engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

integrations: add integration support

Browse source 
* added db migration for integration table, version 55
* added integrations views/templates/models
* added slack integration
* added h.url.current() support for pyramid only contexts
* added issues extractor for commit messages
* renamed repo-deleted/repo-created to repo-delete/repo-create
* removed marshmallow
* added serialization for events to static dicts of data
* added lazy_ugettext for translation of start time string
This commit is contained in:
Daniel Dourvaris 2016-07-10 06:43:20 +03:00
parent 4f7a3ca6e8
commit 7ff09cda95
36 changed files with 5099 additions and 127 deletions
Download patch file Download diff file Expand all files Collapse all files

15
pkgs/python-packages.nix
View file

@ -831,19 +831,6 @@
license = [ pkgs.lib.licenses.bsdOriginal ];
};
};
marshmallow = super.buildPythonPackage {
name = "marshmallow-2.8.0";
buildInputs = with self; [];
doCheck = false;
propagatedBuildInputs = with self; [];
src = fetchurl {
url = "https://pypi.python.org/packages/4f/64/9393d77847d86981c84b88bbea627d30ff71b5ab1402636b366f73737817/marshmallow-2.8.0.tar.gz";
md5 = "204513fc123a3d9bdd7b63b9747f02e6";
};
meta = {
license = [ pkgs.lib.licenses.mit ];
};
};
mccabe = super.buildPythonPackage {
name = "mccabe-0.3";
buildInputs = with self; [];
@ -1368,7 +1355,7 @@
name = "rhodecode-enterprise-ce-4.3.0";
buildInputs = with self; [WebTest configobj cssselect flake8 lxml mock pytest pytest-cov pytest-runner];
doCheck = true;
propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments Pylons Pyro4 Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery colander decorator docutils gunicorn infrae.cache ipython iso8601 kombu marshmallow msgpack-python packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt];
propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments Pylons Pyro4 Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery colander decorator docutils gunicorn infrae.cache ipython iso8601 kombu msgpack-python packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt];
src = ./.;
meta = {
license = [ { fullName = "AGPLv3, and Commercial License"; } ];

1
requirements.txt
View file

@ -84,7 +84,6 @@ iso8601==0.1.11
itsdangerous==0.24
kombu==1.5.1
lxml==3.4.4
marshmallow==2.8.0
mccabe==0.3
meld3==1.0.2
mock==1.0.1

2
rhodecode/__init__.py
View file

@ -47,7 +47,7 @@ CONFIG = {}
EXTENSIONS = {}
__version__ = ('.'.join((str(each) for each in VERSION[:3])))
__dbversion__ = 54 # defines current db version for migrations
__dbversion__ = 55 # defines current db version for migrations
__platform__ = platform.system()
__license__ = 'AGPLv3, and Commercial License'
__author__ = 'RhodeCode GmbH'

2
rhodecode/admin/navigation.py
View file

@ -80,6 +80,8 @@ class NavigationRegistry(object):
NavEntry('email', _('Email'), 'admin_settings_email'),
NavEntry('hooks', _('Hooks'), 'admin_settings_hooks'),
NavEntry('search', _('Full Text Search'), 'admin_settings_search'),
NavEntry('integrations', _('Integrations'),
'global_integrations_home', pyramid=True),
NavEntry('system', _('System Info'), 'admin_settings_system'),
NavEntry('open_source', _('Open Source Licenses'),
'admin_settings_open_source', pyramid=True),

1
rhodecode/config/middleware.py
View file

@ -213,6 +213,7 @@ def includeme(config):
config.include('pyramid_beaker')
config.include('rhodecode.admin')
config.include('rhodecode.authentication')
config.include('rhodecode.integrations')
config.include('rhodecode.login')
config.include('rhodecode.tweens')
config.include('rhodecode.api')

13
rhodecode/config/routing.py
View file

@ -51,6 +51,19 @@ URL_NAME_REQUIREMENTS = {
}
def add_route_requirements(route_path, requirements):
"""
Adds regex requirements to pyramid routes using a mapping dict
>>> add_route_requirements('/{action}/{id}', {'id': r'\d+'})
'/{action}/{id:\d+}'
"""
for key, regex in requirements.items():
route_path = route_path.replace('{%s}' % key, '{%s:%s}' % (key, regex))
return route_path
class JSRoutesMapper(Mapper):
"""
Wrapper for routes.Mapper to make pyroutes compatible url definitions

19
rhodecode/events/__init__.py
View file

@ -19,7 +19,7 @@
from pyramid.threadlocal import get_current_registry
def trigger(event):
def trigger(event, registry=None):
"""
Helper method to send an event. This wraps the pyramid logic to send an
event.
@ -27,9 +27,18 @@ def trigger(event):
# For the first step we are using pyramids thread locals here. If the
# event mechanism works out as a good solution we should think about
# passing the registry as an argument to get rid of it.
registry = get_current_registry()
registry = registry or get_current_registry()
registry.notify(event)
# Until we can work around the problem that VCS operations do not have a
# pyramid context to work with, we send the events to integrations directly
# Later it will be possible to use regular pyramid subscribers ie:
# config.add_subscriber(integrations_event_handler, RhodecodeEvent)
from rhodecode.integrations import integrations_event_handler
if isinstance(event, RhodecodeEvent):
integrations_event_handler(event)
from rhodecode.events.base import RhodecodeEvent
@ -41,8 +50,8 @@ from rhodecode.events.user import (
from rhodecode.events.repo import (
RepoEvent,
RepoPreCreateEvent, RepoCreatedEvent,
RepoPreDeleteEvent, RepoDeletedEvent,
RepoPreCreateEvent, RepoCreateEvent,
RepoPreDeleteEvent, RepoDeleteEvent,
RepoPrePushEvent, RepoPushEvent,
RepoPrePullEvent, RepoPullEvent,
)
@ -54,4 +63,4 @@ from rhodecode.events.pullrequest import (
PullRequestReviewEvent,
PullRequestMergeEvent,
PullRequestCloseEvent,
)
)

30
rhodecode/events/base.py
View file

@ -17,7 +17,6 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
from datetime import datetime
from marshmallow import Schema, fields
from pyramid.threadlocal import get_current_request
from rhodecode.lib.utils2 import AttributeDict
@ -28,29 +27,10 @@ SYSTEM_USER = AttributeDict(dict(
))
class UserSchema(Schema):
"""
Marshmallow schema for a user
"""
username = fields.Str()
class RhodecodeEventSchema(Schema):
"""
Marshmallow schema for a rhodecode event
"""
utc_timestamp = fields.DateTime()
actor = fields.Nested(UserSchema)
actor_ip = fields.Str()
name = fields.Str()
class RhodecodeEvent(object):
"""
Base event class for all Rhodecode events
"""
MarshmallowSchema = RhodecodeEventSchema
def __init__(self):
self.request = get_current_request()
self.utc_timestamp = datetime.utcnow()
@ -68,4 +48,12 @@ class RhodecodeEvent(object):
return '<no ip available>'
def as_dict(self):
return self.MarshmallowSchema().dump(self).data
data = {
'name': self.name,
'utc_timestamp': self.utc_timestamp,
'actor_ip': self.actor_ip,
'actor': {
'username': self.actor.username
}
}
return data

46
rhodecode/events/pullrequest.py
View file

@ -16,44 +16,39 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
from marshmallow import Schema, fields
from rhodecode.translation import lazy_ugettext
from rhodecode.events.repo import RepoEvent
def get_pull_request_url(pull_request):
from rhodecode.model.pull_request import PullRequestModel
return PullRequestModel().get_url(pull_request)
class PullRequestSchema(Schema):
"""
Marshmallow schema for a pull request
"""
pull_request_id = fields.Integer()
url = fields.Function(get_pull_request_url)
title = fields.Str()
class PullRequestEventSchema(RepoEvent.MarshmallowSchema):
"""
Marshmallow schema for a pull request event
"""
pullrequest = fields.Nested(PullRequestSchema)
class PullRequestEvent(RepoEvent):
"""
Base class for pull request events.
:param pullrequest: a :class:`PullRequest` instance
"""
MarshmallowSchema = PullRequestEventSchema
def __init__(self, pullrequest):
super(PullRequestEvent, self).__init__(pullrequest.target_repo)
self.pullrequest = pullrequest
def as_dict(self):
from rhodecode.model.pull_request import PullRequestModel
data = super(PullRequestEvent, self).as_dict()
commits = self._commits_as_dict(self.pullrequest.revisions)
issues = self._issues_as_dict(commits)
data.update({
'pullrequest': {
'title': self.pullrequest.title,
'issues': issues,
'pull_request_id': self.pullrequest.pull_request_id,
'url': PullRequestModel().get_url(self.pullrequest)
}
})
return data
class PullRequestCreateEvent(PullRequestEvent):
"""
@ -61,6 +56,7 @@ class PullRequestCreateEvent(PullRequestEvent):
request is created.
"""
name = 'pullrequest-create'
display_name = lazy_ugettext('pullrequest created')
class PullRequestCloseEvent(PullRequestEvent):
@ -69,6 +65,7 @@ class PullRequestCloseEvent(PullRequestEvent):
request is closed.
"""
name = 'pullrequest-close'
display_name = lazy_ugettext('pullrequest closed')
class PullRequestUpdateEvent(PullRequestEvent):
@ -77,6 +74,7 @@ class PullRequestUpdateEvent(PullRequestEvent):
request is updated.
"""
name = 'pullrequest-update'
display_name = lazy_ugettext('pullrequest updated')
class PullRequestMergeEvent(PullRequestEvent):
@ -85,6 +83,7 @@ class PullRequestMergeEvent(PullRequestEvent):
request is merged.
"""
name = 'pullrequest-merge'
display_name = lazy_ugettext('pullrequest merged')
class PullRequestReviewEvent(PullRequestEvent):
@ -93,5 +92,6 @@ class PullRequestReviewEvent(PullRequestEvent):
request is reviewed.
"""
name = 'pullrequest-review'
display_name = lazy_ugettext('pullrequest reviewed')

122
rhodecode/events/repo.py
View file

@ -16,31 +16,13 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
from marshmallow import Schema, fields
import logging
from rhodecode.translation import lazy_ugettext
from rhodecode.model.db import User, Repository, Session
from rhodecode.events.base import RhodecodeEvent
def get_repo_url(repo):
from rhodecode.model.repo import RepoModel
return RepoModel().get_url(repo)
class RepositorySchema(Schema):
"""
Marshmallow schema for a repository
"""
repo_id = fields.Integer()
repo_name = fields.Str()
url = fields.Function(get_repo_url)
class RepoEventSchema(RhodecodeEvent.MarshmallowSchema):
"""
Marshmallow schema for a repository event
"""
repo = fields.Nested(RepositorySchema)
log = logging.getLogger()
class RepoEvent(RhodecodeEvent):
@ -49,12 +31,68 @@ class RepoEvent(RhodecodeEvent):
:param repo: a :class:`Repository` instance
"""
MarshmallowSchema = RepoEventSchema
def __init__(self, repo):
super(RepoEvent, self).__init__()
self.repo = repo
def as_dict(self):
from rhodecode.model.repo import RepoModel
data = super(RepoEvent, self).as_dict()
data.update({
'repo': {
'repo_id': self.repo.repo_id,
'repo_name': self.repo.repo_name,
'url': RepoModel().get_url(self.repo)
}
})
return data
def _commits_as_dict(self, commit_ids):
""" Helper function to serialize commit_ids """
from rhodecode.lib.utils2 import extract_mentioned_users
from rhodecode.model.db import Repository
from rhodecode.lib import helpers as h
from rhodecode.lib.helpers import process_patterns
from rhodecode.lib.helpers import urlify_commit_message
if not commit_ids:
return []
commits = []
reviewers = []
vcs_repo = self.repo.scm_instance(cache=False)
try:
for commit_id in commit_ids:
cs = vcs_repo.get_changeset(commit_id)
cs_data = cs.__json__()
cs_data['mentions'] = extract_mentioned_users(cs_data['message'])
cs_data['reviewers'] = reviewers
cs_data['url'] = h.url('changeset_home',
repo_name=self.repo.repo_name,
revision=cs_data['raw_id'],
qualified=True
)
urlified_message, issues_data = process_patterns(
cs_data['message'], self.repo.repo_name)
cs_data['issues'] = issues_data
cs_data['message_html'] = urlify_commit_message(cs_data['message'],
self.repo.repo_name)
commits.append(cs_data)
except Exception as e:
log.exception(e)
# we don't send any commits when crash happens, only full list matters
# we short circuit then.
return []
return commits
def _issues_as_dict(self, commits):
""" Helper function to serialize issues from commits """
issues = {}
for commit in commits:
for issue in commit['issues']:
issues[issue['id']] = issue
return issues
class RepoPreCreateEvent(RepoEvent):
"""
@ -62,14 +100,16 @@ class RepoPreCreateEvent(RepoEvent):
created.
"""
name = 'repo-pre-create'
display_name = lazy_ugettext('repository pre create')
class RepoCreatedEvent(RepoEvent):
class RepoCreateEvent(RepoEvent):
"""
An instance of this class is emitted as an :term:`event` whenever a repo is
created.
"""
name = 'repo-created'
name = 'repo-create'
display_name = lazy_ugettext('repository created')
class RepoPreDeleteEvent(RepoEvent):
@ -78,14 +118,16 @@ class RepoPreDeleteEvent(RepoEvent):
created.
"""
name = 'repo-pre-delete'
display_name = lazy_ugettext('repository pre delete')
class RepoDeletedEvent(RepoEvent):
class RepoDeleteEvent(RepoEvent):
"""
An instance of this class is emitted as an :term:`event` whenever a repo is
created.
"""
name = 'repo-deleted'
name = 'repo-delete'
display_name = lazy_ugettext('repository deleted')
class RepoVCSEvent(RepoEvent):
@ -116,6 +158,7 @@ class RepoPrePullEvent(RepoVCSEvent):
are pulled from a repo.
"""
name = 'repo-pre-pull'
display_name = lazy_ugettext('repository pre pull')
class RepoPullEvent(RepoVCSEvent):
@ -124,6 +167,7 @@ class RepoPullEvent(RepoVCSEvent):
are pulled from a repo.
"""
name = 'repo-pull'
display_name = lazy_ugettext('repository pull')
class RepoPrePushEvent(RepoVCSEvent):
@ -132,6 +176,7 @@ class RepoPrePushEvent(RepoVCSEvent):
are pushed to a repo.
"""
name = 'repo-pre-push'
display_name = lazy_ugettext('repository pre push')
class RepoPushEvent(RepoVCSEvent):
@ -142,8 +187,33 @@ class RepoPushEvent(RepoVCSEvent):
:param extras: (optional) dict of data from proxied VCS actions
"""
name = 'repo-push'
display_name = lazy_ugettext('repository push')
def __init__(self, repo_name, pushed_commit_ids, extras):
super(RepoPushEvent, self).__init__(repo_name, extras)
self.pushed_commit_ids = pushed_commit_ids
def as_dict(self):
data = super(RepoPushEvent, self).as_dict()
branch_url = repo_url = data['repo']['url']
commits = self._commits_as_dict(self.pushed_commit_ids)
issues = self._issues_as_dict(commits)
branches = set(
commit['branch'] for commit in commits if commit['branch'])
branches = [
{
'name': branch,
'url': '{}/changelog?branch={}'.format(
data['repo']['url'], branch)
}
for branch in branches
]
data['push'] = {
'commits': commits,
'issues': issues,
'branches': branches,
}
return data

10
rhodecode/events/user.py
View file

@ -18,6 +18,7 @@
from zope.interface import implementer
from rhodecode.translation import lazy_ugettext
from rhodecode.events.base import RhodecodeEvent
from rhodecode.events.interfaces import (
IUserRegistered, IUserPreCreate, IUserPreUpdate)
@ -29,6 +30,9 @@ class UserRegistered(RhodecodeEvent):
An instance of this class is emitted as an :term:`event` whenever a user
account is registered.
"""
name = 'user-register'
display_name = lazy_ugettext('user registered')
def __init__(self, user, session):
self.user = user
self.session = session
@ -40,6 +44,9 @@ class UserPreCreate(RhodecodeEvent):
An instance of this class is emitted as an :term:`event` before a new user
object is created.
"""
name = 'user-pre-create'
display_name = lazy_ugettext('user pre create')
def __init__(self, user_data):
self.user_data = user_data
@ -50,6 +57,9 @@ class UserPreUpdate(RhodecodeEvent):
An instance of this class is emitted as an :term:`event` before a user
object is updated.
"""
name = 'user-pre-update'
display_name = lazy_ugettext('user pre update')
def __init__(self, user, user_data):
self.user = user
self.user_data = user_data

52
rhodecode/integrations/__init__.py Normal file
View file

@ -0,0 +1,52 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2012-2016 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
from rhodecode.integrations.registry import IntegrationTypeRegistry
from rhodecode.integrations.types import slack
log = logging.getLogger(__name__)
# TODO: dan: This is currently global until we figure out what to do about
# VCS's not having a pyramid context - move it to pyramid app configuration
# includeme level later to allow per instance integration setup
integration_type_registry = IntegrationTypeRegistry()
integration_type_registry.register_integration_type(slack.SlackIntegrationType)
def integrations_event_handler(event):
"""
Takes an event and passes it to all enabled integrations
"""
from rhodecode.model.integration import IntegrationModel
integration_model = IntegrationModel()
integrations = integration_model.get_for_event(event)
for integration in integrations:
try:
integration_model.send_event(integration, event)
except Exception:
log.exception(
'failure occured when sending event %s to integration %s' % (
event, integration))
def includeme(config):
config.include('rhodecode.integrations.routes')

37
rhodecode/integrations/registry.py Normal file
View file

@ -0,0 +1,37 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2012-2016 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
log = logging.getLogger()
class IntegrationTypeRegistry(dict):
"""
Registry Class to hold IntegrationTypes
"""
def register_integration_type(self, IntegrationType):
key = IntegrationType.key
if key in self:
log.warning(
'Overriding existing integration type %s (%s) with %s' % (
self[key], key, IntegrationType))
self[key] = IntegrationType

133
rhodecode/integrations/routes.py Normal file
View file

@ -0,0 +1,133 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2012-2016 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
from rhodecode.model.db import Repository, Integration
from rhodecode.config.routing import (
ADMIN_PREFIX, add_route_requirements, URL_NAME_REQUIREMENTS)
from rhodecode.integrations import integration_type_registry
log = logging.getLogger(__name__)
def includeme(config):
config.add_route('global_integrations_home',
ADMIN_PREFIX + '/integrations')
config.add_route('global_integrations_list',
ADMIN_PREFIX + '/integrations/{integration}')
for route_name in ['global_integrations_home', 'global_integrations_list']:
config.add_view('rhodecode.integrations.views.GlobalIntegrationsView',
attr='index',
renderer='rhodecode:templates/admin/integrations/list.html',
request_method='GET',
route_name=route_name)
config.add_route('global_integrations_create',
ADMIN_PREFIX + '/integrations/{integration}/new',
custom_predicates=(valid_integration,))
config.add_route('global_integrations_edit',
ADMIN_PREFIX + '/integrations/{integration}/{integration_id}',
custom_predicates=(valid_integration,))
for route_name in ['global_integrations_create', 'global_integrations_edit']:
config.add_view('rhodecode.integrations.views.GlobalIntegrationsView',
attr='settings_get',
renderer='rhodecode:templates/admin/integrations/edit.html',
request_method='GET',
route_name=route_name)
config.add_view('rhodecode.integrations.views.GlobalIntegrationsView',
attr='settings_post',
renderer='rhodecode:templates/admin/integrations/edit.html',
request_method='POST',
route_name=route_name)
config.add_route('repo_integrations_home',
add_route_requirements(
'{repo_name}/settings/integrations',
URL_NAME_REQUIREMENTS
),
custom_predicates=(valid_repo,))
config.add_route('repo_integrations_list',
add_route_requirements(
'{repo_name}/settings/integrations/{integration}',
URL_NAME_REQUIREMENTS
),
custom_predicates=(valid_repo, valid_integration))
for route_name in ['repo_integrations_home', 'repo_integrations_list']:
config.add_view('rhodecode.integrations.views.RepoIntegrationsView',
attr='index',
request_method='GET',
route_name=route_name)
config.add_route('repo_integrations_create',
add_route_requirements(
'{repo_name}/settings/integrations/{integration}/new',
URL_NAME_REQUIREMENTS
),
custom_predicates=(valid_repo, valid_integration))
config.add_route('repo_integrations_edit',
add_route_requirements(
'{repo_name}/settings/integrations/{integration}/{integration_id}',
URL_NAME_REQUIREMENTS
),
custom_predicates=(valid_repo, valid_integration))
for route_name in ['repo_integrations_edit', 'repo_integrations_create']:
config.add_view('rhodecode.integrations.views.RepoIntegrationsView',
attr='settings_get',
renderer='rhodecode:templates/admin/integrations/edit.html',
request_method='GET',
route_name=route_name)
config.add_view('rhodecode.integrations.views.RepoIntegrationsView',
attr='settings_post',
renderer='rhodecode:templates/admin/integrations/edit.html',
request_method='POST',
route_name=route_name)
def valid_repo(info, request):
repo = Repository.get_by_repo_name(info['match']['repo_name'])
if repo:
return True
def valid_integration(info, request):
integration_type = info['match']['integration']
integration_id = info['match'].get('integration_id')
repo_name = info['match'].get('repo_name')
if integration_type not in integration_type_registry:
return False
repo = None
if repo_name:
repo = Repository.get_by_repo_name(info['match']['repo_name'])
if not repo:
return False
if integration_id:
integration = Integration.get(integration_id)
if not integration:
return False
if integration.integration_type != integration_type:
return False
if repo and repo.repo_id != integration.repo_id:
return False
return True

48
rhodecode/integrations/schema.py Normal file
View file

@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2012-2016 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import colander
from rhodecode.translation import lazy_ugettext
class IntegrationSettingsSchemaBase(colander.MappingSchema):
"""
This base schema is intended for use in integrations.
It adds a few default settings (e.g., "enabled"), so that integration
authors don't have to maintain a bunch of boilerplate.
"""
enabled = colander.SchemaNode(
colander.Bool(),
default=True,
description=lazy_ugettext('Enable or disable this integration.'),
missing=False,
title=lazy_ugettext('Enabled'),
widget='bool',
)
name = colander.SchemaNode(
colander.String(),
description=lazy_ugettext('Short name for this integration.'),
missing=colander.required,
title=lazy_ugettext('Integration name'),
widget='string',
)

19
rhodecode/integrations/types/__init__.py Normal file
View file

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2012-2016 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/

43
rhodecode/integrations/types/base.py Normal file
View file

@ -0,0 +1,43 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2012-2016 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
from rhodecode.integrations.schema import IntegrationSettingsSchemaBase
class IntegrationTypeBase(object):
""" Base class for IntegrationType plugins """
def __init__(self, settings):
"""
:param settings: dict of settings to be used for the integration
"""
self.settings = settings
@classmethod
def settings_schema(cls):
"""
A colander schema of settings for the integration type
Subclasses can return their own schema but should always
inherit from IntegrationSettingsSchemaBase
"""
return IntegrationSettingsSchemaBase()

199
rhodecode/integrations/types/slack.py Normal file
View file

@ -0,0 +1,199 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2012-2016 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
from __future__ import unicode_literals
import re
import logging
import requests
import colander
from celery.task import task
from mako.template import Template
from rhodecode import events
from rhodecode.translation import lazy_ugettext
from rhodecode.lib import helpers as h
from rhodecode.lib.celerylib import run_task
from rhodecode.lib.colander_utils import strip_whitespace
from rhodecode.integrations.types.base import IntegrationTypeBase
from rhodecode.integrations.schema import IntegrationSettingsSchemaBase
log = logging.getLogger()
class SlackSettingsSchema(IntegrationSettingsSchemaBase):
service = colander.SchemaNode(
colander.String(),
title=lazy_ugettext('Slack service URL'),
description=h.literal(lazy_ugettext(
'This can be setup at the '
'<a href="https://my.slack.com/services/new/incoming-webhook/">'
'slack app manager</a>')),
default='',
placeholder='https://hooks.slack.com/services/...',
preparer=strip_whitespace,
validator=colander.url,
widget='string'
)
username = colander.SchemaNode(
colander.String(),
title=lazy_ugettext('Username'),
description=lazy_ugettext('Username to show notifications coming from.'),
missing='Rhodecode',
preparer=strip_whitespace,
widget='string',
placeholder='Rhodecode'
)
channel = colander.SchemaNode(
colander.String(),
title=lazy_ugettext('Channel'),
description=lazy_ugettext('Channel to send notifications to.'),
missing='',
preparer=strip_whitespace,
widget='string',
placeholder='#general'
)
icon_emoji = colander.SchemaNode(
colander.String(),
title=lazy_ugettext('Emoji'),
description=lazy_ugettext('Emoji to use eg. :studio_microphone:'),
missing='',
preparer=strip_whitespace,
widget='string',
placeholder=':studio_microphone:'
)
repo_push_template = Template(r'''
*${data['actor']['username']}* pushed to \
%if data['push']['branches']:
${len(data['push']['branches']) > 1 and 'branches' or 'branch'} \
${', '.join('<%s|%s>' % (branch['url'], branch['name']) for branch in data['push']['branches'])} \
%else:
unknown branch \
%endif
in <${data['repo']['url']}|${data['repo']['repo_name']}>
>>>
%for commit in data['push']['commits']:
<${commit['url']}|${commit['short_id']}> - ${commit['message_html']|html_to_slack_links}
%endfor
''')
class SlackIntegrationType(IntegrationTypeBase):
key = 'slack'
display_name = lazy_ugettext('Slack')
SettingsSchema = SlackSettingsSchema
valid_events = [
events.PullRequestCloseEvent,
events.PullRequestMergeEvent,
events.PullRequestUpdateEvent,
events.PullRequestReviewEvent,
events.PullRequestCreateEvent,
events.RepoPushEvent,
events.RepoCreateEvent,
]
def send_event(self, event):
if event.__class__ not in self.valid_events:
log.debug('event not valid: %r' % event)
return
if event.name not in self.settings['events']:
log.debug('event ignored: %r' % event)
return
data = event.as_dict()
text = '*%s* caused a *%s* event' % (
data['actor']['username'], event.name)
if isinstance(event, events.PullRequestEvent):
text = self.format_pull_request_event(event, data)
elif isinstance(event, events.RepoPushEvent):
text = self.format_repo_push_event(data)
elif isinstance(event, events.RepoCreateEvent):
text = self.format_repo_create_event(data)
else:
log.error('unhandled event type: %r' % event)
run_task(post_text_to_slack, self.settings, text)
@classmethod
def settings_schema(cls):
schema = SlackSettingsSchema()
schema.add(colander.SchemaNode(
colander.Set(),
widget='checkbox_list',
choices=sorted([e.name for e in cls.valid_events]),
description="Events activated for this integration",
default=[e.name for e in cls.valid_events],
name='events'
))
return schema
def format_pull_request_event(self, event, data):
action = {
events.PullRequestCloseEvent: 'closed',
events.PullRequestMergeEvent: 'merged',
events.PullRequestUpdateEvent: 'updated',
events.PullRequestReviewEvent: 'reviewed',
events.PullRequestCreateEvent: 'created',
}.get(event.__class__, '<unknown action>')
return ('Pull request <{url}|#{number}> ({title}) '
'{action} by {user}').format(
user=data['actor']['username'],
number=data['pullrequest']['pull_request_id'],
url=data['pullrequest']['url'],
title=data['pullrequest']['title'],
action=action
)
def format_repo_push_event(self, data):
result = repo_push_template.render(
data=data,
html_to_slack_links=html_to_slack_links,
)
return result
def format_repo_create_msg(self, data):
return '<{}|{}> ({}) repository created by *{}*'.format(
data['repo']['url'],
data['repo']['repo_name'],
data['repo']['repo_type'],
data['actor']['username'],
)
def html_to_slack_links(message):
return re.compile(r'<a .*?href=["\'](.+?)".*?>(.+?)</a>').sub(
r'<\1|\2>', message)
@task(ignore_result=True)
def post_text_to_slack(settings, text):
resp = requests.post(settings['service'], json={
"channel": settings.get('channel', ''),
"username": settings.get('username', 'Rhodecode'),
"text": text,
"icon_emoji": settings.get('icon_emoji', ':studio_microphone:')
})
resp.raise_for_status() # raise exception on a failed request

257
rhodecode/integrations/views.py Normal file
View file

@ -0,0 +1,257 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2012-2016 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import colander
import logging
import pylons
from pyramid.httpexceptions import HTTPFound, HTTPForbidden
from pyramid.renderers import render
from pyramid.response import Response
from rhodecode.lib import auth
from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
from rhodecode.model.db import Repository, Session, Integration
from rhodecode.model.scm import ScmModel
from rhodecode.model.integration import IntegrationModel
from rhodecode.admin.navigation import navigation_list
from rhodecode.translation import _
from rhodecode.integrations import integration_type_registry
log = logging.getLogger(__name__)
class IntegrationSettingsViewBase(object):
""" Base Integration settings view used by both repo / global settings """
def __init__(self, context, request):
self.context = context
self.request = request
self._load_general_context()
if not self.perm_check(request.user):
raise HTTPForbidden()
def _load_general_context(self):
"""
This avoids boilerplate for repo/global+list/edit+views/templates
by doing all possible contexts at the same time however it should
be split up into separate functions once more "contexts" exist
"""
self.IntegrationType = None
self.repo = None
self.integration = None
self.integrations = {}
request = self.request
if 'repo_name' in request.matchdict: # we're in a repo context
repo_name = request.matchdict['repo_name']
self.repo = Repository.get_by_repo_name(repo_name)
if 'integration' in request.matchdict: # we're in integration context
integration_type = request.matchdict['integration']
self.IntegrationType = integration_type_registry[integration_type]
if 'integration_id' in request.matchdict: # single integration context
integration_id = request.matchdict['integration_id']
self.integration = Integration.get(integration_id)
else: # list integrations context
for integration in IntegrationModel().get_integrations(self.repo):
self.integrations.setdefault(integration.integration_type, []
).append(integration)
self.settings = self.integration and self.integration.settings or {}
def _template_c_context(self):
# TODO: dan: this is a stopgap in order to inherit from current pylons
# based admin/repo settings templates - this should be removed entirely
# after port to pyramid
c = pylons.tmpl_context
c.active = 'integrations'
c.rhodecode_user = self.request.user
c.repo = self.repo
c.repo_name = self.repo and self.repo.repo_name or None
if self.repo:
c.repo_info = self.repo
c.rhodecode_db_repo = self.repo
c.repository_pull_requests = ScmModel().get_pull_requests(self.repo)
else:
c.navlist = navigation_list(self.request)
return c
def _form_schema(self):
return self.IntegrationType.settings_schema()
def settings_get(self, defaults=None, errors=None):
"""
View that displays the plugin settings as a form.
"""
defaults = defaults or {}
errors = errors or {}
schema = self._form_schema()
if not defaults:
if self.integration:
defaults['enabled'] = self.integration.enabled
defaults['name'] = self.integration.name
else:
if self.repo:
scope = self.repo.repo_name
else:
scope = _('Global')
defaults['name'] = '{} {} integration'.format(scope,
self.IntegrationType.display_name)
defaults['enabled'] = True
for node in schema:
setting = self.settings.get(node.name)
if setting is not None:
defaults.setdefault(node.name, setting)
else:
if node.default:
defaults.setdefault(node.name, node.default)
template_context = {
'defaults': defaults,
'errors': errors,
'schema': schema,
'current_IntegrationType': self.IntegrationType,
'integration': self.integration,
'settings': self.settings,
'resource': self.context,
'c': self._template_c_context(),
}
return template_context
@auth.CSRFRequired()
def settings_post(self):
"""
View that validates and stores the plugin settings.
"""
if self.request.params.get('delete'):
Session().delete(self.integration)
Session().commit()
self.request.session.flash(
_('Integration {integration_name} deleted successfully.').format(
integration_name=self.integration.name),
queue='success')
if self.repo:
redirect_to = self.request.route_url(
'repo_integrations_home', repo_name=self.repo.repo_name)
else:
redirect_to = self.request.route_url('global_integrations_home')
raise HTTPFound(redirect_to)
schema = self._form_schema()
params = {}
for node in schema.children:
if type(node.typ) in (colander.Set, colander.List):
val = self.request.params.getall(node.name)
else:
val = self.request.params.get(node.name)
if val:
params[node.name] = val
try:
valid_data = schema.deserialize(params)
except colander.Invalid, e:
# Display error message and display form again.
self.request.session.flash(
_('Errors exist when saving plugin settings. '
'Please check the form inputs.'),
queue='error')
return self.settings_get(errors=e.asdict(), defaults=params)
if not self.integration:
self.integration = Integration(
integration_type=self.IntegrationType.key)
if self.repo:
self.integration.repo = self.repo
Session.add(self.integration)
self.integration.enabled = valid_data.pop('enabled', False)
self.integration.name = valid_data.pop('name')
self.integration.settings = valid_data
Session.commit()
# Display success message and redirect.
self.request.session.flash(
_('Integration {integration_name} updated successfully.').format(
integration_name=self.IntegrationType.display_name,
queue='success'))
if self.repo:
redirect_to = self.request.route_url(
'repo_integrations_edit', repo_name=self.repo.repo_name,
integration=self.integration.integration_type,
integration_id=self.integration.integration_id)
else:
redirect_to = self.request.route_url(
'global_integrations_edit',
integration=self.integration.integration_type,
integration_id=self.integration.integration_id)
return HTTPFound(redirect_to)
def index(self):
current_integrations = self.integrations
if self.IntegrationType:
current_integrations = {
self.IntegrationType.key: self.integrations.get(
self.IntegrationType.key, [])
}
template_context = {
'current_IntegrationType': self.IntegrationType,
'current_integrations': current_integrations,
'current_integration': 'none',
'available_integrations': integration_type_registry,
'c': self._template_c_context()
}
if self.repo:
html = render('rhodecode:templates/admin/integrations/list.html',
template_context,
request=self.request)
else:
html = render('rhodecode:templates/admin/integrations/list.html',
template_context,
request=self.request)
return Response(html)
class GlobalIntegrationsView(IntegrationSettingsViewBase):
def perm_check(self, user):
return auth.HasPermissionAll('hg.admin').check_permissions(user=user)
class RepoIntegrationsView(IntegrationSettingsViewBase):
def perm_check(self, user):
return auth.HasRepoPermissionAll('repository.admin'
)(repo_name=self.repo.repo_name, user=user)

3516
rhodecode/lib/dbmigrate/schema/db_4_3_0_0.py Normal file
View file

File diff suppressed because it is too large Load diff

27
rhodecode/lib/dbmigrate/versions/055_version_4_3_0.py Normal file
View file

@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
import logging
import sqlalchemy as sa
from alembic.migration import MigrationContext
from alembic.operations import Operations
from rhodecode.lib.dbmigrate.versions import _reset_base
log = logging.getLogger(__name__)
def upgrade(migrate_engine):
"""
Upgrade operations go here.
Don't create your own engine; bind migrate_engine to your metadata
"""
_reset_base(migrate_engine)
from rhodecode.lib.dbmigrate.schema import db_4_3_0_0
integrations_table = db_4_3_0_0.Integration.__table__
integrations_table.create()
def downgrade(migrate_engine):
pass

59
rhodecode/lib/helpers.py
View file

@ -44,7 +44,7 @@ from pygments.formatters.html import HtmlFormatter
from pygments import highlight as code_highlight
from pygments.lexers import (
get_lexer_by_name, get_lexer_for_filename, get_lexer_for_mimetype)
from pylons import url
from pylons import url as pylons_url
from pylons.i18n.translation import _, ungettext
from pyramid.threadlocal import get_current_request
@ -88,6 +88,22 @@ log = logging.getLogger(__name__)
DEFAULT_USER = User.DEFAULT_USER
DEFAULT_USER_EMAIL = User.DEFAULT_USER_EMAIL
def url(*args, **kw):
return pylons_url(*args, **kw)
def pylons_url_current(*args, **kw):
"""
This function overrides pylons.url.current() which returns the current
path so that it will also work from a pyramid only context. This
should be removed once port to pyramid is complete.
"""
if not args and not kw:
request = get_current_request()
return request.path
return pylons_url.current(*args, **kw)
url.current = pylons_url_current
def html_escape(text, html_escape_table=None):
"""Produce entities within text."""
@ -1614,7 +1630,7 @@ def urlify_commits(text_, repository):
'pref': pref,
'cls': 'revision-link',
'url': url('changeset_home', repo_name=repository,
revision=commit_id),
revision=commit_id, qualified=True),
'commit_id': commit_id,
'suf': suf
}
@ -1624,7 +1640,8 @@ def urlify_commits(text_, repository):
return newtext
def _process_url_func(match_obj, repo_name, uid, entry):
def _process_url_func(match_obj, repo_name, uid, entry,
return_raw_data=False):
pref = ''
if match_obj.group().startswith(' '):
pref = ' '
@ -1650,7 +1667,7 @@ def _process_url_func(match_obj, repo_name, uid, entry):
named_vars.update(match_obj.groupdict())
_url = string.Template(entry['url']).safe_substitute(**named_vars)
return tmpl % {
data = {
'pref': pref,
'cls': 'issue-tracker-link',
'url': _url,
@ -1658,9 +1675,15 @@ def _process_url_func(match_obj, repo_name, uid, entry):
'issue-prefix': entry['pref'],
'serv': entry['url'],
}
if return_raw_data:
return {
'id': issue_id,
'url': _url
}
return tmpl % data
def process_patterns(text_string, repo_name, config):
def process_patterns(text_string, repo_name, config=None):
repo = None
if repo_name:
# Retrieving repo_name to avoid invalid repo_name to explode on
@ -1670,11 +1693,9 @@ def process_patterns(text_string, repo_name, config):
settings_model = IssueTrackerSettingsModel(repo=repo)
active_entries = settings_model.get_settings(cache=True)
issues_data = []
newtext = text_string
for uid, entry in active_entries.items():
url_func = partial(
_process_url_func, repo_name=repo_name, entry=entry, uid=uid)
log.debug('found issue tracker entry with uid %s' % (uid,))
if not (entry['pat'] and entry['url']):
@ -1692,10 +1713,20 @@ def process_patterns(text_string, repo_name, config):
entry['pat'])
continue
data_func = partial(
_process_url_func, repo_name=repo_name, entry=entry, uid=uid,
return_raw_data=True)
for match_obj in pattern.finditer(text_string):
issues_data.append(data_func(match_obj))
url_func = partial(
_process_url_func, repo_name=repo_name, entry=entry, uid=uid)
newtext = pattern.sub(url_func, newtext)
log.debug('processed prefix:uid `%s`' % (uid,))
return newtext
return newtext, issues_data
def urlify_commit_message(commit_text, repository=None):
@ -1707,22 +1738,22 @@ def urlify_commit_message(commit_text, repository=None):
:param repository:
"""
from pylons import url # doh, we need to re-import url to mock it later
from rhodecode import CONFIG
def escaper(string):
return string.replace('<', '&lt;').replace('>', '&gt;')
newtext = escaper(commit_text)
# extract http/https links and make them real urls
newtext = urlify_text(newtext, safe=False)
# urlify commits - extract commit ids and make link out of them, if we have
# the scope of repository present.
if repository:
newtext = urlify_commits(newtext, repository)
# extract http/https links and make them real urls
newtext = urlify_text(newtext, safe=False)
# process issue tracker patterns
newtext = process_patterns(newtext, repository or '', CONFIG)
newtext, issues = process_patterns(newtext, repository or '')
return literal(newtext)

18
rhodecode/lib/hooks_daemon.py
View file

@ -20,11 +20,15 @@
import json
import logging
import urlparse
import threading
from BaseHTTPServer import BaseHTTPRequestHandler
from SocketServer import TCPServer
from routes.util import URLGenerator
import Pyro4
import pylons
import rhodecode
from rhodecode.lib import hooks_base
from rhodecode.lib.utils2 import AttributeDict
@ -236,6 +240,17 @@ class Hooks(object):
def _call_hook(self, hook, extras):
extras = AttributeDict(extras)
netloc = urlparse.urlparse(extras.server_url).netloc
environ = {
'SERVER_NAME': netloc.split(':')[0],
'SERVER_PORT': ':' in netloc and netloc.split(':')[1] or '80',
'SCRIPT_NAME': '',
'PATH_INFO': '/',
'HTTP_HOST': 'localhost',
'REQUEST_METHOD': 'GET',
}
pylons_router = URLGenerator(rhodecode.CONFIG['routes.map'], environ)
pylons.url._push_object(pylons_router)
try:
result = hook(extras)
@ -248,6 +263,9 @@ class Hooks(object):
'exception': type(error).__name__,
'exception_args': error_args,
}
finally:
pylons.url._pop_object()
return {
'status': result.status,
'output': result.output,

39
rhodecode/model/db.py
View file

@ -3475,3 +3475,42 @@ class ExternalIdentity(Base, BaseModel):
query = cls.query()
query = query.filter(cls.local_user_id == local_user_id)
return query
class Integration(Base, BaseModel):
__tablename__ = 'integrations'
__table_args__ = (
{'extend_existing': True, 'mysql_engine': 'InnoDB',
'mysql_charset': 'utf8', 'sqlite_autoincrement': True}
)
integration_id = Column('integration_id', Integer(), primary_key=True)
integration_type = Column('integration_type', String(255))
enabled = Column("enabled", Boolean(), nullable=False)
name = Column('name', String(255), nullable=False)
settings_json = Column('settings_json',
UnicodeText().with_variant(UnicodeText(16384), 'mysql'))
repo_id = Column(
"repo_id", Integer(), ForeignKey('repositories.repo_id'),
nullable=True, unique=None, default=None)
repo = relationship('Repository', lazy='joined')
@hybrid_property
def settings(self):
data = json.loads(self.settings_json or '{}')
return data
@settings.setter
def settings(self, dct):
self.settings_json = json.dumps(dct, indent=2)
def __repr__(self):
if self.repo:
scope = 'repo=%r' % self.repo
else:
scope = 'global'
return '<Integration(%r, %r)>' % (self.integration_type, scope)
def settings_as_dict(self):
return json.loads(self.settings_json)

118
rhodecode/model/integration.py Normal file
View file

@ -0,0 +1,118 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2011-2016 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
"""
Model for integrations
"""
import logging
import traceback
from pylons import tmpl_context as c
from pylons.i18n.translation import _, ungettext
from sqlalchemy import or_
from sqlalchemy.sql.expression import false, true
from mako import exceptions
import rhodecode
from rhodecode import events
from rhodecode.lib import helpers as h
from rhodecode.lib.caching_query import FromCache
from rhodecode.lib.utils import PartialRenderer
from rhodecode.model import BaseModel
from rhodecode.model.db import Integration, User
from rhodecode.model.meta import Session
from rhodecode.integrations import integration_type_registry
log = logging.getLogger(__name__)
class IntegrationModel(BaseModel):
cls = Integration
def __get_integration(self, integration):
if isinstance(integration, Integration):
return integration
elif isinstance(integration, (int, long)):
return self.sa.query(Integration).get(integration)
else:
if integration:
raise Exception('integration must be int, long or Instance'
' of Integration got %s' % type(integration))
def delete(self, integration):
try:
integration = self.__get_integration(integration)
if integration:
Session().delete(integration)
return True
except Exception:
log.error(traceback.format_exc())
raise
return False
def get_integration_handler(self, integration):
TypeClass = integration_type_registry.get(integration.integration_type)
if not TypeClass:
log.error('No class could be found for integration type: {}'.format(
integration.integration_type))
return None
return TypeClass(integration.settings)
def send_event(self, integration, event):
""" Send an event to an integration """
handler = self.get_integration_handler(integration)
if handler:
handler.send_event(event)
def get_integrations(self, repo=None):
if repo:
return self.sa.query(Integration).filter(
Integration.repo_id==repo.repo_id).all()
# global integrations
return self.sa.query(Integration).filter(
Integration.repo_id==None).all()
def get_for_event(self, event, cache=False):
"""
Get integrations that match an event
"""
query = self.sa.query(Integration).filter(Integration.enabled==True)
if isinstance(event, events.RepoEvent): # global + repo integrations
query = query.filter(
or_(Integration.repo_id==None,
Integration.repo_id==event.repo.repo_id))
if cache:
query = query.options(FromCache(
"sql_cache_short",
"get_enabled_repo_integrations_%i" % event.repo.repo_id))
else: # only global integrations
query = query.filter(Integration.repo_id==None)
if cache:
query = query.options(FromCache(
"sql_cache_short", "get_enabled_global_integrations"))
return query.all()

4
rhodecode/model/repo.py
View file

@ -535,7 +535,7 @@ class RepoModel(BaseModel):
# we need to flush here, in order to check if database won't
# throw any exceptions, create filesystem dirs at the very end
self.sa.flush()
events.trigger(events.RepoCreatedEvent(new_repo))
events.trigger(events.RepoCreateEvent(new_repo))
return new_repo
except Exception:
@ -653,7 +653,7 @@ class RepoModel(BaseModel):
'deleted_on': time.time(),
})
log_delete_repository(**old_repo_dict)
events.trigger(events.RepoDeletedEvent(repo))
events.trigger(events.RepoDeleteEvent(repo))
except Exception:
log.error(traceback.format_exc())
raise

40
rhodecode/templates/admin/integrations/base.html Normal file
View file

@ -0,0 +1,40 @@
## -*- coding: utf-8 -*-
<%!
def inherit(context):
if context['c'].repo:
return "/admin/repos/repo_edit.html"
else:
return "/admin/settings/settings.html"
%>
<%inherit file="${inherit(context)}" />
<%def name="title()">
${_('Integrations settings')}
%if c.rhodecode_name:
&middot; ${h.branding(c.rhodecode_name)}
%endif
</%def>
<%def name="breadcrumbs_links()">
${h.link_to(_('Admin'),h.url('admin_home'))}
&raquo;
${_('Integrations')}
</%def>
<%def name="menu_bar_nav()">
%if c.repo:
${self.menu_items(active='repositories')}
%else:
${self.menu_items(active='admin')}
%endif
</%def>
<%def name="menu_bar_subnav()">
%if c.repo:
${self.repo_menu(active='options')}
%endif
</%def>
<%def name="main_content()">
${next.body()}
</%def>

108
rhodecode/templates/admin/integrations/edit.html Normal file
View file

@ -0,0 +1,108 @@
## -*- coding: utf-8 -*-
<%inherit file="base.html"/>
<%def name="breadcrumbs_links()">
%if c.repo:
${h.link_to('Settings',h.url('edit_repo', repo_name=c.repo.repo_name))}
&raquo;
${h.link_to(_('Integrations'),request.route_url(route_name='repo_integrations_home', repo_name=c.repo.repo_name))}
&raquo;
${h.link_to(current_IntegrationType.display_name,
request.route_url(route_name='repo_integrations_list',
repo_name=c.repo.repo_name,
integration=current_IntegrationType.key))}
%else:
${h.link_to(_('Admin'),h.url('admin_home'))}
&raquo;
${h.link_to(_('Settings'),h.url('admin_settings'))}
&raquo;
${h.link_to(_('Integrations'),request.route_url(route_name='global_integrations_home'))}
&raquo;
${h.link_to(current_IntegrationType.display_name,
request.route_url(route_name='global_integrations_list',
integration=current_IntegrationType.key))}
%endif
%if integration:
&raquo;
${integration.name}
%endif
</%def>
<div class="panel panel-default">
<div class="panel-heading">
<h2 class="panel-title">
%if integration:
${current_IntegrationType.display_name} - ${integration.name}
%else:
${_('Create new %(integration_type)s integration') % {'integration_type': current_IntegrationType.display_name}}
%endif
</h2>
</div>
<div class="fields panel-body">
${h.secure_form(request.url)}
<div class="form">
%for node in schema:
<% label_css_class = ("label-checkbox" if (node.widget == "bool") else "") %>
<div class="field">
<div class="label ${label_css_class}"><label for="${node.name}">${node.title}</label></div>
<div class="input">
%if node.widget in ["string", "int", "unicode"]:
${h.text(node.name, defaults.get(node.name), class_="medium", placeholder=hasattr(node, 'placeholder') and node.placeholder or '')}
%elif node.widget in ["text"]:
${h.textarea(node.name, defaults.get(node.name), class_="medium", placeholder=hasattr(node, 'placeholder') and node.placeholder or '')}
%elif node.widget == "password":
${h.password(node.name, defaults.get(node.name), class_="medium")}
%elif node.widget == "bool":
<div class="checkbox">${h.checkbox(node.name, True, checked=defaults.get(node.name))}</div>
%elif node.widget == "select":
${h.select(node.name, defaults.get(node.name), node.choices)}
%elif node.widget == "checkbox_list":
%for i, choice in enumerate(node.choices):
<%
name, value = choice, choice
if isinstance(choice, tuple):
choice, name = choice
%>
<div>
<input id="${node.name}-${choice}"
name="${node.name}"
value="${value}"
type="checkbox"
${value in defaults.get(node.name, []) and 'checked' or ''}>
<label for="${node.name}-${value}">
${name}
</label>
</div>
%endfor
%elif node.widget == "readonly":
${node.default}
%else:
This field is of type ${node.typ}, which cannot be displayed. Must be one of [string|int|bool|select|password|text|checkbox_list].
%endif
%if node.name in errors:
<span class="error-message">${errors.get(node.name)}</span>
<br />
%endif
<p class="help-block">${node.description}</p>
</div>
</div>
%endfor
## Allow derived templates to add something below the form
## input fields
%if hasattr(next, 'below_form_fields'):
${next.below_form_fields()}
%endif
<div class="buttons">
${h.submit('save',_('Save'),class_="btn")}
%if integration:
${h.submit('delete',_('Delete'),class_="btn btn-danger")}
%endif
</div>
</div>
${h.end_form()}
</div>
</div>

147
rhodecode/templates/admin/integrations/list.html Normal file
View file

@ -0,0 +1,147 @@
## -*- coding: utf-8 -*-
<%inherit file="base.html"/>
<%def name="breadcrumbs_links()">
%if c.repo:
${h.link_to('Settings',h.url('edit_repo', repo_name=c.repo.repo_name))}
%else:
${h.link_to(_('Admin'),h.url('admin_home'))}
&raquo;
${h.link_to(_('Settings'),h.url('admin_settings'))}
%endif
%if current_IntegrationType:
&raquo;
%if c.repo:
${h.link_to(_('Integrations'),
request.route_url(route_name='repo_integrations_home',
repo_name=c.repo.repo_name))}
%else:
${h.link_to(_('Integrations'),
request.route_url(route_name='global_integrations_home'))}
%endif
&raquo;
${current_IntegrationType.display_name}
%else:
&raquo;
${_('Integrations')}
%endif
</%def>
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">${_('Create new integration')}</h3>
</div>
<div class="panel-body">
%if not available_integrations:
No integrations available
%else:
%for integration in available_integrations:
<%
if c.repo:
create_url = request.route_url('repo_integrations_create',
repo_name=c.repo.repo_name,
integration=integration)
else:
create_url = request.route_url('global_integrations_create',
integration=integration)
%>
<a href="${create_url}" class="btn">
${integration}
</a>
%endfor
%endif
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">${_('Current integrations')}</h3>
</div>
<div class="panel-body">
<table class="rctable issuetracker">
<thead>
<tr>
<th>${_('Enabled')}</th>
<th>${_('Description')}</th>
<th>${_('Type')}</th>
<th>${_('Actions')}</th>
<th ></th>
</tr>
</thead>
<tbody>
%for integration_type, integrations in sorted(current_integrations.items()):
%for integration in sorted(integrations, key=lambda x: x.name):
<tr id="integration_${integration.integration_id}">
<td class="td-enabled">
%if integration.enabled:
<div class="flag_status approved pull-left"></div>
%else:
<div class="flag_status rejected pull-left"></div>
%endif
</td>
<td class="td-description">
${integration.name}
</td>
<td class="td-regex">
${integration.integration_type}
</td>
<td class="td-action">
%if integration_type not in available_integrations:
${_('unknown integration')}
%else:
<%
if c.repo:
edit_url = request.route_url('repo_integrations_edit',
repo_name=c.repo.repo_name,
integration=integration.integration_type,
integration_id=integration.integration_id)
else:
edit_url = request.route_url('global_integrations_edit',
integration=integration.integration_type,
integration_id=integration.integration_id)
%>
<div class="grid_edit">
<a href="${edit_url}">${_('Edit')}</a>
</div>
<div class="grid_delete">
<a href="${edit_url}"
class="btn btn-link btn-danger delete_integration_entry"
data-desc="${integration.name}"
data-uid="${integration.integration_id}">
${_('Delete')}
</a>
</div>
%endif
</td>
</tr>
%endfor
%endfor
<tr id="last-row"></tr>
</tbody>
</table>
</div>
</div>
<script type="text/javascript">
var delete_integration = function(entry) {
if (confirm("Confirm to remove this integration: "+$(entry).data('desc'))) {
var request = $.ajax({
type: "POST",
url: $(entry).attr('href'),
data: {
'delete': 'delete',
'csrf_token': CSRF_TOKEN
},
success: function(){
location.reload();
},
error: function(data, textStatus, errorThrown){
alert("Error while deleting entry.\nError code {0} ({1}). URL: {2}".format(data.status,data.statusText,$(entry)[0].url));
}
});
};
}
$('.delete_integration_entry').on('click', function(e){
e.preventDefault();
delete_integration(this);
});
</script>

11
rhodecode/templates/admin/repos/repo_edit.html
View file

@ -23,6 +23,10 @@
${self.repo_menu(active='options')}
</%def>
<%def name="main_content()">
<%include file="/admin/repos/repo_edit_${c.active}.html"/>
</%def>
<%def name="main()">
<div class="box">
@ -64,14 +68,17 @@
<li class="${'active' if c.active=='statistics' else ''}">
<a href="${h.url('edit_repo_statistics', repo_name=c.repo_name)}">${_('Statistics')}</a>
</li>
<li class="${'active' if c.active=='integrations' else ''}">
<a href="${h.route_path('repo_integrations_home', repo_name=c.repo_name)}">${_('Integrations')}</a>
</li>
</ul>
</div>
<div class="main-content-full-width">
<%include file="/admin/repos/repo_edit_${c.active}.html"/>
${self.main_content()}
</div>
</div>
</div>
</%def>
</%def>

22
rhodecode/templates/admin/settings/settings.html
View file

@ -18,6 +18,18 @@
${self.menu_items(active='admin')}
</%def>
<%def name="side_bar_nav()">
% for navitem in c.navlist:
<li class="${'active' if c.active==navitem.key else ''}">
<a href="${navitem.url}">${navitem.name}</a>
</li>
% endfor
</%def>
<%def name="main_content()">
<%include file="/admin/settings/settings_${c.active}.html"/>
</%def>
<%def name="main()">
<div class="box">
<div class="title">
@ -28,18 +40,14 @@
<div class='sidebar-col-wrapper'>
<div class="sidebar">
<ul class="nav nav-pills nav-stacked">
% for navitem in c.navlist:
<li class="${'active' if c.active==navitem.key else ''}">
<a href="${navitem.url}">${navitem.name}</a>
</li>
% endfor
${self.side_bar_nav()}
</ul>
</div>
<div class="main-content-full-width">
<%include file="/admin/settings/settings_${c.active}.html"/>
${self.main_content()}
</div>
</div>
</div>
</%def>
</%def>

2
rhodecode/templates/base/base.html
View file

@ -16,7 +16,6 @@
<!-- MENU BAR NAV -->
${self.menu_bar_nav()}
<!-- END MENU BAR NAV -->
${self.body()}
</div>
</div>
${self.menu_bar_subnav()}
@ -82,6 +81,7 @@
<li><a href="${h.url('users_groups')}">${_('User groups')}</a></li>
<li><a href="${h.url('admin_permissions_application')}">${_('Permissions')}</a></li>
<li><a href="${h.route_path('auth_home', traverse='')}">${_('Authentication')}</a></li>
<li><a href="${h.route_path('global_integrations_home')}">${_('Integrations')}</a></li>
<li><a href="${h.url('admin_defaults_repositories')}">${_('Defaults')}</a></li>
<li class="last"><a href="${h.url('admin_settings')}">${_('Settings')}</a></li>
</ul>

12
rhodecode/tests/events/test_repo.py
View file

@ -27,8 +27,8 @@ from rhodecode.model.repo import RepoModel
from rhodecode.events.repo import (
RepoPrePullEvent, RepoPullEvent,
RepoPrePushEvent, RepoPushEvent,
RepoPreCreateEvent, RepoCreatedEvent,
RepoPreDeleteEvent, RepoDeletedEvent,
RepoPreCreateEvent, RepoCreateEvent,
RepoPreDeleteEvent, RepoDeleteEvent,
)
@ -51,8 +51,8 @@ def scm_extras(user_regular, repo_stub):
# TODO: dan: make the serialization tests complete json comparisons
@pytest.mark.parametrize('EventClass', [
RepoPreCreateEvent, RepoCreatedEvent,
RepoPreDeleteEvent, RepoDeletedEvent,
RepoPreCreateEvent, RepoCreateEvent,
RepoPreDeleteEvent, RepoDeleteEvent,
])
def test_repo_events_serialized(repo_stub, EventClass):
event = EventClass(repo_stub)
@ -88,11 +88,11 @@ def test_vcs_repo_push_event_serialize(repo_stub, scm_extras, EventClass):
def test_create_delete_repo_fires_events(backend):
with EventCatcher() as event_catcher:
repo = backend.create_repo()
assert event_catcher.events_types == [RepoPreCreateEvent, RepoCreatedEvent]
assert event_catcher.events_types == [RepoPreCreateEvent, RepoCreateEvent]
with EventCatcher() as event_catcher:
RepoModel().delete(repo)
assert event_catcher.events_types == [RepoPreDeleteEvent, RepoDeletedEvent]
assert event_catcher.events_types == [RepoPreDeleteEvent, RepoDeleteEvent]
def test_pull_fires_events(scm_extras):

40
rhodecode/tests/lib/test_helpers.py
View file

@ -69,6 +69,40 @@ def test_format_binary():
assert helpers.format_byte_size_binary(298489462784) == '278.0 GiB'
@pytest.mark.parametrize('text_string, pattern, expected', [
('No issue here', '(?:#)(?P<issue_id>\d+)', []),
('Fix #42', '(?:#)(?P<issue_id>\d+)',
[{'url': 'http://r.io/{repo}/i/42', 'id': '42'}]),
('Fix #42, #53', '(?:#)(?P<issue_id>\d+)', [
{'url': 'http://r.io/{repo}/i/42', 'id': '42'},
{'url': 'http://r.io/{repo}/i/53', 'id': '53'}]),
('Fix #42', '(?:#)?<issue_id>\d+)', []), # Broken regex
])
def test_extract_issues(backend, text_string, pattern, expected):
repo = backend.create_repo()
config = {
'123': {
'uid': '123',
'pat': pattern,
'url': 'http://r.io/${repo}/i/${issue_id}',
'pref': '#',
}
}
def get_settings_mock(self, cache=True):
return config
with mock.patch.object(IssueTrackerSettingsModel,
'get_settings', get_settings_mock):
text, issues = helpers.process_patterns(text_string, repo.repo_name)
expected = copy.deepcopy(expected)
for item in expected:
item['url'] = item['url'].format(repo=repo.repo_name)
assert issues == expected
@pytest.mark.parametrize('text_string, pattern, expected_text', [
('Fix #42', '(?:#)(?P<issue_id>\d+)',
'Fix <a class="issue-tracker-link" href="http://r.io/{repo}/i/42">#42</a>'
@ -90,7 +124,7 @@ def test_process_patterns_repo(backend, text_string, pattern, expected_text):
with mock.patch.object(IssueTrackerSettingsModel,
'get_settings', get_settings_mock):
processed_text = helpers.process_patterns(
processed_text, issues = helpers.process_patterns(
text_string, repo.repo_name, config)
assert processed_text == expected_text.format(repo=repo.repo_name)
@ -116,7 +150,7 @@ def test_process_patterns_no_repo(text_string, pattern, expected_text):
with mock.patch.object(IssueTrackerSettingsModel,
'get_global_settings', get_settings_mock):
processed_text = helpers.process_patterns(
processed_text, issues = helpers.process_patterns(
text_string, '', config)
assert processed_text == expected_text
@ -140,7 +174,7 @@ def test_process_patterns_non_existent_repo_name(backend):
with mock.patch.object(IssueTrackerSettingsModel,
'get_global_settings', get_settings_mock):
processed_text = helpers.process_patterns(
processed_text, issues = helpers.process_patterns(
text_string, 'do-not-exist', config)
assert processed_text == expected_text

13
rhodecode/translation.py
View file

@ -20,3 +20,16 @@ from pyramid.i18n import TranslationStringFactory
# Create a translation string factory for the 'rhodecode' domain.
_ = TranslationStringFactory('rhodecode')
class LazyString(object):
def __init__(self, *args, **kw):
self.args = args
self.kw = kw
def __str__(self):
return _(*self.args, **self.kw)
def lazy_ugettext(*args, **kw):
""" Lazily evaluated version of _() """
return LazyString(*args, **kw)

1
setup.py
View file

@ -78,7 +78,6 @@ requirements = [
'ipython',
'iso8601',
'kombu',
'marshmallow',
'msgpack-python',
'packaging',
'psycopg2',