engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

integrations: parse pushed tags, and lightweight tags for git.

Browse source 
- now aggregated as 'tags' key
- handles the case for email/webhook integrations
This commit is contained in:
Marcin Kuzminski 2017-12-03 22:37:36 +01:00
parent b816475595
commit 87fecd1bf7
8 changed files with 272 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

94
rhodecode/events/repo.py
View file

@ -18,6 +18,7 @@
import collections
import logging
import datetime
from rhodecode.translation import lazy_ugettext
from rhodecode.model.db import User, Repository, Session
@ -58,22 +59,60 @@ def _commits_as_dict(event, commit_ids, repos):
return commits # return early if we have the commits we need
vcs_repo = repo.scm_instance(cache=False)
try:
# use copy of needed_commits since we modify it while iterating
for commit_id in list(needed_commits):
try:
cs = vcs_repo.get_changeset(commit_id)
except CommitDoesNotExistError:
continue # maybe its in next repo
if commit_id.startswith('tag=>'):
raw_id = commit_id[5:]
cs_data = {
'raw_id': commit_id, 'short_id': commit_id,
'branch': None,
'git_ref_change': 'tag_add',
'message': 'Added new tag {}'.format(raw_id),
'author': event.actor.full_contact,
'date': datetime.datetime.now(),
'refs': {
'branches': [],
'bookmarks': [],
'tags': []
}
}
commits.append(cs_data)
elif commit_id.startswith('delete_branch=>'):
raw_id = commit_id[15:]
cs_data = {
'raw_id': commit_id, 'short_id': commit_id,
'branch': None,
'git_ref_change': 'branch_delete',
'message': 'Deleted branch {}'.format(raw_id),
'author': event.actor.full_contact,
'date': datetime.datetime.now(),
'refs': {
'branches': [],
'bookmarks': [],
'tags': []
}
}
commits.append(cs_data)
else:
try:
cs = vcs_repo.get_changeset(commit_id)
except CommitDoesNotExistError:
continue # maybe its in next repo
cs_data = cs.__json__()
cs_data['refs'] = cs._get_refs()
cs_data = cs.__json__()
cs_data['refs'] = cs._get_refs()
cs_data['mentions'] = extract_mentioned_users(cs_data['message'])
cs_data['reviewers'] = reviewers
cs_data['url'] = RepoModel().get_commit_url(
repo, cs_data['raw_id'], request=event.request)
cs_data['permalink_url'] = RepoModel().get_commit_url(
repo, cs_data['raw_id'], request=event.request, permalink=True)
repo, cs_data['raw_id'], request=event.request,
permalink=True)
urlified_message, issues_data = process_patterns(
cs_data['message'], repo.repo_name)
cs_data['issues'] = issues_data
@ -85,8 +124,8 @@ def _commits_as_dict(event, commit_ids, repos):
needed_commits.remove(commit_id)
except Exception as e:
log.exception(e)
except Exception:
log.exception('Failed to extract commits data')
# we don't send any commits when crash happens, only full list
# matters we short circuit then.
return []
@ -248,6 +287,7 @@ class RepoPushEvent(RepoVCSEvent):
def __init__(self, repo_name, pushed_commit_ids, extras):
super(RepoPushEvent, self).__init__(repo_name, extras)
self.pushed_commit_ids = pushed_commit_ids
self.new_refs = extras.new_refs
def as_dict(self):
data = super(RepoPushEvent, self).as_dict()
@ -256,6 +296,10 @@ class RepoPushEvent(RepoVCSEvent):
return '{}/changelog?branch={}'.format(
data['repo']['url'], branch_name)
def tag_url(tag_name):
return '{}/files/{}/'.format(
data['repo']['url'], tag_name)
commits = _commits_as_dict(
self, commit_ids=self.pushed_commit_ids, repos=[self.repo])
@ -265,8 +309,21 @@ class RepoPushEvent(RepoVCSEvent):
last_branch = commit['branch']
issues = _issues_as_dict(commits)
branches = set(
commit['branch'] for commit in commits if commit['branch'])
branches = set()
tags = set()
for commit in commits:
if commit['refs']['tags']:
for tag in commit['refs']['tags']:
tags.add(tag)
if commit['branch']:
branches.add(commit['branch'])
# maybe we have branches in new_refs ?
try:
branches = branches.union(set(self.new_refs['branches']))
except Exception:
pass
branches = [
{
'name': branch,
@ -275,9 +332,24 @@ class RepoPushEvent(RepoVCSEvent):
for branch in branches
]
# maybe we have branches in new_refs ?
try:
tags = tags.union(set(self.new_refs['tags']))
except Exception:
pass
tags = [
{
'name': tag,
'url': tag_url(tag)
}
for tag in tags
]
data['push'] = {
'commits': commits,
'issues': issues,
'branches': branches,
'tags': tags,
}
return data

8
rhodecode/integrations/types/webhook.py
View file

@ -110,6 +110,11 @@ class WebhookHandler(object):
branches_commits = OrderedDict()
for commit in data['push']['commits']:
if commit.get('git_ref_change'):
# special case for GIT that allows creating tags,
# deleting branches without associated commit
continue
if commit['branch'] not in branches_commits:
branch_commits = {'branch': branch_data[commit['branch']],
'commits': []}
@ -378,7 +383,8 @@ def post_to_webhook(url_calls, settings):
log.debug('calling Webhook with method: %s, and auth:%s',
call_method, auth)
if settings.get('log_data'):
log.debug('calling webhook with data: %s', data)
resp = call_method(url, json={
'token': token,
'event': data

1
rhodecode/integrations/views.py
View file

@ -404,7 +404,6 @@ class RepoIntegrationsView(IntegrationSettingsViewBase):
c.repo_name = self.db_repo.repo_name
c.repository_pull_requests = ScmModel().get_pull_requests(self.repo)
return c
@LoginRequired()

1
rhodecode/model/validation_schema/schemas/integration_schema.py
View file

@ -198,7 +198,6 @@ class IntegrationOptionsSchemaBase(colander.MappingSchema):
)
def make_integration_schema(IntegrationType, settings=None):
"""
Return a colander schema for an integration type

5
rhodecode/tests/integrations/conftest.py
View file

@ -21,6 +21,7 @@
import pytest
from rhodecode import events
from rhodecode.lib.utils2 import AttributeDict
@pytest.fixture
@ -34,7 +35,7 @@ def repo_push_event(backend, user_regular):
]
commit_ids = backend.create_master_repo(commits).values()
repo = backend.create_repo()
scm_extras = {
scm_extras = AttributeDict({
'ip': '127.0.0.1',
'username': user_regular.username,
'user_id': user_regular.user_id,
@ -46,7 +47,7 @@ def repo_push_event(backend, user_regular):
'make_lock': None,
'locked_by': [None],
'commit_ids': commit_ids,
}
})
return events.RepoPushEvent(repo_name=repo.repo_name,
pushed_commit_ids=commit_ids,

24
rhodecode/tests/other/vcs_operations/__init__.py
View file

@ -77,12 +77,13 @@ class Command(object):
assert self.process.returncode == 0
def _add_files_and_push(vcs, dest, clone_url=None, **kwargs):
def _add_files_and_push(vcs, dest, clone_url=None, tags=None, **kwargs):
"""
Generate some files, add it to DEST repo and push back
vcs is git or hg and defines what VCS we want to make those files for
"""
# commit some stuff into this repo
tags = tags or []
cwd = path = jn(dest)
added_file = jn(path, '%ssetup.py' % tempfile._RandomNameSequence().next())
Command(cwd).execute('touch %s' % added_file)
@ -92,7 +93,7 @@ def _add_files_and_push(vcs, dest, clone_url=None, **kwargs):
git_ident = "git config user.name {} && git config user.email {}".format(
'Marcin Kuźminski', 'me@email.com')
for i in xrange(kwargs.get('files_no', 3)):
for i in range(kwargs.get('files_no', 3)):
cmd = """echo 'added_line%s' >> %s""" % (i, added_file)
Command(cwd).execute(cmd)
if vcs == 'hg':
@ -104,6 +105,22 @@ def _add_files_and_push(vcs, dest, clone_url=None, **kwargs):
git_ident, i, added_file)
Command(cwd).execute(cmd)
for tag in tags:
if vcs == 'hg':
stdout, stderr = Command(cwd).execute(
'hg tag', tag['name'])
elif vcs == 'git':
if tag['commit']:
# annotated tag
stdout, stderr = Command(cwd).execute(
"""%s && git tag -a %s -m "%s" """ % (
git_ident, tag['name'], tag['commit']))
else:
# lightweight tag
stdout, stderr = Command(cwd).execute(
"""%s && git tag %s""" % (
git_ident, tag['name']))
# PUSH it back
stdout = stderr = None
if vcs == 'hg':
@ -111,7 +128,8 @@ def _add_files_and_push(vcs, dest, clone_url=None, **kwargs):
'hg push --verbose', clone_url)
elif vcs == 'git':
stdout, stderr = Command(cwd).execute(
"""%s && git push --verbose %s master""" % (
"""%s &&
git push --verbose --tags %s master""" % (
git_ident, clone_url))
return stdout, stderr

27
rhodecode/tests/other/vcs_operations/conftest.py
View file

@ -57,20 +57,24 @@ def assert_no_running_instance(url):
"Port is not free at %s, cannot start web interface" % url)
def get_host_url(pyramid_config):
"""Construct the host url using the port in the test configuration."""
def get_port(pyramid_config):
config = ConfigParser.ConfigParser()
config.read(pyramid_config)
return config.get('server:main', 'port')
return '127.0.0.1:%s' % config.get('server:main', 'port')
def get_host_url(pyramid_config):
"""Construct the host url using the port in the test configuration."""
return '127.0.0.1:%s' % get_port(pyramid_config)
class RcWebServer(object):
"""
Represents a running RCE web server used as a test fixture.
"""
def __init__(self, pyramid_config):
def __init__(self, pyramid_config, log_file):
self.pyramid_config = pyramid_config
self.log_file = log_file
def repo_clone_url(self, repo_name, **kwargs):
params = {
@ -86,6 +90,10 @@ class RcWebServer(object):
def host_url(self):
return 'http://' + get_host_url(self.pyramid_config)
def get_rc_log(self):
with open(self.log_file) as f:
return f.read()
@pytest.fixture(scope="module")
def rcextensions(request, baseapp, tmpdir_factory):
@ -155,12 +163,11 @@ def rc_web_server(
env = os.environ.copy()
env['RC_NO_TMP_PATH'] = '1'
rc_log = RC_LOG
server_out = open(rc_log, 'w')
rc_log = list(RC_LOG.partition('.log'))
rc_log.insert(1, get_port(rc_web_server_config))
rc_log = ''.join(rc_log)
# TODO: Would be great to capture the output and err of the subprocess
# and make it available in a section of the py.test report in case of an
# error.
server_out = open(rc_log, 'w')
host_url = 'http://' + get_host_url(rc_web_server_config)
assert_no_running_instance(host_url)
@ -184,7 +191,7 @@ def rc_web_server(
server_out.flush()
server_out.close()
return RcWebServer(rc_web_server_config)
return RcWebServer(rc_web_server_config, log_file=rc_log)
@pytest.fixture

141
rhodecode/tests/other/vcs_operations/test_vcs_operations_tag_push.py Normal file
View file

@ -0,0 +1,141 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2010-2017 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
"""
Test suite for making push/pull operations, on specially modified INI files
.. important::
You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
to redirect things to stderr instead of stdout.
"""
import pytest
import requests
from rhodecode import events
from rhodecode.model.db import Integration
from rhodecode.model.integration import IntegrationModel
from rhodecode.model.meta import Session
from rhodecode.tests import GIT_REPO, HG_REPO
from rhodecode.tests.other.vcs_operations import Command, _add_files_and_push
from rhodecode.integrations.types.webhook import WebhookIntegrationType
def check_connection():
try:
response = requests.get('http://httpbin.org')
return response.status_code == 200
except Exception as e:
print(e)
return False
connection_available = pytest.mark.skipif(
not check_connection(), reason="No outside internet connection available")
@pytest.fixture
def enable_webhook_push_integration(request):
integration = Integration()
integration.integration_type = WebhookIntegrationType.key
Session().add(integration)
settings = dict(
url='http://httpbin.org',
secret_token='secret',
username=None,
password=None,
custom_header_key=None,
custom_header_val=None,
method_type='get',
events=[events.RepoPushEvent.name],
log_data=True
)
IntegrationModel().update_integration(
integration,
name='IntegrationWebhookTest',
enabled=True,
settings=settings,
repo=None,
repo_group=None,
child_repos_only=False,
)
Session().commit()
integration_id = integration.integration_id
@request.addfinalizer
def cleanup():
integration = Integration.get(integration_id)
Session().delete(integration)
Session().commit()
@pytest.mark.usefixtures(
"disable_locking", "disable_anonymous_user",
"enable_webhook_push_integration")
class TestVCSOperationsOnCustomIniConfig(object):
def test_push_tag_with_commit_hg(self, rc_web_server, tmpdir):
clone_url = rc_web_server.repo_clone_url(HG_REPO)
stdout, stderr = Command('/tmp').execute(
'hg clone', clone_url, tmpdir.strpath)
push_url = rc_web_server.repo_clone_url(HG_REPO)
_add_files_and_push(
'hg', tmpdir.strpath, clone_url=push_url,
tags=[{'name': 'v1.0.0', 'commit': 'added tag v1.0.0'}])
rc_log = rc_web_server.get_rc_log()
assert 'ERROR' not in rc_log
assert "'name': u'v1.0.0'" in rc_log
def test_push_tag_with_commit_git(
self, rc_web_server, tmpdir):
clone_url = rc_web_server.repo_clone_url(GIT_REPO)
stdout, stderr = Command('/tmp').execute(
'git clone', clone_url, tmpdir.strpath)
push_url = rc_web_server.repo_clone_url(GIT_REPO)
_add_files_and_push(
'git', tmpdir.strpath, clone_url=push_url,
tags=[{'name': 'v1.0.0', 'commit': 'added tag v1.0.0'}])
rc_log = rc_web_server.get_rc_log()
assert 'ERROR' not in rc_log
assert "'name': u'v1.0.0'" in rc_log
def test_push_tag_with_no_commit_git(
self, rc_web_server, tmpdir):
clone_url = rc_web_server.repo_clone_url(GIT_REPO)
stdout, stderr = Command('/tmp').execute(
'git clone', clone_url, tmpdir.strpath)
push_url = rc_web_server.repo_clone_url(GIT_REPO)
_add_files_and_push(
'git', tmpdir.strpath, clone_url=push_url,
tags=[{'name': 'v1.0.0', 'commit': 'added tag v1.0.0'}])
rc_log = rc_web_server.get_rc_log()
assert 'ERROR' not in rc_log
assert "'name': u'v1.0.0'" in rc_log