From 2b91837f9c41f3b94695f622d2006e0078233301 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Fri, 11 Aug 2017 23:54:54 +0200 Subject: [PATCH 1/8] pyramid: ported pyramid routing for events --- rhodecode/events/base.py | 24 +++++++++++++++++------- rhodecode/events/pullrequest.py | 10 ++++++---- rhodecode/events/repo.py | 4 ++++ rhodecode/lib/base.py | 30 ++++++++++++++++++++++++++++++ rhodecode/lib/hooks_daemon.py | 13 ++++++++----- 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/rhodecode/events/base.py b/rhodecode/events/base.py index 281cadb8..8dcec569 100644 --- a/rhodecode/events/base.py +++ b/rhodecode/events/base.py @@ -16,9 +16,11 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ import logging +import datetime -from datetime import datetime +from zope.cachedescriptors.property import Lazy as LazyProperty from pyramid.threadlocal import get_current_request + from rhodecode.lib.utils2 import AttributeDict @@ -36,10 +38,20 @@ class RhodecodeEvent(object): Base event class for all RhodeCode events """ name = "RhodeCodeEvent" + no_url_set = '' def __init__(self, request=None): - self.request = request or get_current_request() - self.utc_timestamp = datetime.utcnow() + self._request = request + self.utc_timestamp = datetime.datetime.utcnow() + + def get_request(self): + if self._request: + return self._request + return get_current_request() + + @LazyProperty + def request(self): + return self.get_request() @property def auth_user(self): @@ -57,7 +69,6 @@ class RhodecodeEvent(object): @property def actor(self): auth_user = self.auth_user - if auth_user: instance = auth_user.get_instance() if not instance: @@ -78,15 +89,14 @@ class RhodecodeEvent(object): @property def server_url(self): - default = '' if self.request: try: return self.request.route_url('home') except Exception: log.exception('Failed to fetch URL for server') - return default + return self.no_url_set - return default + return self.no_url_set def as_dict(self): data = { diff --git a/rhodecode/events/pullrequest.py b/rhodecode/events/pullrequest.py index 06704366..cc207d86 100644 --- a/rhodecode/events/pullrequest.py +++ b/rhodecode/events/pullrequest.py @@ -52,9 +52,10 @@ class PullRequestEvent(RepoEvent): 'title': self.pullrequest.title, 'issues': issues, 'pull_request_id': self.pullrequest.pull_request_id, - 'url': PullRequestModel().get_url(self.pullrequest), + 'url': PullRequestModel().get_url( + self.pullrequest, request=self.request), 'permalink_url': PullRequestModel().get_url( - self.pullrequest, permalink=True), + self.pullrequest, request=self.request, permalink=True), 'status': self.pullrequest.calculated_review_status(), 'commits': commits, } @@ -134,9 +135,10 @@ class PullRequestCommentEvent(PullRequestEvent): 'type': self.comment.comment_type, 'file': self.comment.f_path, 'line': self.comment.line_no, - 'url': CommentsModel().get_url(self.comment), + 'url': CommentsModel().get_url( + self.comment, request=self.request), 'permalink_url': CommentsModel().get_url( - self.comment, permalink=True), + self.comment, request=self.request, permalink=True), } }) return data diff --git a/rhodecode/events/repo.py b/rhodecode/events/repo.py index fbf49084..305635e2 100644 --- a/rhodecode/events/repo.py +++ b/rhodecode/events/repo.py @@ -201,6 +201,10 @@ class RepoVCSEvent(RepoEvent): if self.extras.get('server_url'): return self.extras['server_url'] + @property + def request(self): + return self.extras.get('request') or self.get_request() + class RepoPrePullEvent(RepoVCSEvent): """ diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py index 7e532a98..359ab348 100644 --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -511,6 +511,36 @@ class BaseController(WSGIController): return WSGIController.__call__(self, environ, start_response) +def add_events_routes(config): + """ + Adds routing that can be used in events. Because some events are triggered + outside of pyramid context, we need to bootstrap request with some + routing registered + """ + config.add_route(name='home', pattern='/') + + config.add_route(name='repo_summary', pattern='/{repo_name}') + config.add_route(name='repo_summary_explicit', pattern='/{repo_name}/summary') + config.add_route(name='repo_group_home', pattern='/{repo_group_name}') + + config.add_route(name='pullrequest_show', + pattern='/{repo_name}/pull-request/{pull_request_id}') + config.add_route(name='pull_requests_global', + pattern='/pull-request/{pull_request_id}') + + config.add_route(name='repo_commit', + pattern='/{repo_name}/changeset/{commit_id}') + config.add_route(name='repo_files', + pattern='/{repo_name}/files/{commit_id}/{f_path}') + + +def bootstrap_request(): + import pyramid.testing + request = pyramid.testing.DummyRequest() + config = pyramid.testing.setUp(request=request) + add_events_routes(config) + + class BaseRepoController(BaseController): """ Base class for controllers responsible for loading all needed data for diff --git a/rhodecode/lib/hooks_daemon.py b/rhodecode/lib/hooks_daemon.py index 946ac651..4051ddd6 100644 --- a/rhodecode/lib/hooks_daemon.py +++ b/rhodecode/lib/hooks_daemon.py @@ -29,9 +29,11 @@ import pylons import rhodecode from rhodecode.model import meta +from rhodecode.lib.base import bootstrap_request from rhodecode.lib import hooks_base from rhodecode.lib.utils2 import ( AttributeDict, safe_str, get_routes_generator_for_server_url) +from rhodecode.lib.utils2 import AttributeDict log = logging.getLogger(__name__) @@ -188,29 +190,30 @@ class Hooks(object): """ def repo_size(self, extras): - log.debug("Called repo_size of Hooks object") + log.debug("Called repo_size of %s object", self) return self._call_hook(hooks_base.repo_size, extras) def pre_pull(self, extras): - log.debug("Called pre_pull of Hooks object") + log.debug("Called pre_pull of %s object", self) return self._call_hook(hooks_base.pre_pull, extras) def post_pull(self, extras): - log.debug("Called post_pull of Hooks object") + log.debug("Called post_pull of %s object", self) return self._call_hook(hooks_base.post_pull, extras) def pre_push(self, extras): - log.debug("Called pre_push of Hooks object") + log.debug("Called pre_push of %s object", self) return self._call_hook(hooks_base.pre_push, extras) def post_push(self, extras): - log.debug("Called post_push of Hooks object") + log.debug("Called post_push of %s object", self) return self._call_hook(hooks_base.post_push, extras) def _call_hook(self, hook, extras): extras = AttributeDict(extras) pylons_router = get_routes_generator_for_server_url(extras.server_url) pylons.url._push_object(pylons_router) + extras.request = bootstrap_request() try: result = hook(extras) From ad948b5e46bf6ffeb3396c5210905701146de773 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Wed, 26 Jul 2017 16:19:06 +0200 Subject: [PATCH 2/8] events: make sure we propagate our dummy request with proper application_url. - handles proper proxy prefix - handles https and reverse proxies --- rhodecode/lib/base.py | 8 ++++++-- rhodecode/lib/hooks_daemon.py | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py index 359ab348..ac6eb6b5 100644 --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -534,9 +534,13 @@ def add_events_routes(config): pattern='/{repo_name}/files/{commit_id}/{f_path}') -def bootstrap_request(): +def bootstrap_request(**kwargs): import pyramid.testing - request = pyramid.testing.DummyRequest() + request = pyramid.testing.DummyRequest(**kwargs) + request.application_url = kwargs.pop('application_url', 'http://example.com') + request.host = kwargs.pop('host', 'example.com:80') + request.domain = kwargs.pop('domain', 'example.com') + config = pyramid.testing.setUp(request=request) add_events_routes(config) diff --git a/rhodecode/lib/hooks_daemon.py b/rhodecode/lib/hooks_daemon.py index 4051ddd6..e7fdc59e 100644 --- a/rhodecode/lib/hooks_daemon.py +++ b/rhodecode/lib/hooks_daemon.py @@ -213,7 +213,8 @@ class Hooks(object): extras = AttributeDict(extras) pylons_router = get_routes_generator_for_server_url(extras.server_url) pylons.url._push_object(pylons_router) - extras.request = bootstrap_request() + extras.request = bootstrap_request( + application_url=extras['server_url']) try: result = hook(extras) From 0012823054801ab1ef7d47f7fad0dfe2ab5c6e15 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Sat, 12 Aug 2017 19:20:06 +0200 Subject: [PATCH 3/8] docs: updated svn configuration to prevent 400 errors on certain Apache versions. --- docs/admin/svn-http.rst | 19 ++++++++++++++++++- .../templates/mod-dav-svn.conf.mako | 3 +++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/admin/svn-http.rst b/docs/admin/svn-http.rst index 94b96236..7cd05487 100644 --- a/docs/admin/svn-http.rst +++ b/docs/admin/svn-http.rst @@ -15,7 +15,21 @@ Prerequisites ``Apache HTTP Server`` and ``mod_dav_svn``. -Using Ubuntu 14.04 Distribution as an example execute the following: +.. tip:: + + We recommend using Wandisco repositories which provide latest SVN versions + for most platforms. Below is example how to add the wandisco repositories + for Ubuntu. + + .. code-block:: bash + + $ sudo sh -c 'echo "deb http://opensource.wandisco.com/ubuntu `lsb_release -cs` svn19" >> /etc/apt/sources.list.d/subversion19.list' + $ sudo wget -q http://opensource.wandisco.com/wandisco-debian.gpg -O- | sudo apt-key add - + $ sudo apt-get update + + +Using Ubuntu 14.04/16.04 Distribution as an example execute the following to +install required components: .. code-block:: bash @@ -67,6 +81,9 @@ permission issues could occur. To do this edit the ``/etc/apache2/envvars`` DocumentRoot /var/www/html ErrorLog ${'${APACHE_LOG_DIR}'}/error.log CustomLog ${'${APACHE_LOG_DIR}'}/access.log combined + LogLevel info + # allows custom host names, prevents 400 errors on checkout + HttpProtocolOptions Unsafe Include /home/user/.rccontrol/enterprise-1/mod_dav_svn.conf diff --git a/rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako b/rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako index 44fae271..0e359e44 100644 --- a/rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako +++ b/rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako @@ -21,6 +21,9 @@ # DocumentRoot /var/www/html # ErrorLog ${'${APACHE_LOG_DIR}'}/error.log # CustomLog ${'${APACHE_LOG_DIR}'}/access.log combined +# LogLevel info +# # allows custom host names, prevents 400 errors on checkout +# HttpProtocolOptions Unsafe # Include /path/to/generated/mod_dav_svn.conf # # From 7347874011e88f6dd61c2bf7ceb1583b3996f91d Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Sat, 12 Aug 2017 11:48:49 +0200 Subject: [PATCH 4/8] docs: added release notes for 4.9.0 --- docs/release-notes/release-notes-4.9.0.rst | 66 ++++++++++++++++++++++ docs/release-notes/release-notes.rst | 1 + 2 files changed, 67 insertions(+) create mode 100644 docs/release-notes/release-notes-4.9.0.rst diff --git a/docs/release-notes/release-notes-4.9.0.rst b/docs/release-notes/release-notes-4.9.0.rst new file mode 100644 index 00000000..52825db7 --- /dev/null +++ b/docs/release-notes/release-notes-4.9.0.rst @@ -0,0 +1,66 @@ +|RCE| 4.9.0 |RNS| +----------------- + +Release Date +^^^^^^^^^^^^ + +- 2017-08-12 + + +New Features +^^^^^^^^^^^^ + + + +General +^^^^^^^ + +- Off cycle Minor release to fix SCM vulnerabilities. + + +Security +^^^^^^^^ + +- security(critical): Bumped GIT to 2.9.5 fixes CVE-2017-1000117 + https://www.mail-archive.com/linux-kernel@vger.kernel.org/msg1466490.html +- security(critical): Bumped SVN to 1.9.7 fixes CVE-2017-9800 + https://subversion.apache.org/security/CVE-2017-9800-advisory.txt +- security(critical): Bumped Mercurial to 4.2.3 fixes CVE-2017-1000116 + https://www.mercurial-scm.org/wiki/WhatsNew#Mercurial_4.3_.282017-08-10.29 + + +Performance +^^^^^^^^^^^ + +- Fixed Mercurial Stream support for very large repositories. Due to discovered + bug in WebOb library we manage to fix Mercurial stream support. + Now cloning very large repos e.g 100GB, ~1mln commits should be much + faster, and use less memory. + + +Fixes +^^^^^ + +- Fixed problem with default-reviewers in EE package that was missing panel + title and in some occasions generate 500 errors. +- Fixed problem with potential URL generation inside our integration. + This was introduced during pyramid porting. We know ensure that proper + routing generation is done on all events. + + +Upgrade notes +^^^^^^^^^^^^^ + + +- The 4.9.0 release is an off-cycle release. Due to the fact that we needed to + bump Mercurial from 4.1.X to 4.2.X, and Subversion from 1.9.4 to 1.9.7, we + released this version not as 4.8.1 security bug fix but 4.9.0. + We know historically that SVN and Mercurial can have internal api changes. + We tested basic functionality for all 3 vcs-es but due to very short release + time we were unable to test everything. Please report any found problems to us + and we'll for sure address them. + + Note to SVN users: Please make sure to upgrade mod_dav to 1.9.7 version. + At this time we know Wandisco provides 1.9.7 packages for most major distros. + + diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst index ae828d60..461fa1a1 100644 --- a/docs/release-notes/release-notes.rst +++ b/docs/release-notes/release-notes.rst @@ -9,6 +9,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release-notes-4.9.0.rst release-notes-4.8.0.rst release-notes-4.7.2.rst release-notes-4.7.1.rst From ae89e38cf41df782aa517e7337d49dcfb396d87f Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Sat, 12 Aug 2017 19:55:15 +0200 Subject: [PATCH 5/8] release: Start preparation for 4.9.0 --- .bumpversion.cfg | 2 +- .release.cfg | 14 ++++---------- rhodecode/VERSION | 2 +- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cbe58b8b..a9162235 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.8.0 +current_version = 4.9.0 message = release: Bump version {current_version} to {new_version} [bumpversion:file:rhodecode/VERSION] diff --git a/.release.cfg b/.release.cfg index 0134c517..a3670401 100644 --- a/.release.cfg +++ b/.release.cfg @@ -5,26 +5,20 @@ done = false done = true [task:rc_tools_pinned] -done = true [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [task:changelog_updated] -done = true [task:generate_api_docs] -done = true - -[task:updated_translation] -done = true [release] -state = prepared -version = 4.8.0 +state = in_progress +version = 4.9.0 + +[task:updated_translation] [task:generate_js_routes] diff --git a/rhodecode/VERSION b/rhodecode/VERSION index 6ca6df11..b617d997 100644 --- a/rhodecode/VERSION +++ b/rhodecode/VERSION @@ -1 +1 @@ -4.8.0 \ No newline at end of file +4.9.0 \ No newline at end of file From 6b2dc12104bad8e63f0919acec5c74c5ccff76b2 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Sat, 12 Aug 2017 21:55:15 +0200 Subject: [PATCH 6/8] release: updated pip2nix output for 4.9.0 --- .release.cfg | 4 ++++ pkgs/python-packages.nix | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.release.cfg b/.release.cfg index a3670401..ab98a721 100644 --- a/.release.cfg +++ b/.release.cfg @@ -5,14 +5,18 @@ done = false done = true [task:rc_tools_pinned] +done = true [task:fixes_on_stable] +done = true [task:pip2nix_generated] [task:changelog_updated] +done = true [task:generate_api_docs] +done = true [release] state = in_progress diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix index 3be0bb26..f71448c6 100644 --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -1654,7 +1654,7 @@ }; }; rhodecode-enterprise-ce = super.buildPythonPackage { - name = "rhodecode-enterprise-ce-4.8.0"; + name = "rhodecode-enterprise-ce-4.9.0"; buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage configobj]; doCheck = true; propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments pygments-markdown-lexer Pylons Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress cssselect celery channelstream colander decorator deform docutils gevent gunicorn infrae.cache ipython iso8601 kombu lxml msgpack-python nbconvert packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson subprocess32 waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt]; From b6850abfdf6b3d5ce5f4ffb599f77895c2d48183 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Sat, 12 Aug 2017 21:55:17 +0200 Subject: [PATCH 7/8] release: Finish preparation for 4.9.0 --- .release.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.release.cfg b/.release.cfg index ab98a721..d0a975e4 100644 --- a/.release.cfg +++ b/.release.cfg @@ -11,6 +11,7 @@ done = true done = true [task:pip2nix_generated] +done = true [task:changelog_updated] done = true @@ -19,7 +20,7 @@ done = true done = true [release] -state = in_progress +state = prepared version = 4.9.0 [task:updated_translation] From 173061b9a55dc91a79b2922d570f0d7b32930ad8 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Sat, 12 Aug 2017 21:55:17 +0200 Subject: [PATCH 8/8] Added tag v4.9.0 for changeset c5a2b7d0e4bb