From 024d55841794cd639646187c292b81b67379e281 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Tue, 7 Feb 2017 13:16:15 +0100 Subject: [PATCH 01/15] api: cleanup sessions enforce older_then must be a valid INT. --- rhodecode/api/views/server_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rhodecode/api/views/server_api.py b/rhodecode/api/views/server_api.py index c3d604da..ac9c3869 100644 --- a/rhodecode/api/views/server_api.py +++ b/rhodecode/api/views/server_api.py @@ -28,6 +28,7 @@ from rhodecode.api.utils import ( from rhodecode.lib.utils import repo2db_mapper from rhodecode.lib import system_info from rhodecode.lib import user_sessions +from rhodecode.lib.utils2 import safe_int from rhodecode.model.db import UserIpMap from rhodecode.model.scm import ScmModel @@ -223,7 +224,7 @@ def cleanup_sessions(request, apiuser, older_then=Optional(60)): if not has_superadmin_permission(apiuser): raise JSONRPCForbidden() - older_then = Optional.extract(older_then) + older_then = safe_int(Optional.extract(older_then)) or 60 older_than_seconds = 60 * 60 * 24 * older_then config = system_info.rhodecode_config().get_value()['value']['config'] From ece00deae2316ce09a4d4cf911fba2db7c88f10b Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Tue, 7 Feb 2017 19:42:04 +0100 Subject: [PATCH 02/15] search: goto commit search will now use a safe search option and never throw any exceptions even if search is misconfigured. Fixes #5193 --- rhodecode/controllers/home.py | 6 ++++-- rhodecode/lib/index/__init__.py | 4 +++- rhodecode/lib/index/whoosh.py | 7 ++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/rhodecode/controllers/home.py b/rhodecode/controllers/home.py index d1836fd4..5a5c075e 100644 --- a/rhodecode/controllers/home.py +++ b/rhodecode/controllers/home.py @@ -178,7 +178,8 @@ class HomeController(BaseController): user_id=c.rhodecode_user.user_id, ip_addr=self.ip_addr) searcher = searcher_from_config(config) result = searcher.search( - 'commit_id:%s*' % commit_hash_prefix, 'commit', auth_user) + 'commit_id:%s*' % commit_hash_prefix, 'commit', auth_user, + raise_on_exc=False) return [ { @@ -187,7 +188,8 @@ class HomeController(BaseController): 'type': 'commit', 'obj': {'repo': entry['repository']}, 'url': url('changeset_home', - repo_name=entry['repository'], revision=entry['commit_id']) + repo_name=entry['repository'], + revision=entry['commit_id']) } for entry in result['results']] diff --git a/rhodecode/lib/index/__init__.py b/rhodecode/lib/index/__init__.py index f05dfab2..512075b0 100644 --- a/rhodecode/lib/index/__init__.py +++ b/rhodecode/lib/index/__init__.py @@ -39,9 +39,11 @@ class BaseSearch(object): def cleanup(self): pass - def search(self, query, document_type, search_user, repo_name=None): + def search(self, query, document_type, search_user, repo_name=None, + raise_on_exc=True): raise Exception('NotImplemented') + def searcher_from_config(config, prefix='search.'): _config = {} for key in config.keys(): diff --git a/rhodecode/lib/index/whoosh.py b/rhodecode/lib/index/whoosh.py index 051e93ea..00b9be18 100644 --- a/rhodecode/lib/index/whoosh.py +++ b/rhodecode/lib/index/whoosh.py @@ -60,12 +60,12 @@ FRAGMENTER = ContextFragmenter(200) log = logging.getLogger(__name__) - class Search(BaseSearch): name = 'whoosh' def __init__(self, config): + super(Search, self).__init__() self.config = config if not os.path.isdir(self.config['location']): os.makedirs(self.config['location']) @@ -99,8 +99,9 @@ class Search(BaseSearch): query = u'(%s) OR %s' % (query, hashes_or_query) return query - def search(self, query, document_type, search_user, repo_name=None, - requested_page=1, page_limit=10, sort=None): + def search(self, query, document_type, search_user, + repo_name=None, requested_page=1, page_limit=10, sort=None, + raise_on_exc=True): original_query = query query = self._extend_query(query) From 4da7a88228eb7ee960e59c443ad510d100e92e4f Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Tue, 7 Feb 2017 21:54:05 +0100 Subject: [PATCH 03/15] annotations: fixed UI problems in annotation view for newer browsers. --- rhodecode/lib/codeblocks.py | 2 +- rhodecode/public/css/code-block.less | 26 +++----- rhodecode/templates/codeblocks/source.mako | 75 ++++++++++++---------- 3 files changed, 50 insertions(+), 53 deletions(-) diff --git a/rhodecode/lib/codeblocks.py b/rhodecode/lib/codeblocks.py index bc995647..cf901c87 100644 --- a/rhodecode/lib/codeblocks.py +++ b/rhodecode/lib/codeblocks.py @@ -111,7 +111,7 @@ def filenode_as_annotated_lines_tokens(filenode): ] """ - commit_cache = {} # cache commit_getter lookups + commit_cache = {} # cache commit_getter lookups def _get_annotation(commit_id, commit_getter): if commit_id not in commit_cache: diff --git a/rhodecode/public/css/code-block.less b/rhodecode/public/css/code-block.less index 3caa92c8..ec43c947 100644 --- a/rhodecode/public/css/code-block.less +++ b/rhodecode/public/css/code-block.less @@ -969,17 +969,6 @@ table.cb { tr { &.cb-annotate { border-top: 1px solid #eee; - - &+ .cb-line { - border-top: 1px solid #eee; - } - - &:first-child { - border-top: none; - &+ .cb-line { - border-top: none; - } - } } &.cb-hunk { @@ -1114,7 +1103,9 @@ table.cb { background: #ffdddd; } } - + &.cb-annotate-message-spacer { + width:8px; + } &.cb-annotate-info { width: 320px; min-width: 320px; @@ -1122,22 +1113,23 @@ table.cb { padding: 5px 2px; font-size: 13px; - strong.cb-annotate-message { - padding: 5px 0; + .cb-annotate-message { + padding: 2px 0px 0px 0px; white-space: pre-line; - display: inline-block; + overflow: hidden; } .rc-user { float: none; padding: 0 6px 0 17px; - min-width: auto; - min-height: auto; + min-width: unset; + min-height: unset; } } &.cb-annotate-revision { cursor: pointer; text-align: right; + padding: 1px 3px 0px 3px; } } } diff --git a/rhodecode/templates/codeblocks/source.mako b/rhodecode/templates/codeblocks/source.mako index c832b832..bdcdd943 100644 --- a/rhodecode/templates/codeblocks/source.mako +++ b/rhodecode/templates/codeblocks/source.mako @@ -1,16 +1,50 @@ <%def name="render_line(line_num, tokens, annotation=None, - bgcolor=None)"> + bgcolor=None, show_annotation=None)"> <% from rhodecode.lib.codeblocks import render_tokenstream # avoid module lookup for performance html_escape = h.html_escape %> - + + % if annotation: + % if show_annotation: + + ${h.gravatar_with_user(annotation.author, 16) | n} +
${h.chop_at_smart(annotation.message, '\n', suffix_if_chopped='...')}
+ + + + + r${annotation.revision} + + + % else: + + + + + % endif + % else: + + % endif + + @@ -27,40 +61,11 @@ <%def name="render_annotation_lines(annotation, lines, color_hasher)"> - <% - rowspan = len(lines) + 1 # span the line's and annotation - %> - %if not annotation: - - - - - %else: - - - ${h.gravatar_with_user(annotation.author, 16) | n} - ${h.truncate(annotation.message, len(lines) * 30)} - - - - r${annotation.revision} - - - - %endif - - %for line_num, tokens in lines: + % for line_num, tokens in lines: ${render_line(line_num, tokens, bgcolor=color_hasher(annotation and annotation.raw_id or ''), - annotation=annotation, + annotation=annotation, show_annotation=loop.first )} - %endfor + % endfor + From eadfaeb34aa473dca4852bdd8c6fb635b4e5d8f9 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Wed, 8 Feb 2017 21:30:01 +0100 Subject: [PATCH 04/15] backends: make HG repos throw same kind of CommitDoesNotExist errors like other backends. --- rhodecode/lib/vcs/backends/hg/repository.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rhodecode/lib/vcs/backends/hg/repository.py b/rhodecode/lib/vcs/backends/hg/repository.py index fd7499e7..36b34bc2 100644 --- a/rhodecode/lib/vcs/backends/hg/repository.py +++ b/rhodecode/lib/vcs/backends/hg/repository.py @@ -445,7 +445,12 @@ class MercurialRepository(BaseRepository): if isinstance(commit_id, unicode): commit_id = safe_str(commit_id) - raw_id, idx = self._remote.lookup(commit_id, both=True) + try: + raw_id, idx = self._remote.lookup(commit_id, both=True) + except CommitDoesNotExistError: + msg = "Commit %s does not exist for %s" % ( + commit_id, self) + raise CommitDoesNotExistError(msg) return MercurialCommit(self, raw_id, idx, pre_load=pre_load) From 84249d158171a8765fe678bd840e44560290300a Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Wed, 8 Feb 2017 21:30:23 +0100 Subject: [PATCH 05/15] api: validate commit_id when using commit_comment API --- rhodecode/api/tests/test_comment_commit.py | 25 +++++++++++++++++++++- rhodecode/api/views/repo_api.py | 8 ++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/rhodecode/api/tests/test_comment_commit.py b/rhodecode/api/tests/test_comment_commit.py index b2e50467..cceb7705 100644 --- a/rhodecode/api/tests/test_comment_commit.py +++ b/rhodecode/api/tests/test_comment_commit.py @@ -33,9 +33,29 @@ class TestCommentCommit(object): self.apikey, 'comment_commit', repoid=repo.repo_name, commit_id='tip', message='message', status_change=None) response = api_call(self.app, params) - expected = 'failed to set comment on repository `%s`' % repo.repo_name + expected = 'There are no commits yet' assert_error(id_, expected, given=response.body) + @pytest.mark.parametrize("commit_id, expected_err", [ + ('abcabca', {'hg': 'Commit {commit} does not exist for {repo}', + 'git': 'Commit {commit} does not exist for {repo}', + 'svn': 'Commit id {commit} not understood.'}), + ('idontexist', {'hg': 'Commit {commit} does not exist for {repo}', + 'git': 'Commit {commit} does not exist for {repo}', + 'svn': 'Commit id {commit} not understood.'}), + ]) + def test_api_comment_commit_wrong_hash(self, backend, commit_id, expected_err): + repo_name = backend.repo.repo_name + id_, params = build_data( + self.apikey, 'comment_commit', repoid=repo_name, + commit_id=commit_id, message='message', status_change=None) + response = api_call(self.app, params) + + expected_err = expected_err[backend.alias] + expected_err = expected_err.format( + repo=backend.repo.scm_instance(), commit=commit_id) + assert_error(id_, expected_err, given=response.body) + @pytest.mark.parametrize("status_change, message, commit_id", [ (None, 'Hallo', 'tip'), (ChangesetStatus.STATUS_APPROVED, 'Approved', 'tip'), @@ -44,6 +64,9 @@ class TestCommentCommit(object): def test_api_comment_commit( self, backend, status_change, message, commit_id, no_notifications): + + commit_id = backend.repo.scm_instance().get_changeset(commit_id).raw_id + id_, params = build_data( self.apikey, 'comment_commit', repoid=backend.repo_name, commit_id=commit_id, message=message, status=status_change) diff --git a/rhodecode/api/views/repo_api.py b/rhodecode/api/views/repo_api.py index 6dd2ca36..29896575 100644 --- a/rhodecode/api/views/repo_api.py +++ b/rhodecode/api/views/repo_api.py @@ -30,9 +30,9 @@ from rhodecode.api.utils import ( get_perm_or_error, parse_args, get_origin, build_commit_data, validate_set_owner_permissions) from rhodecode.lib.auth import HasPermissionAnyApi, HasUserGroupPermissionAnyApi -from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError from rhodecode.lib.utils2 import str2bool, time_to_datetime from rhodecode.lib.ext_json import json +from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError from rhodecode.model.changeset_status import ChangesetStatusModel from rhodecode.model.comment import CommentsModel from rhodecode.model.db import ( @@ -1426,6 +1426,12 @@ def comment_commit( _perms = ('repository.read', 'repository.write', 'repository.admin') validate_repo_permissions(apiuser, repoid, repo, _perms) + try: + commit_id = repo.scm_instance().get_commit(commit_id=commit_id).raw_id + except Exception as e: + log.exception('Failed to fetch commit') + raise JSONRPCError(e.message) + if isinstance(userid, Optional): userid = apiuser.user_id From 445ef262a743135735b9d0f9f9f1afc002e1e93e Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Mon, 13 Feb 2017 12:02:51 +0100 Subject: [PATCH 06/15] docs: updated contribution and dev setup docs. --- docs/contributing/dev-setup.rst | 39 ++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/docs/contributing/dev-setup.rst b/docs/contributing/dev-setup.rst index 572c65df..df1677ce 100644 --- a/docs/contributing/dev-setup.rst +++ b/docs/contributing/dev-setup.rst @@ -32,7 +32,7 @@ following commands: .. tip:: - Update your channels frequently by running ``nix-channel --upgrade``. + Update your channels frequently by running ``nix-channel --update``. Switch nix to the latest STABLE channel @@ -63,7 +63,10 @@ Clone the required repositories After Nix is set up, clone the RhodeCode Enterprise Community Edition and RhodeCode VCSServer repositories into the same directory. -To do this, use the following example:: +RhodeCode currently is using Mercurial Version Control System, please make sure +you have it installed before continuing. + +To obtain the required sources, use the following commands: mkdir rhodecode-develop && cd rhodecode-develop hg clone https://code.rhodecode.com/rhodecode-enterprise-ce @@ -71,8 +74,7 @@ To do this, use the following example:: .. note:: - If you cannot clone the repository, please request read permissions - via support@rhodecode.com + If you cannot clone the repository, please contact us via support@rhodecode.com Install some required libraries @@ -94,9 +96,14 @@ required libraries:: Enter the Development Shell --------------------------- -The final step is to start the development shell. To do this, run the +The final step is to start the development shells. To do this, run the following command from inside the cloned repository:: + #first, the vcsserver + cd ~/rhodecode-vcsserver + nix-shell + + # then enterprise sources cd ~/rhodecode-enterprise-ce nix-shell @@ -114,19 +121,22 @@ Creating a Development Configuration To create a development environment for RhodeCode Enterprise, use the following steps: -1. Create a copy of `~/rhodecode-enterprise-ce/configs/development.ini` -2. Adjust the configuration settings to your needs +1. Create a copy of vcsserver config: + `cp ~/rhodecode-vcsserver/configs/development.ini ~/rhodecode-vcsserver/configs/dev.ini` +2. Create a copy of rhodocode config: + `cp ~/rhodecode-enterprise-ce/configs/development.ini ~/rhodecode-enterprise-ce/configs/dev.ini` +3. Adjust the configuration settings to your needs if needed. .. note:: - It is recommended to use the name `dev.ini`. + It is recommended to use the name `dev.ini` since it's included in .hgignore file. Setup the Development Database ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ To create a development database, use the following example. This is a one -time operation:: +time operation executed from the nix-shell of rhodecode-enterprise-ce sources :: paster setup-rhodecode dev.ini \ --user=admin --password=secret \ @@ -150,24 +160,23 @@ changes made to the CSS or JavaScript files when developing the code:: This prepares the development (with comments/whitespace) versions of files. -Start the Development Server -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Start the Development Servers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From the rhodecode-vcsserver directory, start the development server in another nix-shell, using the following command:: - pserve configs/development.ini http_port=9900 + pserve configs/dev.ini In the adjacent nix-shell which you created for your development server, you may now start CE with the following command:: - rcserver dev.ini + pserve --reload configs/dev.ini .. note:: - To automatically refresh - and recompile the frontend assets - when changes - are made in the source code, you can use the option `--reload`. + `--reload` flag will automatically reload the server when source file changes. Run the Environment Tests From 163ab1c823c1c4c463b9c836de1f34dc8f8cb8f5 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Mon, 13 Feb 2017 13:04:48 +0100 Subject: [PATCH 07/15] api-events: fix a case events were called from API and we couldn't fetch registered user. - Fixes #5210 --- rhodecode/api/__init__.py | 17 ++++++++++------- rhodecode/events/base.py | 23 +++++++++++++++++++---- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/rhodecode/api/__init__.py b/rhodecode/api/__init__.py index 56fc5420..682b318d 100644 --- a/rhodecode/api/__init__.py +++ b/rhodecode/api/__init__.py @@ -158,33 +158,36 @@ def request_view(request): # search not expired tokens only try: - u = User.get_by_auth_token(request.rpc_api_key) + api_user = User.get_by_auth_token(request.rpc_api_key) - if u is None: + if api_user is None: return jsonrpc_error( request, retid=request.rpc_id, message='Invalid API KEY') - if not u.active: + if not api_user.active: return jsonrpc_error( request, retid=request.rpc_id, message='Request from this user not allowed') # check if we are allowed to use this IP auth_u = AuthUser( - u.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr) + api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr) if not auth_u.ip_allowed: return jsonrpc_error( request, retid=request.rpc_id, message='Request from IP:%s not allowed' % ( - request.rpc_ip_addr,)) + request.rpc_ip_addr,)) else: log.info('Access for IP:%s allowed' % (request.rpc_ip_addr,)) + # register our auth-user + request.rpc_user = auth_u + # now check if token is valid for API role = UserApiKeys.ROLE_API extra_auth_tokens = [ - x.api_key for x in User.extra_valid_auth_tokens(u, role=role)] - active_tokens = [u.api_key] + extra_auth_tokens + x.api_key for x in User.extra_valid_auth_tokens(api_user, role=role)] + active_tokens = [api_user.api_key] + extra_auth_tokens log.debug('Checking if API key has proper role') if request.rpc_api_key not in active_tokens: diff --git a/rhodecode/events/base.py b/rhodecode/events/base.py index 2370a0cb..7fd7e9bb 100644 --- a/rhodecode/events/base.py +++ b/rhodecode/events/base.py @@ -37,16 +37,31 @@ class RhodecodeEvent(object): self.request = get_current_request() self.utc_timestamp = datetime.utcnow() + @property + def auth_user(self): + if not self.request: + return + + user = getattr(self.request, 'user', None) + if user: + return user + + api_user = getattr(self.request, 'rpc_user', None) + if api_user: + return api_user + @property def actor(self): - if self.request: - return self.request.user.get_instance() + auth_user = self.auth_user + if auth_user: + return auth_user.get_instance() return SYSTEM_USER @property def actor_ip(self): - if self.request: - return self.request.user.ip_addr + auth_user = self.auth_user + if auth_user: + return auth_user.ip_addr return '' @property From 8801d9b23cb5bba97ad2d8dda3b1af18188b42cf Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Mon, 13 Feb 2017 16:26:08 +0100 Subject: [PATCH 08/15] events: fixed missing construcors on user based events. --- rhodecode/events/repo.py | 2 ++ rhodecode/events/user.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/rhodecode/events/repo.py b/rhodecode/events/repo.py index cb5fb3c0..2274fb86 100644 --- a/rhodecode/events/repo.py +++ b/rhodecode/events/repo.py @@ -25,6 +25,7 @@ from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError log = logging.getLogger(__name__) + def _commits_as_dict(commit_ids, repos): """ Helper function to serialize commit_ids @@ -102,6 +103,7 @@ def _issues_as_dict(commits): issues[issue['id']] = issue return issues + class RepoEvent(RhodecodeEvent): """ Base class for events acting on a repository. diff --git a/rhodecode/events/user.py b/rhodecode/events/user.py index d5007509..df04cb0b 100644 --- a/rhodecode/events/user.py +++ b/rhodecode/events/user.py @@ -34,6 +34,7 @@ class UserRegistered(RhodecodeEvent): display_name = lazy_ugettext('user registered') def __init__(self, user, session): + super(UserRegistered, self).__init__() self.user = user self.session = session @@ -48,6 +49,7 @@ class UserPreCreate(RhodecodeEvent): display_name = lazy_ugettext('user pre create') def __init__(self, user_data): + super(UserPreCreate, self).__init__() self.user_data = user_data @@ -61,6 +63,7 @@ class UserPostCreate(RhodecodeEvent): display_name = lazy_ugettext('user post create') def __init__(self, user_data): + super(UserPostCreate, self).__init__() self.user_data = user_data @@ -74,5 +77,6 @@ class UserPreUpdate(RhodecodeEvent): display_name = lazy_ugettext('user pre update') def __init__(self, user, user_data): + super(UserPreUpdate, self).__init__() self.user = user self.user_data = user_data From 332dcebbbf0560ea4b484880c4805c6cee0f1657 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Mon, 13 Feb 2017 17:08:28 +0100 Subject: [PATCH 09/15] events: make the System URL extraction safer. Prevents any possible errors inside events. --- rhodecode/events/base.py | 13 +++++++++++-- rhodecode/events/pullrequest.py | 3 +++ rhodecode/events/user.py | 3 +++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/rhodecode/events/base.py b/rhodecode/events/base.py index 7fd7e9bb..1b0188f2 100644 --- a/rhodecode/events/base.py +++ b/rhodecode/events/base.py @@ -15,6 +15,7 @@ # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ +import logging from datetime import datetime from pyramid.threadlocal import get_current_request @@ -26,6 +27,8 @@ SYSTEM_USER = AttributeDict(dict( username='__SYSTEM__' )) +log = logging.getLogger(__name__) + class RhodecodeEvent(object): """ @@ -66,10 +69,16 @@ class RhodecodeEvent(object): @property def server_url(self): + default = '' if self.request: from rhodecode.lib import helpers as h - return h.url('home', qualified=True) - return '' + try: + return h.url('home', qualified=True) + except Exception: + log.exception('Failed to fetch URL for server') + return default + + return default def as_dict(self): data = { diff --git a/rhodecode/events/pullrequest.py b/rhodecode/events/pullrequest.py index b7b12f80..1aad672e 100644 --- a/rhodecode/events/pullrequest.py +++ b/rhodecode/events/pullrequest.py @@ -16,11 +16,14 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ +import logging from rhodecode.translation import lazy_ugettext from rhodecode.events.repo import ( RepoEvent, _commits_as_dict, _issues_as_dict) +log = logging.getLogger(__name__) + class PullRequestEvent(RepoEvent): """ diff --git a/rhodecode/events/user.py b/rhodecode/events/user.py index df04cb0b..a512bc19 100644 --- a/rhodecode/events/user.py +++ b/rhodecode/events/user.py @@ -15,6 +15,7 @@ # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ +import logging from zope.interface import implementer @@ -23,6 +24,8 @@ from rhodecode.events.base import RhodecodeEvent from rhodecode.events.interfaces import ( IUserRegistered, IUserPreCreate, IUserPreUpdate) +log = logging.getLogger(__name__) + @implementer(IUserRegistered) class UserRegistered(RhodecodeEvent): From 1f542f99c0042addac01b4edbb0da5e9abfa2115 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Fri, 10 Feb 2017 16:25:06 +0100 Subject: [PATCH 10/15] http-proto: in case incoming requests come in as chunked stream the data to VCSServer. This should solve a problem of uploading large files to rhodecode. In case of git with small postBuffers GIT client streams data to the server. In such case we want to stream the data back again to vcsserver without reading it fully inside RhodeCode. --- rhodecode/lib/middleware/utils/scm_app_http.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/rhodecode/lib/middleware/utils/scm_app_http.py b/rhodecode/lib/middleware/utils/scm_app_http.py index 6b2137d8..ad88ff69 100644 --- a/rhodecode/lib/middleware/utils/scm_app_http.py +++ b/rhodecode/lib/middleware/utils/scm_app_http.py @@ -93,7 +93,6 @@ class VcsHttpProxy(object): 'X-RC-Locked-Status-Code': rhodecode.CONFIG.get('lock_ret_code') }) - data = environ['wsgi.input'].read() method = environ['REQUEST_METHOD'] # Preserve the query string @@ -104,7 +103,7 @@ class VcsHttpProxy(object): response = session.request( method, url, - data=data, + data=_maybe_stream_request(environ), headers=request_headers, stream=True) @@ -120,10 +119,17 @@ class VcsHttpProxy(object): reason_phrase=response.reason) start_response(status, response_headers) - return _maybe_stream(response) + return _maybe_stream_response(response) -def _maybe_stream(response): +def _maybe_stream_request(environ): + if environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked': + return environ['wsgi.input'] + else: + return environ['wsgi.input'].read() + + +def _maybe_stream_response(response): """ Try to generate chunks from the response if it is chunked. """ From 081b8d5b22a593182f55aef406d8d2a2038843f5 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Mon, 13 Feb 2017 17:49:44 +0100 Subject: [PATCH 11/15] docs: added release notes for 4.6.1 --- docs/release-notes/release-notes-4.6.1.rst | 55 ++++++++++++++++++++++ docs/release-notes/release-notes.rst | 1 + 2 files changed, 56 insertions(+) create mode 100644 docs/release-notes/release-notes-4.6.1.rst diff --git a/docs/release-notes/release-notes-4.6.1.rst b/docs/release-notes/release-notes-4.6.1.rst new file mode 100644 index 00000000..e9b443d0 --- /dev/null +++ b/docs/release-notes/release-notes-4.6.1.rst @@ -0,0 +1,55 @@ +|RCE| 4.6.1 |RNS| +----------------- + +Release Date +^^^^^^^^^^^^ + +- 2017-02-13 + + +New Features +^^^^^^^^^^^^ + + + +General +^^^^^^^ + +- HTTP Protocol: large incoming requests will now properly stream data + into VCSServer. In some cases a large push in GIT can send streaming data. + previously RhodeCode unbundled that data before sending back to VCSServer. + This sometimes caused errors because of wrong headers sent (chunked-encoding) + RhodeCode will now simply stream data back to VCSServer. This should fix the + push problems, and also be much faster for large pushes. + +- Docs: updated contribution and dev setup docs. + + +Security +^^^^^^^^ + + + +Performance +^^^^^^^^^^^ + + + +Fixes +^^^^^ + +- Api: cleanup sessions enforces older_then must be a valid INT. +- Api: validate sent commit_id when using commit_comment API. +- Events: fix a case events were called from API and we + couldn't fetch registered user. + +- Search: goto repository commit search functionality will now use a safe + search option and try not to throw meaningless errors to users from this view. +- Annotations: fixed UI problems in annotation view for newer browsers. + +Upgrade notes +^^^^^^^^^^^^^ + +- Streaming support was changed for push operations. We tested this in several + cases, but please send any feedback if you encounter any problems with it. + diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst index f2df87ea..bf30c304 100644 --- a/docs/release-notes/release-notes.rst +++ b/docs/release-notes/release-notes.rst @@ -9,6 +9,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release-notes-4.6.1.rst release-notes-4.6.0.rst release-notes-4.5.2.rst release-notes-4.5.1.rst From ea1b4588394482fc0e0d3478763c6944492da4b0 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Mon, 13 Feb 2017 18:30:46 +0100 Subject: [PATCH 12/15] release: Start preparation for 4.6.1 --- .bumpversion.cfg | 2 +- .release.cfg | 15 +++++---------- rhodecode/VERSION | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f74293fa..27926f87 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.6.0 +current_version = 4.6.1 message = release: Bump version {current_version} to {new_version} [bumpversion:file:rhodecode/VERSION] diff --git a/.release.cfg b/.release.cfg index 6b283c38..4a0916db 100644 --- a/.release.cfg +++ b/.release.cfg @@ -4,26 +4,21 @@ done = false [task:bump_version] done = true +[task:rc_tools_pinned] + [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [task:changelog_updated] -done = true [task:generate_api_docs] -done = true - -[task:updated_translation] -done = true [release] -state = prepared -version = 4.6.0 +state = in_progress +version = 4.6.1 -[task:rc_tools_pinned] +[task:updated_translation] [task:generate_js_routes] diff --git a/rhodecode/VERSION b/rhodecode/VERSION index 28446a5e..f4fa8fcb 100644 --- a/rhodecode/VERSION +++ b/rhodecode/VERSION @@ -1 +1 @@ -4.6.0 \ No newline at end of file +4.6.1 \ No newline at end of file From 563c39a820030dcf1263bed40744e56102c89135 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Mon, 13 Feb 2017 18:40:15 +0100 Subject: [PATCH 13/15] release: updated pip2nix output for 4.6.1 --- .release.cfg | 4 ++++ pkgs/python-packages.nix | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.release.cfg b/.release.cfg index 4a0916db..5a377e89 100644 --- a/.release.cfg +++ b/.release.cfg @@ -5,14 +5,18 @@ done = false done = true [task:rc_tools_pinned] +done = true [task:fixes_on_stable] +done = true [task:pip2nix_generated] [task:changelog_updated] +done = true [task:generate_api_docs] +done = true [release] state = in_progress diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix index fd1855e0..4022ce65 100644 --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -1498,7 +1498,7 @@ }; }; rhodecode-enterprise-ce = super.buildPythonPackage { - name = "rhodecode-enterprise-ce-4.6.0"; + name = "rhodecode-enterprise-ce-4.6.1"; buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage cssselect lxml configobj]; doCheck = true; propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments pygments-markdown-lexer Pylons Pyro4 Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery channelstream colander decorator deform docutils gevent gunicorn infrae.cache ipython iso8601 kombu msgpack-python packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson subprocess32 waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt]; From 93931f900dbaebf425df44f78188a79d490faa01 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Mon, 13 Feb 2017 18:40:17 +0100 Subject: [PATCH 14/15] release: Finish preparation for 4.6.1 --- .release.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.release.cfg b/.release.cfg index 5a377e89..2fdec30c 100644 --- a/.release.cfg +++ b/.release.cfg @@ -11,6 +11,7 @@ done = true done = true [task:pip2nix_generated] +done = true [task:changelog_updated] done = true @@ -19,7 +20,7 @@ done = true done = true [release] -state = in_progress +state = prepared version = 4.6.1 [task:updated_translation] From 14f3531f1fa6d139c3b75a60fa0e4b42ab07cffc Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Mon, 13 Feb 2017 18:40:17 +0100 Subject: [PATCH 15/15] Added tag v4.6.1 for changeset 5ad13deb9118