From df3eed9bb256a7de121f3332aa35f3391e23b381 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Mon, 5 Feb 2018 13:21:37 +0100 Subject: [PATCH 1/9] meta: fix write condition for meta file. --- rhodecode/subscribers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rhodecode/subscribers.py b/rhodecode/subscribers.py index 269bfa39..e8b2d645 100644 --- a/rhodecode/subscribers.py +++ b/rhodecode/subscribers.py @@ -184,7 +184,7 @@ def write_metadata_if_needed(event): except Exception: age_in_min = 0 - if age_in_min < 60 * 60 * 24: + if age_in_min > 60 * 60 * 24: return try: From a73ab534091e316e8fd0b63735cd62c27742ab72 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Tue, 6 Feb 2018 16:35:46 +0100 Subject: [PATCH 2/9] webhook: fixed extra variable replacement. --- rhodecode/integrations/types/webhook.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rhodecode/integrations/types/webhook.py b/rhodecode/integrations/types/webhook.py index f02e6ed9..6d385e3c 100644 --- a/rhodecode/integrations/types/webhook.py +++ b/rhodecode/integrations/types/webhook.py @@ -93,13 +93,14 @@ class WebhookHandler(object): 'username': data['actor']['username'], 'user_id': data['actor']['user_id'] } + extra_vars = {} for extra_key, extra_val in data['repo']['extra_fields'].items(): - extra_vars['extra:{}'.format(extra_key)] = extra_val + extra_vars['extra__{}'.format(extra_key)] = extra_val common_vars.update(extra_vars) - return string.Template( - self.template_url).safe_substitute(**common_vars) + template_url = self.template_url.replace('${extra:', '${extra__') + return string.Template(template_url).safe_substitute(**common_vars) def repo_push_event_handler(self, event, data): url = self.get_base_parsed_template(data) From db73f0fdd29933e344428b5184c091dbbbba0adf Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Wed, 7 Feb 2018 15:03:42 +0100 Subject: [PATCH 3/9] diffs: in case of text lexers don't do any HL because of pygments newline conversion - split fix for non-standard file endings, e.g only \r - fixes #5422 --- rhodecode/lib/codeblocks.py | 18 +++++++--- rhodecode/lib/diffs.py | 46 ++++++++++++++++++++------ rhodecode/tests/lib/test_codeblocks.py | 26 ++------------- rhodecode/tests/lib/test_diffs.py | 22 ++++++++++-- 4 files changed, 71 insertions(+), 41 deletions(-) diff --git a/rhodecode/lib/codeblocks.py b/rhodecode/lib/codeblocks.py index c90e154d..ad5c4129 100644 --- a/rhodecode/lib/codeblocks.py +++ b/rhodecode/lib/codeblocks.py @@ -24,6 +24,8 @@ from itertools import groupby from pygments import lex from pygments.formatters.html import _get_ttype_class as pygment_token_class +from pygments.lexers.special import TextLexer, Token + from rhodecode.lib.helpers import ( get_lexer_for_filenode, html_escape, get_custom_lexer) from rhodecode.lib.utils2 import AttributeDict @@ -45,7 +47,7 @@ def filenode_as_lines_tokens(filenode, lexer=None): log.debug('Generating file node pygment tokens for %s, %s, org_lexer:%s', lexer, filenode, org_lexer) tokens = tokenize_string(filenode.content, lexer) - lines = split_token_stream(tokens, split_string='\n') + lines = split_token_stream(tokens) rv = list(lines) return rv @@ -59,22 +61,28 @@ def tokenize_string(content, lexer): lexer.stripall = False lexer.stripnl = False lexer.ensurenl = False - for token_type, token_text in lex(content, lexer): + + if isinstance(lexer, TextLexer): + lexed = [(Token.Text, content)] + else: + lexed = lex(content, lexer) + + for token_type, token_text in lexed: yield pygment_token_class(token_type), token_text -def split_token_stream(tokens, split_string=u'\n'): +def split_token_stream(tokens): """ Take a list of (TokenType, text) tuples and split them by a string - >>> split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')]) + split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')]) [(TEXT, 'some'), (TEXT, 'text'), (TEXT, 'more'), (TEXT, 'text')] """ buffer = [] for token_class, token_text in tokens: - parts = token_text.split(split_string) + parts = token_text.split('\n') for part in parts[:-1]: buffer.append((token_class, part)) yield buffer diff --git a/rhodecode/lib/diffs.py b/rhodecode/lib/diffs.py index bf218325..f7cc998f 100644 --- a/rhodecode/lib/diffs.py +++ b/rhodecode/lib/diffs.py @@ -183,13 +183,12 @@ class DiffProcessor(object): :param string: """ - self.cur_diff_size += len(string) if not self.show_full_diff and (self.cur_diff_size > self.diff_limit): raise DiffLimitExceeded('Diff Limit Exceeded') - return safe_unicode(string)\ + return string \ .replace('&', '&')\ .replace('<', '<')\ .replace('>', '>') @@ -278,7 +277,7 @@ class DiffProcessor(object): for chunk in self._diff.chunks(): head = chunk.header - diff = imap(self._escaper, chunk.diff.splitlines(1)) + diff = imap(self._escaper, self.diff_splitter(chunk.diff)) raw_diff = chunk.raw limited_diff = False exceeds_limit = False @@ -529,7 +528,8 @@ class DiffProcessor(object): # a real non-binary diff if head['a_file'] or head['b_file']: - diff = iter(chunk.diff.splitlines(1)) + # simulate splitlines, so we keep the line end part + diff = self.diff_splitter(chunk.diff) # append each file to the diff size raw_chunk_size = len(raw_diff) @@ -608,18 +608,17 @@ class DiffProcessor(object): return diff_container(sorted(_files, key=sorter)) # FIXME: NEWDIFFS: dan: this gets replaced by _new_parse_lines - def _parse_lines(self, diff): + def _parse_lines(self, diff_iter): """ Parse the diff an return data for the template. """ - lineiter = iter(diff) stats = [0, 0] chunks = [] raw_diff = [] try: - line = lineiter.next() + line = diff_iter.next() while line: raw_diff.append(line) @@ -651,7 +650,7 @@ class DiffProcessor(object): 'line': line, }) - line = lineiter.next() + line = diff_iter.next() while old_line < old_end or new_line < new_end: command = ' ' @@ -686,7 +685,7 @@ class DiffProcessor(object): }) raw_diff.append(line) - line = lineiter.next() + line = diff_iter.next() if self._newline_marker.match(line): # we need to append to lines, since this is not @@ -712,13 +711,12 @@ class DiffProcessor(object): chunks = [] raw_diff = [] - diff_iter = imap(lambda s: safe_unicode(s), diff_iter) - try: line = diff_iter.next() while line: raw_diff.append(line) + # match header e.g @@ -0,0 +1 @@\n' match = self._chunk_re.match(line) if not match: @@ -826,6 +824,32 @@ class DiffProcessor(object): idstring = re.sub(r'(?!-)\W', "", idstring).lower() return idstring + @classmethod + def diff_splitter(cls, string): + """ + Diff split that emulates .splitlines() but works only on \n + """ + if not string: + return + elif string == '\n': + yield u'\n' + else: + + has_newline = string.endswith('\n') + elements = string.split('\n') + if has_newline: + # skip last element as it's empty string from newlines + elements = elements[:-1] + + len_elements = len(elements) + + for cnt, line in enumerate(elements, start=1): + last_line = cnt == len_elements + if last_line and not has_newline: + yield safe_unicode(line) + else: + yield safe_unicode(line) + '\n' + def prepare(self, inline_diff=True): """ Prepare the passed udiff for HTML rendering. diff --git a/rhodecode/tests/lib/test_codeblocks.py b/rhodecode/tests/lib/test_codeblocks.py index dc515b32..0914b614 100644 --- a/rhodecode/tests/lib/test_codeblocks.py +++ b/rhodecode/tests/lib/test_codeblocks.py @@ -89,29 +89,9 @@ class TestSplitTokenStream(object): [('type2', u'')], ] - def test_split_token_stream_other_char(self): - lines = list(split_token_stream( - [('type1', 'some\ntext'), ('type2', 'more\n')], - split_string='m')) - - assert lines == [ - [('type1', 'so')], - [('type1', 'e\ntext'), ('type2', '')], - [('type2', 'ore\n')], - ] - - def test_split_token_stream_without_char(self): - lines = list(split_token_stream( - [('type1', 'some\ntext'), ('type2', 'more\n')], - split_string='z')) - - assert lines == [ - [('type1', 'some\ntext'), ('type2', 'more\n')] - ] - def test_split_token_stream_single(self): lines = list(split_token_stream( - [('type1', '\n')], split_string='\n')) + [('type1', '\n')])) assert lines == [ [('type1', '')], @@ -120,7 +100,7 @@ class TestSplitTokenStream(object): def test_split_token_stream_single_repeat(self): lines = list(split_token_stream( - [('type1', '\n\n\n')], split_string='\n')) + [('type1', '\n\n\n')])) assert lines == [ [('type1', '')], @@ -131,7 +111,7 @@ class TestSplitTokenStream(object): def test_split_token_stream_multiple_repeat(self): lines = list(split_token_stream( - [('type1', '\n\n'), ('type2', '\n\n')], split_string='\n')) + [('type1', '\n\n'), ('type2', '\n\n')])) assert lines == [ [('type1', '')], diff --git a/rhodecode/tests/lib/test_diffs.py b/rhodecode/tests/lib/test_diffs.py index f0a659d5..7aff6c41 100644 --- a/rhodecode/tests/lib/test_diffs.py +++ b/rhodecode/tests/lib/test_diffs.py @@ -26,7 +26,7 @@ from rhodecode.lib.diffs import ( DiffProcessor, NEW_FILENODE, DEL_FILENODE, MOD_FILENODE, RENAMED_FILENODE, CHMOD_FILENODE, BIN_FILENODE, COPIED_FILENODE) -from rhodecode.tests.fixture import Fixture +from rhodecode.tests.fixture import Fixture, no_newline_id_generator from rhodecode.lib.vcs.backends.git.repository import GitDiff from rhodecode.lib.vcs.backends.hg.repository import MercurialDiff from rhodecode.lib.vcs.backends.svn.repository import SubversionDiff @@ -162,7 +162,7 @@ def test_diffprocessor_as_html_with_comments(): assert html == expected_html -class TestMixedFilenameEncodings: +class TestMixedFilenameEncodings(object): @pytest.fixture(scope="class") def raw_diff(self): @@ -811,3 +811,21 @@ def test_diff_lib_newlines(diff_fixture_w_content): data = [(x['filename'], x['operation'], x['stats'], x['raw_diff']) for x in diff_proc_d] assert expected_data == data + + +@pytest.mark.parametrize('input_str', [ + '', + '\n', + '\n\n', + 'First\n+second', + 'First\n+second\n', + + '\n\n\n Multi \n\n\n', + '\n\n\n Multi beginning', + 'Multi end \n\n\n', + 'Multi end', + '@@ -0,0 +1 @@\n+test_content \n\n b\n' +], ids=no_newline_id_generator) +def test_splitlines(input_str): + result = DiffProcessor.diff_splitter(input_str) + assert list(result) == input_str.splitlines(True) From 6b097d69a6fada3275a68f21584cb4de27cd8c9f Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Wed, 7 Feb 2018 22:57:10 +0100 Subject: [PATCH 4/9] sec: serialize the repo name in repo checks to prevent potential html injections. --- rhodecode/apps/repository/views/repo_checks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rhodecode/apps/repository/views/repo_checks.py b/rhodecode/apps/repository/views/repo_checks.py index 26651a4c..3014abe4 100644 --- a/rhodecode/apps/repository/views/repo_checks.py +++ b/rhodecode/apps/repository/views/repo_checks.py @@ -27,6 +27,7 @@ from rhodecode.apps._base import BaseAppView from rhodecode.lib import helpers as h from rhodecode.lib.auth import (NotAnonymous, HasRepoPermissionAny) from rhodecode.model.db import Repository +from rhodecode.model.validation_schema.types import RepoNameType log = logging.getLogger(__name__) @@ -43,8 +44,8 @@ class RepoChecksView(BaseAppView): renderer='rhodecode:templates/admin/repos/repo_creating.mako') def repo_creating(self): c = self.load_default_context() - repo_name = self.request.matchdict['repo_name'] + repo_name = RepoNameType().deserialize(None, repo_name) db_repo = Repository.get_by_repo_name(repo_name) # check if maybe repo is already created From 75b4b78e7473c92addb0c3ddf8594a3555d90c5b Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Wed, 7 Feb 2018 23:01:17 +0100 Subject: [PATCH 5/9] docs: added release notes for 4.11.2 --- docs/release-notes/release-notes-4.11.2.rst | 47 +++++++++++++++++++++ docs/release-notes/release-notes.rst | 1 + 2 files changed, 48 insertions(+) create mode 100644 docs/release-notes/release-notes-4.11.2.rst diff --git a/docs/release-notes/release-notes-4.11.2.rst b/docs/release-notes/release-notes-4.11.2.rst new file mode 100644 index 00000000..4f36819c --- /dev/null +++ b/docs/release-notes/release-notes-4.11.2.rst @@ -0,0 +1,47 @@ +|RCE| 4.11.2 |RNS| +------------------ + +Release Date +^^^^^^^^^^^^ + +- 2018-02-08 + + +New Features +^^^^^^^^^^^^ + + + +General +^^^^^^^ + +- git: stabilize long running git processes to prevent potential worker freezes. + + +Security +^^^^^^^^ + +- Repo Checks(low): prevent from html injection on repo-check page. It could + be prepared to fake the displayed text fooling users into unwanted actions. + + +Performance +^^^^^^^^^^^ + + + +Fixes +^^^^^ + +- Diffs: in case of text lexers don't do any HL because of pygments problems. + Fixed newline rendering for certain very unique diffs using Pygments. +- Webhook: fixed extra variable replacement in webhook url. +- Subprocessio: use safe b.read() and prevent potential valueErrors +- Subprocess: use subprocessio helper to run various subprocess commands. +- Git: use subprocessio for hooks execution to use non-blocking subprocess. + + +Upgrade notes +^^^^^^^^^^^^^ + +- Scheduled bugfix release fixing certain requested problems with diffs and, Git. diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst index 7e375495..d5686342 100644 --- a/docs/release-notes/release-notes.rst +++ b/docs/release-notes/release-notes.rst @@ -9,6 +9,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release-notes-4.11.2.rst release-notes-4.11.1.rst release-notes-4.11.0.rst release-notes-4.10.6.rst From 1059058baa8d00591c95f819f89d543b8a8a52f7 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Thu, 8 Feb 2018 20:56:00 +0100 Subject: [PATCH 6/9] release: Start preparation for 4.11.2 --- .bumpversion.cfg | 2 +- .release.cfg | 9 ++------- rhodecode/VERSION | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8c5ea1fb..d72307d2 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.11.1 +current_version = 4.11.2 message = release: Bump version {current_version} to {new_version} [bumpversion:file:rhodecode/VERSION] diff --git a/.release.cfg b/.release.cfg index 89987616..8c487b0c 100644 --- a/.release.cfg +++ b/.release.cfg @@ -5,23 +5,18 @@ done = false done = true [task:rc_tools_pinned] -done = true [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [task:changelog_updated] -done = true [task:generate_api_docs] -done = true [release] -state = prepared -version = 4.11.1 +state = in_progress +version = 4.11.2 [task:updated_translation] diff --git a/rhodecode/VERSION b/rhodecode/VERSION index 012d0f05..13adf3d1 100644 --- a/rhodecode/VERSION +++ b/rhodecode/VERSION @@ -1 +1 @@ -4.11.1 \ No newline at end of file +4.11.2 \ No newline at end of file From 4ee8a8df699ea1a457cb3b1e08855378fe2fdf5a Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Thu, 8 Feb 2018 21:00:18 +0100 Subject: [PATCH 7/9] release: updated pip2nix output for 4.11.2 --- .release.cfg | 4 ++++ pkgs/python-packages.nix | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.release.cfg b/.release.cfg index 8c487b0c..c137ad32 100644 --- a/.release.cfg +++ b/.release.cfg @@ -5,14 +5,18 @@ done = false done = true [task:rc_tools_pinned] +done = true [task:fixes_on_stable] +done = true [task:pip2nix_generated] [task:changelog_updated] +done = true [task:generate_api_docs] +done = true [release] state = in_progress diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix index 51ecb56b..3d4c0f9e 100644 --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -1680,7 +1680,7 @@ }; }; rhodecode-enterprise-ce = super.buildPythonPackage { - name = "rhodecode-enterprise-ce-4.11.1"; + name = "rhodecode-enterprise-ce-4.11.2"; buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage configobj]; doCheck = true; propagatedBuildInputs = with self; [setuptools-scm amqp authomatic Babel Beaker celery Chameleon channelstream click colander configobj cssselect decorator deform docutils dogpile.cache dogpile.core ecdsa FormEncode future futures gnureadline infrae.cache iso8601 itsdangerous Jinja2 billiard kombu lxml Mako Markdown MarkupSafe msgpack-python MySQL-python objgraph packaging Paste PasteDeploy PasteScript pathlib2 peppercorn psutil psycopg2 py-bcrypt pycrypto pycurl pyflakes pygments-markdown-lexer Pygments pyparsing pyramid-beaker pyramid-debugtoolbar pyramid-jinja2 pyramid-mako pyramid pysqlite python-dateutil python-ldap python-memcached python-pam pytz pyzmq py-gfm recaptcha-client redis repoze.lru requests Routes setproctitle simplejson six SQLAlchemy sshpubkeys subprocess32 supervisor Tempita translationstring trollius urllib3 URLObject venusian WebError WebHelpers2 WebHelpers WebOb Whoosh wsgiref zope.cachedescriptors zope.deprecation zope.event zope.interface nbconvert bleach nbformat jupyter-client alembic invoke bumpversion transifex-client gevent greenlet gunicorn waitress uWSGI ipdb ipython CProfileV bottle rhodecode-tools appenlight-client pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage]; From 9fed0a100bc23fb254fb6b6fc13740a2a37cf7ad Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Thu, 8 Feb 2018 21:00:19 +0100 Subject: [PATCH 8/9] release: Finish preparation for 4.11.2 --- .release.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.release.cfg b/.release.cfg index c137ad32..98e6f57b 100644 --- a/.release.cfg +++ b/.release.cfg @@ -11,6 +11,7 @@ done = true done = true [task:pip2nix_generated] +done = true [task:changelog_updated] done = true @@ -19,7 +20,7 @@ done = true done = true [release] -state = in_progress +state = prepared version = 4.11.2 [task:updated_translation] From 1fe35c51dc5a5830c31c5b58f2ee9b16934eee53 Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Thu, 8 Feb 2018 21:00:20 +0100 Subject: [PATCH 9/9] Added tag v4.11.2 for changeset 3982abcfdcc2