release: merge back stable branch into default
This commit is contained in:
Marcin Kuzminski
commit
0db43a1ca4
8 changed files with 125 additions and 45 deletions
47
docs/release-notes/release-notes-4.11.2.rst
Normal file
47
docs/release-notes/release-notes-4.11.2.rst
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
|RCE| 4.11.2 |RNS|
|
||||
------------------
|
||||
|
||||
Release Date
|
||||
^^^^^^^^^^^^
|
||||
|
||||
- 2018-02-08
|
||||
|
||||
|
||||
New Features
|
||||
^^^^^^^^^^^^
|
||||
|
||||
|
||||
|
||||
General
|
||||
^^^^^^^
|
||||
|
||||
- git: stabilize long running git processes to prevent potential worker freezes.
|
||||
|
||||
|
||||
Security
|
||||
^^^^^^^^
|
||||
|
||||
- Repo Checks(low): prevent from html injection on repo-check page. It could
|
||||
be prepared to fake the displayed text fooling users into unwanted actions.
|
||||
|
||||
|
||||
Performance
|
||||
^^^^^^^^^^^
|
||||
|
||||
|
||||
|
||||
Fixes
|
||||
^^^^^
|
||||
|
||||
- Diffs: in case of text lexers don't do any HL because of pygments problems.
|
||||
Fixed newline rendering for certain very unique diffs using Pygments.
|
||||
- Webhook: fixed extra variable replacement in webhook url.
|
||||
- Subprocessio: use safe b.read() and prevent potential valueErrors
|
||||
- Subprocess: use subprocessio helper to run various subprocess commands.
|
||||
- Git: use subprocessio for hooks execution to use non-blocking subprocess.
|
||||
|
||||
|
||||
Upgrade notes
|
||||
^^^^^^^^^^^^^
|
||||
|
||||
- Scheduled bugfix release fixing certain requested problems with diffs and, Git.
|
||||
|
|
@ -9,6 +9,7 @@ Release Notes
|
|||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
release-notes-4.11.2.rst
|
||||
release-notes-4.11.1.rst
|
||||
release-notes-4.11.0.rst
|
||||
release-notes-4.10.6.rst
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from rhodecode.apps._base import BaseAppView
|
|||
from rhodecode.lib import helpers as h
|
||||
from rhodecode.lib.auth import (NotAnonymous, HasRepoPermissionAny)
|
||||
from rhodecode.model.db import Repository
|
||||
from rhodecode.model.validation_schema.types import RepoNameType
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -43,8 +44,8 @@ class RepoChecksView(BaseAppView):
|
|||
renderer='rhodecode:templates/admin/repos/repo_creating.mako')
|
||||
def repo_creating(self):
|
||||
c = self.load_default_context()
|
||||
|
||||
repo_name = self.request.matchdict['repo_name']
|
||||
repo_name = RepoNameType().deserialize(None, repo_name)
|
||||
db_repo = Repository.get_by_repo_name(repo_name)
|
||||
|
||||
# check if maybe repo is already created
|
||||
|
|
|
|||
|
|
@ -93,13 +93,14 @@ class WebhookHandler(object):
|
|||
'username': data['actor']['username'],
|
||||
'user_id': data['actor']['user_id']
|
||||
}
|
||||
|
||||
extra_vars = {}
|
||||
for extra_key, extra_val in data['repo']['extra_fields'].items():
|
||||
extra_vars['extra:{}'.format(extra_key)] = extra_val
|
||||
extra_vars['extra__{}'.format(extra_key)] = extra_val
|
||||
common_vars.update(extra_vars)
|
||||
|
||||
return string.Template(
|
||||
self.template_url).safe_substitute(**common_vars)
|
||||
template_url = self.template_url.replace('${extra:', '${extra__')
|
||||
return string.Template(template_url).safe_substitute(**common_vars)
|
||||
|
||||
def repo_push_event_handler(self, event, data):
|
||||
url = self.get_base_parsed_template(data)
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ from itertools import groupby
|
|||
|
||||
from pygments import lex
|
||||
from pygments.formatters.html import _get_ttype_class as pygment_token_class
|
||||
from pygments.lexers.special import TextLexer, Token
|
||||
|
||||
from rhodecode.lib.helpers import (
|
||||
get_lexer_for_filenode, html_escape, get_custom_lexer)
|
||||
from rhodecode.lib.utils2 import AttributeDict
|
||||
|
|
@ -45,7 +47,7 @@ def filenode_as_lines_tokens(filenode, lexer=None):
|
|||
log.debug('Generating file node pygment tokens for %s, %s, org_lexer:%s',
|
||||
lexer, filenode, org_lexer)
|
||||
tokens = tokenize_string(filenode.content, lexer)
|
||||
lines = split_token_stream(tokens, split_string='\n')
|
||||
lines = split_token_stream(tokens)
|
||||
rv = list(lines)
|
||||
return rv
|
||||
|
||||
|
|
@ -59,22 +61,28 @@ def tokenize_string(content, lexer):
|
|||
lexer.stripall = False
|
||||
lexer.stripnl = False
|
||||
lexer.ensurenl = False
|
||||
for token_type, token_text in lex(content, lexer):
|
||||
|
||||
if isinstance(lexer, TextLexer):
|
||||
lexed = [(Token.Text, content)]
|
||||
else:
|
||||
lexed = lex(content, lexer)
|
||||
|
||||
for token_type, token_text in lexed:
|
||||
yield pygment_token_class(token_type), token_text
|
||||
|
||||
|
||||
def split_token_stream(tokens, split_string=u'\n'):
|
||||
def split_token_stream(tokens):
|
||||
"""
|
||||
Take a list of (TokenType, text) tuples and split them by a string
|
||||
|
||||
>>> split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')])
|
||||
split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')])
|
||||
[(TEXT, 'some'), (TEXT, 'text'),
|
||||
(TEXT, 'more'), (TEXT, 'text')]
|
||||
"""
|
||||
|
||||
buffer = []
|
||||
for token_class, token_text in tokens:
|
||||
parts = token_text.split(split_string)
|
||||
parts = token_text.split('\n')
|
||||
for part in parts[:-1]:
|
||||
buffer.append((token_class, part))
|
||||
yield buffer
|
||||
|
|
|
|||
|
|
@ -183,13 +183,12 @@ class DiffProcessor(object):
|
|||
|
||||
:param string:
|
||||
"""
|
||||
|
||||
self.cur_diff_size += len(string)
|
||||
|
||||
if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
|
||||
raise DiffLimitExceeded('Diff Limit Exceeded')
|
||||
|
||||
return safe_unicode(string)\
|
||||
return string \
|
||||
.replace('&', '&')\
|
||||
.replace('<', '<')\
|
||||
.replace('>', '>')
|
||||
|
|
@ -278,7 +277,7 @@ class DiffProcessor(object):
|
|||
for chunk in self._diff.chunks():
|
||||
head = chunk.header
|
||||
|
||||
diff = imap(self._escaper, chunk.diff.splitlines(1))
|
||||
diff = imap(self._escaper, self.diff_splitter(chunk.diff))
|
||||
raw_diff = chunk.raw
|
||||
limited_diff = False
|
||||
exceeds_limit = False
|
||||
|
|
@ -529,7 +528,8 @@ class DiffProcessor(object):
|
|||
|
||||
# a real non-binary diff
|
||||
if head['a_file'] or head['b_file']:
|
||||
diff = iter(chunk.diff.splitlines(1))
|
||||
# simulate splitlines, so we keep the line end part
|
||||
diff = self.diff_splitter(chunk.diff)
|
||||
|
||||
# append each file to the diff size
|
||||
raw_chunk_size = len(raw_diff)
|
||||
|
|
@ -608,18 +608,17 @@ class DiffProcessor(object):
|
|||
return diff_container(sorted(_files, key=sorter))
|
||||
|
||||
# FIXME: NEWDIFFS: dan: this gets replaced by _new_parse_lines
|
||||
def _parse_lines(self, diff):
|
||||
def _parse_lines(self, diff_iter):
|
||||
"""
|
||||
Parse the diff an return data for the template.
|
||||
"""
|
||||
|
||||
lineiter = iter(diff)
|
||||
stats = [0, 0]
|
||||
chunks = []
|
||||
raw_diff = []
|
||||
|
||||
try:
|
||||
line = lineiter.next()
|
||||
line = diff_iter.next()
|
||||
|
||||
while line:
|
||||
raw_diff.append(line)
|
||||
|
|
@ -651,7 +650,7 @@ class DiffProcessor(object):
|
|||
'line': line,
|
||||
})
|
||||
|
||||
line = lineiter.next()
|
||||
line = diff_iter.next()
|
||||
|
||||
while old_line < old_end or new_line < new_end:
|
||||
command = ' '
|
||||
|
|
@ -686,7 +685,7 @@ class DiffProcessor(object):
|
|||
})
|
||||
raw_diff.append(line)
|
||||
|
||||
line = lineiter.next()
|
||||
line = diff_iter.next()
|
||||
|
||||
if self._newline_marker.match(line):
|
||||
# we need to append to lines, since this is not
|
||||
|
|
@ -712,13 +711,12 @@ class DiffProcessor(object):
|
|||
chunks = []
|
||||
raw_diff = []
|
||||
|
||||
diff_iter = imap(lambda s: safe_unicode(s), diff_iter)
|
||||
|
||||
try:
|
||||
line = diff_iter.next()
|
||||
|
||||
while line:
|
||||
raw_diff.append(line)
|
||||
# match header e.g @@ -0,0 +1 @@\n'
|
||||
match = self._chunk_re.match(line)
|
||||
|
||||
if not match:
|
||||
|
|
@ -826,6 +824,32 @@ class DiffProcessor(object):
|
|||
idstring = re.sub(r'(?!-)\W', "", idstring).lower()
|
||||
return idstring
|
||||
|
||||
@classmethod
|
||||
def diff_splitter(cls, string):
|
||||
"""
|
||||
Diff split that emulates .splitlines() but works only on \n
|
||||
"""
|
||||
if not string:
|
||||
return
|
||||
elif string == '\n':
|
||||
yield u'\n'
|
||||
else:
|
||||
|
||||
has_newline = string.endswith('\n')
|
||||
elements = string.split('\n')
|
||||
if has_newline:
|
||||
# skip last element as it's empty string from newlines
|
||||
elements = elements[:-1]
|
||||
|
||||
len_elements = len(elements)
|
||||
|
||||
for cnt, line in enumerate(elements, start=1):
|
||||
last_line = cnt == len_elements
|
||||
if last_line and not has_newline:
|
||||
yield safe_unicode(line)
|
||||
else:
|
||||
yield safe_unicode(line) + '\n'
|
||||
|
||||
def prepare(self, inline_diff=True):
|
||||
"""
|
||||
Prepare the passed udiff for HTML rendering.
|
||||
|
|
|
|||
|
|
@ -89,29 +89,9 @@ class TestSplitTokenStream(object):
|
|||
[('type2', u'')],
|
||||
]
|
||||
|
||||
def test_split_token_stream_other_char(self):
|
||||
lines = list(split_token_stream(
|
||||
[('type1', 'some\ntext'), ('type2', 'more\n')],
|
||||
split_string='m'))
|
||||
|
||||
assert lines == [
|
||||
[('type1', 'so')],
|
||||
[('type1', 'e\ntext'), ('type2', '')],
|
||||
[('type2', 'ore\n')],
|
||||
]
|
||||
|
||||
def test_split_token_stream_without_char(self):
|
||||
lines = list(split_token_stream(
|
||||
[('type1', 'some\ntext'), ('type2', 'more\n')],
|
||||
split_string='z'))
|
||||
|
||||
assert lines == [
|
||||
[('type1', 'some\ntext'), ('type2', 'more\n')]
|
||||
]
|
||||
|
||||
def test_split_token_stream_single(self):
|
||||
lines = list(split_token_stream(
|
||||
[('type1', '\n')], split_string='\n'))
|
||||
[('type1', '\n')]))
|
||||
|
||||
assert lines == [
|
||||
[('type1', '')],
|
||||
|
|
@ -120,7 +100,7 @@ class TestSplitTokenStream(object):
|
|||
|
||||
def test_split_token_stream_single_repeat(self):
|
||||
lines = list(split_token_stream(
|
||||
[('type1', '\n\n\n')], split_string='\n'))
|
||||
[('type1', '\n\n\n')]))
|
||||
|
||||
assert lines == [
|
||||
[('type1', '')],
|
||||
|
|
@ -131,7 +111,7 @@ class TestSplitTokenStream(object):
|
|||
|
||||
def test_split_token_stream_multiple_repeat(self):
|
||||
lines = list(split_token_stream(
|
||||
[('type1', '\n\n'), ('type2', '\n\n')], split_string='\n'))
|
||||
[('type1', '\n\n'), ('type2', '\n\n')]))
|
||||
|
||||
assert lines == [
|
||||
[('type1', '')],
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from rhodecode.lib.diffs import (
|
|||
DiffProcessor,
|
||||
NEW_FILENODE, DEL_FILENODE, MOD_FILENODE, RENAMED_FILENODE,
|
||||
CHMOD_FILENODE, BIN_FILENODE, COPIED_FILENODE)
|
||||
from rhodecode.tests.fixture import Fixture
|
||||
from rhodecode.tests.fixture import Fixture, no_newline_id_generator
|
||||
from rhodecode.lib.vcs.backends.git.repository import GitDiff
|
||||
from rhodecode.lib.vcs.backends.hg.repository import MercurialDiff
|
||||
from rhodecode.lib.vcs.backends.svn.repository import SubversionDiff
|
||||
|
|
@ -162,7 +162,7 @@ def test_diffprocessor_as_html_with_comments():
|
|||
assert html == expected_html
|
||||
|
||||
|
||||
class TestMixedFilenameEncodings:
|
||||
class TestMixedFilenameEncodings(object):
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def raw_diff(self):
|
||||
|
|
@ -811,3 +811,21 @@ def test_diff_lib_newlines(diff_fixture_w_content):
|
|||
data = [(x['filename'], x['operation'], x['stats'], x['raw_diff'])
|
||||
for x in diff_proc_d]
|
||||
assert expected_data == data
|
||||
|
||||
|
||||
@pytest.mark.parametrize('input_str', [
|
||||
'',
|
||||
'\n',
|
||||
'\n\n',
|
||||
'First\n+second',
|
||||
'First\n+second\n',
|
||||
|
||||
'\n\n\n Multi \n\n\n',
|
||||
'\n\n\n Multi beginning',
|
||||
'Multi end \n\n\n',
|
||||
'Multi end',
|
||||
'@@ -0,0 +1 @@\n+test_content \n\n b\n'
|
||||
], ids=no_newline_id_generator)
|
||||
def test_splitlines(input_str):
|
||||
result = DiffProcessor.diff_splitter(input_str)
|
||||
assert list(result) == input_str.splitlines(True)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue