engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

caching: add option to cache diffs for commits and pull requests.

Browse source 
- new option for vcs settings to enable caching of diffs and commits
- enables auditing of PR after closing it, and removing commits from source repo
- speed up for large PR upto 20x
- reduced memory usage
This commit is contained in:
Bartłomiej Wołyńczyk 2018-04-17 18:46:20 +02:00
parent 9a54021c80
commit 4637dd151e
15 changed files with 400 additions and 161 deletions
Download patch file Download diff file Expand all files Collapse all files

6
rhodecode/apps/_base/__init__.py
View file

@ -33,6 +33,7 @@ from rhodecode.model import user_group
from rhodecode.model import user
from rhodecode.model.db import User
from rhodecode.model.scm import ScmModel
from rhodecode.model.settings import VcsSettingsModel
log = logging.getLogger(__name__)
@ -256,6 +257,11 @@ class RepoAppView(BaseAppView):
f_path_match = self._get_f_path_unchecked(matchdict, default)
return self.path_filter.assert_path_permissions(f_path_match)
def _get_general_setting(self, target_repo, settings_key, default=False):
settings_model = VcsSettingsModel(repo=target_repo)
settings = settings_model.get_general_settings()
return settings.get(settings_key, default)
class PathFilter(object):

89
rhodecode/apps/repository/views/repo_commits.py
View file

@ -34,17 +34,18 @@ from rhodecode.lib.auth import (
LoginRequired, HasRepoPermissionAnyDecorator, NotAnonymous, CSRFRequired)
from rhodecode.lib.compat import OrderedDict
from rhodecode.lib.diffs import cache_diff, load_cached_diff, diff_cache_exist
from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError
import rhodecode.lib.helpers as h
from rhodecode.lib.utils2 import safe_unicode
from rhodecode.lib.utils2 import safe_unicode, str2bool
from rhodecode.lib.vcs.backends.base import EmptyCommit
from rhodecode.lib.vcs.exceptions import (
RepositoryError, CommitDoesNotExistError, NodeDoesNotExistError)
RepositoryError, CommitDoesNotExistError)
from rhodecode.model.db import ChangesetComment, ChangesetStatus
from rhodecode.model.changeset_status import ChangesetStatusModel
from rhodecode.model.comment import CommentsModel
from rhodecode.model.meta import Session
from rhodecode.model.settings import VcsSettingsModel
log = logging.getLogger(__name__)
@ -152,6 +153,12 @@ class RepoCommitsView(RepoAppView):
return c
def _is_diff_cache_enabled(self, target_repo):
caching_enabled = self._get_general_setting(
target_repo, 'rhodecode_diff_cache')
log.debug('Diff caching enabled: %s', caching_enabled)
return caching_enabled
def _commit(self, commit_id_range, method):
_ = self.request.translate
c = self.load_default_context()
@ -240,43 +247,63 @@ class RepoCommitsView(RepoAppView):
commit2 = commit
commit1 = commit.parents[0] if commit.parents else EmptyCommit()
_diff = self.rhodecode_vcs_repo.get_diff(
commit1, commit2,
ignore_whitespace=ign_whitespace_lcl, context=context_lcl)
diff_processor = diffs.DiffProcessor(
_diff, format='newdiff', diff_limit=diff_limit,
file_limit=file_limit, show_full_diff=c.fulldiff)
commit_changes = OrderedDict()
if method == 'show':
_parsed = diff_processor.prepare()
c.limited_diff = isinstance(_parsed, diffs.LimitedDiffContainer)
_parsed = diff_processor.prepare()
def _node_getter(commit):
def get_node(fname):
try:
return commit.get_node(fname)
except NodeDoesNotExistError:
return None
return get_node
inline_comments = CommentsModel().get_inline_comments(
self.db_repo.repo_id, revision=commit.raw_id)
c.inline_cnt = CommentsModel().get_inline_comments_count(
inline_comments)
c.inline_comments = inline_comments
diffset = codeblocks.DiffSet(
repo_name=self.db_repo_name,
source_node_getter=_node_getter(commit1),
target_node_getter=_node_getter(commit2),
comments=inline_comments)
diffset = self.path_filter.render_patchset_filtered(
diffset, _parsed, commit1.raw_id, commit2.raw_id)
cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(
self.db_repo)
cache_file_path = diff_cache_exist(
cache_path, 'diff', commit.raw_id,
ign_whitespace_lcl, context_lcl, c.fulldiff)
caching_enabled = self._is_diff_cache_enabled(self.db_repo)
force_recache = str2bool(self.request.GET.get('force_recache'))
cached_diff = None
if caching_enabled:
cached_diff = load_cached_diff(cache_file_path)
has_proper_diff_cache = cached_diff and cached_diff.get('diff')
if not force_recache and has_proper_diff_cache:
diffset = cached_diff['diff']
else:
vcs_diff = self.rhodecode_vcs_repo.get_diff(
commit1, commit2,
ignore_whitespace=ign_whitespace_lcl,
context=context_lcl)
diff_processor = diffs.DiffProcessor(
vcs_diff, format='newdiff', diff_limit=diff_limit,
file_limit=file_limit, show_full_diff=c.fulldiff)
_parsed = diff_processor.prepare()
diffset = codeblocks.DiffSet(
repo_name=self.db_repo_name,
source_node_getter=codeblocks.diffset_node_getter(commit1),
target_node_getter=codeblocks.diffset_node_getter(commit2))
diffset = self.path_filter.render_patchset_filtered(
diffset, _parsed, commit1.raw_id, commit2.raw_id)
# save cached diff
if caching_enabled:
cache_diff(cache_file_path, diffset, None)
c.limited_diff = diffset.limited_diff
c.changes[commit.raw_id] = diffset
else:
# TODO(marcink): no cache usage here...
_diff = self.rhodecode_vcs_repo.get_diff(
commit1, commit2,
ignore_whitespace=ign_whitespace_lcl, context=context_lcl)
diff_processor = diffs.DiffProcessor(
_diff, format='newdiff', diff_limit=diff_limit,
file_limit=file_limit, show_full_diff=c.fulldiff)
# downloads/raw we only need RAW diff nothing else
diff = self.path_filter.get_raw_patch(diff_processor)
c.changes[commit.raw_id] = [None, None, None, None, diff, None, None]

13
rhodecode/apps/repository/views/repo_compare.py
View file

@ -295,19 +295,10 @@ class RepoCompareView(RepoAppView):
file_limit=file_limit, show_full_diff=c.fulldiff)
_parsed = diff_processor.prepare()
def _node_getter(commit):
""" Returns a function that returns a node for a commit or None """
def get_node(fname):
try:
return commit.get_node(fname)
except NodeDoesNotExistError:
return None
return get_node
diffset = codeblocks.DiffSet(
repo_name=source_repo.repo_name,
source_node_getter=_node_getter(source_commit),
target_node_getter=_node_getter(target_commit),
source_node_getter=codeblocks.diffset_node_getter(source_commit),
target_node_getter=codeblocks.diffset_node_getter(target_commit),
)
c.diffset = self.path_filter.render_patchset_filtered(
diffset, _parsed, source_ref, target_ref)

164
rhodecode/apps/repository/views/repo_pull_requests.py
View file

@ -34,6 +34,7 @@ from rhodecode.apps._base import RepoAppView, DataGridAppView
from rhodecode.lib import helpers as h, diffs, codeblocks, channelstream
from rhodecode.lib.base import vcs_operation_context
from rhodecode.lib.diffs import load_cached_diff, cache_diff, diff_cache_exist
from rhodecode.lib.ext_json import json
from rhodecode.lib.auth import (
LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator,
@ -41,7 +42,7 @@ from rhodecode.lib.auth import (
from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode
from rhodecode.lib.vcs.backends.base import EmptyCommit, UpdateFailureReason
from rhodecode.lib.vcs.exceptions import (CommitDoesNotExistError,
RepositoryRequirementError, NodeDoesNotExistError, EmptyRepositoryError)
RepositoryRequirementError, EmptyRepositoryError)
from rhodecode.model.changeset_status import ChangesetStatusModel
from rhodecode.model.comment import CommentsModel
from rhodecode.model.db import (func, or_, PullRequest, PullRequestVersion,
@ -201,10 +202,16 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
return data
def _is_diff_cache_enabled(self, target_repo):
caching_enabled = self._get_general_setting(
target_repo, 'rhodecode_diff_cache')
log.debug('Diff caching enabled: %s', caching_enabled)
return caching_enabled
def _get_diffset(self, source_repo_name, source_repo,
source_ref_id, target_ref_id,
target_commit, source_commit, diff_limit, fulldiff,
file_limit, display_inline_comments):
target_commit, source_commit, diff_limit, file_limit,
fulldiff):
vcs_diff = PullRequestModel().get_diff(
source_repo, source_ref_id, target_ref_id)
@ -215,21 +222,11 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
_parsed = diff_processor.prepare()
def _node_getter(commit):
def get_node(fname):
try:
return commit.get_node(fname)
except NodeDoesNotExistError:
return None
return get_node
diffset = codeblocks.DiffSet(
repo_name=self.db_repo_name,
source_repo_name=source_repo_name,
source_node_getter=_node_getter(target_commit),
target_node_getter=_node_getter(source_commit),
comments=display_inline_comments
source_node_getter=codeblocks.diffset_node_getter(target_commit),
target_node_getter=codeblocks.diffset_node_getter(source_commit),
)
diffset = self.path_filter.render_patchset_filtered(
diffset, _parsed, target_commit.raw_id, source_commit.raw_id)
@ -443,42 +440,54 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
commits_source_repo = source_scm
c.commits_source_repo = commits_source_repo
commit_cache = {}
try:
pre_load = ["author", "branch", "date", "message"]
show_revs = pull_request_at_ver.revisions
for rev in show_revs:
comm = commits_source_repo.get_commit(
commit_id=rev, pre_load=pre_load)
c.commit_ranges.append(comm)
commit_cache[comm.raw_id] = comm
# Order here matters, we first need to get target, and then
# the source
target_commit = commits_source_repo.get_commit(
commit_id=safe_str(target_ref_id))
source_commit = commits_source_repo.get_commit(
commit_id=safe_str(source_ref_id))
except CommitDoesNotExistError:
log.warning(
'Failed to get commit from `{}` repo'.format(
commits_source_repo), exc_info=True)
except RepositoryRequirementError:
log.warning(
'Failed to get all required data from repo', exc_info=True)
c.missing_requirements = True
c.ancestor = None # set it to None, to hide it from PR view
try:
ancestor_id = source_scm.get_common_ancestor(
source_commit.raw_id, target_commit.raw_id, target_scm)
c.ancestor_commit = source_scm.get_commit(ancestor_id)
except Exception:
c.ancestor_commit = None
# empty version means latest, so we keep this to prevent
# double caching
version_normalized = version or 'latest'
from_version_normalized = from_version or 'latest'
cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(
target_repo)
cache_file_path = diff_cache_exist(
cache_path, 'pull_request', pull_request_id, version_normalized,
from_version_normalized, source_ref_id, target_ref_id, c.fulldiff)
caching_enabled = self._is_diff_cache_enabled(c.target_repo)
force_recache = str2bool(self.request.GET.get('force_recache'))
cached_diff = None
if caching_enabled:
cached_diff = load_cached_diff(cache_file_path)
has_proper_commit_cache = (
cached_diff and cached_diff.get('commits')
and len(cached_diff.get('commits', [])) == 5
and cached_diff.get('commits')[0]
and cached_diff.get('commits')[3])
if not force_recache and has_proper_commit_cache:
diff_commit_cache = \
(ancestor_commit, commit_cache, missing_requirements,
source_commit, target_commit) = cached_diff['commits']
else:
diff_commit_cache = \
(ancestor_commit, commit_cache, missing_requirements,
source_commit, target_commit) = self.get_commits(
commits_source_repo,
pull_request_at_ver,
source_commit,
source_ref_id,
source_scm,
target_commit,
target_ref_id,
target_scm)
# register our commit range
for comm in commit_cache.values():
c.commit_ranges.append(comm)
c.missing_requirements = missing_requirements
c.ancestor_commit = ancestor_commit
c.statuses = source_repo.statuses(
[x.raw_id for x in c.commit_ranges])
@ -500,12 +509,23 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
c.missing_commits = True
else:
c.inline_comments = display_inline_comments
c.diffset = self._get_diffset(
c.source_repo.repo_name, commits_source_repo,
source_ref_id, target_ref_id,
target_commit, source_commit,
diff_limit, c.fulldiff, file_limit, display_inline_comments)
has_proper_diff_cache = cached_diff and cached_diff.get('commits')
if not force_recache and has_proper_diff_cache:
c.diffset = cached_diff['diff']
(ancestor_commit, commit_cache, missing_requirements,
source_commit, target_commit) = cached_diff['commits']
else:
c.diffset = self._get_diffset(
c.source_repo.repo_name, commits_source_repo,
source_ref_id, target_ref_id,
target_commit, source_commit,
diff_limit, file_limit, c.fulldiff)
# save cached diff
if caching_enabled:
cache_diff(cache_file_path, c.diffset, diff_commit_cache)
c.limited_diff = c.diffset.limited_diff
@ -568,7 +588,6 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
if self._rhodecode_user.user_id in allowed_reviewers:
for co in general_comments:
if co.author.user_id == self._rhodecode_user.user_id:
# each comment has a status change
status = co.status_change
if status:
_ver_pr = status[0].comment.pull_request_version_id
@ -576,6 +595,43 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
return self._get_template_context(c)
def get_commits(
self, commits_source_repo, pull_request_at_ver, source_commit,
source_ref_id, source_scm, target_commit, target_ref_id, target_scm):
commit_cache = collections.OrderedDict()
missing_requirements = False
try:
pre_load = ["author", "branch", "date", "message"]
show_revs = pull_request_at_ver.revisions
for rev in show_revs:
comm = commits_source_repo.get_commit(
commit_id=rev, pre_load=pre_load)
commit_cache[comm.raw_id] = comm
# Order here matters, we first need to get target, and then
# the source
target_commit = commits_source_repo.get_commit(
commit_id=safe_str(target_ref_id))
source_commit = commits_source_repo.get_commit(
commit_id=safe_str(source_ref_id))
except CommitDoesNotExistError:
log.warning(
'Failed to get commit from `{}` repo'.format(
commits_source_repo), exc_info=True)
except RepositoryRequirementError:
log.warning(
'Failed to get all required data from repo', exc_info=True)
missing_requirements = True
ancestor_commit = None
try:
ancestor_id = source_scm.get_common_ancestor(
source_commit.raw_id, target_commit.raw_id, target_scm)
ancestor_commit = source_scm.get_commit(ancestor_id)
except Exception:
ancestor_commit = None
return ancestor_commit, commit_cache, missing_requirements, source_commit, target_commit
def assure_not_empty_repo(self):
_ = self.request.translate

49
rhodecode/lib/codeblocks.py
View file

@ -30,6 +30,7 @@ from rhodecode.lib.helpers import (
get_lexer_for_filenode, html_escape, get_custom_lexer)
from rhodecode.lib.utils2 import AttributeDict, StrictAttributeDict
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.lib.vcs.exceptions import VCSError, NodeDoesNotExistError
from rhodecode.lib.diff_match_patch import diff_match_patch
from rhodecode.lib.diffs import LimitedDiffContainer
from pygments.lexers import get_lexer_by_name
@ -351,6 +352,16 @@ def tokens_diff(old_tokens, new_tokens, use_diff_match_patch=True):
return old_tokens_result, new_tokens_result, similarity
def diffset_node_getter(commit):
def get_node(fname):
try:
return commit.get_node(fname)
except NodeDoesNotExistError:
return None
return get_node
class DiffSet(object):
"""
An object for parsing the diff result from diffs.DiffProcessor and
@ -515,6 +526,7 @@ class DiffSet(object):
if target_file_path in self.comments_store:
for lineno, comments in self.comments_store[target_file_path].items():
left_comments[lineno] = comments
# left comments are one that we couldn't place in diff lines.
# could be outdated, or the diff changed and this line is no
# longer available
@ -551,7 +563,7 @@ class DiffSet(object):
result.lines.extend(
self.parse_lines(before, after, source_file, target_file))
result.unified = self.as_unified(result.lines)
result.unified = list(self.as_unified(result.lines))
result.sideside = result.lines
return result
@ -606,8 +618,9 @@ class DiffSet(object):
original.lineno = before['old_lineno']
original.content = before['line']
original.action = self.action_to_op(before['action'])
original.comments = self.get_comments_for('old',
source_file, before['old_lineno'])
original.get_comment_args = (
source_file, 'o', before['old_lineno'])
if after:
if after['action'] == 'new-no-nl':
@ -619,8 +632,9 @@ class DiffSet(object):
modified.lineno = after['new_lineno']
modified.content = after['line']
modified.action = self.action_to_op(after['action'])
modified.comments = self.get_comments_for('new',
target_file, after['new_lineno'])
modified.get_comment_args = (
target_file, 'n', after['new_lineno'])
# diff the lines
if before_tokens and after_tokens:
@ -649,23 +663,6 @@ class DiffSet(object):
return lines
def get_comments_for(self, version, filename, line_number):
if hasattr(filename, 'unicode_path'):
filename = filename.unicode_path
if not isinstance(filename, basestring):
return None
line_key = {
'old': 'o',
'new': 'n',
}[version] + str(line_number)
if filename in self.comments_store:
file_comments = self.comments_store[filename]
if line_key in file_comments:
return file_comments.pop(line_key)
def get_line_tokens(self, line_text, line_number, file=None):
filenode = None
filename = None
@ -722,25 +719,25 @@ class DiffSet(object):
if line.original.action == ' ':
yield (line.original.lineno, line.modified.lineno,
line.original.action, line.original.content,
line.original.comments)
line.original.get_comment_args)
continue
if line.original.action == '-':
yield (line.original.lineno, None,
line.original.action, line.original.content,
line.original.comments)
line.original.get_comment_args)
if line.modified.action == '+':
buf.append((
None, line.modified.lineno,
line.modified.action, line.modified.content,
line.modified.comments))
line.modified.get_comment_args))
continue
if line.modified:
yield (None, line.modified.lineno,
line.modified.action, line.modified.content,
line.modified.comments)
line.modified.get_comment_args)
for b in buf:
yield b

83
rhodecode/lib/diffs.py
View file

@ -23,16 +23,18 @@
Set of diffing helpers, previously part of vcs
"""
import os
import re
import collections
import difflib
import logging
import cPickle as pickle
from itertools import tee, imap
from rhodecode.lib.vcs.exceptions import VCSError
from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode
from rhodecode.lib.utils2 import safe_unicode
from rhodecode.lib.utils2 import safe_unicode, safe_str
log = logging.getLogger(__name__)
@ -1129,3 +1131,82 @@ class LineNotInDiffException(Exception):
class DiffLimitExceeded(Exception):
pass
def cache_diff(cached_diff_file, diff, commits):
struct = {
'version': 'v1',
'diff': diff,
'commits': commits
}
try:
with open(cached_diff_file, 'wb') as f:
pickle.dump(struct, f)
log.debug('Saved diff cache under %s', cached_diff_file)
except Exception:
log.warn('Failed to save cache', exc_info=True)
# cleanup file to not store it "damaged"
try:
os.remove(cached_diff_file)
except Exception:
log.exception('Failed to cleanup path %s', cached_diff_file)
def load_cached_diff(cached_diff_file):
default_struct = {
'version': 'v1',
'diff': None,
'commits': None
}
has_cache = os.path.isfile(cached_diff_file)
if not has_cache:
return default_struct
data = None
try:
with open(cached_diff_file, 'rb') as f:
data = pickle.load(f)
log.debug('Loaded diff cache from %s', cached_diff_file)
except Exception:
log.warn('Failed to read diff cache file', exc_info=True)
if not data:
data = default_struct
if not isinstance(data, dict):
# old version of data ?
data = default_struct
return data
def generate_diff_cache_key(*args):
"""
Helper to generate a cache key using arguments
"""
def arg_mapper(input_param):
input_param = safe_str(input_param)
# we cannot allow '/' in arguments since it would allow
# subdirectory usage
input_param.replace('/', '_')
return input_param or None # prevent empty string arguments
return '_'.join([
'{}' for i in range(len(args))]).format(*map(arg_mapper, args))
def diff_cache_exist(cache_storage, *args):
"""
Based on all generated arguments check and return a cache path
"""
cache_key = generate_diff_cache_key(*args)
cache_file_path = os.path.join(cache_storage, cache_key)
# prevent path traversal attacks using some param that have e.g '../../'
if not os.path.abspath(cache_file_path).startswith(cache_storage):
raise ValueError('Final path must be within {}'.format(cache_storage))
return cache_file_path

21
rhodecode/lib/utils2.py
View file

@ -709,7 +709,19 @@ def extract_mentioned_users(s):
return sorted(list(usrs), key=lambda k: k.lower())
class StrictAttributeDict(dict):
class AttributeDictBase(dict):
def __getstate__(self):
odict = self.__dict__ # get attribute dictionary
return odict
def __setstate__(self, dict):
self.__dict__ = dict
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
class StrictAttributeDict(AttributeDictBase):
"""
Strict Version of Attribute dict which raises an Attribute error when
requested attribute is not set
@ -720,15 +732,12 @@ class StrictAttributeDict(dict):
except KeyError:
raise AttributeError('%s object has no attribute %s' % (
self.__class__, attr))
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
class AttributeDict(dict):
class AttributeDict(AttributeDictBase):
def __getattr__(self, attr):
return self.get(attr, None)
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
def fix_PATH(os_=None):

14
rhodecode/lib/vcs/backends/base.py
View file

@ -206,6 +206,14 @@ class BaseRepository(object):
def __ne__(self, other):
return not self.__eq__(other)
def get_create_shadow_cache_pr_path(self, repo):
path = os.path.join(
os.path.dirname(self.path),
'.__shadow_diff_cache_repo_{}/'.format(repo.repo_id))
if not os.path.exists(path):
os.makedirs(path, 0755)
return path
@classmethod
def get_default_config(cls, default=None):
config = Config()
@ -745,6 +753,12 @@ class BaseCommit(object):
'branch': self.branch
}
def __getstate__(self):
d = self.__dict__.copy()
d.pop('_remote', None)
d.pop('repository', None)
return d
def _get_refs(self):
return {
'branches': [self.branch] if self.branch else [],

3
rhodecode/model/forms.py
View file

@ -430,6 +430,9 @@ class _BaseVcsSettingsForm(formencode.Schema):
vcs_svn_proxy_http_requests_enabled = v.StringBoolean(if_missing=False)
vcs_svn_proxy_http_server_url = v.UnicodeString(strip=True, if_missing=None)
# cache
rhodecode_diff_cache = v.StringBoolean(if_missing=False)
def ApplicationUiSettingsForm(localizer):
_ = localizer

6
rhodecode/model/settings.py
View file

@ -417,7 +417,9 @@ class VcsSettingsModel(object):
'hg_use_rebase_for_merging',
'hg_close_branch_before_merging',
'git_use_rebase_for_merging',
'git_close_branch_before_merging')
'git_close_branch_before_merging',
'diff_cache',
)
HOOKS_SETTINGS = (
('hooks', 'changegroup.repo_size'),
@ -438,6 +440,7 @@ class VcsSettingsModel(object):
GLOBAL_GIT_SETTINGS = (
('vcs_git_lfs', 'enabled'),
('vcs_git_lfs', 'store_location'))
GLOBAL_SVN_SETTINGS = (
('vcs_svn_proxy', 'http_requests_enabled'),
('vcs_svn_proxy', 'http_server_url'))
@ -571,6 +574,7 @@ class VcsSettingsModel(object):
self._create_or_update_ui(
self.repo_settings, *phases, value=safe_str(data[phases_key]))
def create_or_update_global_hg_settings(self, data):
largefiles, largefiles_store, phases, hgsubversion, evolve \
= self.GLOBAL_HG_SETTINGS

14
rhodecode/templates/base/vcs_settings.mako
View file

@ -312,6 +312,20 @@
</div>
% endif
% if display_globals or repo_type in ['hg', 'git', 'svn']:
<div class="panel panel-default">
<div class="panel-heading" id="vcs-pull-requests-options">
<h3 class="panel-title">${_('Diff cache')}<a class="permalink" href="#vcs-pull-requests-options"> ¶</a></h3>
</div>
<div class="panel-body">
<div class="checkbox">
${h.checkbox('rhodecode_diff_cache' + suffix, 'True', **kwargs)}
<label for="rhodecode_diff_cache${suffix}">${_('Enable caching diffs for pull requests cache and commits')}</label>
</div>
</div>
</div>
% endif
% if display_globals or repo_type in ['hg',]:
<div class="panel panel-default">
<div class="panel-heading" id="vcs-pull-requests-options">

2
rhodecode/templates/changeset/changeset.mako
View file

@ -213,7 +213,7 @@
<%namespace name="cbdiffs" file="/codeblocks/diffs.mako"/>
${cbdiffs.render_diffset_menu()}
${cbdiffs.render_diffset(
c.changes[c.commit.raw_id], commit=c.commit, use_comments=True)}
c.changes[c.commit.raw_id], commit=c.commit, use_comments=True,inline_comments=c.inline_comments )}
</div>
## template for inline comment form

93
rhodecode/templates/codeblocks/diffs.mako
View file

@ -44,13 +44,15 @@ return '%s_%s_%i' % (h.safeid(filename), type, line)
# special file-comments that were deleted in previous versions
# it's used for showing outdated comments for deleted files in a PR
deleted_files_comments=None
deleted_files_comments=None,
# for cache purpose
inline_comments=None
)">
%if use_comments:
<div id="cb-comments-inline-container-template" class="js-template">
${inline_comments_container([])}
${inline_comments_container([], inline_comments)}
</div>
<div class="js-template" id="cb-comment-inline-form-template">
<div class="comment-inline-form ac">
@ -211,9 +213,9 @@ collapse_all = len(diffset.files) > collapse_when_files_over
</td>
</tr>
%if c.diffmode == 'unified':
${render_hunk_lines_unified(hunk, use_comments=use_comments)}
${render_hunk_lines_unified(hunk, use_comments=use_comments, inline_comments=inline_comments)}
%elif c.diffmode == 'sideside':
${render_hunk_lines_sideside(hunk, use_comments=use_comments)}
${render_hunk_lines_sideside(hunk, use_comments=use_comments, inline_comments=inline_comments)}
%else:
<tr class="cb-line">
<td>unknown diff mode</td>
@ -230,7 +232,7 @@ collapse_all = len(diffset.files) > collapse_when_files_over
<td class="cb-lineno cb-context"></td>
<td class="cb-lineno cb-context"></td>
<td class="cb-content cb-context">
${inline_comments_container(comments)}
${inline_comments_container(comments, inline_comments)}
</td>
</tr>
%elif c.diffmode == 'sideside':
@ -239,7 +241,7 @@ collapse_all = len(diffset.files) > collapse_when_files_over
<td class="cb-lineno cb-context"></td>
<td class="cb-content cb-context">
% if lineno.startswith('o'):
${inline_comments_container(comments)}
${inline_comments_container(comments, inline_comments)}
% endif
</td>
@ -247,7 +249,7 @@ collapse_all = len(diffset.files) > collapse_when_files_over
<td class="cb-lineno cb-context"></td>
<td class="cb-content cb-context">
% if lineno.startswith('n'):
${inline_comments_container(comments)}
${inline_comments_container(comments, inline_comments)}
% endif
</td>
</tr>
@ -298,7 +300,7 @@ collapse_all = len(diffset.files) > collapse_when_files_over
<td class="cb-lineno cb-context"></td>
<td class="cb-lineno cb-context"></td>
<td class="cb-content cb-context">
${inline_comments_container(comments_dict['comments'])}
${inline_comments_container(comments_dict['comments'], inline_comments)}
</td>
</tr>
%elif c.diffmode == 'sideside':
@ -310,7 +312,7 @@ collapse_all = len(diffset.files) > collapse_when_files_over
<td class="cb-data cb-context"></td>
<td class="cb-lineno cb-context"></td>
<td class="cb-content cb-context">
${inline_comments_container(comments_dict['comments'])}
${inline_comments_container(comments_dict['comments'], inline_comments)}
</td>
</tr>
%endif
@ -484,12 +486,11 @@ from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
</%def>
<%def name="inline_comments_container(comments)">
<%def name="inline_comments_container(comments, inline_comments)">
<div class="inline-comments">
%for comment in comments:
${commentblock.comment_block(comment, inline=True)}
${commentblock.comment_block(comment, inline=True)}
%endfor
% if comments and comments[-1].outdated:
<span class="btn btn-secondary cb-comment-add-button comment-outdated}"
style="display: none;}">
@ -505,8 +506,23 @@ from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
</div>
</%def>
<%!
def get_comments_for(comments, filename, line_version, line_number):
if hasattr(filename, 'unicode_path'):
filename = filename.unicode_path
if not isinstance(filename, basestring):
return None
line_key = '{}{}'.format(line_version, line_number)
if comments and filename in comments:
file_comments = comments[filename]
if line_key in file_comments:
return file_comments[line_key]
%>
<%def name="render_hunk_lines_sideside(hunk, use_comments=False, inline_comments=None)">
<%def name="render_hunk_lines_sideside(hunk, use_comments=False)">
%for i, line in enumerate(hunk.sideside):
<%
old_line_anchor, new_line_anchor = None, None
@ -521,12 +537,16 @@ from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
data-line-no="${line.original.lineno}"
>
<div>
%if line.original.comments:
<% has_outdated = any([x.outdated for x in line.original.comments]) %>
<% loc = None %>
%if line.original.get_comment_args:
<% loc = get_comments_for(inline_comments, *line.original.get_comment_args) %>
%endif
%if loc:
<% has_outdated = any([x.outdated for x in loc]) %>
% if has_outdated:
<i title="${_('comments including outdated')}:${len(line.original.comments)}" class="icon-comment_toggle" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
<i title="${_('comments including outdated')}:${len(loc)}" class="icon-comment_toggle" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
% else:
<i title="${_('comments')}: ${len(line.original.comments)}" class="icon-comment" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
<i title="${_('comments')}: ${len(loc)}" class="icon-comment" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
% endif
%endif
</div>
@ -548,20 +568,28 @@ from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
${render_add_comment_button()}
%endif
<span class="cb-code">${line.original.action} ${line.original.content or '' | n}</span>
%if use_comments and line.original.lineno and line.original.comments:
${inline_comments_container(line.original.comments)}
%if use_comments and line.original.lineno and loc:
${inline_comments_container(loc, inline_comments)}
%endif
</td>
<td class="cb-data ${action_class(line.modified.action)}"
data-line-no="${line.modified.lineno}"
>
<div>
%if line.modified.comments:
<% has_outdated = any([x.outdated for x in line.modified.comments]) %>
%if line.modified.get_comment_args:
<% lmc = get_comments_for(inline_comments, *line.modified.get_comment_args) %>
%else:
<% lmc = None%>
%endif
%if lmc:
<% has_outdated = any([x.outdated for x in lmc]) %>
% if has_outdated:
<i title="${_('comments including outdated')}:${len(line.modified.comments)}" class="icon-comment_toggle" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
<i title="${_('comments including outdated')}:${len(lmc)}" class="icon-comment_toggle" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
% else:
<i title="${_('comments')}: ${len(line.modified.comments)}" class="icon-comment" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
<i title="${_('comments')}: ${len(lmc)}" class="icon-comment" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
% endif
%endif
</div>
@ -583,8 +611,8 @@ from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
${render_add_comment_button()}
%endif
<span class="cb-code">${line.modified.action} ${line.modified.content or '' | n}</span>
%if use_comments and line.modified.lineno and line.modified.comments:
${inline_comments_container(line.modified.comments)}
%if use_comments and line.modified.lineno and lmc:
${inline_comments_container(lmc, inline_comments)}
%endif
</td>
</tr>
@ -592,8 +620,8 @@ from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
</%def>
<%def name="render_hunk_lines_unified(hunk, use_comments=False)">
%for old_line_no, new_line_no, action, content, comments in hunk.unified:
<%def name="render_hunk_lines_unified(hunk, use_comments=False, inline_comments=None)">
%for old_line_no, new_line_no, action, content, comments_args in hunk.unified:
<%
old_line_anchor, new_line_anchor = None, None
if old_line_no:
@ -604,6 +632,13 @@ from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
<tr class="cb-line">
<td class="cb-data ${action_class(action)}">
<div>
%if comments_args:
<% comments = get_comments_for(inline_comments, *comments_args) %>
%else:
<% comments = None%>
%endif
% if comments:
<% has_outdated = any([x.outdated for x in comments]) %>
% if has_outdated:
@ -642,7 +677,7 @@ from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
%endif
<span class="cb-code">${action} ${content or '' | n}</span>
%if use_comments and comments:
${inline_comments_container(comments)}
${inline_comments_container(comments, inline_comments)}
%endif
</td>
</tr>

3
rhodecode/templates/pullrequests/pullrequest_show.mako
View file

@ -570,7 +570,8 @@
c.diffset, use_comments=True,
collapse_when_files_over=30,
disable_new_comments=not c.allowed_to_comment,
deleted_files_comments=c.deleted_files_comments)}
deleted_files_comments=c.deleted_files_comments,
inline_comments=c.inline_comments)}
</div>
% else:
## skipping commits we need to clear the view for missing commits

1
rhodecode/tests/models/settings/test_vcs_settings.py
View file

@ -44,6 +44,7 @@ GENERAL_FORM_DATA = {
'rhodecode_hg_close_branch_before_merging': True,
'rhodecode_git_use_rebase_for_merging': True,
'rhodecode_git_close_branch_before_merging': True,
'rhodecode_diff_cache': True,
}