From f8f3a39d93b1eee2df91101d50bb35a37e999f91 Mon Sep 17 00:00:00 2001 From: Milka Kuzminski Date: Tue, 23 Feb 2021 08:44:25 +0100 Subject: [PATCH 01/28] comments: forbig removal of comments by anyone except the owners. --- rhodecode/apps/repository/views/repo_commits.py | 4 ++++ rhodecode/apps/repository/views/repo_pull_requests.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/rhodecode/apps/repository/views/repo_commits.py b/rhodecode/apps/repository/views/repo_commits.py index c2b45655..c2606b73 100644 --- a/rhodecode/apps/repository/views/repo_commits.py +++ b/rhodecode/apps/repository/views/repo_commits.py @@ -674,6 +674,10 @@ class RepoCommitsView(RepoAppView): is_repo_comment = comment.repo.repo_id == self.db_repo.repo_id comment_repo_admin = is_repo_admin and is_repo_comment + if comment.draft and not comment_owner: + # We never allow to delete draft comments for other than owners + raise HTTPNotFound() + if super_admin or comment_owner or comment_repo_admin: CommentsModel().delete(comment=comment, auth_user=self._rhodecode_user) Session().commit() diff --git a/rhodecode/apps/repository/views/repo_pull_requests.py b/rhodecode/apps/repository/views/repo_pull_requests.py index fa4aa267..baf4d554 100644 --- a/rhodecode/apps/repository/views/repo_pull_requests.py +++ b/rhodecode/apps/repository/views/repo_pull_requests.py @@ -1748,6 +1748,10 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView): is_repo_comment = comment.repo.repo_name == self.db_repo_name comment_repo_admin = is_repo_admin and is_repo_comment + if comment.draft and not comment_owner: + # We never allow to delete draft comments for other than owners + raise HTTPNotFound() + if super_admin or comment_owner or comment_repo_admin: old_calculated_status = comment.pull_request.calculated_review_status() CommentsModel().delete(comment=comment, auth_user=self._rhodecode_user) From 712064268e377489b4ef8dfe489a06987c374703 Mon Sep 17 00:00:00 2001 From: Milka Kuzminski Date: Thu, 4 Mar 2021 22:08:18 +0100 Subject: [PATCH 02/28] ssh: allow clone by ID via SSH operations. --- rhodecode/apps/ssh_support/lib/backends/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rhodecode/apps/ssh_support/lib/backends/__init__.py b/rhodecode/apps/ssh_support/lib/backends/__init__.py index 9d8a589c..d5a31627 100644 --- a/rhodecode/apps/ssh_support/lib/backends/__init__.py +++ b/rhodecode/apps/ssh_support/lib/backends/__init__.py @@ -89,6 +89,14 @@ class SshWrapper(object): return conn + def maybe_translate_repo_uid(self, repo_name): + if repo_name.startswith('_'): + from rhodecode.model.repo import RepoModel + by_id_match = RepoModel().get_repo_by_id(repo_name) + if by_id_match: + repo_name = by_id_match.repo_name + return repo_name + def get_repo_details(self, mode): vcs_type = mode if mode in ['svn', 'hg', 'git'] else None repo_name = None @@ -97,14 +105,14 @@ class SshWrapper(object): hg_match = re.match(hg_pattern, self.command) if hg_match is not None: vcs_type = 'hg' - repo_name = hg_match.group(1).strip('/') + repo_name = self.maybe_translate_repo_uid(hg_match.group(1).strip('/')) return vcs_type, repo_name, mode git_pattern = r'^git-(receive-pack|upload-pack)\s\'[/]?(\S+?)(|\.git)\'$' git_match = re.match(git_pattern, self.command) if git_match is not None: vcs_type = 'git' - repo_name = git_match.group(2).strip('/') + repo_name = self.maybe_translate_repo_uid(git_match.group(2).strip('/')) mode = git_match.group(1) return vcs_type, repo_name, mode From c8ec0b9bcacb1bbeafa37e618cfacf6b4779ec6c Mon Sep 17 00:00:00 2001 From: Milka Kuzminski Date: Tue, 9 Mar 2021 21:16:34 +0100 Subject: [PATCH 03/28] rcextensions: fixed pre-files extractor return code support. --- rhodecode/config/rcextensions/helpers/extract_pre_files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rhodecode/config/rcextensions/helpers/extract_pre_files.py b/rhodecode/config/rcextensions/helpers/extract_pre_files.py index 9485357b..f9c85e3f 100644 --- a/rhodecode/config/rcextensions/helpers/extract_pre_files.py +++ b/rhodecode/config/rcextensions/helpers/extract_pre_files.py @@ -32,6 +32,7 @@ import json from rhodecode.lib import diffs from rhodecode.lib.vcs.backends.hg.diff import MercurialDiff from rhodecode.lib.vcs.backends.git.diff import GitDiff +from vcsserver.utils import safe_int def get_svn_files(repo, vcs_repo, refs): @@ -74,7 +75,7 @@ def get_svn_files(repo, vcs_repo, refs): # skip dirs continue - parsed_entry['file_size'] = int(stdout.strip()) + parsed_entry['file_size'] = safe_int(stdout.strip()) or 0 files.append(parsed_entry) From dcadd22a7d121c376c6a0230184f72dc36eee6ea Mon Sep 17 00:00:00 2001 From: Milka Kuzminski Date: Tue, 9 Mar 2021 21:16:40 +0100 Subject: [PATCH 04/28] release: Bump version 4.24.0 to 4.25.0 --- .bumpversion.cfg | 2 +- pkgs/python-packages.nix | 2 +- rhodecode/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7bbc7c73..a612c39b 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.24.0 +current_version = 4.25.0 message = release: Bump version {current_version} to {new_version} [bumpversion:file:rhodecode/VERSION] diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix index 7cea3d2a..490ce273 100644 --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -1883,7 +1883,7 @@ self: super: { }; }; "rhodecode-enterprise-ce" = super.buildPythonPackage { - name = "rhodecode-enterprise-ce-4.24.0"; + name = "rhodecode-enterprise-ce-4.25.0"; buildInputs = [ self."pytest" self."py" diff --git a/rhodecode/VERSION b/rhodecode/VERSION index 2c4686f5..71cf3665 100644 --- a/rhodecode/VERSION +++ b/rhodecode/VERSION @@ -1 +1 @@ -4.24.0 \ No newline at end of file +4.25.0 \ No newline at end of file From bfa14dbb3a0b7c741caf793bf3d9ca11c7ccd163 Mon Sep 17 00:00:00 2001 From: Milka Kuzminski Date: Thu, 11 Mar 2021 10:06:40 +0100 Subject: [PATCH 05/28] archives: use a special name for non-hashed archives to fix caching issues. --- .../apps/repository/tests/test_repo_files.py | 22 +++++++++++++++++++ rhodecode/apps/repository/views/repo_files.py | 9 ++++---- rhodecode/templates/summary/summary.mako | 3 ++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/rhodecode/apps/repository/tests/test_repo_files.py b/rhodecode/apps/repository/tests/test_repo_files.py index 2c808546..3768cb7b 100644 --- a/rhodecode/apps/repository/tests/test_repo_files.py +++ b/rhodecode/apps/repository/tests/test_repo_files.py @@ -542,6 +542,28 @@ class TestRepositoryArchival(object): for header in headers: assert header in response.headers.items() + def test_archival_no_hash(self, backend): + backend.enable_downloads() + commit = backend.repo.get_commit(commit_idx=173) + for a_type, content_type, extension in settings.ARCHIVE_SPECS: + + short = 'plain' + extension + fname = commit.raw_id + extension + filename = '%s-%s' % (backend.repo_name, short) + response = self.app.get( + route_path('repo_archivefile', + repo_name=backend.repo_name, + fname=fname, params={'with_hash': 0})) + + assert response.status == '200 OK' + headers = [ + ('Content-Disposition', 'attachment; filename=%s' % filename), + ('Content-Type', '%s' % content_type), + ] + + for header in headers: + assert header in response.headers.items() + @pytest.mark.parametrize('arch_ext',[ 'tar', 'rar', 'x', '..ax', '.zipz', 'tar.gz.tar']) def test_archival_wrong_ext(self, backend, arch_ext): diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py index aec275d9..be093aba 100644 --- a/rhodecode/apps/repository/views/repo_files.py +++ b/rhodecode/apps/repository/views/repo_files.py @@ -325,17 +325,18 @@ class RepoFilesView(RepoAppView): return lf_enabled - def _get_archive_name(self, db_repo_name, commit_sha, ext, subrepos=False, path_sha=''): + def _get_archive_name(self, db_repo_name, commit_sha, ext, subrepos=False, path_sha='', with_hash=True): # original backward compat name of archive clean_name = safe_str(db_repo_name.replace('/', '_')) # e.g vcsserver.zip # e.g vcsserver-abcdefgh.zip # e.g vcsserver-abcdefgh-defghijk.zip - archive_name = '{}{}{}{}{}'.format( + archive_name = '{}{}{}{}{}{}'.format( clean_name, '-sub' if subrepos else '', commit_sha, + '-{}'.format('plain') if not with_hash else '', '-{}'.format(path_sha) if path_sha else '', ext) return archive_name @@ -385,7 +386,7 @@ class RepoFilesView(RepoAppView): # used for cache etc archive_name = self._get_archive_name( self.db_repo_name, commit_sha=short_sha, ext=ext, subrepos=subrepos, - path_sha=path_sha) + path_sha=path_sha, with_hash=with_hash) if not with_hash: short_sha = '' @@ -394,7 +395,7 @@ class RepoFilesView(RepoAppView): # what end client gets served response_archive_name = self._get_archive_name( self.db_repo_name, commit_sha=short_sha, ext=ext, subrepos=subrepos, - path_sha=path_sha) + path_sha=path_sha, with_hash=with_hash) # remove extension from our archive directory name archive_dir_name = response_archive_name[:-len(ext)] diff --git a/rhodecode/templates/summary/summary.mako b/rhodecode/templates/summary/summary.mako index 2fe6ed2d..64876f18 100644 --- a/rhodecode/templates/summary/summary.mako +++ b/rhodecode/templates/summary/summary.mako @@ -95,7 +95,8 @@ $(document).ready(function(){ var fname = selectedReference.raw_id + ext; var href = pyroutes.url('repo_archivefile', { 'repo_name': templateContext.repo_name, - 'fname': fname + 'fname': fname, + 'with_hash': '1' }); // set new label $(this).html(ico + ' {0}{1}'.format(escapeHtml(e.added.text), ext)); From 8c12bc34c3f913b0edaea407d67ac549d6a2143a Mon Sep 17 00:00:00 2001 From: Milka Kuzminski Date: Thu, 11 Mar 2021 10:28:52 +0100 Subject: [PATCH 06/28] archives: if implicit ref name was used to obtain archive, redirect to explicit commit sha so we can have the proper caching for references names. --- rhodecode/apps/repository/views/repo_files.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py index be093aba..a2ae4e21 100644 --- a/rhodecode/apps/repository/views/repo_files.py +++ b/rhodecode/apps/repository/views/repo_files.py @@ -373,6 +373,11 @@ class RepoFilesView(RepoAppView): except EmptyRepositoryError: return Response(_('Empty repository')) + # we used a ref, or a shorter version, lets redirect client ot use explicit hash + if commit_id != commit.raw_id: + fname='{}{}'.format(commit.raw_id, ext) + raise HTTPFound(self.request.current_route_path(fname=fname)) + try: at_path = commit.get_node(at_path).path or default_at_path except Exception: @@ -405,9 +410,10 @@ class RepoFilesView(RepoAppView): cached_archive_path = None if archive_cache_enabled: - # check if we it's ok to write + # check if we it's ok to write, and re-create the archive cache if not os.path.isdir(CONFIG['archive_cache_dir']): os.makedirs(CONFIG['archive_cache_dir']) + cached_archive_path = os.path.join( CONFIG['archive_cache_dir'], archive_name) if os.path.isfile(cached_archive_path): From 2637470cf84ee3d54ceac4b0d5623e981c8c26ad Mon Sep 17 00:00:00 2001 From: Milka Kuzminski Date: Thu, 11 Mar 2021 12:50:55 +0100 Subject: [PATCH 07/28] packaging: fixed few packages requirements for a proper builds --- pkgs/patches/channelstream/setuptools.patch | 13 +++++++++++++ pkgs/patches/pyramid_apispec/setuptools.patch | 12 ++++++++++++ pkgs/python-packages-overrides.nix | 12 ++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 pkgs/patches/channelstream/setuptools.patch create mode 100644 pkgs/patches/pyramid_apispec/setuptools.patch diff --git a/pkgs/patches/channelstream/setuptools.patch b/pkgs/patches/channelstream/setuptools.patch new file mode 100644 index 00000000..2e513de3 --- /dev/null +++ b/pkgs/patches/channelstream/setuptools.patch @@ -0,0 +1,13 @@ +diff -rup channelstream-0.6.14-orig/setup.py channelstream-0.6.14/setup.py + +--- channelstream-0.6.14/setup-orig.py 2021-03-11 12:34:45.000000000 +0100 ++++ channelstream-0.6.14/setup.py 2021-03-11 12:34:56.000000000 +0100 +@@ -52,7 +52,7 @@ setup( + include_package_data=True, + install_requires=requires, + python_requires=">=2.7", +- setup_requires=["pytest-runner"], ++ setup_requires=["pytest-runner==5.1.0"], + extras_require={ + "dev": ["coverage", "pytest", "pyramid", "tox", "mock", "webtest"], + "lint": ["black"], diff --git a/pkgs/patches/pyramid_apispec/setuptools.patch b/pkgs/patches/pyramid_apispec/setuptools.patch new file mode 100644 index 00000000..fad7949b --- /dev/null +++ b/pkgs/patches/pyramid_apispec/setuptools.patch @@ -0,0 +1,12 @@ +diff -rup pyramid-apispec-0.3.2-orig/setup.py pyramid-apispec-0.3.2/setup.py +--- pyramid-apispec-0.3.2-orig/setup.py 2021-03-11 11:19:26.000000000 +0100 ++++ pyramid-apispec-0.3.2/setup.py 2021-03-11 11:19:51.000000000 +0100 +@@ -44,7 +44,7 @@ setup( + packages=find_packages(exclude=["contrib", "docs", "tests"]), + package_data={"pyramid_apispec": ["static/*.*"], "": ["LICENSE"]}, + install_requires=["apispec[yaml]==1.0.0"], +- setup_requires=["pytest-runner"], ++ setup_requires=["pytest-runner==5.1"], + extras_require={ + "dev": ["coverage", "pytest", "pyramid", "tox", "webtest"], + "demo": ["marshmallow==2.15.3", "pyramid", "apispec", "webtest"], \ No newline at end of file diff --git a/pkgs/python-packages-overrides.nix b/pkgs/python-packages-overrides.nix index 51f5b3df..01a1e1e1 100644 --- a/pkgs/python-packages-overrides.nix +++ b/pkgs/python-packages-overrides.nix @@ -280,6 +280,18 @@ self: super: { ]; }); + "pyramid-apispec" = super."pyramid-apispec".override (attrs: { + patches = [ + ./patches/pyramid_apispec/setuptools.patch + ]; + }); + + "channelstream" = super."channelstream".override (attrs: { + patches = [ + ./patches/channelstream/setuptools.patch + ]; + }); + # Avoid that base packages screw up the build process inherit (basePythonPackages) setuptools; From ce1015425ed3c53784857800248205fd85c4a967 Mon Sep 17 00:00:00 2001 From: Milka Kuzminski Date: Thu, 11 Mar 2021 22:20:02 +0100 Subject: [PATCH 08/28] pull-requests: fixed multiple repetitions of referenced tickets in pull requests summary sidebar. --- rhodecode/lib/helpers.py | 47 +++++++++++++++---- .../templates/compare/compare_commits.mako | 4 +- .../pullrequests/pullrequest_show.mako | 29 +++++++----- 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py index 2d463f3e..ea670467 100644 --- a/rhodecode/lib/helpers.py +++ b/rhodecode/lib/helpers.py @@ -25,6 +25,7 @@ Consists of functions to typically be used within templates, but also available to Controllers. This module is available to both as 'h'. """ import base64 +import collections import os import random @@ -1733,7 +1734,7 @@ def process_patterns(text_string, repo_name, link_format='html', active_entries= def urlify_commit_message(commit_text, repository=None, active_pattern_entries=None, - issues_container=None, error_container=None): + issues_container_callback=None, error_container=None): """ Parses given text message and makes proper links. issues are linked to given issue-server, and rest is a commit link @@ -1756,8 +1757,9 @@ def urlify_commit_message(commit_text, repository=None, active_pattern_entries=N new_text, issues, errors = process_patterns( new_text, repository or '', active_entries=active_pattern_entries) - if issues_container is not None: - issues_container.extend(issues) + if issues_container_callback is not None: + for issue in issues: + issues_container_callback(issue) if error_container is not None: error_container.extend(errors) @@ -1802,7 +1804,7 @@ def renderer_from_filename(filename, exclude=None): def render(source, renderer='rst', mentions=False, relative_urls=None, - repo_name=None, active_pattern_entries=None, issues_container=None): + repo_name=None, active_pattern_entries=None, issues_container_callback=None): def maybe_convert_relative_links(html_source): if relative_urls: @@ -1819,8 +1821,9 @@ def render(source, renderer='rst', mentions=False, relative_urls=None, source, issues, errors = process_patterns( source, repo_name, link_format='rst', active_entries=active_pattern_entries) - if issues_container is not None: - issues_container.extend(issues) + if issues_container_callback is not None: + for issue in issues: + issues_container_callback(issue) return literal( '
%s
' % @@ -1833,8 +1836,10 @@ def render(source, renderer='rst', mentions=False, relative_urls=None, source, issues, errors = process_patterns( source, repo_name, link_format='markdown', active_entries=active_pattern_entries) - if issues_container is not None: - issues_container.extend(issues) + if issues_container_callback is not None: + for issue in issues: + issues_container_callback(issue) + return literal( '
%s
' % @@ -2115,3 +2120,29 @@ def is_active(menu_entry, selected): if selected in menu_entry: return "active" + + +class IssuesRegistry(object): + """ + issue_registry = IssuesRegistry() + some_func(issues_callback=issues_registry(...)) + """ + + def __init__(self): + self.issues = [] + self.unique_issues = collections.defaultdict(lambda: []) + + def __call__(self, commit_dict=None): + def callback(issue): + if commit_dict and issue: + issue['commit'] = commit_dict + self.issues.append(issue) + self.unique_issues[issue['id']].append(issue) + return callback + + def get_issues(self): + return self.issues + + @property + def issues_unique_count(self): + return len(set(i['id'] for i in self.issues)) diff --git a/rhodecode/templates/compare/compare_commits.mako b/rhodecode/templates/compare/compare_commits.mako index fe0338a6..e13dbb90 100644 --- a/rhodecode/templates/compare/compare_commits.mako +++ b/rhodecode/templates/compare/compare_commits.mako @@ -24,7 +24,7 @@ ## to speed up lookups cache some functions before the loop <% active_patterns = h.get_active_pattern_entries(c.repo_name) - urlify_commit_message = h.partial(h.urlify_commit_message, active_pattern_entries=active_patterns, issues_container=getattr(c, 'referenced_commit_issues', None)) + urlify_commit_message = h.partial(h.urlify_commit_message, active_pattern_entries=active_patterns) %> %for commit in c.commit_ranges: @@ -57,7 +57,7 @@
-
${urlify_commit_message(commit.message, c.repo_name)}
+
${urlify_commit_message(commit.message, c.repo_name, issues_container_callback=getattr(c, 'referenced_commit_issues', lambda p:p)(commit.serialize()))}
diff --git a/rhodecode/templates/pullrequests/pullrequest_show.mako b/rhodecode/templates/pullrequests/pullrequest_show.mako index b171588e..723cd864 100644 --- a/rhodecode/templates/pullrequests/pullrequest_show.mako +++ b/rhodecode/templates/pullrequests/pullrequest_show.mako @@ -27,8 +27,8 @@ <%def name="main()"> ## Container to gather extracted Tickets <% - c.referenced_commit_issues = [] - c.referenced_desc_issues = [] + c.referenced_commit_issues = h.IssuesRegistry() + c.referenced_desc_issues = h.IssuesRegistry() %>