From b6c2ac4521a36c853bdb1c4be587289322950dfe Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Mon, 2 Jun 2025 13:50:04 +0200 Subject: [PATCH 1/8] search: remove whoosh, and add support for ES 8 & 9 --- docs/admin/system_admin/indexing.rst | 25 ++---- rhodecode/api/views/repo_api.py | 5 +- rhodecode/apps/home/views.py | 6 +- rhodecode/apps/search/views.py | 6 +- rhodecode/lib/index/__init__.py | 27 +++--- rhodecode/templates/search/search.mako | 49 +++-------- .../templates/search/search_content.mako | 86 ++++++++++--------- 7 files changed, 88 insertions(+), 116 deletions(-) diff --git a/docs/admin/system_admin/indexing.rst b/docs/admin/system_admin/indexing.rst index db9ee842..88b29fee 100644 --- a/docs/admin/system_admin/indexing.rst +++ b/docs/admin/system_admin/indexing.rst @@ -315,9 +315,7 @@ Enabling ElasticSearch ^^^^^^^^^^^^^^^^^^^^^^ ElasticSearch is available in EE edition only. It provides much scalable and more advanced -search capabilities. While Whoosh is fine for upto 1-2GB of data, beyond that amount it -starts slowing down, and can cause other problems. -New ElasticSearch 6 also provides much more advanced query language. +search capabilities. New ElasticSearch also provides much more advanced query language. It allows advanced filtering by file paths, extensions, use OR statements, ranges etc. Please check query language examples in the search field for some advanced query language usage. @@ -325,25 +323,15 @@ Please check query language examples in the search field for some advanced query 1. Open the :file:`rhodecode.ini` file for the instance you wish to edit. The default location is :file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini` -2. Find the search configuration section: -.. code-block:: ini - - ################################### - ## SEARCH INDEXING CONFIGURATION ## - ################################### - - search.module = rhodecode.lib.index.whoosh - search.location = %(here)s/data/index - -and change it to: +2. Find the search configuration section and change it to: .. code-block:: ini search.module = rc_elasticsearch - search.location = http://localhost:9200 - ## specify Elastic Search version, 6 for latest or 2 for legacy - search.es_version = 6 + search.location = http://elasticsearch:9200 + ## specify Elastic Search version, 8 for latest + search.es_version = 8 where ``search.location`` points to the ElasticSearch server by default running on port 9200. @@ -352,8 +340,7 @@ Index invocation also needs change. Please provide --es-version= and --engine-location= parameters to define ElasticSearch server location and it's version. For example:: - rhodecode-index --instace-name=enterprise-1 --es-version=6 --engine-location=http://localhost:9200 + rhodecode-index --instance-name=enterprise-1 --es-version=8 --engine-location=http://elasticsearch:9200 -.. _Whoosh: https://pypi.python.org/pypi/Whoosh/ .. _ElasticSearch 6: https://www.elastic.co/ diff --git a/rhodecode/api/views/repo_api.py b/rhodecode/api/views/repo_api.py index bdd90be6..593a1fa6 100644 --- a/rhodecode/api/views/repo_api.py +++ b/rhodecode/api/views/repo_api.py @@ -39,6 +39,7 @@ from rhodecode.lib import audit_logger, rc_cache, channelstream from rhodecode.lib import repo_maintenance from rhodecode.lib.auth import HasPermissionAnyApi, HasUserGroupPermissionAnyApi, HasRepoPermissionAnyApi from rhodecode.lib.celerylib.utils import get_task_id +from rhodecode.lib.str_utils import safe_bytes from rhodecode.lib.utils2 import str2bool, time_to_datetime, safe_str, safe_int from rhodecode.lib.ext_json import json from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError, CommentVersionMismatch @@ -615,7 +616,7 @@ def get_repo_file( elif details == "full": extended_info = content = True - file_path = safe_str(file_path) + bytes_path = safe_bytes(file_path) try: # check if repo is not empty by any chance, skip quicker if it is. _scm = repo.scm_instance() @@ -625,7 +626,7 @@ def get_repo_file( node = ScmModel().get_node( repo, commit_id, - file_path, + bytes_path, extended_info=extended_info, content=content, max_file_bytes=max_file_bytes, diff --git a/rhodecode/apps/home/views.py b/rhodecode/apps/home/views.py index 78b84b72..583b1a76 100644 --- a/rhodecode/apps/home/views.py +++ b/rhodecode/apps/home/views.py @@ -483,7 +483,7 @@ class HomeView(BaseAppView, DataGridAppView): if not searcher: return [] - is_es_6 = searcher.is_es_6 + is_es_8 = searcher.is_es_8 queries = [] repo_group_name, repo_name, repo_context = None, None, None @@ -495,7 +495,7 @@ class HomeView(BaseAppView, DataGridAppView): repo_name = search_context.get("search_context[repo_name]") repo_context = search_context.get("search_context[repo_view_type]") - if is_es_6 and repo_name: + if is_es_8 and repo_name: # files def query_modifier(): qry = query @@ -535,7 +535,7 @@ class HomeView(BaseAppView, DataGridAppView): else: queries.extend([commit_qry, file_qry]) - elif is_es_6 and repo_group_name: + elif is_es_8 and repo_group_name: # files def query_modifier(): qry = query diff --git a/rhodecode/apps/search/views.py b/rhodecode/apps/search/views.py index 3bccfe51..0dc3c125 100644 --- a/rhodecode/apps/search/views.py +++ b/rhodecode/apps/search/views.py @@ -90,10 +90,12 @@ def perform_search(request, tmpl_context, repo_name=None, repo_group_name=None): sort=search_sort, ) + search_result_count = search_result["count"]["value"] + formatted_results = Page( search_result["results"], page=requested_page, - item_count=search_result["count"], + item_count=search_result_count, items_per_page=page_limit, url_maker=url_generator, ) @@ -103,7 +105,7 @@ def perform_search(request, tmpl_context, repo_name=None, repo_group_name=None): search_tags = searcher.extract_search_tags(search_query) if not search_result["error"]: - execution_time = "{} results ({:.4f} seconds)".format(search_result["count"], search_result["runtime"]) + execution_time = "{} results ({:.4f} seconds)".format(search_result_count, search_result["runtime"]) elif not errors: node = schema["search_query"] errors = [validation_schema.Invalid(node, search_result["error"])] diff --git a/rhodecode/lib/index/__init__.py b/rhodecode/lib/index/__init__.py index 122c8d5e..55e4ece8 100644 --- a/rhodecode/lib/index/__init__.py +++ b/rhodecode/lib/index/__init__.py @@ -28,16 +28,19 @@ from rhodecode.lib.index.search_utils import normalize_text_for_matching log = logging.getLogger(__name__) # leave defaults for backward compat -default_searcher = "rhodecode.lib.index.whoosh" -default_location = "%(here)s/data/index" +default_searcher = "rc_elasticsearch" + +ES_VERSION_8 = 8 +ES_VERSION_9 = 9 +OS_VERSION_2 = 2 + +# for legacy reasons we keep 8 compat as default +DEFAULT_ES_VERSION = ES_VERSION_8 +DEFAULT_OS_VERSION = OS_VERSION_2 -ES_VERSION_2 = "2" -ES_VERSION_6 = "6" -# for legacy reasons we keep 2 compat as default -DEFAULT_ES_VERSION = ES_VERSION_2 try: - from rhodecode_tools.lib.fts_index.elasticsearch_engine_6 import ES_CONFIG # pragma: no cover + from rhodecode_tools.lib.fts_index.elasticsearch_engine_8 import ES_CONFIG # pragma: no cover except ImportError: log.warning("rhodecode_tools not available, use of full text search is limited") pass @@ -71,8 +74,8 @@ class BaseSearcher(object): return " ".join(normalize_text_for_matching(query).split()) @property - def is_es_6(self): - return self.es_version == ES_VERSION_6 + def is_es_8(self): + return self.es_version >= ES_VERSION_8 def get_handlers(self): return {} @@ -140,11 +143,9 @@ def search_config(config, prefix="search."): def searcher_from_config(config, prefix="search."): _config = search_config(config, prefix) - if "location" not in _config: - _config["location"] = default_location if "es_version" not in _config: - # use an old legacy ES version set to 2 - _config["es_version"] = "2" + # use an old legacy ES version set to 8 + _config["es_version"] = DEFAULT_ES_VERSION imported = importlib.import_module(_config.get("module", default_searcher)) searcher = imported.Searcher(config=_config) diff --git a/rhodecode/templates/search/search.mako b/rhodecode/templates/search/search.mako index 9d20c3d4..1b89e409 100644 --- a/rhodecode/templates/search/search.mako +++ b/rhodecode/templates/search/search.mako @@ -154,53 +154,32 @@ % endfor

${_('Query Language examples')}

- +
${c.runtime}
diff --git a/rhodecode/templates/search/search_content.mako b/rhodecode/templates/search/search_content.mako index 9926c383..fe981192 100644 --- a/rhodecode/templates/search/search_content.mako +++ b/rhodecode/templates/search/search_content.mako @@ -1,8 +1,39 @@ <%namespace name="search" file="/search/search.mako"/> -<%def name="highlight_text_file(has_matched_content, file_content, lexer, html_formatter, matching_lines, shown_matching_lines, url, use_hl_filter)"> +<%def name="highlight_text_file(file_path, full_file_content, hl_file_content, mimetype, url, use_hl_filter)"> +<% + +matching_lines = [] +shown_matching_lines = 0 + +if c.searcher.is_es_8: + # use empty terms so we default to markers usage + total_lines, matching_lines = h.get_matching_line_offsets(hl_file_content, terms=None) + +lines_of_interest = set() +for line_number in matching_lines: + if len(lines_of_interest) < max_lines: + lines_of_interest |= set(range( + max(line_number - line_context, 0), + min(line_number + line_context, total_lines + 1))) + shown_matching_lines += 1 + +has_matched_content = len(lines_of_interest) >= 1 +%> + + % if has_matched_content: - ${h.code_highlight(file_content, lexer, html_formatter, use_hl_filter=use_hl_filter)|n} + <% + lexer = h.get_lexer_safe(mimetype=mimetype, filepath=filepath) + html_formatter = h.SearchContentCodeHtmlFormatter( + linenos=True, + cssclass="code-highlight", + url=match_file_url, + query_terms=terms, + only_line_numbers=lines_of_interest + ) + %> + ${h.code_highlight(hl_file_content, lexer, html_formatter, use_hl_filter=use_hl_filter)|n} % else: ${_('No content matched')}
% endif @@ -20,41 +51,15 @@ %for entry in c.formatted_results: <% - file_content = entry['content_highlight'] or entry['content'] - mimetype = entry.get('mimetype') + hl_file_content = entry['content_highlight'] + full_file_content = entry['content'] filepath = entry.get('path') + mimetype = entry.get('mimetype') max_lines = h.safe_int(request.GET.get('max_lines', '10')) - line_context = h.safe_int(request.GET.get('line_contenxt', '3')) + line_context = h.safe_int(request.GET.get('line_context', '3')) match_file_url=h.route_path('repo_files',repo_name=entry['repository'], commit_id=entry.get('commit_id', 'tip'),f_path=entry['f_path'], _query={"mark": query_mark}) terms = c.cur_query - - if c.searcher.is_es_6: - # use empty terms so we default to markers usage - total_lines, matching_lines = h.get_matching_line_offsets(file_content, terms=None) - else: - total_lines, matching_lines = h.get_matching_line_offsets(file_content, terms) - - shown_matching_lines = 0 - lines_of_interest = set() - for line_number in matching_lines: - if len(lines_of_interest) < max_lines: - lines_of_interest |= set(range( - max(line_number - line_context, 0), - min(line_number + line_context, total_lines + 1))) - shown_matching_lines += 1 - lexer = h.get_lexer_safe(mimetype=mimetype, filepath=filepath) - - html_formatter = h.SearchContentCodeHtmlFormatter( - linenos=True, - cssclass="code-highlight", - url=match_file_url, - query_terms=terms, - only_line_numbers=lines_of_interest - ) - - has_matched_content = len(lines_of_interest) >= 1 - %> ## search results are additionally filtered, and this check is just a safe gate % if c.rhodecode_user.is_admin or h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results content check'): @@ -75,7 +80,7 @@
${entry.get('lines', 0.)} ${_ungettext('line', 'lines', entry.get('lines', 0.))} - (${len(matching_lines)} ${_ungettext('matched', 'matched', len(matching_lines))}) + ##(${len(matching_lines)} ${_ungettext('matched', 'matched', len(matching_lines))}) % if entry.get('size'): @@ -112,7 +117,7 @@ ${_('Narrow to this repository group')} % endif - ## hiden if in repo view + ## hidden if in repo view % if not c.repo_name: ${search.repo_icon(repo_type)} @@ -125,17 +130,14 @@
-
${highlight_text_file( - has_matched_content=has_matched_content, - file_content=file_content, - lexer=lexer, - html_formatter=html_formatter, - matching_lines=matching_lines, - shown_matching_lines=shown_matching_lines, + file_path=file_path, + full_file_content=full_file_content, + hl_file_content=hl_file_content, + mimetype=mimetype, url=match_file_url, - use_hl_filter=c.searcher.is_es_6 + use_hl_filter=c.searcher.is_es_8 )}
From 65cd564db6893f2ce994578f9270a3a04fae4c1f Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Wed, 4 Jun 2025 08:31:17 +0200 Subject: [PATCH 2/8] deps: bumped rhodecode-tools to 4.2.0 and its dependencies --- requirements.txt | 50 +++++++++++++++++++-------------------- requirements_rc_tools.txt | 4 ++-- requirements_test.txt | 2 +- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/requirements.txt b/requirements.txt index f2c82d1c..b048ad11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,23 @@ # deps, generated via pipdeptree --exclude setuptools,wheel,pipdeptree,pip -f | tr '[:upper:]' '[:lower:]' alembic==1.13.1 - mako==1.3.6 + mako==1.3.10 markupsafe==3.0.2 sqlalchemy==1.4.52 greenlet==3.1.1 - typing_extensions==4.12.2 + typing_extensions==4.13.2 async-timeout==4.0.3 babel==2.12.1 beaker==1.13.0 celery==5.4.0 billiard==4.2.1 - click==8.1.7 + click==8.1.8 click-didyoumean==0.3.1 - click==8.1.7 + click==8.1.8 click-plugins==1.1.1 - click==8.1.7 + click==8.1.8 click-repl==0.3.0 - click==8.1.7 + click==8.1.8 prompt_toolkit==3.0.48 wcwidth==0.2.13 kombu==5.4.2 @@ -26,7 +26,7 @@ celery==5.4.0 tzdata==2024.2 vine==5.1.0 python-dateutil==2.9.0.post0 - six==1.16.0 + six==1.17.0 tzdata==2024.2 vine==5.1.0 channelstream==0.7.1 @@ -64,10 +64,10 @@ channelstream==0.7.1 zope.interface==7.2.0 zope.deprecation==5.1.0 python-dateutil==2.9.0.post0 - six==1.16.0 + six==1.17.0 requests==2.32.3 - certifi==2024.8.30 - charset-normalizer==3.4.0 + certifi==2025.1.31 + charset-normalizer==3.4.1 idna==3.10 urllib3==1.26.20 ws4py==0.5.1 @@ -86,7 +86,7 @@ dogpile.cache==1.3.4 stevedore==5.1.0 pbr==5.11.1 formencode==2.1.0 - six==1.16.0 + six==1.17.0 fsspec==2024.12.0 gunicorn==23.0.0 packaging==24.1 @@ -107,11 +107,11 @@ ipython==8.26.0 pygments==2.18.0 stack-data==0.6.3 asttokens==2.4.1 - six==1.16.0 + six==1.17.0 executing==2.0.1 pure_eval==0.2.3 traitlets==5.14.3 - typing_extensions==4.12.2 + typing_extensions==4.13.2 markdown==3.4.3 msgpack==1.1.0 mysqlclient==2.1.1 @@ -119,7 +119,7 @@ nbconvert==7.7.3 beautifulsoup4==4.12.3 soupsieve==2.5 bleach==6.1.0 - six==1.16.0 + six==1.17.0 webencodings==0.5.1 defusedxml==0.7.1 jinja2==3.1.6 @@ -136,7 +136,7 @@ nbconvert==7.7.3 platformdirs==3.10.0 traitlets==5.14.3 python-dateutil==2.9.0.post0 - six==1.16.0 + six==1.17.0 pyzmq==26.2.0 tornado==6.4.2 traitlets==5.14.3 @@ -175,8 +175,8 @@ premailer==3.10.0 cssutils==2.6.0 lxml==5.3.0 requests==2.32.3 - certifi==2024.8.30 - charset-normalizer==3.4.0 + certifi==2025.1.31 + charset-normalizer==3.4.1 idna==3.10 urllib3==1.26.20 psutil==5.9.8 @@ -209,7 +209,7 @@ pyramid-mailer==0.15.1 transaction==5.0.0 zope.interface==7.2.0 pyramid-mako==1.1.0 - mako==1.3.6 + mako==1.3.10 markupsafe==3.0.2 pyramid==2.0.2 hupper==1.12 @@ -230,7 +230,7 @@ python-memcached==1.62 python-pam==2.0.2 python3-saml==1.16.0 isodate==0.6.1 - six==1.16.0 + six==1.17.0 lxml==5.3.0 xmlsec==1.3.14 lxml==5.3.0 @@ -240,7 +240,7 @@ redis==5.2.0 regex==2022.10.31 routes==2.5.1 repoze.lru==0.7 - six==1.16.0 + six==1.17.0 s3fs==2024.12.0 aiobotocore==2.17.0 aiohttp==3.11.11 @@ -259,12 +259,12 @@ s3fs==2024.12.0 botocore==1.35.93 jmespath==1.0.1 python-dateutil==2.9.0.post0 - six==1.16.0 + six==1.17.0 urllib3==1.26.20 jmespath==1.0.1 multidict==6.1.0 python-dateutil==2.9.0.post0 - six==1.16.0 + six==1.17.0 urllib3==1.26.20 wrapt==1.17.0 aiohttp==3.11.11 @@ -286,10 +286,10 @@ sshpubkeys==3.3.1 cffi==1.17.1 pycparser==2.22 ecdsa==0.19.0 - six==1.16.0 + six==1.17.0 sqlalchemy==1.4.52 greenlet==3.1.1 - typing_extensions==4.12.2 + typing_extensions==4.13.2 supervisor==4.2.5 tzlocal==4.3 pytz-deprecation-shim==0.1.0.post0 @@ -300,7 +300,7 @@ urlobject==2.4.3 waitress==3.0.2 webhelpers2==2.1 markupsafe==3.0.2 - six==1.16.0 + six==1.17.0 whoosh==2.7.4 zope.cachedescriptors==5.1.0 qrcode==7.4.2 diff --git a/requirements_rc_tools.txt b/requirements_rc_tools.txt index 088bd984..23d46fb5 100644 --- a/requirements_rc_tools.txt +++ b/requirements_rc_tools.txt @@ -1,3 +1,3 @@ ## rhodecode-tools, special case, use file://PATH.tar.gz#egg=rhodecode-tools==X.Y.Z, to test local version -rhodecode-tools @ https://code.rhodecode.com/_file_store/download/0-51fb13c9-6175-480c-b9dc-630c7601d47c.1.0.tar.gz#egg=rhodecode-tools -rhodecode-tools==4.1.0 +rhodecode-tools @ https://code.rhodecode.com/_file_store/download/0-c84a5d20-64c3-4ef8-9c83-37e2b2dbddd9.2.0.tar.gz#egg=rhodecode-tools +rhodecode-tools==4.2.0 diff --git a/requirements_test.txt b/requirements_test.txt index 99786ec0..190ec9ec 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -23,7 +23,7 @@ pytest-profiling==1.7.0 iniconfig==2.0.0 packaging==24.1 pluggy==1.5.0 - six==1.16.0 + six==1.17.0 pytest-rerunfailures==14.0 packaging==24.1 pytest==8.3.3 From df765294283b90e76dfc12569a6f70b43e4e6934 Mon Sep 17 00:00:00 2001 From: RhodeCode Admin Date: Wed, 4 Jun 2025 08:43:07 +0200 Subject: [PATCH 3/8] search: added deprecation for whoosh searcher --- rhodecode/lib/index/__init__.py | 7 ++++++- rhodecode/lib/index/whoosh.py | 29 +++++++---------------------- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/rhodecode/lib/index/__init__.py b/rhodecode/lib/index/__init__.py index 55e4ece8..833aef02 100644 --- a/rhodecode/lib/index/__init__.py +++ b/rhodecode/lib/index/__init__.py @@ -147,6 +147,11 @@ def searcher_from_config(config, prefix="search."): # use an old legacy ES version set to 8 _config["es_version"] = DEFAULT_ES_VERSION - imported = importlib.import_module(_config.get("module", default_searcher)) + search_module = _config.get("module", default_searcher) + if search_module == "rhodecode.lib.index.whoosh": + log.warning("rhodecode.lib.index.whoosh module is no longer supported, using default searcher rc_elasticsearch") + search_module = "rc_elasticsearch" + + imported = importlib.import_module(search_module) searcher = imported.Searcher(config=_config) return searcher diff --git a/rhodecode/lib/index/whoosh.py b/rhodecode/lib/index/whoosh.py index b3c2963a..31a4cc78 100644 --- a/rhodecode/lib/index/whoosh.py +++ b/rhodecode/lib/index/whoosh.py @@ -36,28 +36,13 @@ from rhodecode.lib.str_utils import safe_str log = logging.getLogger(__name__) -try: - # we first try to import from rhodecode tools, fallback to copies if - # we're unable to - from rhodecode_tools.lib.fts_index.whoosh_schema import ( - ANALYZER, - FILE_INDEX_NAME, - FILE_SCHEMA, - COMMIT_INDEX_NAME, - COMMIT_SCHEMA, - ) -except ImportError: - log.warning( - "rhodecode_tools schema not available, doing a fallback " - "import from `rhodecode.lib.index.whoosh_fallback_schema`" - ) - from rhodecode.lib.index.whoosh_fallback_schema import ( - ANALYZER, - FILE_INDEX_NAME, - FILE_SCHEMA, - COMMIT_INDEX_NAME, - COMMIT_SCHEMA, - ) +from rhodecode.lib.index.whoosh_fallback_schema import ( + ANALYZER, + FILE_INDEX_NAME, + FILE_SCHEMA, + COMMIT_INDEX_NAME, + COMMIT_SCHEMA, +) FORMATTER = HtmlFormatter("span", between='\n...\n') From 5819a0ec6176142c9a07e6af7c5766bdb8388c16 Mon Sep 17 00:00:00 2001 From: ievgenii vdovenko Date: Fri, 6 Jun 2025 13:13:21 +0200 Subject: [PATCH 4/8] fix: calls vcs server cache invalidation on repository delete --- .../apps/repository/views/repo_settings_advanced.py | 11 ++++++++++- rhodecode/lib/vcs/backends/svn/repository.py | 6 +++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/rhodecode/apps/repository/views/repo_settings_advanced.py b/rhodecode/apps/repository/views/repo_settings_advanced.py index 87a0c098..402df14d 100644 --- a/rhodecode/apps/repository/views/repo_settings_advanced.py +++ b/rhodecode/apps/repository/views/repo_settings_advanced.py @@ -134,6 +134,10 @@ class RepoSettingsAdvancedView(RepoAppView): repo_advanced_url = h.route_path("edit_repo_advanced", repo_name=self.db_repo_name, _anchor="advanced-delete") try: old_data = self.db_repo.get_api_data() + + delete_cache = True + self._invalidate_remote_cache(delete=delete_cache) + RepoModel().delete(self.db_repo, forks=handle_forks) _forks = self.db_repo.forks.count() @@ -148,7 +152,7 @@ class RepoSettingsAdvancedView(RepoAppView): "repo.delete", action_data={"old_data": old_data}, user=self._rhodecode_user, repo=repo ) - ScmModel().mark_for_invalidation(self.db_repo_name, delete=True) + ScmModel().mark_for_invalidation(self.db_repo_name, delete=delete_cache) h.flash(_("Deleted repository `%s`") % self.db_repo_name, category="success") Session().commit() except AttachedForksError: @@ -196,6 +200,11 @@ class RepoSettingsAdvancedView(RepoAppView): raise HTTPFound(h.route_path("home")) + def _invalidate_remote_cache(self, delete: bool): + log.debug(f"Invalidating remote cache, for repository: {self.db_repo_name}") + scm_repo = self.db_repo.scm_instance(cache=False) + scm_repo.vcsserver_invalidate_cache(delete=delete) + @LoginRequired() @HasRepoPermissionAnyDecorator("repository.admin") @CSRFRequired() diff --git a/rhodecode/lib/vcs/backends/svn/repository.py b/rhodecode/lib/vcs/backends/svn/repository.py index d0ae04de..101cb8da 100644 --- a/rhodecode/lib/vcs/backends/svn/repository.py +++ b/rhodecode/lib/vcs/backends/svn/repository.py @@ -235,8 +235,8 @@ class SubversionRepository(base.BaseRepository): try: SubversionRepository(path) return True - except VCSError: - pass + except VCSError as e: + log.warning(f"VCSError: {repr(e)}") return False @LazyProperty @@ -307,7 +307,7 @@ class SubversionRepository(base.BaseRepository): start_raw_id = self._sanitize_commit_id(start_id) end_raw_id = self._sanitize_commit_id(end_id) - + # for empty revisions in SVN if start_raw_id not in self.commit_ids or end_raw_id not in self.commit_ids: return base.CollectionGenerator(self, []) From 1ede0669d946285a1ad86b2a670212efb6f59f90 Mon Sep 17 00:00:00 2001 From: ievgenii vdovenko Date: Fri, 6 Jun 2025 17:14:48 +0200 Subject: [PATCH 5/8] fix: adds test --- .../tests/test_repo_settings_advanced.py | 33 ++++++++++++++++++- .../views/repo_settings_advanced.py | 1 - 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/rhodecode/apps/repository/tests/test_repo_settings_advanced.py b/rhodecode/apps/repository/tests/test_repo_settings_advanced.py index 7a484d61..68fcf0ae 100644 --- a/rhodecode/apps/repository/tests/test_repo_settings_advanced.py +++ b/rhodecode/apps/repository/tests/test_repo_settings_advanced.py @@ -15,9 +15,10 @@ # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ - import pytest +from unittest.mock import patch, MagicMock +from rhodecode.apps.repository.views.repo_settings_advanced import RepoSettingsAdvancedView from rhodecode.lib.str_utils import safe_str from rhodecode.model.db import Repository from rhodecode.model.repo import RepoModel @@ -110,6 +111,36 @@ class TestAdminRepoSettingsAdvanced(object): assert RepoModel().get_by_repo_name(repo_name) is None assert not repo_on_filesystem(repo_name_str) + @patch.object(RepoSettingsAdvancedView, "_maybe_needs_password_change") + @patch.object(RepoSettingsAdvancedView, "_maybe_needs_2fa_configuration") + @patch.object(RepoSettingsAdvancedView, "_maybe_needs_2fa_check") + @patch("rhodecode.apps._base.ScmModel") + @patch("rhodecode.apps._base.IssueTrackerSettingsModel") + @patch("rhodecode.lib.helpers.route_path") + def test_advanced_repo_delete__call_remote_cache_invalidation(self, *ignore): + request = MagicMock() + + db_repo = MagicMock() + scm_instance = MagicMock() + + db_repo.scm_instance.return_value = scm_instance + request.db_repo = db_repo + + obj = RepoSettingsAdvancedView(MagicMock(), request) + # bypass decorators + unwrapped__edit_advanced_delete = RepoSettingsAdvancedView.__dict__[ + "edit_advanced_delete" + ].__wrapped__.__wrapped__.__wrapped__ + + try: + unwrapped__edit_advanced_delete(obj) + except Exception as ignore: + # ignore error, because of mocks method will fail with exception which is not related to the current test + pass + + db_repo.scm_instance.assert_called_once_with(cache=False) + scm_instance.vcsserver_invalidate_cache(delete=True) + @pytest.mark.parametrize("suffix", ["", "ąęł", "123"], ids=no_newline_id_generator) def test_advanced_repo_archive(self, autologin_user, backend, suffix, csrf_token): repo = backend.create_repo(name_suffix=suffix) diff --git a/rhodecode/apps/repository/views/repo_settings_advanced.py b/rhodecode/apps/repository/views/repo_settings_advanced.py index 402df14d..d6d18cd4 100644 --- a/rhodecode/apps/repository/views/repo_settings_advanced.py +++ b/rhodecode/apps/repository/views/repo_settings_advanced.py @@ -22,7 +22,6 @@ import logging from pyramid.httpexceptions import HTTPFound from packaging.version import Version -from rhodecode import events from rhodecode.apps._base import RepoAppView from rhodecode.lib import helpers as h from rhodecode.lib import audit_logger From 69da253b1ee4205f169d213833fcb169b1d665c0 Mon Sep 17 00:00:00 2001 From: ievgenii vdovenko Date: Fri, 6 Jun 2025 17:44:55 +0200 Subject: [PATCH 6/8] fix: fixes assert in the test --- rhodecode/apps/repository/tests/test_repo_settings_advanced.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rhodecode/apps/repository/tests/test_repo_settings_advanced.py b/rhodecode/apps/repository/tests/test_repo_settings_advanced.py index 68fcf0ae..b32a9cad 100644 --- a/rhodecode/apps/repository/tests/test_repo_settings_advanced.py +++ b/rhodecode/apps/repository/tests/test_repo_settings_advanced.py @@ -119,7 +119,6 @@ class TestAdminRepoSettingsAdvanced(object): @patch("rhodecode.lib.helpers.route_path") def test_advanced_repo_delete__call_remote_cache_invalidation(self, *ignore): request = MagicMock() - db_repo = MagicMock() scm_instance = MagicMock() @@ -139,7 +138,7 @@ class TestAdminRepoSettingsAdvanced(object): pass db_repo.scm_instance.assert_called_once_with(cache=False) - scm_instance.vcsserver_invalidate_cache(delete=True) + scm_instance.vcsserver_invalidate_cache.assert_called_once_with(delete=True) @pytest.mark.parametrize("suffix", ["", "ąęł", "123"], ids=no_newline_id_generator) def test_advanced_repo_archive(self, autologin_user, backend, suffix, csrf_token): From d00249de71a87abfcfec7bf32d90b6edb71e398d Mon Sep 17 00:00:00 2001 From: ievgenii vdovenko Date: Wed, 11 Jun 2025 10:09:19 +0200 Subject: [PATCH 7/8] remote debugger: adds pycharm remote debugger --- requirements_debug.txt | 3 ++ rhodecode/config/constants.py | 2 + rhodecode/config/middleware.py | 13 +++++++ .../lib/middleware/pycharm_remote_debugger.py | 37 +++++++++++++++++++ 4 files changed, 55 insertions(+) create mode 100644 rhodecode/config/constants.py create mode 100644 rhodecode/lib/middleware/pycharm_remote_debugger.py diff --git a/requirements_debug.txt b/requirements_debug.txt index 57f1d02d..be3fb446 100644 --- a/requirements_debug.txt +++ b/requirements_debug.txt @@ -25,3 +25,6 @@ types-sqlalchemy types-psutil types-pycurl types-ujson + +#remote debugger +pydevd-pycharm~=251.25410.159 diff --git a/rhodecode/config/constants.py b/rhodecode/config/constants.py new file mode 100644 index 00000000..e8d0e984 --- /dev/null +++ b/rhodecode/config/constants.py @@ -0,0 +1,2 @@ +PYCHARM_DEBUG = "PYCHARM_DEBUG" +PYCHARM_DEBUG_PAUSE_AT_STARTUP = "PYCHARM_DEBUG_PAUSE_AT_STARTUP" diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py index 40e77751..2cd20d31 100644 --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -31,6 +31,8 @@ from pyramid.settings import asbool, aslist from pyramid.httpexceptions import HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound from pyramid.renderers import render_to_response +from rhodecode.config.constants import PYCHARM_DEBUG +from rhodecode.lib.middleware.pycharm_remote_debugger import PyCharmDebugMiddleware from rhodecode.model import meta from rhodecode.config import patches @@ -420,6 +422,16 @@ def includeme(config, auth_resources=None): config.add_view(error_handler, context=HTTPError) +def wrap_app_in_pycharm_remote_debugger_if_enabled(pyramid_app): + pycharm_debugger_enabled = os.getenv(PYCHARM_DEBUG, "0") == "1" + log.debug(f"Pycharm remote debugger enabled: {pycharm_debugger_enabled}") + + if pycharm_debugger_enabled: + return PyCharmDebugMiddleware(pyramid_app) + + return pyramid_app + + def wrap_app_in_wsgi_middlewares(pyramid_app, config): """ Apply outer WSGI middlewares around the application. @@ -429,6 +441,7 @@ def wrap_app_in_wsgi_middlewares(pyramid_app, config): # enable https redirects based on HTTP_X_URL_SCHEME set by proxy pyramid_app = HttpsFixup(pyramid_app, settings) + pyramid_app = wrap_app_in_pycharm_remote_debugger_if_enabled(pyramid_app) pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(pyramid_app, settings) registry.ae_client = _ae_client diff --git a/rhodecode/lib/middleware/pycharm_remote_debugger.py b/rhodecode/lib/middleware/pycharm_remote_debugger.py new file mode 100644 index 00000000..fab71ec8 --- /dev/null +++ b/rhodecode/lib/middleware/pycharm_remote_debugger.py @@ -0,0 +1,37 @@ +import os +import logging +import socket + +from rhodecode.config.constants import PYCHARM_DEBUG_PAUSE_AT_STARTUP + +log = logging.getLogger(__name__) + + +class PyCharmDebugMiddleware: + def __init__(self, handler): + self.handler = handler + self._start_debugger() + + def _start_debugger(self): + try: + import pydevd_pycharm + + suspend = os.getenv(PYCHARM_DEBUG_PAUSE_AT_STARTUP, "0") == "1" + host = "host.docker.internal" # assuming that app is running inside docker, and the debug server is on the same machine + + pydevd_pycharm.settrace( + host, + suspend=suspend, + stdoutToServer=True, + stderrToServer=True, + ) + log.debug("PyCharm debugger attached successfully!") + except ImportError: + log.warning("pydevd_pycharm not installed. Debugging disabled.") + except ConnectionRefusedError: + ip = socket.gethostbyname(host) + log.warning(f"debug server is not running on host[ip]: {host}[{ip}], shutdown remote debugger.") + pydevd_pycharm.stoptrace() + + def __call__(self, environ, start_response): + return self.handler(environ, start_response) From 6da86f12028da74d8f435b44bbdd9dd179cefa00 Mon Sep 17 00:00:00 2001 From: ievgenii vdovenko Date: Wed, 11 Jun 2025 12:54:31 +0200 Subject: [PATCH 8/8] release notes: adds release notes --- docs/release-notes/release-notes-5.6.0.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/release-notes-5.6.0.rst b/docs/release-notes/release-notes-5.6.0.rst index 554ef65d..bd3ec256 100644 --- a/docs/release-notes/release-notes-5.6.0.rst +++ b/docs/release-notes/release-notes-5.6.0.rst @@ -9,7 +9,8 @@ Release Date New Features ^^^^^^^^^^^^ -- git: added server-side support for partial and shallow cloning, so "--depth", "--filter" arguments are supported +- git: Added server-side support for partial and shallow cloning, so "--depth", "--filter" arguments are supported. +- PyCharm Debugger Integration: Added support for remote debugging with PyCharm. General ^^^^^^^ @@ -28,6 +29,8 @@ Performance Fixes ^^^^^ +- Resolved an issue with SVN repositories by implementing proper cache invalidation. +- Fixed a regression affecting push capabilities. Upgrade notes