Merge branch 'main' into RCCE-247-group-permissions
This commit is contained in:
Andrii V
commit
ed71e815ff
19 changed files with 232 additions and 174 deletions
|
|
@ -315,9 +315,7 @@ Enabling ElasticSearch
|
|||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ElasticSearch is available in EE edition only. It provides much scalable and more advanced
|
||||
search capabilities. While Whoosh is fine for upto 1-2GB of data, beyond that amount it
|
||||
starts slowing down, and can cause other problems.
|
||||
New ElasticSearch 6 also provides much more advanced query language.
|
||||
search capabilities. New ElasticSearch also provides much more advanced query language.
|
||||
It allows advanced filtering by file paths, extensions, use OR statements, ranges etc.
|
||||
Please check query language examples in the search field for some advanced query language usage.
|
||||
|
||||
|
|
@ -325,25 +323,15 @@ Please check query language examples in the search field for some advanced query
|
|||
1. Open the :file:`rhodecode.ini` file for the instance you wish to edit. The
|
||||
default location is
|
||||
:file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
|
||||
2. Find the search configuration section:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
###################################
|
||||
## SEARCH INDEXING CONFIGURATION ##
|
||||
###################################
|
||||
|
||||
search.module = rhodecode.lib.index.whoosh
|
||||
search.location = %(here)s/data/index
|
||||
|
||||
and change it to:
|
||||
2. Find the search configuration section and change it to:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
search.module = rc_elasticsearch
|
||||
search.location = http://localhost:9200
|
||||
## specify Elastic Search version, 6 for latest or 2 for legacy
|
||||
search.es_version = 6
|
||||
search.location = http://elasticsearch:9200
|
||||
## specify Elastic Search version, 8 for latest
|
||||
search.es_version = 8
|
||||
|
||||
where ``search.location`` points to the ElasticSearch server
|
||||
by default running on port 9200.
|
||||
|
|
@ -352,8 +340,7 @@ Index invocation also needs change. Please provide --es-version= and
|
|||
--engine-location= parameters to define ElasticSearch server location and it's version.
|
||||
For example::
|
||||
|
||||
rhodecode-index --instace-name=enterprise-1 --es-version=6 --engine-location=http://localhost:9200
|
||||
rhodecode-index --instance-name=enterprise-1 --es-version=8 --engine-location=http://elasticsearch:9200
|
||||
|
||||
|
||||
.. _Whoosh: https://pypi.python.org/pypi/Whoosh/
|
||||
.. _ElasticSearch 6: https://www.elastic.co/
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ Release Date
|
|||
New Features
|
||||
^^^^^^^^^^^^
|
||||
|
||||
- git: added server-side support for partial and shallow cloning, so "--depth", "--filter" arguments are supported
|
||||
- git: Added server-side support for partial and shallow cloning, so "--depth", "--filter" arguments are supported.
|
||||
- PyCharm Debugger Integration: Added support for remote debugging with PyCharm.
|
||||
|
||||
General
|
||||
^^^^^^^
|
||||
|
|
@ -28,6 +29,8 @@ Performance
|
|||
Fixes
|
||||
^^^^^
|
||||
|
||||
- Resolved an issue with SVN repositories by implementing proper cache invalidation.
|
||||
- Fixed a regression affecting push capabilities.
|
||||
|
||||
|
||||
Upgrade notes
|
||||
|
|
|
|||
|
|
@ -1,23 +1,23 @@
|
|||
# deps, generated via pipdeptree --exclude setuptools,wheel,pipdeptree,pip -f | tr '[:upper:]' '[:lower:]'
|
||||
|
||||
alembic==1.13.1
|
||||
mako==1.3.6
|
||||
mako==1.3.10
|
||||
markupsafe==3.0.2
|
||||
sqlalchemy==1.4.52
|
||||
greenlet==3.1.1
|
||||
typing_extensions==4.12.2
|
||||
typing_extensions==4.13.2
|
||||
async-timeout==4.0.3
|
||||
babel==2.12.1
|
||||
beaker==1.13.0
|
||||
celery==5.4.0
|
||||
billiard==4.2.1
|
||||
click==8.1.7
|
||||
click==8.1.8
|
||||
click-didyoumean==0.3.1
|
||||
click==8.1.7
|
||||
click==8.1.8
|
||||
click-plugins==1.1.1
|
||||
click==8.1.7
|
||||
click==8.1.8
|
||||
click-repl==0.3.0
|
||||
click==8.1.7
|
||||
click==8.1.8
|
||||
prompt_toolkit==3.0.48
|
||||
wcwidth==0.2.13
|
||||
kombu==5.4.2
|
||||
|
|
@ -26,7 +26,7 @@ celery==5.4.0
|
|||
tzdata==2024.2
|
||||
vine==5.1.0
|
||||
python-dateutil==2.9.0.post0
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
tzdata==2024.2
|
||||
vine==5.1.0
|
||||
channelstream==0.7.1
|
||||
|
|
@ -64,10 +64,10 @@ channelstream==0.7.1
|
|||
zope.interface==7.2.0
|
||||
zope.deprecation==5.1.0
|
||||
python-dateutil==2.9.0.post0
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
requests==2.32.3
|
||||
certifi==2024.8.30
|
||||
charset-normalizer==3.4.0
|
||||
certifi==2025.1.31
|
||||
charset-normalizer==3.4.1
|
||||
idna==3.10
|
||||
urllib3==1.26.20
|
||||
ws4py==0.5.1
|
||||
|
|
@ -86,7 +86,7 @@ dogpile.cache==1.3.4
|
|||
stevedore==5.1.0
|
||||
pbr==5.11.1
|
||||
formencode==2.1.0
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
fsspec==2024.12.0
|
||||
gunicorn==23.0.0
|
||||
packaging==24.1
|
||||
|
|
@ -107,11 +107,11 @@ ipython==8.26.0
|
|||
pygments==2.18.0
|
||||
stack-data==0.6.3
|
||||
asttokens==2.4.1
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
executing==2.0.1
|
||||
pure_eval==0.2.3
|
||||
traitlets==5.14.3
|
||||
typing_extensions==4.12.2
|
||||
typing_extensions==4.13.2
|
||||
markdown==3.4.3
|
||||
msgpack==1.1.0
|
||||
mysqlclient==2.1.1
|
||||
|
|
@ -119,7 +119,7 @@ nbconvert==7.7.3
|
|||
beautifulsoup4==4.12.3
|
||||
soupsieve==2.5
|
||||
bleach==6.1.0
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
webencodings==0.5.1
|
||||
defusedxml==0.7.1
|
||||
jinja2==3.1.6
|
||||
|
|
@ -136,7 +136,7 @@ nbconvert==7.7.3
|
|||
platformdirs==3.10.0
|
||||
traitlets==5.14.3
|
||||
python-dateutil==2.9.0.post0
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
pyzmq==26.2.0
|
||||
tornado==6.4.2
|
||||
traitlets==5.14.3
|
||||
|
|
@ -175,8 +175,8 @@ premailer==3.10.0
|
|||
cssutils==2.6.0
|
||||
lxml==5.3.0
|
||||
requests==2.32.3
|
||||
certifi==2024.8.30
|
||||
charset-normalizer==3.4.0
|
||||
certifi==2025.1.31
|
||||
charset-normalizer==3.4.1
|
||||
idna==3.10
|
||||
urllib3==1.26.20
|
||||
psutil==5.9.8
|
||||
|
|
@ -209,7 +209,7 @@ pyramid-mailer==0.15.1
|
|||
transaction==5.0.0
|
||||
zope.interface==7.2.0
|
||||
pyramid-mako==1.1.0
|
||||
mako==1.3.6
|
||||
mako==1.3.10
|
||||
markupsafe==3.0.2
|
||||
pyramid==2.0.2
|
||||
hupper==1.12
|
||||
|
|
@ -230,7 +230,7 @@ python-memcached==1.62
|
|||
python-pam==2.0.2
|
||||
python3-saml==1.16.0
|
||||
isodate==0.6.1
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
lxml==5.3.0
|
||||
xmlsec==1.3.14
|
||||
lxml==5.3.0
|
||||
|
|
@ -240,7 +240,7 @@ redis==5.2.0
|
|||
regex==2022.10.31
|
||||
routes==2.5.1
|
||||
repoze.lru==0.7
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
s3fs==2024.12.0
|
||||
aiobotocore==2.17.0
|
||||
aiohttp==3.11.11
|
||||
|
|
@ -259,12 +259,12 @@ s3fs==2024.12.0
|
|||
botocore==1.35.93
|
||||
jmespath==1.0.1
|
||||
python-dateutil==2.9.0.post0
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
urllib3==1.26.20
|
||||
jmespath==1.0.1
|
||||
multidict==6.1.0
|
||||
python-dateutil==2.9.0.post0
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
urllib3==1.26.20
|
||||
wrapt==1.17.0
|
||||
aiohttp==3.11.11
|
||||
|
|
@ -286,10 +286,10 @@ sshpubkeys==3.3.1
|
|||
cffi==1.17.1
|
||||
pycparser==2.22
|
||||
ecdsa==0.19.0
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
sqlalchemy==1.4.52
|
||||
greenlet==3.1.1
|
||||
typing_extensions==4.12.2
|
||||
typing_extensions==4.13.2
|
||||
supervisor==4.2.5
|
||||
tzlocal==4.3
|
||||
pytz-deprecation-shim==0.1.0.post0
|
||||
|
|
@ -300,7 +300,7 @@ urlobject==2.4.3
|
|||
waitress==3.0.2
|
||||
webhelpers2==2.1
|
||||
markupsafe==3.0.2
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
whoosh==2.7.4
|
||||
zope.cachedescriptors==5.1.0
|
||||
qrcode==7.4.2
|
||||
|
|
|
|||
|
|
@ -25,3 +25,6 @@ types-sqlalchemy
|
|||
types-psutil
|
||||
types-pycurl
|
||||
types-ujson
|
||||
|
||||
#remote debugger
|
||||
pydevd-pycharm~=251.25410.159
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
## rhodecode-tools, special case, use file://PATH.tar.gz#egg=rhodecode-tools==X.Y.Z, to test local version
|
||||
rhodecode-tools @ https://code.rhodecode.com/_file_store/download/0-51fb13c9-6175-480c-b9dc-630c7601d47c.1.0.tar.gz#egg=rhodecode-tools
|
||||
rhodecode-tools==4.1.0
|
||||
rhodecode-tools @ https://code.rhodecode.com/_file_store/download/0-c84a5d20-64c3-4ef8-9c83-37e2b2dbddd9.2.0.tar.gz#egg=rhodecode-tools
|
||||
rhodecode-tools==4.2.0
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ pytest-profiling==1.7.0
|
|||
iniconfig==2.0.0
|
||||
packaging==24.1
|
||||
pluggy==1.5.0
|
||||
six==1.16.0
|
||||
six==1.17.0
|
||||
pytest-rerunfailures==14.0
|
||||
packaging==24.1
|
||||
pytest==8.3.3
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from rhodecode.lib import audit_logger, rc_cache, channelstream
|
|||
from rhodecode.lib import repo_maintenance
|
||||
from rhodecode.lib.auth import HasPermissionAnyApi, HasUserGroupPermissionAnyApi, HasRepoPermissionAnyApi
|
||||
from rhodecode.lib.celerylib.utils import get_task_id
|
||||
from rhodecode.lib.str_utils import safe_bytes
|
||||
from rhodecode.lib.utils2 import str2bool, time_to_datetime, safe_str, safe_int
|
||||
from rhodecode.lib.ext_json import json
|
||||
from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError, CommentVersionMismatch
|
||||
|
|
@ -615,7 +616,7 @@ def get_repo_file(
|
|||
elif details == "full":
|
||||
extended_info = content = True
|
||||
|
||||
file_path = safe_str(file_path)
|
||||
bytes_path = safe_bytes(file_path)
|
||||
try:
|
||||
# check if repo is not empty by any chance, skip quicker if it is.
|
||||
_scm = repo.scm_instance()
|
||||
|
|
@ -625,7 +626,7 @@ def get_repo_file(
|
|||
node = ScmModel().get_node(
|
||||
repo,
|
||||
commit_id,
|
||||
file_path,
|
||||
bytes_path,
|
||||
extended_info=extended_info,
|
||||
content=content,
|
||||
max_file_bytes=max_file_bytes,
|
||||
|
|
|
|||
|
|
@ -483,7 +483,7 @@ class HomeView(BaseAppView, DataGridAppView):
|
|||
if not searcher:
|
||||
return []
|
||||
|
||||
is_es_6 = searcher.is_es_6
|
||||
is_es_8 = searcher.is_es_8
|
||||
|
||||
queries = []
|
||||
repo_group_name, repo_name, repo_context = None, None, None
|
||||
|
|
@ -495,7 +495,7 @@ class HomeView(BaseAppView, DataGridAppView):
|
|||
repo_name = search_context.get("search_context[repo_name]")
|
||||
repo_context = search_context.get("search_context[repo_view_type]")
|
||||
|
||||
if is_es_6 and repo_name:
|
||||
if is_es_8 and repo_name:
|
||||
# files
|
||||
def query_modifier():
|
||||
qry = query
|
||||
|
|
@ -535,7 +535,7 @@ class HomeView(BaseAppView, DataGridAppView):
|
|||
else:
|
||||
queries.extend([commit_qry, file_qry])
|
||||
|
||||
elif is_es_6 and repo_group_name:
|
||||
elif is_es_8 and repo_group_name:
|
||||
# files
|
||||
def query_modifier():
|
||||
qry = query
|
||||
|
|
|
|||
|
|
@ -15,9 +15,10 @@
|
|||
# This program is dual-licensed. If you wish to learn more about the
|
||||
# RhodeCode Enterprise Edition, including its added features, Support services,
|
||||
# and proprietary license terms, please see https://rhodecode.com/licenses/
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from rhodecode.apps.repository.views.repo_settings_advanced import RepoSettingsAdvancedView
|
||||
from rhodecode.lib.str_utils import safe_str
|
||||
from rhodecode.model.db import Repository
|
||||
from rhodecode.model.repo import RepoModel
|
||||
|
|
@ -110,6 +111,35 @@ class TestAdminRepoSettingsAdvanced(object):
|
|||
assert RepoModel().get_by_repo_name(repo_name) is None
|
||||
assert not repo_on_filesystem(repo_name_str)
|
||||
|
||||
@patch.object(RepoSettingsAdvancedView, "_maybe_needs_password_change")
|
||||
@patch.object(RepoSettingsAdvancedView, "_maybe_needs_2fa_configuration")
|
||||
@patch.object(RepoSettingsAdvancedView, "_maybe_needs_2fa_check")
|
||||
@patch("rhodecode.apps._base.ScmModel")
|
||||
@patch("rhodecode.apps._base.IssueTrackerSettingsModel")
|
||||
@patch("rhodecode.lib.helpers.route_path")
|
||||
def test_advanced_repo_delete__call_remote_cache_invalidation(self, *ignore):
|
||||
request = MagicMock()
|
||||
db_repo = MagicMock()
|
||||
scm_instance = MagicMock()
|
||||
|
||||
db_repo.scm_instance.return_value = scm_instance
|
||||
request.db_repo = db_repo
|
||||
|
||||
obj = RepoSettingsAdvancedView(MagicMock(), request)
|
||||
# bypass decorators
|
||||
unwrapped__edit_advanced_delete = RepoSettingsAdvancedView.__dict__[
|
||||
"edit_advanced_delete"
|
||||
].__wrapped__.__wrapped__.__wrapped__
|
||||
|
||||
try:
|
||||
unwrapped__edit_advanced_delete(obj)
|
||||
except Exception as ignore:
|
||||
# ignore error, because of mocks method will fail with exception which is not related to the current test
|
||||
pass
|
||||
|
||||
db_repo.scm_instance.assert_called_once_with(cache=False)
|
||||
scm_instance.vcsserver_invalidate_cache.assert_called_once_with(delete=True)
|
||||
|
||||
@pytest.mark.parametrize("suffix", ["", "ąęł", "123"], ids=no_newline_id_generator)
|
||||
def test_advanced_repo_archive(self, autologin_user, backend, suffix, csrf_token):
|
||||
repo = backend.create_repo(name_suffix=suffix)
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import logging
|
|||
from pyramid.httpexceptions import HTTPFound
|
||||
from packaging.version import Version
|
||||
|
||||
from rhodecode import events
|
||||
from rhodecode.apps._base import RepoAppView
|
||||
from rhodecode.lib import helpers as h
|
||||
from rhodecode.lib import audit_logger
|
||||
|
|
@ -134,6 +133,10 @@ class RepoSettingsAdvancedView(RepoAppView):
|
|||
repo_advanced_url = h.route_path("edit_repo_advanced", repo_name=self.db_repo_name, _anchor="advanced-delete")
|
||||
try:
|
||||
old_data = self.db_repo.get_api_data()
|
||||
|
||||
delete_cache = True
|
||||
self._invalidate_remote_cache(delete=delete_cache)
|
||||
|
||||
RepoModel().delete(self.db_repo, forks=handle_forks)
|
||||
|
||||
_forks = self.db_repo.forks.count()
|
||||
|
|
@ -148,7 +151,7 @@ class RepoSettingsAdvancedView(RepoAppView):
|
|||
"repo.delete", action_data={"old_data": old_data}, user=self._rhodecode_user, repo=repo
|
||||
)
|
||||
|
||||
ScmModel().mark_for_invalidation(self.db_repo_name, delete=True)
|
||||
ScmModel().mark_for_invalidation(self.db_repo_name, delete=delete_cache)
|
||||
h.flash(_("Deleted repository `%s`") % self.db_repo_name, category="success")
|
||||
Session().commit()
|
||||
except AttachedForksError:
|
||||
|
|
@ -196,6 +199,11 @@ class RepoSettingsAdvancedView(RepoAppView):
|
|||
|
||||
raise HTTPFound(h.route_path("home"))
|
||||
|
||||
def _invalidate_remote_cache(self, delete: bool):
|
||||
log.debug(f"Invalidating remote cache, for repository: {self.db_repo_name}")
|
||||
scm_repo = self.db_repo.scm_instance(cache=False)
|
||||
scm_repo.vcsserver_invalidate_cache(delete=delete)
|
||||
|
||||
@LoginRequired()
|
||||
@HasRepoPermissionAnyDecorator("repository.admin")
|
||||
@CSRFRequired()
|
||||
|
|
|
|||
|
|
@ -90,10 +90,12 @@ def perform_search(request, tmpl_context, repo_name=None, repo_group_name=None):
|
|||
sort=search_sort,
|
||||
)
|
||||
|
||||
search_result_count = search_result["count"]["value"]
|
||||
|
||||
formatted_results = Page(
|
||||
search_result["results"],
|
||||
page=requested_page,
|
||||
item_count=search_result["count"],
|
||||
item_count=search_result_count,
|
||||
items_per_page=page_limit,
|
||||
url_maker=url_generator,
|
||||
)
|
||||
|
|
@ -103,7 +105,7 @@ def perform_search(request, tmpl_context, repo_name=None, repo_group_name=None):
|
|||
search_tags = searcher.extract_search_tags(search_query)
|
||||
|
||||
if not search_result["error"]:
|
||||
execution_time = "{} results ({:.4f} seconds)".format(search_result["count"], search_result["runtime"])
|
||||
execution_time = "{} results ({:.4f} seconds)".format(search_result_count, search_result["runtime"])
|
||||
elif not errors:
|
||||
node = schema["search_query"]
|
||||
errors = [validation_schema.Invalid(node, search_result["error"])]
|
||||
|
|
|
|||
2
rhodecode/config/constants.py
Normal file
2
rhodecode/config/constants.py
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
PYCHARM_DEBUG = "PYCHARM_DEBUG"
|
||||
PYCHARM_DEBUG_PAUSE_AT_STARTUP = "PYCHARM_DEBUG_PAUSE_AT_STARTUP"
|
||||
|
|
@ -31,6 +31,8 @@ from pyramid.settings import asbool, aslist
|
|||
from pyramid.httpexceptions import HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound
|
||||
from pyramid.renderers import render_to_response
|
||||
|
||||
from rhodecode.config.constants import PYCHARM_DEBUG
|
||||
from rhodecode.lib.middleware.pycharm_remote_debugger import PyCharmDebugMiddleware
|
||||
from rhodecode.model import meta
|
||||
from rhodecode.config import patches
|
||||
|
||||
|
|
@ -420,6 +422,16 @@ def includeme(config, auth_resources=None):
|
|||
config.add_view(error_handler, context=HTTPError)
|
||||
|
||||
|
||||
def wrap_app_in_pycharm_remote_debugger_if_enabled(pyramid_app):
|
||||
pycharm_debugger_enabled = os.getenv(PYCHARM_DEBUG, "0") == "1"
|
||||
log.debug(f"Pycharm remote debugger enabled: {pycharm_debugger_enabled}")
|
||||
|
||||
if pycharm_debugger_enabled:
|
||||
return PyCharmDebugMiddleware(pyramid_app)
|
||||
|
||||
return pyramid_app
|
||||
|
||||
|
||||
def wrap_app_in_wsgi_middlewares(pyramid_app, config):
|
||||
"""
|
||||
Apply outer WSGI middlewares around the application.
|
||||
|
|
@ -429,6 +441,7 @@ def wrap_app_in_wsgi_middlewares(pyramid_app, config):
|
|||
|
||||
# enable https redirects based on HTTP_X_URL_SCHEME set by proxy
|
||||
pyramid_app = HttpsFixup(pyramid_app, settings)
|
||||
pyramid_app = wrap_app_in_pycharm_remote_debugger_if_enabled(pyramid_app)
|
||||
|
||||
pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(pyramid_app, settings)
|
||||
registry.ae_client = _ae_client
|
||||
|
|
|
|||
|
|
@ -28,16 +28,19 @@ from rhodecode.lib.index.search_utils import normalize_text_for_matching
|
|||
log = logging.getLogger(__name__)
|
||||
|
||||
# leave defaults for backward compat
|
||||
default_searcher = "rhodecode.lib.index.whoosh"
|
||||
default_location = "%(here)s/data/index"
|
||||
default_searcher = "rc_elasticsearch"
|
||||
|
||||
ES_VERSION_8 = 8
|
||||
ES_VERSION_9 = 9
|
||||
OS_VERSION_2 = 2
|
||||
|
||||
# for legacy reasons we keep 8 compat as default
|
||||
DEFAULT_ES_VERSION = ES_VERSION_8
|
||||
DEFAULT_OS_VERSION = OS_VERSION_2
|
||||
|
||||
ES_VERSION_2 = "2"
|
||||
ES_VERSION_6 = "6"
|
||||
# for legacy reasons we keep 2 compat as default
|
||||
DEFAULT_ES_VERSION = ES_VERSION_2
|
||||
|
||||
try:
|
||||
from rhodecode_tools.lib.fts_index.elasticsearch_engine_6 import ES_CONFIG # pragma: no cover
|
||||
from rhodecode_tools.lib.fts_index.elasticsearch_engine_8 import ES_CONFIG # pragma: no cover
|
||||
except ImportError:
|
||||
log.warning("rhodecode_tools not available, use of full text search is limited")
|
||||
pass
|
||||
|
|
@ -71,8 +74,8 @@ class BaseSearcher(object):
|
|||
return " ".join(normalize_text_for_matching(query).split())
|
||||
|
||||
@property
|
||||
def is_es_6(self):
|
||||
return self.es_version == ES_VERSION_6
|
||||
def is_es_8(self):
|
||||
return self.es_version >= ES_VERSION_8
|
||||
|
||||
def get_handlers(self):
|
||||
return {}
|
||||
|
|
@ -140,12 +143,15 @@ def search_config(config, prefix="search."):
|
|||
def searcher_from_config(config, prefix="search."):
|
||||
_config = search_config(config, prefix)
|
||||
|
||||
if "location" not in _config:
|
||||
_config["location"] = default_location
|
||||
if "es_version" not in _config:
|
||||
# use an old legacy ES version set to 2
|
||||
_config["es_version"] = "2"
|
||||
# use an old legacy ES version set to 8
|
||||
_config["es_version"] = DEFAULT_ES_VERSION
|
||||
|
||||
imported = importlib.import_module(_config.get("module", default_searcher))
|
||||
search_module = _config.get("module", default_searcher)
|
||||
if search_module == "rhodecode.lib.index.whoosh":
|
||||
log.warning("rhodecode.lib.index.whoosh module is no longer supported, using default searcher rc_elasticsearch")
|
||||
search_module = "rc_elasticsearch"
|
||||
|
||||
imported = importlib.import_module(search_module)
|
||||
searcher = imported.Searcher(config=_config)
|
||||
return searcher
|
||||
|
|
|
|||
|
|
@ -36,28 +36,13 @@ from rhodecode.lib.str_utils import safe_str
|
|||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
try:
|
||||
# we first try to import from rhodecode tools, fallback to copies if
|
||||
# we're unable to
|
||||
from rhodecode_tools.lib.fts_index.whoosh_schema import (
|
||||
ANALYZER,
|
||||
FILE_INDEX_NAME,
|
||||
FILE_SCHEMA,
|
||||
COMMIT_INDEX_NAME,
|
||||
COMMIT_SCHEMA,
|
||||
)
|
||||
except ImportError:
|
||||
log.warning(
|
||||
"rhodecode_tools schema not available, doing a fallback "
|
||||
"import from `rhodecode.lib.index.whoosh_fallback_schema`"
|
||||
)
|
||||
from rhodecode.lib.index.whoosh_fallback_schema import (
|
||||
ANALYZER,
|
||||
FILE_INDEX_NAME,
|
||||
FILE_SCHEMA,
|
||||
COMMIT_INDEX_NAME,
|
||||
COMMIT_SCHEMA,
|
||||
)
|
||||
from rhodecode.lib.index.whoosh_fallback_schema import (
|
||||
ANALYZER,
|
||||
FILE_INDEX_NAME,
|
||||
FILE_SCHEMA,
|
||||
COMMIT_INDEX_NAME,
|
||||
COMMIT_SCHEMA,
|
||||
)
|
||||
|
||||
|
||||
FORMATTER = HtmlFormatter("span", between='\n<span class="break">...</span>\n')
|
||||
|
|
|
|||
37
rhodecode/lib/middleware/pycharm_remote_debugger.py
Normal file
37
rhodecode/lib/middleware/pycharm_remote_debugger.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import os
|
||||
import logging
|
||||
import socket
|
||||
|
||||
from rhodecode.config.constants import PYCHARM_DEBUG_PAUSE_AT_STARTUP
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PyCharmDebugMiddleware:
|
||||
def __init__(self, handler):
|
||||
self.handler = handler
|
||||
self._start_debugger()
|
||||
|
||||
def _start_debugger(self):
|
||||
try:
|
||||
import pydevd_pycharm
|
||||
|
||||
suspend = os.getenv(PYCHARM_DEBUG_PAUSE_AT_STARTUP, "0") == "1"
|
||||
host = "host.docker.internal" # assuming that app is running inside docker, and the debug server is on the same machine
|
||||
|
||||
pydevd_pycharm.settrace(
|
||||
host,
|
||||
suspend=suspend,
|
||||
stdoutToServer=True,
|
||||
stderrToServer=True,
|
||||
)
|
||||
log.debug("PyCharm debugger attached successfully!")
|
||||
except ImportError:
|
||||
log.warning("pydevd_pycharm not installed. Debugging disabled.")
|
||||
except ConnectionRefusedError:
|
||||
ip = socket.gethostbyname(host)
|
||||
log.warning(f"debug server is not running on host[ip]: {host}[{ip}], shutdown remote debugger.")
|
||||
pydevd_pycharm.stoptrace()
|
||||
|
||||
def __call__(self, environ, start_response):
|
||||
return self.handler(environ, start_response)
|
||||
|
|
@ -235,8 +235,8 @@ class SubversionRepository(base.BaseRepository):
|
|||
try:
|
||||
SubversionRepository(path)
|
||||
return True
|
||||
except VCSError:
|
||||
pass
|
||||
except VCSError as e:
|
||||
log.warning(f"VCSError: {repr(e)}")
|
||||
return False
|
||||
|
||||
@LazyProperty
|
||||
|
|
@ -307,7 +307,7 @@ class SubversionRepository(base.BaseRepository):
|
|||
|
||||
start_raw_id = self._sanitize_commit_id(start_id)
|
||||
end_raw_id = self._sanitize_commit_id(end_id)
|
||||
|
||||
|
||||
# for empty revisions in SVN
|
||||
if start_raw_id not in self.commit_ids or end_raw_id not in self.commit_ids:
|
||||
return base.CollectionGenerator(self, [])
|
||||
|
|
|
|||
|
|
@ -154,53 +154,32 @@
|
|||
% endfor
|
||||
<div class="field">
|
||||
<p class="filterexample" style="position: inherit" onclick="$('#search-help').toggle()">${_('Query Language examples')}</p>
|
||||
<pre id="search-help" style="display: none">\
|
||||
|
||||
% if c.searcher.name == 'whoosh':
|
||||
Example filter terms for `Whoosh` search:
|
||||
query lang: <a href="${c.searcher.query_lang_doc}">Whoosh Query Language</a>
|
||||
Whoosh has limited query capabilities. For advanced search use ElasticSearch 6 from RhodeCode EE edition.
|
||||
<div id="search-help" style="display: none">\
|
||||
|
||||
Generate wildcards using '*' character:
|
||||
"repo_name:vcs*" - search everything starting with 'vcs'
|
||||
"repo_name:*vcs*" - search for repository containing 'vcs'
|
||||
% if c.searcher.name == 'elasticsearch':
|
||||
|
||||
Optional AND / OR operators in queries
|
||||
"repo_name:vcs OR repo_name:test"
|
||||
"owner:test AND repo_name:test*" AND extension:py
|
||||
|
||||
Move advanced search is available via ElasticSearch6 backend in EE edition.
|
||||
% elif c.searcher.name == 'elasticsearch' and c.searcher.es_version == '2':
|
||||
Example filter terms for `ElasticSearch-${c.searcher.es_version}`search:
|
||||
ElasticSearch-2 has limited query capabilities. For advanced search use ElasticSearch 6 from RhodeCode EE edition.
|
||||
<h3>Example filter terms for `ElasticSearch-${c.searcher.es_version}`search:</h3>
|
||||
query lang: <a href="${c.searcher.query_lang_doc}">ES ${c.searcher.es_version} Query Language</a></br>
|
||||
The reserved characters require escaping by `\`: <pre>+ - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /</pre>
|
||||
</br>
|
||||
|
||||
search type: content (File Content)
|
||||
indexed fields: content
|
||||
|
||||
# search for `fix` string in all files
|
||||
fix
|
||||
|
||||
search type: commit (Commit message)
|
||||
indexed fields: message
|
||||
|
||||
search type: path (File name)
|
||||
indexed fields: path
|
||||
|
||||
% else:
|
||||
Example filter terms for `ElasticSearch-${c.searcher.es_version}`search:
|
||||
query lang: <a href="${c.searcher.query_lang_doc}">ES 6 Query Language</a>
|
||||
The reserved characters needed espace by `\`: + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
|
||||
% for handler in c.searcher.get_handlers().values():
|
||||
|
||||
search type: ${handler.search_type_label}
|
||||
*indexed fields*: ${', '.join( [('\n ' if x[0]%4==0 else '')+x[1] for x in enumerate(handler.es_6_field_names)])}
|
||||
<h4>Search type: ${handler.search_type_label}</h4>
|
||||
<strong>indexed fields:</strong>
|
||||
<pre>${', '.join( [('\n' if x[0]%4==0 else '')+x[1] for x in enumerate(handler.es_6_field_names)])}</pre>
|
||||
|
||||
<pre>
|
||||
% for entry in handler.es_6_example_queries:
|
||||
${entry.rstrip()}
|
||||
${h.textwrap.dedent(entry.rstrip())}
|
||||
% endfor
|
||||
</pre>
|
||||
% endfor
|
||||
|
||||
% endif
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="field">${c.runtime}</div>
|
||||
|
|
|
|||
|
|
@ -1,8 +1,39 @@
|
|||
<%namespace name="search" file="/search/search.mako"/>
|
||||
|
||||
<%def name="highlight_text_file(has_matched_content, file_content, lexer, html_formatter, matching_lines, shown_matching_lines, url, use_hl_filter)">
|
||||
<%def name="highlight_text_file(file_path, full_file_content, hl_file_content, mimetype, url, use_hl_filter)">
|
||||
<%
|
||||
|
||||
matching_lines = []
|
||||
shown_matching_lines = 0
|
||||
|
||||
if c.searcher.is_es_8:
|
||||
# use empty terms so we default to markers usage
|
||||
total_lines, matching_lines = h.get_matching_line_offsets(hl_file_content, terms=None)
|
||||
|
||||
lines_of_interest = set()
|
||||
for line_number in matching_lines:
|
||||
if len(lines_of_interest) < max_lines:
|
||||
lines_of_interest |= set(range(
|
||||
max(line_number - line_context, 0),
|
||||
min(line_number + line_context, total_lines + 1)))
|
||||
shown_matching_lines += 1
|
||||
|
||||
has_matched_content = len(lines_of_interest) >= 1
|
||||
%>
|
||||
|
||||
|
||||
% if has_matched_content:
|
||||
${h.code_highlight(file_content, lexer, html_formatter, use_hl_filter=use_hl_filter)|n}
|
||||
<%
|
||||
lexer = h.get_lexer_safe(mimetype=mimetype, filepath=filepath)
|
||||
html_formatter = h.SearchContentCodeHtmlFormatter(
|
||||
linenos=True,
|
||||
cssclass="code-highlight",
|
||||
url=match_file_url,
|
||||
query_terms=terms,
|
||||
only_line_numbers=lines_of_interest
|
||||
)
|
||||
%>
|
||||
${h.code_highlight(hl_file_content, lexer, html_formatter, use_hl_filter=use_hl_filter)|n}
|
||||
% else:
|
||||
${_('No content matched')} <br/>
|
||||
% endif
|
||||
|
|
@ -20,41 +51,15 @@
|
|||
%for entry in c.formatted_results:
|
||||
|
||||
<%
|
||||
file_content = entry['content_highlight'] or entry['content']
|
||||
mimetype = entry.get('mimetype')
|
||||
hl_file_content = entry['content_highlight']
|
||||
full_file_content = entry['content']
|
||||
filepath = entry.get('path')
|
||||
mimetype = entry.get('mimetype')
|
||||
max_lines = h.safe_int(request.GET.get('max_lines', '10'))
|
||||
line_context = h.safe_int(request.GET.get('line_contenxt', '3'))
|
||||
line_context = h.safe_int(request.GET.get('line_context', '3'))
|
||||
|
||||
match_file_url=h.route_path('repo_files',repo_name=entry['repository'], commit_id=entry.get('commit_id', 'tip'),f_path=entry['f_path'], _query={"mark": query_mark})
|
||||
terms = c.cur_query
|
||||
|
||||
if c.searcher.is_es_6:
|
||||
# use empty terms so we default to markers usage
|
||||
total_lines, matching_lines = h.get_matching_line_offsets(file_content, terms=None)
|
||||
else:
|
||||
total_lines, matching_lines = h.get_matching_line_offsets(file_content, terms)
|
||||
|
||||
shown_matching_lines = 0
|
||||
lines_of_interest = set()
|
||||
for line_number in matching_lines:
|
||||
if len(lines_of_interest) < max_lines:
|
||||
lines_of_interest |= set(range(
|
||||
max(line_number - line_context, 0),
|
||||
min(line_number + line_context, total_lines + 1)))
|
||||
shown_matching_lines += 1
|
||||
lexer = h.get_lexer_safe(mimetype=mimetype, filepath=filepath)
|
||||
|
||||
html_formatter = h.SearchContentCodeHtmlFormatter(
|
||||
linenos=True,
|
||||
cssclass="code-highlight",
|
||||
url=match_file_url,
|
||||
query_terms=terms,
|
||||
only_line_numbers=lines_of_interest
|
||||
)
|
||||
|
||||
has_matched_content = len(lines_of_interest) >= 1
|
||||
|
||||
%>
|
||||
## search results are additionally filtered, and this check is just a safe gate
|
||||
% if c.rhodecode_user.is_admin or h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results content check'):
|
||||
|
|
@ -75,7 +80,7 @@
|
|||
<div class="stats-info">
|
||||
<span class="stats-first-item">
|
||||
${entry.get('lines', 0.)} ${_ungettext('line', 'lines', entry.get('lines', 0.))}
|
||||
(${len(matching_lines)} ${_ungettext('matched', 'matched', len(matching_lines))})
|
||||
##(${len(matching_lines)} ${_ungettext('matched', 'matched', len(matching_lines))})
|
||||
</span>
|
||||
<span>
|
||||
% if entry.get('size'):
|
||||
|
|
@ -112,7 +117,7 @@
|
|||
<a href="${h.route_path('search_repo_group', repo_group_name=repo_group, _query={'q': c.cur_query})}">${_('Narrow to this repository group')}</a>
|
||||
</span>
|
||||
% endif
|
||||
## hiden if in repo view
|
||||
## hidden if in repo view
|
||||
% if not c.repo_name:
|
||||
<span class="tag tag8">
|
||||
${search.repo_icon(repo_type)}
|
||||
|
|
@ -125,17 +130,14 @@
|
|||
<div class="clear-fix"></div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="code-body search-code-body clear-fix">
|
||||
${highlight_text_file(
|
||||
has_matched_content=has_matched_content,
|
||||
file_content=file_content,
|
||||
lexer=lexer,
|
||||
html_formatter=html_formatter,
|
||||
matching_lines=matching_lines,
|
||||
shown_matching_lines=shown_matching_lines,
|
||||
file_path=file_path,
|
||||
full_file_content=full_file_content,
|
||||
hl_file_content=hl_file_content,
|
||||
mimetype=mimetype,
|
||||
url=match_file_url,
|
||||
use_hl_filter=c.searcher.is_es_6
|
||||
use_hl_filter=c.searcher.is_es_8
|
||||
)}
|
||||
</div>
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue