diff --git a/docs/admin/system_admin/indexing.rst b/docs/admin/system_admin/indexing.rst index db9ee842..88b29fee 100644 --- a/docs/admin/system_admin/indexing.rst +++ b/docs/admin/system_admin/indexing.rst @@ -315,9 +315,7 @@ Enabling ElasticSearch ^^^^^^^^^^^^^^^^^^^^^^ ElasticSearch is available in EE edition only. It provides much scalable and more advanced -search capabilities. While Whoosh is fine for upto 1-2GB of data, beyond that amount it -starts slowing down, and can cause other problems. -New ElasticSearch 6 also provides much more advanced query language. +search capabilities. New ElasticSearch also provides much more advanced query language. It allows advanced filtering by file paths, extensions, use OR statements, ranges etc. Please check query language examples in the search field for some advanced query language usage. @@ -325,25 +323,15 @@ Please check query language examples in the search field for some advanced query 1. Open the :file:`rhodecode.ini` file for the instance you wish to edit. The default location is :file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini` -2. Find the search configuration section: -.. code-block:: ini - - ################################### - ## SEARCH INDEXING CONFIGURATION ## - ################################### - - search.module = rhodecode.lib.index.whoosh - search.location = %(here)s/data/index - -and change it to: +2. Find the search configuration section and change it to: .. code-block:: ini search.module = rc_elasticsearch - search.location = http://localhost:9200 - ## specify Elastic Search version, 6 for latest or 2 for legacy - search.es_version = 6 + search.location = http://elasticsearch:9200 + ## specify Elastic Search version, 8 for latest + search.es_version = 8 where ``search.location`` points to the ElasticSearch server by default running on port 9200. @@ -352,8 +340,7 @@ Index invocation also needs change. Please provide --es-version= and --engine-location= parameters to define ElasticSearch server location and it's version. For example:: - rhodecode-index --instace-name=enterprise-1 --es-version=6 --engine-location=http://localhost:9200 + rhodecode-index --instance-name=enterprise-1 --es-version=8 --engine-location=http://elasticsearch:9200 -.. _Whoosh: https://pypi.python.org/pypi/Whoosh/ .. _ElasticSearch 6: https://www.elastic.co/ diff --git a/rhodecode/api/views/repo_api.py b/rhodecode/api/views/repo_api.py index bdd90be6..593a1fa6 100644 --- a/rhodecode/api/views/repo_api.py +++ b/rhodecode/api/views/repo_api.py @@ -39,6 +39,7 @@ from rhodecode.lib import audit_logger, rc_cache, channelstream from rhodecode.lib import repo_maintenance from rhodecode.lib.auth import HasPermissionAnyApi, HasUserGroupPermissionAnyApi, HasRepoPermissionAnyApi from rhodecode.lib.celerylib.utils import get_task_id +from rhodecode.lib.str_utils import safe_bytes from rhodecode.lib.utils2 import str2bool, time_to_datetime, safe_str, safe_int from rhodecode.lib.ext_json import json from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError, CommentVersionMismatch @@ -615,7 +616,7 @@ def get_repo_file( elif details == "full": extended_info = content = True - file_path = safe_str(file_path) + bytes_path = safe_bytes(file_path) try: # check if repo is not empty by any chance, skip quicker if it is. _scm = repo.scm_instance() @@ -625,7 +626,7 @@ def get_repo_file( node = ScmModel().get_node( repo, commit_id, - file_path, + bytes_path, extended_info=extended_info, content=content, max_file_bytes=max_file_bytes, diff --git a/rhodecode/apps/home/views.py b/rhodecode/apps/home/views.py index 78b84b72..583b1a76 100644 --- a/rhodecode/apps/home/views.py +++ b/rhodecode/apps/home/views.py @@ -483,7 +483,7 @@ class HomeView(BaseAppView, DataGridAppView): if not searcher: return [] - is_es_6 = searcher.is_es_6 + is_es_8 = searcher.is_es_8 queries = [] repo_group_name, repo_name, repo_context = None, None, None @@ -495,7 +495,7 @@ class HomeView(BaseAppView, DataGridAppView): repo_name = search_context.get("search_context[repo_name]") repo_context = search_context.get("search_context[repo_view_type]") - if is_es_6 and repo_name: + if is_es_8 and repo_name: # files def query_modifier(): qry = query @@ -535,7 +535,7 @@ class HomeView(BaseAppView, DataGridAppView): else: queries.extend([commit_qry, file_qry]) - elif is_es_6 and repo_group_name: + elif is_es_8 and repo_group_name: # files def query_modifier(): qry = query diff --git a/rhodecode/apps/search/views.py b/rhodecode/apps/search/views.py index 3bccfe51..0dc3c125 100644 --- a/rhodecode/apps/search/views.py +++ b/rhodecode/apps/search/views.py @@ -90,10 +90,12 @@ def perform_search(request, tmpl_context, repo_name=None, repo_group_name=None): sort=search_sort, ) + search_result_count = search_result["count"]["value"] + formatted_results = Page( search_result["results"], page=requested_page, - item_count=search_result["count"], + item_count=search_result_count, items_per_page=page_limit, url_maker=url_generator, ) @@ -103,7 +105,7 @@ def perform_search(request, tmpl_context, repo_name=None, repo_group_name=None): search_tags = searcher.extract_search_tags(search_query) if not search_result["error"]: - execution_time = "{} results ({:.4f} seconds)".format(search_result["count"], search_result["runtime"]) + execution_time = "{} results ({:.4f} seconds)".format(search_result_count, search_result["runtime"]) elif not errors: node = schema["search_query"] errors = [validation_schema.Invalid(node, search_result["error"])] diff --git a/rhodecode/lib/index/__init__.py b/rhodecode/lib/index/__init__.py index 122c8d5e..55e4ece8 100644 --- a/rhodecode/lib/index/__init__.py +++ b/rhodecode/lib/index/__init__.py @@ -28,16 +28,19 @@ from rhodecode.lib.index.search_utils import normalize_text_for_matching log = logging.getLogger(__name__) # leave defaults for backward compat -default_searcher = "rhodecode.lib.index.whoosh" -default_location = "%(here)s/data/index" +default_searcher = "rc_elasticsearch" + +ES_VERSION_8 = 8 +ES_VERSION_9 = 9 +OS_VERSION_2 = 2 + +# for legacy reasons we keep 8 compat as default +DEFAULT_ES_VERSION = ES_VERSION_8 +DEFAULT_OS_VERSION = OS_VERSION_2 -ES_VERSION_2 = "2" -ES_VERSION_6 = "6" -# for legacy reasons we keep 2 compat as default -DEFAULT_ES_VERSION = ES_VERSION_2 try: - from rhodecode_tools.lib.fts_index.elasticsearch_engine_6 import ES_CONFIG # pragma: no cover + from rhodecode_tools.lib.fts_index.elasticsearch_engine_8 import ES_CONFIG # pragma: no cover except ImportError: log.warning("rhodecode_tools not available, use of full text search is limited") pass @@ -71,8 +74,8 @@ class BaseSearcher(object): return " ".join(normalize_text_for_matching(query).split()) @property - def is_es_6(self): - return self.es_version == ES_VERSION_6 + def is_es_8(self): + return self.es_version >= ES_VERSION_8 def get_handlers(self): return {} @@ -140,11 +143,9 @@ def search_config(config, prefix="search."): def searcher_from_config(config, prefix="search."): _config = search_config(config, prefix) - if "location" not in _config: - _config["location"] = default_location if "es_version" not in _config: - # use an old legacy ES version set to 2 - _config["es_version"] = "2" + # use an old legacy ES version set to 8 + _config["es_version"] = DEFAULT_ES_VERSION imported = importlib.import_module(_config.get("module", default_searcher)) searcher = imported.Searcher(config=_config) diff --git a/rhodecode/templates/search/search.mako b/rhodecode/templates/search/search.mako index 9d20c3d4..1b89e409 100644 --- a/rhodecode/templates/search/search.mako +++ b/rhodecode/templates/search/search.mako @@ -154,53 +154,32 @@ % endfor

${_('Query Language examples')}

- +
${c.runtime}
diff --git a/rhodecode/templates/search/search_content.mako b/rhodecode/templates/search/search_content.mako index 9926c383..fe981192 100644 --- a/rhodecode/templates/search/search_content.mako +++ b/rhodecode/templates/search/search_content.mako @@ -1,8 +1,39 @@ <%namespace name="search" file="/search/search.mako"/> -<%def name="highlight_text_file(has_matched_content, file_content, lexer, html_formatter, matching_lines, shown_matching_lines, url, use_hl_filter)"> +<%def name="highlight_text_file(file_path, full_file_content, hl_file_content, mimetype, url, use_hl_filter)"> +<% + +matching_lines = [] +shown_matching_lines = 0 + +if c.searcher.is_es_8: + # use empty terms so we default to markers usage + total_lines, matching_lines = h.get_matching_line_offsets(hl_file_content, terms=None) + +lines_of_interest = set() +for line_number in matching_lines: + if len(lines_of_interest) < max_lines: + lines_of_interest |= set(range( + max(line_number - line_context, 0), + min(line_number + line_context, total_lines + 1))) + shown_matching_lines += 1 + +has_matched_content = len(lines_of_interest) >= 1 +%> + + % if has_matched_content: - ${h.code_highlight(file_content, lexer, html_formatter, use_hl_filter=use_hl_filter)|n} + <% + lexer = h.get_lexer_safe(mimetype=mimetype, filepath=filepath) + html_formatter = h.SearchContentCodeHtmlFormatter( + linenos=True, + cssclass="code-highlight", + url=match_file_url, + query_terms=terms, + only_line_numbers=lines_of_interest + ) + %> + ${h.code_highlight(hl_file_content, lexer, html_formatter, use_hl_filter=use_hl_filter)|n} % else: ${_('No content matched')}
% endif @@ -20,41 +51,15 @@ %for entry in c.formatted_results: <% - file_content = entry['content_highlight'] or entry['content'] - mimetype = entry.get('mimetype') + hl_file_content = entry['content_highlight'] + full_file_content = entry['content'] filepath = entry.get('path') + mimetype = entry.get('mimetype') max_lines = h.safe_int(request.GET.get('max_lines', '10')) - line_context = h.safe_int(request.GET.get('line_contenxt', '3')) + line_context = h.safe_int(request.GET.get('line_context', '3')) match_file_url=h.route_path('repo_files',repo_name=entry['repository'], commit_id=entry.get('commit_id', 'tip'),f_path=entry['f_path'], _query={"mark": query_mark}) terms = c.cur_query - - if c.searcher.is_es_6: - # use empty terms so we default to markers usage - total_lines, matching_lines = h.get_matching_line_offsets(file_content, terms=None) - else: - total_lines, matching_lines = h.get_matching_line_offsets(file_content, terms) - - shown_matching_lines = 0 - lines_of_interest = set() - for line_number in matching_lines: - if len(lines_of_interest) < max_lines: - lines_of_interest |= set(range( - max(line_number - line_context, 0), - min(line_number + line_context, total_lines + 1))) - shown_matching_lines += 1 - lexer = h.get_lexer_safe(mimetype=mimetype, filepath=filepath) - - html_formatter = h.SearchContentCodeHtmlFormatter( - linenos=True, - cssclass="code-highlight", - url=match_file_url, - query_terms=terms, - only_line_numbers=lines_of_interest - ) - - has_matched_content = len(lines_of_interest) >= 1 - %> ## search results are additionally filtered, and this check is just a safe gate % if c.rhodecode_user.is_admin or h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results content check'): @@ -75,7 +80,7 @@
${entry.get('lines', 0.)} ${_ungettext('line', 'lines', entry.get('lines', 0.))} - (${len(matching_lines)} ${_ungettext('matched', 'matched', len(matching_lines))}) + ##(${len(matching_lines)} ${_ungettext('matched', 'matched', len(matching_lines))}) % if entry.get('size'): @@ -112,7 +117,7 @@ ${_('Narrow to this repository group')} % endif - ## hiden if in repo view + ## hidden if in repo view % if not c.repo_name: ${search.repo_icon(repo_type)} @@ -125,17 +130,14 @@
-
${highlight_text_file( - has_matched_content=has_matched_content, - file_content=file_content, - lexer=lexer, - html_formatter=html_formatter, - matching_lines=matching_lines, - shown_matching_lines=shown_matching_lines, + file_path=file_path, + full_file_content=full_file_content, + hl_file_content=hl_file_content, + mimetype=mimetype, url=match_file_url, - use_hl_filter=c.searcher.is_es_6 + use_hl_filter=c.searcher.is_es_8 )}