search: remove whoosh, and add support for ES 8 & 9
This commit is contained in:
RhodeCode Admin
parent
aa989aeadd
commit
b6c2ac4521
7 changed files with 88 additions and 116 deletions
|
|
@ -315,9 +315,7 @@ Enabling ElasticSearch
|
|||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ElasticSearch is available in EE edition only. It provides much scalable and more advanced
|
||||
search capabilities. While Whoosh is fine for upto 1-2GB of data, beyond that amount it
|
||||
starts slowing down, and can cause other problems.
|
||||
New ElasticSearch 6 also provides much more advanced query language.
|
||||
search capabilities. New ElasticSearch also provides much more advanced query language.
|
||||
It allows advanced filtering by file paths, extensions, use OR statements, ranges etc.
|
||||
Please check query language examples in the search field for some advanced query language usage.
|
||||
|
||||
|
|
@ -325,25 +323,15 @@ Please check query language examples in the search field for some advanced query
|
|||
1. Open the :file:`rhodecode.ini` file for the instance you wish to edit. The
|
||||
default location is
|
||||
:file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
|
||||
2. Find the search configuration section:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
###################################
|
||||
## SEARCH INDEXING CONFIGURATION ##
|
||||
###################################
|
||||
|
||||
search.module = rhodecode.lib.index.whoosh
|
||||
search.location = %(here)s/data/index
|
||||
|
||||
and change it to:
|
||||
2. Find the search configuration section and change it to:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
search.module = rc_elasticsearch
|
||||
search.location = http://localhost:9200
|
||||
## specify Elastic Search version, 6 for latest or 2 for legacy
|
||||
search.es_version = 6
|
||||
search.location = http://elasticsearch:9200
|
||||
## specify Elastic Search version, 8 for latest
|
||||
search.es_version = 8
|
||||
|
||||
where ``search.location`` points to the ElasticSearch server
|
||||
by default running on port 9200.
|
||||
|
|
@ -352,8 +340,7 @@ Index invocation also needs change. Please provide --es-version= and
|
|||
--engine-location= parameters to define ElasticSearch server location and it's version.
|
||||
For example::
|
||||
|
||||
rhodecode-index --instace-name=enterprise-1 --es-version=6 --engine-location=http://localhost:9200
|
||||
rhodecode-index --instance-name=enterprise-1 --es-version=8 --engine-location=http://elasticsearch:9200
|
||||
|
||||
|
||||
.. _Whoosh: https://pypi.python.org/pypi/Whoosh/
|
||||
.. _ElasticSearch 6: https://www.elastic.co/
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from rhodecode.lib import audit_logger, rc_cache, channelstream
|
|||
from rhodecode.lib import repo_maintenance
|
||||
from rhodecode.lib.auth import HasPermissionAnyApi, HasUserGroupPermissionAnyApi, HasRepoPermissionAnyApi
|
||||
from rhodecode.lib.celerylib.utils import get_task_id
|
||||
from rhodecode.lib.str_utils import safe_bytes
|
||||
from rhodecode.lib.utils2 import str2bool, time_to_datetime, safe_str, safe_int
|
||||
from rhodecode.lib.ext_json import json
|
||||
from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError, CommentVersionMismatch
|
||||
|
|
@ -615,7 +616,7 @@ def get_repo_file(
|
|||
elif details == "full":
|
||||
extended_info = content = True
|
||||
|
||||
file_path = safe_str(file_path)
|
||||
bytes_path = safe_bytes(file_path)
|
||||
try:
|
||||
# check if repo is not empty by any chance, skip quicker if it is.
|
||||
_scm = repo.scm_instance()
|
||||
|
|
@ -625,7 +626,7 @@ def get_repo_file(
|
|||
node = ScmModel().get_node(
|
||||
repo,
|
||||
commit_id,
|
||||
file_path,
|
||||
bytes_path,
|
||||
extended_info=extended_info,
|
||||
content=content,
|
||||
max_file_bytes=max_file_bytes,
|
||||
|
|
|
|||
|
|
@ -483,7 +483,7 @@ class HomeView(BaseAppView, DataGridAppView):
|
|||
if not searcher:
|
||||
return []
|
||||
|
||||
is_es_6 = searcher.is_es_6
|
||||
is_es_8 = searcher.is_es_8
|
||||
|
||||
queries = []
|
||||
repo_group_name, repo_name, repo_context = None, None, None
|
||||
|
|
@ -495,7 +495,7 @@ class HomeView(BaseAppView, DataGridAppView):
|
|||
repo_name = search_context.get("search_context[repo_name]")
|
||||
repo_context = search_context.get("search_context[repo_view_type]")
|
||||
|
||||
if is_es_6 and repo_name:
|
||||
if is_es_8 and repo_name:
|
||||
# files
|
||||
def query_modifier():
|
||||
qry = query
|
||||
|
|
@ -535,7 +535,7 @@ class HomeView(BaseAppView, DataGridAppView):
|
|||
else:
|
||||
queries.extend([commit_qry, file_qry])
|
||||
|
||||
elif is_es_6 and repo_group_name:
|
||||
elif is_es_8 and repo_group_name:
|
||||
# files
|
||||
def query_modifier():
|
||||
qry = query
|
||||
|
|
|
|||
|
|
@ -90,10 +90,12 @@ def perform_search(request, tmpl_context, repo_name=None, repo_group_name=None):
|
|||
sort=search_sort,
|
||||
)
|
||||
|
||||
search_result_count = search_result["count"]["value"]
|
||||
|
||||
formatted_results = Page(
|
||||
search_result["results"],
|
||||
page=requested_page,
|
||||
item_count=search_result["count"],
|
||||
item_count=search_result_count,
|
||||
items_per_page=page_limit,
|
||||
url_maker=url_generator,
|
||||
)
|
||||
|
|
@ -103,7 +105,7 @@ def perform_search(request, tmpl_context, repo_name=None, repo_group_name=None):
|
|||
search_tags = searcher.extract_search_tags(search_query)
|
||||
|
||||
if not search_result["error"]:
|
||||
execution_time = "{} results ({:.4f} seconds)".format(search_result["count"], search_result["runtime"])
|
||||
execution_time = "{} results ({:.4f} seconds)".format(search_result_count, search_result["runtime"])
|
||||
elif not errors:
|
||||
node = schema["search_query"]
|
||||
errors = [validation_schema.Invalid(node, search_result["error"])]
|
||||
|
|
|
|||
|
|
@ -28,16 +28,19 @@ from rhodecode.lib.index.search_utils import normalize_text_for_matching
|
|||
log = logging.getLogger(__name__)
|
||||
|
||||
# leave defaults for backward compat
|
||||
default_searcher = "rhodecode.lib.index.whoosh"
|
||||
default_location = "%(here)s/data/index"
|
||||
default_searcher = "rc_elasticsearch"
|
||||
|
||||
ES_VERSION_8 = 8
|
||||
ES_VERSION_9 = 9
|
||||
OS_VERSION_2 = 2
|
||||
|
||||
# for legacy reasons we keep 8 compat as default
|
||||
DEFAULT_ES_VERSION = ES_VERSION_8
|
||||
DEFAULT_OS_VERSION = OS_VERSION_2
|
||||
|
||||
ES_VERSION_2 = "2"
|
||||
ES_VERSION_6 = "6"
|
||||
# for legacy reasons we keep 2 compat as default
|
||||
DEFAULT_ES_VERSION = ES_VERSION_2
|
||||
|
||||
try:
|
||||
from rhodecode_tools.lib.fts_index.elasticsearch_engine_6 import ES_CONFIG # pragma: no cover
|
||||
from rhodecode_tools.lib.fts_index.elasticsearch_engine_8 import ES_CONFIG # pragma: no cover
|
||||
except ImportError:
|
||||
log.warning("rhodecode_tools not available, use of full text search is limited")
|
||||
pass
|
||||
|
|
@ -71,8 +74,8 @@ class BaseSearcher(object):
|
|||
return " ".join(normalize_text_for_matching(query).split())
|
||||
|
||||
@property
|
||||
def is_es_6(self):
|
||||
return self.es_version == ES_VERSION_6
|
||||
def is_es_8(self):
|
||||
return self.es_version >= ES_VERSION_8
|
||||
|
||||
def get_handlers(self):
|
||||
return {}
|
||||
|
|
@ -140,11 +143,9 @@ def search_config(config, prefix="search."):
|
|||
def searcher_from_config(config, prefix="search."):
|
||||
_config = search_config(config, prefix)
|
||||
|
||||
if "location" not in _config:
|
||||
_config["location"] = default_location
|
||||
if "es_version" not in _config:
|
||||
# use an old legacy ES version set to 2
|
||||
_config["es_version"] = "2"
|
||||
# use an old legacy ES version set to 8
|
||||
_config["es_version"] = DEFAULT_ES_VERSION
|
||||
|
||||
imported = importlib.import_module(_config.get("module", default_searcher))
|
||||
searcher = imported.Searcher(config=_config)
|
||||
|
|
|
|||
|
|
@ -154,53 +154,32 @@
|
|||
% endfor
|
||||
<div class="field">
|
||||
<p class="filterexample" style="position: inherit" onclick="$('#search-help').toggle()">${_('Query Language examples')}</p>
|
||||
<pre id="search-help" style="display: none">\
|
||||
|
||||
% if c.searcher.name == 'whoosh':
|
||||
Example filter terms for `Whoosh` search:
|
||||
query lang: <a href="${c.searcher.query_lang_doc}">Whoosh Query Language</a>
|
||||
Whoosh has limited query capabilities. For advanced search use ElasticSearch 6 from RhodeCode EE edition.
|
||||
<div id="search-help" style="display: none">\
|
||||
|
||||
Generate wildcards using '*' character:
|
||||
"repo_name:vcs*" - search everything starting with 'vcs'
|
||||
"repo_name:*vcs*" - search for repository containing 'vcs'
|
||||
% if c.searcher.name == 'elasticsearch':
|
||||
|
||||
Optional AND / OR operators in queries
|
||||
"repo_name:vcs OR repo_name:test"
|
||||
"owner:test AND repo_name:test*" AND extension:py
|
||||
|
||||
Move advanced search is available via ElasticSearch6 backend in EE edition.
|
||||
% elif c.searcher.name == 'elasticsearch' and c.searcher.es_version == '2':
|
||||
Example filter terms for `ElasticSearch-${c.searcher.es_version}`search:
|
||||
ElasticSearch-2 has limited query capabilities. For advanced search use ElasticSearch 6 from RhodeCode EE edition.
|
||||
<h3>Example filter terms for `ElasticSearch-${c.searcher.es_version}`search:</h3>
|
||||
query lang: <a href="${c.searcher.query_lang_doc}">ES ${c.searcher.es_version} Query Language</a></br>
|
||||
The reserved characters require escaping by `\`: <pre>+ - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /</pre>
|
||||
</br>
|
||||
|
||||
search type: content (File Content)
|
||||
indexed fields: content
|
||||
|
||||
# search for `fix` string in all files
|
||||
fix
|
||||
|
||||
search type: commit (Commit message)
|
||||
indexed fields: message
|
||||
|
||||
search type: path (File name)
|
||||
indexed fields: path
|
||||
|
||||
% else:
|
||||
Example filter terms for `ElasticSearch-${c.searcher.es_version}`search:
|
||||
query lang: <a href="${c.searcher.query_lang_doc}">ES 6 Query Language</a>
|
||||
The reserved characters needed espace by `\`: + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
|
||||
% for handler in c.searcher.get_handlers().values():
|
||||
|
||||
search type: ${handler.search_type_label}
|
||||
*indexed fields*: ${', '.join( [('\n ' if x[0]%4==0 else '')+x[1] for x in enumerate(handler.es_6_field_names)])}
|
||||
<h4>Search type: ${handler.search_type_label}</h4>
|
||||
<strong>indexed fields:</strong>
|
||||
<pre>${', '.join( [('\n' if x[0]%4==0 else '')+x[1] for x in enumerate(handler.es_6_field_names)])}</pre>
|
||||
|
||||
<pre>
|
||||
% for entry in handler.es_6_example_queries:
|
||||
${entry.rstrip()}
|
||||
${h.textwrap.dedent(entry.rstrip())}
|
||||
% endfor
|
||||
</pre>
|
||||
% endfor
|
||||
|
||||
% endif
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="field">${c.runtime}</div>
|
||||
|
|
|
|||
|
|
@ -1,8 +1,39 @@
|
|||
<%namespace name="search" file="/search/search.mako"/>
|
||||
|
||||
<%def name="highlight_text_file(has_matched_content, file_content, lexer, html_formatter, matching_lines, shown_matching_lines, url, use_hl_filter)">
|
||||
<%def name="highlight_text_file(file_path, full_file_content, hl_file_content, mimetype, url, use_hl_filter)">
|
||||
<%
|
||||
|
||||
matching_lines = []
|
||||
shown_matching_lines = 0
|
||||
|
||||
if c.searcher.is_es_8:
|
||||
# use empty terms so we default to markers usage
|
||||
total_lines, matching_lines = h.get_matching_line_offsets(hl_file_content, terms=None)
|
||||
|
||||
lines_of_interest = set()
|
||||
for line_number in matching_lines:
|
||||
if len(lines_of_interest) < max_lines:
|
||||
lines_of_interest |= set(range(
|
||||
max(line_number - line_context, 0),
|
||||
min(line_number + line_context, total_lines + 1)))
|
||||
shown_matching_lines += 1
|
||||
|
||||
has_matched_content = len(lines_of_interest) >= 1
|
||||
%>
|
||||
|
||||
|
||||
% if has_matched_content:
|
||||
${h.code_highlight(file_content, lexer, html_formatter, use_hl_filter=use_hl_filter)|n}
|
||||
<%
|
||||
lexer = h.get_lexer_safe(mimetype=mimetype, filepath=filepath)
|
||||
html_formatter = h.SearchContentCodeHtmlFormatter(
|
||||
linenos=True,
|
||||
cssclass="code-highlight",
|
||||
url=match_file_url,
|
||||
query_terms=terms,
|
||||
only_line_numbers=lines_of_interest
|
||||
)
|
||||
%>
|
||||
${h.code_highlight(hl_file_content, lexer, html_formatter, use_hl_filter=use_hl_filter)|n}
|
||||
% else:
|
||||
${_('No content matched')} <br/>
|
||||
% endif
|
||||
|
|
@ -20,41 +51,15 @@
|
|||
%for entry in c.formatted_results:
|
||||
|
||||
<%
|
||||
file_content = entry['content_highlight'] or entry['content']
|
||||
mimetype = entry.get('mimetype')
|
||||
hl_file_content = entry['content_highlight']
|
||||
full_file_content = entry['content']
|
||||
filepath = entry.get('path')
|
||||
mimetype = entry.get('mimetype')
|
||||
max_lines = h.safe_int(request.GET.get('max_lines', '10'))
|
||||
line_context = h.safe_int(request.GET.get('line_contenxt', '3'))
|
||||
line_context = h.safe_int(request.GET.get('line_context', '3'))
|
||||
|
||||
match_file_url=h.route_path('repo_files',repo_name=entry['repository'], commit_id=entry.get('commit_id', 'tip'),f_path=entry['f_path'], _query={"mark": query_mark})
|
||||
terms = c.cur_query
|
||||
|
||||
if c.searcher.is_es_6:
|
||||
# use empty terms so we default to markers usage
|
||||
total_lines, matching_lines = h.get_matching_line_offsets(file_content, terms=None)
|
||||
else:
|
||||
total_lines, matching_lines = h.get_matching_line_offsets(file_content, terms)
|
||||
|
||||
shown_matching_lines = 0
|
||||
lines_of_interest = set()
|
||||
for line_number in matching_lines:
|
||||
if len(lines_of_interest) < max_lines:
|
||||
lines_of_interest |= set(range(
|
||||
max(line_number - line_context, 0),
|
||||
min(line_number + line_context, total_lines + 1)))
|
||||
shown_matching_lines += 1
|
||||
lexer = h.get_lexer_safe(mimetype=mimetype, filepath=filepath)
|
||||
|
||||
html_formatter = h.SearchContentCodeHtmlFormatter(
|
||||
linenos=True,
|
||||
cssclass="code-highlight",
|
||||
url=match_file_url,
|
||||
query_terms=terms,
|
||||
only_line_numbers=lines_of_interest
|
||||
)
|
||||
|
||||
has_matched_content = len(lines_of_interest) >= 1
|
||||
|
||||
%>
|
||||
## search results are additionally filtered, and this check is just a safe gate
|
||||
% if c.rhodecode_user.is_admin or h.HasRepoPermissionAny('repository.write','repository.read','repository.admin')(entry['repository'], 'search results content check'):
|
||||
|
|
@ -75,7 +80,7 @@
|
|||
<div class="stats-info">
|
||||
<span class="stats-first-item">
|
||||
${entry.get('lines', 0.)} ${_ungettext('line', 'lines', entry.get('lines', 0.))}
|
||||
(${len(matching_lines)} ${_ungettext('matched', 'matched', len(matching_lines))})
|
||||
##(${len(matching_lines)} ${_ungettext('matched', 'matched', len(matching_lines))})
|
||||
</span>
|
||||
<span>
|
||||
% if entry.get('size'):
|
||||
|
|
@ -112,7 +117,7 @@
|
|||
<a href="${h.route_path('search_repo_group', repo_group_name=repo_group, _query={'q': c.cur_query})}">${_('Narrow to this repository group')}</a>
|
||||
</span>
|
||||
% endif
|
||||
## hiden if in repo view
|
||||
## hidden if in repo view
|
||||
% if not c.repo_name:
|
||||
<span class="tag tag8">
|
||||
${search.repo_icon(repo_type)}
|
||||
|
|
@ -125,17 +130,14 @@
|
|||
<div class="clear-fix"></div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="code-body search-code-body clear-fix">
|
||||
${highlight_text_file(
|
||||
has_matched_content=has_matched_content,
|
||||
file_content=file_content,
|
||||
lexer=lexer,
|
||||
html_formatter=html_formatter,
|
||||
matching_lines=matching_lines,
|
||||
shown_matching_lines=shown_matching_lines,
|
||||
file_path=file_path,
|
||||
full_file_content=full_file_content,
|
||||
hl_file_content=hl_file_content,
|
||||
mimetype=mimetype,
|
||||
url=match_file_url,
|
||||
use_hl_filter=c.searcher.is_es_6
|
||||
use_hl_filter=c.searcher.is_es_8
|
||||
)}
|
||||
</div>
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue