diff --git a/pkgs/python-packages-overrides.nix b/pkgs/python-packages-overrides.nix index cd157c3b..fbdac0f3 100644 --- a/pkgs/python-packages-overrides.nix +++ b/pkgs/python-packages-overrides.nix @@ -164,12 +164,6 @@ self: super: { }; }); - "pytest-runner" = super."pytest-runner".override (attrs: { - propagatedBuildInputs = [ - self."setuptools-scm" - ]; - }); - "python-ldap" = super."python-ldap".override (attrs: { propagatedBuildInputs = attrs.propagatedBuildInputs ++ [ pkgs.openldap diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix index ac78740e..b36db794 100644 --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -5,7 +5,7 @@ self: super: { "alembic" = super.buildPythonPackage { - name = "alembic-1.0.5"; + name = "alembic-1.0.8"; doCheck = false; propagatedBuildInputs = [ self."sqlalchemy" @@ -14,8 +14,8 @@ self: super: { self."python-dateutil" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/1c/65/b8e4f5b2f345bb13b5e0a3fddd892b0b3f0e8ad4880e954fdc6a50d00d84/alembic-1.0.5.tar.gz"; - sha256 = "0rpjqp2iq6p49x1nli18ivak1izz547nnjxi110mzrgc1v7dxzz9"; + url = "https://files.pythonhosted.org/packages/d6/bb/ec1e21f2e303689ad2170eb47fc67df9ad4199ade6759a99474c4d3535c8/alembic-1.0.8.tar.gz"; + sha256 = "1s34i1j0dsxbflxligwhnkf37a5hvcshsv8ibkcfdjf03ph42pah"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -1051,14 +1051,14 @@ self: super: { }; }; "paste" = super.buildPythonPackage { - name = "paste-3.0.5"; + name = "paste-3.0.8"; doCheck = false; propagatedBuildInputs = [ self."six" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/d4/41/91bde422400786b1b06357c1e6e3a5379f54dc3002aeb337cb767233304e/Paste-3.0.5.tar.gz"; - sha256 = "1a6i8fh1fg8r4x800fvy9r82m15clwjim6yf2g9r4dff0y40dchv"; + url = "https://files.pythonhosted.org/packages/66/65/e3acf1663438483c1f6ced0b6c6f3b90da9f0faacb0a6e2aa0f3f9f4b235/Paste-3.0.8.tar.gz"; + sha256 = "05w1sh6ky4d7pmdb8nv82n13w22jcn3qsagg5ih3hjmbws9kkwf4"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -1076,7 +1076,7 @@ self: super: { }; }; "pastescript" = super.buildPythonPackage { - name = "pastescript-3.0.0"; + name = "pastescript-3.1.0"; doCheck = false; propagatedBuildInputs = [ self."paste" @@ -1084,8 +1084,8 @@ self: super: { self."six" ]; src = fetchurl { - url = "https://files.pythonhosted.org/packages/08/2a/3797377a884ab9a064ad4d564ed612e54d26d7997caa8229c9c9df4eac31/PasteScript-3.0.0.tar.gz"; - sha256 = "1hvmyz1sbn7ws1syw567ph7km9fi0wi75r3vlyzx6sk0z26xkm6r"; + url = "https://files.pythonhosted.org/packages/9e/1d/14db1c283eb21a5d36b6ba1114c13b709629711e64acab653d9994fe346f/PasteScript-3.1.0.tar.gz"; + sha256 = "02qcxjjr32ks7a6d4f533wl34ysc7yhwlrfcyqwqbzr52250v4fs"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -1752,7 +1752,6 @@ self: super: { ]; doCheck = true; propagatedBuildInputs = [ - self."setuptools-scm" self."amqp" self."authomatic" self."babel" @@ -1942,17 +1941,6 @@ self: super: { license = [ pkgs.lib.licenses.mit ]; }; }; - "setuptools-scm" = super.buildPythonPackage { - name = "setuptools-scm-2.1.0"; - doCheck = false; - src = fetchurl { - url = "https://files.pythonhosted.org/packages/e5/62/f9e1ac314464eb5945c97542acb6bf6f3381dfa5d7a658de7730c36f31a1/setuptools_scm-2.1.0.tar.gz"; - sha256 = "0yb364cgk15sfw3x8ln4ssh98z1dj6n8iiz4r2rw1cfsxhgi8rx7"; - }; - meta = { - license = [ pkgs.lib.licenses.mit ]; - }; - }; "simplegeneric" = super.buildPythonPackage { name = "simplegeneric-0.8.1"; doCheck = false; diff --git a/requirements.txt b/requirements.txt index c388c62e..ed28842d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ ## dependencies -setuptools-scm==2.1.0 amqp==2.3.1 # not released authomatic that has updated some oauth providers https://code.rhodecode.com/upstream/authomatic/archive/90a9ce60cc405ae8a2bf5c3713acd5d78579a04e.tar.gz?md5=3c68720a1322b25254009518d1ff6801#egg=authomatic==0.1.0.post1 @@ -35,9 +34,9 @@ markupsafe==1.1.0 msgpack-python==0.5.6 pyotp==2.2.7 packaging==15.2 -paste==3.0.5 +paste==3.0.8 pastedeploy==2.0.1 -pastescript==3.0.0 +pastescript==3.1.0 peppercorn==0.6 psutil==5.5.1 py-bcrypt==0.4 @@ -98,7 +97,7 @@ nbformat==4.4.0 jupyter_client==5.0.0 ## cli tools -alembic==1.0.5 +alembic==1.0.8 invoke==0.13.0 bumpversion==0.5.3 diff --git a/rhodecode/api/tests/test_fts_search.py b/rhodecode/api/tests/test_fts_search.py new file mode 100644 index 00000000..7662cc08 --- /dev/null +++ b/rhodecode/api/tests/test_fts_search.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import pytest +from rhodecode.tests import HG_REPO +from rhodecode.api.tests.utils import ( + build_data, api_call, assert_error, assert_ok) + + +@pytest.mark.usefixtures("testuser_api", "app") +class TestApiSearch(object): + + @pytest.mark.parametrize("query, expected_hits, expected_paths", [ + ('todo', 23, [ + 'vcs/backends/hg/inmemory.py', + 'vcs/tests/test_git.py']), + ('extension:rst installation', 6, [ + 'docs/index.rst', + 'docs/installation.rst']), + ('def repo', 87, [ + 'vcs/tests/test_git.py', + 'vcs/tests/test_changesets.py']), + ('repository:%s def test' % HG_REPO, 18, [ + 'vcs/tests/test_git.py', + 'vcs/tests/test_changesets.py']), + ('"def main"', 9, [ + 'vcs/__init__.py', + 'vcs/tests/__init__.py', + 'vcs/utils/progressbar.py']), + ('owner:test_admin', 358, [ + 'vcs/tests/base.py', + 'MANIFEST.in', + 'vcs/utils/termcolors.py', + 'docs/theme/ADC/static/documentation.png']), + ('owner:test_admin def main', 72, [ + 'vcs/__init__.py', + 'vcs/tests/test_utils_filesize.py', + 'vcs/tests/test_cli.py']), + ('owner:michaƂ test', 0, []), + ]) + def test_search_content_results(self, query, expected_hits, expected_paths): + id_, params = build_data( + self.apikey_regular, 'search', + search_query=query, + search_type='content') + + response = api_call(self.app, params) + json_response = response.json + + assert json_response['result']['item_count'] == expected_hits + paths = [x['f_path'] for x in json_response['result']['results']] + + for expected_path in expected_paths: + assert expected_path in paths + + @pytest.mark.parametrize("query, expected_hits, expected_paths", [ + ('readme.rst', 3, []), + ('test*', 75, []), + ('*model*', 1, []), + ('extension:rst', 48, []), + ('extension:rst api', 24, []), + ]) + def test_search_file_paths(self, query, expected_hits, expected_paths): + id_, params = build_data( + self.apikey_regular, 'search', + search_query=query, + search_type='path') + + response = api_call(self.app, params) + json_response = response.json + + assert json_response['result']['item_count'] == expected_hits + paths = [x['f_path'] for x in json_response['result']['results']] + + for expected_path in expected_paths: + assert expected_path in paths diff --git a/rhodecode/api/views/search_api.py b/rhodecode/api/views/search_api.py new file mode 100644 index 00000000..b7213cb7 --- /dev/null +++ b/rhodecode/api/views/search_api.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2011-2019 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + + +import logging + +from rhodecode.api import jsonrpc_method +from rhodecode.api.exc import JSONRPCValidationError +from rhodecode.api.utils import Optional +from rhodecode.lib.index import searcher_from_config +from rhodecode.model import validation_schema +from rhodecode.model.validation_schema.schemas import search_schema + +log = logging.getLogger(__name__) + + +@jsonrpc_method() +def search(request, apiuser, search_query, search_type, page_limit=Optional(10), + page=Optional(1), search_sort=Optional('newfirst'), + repo_name=Optional(None), repo_group_name=Optional(None)): + """ + Fetch Full Text Search results using API. + + :param apiuser: This is filled automatically from the |authtoken|. + :type apiuser: AuthUser + :param search_query: Search query. + :type search_query: str + :param search_type: Search type. The following are valid options: + * commit + * content + * path + :type search_type: str + :param page_limit: Page item limit, from 1 to 500. Default 10 items. + :type page_limit: Optional(int) + :param page: Page number. Default first page. + :type page: Optional(int) + :param search_sort: Search sort order. Default newfirst. The following are valid options: + * newfirst + * oldfirst + :type search_sort: Optional(str) + :param repo_name: Filter by one repo. Default is all. + :type repo_name: Optional(str) + :param repo_group_name: Filter by one repo group. Default is all. + :type repo_group_name: Optional(str) + """ + + data = {'execution_time': ''} + repo_name = Optional.extract(repo_name) + repo_group_name = Optional.extract(repo_group_name) + + schema = search_schema.SearchParamsSchema() + + try: + search_params = schema.deserialize( + dict(search_query=search_query, + search_type=search_type, + search_sort=Optional.extract(search_sort), + page_limit=Optional.extract(page_limit), + requested_page=Optional.extract(page)) + ) + except validation_schema.Invalid as err: + raise JSONRPCValidationError(colander_exc=err) + + search_query = search_params.get('search_query') + search_type = search_params.get('search_type') + search_sort = search_params.get('search_sort') + + if search_params.get('search_query'): + page_limit = search_params['page_limit'] + requested_page = search_params['requested_page'] + + searcher = searcher_from_config(request.registry.settings) + + try: + search_result = searcher.search( + search_query, search_type, apiuser, repo_name, repo_group_name, + requested_page=requested_page, page_limit=page_limit, sort=search_sort) + + data.update(dict( + results=list(search_result['results']), page=requested_page, + item_count=search_result['count'], + items_per_page=page_limit)) + finally: + searcher.cleanup() + + if not search_result['error']: + data['execution_time'] = '%s results (%.3f seconds)' % ( + search_result['count'], + search_result['runtime']) + else: + node = schema['search_query'] + raise JSONRPCValidationError( + colander_exc=validation_schema.Invalid(node, search_result['error'])) + + return data diff --git a/rhodecode/config/licenses.json b/rhodecode/config/licenses.json index 7e3ea426..aa2e0c1e 100644 --- a/rhodecode/config/licenses.json +++ b/rhodecode/config/licenses.json @@ -269,17 +269,6 @@ ], "name": "python2.7-pytest-runner-4.2" }, - { - "license": [ - { - "fullName": "MIT License", - "shortName": "mit", - "spdxId": "MIT", - "url": "http://spdx.org/licenses/MIT.html" - } - ], - "name": "python2.7-setuptools-scm-2.1.0" - }, { "license": [ {