diff --git a/pkgs/python-packages-overrides.nix b/pkgs/python-packages-overrides.nix
index cd157c3b..fbdac0f3 100644
--- a/pkgs/python-packages-overrides.nix
+++ b/pkgs/python-packages-overrides.nix
@@ -164,12 +164,6 @@ self: super: {
};
});
- "pytest-runner" = super."pytest-runner".override (attrs: {
- propagatedBuildInputs = [
- self."setuptools-scm"
- ];
- });
-
"python-ldap" = super."python-ldap".override (attrs: {
propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
pkgs.openldap
diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix
index ac78740e..b36db794 100644
--- a/pkgs/python-packages.nix
+++ b/pkgs/python-packages.nix
@@ -5,7 +5,7 @@
self: super: {
"alembic" = super.buildPythonPackage {
- name = "alembic-1.0.5";
+ name = "alembic-1.0.8";
doCheck = false;
propagatedBuildInputs = [
self."sqlalchemy"
@@ -14,8 +14,8 @@ self: super: {
self."python-dateutil"
];
src = fetchurl {
- url = "https://files.pythonhosted.org/packages/1c/65/b8e4f5b2f345bb13b5e0a3fddd892b0b3f0e8ad4880e954fdc6a50d00d84/alembic-1.0.5.tar.gz";
- sha256 = "0rpjqp2iq6p49x1nli18ivak1izz547nnjxi110mzrgc1v7dxzz9";
+ url = "https://files.pythonhosted.org/packages/d6/bb/ec1e21f2e303689ad2170eb47fc67df9ad4199ade6759a99474c4d3535c8/alembic-1.0.8.tar.gz";
+ sha256 = "1s34i1j0dsxbflxligwhnkf37a5hvcshsv8ibkcfdjf03ph42pah";
};
meta = {
license = [ pkgs.lib.licenses.mit ];
@@ -1051,14 +1051,14 @@ self: super: {
};
};
"paste" = super.buildPythonPackage {
- name = "paste-3.0.5";
+ name = "paste-3.0.8";
doCheck = false;
propagatedBuildInputs = [
self."six"
];
src = fetchurl {
- url = "https://files.pythonhosted.org/packages/d4/41/91bde422400786b1b06357c1e6e3a5379f54dc3002aeb337cb767233304e/Paste-3.0.5.tar.gz";
- sha256 = "1a6i8fh1fg8r4x800fvy9r82m15clwjim6yf2g9r4dff0y40dchv";
+ url = "https://files.pythonhosted.org/packages/66/65/e3acf1663438483c1f6ced0b6c6f3b90da9f0faacb0a6e2aa0f3f9f4b235/Paste-3.0.8.tar.gz";
+ sha256 = "05w1sh6ky4d7pmdb8nv82n13w22jcn3qsagg5ih3hjmbws9kkwf4";
};
meta = {
license = [ pkgs.lib.licenses.mit ];
@@ -1076,7 +1076,7 @@ self: super: {
};
};
"pastescript" = super.buildPythonPackage {
- name = "pastescript-3.0.0";
+ name = "pastescript-3.1.0";
doCheck = false;
propagatedBuildInputs = [
self."paste"
@@ -1084,8 +1084,8 @@ self: super: {
self."six"
];
src = fetchurl {
- url = "https://files.pythonhosted.org/packages/08/2a/3797377a884ab9a064ad4d564ed612e54d26d7997caa8229c9c9df4eac31/PasteScript-3.0.0.tar.gz";
- sha256 = "1hvmyz1sbn7ws1syw567ph7km9fi0wi75r3vlyzx6sk0z26xkm6r";
+ url = "https://files.pythonhosted.org/packages/9e/1d/14db1c283eb21a5d36b6ba1114c13b709629711e64acab653d9994fe346f/PasteScript-3.1.0.tar.gz";
+ sha256 = "02qcxjjr32ks7a6d4f533wl34ysc7yhwlrfcyqwqbzr52250v4fs";
};
meta = {
license = [ pkgs.lib.licenses.mit ];
@@ -1752,7 +1752,6 @@ self: super: {
];
doCheck = true;
propagatedBuildInputs = [
- self."setuptools-scm"
self."amqp"
self."authomatic"
self."babel"
@@ -1942,17 +1941,6 @@ self: super: {
license = [ pkgs.lib.licenses.mit ];
};
};
- "setuptools-scm" = super.buildPythonPackage {
- name = "setuptools-scm-2.1.0";
- doCheck = false;
- src = fetchurl {
- url = "https://files.pythonhosted.org/packages/e5/62/f9e1ac314464eb5945c97542acb6bf6f3381dfa5d7a658de7730c36f31a1/setuptools_scm-2.1.0.tar.gz";
- sha256 = "0yb364cgk15sfw3x8ln4ssh98z1dj6n8iiz4r2rw1cfsxhgi8rx7";
- };
- meta = {
- license = [ pkgs.lib.licenses.mit ];
- };
- };
"simplegeneric" = super.buildPythonPackage {
name = "simplegeneric-0.8.1";
doCheck = false;
diff --git a/requirements.txt b/requirements.txt
index c388c62e..ed28842d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,5 @@
## dependencies
-setuptools-scm==2.1.0
amqp==2.3.1
# not released authomatic that has updated some oauth providers
https://code.rhodecode.com/upstream/authomatic/archive/90a9ce60cc405ae8a2bf5c3713acd5d78579a04e.tar.gz?md5=3c68720a1322b25254009518d1ff6801#egg=authomatic==0.1.0.post1
@@ -35,9 +34,9 @@ markupsafe==1.1.0
msgpack-python==0.5.6
pyotp==2.2.7
packaging==15.2
-paste==3.0.5
+paste==3.0.8
pastedeploy==2.0.1
-pastescript==3.0.0
+pastescript==3.1.0
peppercorn==0.6
psutil==5.5.1
py-bcrypt==0.4
@@ -98,7 +97,7 @@ nbformat==4.4.0
jupyter_client==5.0.0
## cli tools
-alembic==1.0.5
+alembic==1.0.8
invoke==0.13.0
bumpversion==0.5.3
diff --git a/rhodecode/api/tests/test_fts_search.py b/rhodecode/api/tests/test_fts_search.py
new file mode 100644
index 00000000..7662cc08
--- /dev/null
+++ b/rhodecode/api/tests/test_fts_search.py
@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2019 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import pytest
+from rhodecode.tests import HG_REPO
+from rhodecode.api.tests.utils import (
+ build_data, api_call, assert_error, assert_ok)
+
+
+@pytest.mark.usefixtures("testuser_api", "app")
+class TestApiSearch(object):
+
+ @pytest.mark.parametrize("query, expected_hits, expected_paths", [
+ ('todo', 23, [
+ 'vcs/backends/hg/inmemory.py',
+ 'vcs/tests/test_git.py']),
+ ('extension:rst installation', 6, [
+ 'docs/index.rst',
+ 'docs/installation.rst']),
+ ('def repo', 87, [
+ 'vcs/tests/test_git.py',
+ 'vcs/tests/test_changesets.py']),
+ ('repository:%s def test' % HG_REPO, 18, [
+ 'vcs/tests/test_git.py',
+ 'vcs/tests/test_changesets.py']),
+ ('"def main"', 9, [
+ 'vcs/__init__.py',
+ 'vcs/tests/__init__.py',
+ 'vcs/utils/progressbar.py']),
+ ('owner:test_admin', 358, [
+ 'vcs/tests/base.py',
+ 'MANIFEST.in',
+ 'vcs/utils/termcolors.py',
+ 'docs/theme/ADC/static/documentation.png']),
+ ('owner:test_admin def main', 72, [
+ 'vcs/__init__.py',
+ 'vcs/tests/test_utils_filesize.py',
+ 'vcs/tests/test_cli.py']),
+ ('owner:michaĆ test', 0, []),
+ ])
+ def test_search_content_results(self, query, expected_hits, expected_paths):
+ id_, params = build_data(
+ self.apikey_regular, 'search',
+ search_query=query,
+ search_type='content')
+
+ response = api_call(self.app, params)
+ json_response = response.json
+
+ assert json_response['result']['item_count'] == expected_hits
+ paths = [x['f_path'] for x in json_response['result']['results']]
+
+ for expected_path in expected_paths:
+ assert expected_path in paths
+
+ @pytest.mark.parametrize("query, expected_hits, expected_paths", [
+ ('readme.rst', 3, []),
+ ('test*', 75, []),
+ ('*model*', 1, []),
+ ('extension:rst', 48, []),
+ ('extension:rst api', 24, []),
+ ])
+ def test_search_file_paths(self, query, expected_hits, expected_paths):
+ id_, params = build_data(
+ self.apikey_regular, 'search',
+ search_query=query,
+ search_type='path')
+
+ response = api_call(self.app, params)
+ json_response = response.json
+
+ assert json_response['result']['item_count'] == expected_hits
+ paths = [x['f_path'] for x in json_response['result']['results']]
+
+ for expected_path in expected_paths:
+ assert expected_path in paths
diff --git a/rhodecode/api/views/search_api.py b/rhodecode/api/views/search_api.py
new file mode 100644
index 00000000..b7213cb7
--- /dev/null
+++ b/rhodecode/api/views/search_api.py
@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2011-2019 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+
+import logging
+
+from rhodecode.api import jsonrpc_method
+from rhodecode.api.exc import JSONRPCValidationError
+from rhodecode.api.utils import Optional
+from rhodecode.lib.index import searcher_from_config
+from rhodecode.model import validation_schema
+from rhodecode.model.validation_schema.schemas import search_schema
+
+log = logging.getLogger(__name__)
+
+
+@jsonrpc_method()
+def search(request, apiuser, search_query, search_type, page_limit=Optional(10),
+ page=Optional(1), search_sort=Optional('newfirst'),
+ repo_name=Optional(None), repo_group_name=Optional(None)):
+ """
+ Fetch Full Text Search results using API.
+
+ :param apiuser: This is filled automatically from the |authtoken|.
+ :type apiuser: AuthUser
+ :param search_query: Search query.
+ :type search_query: str
+ :param search_type: Search type. The following are valid options:
+ * commit
+ * content
+ * path
+ :type search_type: str
+ :param page_limit: Page item limit, from 1 to 500. Default 10 items.
+ :type page_limit: Optional(int)
+ :param page: Page number. Default first page.
+ :type page: Optional(int)
+ :param search_sort: Search sort order. Default newfirst. The following are valid options:
+ * newfirst
+ * oldfirst
+ :type search_sort: Optional(str)
+ :param repo_name: Filter by one repo. Default is all.
+ :type repo_name: Optional(str)
+ :param repo_group_name: Filter by one repo group. Default is all.
+ :type repo_group_name: Optional(str)
+ """
+
+ data = {'execution_time': ''}
+ repo_name = Optional.extract(repo_name)
+ repo_group_name = Optional.extract(repo_group_name)
+
+ schema = search_schema.SearchParamsSchema()
+
+ try:
+ search_params = schema.deserialize(
+ dict(search_query=search_query,
+ search_type=search_type,
+ search_sort=Optional.extract(search_sort),
+ page_limit=Optional.extract(page_limit),
+ requested_page=Optional.extract(page))
+ )
+ except validation_schema.Invalid as err:
+ raise JSONRPCValidationError(colander_exc=err)
+
+ search_query = search_params.get('search_query')
+ search_type = search_params.get('search_type')
+ search_sort = search_params.get('search_sort')
+
+ if search_params.get('search_query'):
+ page_limit = search_params['page_limit']
+ requested_page = search_params['requested_page']
+
+ searcher = searcher_from_config(request.registry.settings)
+
+ try:
+ search_result = searcher.search(
+ search_query, search_type, apiuser, repo_name, repo_group_name,
+ requested_page=requested_page, page_limit=page_limit, sort=search_sort)
+
+ data.update(dict(
+ results=list(search_result['results']), page=requested_page,
+ item_count=search_result['count'],
+ items_per_page=page_limit))
+ finally:
+ searcher.cleanup()
+
+ if not search_result['error']:
+ data['execution_time'] = '%s results (%.3f seconds)' % (
+ search_result['count'],
+ search_result['runtime'])
+ else:
+ node = schema['search_query']
+ raise JSONRPCValidationError(
+ colander_exc=validation_schema.Invalid(node, search_result['error']))
+
+ return data
diff --git a/rhodecode/config/licenses.json b/rhodecode/config/licenses.json
index 7e3ea426..aa2e0c1e 100644
--- a/rhodecode/config/licenses.json
+++ b/rhodecode/config/licenses.json
@@ -269,17 +269,6 @@
],
"name": "python2.7-pytest-runner-4.2"
},
- {
- "license": [
- {
- "fullName": "MIT License",
- "shortName": "mit",
- "spdxId": "MIT",
- "url": "http://spdx.org/licenses/MIT.html"
- }
- ],
- "name": "python2.7-setuptools-scm-2.1.0"
- },
{
"license": [
{