goto-switcher: optimized performance and query capabilities.

- Previous implementation had on significant bug. The use of LIMIT 20
was limiting results BEFORE auth checks. In case of large ammount of similarly
named repositories user didn't had access too, the result goto search was empty.

This was becuase first 20 items fetched didn't pass permission checks and final
auth list was empty.

To fix this we now use proper filtering for auth using SQL. It means we first check
user allowed repositories, and add this as a filter so end result from database is already
to only the accessible repositories.

rhodecode/apps/home/views.py

@@ -30,10 +30,11 @@ from rhodecode.lib.auth import LoginRequired, NotAnonymous, \
 from rhodecode.lib.index import searcher_from_config
 from rhodecode.lib.utils2 import safe_unicode, str2bool
 from rhodecode.lib.ext_json import json
-from rhodecode.model.db import func, Repository, RepoGroup
+from rhodecode.model.db import (
+    func, or_, in_filter_generator, Repository, RepoGroup)
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
-from rhodecode.model.scm import ScmModel, RepoGroupList, RepoList
+from rhodecode.model.scm import RepoGroupList, RepoList
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 
@@ -101,9 +102,17 @@ class HomeView(BaseAppView):
         return {'suggestions': _user_groups}
 
     def _get_repo_list(self, name_contains=None, repo_type=None, limit=20):
+        allowed_ids = self._rhodecode_user.repo_acl_ids(
+            ['repository.read', 'repository.write', 'repository.admin'],
+            cache=False, name_filter=name_contains)
+
         query = Repository.query()\
             .order_by(func.length(Repository.repo_name))\
-            .order_by(Repository.repo_name)
+            .order_by(Repository.repo_name)\
+            .filter(or_(
+                # generate multiple IN to fix limitation problems
+                *in_filter_generator(Repository.repo_id, allowed_ids)
+            ))
 
         if repo_type:
             query = query.filter(Repository.repo_type == repo_type)
@@ -114,23 +123,30 @@ class HomeView(BaseAppView):
                 Repository.repo_name.ilike(ilike_expression))
             query = query.limit(limit)
 
-        all_repos = query.all()
-        # permission checks are inside this function
-        repo_iter = ScmModel().get_repos(all_repos)
+        acl_repo_iter = query
+
         return [
             {
-                'id': obj['name'],
-                'text': obj['name'],
+                'id': obj.repo_name,
+                'text': obj.repo_name,
                 'type': 'repo',
-                'obj': obj['dbrepo'],
-                'url': h.route_path('repo_summary', repo_name=obj['name'])
+                'obj': {'repo_type': obj.repo_type, 'private': obj.private},
+                'url': h.route_path('repo_summary', repo_name=obj.repo_name)
             }
-            for obj in repo_iter]
+            for obj in acl_repo_iter]
 
     def _get_repo_group_list(self, name_contains=None, limit=20):
+        allowed_ids = self._rhodecode_user.repo_group_acl_ids(
+            ['group.read', 'group.write', 'group.admin'],
+            cache=False, name_filter=name_contains)
+
         query = RepoGroup.query()\
             .order_by(func.length(RepoGroup.group_name))\
-            .order_by(RepoGroup.group_name)
+            .order_by(RepoGroup.group_name) \
+            .filter(or_(
+                # generate multiple IN to fix limitation problems
+                *in_filter_generator(RepoGroup.group_id, allowed_ids)
+            ))
 
         if name_contains:
             ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
@@ -138,8 +154,8 @@ class HomeView(BaseAppView):
                 RepoGroup.group_name.ilike(ilike_expression))
             query = query.limit(limit)
 
-        all_groups = query.all()
-        repo_groups_iter = ScmModel().get_repo_groups(all_groups)
+        acl_repo_iter = query
+
         return [
             {
                 'id': obj.group_name,
@@ -149,7 +165,7 @@ class HomeView(BaseAppView):
                 'url': h.route_path(
                     'repo_group_home', repo_group_name=obj.group_name)
             }
-            for obj in repo_groups_iter]
+            for obj in acl_repo_iter]
 
     def _get_hash_commit_list(self, auth_user, query=None):
         if not query or len(query) < 3:

rhodecode/model/db.py

@@ -100,6 +100,24 @@ def _hash_key(k):
     return md5_safe(k)
 
 
+def in_filter_generator(qry, items, limit=500):
+    """
+    Splits IN() into multiple with OR
+    e.g.::
+    cnt = Repository.query().filter(
+        or_(
+            *in_filter_generator(Repository.repo_id, range(100000))
+        )).count()
+    """
+    parts = []
+    for chunk in xrange(0, len(items), limit):
+        parts.append(
+            qry.in_(items[chunk: chunk + limit])
+        )
+
+    return parts
+
+
 class EncryptedTextValue(TypeDecorator):
     """
     Special column for encrypted long text data, use like::