engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

repository-groups: use lazy loaded admin dashboard

Browse source
This commit is contained in:
Marcin Kuzminski 2019-04-03 23:15:02 +02:00
parent 8920170d1d
commit 5ac3be4f4e
10 changed files with 364 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

2
rhodecode/__init__.py
View file

@ -45,7 +45,7 @@ PYRAMID_SETTINGS = {}
EXTENSIONS = {}
__version__ = ('.'.join((str(each) for each in VERSION[:3])))
__dbversion__ = 95 # defines current db version for migrations
__dbversion__ = 97 # defines current db version for migrations
__platform__ = platform.system()
__license__ = 'AGPLv3, and Commercial License'
__author__ = 'RhodeCode GmbH'

4
rhodecode/apps/admin/__init__.py
View file

@ -423,6 +423,10 @@ def admin_routes(config):
name='repo_groups',
pattern='/repo_groups')
config.add_route(
name='repo_groups_data',
pattern='/repo_groups_data')
config.add_route(
name='repo_group_new',
pattern='/repo_group/new')

32
rhodecode/apps/admin/tests/test_admin_repository_groups.py
View file

@ -23,11 +23,11 @@ import pytest
from rhodecode.apps._base import ADMIN_PREFIX
from rhodecode.lib import helpers as h
from rhodecode.model.db import Repository, UserRepoToPerm, User
from rhodecode.model.db import Repository, UserRepoToPerm, User, RepoGroup
from rhodecode.model.meta import Session
from rhodecode.model.repo_group import RepoGroupModel
from rhodecode.tests import (
assert_session_flash, TEST_USER_REGULAR_LOGIN, TESTS_TMP_PATH, TestController)
assert_session_flash, TEST_USER_REGULAR_LOGIN, TESTS_TMP_PATH)
from rhodecode.tests.fixture import Fixture
fixture = Fixture()
@ -38,6 +38,7 @@ def route_path(name, params=None, **kwargs):
base_url = {
'repo_groups': ADMIN_PREFIX + '/repo_groups',
'repo_groups_data': ADMIN_PREFIX + '/repo_groups_data',
'repo_group_new': ADMIN_PREFIX + '/repo_group/new',
'repo_group_create': ADMIN_PREFIX + '/repo_group/create',
@ -59,13 +60,30 @@ def _get_permission_for_user(user, repo):
@pytest.mark.usefixtures("app")
class TestAdminRepositoryGroups(object):
def test_show_repo_groups(self, autologin_user):
response = self.app.get(route_path('repo_groups'))
response.mustcontain('data: []')
def test_show_repo_groups_after_creating_group(self, autologin_user):
def test_show_repo_groups(self, autologin_user):
self.app.get(route_path('repo_groups'))
def test_show_repo_groups_data(self, autologin_user, xhr_header):
response = self.app.get(route_path(
'repo_groups_data'), extra_environ=xhr_header)
all_repo_groups = RepoGroup.query().count()
assert response.json['recordsTotal'] == all_repo_groups
def test_show_repo_groups_data_filtered(self, autologin_user, xhr_header):
response = self.app.get(route_path(
'repo_groups_data', params={'search[value]': 'empty_search'}),
extra_environ=xhr_header)
all_repo_groups = RepoGroup.query().count()
assert response.json['recordsTotal'] == all_repo_groups
assert response.json['recordsFiltered'] == 0
def test_show_repo_groups_after_creating_group(self, autologin_user, xhr_header):
fixture.create_repo_group('test_repo_group')
response = self.app.get(route_path('repo_groups'))
response = self.app.get(route_path(
'repo_groups_data'), extra_environ=xhr_header)
response.mustcontain('"name_raw": "test_repo_group"')
fixture.destroy_repo_group('test_repo_group')

170
rhodecode/apps/admin/views/repo_groups.py
View file

@ -17,7 +17,7 @@
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import datetime
import logging
import formencode
import formencode.htmlfill
@ -30,16 +30,16 @@ from pyramid.response import Response
from rhodecode import events
from rhodecode.apps._base import BaseAppView, DataGridAppView
from rhodecode.lib.ext_json import json
from rhodecode.lib.auth import (
LoginRequired, CSRFRequired, NotAnonymous,
HasPermissionAny, HasRepoGroupPermissionAny)
from rhodecode.lib import helpers as h, audit_logger
from rhodecode.lib.utils2 import safe_int, safe_unicode
from rhodecode.lib.utils2 import safe_int, safe_unicode, datetime_to_time
from rhodecode.model.forms import RepoGroupForm
from rhodecode.model.repo_group import RepoGroupModel
from rhodecode.model.scm import RepoGroupList
from rhodecode.model.db import Session, RepoGroup
from rhodecode.model.db import (
or_, count, func, in_filter_generator, Session, RepoGroup, User, Repository)
log = logging.getLogger(__name__)
@ -88,23 +88,169 @@ class AdminRepoGroupsView(BaseAppView, DataGridAppView):
return False
return False
# permission check in data loading of
# `repo_group_list_data` via RepoGroupList
@LoginRequired()
@NotAnonymous()
# perms check inside
@view_config(
route_name='repo_groups', request_method='GET',
renderer='rhodecode:templates/admin/repo_groups/repo_groups.mako')
def repo_group_list(self):
c = self.load_default_context()
repo_group_list = RepoGroup.get_all_repo_groups()
repo_group_list_acl = RepoGroupList(
repo_group_list, perm_set=['group.admin'])
repo_group_data = RepoGroupModel().get_repo_groups_as_dict(
repo_group_list=repo_group_list_acl, admin=True)
c.data = json.dumps(repo_group_data)
return self._get_template_context(c)
# permission check inside
@LoginRequired()
@NotAnonymous()
@view_config(
route_name='repo_groups_data', request_method='GET',
renderer='json_ext', xhr=True)
def repo_group_list_data(self):
self.load_default_context()
column_map = {
'name_raw': 'group_name_hash',
'desc': 'group_description',
'last_change_raw': 'updated_on',
'top_level_repos': 'repos_total',
'owner': 'user_username',
}
draw, start, limit = self._extract_chunk(self.request)
search_q, order_by, order_dir = self._extract_ordering(
self.request, column_map=column_map)
_render = self.request.get_partial_renderer(
'rhodecode:templates/data_table/_dt_elements.mako')
c = _render.get_call_context()
def quick_menu(repo_group_name):
return _render('quick_repo_group_menu', repo_group_name)
def repo_group_lnk(repo_group_name):
return _render('repo_group_name', repo_group_name)
def last_change(last_change):
if isinstance(last_change, datetime.datetime) and not last_change.tzinfo:
delta = datetime.timedelta(
seconds=(datetime.datetime.now() - datetime.datetime.utcnow()).seconds)
last_change = last_change + delta
return _render("last_change", last_change)
def desc(desc, personal):
return _render(
'repo_group_desc', desc, personal, c.visual.stylify_metatags)
def repo_group_actions(repo_group_id, repo_group_name, gr_count):
return _render(
'repo_group_actions', repo_group_id, repo_group_name, gr_count)
def user_profile(username):
return _render('user_profile', username)
auth_repo_group_list = RepoGroupList(
RepoGroup.query().all(), perm_set=['group.admin'])
allowed_ids = [-1]
for repo_group in auth_repo_group_list:
allowed_ids.append(repo_group.group_id)
repo_groups_data_total_count = RepoGroup.query()\
.filter(or_(
# generate multiple IN to fix limitation problems
*in_filter_generator(RepoGroup.group_id, allowed_ids)
)) \
.count()
repo_groups_data_total_inactive_count = RepoGroup.query()\
.filter(RepoGroup.group_id.in_(allowed_ids))\
.count()
repo_count = count(Repository.repo_id)
base_q = Session.query(
RepoGroup.group_name,
RepoGroup.group_name_hash,
RepoGroup.group_description,
RepoGroup.group_id,
RepoGroup.personal,
RepoGroup.updated_on,
User,
repo_count.label('repos_count')
) \
.filter(or_(
# generate multiple IN to fix limitation problems
*in_filter_generator(RepoGroup.group_id, allowed_ids)
)) \
.outerjoin(Repository) \
.join(User, User.user_id == RepoGroup.user_id) \
.group_by(RepoGroup, User)
if search_q:
like_expression = u'%{}%'.format(safe_unicode(search_q))
base_q = base_q.filter(or_(
RepoGroup.group_name.ilike(like_expression),
))
repo_groups_data_total_filtered_count = base_q.count()
# the inactive isn't really used, but we still make it same as other data grids
# which use inactive (users,user groups)
repo_groups_data_total_filtered_inactive_count = repo_groups_data_total_filtered_count
sort_defined = False
if order_by == 'group_name':
sort_col = func.lower(RepoGroup.group_name)
sort_defined = True
elif order_by == 'repos_total':
sort_col = repo_count
sort_defined = True
elif order_by == 'user_username':
sort_col = User.username
else:
sort_col = getattr(RepoGroup, order_by, None)
if sort_defined or sort_col:
if order_dir == 'asc':
sort_col = sort_col.asc()
else:
sort_col = sort_col.desc()
base_q = base_q.order_by(sort_col)
base_q = base_q.offset(start).limit(limit)
# authenticated access to user groups
auth_repo_group_list = base_q.all()
repo_groups_data = []
for repo_gr in auth_repo_group_list:
row = {
"menu": quick_menu(repo_gr.group_name),
"name": repo_group_lnk(repo_gr.group_name),
"name_raw": repo_gr.group_name,
"last_change": last_change(repo_gr.updated_on),
"last_change_raw": datetime_to_time(repo_gr.updated_on),
"last_changeset": "",
"last_changeset_raw": "",
"desc": desc(repo_gr.group_description, repo_gr.personal),
"owner": user_profile(repo_gr.User.username),
"top_level_repos": repo_gr.repos_count,
"action": repo_group_actions(
repo_gr.group_id, repo_gr.group_name, repo_gr.repos_count),
}
repo_groups_data.append(row)
data = ({
'draw': draw,
'data': repo_groups_data,
'recordsTotal': repo_groups_data_total_count,
'recordsTotalInactive': repo_groups_data_total_inactive_count,
'recordsFiltered': repo_groups_data_total_filtered_count,
'recordsFilteredInactive': repo_groups_data_total_filtered_inactive_count,
})
return data
@LoginRequired()
@NotAnonymous()
# perm checks inside

54
rhodecode/lib/dbmigrate/versions/096_version_4_17_0.py Normal file
View file

@ -0,0 +1,54 @@
# -*- coding: utf-8 -*-
import logging
from alembic.migration import MigrationContext
from alembic.operations import Operations
from sqlalchemy import String, Column
from sqlalchemy.sql import text
from rhodecode.lib.dbmigrate.versions import _reset_base
from rhodecode.model import meta, init_model_encryption
from rhodecode.model.db import RepoGroup
log = logging.getLogger(__name__)
def upgrade(migrate_engine):
"""
Upgrade operations go here.
Don't create your own engine; bind migrate_engine to your metadata
"""
_reset_base(migrate_engine)
from rhodecode.lib.dbmigrate.schema import db_4_16_0_2
init_model_encryption(db_4_16_0_2)
context = MigrationContext.configure(migrate_engine.connect())
op = Operations(context)
repo_group = db_4_16_0_2.RepoGroup.__table__
with op.batch_alter_table(repo_group.name) as batch_op:
batch_op.add_column(
Column("repo_group_name_hash", String(1024), nullable=True, unique=False))
_generate_repo_group_name_hashes(db_4_16_0_2, op, meta.Session)
def downgrade(migrate_engine):
pass
def _generate_repo_group_name_hashes(models, op, session):
repo_groups = models.RepoGroup.get_all()
for repo_group in repo_groups:
print(repo_group.group_name)
hash_ = RepoGroup.hash_repo_group_name(repo_group.group_name)
params = {'hash': hash_, 'id': repo_group.group_id}
query = text(
'UPDATE groups SET repo_group_name_hash = :hash'
' WHERE group_id = :id').bindparams(**params)
op.execute(query)
session().commit()

39
rhodecode/lib/dbmigrate/versions/097_version_4_17_0.py Normal file
View file

@ -0,0 +1,39 @@
# -*- coding: utf-8 -*-
import logging
from alembic.migration import MigrationContext
from alembic.operations import Operations
from rhodecode.lib.dbmigrate.versions import _reset_base
from rhodecode.model import init_model_encryption
log = logging.getLogger(__name__)
def upgrade(migrate_engine):
"""
Upgrade operations go here.
Don't create your own engine; bind migrate_engine to your metadata
"""
_reset_base(migrate_engine)
from rhodecode.lib.dbmigrate.schema import db_4_16_0_2
init_model_encryption(db_4_16_0_2)
context = MigrationContext.configure(migrate_engine.connect())
op = Operations(context)
repo_group = db_4_16_0_2.RepoGroup.__table__
with op.batch_alter_table(repo_group.name) as batch_op:
batch_op.alter_column("repo_group_name_hash", nullable=False)
def downgrade(migrate_engine):
pass
def _generate_repo_group_name_hashes(models, op, session):
pass

33
rhodecode/model/db.py
View file

@ -25,6 +25,7 @@ Database Models for RhodeCode Enterprise
import re
import os
import time
import string
import hashlib
import logging
import datetime
@ -50,6 +51,7 @@ from sqlalchemy.dialects.mysql import LONGTEXT
from zope.cachedescriptors.property import Lazy as LazyProperty
from pyramid import compat
from pyramid.threadlocal import get_current_request
from webhelpers.text import collapse, remove_formatting
from rhodecode.translation import _
from rhodecode.lib.vcs import get_vcs_instance
@ -2469,7 +2471,8 @@ class RepoGroup(Base, BaseModel):
CHOICES_SEPARATOR = '/' # used to generate select2 choices for nested groups
group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
group_name = Column("group_name", String(255), nullable=False, unique=True, default=None)
_group_name = Column("group_name", String(255), nullable=False, unique=True, default=None)
group_name_hash = Column("repo_group_name_hash", String(1024), nullable=False, unique=False)
group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)
group_description = Column("group_description", String(10000), nullable=True, unique=None, default=None)
enable_locking = Column("enable_locking", Boolean(), nullable=False, unique=None, default=False)
@ -2492,6 +2495,15 @@ class RepoGroup(Base, BaseModel):
return u"<%s('id:%s:%s')>" % (
self.__class__.__name__, self.group_id, self.group_name)
@hybrid_property
def group_name(self):
return self._group_name
@group_name.setter
def group_name(self, value):
self._group_name = value
self.group_name_hash = self.hash_repo_group_name(value)
@validates('group_parent_id')
def validate_group_parent_id(self, key, val):
"""
@ -2507,6 +2519,18 @@ class RepoGroup(Base, BaseModel):
from rhodecode.lib import helpers as h
return h.escape(self.group_description)
@classmethod
def hash_repo_group_name(cls, repo_group_name):
val = remove_formatting(repo_group_name)
val = safe_str(val).lower()
chars = []
for c in val:
if c not in string.ascii_letters:
c = str(ord(c))
chars.append(c)
return ''.join(chars)
@classmethod
def _generate_choice(cls, repo_group):
from webhelpers.html import literal as _literal
@ -2770,6 +2794,13 @@ class RepoGroup(Base, BaseModel):
}
return data
def get_dict(self):
# Since we transformed `group_name` to a hybrid property, we need to
# keep compatibility with the code which uses `group_name` field.
result = super(RepoGroup, self).get_dict()
result['group_name'] = result.pop('_group_name', None)
return result
class Permission(Base, BaseModel):
__tablename__ = 'permissions'

67
rhodecode/templates/admin/repo_groups/repo_groups.mako
View file

@ -10,7 +10,7 @@
<%def name="breadcrumbs_links()">
<input class="q_filter_box" id="q_filter" size="15" type="text" name="filter" placeholder="${_('quick filter...')}" value=""/>
${h.link_to(_('Admin'),h.route_path('admin_home'))} &raquo; <span id="repo_group_count">0</span> ${_('repository groups')}
${h.link_to(_('Admin'),h.route_path('admin_home'))} &raquo; <span id="repo_group_count">0</span>
</%def>
<%def name="menu_bar_nav()">
@ -36,15 +36,32 @@
<script>
$(document).ready(function() {
var get_datatable_count = function(){
var api = $('#group_list_table').dataTable().api();
$('#repo_group_count').text(api.page.info().recordsDisplay);
};
var $repoGroupsListTable = $('#group_list_table');
// repo group list
$('#group_list_table').DataTable({
data: ${c.data|n},
$repoGroupsListTable.DataTable({
processing: true,
serverSide: true,
ajax: {
"url": "${h.route_path('repo_groups_data')}",
"dataSrc": function (json) {
var filteredCount = json.recordsFiltered;
var filteredInactiveCount = json.recordsFilteredInactive;
var totalInactive = json.recordsTotalInactive;
var total = json.recordsTotal;
var _text = _gettext(
"{0} of {1} repository groups").format(
filteredCount, total);
if (total === filteredCount) {
_text = _gettext("{0} repository groups").format(total);
}
$('#repo_group_count').text(_text);
return json.data;
},
},
dom: 'rtp',
pageLength: ${c.visual.admin_grid_items},
order: [[ 0, "asc" ]],
@ -62,36 +79,34 @@ $(document).ready(function() {
{ data: {"_": "owner",
"sort": "owner"}, title: "${_('Owner')}", className: "td-user" },
{ data: {"_": "action",
"sort": "action"}, title: "${_('Action')}", className: "td-action" }
"sort": "action"}, title: "${_('Action')}", className: "td-action", orderable: false }
],
language: {
paginate: DEFAULT_GRID_PAGINATION,
sProcessing: _gettext('loading...'),
emptyTable: _gettext("No repository groups available yet.")
},
"initComplete": function( settings, json ) {
get_datatable_count();
quick_repo_menu();
}
});
// update the counter when doing search
$('#group_list_table').on( 'search.dt', function (e,settings) {
get_datatable_count();
$repoGroupsListTable.on('xhr.dt', function(e, settings, json, xhr){
$repoGroupsListTable.css('opacity', 1);
});
// filter, filter both grids
$('#q_filter').on( 'keyup', function () {
var repo_group_api = $('#group_list_table').dataTable().api();
repo_group_api
.columns(0)
.search(this.value)
.draw();
$repoGroupsListTable.on('preXhr.dt', function(e, settings, data){
$repoGroupsListTable.css('opacity', 0.3);
});
// refilter table if page load via back button
$("#q_filter").trigger('keyup');
// filter
$('#q_filter').on('keyup',
$.debounce(250, function() {
$repoGroupsListTable.DataTable().search(
$('#q_filter').val()
).draw();
})
);
});
</script>
</%def>

2
rhodecode/templates/admin/repos/repos.mako
View file

@ -66,7 +66,7 @@ $(document).ready(function() {
{ data: {"_": "state",
"sort": "state"}, title: "${_('State')}", className: "td-tags td-state" },
{ data: {"_": "action",
"sort": "action"}, title: "${_('Action')}", className: "td-action" }
"sort": "action"}, title: "${_('Action')}", className: "td-action", orderable: false }
],
language: {
paginate: DEFAULT_GRID_PAGINATION,

13
rhodecode/tests/functional/test_delegated_admin.py
View file

@ -34,6 +34,8 @@ def route_path(name, params=None, **kwargs):
ADMIN_PREFIX + '/repos',
'repo_groups':
ADMIN_PREFIX + '/repo_groups',
'repo_groups_data':
ADMIN_PREFIX + '/repo_groups_data',
'user_groups':
ADMIN_PREFIX + '/user_groups',
'user_groups_data':
@ -67,8 +69,9 @@ class TestAdminDelegatedUser(TestController):
response = self.app.get(route_path('repos'), status=200)
response.mustcontain('data: []')
response = self.app.get(route_path('repo_groups'), status=200)
response.mustcontain('data: []')
response = self.app.get(route_path('repo_groups_data'),
status=200, extra_environ=xhr_header)
assert response.json['data'] == []
response = self.app.get(route_path('user_groups_data'),
status=200, extra_environ=xhr_header)
@ -102,7 +105,8 @@ class TestAdminDelegatedUser(TestController):
response = self.app.get(route_path('repos'), status=200)
response.mustcontain('"name_raw": "{}"'.format(repo_name))
response = self.app.get(route_path('repo_groups'), status=200)
response = self.app.get(route_path('repo_groups_data'),
extra_environ=xhr_header, status=200)
response.mustcontain('"name_raw": "{}"'.format(repo_group_name))
response = self.app.get(route_path('user_groups_data'),
@ -144,7 +148,8 @@ class TestAdminDelegatedUser(TestController):
response = self.app.get(route_path('repos'), status=200)
response.mustcontain('"name_raw": "{}"'.format(repo_name))
response = self.app.get(route_path('repo_groups'), status=200)
response = self.app.get(route_path('repo_groups_data'),
extra_environ=xhr_header, status=200)
response.mustcontain('"name_raw": "{}"'.format(repo_group_name))
response = self.app.get(route_path('user_groups_data'),