engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

admin-users: moved grid browsing to pyramid.

Browse source 
- fixed loading data to be lazy fetched.
- speeds up loading large user sets significantly
This commit is contained in:
Marcin Kuzminski 2017-03-07 21:17:02 +01:00
parent ae271af5cc
commit f8631f5fec
11 changed files with 209 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

9
rhodecode/apps/admin/__init__.py
View file

@ -53,6 +53,15 @@ def includeme(config):
name='admin_settings_sessions_cleanup',
pattern=ADMIN_PREFIX + '/settings/sessions/cleanup')
# users admin
config.add_route(
name='users',
pattern=ADMIN_PREFIX + '/users')
config.add_route(
name='users_data',
pattern=ADMIN_PREFIX + '/users_data')
# user auth tokens
config.add_route(
name='edit_user_auth_tokens',

36
rhodecode/apps/admin/tests/test_admin_users.py
View file

@ -22,18 +22,18 @@ import pytest
from rhodecode.model.db import User, UserApiKeys
from rhodecode.apps._base import ADMIN_PREFIX
from rhodecode.tests import (
TestController, TEST_USER_REGULAR_LOGIN, assert_session_flash)
from rhodecode.tests.fixture import Fixture
from rhodecode.tests.utils import AssertResponse
fixture = Fixture()
def route_path(name, params=None, **kwargs):
import urllib
from rhodecode.apps._base import ADMIN_PREFIX
def route_path(name, **kwargs):
return {
base_url = {
'users':
ADMIN_PREFIX + '/users',
'users_data':
@ -46,9 +46,37 @@ def route_path(name, **kwargs):
ADMIN_PREFIX + '/users/{user_id}/edit/auth_tokens/delete',
}[name].format(**kwargs)
if params:
base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
return base_url
class TestAdminUsersView(TestController):
def test_show_users(self):
self.log_user()
self.app.get(route_path('users'))
def test_show_users_data(self, xhr_header):
self.log_user()
response = self.app.get(route_path(
'users_data'), extra_environ=xhr_header)
all_users = User.query().filter(
User.username != User.DEFAULT_USER).count()
assert response.json['recordsTotal'] == all_users
def test_show_users_data_filtered(self, xhr_header):
self.log_user()
response = self.app.get(route_path(
'users_data', params={'search[value]': 'empty_search'}),
extra_environ=xhr_header)
all_users = User.query().filter(
User.username != User.DEFAULT_USER).count()
assert response.json['recordsTotal'] == all_users
assert response.json['recordsFiltered'] == 0
def test_auth_tokens_default_user(self):
self.log_user()
user = User.get_default_user()

103
rhodecode/apps/admin/views/users.py
View file

@ -28,9 +28,9 @@ from rhodecode.lib.auth import (
LoginRequired, HasPermissionAllDecorator, CSRFRequired)
from rhodecode.lib import helpers as h
from rhodecode.lib.utils import PartialRenderer
from rhodecode.lib.utils2 import safe_int
from rhodecode.lib.utils2 import safe_int, safe_unicode
from rhodecode.model.auth_token import AuthTokenModel
from rhodecode.model.db import User
from rhodecode.model.db import User, or_
from rhodecode.model.meta import Session
log = logging.getLogger(__name__)
@ -58,6 +58,105 @@ class AdminUsersView(BaseAppView):
# is a pyramid view
raise HTTPFound('/')
def _extract_ordering(self, request):
column_index = safe_int(request.GET.get('order[0][column]'))
order_dir = request.GET.get(
'order[0][dir]', 'desc')
order_by = request.GET.get(
'columns[%s][data][sort]' % column_index, 'name_raw')
# translate datatable to DB columns
order_by = {
'first_name': 'name',
'last_name': 'lastname',
'last_activity': ''
}.get(order_by) or order_by
search_q = request.GET.get('search[value]')
return search_q, order_by, order_dir
def _extract_chunk(self, request):
start = safe_int(request.GET.get('start'), 0)
length = safe_int(request.GET.get('length'), 25)
draw = safe_int(request.GET.get('draw'))
return draw, start, length
@HasPermissionAllDecorator('hg.admin')
@view_config(
route_name='users', request_method='GET',
renderer='rhodecode:templates/admin/users/users.mako')
def users_list(self):
c = self.load_default_context()
return self._get_template_context(c)
@HasPermissionAllDecorator('hg.admin')
@view_config(
# renderer defined below
route_name='users_data', request_method='GET', renderer='json',
xhr=True)
def users_list_data(self):
draw, start, limit = self._extract_chunk(self.request)
search_q, order_by, order_dir = self._extract_ordering(self.request)
_render = PartialRenderer('data_table/_dt_elements.mako')
def user_actions(user_id, username):
return _render("user_actions", user_id, username)
users_data_total_count = User.query()\
.filter(User.username != User.DEFAULT_USER) \
.count()
# json generate
base_q = User.query().filter(User.username != User.DEFAULT_USER)
if search_q:
like_expression = u'{}%'.format(safe_unicode(search_q))
base_q = base_q.filter(or_(
User.username.ilike(like_expression),
User._email.ilike(like_expression),
User.name.ilike(like_expression),
User.lastname.ilike(like_expression),
))
users_data_total_filtered_count = base_q.count()
sort_col = getattr(User, order_by, None)
if sort_col and order_dir == 'asc':
base_q = base_q.order_by(sort_col.asc())
elif sort_col:
base_q = base_q.order_by(sort_col.desc())
base_q = base_q.offset(start).limit(limit)
users_list = base_q.all()
users_data = []
for user in users_list:
users_data.append({
"username": h.gravatar_with_user(user.username),
"email": user.email,
"first_name": h.escape(user.name),
"last_name": h.escape(user.lastname),
"last_login": h.format_date(user.last_login),
"last_activity": h.format_date(
h.time_to_datetime(user.user_data.get('last_activity', 0))),
"active": h.bool2icon(user.active),
"active_raw": user.active,
"admin": h.bool2icon(user.admin),
"extern_type": user.extern_type,
"extern_name": user.extern_name,
"action": user_actions(user.user_id, user.username),
})
data = ({
'draw': draw,
'data': users_data,
'recordsTotal': users_data_total_count,
'recordsFiltered': users_data_total_filtered_count,
})
return data
@LoginRequired()
@HasPermissionAllDecorator('hg.admin')
@view_config(

2
rhodecode/config/routing.py
View file

@ -292,8 +292,6 @@ def make_map(config):
controller='admin/users') as m:
m.connect('users', '/users',
action='create', conditions={'method': ['POST']})
m.connect('users', '/users',
action='index', conditions={'method': ['GET']})
m.connect('new_user', '/users/new',
action='new', conditions={'method': ['GET']})
m.connect('update_user', '/users/{user_id}',

2
rhodecode/controllers/admin/my_account.py
View file

@ -77,7 +77,7 @@ class MyAccountController(BaseController):
if c.user.username == User.DEFAULT_USER:
h.flash(_("You can't edit this user since it's"
" crucial for entire application"), category='warning')
return redirect(url('users'))
return redirect(h.route_path('users'))
c.auth_user = AuthUser(
user_id=c.rhodecode_user.user_id, ip_addr=self.ip_addr)

63
rhodecode/controllers/admin/users.py
View file

@ -50,7 +50,6 @@ from rhodecode.model.user import UserModel
from rhodecode.model.meta import Session
from rhodecode.model.permission import PermissionModel
from rhodecode.lib.utils import action_logger
from rhodecode.lib.ext_json import json
from rhodecode.lib.utils2 import datetime_to_time, safe_int, AttributeDict
log = logging.getLogger(__name__)
@ -76,51 +75,6 @@ class UsersController(BaseController):
]
PermissionModel().set_global_permission_choices(c, gettext_translator=_)
@HasPermissionAllDecorator('hg.admin')
def index(self):
"""GET /users: All items in the collection"""
# url('users')
from rhodecode.lib.utils import PartialRenderer
_render = PartialRenderer('data_table/_dt_elements.mako')
def username(user_id, username):
return _render("user_name", user_id, username)
def user_actions(user_id, username):
return _render("user_actions", user_id, username)
# json generate
c.users_list = User.query()\
.filter(User.username != User.DEFAULT_USER) \
.all()
users_data = []
for user in c.users_list:
users_data.append({
"username": h.gravatar_with_user(user.username),
"username_raw": user.username,
"email": user.email,
"first_name": h.escape(user.name),
"last_name": h.escape(user.lastname),
"last_login": h.format_date(user.last_login),
"last_login_raw": datetime_to_time(user.last_login),
"last_activity": h.format_date(
h.time_to_datetime(user.user_data.get('last_activity', 0))),
"last_activity_raw": user.user_data.get('last_activity', 0),
"active": h.bool2icon(user.active),
"active_raw": user.active,
"admin": h.bool2icon(user.admin),
"admin_raw": user.admin,
"extern_type": user.extern_type,
"extern_name": user.extern_name,
"action": user_actions(user.user_id, user.username),
})
c.data = json.dumps(users_data)
return render('admin/users/users.mako')
def _get_personal_repo_group_template_vars(self):
DummyUser = AttributeDict({
'username': '${username}',
@ -135,7 +89,6 @@ class UsersController(BaseController):
@auth.CSRFRequired()
def create(self):
"""POST /users: Create a new item"""
# url('users')
c.default_extern_type = auth_rhodecode.RhodeCodeAuthPlugin.name
user_model = UserModel()
user_form = UserForm()()
@ -168,7 +121,7 @@ class UsersController(BaseController):
log.exception("Exception creation of user")
h.flash(_('Error occurred during creation of user %s')
% request.POST.get('username'), category='error')
return redirect(url('users'))
return redirect(h.route_path('users'))
@HasPermissionAllDecorator('hg.admin')
def new(self):
@ -312,7 +265,7 @@ class UsersController(BaseController):
log.exception("Exception during deletion of user")
h.flash(_('An error occurred during deletion of user'),
category='error')
return redirect(url('users'))
return redirect(h.route_path('users'))
@HasPermissionAllDecorator('hg.admin')
@auth.CSRFRequired()
@ -404,7 +357,7 @@ class UsersController(BaseController):
c.user = User.get_or_404(user_id)
if c.user.username == User.DEFAULT_USER:
h.flash(_("You can't edit this user"), category='warning')
return redirect(url('users'))
return redirect(h.route_path('users'))
c.active = 'profile'
c.extern_type = c.user.extern_type
@ -425,7 +378,7 @@ class UsersController(BaseController):
user = c.user = User.get_or_404(user_id)
if user.username == User.DEFAULT_USER:
h.flash(_("You can't edit this user"), category='warning')
return redirect(url('users'))
return redirect(h.route_path('users'))
c.active = 'advanced'
c.perm_user = AuthUser(user_id=user_id, ip_addr=self.ip_addr)
@ -457,7 +410,7 @@ class UsersController(BaseController):
c.user = User.get_or_404(user_id)
if c.user.username == User.DEFAULT_USER:
h.flash(_("You can't edit this user"), category='warning')
return redirect(url('users'))
return redirect(h.route_path('users'))
c.active = 'global_perms'
@ -531,7 +484,7 @@ class UsersController(BaseController):
c.user = User.get_or_404(user_id)
if c.user.username == User.DEFAULT_USER:
h.flash(_("You can't edit this user"), category='warning')
return redirect(url('users'))
return redirect(h.route_path('users'))
c.active = 'perms_summary'
c.perm_user = AuthUser(user_id=user_id, ip_addr=self.ip_addr)
@ -544,7 +497,7 @@ class UsersController(BaseController):
c.user = User.get_or_404(user_id)
if c.user.username == User.DEFAULT_USER:
h.flash(_("You can't edit this user"), category='warning')
return redirect(url('users'))
return redirect(h.route_path('users'))
c.active = 'emails'
c.user_email_map = UserEmailMap.query() \
@ -602,7 +555,7 @@ class UsersController(BaseController):
c.user = User.get_or_404(user_id)
if c.user.username == User.DEFAULT_USER:
h.flash(_("You can't edit this user"), category='warning')
return redirect(url('users'))
return redirect(h.route_path('users'))
c.active = 'ips'
c.user_ip_map = UserIpMap.query() \

2
rhodecode/templates/admin/users/user_add.mako
View file

@ -10,7 +10,7 @@
<%def name="breadcrumbs_links()">
${h.link_to(_('Admin'),h.url('admin_home'))}
&raquo;
${h.link_to(_('Users'),h.url('users'))}
${h.link_to(_('Users'),h.route_path('users'))}
&raquo;
${_('Add User')}
</%def>

2
rhodecode/templates/admin/users/user_edit.mako
View file

@ -11,7 +11,7 @@
<%def name="breadcrumbs_links()">
${h.link_to(_('Admin'),h.url('admin_home'))}
&raquo;
${h.link_to(_('Users'),h.url('users'))}
${h.link_to(_('Users'),h.route_path('users'))}
&raquo;
${c.user.username}
</%def>

97
rhodecode/templates/admin/users/users.mako
View file

@ -34,60 +34,31 @@
</div>
</div>
<script>
<script type="text/javascript">
$(document).ready(function() {
var get_datatable_count = function(){
var datatable = $('#user_list_table').dataTable();
var api = datatable.api();
var total = api.page.info().recordsDisplay;
var active = datatable.fnGetFilteredData();
var _text = _gettext("{0} active out of {1} users").format(active, total);
var getDatatableCount = function(){
var table = $('#user_list_table').dataTable();
var page = table.api().page.info();
var active = page.recordsDisplay;
var total = page.recordsTotal;
var _text = _gettext("{0} out of {1} users").format(active, total);
$('#user_count').text(_text);
};
// custom filter that filters by username OR email
$.fn.dataTable.ext.search.push(
function( settings, data, dataIndex ) {
var query = $('#q_filter').val();
var username = data[0];
var email = data[1];
var first_name = data[2];
var last_name = data[3];
var query_str = username + " " +
email + " " +
first_name + " " +
last_name;
if ((query_str).indexOf(query) !== -1) {
return true;
}
return false;
}
);
// filtered data plugin
$.fn.dataTableExt.oApi.fnGetFilteredData = function ( oSettings ) {
var res = [];
for ( var i=0, iLen=oSettings.fnRecordsDisplay() ; i<iLen ; i++ ) {
var record = oSettings.aoData[i]._aData;
if(record['active_raw']){
res.push(record);
}
}
return res.length;
};
// user list
$('#user_list_table').DataTable({
data: ${c.data|n},
processing: false,
serverSide: true,
ajax: "${h.route_path('users_data')}",
dom: 'rtp',
pageLength: ${c.visual.admin_grid_items},
order: [[ 1, "asc" ]],
order: [[ 0, "asc" ]],
columns: [
{ data: {"_": "username",
"sort": "username_raw"}, title: "${_('Username')}", className: "td-user" },
"sort": "username"}, title: "${_('Username')}", className: "td-user" },
{ data: {"_": "email",
"sort": "email"}, title: "${_('Email')}", className: "td-email" },
{ data: {"_": "first_name",
@ -95,12 +66,12 @@ $(document).ready(function() {
{ data: {"_": "last_name",
"sort": "last_name"}, title: "${_('Last Name')}", className: "td-user" },
{ data: {"_": "last_activity",
"sort": "last_activity_raw",
"type": Number}, title: "${_('Last activity')}", className: "td-time" },
"sort": "last_activity",
"type": Number}, title: "${_('Last activity')}", className: "td-time", orderable: false },
{ data: {"_": "active",
"sort": "active_raw"}, title: "${_('Active')}", className: "td-active" },
"sort": "active"}, title: "${_('Active')}", className: "td-active" },
{ data: {"_": "admin",
"sort": "admin_raw"}, title: "${_('Admin')}", className: "td-admin" },
"sort": "admin"}, title: "${_('Admin')}", className: "td-admin" },
{ data: {"_": "extern_type",
"sort": "extern_type"}, title: "${_('Auth type')}", className: "td-type" },
{ data: {"_": "action",
@ -110,9 +81,7 @@ $(document).ready(function() {
paginate: DEFAULT_GRID_PAGINATION,
emptyTable: _gettext("No users available yet.")
},
"initComplete": function( settings, json ) {
get_datatable_count();
},
"createdRow": function ( row, data, index ) {
if (!data['active_raw']){
$(row).addClass('closed')
@ -120,23 +89,29 @@ $(document).ready(function() {
}
});
// update the counter when doing search
$('#user_list_table').on( 'search.dt', function (e,settings) {
get_datatable_count();
$('#user_list_table').on('xhr.dt', function(e, settings, json, xhr){
$('#user_list_table').css('opacity', 1);
});
// filter, filter both grids
$('#q_filter').on( 'keyup', function () {
var user_api = $('#user_list_table').dataTable().api();
user_api
.draw();
$('#user_list_table').on('preXhr.dt', function(e, settings, data){
$('#user_list_table').css('opacity', 0.3);
});
// refilter table if page load via back button
$("#q_filter").trigger('keyup');
// refresh counters on draw
$('#user_list_table').on('draw.dt', function(){
getDatatableCount();
});
// filter
$('#q_filter').on('keyup',
$.debounce(250, function() {
$('#user_list_table').DataTable().search(
$('#q_filter').val()
).draw();
})
);
});
</script>
</%def>

2
rhodecode/templates/base/base.mako
View file

@ -75,7 +75,7 @@
<li><a href="${h.url('admin_home')}">${_('Admin journal')}</a></li>
<li><a href="${h.url('repos')}">${_('Repositories')}</a></li>
<li><a href="${h.url('repo_groups')}">${_('Repository groups')}</a></li>
<li><a href="${h.url('users')}">${_('Users')}</a></li>
<li><a href="${h.route_path('users')}">${_('Users')}</a></li>
<li><a href="${h.url('users_groups')}">${_('User groups')}</a></li>
<li><a href="${h.url('admin_permissions_application')}">${_('Permissions')}</a></li>
<li><a href="${h.route_path('auth_home', traverse='')}">${_('Authentication')}</a></li>

24
rhodecode/tests/functional/test_admin_users.py
View file

@ -36,6 +36,20 @@ from rhodecode.tests.utils import AssertResponse
fixture = Fixture()
def route_path(name, params=None, **kwargs):
import urllib
from rhodecode.apps._base import ADMIN_PREFIX
base_url = {
'users_data':
ADMIN_PREFIX + '/users_data',
}[name].format(**kwargs)
if params:
base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
return base_url
class TestAdminUsersController(TestController):
test_user_1 = 'testme'
destroy_users = set()
@ -44,7 +58,7 @@ class TestAdminUsersController(TestController):
def teardown_method(cls, method):
fixture.destroy_users(cls.destroy_users)
def test_create(self):
def test_create(self, xhr_header):
self.log_user()
username = 'newtestuser'
password = 'test12'
@ -53,7 +67,7 @@ class TestAdminUsersController(TestController):
lastname = 'lastname'
email = 'mail@mail.com'
response = self.app.get(url('new_user'))
self.app.get(url('new_user'))
response = self.app.post(url('users'), params={
'username': username,
@ -81,8 +95,8 @@ class TestAdminUsersController(TestController):
assert new_user.lastname == lastname
assert new_user.email == email
response.follow()
response = response.follow()
response = self.app.get(route_path('users_data'),
extra_environ=xhr_header)
response.mustcontain(username)
def test_create_err(self):
@ -93,7 +107,7 @@ class TestAdminUsersController(TestController):
lastname = 'lastname'
email = 'errmail.com'
response = self.app.get(url('new_user'))
self.app.get(url('new_user'))
response = self.app.post(url('users'), params={
'username': username,