release: Merge default into stable for release preparation

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.17.4
+current_version = 4.18.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:rhodecode/VERSION]

.coveragerc

@@ -8,6 +8,7 @@ include =
 omit =
     rhodecode/lib/dbmigrate/*
     rhodecode/lib/paster_commands/*
+    rhodecode/lib/_vendor/*
 
 [report]
 

.hgignore

@@ -45,6 +45,7 @@ syntax: regexp
 ^rhodecode/public/js/rhodecode-components.html$
 ^rhodecode/public/js/rhodecode-components.js$
 ^rhodecode/public/js/scripts.js$
+^rhodecode/public/js/scripts.min.js$
 ^rhodecode/public/js/src/components/root-styles.gen.html$
 ^rhodecode/public/js/vendors/webcomponentsjs/
 ^rhodecode\.db$

.release.cfg

@@ -5,26 +5,21 @@ done = false
 done = true
 
 [task:rc_tools_pinned]
-done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [task:changelog_updated]
-done = true
 
 [task:generate_api_docs]
-done = true
-
-[release]
-state = prepared
-version = 4.17.4
 
 [task:updated_translation]
 
+[release]
+state = in_progress
+version = 4.18.0
+
 [task:generate_js_routes]
 
 [task:updated_trial_license]

Gruntfile.js

@@ -13,9 +13,10 @@ module.exports = function(grunt) {
 
   grunt.loadNpmTasks('grunt-contrib-less');
   grunt.loadNpmTasks('grunt-contrib-concat');
+  grunt.loadNpmTasks('grunt-contrib-uglify');
   grunt.loadNpmTasks('grunt-contrib-watch');
   grunt.loadNpmTasks('grunt-contrib-jshint');
   grunt.loadNpmTasks('grunt-contrib-copy');
   grunt.loadNpmTasks('grunt-webpack');
-  grunt.registerTask('default', ['less:production', 'less:components', 'copy', 'webpack', 'concat:dist']);
+  grunt.registerTask('default', ['less:production', 'less:components', 'copy', 'webpack', 'concat:dist', 'uglify:dist']);
 };

Makefile

@@ -17,6 +17,7 @@ test:
 test-clean:
 	rm -rf coverage.xml htmlcov junit.xml pylint.log result
 	find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
+	find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';'
 
 test-only:
 	PYTHONHASHSEED=random \
@@ -28,7 +29,7 @@ test-only-mysql:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar --cov=rhodecode \
     --cov-report=term-missing --cov-report=html \
-    --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "mysql://root:qweqwe@localhost/rhodecode_test"}}' \
+    --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "mysql://root:qweqwe@localhost/rhodecode_test?charset=utf8"}}' \
     rhodecode
 
 test-only-postgres:

README.rst

@@ -5,20 +5,24 @@ RhodeCode
 About
 -----
 
-``RhodeCode`` is a fast and powerful management tool for Mercurial_ and GIT_
-and Subversion_ with a built in push/pull server, full text search,
-pull requests and powerful code-review system. It works on http/https, SSH and
-has a few unique features like:
+``RhodeCode`` is a fast and powerful source code management tool for
+Mercurial_, GIT_ and Subversion_. It's main features are:
 
-- plugable architecture from Pyramid web-framework.
-- advanced permission system with IP restrictions, inheritation, and user-groups.
+
+- built in push/pull server
+- SSH with key management support
+- full text search.
+- plugable authentication.
+- pull requests and powerful code-review system.
+- advanced permission system with IP restrictions, permission inheritation, and user-groups.
 - rich set of authentication plugins including LDAP, ActiveDirectory, SAML 2.0,
   Atlassian Crowd, Http-Headers, Pam, Token-Auth, OAuth.
 - live code-review chat, and reviewer rules.
 - full web based file editing.
 - unified multi vcs support.
 - snippets (gist) system.
-- integration framework for Slack, CI systems, Webhooks.
+- artfacts store for binaries.
+- integration framework for Slack, CI systems, Webhooks, Jira, Redmine etc.
 - integration with all 3rd party issue trackers.
 
 
@@ -41,7 +45,8 @@ Source code
 -----------
 
 The latest sources can be obtained from official RhodeCode instance
-https://code.rhodecode.com
+https://code.rhodecode.com/rhodecode-enterprise-ce
+https://code.rhodecode.com/rhodecode-vcsserver
 
 
 Contributions

configs/gunicorn_config.py

@@ -1,58 +1,26 @@
 """
-gunicorn config extension and hooks. Sets additional configuration that is
-available post the .ini config.
-
-- workers = ${cpu_number}
-- threads = 1
-- proc_name = ${gunicorn_proc_name}
-- worker_class = sync
-- worker_connections = 10
-- max_requests = 1000
-- max_requests_jitter = 30
-- timeout = 21600
-
+Gunicorn config extension and hooks. This config file adds some extra settings and memory management.
+Gunicorn configuration should be managed by .ini files entries of RhodeCode or VCSServer
 """
 
-import multiprocessing
+import gc
+import os
 import sys
+import math
 import time
-import datetime
 import threading
 import traceback
+import random
 from gunicorn.glogging import Logger
 
 
+def get_workers():
+    import multiprocessing
+    return multiprocessing.cpu_count() * 2 + 1
+
 # GLOBAL
 errorlog = '-'
 accesslog = '-'
-loglevel = 'debug'
-
-# SECURITY
-
-# The maximum size of HTTP request line in bytes.
-# 0 for unlimited
-limit_request_line = 0
-
-# Limit the number of HTTP headers fields in a request.
-# By default this value is 100 and can't be larger than 32768.
-limit_request_fields = 10240
-
-# Limit the allowed size of an HTTP request header field.
-# Value is a positive number or 0.
-# Setting it to 0 will allow unlimited header field sizes.
-limit_request_field_size = 0
-
-
-# Timeout for graceful workers restart.
-# After receiving a restart signal, workers have this much time to finish
-# serving requests. Workers still alive after the timeout (starting from the
-# receipt of the restart signal) are force killed.
-graceful_timeout = 30
-
-
-# The number of seconds to wait for requests on a Keep-Alive connection.
-# Generally set in the 1-5 seconds range.
-keepalive = 2
 
 
 # SERVER MECHANICS
@@ -63,38 +31,178 @@ tmp_upload_dir = None
 
 # Custom log format
 access_log_format = (
-    '%(t)s [%(p)-8s] GNCRN %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
+    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
 
 # self adjust workers based on CPU count
-# workers = multiprocessing.cpu_count() * 2 + 1
+# workers = get_workers()
 
 
-def post_fork(server, worker):
-    server.log.info("[<%-10s>] WORKER spawned", worker.pid)
+def _get_process_rss(pid=None):
+    try:
+        import psutil
+        if pid:
+            proc = psutil.Process(pid)
+        else:
+            proc = psutil.Process()
+        return proc.memory_info().rss
+    except Exception:
+        return None
+
+
+def _get_config(ini_path):
+
+    try:
+        import configparser
+    except ImportError:
+        import ConfigParser as configparser
+    try:
+        config = configparser.RawConfigParser()
+        config.read(ini_path)
+        return config
+    except Exception:
+        return None
+
+
+def _time_with_offset(memory_usage_check_interval):
+    return time.time() - random.randint(0, memory_usage_check_interval/2.0)
 
 
 def pre_fork(server, worker):
     pass
 
 
+def post_fork(server, worker):
+
+    # memory spec defaults
+    _memory_max_usage = 0
+    _memory_usage_check_interval = 60
+    _memory_usage_recovery_threshold = 0.8
+
+    ini_path = os.path.abspath(server.cfg.paste)
+    conf = _get_config(ini_path)
+
+    section = 'server:main'
+    if conf and conf.has_section(section):
+
+        if conf.has_option(section, 'memory_max_usage'):
+            _memory_max_usage = conf.getint(section, 'memory_max_usage')
+
+        if conf.has_option(section, 'memory_usage_check_interval'):
+            _memory_usage_check_interval = conf.getint(section, 'memory_usage_check_interval')
+
+        if conf.has_option(section, 'memory_usage_recovery_threshold'):
+            _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
+
+    worker._memory_max_usage = _memory_max_usage
+    worker._memory_usage_check_interval = _memory_usage_check_interval
+    worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold
+
+    # register memory last check time, with some random offset so we don't recycle all
+    # at once
+    worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
+
+    if _memory_max_usage:
+        server.log.info("[%-10s] WORKER spawned with max memory set at %s", worker.pid,
+                        _format_data_size(_memory_max_usage))
+    else:
+        server.log.info("[%-10s] WORKER spawned", worker.pid)
+
+
 def pre_exec(server):
     server.log.info("Forked child, re-executing.")
 
 
 def on_starting(server):
-    server.log.info("Server is starting.")
+    server_lbl = '{} {}'.format(server.proc_name, server.address)
+    server.log.info("Server %s is starting.", server_lbl)
 
 
 def when_ready(server):
-    server.log.info("Server is ready. Spawning workers")
+    server.log.info("Server %s is ready. Spawning workers", server)
 
 
 def on_reload(server):
     pass
 
 
+def _format_data_size(size, unit="B", precision=1, binary=True):
+    """Format a number using SI units (kilo, mega, etc.).
+
+    ``size``: The number as a float or int.
+
+    ``unit``: The unit name in plural form. Examples: "bytes", "B".
+
+    ``precision``: How many digits to the right of the decimal point. Default
+    is 1.  0 suppresses the decimal point.
+
+    ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
+    If true, use base-2 binary prefixes (kibi = Ki = 1024).
+
+    ``full_name``: If false (default), use the prefix abbreviation ("k" or
+    "Ki").  If true, use the full prefix ("kilo" or "kibi"). If false,
+    use abbreviation ("k" or "Ki").
+
+    """
+
+    if not binary:
+        base = 1000
+        multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
+    else:
+        base = 1024
+        multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
+
+    sign = ""
+    if size > 0:
+        m = int(math.log(size, base))
+    elif size < 0:
+        sign = "-"
+        size = -size
+        m = int(math.log(size, base))
+    else:
+        m = 0
+    if m > 8:
+        m = 8
+
+    if m == 0:
+        precision = '%.0f'
+    else:
+        precision = '%%.%df' % precision
+
+    size = precision % (size / math.pow(base, m))
+
+    return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
+
+
+def _check_memory_usage(worker):
+    memory_max_usage = worker._memory_max_usage
+    if not memory_max_usage:
+        return
+
+    memory_usage_check_interval = worker._memory_usage_check_interval
+    memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
+
+    elapsed = time.time() - worker._last_memory_check_time
+    if elapsed > memory_usage_check_interval:
+        mem_usage = _get_process_rss()
+        if mem_usage and mem_usage > memory_max_usage:
+            worker.log.info(
+                "memory usage %s > %s, forcing gc",
+                _format_data_size(mem_usage), _format_data_size(memory_max_usage))
+            # Try to clean it up by forcing a full collection.
+            gc.collect()
+            mem_usage = _get_process_rss()
+            if mem_usage > memory_usage_recovery_threshold:
+                # Didn't clean up enough, we'll have to terminate.
+                worker.log.warning(
+                    "memory usage %s > %s after gc, quitting",
+                    _format_data_size(mem_usage), _format_data_size(memory_max_usage))
+                # This will cause worker to auto-restart itself
+                worker.alive = False
+        worker._last_memory_check_time = time.time()
+
+
 def worker_int(worker):
-    worker.log.info("[<%-10s>] worker received INT or QUIT signal", worker.pid)
+    worker.log.info("[%-10s] worker received INT or QUIT signal", worker.pid)
 
     # get traceback info, on worker crash
     id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
@@ -110,15 +218,15 @@ def worker_int(worker):
 
 
 def worker_abort(worker):
-    worker.log.info("[<%-10s>] worker received SIGABRT signal", worker.pid)
+    worker.log.info("[%-10s] worker received SIGABRT signal", worker.pid)
 
 
 def worker_exit(server, worker):
-    worker.log.info("[<%-10s>] worker exit", worker.pid)
+    worker.log.info("[%-10s] worker exit", worker.pid)
 
 
 def child_exit(server, worker):
-    worker.log.info("[<%-10s>] worker child exit", worker.pid)
+    worker.log.info("[%-10s] worker child exit", worker.pid)
 
 
 def pre_request(worker, req):
@@ -129,9 +237,12 @@ def pre_request(worker, req):
 
 def post_request(worker, req, environ, resp):
     total_time = time.time() - worker.start_time
+    # Gunicorn sometimes has problems with reading the status_code
+    status_code = getattr(resp, 'status_code', '')
     worker.log.debug(
-        "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.3fs",
-        worker.nr, req.method, req.path, resp.status_code, total_time)
+        "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
+        worker.nr, req.method, req.path, status_code, total_time)
+    _check_memory_usage(worker)
 
 
 class RhodeCodeLogger(Logger):

default.nix

@@ -190,7 +190,7 @@ let
       postInstall = ''
         # check required files
         STATIC_CHECK="/robots.txt /502.html
-                      /js/scripts.js /js/rhodecode-components.js
+                      /js/scripts.min.js /js/rhodecode-components.js
                       /css/style.css /css/style-polymer.css /css/style-ipython.css"
 
         for file in $STATIC_CHECK;

docs/admin/repo_admin/repo-admin-tasks.rst

@@ -12,13 +12,50 @@ Here is how to force delete a repository and remove all dependent objects
 
 
 .. code-block:: bash
+   :dedent: 1
 
     # starts the ishell interactive prompt
     $ rccontrol ishell enterprise-1
 
 .. code-block:: python
+   :dedent: 1
 
     In [4]: from rhodecode.model.repo import RepoModel
     In [3]: repo = Repository.get_by_repo_name('test_repos/repo_with_prs')
     In [5]: RepoModel().delete(repo, forks='detach', pull_requests='delete')
     In [6]: Session().commit()
+
+
+Below is a fully automated example to force delete repositories reading from a
+file where each line is a repository name. This can be executed via simple CLI command
+without entering the interactive shell.
+
+Save the below content as a file named `repo_delete_task.py`
+
+
+.. code-block:: python
+   :dedent: 1
+
+    from rhodecode.model.db import *
+    from rhodecode.model.repo import RepoModel
+    with open('delete_repos.txt', 'rb') as f:
+        # read all lines from file
+        repos = f.readlines()
+        for repo_name in repos:
+            repo_name = repo_name.strip()  # cleanup the name just in case
+            repo = Repository.get_by_repo_name(repo_name)
+            if not repo:
+                raise Exception('Repo with name {} not found'.format(repo_name))
+            RepoModel().delete(repo, forks='detach', pull_requests='delete')
+            Session().commit()
+            print('Removed repository {}'.format(repo_name))
+
+
+The code above will read the names of repositories from a file called `delete_repos.txt`
+Each lines should represent a single name e.g `repo_name_1` or `repo_group/repo_name_2`
+
+Run this line from CLI to execute the code from the `repo_delete_task.py` file and
+exit the ishell after the execution::
+
+    echo "%run repo_delete_task.py" | rccontrol ishell Enterprise-1
+

docs/admin/system_admin/nginx/nginx-config-example.rst

@@ -110,6 +110,7 @@ Use the following example to configure Nginx as a your web server.
         #    gzip_types  text/css text/javascript text/xml text/plain text/x-component application/javascript application/json application/xml application/rss+xml font/truetype font/opentype application/vnd.ms-fontobject image/svg+xml;
         #    gzip_vary on;
         #    gzip_disable     "msie6";
+        #    expires 60d;
         #    alias /path/to/.rccontrol/community-1/static;
         #    alias /path/to/.rccontrol/enterprise-1/static;
         # }

docs/admin/system_admin/tuning-rhodecode.rst

@@ -9,7 +9,8 @@ may find some of the following methods useful.
 .. toctree::
 
    tuning/tuning-gunicorn
-   tuning/tuning-vcs-memory-cache
+   tuning/tuning-vcs-server-memory-usage
+   tuning/tuning-rhodecode-memory-usage
    tuning/tuning-user-sessions-performance
    tuning/tuning-increase-db-performance
    tuning/tuning-scale-horizontally-cluster

docs/admin/system_admin/tuning/tuning-gunicorn.rst

@@ -25,26 +25,22 @@ To do this, use the following steps:
 
 .. code-block:: ini
 
-    use = egg:gunicorn#main
-    ## Sets the number of process workers. You must set `instance_id = *`
-    ## when this option is set to more than one worker, recommended
-    ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
-    ## The `instance_id = *` must be set in the [app:main] section below
-    workers = 4
-    ## process name
-    proc_name = rhodecode
-    ## type of worker class, one of sync, gevent
-    ## recommended for bigger setup is using of of other than sync one
-    worker_class = sync
-    ## The maximum number of simultaneous clients. Valid only for Gevent
-    #worker_connections = 10
-    ## max number of requests that worker will handle before being gracefully
-    ## restarted, could prevent memory leaks
-    max_requests = 1000
-    max_requests_jitter = 30
-    ## amount of time a worker can spend with handling a request tuning-change-lfs-dir.before it
-    ## gets killed and restarted. Set to 6hrs
-    timeout = 21600
+    ; Sets the number of process workers. More workers means more concurrent connections
+    ; RhodeCode can handle at the same time. Each additional worker also it increases
+    ; memory usage as each has it's own set of caches.
+    ; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
+    ; than 8-10 unless for really big deployments .e.g 700-1000 users.
+    ; `instance_id = *` must be set in the [app:main] section below (which is the default)
+    ; when using more than 1 worker.
+    workers = 6
+
+    ; Type of worker class, one of `sync`, `gevent`
+    ; Use `gevent` for rhodecode
+    worker_class = gevent
+
+    ; The maximum number of simultaneous clients per worker. Valid only for gevent
+    worker_connections = 10
+
 
 3. In the ``[app:main]`` section, set the ``instance_id`` property to ``*``.
 
@@ -63,24 +59,19 @@ To do this, use the following steps:
 
 .. code-block:: ini
 
-    ## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
-    use = egg:gunicorn#main
-    ## Sets the number of process workers. Recommended
-    ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
-    workers = 4
-    ## process name
-    proc_name = rhodecode_vcsserver
-    ## type of worker class, currently `sync` is the only option allowed.
+    ; Sets the number of process workers. More workers means more concurrent connections
+    ; RhodeCode can handle at the same time. Each additional worker also it increases
+    ; memory usage as each has it's own set of caches.
+    ; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
+    ; than 8-10 unless for really big deployments .e.g 700-1000 users.
+    ; `instance_id = *` must be set in the [app:main] section below (which is the default)
+    ; when using more than 1 worker.
+    workers = 8
+
+    ; Type of worker class, one of `sync`, `gevent`
+    ; Use `sync` for vcsserver
     worker_class = sync
-    ## The maximum number of simultaneous clients. Valid only for Gevent
-    #worker_connections = 10
-    ## max number of requests that worker will handle before being gracefully
-    ## restarted, could prevent memory leaks
-    max_requests = 1000
-    max_requests_jitter = 30
-    ## amount of time a worker can spend with handling a request before it
-    ## gets killed and restarted. Set to 6hrs
-    timeout = 21600
+
 
 6. Save your changes.
 7. Restart your |RCE| instances, using the following command:
@@ -109,17 +100,18 @@ To enable `Gevent` on |RCE| do the following:
 
 .. code-block:: ini
 
-    ## type of worker class, one of sync, gevent
-    ## recommended for bigger setup is using of of other than sync one
+    ; Type of worker class, one of `sync`, `gevent`
+    ; Use `gevent` for rhodecode
     worker_class = gevent
-    ## The maximum number of simultaneous clients. Valid only for Gevent
+
+    ; The maximum number of simultaneous clients per worker. Valid only for gevent
     worker_connections = 30
 
 
 .. note::
 
     `Gevent` is currently only supported for Enterprise/Community instances.
-    VCSServer doesn't yet support gevent.
+    VCSServer doesn't support gevent.
 
 
 

docs/admin/system_admin/tuning/tuning-rhodecode-memory-usage.rst

@@ -0,0 +1,26 @@
+.. _adjust-rhodecode-mem:
+
+RhodeCode Memory Usage
+----------------------
+
+Starting from Version 4.18.X RhodeCode has a builtin memory monitor for gunicorn workers.
+Enabling this can limit the maximum amount of memory system can use. Each worker
+for RhodeCode is monitored independently.
+To enable Memory management make sure to have following settings inside `[app:main] section` of
+:file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+
+
+
+    ; Maximum memory usage that each worker can use before it will receive a
+    ; graceful restart signal 0 = memory monitoring is disabled
+    ; Examples: 268435456 (256MB), 536870912 (512MB)
+    ; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
+    memory_max_usage = 1073741824
+
+    ; How often in seconds to check for memory usage for each gunicorn worker
+    memory_usage_check_interval = 60
+
+    ; Threshold value for which we don't recycle worker if GarbageCollection
+    ; frees up enough resources. Before each restart we try to run GC on worker
+    ; in case we get enough free memory after that, restart will not happen.
+    memory_usage_recovery_threshold = 0.8

docs/admin/system_admin/tuning/tuning-scale-horizontally-cluster.rst

@@ -57,7 +57,7 @@ Here's an overview what components should be installed/setup on each server in o
  - `nginx` acting as a load-balancer.
  - `postgresql-server` used for database and sessions.
  - `redis-server` used for storing shared caches.
- - optionally `rabbitmq-server` for `Celery` if used.
+ - optionally `rabbitmq-server` or `redis` for `Celery` if used.
  - optionally if `Celery` is used Enterprise/Community instance + VCSServer.
  - optionally mailserver that can be shared by other instances.
  - optionally channelstream server to handle live communication for all instances.
@@ -263,6 +263,7 @@ persistent sessions across nodes. Please generate other one then in this example
             gzip_types  text/css text/javascript text/xml text/plain text/x-component application/javascript application/json application/xml application/rss+xml font/truetype font/opentype application/vnd.ms-fontobject image/svg+xml;
             gzip_vary on;
             gzip_disable     "msie6";
+            expires 60d;
             #alias /home/rcdev/.rccontrol/community-1/static;
             alias /home/rcdev/.rccontrol/enterprise-1/static;
         }
@@ -372,16 +373,16 @@ Using Celery with cluster
 
 
 If `Celery` is used we recommend setting also an instance of Enterprise/Community+VCSserver
-on the node that is running `RabbitMQ`_. Those instances will be used to executed async
-tasks on the `rc-node-1`. This is the most efficient setup. `Celery` usually
-handles tasks such as sending emails, forking repositories, importing
+on the node that is running `RabbitMQ`_ or `Redis`_. Those instances will be used to
+executed async tasks on the `rc-node-1`. This is the most efficient setup.
+`Celery` usually handles tasks such as sending emails, forking repositories, importing
 repositories from external location etc. Using workers on instance that has
 the direct access to disks used by NFS as well as email server gives noticeable
 performance boost. Running local workers to the NFS storage results in faster
 execution of forking large repositories or sending lots of emails.
 
 Those instances need to be configured in the same way as for other nodes.
-The instance in rc-node-1 can be added to the cluser, but we don't recommend doing it.
+The instance in rc-node-1 can be added to the cluster, but we don't recommend doing it.
 For best results let it be isolated to only executing `Celery` tasks in the cluster setup.
 
 

docs/admin/system_admin/tuning/tuning-vcs-memory-cache.rst

@@ -1,8 +0,0 @@
-.. _adjust-vcs-mem-cache:
-
-VCSServer Memory Cache
-----------------------
-
-The VCS Server mamory cache can be adjusted to work best with the resources
-available to your |RCE| instance. If you find that memory resources are under
-pressure, see the :ref:`vcs-server-maintain` section for details.

docs/admin/system_admin/tuning/tuning-vcs-server-memory-usage.rst

@@ -0,0 +1,26 @@
+.. _adjust-vcs-server-mem:
+
+VCSServer Memory Usage
+----------------------
+
+Starting from Version 4.18.X RhodeCode has a builtin memory monitor for gunicorn workers.
+Enabling this can limit the maximum amount of memory system can use. Each worker
+for VCS Server is monitored independently.
+To enable Memory management make sure to have following settings inside `[app:main] section` of
+:file:`home/{user}/.rccontrol/{instance-id}/vcsserver.ini` file.
+
+
+
+    ; Maximum memory usage that each worker can use before it will receive a
+    ; graceful restart signal 0 = memory monitoring is disabled
+    ; Examples: 268435456 (256MB), 536870912 (512MB)
+    ; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
+    memory_max_usage = 1073741824
+
+    ; How often in seconds to check for memory usage for each gunicorn worker
+    memory_usage_check_interval = 60
+
+    ; Threshold value for which we don't recycle worker if GarbageCollection
+    ; frees up enough resources. Before each restart we try to run GC on worker
+    ; in case we get enough free memory after that, restart will not happen.
+    memory_usage_recovery_threshold = 0.8

docs/admin/system_admin/vcs-server.rst

@@ -110,35 +110,39 @@ match, for example:
 
 .. _vcs-server-maintain:
 
-VCS Server Memory Optimization
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+VCS Server Cache Optimization
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-To optimize the VCS server to manage the cache and memory usage efficiently, you need to
-configure the following options in the
-:file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.ini` file. Once
-configured, restart the VCS Server. By default we use an optimal settings, but in certain
-conditions tunning expiration_time and max_size can affect memory usage and performance
+To optimize the VCS server to manage the cache and memory usage efficiently, it's recommended to
+configure the Redis backend for VCSServer caches.
+Once configured, restart the VCS Server.
+
+Make sure Redis is installed and running.
+Open :file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.ini`
+file and ensure the below settings for `repo_object` type cache are set:
 
 .. code-block:: ini
 
-    ## cache region for storing repo_objects cache
-    rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
+    ; ensure the default file based cache is *commented out*
+    ##rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
+    ##rc_cache.repo_object.expiration_time = 2592000
 
-    ## cache auto-expires after N seconds, setting this to 0 disabled cache
-    rc_cache.repo_object.expiration_time = 300
+    ; `repo_object` cache settings for vcs methods for repositories
+    rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
 
-    ## max size of LRU, old values will be discarded if the size of cache reaches max_size
-    ## Sets the maximum number of items stored in the cache, before the cache
-    ## starts to be cleared.
+    ; cache auto-expires after N seconds
+    ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
+    rc_cache.repo_object.expiration_time = 2592000
 
-    ## As a general rule of thumb, running this value at 120 resulted in a
-    ## 5GB cache. Running it at 240 resulted in a 9GB cache. Your results
-    ## will differ based on usage patterns and |repo| sizes.
+    ; redis_expiration_time needs to be greater then expiration_time
+    rc_cache.repo_object.arguments.redis_expiration_time = 3592000
 
-    ## Tweaking this value to run at a fairly constant memory load on your
-    ## server will help performance.
-
-    rc_cache.repo_object.max_size = 120
+    rc_cache.repo_object.arguments.host = localhost
+    rc_cache.repo_object.arguments.port = 6379
+    rc_cache.repo_object.arguments.db = 5
+    rc_cache.repo_object.arguments.socket_timeout = 30
+    ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
+    rc_cache.repo_object.arguments.distributed_lock = true
 
 
 To clear the cache completely, you can restart the VCS Server.
@@ -190,25 +194,6 @@ For a more detailed explanation of the logger levers, see :ref:`debug-mode`.
     \port <int>
         Set the port number on which the VCS Server will be available.
 
-    \locale <locale_utf>
-        Set the locale the VCS Server expects.
-
-    \workers <int>
-        Set the number of process workers.Recommended
-        value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
-
-    \max_requests <int>
-        The maximum number of requests a worker will process before restarting.
-        Any value greater than zero will limit the number of requests a work
-        will process before automatically restarting. This is a simple method
-        to help limit the damage of memory leaks.
-
-    \max_requests_jitter <int>
-        The maximum jitter to add to the max_requests setting.
-        The jitter causes the restart per worker to be randomized by
-        randint(0, max_requests_jitter). This is intended to stagger worker
-        restarts to avoid all workers restarting at the same time.
-
 
 .. note::
 
@@ -216,63 +201,139 @@ For a more detailed explanation of the logger levers, see :ref:`debug-mode`.
 
 .. code-block:: ini
 
-    ################################################################################
-    # RhodeCode VCSServer with HTTP Backend - configuration                        #
-    #                                                                              #
-    ################################################################################
-
+    ; #################################
+    ; RHODECODE VCSSERVER CONFIGURATION
+    ; #################################
 
     [server:main]
-    ## COMMON ##
+    ; COMMON HOST/IP CONFIG
     host = 127.0.0.1
     port = 10002
 
-    ##########################
-    ## GUNICORN WSGI SERVER ##
-    ##########################
-    ## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
+    ; ###########################
+    ; GUNICORN APPLICATION SERVER
+    ; ###########################
+
+    ; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
+
+    ; Module to use, this setting shouldn't be changed
     use = egg:gunicorn#main
-    ## Sets the number of process workers. Recommended
-    ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
-    workers = 3
-    ## process name
+
+    ; Sets the number of process workers. More workers means more concurrent connections
+    ; RhodeCode can handle at the same time. Each additional worker also it increases
+    ; memory usage as each has it's own set of caches.
+    ; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
+    ; than 8-10 unless for really big deployments .e.g 700-1000 users.
+    ; `instance_id = *` must be set in the [app:main] section below (which is the default)
+    ; when using more than 1 worker.
+    workers = 6
+
+    ; Gunicorn access log level
+    loglevel = info
+
+    ; Process name visible in process list
     proc_name = rhodecode_vcsserver
-    ## type of worker class, one of sync, gevent
-    ## recommended for bigger setup is using of of other than sync one
+
+    ; Type of worker class, one of sync, gevent
+    ; currently `sync` is the only option allowed.
     worker_class = sync
-    ## The maximum number of simultaneous clients. Valid only for Gevent
-    #worker_connections = 10
-    ## max number of requests that worker will handle before being gracefully
-    ## restarted, could prevent memory leaks
+
+    ; The maximum number of simultaneous clients. Valid only for gevent
+    worker_connections = 10
+
+    ; Max number of requests that worker will handle before being gracefully restarted.
+    ; Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
     max_requests = 1000
     max_requests_jitter = 30
-    ## amount of time a worker can spend with handling a request before it
-    ## gets killed and restarted. Set to 6hrs
+
+    ; Amount of time a worker can spend with handling a request before it
+    ; gets killed and restarted. By default set to 21600 (6hrs)
+    ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
     timeout = 21600
 
+    ; The maximum size of HTTP request line in bytes.
+    ; 0 for unlimited
+    limit_request_line = 0
+
+    ; Limit the number of HTTP headers fields in a request.
+    ; By default this value is 100 and can't be larger than 32768.
+    limit_request_fields = 32768
+
+    ; Limit the allowed size of an HTTP request header field.
+    ; Value is a positive number or 0.
+    ; Setting it to 0 will allow unlimited header field sizes.
+    limit_request_field_size = 0
+
+    ; Timeout for graceful workers restart.
+    ; After receiving a restart signal, workers have this much time to finish
+    ; serving requests. Workers still alive after the timeout (starting from the
+    ; receipt of the restart signal) are force killed.
+    ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
+    graceful_timeout = 3600
+
+    # The number of seconds to wait for requests on a Keep-Alive connection.
+    # Generally set in the 1-5 seconds range.
+    keepalive = 2
+
+    ; Maximum memory usage that each worker can use before it will receive a
+    ; graceful restart signal 0 = memory monitoring is disabled
+    ; Examples: 268435456 (256MB), 536870912 (512MB)
+    ; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
+    memory_max_usage = 1073741824
+
+    ; How often in seconds to check for memory usage for each gunicorn worker
+    memory_usage_check_interval = 60
+
+    ; Threshold value for which we don't recycle worker if GarbageCollection
+    ; frees up enough resources. Before each restart we try to run GC on worker
+    ; in case we get enough free memory after that, restart will not happen.
+    memory_usage_recovery_threshold = 0.8
+
+
     [app:main]
     use = egg:rhodecode-vcsserver
 
     pyramid.default_locale_name = en
     pyramid.includes =
 
-    ## default locale used by VCS systems
+    ; default locale used by VCS systems
     locale = en_US.UTF-8
 
-    # cache regions, please don't change
-    beaker.cache.regions = repo_object
-    beaker.cache.repo_object.type = memorylru
-    beaker.cache.repo_object.max_items = 100
-    # cache auto-expires after N seconds
-    beaker.cache.repo_object.expire = 300
-    beaker.cache.repo_object.enabled = true
+    ; #############
+    ; DOGPILE CACHE
+    ; #############
 
+    ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
+    ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
+    cache_dir = %(here)s/data
 
-    ################################
-    ### LOGGING CONFIGURATION   ####
-    ################################
+    ; **********************************************************
+    ; `repo_object` cache with redis backend
+    ; recommended for larger instance, or for better performance
+    ; **********************************************************
+
+    ; `repo_object` cache settings for vcs methods for repositories
+    rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
+
+    ; cache auto-expires after N seconds
+    ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
+    rc_cache.repo_object.expiration_time = 2592000
+
+    ; redis_expiration_time needs to be greater then expiration_time
+    rc_cache.repo_object.arguments.redis_expiration_time = 3592000
+
+    rc_cache.repo_object.arguments.host = localhost
+    rc_cache.repo_object.arguments.port = 6379
+    rc_cache.repo_object.arguments.db = 5
+    rc_cache.repo_object.arguments.socket_timeout = 30
+    ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
+    rc_cache.repo_object.arguments.distributed_lock = true
+
+    ; #####################
+    ; LOGGING CONFIGURATION
+    ; #####################
     [loggers]
-    keys = root, vcsserver, beaker
+    keys = root, vcsserver
 
     [handlers]
     keys = console
@@ -280,9 +341,9 @@ For a more detailed explanation of the logger levers, see :ref:`debug-mode`.
     [formatters]
     keys = generic
 
-    #############
-    ## LOGGERS ##
-    #############
+    ; #######
+    ; LOGGERS
+    ; #######
     [logger_root]
     level = NOTSET
     handlers = console
@@ -293,29 +354,23 @@ For a more detailed explanation of the logger levers, see :ref:`debug-mode`.
     qualname = vcsserver
     propagate = 1
 
-    [logger_beaker]
-    level = DEBUG
-    handlers =
-    qualname = beaker
-    propagate = 1
 
-
-    ##############
-    ## HANDLERS ##
-    ##############
+    ; ########
+    ; HANDLERS
+    ; ########
 
     [handler_console]
     class = StreamHandler
-    args = (sys.stderr,)
-    level = DEBUG
+    args = (sys.stderr, )
+    level = INFO
     formatter = generic
 
-    ################
-    ## FORMATTERS ##
-    ################
+    ; ##########
+    ; FORMATTERS
+    ; ##########
 
     [formatter_generic]
-    format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
     datefmt = %Y-%m-%d %H:%M:%S
 
 

docs/api/methods/pull-request-methods.rst

@@ -39,7 +39,7 @@ close_pull_request
 comment_pull_request 
 --------------------
 
-.. py:function:: comment_pull_request(apiuser, pullrequestid, repoid=<Optional:None>, message=<Optional:None>, commit_id=<Optional:None>, status=<Optional:None>, comment_type=<Optional:u'note'>, resolves_comment_id=<Optional:None>, userid=<Optional:<OptionalAttr:apiuser>>)
+.. py:function:: comment_pull_request(apiuser, pullrequestid, repoid=<Optional:None>, message=<Optional:None>, commit_id=<Optional:None>, status=<Optional:None>, comment_type=<Optional:u'note'>, resolves_comment_id=<Optional:None>, extra_recipients=<Optional:[]>, userid=<Optional:<OptionalAttr:apiuser>>)
 
    Comment on the pull request specified with the `pullrequestid`,
    in the |repo| specified by the `repoid`, and optionally change the
@@ -63,6 +63,11 @@ comment_pull_request
    :type status: str
    :param comment_type: Comment type, one of: 'note', 'todo'
    :type comment_type: Optional(str), default: 'note'
+   :param resolves_comment_id: id of comment which this one will resolve
+   :type resolves_comment_id: Optional(int)
+   :param extra_recipients: list of user ids or usernames to add
+       notifications for this comment. Acts like a CC for notification
+   :type extra_recipients: Optional(list)
    :param userid: Comment on the pull request as this user
    :type userid: Optional(str or int)
 
@@ -126,7 +131,7 @@ create_pull_request
 get_pull_request 
 ----------------
 
-.. py:function:: get_pull_request(apiuser, pullrequestid, repoid=<Optional:None>)
+.. py:function:: get_pull_request(apiuser, pullrequestid, repoid=<Optional:None>, merge_state=<Optional:False>)
 
    Get a pull request based on the given ID.
 
@@ -137,6 +142,9 @@ get_pull_request
    :type repoid: str or int
    :param pullrequestid: ID of the requested pull request.
    :type pullrequestid: int
+   :param merge_state: Optional calculate merge state for each repository.
+       This could result in longer time to fetch the data
+   :type merge_state: bool
 
    Example output:
 
@@ -250,7 +258,7 @@ get_pull_request_comments
 get_pull_requests 
 -----------------
 
-.. py:function:: get_pull_requests(apiuser, repoid, status=<Optional:'new'>, merge_state=<Optional:True>)
+.. py:function:: get_pull_requests(apiuser, repoid, status=<Optional:'new'>, merge_state=<Optional:False>)
 
    Get all pull requests from the repository specified in `repoid`.
 

docs/api/methods/repo-methods.rst

@@ -28,7 +28,7 @@ add_field_to_repo
 comment_commit 
 --------------
 
-.. py:function:: comment_commit(apiuser, repoid, commit_id, message, status=<Optional:None>, comment_type=<Optional:u'note'>, resolves_comment_id=<Optional:None>, userid=<Optional:<OptionalAttr:apiuser>>)
+.. py:function:: comment_commit(apiuser, repoid, commit_id, message, status=<Optional:None>, comment_type=<Optional:u'note'>, resolves_comment_id=<Optional:None>, extra_recipients=<Optional:[]>, userid=<Optional:<OptionalAttr:apiuser>>)
 
    Set a commit comment, and optionally change the status of the commit.
 
@@ -45,6 +45,11 @@ comment_commit
    :type status: str
    :param comment_type: Comment type, one of: 'note', 'todo'
    :type comment_type: Optional(str), default: 'note'
+   :param resolves_comment_id: id of comment which this one will resolve
+   :type resolves_comment_id: Optional(int)
+   :param extra_recipients: list of user ids or usernames to add
+       notifications for this comment. Acts like a CC for notification
+   :type extra_recipients: Optional(list)
    :param userid: Set the user name of the comment creator.
    :type userid: Optional(str or int)
 
@@ -66,7 +71,7 @@ comment_commit
 create_repo 
 -----------
 
-.. py:function:: create_repo(apiuser, repo_name, repo_type, owner=<Optional:<OptionalAttr:apiuser>>, description=<Optional:''>, private=<Optional:False>, clone_uri=<Optional:None>, push_uri=<Optional:None>, landing_rev=<Optional:'rev:tip'>, enable_statistics=<Optional:False>, enable_locking=<Optional:False>, enable_downloads=<Optional:False>, copy_permissions=<Optional:False>)
+.. py:function:: create_repo(apiuser, repo_name, repo_type, owner=<Optional:<OptionalAttr:apiuser>>, description=<Optional:''>, private=<Optional:False>, clone_uri=<Optional:None>, push_uri=<Optional:None>, landing_rev=<Optional:None>, enable_statistics=<Optional:False>, enable_locking=<Optional:False>, enable_downloads=<Optional:False>, copy_permissions=<Optional:False>)
 
    Creates a repository.
 
@@ -97,7 +102,7 @@ create_repo
    :type clone_uri: str
    :param push_uri: set push_uri
    :type push_uri: str
-   :param landing_rev: <rev_type>:<rev>
+   :param landing_rev: <rev_type>:<rev>, e.g branch:default, book:dev, rev:abcd
    :type landing_rev: str
    :param enable_locking:
    :type enable_locking: bool
@@ -169,7 +174,7 @@ delete_repo
 fork_repo 
 ---------
 
-.. py:function:: fork_repo(apiuser, repoid, fork_name, owner=<Optional:<OptionalAttr:apiuser>>, description=<Optional:''>, private=<Optional:False>, clone_uri=<Optional:None>, landing_rev=<Optional:'rev:tip'>, copy_permissions=<Optional:False>)
+.. py:function:: fork_repo(apiuser, repoid, fork_name, owner=<Optional:<OptionalAttr:apiuser>>, description=<Optional:''>, private=<Optional:False>, clone_uri=<Optional:None>, landing_rev=<Optional:None>, copy_permissions=<Optional:False>)
 
    Creates a fork of the specified |repo|.
 
@@ -198,7 +203,7 @@ fork_repo
    :type copy_permissions: bool
    :param private: Make the fork private. The default is False.
    :type private: bool
-   :param landing_rev: Set the landing revision. The default is tip.
+   :param landing_rev: Set the landing revision. E.g branch:default, book:dev, rev:abcd
 
    Example output:
 
@@ -1085,7 +1090,7 @@ strip
 update_repo 
 -----------
 
-.. py:function:: update_repo(apiuser, repoid, repo_name=<Optional:None>, owner=<Optional:<OptionalAttr:apiuser>>, description=<Optional:''>, private=<Optional:False>, clone_uri=<Optional:None>, push_uri=<Optional:None>, landing_rev=<Optional:'rev:tip'>, fork_of=<Optional:None>, enable_statistics=<Optional:False>, enable_locking=<Optional:False>, enable_downloads=<Optional:False>, fields=<Optional:''>)
+.. py:function:: update_repo(apiuser, repoid, repo_name=<Optional:None>, owner=<Optional:<OptionalAttr:apiuser>>, description=<Optional:''>, private=<Optional:False>, clone_uri=<Optional:None>, push_uri=<Optional:None>, landing_rev=<Optional:None>, fork_of=<Optional:None>, enable_statistics=<Optional:False>, enable_locking=<Optional:False>, enable_downloads=<Optional:False>, fields=<Optional:''>)
 
    Updates a repository with the given information.
 
@@ -1117,7 +1122,7 @@ update_repo
    :type private: bool
    :param clone_uri: Update the |repo| clone URI.
    :type clone_uri: str
-   :param landing_rev: Set the |repo| landing revision. Default is ``rev:tip``.
+   :param landing_rev: Set the |repo| landing revision. e.g branch:default, book:dev, rev:abcd
    :type landing_rev: str
    :param enable_statistics: Enable statistics on the |repo|, (True | False).
    :type enable_statistics: bool

docs/api/methods/search-methods.rst

@@ -6,7 +6,7 @@ search methods
 search 
 ------
 
-.. py:function:: search(apiuser, search_query, search_type, page_limit=<Optional:10>, page=<Optional:1>, search_sort=<Optional:'newfirst'>, repo_name=<Optional:None>, repo_group_name=<Optional:None>)
+.. py:function:: search(apiuser, search_query, search_type, page_limit=<Optional:10>, page=<Optional:1>, search_sort=<Optional:'desc:date'>, repo_name=<Optional:None>, repo_group_name=<Optional:None>)
 
    Fetch Full Text Search results using API.
 
@@ -23,9 +23,15 @@ search
    :type page_limit: Optional(int)
    :param page: Page number. Default first page.
    :type page: Optional(int)
-   :param search_sort: Search sort order. Default newfirst. The following are valid options:
-       * newfirst
-       * oldfirst
+   :param search_sort: Search sort order.Must start with asc: or desc: Default desc:date.
+       The following are valid options:
+       * asc|desc:message.raw
+       * asc|desc:date
+       * asc|desc:author.email.raw
+       * asc|desc:message.raw
+       * newfirst (old legacy equal to desc:date)
+       * oldfirst (old legacy equal to asc:date)
+
    :type search_sort: Optional(str)
    :param repo_name: Filter by one repo. Default is all.
    :type repo_name: Optional(str)

docs/api/methods/store-methods.rst

@@ -6,7 +6,7 @@ store methods
 file_store_add (EE only)
 ------------------------
 
-.. py:function:: file_store_add(apiuser, filename, content)
+.. py:function:: file_store_add(apiuser, filename, content, description=<Optional:''>)
 
    Upload API for the file_store
 
@@ -19,6 +19,8 @@ file_store_add (EE only)
    :type apiuser: AuthUser
    :param filename: name of the file uploaded
    :type filename: str
+   :param description: Optional description for added file
+   :type description: str
    :param content: base64 encoded content of the uploaded file
    :type content: str
 
@@ -35,3 +37,148 @@ file_store_add (EE only)
      error :  null
 
 
+file_store_add_with_acl (EE only)
+---------------------------------
+
+.. py:function:: file_store_add_with_acl(apiuser, filename, content, description=<Optional:''>, scope_user_id=<Optional:None>, scope_repo_id=<Optional:None>, scope_repo_group_id=<Optional:None>)
+
+   Upload API for the file_store
+
+   Example usage from CLI::
+       rhodecode-api --instance-name=enterprise-1 upload_file "{"content": "$(cat image.jpg | base64)", "filename":"image.jpg", "scope_repo_id":101}"
+
+   This command takes the following options:
+
+   :param apiuser: This is filled automatically from the |authtoken|.
+   :type apiuser: AuthUser
+   :param filename: name of the file uploaded
+   :type filename: str
+   :param description: Optional description for added file
+   :type description: str
+   :param content: base64 encoded content of the uploaded file
+   :type content: str
+
+   :param scope_user_id: Optionally bind this file to user.
+       This will check ACL in such way only this user can access the file.
+   :type scope_user_id: int
+   :param scope_repo_id: Optionally bind this file to repository.
+       This will check ACL in such way only user with proper access to such
+       repository can access the file.
+   :type scope_repo_id: int
+   :param scope_repo_group_id:  Optionally bind this file to repository group.
+       This will check ACL in such way only user with proper access to such
+       repository group can access the file.
+   :type scope_repo_group_id: int
+
+   Example output:
+
+   .. code-block:: bash
+
+     id : <id_given_in_input>
+     result: {
+       "access_path": "/_file_store/download/84d156f7-8323-4ad3-9fce-4a8e88e1deaf-0.jpg",
+       "access_path_fqn": "http://server.domain.com/_file_store/download/84d156f7-8323-4ad3-9fce-4a8e88e1deaf-0.jpg",
+       "store_fid": "84d156f7-8323-4ad3-9fce-4a8e88e1deaf-0.jpg"
+     }
+     error :  null
+
+
+file_store_get_info (EE only)
+-----------------------------
+
+.. py:function:: file_store_get_info(apiuser, store_fid)
+
+   Get artifact data.
+
+   Example output:
+
+   .. code-block:: bash
+
+     id : <id_given_in_input>
+     result: {
+         "artifact": {
+           "access_path_fqn": "https://rhodecode.example.com/_file_store/download/0-031c2aa0-0d56-49a7-9ba3-b570bdd342ab.jpg",
+           "created_on": "2019-10-15T16:25:35.491",
+           "description": "my upload",
+           "downloaded_times": 1,
+           "file_uid": "0-031c2aa0-0d56-49a7-9ba3-b570bdd342ab.jpg",
+           "filename": "example.jpg",
+           "filename_org": "0-031c2aa0-0d56-49a7-9ba3-b570bdd342ab.jpg",
+           "hidden": false,
+           "metadata": [
+             {
+               "artifact": "0-031c2aa0-0d56-49a7-9ba3-b570bdd342ab.jpg",
+               "key": "yellow",
+               "section": "tags",
+               "value": "bar"
+             }
+           ],
+           "sha256": "818dff0f44574dfb6814d38e6bf3c60c5943d1d13653398ecddaedf2f6a5b04d",
+           "size": 18599,
+           "uploaded_by": {
+             "email": "admin@rhodecode.com",
+             "emails": [
+               "admin@rhodecode.com"
+             ],
+             "firstname": "Admin",
+             "lastname": "LastName",
+             "user_id": 2,
+             "username": "admin"
+           }
+         }
+     }
+     error :  null
+
+
+file_store_add_metadata (EE only)
+---------------------------------
+
+.. py:function:: file_store_add_metadata(apiuser, store_fid, section, key, value, value_type=<Optional:'unicode'>)
+
+   Add metadata into artifact. The metadata consist of section, key, value. eg.
+   section='tags', 'key'='tag_name', value='1'
+
+   :param apiuser: This is filled automatically from the |authtoken|.
+   :type apiuser: AuthUser
+
+   :param store_fid: file uid, e.g 0-d054cb71-91ab-44e2-9e4b-23fe14b4d74a.mp4
+   :type store_fid: str
+
+   :param section: Section name to add metadata
+   :type section: str
+
+   :param key: Key to add as metadata
+   :type key: str
+
+   :param value: Value to add as metadata
+   :type value: str
+
+   :param value_type: Optional type, default is 'unicode' other types are:
+       int, list, bool, unicode, str
+
+   :type value_type: str
+
+   Example output:
+
+   .. code-block:: bash
+
+     id : <id_given_in_input>
+     result: {
+           "metadata": [
+             {
+               "artifact": "0-d054cb71-91ab-44e2-9e4b-23fe14b4d74a.mp4",
+               "key": "secret",
+               "section": "tags",
+               "value": "1"
+             },
+             {
+               "artifact": "0-d054cb71-91ab-44e2-9e4b-23fe14b4d74a.mp4",
+               "key": "video",
+               "section": "tags",
+               "value": "1"
+             }
+           ]
+     }
+     error :  null
+
+

docs/api/methods/user-group-methods.rst

@@ -72,7 +72,9 @@ create_user_group
    :param active: Set this group as active.
    :type active: Optional(``True`` | ``False``)
    :param sync: Set enabled or disabled the automatically sync from
-       external authentication types like ldap.
+       external authentication types like ldap. If User Group will be named like
+       one from e.g ldap and sync flag is enabled members will be synced automatically.
+       Sync type when enabled via API is set to `manual_api`
    :type sync: Optional(``True`` | ``False``)
 
    Example output:
@@ -391,7 +393,9 @@ update_user_group
    :param active: Set the group as active.
    :type active: Optional(``True`` | ``False``)
    :param sync: Set enabled or disabled the automatically sync from
-       external authentication types like ldap.
+       external authentication types like ldap. If User Group will be named like
+       one from e.g ldap and sync flag is enabled members will be synced automatically.
+       Sync type when enabled via API is set to `manual_api`
    :type sync: Optional(``True`` | ``False``)
 
    Example output:

docs/api/methods/user-methods.rst

@@ -6,7 +6,7 @@ user methods
 create_user 
 -----------
 
-.. py:function:: create_user(apiuser, username, email, password=<Optional:''>, firstname=<Optional:''>, lastname=<Optional:''>, active=<Optional:True>, admin=<Optional:False>, extern_name=<Optional:'rhodecode'>, extern_type=<Optional:'rhodecode'>, force_password_change=<Optional:False>, create_personal_repo_group=<Optional:None>)
+.. py:function:: create_user(apiuser, username, email, password=<Optional:''>, firstname=<Optional:''>, lastname=<Optional:''>, description=<Optional:''>, active=<Optional:True>, admin=<Optional:False>, extern_name=<Optional:'rhodecode'>, extern_type=<Optional:'rhodecode'>, force_password_change=<Optional:False>, create_personal_repo_group=<Optional:None>)
 
    Creates a new user and returns the new user object.
 
@@ -27,6 +27,8 @@ create_user
    :type firstname: Optional(str)
    :param lastname: Set the new user surname.
    :type lastname: Optional(str)
+   :param description: Set user description, or short bio. Metatags are allowed.
+   :type description: Optional(str)
    :param active: Set the user as active.
    :type active: Optional(``True`` | ``False``)
    :param admin: Give the new user admin rights.
@@ -155,6 +157,7 @@ get_user
            "extern_name": "rhodecode",
            "extern_type": "rhodecode",
            "firstname": "username",
+           "description": "user description",
            "ip_addresses": [],
            "language": null,
            "last_login": "Timestamp",
@@ -268,7 +271,7 @@ get_users
 update_user 
 -----------
 
-.. py:function:: update_user(apiuser, userid, username=<Optional:None>, email=<Optional:None>, password=<Optional:None>, firstname=<Optional:None>, lastname=<Optional:None>, active=<Optional:None>, admin=<Optional:None>, extern_type=<Optional:None>, extern_name=<Optional:None>)
+.. py:function:: update_user(apiuser, userid, username=<Optional:None>, email=<Optional:None>, password=<Optional:None>, firstname=<Optional:None>, lastname=<Optional:None>, description=<Optional:None>, active=<Optional:None>, admin=<Optional:None>, extern_type=<Optional:None>, extern_name=<Optional:None>)
 
    Updates the details for the specified user, if that user exists.
 
@@ -291,6 +294,8 @@ update_user
    :type firstname: Optional(str)
    :param lastname: Set the new surname.
    :type lastname: Optional(str)
+   :param description: Set user description, or short bio. Metatags are allowed.
+   :type description: Optional(str)
    :param active: Set the new user as active.
    :type active: Optional(``True`` | ``False``)
    :param admin: Give the user admin rights.

docs/contributing/dev-setup.rst

@@ -19,7 +19,7 @@ Setup Nix Package Manager
 
 To install the Nix Package Manager, please run::
 
-   $ curl https://nixos.org/nix/install | sh
+   $ curl https://nixos.org/releases/nix/nix-2.0.4/install | sh
 
 or go to https://nixos.org/nix/ and follow the installation instructions.
 Once this is correctly set up on your system, you should be able to use the

docs/index.rst

@@ -78,6 +78,12 @@ and commit files and |repos| while managing their security permissions.
    extensions/extensions-hooks
    contributing/contributing
 
+.. toctree::
+   :maxdepth: 2
+   :caption: RhodeCode Control Documentation
+
+   RhodeCode Installer <https://docs.rhodecode.com/RhodeCode-Control/>
+
 .. toctree::
    :maxdepth: 1
    :caption: About

docs/install/configure-celery.rst

@@ -11,16 +11,20 @@ and import repositories in async way. It is also used for bi-directional
 repository sync in scheduler.
 
 If you decide to use Celery you also need a working message queue.
-The recommended and fully supported message broker is rabbitmq_.
+There are two fully supported message brokers is rabbitmq_ and redis_ (recommended).
+
+Since release 4.18.X we recommend using redis_ as a backend since it's generally
+easier to work with, and results in simpler stack as redis is generally recommended
+for caching purposes.
 
 
 In order to install and configure Celery, follow these steps:
 
-1. Install RabbitMQ, see the documentation on the Celery website for
-   `rabbitmq installation`_, or `rabbitmq website installation`_
+1. Install RabbitMQ or Redis for a message queue, see the documentation on the Celery website for
+   `redis installation`_ or `rabbitmq installation`_
 
 
-1a. As en example configuration after installation, you can run::
+1a. If you choose RabbitMQ example configuration after installation would look like that::
 
    sudo rabbitmqctl add_user rcuser secret_password
    sudo rabbitmqctl add_vhost rhodevhost
@@ -45,6 +49,10 @@ In order to install and configure Celery, follow these steps:
    Set the broker_url as minimal settings required to enable operation.
    If used our example data from pt 1a, here is how the broker url should look like::
 
+        # for Redis
+        celery.broker_url = redis://localhost:6379/8
+
+        # for RabbitMQ
         celery.broker_url = amqp://rcuser:secret_password@localhost:5672/rhodevhost
 
    Full configuration example is below:
@@ -57,7 +65,7 @@ In order to install and configure Celery, follow these steps:
         ####################################
 
         use_celery = true
-        celery.broker_url = amqp://rcuser:secret@localhost:5672/rhodevhost
+        celery.broker_url = redis://localhost:6379/8
 
         # maximum tasks to execute before worker restart
         celery.max_tasks_per_child = 100
@@ -69,6 +77,8 @@ In order to install and configure Celery, follow these steps:
 .. _python: http://www.python.org/
 .. _mercurial: http://mercurial.selenic.com/
 .. _celery: http://celeryproject.org/
+.. _redis: http://redis.io
+.. _redis installation: https://redis.io/topics/quickstart
 .. _rabbitmq: http://www.rabbitmq.com/
 .. _rabbitmq installation: http://docs.celeryproject.org/en/latest/getting-started/brokers/rabbitmq.html
 .. _rabbitmq website installation: http://www.rabbitmq.com/download.html

docs/install/database-string.rst

@@ -35,4 +35,3 @@ use the following steps:
     # see sqlalchemy docs for other advanced settings
     sqlalchemy.db1.echo = false
     sqlalchemy.db1.pool_recycle = 3600
-    sqlalchemy.db1.convert_unicode = true

docs/issue-trackers/issue-trackers.rst

@@ -38,44 +38,39 @@ default one. See the instructions in :ref:`repo-it`
 
 .. _issue-tr-eg-ref:
 
+
 Jira Integration
 ----------------
 
-* Regex = ``(?:^#|\s#)(\w+-\d+)``
-* URL = ``https://myissueserver.com/browse/${id}``
-* Issue Prefix = ``#``
+Please check examples in the view for configuration the issue trackers.
+
 
 Confluence (Wiki)
 -----------------
 
-* Regex = ``(?:conf-)([A-Z0-9]+)``
-* URL = ``https://example.atlassian.net/display/wiki/${id}/${repo_name}``
-* issue prefix = ``CONF-``
+Please check examples in the view for configuration the issue trackers.
+
 
 Redmine Integration
 -------------------
 
-* Regex = ``(issue-+\d+)``
-* URL = ``https://myissueserver.com/redmine/issue/${id}``
-* Issue Prefix = ``issue-``
+Please check examples in the view for configuration the issue trackers.
 
-Redmine (wiki)
---------------
 
-* Regex = ``(?:wiki-)([a-zA-Z0-9]+)``
-* URL = ``https://example.com/redmine/projects/wiki/${repo_name}``
-* Issue prefix = ``Issue-``
+Redmine wiki Integration
+------------------------
+
+Please check examples in the view for configuration the issue trackers.
+
 
 Pivotal Tracker
 ---------------
 
-* Regex = ``(?:pivot-)(?<project_id>\d+)-(?<story>\d+)``
-* URL = ``https://www.pivotaltracker.com/s/projects/${project_id}/stories/${story}``
-* Issue prefix = ``Piv-``
+Please check examples in the view for configuration the issue trackers.
+
 
 Trello
 ------
 
-* Regex = ``(?:trello-)(?<card_id>[a-zA-Z0-9]+)``
-* URL = ``https://trello.com/example.com/${card_id}``
-* Issue prefix = ``Trello-``
+Please check examples in the view for configuration the issue trackers.
+

docs/release-notes/release-notes-4.17.0.rst

@@ -62,7 +62,7 @@ Performance
 Fixes
 ^^^^^
 
-- hHooks: fixed more unicode problems with new pull-request link generator.
+- Hooks: fixed more unicode problems with new pull-request link generator.
 - Mercurial: fix ssh-server support for mercurial custom options.
 - Pull requests: updated metadata information for failed merges with multiple heads.
 - Pull requests: calculate ancestor in the same way as creation mode.

docs/release-notes/release-notes-4.18.0.rst

@@ -0,0 +1,230 @@
+|RCE| 4.18.0 |RNS|
+------------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2020-01-05
+
+
+New Features
+^^^^^^^^^^^^
+
+- Artifacts: are no longer in BETA. New info page is available for uploaded artifacts
+  which exposes some useful information like sha256, various access urls etc, and also
+  allows deletion of artifacts, and updating their description.
+- Artifacts: support new download url based on access to artifacts using new auth-token types.
+- Artifacts: added ability to store artifacts using API, and internal cli upload.
+  This allows uploading of artifacts that can have 100s of GBs in size efficiently.
+- Artifacts: added metadata logic to store various extra custom data for artifacts.
+- Comments: added support for adding comment attachments using the artifacts logic.
+  Logged in users can now pick or drag and drop attachments into comment forms.
+- Comments: enable linkification of certain patterns on comments in repo/pull request scopes.
+  This will render now active links to commits, pull-requests mentioned in comments body.
+- Jira: new update integration plugin.
+  Plugin now fetches possible transitions from tickets and show them to users in the interface.
+  Allow sending extra attributes during a transition like `resolution` message.
+- Navigation: Added new consistent and contextual way of creating new objects
+  likes gists, repositories, and repository groups using dedicated action (with a `+` sign)
+  available in the top navigation.
+- Hovercards: added new tooltips and hovercards to expose certain information for objects shown in UI.
+  RhodeCode usernames, issues, pull-requests will have active hovercard logic that will
+  load extra information about them and exposing them to users.
+- Files: all readme files found in repository file browser will be now rendered, allowing having readme per directory.
+- Search: expose line counts in search files information.
+- Audit-logs: expose download user audit logs as JSON file.
+- Users: added description field for users.
+  Allows users to write a short BIO, or description of their role in the organization.
+- Users: allow super-admins to change bound authentication type for users.
+  E.g internal rhodecode accounts can be changed to ldap easily from user settings page.
+- Pull requests: simplified the UI for display view, hide less important information and expose the most important ones.
+- Pull requests: add merge check that detects WIP marker in title.
+  Usually WIP in title means unfinished task that needs still some work, such marker will prevent accidental merges.
+- Pull requests: TODO comments have now a dedicated box below reviewers to keep track
+  of important TODOs that still need attention before review process is finalized.
+- Pull requests: participants of pull request will receive an email about update of a
+  pull requests with a small summary of changes made.
+- Pull requests: change the naming from #NUM into !NUM.
+  !NUM format is now parsed and linkified in comments and commit messages.
+- Pull requests: pull requests which state is changing can now be viewed with a limited view.
+- Pull requests: re-organize merge/close buttons and merge checks according to the new UI.
+- Pull requests: update commits button allows a force-refresh update now using dropdown option.
+- Pull requests: added quick filter to grid view to filter/search pull requests in a repository.
+- Pull requests: closing a pull-request without a merge requires additional confirmation now.
+- Pull requests: merge checks will now show which files caused conflicts and are blocking the merge.
+- Emails: updated all generated emails design and cleanup the data fields they expose.
+  a) More consistent UI for all types of emails. b) Improved formatting of plaintext emails
+  c) Added reply link to comment type emails for quicker response action.
+
+
+General
+^^^^^^^
+
+- Artifacts: don't show hidden artifacts, allow showing them via a GET ?hidden=1 flag.
+  Hidden artifacts are for example comment attachments.
+- UI: new commits page, according to the new design, which started on 4.17.X release lines
+- UI: use explicit named actions like "create user" instead of generic "save" which is bad UX.
+- UI: fixed problems with generating last change in repository groups.
+  There's now a new logic that checks all objects inside group for latest update time.
+- API: add artifact `get_info`, and `store_metadata` methods.
+- API: allowed to specify extra recipients for pr/commit comments api methods.
+- Vcsserver: set file based cache as default for vcsserver which can be shared
+  across multiple workers saving memory usage.
+- Vcsserver: added redis as possible cache backend for even greater performance.
+- Dependencies: bumped GIT version to 2.23.0
+- Dependencies: bumped SVN version to 1.12.2
+- Dependencies: bumped Mercurial version to 5.1.1 and hg-evolve to 9.1.0
+- Search: added logic for sorting ElasticSearch6 backend search results.
+- User bookmarks: make it easier to re-organize existing entries.
+- Data grids: hide pagination for single pages in grids.
+- Gists: UX, removed private/public gist buttons and replaced them with radio group.
+- Gunicorn: moved all configuration of gunicorn workers to .ini files.
+- Gunicorn: added worker memory management allowing setting maximum per-worker memory usage.
+- Automation: moved update groups task into celery task
+- Cache commits: add option to refresh caches manually from advanced pages.
+- Pull requests: add indication of state change in list of pull-requests and actually show them in the list.
+- Cache keys: register and self cleanup cache keys used for invalidation to prevent leaking lot of them into DB on worker recycle
+- Repo groups: removed locking inheritance flag from repo-groups. We'll deprecate this soon and this only brings in confusion
+- System snapshot: improved formatting for better readability
+- System info: expose data about vcsserver.
+- Packages: updated celery to 4.3.0 and switch default backend to redis instead of RabbitMQ.
+  Redis is stable enough and easier to install. Having Redis simplifies the stack as it's used in other parts of RhodeCode.
+- Dependencies: bumped alembic to 1.2.1
+- Dependencies: bumped amqp==2.5.2 and kombu==4.6.6
+- Dependencies: bumped atomicwrites==1.3.0
+- Dependencies: bumped cffi==1.12.3
+- Dependencies: bumped configparser==4.0.2
+- Dependencies: bumped deform==2.0.8
+- Dependencies: bumped dogpile.cache==0.9.0
+- Dependencies: bumped hupper==1.8.1
+- Dependencies: bumped mako to 1.1.0
+- Dependencies: bumped markupsafe to 1.1.1
+- Dependencies: bumped packaging==19.2
+- Dependencies: bumped paste==3.2.1
+- Dependencies: bumped pastescript==3.2.0
+- Dependencies: bumped pathlib2 to 2.3.4
+- Dependencies: bumped pluggy==0.13.0
+- Dependencies: bumped psutil to 5.6.3
+- Dependencies: bumped psutil==5.6.5
+- Dependencies: bumped psycopg2==2.8.4
+- Dependencies: bumped pycurl to 7.43.0.3
+- Dependencies: bumped pyotp==2.3.0
+- Dependencies: bumped pyparsing to 2.4.2
+- Dependencies: bumped pyramid-debugtoolbar==4.5.1
+- Dependencies: bumped pyramid-mako to 1.1.0
+- Dependencies: bumped redis to 3.3.8
+- Dependencies: bumped sqlalchemy to 1.3.8
+- Dependencies: bumped sqlalchemy==1.3.11
+- Dependencies: bumped test libraries.
+- Dependencies: freeze alembic==1.3.1
+- Dependencies: freeze python-dateutil
+- Dependencies: freeze redis==3.3.11
+- Dependencies: freeze supervisor==4.1.0
+
+
+Security
+^^^^^^^^
+
+- Security: fixed issues with exposing wrong http status (403) indicating repository with
+  given name exists and we don't have permissions to it. This was exposed in the redirection
+  logic of the global pull-request page. In case of redirection we also exposed
+  repository name in the URL.
+
+
+Performance
+^^^^^^^^^^^
+
+- Core: many various small improvements and optimizations to make rhodecode faster then before.
+- VCSServer: new cache implementation for remote functions.
+  Single worker shared caches that can use redis/file-cache.
+  This greatly improves performance on larger instances, and doesn't trigger cache
+  re-calculation on worker restarts.
+- GIT: switched internal git operations from Dulwich to libgit2 in order to obtain better performance and scalability.
+- SSH: skip loading unneeded application parts for SSH to make execution of ssh commands faster.
+- Main page: main page will now load repositories and repositories groups using partial DB calls instead of big JSON files.
+  In case of many repositories in root this could lead to very slow page rendering.
+- Admin pages: made all grids use same DB based partial loading logic. We'll no longer fetch
+  all objects into JSON for display purposes. This significantly improves speed of those pages in case
+  of many objects shown in them.
+- Summary page: use non-memory cache for readme, and cleanup cache for repo stats.
+  This change won't re-cache after worker restarts and can be shared across all workers
+- Files: only check for git_lfs/hg_largefiles if they are enabled.
+  This speeds up fetching of files if they are not LF and very big.
+- Vcsserver: added support for streaming data from the remote methods. This allows
+  to stream very large files without taking up memory, mostly for usage in SVN when
+  downloading large binaries from vcs system.
+- Files: added streaming remote attributes for vcsserver.
+  This change enables streaming raw content or raw downloads of large files without
+  transferring them over to enterprise for pack & repack using msgpack.
+  Msgpack has a limit of 2gb and generally pack+repack for ~2gb is very slow.
+- Files: ensure over size limit files never do any content fetching when viewing such files.
+- VCSServer: skip host verification to speed up pycurl calls.
+- User-bookmarks: cache fetching of bookmarks since this is quite expensive query to
+  make with joinedload on repos/repo groups.
+- Goto-switcher: reduce query data to only required attributes for speedups.
+- My account: owner/watched repos are now loaded only using DB queries.
+
+
+Fixes
+^^^^^
+
+- Mercurial: move imports from top-level to prevent from loading mercurial code on hook execution for svn/git.
+- GIT: limit sync-fetch logic to only retrieve tags/ and heads/ with default execution arguments.
+- GIT: fixed issue with git submodules detection.
+- SVN: fix checkout url for ssh+svn backend not having special prefix resulting in incorrect command shown.
+- SVN: fixed problem with showing empty directories.
+- OAuth: use a vendored version of `authomatic` library, and switch Bitbucket authentication to use oauth2.
+- Diffs: handle paths with quotes in diffs.
+- Diffs: fixed outdated files in pull-requests re-using the filediff raw_id for anchor generation. Fixes #5567
+- Diffs: toggle race condition on sticky vs wide-diff-mode that caused some display problems on larger diffs.
+- Pull requests: handle exceptions in state change and improve logging.
+- Pull requests: fixed title/description generation for single commits which are numbers.
+- Pull requests: changed the source of changes to be using shadow repos if it exists.
+  In case of `git push -f` and rebase we lost commits in the repo resulting in
+  problems of displaying versions of pull-requests.
+- Pull requests: handle case when removing existing files from a repository in compare versions diff.
+- Files: don't expose copy content helper in case of binary files.
+- Registration: properly expose first_name/last_name into email on user registration.
+- Markup renderers: fixed broken code highlight for rst files.
+- Ui: make super admin be named consistently across ui.
+- Audit logs: fixed search cases with special chars such as `-`.
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- New Automation task. We've changed the logic for updating latest change inside repository group.
+  New logic includes scanning for changes in all nested objects. Since this is a heavy task
+  a new dedicated scheduler task has been created to update it automatically on a scheduled base.
+  Please review in `admin > settings > automation` to enable this task.
+
+- New safer encryption algorithm. Some setting values are encrypted before storing it inside the database.
+  To keep full backward compatibility old AES algorithm is used.
+  If you wish to enable a safer option set fernet encryption instead inside rhodecode.ini
+  `rhodecode.encrypted_values.algorithm = fernet`
+
+- Pull requests UI changes. We've simplified the UI on pull requests page.
+  Please review the new UI to prevent surprises. All actions from old UI should be still possible with the new one.
+
+- Redis is now a default recommended backend for Celery and replaces previous rabbitmq.
+  Redis is generally easier to manage and install, and it's also very stable for usage
+  in the scheduler/celery async tasks. Since we also recommend Redis for caches the application
+  stack can be simplified by removing rabbitmq and replacing it with single Redis instance.
+
+- Recommendation for using Redis as the new cache backend on vcsserver.
+  Since Version 4.18.0 VCSServer has a new cache implementation for VCS data.
+  By default, for simplicity the cache type is file based. We strongly recommend using
+  Redis instead for better Performance and scalability
+  Please review vcsserver.ini settings under:
+  `rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack`
+
+- New memory monitoring for Gunicorn workers. Starting from 4.18 release a option was added
+  to limit the maximum amount of memory used by a worker.
+  Please review new settings in `[server:main]` section for memory management in both
+  rhodecode.ini and vcsserver.ini::
+
+    ; Maximum memory usage that each worker can use before it will receive a
+    ; graceful restart signal 0 = memory monitoring is disabled
+    ; Examples: 268435456 (256MB), 536870912 (512MB)
+    ; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
+    memory_max_usage = 0

docs/release-notes/release-notes.rst

@@ -9,6 +9,7 @@ Release Notes
 .. toctree::
    :maxdepth: 1
 
+   release-notes-4.18.0.rst
    release-notes-4.17.4.rst
    release-notes-4.17.3.rst
    release-notes-4.17.2.rst

docs/tools/install-tools.rst

@@ -57,9 +57,12 @@ To install |RCT|, use the following steps:
 
 1. Set up a ``virtualenv`` on your local machine, see virtualenv_ instructions
    here.
-2. Install |RCT| using pip. Full url with token is available at https://rhodecode.com/u/#rhodecode-tools
-   ``pip install -I https://dls.rhodecode.com/dls/<token>/rhodecode-tools/latest``
+2. Install |RCT| using pip. All downloadable versions of |RCT| are available at:
+   `https://code.rhodecode.com/rhodecode-tools-ce/artifacts`
 
+   Example installation::
+
+    pip install -I https://code.rhodecode.com/rhodecode-tools-ce/artifacts/download/0-10ac93f4-bb7d-4b97-baea-68110743dd5a.tar.gz
 
 Once |RCT| is installed using these steps there are a few extra
 configuration changes you can make. These are explained in more detail in the

docs/usage/keyboard-shortcuts.rst

@@ -56,4 +56,4 @@ Repository Shortcuts
         Go to the repository settings page.
 
     \--:kbd:`gO`
-        Go to the repository permissions settings.
+        Go to the repository access permissions settings.

grunt_config.json

@@ -99,6 +99,12 @@
       "nonull": true
     }
   },
+  "uglify": {
+    "dist": {
+      "src": "<%= dirs.js.dest %>/scripts.js",
+      "dest": "<%= dirs.js.dest %>/scripts.min.js"
+    }
+  },
   "less": {
     "development": {
       "options": {

package.json

@@ -22,6 +22,7 @@
     "grunt-contrib-less": "^1.1.0",
     "grunt-contrib-watch": "^0.6.1",
     "grunt-webpack": "^3.1.3",
+    "grunt-contrib-uglify": "^4.0.1",
     "jquery": "1.11.3",
     "mark.js": "8.11.1",
     "jshint": "^2.9.1-rc3",

pytest.ini

@@ -9,8 +9,11 @@ vcsserver_config_http = rhodecode/tests/vcsserver_http.ini
 
 addopts =
   --pdbcls=IPython.terminal.debugger:TerminalPdb
+  --strict-markers
 
 markers =
   vcs_operations: Mark tests depending on a running RhodeCode instance.
   xfail_backends: Mark tests as xfail for given backends.
   skip_backends: Mark tests as skipped for given backends.
+  backends: Mark backends
+  dbs: database markers for running tests for given DB

requirements.txt

@@ -1,13 +1,10 @@
 ## dependencies
 
-amqp==2.3.1
-# not released authomatic that has updated some oauth providers
-https://code.rhodecode.com/upstream/authomatic/artifacts/download/0-4fe9c041-a567-4f84-be4c-7efa2a606d3c.tar.gz?md5=f6bdc3c769688212db68233e8d2b0383#egg=authomatic==0.1.0.post1
-
+amqp==2.5.2
 babel==1.3
 beaker==1.9.1
 bleach==3.1.0
-celery==4.1.1
+celery==4.3.0
 channelstream==0.5.2
 click==7.0
 colander==1.7.0
@@ -16,9 +13,9 @@ https://code.rhodecode.com/upstream/configobj/artifacts/download/0-012de99a-b1e1
 cssselect==1.0.3
 cryptography==2.6.1
 decorator==4.1.2
-deform==2.0.7
+deform==2.0.8
 docutils==0.14.0
-dogpile.cache==0.7.1
+dogpile.cache==0.9.0
 dogpile.core==0.4.1
 formencode==1.2.4
 future==0.14.3
@@ -26,55 +23,54 @@ futures==3.0.2
 infrae.cache==1.0.1
 iso8601==0.1.12
 itsdangerous==0.24
-kombu==4.2.1
+kombu==4.6.6
 lxml==4.2.5
-mako==1.0.7
+mako==1.1.0
 markdown==2.6.11
-markupsafe==1.1.0
+markupsafe==1.1.1
 msgpack-python==0.5.6
-pyotp==2.2.7
-packaging==15.2
-pathlib2==2.3.4
-paste==3.0.8
+pyotp==2.3.0
+packaging==19.2
+pathlib2==2.3.5
+paste==3.2.1
 pastedeploy==2.0.1
-pastescript==3.1.0
+pastescript==3.2.0
 peppercorn==0.6
-psutil==5.5.1
+psutil==5.6.5
 py-bcrypt==0.4
-pycurl==7.43.0.2
+pycurl==7.43.0.3
 pycrypto==2.6.1
 pygments==2.4.2
-pyparsing==2.3.0
-pyramid-debugtoolbar==4.5.0
-pyramid-mako==1.0.2
+pyparsing==2.4.5
+pyramid-debugtoolbar==4.5.1
+pyramid-mako==1.1.0
 pyramid==1.10.4
 pyramid_mailer==0.15.1
-python-dateutil
+python-dateutil==2.8.1
 python-ldap==3.1.0
 python-memcached==1.59
 python-pam==1.8.4
 python-saml==2.4.2
-pytz==2018.4
+pytz==2019.2
 tzlocal==1.5.1
 pyzmq==14.6.0
 py-gfm==0.1.4
-redis==2.10.6
+redis==3.3.11
 repoze.lru==0.7
-requests==2.9.1
+requests==2.22.0
 routes==2.4.1
 simplejson==3.16.0
 six==1.11.0
-sqlalchemy==1.1.18
+sqlalchemy==1.3.11
 sshpubkeys==3.1.0
 subprocess32==3.5.4
-supervisor==4.0.3
+supervisor==4.1.0
 translationstring==1.3
-urllib3==1.24.1
+urllib3==1.25.2
 urlobject==2.4.3
 venusian==1.2.0
 weberror==0.10.3
 webhelpers2==2.0
-webhelpers==1.3
 webob==1.8.5
 whoosh==2.7.4
 wsgiref==0.1.2
@@ -87,17 +83,18 @@ zope.interface==4.6.0
 mysql-python==1.2.5
 pymysql==0.8.1
 pysqlite==2.8.3
-psycopg2==2.8.3
+psycopg2==2.8.4
 
 # IPYTHON RENDERING
 # entrypoints backport, pypi version doesn't support egg installs
 https://code.rhodecode.com/upstream/entrypoints/artifacts/download/0-8e9ee9e4-c4db-409c-b07e-81568fd1832d.tar.gz?md5=3a027b8ff1d257b91fe257de6c43357d#egg=entrypoints==0.2.2.rhodecode-upstream1
 nbconvert==5.3.1
 nbformat==4.4.0
-jupyter_client==5.0.0
+jupyter-client==5.0.0
+jupyter-core==4.5.0
 
 ## cli tools
-alembic==1.0.10
+alembic==1.3.1
 invoke==0.13.0
 bumpversion==0.5.3
 
@@ -105,14 +102,15 @@ bumpversion==0.5.3
 gevent==1.4.0
 greenlet==0.4.15
 gunicorn==19.9.0
-waitress==1.3.0
+waitress==1.3.1
 
 ## debug
 ipdb==0.12.0
 ipython==5.1.0
 
-## rhodecode-tools, special case
-https://code.rhodecode.com/rhodecode-tools-ce/artifacts/download/0-10ac93f4-bb7d-4b97-baea-68110743dd5a.tar.gz?md5=962dc77c06aceee62282b98d33149661#egg=rhodecode-tools==1.2.1
+## rhodecode-tools, special case, use file://PATH.tar.gz#egg=rhodecode-tools==X.Y.Z, to test local version
+https://code.rhodecode.com/rhodecode-tools-ce/artifacts/download/0-ed54e749-2ef5-4bc7-ae7f-7900e3c2aa15.tar.gz?sha256=76f024bad3a1e55fdb3d64f13f5b77ff21a12fee699918de2110fe21effd5a3a#egg=rhodecode-tools==1.4.0
+
 
 ## appenlight
 appenlight-client==0.6.26

requirements_pinned.txt

@@ -1,19 +1,27 @@
 # contains not directly required libraries we want to pin the version.
 
-atomicwrites==1.2.1
-attrs==18.2.0
-billiard==3.5.0.3
+atomicwrites==1.3.0
+attrs==19.3.0
+asn1crypto==0.24.0
+billiard==3.6.1.0
+cffi==1.12.3
 chameleon==2.24
-cffi==1.12.2
+configparser==4.0.2
+contextlib2==0.6.0.post1
 ecdsa==0.13.2
-hupper==1.6.1
 gnureadline==6.3.8
+hupper==1.9.1
+ipaddress==1.0.23
+importlib-metadata==0.23
 jinja2==2.9.6
 jsonschema==2.6.0
+pluggy==0.13.1
+pyasn1-modules==0.2.6
 pyramid-jinja2==2.7
-pluggy==0.11.0
-setproctitle==1.1.10
 scandir==1.10.0
+setproctitle==1.1.10
 tempita==0.5.2
+testpath==0.4.4
+transaction==2.4.0
 vine==1.3.0
-configparser==3.7.4
+wcwidth==0.1.7

requirements_test.txt

@@ -1,16 +1,16 @@
 # test related requirements
-pytest==3.8.2
-py==1.6.0
-pytest-cov==2.6.0
-pytest-sugar==0.9.1
-pytest-runner==4.2.0
-pytest-profiling==1.3.0
-pytest-timeout==1.3.2
+pytest==4.6.5
+py==1.8.0
+pytest-cov==2.7.1
+pytest-sugar==0.9.2
+pytest-runner==5.1.0
+pytest-profiling==1.7.0
+pytest-timeout==1.3.3
 gprof2dot==2017.9.19
 
-mock==1.0.1
+mock==3.0.5
 cov-core==1.15.0
-coverage==4.5.3
+coverage==4.5.4
 
 webtest==2.0.33
 beautifulsoup4==4.6.3

rhodecode/VERSION

@@ -1 +1 @@
-4.17.4
+4.18.0

rhodecode/__init__.py

@@ -45,7 +45,7 @@ PYRAMID_SETTINGS = {}
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 98  # defines current db version for migrations
+__dbversion__ = 103  # defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'

rhodecode/api/__init__.py

@@ -122,7 +122,7 @@ def jsonrpc_response(request, result):
     return response
 
 
-def jsonrpc_error(request, message, retid=None, code=None):
+def jsonrpc_error(request, message, retid=None, code=None, headers=None):
     """
     Generate a Response object with a JSON-RPC error body
 
@@ -132,10 +132,12 @@ def jsonrpc_error(request, message, retid=None, code=None):
     """
     err_dict = {'id': retid, 'result': None, 'error': message}
     body = render(DEFAULT_RENDERER, err_dict, request=request).encode('utf-8')
+
     return Response(
         body=body,
         status=code,
-        content_type='application/json'
+        content_type='application/json',
+        headerlist=headers
     )
 
 
@@ -287,8 +289,7 @@ def request_view(request):
     })
 
     # register some common functions for usage
-    attach_context_attributes(
-        TemplateArgs(), request, request.rpc_user.user_id)
+    attach_context_attributes(TemplateArgs(), request, request.rpc_user.user_id)
 
     try:
         ret_value = func(**call_params)
@@ -298,9 +299,13 @@ def request_view(request):
     except Exception:
         log.exception('Unhandled exception occurred on api call: %s', func)
         exc_info = sys.exc_info()
-        store_exception(id(exc_info), exc_info, prefix='rhodecode-api')
+        exc_id, exc_type_name = store_exception(
+            id(exc_info), exc_info, prefix='rhodecode-api')
+        error_headers = [('RhodeCode-Exception-Id', str(exc_id)),
+                         ('RhodeCode-Exception-Type', str(exc_type_name))]
         return jsonrpc_error(
-            request, retid=request.rpc_id, message='Internal server error')
+            request, retid=request.rpc_id, message='Internal server error',
+            headers=error_headers)
 
 
 def setup_request(request):
@@ -333,6 +338,7 @@ def setup_request(request):
             raise JSONRPCError("Content-Length is 0")
 
     raw_body = request.body
+    log.debug("Loading JSON body now")
     try:
         json_body = json.loads(raw_body)
     except ValueError as e:
@@ -359,7 +365,7 @@ def setup_request(request):
         request.rpc_params = json_body['args'] \
             if isinstance(json_body['args'], dict) else {}
 
-        log.debug('method: %s, params: %s', request.rpc_method, request.rpc_params)
+        log.debug('method: %s, params: %.10240r', request.rpc_method, request.rpc_params)
     except KeyError as e:
         raise JSONRPCError('Incorrect JSON data. Missing %s' % e)
 

rhodecode/api/tests/test_close_pull_request.py

@@ -49,7 +49,7 @@ class TestClosePullRequest(object):
         assert_ok(id_, expected, response.body)
         journal = UserLog.query()\
             .filter(UserLog.user_id == author) \
-            .order_by('user_log_id') \
+            .order_by(UserLog.user_log_id.asc()) \
             .filter(UserLog.repository_id == repo)\
             .all()
         assert journal[-1].action == 'repo.pull_request.close'

rhodecode/api/tests/test_comment_commit.py

@@ -20,7 +20,7 @@
 
 import pytest
 
-from rhodecode.model.db import ChangesetStatus
+from rhodecode.model.db import ChangesetStatus, User
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_error, assert_ok)
 
@@ -79,3 +79,38 @@ class TestCommentCommit(object):
             'success': True
         }
         assert_ok(id_, expected, given=response.body)
+
+    def test_api_comment_commit_with_extra_recipients(self, backend, user_util):
+
+        commit_id = backend.repo.scm_instance().get_commit('tip').raw_id
+
+        user1 = user_util.create_user()
+        user1_id = user1.user_id
+        user2 = user_util.create_user()
+        user2_id = user2.user_id
+
+        id_, params = build_data(
+            self.apikey, 'comment_commit', repoid=backend.repo_name,
+            commit_id=commit_id,
+            message='abracadabra',
+            extra_recipients=[user1.user_id, user2.username])
+
+        response = api_call(self.app, params)
+        repo = backend.repo.scm_instance()
+
+        expected = {
+            'msg': 'Commented on commit `%s` for repository `%s`' % (
+                repo.get_commit().raw_id, backend.repo_name),
+            'status_change': None,
+            'success': True
+        }
+
+        assert_ok(id_, expected, given=response.body)
+        # check user1/user2 inbox for notification
+        user1 = User.get(user1_id)
+        assert 1 == len(user1.notifications)
+        assert 'abracadabra' in user1.notifications[0].notification.body
+
+        user2 = User.get(user2_id)
+        assert 1 == len(user2.notifications)
+        assert 'abracadabra' in user2.notifications[0].notification.body

rhodecode/api/tests/test_comment_pull_request.py

@@ -21,7 +21,7 @@
 import pytest
 
 from rhodecode.model.comment import CommentsModel
-from rhodecode.model.db import UserLog
+from rhodecode.model.db import UserLog, User
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN
 from rhodecode.api.tests.utils import (
@@ -65,10 +65,47 @@ class TestCommentPullRequest(object):
         journal = UserLog.query()\
             .filter(UserLog.user_id == author)\
             .filter(UserLog.repository_id == repo) \
-            .order_by('user_log_id') \
+            .order_by(UserLog.user_log_id.asc()) \
             .all()
         assert journal[-1].action == 'repo.pull_request.comment.create'
 
+    @pytest.mark.backends("git", "hg")
+    def test_api_comment_pull_request_with_extra_recipients(self, pr_util, user_util):
+        pull_request = pr_util.create_pull_request()
+
+        user1 = user_util.create_user()
+        user1_id = user1.user_id
+        user2 = user_util.create_user()
+        user2_id = user2.user_id
+
+        id_, params = build_data(
+            self.apikey, 'comment_pull_request',
+            repoid=pull_request.target_repo.repo_name,
+            pullrequestid=pull_request.pull_request_id,
+            message='test message',
+            extra_recipients=[user1.user_id, user2.username]
+        )
+        response = api_call(self.app, params)
+        pull_request = PullRequestModel().get(pull_request.pull_request_id)
+
+        comments = CommentsModel().get_comments(
+            pull_request.target_repo.repo_id, pull_request=pull_request)
+
+        expected = {
+            'pull_request_id': pull_request.pull_request_id,
+            'comment_id': comments[-1].comment_id,
+            'status': {'given': None, 'was_changed': None}
+        }
+        assert_ok(id_, expected, response.body)
+        # check user1/user2 inbox for notification
+        user1 = User.get(user1_id)
+        assert 1 == len(user1.notifications)
+        assert 'test message' in user1.notifications[0].notification.body
+
+        user2 = User.get(user2_id)
+        assert 1 == len(user2.notifications)
+        assert 'test message' in user2.notifications[0].notification.body
+
     @pytest.mark.backends("git", "hg")
     def test_api_comment_pull_request_change_status(
             self, pr_util, no_notifications):

rhodecode/api/tests/test_create_user.py

@@ -82,6 +82,7 @@ class TestCreateUser(object):
             self.apikey, 'create_user',
             username=username,
             email=email,
+            description='CTO of Things',
             password='example')
         response = api_call(self.app, params)
 

rhodecode/api/tests/test_fts_search.py

@@ -27,6 +27,16 @@ from rhodecode.api.tests.utils import (
 @pytest.mark.usefixtures("testuser_api", "app")
 class TestApiSearch(object):
 
+    @pytest.mark.parametrize("sort_dir", [
+        "asc",
+        "desc",
+    ])
+    @pytest.mark.parametrize("sort", [
+        "xxx",
+        "author_email",
+        "date",
+        "message",
+    ])
     @pytest.mark.parametrize("query, expected_hits, expected_paths", [
         ('todo', 23, [
             'vcs/backends/hg/inmemory.py',
@@ -55,10 +65,11 @@ class TestApiSearch(object):
             'vcs/tests/test_cli.py']),
         ('owner:michał test', 0, []),
     ])
-    def test_search_content_results(self, query, expected_hits, expected_paths):
+    def test_search_content_results(self, sort_dir, sort, query, expected_hits, expected_paths):
         id_, params = build_data(
             self.apikey_regular, 'search',
             search_query=query,
+            search_sort='{}:{}'.format(sort_dir, sort),
             search_type='content')
 
         response = api_call(self.app, params)
@@ -70,6 +81,16 @@ class TestApiSearch(object):
         for expected_path in expected_paths:
             assert expected_path in paths
 
+    @pytest.mark.parametrize("sort_dir", [
+        "asc",
+        "desc",
+    ])
+    @pytest.mark.parametrize("sort", [
+        "xxx",
+        "date",
+        "file",
+        "size",
+    ])
     @pytest.mark.parametrize("query, expected_hits, expected_paths", [
         ('readme.rst', 3, []),
         ('test*', 75, []),
@@ -77,10 +98,11 @@ class TestApiSearch(object):
         ('extension:rst', 48, []),
         ('extension:rst api', 24, []),
     ])
-    def test_search_file_paths(self, query, expected_hits, expected_paths):
+    def test_search_file_paths(self, sort_dir, sort, query, expected_hits, expected_paths):
         id_, params = build_data(
             self.apikey_regular, 'search',
             search_query=query,
+            search_sort='{}:{}'.format(sort_dir, sort),
             search_type='path')
 
         response = api_call(self.app, params)

rhodecode/api/tests/test_get_method.py

@@ -50,6 +50,7 @@ class TestGetMethod(object):
                     {'apiuser': '<RequiredType>',
                      'comment_type': "<Optional:u'note'>",
                      'commit_id': '<RequiredType>',
+                     'extra_recipients': '<Optional:[]>',
                      'message': '<RequiredType>',
                      'repoid': '<RequiredType>',
                      'request': '<RequiredType>',

rhodecode/api/tests/test_get_repo_changeset.py

@@ -54,7 +54,7 @@ class TestGetRepoChangeset(object):
             details=details,
         )
         response = api_call(self.app, params)
-        expected = 'commit_id must be a string value'
+        expected = "commit_id must be a string value got <type 'int'> instead"
         assert_error(id_, expected, given=response.body)
 
     @pytest.mark.parametrize("details", ['basic', 'extended', 'full'])
@@ -137,5 +137,5 @@ class TestGetRepoChangeset(object):
             details=details,
         )
         response = api_call(self.app, params)
-        expected = 'commit_id must be a string value'
+        expected = "commit_id must be a string value got <type 'int'> instead"
         assert_error(id_, expected, given=response.body)

rhodecode/api/tests/test_get_repo_comments.py

@@ -31,36 +31,38 @@ from rhodecode.api.tests.utils import (
 @pytest.fixture()
 def make_repo_comments_factory(request):
 
-    def maker(repo):
-        user = User.get_first_super_admin()
-        commit = repo.scm_instance()[0]
+    class Make(object):
 
-        commit_id = commit.raw_id
-        file_0 = commit.affected_files[0]
-        comments = []
+        def make_comments(self, repo):
+            user = User.get_first_super_admin()
+            commit = repo.scm_instance()[0]
 
-        # general
-        CommentsModel().create(
-            text='General Comment', repo=repo, user=user, commit_id=commit_id,
-            comment_type=ChangesetComment.COMMENT_TYPE_NOTE, send_email=False)
+            commit_id = commit.raw_id
+            file_0 = commit.affected_files[0]
+            comments = []
 
-        # inline
-        CommentsModel().create(
-            text='Inline Comment', repo=repo, user=user, commit_id=commit_id,
-            f_path=file_0, line_no='n1',
-            comment_type=ChangesetComment.COMMENT_TYPE_NOTE, send_email=False)
+            # general
+            CommentsModel().create(
+                text='General Comment', repo=repo, user=user, commit_id=commit_id,
+                comment_type=ChangesetComment.COMMENT_TYPE_NOTE, send_email=False)
 
-        # todo
-        CommentsModel().create(
-            text='INLINE TODO Comment', repo=repo, user=user, commit_id=commit_id,
-            f_path=file_0, line_no='n1',
-            comment_type=ChangesetComment.COMMENT_TYPE_TODO, send_email=False)
+            # inline
+            CommentsModel().create(
+                text='Inline Comment', repo=repo, user=user, commit_id=commit_id,
+                f_path=file_0, line_no='n1',
+                comment_type=ChangesetComment.COMMENT_TYPE_NOTE, send_email=False)
 
-        @request.addfinalizer
-        def cleanup():
-            for comment in comments:
-                Session().delete(comment)
-    return maker
+            # todo
+            CommentsModel().create(
+                text='INLINE TODO Comment', repo=repo, user=user, commit_id=commit_id,
+                f_path=file_0, line_no='n1',
+                comment_type=ChangesetComment.COMMENT_TYPE_TODO, send_email=False)
+
+            @request.addfinalizer
+            def cleanup():
+                for comment in comments:
+                    Session().delete(comment)
+    return Make()
 
 
 @pytest.mark.usefixtures("testuser_api", "app")
@@ -76,7 +78,7 @@ class TestGetRepo(object):
                                    make_repo_comments_factory, filters, expected_count):
         commits = [{'message': 'A'}, {'message': 'B'}]
         repo = backend.create_repo(commits=commits)
-        make_repo_comments_factory(repo)
+        make_repo_comments_factory.make_comments(repo)
 
         api_call_params = {'repoid': repo.repo_name,}
         api_call_params.update(filters)
@@ -92,12 +94,13 @@ class TestGetRepo(object):
 
         assert len(result) == expected_count
 
-    def test_api_get_repo_comments_wrong_comment_typ(self, backend_hg):
+    def test_api_get_repo_comments_wrong_comment_type(
+            self, make_repo_comments_factory, backend_hg):
+        commits = [{'message': 'A'}, {'message': 'B'}]
+        repo = backend_hg.create_repo(commits=commits)
+        make_repo_comments_factory.make_comments(repo)
 
-        repo = backend_hg.create_repo()
-        make_repo_comments_factory(repo)
-
-        api_call_params = {'repoid': repo.repo_name,}
+        api_call_params = {'repoid': repo.repo_name}
         api_call_params.update({'comment_type': 'bogus'})
 
         expected = 'comment_type must be one of `{}` got {}'.format(

rhodecode/api/tests/test_get_server_info.py

@@ -25,7 +25,7 @@ from rhodecode.model.scm import ScmModel
 from rhodecode.api.tests.utils import build_data, api_call, assert_ok
 
 
-@pytest.fixture
+@pytest.fixture()
 def http_host_stub():
     """
     To ensure that we can get an IP address, this test shall run with a

rhodecode/api/tests/test_merge_pull_request.py

@@ -120,7 +120,7 @@ class TestMergePullRequest(object):
         journal = UserLog.query()\
             .filter(UserLog.user_id == author)\
             .filter(UserLog.repository_id == repo) \
-            .order_by('user_log_id') \
+            .order_by(UserLog.user_log_id.asc()) \
             .all()
         assert journal[-2].action == 'repo.pull_request.merge'
         assert journal[-1].action == 'repo.pull_request.close'
@@ -221,7 +221,7 @@ class TestMergePullRequest(object):
         journal = UserLog.query() \
             .filter(UserLog.user_id == merge_user_id) \
             .filter(UserLog.repository_id == repo) \
-            .order_by('user_log_id') \
+            .order_by(UserLog.user_log_id.asc()) \
             .all()
         assert journal[-2].action == 'repo.pull_request.merge'
         assert journal[-1].action == 'repo.pull_request.close'

rhodecode/api/tests/test_update_user.py

@@ -42,7 +42,8 @@ class TestUpdateUser(object):
         ('extern_name', None),
         ('active', False),
         ('active', True),
-        ('password', 'newpass')
+        ('password', 'newpass'),
+        ('description', 'CTO 4 Life')
     ])
     def test_api_update_user(self, name, expected, user_util):
         usr = user_util.create_user()

rhodecode/api/tests/utils.py

@@ -20,11 +20,16 @@
 
 
 import random
+import pytest
 
 from rhodecode.api.utils import get_origin
 from rhodecode.lib.ext_json import json
 
 
+def jsonify(obj):
+    return json.loads(json.dumps(obj))
+
+
 API_URL = '/_admin/api'
 
 
@@ -42,12 +47,16 @@ def assert_call_ok(id_, given):
 
 
 def assert_ok(id_, expected, given):
+    given = json.loads(given)
+    if given.get('error'):
+        pytest.fail("Unexpected ERROR in success response: {}".format(given['error']))
+
     expected = jsonify({
         'id': id_,
         'error': None,
         'result': expected
     })
-    given = json.loads(given)
+
     assert expected == given
 
 
@@ -61,10 +70,6 @@ def assert_error(id_, expected, given):
     assert expected == given
 
 
-def jsonify(obj):
-    return json.loads(json.dumps(obj))
-
-
 def build_data(apikey, method, **kw):
     """
     Builds API data with given random ID

rhodecode/api/views/pull_request_api.py

@@ -451,7 +451,7 @@ def comment_pull_request(
         request, apiuser, pullrequestid, repoid=Optional(None),
         message=Optional(None), commit_id=Optional(None), status=Optional(None),
         comment_type=Optional(ChangesetComment.COMMENT_TYPE_NOTE),
-        resolves_comment_id=Optional(None),
+        resolves_comment_id=Optional(None), extra_recipients=Optional([]),
         userid=Optional(OAttr('apiuser'))):
     """
     Comment on the pull request specified with the `pullrequestid`,
@@ -476,6 +476,11 @@ def comment_pull_request(
     :type status: str
     :param comment_type: Comment type, one of: 'note', 'todo'
     :type comment_type: Optional(str), default: 'note'
+    :param resolves_comment_id: id of comment which this one will resolve
+    :type resolves_comment_id: Optional(int)
+    :param extra_recipients: list of user ids or usernames to add
+        notifications for this comment. Acts like a CC for notification
+    :type extra_recipients: Optional(list)
     :param userid: Comment on the pull request as this user
     :type userid: Optional(str or int)
 
@@ -521,6 +526,7 @@ def comment_pull_request(
     commit_id = Optional.extract(commit_id)
     comment_type = Optional.extract(comment_type)
     resolves_comment_id = Optional.extract(resolves_comment_id)
+    extra_recipients = Optional.extract(extra_recipients)
 
     if not message and not status:
         raise JSONRPCError(
@@ -580,7 +586,8 @@ def comment_pull_request(
         renderer=renderer,
         comment_type=comment_type,
         resolves_comment_id=resolves_comment_id,
-        auth_user=auth_user
+        auth_user=auth_user,
+        extra_recipients=extra_recipients
     )
 
     if allowed_to_change_status and status:
@@ -888,7 +895,9 @@ def update_pull_request(
 
         with pull_request.set_state(PullRequest.STATE_UPDATING):
             if PullRequestModel().has_valid_update_type(pull_request):
-                update_response = PullRequestModel().update_commits(pull_request)
+                db_user = apiuser.get_instance()
+                update_response = PullRequestModel().update_commits(
+                    pull_request, db_user)
                 commit_changes = update_response.changes or commit_changes
             Session().commit()
 

rhodecode/api/views/repo_api.py

@@ -617,9 +617,7 @@ def get_repo_fts_tree(request, apiuser, repoid, commit_id, root_path):
     cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
     region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
-    @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
-                                           condition=cache_on)
-    def compute_fts_tree(repo_id, commit_id, root_path, cache_ver):
+    def compute_fts_tree(cache_ver, repo_id, commit_id, root_path):
         return ScmModel().get_fts_data(repo_id, commit_id, root_path)
 
     try:
@@ -640,7 +638,7 @@ def get_repo_fts_tree(request, apiuser, repoid, commit_id, root_path):
             'with caching: %s[TTL: %ss]' % (
                 repo_id, commit_id, cache_on, cache_seconds or 0))
 
-        tree_files = compute_fts_tree(repo_id, commit_id, root_path, 'v1')
+        tree_files = compute_fts_tree(rc_cache.FILE_TREE_CACHE_VER, repo_id, commit_id, root_path)
         return tree_files
 
     except Exception:
@@ -714,7 +712,7 @@ def create_repo(
         private=Optional(False),
         clone_uri=Optional(None),
         push_uri=Optional(None),
-        landing_rev=Optional('rev:tip'),
+        landing_rev=Optional(None),
         enable_statistics=Optional(False),
         enable_locking=Optional(False),
         enable_downloads=Optional(False),
@@ -749,7 +747,7 @@ def create_repo(
     :type clone_uri: str
     :param push_uri: set push_uri
     :type push_uri: str
-    :param landing_rev: <rev_type>:<rev>
+    :param landing_rev: <rev_type>:<rev>, e.g branch:default, book:dev, rev:abcd
     :type landing_rev: str
     :param enable_locking:
     :type enable_locking: bool
@@ -793,7 +791,6 @@ def create_repo(
     copy_permissions = Optional.extract(copy_permissions)
     clone_uri = Optional.extract(clone_uri)
     push_uri = Optional.extract(push_uri)
-    landing_commit_ref = Optional.extract(landing_rev)
 
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     if isinstance(private, Optional):
@@ -807,8 +804,15 @@ def create_repo(
     if isinstance(enable_downloads, Optional):
         enable_downloads = defs.get('repo_enable_downloads')
 
+    landing_ref, _label = ScmModel.backend_landing_ref(repo_type)
+    ref_choices, _labels = ScmModel().get_repo_landing_revs(request.translate)
+    ref_choices = list(set(ref_choices + [landing_ref]))
+
+    landing_commit_ref = Optional.extract(landing_rev) or landing_ref
+
     schema = repo_schema.RepoSchema().bind(
         repo_type_options=rhodecode.BACKENDS.keys(),
+        repo_ref_options=ref_choices,
         repo_type=repo_type,
         # user caller
         user=apiuser)
@@ -958,7 +962,7 @@ def update_repo(
         owner=Optional(OAttr('apiuser')), description=Optional(''),
         private=Optional(False),
         clone_uri=Optional(None), push_uri=Optional(None),
-        landing_rev=Optional('rev:tip'), fork_of=Optional(None),
+        landing_rev=Optional(None), fork_of=Optional(None),
         enable_statistics=Optional(False),
         enable_locking=Optional(False),
         enable_downloads=Optional(False), fields=Optional('')):
@@ -993,7 +997,7 @@ def update_repo(
     :type private: bool
     :param clone_uri: Update the |repo| clone URI.
     :type clone_uri: str
-    :param landing_rev: Set the |repo| landing revision. Default is ``rev:tip``.
+    :param landing_rev: Set the |repo| landing revision. e.g branch:default, book:dev, rev:abcd
     :type landing_rev: str
     :param enable_statistics: Enable statistics on the |repo|, (True | False).
     :type enable_statistics: bool
@@ -1049,8 +1053,10 @@ def update_repo(
         repo_enable_downloads=enable_downloads
         if not isinstance(enable_downloads, Optional) else repo.enable_downloads)
 
+    landing_ref, _label = ScmModel.backend_landing_ref(repo.repo_type)
     ref_choices, _labels = ScmModel().get_repo_landing_revs(
         request.translate, repo=repo)
+    ref_choices = list(set(ref_choices + [landing_ref]))
 
     old_values = repo.get_api_data()
     repo_type = repo.repo_type
@@ -1128,7 +1134,7 @@ def fork_repo(request, apiuser, repoid, fork_name,
               description=Optional(''),
               private=Optional(False),
               clone_uri=Optional(None),
-              landing_rev=Optional('rev:tip'),
+              landing_rev=Optional(None),
               copy_permissions=Optional(False)):
     """
     Creates a fork of the specified |repo|.
@@ -1158,7 +1164,7 @@ def fork_repo(request, apiuser, repoid, fork_name,
     :type copy_permissions: bool
     :param private: Make the fork private. The default is False.
     :type private: bool
-    :param landing_rev: Set the landing revision. The default is tip.
+    :param landing_rev: Set the landing revision. E.g branch:default, book:dev, rev:abcd
 
     Example output:
 
@@ -1210,11 +1216,17 @@ def fork_repo(request, apiuser, repoid, fork_name,
     description = Optional.extract(description)
     copy_permissions = Optional.extract(copy_permissions)
     clone_uri = Optional.extract(clone_uri)
-    landing_commit_ref = Optional.extract(landing_rev)
+
+    landing_ref, _label = ScmModel.backend_landing_ref(repo.repo_type)
+    ref_choices, _labels = ScmModel().get_repo_landing_revs(request.translate)
+    ref_choices = list(set(ref_choices + [landing_ref]))
+    landing_commit_ref = Optional.extract(landing_rev) or landing_ref
+
     private = Optional.extract(private)
 
     schema = repo_schema.RepoSchema().bind(
         repo_type_options=rhodecode.BACKENDS.keys(),
+        repo_ref_options=ref_choices,
         repo_type=repo.repo_type,
         # user caller
         user=apiuser)
@@ -1538,7 +1550,7 @@ def lock(request, apiuser, repoid, locked=Optional(None),
 def comment_commit(
         request, apiuser, repoid, commit_id, message, status=Optional(None),
         comment_type=Optional(ChangesetComment.COMMENT_TYPE_NOTE),
-        resolves_comment_id=Optional(None),
+        resolves_comment_id=Optional(None), extra_recipients=Optional([]),
         userid=Optional(OAttr('apiuser'))):
     """
     Set a commit comment, and optionally change the status of the commit.
@@ -1556,6 +1568,11 @@ def comment_commit(
     :type status: str
     :param comment_type: Comment type, one of: 'note', 'todo'
     :type comment_type: Optional(str), default: 'note'
+    :param resolves_comment_id: id of comment which this one will resolve
+    :type resolves_comment_id: Optional(int)
+    :param extra_recipients: list of user ids or usernames to add
+        notifications for this comment. Acts like a CC for notification
+    :type extra_recipients: Optional(list)
     :param userid: Set the user name of the comment creator.
     :type userid: Optional(str or int)
 
@@ -1592,6 +1609,7 @@ def comment_commit(
     status = Optional.extract(status)
     comment_type = Optional.extract(comment_type)
     resolves_comment_id = Optional.extract(resolves_comment_id)
+    extra_recipients = Optional.extract(extra_recipients)
 
     allowed_statuses = [x[0] for x in ChangesetStatus.STATUSES]
     if status and status not in allowed_statuses:
@@ -1620,7 +1638,8 @@ def comment_commit(
             renderer=renderer,
             comment_type=comment_type,
             resolves_comment_id=resolves_comment_id,
-            auth_user=apiuser
+            auth_user=apiuser,
+            extra_recipients=extra_recipients
         )
         if status:
             # also do a status change

rhodecode/api/views/search_api.py

@@ -33,7 +33,7 @@ log = logging.getLogger(__name__)
 
 @jsonrpc_method()
 def search(request, apiuser, search_query, search_type, page_limit=Optional(10),
-           page=Optional(1), search_sort=Optional('newfirst'),
+           page=Optional(1), search_sort=Optional('desc:date'),
            repo_name=Optional(None), repo_group_name=Optional(None)):
     """
     Fetch Full Text Search results using API.
@@ -51,9 +51,15 @@ def search(request, apiuser, search_query, search_type, page_limit=Optional(10),
     :type page_limit: Optional(int)
     :param page: Page number. Default first page.
     :type page: Optional(int)
-    :param search_sort: Search sort order. Default newfirst. The following are valid options:
-        * newfirst
-        * oldfirst
+    :param search_sort: Search sort order.Must start with asc: or desc: Default desc:date.
+        The following are valid options:
+        * asc|desc:message.raw
+        * asc|desc:date
+        * asc|desc:author.email.raw
+        * asc|desc:message.raw
+        * newfirst (old legacy equal to desc:date)
+        * oldfirst (old legacy equal to asc:date)
+
     :type search_sort: Optional(str)
     :param repo_name: Filter by one repo. Default is all.
     :type repo_name: Optional(str)
@@ -101,7 +107,7 @@ def search(request, apiuser, search_query, search_type, page_limit=Optional(10),
             searcher.cleanup()
 
         if not search_result['error']:
-            data['execution_time'] = '%s results (%.3f seconds)' % (
+            data['execution_time'] = '%s results (%.4f seconds)' % (
                 search_result['count'],
                 search_result['runtime'])
         else:

rhodecode/api/views/user_api.py

@@ -75,6 +75,7 @@ def get_user(request, apiuser, userid=Optional(OAttr('apiuser'))):
             "extern_name": "rhodecode",
             "extern_type": "rhodecode",
             "firstname": "username",
+            "description": "user description",
             "ip_addresses": [],
             "language": null,
             "last_login": "Timestamp",
@@ -159,7 +160,7 @@ def get_users(request, apiuser):
 
 @jsonrpc_method()
 def create_user(request, apiuser, username, email, password=Optional(''),
-                firstname=Optional(''), lastname=Optional(''),
+                firstname=Optional(''), lastname=Optional(''), description=Optional(''),
                 active=Optional(True), admin=Optional(False),
                 extern_name=Optional('rhodecode'),
                 extern_type=Optional('rhodecode'),
@@ -185,6 +186,8 @@ def create_user(request, apiuser, username, email, password=Optional(''),
     :type firstname: Optional(str)
     :param lastname: Set the new user surname.
     :type lastname: Optional(str)
+    :param description: Set user description, or short bio. Metatags are allowed.
+    :type description: Optional(str)
     :param active: Set the user as active.
     :type active: Optional(``True`` | ``False``)
     :param admin: Give the new user admin rights.
@@ -250,6 +253,7 @@ def create_user(request, apiuser, username, email, password=Optional(''),
     email = Optional.extract(email)
     first_name = Optional.extract(firstname)
     last_name = Optional.extract(lastname)
+    description = Optional.extract(description)
     active = Optional.extract(active)
     admin = Optional.extract(admin)
     extern_type = Optional.extract(extern_type)
@@ -267,6 +271,7 @@ def create_user(request, apiuser, username, email, password=Optional(''),
             last_name=last_name,
             active=active,
             admin=admin,
+            description=description,
             extern_type=extern_type,
             extern_name=extern_name,
             ))
@@ -280,6 +285,7 @@ def create_user(request, apiuser, username, email, password=Optional(''),
             email=schema_data['email'],
             firstname=schema_data['first_name'],
             lastname=schema_data['last_name'],
+            description=schema_data['description'],
             active=schema_data['active'],
             admin=schema_data['admin'],
             extern_type=schema_data['extern_type'],
@@ -307,7 +313,7 @@ def create_user(request, apiuser, username, email, password=Optional(''),
 def update_user(request, apiuser, userid, username=Optional(None),
                 email=Optional(None), password=Optional(None),
                 firstname=Optional(None), lastname=Optional(None),
-                active=Optional(None), admin=Optional(None),
+                description=Optional(None), active=Optional(None), admin=Optional(None),
                 extern_type=Optional(None), extern_name=Optional(None), ):
     """
     Updates the details for the specified user, if that user exists.
@@ -331,6 +337,8 @@ def update_user(request, apiuser, userid, username=Optional(None),
     :type firstname: Optional(str)
     :param lastname: Set the new surname.
     :type lastname: Optional(str)
+    :param description: Set user description, or short bio. Metatags are allowed.
+    :type description: Optional(str)
     :param active: Set the new user as active.
     :type active: Optional(``True`` | ``False``)
     :param admin: Give the user admin rights.
@@ -379,6 +387,7 @@ def update_user(request, apiuser, userid, username=Optional(None),
         store_update(updates, email, 'email')
         store_update(updates, firstname, 'name')
         store_update(updates, lastname, 'lastname')
+        store_update(updates, description, 'description')
         store_update(updates, active, 'active')
         store_update(updates, admin, 'admin')
         store_update(updates, extern_name, 'extern_name')

rhodecode/api/views/user_group_api.py

@@ -191,7 +191,9 @@ def create_user_group(
     :param active: Set this group as active.
     :type active: Optional(``True`` | ``False``)
     :param sync: Set enabled or disabled the automatically sync from
-        external authentication types like ldap.
+        external authentication types like ldap. If User Group will be named like
+        one from e.g ldap and sync flag is enabled members will be synced automatically.
+        Sync type when enabled via API is set to `manual_api`
     :type sync: Optional(``True`` | ``False``)
 
     Example output:
@@ -303,7 +305,9 @@ def update_user_group(request, apiuser, usergroupid, group_name=Optional(''),
     :param active: Set the group as active.
     :type active: Optional(``True`` | ``False``)
     :param sync: Set enabled or disabled the automatically sync from
-        external authentication types like ldap.
+        external authentication types like ldap. If User Group will be named like
+        one from e.g ldap and sync flag is enabled members will be synced automatically.
+        Sync type when enabled via API is set to `manual_api`
     :type sync: Optional(``True`` | ``False``)
 
     Example output:

rhodecode/apps/_base/__init__.py

@@ -25,9 +25,11 @@ import operator
 from pyramid import compat
 from pyramid.httpexceptions import HTTPFound, HTTPForbidden, HTTPBadRequest
 
-from rhodecode.lib import helpers as h, diffs
+from rhodecode.lib import helpers as h, diffs, rc_cache
 from rhodecode.lib.utils2 import (
     StrictAttributeDict, str2bool, safe_int, datetime_to_time, safe_unicode)
+from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
+from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import RepositoryRequirementError
 from rhodecode.model import repo
 from rhodecode.model import repo_group
@@ -36,6 +38,7 @@ from rhodecode.model import user
 from rhodecode.model.db import User
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel
+from rhodecode.model.repo import ReadmeFinder
 
 log = logging.getLogger(__name__)
 
@@ -222,6 +225,7 @@ class RepoAppView(BaseAppView):
         self.db_repo = request.db_repo
         self.db_repo_name = self.db_repo.repo_name
         self.db_repo_pull_requests = ScmModel().get_pull_requests(self.db_repo)
+        self.db_repo_artifacts = ScmModel().get_artifacts(self.db_repo)
 
     def _handle_missing_requirements(self, error):
         log.error(
@@ -237,6 +241,7 @@ class RepoAppView(BaseAppView):
         c.rhodecode_db_repo = self.db_repo
         c.repo_name = self.db_repo_name
         c.repository_pull_requests = self.db_repo_pull_requests
+        c.repository_artifacts = self.db_repo_artifacts
         c.repository_is_user_following = ScmModel().is_following_repo(
             self.db_repo_name, self._rhodecode_user.user_id)
         self.path_filter = PathFilter(None)
@@ -305,6 +310,69 @@ class RepoAppView(BaseAppView):
         settings = settings_model.get_general_settings()
         return settings.get(settings_key, default)
 
+    def _get_repo_setting(self, target_repo, settings_key, default=False):
+        settings_model = VcsSettingsModel(repo=target_repo)
+        settings = settings_model.get_repo_settings_inherited()
+        return settings.get(settings_key, default)
+
+    def _get_readme_data(self, db_repo, renderer_type, commit_id=None, path='/'):
+        log.debug('Looking for README file at path %s', path)
+        if commit_id:
+            landing_commit_id = commit_id
+        else:
+            landing_commit = db_repo.get_landing_commit()
+            if isinstance(landing_commit, EmptyCommit):
+                return None, None
+            landing_commit_id = landing_commit.raw_id
+
+        cache_namespace_uid = 'cache_repo.{}'.format(db_repo.repo_id)
+        region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
+        start = time.time()
+
+        @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
+        def generate_repo_readme(repo_id, _commit_id, _repo_name, _readme_search_path, _renderer_type):
+            readme_data = None
+            readme_filename = None
+
+            commit = db_repo.get_commit(_commit_id)
+            log.debug("Searching for a README file at commit %s.", _commit_id)
+            readme_node = ReadmeFinder(_renderer_type).search(commit, path=_readme_search_path)
+
+            if readme_node:
+                log.debug('Found README node: %s', readme_node)
+                relative_urls = {
+                    'raw': h.route_path(
+                        'repo_file_raw', repo_name=_repo_name,
+                        commit_id=commit.raw_id, f_path=readme_node.path),
+                    'standard': h.route_path(
+                        'repo_files', repo_name=_repo_name,
+                        commit_id=commit.raw_id, f_path=readme_node.path),
+                }
+                readme_data = self._render_readme_or_none(commit, readme_node, relative_urls)
+                readme_filename = readme_node.unicode_path
+
+            return readme_data, readme_filename
+
+        readme_data, readme_filename = generate_repo_readme(
+            db_repo.repo_id, landing_commit_id, db_repo.repo_name, path, renderer_type,)
+        compute_time = time.time() - start
+        log.debug('Repo README for path %s generated and computed in %.4fs',
+                  path, compute_time)
+        return readme_data, readme_filename
+
+    def _render_readme_or_none(self, commit, readme_node, relative_urls):
+        log.debug('Found README file `%s` rendering...', readme_node.path)
+        renderer = MarkupRenderer()
+        try:
+            html_source = renderer.render(
+                readme_node.content, filename=readme_node.path)
+            if relative_urls:
+                return relative_links(html_source, relative_urls)
+            return html_source
+        except Exception:
+            log.exception(
+                "Exception while trying to render the README")
+
     def get_recache_flag(self):
         for flag_name in ['force_recache', 'force-recache', 'no-cache']:
             flag_val = self.request.GET.get(flag_name)
@@ -464,7 +532,7 @@ class BaseReferencesView(RepoAppView):
 
     def load_refs_context(self, ref_items, partials_template):
         _render = self.request.get_partial_renderer(partials_template)
-        pre_load = ["author", "date", "message"]
+        pre_load = ["author", "date", "message", "parents"]
 
         is_svn = h.is_svn(self.rhodecode_vcs_repo)
         is_hg = h.is_hg(self.rhodecode_vcs_repo)

rhodecode/apps/admin/__init__.py

@@ -140,7 +140,6 @@ def admin_routes(config):
         name='admin_settings_visual_update',
         pattern='/settings/visual/update')
 
-
     config.add_route(
         name='admin_settings_issuetracker',
         pattern='/settings/issue-tracker')
@@ -378,6 +377,10 @@ def admin_routes(config):
         name='edit_user_audit_logs',
         pattern='/users/{user_id:\d+}/edit/audit', user_route=True)
 
+    config.add_route(
+        name='edit_user_audit_logs_download',
+        pattern='/users/{user_id:\d+}/edit/audit/download', user_route=True)
+
     # user caches
     config.add_route(
         name='edit_user_caches',
@@ -410,6 +413,10 @@ def admin_routes(config):
         name='repos',
         pattern='/repos')
 
+    config.add_route(
+        name='repos_data',
+        pattern='/repos_data')
+
     config.add_route(
         name='repo_new',
         pattern='/repos/new')

rhodecode/apps/admin/tests/test_admin_auth_settings.py

@@ -155,7 +155,7 @@ class TestAuthSettingsView(object):
         response = self._post_ldap_settings(params, override={
             'port': invalid_port_value,
         })
-        assertr = AssertResponse(response)
+        assertr = response.assert_response()
         assertr.element_contains(
             '.form .field #port ~ .error-message',
             invalid_port_value)

rhodecode/apps/admin/tests/test_admin_repos.py

@@ -47,6 +47,7 @@ def route_path(name, params=None, **kwargs):
 
     base_url = {
         'repos': ADMIN_PREFIX + '/repos',
+        'repos_data': ADMIN_PREFIX + '/repos_data',
         'repo_new': ADMIN_PREFIX + '/repos/new',
         'repo_create': ADMIN_PREFIX + '/repos/create',
 
@@ -70,24 +71,25 @@ def _get_permission_for_user(user, repo):
 @pytest.mark.usefixtures("app")
 class TestAdminRepos(object):
 
-    def test_repo_list(self, autologin_user, user_util):
+    def test_repo_list(self, autologin_user, user_util, xhr_header):
         repo = user_util.create_repo()
         repo_name = repo.repo_name
         response = self.app.get(
-            route_path('repos'), status=200)
+            route_path('repos_data'), status=200,
+            extra_environ=xhr_header)
 
         response.mustcontain(repo_name)
 
     def test_create_page_restricted_to_single_backend(self, autologin_user, backend):
         with mock.patch('rhodecode.BACKENDS', {'git': 'git'}):
             response = self.app.get(route_path('repo_new'), status=200)
-        assert_response = AssertResponse(response)
+        assert_response = response.assert_response()
         element = assert_response.get_element('#repo_type')
         assert element.text_content() == '\ngit\n'
 
     def test_create_page_non_restricted_backends(self, autologin_user, backend):
         response = self.app.get(route_path('repo_new'), status=200)
-        assert_response = AssertResponse(response)
+        assert_response = response.assert_response()
         assert_response.element_contains('#repo_type', 'git')
         assert_response.element_contains('#repo_type', 'svn')
         assert_response.element_contains('#repo_type', 'hg')

rhodecode/apps/admin/tests/test_admin_repository_groups.py

@@ -84,7 +84,7 @@ class TestAdminRepositoryGroups(object):
         fixture.create_repo_group('test_repo_group')
         response = self.app.get(route_path(
             'repo_groups_data'), extra_environ=xhr_header)
-        response.mustcontain('"name_raw": "test_repo_group"')
+        response.mustcontain('<a href=\\"/{}/_edit\\" title=\\"Edit\\">Edit</a>'.format('test_repo_group'))
         fixture.destroy_repo_group('test_repo_group')
 
     def test_new(self, autologin_user):

rhodecode/apps/admin/tests/test_admin_settings.py

@@ -367,7 +367,7 @@ class TestAdminSettingsVcs(object):
 
     def test_has_an_input_for_invalidation_of_inline_comments(self):
         response = self.app.get(route_path('admin_settings_vcs'))
-        assert_response = AssertResponse(response)
+        assert_response = response.assert_response()
         assert_response.one_element_exists(
             '[name=rhodecode_use_outdated_comments]')
 
@@ -412,14 +412,14 @@ class TestAdminSettingsVcs(object):
         setting = SettingsModel().get_setting_by_name(setting_key)
         assert setting.app_settings_value is new_value
 
-    @pytest.fixture
+    @pytest.fixture()
     def disable_sql_cache(self, request):
         patcher = mock.patch(
             'rhodecode.lib.caching_query.FromCache.process_query')
         request.addfinalizer(patcher.stop)
         patcher.start()
 
-    @pytest.fixture
+    @pytest.fixture()
     def form_defaults(self):
         from rhodecode.apps.admin.views.settings import AdminSettingsView
         return AdminSettingsView._form_defaults()
@@ -518,7 +518,7 @@ class TestOpenSourceLicenses(object):
             response = self.app.get(
                 route_path('admin_settings_open_source'), status=200)
 
-        assert_response = AssertResponse(response)
+        assert_response = response.assert_response()
         assert_response.element_contains(
             '.panel-heading', 'Licenses of Third Party Packages')
         for license_data in sample_licenses:
@@ -528,7 +528,7 @@ class TestOpenSourceLicenses(object):
     def test_records_can_be_read(self, autologin_user):
         response = self.app.get(
             route_path('admin_settings_open_source'), status=200)
-        assert_response = AssertResponse(response)
+        assert_response = response.assert_response()
         assert_response.element_contains(
             '.panel-heading', 'Licenses of Third Party Packages')
 
@@ -726,7 +726,7 @@ class TestAdminSettingsIssueTracker(object):
             IssueTrackerSettingsModel().delete_entries(self.uid)
 
     def test_delete_issuetracker_pattern(
-            self, autologin_user, backend, csrf_token, settings_util):
+            self, autologin_user, backend, csrf_token, settings_util, xhr_header):
         pattern = 'issuetracker_pat'
         uid = md5(pattern)
         settings_util.create_rhodecode_setting(
@@ -734,10 +734,9 @@ class TestAdminSettingsIssueTracker(object):
 
         post_url = route_path('admin_settings_issuetracker_delete')
         post_data = {
-            '_method': 'delete',
             'uid': uid,
             'csrf_token': csrf_token
         }
-        self.app.post(post_url, post_data, status=302)
+        self.app.post(post_url, post_data, extra_environ=xhr_header, status=200)
         settings = SettingsModel().get_all_settings()
         assert 'rhodecode_%s%s' % (self.SHORT_PATTERN_KEY, uid) not in settings

rhodecode/apps/admin/tests/test_admin_users.py

@@ -91,6 +91,9 @@ def route_path(name, params=None, **kwargs):
         'edit_user_audit_logs':
             ADMIN_PREFIX + '/users/{user_id}/edit/audit',
 
+        'edit_user_audit_logs_download':
+            ADMIN_PREFIX + '/users/{user_id}/edit/audit/download',
+
     }[name].format(**kwargs)
 
     if params:
@@ -318,7 +321,6 @@ class TestAdminUsersView(TestController):
             route_path('edit_user_emails', user_id=user_id))
         response.mustcontain(no=['example@rhodecode.com'])
 
-
     def test_create(self, request, xhr_header):
         self.log_user()
         username = 'newtestuser'
@@ -333,6 +335,7 @@ class TestAdminUsersView(TestController):
         response = self.app.post(route_path('users_create'), params={
             'username': username,
             'password': password,
+            'description': 'mr CTO',
             'password_confirmation': password_confirmation,
             'firstname': name,
             'active': True,
@@ -381,6 +384,7 @@ class TestAdminUsersView(TestController):
             'name': name,
             'active': False,
             'lastname': lastname,
+            'description': 'mr CTO',
             'email': email,
             'csrf_token': self.csrf_token,
         })
@@ -418,6 +422,7 @@ class TestAdminUsersView(TestController):
         ('email', {'email': 'some@email.com'}),
         ('language', {'language': 'de'}),
         ('language', {'language': 'en'}),
+        ('description', {'description': 'hello CTO'}),
         # ('new_password', {'new_password': 'foobar123',
         #                   'password_confirmation': 'foobar123'})
         ])
@@ -515,7 +520,7 @@ class TestAdminUsersView(TestController):
             route_path('user_delete', user_id=new_user.user_id),
             params={'csrf_token': self.csrf_token})
 
-        assert_session_flash(response, 'Successfully deleted user')
+        assert_session_flash(response, 'Successfully deleted user `{}`'.format(username))
 
     def test_delete_owner_of_repository(self, request, user_util):
         self.log_user()
@@ -531,8 +536,7 @@ class TestAdminUsersView(TestController):
             params={'csrf_token': self.csrf_token})
 
         msg = 'user "%s" still owns 1 repositories and cannot be removed. ' \
-              'Switch owners or remove those repositories:%s' % (username,
-                                                                 obj_name)
+              'Switch owners or remove those repositories:%s' % (username, obj_name)
         assert_session_flash(response, msg)
         fixture.destroy_repo(obj_name)
 
@@ -542,6 +546,7 @@ class TestAdminUsersView(TestController):
         usr = user_util.create_user(auto_cleanup=False)
         username = usr.username
         fixture.create_repo(obj_name, cur_user=usr.username)
+        Session().commit()
 
         new_user = Session().query(User)\
             .filter(User.username == username).one()
@@ -583,8 +588,7 @@ class TestAdminUsersView(TestController):
             params={'csrf_token': self.csrf_token})
 
         msg = 'user "%s" still owns 1 repository groups and cannot be removed. ' \
-              'Switch owners or remove those repository groups:%s' % (username,
-                                                                      obj_name)
+              'Switch owners or remove those repository groups:%s' % (username, obj_name)
         assert_session_flash(response, msg)
         fixture.destroy_repo_group(obj_name)
 
@@ -635,8 +639,7 @@ class TestAdminUsersView(TestController):
             params={'csrf_token': self.csrf_token})
 
         msg = 'user "%s" still owns 1 user groups and cannot be removed. ' \
-              'Switch owners or remove those user groups:%s' % (username,
-                                                                obj_name)
+              'Switch owners or remove those user groups:%s' % (username, obj_name)
         assert_session_flash(response, msg)
         fixture.destroy_user_group(obj_name)
 
@@ -779,3 +782,13 @@ class TestAdminUsersView(TestController):
         user = self.log_user()
         self.app.get(
             route_path('edit_user_audit_logs', user_id=user['user_id']))
+
+    def test_audit_log_page_download(self):
+        user = self.log_user()
+        user_id = user['user_id']
+        response = self.app.get(
+            route_path('edit_user_audit_logs_download', user_id=user_id))
+
+        assert response.content_disposition == \
+               'attachment; filename=user_{}_audit_logs.json'.format(user_id)
+        assert response.content_type == "application/json"

rhodecode/apps/admin/views/audit_logs.py

@@ -28,7 +28,7 @@ from rhodecode.model.db import joinedload, UserLog
 from rhodecode.lib.user_log_filter import user_log_filter
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
 from rhodecode.lib.utils2 import safe_int
-from rhodecode.lib.helpers import Page
+from rhodecode.lib.helpers import SqlPage
 
 log = logging.getLogger(__name__)
 
@@ -62,13 +62,16 @@ class AdminAuditLogsView(BaseAppView):
 
         p = safe_int(self.request.GET.get('page', 1), 1)
 
-        def url_generator(**kw):
+        def url_generator(page_num):
+            query_params = {
+                'page': page_num
+            }
             if c.search_term:
-                kw['filter'] = c.search_term
-            return self.request.current_route_path(_query=kw)
+                query_params['filter'] = c.search_term
+            return self.request.current_route_path(_query=query_params)
 
-        c.audit_logs = Page(users_log, page=p, items_per_page=10,
-                            url=url_generator)
+        c.audit_logs = SqlPage(users_log, page=p, items_per_page=10,
+                               url_maker=url_generator)
         return self._get_template_context(c)
 
     @LoginRequired()

rhodecode/apps/admin/views/main_views.py

@@ -25,7 +25,7 @@ from pyramid.view import view_config
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.lib import helpers as h
-from rhodecode.lib.auth import (LoginRequired, NotAnonymous)
+from rhodecode.lib.auth import (LoginRequired, NotAnonymous, HasRepoPermissionAny)
 from rhodecode.model.db import PullRequest
 
 
@@ -66,6 +66,13 @@ class AdminMainView(BaseAppView):
         pull_request_id = pull_request.pull_request_id
 
         repo_name = pull_request.target_repo.repo_name
+        # NOTE(marcink):
+        # check permissions so we don't redirect to repo that we don't have access to
+        # exposing it's name
+        target_repo_perm = HasRepoPermissionAny(
+            'repository.read', 'repository.write', 'repository.admin')(repo_name)
+        if not target_repo_perm:
+            raise HTTPNotFound()
 
         raise HTTPFound(
             h.route_path('pullrequest_show', repo_name=repo_name,

rhodecode/apps/admin/views/permissions.py

@@ -68,7 +68,8 @@ class AdminPermissionsView(BaseAppView, DataGridAppView):
 
         c.user = User.get_default_user(refresh=True)
 
-        app_settings = SettingsModel().get_all_settings()
+        app_settings = c.rc_config
+
         defaults = {
             'anonymous': c.user.active,
             'default_register_message': app_settings.get(

rhodecode/apps/admin/views/process_management.py

@@ -47,7 +47,7 @@ class AdminProcessManagementView(BaseAppView):
                 'name': proc.name(),
                 'mem_rss': mem.rss,
                 'mem_vms': mem.vms,
-                'cpu_percent': proc.cpu_percent(),
+                'cpu_percent': proc.cpu_percent(interval=0.1),
                 'create_time': proc.create_time(),
                 'cmd': ' '.join(proc.cmdline()),
             })

rhodecode/apps/admin/views/repo_groups.py

@@ -19,6 +19,8 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import datetime
 import logging
+import time
+
 import formencode
 import formencode.htmlfill
 
@@ -63,7 +65,7 @@ class AdminRepoGroupsView(BaseAppView, DataGridAppView):
         # and display only those we have ADMIN right
         groups_with_admin_rights = RepoGroupList(
             RepoGroup.query().all(),
-            perm_set=['group.admin'])
+            perm_set=['group.admin'], extra_kwargs=dict(user=self._rhodecode_user))
         c.repo_groups = RepoGroup.groups_choices(
             groups=groups_with_admin_rights,
             show_empty_group=allow_empty_group)
@@ -109,9 +111,9 @@ class AdminRepoGroupsView(BaseAppView, DataGridAppView):
     def repo_group_list_data(self):
         self.load_default_context()
         column_map = {
-            'name_raw': 'group_name_hash',
+            'name': 'group_name_hash',
             'desc': 'group_description',
-            'last_change_raw': 'updated_on',
+            'last_change': 'updated_on',
             'top_level_repos': 'repos_total',
             'owner': 'user_username',
         }
@@ -131,9 +133,10 @@ class AdminRepoGroupsView(BaseAppView, DataGridAppView):
 
         def last_change(last_change):
             if isinstance(last_change, datetime.datetime) and not last_change.tzinfo:
-                delta = datetime.timedelta(
-                    seconds=(datetime.datetime.now() - datetime.datetime.utcnow()).seconds)
-                last_change = last_change + delta
+                ts = time.time()
+                utc_offset = (datetime.datetime.fromtimestamp(ts)
+                              - datetime.datetime.utcfromtimestamp(ts)).total_seconds()
+                last_change = last_change + datetime.timedelta(seconds=utc_offset)
             return _render("last_change", last_change)
 
         def desc(desc, personal):
@@ -147,12 +150,8 @@ class AdminRepoGroupsView(BaseAppView, DataGridAppView):
         def user_profile(username):
             return _render('user_profile', username)
 
-        auth_repo_group_list = RepoGroupList(
-            RepoGroup.query().all(), perm_set=['group.admin'])
-
-        allowed_ids = [-1]
-        for repo_group in auth_repo_group_list:
-                allowed_ids.append(repo_group.group_id)
+        _perms = ['group.admin']
+        allowed_ids = [-1] + self._rhodecode_user.repo_group_acl_ids_from_stack(_perms)
 
         repo_groups_data_total_count = RepoGroup.query()\
             .filter(or_(
@@ -180,7 +179,7 @@ class AdminRepoGroupsView(BaseAppView, DataGridAppView):
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(RepoGroup.group_id, allowed_ids)
             )) \
-            .outerjoin(Repository) \
+            .outerjoin(Repository,  Repository.group_id == RepoGroup.group_id) \
             .join(User, User.user_id == RepoGroup.user_id) \
             .group_by(RepoGroup, User)
 
@@ -224,9 +223,8 @@ class AdminRepoGroupsView(BaseAppView, DataGridAppView):
             row = {
                 "menu": quick_menu(repo_gr.group_name),
                 "name": repo_group_lnk(repo_gr.group_name),
-                "name_raw": repo_gr.group_name,
+
                 "last_change": last_change(repo_gr.updated_on),
-                "last_change_raw": datetime_to_time(repo_gr.updated_on),
 
                 "last_changeset": "",
                 "last_changeset_raw": "",

rhodecode/apps/admin/views/repositories.py

@@ -31,7 +31,6 @@ from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView
 from rhodecode.lib.celerylib.utils import get_task_id
 
-from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, CSRFRequired, NotAnonymous,
     HasPermissionAny, HasRepoGroupPermissionAny)
@@ -43,7 +42,8 @@ from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.scm import RepoList, RepoGroupList, ScmModel
 from rhodecode.model.settings import SettingsModel
-from rhodecode.model.db import Repository, RepoGroup
+from rhodecode.model.db import (
+    in_filter_generator, or_, func, Session, Repository, RepoGroup, User)
 
 log = logging.getLogger(__name__)
 
@@ -60,8 +60,6 @@ class AdminReposView(BaseAppView, DataGridAppView):
                                    perm_set=['group.write', 'group.admin'])
         c.repo_groups = RepoGroup.groups_choices(groups=acl_groups)
         c.repo_groups_choices = map(lambda k: safe_unicode(k[0]), c.repo_groups)
-        c.landing_revs_choices, c.landing_revs = \
-            ScmModel().get_repo_landing_revs(self.request.translate)
         c.personal_repo_group = self._rhodecode_user.personal_repo_group
 
     @LoginRequired()
@@ -72,16 +70,95 @@ class AdminReposView(BaseAppView, DataGridAppView):
         renderer='rhodecode:templates/admin/repos/repos.mako')
     def repository_list(self):
         c = self.load_default_context()
-
-        repo_list = Repository.get_all_repos()
-        c.repo_list = RepoList(repo_list, perm_set=['repository.admin'])
-        repos_data = RepoModel().get_repos_as_dict(
-            repo_list=c.repo_list, admin=True, super_user_actions=True)
-        # json used to render the grid
-        c.data = json.dumps(repos_data)
-
         return self._get_template_context(c)
 
+    @LoginRequired()
+    @NotAnonymous()
+    # perms check inside
+    @view_config(
+        route_name='repos_data', request_method='GET',
+        renderer='json_ext', xhr=True)
+    def repository_list_data(self):
+        self.load_default_context()
+        column_map = {
+            'name': 'repo_name',
+            'desc': 'description',
+            'last_change': 'updated_on',
+            'owner': 'user_username',
+        }
+        draw, start, limit = self._extract_chunk(self.request)
+        search_q, order_by, order_dir = self._extract_ordering(
+            self.request, column_map=column_map)
+
+        _perms = ['repository.admin']
+        allowed_ids = [-1] + self._rhodecode_user.repo_acl_ids_from_stack(_perms)
+
+        repos_data_total_count = Repository.query() \
+            .filter(or_(
+                # generate multiple IN to fix limitation problems
+                *in_filter_generator(Repository.repo_id, allowed_ids))
+            ) \
+            .count()
+
+        base_q = Session.query(
+            Repository.repo_id,
+            Repository.repo_name,
+            Repository.description,
+            Repository.repo_type,
+            Repository.repo_state,
+            Repository.private,
+            Repository.archived,
+            Repository.fork,
+            Repository.updated_on,
+            Repository._changeset_cache,
+            User,
+            ) \
+            .filter(or_(
+                # generate multiple IN to fix limitation problems
+                *in_filter_generator(Repository.repo_id, allowed_ids))
+            ) \
+            .join(User, User.user_id == Repository.user_id) \
+            .group_by(Repository, User)
+
+        if search_q:
+            like_expression = u'%{}%'.format(safe_unicode(search_q))
+            base_q = base_q.filter(or_(
+                Repository.repo_name.ilike(like_expression),
+            ))
+
+        repos_data_total_filtered_count = base_q.count()
+
+        sort_defined = False
+        if order_by == 'repo_name':
+            sort_col = func.lower(Repository.repo_name)
+            sort_defined = True
+        elif order_by == 'user_username':
+            sort_col = User.username
+        else:
+            sort_col = getattr(Repository, order_by, None)
+
+        if sort_defined or sort_col:
+            if order_dir == 'asc':
+                sort_col = sort_col.asc()
+            else:
+                sort_col = sort_col.desc()
+
+        base_q = base_q.order_by(sort_col)
+        base_q = base_q.offset(start).limit(limit)
+
+        repos_list = base_q.all()
+
+        repos_data = RepoModel().get_repos_as_dict(
+            repo_list=repos_list, admin=True, super_user_actions=True)
+
+        data = ({
+            'draw': draw,
+            'data': repos_data,
+            'recordsTotal': repos_data_total_count,
+            'recordsFiltered': repos_data_total_filtered_count,
+        })
+        return data
+
     @LoginRequired()
     @NotAnonymous()
     # perms check inside
@@ -151,8 +228,7 @@ class AdminReposView(BaseAppView, DataGridAppView):
         try:
             # CanWriteToGroup validators checks permissions of this POST
             form = RepoForm(
-                self.request.translate, repo_groups=c.repo_groups_choices,
-                landing_revs=c.landing_revs_choices)()
+                self.request.translate, repo_groups=c.repo_groups_choices)()
             form_result = form.to_python(dict(self.request.POST))
             copy_permissions = form_result.get('repo_copy_permissions')
             # create is done sometimes async on celery, db transaction

rhodecode/apps/admin/views/settings.py

@@ -445,7 +445,7 @@ class AdminSettingsView(BaseAppView):
     def settings_issuetracker(self):
         c = self.load_default_context()
         c.active = 'issuetracker'
-        defaults = SettingsModel().get_all_settings()
+        defaults = c.rc_config
 
         entry_key = 'rhodecode_issuetracker_pat_'
 
@@ -518,7 +518,7 @@ class AdminSettingsView(BaseAppView):
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_issuetracker_delete', request_method='POST',
-        renderer='rhodecode:templates/admin/settings/settings.mako')
+        renderer='json_ext', xhr=True)
     def settings_issuetracker_delete(self):
         _ = self.request.translate
         self.load_default_context()
@@ -528,8 +528,11 @@ class AdminSettingsView(BaseAppView):
         except Exception:
             log.exception('Failed to delete issue tracker setting %s', uid)
             raise HTTPNotFound()
-        h.flash(_('Removed issue tracker entry'), category='success')
-        raise HTTPFound(h.route_path('admin_settings_issuetracker'))
+
+        SettingsModel().invalidate_settings_cache()
+        h.flash(_('Removed issue tracker entry.'), category='success')
+
+        return {'deleted': uid}
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
@@ -570,8 +573,7 @@ class AdminSettingsView(BaseAppView):
 
         email_kwargs = {
             'date': datetime.datetime.now(),
-            'user': c.rhodecode_user,
-            'rhodecode_version': c.rhodecode_version
+            'user': c.rhodecode_user
         }
 
         (subject, headers, email_body,

rhodecode/apps/admin/views/system_info.py

@@ -155,6 +155,10 @@ class AdminSystemInfoSettingsView(BaseAppView):
 
         ]
 
+        c.vcsserver_data_items = [
+            (k, v) for k,v in (val('vcs_server_config') or {}).items()
+        ]
+
         if snapshot:
             if c.allowed_to_snapshot:
                 c.data_items.pop(0)  # remove server info

rhodecode/apps/admin/views/user_groups.py

@@ -99,12 +99,8 @@ class AdminUserGroupsView(BaseAppView, DataGridAppView):
         def user_profile(username):
             return _render('user_profile', username)
 
-        auth_user_group_list = UserGroupList(
-            UserGroup.query().all(), perm_set=['usergroup.admin'])
-
-        allowed_ids = [-1]
-        for user_group in auth_user_group_list:
-                allowed_ids.append(user_group.users_group_id)
+        _perms = ['usergroup.admin']
+        allowed_ids = [-1] + self._rhodecode_user.user_group_acl_ids_from_stack(_perms)
 
         user_groups_data_total_count = UserGroup.query()\
             .filter(or_(
@@ -134,7 +130,7 @@ class AdminUserGroupsView(BaseAppView, DataGridAppView):
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(UserGroup.users_group_id, allowed_ids)
             )) \
-            .outerjoin(UserGroupMember) \
+            .outerjoin(UserGroupMember, UserGroupMember.users_group_id == UserGroup.users_group_id) \
             .join(User, User.user_id == UserGroup.user_id) \
             .group_by(UserGroup, User)
 
@@ -175,7 +171,6 @@ class AdminUserGroupsView(BaseAppView, DataGridAppView):
         for user_gr in auth_user_group_list:
             row = {
                 "users_group_name": user_group_name(user_gr.users_group_name),
-                "name_raw": h.escape(user_gr.users_group_name),
                 "description": h.escape(user_gr.user_group_description),
                 "members": user_gr.member_count,
                 # NOTE(marcink): because of advanced query we

rhodecode/apps/admin/views/users.py

@@ -31,6 +31,7 @@ from pyramid.response import Response
 from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView, UserAppView
 from rhodecode.apps.ssh_support import SshKeyFileChangeEvent
+from rhodecode.authentication.base import get_authn_registry, RhodeCodeExternalAuthPlugin
 from rhodecode.authentication.plugins import auth_rhodecode
 from rhodecode.events import trigger
 from rhodecode.model.db import true
@@ -43,6 +44,7 @@ from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib import helpers as h
+from rhodecode.lib.helpers import SqlPage
 from rhodecode.lib.utils2 import safe_int, safe_unicode, AttributeDict
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.forms import (
@@ -249,7 +251,32 @@ class UsersView(UserAppView):
     in there as well.
     """
 
+    def get_auth_plugins(self):
+        valid_plugins = []
+        authn_registry = get_authn_registry(self.request.registry)
+        for plugin in authn_registry.get_plugins_for_authentication():
+            if isinstance(plugin, RhodeCodeExternalAuthPlugin):
+                valid_plugins.append(plugin)
+            elif plugin.name == 'rhodecode':
+                valid_plugins.append(plugin)
+
+        # extend our choices if user has set a bound plugin which isn't enabled at the
+        # moment
+        extern_type = self.db_user.extern_type
+        if extern_type not in [x.uid for x in valid_plugins]:
+            try:
+                plugin = authn_registry.get_plugin_by_uid(extern_type)
+                if plugin:
+                    valid_plugins.append(plugin)
+
+            except Exception:
+                log.exception(
+                    'Could not extend user plugins with `{}`'.format(extern_type))
+        return valid_plugins
+
     def load_default_context(self):
+        req = self.request
+
         c = self._get_local_tmpl_context()
         c.allow_scoped_tokens = self.ALLOW_SCOPED_TOKENS
         c.allowed_languages = [
@@ -263,7 +290,10 @@ class UsersView(UserAppView):
             ('ru', 'Russian (ru)'),
             ('zh', 'Chinese (zh)'),
         ]
-        req = self.request
+
+        c.allowed_extern_types = [
+            (x.uid, x.get_display_name()) for x in self.get_auth_plugins()
+        ]
 
         c.available_permissions = req.registry.settings['available_permissions']
         PermissionModel().set_global_permission_choices(
@@ -297,7 +327,7 @@ class UsersView(UserAppView):
         old_values = c.user.get_api_data()
         try:
             form_result = _form.to_python(dict(self.request.POST))
-            skip_attrs = ['extern_type', 'extern_name']
+            skip_attrs = ['extern_name']
             # TODO: plugin should define if username can be updated
             if c.extern_type != "rhodecode":
                 # forbid updating username for external accounts
@@ -347,59 +377,69 @@ class UsersView(UserAppView):
         _repos = c.user.repositories
         _repo_groups = c.user.repository_groups
         _user_groups = c.user.user_groups
+        _artifacts = c.user.artifacts
 
         handle_repos = None
         handle_repo_groups = None
         handle_user_groups = None
-        # dummy call for flash of handle
-        set_handle_flash_repos = lambda: None
-        set_handle_flash_repo_groups = lambda: None
-        set_handle_flash_user_groups = lambda: None
+        handle_artifacts = None
+
+        # calls for flash of handle based on handle case detach or delete
+        def set_handle_flash_repos():
+            handle = handle_repos
+            if handle == 'detach':
+                h.flash(_('Detached %s repositories') % len(_repos),
+                        category='success')
+            elif handle == 'delete':
+                h.flash(_('Deleted %s repositories') % len(_repos),
+                        category='success')
+
+        def set_handle_flash_repo_groups():
+            handle = handle_repo_groups
+            if handle == 'detach':
+                h.flash(_('Detached %s repository groups') % len(_repo_groups),
+                        category='success')
+            elif handle == 'delete':
+                h.flash(_('Deleted %s repository groups') % len(_repo_groups),
+                        category='success')
+
+        def set_handle_flash_user_groups():
+            handle = handle_user_groups
+            if handle == 'detach':
+                h.flash(_('Detached %s user groups') % len(_user_groups),
+                        category='success')
+            elif handle == 'delete':
+                h.flash(_('Deleted %s user groups') % len(_user_groups),
+                        category='success')
+
+        def set_handle_flash_artifacts():
+            handle = handle_artifacts
+            if handle == 'detach':
+                h.flash(_('Detached %s artifacts') % len(_artifacts),
+                        category='success')
+            elif handle == 'delete':
+                h.flash(_('Deleted %s artifacts') % len(_artifacts),
+                        category='success')
 
         if _repos and self.request.POST.get('user_repos'):
-            do = self.request.POST['user_repos']
-            if do == 'detach':
-                handle_repos = 'detach'
-                set_handle_flash_repos = lambda: h.flash(
-                    _('Detached %s repositories') % len(_repos),
-                    category='success')
-            elif do == 'delete':
-                handle_repos = 'delete'
-                set_handle_flash_repos = lambda: h.flash(
-                    _('Deleted %s repositories') % len(_repos),
-                    category='success')
+            handle_repos = self.request.POST['user_repos']
 
         if _repo_groups and self.request.POST.get('user_repo_groups'):
-            do = self.request.POST['user_repo_groups']
-            if do == 'detach':
-                handle_repo_groups = 'detach'
-                set_handle_flash_repo_groups = lambda: h.flash(
-                    _('Detached %s repository groups') % len(_repo_groups),
-                    category='success')
-            elif do == 'delete':
-                handle_repo_groups = 'delete'
-                set_handle_flash_repo_groups = lambda: h.flash(
-                    _('Deleted %s repository groups') % len(_repo_groups),
-                    category='success')
+            handle_repo_groups = self.request.POST['user_repo_groups']
 
         if _user_groups and self.request.POST.get('user_user_groups'):
-            do = self.request.POST['user_user_groups']
-            if do == 'detach':
-                handle_user_groups = 'detach'
-                set_handle_flash_user_groups = lambda: h.flash(
-                    _('Detached %s user groups') % len(_user_groups),
-                    category='success')
-            elif do == 'delete':
-                handle_user_groups = 'delete'
-                set_handle_flash_user_groups = lambda: h.flash(
-                    _('Deleted %s user groups') % len(_user_groups),
-                    category='success')
+            handle_user_groups = self.request.POST['user_user_groups']
+
+        if _artifacts and self.request.POST.get('user_artifacts'):
+            handle_artifacts = self.request.POST['user_artifacts']
 
         old_values = c.user.get_api_data()
+
         try:
             UserModel().delete(c.user, handle_repos=handle_repos,
                                handle_repo_groups=handle_repo_groups,
-                               handle_user_groups=handle_user_groups)
+                               handle_user_groups=handle_user_groups,
+                               handle_artifacts=handle_artifacts)
 
             audit_logger.store_web(
                 'user.delete', action_data={'old_data': old_values},
@@ -409,7 +449,9 @@ class UsersView(UserAppView):
             set_handle_flash_repos()
             set_handle_flash_repo_groups()
             set_handle_flash_user_groups()
-            h.flash(_('Successfully deleted user'), category='success')
+            set_handle_flash_artifacts()
+            username = h.escape(old_values['username'])
+            h.flash(_('Successfully deleted user `{}`').format(username), category='success')
         except (UserOwnsReposException, UserOwnsRepoGroupsException,
                 UserOwnsUserGroupsException, DefaultUserException) as e:
             h.flash(e, category='warning')
@@ -1187,16 +1229,42 @@ class UsersView(UserAppView):
         filter_term = self.request.GET.get('filter')
         user_log = UserModel().get_user_log(c.user, filter_term)
 
-        def url_generator(**kw):
+        def url_generator(page_num):
+            query_params = {
+                'page': page_num
+            }
             if filter_term:
-                kw['filter'] = filter_term
-            return self.request.current_route_path(_query=kw)
+                query_params['filter'] = filter_term
+            return self.request.current_route_path(_query=query_params)
 
-        c.audit_logs = h.Page(
-            user_log, page=p, items_per_page=10, url=url_generator)
+        c.audit_logs = SqlPage(
+            user_log, page=p, items_per_page=10, url_maker=url_generator)
         c.filter_term = filter_term
         return self._get_template_context(c)
 
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    @view_config(
+        route_name='edit_user_audit_logs_download', request_method='GET',
+        renderer='string')
+    def user_audit_logs_download(self):
+        _ = self.request.translate
+        c = self.load_default_context()
+        c.user = self.db_user
+
+        user_log = UserModel().get_user_log(c.user, filter_term=None)
+
+        audit_log_data = {}
+        for entry in user_log:
+            audit_log_data[entry.user_log_id] = entry.get_dict()
+
+        response = Response(json.dumps(audit_log_data, indent=4))
+        response.content_disposition = str(
+            'attachment; filename=%s' % 'user_{}_audit_logs.json'.format(c.user.user_id))
+        response.content_type = 'application/json'
+
+        return response
+
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(

rhodecode/apps/debug_style/__init__.py

@@ -42,6 +42,14 @@ def includeme(config):
         name='debug_style_home',
         pattern=ADMIN_PREFIX + '/debug_style',
         debug_style=True)
+    config.add_route(
+        name='debug_style_email',
+        pattern=ADMIN_PREFIX + '/debug_style/email/{email_id}',
+        debug_style=True)
+    config.add_route(
+        name='debug_style_email_plain_rendered',
+        pattern=ADMIN_PREFIX + '/debug_style/email-rendered/{email_id}',
+        debug_style=True)
     config.add_route(
         name='debug_style_template',
         pattern=ADMIN_PREFIX + '/debug_style/t/{t_path}',

rhodecode/apps/debug_style/views.py

@@ -20,10 +20,15 @@
 
 import os
 import logging
+import datetime
 
 from pyramid.view import view_config
 from pyramid.renderers import render_to_response
 from rhodecode.apps._base import BaseAppView
+from rhodecode.lib.celerylib import run_task, tasks
+from rhodecode.lib.utils2 import AttributeDict
+from rhodecode.model.db import User
+from rhodecode.model.notification import EmailNotificationModel
 
 log = logging.getLogger(__name__)
 
@@ -45,6 +50,317 @@ class DebugStyleView(BaseAppView):
             'debug_style/index.html', self._get_template_context(c),
             request=self.request)
 
+    @view_config(
+        route_name='debug_style_email', request_method='GET',
+        renderer=None)
+    @view_config(
+        route_name='debug_style_email_plain_rendered', request_method='GET',
+        renderer=None)
+    def render_email(self):
+        c = self.load_default_context()
+        email_id = self.request.matchdict['email_id']
+        c.active = 'emails'
+
+        pr = AttributeDict(
+            pull_request_id=123,
+            title='digital_ocean: fix redis, elastic search start on boot, '
+                  'fix fd limits on supervisor, set postgres 11 version',
+            description='''
+Check if we should use full-topic or mini-topic.
+
+- full topic produces some problems with merge states etc
+- server-mini-topic needs probably tweeks.            
+            ''',
+            repo_name='foobar',
+            source_ref_parts=AttributeDict(type='branch', name='fix-ticket-2000'),
+            target_ref_parts=AttributeDict(type='branch', name='master'),
+        )
+        target_repo = AttributeDict(repo_name='repo_group/target_repo')
+        source_repo = AttributeDict(repo_name='repo_group/source_repo')
+        user = User.get_by_username(self.request.GET.get('user')) or self._rhodecode_db_user
+        # file/commit changes for PR update
+        commit_changes = AttributeDict({
+            'added': ['aaaaaaabbbbb', 'cccccccddddddd'],
+            'removed': ['eeeeeeeeeee'],
+        })
+        file_changes = AttributeDict({
+            'added': ['a/file1.md', 'file2.py'],
+            'modified': ['b/modified_file.rst'],
+            'removed': ['.idea'],
+        })
+        email_kwargs = {
+            'test': {},
+            'message': {
+                'body': 'message body !'
+            },
+            'email_test': {
+                'user': user,
+                'date': datetime.datetime.now(),
+            },
+            'password_reset': {
+                'password_reset_url': 'http://example.com/reset-rhodecode-password/token',
+
+                'user': user,
+                'date': datetime.datetime.now(),
+                'email': 'test@rhodecode.com',
+                'first_admin_email': User.get_first_super_admin().email
+            },
+            'password_reset_confirmation': {
+                'new_password': 'new-password-example',
+                'user': user,
+                'date': datetime.datetime.now(),
+                'email': 'test@rhodecode.com',
+                'first_admin_email': User.get_first_super_admin().email
+            },
+            'registration': {
+                'user': user,
+                'date': datetime.datetime.now(),
+            },
+
+            'pull_request_comment': {
+                'user': user,
+
+                'status_change': None,
+                'status_change_type': None,
+
+                'pull_request': pr,
+                'pull_request_commits': [],
+
+                'pull_request_target_repo': target_repo,
+                'pull_request_target_repo_url': 'http://target-repo/url',
+
+                'pull_request_source_repo': source_repo,
+                'pull_request_source_repo_url': 'http://source-repo/url',
+
+                'pull_request_url': 'http://localhost/pr1',
+                'pr_comment_url': 'http://comment-url',
+                'pr_comment_reply_url': 'http://comment-url#reply',
+
+                'comment_file': None,
+                'comment_line': None,
+                'comment_type': 'note',
+                'comment_body': 'This is my comment body. *I like !*',
+                'comment_id': 2048,
+                'renderer_type': 'markdown',
+                'mention': True,
+
+            },
+            'pull_request_comment+status': {
+                'user': user,
+
+                'status_change': 'approved',
+                'status_change_type': 'approved',
+
+                'pull_request': pr,
+                'pull_request_commits': [],
+
+                'pull_request_target_repo': target_repo,
+                'pull_request_target_repo_url': 'http://target-repo/url',
+
+                'pull_request_source_repo': source_repo,
+                'pull_request_source_repo_url': 'http://source-repo/url',
+
+                'pull_request_url': 'http://localhost/pr1',
+                'pr_comment_url': 'http://comment-url',
+                'pr_comment_reply_url': 'http://comment-url#reply',
+
+                'comment_type': 'todo',
+                'comment_file': None,
+                'comment_line': None,
+                'comment_body': '''
+I think something like this would be better
+
+```py
+
+def db():
+   global connection
+   return connection
+
+```
+                
+                ''',
+                'comment_id': 2048,
+                'renderer_type': 'markdown',
+                'mention': True,
+
+            },
+            'pull_request_comment+file': {
+                'user': user,
+
+                'status_change': None,
+                'status_change_type': None,
+
+                'pull_request': pr,
+                'pull_request_commits': [],
+
+                'pull_request_target_repo': target_repo,
+                'pull_request_target_repo_url': 'http://target-repo/url',
+
+                'pull_request_source_repo': source_repo,
+                'pull_request_source_repo_url': 'http://source-repo/url',
+
+                'pull_request_url': 'http://localhost/pr1',
+
+                'pr_comment_url': 'http://comment-url',
+                'pr_comment_reply_url': 'http://comment-url#reply',
+
+                'comment_file': 'rhodecode/model/db.py',
+                'comment_line': 'o1210',
+                'comment_type': 'todo',
+                'comment_body': '''
+I like this !
+
+But please check this code::
+    
+    def main():
+        print 'ok'
+
+This should work better !
+                ''',
+                'comment_id': 2048,
+                'renderer_type': 'rst',
+                'mention': True,
+
+            },
+
+            'pull_request_update': {
+                'updating_user': user,
+
+                'status_change': None,
+                'status_change_type': None,
+
+                'pull_request': pr,
+                'pull_request_commits': [],
+
+                'pull_request_target_repo': target_repo,
+                'pull_request_target_repo_url': 'http://target-repo/url',
+
+                'pull_request_source_repo': source_repo,
+                'pull_request_source_repo_url': 'http://source-repo/url',
+
+                'pull_request_url': 'http://localhost/pr1',
+
+                # update comment links
+                'pr_comment_url': 'http://comment-url',
+                'pr_comment_reply_url': 'http://comment-url#reply',
+                'ancestor_commit_id': 'f39bd443',
+                'added_commits': commit_changes.added,
+                'removed_commits': commit_changes.removed,
+                'changed_files': (file_changes.added + file_changes.modified + file_changes.removed),
+                'added_files': file_changes.added,
+                'modified_files': file_changes.modified,
+                'removed_files': file_changes.removed,
+            },
+
+            'cs_comment': {
+                'user': user,
+                'commit': AttributeDict(idx=123, raw_id='a'*40, message='Commit message'),
+                'status_change': None,
+                'status_change_type': None,
+
+                'commit_target_repo_url': 'http://foo.example.com/#comment1',
+                'repo_name': 'test-repo',
+                'comment_type': 'note',
+                'comment_file': None,
+                'comment_line': None,
+                'commit_comment_url': 'http://comment-url',
+                'commit_comment_reply_url': 'http://comment-url#reply',
+                'comment_body': 'This is my comment body. *I like !*',
+                'comment_id': 2048,
+                'renderer_type': 'markdown',
+                'mention': True,
+            },
+            'cs_comment+status': {
+                'user': user,
+                'commit': AttributeDict(idx=123, raw_id='a' * 40, message='Commit message'),
+                'status_change': 'approved',
+                'status_change_type': 'approved',
+
+                'commit_target_repo_url': 'http://foo.example.com/#comment1',
+                'repo_name': 'test-repo',
+                'comment_type': 'note',
+                'comment_file': None,
+                'comment_line': None,
+                'commit_comment_url': 'http://comment-url',
+                'commit_comment_reply_url': 'http://comment-url#reply',
+                'comment_body': '''
+Hello **world**
+
+This is a multiline comment :)
+
+- list
+- list2
+                ''',
+                'comment_id': 2048,
+                'renderer_type': 'markdown',
+                'mention': True,
+            },
+            'cs_comment+file': {
+                'user': user,
+                'commit': AttributeDict(idx=123, raw_id='a' * 40, message='Commit message'),
+                'status_change': None,
+                'status_change_type': None,
+
+                'commit_target_repo_url': 'http://foo.example.com/#comment1',
+                'repo_name': 'test-repo',
+
+                'comment_type': 'note',
+                'comment_file': 'test-file.py',
+                'comment_line': 'n100',
+
+                'commit_comment_url': 'http://comment-url',
+                'commit_comment_reply_url': 'http://comment-url#reply',
+                'comment_body': 'This is my comment body. *I like !*',
+                'comment_id': 2048,
+                'renderer_type': 'markdown',
+                'mention': True,
+            },
+            
+            'pull_request': {
+                'user': user,
+                'pull_request': pr,
+                'pull_request_commits': [
+                    ('472d1df03bf7206e278fcedc6ac92b46b01c4e21', '''\
+my-account: moved email closer to profile as it's similar data just moved outside.                    
+                    '''),
+                    ('cbfa3061b6de2696c7161ed15ba5c6a0045f90a7', '''\
+users: description edit fixes
+
+- tests
+- added metatags info                    
+                    '''),
+                ],
+
+                'pull_request_target_repo': target_repo,
+                'pull_request_target_repo_url': 'http://target-repo/url',
+
+                'pull_request_source_repo': source_repo,
+                'pull_request_source_repo_url': 'http://source-repo/url',
+
+                'pull_request_url': 'http://code.rhodecode.com/_pull-request/123',
+            }
+
+        }
+
+        template_type = email_id.split('+')[0]
+        (c.subject, c.headers, c.email_body,
+         c.email_body_plaintext) = EmailNotificationModel().render_email(
+            template_type, **email_kwargs.get(email_id, {}))
+
+        test_email = self.request.GET.get('email')
+        if test_email:
+            recipients = [test_email]
+            run_task(tasks.send_email, recipients, c.subject,
+                     c.email_body_plaintext, c.email_body)
+
+        if self.request.matched_route.name == 'debug_style_email_plain_rendered':
+            template = 'debug_style/email_plain_rendered.mako'
+        else:
+            template = 'debug_style/email.mako'
+        return render_to_response(
+            template, self._get_template_context(c),
+            request=self.request)
+
     @view_config(
         route_name='debug_style_template', request_method='GET',
         renderer=None)
@@ -53,7 +369,18 @@ class DebugStyleView(BaseAppView):
         c = self.load_default_context()
         c.active = os.path.splitext(t_path)[0]
         c.came_from = ''
+        c.email_types = {
+            'cs_comment+file': {},
+            'cs_comment+status': {},
+
+            'pull_request_comment+file': {},
+            'pull_request_comment+status': {},
+
+            'pull_request_update': {},
+        }
+        c.email_types.update(EmailNotificationModel.email_types)
 
         return render_to_response(
             'debug_style/' + t_path, self._get_template_context(c),
-            request=self.request)
+            request=self.request)
+

rhodecode/apps/file_store/__init__.py

@@ -44,6 +44,9 @@ def includeme(config):
     config.add_route(
         name='download_file',
         pattern='/_file_store/download/{fid}')
+    config.add_route(
+        name='download_file_by_token',
+        pattern='/_file_store/token-download/{_auth_token}/{fid}')
 
     # Scan module for configuration decorators.
     config.scan('.views', ignore='.tests')

rhodecode/apps/file_store/backends/__init__.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2016-2019 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/

rhodecode/apps/file_store/backends/local_store.py

@@ -26,7 +26,8 @@ import hashlib
 from rhodecode.lib.ext_json import json
 from rhodecode.apps.file_store import utils
 from rhodecode.apps.file_store.extensions import resolve_extensions
-from rhodecode.apps.file_store.exceptions import FileNotAllowedException
+from rhodecode.apps.file_store.exceptions import (
+    FileNotAllowedException, FileOverSizeException)
 
 METADATA_VER = 'v1'
 
@@ -91,6 +92,9 @@ class LocalFileStorage(object):
         self.base_path = base_path
         self.extensions = resolve_extensions([], groups=extension_groups)
 
+    def __repr__(self):
+        return '{}@{}'.format(self.__class__, self.base_path)
+
     def store_path(self, filename):
         """
         Returns absolute file path of the filename, joined to the
@@ -140,16 +144,21 @@ class LocalFileStorage(object):
         :param ext: extension to check
         :param extensions: iterable of extensions to validate against (or self.extensions)
         """
+        def normalize_ext(_ext):
+            if _ext.startswith('.'):
+                _ext = _ext[1:]
+            return _ext.lower()
 
         extensions = extensions or self.extensions
         if not extensions:
             return True
-        if ext.startswith('.'):
-            ext = ext[1:]
-        return ext.lower() in extensions
+
+        ext = normalize_ext(ext)
+
+        return ext in [normalize_ext(x) for x in extensions]
 
     def save_file(self, file_obj, filename, directory=None, extensions=None,
-                  extra_metadata=None, **kwargs):
+                  extra_metadata=None, max_filesize=None, **kwargs):
         """
         Saves a file object to the uploads location.
         Returns the resolved filename, i.e. the directory +
@@ -159,7 +168,9 @@ class LocalFileStorage(object):
         :param filename: original filename
         :param directory: relative path of sub-directory
         :param extensions: iterable of allowed extensions, if not default
+        :param max_filesize: maximum size of file that should be allowed
         :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
+
         """
 
         extensions = extensions or self.extensions
@@ -191,6 +202,12 @@ class LocalFileStorage(object):
             metadata = extra_metadata
 
         size = os.stat(path).st_size
+
+        if max_filesize and size > max_filesize:
+            # free up the copied file, and raise exc
+            os.remove(path)
+            raise FileOverSizeException()
+
         file_hash = self.calculate_path_hash(path)
 
         metadata.update(

rhodecode/apps/file_store/tests/test_upload_file.py

@@ -21,7 +21,8 @@ import os
 import pytest
 
 from rhodecode.lib.ext_json import json
-from rhodecode.model.db import Session, FileStore
+from rhodecode.model.auth_token import AuthTokenModel
+from rhodecode.model.db import Session, FileStore, Repository, User
 from rhodecode.tests import TestController
 from rhodecode.apps.file_store import utils, config_keys
 
@@ -32,6 +33,7 @@ def route_path(name, params=None, **kwargs):
     base_url = {
         'upload_file': '/_file_store/upload',
         'download_file': '/_file_store/download/{fid}',
+        'download_file_by_token': '/_file_store/token-download/{_auth_token}/{fid}'
 
     }[name].format(**kwargs)
 
@@ -124,3 +126,136 @@ class TestFileStoreViews(TestController):
             status=200)
 
         assert response.json['store_fid']
+
+    @pytest.fixture()
+    def create_artifact_factory(self, tmpdir):
+        def factory(user_id, content):
+            store_path = self.app._pyramid_settings[config_keys.store_path]
+            store = utils.get_file_storage({config_keys.store_path: store_path})
+            fid = 'example.txt'
+
+            filesystem_file = os.path.join(str(tmpdir), fid)
+            with open(filesystem_file, 'wb') as f:
+                f.write(content)
+
+            with open(filesystem_file, 'rb') as f:
+                store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
+
+            entry = FileStore.create(
+                file_uid=store_uid, filename=metadata["filename"],
+                file_hash=metadata["sha256"], file_size=metadata["size"],
+                file_display_name='file_display_name',
+                file_description='repo artifact `{}`'.format(metadata["filename"]),
+                check_acl=True, user_id=user_id,
+            )
+            Session().add(entry)
+            Session().commit()
+            return entry
+        return factory
+
+    def test_download_file_non_scoped(self, user_util, create_artifact_factory):
+        user = self.log_user()
+        user_id = user['user_id']
+        content = 'HELLO MY NAME IS ARTIFACT !'
+
+        artifact = create_artifact_factory(user_id, content)
+        file_uid = artifact.file_uid
+        response = self.app.get(route_path('download_file', fid=file_uid), status=200)
+        assert response.text == content
+
+        # log-in to new user and test download again
+        user = user_util.create_user(password='qweqwe')
+        self.log_user(user.username, 'qweqwe')
+        response = self.app.get(route_path('download_file', fid=file_uid), status=200)
+        assert response.text == content
+
+    def test_download_file_scoped_to_repo(self, user_util, create_artifact_factory):
+        user = self.log_user()
+        user_id = user['user_id']
+        content = 'HELLO MY NAME IS ARTIFACT !'
+
+        artifact = create_artifact_factory(user_id, content)
+        # bind to repo
+        repo = user_util.create_repo()
+        repo_id = repo.repo_id
+        artifact.scope_repo_id = repo_id
+        Session().add(artifact)
+        Session().commit()
+
+        file_uid = artifact.file_uid
+        response = self.app.get(route_path('download_file', fid=file_uid), status=200)
+        assert response.text == content
+
+        # log-in to new user and test download again
+        user = user_util.create_user(password='qweqwe')
+        self.log_user(user.username, 'qweqwe')
+        response = self.app.get(route_path('download_file', fid=file_uid), status=200)
+        assert response.text == content
+
+        # forbid user the rights to repo
+        repo = Repository.get(repo_id)
+        user_util.grant_user_permission_to_repo(repo, user, 'repository.none')
+        self.app.get(route_path('download_file', fid=file_uid), status=404)
+
+    def test_download_file_scoped_to_user(self, user_util, create_artifact_factory):
+        user = self.log_user()
+        user_id = user['user_id']
+        content = 'HELLO MY NAME IS ARTIFACT !'
+
+        artifact = create_artifact_factory(user_id, content)
+        # bind to user
+        user = user_util.create_user(password='qweqwe')
+
+        artifact.scope_user_id = user.user_id
+        Session().add(artifact)
+        Session().commit()
+
+        # artifact creator doesn't have access since it's bind to another user
+        file_uid = artifact.file_uid
+        self.app.get(route_path('download_file', fid=file_uid), status=404)
+
+        # log-in to new user and test download again, should be ok since we're bind to this artifact
+        self.log_user(user.username, 'qweqwe')
+        response = self.app.get(route_path('download_file', fid=file_uid), status=200)
+        assert response.text == content
+
+    def test_download_file_scoped_to_repo_with_bad_token(self, user_util, create_artifact_factory):
+        user_id = User.get_first_super_admin().user_id
+        content = 'HELLO MY NAME IS ARTIFACT !'
+
+        artifact = create_artifact_factory(user_id, content)
+        # bind to repo
+        repo = user_util.create_repo()
+        repo_id = repo.repo_id
+        artifact.scope_repo_id = repo_id
+        Session().add(artifact)
+        Session().commit()
+
+        file_uid = artifact.file_uid
+        self.app.get(route_path('download_file_by_token',
+                                _auth_token='bogus', fid=file_uid), status=302)
+
+    def test_download_file_scoped_to_repo_with_token(self, user_util, create_artifact_factory):
+        user = User.get_first_super_admin()
+        AuthTokenModel().create(user, 'test artifact token',
+                                role=AuthTokenModel.cls.ROLE_ARTIFACT_DOWNLOAD)
+
+        user = User.get_first_super_admin()
+        artifact_token = user.artifact_token
+
+        user_id = User.get_first_super_admin().user_id
+        content = 'HELLO MY NAME IS ARTIFACT !'
+
+        artifact = create_artifact_factory(user_id, content)
+        # bind to repo
+        repo = user_util.create_repo()
+        repo_id = repo.repo_id
+        artifact.scope_repo_id = repo_id
+        Session().add(artifact)
+        Session().commit()
+
+        file_uid = artifact.file_uid
+        response = self.app.get(
+            route_path('download_file_by_token',
+                       _auth_token=artifact_token, fid=file_uid), status=200)
+        assert response.text == content

rhodecode/apps/file_store/utils.py

@@ -25,7 +25,7 @@ import pathlib2
 
 
 def get_file_storage(settings):
-    from rhodecode.apps.file_store.local_store import LocalFileStorage
+    from rhodecode.apps.file_store.backends.local_store import LocalFileStorage
     from rhodecode.apps.file_store import config_keys
     store_path = settings.get(config_keys.store_path)
     return LocalFileStorage(base_path=store_path)

rhodecode/apps/file_store/views.py

@@ -30,8 +30,10 @@ from rhodecode.apps.file_store.exceptions import (
 
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
-from rhodecode.lib.auth import (CSRFRequired, NotAnonymous, HasRepoPermissionAny, HasRepoGroupPermissionAny)
-from rhodecode.model.db import Session, FileStore
+from rhodecode.lib.auth import (
+    CSRFRequired, NotAnonymous, HasRepoPermissionAny, HasRepoGroupPermissionAny,
+    LoginRequired)
+from rhodecode.model.db import Session, FileStore, UserApiKeys
 
 log = logging.getLogger(__name__)
 
@@ -44,6 +46,55 @@ class FileStoreView(BaseAppView):
         self.storage = utils.get_file_storage(self.request.registry.settings)
         return c
 
+    def _serve_file(self, file_uid):
+
+        if not self.storage.exists(file_uid):
+            store_path = self.storage.store_path(file_uid)
+            log.debug('File with FID:%s not found in the store under `%s`',
+                      file_uid, store_path)
+            raise HTTPNotFound()
+
+        db_obj = FileStore().query().filter(FileStore.file_uid == file_uid).scalar()
+        if not db_obj:
+            raise HTTPNotFound()
+
+        # private upload for user
+        if db_obj.check_acl and db_obj.scope_user_id:
+            log.debug('Artifact: checking scope access for bound artifact user: `%s`',
+                      db_obj.scope_user_id)
+            user = db_obj.user
+            if self._rhodecode_db_user.user_id != user.user_id:
+                log.warning('Access to file store object forbidden')
+                raise HTTPNotFound()
+
+        # scoped to repository permissions
+        if db_obj.check_acl and db_obj.scope_repo_id:
+            log.debug('Artifact: checking scope access for bound artifact repo: `%s`',
+                      db_obj.scope_repo_id)
+            repo = db_obj.repo
+            perm_set = ['repository.read', 'repository.write', 'repository.admin']
+            has_perm = HasRepoPermissionAny(*perm_set)(repo.repo_name, 'FileStore check')
+            if not has_perm:
+                log.warning('Access to file store object `%s` forbidden', file_uid)
+                raise HTTPNotFound()
+
+        # scoped to repository group permissions
+        if db_obj.check_acl and db_obj.scope_repo_group_id:
+            log.debug('Artifact: checking scope access for bound artifact repo group: `%s`',
+                      db_obj.scope_repo_group_id)
+            repo_group = db_obj.repo_group
+            perm_set = ['group.read', 'group.write', 'group.admin']
+            has_perm = HasRepoGroupPermissionAny(*perm_set)(repo_group.group_name, 'FileStore check')
+            if not has_perm:
+                log.warning('Access to file store object `%s` forbidden', file_uid)
+                raise HTTPNotFound()
+
+        FileStore.bump_access_counter(file_uid)
+
+        file_path = self.storage.store_path(file_uid)
+        return FileResponse(file_path)
+
+    @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     @view_config(route_name='upload_file', request_method='POST', renderer='json_ext')
@@ -84,7 +135,7 @@ class FileStoreView(BaseAppView):
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
-                file_description='upload attachment',
+                file_description=u'upload attachment',
                 check_acl=False, user_id=self._rhodecode_user.user_id
             )
             Session().add(entry)
@@ -99,46 +150,25 @@ class FileStoreView(BaseAppView):
         return {'store_fid': store_uid,
                 'access_path': h.route_path('download_file', fid=store_uid)}
 
+    # ACL is checked by scopes, if no scope the file is accessible to all
     @view_config(route_name='download_file')
     def download_file(self):
         self.load_default_context()
         file_uid = self.request.matchdict['fid']
         log.debug('Requesting FID:%s from store %s', file_uid, self.storage)
+        return self._serve_file(file_uid)
 
-        if not self.storage.exists(file_uid):
-            log.debug('File with FID:%s not found in the store', file_uid)
-            raise HTTPNotFound()
+    # in addition to @LoginRequired ACL is checked by scopes
+    @LoginRequired(auth_token_access=[UserApiKeys.ROLE_ARTIFACT_DOWNLOAD])
+    @NotAnonymous()
+    @view_config(route_name='download_file_by_token')
+    def download_file_by_token(self):
+        """
+        Special view that allows to access the download file by special URL that
+        is stored inside the URL.
 
-        db_obj = FileStore().query().filter(FileStore.file_uid == file_uid).scalar()
-        if not db_obj:
-            raise HTTPNotFound()
-
-        # private upload for user
-        if db_obj.check_acl and db_obj.scope_user_id:
-            user = db_obj.user
-            if self._rhodecode_db_user.user_id != user.user_id:
-                log.warning('Access to file store object forbidden')
-                raise HTTPNotFound()
-
-        # scoped to repository permissions
-        if db_obj.check_acl and db_obj.scope_repo_id:
-            repo = db_obj.repo
-            perm_set = ['repository.read', 'repository.write', 'repository.admin']
-            has_perm = HasRepoPermissionAny(*perm_set)(repo.repo_name, 'FileStore check')
-            if not has_perm:
-                log.warning('Access to file store object forbidden')
-                raise HTTPNotFound()
-
-        # scoped to repository group permissions
-        if db_obj.check_acl and db_obj.scope_repo_group_id:
-            repo_group = db_obj.repo_group
-            perm_set = ['group.read', 'group.write', 'group.admin']
-            has_perm = HasRepoGroupPermissionAny(*perm_set)(repo_group.group_name, 'FileStore check')
-            if not has_perm:
-                log.warning('Access to file store object forbidden')
-                raise HTTPNotFound()
-
-        FileStore.bump_access_counter(file_uid)
-
-        file_path = self.storage.store_path(file_uid)
-        return FileResponse(file_path)
+        http://example.com/_file_store/token-download/TOKEN/FILE_UID
+        """
+        self.load_default_context()
+        file_uid = self.request.matchdict['fid']
+        return self._serve_file(file_uid)

rhodecode/apps/gist/tests/test_admin_gists.py

@@ -83,7 +83,7 @@ class GistUtility(object):
         Session().commit()
 
 
-@pytest.fixture
+@pytest.fixture()
 def create_gist(request):
     gist_utility = GistUtility()
     request.addfinalizer(gist_utility.cleanup)
@@ -159,7 +159,7 @@ class TestGistsController(TestController):
             params={'lifetime': -1,
                     'content': 'gist test',
                     'filename': 'foo',
-                    'public': 'public',
+                    'gist_type': 'public',
                     'gist_acl_level': Gist.ACL_LEVEL_PUBLIC,
                     'csrf_token': self.csrf_token},
             status=302)
@@ -174,7 +174,7 @@ class TestGistsController(TestController):
             params={'lifetime': -1,
                     'content': 'gist test',
                     'filename': '/home/foo',
-                    'public': 'public',
+                    'gist_type': 'public',
                     'gist_acl_level': Gist.ACL_LEVEL_PUBLIC,
                     'csrf_token': self.csrf_token},
             status=200)
@@ -197,7 +197,7 @@ class TestGistsController(TestController):
             params={'lifetime': -1,
                     'content': 'private gist test',
                     'filename': 'private-foo',
-                    'private': 'private',
+                    'gist_type': 'private',
                     'gist_acl_level': Gist.ACL_LEVEL_PUBLIC,
                     'csrf_token': self.csrf_token},
             status=302)
@@ -216,7 +216,7 @@ class TestGistsController(TestController):
             params={'lifetime': -1,
                     'content': 'private gist test',
                     'filename': 'private-foo',
-                    'private': 'private',
+                    'gist_type': 'private',
                     'gist_acl_level': Gist.ACL_LEVEL_PRIVATE,
                     'csrf_token': self.csrf_token},
             status=302)
@@ -236,7 +236,7 @@ class TestGistsController(TestController):
                     'content': 'gist test',
                     'filename': 'foo-desc',
                     'description': 'gist-desc',
-                    'public': 'public',
+                    'gist_type': 'public',
                     'gist_acl_level': Gist.ACL_LEVEL_PUBLIC,
                     'csrf_token': self.csrf_token},
              status=302)
@@ -252,7 +252,7 @@ class TestGistsController(TestController):
             'content': 'gist test',
             'filename': 'foo-desc',
             'description': 'gist-desc',
-            'public': 'public',
+            'gist_type': 'public',
             'gist_acl_level': Gist.ACL_LEVEL_PUBLIC,
             'csrf_token': self.csrf_token
         }

rhodecode/apps/gist/views.py

@@ -72,7 +72,7 @@ class GistView(BaseAppView):
     @LoginRequired()
     @view_config(
         route_name='gists_show', request_method='GET',
-        renderer='rhodecode:templates/admin/gists/index.mako')
+        renderer='rhodecode:templates/admin/gists/gist_index.mako')
     def gist_show_all(self):
         c = self.load_default_context()
 
@@ -136,7 +136,7 @@ class GistView(BaseAppView):
     @NotAnonymous()
     @view_config(
         route_name='gists_new', request_method='GET',
-        renderer='rhodecode:templates/admin/gists/new.mako')
+        renderer='rhodecode:templates/admin/gists/gist_new.mako')
     def gist_new(self):
         c = self.load_default_context()
         return self._get_template_context(c)
@@ -146,21 +146,26 @@ class GistView(BaseAppView):
     @CSRFRequired()
     @view_config(
         route_name='gists_create', request_method='POST',
-        renderer='rhodecode:templates/admin/gists/new.mako')
+        renderer='rhodecode:templates/admin/gists/gist_new.mako')
     def gist_create(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         data = dict(self.request.POST)
         data['filename'] = data.get('filename') or Gist.DEFAULT_FILENAME
+
         data['nodes'] = [{
             'filename': data['filename'],
             'content': data.get('content'),
             'mimetype': data.get('mimetype')  # None is autodetect
         }]
+        gist_type = {
+            'public': Gist.GIST_PUBLIC,
+            'private': Gist.GIST_PRIVATE
+        }.get(data.get('gist_type')) or Gist.GIST_PRIVATE
+
+        data['gist_type'] = gist_type
 
-        data['gist_type'] = (
-            Gist.GIST_PUBLIC if data.get('public') else Gist.GIST_PRIVATE)
         data['gist_acl_level'] = (
             data.get('gist_acl_level') or Gist.ACL_LEVEL_PRIVATE)
 
@@ -196,7 +201,7 @@ class GistView(BaseAppView):
                 errors['filename'] = errors['nodes.0.filename']
                 del errors['nodes.0.filename']
 
-            data = render('rhodecode:templates/admin/gists/new.mako',
+            data = render('rhodecode:templates/admin/gists/gist_new.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
@@ -260,10 +265,10 @@ class GistView(BaseAppView):
     @LoginRequired()
     @view_config(
         route_name='gist_show', request_method='GET',
-        renderer='rhodecode:templates/admin/gists/show.mako')
+        renderer='rhodecode:templates/admin/gists/gist_show.mako')
     @view_config(
         route_name='gist_show_rev', request_method='GET',
-        renderer='rhodecode:templates/admin/gists/show.mako')
+        renderer='rhodecode:templates/admin/gists/gist_show.mako')
     @view_config(
         route_name='gist_show_formatted', request_method='GET',
         renderer=None)
@@ -304,7 +309,7 @@ class GistView(BaseAppView):
     @NotAnonymous()
     @view_config(
         route_name='gist_edit', request_method='GET',
-        renderer='rhodecode:templates/admin/gists/edit.mako')
+        renderer='rhodecode:templates/admin/gists/gist_edit.mako')
     def gist_edit(self):
         _ = self.request.translate
         gist_id = self.request.matchdict['gist_id']
@@ -338,7 +343,7 @@ class GistView(BaseAppView):
     @CSRFRequired()
     @view_config(
         route_name='gist_update', request_method='POST',
-        renderer='rhodecode:templates/admin/gists/edit.mako')
+        renderer='rhodecode:templates/admin/gists/gist_edit.mako')
     def gist_update(self):
         _ = self.request.translate
         gist_id = self.request.matchdict['gist_id']

rhodecode/apps/home/__init__.py

@@ -43,6 +43,14 @@ def includeme(config):
         name='home',
         pattern='/')
 
+    config.add_route(
+        name='main_page_repos_data',
+        pattern='/_home_repos')
+
+    config.add_route(
+        name='main_page_repo_groups_data',
+        pattern='/_home_repo_groups')
+
     config.add_route(
         name='user_autocomplete_data',
         pattern='/_users')

rhodecode/apps/home/tests/test_home.py

@@ -22,7 +22,7 @@
 import pytest
 
 import rhodecode
-from rhodecode.model.db import Repository
+from rhodecode.model.db import Repository, RepoGroup, User
 from rhodecode.model.meta import Session
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
@@ -37,6 +37,8 @@ fixture = Fixture()
 def route_path(name, **kwargs):
     return {
         'home': '/',
+        'main_page_repos_data': '/_home_repos',
+        'main_page_repo_groups_data': '/_home_repo_groups',
         'repo_group_home': '/{repo_group_name}'
     }[name].format(**kwargs)
 
@@ -47,11 +49,42 @@ class TestHomeController(TestController):
         self.log_user()
         response = self.app.get(route_path('home'))
         # if global permission is set
-        response.mustcontain('Add Repository')
+        response.mustcontain('New Repository')
+
+    def test_index_grid_repos(self, xhr_header):
+        self.log_user()
+        response = self.app.get(route_path('main_page_repos_data'), extra_environ=xhr_header)
+        # search for objects inside the JavaScript JSON
+        for obj in Repository.getAll():
+            response.mustcontain('<a href=\\"/{}\\">'.format(obj.repo_name))
+
+    def test_index_grid_repo_groups(self, xhr_header):
+        self.log_user()
+        response = self.app.get(route_path('main_page_repo_groups_data'),
+                                extra_environ=xhr_header,)
 
         # search for objects inside the JavaScript JSON
-        for repo in Repository.getAll():
-            response.mustcontain('"name_raw": "%s"' % repo.repo_name)
+        for obj in RepoGroup.getAll():
+            response.mustcontain('<a href=\\"/{}\\">'.format(obj.group_name))
+
+    def test_index_grid_repo_groups_without_access(self, xhr_header, user_util):
+        user = user_util.create_user(password='qweqwe')
+        group_ok = user_util.create_repo_group(owner=user)
+        group_id_ok = group_ok.group_id
+
+        group_forbidden = user_util.create_repo_group(owner=User.get_first_super_admin())
+        group_id_forbidden = group_forbidden.group_id
+
+        user_util.grant_user_permission_to_repo_group(group_forbidden, user, 'group.none')
+        self.log_user(user.username, 'qweqwe')
+
+        self.app.get(route_path('main_page_repo_groups_data'),
+                     extra_environ=xhr_header,
+                     params={'repo_group_id': group_id_ok}, status=200)
+
+        self.app.get(route_path('main_page_repo_groups_data'),
+                     extra_environ=xhr_header,
+                     params={'repo_group_id': group_id_forbidden}, status=404)
 
     def test_index_contains_statics_with_ver(self):
         from rhodecode.lib.base import calculate_version_hash
@@ -62,11 +95,11 @@ class TestHomeController(TestController):
         rhodecode_version_hash = calculate_version_hash(
             {'beaker.session.secret': 'test-rc-uytcxaz'})
         response.mustcontain('style.css?ver={0}'.format(rhodecode_version_hash))
-        response.mustcontain('scripts.js?ver={0}'.format(rhodecode_version_hash))
+        response.mustcontain('scripts.min.js?ver={0}'.format(rhodecode_version_hash))
 
-    def test_index_contains_backend_specific_details(self, backend):
+    def test_index_contains_backend_specific_details(self, backend, xhr_header):
         self.log_user()
-        response = self.app.get(route_path('home'))
+        response = self.app.get(route_path('main_page_repos_data'), extra_environ=xhr_header)
         tip = backend.repo.get_commit().raw_id
 
         # html in javascript variable:
@@ -81,39 +114,44 @@ class TestHomeController(TestController):
             response = self.app.get(route_path('home'), status=302)
             assert 'login' in response.location
 
-    def test_index_page_on_groups(self, autologin_user, repo_group):
-        response = self.app.get(route_path('repo_group_home', repo_group_name='gr1'))
-        response.mustcontain("gr1/repo_in_group")
+    def test_index_page_on_groups_with_wrong_group_id(self, autologin_user, xhr_header):
+        group_id = 918123
+        self.app.get(
+            route_path('main_page_repo_groups_data'),
+            params={'repo_group_id': group_id},
+            status=404, extra_environ=xhr_header)
 
-    def test_index_page_on_group_with_trailing_slash(
-            self, autologin_user, repo_group):
-        response = self.app.get(route_path('repo_group_home', repo_group_name='gr1') + '/')
-        response.mustcontain("gr1/repo_in_group")
+    def test_index_page_on_groups(self, autologin_user, user_util, xhr_header):
+        gr = user_util.create_repo_group()
+        repo = user_util.create_repo(parent=gr)
+        repo_name = repo.repo_name
+        group_id = gr.group_id
 
-    @pytest.fixture(scope='class')
-    def repo_group(self, request):
-        gr = fixture.create_repo_group('gr1')
-        fixture.create_repo(name='gr1/repo_in_group', repo_group=gr)
+        response = self.app.get(route_path(
+            'repo_group_home', repo_group_name=gr.group_name))
+        response.mustcontain('d.repo_group_id = {}'.format(group_id))
 
-        @request.addfinalizer
-        def cleanup():
-            RepoModel().delete('gr1/repo_in_group')
-            RepoGroupModel().delete(repo_group='gr1', force_delete=True)
-            Session().commit()
+        response = self.app.get(
+            route_path('main_page_repos_data'),
+            params={'repo_group_id': group_id},
+            extra_environ=xhr_header,)
+        response.mustcontain(repo_name)
 
-    def test_index_with_name_with_tags(self, user_util, autologin_user):
-        user = user_util.create_user()
-        username = user.username
-        user.name = '<img src="/image1" onload="alert(\'Hello, World!\');">'
-        user.lastname = '#"><img src=x onerror=prompt(document.cookie);>'
+    def test_index_page_on_group_with_trailing_slash(self, autologin_user, user_util, xhr_header):
+        gr = user_util.create_repo_group()
+        repo = user_util.create_repo(parent=gr)
+        repo_name = repo.repo_name
+        group_id = gr.group_id
 
-        Session().add(user)
-        Session().commit()
-        user_util.create_repo(owner=username)
+        response = self.app.get(route_path(
+            'repo_group_home', repo_group_name=gr.group_name+'/'))
+        response.mustcontain('d.repo_group_id = {}'.format(group_id))
 
-        response = self.app.get(route_path('home'))
-        response.mustcontain(h.html_escape(user.first_name))
-        response.mustcontain(h.html_escape(user.last_name))
+        response = self.app.get(
+            route_path('main_page_repos_data'),
+            params={'repo_group_id': group_id},
+            extra_environ=xhr_header, )
+        response.mustcontain(repo_name)
 
     @pytest.mark.parametrize("name, state", [
         ('Disabled', False),
@@ -137,5 +175,5 @@ class TestHomeController(TestController):
     def test_logout_form_contains_csrf(self, autologin_user, csrf_token):
         response = self.app.get(route_path('home'))
         assert_response = response.assert_response()
-        element = assert_response.get_element('.logout #csrf_token')
+        element = assert_response.get_element('.logout [name=csrf_token]')
         assert element.value == csrf_token

rhodecode/apps/home/views.py

@@ -22,29 +22,30 @@ import re
 import logging
 import collections
 
+from pyramid.httpexceptions import HTTPNotFound
 from pyramid.view import view_config
 
-from rhodecode.apps._base import BaseAppView
+from rhodecode.apps._base import BaseAppView, DataGridAppView
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
-    LoginRequired, NotAnonymous, HasRepoGroupPermissionAnyDecorator, CSRFRequired)
+    LoginRequired, NotAnonymous, HasRepoGroupPermissionAnyDecorator, CSRFRequired,
+    HasRepoGroupPermissionAny, AuthUser)
 from rhodecode.lib.codeblocks import filenode_as_lines_tokens
 from rhodecode.lib.index import searcher_from_config
 from rhodecode.lib.utils2 import safe_unicode, str2bool, safe_int
-from rhodecode.lib.ext_json import json
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.model.db import (
-    func, true, or_, case, in_filter_generator, Repository, RepoGroup, User, UserGroup)
+    func, true, or_, case, in_filter_generator, Session,
+    Repository, RepoGroup, User, UserGroup)
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
-from rhodecode.model.scm import RepoGroupList, RepoList
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 
 log = logging.getLogger(__name__)
 
 
-class HomeView(BaseAppView):
+class HomeView(BaseAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
@@ -112,7 +113,12 @@ class HomeView(BaseAppView):
             ['repository.read', 'repository.write', 'repository.admin'],
             cache=False, name_filter=name_contains) or [-1]
 
-        query = Repository.query()\
+        query = Session().query(
+                Repository.repo_name,
+                Repository.repo_id,
+                Repository.repo_type,
+                Repository.private,
+            )\
             .filter(Repository.archived.isnot(true()))\
             .filter(or_(
                 # generate multiple IN to fix limitation problems
@@ -158,7 +164,10 @@ class HomeView(BaseAppView):
             ['group.read', 'group.write', 'group.admin'],
             cache=False, name_filter=name_contains) or [-1]
 
-        query = RepoGroup.query()\
+        query = Session().query(
+                RepoGroup.group_id,
+                RepoGroup.group_name,
+            )\
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(RepoGroup.group_id, allowed_ids)
@@ -449,6 +458,7 @@ class HomeView(BaseAppView):
                 'id': -10,
                 'value': query,
                 'value_display': label,
+                'value_icon': '<i class="icon-code"></i>',
                 'type': 'search',
                 'subtype': 'repo',
                 'url': h.route_path('search_repo',
@@ -466,6 +476,7 @@ class HomeView(BaseAppView):
                 'id': -20,
                 'value': query,
                 'value_display': label,
+                'value_icon': '<i class="icon-history"></i>',
                 'type': 'search',
                 'subtype': 'repo',
                 'url': h.route_path('search_repo',
@@ -491,6 +502,7 @@ class HomeView(BaseAppView):
                 'id': -30,
                 'value': query,
                 'value_display': label,
+                'value_icon': '<i class="icon-code"></i>',
                 'type': 'search',
                 'subtype': 'repo_group',
                 'url': h.route_path('search_repo_group',
@@ -508,6 +520,7 @@ class HomeView(BaseAppView):
                 'id': -40,
                 'value': query,
                 'value_display': label,
+                'value_icon': '<i class="icon-history"></i>',
                 'type': 'search',
                 'subtype': 'repo_group',
                 'url': h.route_path('search_repo_group',
@@ -529,6 +542,7 @@ class HomeView(BaseAppView):
                     'id': -1,
                     'value': query,
                     'value_display': u'File search for: `{}`'.format(query),
+                    'value_icon': '<i class="icon-code"></i>',
                     'type': 'search',
                     'subtype': 'global',
                     'url': h.route_path('search',
@@ -539,6 +553,7 @@ class HomeView(BaseAppView):
                     'id': -2,
                     'value': query,
                     'value_display': u'Commit search for: `{}`'.format(query),
+                    'value_icon': '<i class="icon-history"></i>',
                     'type': 'search',
                     'subtype': 'global',
                     'url': h.route_path('search',
@@ -667,23 +682,6 @@ class HomeView(BaseAppView):
 
         return {'suggestions': res}
 
-    def _get_groups_and_repos(self, repo_group_id=None):
-        # repo groups groups
-        repo_group_list = RepoGroup.get_all_repo_groups(group_id=repo_group_id)
-        _perms = ['group.read', 'group.write', 'group.admin']
-        repo_group_list_acl = RepoGroupList(repo_group_list, perm_set=_perms)
-        repo_group_data = RepoGroupModel().get_repo_groups_as_dict(
-            repo_group_list=repo_group_list_acl, admin=False)
-
-        # repositories
-        repo_list = Repository.get_all_repos(group_id=repo_group_id)
-        _perms = ['repository.read', 'repository.write', 'repository.admin']
-        repo_list_acl = RepoList(repo_list, perm_set=_perms)
-        repo_data = RepoModel().get_repos_as_dict(
-            repo_list=repo_list_acl, admin=False)
-
-        return repo_data, repo_group_data
-
     @LoginRequired()
     @view_config(
         route_name='home', request_method='GET',
@@ -691,17 +689,74 @@ class HomeView(BaseAppView):
     def main_page(self):
         c = self.load_default_context()
         c.repo_group = None
-
-        repo_data, repo_group_data = self._get_groups_and_repos()
-        # json used to render the grids
-        c.repos_data = json.dumps(repo_data)
-        c.repo_groups_data = json.dumps(repo_group_data)
-
         return self._get_template_context(c)
 
+    def _main_page_repo_groups_data(self, repo_group_id):
+        column_map = {
+            'name': 'group_name_hash',
+            'desc': 'group_description',
+            'last_change': 'updated_on',
+            'owner': 'user_username',
+        }
+        draw, start, limit = self._extract_chunk(self.request)
+        search_q, order_by, order_dir = self._extract_ordering(
+            self.request, column_map=column_map)
+        return RepoGroupModel().get_repo_groups_data_table(
+            draw, start, limit,
+            search_q, order_by, order_dir,
+            self._rhodecode_user, repo_group_id)
+
+    def _main_page_repos_data(self, repo_group_id):
+        column_map = {
+            'name': 'repo_name',
+            'desc': 'description',
+            'last_change': 'updated_on',
+            'owner': 'user_username',
+        }
+        draw, start, limit = self._extract_chunk(self.request)
+        search_q, order_by, order_dir = self._extract_ordering(
+            self.request, column_map=column_map)
+        return RepoModel().get_repos_data_table(
+            draw, start, limit,
+            search_q, order_by, order_dir,
+            self._rhodecode_user, repo_group_id)
+
     @LoginRequired()
-    @HasRepoGroupPermissionAnyDecorator(
-        'group.read', 'group.write', 'group.admin')
+    @view_config(
+        route_name='main_page_repo_groups_data',
+        request_method='GET', renderer='json_ext', xhr=True)
+    def main_page_repo_groups_data(self):
+        self.load_default_context()
+        repo_group_id = safe_int(self.request.GET.get('repo_group_id'))
+
+        if repo_group_id:
+            group = RepoGroup.get_or_404(repo_group_id)
+            _perms = AuthUser.repo_group_read_perms
+            if not HasRepoGroupPermissionAny(*_perms)(
+                    group.group_name, 'user is allowed to list repo group children'):
+                raise HTTPNotFound()
+
+        return self._main_page_repo_groups_data(repo_group_id)
+
+    @LoginRequired()
+    @view_config(
+        route_name='main_page_repos_data',
+        request_method='GET', renderer='json_ext', xhr=True)
+    def main_page_repos_data(self):
+        self.load_default_context()
+        repo_group_id = safe_int(self.request.GET.get('repo_group_id'))
+
+        if repo_group_id:
+            group = RepoGroup.get_or_404(repo_group_id)
+            _perms = AuthUser.repo_group_read_perms
+            if not HasRepoGroupPermissionAny(*_perms)(
+                    group.group_name, 'user is allowed to list repo group children'):
+                raise HTTPNotFound()
+
+        return self._main_page_repos_data(repo_group_id)
+
+    @LoginRequired()
+    @HasRepoGroupPermissionAnyDecorator(*AuthUser.repo_group_read_perms)
     @view_config(
         route_name='repo_group_home', request_method='GET',
         renderer='rhodecode:templates/index_repo_group.mako')
@@ -711,16 +766,6 @@ class HomeView(BaseAppView):
     def repo_group_main_page(self):
         c = self.load_default_context()
         c.repo_group = self.request.db_repo_group
-        repo_data, repo_group_data = self._get_groups_and_repos(c.repo_group.group_id)
-
-        # update every 5 min
-        if self.request.db_repo_group.last_commit_cache_update_diff > 60 * 5:
-            self.request.db_repo_group.update_commit_cache()
-
-        # json used to render the grids
-        c.repos_data = json.dumps(repo_data)
-        c.repo_groups_data = json.dumps(repo_group_data)
-
         return self._get_template_context(c)
 
     @LoginRequired()

rhodecode/apps/hovercards/__init__.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2018-2019 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+
+def includeme(config):
+
+    config.add_route(
+        name='hovercard_user',
+        pattern='/_hovercard/user/{user_id}')
+
+    config.add_route(
+        name='hovercard_user_group',
+        pattern='/_hovercard/user_group/{user_group_id}')
+
+    config.add_route(
+        name='hovercard_pull_request',
+        pattern='/_hovercard/pull_request/{pull_request_id}')
+
+    config.add_route(
+        name='hovercard_repo_commit',
+        pattern='/_hovercard/commit/{repo_name:.*?[^/]}/{commit_id}', repo_route=True)
+
+    # Scan module for configuration decorators.
+    config.scan('.views', ignore='.tests')

rhodecode/apps/hovercards/views.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2016-2019 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import re
+import logging
+import collections
+
+from pyramid.httpexceptions import HTTPNotFound
+from pyramid.view import view_config
+
+from rhodecode.apps._base import BaseAppView, RepoAppView
+from rhodecode.lib import helpers as h
+from rhodecode.lib.auth import (
+    LoginRequired, NotAnonymous, HasRepoGroupPermissionAnyDecorator, CSRFRequired,
+    HasRepoPermissionAnyDecorator)
+from rhodecode.lib.codeblocks import filenode_as_lines_tokens
+from rhodecode.lib.index import searcher_from_config
+from rhodecode.lib.utils2 import safe_unicode, str2bool, safe_int
+from rhodecode.lib.ext_json import json
+from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError, EmptyRepositoryError
+from rhodecode.lib.vcs.nodes import FileNode
+from rhodecode.model.db import (
+    func, true, or_, case, in_filter_generator, Repository, RepoGroup, User, UserGroup, PullRequest)
+from rhodecode.model.repo import RepoModel
+from rhodecode.model.repo_group import RepoGroupModel
+from rhodecode.model.scm import RepoGroupList, RepoList
+from rhodecode.model.user import UserModel
+from rhodecode.model.user_group import UserGroupModel
+
+log = logging.getLogger(__name__)
+
+
+class HoverCardsView(BaseAppView):
+
+    def load_default_context(self):
+        c = self._get_local_tmpl_context()
+        return c
+
+    @LoginRequired()
+    @view_config(
+        route_name='hovercard_user', request_method='GET', xhr=True,
+        renderer='rhodecode:templates/hovercards/hovercard_user.mako')
+    def hovercard_user(self):
+        c = self.load_default_context()
+        user_id = self.request.matchdict['user_id']
+        c.user = User.get_or_404(user_id)
+        return self._get_template_context(c)
+
+    @LoginRequired()
+    @view_config(
+        route_name='hovercard_user_group', request_method='GET', xhr=True,
+        renderer='rhodecode:templates/hovercards/hovercard_user_group.mako')
+    def hovercard_user_group(self):
+        c = self.load_default_context()
+        user_group_id = self.request.matchdict['user_group_id']
+        c.user_group = UserGroup.get_or_404(user_group_id)
+        return self._get_template_context(c)
+
+    @LoginRequired()
+    @view_config(
+        route_name='hovercard_pull_request', request_method='GET', xhr=True,
+        renderer='rhodecode:templates/hovercards/hovercard_pull_request.mako')
+    def hovercard_pull_request(self):
+        c = self.load_default_context()
+        c.pull_request = PullRequest.get_or_404(
+            self.request.matchdict['pull_request_id'])
+        perms = ['repository.read', 'repository.write', 'repository.admin']
+        c.can_view_pr = h.HasRepoPermissionAny(*perms)(
+            c.pull_request.target_repo.repo_name)
+        return self._get_template_context(c)
+
+
+class HoverCardsRepoView(RepoAppView):
+    def load_default_context(self):
+        c = self._get_local_tmpl_context()
+        return c
+
+    @LoginRequired()
+    @HasRepoPermissionAnyDecorator('repository.read', 'repository.write', 'repository.admin')
+    @view_config(
+        route_name='hovercard_repo_commit', request_method='GET', xhr=True,
+        renderer='rhodecode:templates/hovercards/hovercard_repo_commit.mako')
+    def hovercard_repo_commit(self):
+        c = self.load_default_context()
+        commit_id = self.request.matchdict['commit_id']
+        pre_load = ['author', 'branch', 'date', 'message']
+        try:
+            c.commit = self.rhodecode_vcs_repo.get_commit(
+                commit_id=commit_id, pre_load=pre_load)
+        except (CommitDoesNotExistError, EmptyRepositoryError):
+            raise HTTPNotFound()
+
+        return self._get_template_context(c)

rhodecode/apps/journal/views.py

@@ -22,7 +22,7 @@
 import logging
 import itertools
 
-from webhelpers.feedgenerator import Atom1Feed, Rss201rev2Feed
+
 
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPBadRequest
@@ -34,10 +34,11 @@ from rhodecode.model.db import (
     or_, joinedload, Repository, UserLog, UserFollowing, User, UserApiKeys)
 from rhodecode.model.meta import Session
 import rhodecode.lib.helpers as h
-from rhodecode.lib.helpers import Page
+from rhodecode.lib.helpers import SqlPage
 from rhodecode.lib.user_log_filter import user_log_filter
 from rhodecode.lib.auth import LoginRequired, NotAnonymous, CSRFRequired, HasRepoPermissionAny
 from rhodecode.lib.utils2 import safe_int, AttributeDict, md5_safe
+from rhodecode.lib.feedgenerator.feedgenerator import Atom1Feed, Rss201rev2Feed
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
@@ -166,7 +167,7 @@ class JournalView(BaseAppView):
                 description=desc)
 
         response = Response(feed.writeString('utf-8'))
-        response.content_type = feed.mime_type
+        response.content_type = feed.content_type
         return response
 
     def _rss_feed(self, repos, search_term, public=True):
@@ -212,7 +213,7 @@ class JournalView(BaseAppView):
                 description=desc)
 
         response = Response(feed.writeString('utf-8'))
-        response.content_type = feed.mime_type
+        response.content_type = feed.content_type
         return response
 
     @LoginRequired()
@@ -232,15 +233,15 @@ class JournalView(BaseAppView):
 
         journal = self._get_journal_data(following, c.search_term)
 
-        def url_generator(**kw):
+        def url_generator(page_num):
             query_params = {
+                'page': page_num,
                 'filter': c.search_term
             }
-            query_params.update(kw)
             return self.request.current_route_path(_query=query_params)
 
-        c.journal_pager = Page(
-            journal, page=p, items_per_page=20, url=url_generator)
+        c.journal_pager = SqlPage(
+            journal, page=p, items_per_page=20, url_maker=url_generator)
         c.journal_day_aggreagate = self._get_daily_aggregate(c.journal_pager)
 
         c.journal_data = render(
@@ -333,13 +334,14 @@ class JournalView(BaseAppView):
 
         journal = self._get_journal_data(c.following, c.search_term)
 
-        def url_generator(**kw):
-            query_params = {}
-            query_params.update(kw)
+        def url_generator(page_num):
+            query_params = {
+                'page': page_num
+            }
             return self.request.current_route_path(_query=query_params)
 
-        c.journal_pager = Page(
-            journal, page=p, items_per_page=20, url=url_generator)
+        c.journal_pager = SqlPage(
+            journal, page=p, items_per_page=20, url_maker=url_generator)
         c.journal_day_aggreagate = self._get_daily_aggregate(c.journal_pager)
 
         c.journal_data = render(

rhodecode/apps/login/tests/test_login.py

@@ -93,7 +93,7 @@ class TestLoginController(object):
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == 'test_admin'
-        response.mustcontain('/%s' % HG_REPO)
+        response.mustcontain('logout')
 
     def test_login_regular_ok(self):
         response = self.app.post(route_path('login'),
@@ -104,8 +104,7 @@ class TestLoginController(object):
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == 'test_regular'
-
-        response.mustcontain('/%s' % HG_REPO)
+        response.mustcontain('logout')
 
     def test_login_regular_forbidden_when_super_admin_restriction(self):
         from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin
@@ -225,7 +224,7 @@ class TestLoginController(object):
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == temp_user
-        response.mustcontain('/%s' % HG_REPO)
+        response.mustcontain('logout')
 
         # new password should be bcrypted, after log-in and transfer
         user = User.get_by_username(temp_user)
@@ -401,7 +400,7 @@ class TestLoginController(object):
         )  # This should be overridden
 
         assert_session_flash(
-            response, 'You have successfully registered with RhodeCode')
+            response, 'You have successfully registered with RhodeCode. You can log-in now.')
 
         ret = Session().query(User).filter(
             User.username == 'test_regular4').one()

rhodecode/apps/login/tests/test_password_reset.py

@@ -88,7 +88,7 @@ class TestPasswordReset(TestController):
 
         response = self.app.get(route_path('reset_password'))
         
-        assert_response = AssertResponse(response)
+        assert_response = response.assert_response()
         if show_reset:
             response.mustcontain('Send password reset email')
             assert_response.one_element_exists('#email')

rhodecode/apps/login/tests/test_register_captcha.py

@@ -90,7 +90,7 @@ class TestRegisterCaptcha(object):
 
         response = app.get(ADMIN_PREFIX + '/register')
 
-        assertr = AssertResponse(response)
+        assertr = response.assert_response()
         if active:
             assertr.one_element_exists('#recaptcha_field')
         else:
@@ -128,6 +128,6 @@ class TestRegisterCaptcha(object):
         else:
             # If captche input is invalid we expect to stay on the registration
             # page with an error message displayed.
-            assertr = AssertResponse(response)
+            assertr = response.assert_response()
             assert response.status_int == 200
             assertr.one_element_exists('#recaptcha_field ~ span.error-message')