From 6f6bb8dccc7af4e6b2f93d37dc1b8f939c5f4ccb Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Mon, 10 Sep 2018 18:59:36 +0200
Subject: [PATCH 01/19] docs: fixed not supported code-block

---
 docs/api/methods/deprecated-methods.rst | 2 +-
 rhodecode/api/views/deprecated_api.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/api/methods/deprecated-methods.rst b/docs/api/methods/deprecated-methods.rst
index ea5f069f..8bc49cf9 100644
--- a/docs/api/methods/deprecated-methods.rst
+++ b/docs/api/methods/deprecated-methods.rst
@@ -38,7 +38,7 @@ changeset_comment
 
    Example error output:
 
-   .. code-block:: json
+   .. code-block:: javascript
 
        {
            "id" : <id_given_in_input>,
diff --git a/rhodecode/api/views/deprecated_api.py b/rhodecode/api/views/deprecated_api.py
index e6e4d9ce..904f9a41 100644
--- a/rhodecode/api/views/deprecated_api.py
+++ b/rhodecode/api/views/deprecated_api.py
@@ -66,7 +66,7 @@ def changeset_comment(request, apiuser, repoid, revision, message,
 
     Example error output:
 
-    .. code-block:: json
+    .. code-block:: javascript
 
         {
             "id" : <id_given_in_input>,

From 817c7150c115f44695dc846b3a873fd63e1558fe Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Mon, 10 Sep 2018 19:49:33 +0200
Subject: [PATCH 02/19] docs: updated scaling/cluster docs

---
 docs/admin/apache-config.rst                  |   2 +-
 docs/admin/nginx-config-example.rst           |  13 +-
 docs/admin/nginx-config.rst                   |   4 +-
 ...{nginx-tuning.rst => nginx-proxy-conf.rst} |   7 +-
 docs/admin/tuning-change-large-file-dir.rst   |   4 +-
 docs/admin/tuning-change-lfs-dir.rst          |   4 +-
 docs/admin/tuning-gunicorn.rst                |  16 +-
 docs/admin/tuning-increase-cache-size.rst     |  17 -
 docs/admin/tuning-increase-db-performance.rst |   4 +-
 docs/admin/tuning-rhodecode.rst               |   3 +-
 .../tuning-scale-horizontally-cluster.rst     | 383 ++++++++++++++++++
 docs/admin/tuning-scale-horizontally.rst      |  58 ---
 .../tuning-user-sessions-performance.rst      |   7 +-
 docs/admin/tuning-vcs-memory-cache.rst        |   4 +-
 14 files changed, 420 insertions(+), 106 deletions(-)
 rename docs/admin/{nginx-tuning.rst => nginx-proxy-conf.rst} (95%)
 delete mode 100644 docs/admin/tuning-increase-cache-size.rst
 create mode 100644 docs/admin/tuning-scale-horizontally-cluster.rst
 delete mode 100644 docs/admin/tuning-scale-horizontally.rst

diff --git a/docs/admin/apache-config.rst b/docs/admin/apache-config.rst
index 7af90abc..7a649851 100644
--- a/docs/admin/apache-config.rst
+++ b/docs/admin/apache-config.rst
@@ -8,7 +8,7 @@ the information in the following sections.
 
 .. toctree::
 
-   apache-diffie-hellman
    apache-conf-example
+   apache-diffie-hellman
    apache-subdirectory
    apache-wsgi-coding
diff --git a/docs/admin/nginx-config-example.rst b/docs/admin/nginx-config-example.rst
index bac449d5..02c76350 100644
--- a/docs/admin/nginx-config-example.rst
+++ b/docs/admin/nginx-config-example.rst
@@ -7,7 +7,7 @@ Use the following example to configure Nginx as a your web server.
 .. code-block:: nginx
 
     ## rate limiter for certain pages to prevent brute force attacks
-    limit_req_zone  $binary_remote_addr  zone=dl_limit:10m   rate=1r/s;
+    limit_req_zone  $binary_remote_addr  zone=req_limit:10m   rate=1r/s;
 
     ## custom log format
     log_format log_custom '$remote_addr - $remote_user [$time_local] '
@@ -18,7 +18,7 @@ Use the following example to configure Nginx as a your web server.
     ## define upstream (local RhodeCode instance) to connect to
     upstream rc {
         # Url to running RhodeCode instance.
-        # This is shown as `- URL:` in output from rccontrol status.
+        # This is shown as `- URL: <host>` in output from rccontrol status.
         server 127.0.0.1:10002;
 
         # add more instances for load balancing
@@ -85,9 +85,10 @@ Use the following example to configure Nginx as a your web server.
         # Diffie-Hellman parameter for DHE ciphersuites, recommended 2048 bits
         #ssl_dhparam /etc/nginx/ssl/dhparam.pem;
 
+        # example of proxy.conf can be found in our docs.
         include     /etc/nginx/proxy.conf;
 
-        ## serve static files by Nginx, recommended for performance
+        ## uncomment to serve static files by Nginx, recommended for performance
         # location /_static/rhodecode {
         #    gzip on;
         #    gzip_min_length  500;
@@ -96,6 +97,7 @@ Use the following example to configure Nginx as a your web server.
         #    gzip_types  text/css text/javascript text/xml text/plain text/x-component application/javascript application/json application/xml application/rss+xml font/truetype font/opentype application/vnd.ms-fontobject image/svg+xml;
         #    gzip_vary on;
         #    gzip_disable     "msie6";
+        #    alias /path/to/.rccontrol/community-1/static;
         #    alias /path/to/.rccontrol/enterprise-1/static;
         # }
 
@@ -120,9 +122,9 @@ Use the following example to configure Nginx as a your web server.
             proxy_set_header Connection  "upgrade";
         }
 
+        ## rate limit this endpoint to prevent login page brute-force attacks
         location /_admin/login {
-            ## rate limit this endpoint
-            limit_req  zone=dl_limit  burst=10  nodelay;
+            limit_req  zone=req_limit  burst=10  nodelay;
             try_files $uri @rhode;
         }
 
@@ -138,6 +140,7 @@ Use the following example to configure Nginx as a your web server.
         ## is turned off
         error_page 502 /502.html;
         location = /502.html {
+           #root  /path/to/.rccontrol/community-1/static;
            root  /path/to/.rccontrol/enterprise-1/static;
         }
     }
\ No newline at end of file
diff --git a/docs/admin/nginx-config.rst b/docs/admin/nginx-config.rst
index 31477588..a81dbae3 100644
--- a/docs/admin/nginx-config.rst
+++ b/docs/admin/nginx-config.rst
@@ -8,7 +8,7 @@ the information in the following sections.
 
 .. toctree::
 
-   nginx-diffie-hellman
    nginx-config-example
-   nginx-tuning
+   nginx-diffie-hellman
+   nginx-proxy-conf
    nginx-url-prefix
diff --git a/docs/admin/nginx-tuning.rst b/docs/admin/nginx-proxy-conf.rst
similarity index 95%
rename from docs/admin/nginx-tuning.rst
rename to docs/admin/nginx-proxy-conf.rst
index 1478cbf8..e468c027 100644
--- a/docs/admin/nginx-tuning.rst
+++ b/docs/admin/nginx-proxy-conf.rst
@@ -1,7 +1,8 @@
-.. _nginx-tuning:
+.. _nginx-proxy-conf:
+
+Nginx Proxy Config
+------------------
 
-Nginx Tuning
-------------
 
 Set the following properties in your ``/etc/nginx/proxy.conf`` so it does not
 timeout during large pushes.
diff --git a/docs/admin/tuning-change-large-file-dir.rst b/docs/admin/tuning-change-large-file-dir.rst
index 0eaabde6..a248d13f 100644
--- a/docs/admin/tuning-change-large-file-dir.rst
+++ b/docs/admin/tuning-change-large-file-dir.rst
@@ -1,7 +1,7 @@
 .. _hg-lrg-loc:
 
-Change the |hg| Large Files Location
-------------------------------------
+|hg| Large Files Location
+-------------------------
 
 |RCE| manages |hg| larges files from the following default location
 :file:`/home/{user}/repos/.cache/largefiles`. If you wish to change this, use
diff --git a/docs/admin/tuning-change-lfs-dir.rst b/docs/admin/tuning-change-lfs-dir.rst
index 55e6734a..afc9cb35 100644
--- a/docs/admin/tuning-change-lfs-dir.rst
+++ b/docs/admin/tuning-change-lfs-dir.rst
@@ -1,7 +1,7 @@
 .. _git-lfs-loc:
 
-Change the |git| LFS storage Location
--------------------------------------
+|git| LFS storage Location
+--------------------------
 
 |RCE| manages |git| LFS files from the following default location
 :file:`/home/{user}/repos/.cache/lfs_store`. If you wish to change this, use
diff --git a/docs/admin/tuning-gunicorn.rst b/docs/admin/tuning-gunicorn.rst
index 4b3156e5..7687e61e 100644
--- a/docs/admin/tuning-gunicorn.rst
+++ b/docs/admin/tuning-gunicorn.rst
@@ -1,14 +1,13 @@
 .. _increase-gunicorn:
 
-Increase Gunicorn Workers
--------------------------
+Configure Gunicorn Workers
+--------------------------
 
 
-|RCE| comes with `Gunicorn`_ packaged in its Nix environment.
-Gunicorn is a Python WSGI HTTP Server for UNIX.
+|RCE| comes with `Gunicorn`_ which is a Python WSGI HTTP Server for UNIX.
 
 To improve |RCE| performance you can increase the number of `Gunicorn`_  workers.
-This allows to handle more connections concurently, and provide better
+This allows to handle more connections concurrently, and provide better
 responsiveness and performance.
 
 By default during installation |RCC|  tries to detect how many CPUs are
@@ -18,8 +17,11 @@ However sometimes it's better to manually set the number of workers.
 To do this, use the following steps:
 
 1. Open the :file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
-2. In the ``[server:main]`` section, increase the number of Gunicorn
-   ``workers`` using the following formula :math:`(2 * Cores) + 1`.
+2. In the ``[server:main]`` section, change the number of Gunicorn
+   ``workers`` using the following default formula :math:`(2 * Cores) + 1`.
+   We however not recommend using more than 8-12 workers per server. It's better
+   to start using the :ref:`scale-horizontal-cluster` in case that performance
+   with 8-12 workers is not enough.
 
 .. code-block:: ini
 
diff --git a/docs/admin/tuning-increase-cache-size.rst b/docs/admin/tuning-increase-cache-size.rst
deleted file mode 100644
index f0a5b047..00000000
--- a/docs/admin/tuning-increase-cache-size.rst
+++ /dev/null
@@ -1,17 +0,0 @@
-.. _cache-size:
-
-Increase Cache Size
--------------------
-
-When managing hundreds of |repos| from the main |RCE| interface the system
-can become slow when the cache expires. Increasing the cache expiration
-option improves the response times of the main user interface.
-To increase your cache size, change the following default value in the
-:file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file. The value
-is specified in seconds.
-
-.. code-block:: ini
-
-      beaker.cache.long_term.expire=3600  # day (86400) week (604800)
-
-.. note:: The |RCE| cache automatically expires for changed |repos|.
diff --git a/docs/admin/tuning-increase-db-performance.rst b/docs/admin/tuning-increase-db-performance.rst
index be0074aa..d56088fc 100644
--- a/docs/admin/tuning-increase-db-performance.rst
+++ b/docs/admin/tuning-increase-db-performance.rst
@@ -1,7 +1,7 @@
 .. _db-session-ref:
 
-Increase Database Performance
------------------------------
+Database Performance
+--------------------
 
 For tuning PostgreSQL we recommend reading: http://www.revsys.com/writings/postgresql-performance.html
 
diff --git a/docs/admin/tuning-rhodecode.rst b/docs/admin/tuning-rhodecode.rst
index e15fd711..10451191 100644
--- a/docs/admin/tuning-rhodecode.rst
+++ b/docs/admin/tuning-rhodecode.rst
@@ -12,8 +12,7 @@ may find some of the following methods useful.
    tuning-vcs-memory-cache
    tuning-user-sessions-performance
    tuning-increase-db-performance
-   tuning-scale-horizontally
-   tuning-increase-cache-size
+   tuning-scale-horizontally-cluster
    tuning-mount-cache-memory
    tuning-change-encoding
    tuning-change-large-file-dir
diff --git a/docs/admin/tuning-scale-horizontally-cluster.rst b/docs/admin/tuning-scale-horizontally-cluster.rst
new file mode 100644
index 00000000..441c4c53
--- /dev/null
+++ b/docs/admin/tuning-scale-horizontally-cluster.rst
@@ -0,0 +1,383 @@
+.. _scale-horizontal-cluster:
+
+
+Scale Horizontally / RhodeCode Cluster
+--------------------------------------
+
+|RCE| is built in a way it support horizontal scaling across multiple machines.
+There are three main pre-requisites for that:
+
+- Shared storage that each machine can access. Using NFS or other shared storage system.
+- Shared DB connection across machines. Using `MySQL`/`PostgreSQL` that each node can access.
+- |RCE| user sessions and caches need to use a shared storage (e.g `Redis`_/`Memcached`)
+
+
+Horizontal scaling means adding more machines or workers into your pool of
+resources. Horizontally scaling |RCE| gives a huge performance increase,
+especially under large traffic scenarios with a high number of requests.
+This is very beneficial when |RCE| is serving many users simultaneously,
+or if continuous integration servers are automatically pulling and pushing code.
+It also adds High-Availability to your running system.
+
+
+Cluster Overview
+^^^^^^^^^^^^^^^^
+
+Below we'll present a configuration example that will use two separate nodes to serve
+|RCE| in a load-balanced environment. The 3rd node will act as a shared storage/cache
+and handle load-balancing. In addition 3rd node will be used as shared database instance.
+
+This setup can be used both in Docker based configuration or with individual
+physical/virtual machines. Using the 3rd node for Storage/Redis/PostgreSQL/Nginx is
+optional. All those components can be installed on one of the two nodes used for |RCE|.
+We'll use following naming for our nodes:
+
+ - `rc-node-1` (NFS, DB, Cache node)
+ - `rc-node-2` (Worker node1)
+ - `rc-node-3` (Worker node2)
+
+Our shares NFS storage in the example is located on `/home/rcdev/storage` and
+it's RW accessible on **each** node.
+
+In this example we used certain recommended components, however many
+of those can be replaced by other, in case your organization already uses them, for example:
+
+- `MySQL`/`PostgreSQL`: Aren't replaceable and are the two only supported databases.
+- `Nginx`_ on `rc-node-1` can be replaced by: `Hardware Load Balancer (F5)`, `Apache`_, `HA-Proxy` etc.
+- `Nginx`_ on rc-node-2/3 acts as a reverse proxy and can be replaced by other HTTP server
+  acting as reverse proxy such as `Apache`_.
+- `Redis`_ on `rc-node-1` can be replaced by: `Memcached`
+
+
+Here's an overview what components should be installed/setup on each server in our example:
+
+- **rc-node-1**:
+
+ - main storage acting as NFS host.
+ - `nginx` acting as a load-balancer.
+ - `postgresql-server` used for database and sessions.
+ - `redis-server` used for storing shared caches.
+ - optionally `rabbitmq-server` for `Celery` if used.
+ - optionally if `Celery` is used Enterprise/Community instance + VCSServer.
+ - optionally mailserver that can be shared by other instances.
+ - optionally channelstream server to handle live communication for all instances.
+
+
+- **rc-node-2/3**:
+
+ - `nginx` acting as a reverse proxy to handle requests to |RCE|.
+ - 1x RhodeCode Enterprise/Community instance.
+ - 1x VCSServer instance.
+ - optionally for testing connection: postgresql-client, redis-client (redis-tools).
+
+
+Before we start here are few assumptions that should be fulfilled:
+
+- make sure each node can access each other.
+- make sure `Redis`_/`MySQL`/`PostgreSQL`/`RabbitMQ`_ are running on `rc-node-1`
+- make sure both `rc-node-2`/`3` can access NFS storage with RW access
+- make sure rc-node-2/3 can access `Redis`_/`PostgreSQL`, `MySQL` database on `rc-node-1`.
+- make sure `Redis`_/Database/`RabbitMQ`_ are password protected and accessible only from rc-node-2/3.
+
+
+
+Setup rc-node-2/3
+^^^^^^^^^^^^^^^^^
+
+Initially before `rc-node-1` we'll configure both nodes 2 and 3 to operate as standalone
+nodes with their own hostnames. Use a default installation settings, and use
+the default local addresses (127.0.0.1) to configure VCSServer and Community/Enterprise instances.
+All external connectivity will be handled by the reverse proxy (`Nginx`_ in our example).
+
+This way we can ensure each individual host works,
+accepts connections, or do some operations explicitly on chosen node.
+
+In addition this would allow use to explicitly direct certain traffic to a node, e.g
+CI server will only call directly `rc-node-3`. This should be done similar to normal
+installation so check out `Nginx`_/`Apache`_ configuration example to configure each host.
+Each one should already connect to shared database during installation.
+
+
+1) Assuming our final url will be http://rc-node-1, Configure `instances_id`, `app.base_url`
+
+a) On **rc-node-2** find the following settings and edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+
+.. code-block:: ini
+
+    ## required format is: *NAME-
+    instance_id = *rc-node-2-
+    app.base_url = http://rc-node-1
+
+
+b) On **rc-node-3** find the following settings and edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+
+.. code-block:: ini
+
+    ## required format is: *NAME-
+    instance_id = *rc-node-3-
+    app.base_url = http://rc-node-1
+
+
+
+2) Configure `User Session` to use a shared database. Example config that should be
+   changed on both node 2 and 3. Edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+
+.. code-block:: ini
+
+    ####################################
+    ###       BEAKER SESSION        ####
+    ####################################
+
+    ## Disable the default `file` sessions
+    #beaker.session.type = file
+    #beaker.session.data_dir = %(here)s/data/sessions
+
+    ## use shared db based session, fast, and allows easy management over logged in users
+    beaker.session.type = ext:database
+    beaker.session.table_name = db_session
+    # use our rc-node-1 here
+    beaker.session.sa.url = postgresql://postgres:qweqwe@rc-node-1/rhodecode
+    beaker.session.sa.pool_recycle = 3600
+    beaker.session.sa.echo = false
+
+In addition make sure both instances use the same `session.secret` so users have
+persistent sessions across nodes. Please generate other one then in this example.
+
+.. code-block:: ini
+
+    # use an unique generated long string
+    beaker.session.secret = 70e116cae2274656ba7265fd860aebbd
+
+3) Configure stored cached/archive cache to our shared NFS `rc-node-1`
+
+.. code-block:: ini
+
+    # note the `_` prefix that allows using a directory without
+    # remap and rescan checking for vcs inside it.
+    cache_dir = /home/rcdev/storage/_cache_dir/data
+    # note archive cache dir is disabled by default, however if you enable
+    # it also needs to be shared
+    #archive_cache_dir = /home/rcdev/storage/_tarball_cache_dir
+
+
+4) Change cache backends to use `Redis`_ based caches. Below full example config
+   that replaces default file-based cache to shared `Redis`_ with Distributed Lock.
+
+
+.. code-block:: ini
+
+    #####################################
+    ###         DOGPILE CACHE        ####
+    #####################################
+
+    ## `cache_perms` cache settings for permission tree, auth TTL.
+    #rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
+    #rc_cache.cache_perms.expiration_time = 300
+
+    ## alternative `cache_perms` redis backend with distributed lock
+    rc_cache.cache_perms.backend = dogpile.cache.rc.redis
+    rc_cache.cache_perms.expiration_time = 300
+    ## redis_expiration_time needs to be greater then expiration_time
+    rc_cache.cache_perms.arguments.redis_expiration_time = 7200
+    rc_cache.cache_perms.arguments.socket_timeout = 30
+    rc_cache.cache_perms.arguments.host = rc-node-1
+    rc_cache.cache_perms.arguments.password = qweqwe
+    rc_cache.cache_perms.arguments.port = 6379
+    rc_cache.cache_perms.arguments.db = 0
+    rc_cache.cache_perms.arguments.distributed_lock = true
+
+    ## `cache_repo` cache settings for FileTree, Readme, RSS FEEDS
+    #rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
+    #rc_cache.cache_repo.expiration_time = 2592000
+
+    ## alternative `cache_repo` redis backend with distributed lock
+    rc_cache.cache_repo.backend = dogpile.cache.rc.redis
+    rc_cache.cache_repo.expiration_time = 2592000
+    ## redis_expiration_time needs to be greater then expiration_time
+    rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
+    rc_cache.cache_repo.arguments.socket_timeout = 30
+    rc_cache.cache_repo.arguments.host = rc-node-1
+    rc_cache.cache_repo.arguments.password = qweqwe
+    rc_cache.cache_repo.arguments.port = 6379
+    rc_cache.cache_repo.arguments.db = 1
+    rc_cache.cache_repo.arguments.distributed_lock = true
+
+    ## cache settings for SQL queries, this needs to use memory type backend
+    rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
+    rc_cache.sql_cache_short.expiration_time = 30
+
+    ## `cache_repo_longterm` cache for repo object instances, this needs to use memory
+    ## type backend as the objects kept are not pickle serializable
+    rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
+    ## by default we use 96H, this is using invalidation on push anyway
+    rc_cache.cache_repo_longterm.expiration_time = 345600
+    ## max items in LRU cache, reduce this number to save memory, and expire last used
+    ## cached objects
+    rc_cache.cache_repo_longterm.max_size = 10000
+
+
+4) Configure `Nginx`_ as reverse proxy on `rc-node-2/3`:
+   Minimal `Nginx`_ config used:
+
+
+.. code-block:: nginx
+
+    ## rate limiter for certain pages to prevent brute force attacks
+    limit_req_zone  $binary_remote_addr  zone=req_limit:10m   rate=1r/s;
+
+    ## custom log format
+    log_format log_custom '$remote_addr - $remote_user [$time_local] '
+                          '"$request" $status $body_bytes_sent '
+                          '"$http_referer" "$http_user_agent" '
+                          '$request_time $upstream_response_time $pipe';
+
+    server {
+        listen          80;
+        server_name     rc-node-2;
+        #server_name     rc-node-3;
+
+        access_log   /var/log/nginx/rhodecode.access.log log_custom;
+        error_log    /var/log/nginx/rhodecode.error.log;
+
+        # example of proxy.conf can be found in our docs.
+        include     /etc/nginx/proxy.conf;
+
+        ## serve static files by Nginx, recommended for performance
+        location /_static/rhodecode {
+            gzip on;
+            gzip_min_length  500;
+            gzip_proxied     any;
+            gzip_comp_level 4;
+            gzip_types  text/css text/javascript text/xml text/plain text/x-component application/javascript application/json application/xml application/rss+xml font/truetype font/opentype application/vnd.ms-fontobject image/svg+xml;
+            gzip_vary on;
+            gzip_disable     "msie6";
+            #alias /home/rcdev/.rccontrol/community-1/static;
+            alias /home/rcdev/.rccontrol/enterprise-1/static;
+        }
+
+
+        location /_admin/login {
+            limit_req  zone=req_limit  burst=10  nodelay;
+            try_files $uri @rhode;
+        }
+
+        location / {
+            try_files $uri @rhode;
+        }
+
+        location @rhode {
+            # Url to running RhodeCode instance.
+            # This is shown as `- URL: <host>` in output from rccontrol status.
+            proxy_pass      http://127.0.0.1:10020;
+        }
+
+        ## custom 502 error page. Will be displayed while RhodeCode server
+        ## is turned off
+        error_page 502 /502.html;
+        location = /502.html {
+           #root  /home/rcdev/.rccontrol/community-1/static;
+           root  /home/rcdev/.rccontrol/enterprise-1/static;
+        }
+    }
+
+
+5) Optional: Full text search, in case you use `Whoosh` full text search we also need a
+   shared storage for the index. In our example our NFS is mounted at `/home/rcdev/storage`
+   which represents out storage so we can use the following:
+
+.. code-block:: ini
+
+    # note the `_` prefix that allows using a directory without
+    # remap and rescan checking for vcs inside it.
+    search.location = /home/rcdev/storage/_index_data/index
+
+
+.. note::
+
+    If you use ElasticSearch it's by default shared, and simply running ES node is
+    by default cluster compatible.
+
+
+6) Optional: If you intend to use mailing all instances need to use either a shared
+   mailing node, or each will use individual local mailagent. Simply put node-1/2/3 needs
+   to use same mailing configuration.
+
+
+
+Setup rc-node-1
+^^^^^^^^^^^^^^^
+
+
+Configure `Nginx`_ as Load Balancer to rc-node-2/3.
+Minimal `Nginx`_ example below:
+
+.. code-block:: nginx
+
+    ## define rc-cluster which contains a pool of our instances to connect to
+    upstream rc-cluster {
+        # rc-node-2/3 are stored in /etc/hosts with correct IP addresses
+        server rc-node-2:80;
+        server rc-node-3:80;
+    }
+
+    server {
+        listen          80;
+        server_name     rc-node-1;
+
+        location / {
+            proxy_pass http://rc-cluster;
+        }
+    }
+
+
+.. note::
+
+   You should configure your load balancing accordingly. We recommend writing
+   load balancing rules that will separate regular user traffic from
+   automated process traffic like continuous servers or build bots. Sticky sessions
+   are not required.
+
+
+Show which instance handles a request
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can easily check if load-balancing is working as expected. Visit our main node
+`rc-node-1` URL which at that point should already handle incoming requests and balance
+it across node-2/3.
+
+Add a special GET param `?showrcid=1` to show current instance handling your request.
+
+For example: visiting url `http://rc-node-1/?showrcid=1` will show, in the bottom
+of the screen` cluster instance info.
+e.g: `RhodeCode instance id: rc-node-3-rc-node-3-3246`
+which is generated from::
+
+    <NODE_HOSTNAME>-<INSTANCE_ID>-<WORKER_PID>
+
+
+Using Celery with cluster
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+
+If `Celery` is used we recommend setting also an instance of Enterprise/Community+VCSserver
+on the node that is running `RabbitMQ`_. Those instances will be used to executed async
+tasks on the `rc-node-1`. This is the most efficient setup. `Celery` usually
+handles tasks such as sending emails, forking repositories, importing
+repositories from external location etc. Using workers on instance that has
+the direct access to disks used by NFS as well as email server gives noticeable
+performance boost. Running local workers to the NFS storage results in faster
+execution of forking large repositories or sending lots of emails.
+
+Those instances need to be configured in the same way as for other nodes.
+The instance in rc-node-1 can be added to the cluser, but we don't recommend doing it.
+For best results let it be isolated to only executing `Celery` tasks in the cluster setup.
+
+
+.. _Gunicorn: http://gunicorn.org/
+.. _Whoosh: https://pypi.python.org/pypi/Whoosh/
+.. _Elasticsearch: https://www.elastic.co/..
+.. _RabbitMQ: http://www.rabbitmq.com/
+.. _Nginx: http://nginx.io
+.. _Apache: http://nginx.io
+.. _Redis: http://redis.io
+
diff --git a/docs/admin/tuning-scale-horizontally.rst b/docs/admin/tuning-scale-horizontally.rst
deleted file mode 100644
index ae0f06a6..00000000
--- a/docs/admin/tuning-scale-horizontally.rst
+++ /dev/null
@@ -1,58 +0,0 @@
-.. _scale-horizontal:
-
-Scale Horizontally
-------------------
-
-|RCE| is built in a way it support horizontal scaling across multiple machines.
-There are two main pre-requisites for that:
-
-- Shared storage that each machine can access.
-- Shared DB connection across machines.
-
-
-Horizontal scaling means adding more machines or workers into your pool of
-resources. Horizontally scaling |RCE| gives a huge performance increase,
-especially under large traffic scenarios with a high number of requests. This
-is very beneficial when |RCE| is serving many users simultaneously,
-or if continuous integration servers are automatically pulling and pushing code.
-
-
-If you scale across different machines, each |RCM| instance
-needs to store its data on a shared disk, preferably together with your
-|repos|. This data directory contains template caches, a full text search index,
-and is used for task locking to ensure safety across multiple instances.
-To do this, set the following properties in the :file:`rhodecode.ini` file to
-set the shared location across all |RCM| instances.
-
-.. code-block:: ini
-
-    cache_dir = /shared/path/caches                 # set to shared location
-    search.location = /shared/path/search_index     # set to shared location
-
-    ####################################
-    ###         BEAKER CACHE        ####
-    ####################################
-    beaker.cache.data_dir = /shared/path/data       # set to shared location
-    beaker.cache.lock_dir = /shared/path/lock       # set to shared location
-
-
-.. note::
-
-    If you use custom caches such as `beaker.cache.auth_plugins.` it's recommended
-    to set it to the memcached/redis or database backend so it can be shared
-    across machines.
-
-
-It is recommended to create another dedicated |RCE| instance to handle
-traffic from build farms or continuous integration servers.
-
-.. note::
-
-   You should configure your load balancing accordingly. We recommend writing
-   load balancing rules that will separate regular user traffic from
-   automated process traffic like continuous servers or build bots.
-
-.. note::
-
-     If Celery is used on each instance then you should run separate Celery
-     instances, but the message broker should be the same for all of them.
diff --git a/docs/admin/tuning-user-sessions-performance.rst b/docs/admin/tuning-user-sessions-performance.rst
index 6bbe41eb..a6c80fc7 100644
--- a/docs/admin/tuning-user-sessions-performance.rst
+++ b/docs/admin/tuning-user-sessions-performance.rst
@@ -1,7 +1,7 @@
 .. _user-session-ref:
 
-Increase User Session Performance
----------------------------------
+User Session Performance
+------------------------
 
 The default file-based sessions are only suitable for smaller setups, or
 instances that doesn't have a lot of users or traffic.
@@ -24,8 +24,9 @@ your :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
       beaker.session.type = ext:database
       beaker.session.table_name = db_session
 
-      # use just one of the following accoring to the type of database
+      # use just one of the following according to the type of database
       beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
+      # or
       beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 
       beaker.session.sa.pool_recycle = 3600
diff --git a/docs/admin/tuning-vcs-memory-cache.rst b/docs/admin/tuning-vcs-memory-cache.rst
index 16ee07cd..ba42504b 100644
--- a/docs/admin/tuning-vcs-memory-cache.rst
+++ b/docs/admin/tuning-vcs-memory-cache.rst
@@ -1,7 +1,7 @@
 .. _adjust-vcs-mem-cache:
 
-Adjusting VCS Memory Cache
---------------------------
+VCSServer Memory Cache
+----------------------
 
 The VCS Server mamory cache can be adjusted to work best with the resources
 available to your |RCE| instance. If you find that memory resources are under

From 4a46219a597cd476e4cc7fdcc6441bb67cb28e51 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Mon, 10 Sep 2018 22:18:53 +0200
Subject: [PATCH 03/19] exc_tracker: allow setting custom store via .ini file

- use safer exc read when listing exceptions.
---
 configs/development.ini                         |  4 ++++
 configs/production.ini                          |  4 ++++
 rhodecode/apps/admin/views/exception_tracker.py | 14 ++++++++++----
 rhodecode/config/middleware.py                  |  7 +++++++
 rhodecode/lib/exc_tracking.py                   | 10 ++++------
 5 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/configs/development.ini b/configs/development.ini
index 8715e5cb..624efc84 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -292,6 +292,10 @@ supervisor.group_id = dev
 ## Display extended labs settings
 labs_settings_active = true
 
+## custom exception store path, defaults to TMPDIR
+exception_store_path =
+
+
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
diff --git a/configs/production.ini b/configs/production.ini
index dd49e349..9a0986ae 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -267,6 +267,10 @@ supervisor.group_id = prod
 ## Display extended labs settings
 labs_settings_active = true
 
+## custom exception store path, defaults to TMPDIR
+exception_store_path =
+
+
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
diff --git a/rhodecode/apps/admin/views/exception_tracker.py b/rhodecode/apps/admin/views/exception_tracker.py
index 3ef0d320..d832d0aa 100644
--- a/rhodecode/apps/admin/views/exception_tracker.py
+++ b/rhodecode/apps/admin/views/exception_tracker.py
@@ -73,10 +73,16 @@ class ExceptionsTrackerView(BaseAppView):
 
             if read_metadata:
                 full_path = os.path.join(exc_store_path, fname)
-                # we can read our metadata
-                with open(full_path, 'rb') as f:
-                    exc_metadata = exc_tracking.exc_unserialize(f.read())
-                    exc.update(exc_metadata)
+                if not os.path.isfile(full_path):
+                    continue
+                try:
+                    # we can read our metadata
+                    with open(full_path, 'rb') as f:
+                        exc_metadata = exc_tracking.exc_unserialize(f.read())
+                        exc.update(exc_metadata)
+                except Exception:
+                    log.exception('Failed to read exc data from:{}'.format(full_path))
+                    pass
 
             # convert our timestamp to a date obj, for nicer representation
             exc['exc_utc_date'] = time_to_utcdatetime(exc['exc_timestamp'])
diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py
index a64e2664..2ea487bb 100644
--- a/rhodecode/config/middleware.py
+++ b/rhodecode/config/middleware.py
@@ -436,6 +436,7 @@ def _sanitize_vcs_settings(settings):
 
 
 def _sanitize_cache_settings(settings):
+
     default_cache_dir = os.path.join(tempfile.gettempdir(), 'rc_cache')
 
     # save default, cache dir, and use it for all backends later.
@@ -448,6 +449,12 @@ def _sanitize_cache_settings(settings):
     if not os.path.isdir(default_cache_dir):
         os.makedirs(default_cache_dir, mode=0755)
 
+    # exception store cache
+    _string_setting(
+        settings,
+        'exception_store_path',
+        default_cache_dir, lower=False)
+
     # cache_perms
     _string_setting(
         settings,
diff --git a/rhodecode/lib/exc_tracking.py b/rhodecode/lib/exc_tracking.py
index f803e08b..de47f06f 100644
--- a/rhodecode/lib/exc_tracking.py
+++ b/rhodecode/lib/exc_tracking.py
@@ -31,6 +31,7 @@ log = logging.getLogger(__name__)
 
 # NOTE: Any changes should be synced with exc_tracking at vcsserver.lib.exc_tracking
 global_prefix = 'rhodecode'
+exc_store_dir_name = 'rc_exception_store_v1'
 
 
 def exc_serialize(exc_id, tb, exc_type):
@@ -54,13 +55,10 @@ def get_exc_store():
     """
     Get and create exception store if it's not existing
     """
-    exc_store_dir = 'rc_exception_store_v1'
-    # fallback
-    _exc_store_path = os.path.join(tempfile.gettempdir(), exc_store_dir)
+    import rhodecode as app
 
-    exc_store_dir = ''  # TODO: need a persistent cross instance store here
-    if exc_store_dir:
-        _exc_store_path = os.path.join(exc_store_dir, exc_store_dir)
+    exc_store_dir = app.CONFIG.get('exception_store_path', '') or tempfile.gettempdir()
+    _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
 
     _exc_store_path = os.path.abspath(_exc_store_path)
     if not os.path.isdir(_exc_store_path):

From fcdd57f4417795b908e02355f2792c72c98e3ef7 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Mon, 10 Sep 2018 21:05:38 +0200
Subject: [PATCH 04/19] celery: use exc_tracker to store tasks exceptions for
 easier debugging.

---
 rhodecode/lib/celerylib/loader.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/rhodecode/lib/celerylib/loader.py b/rhodecode/lib/celerylib/loader.py
index 10925c99..b847c5ff 100644
--- a/rhodecode/lib/celerylib/loader.py
+++ b/rhodecode/lib/celerylib/loader.py
@@ -164,7 +164,14 @@ def task_retry_signal(
 @signals.task_failure.connect
 def task_failure_signal(
         task_id, exception, args, kwargs, traceback, einfo, **kargs):
+    from rhodecode.lib.exc_tracking import store_exception
+
     meta.Session.remove()
+
+    # simulate sys.exc_info()
+    exc_info = (einfo.type, einfo.exception, einfo.tb)
+    store_exception(id(exc_info), exc_info, prefix='celery_rhodecode')
+
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()

From 98133036cd2d725de24291a1db5b893bb58bcd90 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Tue, 11 Sep 2018 11:31:53 +0200
Subject: [PATCH 05/19] svn: use shared configurable storage for svn_txn_id
 interception logic.

---
 rhodecode/lib/hooks_daemon.py         | 10 ++++++++--
 rhodecode/lib/middleware/simplesvn.py |  4 ++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/rhodecode/lib/hooks_daemon.py b/rhodecode/lib/hooks_daemon.py
index 9fc5ed64..bb5a225a 100644
--- a/rhodecode/lib/hooks_daemon.py
+++ b/rhodecode/lib/hooks_daemon.py
@@ -190,8 +190,14 @@ class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
 
 
 def get_txn_id_data_path(txn_id):
-    root = tempfile.gettempdir()
-    return os.path.join(root, 'rc_txn_id_{}'.format(txn_id))
+    import rhodecode
+
+    root = rhodecode.CONFIG.get('cache_dir') or tempfile.gettempdir()
+    final_dir = os.path.join(root, 'svn_txn_id')
+
+    if not os.path.isdir(final_dir):
+        os.makedirs(final_dir)
+    return os.path.join(final_dir, 'rc_txn_id_{}'.format(txn_id))
 
 
 def store_txn_id_data(txn_id, data_dict):
diff --git a/rhodecode/lib/middleware/simplesvn.py b/rhodecode/lib/middleware/simplesvn.py
index d59eb730..d62a8263 100644
--- a/rhodecode/lib/middleware/simplesvn.py
+++ b/rhodecode/lib/middleware/simplesvn.py
@@ -74,10 +74,10 @@ class SimpleSvnApp(object):
 
         if response.status_code not in [200, 401]:
             if response.status_code >= 500:
-                log.error('Got SVN response:%s with text:`%s`',
+                log.error('Got SVN response:%s with text:\n`%s`',
                           response, response.text)
             else:
-                log.debug('Got SVN response:%s with text:`%s`',
+                log.debug('Got SVN response:%s with text:\n`%s`',
                           response, response.text)
         else:
             log.debug('got response code: %s', response.status_code)

From d62c9827496f078c22c50fa138e5ce58afdb0492 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Tue, 11 Sep 2018 14:22:22 +0200
Subject: [PATCH 06/19] svn: use streaming uploads/downloads of files.

- before we buffered whole content before uploading/download
- this could lead to OOM errors on big files
- now instead we stream when PUT/GET are sent from SVN resulting in faster more efficient data transfer
---
 rhodecode/lib/middleware/simplesvn.py | 49 +++++++++++++++++++--------
 1 file changed, 35 insertions(+), 14 deletions(-)

diff --git a/rhodecode/lib/middleware/simplesvn.py b/rhodecode/lib/middleware/simplesvn.py
index d62a8263..66233344 100644
--- a/rhodecode/lib/middleware/simplesvn.py
+++ b/rhodecode/lib/middleware/simplesvn.py
@@ -21,7 +21,7 @@
 import base64
 import logging
 import urllib
-from urlparse import urljoin
+import urlparse
 
 import requests
 from pyramid.httpexceptions import HTTPNotAcceptable
@@ -48,29 +48,50 @@ class SimpleSvnApp(object):
 
     def __call__(self, environ, start_response):
         request_headers = self._get_request_headers(environ)
-
         data = environ['wsgi.input']
-        # johbo: Avoid that we end up with sending the request in chunked
-        # transfer encoding (mainly on Gunicorn). If we know the content
-        # length, then we should transfer the payload in one request.
-        if environ['REQUEST_METHOD'] == 'MKCOL' or 'CONTENT_LENGTH' in environ:
-            data = data.read()
-            if data.startswith('(create-txn-with-props'):
+        req_method = environ['REQUEST_METHOD']
+        has_content_length = 'CONTENT_LENGTH' in environ
+        path_info = self._get_url(environ['PATH_INFO'])
+        transfer_encoding = environ.get('HTTP_TRANSFER_ENCODING', '')
+        log.debug('Handling: %s method via `%s`', req_method, path_info)
+
+        # stream control flag, based on request and content type...
+        stream = False
+
+        if req_method in ['MKCOL'] or has_content_length:
+            data_processed = False
+            # read chunk to check if we have txn-with-props
+            initial_data = data.read(1024)
+            if initial_data.startswith('(create-txn-with-props'):
+                data = initial_data + data.read()
                 # store on-the-fly our rc_extra using svn revision properties
                 # those can be read later on in hooks executed so we have a way
                 # to pass in the data into svn hooks
                 rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
                 rc_data_len = len(rc_data)
-                # header defines data lenght, and serialized data
+                # header defines data length, and serialized data
                 skel = ' rc-scm-extras {} {}'.format(rc_data_len, rc_data)
                 data = data[:-2] + skel + '))'
+                data_processed = True
 
-        log.debug('Calling: %s method via `%s`', environ['REQUEST_METHOD'],
-                  self._get_url(environ['PATH_INFO']))
+            if not data_processed:
+                # NOTE(johbo): Avoid that we end up with sending the request in chunked
+                # transfer encoding (mainly on Gunicorn). If we know the content
+                # length, then we should transfer the payload in one request.
+                data = initial_data + data.read()
 
+        if req_method in ['GET', 'PUT'] or transfer_encoding == 'chunked':
+            # NOTE(marcink): when getting/uploading files we want to STREAM content
+            # back to the client/proxy instead of buffering it here...
+            stream = True
+
+        stream = stream
+        log.debug(
+            'Calling SVN PROXY: method:%s via `%s`, Stream: %s',
+            req_method, path_info, stream)
         response = requests.request(
-            environ['REQUEST_METHOD'], self._get_url(environ['PATH_INFO']),
-            data=data, headers=request_headers)
+            req_method, path_info,
+            data=data, headers=request_headers, stream=stream)
 
         if response.status_code not in [200, 401]:
             if response.status_code >= 500:
@@ -97,7 +118,7 @@ class SimpleSvnApp(object):
         return response.iter_content(chunk_size=1024)
 
     def _get_url(self, path):
-        url_path = urljoin(
+        url_path = urlparse.urljoin(
             self.config.get('subversion_http_server_url', ''), path)
         url_path = urllib.quote(url_path, safe="/:=~+!$,;'")
         return url_path

From ad23d1ec1ff555998de62243892d4b700a71fb53 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Wed, 12 Sep 2018 10:43:48 +0200
Subject: [PATCH 07/19] docs: deprecate beaker.cache in docs

---
 docs/admin/enable-debug.rst |  8 +-------
 docs/admin/vcs-server.rst   | 38 ++++++++++++++++++-------------------
 2 files changed, 20 insertions(+), 26 deletions(-)

diff --git a/docs/admin/enable-debug.rst b/docs/admin/enable-debug.rst
index 0d61f8f6..19cfa039 100644
--- a/docs/admin/enable-debug.rst
+++ b/docs/admin/enable-debug.rst
@@ -47,7 +47,7 @@ the ``debug`` level.
     ### LOGGING CONFIGURATION   ####
     ################################
     [loggers]
-    keys = root, sqlalchemy, beaker, rhodecode, ssh_wrapper
+    keys = root, sqlalchemy, rhodecode, ssh_wrapper
 
     [handlers]
     keys = console, console_sql, file, file_rotating
@@ -69,12 +69,6 @@ the ``debug`` level.
     ## "level = DEBUG" logs the route matched and routing variables.
     propagate = 1
 
-    [logger_beaker]
-    level = DEBUG
-    handlers =
-    qualname = beaker.container
-    propagate = 1
-
     [logger_rhodecode]
     level = DEBUG
     handlers =
diff --git a/docs/admin/vcs-server.rst b/docs/admin/vcs-server.rst
index 8f07438e..8bd9e368 100644
--- a/docs/admin/vcs-server.rst
+++ b/docs/admin/vcs-server.rst
@@ -113,33 +113,33 @@ match, for example:
 VCS Server Memory Optimization
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-To configure the VCS server to manage the cache efficiently, you need to
+To optimize the VCS server to manage the cache and memory usage efficiently, you need to
 configure the following options in the
 :file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.ini` file. Once
-configured, restart the VCS Server.
+configured, restart the VCS Server. By default we use an optimal settings, but in certain
+conditions tunning expiration_time and max_size can affect memory usage and performance
 
-.. rst-class:: dl-horizontal
+.. code-block:: ini
 
-    \beaker.cache.repo_object.type = memorylru
-        Configures the cache to discard the least recently used items.
-        This setting takes the following valid options:
+    ## cache region for storing repo_objects cache
+    rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
 
-        * ``memorylru``: The default setting, which removes the least recently
-          used items from the cache.
-        * ``memory``: Runs the VCS Server without clearing the cache.
-        * ``nocache``: Runs the VCS Server without a cache. This will
-          dramatically reduce the VCS Server performance.
+    ## cache auto-expires after N seconds, setting this to 0 disabled cache
+    rc_cache.repo_object.expiration_time = 300
 
-    \beaker.cache.repo_object.max_items = 100
-        Sets the maximum number of items stored in the cache, before the cache
-        starts to be cleared.
+    ## max size of LRU, old values will be discarded if the size of cache reaches max_size
+    ## Sets the maximum number of items stored in the cache, before the cache
+    ## starts to be cleared.
 
-        As a general rule of thumb, running this value at 120 resulted in a
-        5GB cache. Running it at 240 resulted in a 9GB cache. Your results
-        will differ based on usage patterns and |repo| sizes.
+    ## As a general rule of thumb, running this value at 120 resulted in a
+    ## 5GB cache. Running it at 240 resulted in a 9GB cache. Your results
+    ## will differ based on usage patterns and |repo| sizes.
+
+    ## Tweaking this value to run at a fairly constant memory load on your
+    ## server will help performance.
+
+    rc_cache.repo_object.max_size = 120
 
-        Tweaking this value to run at a fairly constant memory load on your
-        server will help performance.
 
 To clear the cache completely, you can restart the VCS Server.
 

From ec2a3553cd1fcc5eb3876b121d5ab729926ae7ad Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Thu, 13 Sep 2018 14:35:14 +0200
Subject: [PATCH 08/19] exception_store: rename .ini option for future

---
 configs/development.ini        | 14 +-------------
 configs/production.ini         | 13 +------------
 rhodecode/config/middleware.py |  2 +-
 rhodecode/lib/exc_tracking.py  |  2 +-
 4 files changed, 4 insertions(+), 27 deletions(-)

diff --git a/configs/development.ini b/configs/development.ini
index 624efc84..0e5a7c6a 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -20,17 +20,6 @@ debug = true
 ## email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
-## Uncomment and replace with the address which should receive any error report
-## note: using appenlight for error handling doesn't need this to be uncommented
-#email_to = admin@localhost
-
-## in case of Application errors, sent an error email form
-#error_email_from = rhodecode_error@localhost
-
-## additional error message to be send in case of server crash
-#error_message =
-
-
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
@@ -293,8 +282,7 @@ supervisor.group_id = dev
 labs_settings_active = true
 
 ## custom exception store path, defaults to TMPDIR
-exception_store_path =
-
+exception_tracker.store_path =
 
 ####################################
 ###        CELERY CONFIG        ####
diff --git a/configs/production.ini b/configs/production.ini
index 9a0986ae..058e9a97 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -20,17 +20,6 @@ debug = true
 ## email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
-## Uncomment and replace with the address which should receive any error report
-## note: using appenlight for error handling doesn't need this to be uncommented
-#email_to = admin@localhost
-
-## in case of Application errors, sent an error email form
-#error_email_from = rhodecode_error@localhost
-
-## additional error message to be send in case of server crash
-#error_message =
-
-
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
@@ -268,7 +257,7 @@ supervisor.group_id = prod
 labs_settings_active = true
 
 ## custom exception store path, defaults to TMPDIR
-exception_store_path =
+exception_tracker.store_path =
 
 
 ####################################
diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py
index 2ea487bb..58bdc8d1 100644
--- a/rhodecode/config/middleware.py
+++ b/rhodecode/config/middleware.py
@@ -452,7 +452,7 @@ def _sanitize_cache_settings(settings):
     # exception store cache
     _string_setting(
         settings,
-        'exception_store_path',
+        'exception_tracker.store_path',
         default_cache_dir, lower=False)
 
     # cache_perms
diff --git a/rhodecode/lib/exc_tracking.py b/rhodecode/lib/exc_tracking.py
index de47f06f..368cdda3 100644
--- a/rhodecode/lib/exc_tracking.py
+++ b/rhodecode/lib/exc_tracking.py
@@ -57,7 +57,7 @@ def get_exc_store():
     """
     import rhodecode as app
 
-    exc_store_dir = app.CONFIG.get('exception_store_path', '') or tempfile.gettempdir()
+    exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
     _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
 
     _exc_store_path = os.path.abspath(_exc_store_path)

From 26ec398df0a2127dc07fd83089b81dd42471ff6b Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Fri, 14 Sep 2018 18:24:51 +0200
Subject: [PATCH 09/19] audit-logs: added entries for branch permissions

---
 rhodecode/lib/audit_logger.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/rhodecode/lib/audit_logger.py b/rhodecode/lib/audit_logger.py
index 172d611e..baf9773a 100644
--- a/rhodecode/lib/audit_logger.py
+++ b/rhodecode/lib/audit_logger.py
@@ -64,10 +64,16 @@ ACTIONS_V1 = {
     'repo.fork': {'data': {}},
     'repo.edit': {'old_data': {}},
     'repo.edit.permissions': {},
+    'repo.edit.permissions.branch': {},
     'repo.delete': {'old_data': {}},
-    'repo.commit.strip': {'commit_id': ''},
+
     'repo.archive.download': {'user_agent': '', 'archive_name': '',
                               'archive_spec': '', 'archive_cached': ''},
+
+    'repo.permissions.branch_rule.create': {},
+    'repo.permissions.branch_rule.edit': {},
+    'repo.permissions.branch_rule.delete': {},
+
     'repo.pull_request.create': '',
     'repo.pull_request.edit': '',
     'repo.pull_request.delete': '',
@@ -80,6 +86,7 @@ ACTIONS_V1 = {
     'repo.pull_request.reviewer.add': '',
     'repo.pull_request.reviewer.delete': '',
 
+    'repo.commit.strip': {'commit_id': ''},
     'repo.commit.comment.create': {'data': {}},
     'repo.commit.comment.delete': {'data': {}},
     'repo.commit.vote': '',
@@ -89,6 +96,7 @@ ACTIONS_V1 = {
     'repo_group.edit.permissions': {},
     'repo_group.delete': {'old_data': {}},
 }
+
 ACTIONS = ACTIONS_V1
 
 SOURCE_WEB = 'source_web'

From 5c62c69d2aa8a35776523ada87bbdd0d3de50703 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Mon, 17 Sep 2018 19:05:34 +0200
Subject: [PATCH 10/19] comments: fix extracing auth_user from the passed in
 objects. Before if auth_user is empty we could relly on INT or STR passed in

---
 rhodecode/model/comment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rhodecode/model/comment.py b/rhodecode/model/comment.py
index 687567b6..4d6f6455 100644
--- a/rhodecode/model/comment.py
+++ b/rhodecode/model/comment.py
@@ -191,7 +191,6 @@ class CommentsModel(BaseModel):
         :param renderer: pick renderer for this comment
         """
 
-        auth_user = auth_user or user
         if not text:
             log.warning('Missing text for comment, skipping...')
             return
@@ -203,6 +202,7 @@ class CommentsModel(BaseModel):
 
         repo = self._get_repo(repo)
         user = self._get_user(user)
+        auth_user = auth_user or user
 
         schema = comment_schema.CommentSchema()
         validated_kwargs = schema.deserialize(dict(

From f466a753216c0cf39b1422c07e63d69396badc91 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Tue, 18 Sep 2018 09:49:18 +0200
Subject: [PATCH 11/19] comments: use proper auth user for close PR action.

---
 rhodecode/api/views/pull_request_api.py               | 2 +-
 rhodecode/apps/repository/views/repo_pull_requests.py | 3 ++-
 rhodecode/model/pull_request.py                       | 5 +++--
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/rhodecode/api/views/pull_request_api.py b/rhodecode/api/views/pull_request_api.py
index 4bf87862..a74b5dc9 100644
--- a/rhodecode/api/views/pull_request_api.py
+++ b/rhodecode/api/views/pull_request_api.py
@@ -924,7 +924,7 @@ def close_pull_request(
 
     # finally close the PR, with proper message comment
     comment, status = PullRequestModel().close_pull_request_with_comment(
-        pull_request, apiuser, repo, message=message)
+        pull_request, apiuser, repo, message=message, auth_user=apiuser)
     status_lbl = ChangesetStatus.get_status_lbl(status)
 
     Session().commit()
diff --git a/rhodecode/apps/repository/views/repo_pull_requests.py b/rhodecode/apps/repository/views/repo_pull_requests.py
index 212fa21a..ee5a0aad 100644
--- a/rhodecode/apps/repository/views/repo_pull_requests.py
+++ b/rhodecode/apps/repository/views/repo_pull_requests.py
@@ -1195,7 +1195,8 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
                           'pull request %s', pull_request_id)
                 raise HTTPForbidden()
             comment, status = PullRequestModel().close_pull_request_with_comment(
-                pull_request, self._rhodecode_user, self.db_repo, message=text)
+                pull_request, self._rhodecode_user, self.db_repo, message=text,
+                auth_user=self._rhodecode_user)
             Session().flush()
             events.trigger(
                 events.PullRequestCommentEvent(pull_request, comment))
diff --git a/rhodecode/model/pull_request.py b/rhodecode/model/pull_request.py
index 37398adf..e537cd68 100644
--- a/rhodecode/model/pull_request.py
+++ b/rhodecode/model/pull_request.py
@@ -1174,7 +1174,7 @@ class PullRequestModel(BaseModel):
             'repo.pull_request.close', {'data': pr_data}, user, pull_request)
 
     def close_pull_request_with_comment(
-            self, pull_request, user, repo, message=None):
+            self, pull_request, user, repo, message=None, auth_user=None):
 
         pull_request_review_status = pull_request.calculated_review_status()
 
@@ -1198,7 +1198,8 @@ class PullRequestModel(BaseModel):
             pull_request=pull_request.pull_request_id,
             status_change=status_lbl,
             status_change_type=status,
-            closing_pr=True
+            closing_pr=True,
+            auth_user=auth_user,
         )
 
         # calculate old status before we change it

From 169bb210f03b130b08ec461abaee41e2bb96dd63 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Tue, 18 Sep 2018 10:30:58 +0200
Subject: [PATCH 12/19] exception_tracker: use a default value of exception
 store that is working across all instances e.g vcsserver and enterprise.

---
 rhodecode/config/middleware.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py
index 58bdc8d1..5915f326 100644
--- a/rhodecode/config/middleware.py
+++ b/rhodecode/config/middleware.py
@@ -436,8 +436,8 @@ def _sanitize_vcs_settings(settings):
 
 
 def _sanitize_cache_settings(settings):
-
-    default_cache_dir = os.path.join(tempfile.gettempdir(), 'rc_cache')
+    temp_store = tempfile.gettempdir()
+    default_cache_dir = os.path.join(temp_store, 'rc_cache')
 
     # save default, cache dir, and use it for all backends later.
     default_cache_dir = _string_setting(
@@ -453,7 +453,7 @@ def _sanitize_cache_settings(settings):
     _string_setting(
         settings,
         'exception_tracker.store_path',
-        default_cache_dir, lower=False)
+        temp_store, lower=False, default_when_empty=True)
 
     # cache_perms
     _string_setting(

From a25f2b21af180aa4d8a8c520cf3b5f9f56317507 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Tue, 18 Sep 2018 10:01:06 +0200
Subject: [PATCH 13/19] docs: added 4.13 release note about potential encoding
 problems

---
 docs/release-notes/release-notes-4.13.0.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/release-notes/release-notes-4.13.0.rst b/docs/release-notes/release-notes-4.13.0.rst
index c850bfce..bbb2d20d 100644
--- a/docs/release-notes/release-notes-4.13.0.rst
+++ b/docs/release-notes/release-notes-4.13.0.rst
@@ -131,3 +131,7 @@ Upgrade notes
   generated rhodecode.template.ini file. This file is stored next to rhodecode.ini
 - SVN 1.10.2 was introduced in this release. Please make sure to update your
   mod_dav to the same version for best compatibility.
+- This release brings new Database drivers. We discovered that in some setups it is now
+  required to explicitly define a encoding charset in the database connection string.
+  If you're getting a permanent exception such as `"UnicodeDecodeError: 'utf8' codec can't decode byte 0xfc in position 15: invalid start byte"`
+  please follow a fix from here: https://community.rhodecode.com/t/unicodedecodeerror-utf8-codec-cant-decode-byte-0xfc-in-position-15-invalid-start-byte/246

From a3e123e7d093cf8c2d7be781f97822330cac296b Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Tue, 18 Sep 2018 10:09:37 +0200
Subject: [PATCH 14/19] docs: update instructions for shared exception store in
 cluster setup

---
 .../tuning-scale-horizontally-cluster.rst     | 26 ++++++++++++++-----
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/docs/admin/tuning-scale-horizontally-cluster.rst b/docs/admin/tuning-scale-horizontally-cluster.rst
index 441c4c53..3c7b55b5 100644
--- a/docs/admin/tuning-scale-horizontally-cluster.rst
+++ b/docs/admin/tuning-scale-horizontally-cluster.rst
@@ -120,7 +120,8 @@ b) On **rc-node-3** find the following settings and edit :file:`/home/{user}/.rc
 
 
 2) Configure `User Session` to use a shared database. Example config that should be
-   changed on both node 2 and 3. Edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+   changed on both **rc-node-2** and **rc-node-3** .
+   Edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
 
 .. code-block:: ini
 
@@ -160,7 +161,18 @@ persistent sessions across nodes. Please generate other one then in this example
     #archive_cache_dir = /home/rcdev/storage/_tarball_cache_dir
 
 
-4) Change cache backends to use `Redis`_ based caches. Below full example config
+4) Use shared exception store. Example config that should be
+   changed on both **rc-node-2** and **rc-node-3**, and also for VCSServer.
+   Edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` and
+   :file:`/home/{user}/.rccontrol/{vcsserver-instance-id}/vcsserver.ini`
+   and add/change following setting.
+
+.. code-block:: ini
+
+    exception_tracker.store_path = /home/rcdev/storage/_exception_store_data
+
+
+5) Change cache backends to use `Redis`_ based caches. Below full example config
    that replaces default file-based cache to shared `Redis`_ with Distributed Lock.
 
 
@@ -216,7 +228,7 @@ persistent sessions across nodes. Please generate other one then in this example
     rc_cache.cache_repo_longterm.max_size = 10000
 
 
-4) Configure `Nginx`_ as reverse proxy on `rc-node-2/3`:
+6) Configure `Nginx`_ as reverse proxy on `rc-node-2/3`:
    Minimal `Nginx`_ config used:
 
 
@@ -281,7 +293,7 @@ persistent sessions across nodes. Please generate other one then in this example
     }
 
 
-5) Optional: Full text search, in case you use `Whoosh` full text search we also need a
+7) Optional: Full text search, in case you use `Whoosh` full text search we also need a
    shared storage for the index. In our example our NFS is mounted at `/home/rcdev/storage`
    which represents out storage so we can use the following:
 
@@ -298,9 +310,9 @@ persistent sessions across nodes. Please generate other one then in this example
     by default cluster compatible.
 
 
-6) Optional: If you intend to use mailing all instances need to use either a shared
-   mailing node, or each will use individual local mailagent. Simply put node-1/2/3 needs
-   to use same mailing configuration.
+8) Optional: If you intend to use mailing all instances need to use either a shared
+   mailing node, or each will use individual local mail agent. Simply put node-1/2/3
+   needs to use same mailing configuration.
 
 
 

From 3e18692b8d796c5284a2d7e5268a535b50593221 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Tue, 18 Sep 2018 12:15:05 +0200
Subject: [PATCH 15/19] docs: added release notes for 4.13.2 release

---
 docs/release-notes/release-notes-4.13.2.rst | 51 +++++++++++++++++++++
 docs/release-notes/release-notes.rst        |  1 +
 2 files changed, 52 insertions(+)
 create mode 100644 docs/release-notes/release-notes-4.13.2.rst

diff --git a/docs/release-notes/release-notes-4.13.2.rst b/docs/release-notes/release-notes-4.13.2.rst
new file mode 100644
index 00000000..a74b79a7
--- /dev/null
+++ b/docs/release-notes/release-notes-4.13.2.rst
@@ -0,0 +1,51 @@
+|RCE| 4.13.2 |RNS|
+------------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2018-08-18
+
+
+New Features
+^^^^^^^^^^^^
+
+- audit-logs: add audit logs for branch permissions.
+
+
+General
+^^^^^^^
+
+- svn: properly handle credentials from URL during remote repository import.
+- svn: use more detailed logs/errors so `exception_tracker` can show it with details.
+- svn: use streaming uploads/downloads of files.
+- svn: use shared configurable storage for svn_txn_id interception logic.
+- celery: use `exception_tracker` to store tasks exceptions for easier debugging.
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+
+
+Fixes
+^^^^^
+
+- exception_tracker: use a default value of exception store that is working across
+  all instances e.g vcsserver and enterprise. This fixed problem with usage in
+  cluster type setup.
+- Comments: fixed problem with audit-logs receiving a empty value in certain
+  conditions resulting in 500 error on commenting.
+- Branch Permissions: during creation of rule add special check for existing entries to
+  prevent duplicates if two users edit branch permissions at the same time.
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- Scheduled release addressing reported problems, and improving stability.
diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst
index 5ddded51..8fc7d6b4 100644
--- a/docs/release-notes/release-notes.rst
+++ b/docs/release-notes/release-notes.rst
@@ -9,6 +9,7 @@ Release Notes
 .. toctree::
    :maxdepth: 1
 
+   release-notes-4.13.2.rst
    release-notes-4.13.1.rst
    release-notes-4.13.0.rst
    release-notes-4.12.4.rst

From 326fb491eb50111cb69b0abecd90d75434a5d2ee Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Tue, 18 Sep 2018 13:19:01 +0200
Subject: [PATCH 16/19] release: Start preparation for 4.13.2

---
 .bumpversion.cfg  | 2 +-
 .release.cfg      | 9 ++-------
 rhodecode/VERSION | 2 +-
 3 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 1eb1b8f0..357a2f9c 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.13.1
+current_version = 4.13.2
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:rhodecode/VERSION]
diff --git a/.release.cfg b/.release.cfg
index f673b1c8..131a5c78 100644
--- a/.release.cfg
+++ b/.release.cfg
@@ -5,23 +5,18 @@ done = false
 done = true
 
 [task:rc_tools_pinned]
-done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [task:changelog_updated]
-done = true
 
 [task:generate_api_docs]
-done = true
 
 [release]
-state = prepared
-version = 4.13.1
+state = in_progress
+version = 4.13.2
 
 [task:updated_translation]
 
diff --git a/rhodecode/VERSION b/rhodecode/VERSION
index 6a6a6933..79f0f240 100644
--- a/rhodecode/VERSION
+++ b/rhodecode/VERSION
@@ -1 +1 @@
-4.13.1
\ No newline at end of file
+4.13.2
\ No newline at end of file

From c143a872e413f2bbf90a4b06ed641f66629a0446 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Tue, 18 Sep 2018 13:27:43 +0200
Subject: [PATCH 17/19] release: updated pip2nix output for 4.13.2

---
 .release.cfg             | 4 ++++
 pkgs/python-packages.nix | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.release.cfg b/.release.cfg
index 131a5c78..13ada4a5 100644
--- a/.release.cfg
+++ b/.release.cfg
@@ -5,14 +5,18 @@ done = false
 done = true
 
 [task:rc_tools_pinned]
+done = true
 
 [task:fixes_on_stable]
+done = true
 
 [task:pip2nix_generated]
 
 [task:changelog_updated]
+done = true
 
 [task:generate_api_docs]
+done = true
 
 [release]
 state = in_progress
diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix
index 0ef0b61e..79777196 100644
--- a/pkgs/python-packages.nix
+++ b/pkgs/python-packages.nix
@@ -1632,7 +1632,7 @@ self: super: {
     };
   };
   "rhodecode-enterprise-ce" = super.buildPythonPackage {
-    name = "rhodecode-enterprise-ce-4.13.1";
+    name = "rhodecode-enterprise-ce-4.13.2";
     buildInputs = [
       self."pytest"
       self."py"

From 62bf76f9f1dfb530ec27198fab838cff892d7a34 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Tue, 18 Sep 2018 13:27:46 +0200
Subject: [PATCH 18/19] release: Finish preparation for 4.13.2

---
 .release.cfg | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.release.cfg b/.release.cfg
index 13ada4a5..86972d43 100644
--- a/.release.cfg
+++ b/.release.cfg
@@ -11,6 +11,7 @@ done = true
 done = true
 
 [task:pip2nix_generated]
+done = true
 
 [task:changelog_updated]
 done = true
@@ -19,7 +20,7 @@ done = true
 done = true
 
 [release]
-state = in_progress
+state = prepared
 version = 4.13.2
 
 [task:updated_translation]

From 81ca38390064a5a3fd8b6077455ddd3c361440b4 Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Tue, 18 Sep 2018 13:27:47 +0200
Subject: [PATCH 19/19] Added tag v4.13.2 for changeset 610d62155052