From 633c58187ead8ea7d238a79aa276324445d9156c Mon Sep 17 00:00:00 2001
From: Marcin Kuzminski <marcin@rhodecode.com>
Date: Tue, 3 Sep 2019 11:42:10 +0200
Subject: [PATCH] packages: updated celery to 4.3.0 and switch default backend
 to redis

- we usually have redis, and it makes our stack simpler and easier to debug.
---
 configs/development.ini                       |  7 +-
 configs/production.ini                        |  7 +-
 .../tuning-scale-horizontally-cluster.rst     | 10 +--
 docs/install/configure-celery.rst             | 20 +++--
 pkgs/python-packages.nix                      | 74 +++++++++++++++----
 requirements.txt                              |  8 +-
 requirements_pinned.txt                       |  2 +-
 7 files changed, 94 insertions(+), 34 deletions(-)

diff --git a/configs/development.ini b/configs/development.ini
index 3bd3c90f..bfb97ea3 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -307,8 +307,11 @@ file_store.storage_path = %(here)s/data/file_store
 
 use_celery = false
 
-## connection url to the message broker (default rabbitmq)
-celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
+## connection url to the message broker (default redis)
+celery.broker_url = redis://localhost:6379/8
+
+## rabbitmq example
+#celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ## maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 100
diff --git a/configs/production.ini b/configs/production.ini
index 2f24f63f..50391a1a 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -282,8 +282,11 @@ file_store.storage_path = %(here)s/data/file_store
 
 use_celery = false
 
-## connection url to the message broker (default rabbitmq)
-celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
+## connection url to the message broker (default redis)
+celery.broker_url = redis://localhost:6379/8
+
+## rabbitmq example
+#celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ## maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 100
diff --git a/docs/admin/system_admin/tuning/tuning-scale-horizontally-cluster.rst b/docs/admin/system_admin/tuning/tuning-scale-horizontally-cluster.rst
index d7e57c93..64ff2234 100644
--- a/docs/admin/system_admin/tuning/tuning-scale-horizontally-cluster.rst
+++ b/docs/admin/system_admin/tuning/tuning-scale-horizontally-cluster.rst
@@ -57,7 +57,7 @@ Here's an overview what components should be installed/setup on each server in o
  - `nginx` acting as a load-balancer.
  - `postgresql-server` used for database and sessions.
  - `redis-server` used for storing shared caches.
- - optionally `rabbitmq-server` for `Celery` if used.
+ - optionally `rabbitmq-server` or `redis` for `Celery` if used.
  - optionally if `Celery` is used Enterprise/Community instance + VCSServer.
  - optionally mailserver that can be shared by other instances.
  - optionally channelstream server to handle live communication for all instances.
@@ -372,16 +372,16 @@ Using Celery with cluster
 
 
 If `Celery` is used we recommend setting also an instance of Enterprise/Community+VCSserver
-on the node that is running `RabbitMQ`_. Those instances will be used to executed async
-tasks on the `rc-node-1`. This is the most efficient setup. `Celery` usually
-handles tasks such as sending emails, forking repositories, importing
+on the node that is running `RabbitMQ`_ or `Redis`_. Those instances will be used to
+executed async tasks on the `rc-node-1`. This is the most efficient setup.
+`Celery` usually handles tasks such as sending emails, forking repositories, importing
 repositories from external location etc. Using workers on instance that has
 the direct access to disks used by NFS as well as email server gives noticeable
 performance boost. Running local workers to the NFS storage results in faster
 execution of forking large repositories or sending lots of emails.
 
 Those instances need to be configured in the same way as for other nodes.
-The instance in rc-node-1 can be added to the cluser, but we don't recommend doing it.
+The instance in rc-node-1 can be added to the cluster, but we don't recommend doing it.
 For best results let it be isolated to only executing `Celery` tasks in the cluster setup.
 
 
diff --git a/docs/install/configure-celery.rst b/docs/install/configure-celery.rst
index b26eda00..ea38cfd7 100644
--- a/docs/install/configure-celery.rst
+++ b/docs/install/configure-celery.rst
@@ -11,16 +11,20 @@ and import repositories in async way. It is also used for bi-directional
 repository sync in scheduler.
 
 If you decide to use Celery you also need a working message queue.
-The recommended and fully supported message broker is rabbitmq_.
+There are two fully supported message brokers is rabbitmq_ and redis_ (recommended).
+
+Since release 4.18.X we recommend using redis_ as a backend since it's generally
+easier to work with, and results in simpler stack as redis is generally recommended
+for caching purposes.
 
 
 In order to install and configure Celery, follow these steps:
 
-1. Install RabbitMQ, see the documentation on the Celery website for
-   `rabbitmq installation`_, or `rabbitmq website installation`_
+1. Install RabbitMQ or Redis for a message queue, see the documentation on the Celery website for
+   `redis installation`_ or `rabbitmq installation`_
 
 
-1a. As en example configuration after installation, you can run::
+1a. If you choose RabbitMQ example configuration after installation would look like that::
 
    sudo rabbitmqctl add_user rcuser secret_password
    sudo rabbitmqctl add_vhost rhodevhost
@@ -45,6 +49,10 @@ In order to install and configure Celery, follow these steps:
    Set the broker_url as minimal settings required to enable operation.
    If used our example data from pt 1a, here is how the broker url should look like::
 
+        # for Redis
+        celery.broker_url = redis://localhost:6379/8
+
+        # for RabbitMQ
         celery.broker_url = amqp://rcuser:secret_password@localhost:5672/rhodevhost
 
    Full configuration example is below:
@@ -57,7 +65,7 @@ In order to install and configure Celery, follow these steps:
         ####################################
 
         use_celery = true
-        celery.broker_url = amqp://rcuser:secret@localhost:5672/rhodevhost
+        celery.broker_url = redis://localhost:6379/8
 
         # maximum tasks to execute before worker restart
         celery.max_tasks_per_child = 100
@@ -69,6 +77,8 @@ In order to install and configure Celery, follow these steps:
 .. _python: http://www.python.org/
 .. _mercurial: http://mercurial.selenic.com/
 .. _celery: http://celeryproject.org/
+.. _redis: http://redis.io
+.. _redis installation: https://redis.io/topics/quickstart
 .. _rabbitmq: http://www.rabbitmq.com/
 .. _rabbitmq installation: http://docs.celeryproject.org/en/latest/getting-started/brokers/rabbitmq.html
 .. _rabbitmq website installation: http://www.rabbitmq.com/download.html
diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix
index 8e03a163..d55ea99f 100644
--- a/pkgs/python-packages.nix
+++ b/pkgs/python-packages.nix
@@ -22,14 +22,14 @@ self: super: {
     };
   };
   "amqp" = super.buildPythonPackage {
-    name = "amqp-2.3.1";
+    name = "amqp-2.5.1";
     doCheck = false;
     propagatedBuildInputs = [
       self."vine"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/1b/32/242ff76cd802766f11c89c72f3389b5c8de4bdfbab406137b90c5fae8b05/amqp-2.3.1.tar.gz";
-      sha256 = "0wlfnvhmfrn7c8qif2jyvsm63ibdxp02ss564qwrvqfhz0di72s0";
+      url = "https://files.pythonhosted.org/packages/b5/f5/70e364a1f5fbafc742c098ad88a064b801b0d69cf56bfad13be2c08be4e2/amqp-2.5.1.tar.gz";
+      sha256 = "0s2yxnnhhx9hww0n33yn22q6sgnbd6n2nw92050qv2qpc3i1ga8r";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -135,11 +135,11 @@ self: super: {
     };
   };
   "billiard" = super.buildPythonPackage {
-    name = "billiard-3.5.0.3";
+    name = "billiard-3.6.1.0";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/39/ac/f5571210cca2e4f4532e38aaff242f26c8654c5e2436bee966c230647ccc/billiard-3.5.0.3.tar.gz";
-      sha256 = "1riwiiwgb141151md4ykx49qrz749akj5k8g290ji9bsqjyj4yqx";
+      url = "https://files.pythonhosted.org/packages/68/1d/2aea8fbb0b1e1260a8a2e77352de2983d36d7ac01207cf14c2b9c6cc860e/billiard-3.6.1.0.tar.gz";
+      sha256 = "09hzy3aqi7visy4vmf4xiish61n0rq5nd3iwjydydps8yrs9r05q";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -172,16 +172,17 @@ self: super: {
     };
   };
   "celery" = super.buildPythonPackage {
-    name = "celery-4.1.1";
+    name = "celery-4.3.0";
     doCheck = false;
     propagatedBuildInputs = [
       self."pytz"
       self."billiard"
       self."kombu"
+      self."vine"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/e9/cf/a4c0597effca20c57eb586324e41d1180bc8f13a933da41e0646cff69f02/celery-4.1.1.tar.gz";
-      sha256 = "1xbir4vw42n2ir9lanhwl7w69zpmj7lbi66fxm2b7pyvkcss7wni";
+      url = "https://files.pythonhosted.org/packages/a2/4b/d020836f751617e907e84753a41c92231cd4b673ff991b8ee9da52361323/celery-4.3.0.tar.gz";
+      sha256 = "1y8y0gbgkwimpxqnxq2rm5qz2vy01fvjiybnpm00y5rzd2m34iac";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -284,6 +285,17 @@ self: super: {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  "contextlib2" = super.buildPythonPackage {
+    name = "contextlib2-0.5.5";
+    doCheck = false;
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/6e/db/41233498c210b03ab8b072c8ee49b1cd63b3b0c76f8ea0a0e5d02df06898/contextlib2-0.5.5.tar.gz";
+      sha256 = "0j6ad6lwwyc9kv71skj098v5l7x5biyj2hs4lc5x1kcixqcr97sh";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.psfl ];
+    };
+  };
   "cov-core" = super.buildPythonPackage {
     name = "cov-core-1.15.0";
     doCheck = false;
@@ -665,6 +677,23 @@ self: super: {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
+  "importlib-metadata" = super.buildPythonPackage {
+    name = "importlib-metadata-0.20";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."zipp"
+      self."contextlib2"
+      self."configparser"
+      self."pathlib2"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/05/41/7d339dd7b507e97f67be812fdf29c4ad991ddd34b1ed0f3c54e8f1c4e0b3/importlib_metadata-0.20.tar.gz";
+      sha256 = "13bshj8i98l9gxi6df4xbw1262phmawgr527as20brblwf93a55p";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.asl20 ];
+    };
+  };
   "infrae.cache" = super.buildPythonPackage {
     name = "infrae.cache-1.0.1";
     doCheck = false;
@@ -848,14 +877,15 @@ self: super: {
     };
   };
   "kombu" = super.buildPythonPackage {
-    name = "kombu-4.2.1";
+    name = "kombu-4.6.4";
     doCheck = false;
     propagatedBuildInputs = [
       self."amqp"
+      self."importlib-metadata"
     ];
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/39/9f/556b988833abede4a80dbd18b2bdf4e8ff4486dd482ed45da961347e8ed2/kombu-4.2.1.tar.gz";
-      sha256 = "10lh3hncvw67fz0k5vgbx3yh9gjfpqdlia1f13i28cgnc1nfrbc6";
+      url = "https://files.pythonhosted.org/packages/52/f2/5a64fc850b0533d2daf09a523406e51e85a8b2a4a2bc87a922a8906ba2aa/kombu-4.6.4.tar.gz";
+      sha256 = "16w02mvkxchz7041yia4h8xmqavci88szk18ynxvw4chzcnk3w75";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
@@ -1636,11 +1666,11 @@ self: super: {
     };
   };
   "pytz" = super.buildPythonPackage {
-    name = "pytz-2018.4";
+    name = "pytz-2019.2";
     doCheck = false;
     src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/10/76/52efda4ef98e7544321fd8d5d512e11739c1df18b0649551aeccfb1c8376/pytz-2018.4.tar.gz";
-      sha256 = "0jgpqx3kk2rhv81j1izjxvmx8d0x7hzs1857pgqnixic5wq2ar60";
+      url = "https://files.pythonhosted.org/packages/27/c0/fbd352ca76050952a03db776d241959d5a2ee1abddfeb9e2a53fdb489be4/pytz-2019.2.tar.gz";
+      sha256 = "0ckb27hhjc8i8gcdvk4d9avld62b7k52yjijc60s2m3y8cpb7h16";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
@@ -2271,6 +2301,20 @@ self: super: {
       license = [ { fullName = "PSF or ZPL"; } ];
     };
   };
+  "zipp" = super.buildPythonPackage {
+    name = "zipp-0.6.0";
+    doCheck = false;
+    propagatedBuildInputs = [
+      self."more-itertools"
+    ];
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/57/dd/585d728479d97d25aeeb9aa470d36a4ad8d0ba5610f84e14770128ce6ff7/zipp-0.6.0.tar.gz";
+      sha256 = "13ndkf7vklw978a4gdl1yfvn8hch28429a0iam67sg4nrp5v261p";
+    };
+    meta = {
+      license = [ pkgs.lib.licenses.mit ];
+    };
+  };
   "zope.cachedescriptors" = super.buildPythonPackage {
     name = "zope.cachedescriptors-4.3.1";
     doCheck = false;
diff --git a/requirements.txt b/requirements.txt
index 741474b4..937c5732 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,11 +1,11 @@
 ## dependencies
 
-amqp==2.3.1
+amqp==2.5.1
 
 babel==1.3
 beaker==1.9.1
 bleach==3.1.0
-celery==4.1.1
+celery==4.3.0
 channelstream==0.5.2
 click==7.0
 colander==1.7.0
@@ -24,7 +24,7 @@ futures==3.0.2
 infrae.cache==1.0.1
 iso8601==0.1.12
 itsdangerous==0.24
-kombu==4.2.1
+kombu==4.6.4
 lxml==4.2.5
 mako==1.0.7
 markdown==2.6.11
@@ -52,7 +52,7 @@ python-ldap==3.1.0
 python-memcached==1.59
 python-pam==1.8.4
 python-saml==2.4.2
-pytz==2018.4
+pytz==2019.2
 tzlocal==1.5.1
 pyzmq==14.6.0
 py-gfm==0.1.4
diff --git a/requirements_pinned.txt b/requirements_pinned.txt
index 9a3f1de5..f85090ef 100644
--- a/requirements_pinned.txt
+++ b/requirements_pinned.txt
@@ -2,7 +2,7 @@
 
 atomicwrites==1.2.1
 attrs==18.2.0
-billiard==3.5.0.3
+billiard==3.6.1.0
 chameleon==2.24
 cffi==1.12.2
 ecdsa==0.13.2