integrations: implement retry to HTTP[S] calls for integrations.

rhodecode/integrations/types/base.py

@@ -22,6 +22,9 @@ import colander
 import string
 import collections
 import logging
+import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
 
 from mako import exceptions
 
@@ -320,3 +323,33 @@ def render_with_traceback(template, *args, **kwargs):
     except Exception:
         log.error(exceptions.text_error_template().render())
         raise
+
+
+STATUS_400 = (400, 401, 403)
+STATUS_500 = (500, 502, 504)
+
+
+def requests_retry_call(
+        retries=3, backoff_factor=0.3, status_forcelist=STATUS_400+STATUS_500,
+        session=None):
+    """
+    session = requests_retry_session()
+    response = session.get('http://example.com')
+
+    :param retries:
+    :param backoff_factor:
+    :param status_forcelist:
+    :param session:
+    """
+    session = session or requests.Session()
+    retry = Retry(
+        total=retries,
+        read=retries,
+        connect=retries,
+        backoff_factor=backoff_factor,
+        status_forcelist=status_forcelist,
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
+    return session

rhodecode/integrations/types/hipchat.py

@@ -31,7 +31,8 @@ from rhodecode.lib import helpers as h
 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
 from rhodecode.lib.colander_utils import strip_whitespace
 from rhodecode.integrations.types.base import (
-    IntegrationTypeBase, CommitParsingDataHandler, render_with_traceback)
+    IntegrationTypeBase, CommitParsingDataHandler, render_with_traceback,
+    requests_retry_call)
 
 log = logging.getLogger(__name__)
 
@@ -248,6 +249,6 @@ def post_text_to_hipchat(settings, text):
         "color": settings.get('color', 'yellow'),
         "notify": settings.get('notify', False),
     }
-
-    resp = requests.post(settings['server_url'], json=json_message, timeout=60)
+    req_session = requests_retry_call()
+    resp = req_session.post(settings['server_url'], json=json_message, timeout=60)
     resp.raise_for_status()  # raise exception on a failed request

rhodecode/integrations/types/slack.py

@@ -35,7 +35,8 @@ from rhodecode.lib import helpers as h
 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
 from rhodecode.lib.colander_utils import strip_whitespace
 from rhodecode.integrations.types.base import (
-    IntegrationTypeBase, CommitParsingDataHandler, render_with_traceback)
+    IntegrationTypeBase, CommitParsingDataHandler, render_with_traceback,
+    requests_retry_call)
 
 log = logging.getLogger(__name__)
 
@@ -344,6 +345,6 @@ def post_text_to_slack(settings, title, text, fields=None, overrides=None):
         "username": settings.get('username', 'Rhodecode'),
         "attachments": [message_data]
     }
-
-    resp = requests.post(settings['service'], json=json_message, timeout=60)
+    req_session = requests_retry_call()
+    resp = req_session.post(settings['service'], json=json_message, timeout=60)
     resp.raise_for_status()  # raise exception on a failed request

rhodecode/integrations/types/webhook.py

@@ -23,17 +23,14 @@ from __future__ import unicode_literals
 import deform
 import deform.widget
 import logging
-import requests
-import requests.adapters
 import colander
-from requests.packages.urllib3.util.retry import Retry
 
 import rhodecode
 from rhodecode import events
 from rhodecode.translation import _
 from rhodecode.integrations.types.base import (
     IntegrationTypeBase, get_auth, get_web_token, get_url_vars,
-    WebhookDataHandler, WEBHOOK_URL_VARS)
+    WebhookDataHandler, WEBHOOK_URL_VARS, requests_retry_call)
 from rhodecode.lib.celerylib import run_task, async_task, RequestContextTask
 from rhodecode.model.validation_schema import widgets
 
@@ -233,11 +230,6 @@ def post_to_webhook(url_calls, settings):
          'utc_timestamp': datetime.datetime(2017, 11, 30, 13, 0, 1, 569276)
 
     """
-    max_retries = 3
-    retries = Retry(
-        total=max_retries,
-        backoff_factor=0.15,
-        status_forcelist=[500, 502, 503, 504])
     call_headers = {
         'User-Agent': 'RhodeCode-webhook-caller/{}'.format(
             rhodecode.__version__)
@@ -247,9 +239,7 @@ def post_to_webhook(url_calls, settings):
     token = get_web_token(settings)
 
     for url, headers, data in url_calls:
-        req_session = requests.Session()
-        req_session.mount(  # retry max N times
-            'http://', requests.adapters.HTTPAdapter(max_retries=retries))
+        req_session = requests_retry_call()
 
         method = settings.get('method_type') or 'post'
         call_method = getattr(req_session, method)