release: merge back stable branch into default

docs/release-notes/release-notes-4.24.0.rst

@@ -1,4 +1,4 @@
-|RCE| 4.23.0 |RNS|
+|RCE| 4.24.0 |RNS|
 ------------------
 
 Release Date
@@ -16,14 +16,16 @@ New Features
   Can be used for backups etc.
 - Pull requests: expose commit versions in the pull-request commit list.
 
+
 General
 ^^^^^^^
 
 - Deps: bumped redis to 3.5.3
-- rcextensions: improve examples
+- Rcextensions: improve examples for some usage.
 - Setup: added optional parameters to apply a default license, or skip re-creation of database at install.
 - Docs: update headers for NGINX
 - Beaker cache: remove no longer used beaker cache init
+- Installation: the installer no longer requires gzip and bzip packages, and works on python 2 and 3
 
 
 Security

docs/release-notes/release-notes-4.24.1.rst

@@ -0,0 +1,46 @@
+|RCE| 4.24.1 |RNS|
+------------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2021-02-04
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+- Core: added statsd client for statistics usage.
+- Clone urls: allow custom clone by id template so users can set clone-by-id as default.
+- Automation: enable check for new version for EE edition as automation task that will send notifications when new RhodeCode version is available
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+- Core: bumped git to 2.30.0
+
+
+Fixes
+^^^^^
+
+- Comments: add ability to resolve todos from the side-bar. This should prevent situations
+  when a TODO was left over in outdated/removed code pieces, and users needs to search to resolve them.
+- Pull requests: fixed a case when template marker was used in description field causing 500 errors on commenting.
+- Merges: fixed excessive data saved in merge metadata that could not fit inside the DB table.
+- Exceptions: fixed problem with exceptions formatting resulting in limited exception data reporting.
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- Un-scheduled release addressing problems in 4.24.X releases.

docs/release-notes/release-notes.rst

@@ -9,6 +9,7 @@ Release Notes
 .. toctree::
    :maxdepth: 1
 
+   release-notes-4.24.1.rst
    release-notes-4.24.0.rst
    release-notes-4.23.2.rst
    release-notes-4.23.1.rst

pkgs/patches/pytest/setuptools.patch

@@ -0,0 +1,12 @@
+diff -rup pytest-4.6.5-orig/setup.py pytest-4.6.5/setup.py
+--- pytest-4.6.5-orig/setup.py	2018-04-10 10:23:04.000000000 +0200
++++  pytest-4.6.5/setup.py	2018-04-10 10:23:34.000000000 +0200
+@@ -24,7 +24,7 @@ INSTALL_REQUIRES = [
+ def main():
+     setup(
+         use_scm_version={"write_to": "src/_pytest/_version.py"},
+-        setup_requires=["setuptools-scm", "setuptools>=40.0"],
++        setup_requires=["setuptools-scm", "setuptools<=42.0"],
+         package_dir={"": "src"},
+         # fmt: off
+         extras_require={

pkgs/python-packages-overrides.nix

@@ -274,6 +274,12 @@ self: super: {
     ];
   });
 
+  "pytest" = super."pytest".override (attrs: {
+    patches = [
+      ./patches/pytest/setuptools.patch
+    ];
+  });
+
   # Avoid that base packages screw up the build process
   inherit (basePythonPackages)
     setuptools;

rhodecode/__init__.py

@@ -48,7 +48,7 @@ PYRAMID_SETTINGS = {}
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 112  # defines current db version for migrations
+__dbversion__ = 113  # defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'

rhodecode/apps/admin/views/settings.py

@@ -384,6 +384,7 @@ class AdminSettingsView(BaseAppView):
                 ('markup_renderer', 'rhodecode_markup_renderer', 'unicode'),
                 ('gravatar_url', 'rhodecode_gravatar_url', 'unicode'),
                 ('clone_uri_tmpl', 'rhodecode_clone_uri_tmpl', 'unicode'),
+                ('clone_uri_id_tmpl', 'rhodecode_clone_uri_id_tmpl', 'unicode'),
                 ('clone_uri_ssh_tmpl', 'rhodecode_clone_uri_ssh_tmpl', 'unicode'),
                 ('support_url', 'rhodecode_support_url', 'unicode'),
                 ('show_revision_number', 'rhodecode_show_revision_number', 'bool'),

rhodecode/apps/debug_style/views.py

@@ -102,6 +102,11 @@ Check if we should use full-topic or mini-topic.
                 'date': datetime.datetime.now(),
             },
 
+            'update_available': {
+                'current_ver': '4.23.0',
+                'latest_ver': '4.24.0',
+            },
+
             'exception': {
                 'email_prefix': '[RHODECODE ERROR]',
                 'exc_id': exc_traceback['exc_id'],

rhodecode/apps/repository/tests/test_repo_pullrequests.py

@@ -420,6 +420,27 @@ class TestPullrequestsView(object):
         assert pull_request.title == 'New title'
         assert pull_request.description == 'New description'
 
+    def test_edit_title_description(self, pr_util, csrf_token):
+        pull_request = pr_util.create_pull_request()
+        pull_request_id = pull_request.pull_request_id
+
+        response = self.app.post(
+            route_path('pullrequest_update',
+                repo_name=pull_request.target_repo.repo_name,
+                pull_request_id=pull_request_id),
+            params={
+                'edit_pull_request': 'true',
+                'title': 'New title {} {2} {foo}',
+                'description': 'New description',
+                'csrf_token': csrf_token})
+
+        assert_session_flash(
+            response, u'Pull request title & description updated.',
+            category='success')
+
+        pull_request = PullRequest.get(pull_request_id)
+        assert pull_request.title_safe == 'New title {{}} {{2}} {{foo}}'
+
     def test_edit_title_description_closed(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id

rhodecode/apps/repository/views/repo_summary.py

@@ -83,14 +83,10 @@ class RepoSummaryView(RepoAppView):
         if self._rhodecode_user.username != User.DEFAULT_USER:
             username = safe_str(self._rhodecode_user.username)
 
-        _def_clone_uri = _def_clone_uri_id = c.clone_uri_tmpl
+        _def_clone_uri = c.clone_uri_tmpl
+        _def_clone_uri_id = c.clone_uri_id_tmpl
         _def_clone_uri_ssh = c.clone_uri_ssh_tmpl
 
-        if '{repo}' in _def_clone_uri:
-            _def_clone_uri_id = _def_clone_uri.replace('{repo}', '_{repoid}')
-        elif '{repoid}' in _def_clone_uri:
-            _def_clone_uri_id = _def_clone_uri.replace('_{repoid}', '{repo}')
-
         c.clone_repo_url = self.db_repo.clone_url(
             user=username, uri_tmpl=_def_clone_uri)
         c.clone_repo_url_id = self.db_repo.clone_url(

rhodecode/config/middleware.py

@@ -340,6 +340,10 @@ def includeme(config, auth_resources=None):
         'rhodecode.lib.request_counter.get_request_counter',
         'request_count')
 
+    config.add_request_method(
+        'rhodecode.lib._vendor.statsd.get_statsd_client',
+        'statsd', reify=True)
+
     # Set the authorization policy.
     authz_policy = ACLAuthorizationPolicy()
     config.set_authorization_policy(authz_policy)

rhodecode/lib/_vendor/statsd/__init__.py

@@ -0,0 +1,46 @@
+from __future__ import absolute_import, division, unicode_literals
+
+import logging
+
+from .stream import TCPStatsClient, UnixSocketStatsClient  # noqa
+from .udp import StatsClient  # noqa
+
+HOST = 'localhost'
+PORT = 8125
+IPV6 = False
+PREFIX = None
+MAXUDPSIZE = 512
+
+log = logging.getLogger('rhodecode.statsd')
+
+
+def statsd_config(config, prefix='statsd.'):
+    _config = {}
+    for key in config.keys():
+        if key.startswith(prefix):
+            _config[key[len(prefix):]] = config[key]
+    return _config
+
+
+def client_from_config(configuration, prefix='statsd.', **kwargs):
+    from pyramid.settings import asbool
+
+    _config = statsd_config(configuration, prefix)
+    statsd_enabled = asbool(_config.pop('enabled', False))
+    if not statsd_enabled:
+        log.debug('statsd client not enabled by statsd.enabled =  flag, skipping...')
+        return
+
+    host = _config.pop('statsd_host', HOST)
+    port = _config.pop('statsd_port', PORT)
+    prefix = _config.pop('statsd_prefix', PREFIX)
+    maxudpsize = _config.pop('statsd_maxudpsize', MAXUDPSIZE)
+    ipv6 = asbool(_config.pop('statsd_ipv6', IPV6))
+    log.debug('configured statsd client %s:%s', host, port)
+
+    return StatsClient(
+        host=host, port=port, prefix=prefix, maxudpsize=maxudpsize, ipv6=ipv6)
+
+
+def get_statsd_client(request):
+    return client_from_config(request.registry.settings)

rhodecode/lib/_vendor/statsd/base.py

@@ -0,0 +1,107 @@
+from __future__ import absolute_import, division, unicode_literals
+
+import random
+from collections import deque
+from datetime import timedelta
+
+from .timer import Timer
+
+
+class StatsClientBase(object):
+    """A Base class for various statsd clients."""
+
+    def close(self):
+        """Used to close and clean up any underlying resources."""
+        raise NotImplementedError()
+
+    def _send(self):
+        raise NotImplementedError()
+
+    def pipeline(self):
+        raise NotImplementedError()
+
+    def timer(self, stat, rate=1):
+        return Timer(self, stat, rate)
+
+    def timing(self, stat, delta, rate=1):
+        """
+        Send new timing information.
+
+        `delta` can be either a number of milliseconds or a timedelta.
+        """
+        if isinstance(delta, timedelta):
+            # Convert timedelta to number of milliseconds.
+            delta = delta.total_seconds() * 1000.
+        self._send_stat(stat, '%0.6f|ms' % delta, rate)
+
+    def incr(self, stat, count=1, rate=1):
+        """Increment a stat by `count`."""
+        self._send_stat(stat, '%s|c' % count, rate)
+
+    def decr(self, stat, count=1, rate=1):
+        """Decrement a stat by `count`."""
+        self.incr(stat, -count, rate)
+
+    def gauge(self, stat, value, rate=1, delta=False):
+        """Set a gauge value."""
+        if value < 0 and not delta:
+            if rate < 1:
+                if random.random() > rate:
+                    return
+            with self.pipeline() as pipe:
+                pipe._send_stat(stat, '0|g', 1)
+                pipe._send_stat(stat, '%s|g' % value, 1)
+        else:
+            prefix = '+' if delta and value >= 0 else ''
+            self._send_stat(stat, '%s%s|g' % (prefix, value), rate)
+
+    def set(self, stat, value, rate=1):
+        """Set a set value."""
+        self._send_stat(stat, '%s|s' % value, rate)
+
+    def _send_stat(self, stat, value, rate):
+        self._after(self._prepare(stat, value, rate))
+
+    def _prepare(self, stat, value, rate):
+        if rate < 1:
+            if random.random() > rate:
+                return
+            value = '%s|@%s' % (value, rate)
+
+        if self._prefix:
+            stat = '%s.%s' % (self._prefix, stat)
+
+        return '%s:%s' % (stat, value)
+
+    def _after(self, data):
+        if data:
+            self._send(data)
+
+
+class PipelineBase(StatsClientBase):
+
+    def __init__(self, client):
+        self._client = client
+        self._prefix = client._prefix
+        self._stats = deque()
+
+    def _send(self):
+        raise NotImplementedError()
+
+    def _after(self, data):
+        if data is not None:
+            self._stats.append(data)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, typ, value, tb):
+        self.send()
+
+    def send(self):
+        if not self._stats:
+            return
+        self._send()
+
+    def pipeline(self):
+        return self.__class__(self)

rhodecode/lib/_vendor/statsd/stream.py

@@ -0,0 +1,75 @@
+from __future__ import absolute_import, division, unicode_literals
+
+import socket
+
+from .base import StatsClientBase, PipelineBase
+
+
+class StreamPipeline(PipelineBase):
+    def _send(self):
+        self._client._after('\n'.join(self._stats))
+        self._stats.clear()
+
+
+class StreamClientBase(StatsClientBase):
+    def connect(self):
+        raise NotImplementedError()
+
+    def close(self):
+        if self._sock and hasattr(self._sock, 'close'):
+            self._sock.close()
+        self._sock = None
+
+    def reconnect(self):
+        self.close()
+        self.connect()
+
+    def pipeline(self):
+        return StreamPipeline(self)
+
+    def _send(self, data):
+        """Send data to statsd."""
+        if not self._sock:
+            self.connect()
+        self._do_send(data)
+
+    def _do_send(self, data):
+        self._sock.sendall(data.encode('ascii') + b'\n')
+
+
+class TCPStatsClient(StreamClientBase):
+    """TCP version of StatsClient."""
+
+    def __init__(self, host='localhost', port=8125, prefix=None,
+                 timeout=None, ipv6=False):
+        """Create a new client."""
+        self._host = host
+        self._port = port
+        self._ipv6 = ipv6
+        self._timeout = timeout
+        self._prefix = prefix
+        self._sock = None
+
+    def connect(self):
+        fam = socket.AF_INET6 if self._ipv6 else socket.AF_INET
+        family, _, _, _, addr = socket.getaddrinfo(
+            self._host, self._port, fam, socket.SOCK_STREAM)[0]
+        self._sock = socket.socket(family, socket.SOCK_STREAM)
+        self._sock.settimeout(self._timeout)
+        self._sock.connect(addr)
+
+
+class UnixSocketStatsClient(StreamClientBase):
+    """Unix domain socket version of StatsClient."""
+
+    def __init__(self, socket_path, prefix=None, timeout=None):
+        """Create a new client."""
+        self._socket_path = socket_path
+        self._timeout = timeout
+        self._prefix = prefix
+        self._sock = None
+
+    def connect(self):
+        self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self._sock.settimeout(self._timeout)
+        self._sock.connect(self._socket_path)

rhodecode/lib/_vendor/statsd/timer.py

@@ -0,0 +1,71 @@
+from __future__ import absolute_import, division, unicode_literals
+
+import functools
+
+# Use timer that's not susceptible to time of day adjustments.
+try:
+    # perf_counter is only present on Py3.3+
+    from time import perf_counter as time_now
+except ImportError:
+    # fall back to using time
+    from time import time as time_now
+
+
+def safe_wraps(wrapper, *args, **kwargs):
+    """Safely wraps partial functions."""
+    while isinstance(wrapper, functools.partial):
+        wrapper = wrapper.func
+    return functools.wraps(wrapper, *args, **kwargs)
+
+
+class Timer(object):
+    """A context manager/decorator for statsd.timing()."""
+
+    def __init__(self, client, stat, rate=1):
+        self.client = client
+        self.stat = stat
+        self.rate = rate
+        self.ms = None
+        self._sent = False
+        self._start_time = None
+
+    def __call__(self, f):
+        """Thread-safe timing function decorator."""
+        @safe_wraps(f)
+        def _wrapped(*args, **kwargs):
+            start_time = time_now()
+            try:
+                return f(*args, **kwargs)
+            finally:
+                elapsed_time_ms = 1000.0 * (time_now() - start_time)
+                self.client.timing(self.stat, elapsed_time_ms, self.rate)
+        return _wrapped
+
+    def __enter__(self):
+        return self.start()
+
+    def __exit__(self, typ, value, tb):
+        self.stop()
+
+    def start(self):
+        self.ms = None
+        self._sent = False
+        self._start_time = time_now()
+        return self
+
+    def stop(self, send=True):
+        if self._start_time is None:
+            raise RuntimeError('Timer has not started.')
+        dt = time_now() - self._start_time
+        self.ms = 1000.0 * dt  # Convert to milliseconds.
+        if send:
+            self.send()
+        return self
+
+    def send(self):
+        if self.ms is None:
+            raise RuntimeError('No data recorded.')
+        if self._sent:
+            raise RuntimeError('Already sent data.')
+        self._sent = True
+        self.client.timing(self.stat, self.ms, self.rate)

rhodecode/lib/_vendor/statsd/udp.py

@@ -0,0 +1,55 @@
+from __future__ import absolute_import, division, unicode_literals
+
+import socket
+
+from .base import StatsClientBase, PipelineBase
+
+
+class Pipeline(PipelineBase):
+
+    def __init__(self, client):
+        super(Pipeline, self).__init__(client)
+        self._maxudpsize = client._maxudpsize
+
+    def _send(self):
+        data = self._stats.popleft()
+        while self._stats:
+            # Use popleft to preserve the order of the stats.
+            stat = self._stats.popleft()
+            if len(stat) + len(data) + 1 >= self._maxudpsize:
+                self._client._after(data)
+                data = stat
+            else:
+                data += '\n' + stat
+        self._client._after(data)
+
+
+class StatsClient(StatsClientBase):
+    """A client for statsd."""
+
+    def __init__(self, host='localhost', port=8125, prefix=None,
+                 maxudpsize=512, ipv6=False):
+        """Create a new client."""
+        fam = socket.AF_INET6 if ipv6 else socket.AF_INET
+        family, _, _, _, addr = socket.getaddrinfo(
+            host, port, fam, socket.SOCK_DGRAM)[0]
+        self._addr = addr
+        self._sock = socket.socket(family, socket.SOCK_DGRAM)
+        self._prefix = prefix
+        self._maxudpsize = maxudpsize
+
+    def _send(self, data):
+        """Send data to statsd."""
+        try:
+            self._sock.sendto(data.encode('ascii'), self._addr)
+        except (socket.error, RuntimeError):
+            # No time for love, Dr. Jones!
+            pass
+
+    def close(self):
+        if self._sock and hasattr(self._sock, 'close'):
+            self._sock.close()
+        self._sock = None
+
+    def pipeline(self):
+        return Pipeline(self)

rhodecode/lib/base.py

@@ -342,6 +342,7 @@ def attach_context_attributes(context, request, user_id=None, is_api=None):
     if request.GET.get('default_encoding'):
         context.default_encodings.insert(0, request.GET.get('default_encoding'))
     context.clone_uri_tmpl = rc_config.get('rhodecode_clone_uri_tmpl')
+    context.clone_uri_id_tmpl = rc_config.get('rhodecode_clone_uri_id_tmpl')
     context.clone_uri_ssh_tmpl = rc_config.get('rhodecode_clone_uri_ssh_tmpl')
 
     # INI stored

rhodecode/lib/celerylib/tasks.py

@@ -33,9 +33,9 @@ from email.utils import formatdate
 
 import rhodecode
 from rhodecode.lib import audit_logger
-from rhodecode.lib.celerylib import get_logger, async_task, RequestContextTask
+from rhodecode.lib.celerylib import get_logger, async_task, RequestContextTask, run_task
 from rhodecode.lib import hooks_base
-from rhodecode.lib.utils2 import safe_int, str2bool
+from rhodecode.lib.utils2 import safe_int, str2bool, aslist
 from rhodecode.model.db import (
     Session, IntegrityError, true, Repository, RepoGroup, User)
 
@@ -338,15 +338,39 @@ def repo_maintenance(repoid):
 
 
 @async_task(ignore_result=True)
-def check_for_update():
+def check_for_update(send_email_notification=True, email_recipients=None):
     from rhodecode.model.update import UpdateModel
+    from rhodecode.model.notification import EmailNotificationModel
+
+    log = get_logger(check_for_update)
     update_url = UpdateModel().get_update_url()
     cur_ver = rhodecode.__version__
 
     try:
         data = UpdateModel().get_update_data(update_url)
-        latest = data['versions'][0]
-        UpdateModel().store_version(latest['version'])
+
+        current_ver = UpdateModel().get_stored_version(fallback=cur_ver)
+        latest_ver = data['versions'][0]['version']
+        UpdateModel().store_version(latest_ver)
+
+        if send_email_notification:
+            log.debug('Send email notification is enabled. '
+                      'Current RhodeCode version: %s, latest known: %s', current_ver, latest_ver)
+            if UpdateModel().is_outdated(current_ver, latest_ver):
+
+                email_kwargs = {
+                    'current_ver': current_ver,
+                    'latest_ver': latest_ver,
+                }
+
+                (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
+                    EmailNotificationModel.TYPE_UPDATE_AVAILABLE, **email_kwargs)
+
+                email_recipients = aslist(email_recipients, sep=',') or \
+                                   [user.email for user in User.get_all_super_admins()]
+                run_task(send_email, email_recipients, subject,
+                         email_body_plaintext, email_body)
+
     except Exception:
         pass
 

rhodecode/lib/db_manage.py

@@ -595,12 +595,13 @@ class DbManage(object):
             # Visual
             ('show_public_icon', True, 'bool'),
             ('show_private_icon', True, 'bool'),
-            ('stylify_metatags', False, 'bool'),
+            ('stylify_metatags', True, 'bool'),
             ('dashboard_items', 100, 'int'),
             ('admin_grid_items', 25, 'int'),
 
             ('markup_renderer', 'markdown', 'unicode'),
 
+            ('repository_fields', True, 'bool'),
             ('show_version', True, 'bool'),
             ('show_revision_number', True, 'bool'),
             ('show_sha_length', 12, 'int'),
@@ -609,6 +610,7 @@ class DbManage(object):
             ('gravatar_url', User.DEFAULT_GRAVATAR_URL, 'unicode'),
 
             ('clone_uri_tmpl', Repository.DEFAULT_CLONE_URI, 'unicode'),
+            ('clone_uri_id_tmpl', Repository.DEFAULT_CLONE_URI_ID, 'unicode'),
             ('clone_uri_ssh_tmpl', Repository.DEFAULT_CLONE_URI_SSH, 'unicode'),
             ('support_url', '', 'unicode'),
             ('update_url', RhodeCodeSetting.DEFAULT_UPDATE_URL, 'unicode'),

rhodecode/lib/dbmigrate/versions/113_version_4_24_0.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+
+import logging
+from sqlalchemy import *
+
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+
+from rhodecode.lib.dbmigrate.versions import _reset_base
+from rhodecode.model import meta, init_model_encryption
+
+
+log = logging.getLogger(__name__)
+
+
+def upgrade(migrate_engine):
+    """
+    Upgrade operations go here.
+    Don't create your own engine; bind migrate_engine to your metadata
+    """
+    _reset_base(migrate_engine)
+    from rhodecode.lib.dbmigrate.schema import db_4_20_0_0 as db
+
+    init_model_encryption(db)
+
+    # issue fixups
+    fixups(db, meta.Session)
+
+
+def downgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+
+def fixups(models, _SESSION):
+    # now create new changed value of clone_url
+    Optional = models.Optional
+
+    def get_by_name(cls, key):
+        return cls.query().filter(cls.app_settings_name == key).scalar()
+
+    def create_or_update(cls, key, val=Optional(''), type_=Optional('unicode')):
+        res = get_by_name(cls, key)
+        if not res:
+            val = Optional.extract(val)
+            type_ = Optional.extract(type_)
+            res = cls(key, val, type_)
+        else:
+            res.app_settings_name = key
+            if not isinstance(val, Optional):
+                # update if set
+                res.app_settings_value = val
+            if not isinstance(type_, Optional):
+                # update if set
+                res.app_settings_type = type_
+        return res
+
+    clone_uri_tmpl = models.Repository.DEFAULT_CLONE_URI_ID
+    print('settings new clone by url template to %s' % clone_uri_tmpl)
+
+    sett = create_or_update(models.RhodeCodeSetting,
+        'clone_uri_id_tmpl', clone_uri_tmpl, 'unicode')
+    _SESSION().add(sett)
+    _SESSION.commit()

rhodecode/lib/middleware/request_wrapper.py

@@ -37,21 +37,29 @@ class RequestWrapperTween(object):
 
         # one-time configuration code goes here
 
+    def _get_user_info(self, request):
+        user = get_current_rhodecode_user(request)
+        if not user:
+            user = AuthUser.repr_user(ip=get_ip_addr(request.environ))
+        return user
+
     def __call__(self, request):
         start = time.time()
         log.debug('Starting request time measurement')
         try:
             response = self.handler(request)
         finally:
-            end = time.time()
-            total = end - start
             count = request.request_count()
             _ver_ = rhodecode.__version__
-            default_user_info = AuthUser.repr_user(ip=get_ip_addr(request.environ))
-            user_info = get_current_rhodecode_user(request) or default_user_info
+            statsd = request.statsd
+            total = time.time() - start
+            if statsd:
+                statsd.timing('rhodecode.req.timing', total)
+                statsd.incr('rhodecode.req.count')
+
             log.info(
                 'Req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s',
-                count, user_info, request.environ.get('REQUEST_METHOD'),
+                count, self._get_user_info(request), request.environ.get('REQUEST_METHOD'),
                 safe_str(get_access_path(request.environ)), total,
                 get_user_agent(request. environ), _ver_
             )

rhodecode/lib/vcs/backends/hg/repository.py

@@ -765,7 +765,14 @@ class MercurialRepository(BaseRepository):
 
         try:
             if target_ref.type == 'branch' and len(self._heads(target_ref.name)) != 1:
-                heads = '\n,'.join(self._heads(target_ref.name))
+                heads_all = self._heads(target_ref.name)
+                max_heads = 10
+                if len(heads_all) > max_heads:
+                    heads = '\n,'.join(
+                        heads_all[:max_heads] +
+                        ['and {} more.'.format(len(heads_all)-max_heads)])
+                else:
+                    heads = '\n,'.join(heads_all)
                 metadata = {
                     'target_ref': target_ref,
                     'source_ref': source_ref,
@@ -854,7 +861,16 @@ class MercurialRepository(BaseRepository):
             except RepositoryError as e:
                 log.exception('Failure when doing local merge on hg shadow repo')
                 if isinstance(e, UnresolvedFilesInRepo):
-                    metadata['unresolved_files'] = '\n* conflict: ' + ('\n * conflict: '.join(e.args[0]))
+                    all_conflicts = list(e.args[0])
+                    max_conflicts = 20
+                    if len(all_conflicts) > max_conflicts:
+                        conflicts = all_conflicts[:max_conflicts] \
+                                    + ['and {} more.'.format(len(all_conflicts)-max_conflicts)]
+                    else:
+                        conflicts = all_conflicts
+                    metadata['unresolved_files'] = \
+                        '\n* conflict: ' + \
+                        ('\n * conflict: '.join(conflicts))
 
                 merge_possible = False
                 merge_failure_reason = MergeFailureReason.MERGE_FAILED

rhodecode/lib/vcs/exceptions.py

@@ -220,7 +220,7 @@ def map_vcs_exceptions(func):
                 if any(e.args):
                     _args = [a for a in e.args]
                     # replace the first argument with a prefix exc name
-                    args = ['{}:'.format(exc_name, _args[0] if _args else '?')] + _args[1:]
+                    args = ['{}:{}'.format(exc_name, _args[0] if _args else '?')] + _args[1:]
                 else:
                     args = [__traceback_info__ or '{}: UnhandledException'.format(exc_name)]
                 if debug or __traceback_info__ and kind not in ['unhandled', 'lookup']:

rhodecode/model/comment.py

@@ -336,6 +336,7 @@ class CommentsModel(BaseModel):
         comment.author = user
         resolved_comment = self.__get_commit_comment(
             validated_kwargs['resolves_comment_id'])
+
         # check if the comment actually belongs to this PR
         if resolved_comment and resolved_comment.pull_request and \
                 resolved_comment.pull_request != pull_request:
@@ -351,6 +352,10 @@ class CommentsModel(BaseModel):
             # comment not bound to this repo, forbid
             resolved_comment = None
 
+        if resolved_comment and resolved_comment.resolved_by:
+            # if this comment is already resolved, don't mark it again!
+            resolved_comment = None
+
         comment.resolved_comment = resolved_comment
 
         pull_request_id = pull_request

rhodecode/model/db.py

@@ -4220,6 +4220,12 @@ class _PullRequestBase(BaseModel):
             return True
         return False
 
+    @property
+    def title_safe(self):
+        return self.title\
+            .replace('{', '{{')\
+            .replace('}', '}}')
+
     @hybrid_property
     def description_safe(self):
         from rhodecode.lib import helpers as h

rhodecode/model/forms.py

@@ -390,6 +390,7 @@ def ApplicationVisualisationForm(localizer):
         rhodecode_markup_renderer = v.OneOf(['markdown', 'rst'])
         rhodecode_gravatar_url = v.UnicodeString(min=3)
         rhodecode_clone_uri_tmpl = v.UnicodeString(not_empty=False, if_empty=Repository.DEFAULT_CLONE_URI)
+        rhodecode_clone_uri_id_tmpl = v.UnicodeString(not_empty=False, if_empty=Repository.DEFAULT_CLONE_URI_ID)
         rhodecode_clone_uri_ssh_tmpl = v.UnicodeString(not_empty=False, if_empty=Repository.DEFAULT_CLONE_URI_SSH)
         rhodecode_support_url = v.UnicodeString()
         rhodecode_show_revision_number = v.StringBoolean(if_missing=False)

rhodecode/model/notification.py

@@ -343,6 +343,7 @@ class EmailNotificationModel(BaseModel):
     TYPE_PASSWORD_RESET_CONFIRMATION = 'password_reset_confirmation'
     TYPE_EMAIL_TEST = 'email_test'
     TYPE_EMAIL_EXCEPTION = 'exception'
+    TYPE_UPDATE_AVAILABLE = 'update_available'
     TYPE_TEST = 'test'
 
     email_types = {
@@ -352,6 +353,8 @@ class EmailNotificationModel(BaseModel):
             'rhodecode:templates/email_templates/test.mako',
         TYPE_EMAIL_EXCEPTION:
             'rhodecode:templates/email_templates/exception_tracker.mako',
+        TYPE_UPDATE_AVAILABLE:
+            'rhodecode:templates/email_templates/update_available.mako',
         TYPE_EMAIL_TEST:
             'rhodecode:templates/email_templates/email_test.mako',
         TYPE_REGISTRATION:

rhodecode/model/update.py

@@ -60,11 +60,11 @@ class UpdateModel(BaseModel):
         Session().add(setting)
         Session().commit()
 
-    def get_stored_version(self):
+    def get_stored_version(self, fallback=None):
         obj = SettingsModel().get_setting_by_name(self.UPDATE_SETTINGS_KEY)
         if obj:
             return obj.app_settings_value
-        return '0.0.0'
+        return fallback or '0.0.0'
 
     def _sanitize_version(self, version):
         """

rhodecode/public/js/src/rhodecode/comments.js

@@ -25,6 +25,7 @@ var firefoxAnchorFix = function() {
   }
 };
 
+
 var linkifyComments = function(comments) {
   var firstCommentId = null;
   if (comments) {
@@ -36,6 +37,7 @@ var linkifyComments = function(comments) {
   }
 };
 
+
 var bindToggleButtons = function() {
   $('.comment-toggle').on('click', function() {
         $(this).parent().nextUntil('tr.line').toggle('inline-comments');
@@ -43,7 +45,6 @@ var bindToggleButtons = function() {
 };
 
 
-
 var _submitAjaxPOST = function(url, postData, successHandler, failHandler) {
     failHandler = failHandler || function() {};
     postData = toQueryString(postData);
@@ -63,8 +64,6 @@ var _submitAjaxPOST = function(url, postData, successHandler, failHandler) {
 };
 
 
-
-
 /* Comment form for main and inline comments */
 (function(mod) {
 
@@ -239,8 +238,7 @@ var _submitAjaxPOST = function(url, postData, successHandler, failHandler) {
         };
 
         this.markCommentResolved = function(resolvedCommentId){
-            $('#comment-label-{0}'.format(resolvedCommentId)).find('.resolved').show();
-            $('#comment-label-{0}'.format(resolvedCommentId)).find('.resolve').hide();
+            Rhodecode.comments.markCommentResolved(resolvedCommentId)
         };
 
         this.isAllowedToSubmit = function() {
@@ -1308,6 +1306,11 @@ var CommentsController = function() {
       return $(tmpl);
   }
 
+  this.markCommentResolved = function(commentId) {
+    $('#comment-label-{0}'.format(commentId)).find('.resolved').show();
+    $('#comment-label-{0}'.format(commentId)).find('.resolve').hide();
+  };
+
   this.createComment = function(node, f_path, line_no, resolutionComment) {
       self.edit = false;
       var $node = $(node);
@@ -1403,7 +1406,7 @@ var CommentsController = function() {
 
                     //mark visually which comment was resolved
                     if (resolvesCommentId) {
-                        commentForm.markCommentResolved(resolvesCommentId);
+                        self.markCommentResolved(resolvesCommentId);
                     }
 
                     // run global callback on submit
@@ -1462,7 +1465,6 @@ var CommentsController = function() {
 
     var comment = $('#comment-'+commentId);
     var commentData = comment.data();
-    console.log(commentData);
 
     if (commentData.commentInline) {
         var f_path = commentData.commentFPath;
@@ -1494,9 +1496,144 @@ var CommentsController = function() {
     return false;
   };
 
+  this.resolveTodo = function (elem, todoId) {
+      var commentId = todoId;
+
+      SwalNoAnimation.fire({
+          title: 'Resolve TODO {0}'.format(todoId),
+          showCancelButton: true,
+          confirmButtonText: _gettext('Yes'),
+          showLoaderOnConfirm: true,
+
+          allowOutsideClick: function () {
+              !Swal.isLoading()
+          },
+          preConfirm: function () {
+              var comment = $('#comment-' + commentId);
+              var commentData = comment.data();
+
+              var f_path = null
+              var line_no = null
+              if (commentData.commentInline) {
+                  f_path = commentData.commentFPath;
+                  line_no = commentData.commentLineNo;
+              }
+
+              var renderer = templateContext.visual.default_renderer;
+              var commentBoxUrl = '{1}#comment-{0}'.format(commentId);
+
+              // Pull request case
+              if (templateContext.pull_request_data.pull_request_id !== null) {
+                  var commentUrl = pyroutes.url('pullrequest_comment_create',
+                          {
+                              'repo_name': templateContext.repo_name,
+                              'pull_request_id': templateContext.pull_request_data.pull_request_id,
+                              'comment_id': commentId
+                          });
+              } else {
+                  var commentUrl = pyroutes.url('repo_commit_comment_create',
+                          {
+                              'repo_name': templateContext.repo_name,
+                              'commit_id': templateContext.commit_data.commit_id,
+                              'comment_id': commentId
+                          });
+              }
+
+              if (renderer === 'rst') {
+                  commentBoxUrl = '`#{0} <{1}#comment-{0}>`_'.format(commentId, commentUrl);
+              } else if (renderer === 'markdown') {
+                  commentBoxUrl = '[#{0}]({1}#comment-{0})'.format(commentId, commentUrl);
+              }
+              var resolveText = _gettext('TODO from comment {0} was fixed.').format(commentBoxUrl);
+
+              var postData = {
+                  text: resolveText,
+                  comment_type: 'note',
+                  draft: false,
+                  csrf_token: CSRF_TOKEN,
+                  resolves_comment_id: commentId
+              }
+              if (commentData.commentInline) {
+                  postData['f_path'] = f_path;
+                  postData['line'] = line_no;
+              }
+
+              return new Promise(function (resolve, reject) {
+                  $.ajax({
+                      type: 'POST',
+                      data: postData,
+                      url: commentUrl,
+                      headers: {'X-PARTIAL-XHR': true}
+                  })
+                  .done(function (data) {
+                      resolve(data);
+                  })
+                  .fail(function (jqXHR, textStatus, errorThrown) {
+                      var prefix = "Error while resolving TODO.\n"
+                      var message = formatErrorMessage(jqXHR, textStatus, errorThrown, prefix);
+                      ajaxErrorSwal(message);
+                  });
+              })
+          }
+
+      })
+      .then(function (result) {
+          var success = function (json_data) {
+              resolvesCommentId = commentId;
+              var commentResolved = json_data[Object.keys(json_data)[0]]
+
+              try {
+
+                  if (commentResolved.f_path) {
+                      // inject newly created comments, json_data is {<comment_id>: {}}
+                      self.attachInlineComment(json_data)
+                  } else {
+                      self.attachGeneralComment(json_data)
+                  }
+
+                  //mark visually which comment was resolved
+                  if (resolvesCommentId) {
+                      self.markCommentResolved(resolvesCommentId);
+                  }
+
+                  // run global callback on submit
+                  if (window.commentFormGlobalSubmitSuccessCallback !== undefined) {
+                      commentFormGlobalSubmitSuccessCallback({
+                          draft: false,
+                          comment_id: commentId
+                      });
+                  }
+
+              } catch (e) {
+                  console.error(e);
+              }
+
+              if (window.updateSticky !== undefined) {
+                  // potentially our comments change the active window size, so we
+                  // notify sticky elements
+                  updateSticky()
+              }
+
+              if (window.refreshAllComments !== undefined) {
+                  // if we have this handler, run it, and refresh all comments boxes
+                  refreshAllComments()
+              }
+              // re trigger the linkification of next/prev navigation
+              linkifyComments($('.inline-comment-injected'));
+              timeagoActivate();
+              tooltipActivate();
+          };
+
+          if (result.value) {
+            $(elem).remove();
+            success(result.value)
+          }
+      })
+  };
+
 };
 
 window.commentHelp = function(renderer) {
     var funcData = {'renderer': renderer}
     return renderTemplate('commentHelpHovercard', funcData)
-}
+}

rhodecode/templates/admin/settings/settings_visual.mako

@@ -173,6 +173,9 @@ ${h.secure_form(h.route_path('admin_settings_visual_update'), request=request)}
         <div class="field">
            ${h.text('rhodecode_clone_uri_tmpl', size=60)} HTTP[S]
         </div>
+        <div class="field">
+            ${h.text('rhodecode_clone_uri_id_tmpl', size=60)} HTTP UID
+        </div>
         <div class="field">
            ${h.text('rhodecode_clone_uri_ssh_tmpl', size=60)} SSH
         </div>

rhodecode/templates/ejs_templates/templates.html

@@ -236,6 +236,14 @@ if (show_disabled) {
     Created:
     <time class="timeago" title="<%= created_on %>" datetime="<%= datetime %>"><%= $.timeago(datetime) %></time>
 
+    <% if (is_todo) { %>
+    <div style="text-align: center; padding-top: 5px">
+    <a class="btn btn-sm" href="#resolveTodo<%- comment_id -%>" onclick="Rhodecode.comments.resolveTodo(this, '<%- comment_id -%>'); return false">
+        <strong>Resolve TODO</strong>
+    </a>
+    </div>
+    <% } %>
+
 </div>
 
 </script>

rhodecode/templates/email_templates/pull_request_comment.mako

@@ -14,7 +14,7 @@ data = {
     'comment_type': comment_type,
     'comment_id': comment_id,
 
-    'pr_title': pull_request.title,
+    'pr_title': pull_request.title_safe,
     'pr_id': pull_request.pull_request_id,
     'mention_prefix': '[mention] ' if mention else '',
 }
@@ -31,7 +31,6 @@ else:
         _('{mention_prefix}{user} left a {comment_type} on pull request !{pr_id}: "{pr_title}"').format(**data)
 %>
 
-
 ${subject_template.format(**data) |n}
 </%def>
 
@@ -47,7 +46,7 @@ data = {
     'comment_type': comment_type,
     'comment_id': comment_id,
 
-    'pr_title': pull_request.title,
+    'pr_title': pull_request.title_safe,
     'pr_id': pull_request.pull_request_id,
     'source_ref_type': pull_request.source_ref_parts.type,
     'source_ref_name': pull_request.source_ref_parts.name,
@@ -99,7 +98,7 @@ data = {
     'comment_id': comment_id,
     'renderer_type': renderer_type or 'plain',
 
-    'pr_title': pull_request.title,
+    'pr_title': pull_request.title_safe,
     'pr_id': pull_request.pull_request_id,
     'status': status_change,
     'source_ref_type': pull_request.source_ref_parts.type,

rhodecode/templates/email_templates/pull_request_review.mako

@@ -8,7 +8,7 @@
 data = {
     'user': '@'+h.person(user),
     'pr_id': pull_request.pull_request_id,
-    'pr_title': pull_request.title,
+    'pr_title': pull_request.title_safe,
 }
 
 if user_role == 'observer':
@@ -26,7 +26,7 @@ ${subject_template.format(**data) |n}
 data = {
     'user': h.person(user),
     'pr_id': pull_request.pull_request_id,
-    'pr_title': pull_request.title,
+    'pr_title': pull_request.title_safe,
     'source_ref_type': pull_request.source_ref_parts.type,
     'source_ref_name': pull_request.source_ref_parts.name,
     'target_ref_type': pull_request.target_ref_parts.type,
@@ -66,7 +66,7 @@ ${self.plaintext_footer()}
 data = {
     'user': h.person(user),
     'pr_id': pull_request.pull_request_id,
-    'pr_title': pull_request.title,
+    'pr_title': pull_request.title_safe,
     'source_ref_type': pull_request.source_ref_parts.type,
     'source_ref_name': pull_request.source_ref_parts.name,
     'target_ref_type': pull_request.target_ref_parts.type,

rhodecode/templates/email_templates/pull_request_update.mako

@@ -8,7 +8,7 @@
 data = {
     'updating_user': '@'+h.person(updating_user),
     'pr_id': pull_request.pull_request_id,
-    'pr_title': pull_request.title,
+    'pr_title': pull_request.title_safe,
 }
 
 subject_template = email_pr_update_subject_template or _('{updating_user} updated pull request. !{pr_id}: "{pr_title}"')
@@ -23,7 +23,7 @@ ${subject_template.format(**data) |n}
 data = {
     'updating_user': h.person(updating_user),
     'pr_id': pull_request.pull_request_id,
-    'pr_title': pull_request.title,
+    'pr_title': pull_request.title_safe,
     'source_ref_type': pull_request.source_ref_parts.type,
     'source_ref_name': pull_request.source_ref_parts.name,
     'target_ref_type': pull_request.target_ref_parts.type,
@@ -74,7 +74,7 @@ ${self.plaintext_footer()}
 data = {
     'updating_user': h.person(updating_user),
     'pr_id': pull_request.pull_request_id,
-    'pr_title': pull_request.title,
+    'pr_title': pull_request.title_safe,
     'source_ref_type': pull_request.source_ref_parts.type,
     'source_ref_name': pull_request.source_ref_parts.name,
     'target_ref_type': pull_request.target_ref_parts.type,

rhodecode/templates/email_templates/update_available.mako

@@ -0,0 +1,32 @@
+## -*- coding: utf-8 -*-
+<%inherit file="base.mako"/>
+
+<%def name="subject()" filter="n,trim,whitespace_filter">
+New Version of RhodeCode is available !
+</%def>
+
+## plain text version of the email. Empty by default
+<%def name="body_plaintext()" filter="n,trim">
+A new version of RhodeCode is available!
+
+Your version: ${current_ver}
+New version: ${latest_ver}
+
+Release notes:
+
+https://docs.rhodecode.com/RhodeCode-Enterprise/release-notes/release-notes-${latest_ver}.html
+</%def>
+
+## BODY GOES BELOW
+
+<h3>A new version of RhodeCode is available!</h3>
+<br/>
+Your version: ${current_ver}<br/>
+New version: <strong>${latest_ver}</strong><br/>
+
+<h4>Release notes</h4>
+
+<a href="https://docs.rhodecode.com/RhodeCode-Enterprise/release-notes/release-notes-${latest_ver}.html">
+    https://docs.rhodecode.com/RhodeCode-Enterprise/release-notes/release-notes-${latest_ver}.html
+</a>
+

rhodecode/tests/lib/test_mako_emails.py

@@ -27,6 +27,16 @@ from rhodecode.model.db import User, PullRequestReviewers
 from rhodecode.model.notification import EmailNotificationModel
 
 
+@pytest.fixture()
+def pr():
+    def factory(ref):
+        return collections.namedtuple(
+            'PullRequest',
+            'pull_request_id, title, title_safe, description, source_ref_parts, source_ref_name, target_ref_parts, target_ref_name')\
+            (200, 'Example Pull Request', 'Example Pull Request', 'Desc of PR', ref, 'bookmark', ref, 'Branch')
+    return factory
+
+
 def test_get_template_obj(app, request_stub):
     template = EmailNotificationModel().get_renderer(
         EmailNotificationModel.TYPE_TEST, request_stub)
@@ -53,14 +63,10 @@ def test_render_email(app, http_host_only_stub):
 
 
 @pytest.mark.parametrize('role', PullRequestReviewers.ROLES)
-def test_render_pr_email(app, user_admin, role):
+def test_render_pr_email(app, user_admin, role, pr):
     ref = collections.namedtuple(
         'Ref', 'name, type')('fxies123', 'book')
-
-    pr = collections.namedtuple('PullRequest',
-        'pull_request_id, title, description, source_ref_parts, source_ref_name, target_ref_parts, target_ref_name')(
-        200, 'Example Pull Request', 'Desc of PR', ref, 'bookmark', ref, 'Branch')
-
+    pr = pr(ref)
     source_repo = target_repo = collections.namedtuple(
         'Repo', 'type, repo_name')('hg', 'pull_request_1')
 
@@ -89,13 +95,11 @@ def test_render_pr_email(app, user_admin, role):
         assert subject == '@test_admin (RhodeCode Admin) added you as observer to pull request. !200: "Example Pull Request"'
 
 
-def test_render_pr_update_email(app, user_admin):
+def test_render_pr_update_email(app, user_admin, pr):
     ref = collections.namedtuple(
         'Ref', 'name, type')('fxies123', 'book')
 
-    pr = collections.namedtuple('PullRequest',
-        'pull_request_id, title, description, source_ref_parts, source_ref_name, target_ref_parts, target_ref_name')(
-        200, 'Example Pull Request', 'Desc of PR', ref, 'bookmark', ref, 'Branch')
+    pr = pr(ref)
 
     source_repo = target_repo = collections.namedtuple(
         'Repo', 'type, repo_name')('hg', 'pull_request_1')
@@ -150,13 +154,11 @@ def test_render_pr_update_email(app, user_admin):
     EmailNotificationModel.TYPE_COMMIT_COMMENT,
     EmailNotificationModel.TYPE_PULL_REQUEST_COMMENT
 ])
-def test_render_comment_subject_no_newlines(app, mention, email_type):
+def test_render_comment_subject_no_newlines(app, mention, email_type, pr):
     ref = collections.namedtuple(
         'Ref', 'name, type')('fxies123', 'book')
 
-    pr = collections.namedtuple('PullRequest',
-        'pull_request_id, title, description, source_ref_parts, source_ref_name, target_ref_parts, target_ref_name')(
-        200, 'Example Pull Request', 'Desc of PR', ref, 'bookmark', ref, 'Branch')
+    pr = pr(ref)
 
     source_repo = target_repo = collections.namedtuple(
         'Repo', 'type, repo_name')('hg', 'pull_request_1')