exceptions: added new exception tracking capability.

- store exceptions
- expose super admin interface to browse latest errors

rhodecode/apps/admin/__init__.py

@@ -59,6 +59,19 @@ def admin_routes(config):
         name='admin_settings_system_update',
         pattern='/settings/system/updates')
 
+    config.add_route(
+        name='admin_settings_exception_tracker',
+        pattern='/settings/exceptions')
+    config.add_route(
+        name='admin_settings_exception_tracker_delete_all',
+        pattern='/settings/exceptions/delete')
+    config.add_route(
+        name='admin_settings_exception_tracker_show',
+        pattern='/settings/exceptions/{exception_id}')
+    config.add_route(
+        name='admin_settings_exception_tracker_delete',
+        pattern='/settings/exceptions/{exception_id}/delete')
+
     config.add_route(
         name='admin_settings_sessions',
         pattern='/settings/sessions')

rhodecode/apps/admin/navigation.py

@@ -89,6 +89,9 @@ class NavigationRegistry(object):
                  'global_integrations_home'),
         NavEntry('system', _('System Info'),
                  'admin_settings_system'),
+        NavEntry('exceptions', _('Exceptions Tracker'),
+                 'admin_settings_exception_tracker',
+                 active_list=['exceptions', 'exceptions_browse']),
         NavEntry('process_management', _('Processes'),
                  'admin_settings_process_management'),
         NavEntry('sessions', _('User Sessions'),

rhodecode/apps/admin/views/exception_tracker.py

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2018-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import os
+import logging
+
+from pyramid.httpexceptions import HTTPFound
+from pyramid.view import view_config
+
+from rhodecode.apps._base import BaseAppView
+from rhodecode.apps.admin.navigation import navigation_list
+from rhodecode.lib import helpers as h
+from rhodecode.lib.auth import (
+    LoginRequired, HasPermissionAllDecorator, CSRFRequired)
+from rhodecode.lib.utils2 import time_to_utcdatetime
+from rhodecode.lib import exc_tracking
+
+log = logging.getLogger(__name__)
+
+
+class ExceptionsTrackerView(BaseAppView):
+    def load_default_context(self):
+        c = self._get_local_tmpl_context()
+        c.navlist = navigation_list(self.request)
+        return c
+
+    def count_all_exceptions(self):
+        exc_store_path = exc_tracking.get_exc_store()
+        count = 0
+        for fname in os.listdir(exc_store_path):
+            parts = fname.split('_', 2)
+            if not len(parts) == 3:
+                continue
+            count +=1
+        return count
+
+    def get_all_exceptions(self, read_metadata=False, limit=None):
+        exc_store_path = exc_tracking.get_exc_store()
+        exception_list = []
+
+        def key_sorter(val):
+            try:
+                return val.split('_')[-1]
+            except Exception:
+                return 0
+        count = 0
+        for fname in reversed(sorted(os.listdir(exc_store_path), key=key_sorter)):
+
+            parts = fname.split('_', 2)
+            if not len(parts) == 3:
+                continue
+
+            exc_id, app_type, exc_timestamp = parts
+
+            exc = {'exc_id': exc_id, 'app_type': app_type, 'exc_type': 'unknown',
+                   'exc_utc_date': '', 'exc_timestamp': exc_timestamp}
+
+            if read_metadata:
+                full_path = os.path.join(exc_store_path, fname)
+                # we can read our metadata
+                with open(full_path, 'rb') as f:
+                    exc_metadata = exc_tracking.exc_unserialize(f.read())
+                    exc.update(exc_metadata)
+
+            # convert our timestamp to a date obj, for nicer representation
+            exc['exc_utc_date'] = time_to_utcdatetime(exc['exc_timestamp'])
+            exception_list.append(exc)
+
+            count += 1
+            if limit and count >= limit:
+                break
+        return exception_list
+
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    @view_config(
+        route_name='admin_settings_exception_tracker', request_method='GET',
+        renderer='rhodecode:templates/admin/settings/settings.mako')
+    def browse_exceptions(self):
+        _ = self.request.translate
+        c = self.load_default_context()
+        c.active = 'exceptions_browse'
+        c.limit = self.request.GET.get('limit', 50)
+        c.exception_list = self.get_all_exceptions(read_metadata=True, limit=c.limit)
+        c.exception_list_count = self.count_all_exceptions()
+        c.exception_store_dir = exc_tracking.get_exc_store()
+        return self._get_template_context(c)
+
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    @view_config(
+        route_name='admin_settings_exception_tracker_show', request_method='GET',
+        renderer='rhodecode:templates/admin/settings/settings.mako')
+    def exception_show(self):
+        _ = self.request.translate
+        c = self.load_default_context()
+
+        c.active = 'exceptions'
+        c.exception_id = self.request.matchdict['exception_id']
+        c.traceback = exc_tracking.read_exception(c.exception_id, prefix=None)
+        return self._get_template_context(c)
+
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    @CSRFRequired()
+    @view_config(
+        route_name='admin_settings_exception_tracker_delete_all', request_method='POST',
+        renderer='rhodecode:templates/admin/settings/settings.mako')
+    def exception_delete_all(self):
+        _ = self.request.translate
+        c = self.load_default_context()
+
+        c.active = 'exceptions'
+        all_exc = self.get_all_exceptions()
+        exc_count = len(all_exc)
+        for exc in all_exc:
+            exc_tracking.delete_exception(exc['exc_id'], prefix=None)
+
+        h.flash(_('Removed {} Exceptions').format(exc_count), category='success')
+        raise HTTPFound(h.route_path('admin_settings_exception_tracker'))
+
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    @CSRFRequired()
+    @view_config(
+        route_name='admin_settings_exception_tracker_delete', request_method='POST',
+        renderer='rhodecode:templates/admin/settings/settings.mako')
+    def exception_delete(self):
+        _ = self.request.translate
+        c = self.load_default_context()
+
+        c.active = 'exceptions'
+        c.exception_id = self.request.matchdict['exception_id']
+        exc_tracking.delete_exception(c.exception_id, prefix=None)
+
+        h.flash(_('Removed Exception {}').format(c.exception_id), category='success')
+        raise HTTPFound(h.route_path('admin_settings_exception_tracker'))

rhodecode/config/middleware.py

@@ -19,6 +19,7 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
+import sys
 import logging
 import traceback
 import collections
@@ -48,6 +49,7 @@ from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.celerylib.loader import configure_celery
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
 from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict
+from rhodecode.lib.exc_tracking import store_exception
 from rhodecode.subscribers import (
     scan_repositories_if_enabled, write_js_routes_if_enabled,
     write_metadata_if_needed, inject_app_settings)
@@ -172,7 +174,17 @@ def error_handler(exception, request):
         c.causes = base_response.causes
 
     c.messages = helpers.flash.pop_messages(request=request)
-    c.traceback = traceback.format_exc()
+
+    exc_info = sys.exc_info()
+    c.exception_id = id(exc_info)
+    c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
+                          or base_response.status_code > 499
+    c.exception_id_url = request.route_url(
+        'admin_settings_exception_tracker_show', exception_id=c.exception_id)
+
+    if c.show_exception_id:
+        store_exception(c.exception_id, exc_info)
+
     response = render_to_response(
         '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
         response=base_response)

rhodecode/lib/exc_tracking.py

@@ -0,0 +1,146 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2018 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import time
+import datetime
+import msgpack
+import logging
+import traceback
+import tempfile
+
+
+log = logging.getLogger(__name__)
+
+# NOTE: Any changes should be synced with exc_tracking at vcsserver.lib.exc_tracking
+global_prefix = 'rhodecode'
+
+
+def exc_serialize(exc_id, tb, exc_type):
+
+    data = {
+        'version': 'v1',
+        'exc_id': exc_id,
+        'exc_utc_date': datetime.datetime.utcnow().isoformat(),
+        'exc_timestamp': repr(time.time()),
+        'exc_message': tb,
+        'exc_type': exc_type,
+    }
+    return msgpack.packb(data), data
+
+
+def exc_unserialize(tb):
+    return msgpack.unpackb(tb)
+
+
+def get_exc_store():
+    """
+    Get and create exception store if it's not existing
+    """
+    exc_store_dir = 'rc_exception_store_v1'
+    # fallback
+    _exc_store_path = os.path.join(tempfile.gettempdir(), exc_store_dir)
+
+    exc_store_dir = ''  # TODO: need a persistent cross instance store here
+    if exc_store_dir:
+        _exc_store_path = os.path.join(exc_store_dir, exc_store_dir)
+
+    _exc_store_path = os.path.abspath(_exc_store_path)
+    if not os.path.isdir(_exc_store_path):
+        os.makedirs(_exc_store_path)
+        log.debug('Initializing exceptions store at %s', _exc_store_path)
+    return _exc_store_path
+
+
+def _store_exception(exc_id, exc_info, prefix):
+    exc_type, exc_value, exc_traceback = exc_info
+    tb = ''.join(traceback.format_exception(
+        exc_type, exc_value, exc_traceback, None))
+
+    exc_type_name = exc_type.__name__
+    exc_store_path = get_exc_store()
+    exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name)
+    exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
+    if not os.path.isdir(exc_store_path):
+        os.makedirs(exc_store_path)
+    stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
+    with open(stored_exc_path, 'wb') as f:
+        f.write(exc_data)
+    log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
+
+
+def store_exception(exc_id, exc_info, prefix=global_prefix):
+    try:
+        _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
+    except Exception:
+        log.exception('Failed to store exception `%s` information', exc_id)
+        # there's no way this can fail, it will crash server badly if it does.
+        pass
+
+
+def _find_exc_file(exc_id, prefix=global_prefix):
+    exc_store_path = get_exc_store()
+    if prefix:
+        exc_id = '{}_{}'.format(exc_id, prefix)
+    else:
+        # search without a prefix
+        exc_id = '{}'.format(exc_id)
+
+    # we need to search the store for such start pattern as above
+    for fname in os.listdir(exc_store_path):
+        if fname.startswith(exc_id):
+            exc_id = os.path.join(exc_store_path, fname)
+            break
+        continue
+    else:
+        exc_id = None
+
+    return exc_id
+
+
+def _read_exception(exc_id, prefix):
+    exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
+    if exc_id_file_path:
+        with open(exc_id_file_path, 'rb') as f:
+            return exc_unserialize(f.read())
+    else:
+        log.debug('Exception File `%s` not found', exc_id_file_path)
+    return None
+
+
+def read_exception(exc_id, prefix=global_prefix):
+    try:
+        return _read_exception(exc_id=exc_id, prefix=prefix)
+    except Exception:
+        log.exception('Failed to read exception `%s` information', exc_id)
+        # there's no way this can fail, it will crash server badly if it does.
+    return None
+
+
+def delete_exception(exc_id, prefix=global_prefix):
+    try:
+        exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
+        if exc_id_file_path:
+            os.remove(exc_id_file_path)
+
+    except Exception:
+        log.exception('Failed to remove exception `%s` information', exc_id)
+        # there's no way this can fail, it will crash server badly if it does.
+        pass

rhodecode/public/css/main-content.less

@@ -77,6 +77,13 @@
     min-width: 100%;
 }
 
+.main-content-auto-width {
+    .main-content;
+    width: auto;
+    min-width: 100%;
+    max-width: inherit;
+}
+
 .field {
     clear: left;
     margin-bottom: @padding;

rhodecode/templates/admin/settings/settings.mako

@@ -44,7 +44,7 @@
           </ul>
       </div>
 
-      <div class="main-content-full-width">
+      <div class="main-content-auto-width">
         ${self.main_content()}
       </div>
     </div>

rhodecode/templates/admin/settings/settings_exceptions.mako

@@ -0,0 +1,38 @@
+<div class="panel panel-default">
+    <div class="panel-heading">
+        <h3 class="panel-title">${_('Exceptions Tracker - Exception ID')}: ${c.exception_id}</h3>
+    </div>
+    <div class="panel-body">
+    % if c.traceback:
+
+    <h4>${_('Exception `{}` generated on UTC date: {}').format(c.traceback.get('exc_type', 'NO_TYPE'), c.traceback.get('exc_utc_date', 'NO_DATE'))}</h4>
+    <pre>${c.traceback.get('exc_message', 'NO_MESSAGE')}</pre>
+
+    % else:
+        ${_('Unable to Read Exception. It might be removed or non-existing.')}
+    % endif
+    </div>
+</div>
+
+
+% if c.traceback:
+<div class="panel panel-danger">
+    <div class="panel-heading" id="advanced-delete">
+        <h3 class="panel-title">${_('Delete this Exception')}</h3>
+    </div>
+    <div class="panel-body">
+      ${h.secure_form(h.route_path('admin_settings_exception_tracker_delete', exception_id=c.exception_id), request=request)}
+        <div style="margin: 0 0 20px 0" class="fake-space"></div>
+
+        <div class="field">
+            <button class="btn btn-small btn-danger" type="submit"
+                    onclick="return confirm('${_('Confirm to delete this exception')}');">
+                <i class="icon-remove-sign"></i>
+                ${_('Delete This Exception')}
+            </button>
+        </div>
+
+      ${h.end_form()}
+    </div>
+</div>
+% endif

rhodecode/templates/admin/settings/settings_exceptions_browse.mako

@@ -0,0 +1,52 @@
+<div class="panel panel-default">
+    <div class="panel-heading">
+        <h3 class="panel-title">${_('Exceptions Tracker ')}</h3>
+    </div>
+    <div class="panel-body">
+        % if c.exception_list_count == 1:
+            ${_('There is {} stored exception.').format(c.exception_list_count)}
+        % else:
+            ${_('There are {} stored exceptions.').format(c.exception_list_count)}
+        % endif
+        ${_('Store directory')}: ${c.exception_store_dir}
+
+      ${h.secure_form(h.route_path('admin_settings_exception_tracker_delete_all'), request=request)}
+        <div style="margin: 0 0 20px 0" class="fake-space"></div>
+
+        <div class="field">
+            <button class="btn btn-small btn-danger" type="submit"
+                    onclick="return confirm('${_('Confirm to delete all exceptions')}');">
+                <i class="icon-remove-sign"></i>
+                ${_('Delete All')}
+            </button>
+        </div>
+
+      ${h.end_form()}
+
+    </div>
+</div>
+
+
+<div class="panel panel-default">
+    <div class="panel-heading">
+        <h3 class="panel-title">${_('Exceptions Tracker - Showing the last {} Exceptions').format(c.limit)}</h3>
+    </div>
+    <div class="panel-body">
+        <table class="rctable">
+        <tr>
+            <th>Exception ID</th>
+            <th>Date</th>
+            <th>App Type</th>
+            <th>Exc Type</th>
+        </tr>
+        % for tb in c.exception_list:
+            <tr>
+                <td><a href="${h.route_path('admin_settings_exception_tracker_show', exception_id=tb['exc_id'])}"><code>${tb['exc_id']}</code></a></td>
+                <td>${h.format_date(tb['exc_utc_date'])}</td>
+                <td>${tb['app_type']}</td>
+                <td>${tb['exc_type']}</td>
+            </tr>
+        % endfor
+        </table>
+    </div>
+</div>

rhodecode/templates/errors/error_document.mako

@@ -60,9 +60,17 @@
                 </div>
                 <div class="inner-column">
                     <h4>Support</h4>
-                    <p>For support, go to <a href="${c.visual.rhodecode_support_url}" target="_blank">${_('Support')}</a>.
+                    <p>For help and support, go to the <a href="${c.visual.rhodecode_support_url}" target="_blank">${_('Support Page')}</a>.
                        It may be useful to include your log file; see the log file locations <a href="${h.route_url('enterprise_log_file_locations')}">here</a>.
                     </p>
+                    % if c.show_exception_id:
+                    <p>
+                        Super Admin can see detailed traceback information from this exception by checking the below Exception ID.
+                        Please include the below link for further details of this exception.
+
+                        Exception ID: <a href="${c.exception_id_url}">${c.exception_id}</a>
+                    </p>
+                    % endif
                 </div>
                 <div class="inner-column">
                     <h4>Documentation</h4>