engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

release: merged default into stable

Browse source
This commit is contained in:
Marcin Kuzminski 2018-09-06 21:30:04 +02:00
commit ae3f64f60f
10 changed files with 172 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

43
docs/release-notes/release-notes-4.13.1.rst Normal file
View file

@ -0,0 +1,43 @@
|RCE| 4.13.1 |RNS|
------------------
Release Date
^^^^^^^^^^^^
- 2018-08-06
New Features
^^^^^^^^^^^^
General
^^^^^^^
- core: added option to prefix cache keys for usage in cluster.
- exception-tracker: store event sending exception for easier event fail debugging.
- maintenance: add repack and fsck for git maintenance execution list.
Security
^^^^^^^^
Performance
^^^^^^^^^^^
Fixes
^^^^^
- caches: use single default cache dir for all backends.
- caches: don't use lower in cache settings to support uppercase PATHS
Upgrade notes
^^^^^^^^^^^^^
- Unscheduled release addressing reported problems, and improving stability.

1
docs/release-notes/release-notes.rst
View file

@ -9,6 +9,7 @@ Release Notes
.. toctree::
:maxdepth: 1
release-notes-4.13.1.rst
release-notes-4.13.0.rst
release-notes-4.12.4.rst
release-notes-4.12.3.rst

2
pkgs/python-packages.nix
View file

@ -1653,6 +1653,8 @@ self: super: {
self."setuptools-scm"
self."amqp"
self."authomatic"
self."atomicwrites"
self."attrs"
self."babel"
self."beaker"
self."celery"

2
requirements.txt
View file

@ -4,6 +4,8 @@ setuptools-scm==2.1.0
amqp==2.3.1
# not released authomatic that has updated some oauth providers
https://code.rhodecode.com/upstream/authomatic/archive/90a9ce60cc405ae8a2bf5c3713acd5d78579a04e.tar.gz?md5=3c68720a1322b25254009518d1ff6801#egg=authomatic==0.1.0.post1
atomicwrites==1.1.5
attrs==18.1.0
babel==1.3
beaker==1.9.1
celery==4.1.1

2
rhodecode/apps/repository/views/repo_maintainance.py
View file

@ -32,8 +32,6 @@ log = logging.getLogger(__name__)
class RepoMaintenanceView(RepoAppView):
def load_default_context(self):
c = self._get_local_tmpl_context()
return c
@LoginRequired()

41
rhodecode/config/middleware.py
View file

@ -436,13 +436,23 @@ def _sanitize_vcs_settings(settings):
def _sanitize_cache_settings(settings):
_string_setting(settings, 'cache_dir',
os.path.join(tempfile.gettempdir(), 'rc_cache'))
default_cache_dir = os.path.join(tempfile.gettempdir(), 'rc_cache')
# save default, cache dir, and use it for all backends later.
default_cache_dir = _string_setting(
settings,
'cache_dir',
default_cache_dir, lower=False, default_when_empty=True)
# ensure we have our dir created
if not os.path.isdir(default_cache_dir):
os.makedirs(default_cache_dir, mode=0755)
# cache_perms
_string_setting(
settings,
'rc_cache.cache_perms.backend',
'dogpile.cache.rc.file_namespace')
'dogpile.cache.rc.file_namespace', lower=False)
_int_setting(
settings,
'rc_cache.cache_perms.expiration_time',
@ -450,13 +460,13 @@ def _sanitize_cache_settings(settings):
_string_setting(
settings,
'rc_cache.cache_perms.arguments.filename',
os.path.join(tempfile.gettempdir(), 'rc_cache_1'))
os.path.join(default_cache_dir, 'rc_cache_1'), lower=False)
# cache_repo
_string_setting(
settings,
'rc_cache.cache_repo.backend',
'dogpile.cache.rc.file_namespace')
'dogpile.cache.rc.file_namespace', lower=False)
_int_setting(
settings,
'rc_cache.cache_repo.expiration_time',
@ -464,13 +474,13 @@ def _sanitize_cache_settings(settings):
_string_setting(
settings,
'rc_cache.cache_repo.arguments.filename',
os.path.join(tempfile.gettempdir(), 'rc_cache_2'))
os.path.join(default_cache_dir, 'rc_cache_2'), lower=False)
# cache_license
_string_setting(
settings,
'rc_cache.cache_license.backend',
'dogpile.cache.rc.file_namespace')
'dogpile.cache.rc.file_namespace', lower=False)
_int_setting(
settings,
'rc_cache.cache_license.expiration_time',
@ -478,13 +488,13 @@ def _sanitize_cache_settings(settings):
_string_setting(
settings,
'rc_cache.cache_license.arguments.filename',
os.path.join(tempfile.gettempdir(), 'rc_cache_3'))
os.path.join(default_cache_dir, 'rc_cache_3'), lower=False)
# cache_repo_longterm memory, 96H
_string_setting(
settings,
'rc_cache.cache_repo_longterm.backend',
'dogpile.cache.rc.memory_lru')
'dogpile.cache.rc.memory_lru', lower=False)
_int_setting(
settings,
'rc_cache.cache_repo_longterm.expiration_time',
@ -498,7 +508,7 @@ def _sanitize_cache_settings(settings):
_string_setting(
settings,
'rc_cache.sql_cache_short.backend',
'dogpile.cache.rc.memory_lru')
'dogpile.cache.rc.memory_lru', lower=False)
_int_setting(
settings,
'rc_cache.sql_cache_short.expiration_time',
@ -511,6 +521,7 @@ def _sanitize_cache_settings(settings):
def _int_setting(settings, name, default):
settings[name] = int(settings.get(name, default))
return settings[name]
def _bool_setting(settings, name, default):
@ -518,6 +529,7 @@ def _bool_setting(settings, name, default):
if isinstance(input_val, unicode):
input_val = input_val.encode('utf8')
settings[name] = asbool(input_val)
return settings[name]
def _list_setting(settings, name, default):
@ -530,13 +542,20 @@ def _list_setting(settings, name, default):
else:
# Otherwise we assume it uses pyramids space/newline separation.
settings[name] = aslist(raw_value)
return settings[name]
def _string_setting(settings, name, default, lower=True):
def _string_setting(settings, name, default, lower=True, default_when_empty=False):
value = settings.get(name, default)
if default_when_empty and not value:
# use default value when value is empty
value = default
if lower:
value = value.lower()
settings[name] = value
return settings[name]
def _substitute_values(mapping, substitutions):

11
rhodecode/config/utils.py
View file

@ -81,9 +81,14 @@ def get_vcs_server_protocol(config):
def set_instance_id(config):
""" Sets a dynamic generated config['instance_id'] if missing or '*' """
"""
Sets a dynamic generated config['instance_id'] if missing or '*'
E.g instance_id = *cluster-1 or instance_id = *
"""
config['instance_id'] = config.get('instance_id') or ''
if config['instance_id'] == '*' or not config['instance_id']:
instance_id = config['instance_id']
if instance_id.startswith('*') or not instance_id:
prefix = instance_id.lstrip('*')
_platform_id = platform.uname()[1] or 'instance'
config['instance_id'] = '%s-%s' % (_platform_id, os.getpid())
config['instance_id'] = '%s%s-%s' % (prefix, _platform_id, os.getpid())

6
rhodecode/integrations/__init__.py
View file

@ -17,11 +17,13 @@
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import sys
import logging
from rhodecode.integrations.registry import IntegrationTypeRegistry
from rhodecode.integrations.types import webhook, slack, hipchat, email, base
from rhodecode.lib.exc_tracking import store_exception
log = logging.getLogger(__name__)
@ -61,6 +63,8 @@ def integrations_event_handler(event):
try:
integration_model.send_event(integration, event)
except Exception:
exc_info = sys.exc_info()
store_exception(id(exc_info), exc_info)
log.exception(
'failure occurred when sending event %s to integration %s' % (
event, integration))

7
rhodecode/lib/exc_tracking.py
View file

@ -87,6 +87,13 @@ def _store_exception(exc_id, exc_info, prefix):
def store_exception(exc_id, exc_info, prefix=global_prefix):
"""
Example usage::
exc_info = sys.exc_info()
store_exception(id(exc_info), exc_info)
"""
try:
_store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
except Exception:

91
rhodecode/lib/repo_maintenance.py
View file

@ -51,25 +51,81 @@ class GitGC(MaintenanceTask):
output = []
instance = self.db_repo.scm_instance()
objects = self._count_objects(instance)
output.append(objects)
log.debug('GIT objects:%s', objects)
objects_before = self._count_objects(instance)
stdout, stderr = instance.run_git_command(
['gc', '--aggressive'], fail_on_stderr=False)
out = 'executed git gc --aggressive'
if stderr:
out = ''.join(stderr.splitlines())
elif stdout:
out = ''.join(stdout.splitlines())
log.debug('GIT objects:%s', objects_before)
cmd = ['gc', '--aggressive']
stdout, stderr = instance.run_git_command(cmd, fail_on_stderr=False)
out = 'executed {}'.format(' '.join(cmd))
output.append(out)
objects = self._count_objects(instance)
log.debug('GIT objects:%s', objects)
output.append(objects)
out = ''
if stderr:
out += ''.join(stderr.splitlines())
if stdout:
out += ''.join(stdout.splitlines())
if out:
output.append(out)
objects_after = self._count_objects(instance)
log.debug('GIT objects:%s', objects_after)
output.append('objects before :' + objects_before)
output.append('objects after :' + objects_after)
return '\n'.join(output)
class GitFSCK(MaintenanceTask):
human_name = 'GIT FSCK'
def run(self):
output = []
instance = self.db_repo.scm_instance()
cmd = ['fsck', '--full']
stdout, stderr = instance.run_git_command(cmd, fail_on_stderr=False)
out = 'executed {}'.format(' '.join(cmd))
output.append(out)
out = ''
if stderr:
out += ''.join(stderr.splitlines())
if stdout:
out += ''.join(stdout.splitlines())
if out:
output.append(out)
return '\n'.join(output)
class GitRepack(MaintenanceTask):
human_name = 'GIT Repack'
def run(self):
output = []
instance = self.db_repo.scm_instance()
cmd = ['repack', '-a', '-d',
'--window-memory', '10m', '--max-pack-size', '100m']
stdout, stderr = instance.run_git_command(cmd, fail_on_stderr=False)
out = 'executed {}'.format(' '.join(cmd))
output.append(out)
out = ''
if stderr:
out += ''.join(stderr.splitlines())
if stdout:
out += ''.join(stdout.splitlines())
if out:
output.append(out)
return '\n'.join(output)
@ -98,7 +154,7 @@ class RepoMaintenance(object):
"""
tasks = {
'hg': [HGVerify],
'git': [GitGC],
'git': [GitFSCK, GitGC, GitRepack],
'svn': [SVNVerify],
}
@ -114,5 +170,6 @@ class RepoMaintenance(object):
def execute(self, db_repo):
executed_tasks = []
for task in self.tasks[db_repo.repo_type]:
executed_tasks.append(task(db_repo).run())
output = task.human_name + ':\n' + task(db_repo).run() + '\n--\n'
executed_tasks.append(output)
return executed_tasks