engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

system-info: fetch vcs settings from vcsserver. Fixes #4276

Browse source 
- redone the system info page. Errors/Warnings now show next to the
data, and also show up in API call.
- implemented fetching of git/hg/svn version from running vcsserver
- implemented fetching of vcsserver version from itself
- added more warnings/errors detection on RAM/Inodes etc
- few cleanups
This commit is contained in:
Marcin Kuzminski 2016-11-14 21:14:29 +01:00
parent edcc18406a
commit c68c339c0b
13 changed files with 756 additions and 356 deletions
Download patch file Download diff file Expand all files Collapse all files

12
rhodecode/api/tests/test_get_server_info.py
View file

@ -45,9 +45,9 @@ class TestGetServerInfo(object):
expected['uptime'] = resp['result']['uptime']
expected['load'] = resp['result']['load']
expected['cpu'] = resp['result']['cpu']
expected['disk'] = resp['result']['disk']
expected['disk_inodes'] = resp['result']['disk_inodes']
expected['server_ip'] = '127.0.0.1:80'
expected['storage'] = resp['result']['storage']
expected['storage_inodes'] = resp['result']['storage_inodes']
expected['server'] = resp['result']['server']
assert_ok(id_, expected, given=response.body)
@ -60,8 +60,8 @@ class TestGetServerInfo(object):
expected['uptime'] = resp['result']['uptime']
expected['load'] = resp['result']['load']
expected['cpu'] = resp['result']['cpu']
expected['disk'] = resp['result']['disk']
expected['disk_inodes'] = resp['result']['disk_inodes']
expected['server_ip'] = '127.0.0.1:80'
expected['storage'] = resp['result']['storage']
expected['storage_inodes'] = resp['result']['storage_inodes']
expected['server'] = resp['result']['server']
assert_ok(id_, expected, given=response.body)

111
rhodecode/controllers/admin/settings.py
View file

@ -548,62 +548,85 @@ class SettingsController(BaseController):
"""GET /admin/settings/system: All items in the collection"""
# url('admin_settings_system')
snapshot = str2bool(request.GET.get('snapshot'))
c.active = 'system'
defaults = self._form_defaults()
c.rhodecode_ini = rhodecode.CONFIG
c.active = 'system'
c.rhodecode_update_url = defaults.get('rhodecode_update_url')
server_info = ScmModel().get_server_info(request.environ)
for key, val in server_info.iteritems():
setattr(c, key, val)
if c.disk['percent'] > 90:
h.flash(h.literal(_(
'Critical: your disk space is very low <b>%s%%</b> used' %
c.disk['percent'])), 'error')
elif c.disk['percent'] > 70:
h.flash(h.literal(_(
'Warning: your disk space is running low <b>%s%%</b> used' %
c.disk['percent'])), 'warning')
def val(name, subkey='human_value'):
return server_info[name][subkey]
try:
c.uptime_age = h._age(
h.time_to_datetime(c.boot_time), False, show_suffix=False)
except TypeError:
c.uptime_age = c.boot_time
def state(name):
return server_info[name]['state']
try:
c.system_memory = '%s/%s, %s%% (%s%%) used%s' % (
h.format_byte_size_binary(c.memory['used']),
h.format_byte_size_binary(c.memory['total']),
c.memory['percent2'],
c.memory['percent'],
' %s' % c.memory['error'] if 'error' in c.memory else '')
except TypeError:
c.system_memory = 'NOT AVAILABLE'
def val2(name):
val = server_info[name]['human_value']
state = server_info[name]['state']
return val, state
c.data_items = [
# update info
(_('Update info'), h.literal(
'<span class="link" id="check_for_update" >%s.</span>' % (
_('Check for updates')) +
'<br/> <span >%s.</span>' % (_('Note: please make sure this server can access `%s` for the update link to work') % c.rhodecode_update_url)
), ''),
# RhodeCode specific
(_('RhodeCode Version'), c.rhodecode_version, ''),
(_('RhodeCode Server IP'), val('server')['server_ip'], state('server')),
(_('RhodeCode Server ID'), val('server')['server_id'], state('server')),
(_('RhodeCode Configuration'), val('rhodecode_config')['path'], state('rhodecode_config')),
('', '', ''), # spacer
# Database
(_('Database'), val('database')['url'], state('database')),
(_('Database version'), val('database')['version'], state('database')),
('', '', ''), # spacer
# Platform/Python
(_('Platform'), val('platform'), state('platform')),
(_('Python version'), val('python')['version'], state('python')),
(_('Python path'), val('python')['executable'], state('python')),
('', '', ''), # spacer
# Systems stats
(_('CPU'), val('cpu'), state('cpu')),
(_('Load'), val('load'), state('load')),
(_('Memory'), val('memory'), state('memory')),
(_('Uptime'), val('uptime')['uptime'], state('uptime')),
('', '', ''), # spacer
# Repo storage
(_('Storage location'), val('storage')['path'], state('storage')),
(_('Storage info'), val('storage')['text'], state('storage')),
(_('Storage inodes'), val('storage_inodes')['text'], state('storage_inodes')),
(_('Gist storage location'), val('storage_gist')['path'], state('storage_gist')),
(_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
(_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
(_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
(_('Search storage'), val('storage_search')['path'], state('storage_search')),
(_('Search info'), val('storage_search')['text'], state('storage_search')),
('', '', ''), # spacer
# VCS specific
(_('VCS Backends'), val('vcs_backends'), state('vcs_backends')),
(_('VCS Server'), val('vcs_server')['text'], state('vcs_server')),
(_('GIT'), val('git'), state('git')),
(_('HG'), val('hg'), state('hg')),
(_('SVN'), val('svn'), state('svn')),
rhodecode_ini_safe = rhodecode.CONFIG.copy()
blacklist = [
'rhodecode_license_key',
'routes.map',
'pylons.h',
'pylons.app_globals',
'pylons.environ_config',
'sqlalchemy.db1.url',
('app_conf', 'sqlalchemy.db1.url')
]
for k in blacklist:
if isinstance(k, tuple):
section, key = k
if section in rhodecode_ini_safe:
rhodecode_ini_safe[section].pop(key, None)
else:
rhodecode_ini_safe.pop(k, None)
c.rhodecode_ini_safe = rhodecode_ini_safe
# TODO: marcink, figure out how to allow only selected users to do this
c.allowed_to_snapshot = False
c.allowed_to_snapshot = c.rhodecode_user.admin
if snapshot:
if c.allowed_to_snapshot:

582
rhodecode/lib/system_info.py Normal file
View file

@ -0,0 +1,582 @@
import os
import sys
import time
import platform
import pkg_resources
import logging
log = logging.getLogger(__name__)
psutil = None
try:
# cygwin cannot have yet psutil support.
import psutil as psutil
except ImportError:
pass
_NA = 'NOT AVAILABLE'
STATE_OK = 'ok'
STATE_ERR = 'error'
STATE_WARN = 'warning'
STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK}
# HELPERS
def percentage(part, whole):
whole = float(whole)
if whole > 0:
return 100 * float(part) / whole
return 0
def get_storage_size(storage_path):
sizes = []
for file_ in os.listdir(storage_path):
storage_file = os.path.join(storage_path, file_)
if os.path.isfile(storage_file):
try:
sizes.append(os.path.getsize(storage_file))
except OSError:
log.exception('Failed to get size of storage file %s',
storage_file)
pass
return sum(sizes)
class SysInfoRes(object):
def __init__(self, value, state=STATE_OK_DEFAULT, human_value=None):
self.value = value
self.state = state
self.human_value = human_value or value
def __json__(self):
return {
'value': self.value,
'state': self.state,
'human_value': self.human_value,
}
def __str__(self):
return '<SysInfoRes({})>'.format(self.__json__())
class SysInfo(object):
def __init__(self, func_name, **kwargs):
self.func_name = func_name
self.value = _NA
self.state = None
self.kwargs = kwargs or {}
def __call__(self):
computed = self.compute(**self.kwargs)
if not isinstance(computed, SysInfoRes):
raise ValueError(
'computed value for {} is not instance of '
'{}, got {} instead'.format(
self.func_name, SysInfoRes, type(computed)))
return computed.__json__()
def __str__(self):
return '<SysInfo({})>'.format(self.func_name)
def compute(self, **kwargs):
return self.func_name(**kwargs)
# SysInfo functions
def python_info():
value = {
'version': ' '.join(platform._sys_version()),
'executable': sys.executable
}
return SysInfoRes(value=value)
def py_modules():
mods = dict([(p.project_name, p.version)
for p in pkg_resources.working_set])
value = sorted(mods.items(), key=lambda k: k[0].lower())
return SysInfoRes(value=value)
def platform_type():
from rhodecode.lib.utils import safe_unicode
value = safe_unicode(platform.platform())
return SysInfoRes(value=value)
def uptime():
from rhodecode.lib.helpers import age, time_to_datetime
from rhodecode.translation import _
_uptime = _uptime_human = {'boot_time': 0, 'uptime': 0}
state = STATE_OK_DEFAULT
if not psutil:
return SysInfoRes(value=_uptime, state=state)
boot_time = psutil.boot_time()
_uptime['boot_time'] = boot_time
_uptime['uptime'] = time.time() - boot_time
_uptime_human['boot_time'] = time_to_datetime(boot_time)
_uptime_human['uptime'] = _('Server started {}').format(
age(time_to_datetime(boot_time)))
return SysInfoRes(value=_uptime, human_value=_uptime_human)
def memory():
from rhodecode.lib.helpers import format_byte_size_binary
_memory = {'available': 0, 'used': 0, 'cached': 0, 'percent': 0,
'percent_used': 0, 'free': 0, 'inactive': 0, 'active': 0,
'shared': 0, 'total': 0, 'buffers': 0}
state = STATE_OK_DEFAULT
if not psutil:
return SysInfoRes(value=_memory, state=state)
# memory
_memory = dict(psutil.virtual_memory()._asdict())
_memory['percent_used'] = psutil._common.usage_percent(
(_memory['total'] - _memory['free']), _memory['total'], 1)
try:
human_value = '%s/%s, %s%% used' % (
format_byte_size_binary(_memory['used']),
format_byte_size_binary(_memory['total']),
_memory['percent_used'],)
except TypeError:
human_value = 'NOT AVAILABLE'
if state['type'] == STATE_OK and _memory['percent_used'] > 90:
msg = 'Critical: your available RAM memory is very low.'
state = {'message': msg, 'type': STATE_ERR}
elif state['type'] == STATE_OK and _memory['percent_used'] > 70:
msg = 'Warning: your available RAM memory is running low.'
state = {'message': msg, 'type': STATE_WARN}
return SysInfoRes(value=_memory, state=state, human_value=human_value)
def machine_load():
_load = {'1_min': _NA, '5_min': _NA, '15_min': _NA}
state = STATE_OK_DEFAULT
if not psutil:
return SysInfoRes(value=_load, state=state)
# load averages
if hasattr(psutil.os, 'getloadavg'):
_load = dict(zip(['1_min', '5_min', '15_min'], psutil.os.getloadavg()))
human_value = '1min: %s, 5min: %s, 15min: %s' % (
_load['1_min'], _load['5_min'], _load['15_min'])
# TODO: warn about too-much load 15 min
return SysInfoRes(value=_load, state=state, human_value=human_value)
def cpu():
state = STATE_OK_DEFAULT
cpu_value = 0
if not psutil:
return SysInfoRes(value=cpu_value, state=state)
cpu_value = psutil.cpu_percent(0.5)
human_value = '{} %'.format(cpu_value)
return SysInfoRes(value=cpu_value, state=state, human_value=human_value)
def storage():
from rhodecode.lib.helpers import format_byte_size_binary
from rhodecode.model.settings import VcsSettingsModel
path = VcsSettingsModel().get_repos_location()
# disk storage
disk = {'percent': 0, 'used': 0, 'total': 0, 'path': path, 'text': ''}
state = STATE_OK_DEFAULT
if not psutil:
return SysInfoRes(value=disk, state=state)
try:
disk.update(dict(psutil.disk_usage(path)._asdict()))
except Exception as e:
log.exception('Failed to fetch disk info')
state = {'message': str(e), 'type': STATE_ERR}
human_value = disk
human_value['text'] = "{}/{}, {}% used".format(
format_byte_size_binary(disk['used']),
format_byte_size_binary(disk['total']),
(disk['percent']))
if state['type'] == STATE_OK and disk['percent'] > 90:
msg = 'Critical: your disk space is very low.'
state = {'message': msg, 'type': STATE_ERR}
elif state['type'] == STATE_OK and disk['percent'] > 70:
msg = 'Warning: your disk space is running low.'
state = {'message': msg, 'type': STATE_WARN}
return SysInfoRes(value=disk, state=state, human_value=human_value)
def storage_inodes():
from rhodecode.model.settings import VcsSettingsModel
path = VcsSettingsModel().get_repos_location()
_disk_inodes = dict(percent=0, free=0, used=0, total=0, path=path, text='')
state = STATE_OK_DEFAULT
if not psutil:
return SysInfoRes(value=_disk_inodes, state=state)
try:
i_stat = os.statvfs(path)
_disk_inodes['used'] = i_stat.f_ffree
_disk_inodes['free'] = i_stat.f_favail
_disk_inodes['total'] = i_stat.f_files
_disk_inodes['percent'] = percentage(
_disk_inodes['used'], _disk_inodes['total'])
except Exception as e:
log.exception('Failed to fetch disk inodes info')
state = {'message': str(e), 'type': STATE_ERR}
human_value = _disk_inodes
human_value['text'] = "{}/{}, {}% used".format(
_disk_inodes['used'], _disk_inodes['total'], _disk_inodes['percent'])
if state['type'] == STATE_OK and _disk_inodes['percent'] > 90:
msg = 'Critical: your disk free inodes are very low.'
state = {'message': msg, 'type': STATE_ERR}
elif state['type'] == STATE_OK and _disk_inodes['percent'] > 70:
msg = 'Warning: your disk free inodes are running low.'
state = {'message': msg, 'type': STATE_WARN}
return SysInfoRes(value=_disk_inodes, state=state)
def storage_archives():
import rhodecode
from rhodecode.lib.utils import safe_str
from rhodecode.lib.helpers import format_byte_size_binary
msg = 'Enable this by setting ' \
'archive_cache_dir=/path/to/cache option in the .ini file'
path = safe_str(rhodecode.CONFIG.get('archive_cache_dir', msg))
disk_archive = dict(percent=0, used=0, total=0, items=0, path=path, text='')
state = STATE_OK_DEFAULT
try:
items_count = 0
used = 0
for root, dirs, files in os.walk(path):
if root == path:
items_count = len(dirs)
for f in files:
try:
used += os.path.getsize(os.path.join(root, f))
except OSError:
pass
disk_archive.update({
'percent': 100,
'used': used,
'total': used,
'items': items_count
})
except Exception as e:
log.exception('failed to fetch archive cache storage')
state = {'message': str(e), 'type': STATE_ERR}
human_value = disk_archive
human_value['text'] = "{} ({} items)".format(format_byte_size_binary(
disk_archive['used']), disk_archive['total'])
return SysInfoRes(value=disk_archive, state=state, human_value=human_value)
def storage_gist():
from rhodecode.model.gist import GIST_STORE_LOC
from rhodecode.model.settings import VcsSettingsModel
from rhodecode.lib.utils import safe_str
from rhodecode.lib.helpers import format_byte_size_binary
path = safe_str(os.path.join(
VcsSettingsModel().get_repos_location(), GIST_STORE_LOC))
# gist storage
_disk_gist = dict(percent=0, used=0, total=0, items=0, path=path, text='')
state = STATE_OK_DEFAULT
try:
items_count = 0
used = 0
for root, dirs, files in os.walk(path):
if root == path:
items_count = len(dirs)
for f in files:
try:
used += os.path.getsize(os.path.join(root, f))
except OSError:
pass
_disk_gist.update({
'percent': 100,
'used': used,
'total': used,
'items': items_count
})
except Exception as e:
log.exception('failed to fetch gist storage items')
state = {'message': str(e), 'type': STATE_ERR}
human_value = _disk_gist
human_value['text'] = "{} ({} items)".format(format_byte_size_binary(
_disk_gist['used']), _disk_gist['items'])
return SysInfoRes(value=_disk_gist, state=state, human_value=human_value)
def storage_search():
import rhodecode
path = rhodecode.CONFIG.get('search.location', '')
# search index storage
_disk_index = dict(percent=0, used=0, total=0, path=path, text='')
state = STATE_OK_DEFAULT
try:
search_index_storage_path_exists = os.path.isdir(path)
if search_index_storage_path_exists:
used = get_storage_size(path)
_disk_index.update({
'percent': 100,
'used': used,
'total': used,
})
except Exception as e:
log.exception('failed to fetch search index storage')
state = {'message': str(e), 'type': STATE_ERR}
human_value = _disk_index
human_value['text'] = "{}/{}, {}% used".format(
_disk_index['used'], _disk_index['total'], _disk_index['percent'])
return SysInfoRes(value=_disk_index, state=state, human_value=human_value)
def git_info():
from rhodecode.lib.vcs.backends import git
state = STATE_OK_DEFAULT
value = human_value = ''
try:
value = git.discover_git_version(raise_on_exc=True)
human_value = 'version reported from VCSServer: {}'.format(value)
except Exception as e:
state = {'message': str(e), 'type': STATE_ERR}
return SysInfoRes(value=value, state=state, human_value=human_value)
def hg_info():
from rhodecode.lib.vcs.backends import hg
state = STATE_OK_DEFAULT
value = human_value = ''
try:
value = hg.discover_hg_version(raise_on_exc=True)
human_value = 'version reported from VCSServer: {}'.format(value)
except Exception as e:
state = {'message': str(e), 'type': STATE_ERR}
return SysInfoRes(value=value, state=state, human_value=human_value)
def svn_info():
from rhodecode.lib.vcs.backends import svn
state = STATE_OK_DEFAULT
value = human_value = ''
try:
value = svn.discover_svn_version(raise_on_exc=True)
human_value = 'version reported from VCSServer: {}'.format(value)
except Exception as e:
state = {'message': str(e), 'type': STATE_ERR}
return SysInfoRes(value=value, state=state, human_value=human_value)
def vcs_backends():
import rhodecode
value = rhodecode.CONFIG.get('vcs.backends', '').split(',')
human_value = 'Enabled backends in order: {}'.format(','.join(value))
return SysInfoRes(value=value, human_value=human_value)
def vcs_server():
import rhodecode
from rhodecode.lib.vcs.backends import get_vcsserver_version
server_url = rhodecode.CONFIG.get('vcs.server')
enabled = rhodecode.CONFIG.get('vcs.server.enable')
protocol = rhodecode.CONFIG.get('vcs.server.protocol')
state = STATE_OK_DEFAULT
version = None
try:
version = get_vcsserver_version()
connection = 'connected'
except Exception as e:
connection = 'failed'
state = {'message': str(e), 'type': STATE_ERR}
value = dict(
url=server_url,
enabled=enabled,
protocol=protocol,
connection=connection,
version=version,
text='',
)
human_value = value
human_value['text'] = \
'{url}@ver:{ver} via {mode} mode, connection:{conn}'.format(
url=server_url, ver=version, mode=protocol, conn=connection)
return SysInfoRes(value='', state=state, human_value=human_value)
def rhodecode_app_info():
import rhodecode
return SysInfoRes(value={'rhodecode_version': rhodecode.__version__})
def rhodecode_config():
import rhodecode
path = rhodecode.CONFIG.get('__file__')
rhodecode_ini_safe = rhodecode.CONFIG.copy()
blacklist = [
'rhodecode_license_key',
'routes.map',
'pylons.h',
'pylons.app_globals',
'pylons.environ_config',
'sqlalchemy.db1.url',
'channelstream.secret',
'beaker.session.secret',
'rhodecode.encrypted_values.secret',
'rhodecode_auth_github_consumer_key',
'rhodecode_auth_github_consumer_secret',
'rhodecode_auth_google_consumer_key',
'rhodecode_auth_google_consumer_secret',
'rhodecode_auth_bitbucket_consumer_secret',
'rhodecode_auth_bitbucket_consumer_key',
'rhodecode_auth_twitter_consumer_secret',
'rhodecode_auth_twitter_consumer_key',
'rhodecode_auth_twitter_secret',
'rhodecode_auth_github_secret',
'rhodecode_auth_google_secret',
'rhodecode_auth_bitbucket_secret',
'appenlight.api_key',
('app_conf', 'sqlalchemy.db1.url')
]
for k in blacklist:
if isinstance(k, tuple):
section, key = k
if section in rhodecode_ini_safe:
rhodecode_ini_safe[section] = '**OBFUSCATED**'
else:
rhodecode_ini_safe.pop(k, None)
# TODO: maybe put some CONFIG checks here ?
return SysInfoRes(value={'config': rhodecode_ini_safe, 'path': path})
def database_info():
import rhodecode
from sqlalchemy.engine import url as engine_url
from rhodecode.model.meta import Base as sql_base, Session
from rhodecode.model.db import DbMigrateVersion
state = STATE_OK_DEFAULT
db_migrate = DbMigrateVersion.query().filter(
DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
db_url_obj = engine_url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
try:
engine = sql_base.metadata.bind
db_server_info = engine.dialect._get_server_version_info(
Session.connection(bind=engine))
db_version = '.'.join(map(str, db_server_info))
except Exception:
log.exception('failed to fetch db version')
db_version = 'UNKNOWN'
db_info = dict(
migrate_version=db_migrate.version,
type=db_url_obj.get_backend_name(),
version=db_version,
url=repr(db_url_obj)
)
human_value = db_info
human_value['url'] = "{} @ migration version: {}".format(
db_info['url'], db_info['migrate_version'])
human_value['version'] = "{} {}".format(db_info['type'], db_info['version'])
return SysInfoRes(value=db_info, state=state, human_value=human_value)
def server_info(environ):
import rhodecode
from rhodecode.lib.base import get_server_ip_addr, get_server_port
value = {
'server_ip': '%s:%s' % (
get_server_ip_addr(environ, log_errors=False),
get_server_port(environ)
),
'server_id': rhodecode.CONFIG.get('instance_id'),
}
return SysInfoRes(value=value)
def get_system_info(environ):
environ = environ or {}
return {
'rhodecode_app': SysInfo(rhodecode_app_info)(),
'rhodecode_config': SysInfo(rhodecode_config)(),
'python': SysInfo(python_info)(),
'py_modules': SysInfo(py_modules)(),
'platform': SysInfo(platform_type)(),
'server': SysInfo(server_info, environ=environ)(),
'database': SysInfo(database_info)(),
'storage': SysInfo(storage)(),
'storage_inodes': SysInfo(storage_inodes)(),
'storage_archive': SysInfo(storage_archives)(),
'storage_search': SysInfo(storage_search)(),
'storage_gist': SysInfo(storage_gist)(),
'uptime': SysInfo(uptime)(),
'load': SysInfo(machine_load)(),
'cpu': SysInfo(cpu)(),
'memory': SysInfo(memory)(),
'vcs_backends': SysInfo(vcs_backends)(),
'vcs_server': SysInfo(vcs_server)(),
'git': SysInfo(git_info)(),
'hg': SysInfo(hg_info)(),
'svn': SysInfo(svn_info)(),
}

4
rhodecode/lib/vcs/__init__.py
View file

@ -99,6 +99,7 @@ def connect_pyro4(server_and_port):
connection.Git = None
connection.Hg = None
connection.Svn = None
connection.Service = None
def connect_http(server_and_port):
@ -113,6 +114,8 @@ def connect_http(server_and_port):
server_and_port, '/hg', session_factory)
connection.Svn = client_http.RepoMaker(
server_and_port, '/svn', session_factory)
connection.Service = client_http.ServiceConnection(
server_and_port, '/_service', session_factory)
scm_app.HG_REMOTE_WSGI = client_http.VcsHttpProxy(
server_and_port, '/proxy/hg')
@ -124,6 +127,7 @@ def connect_http(server_and_port):
connection.Git = None
connection.Hg = None
connection.Svn = None
connection.Service = None
def connect_vcs(server_and_port, protocol):

9
rhodecode/lib/vcs/backends/__init__.py
View file

@ -76,3 +76,12 @@ def get_supported_backends():
Returns list of aliases of supported backends.
"""
return settings.BACKENDS.keys()
def get_vcsserver_version():
from rhodecode.lib.vcs import connection
data = connection.Service.get_vcsserver_service_data()
if data and 'version' in data:
return data['version']
return None

4
rhodecode/lib/vcs/backends/git/__init__.py
View file

@ -33,7 +33,7 @@ from rhodecode.lib.vcs.backends.git.inmemory import GitInMemoryCommit
log = logging.getLogger(__name__)
def discover_git_version():
def discover_git_version(raise_on_exc=False):
"""
Returns the string as it was returned by running 'git --version'
@ -44,4 +44,6 @@ def discover_git_version():
return connection.Git.discover_git_version()
except Exception:
log.warning("Failed to discover the Git version", exc_info=True)
if raise_on_exc:
raise
return ''

21
rhodecode/lib/vcs/backends/hg/__init__.py
View file

@ -21,7 +21,28 @@
"""
HG module
"""
import logging
from rhodecode.lib.vcs import connection
from rhodecode.lib.vcs.backends.hg.commit import MercurialCommit
from rhodecode.lib.vcs.backends.hg.inmemory import MercurialInMemoryCommit
from rhodecode.lib.vcs.backends.hg.repository import MercurialRepository
log = logging.getLogger(__name__)
def discover_hg_version(raise_on_exc=False):
"""
Returns the string as it was returned by running 'git --version'
It will return an empty string in case the connection is not initialized
or no vcsserver is available.
"""
try:
return connection.Hg.discover_hg_version()
except Exception:
log.warning("Failed to discover the HG version", exc_info=True)
if raise_on_exc:
raise
return ''

21
rhodecode/lib/vcs/backends/svn/__init__.py
View file

@ -21,6 +21,27 @@
"""
SVN module
"""
import logging
from rhodecode.lib.vcs import connection
from rhodecode.lib.vcs.backends.svn.commit import SubversionCommit
from rhodecode.lib.vcs.backends.svn.repository import SubversionRepository
log = logging.getLogger(__name__)
def discover_svn_version(raise_on_exc=False):
"""
Returns the string as it was returned by running 'git --version'
It will return an empty string in case the connection is not initialized
or no vcsserver is available.
"""
try:
return connection.Svn.discover_svn_version()
except Exception:
log.warning("Failed to discover the SVN version", exc_info=True)
if raise_on_exc:
raise
return ''

2
rhodecode/lib/vcs/connection.py
View file

@ -30,6 +30,8 @@ def _not_initialized(*args, **kwargs):
"Make sure `vcs.server` is enabled in your configuration.")
# TODO: figure out a nice default value for these things
Service = _not_initialized
Git = _not_initialized
Hg = _not_initialized
Svn = _not_initialized

222
rhodecode/model/scm.py
View file

@ -25,13 +25,11 @@ Scm model for RhodeCode
import os.path
import re
import sys
import time
import traceback
import logging
import cStringIO
import pkg_resources
import pylons
from pylons.i18n.translation import _
from sqlalchemy import func
from zope.cachedescriptors.property import Lazy as LazyProperty
@ -50,12 +48,12 @@ from rhodecode.lib.exceptions import NonRelativePathError, IMCCommitError
from rhodecode.lib import hooks_utils, caches
from rhodecode.lib.utils import (
get_filesystem_repos, action_logger, make_db_config)
from rhodecode.lib.utils2 import (
safe_str, safe_unicode, get_server_url, md5)
from rhodecode.lib.utils2 import (safe_str, safe_unicode)
from rhodecode.lib.system_info import get_system_info
from rhodecode.model import BaseModel
from rhodecode.model.db import (
Repository, CacheKey, UserFollowing, UserLog, User, RepoGroup,
PullRequest, DbMigrateVersion)
PullRequest)
from rhodecode.model.settings import VcsSettingsModel
log = logging.getLogger(__name__)
@ -882,218 +880,8 @@ class ScmModel(BaseModel):
self.install_svn_hooks(repo)
def get_server_info(self, environ=None):
import platform
import rhodecode
import pkg_resources
from rhodecode.model.meta import Base as sql_base, Session
from sqlalchemy.engine import url
from rhodecode.lib.base import get_server_ip_addr, get_server_port
from rhodecode.lib.vcs.backends.git import discover_git_version
from rhodecode.model.gist import GIST_STORE_LOC
def percentage(part, whole):
whole = float(whole)
if whole > 0:
return 100 * float(part) / whole
return 0
try:
# cygwin cannot have yet psutil support.
import psutil
except ImportError:
psutil = None
environ = environ or {}
_NA = 'NOT AVAILABLE'
_memory = _NA
_uptime = _NA
_boot_time = _NA
_cpu = _NA
_disk = dict(percent=0, used=0, total=0, error='')
_disk_inodes = dict(percent=0, free=0, used=0, total=0, error='')
_load = {'1_min': _NA, '5_min': _NA, '15_min': _NA}
model = VcsSettingsModel()
storage_path = model.get_repos_location()
gist_storage_path = os.path.join(storage_path, GIST_STORE_LOC)
archive_storage_path = rhodecode.CONFIG.get('archive_cache_dir', '')
search_index_storage_path = rhodecode.CONFIG.get('search.location', '')
if psutil:
# disk storage
try:
_disk = dict(psutil.disk_usage(storage_path)._asdict())
except Exception as e:
log.exception('Failed to fetch disk info')
_disk = {'percent': 0, 'used': 0, 'total': 0, 'error': str(e)}
# disk inodes usage
try:
i_stat = os.statvfs(storage_path)
_disk_inodes['used'] = i_stat.f_ffree
_disk_inodes['free'] = i_stat.f_favail
_disk_inodes['total'] = i_stat.f_files
_disk_inodes['percent'] = percentage(
_disk_inodes['used'], _disk_inodes['total'])
except Exception as e:
log.exception('Failed to fetch disk inodes info')
_disk_inodes['error'] = str(e)
# memory
_memory = dict(psutil.virtual_memory()._asdict())
_memory['percent2'] = psutil._common.usage_percent(
(_memory['total'] - _memory['free']),
_memory['total'], 1)
# load averages
if hasattr(psutil.os, 'getloadavg'):
_load = dict(zip(
['1_min', '5_min', '15_min'], psutil.os.getloadavg()))
_uptime = time.time() - psutil.boot_time()
_boot_time = psutil.boot_time()
_cpu = psutil.cpu_percent(0.5)
mods = dict([(p.project_name, p.version)
for p in pkg_resources.working_set])
def get_storage_size(storage_path):
sizes = []
for file_ in os.listdir(storage_path):
storage_file = os.path.join(storage_path, file_)
if os.path.isfile(storage_file):
try:
sizes.append(os.path.getsize(storage_file))
except OSError:
log.exception('Failed to get size of storage file %s',
storage_file)
pass
return sum(sizes)
# archive cache storage
_disk_archive = {'percent': 0, 'used': 0, 'total': 0}
try:
archive_storage_path_exists = os.path.isdir(
archive_storage_path)
if archive_storage_path and archive_storage_path_exists:
used = get_storage_size(archive_storage_path)
_disk_archive.update({
'used': used,
'total': used,
})
except Exception as e:
log.exception('failed to fetch archive cache storage')
_disk_archive['error'] = str(e)
# search index storage
_disk_index = {'percent': 0, 'used': 0, 'total': 0}
try:
search_index_storage_path_exists = os.path.isdir(
search_index_storage_path)
if search_index_storage_path_exists:
used = get_storage_size(search_index_storage_path)
_disk_index.update({
'percent': 100,
'used': used,
'total': used,
})
except Exception as e:
log.exception('failed to fetch search index storage')
_disk_index['error'] = str(e)
# gist storage
_disk_gist = {'percent': 0, 'used': 0, 'total': 0, 'items': 0}
try:
items_count = 0
used = 0
for root, dirs, files in os.walk(safe_str(gist_storage_path)):
if root == gist_storage_path:
items_count = len(dirs)
for f in files:
try:
used += os.path.getsize(os.path.join(root, f))
except OSError:
pass
_disk_gist.update({
'percent': 100,
'used': used,
'total': used,
'items': items_count
})
except Exception as e:
log.exception('failed to fetch gist storage items')
_disk_gist['error'] = str(e)
# GIT info
git_ver = discover_git_version()
# SVN info
# TODO: johbo: Add discover_svn_version to replace this code.
try:
import svn.core
svn_ver = svn.core.SVN_VERSION
except ImportError:
svn_ver = None
# DB stuff
db_info = url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
db_type = db_info.__to_string__()
try:
engine = sql_base.metadata.bind
db_server_info = engine.dialect._get_server_version_info(
Session.connection(bind=engine))
db_version = '%s %s' % (db_info.drivername,
'.'.join(map(str, db_server_info)))
except Exception:
log.exception('failed to fetch db version')
db_version = '%s %s' % (db_info.drivername, '?')
db_migrate = DbMigrateVersion.query().filter(
DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
db_migrate_version = db_migrate.version
info = {
'py_version': ' '.join(platform._sys_version()),
'py_path': sys.executable,
'py_modules': sorted(mods.items(), key=lambda k: k[0].lower()),
'platform': safe_unicode(platform.platform()),
'storage': storage_path,
'archive_storage': archive_storage_path,
'index_storage': search_index_storage_path,
'gist_storage': gist_storage_path,
'db_type': db_type,
'db_version': db_version,
'db_migrate_version': db_migrate_version,
'rhodecode_version': rhodecode.__version__,
'rhodecode_config_ini': rhodecode.CONFIG.get('__file__'),
'server_ip': '%s:%s' % (
get_server_ip_addr(environ, log_errors=False),
get_server_port(environ)
),
'server_id': rhodecode.CONFIG.get('instance_id'),
'git_version': safe_unicode(git_ver),
'hg_version': mods.get('mercurial'),
'svn_version': svn_ver,
'uptime': _uptime,
'boot_time': _boot_time,
'load': _load,
'cpu': _cpu,
'memory': _memory,
'disk': _disk,
'disk_inodes': _disk_inodes,
'disk_archive': _disk_archive,
'disk_gist': _disk_gist,
'disk_index': _disk_index,
}
return info
server_info = get_system_info(environ)
return server_info
def _check_rhodecode_hook(hook_path):

59
rhodecode/templates/admin/settings/settings_system.html
View file

@ -1,42 +1,3 @@
<%
elems = [
## general
(_('RhodeCode Enterprise version'), h.literal('%s <div class="link" id="check_for_update" >%s</div>' % (c.rhodecode_version, _('check for updates'))), ''),
(_('Upgrade info endpoint'), h.literal('%s <br/><span >%s.</span>' % (c.rhodecode_update_url, _('Note: please make sure this server can access this url'))), ''),
(_('Configuration INI file'), c.rhodecode_config_ini, ''),
## systems stats
(_('RhodeCode Enterprise Server IP'), c.server_ip, ''),
(_('RhodeCode Enterprise Server ID'), c.server_id, ''),
(_('Platform'), c.platform, ''),
(_('Uptime'), c.uptime_age, ''),
(_('Storage location'), c.storage, ''),
(_('Storage disk space'), "%s/%s, %s%% used%s" % (h.format_byte_size_binary(c.disk['used']), h.format_byte_size_binary(c.disk['total']),(c.disk['percent']), ' %s' % c.disk['error'] if 'error' in c.disk else ''), ''),
(_('Storage file limit (inodes)'), "%s/%s, %.1f%% used%s" % (c.disk_inodes['used'], c.disk_inodes['total'],(c.disk_inodes['percent']), ' %s' % c.disk_inodes['error'] if 'error' in c.disk_inodes else ''), ''),
(_('Search index storage'), c.index_storage, ''),
(_('Search index size'), "%s %s" % (h.format_byte_size_binary(c.disk_index['used']), ' %s' % c.disk_index['error'] if 'error' in c.disk_index else ''), ''),
(_('Gist storage'), c.gist_storage, ''),
(_('Gist storage size'), "%s (%s items)%s" % (h.format_byte_size_binary(c.disk_gist['used']),c.disk_gist['items'], ' %s' % c.disk_gist['error'] if 'error' in c.disk_gist else ''), ''),
(_('Archive cache'), h.literal('%s <br/><span >%s.</span>' % (c.archive_storage, _('Enable this by setting archive_cache_dir=/path/to/cache option in the .ini file'))), ''),
(_('Archive cache size'), "%s%s" % (h.format_byte_size_binary(c.disk_archive['used']), ' %s' % c.disk_archive['error'] if 'error' in c.disk_archive else ''), ''),
(_('System memory'), c.system_memory, ''),
(_('CPU'), '%s %%' %(c.cpu), ''),
(_('Load'), '1min: %s, 5min: %s, 15min: %s' %(c.load['1_min'],c.load['5_min'],c.load['15_min']), ''),
## rhodecode stuff
(_('Python version'), c.py_version, ''),
(_('Python path'), c.py_path, ''),
(_('GIT version'), c.git_version, ''),
(_('HG version'), c.hg_version, ''),
(_('SVN version'), c.svn_version, ''),
(_('Database'), "%s @ version: %s" % (c.db_type, c.db_migrate_version), ''),
(_('Database version'), c.db_version, ''),
]
%>
<div id="update_notice" style="display: none; margin: -40px 0px 20px 0px">
<div>${_('Checking for updates...')}</div>
@ -47,15 +8,21 @@
<div class="panel-heading">
<h3 class="panel-title">${_('System Info')}</h3>
% if c.allowed_to_snapshot:
<a href="${url('admin_settings_system', snapshot=1)}" class="panel-edit">${_('create snapshot')}</a>
<a href="${url('admin_settings_system', snapshot=1)}" class="panel-edit">${_('create summary snapshot')}</a>
% endif
</div>
<div class="panel-body">
<dl class="dl-horizontal settings">
%for dt, dd, tt in elems:
<dt>${dt}:</dt>
<dd title="${tt}">${dd}</dd>
%endfor
% for dt, dd, warn in c.data_items:
<dt>${dt}${':' if dt else '---'}</dt>
<dd>${dd}${'' if dt else '---'}
% if warn and warn['message']:
<div class="alert-${warn['type']}">
<strong>${warn['message']}</strong>
</div>
% endif
</dd>
% endfor
</dl>
</div>
</div>
@ -71,12 +38,12 @@
<col class='content'>
</colgroup>
<tbody>
%for key, value in c.py_modules:
% for key, value in c.py_modules['human_value']:
<tr>
<td>${key}</td>
<td>${value}</td>
</tr>
%endfor
% endfor
</tbody>
</table>
</div>

50
rhodecode/templates/admin/settings/settings_system_snapshot.html
View file

@ -1,61 +1,27 @@
<%
elems = [
## general
(_('RhodeCode Enterprise version'), c.rhodecode_version, ''),
(_('Upgrade info endpoint'), c.rhodecode_update_url, ''),
(_('Configuration INI file'), c.rhodecode_config_ini, ''),
## systems stats
(_('RhodeCode Enterprise Server IP'), c.server_ip, ''),
(_('RhodeCode Enterprise Server ID'), c.server_id, ''),
(_('Platform'), c.platform, ''),
(_('Uptime'), c.uptime_age, ''),
(_('Storage location'), c.storage, ''),
(_('Storage disk space'), "%s/%s, %s%% used%s" % (h.format_byte_size_binary(c.disk['used']), h.format_byte_size_binary(c.disk['total']),(c.disk['percent']), ' %s' % c.disk['error'] if 'error' in c.disk else ''), ''),
(_('Search index storage'), c.index_storage, ''),
(_('Search index size'), "%s %s" % (h.format_byte_size_binary(c.disk_index['used']), ' %s' % c.disk_index['error'] if 'error' in c.disk_index else ''), ''),
(_('Gist storage'), c.gist_storage, ''),
(_('Gist storage size'), "%s (%s items)%s" % (h.format_byte_size_binary(c.disk_gist['used']),c.disk_gist['items'], ' %s' % c.disk_gist['error'] if 'error' in c.disk_gist else ''), ''),
(_('Archive cache'), c.archive_storage, ''),
(_('Archive cache size'), "%s%s" % (h.format_byte_size_binary(c.disk_archive['used']), ' %s' % c.disk_archive['error'] if 'error' in c.disk_archive else ''), ''),
(_('System memory'), c.system_memory, ''),
(_('CPU'), '%s %%' %(c.cpu), ''),
(_('Load'), '1min: %s, 5min: %s, 15min: %s' %(c.load['1_min'],c.load['5_min'],c.load['15_min']), ''),
## rhodecode stuff
(_('Python version'), c.py_version, ''),
(_('Python path'), c.py_path, ''),
(_('GIT version'), c.git_version, ''),
(_('HG version'), c.hg_version, ''),
(_('SVN version'), c.svn_version, ''),
(_('Database'), "%s @ version: %s" % (c.db_type, c.db_migrate_version), ''),
(_('Database version'), c.db_version, ''),
]
%>
<pre>
SYSTEM INFO
-----------
% for dt, dd, tt in elems:
${dt}: ${dd}
% for dt, dd, warn in c.data_items:
${dt}${':' if dt else '---'}
${dd}
% if warn and warn['message']:
ALERT_${warn['type'].upper()} ${warn['message']}
% endif
% endfor
PYTHON PACKAGES
---------------
% for key, value in c.py_modules:
% for key, value in c.py_modules['human_value']:
${key}: ${value}
% endfor
SYSTEM SETTINGS
---------------
% for key, value in sorted(c.rhodecode_ini_safe.items()):
% for key, value in sorted(c.rhodecode_config['human_value'].items()):
% if isinstance(value, dict):
% for key2, value2 in value.items():

15
rhodecode/tests/lib/test_system_info.py Normal file
View file

@ -0,0 +1,15 @@
import py.test
from rhodecode.lib.system_info import get_system_info
def test_system_info(app):
info = get_system_info({})
assert info['load']['value']['15_min'] != 'NOT AVAILABLE'
def test_system_info_without_psutil(monkeypatch, app):
import rhodecode.lib.system_info
monkeypatch.setattr(rhodecode.lib.system_info, 'psutil', None)
info = get_system_info({})
assert info['load']['value']['15_min'] == 'NOT AVAILABLE'