caches: don't use beaker for file caches anymore
This commit is contained in:
Marcin Kuzminski
parent
ddc445856e
commit
a48ab8a32d
15 changed files with 245 additions and 201 deletions
|
|
@ -308,47 +308,72 @@ celery.max_tasks_per_child = 100
|
|||
## tasks will never be sent to the queue, but executed locally instead.
|
||||
celery.task_always_eager = false
|
||||
|
||||
#####################################
|
||||
### DOGPILE CACHE ####
|
||||
#####################################
|
||||
## Default cache dir for caches. Putting this into a ramdisk
|
||||
## can boost performance, eg. /tmpfs/data_ramdisk, however this might require lots
|
||||
## of space
|
||||
cache_dir = /tmp/rcdev/data
|
||||
|
||||
## cache settings for permission tree, auth TTL.
|
||||
rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
|
||||
rc_cache.cache_perms.expiration_time = 300
|
||||
rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1
|
||||
|
||||
## redis backend with distributed locks
|
||||
#rc_cache.cache_perms.backend = dogpile.cache.rc.redis
|
||||
#rc_cache.cache_perms.expiration_time = 300
|
||||
#rc_cache.cache_perms.arguments.host = localhost
|
||||
#rc_cache.cache_perms.arguments.port = 6379
|
||||
#rc_cache.cache_perms.arguments.db = 0
|
||||
#rc_cache.cache_perms.arguments.redis_expiration_time = 7200
|
||||
#rc_cache.cache_perms.arguments.distributed_lock = true
|
||||
|
||||
|
||||
rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
|
||||
rc_cache.cache_repo.expiration_time = 2592000
|
||||
rc_cache.cache_repo.arguments.filename = /tmp/rc_cache_2
|
||||
|
||||
## redis backend with distributed locks
|
||||
#rc_cache.cache_repo.backend = dogpile.cache.rc.redis
|
||||
#rc_cache.cache_repo.expiration_time = 2592000
|
||||
## this needs to be greater then expiration_time
|
||||
#rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
|
||||
#rc_cache.cache_repo.arguments.host = localhost
|
||||
#rc_cache.cache_repo.arguments.port = 6379
|
||||
#rc_cache.cache_repo.arguments.db = 1
|
||||
#rc_cache.cache_repo.arguments.distributed_lock = true
|
||||
|
||||
|
||||
####################################
|
||||
### BEAKER CACHE ####
|
||||
####################################
|
||||
# default cache dir for templates. Putting this into a ramdisk
|
||||
## can boost performance, eg. %(here)s/data_ramdisk
|
||||
cache_dir = %(here)s/data
|
||||
|
||||
## locking and default file storage for Beaker. Putting this into a ramdisk
|
||||
## can boost performance, eg. %(here)s/data_ramdisk/cache/beaker_data
|
||||
beaker.cache.data_dir = %(here)s/data/cache/beaker_data
|
||||
beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
|
||||
|
||||
beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
|
||||
beaker.cache.regions = long_term, sql_cache_short
|
||||
|
||||
beaker.cache.long_term.type = memory
|
||||
beaker.cache.long_term.expire = 36000
|
||||
beaker.cache.long_term.type = memorylru_base
|
||||
beaker.cache.long_term.expire = 172800
|
||||
beaker.cache.long_term.key_length = 256
|
||||
|
||||
beaker.cache.sql_cache_short.type = memory
|
||||
beaker.cache.sql_cache_short.type = memorylru_base
|
||||
beaker.cache.sql_cache_short.expire = 10
|
||||
beaker.cache.sql_cache_short.key_length = 256
|
||||
|
||||
beaker.cache.repo_cache_long.type = memorylru_base
|
||||
beaker.cache.repo_cache_long.max_items = 4096
|
||||
beaker.cache.repo_cache_long.expire = 2592000
|
||||
|
||||
## default is memorylru_base cache, configure only if required
|
||||
## using multi-node or multi-worker setup
|
||||
#beaker.cache.repo_cache_long.type = ext:memcached
|
||||
#beaker.cache.repo_cache_long.url = localhost:11211
|
||||
#beaker.cache.repo_cache_long.expire = 1209600
|
||||
#beaker.cache.repo_cache_long.key_length = 256
|
||||
|
||||
####################################
|
||||
### BEAKER SESSION ####
|
||||
####################################
|
||||
|
||||
## .session.type is type of storage options for the session, current allowed
|
||||
## types are file, ext:memcached, ext:database, and memory (default).
|
||||
## types are file, ext:memcached, ext:redis, ext:database, and memory (default).
|
||||
beaker.session.type = file
|
||||
beaker.session.data_dir = %(here)s/data/sessions/data
|
||||
beaker.session.data_dir = %(here)s/data/sessions
|
||||
|
||||
## db based session, fast, and allows easy management over logged in users
|
||||
#beaker.session.type = ext:database
|
||||
|
|
@ -496,6 +521,8 @@ debug_style = true
|
|||
#sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
|
||||
#sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
|
||||
#sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode
|
||||
#sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
|
||||
|
||||
sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
|
||||
|
||||
# see sqlalchemy docs for other advanced settings
|
||||
|
|
@ -515,6 +542,9 @@ sqlalchemy.db1.convert_unicode = true
|
|||
## which defaults to five.
|
||||
#sqlalchemy.db1.max_overflow = 10
|
||||
|
||||
## Connection check ping, used to detect broken database connections
|
||||
## could be enabled to better handle cases if MySQL has gone away errors
|
||||
#sqlalchemy.db1.ping_connection = true
|
||||
|
||||
##################
|
||||
### VCS CONFIG ###
|
||||
|
|
@ -624,7 +654,7 @@ custom.conf = 1
|
|||
### LOGGING CONFIGURATION ####
|
||||
################################
|
||||
[loggers]
|
||||
keys = root, sqlalchemy, beaker, rhodecode, ssh_wrapper, celery
|
||||
keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
|
||||
|
||||
[handlers]
|
||||
keys = console, console_sql
|
||||
|
|
@ -680,9 +710,12 @@ level = DEBUG
|
|||
formatter = color_formatter
|
||||
|
||||
[handler_console_sql]
|
||||
# "level = DEBUG" logs SQL queries and results.
|
||||
# "level = INFO" logs SQL queries.
|
||||
# "level = WARN" logs neither. (Recommended for production systems.)
|
||||
class = StreamHandler
|
||||
args = (sys.stderr, )
|
||||
level = DEBUG
|
||||
level = WARN
|
||||
formatter = color_formatter_sql
|
||||
|
||||
################
|
||||
|
|
|
|||
|
|
@ -283,47 +283,72 @@ celery.max_tasks_per_child = 100
|
|||
## tasks will never be sent to the queue, but executed locally instead.
|
||||
celery.task_always_eager = false
|
||||
|
||||
#####################################
|
||||
### DOGPILE CACHE ####
|
||||
#####################################
|
||||
## Default cache dir for caches. Putting this into a ramdisk
|
||||
## can boost performance, eg. /tmpfs/data_ramdisk, however this might require lots
|
||||
## of space
|
||||
cache_dir = /tmp/rcdev/data
|
||||
|
||||
## cache settings for permission tree, auth TTL.
|
||||
rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
|
||||
rc_cache.cache_perms.expiration_time = 300
|
||||
rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1
|
||||
|
||||
## redis backend with distributed locks
|
||||
#rc_cache.cache_perms.backend = dogpile.cache.rc.redis
|
||||
#rc_cache.cache_perms.expiration_time = 300
|
||||
#rc_cache.cache_perms.arguments.host = localhost
|
||||
#rc_cache.cache_perms.arguments.port = 6379
|
||||
#rc_cache.cache_perms.arguments.db = 0
|
||||
#rc_cache.cache_perms.arguments.redis_expiration_time = 7200
|
||||
#rc_cache.cache_perms.arguments.distributed_lock = true
|
||||
|
||||
|
||||
rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
|
||||
rc_cache.cache_repo.expiration_time = 2592000
|
||||
rc_cache.cache_repo.arguments.filename = /tmp/rc_cache_2
|
||||
|
||||
## redis backend with distributed locks
|
||||
#rc_cache.cache_repo.backend = dogpile.cache.rc.redis
|
||||
#rc_cache.cache_repo.expiration_time = 2592000
|
||||
## this needs to be greater then expiration_time
|
||||
#rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
|
||||
#rc_cache.cache_repo.arguments.host = localhost
|
||||
#rc_cache.cache_repo.arguments.port = 6379
|
||||
#rc_cache.cache_repo.arguments.db = 1
|
||||
#rc_cache.cache_repo.arguments.distributed_lock = true
|
||||
|
||||
|
||||
####################################
|
||||
### BEAKER CACHE ####
|
||||
####################################
|
||||
# default cache dir for templates. Putting this into a ramdisk
|
||||
## can boost performance, eg. %(here)s/data_ramdisk
|
||||
cache_dir = %(here)s/data
|
||||
|
||||
## locking and default file storage for Beaker. Putting this into a ramdisk
|
||||
## can boost performance, eg. %(here)s/data_ramdisk/cache/beaker_data
|
||||
beaker.cache.data_dir = %(here)s/data/cache/beaker_data
|
||||
beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
|
||||
|
||||
beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
|
||||
beaker.cache.regions = long_term, sql_cache_short
|
||||
|
||||
beaker.cache.long_term.type = memory
|
||||
beaker.cache.long_term.expire = 36000
|
||||
beaker.cache.long_term.expire = 172800
|
||||
beaker.cache.long_term.key_length = 256
|
||||
|
||||
beaker.cache.sql_cache_short.type = memory
|
||||
beaker.cache.sql_cache_short.expire = 10
|
||||
beaker.cache.sql_cache_short.key_length = 256
|
||||
|
||||
beaker.cache.repo_cache_long.type = memorylru_base
|
||||
beaker.cache.repo_cache_long.max_items = 4096
|
||||
beaker.cache.repo_cache_long.expire = 2592000
|
||||
|
||||
## default is memorylru_base cache, configure only if required
|
||||
## using multi-node or multi-worker setup
|
||||
#beaker.cache.repo_cache_long.type = ext:memcached
|
||||
#beaker.cache.repo_cache_long.url = localhost:11211
|
||||
#beaker.cache.repo_cache_long.expire = 1209600
|
||||
#beaker.cache.repo_cache_long.key_length = 256
|
||||
|
||||
####################################
|
||||
### BEAKER SESSION ####
|
||||
####################################
|
||||
|
||||
## .session.type is type of storage options for the session, current allowed
|
||||
## types are file, ext:memcached, ext:database, and memory (default).
|
||||
## types are file, ext:memcached, ext:redis, ext:database, and memory (default).
|
||||
beaker.session.type = file
|
||||
beaker.session.data_dir = %(here)s/data/sessions/data
|
||||
beaker.session.data_dir = %(here)s/data/sessions
|
||||
|
||||
## db based session, fast, and allows easy management over logged in users
|
||||
#beaker.session.type = ext:database
|
||||
|
|
@ -466,6 +491,8 @@ set debug = false
|
|||
#sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
|
||||
#sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
|
||||
#sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode
|
||||
#sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
|
||||
|
||||
sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
|
||||
|
||||
# see sqlalchemy docs for other advanced settings
|
||||
|
|
@ -485,6 +512,9 @@ sqlalchemy.db1.convert_unicode = true
|
|||
## which defaults to five.
|
||||
#sqlalchemy.db1.max_overflow = 10
|
||||
|
||||
## Connection check ping, used to detect broken database connections
|
||||
## could be enabled to better handle cases if MySQL has gone away errors
|
||||
#sqlalchemy.db1.ping_connection = true
|
||||
|
||||
##################
|
||||
### VCS CONFIG ###
|
||||
|
|
@ -593,7 +623,7 @@ custom.conf = 1
|
|||
### LOGGING CONFIGURATION ####
|
||||
################################
|
||||
[loggers]
|
||||
keys = root, sqlalchemy, beaker, rhodecode, ssh_wrapper, celery
|
||||
keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
|
||||
|
||||
[handlers]
|
||||
keys = console, console_sql
|
||||
|
|
@ -649,6 +679,9 @@ level = INFO
|
|||
formatter = generic
|
||||
|
||||
[handler_console_sql]
|
||||
# "level = DEBUG" logs SQL queries and results.
|
||||
# "level = INFO" logs SQL queries.
|
||||
# "level = WARN" logs neither. (Recommended for production systems.)
|
||||
class = StreamHandler
|
||||
args = (sys.stderr, )
|
||||
level = WARN
|
||||
|
|
|
|||
|
|
@ -1199,13 +1199,6 @@ class UsersView(UserAppView):
|
|||
|
||||
return perm_user.permissions
|
||||
|
||||
def _get_user_cache_keys(self, cache_namespace_uid, keys):
|
||||
user_keys = []
|
||||
for k in sorted(keys):
|
||||
if k.startswith(cache_namespace_uid):
|
||||
user_keys.append(k)
|
||||
return user_keys
|
||||
|
||||
@LoginRequired()
|
||||
@HasPermissionAllDecorator('hg.admin')
|
||||
@view_config(
|
||||
|
|
@ -1222,8 +1215,7 @@ class UsersView(UserAppView):
|
|||
cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
|
||||
c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
|
||||
c.backend = c.region.backend
|
||||
c.user_keys = self._get_user_cache_keys(
|
||||
cache_namespace_uid, c.region.backend.list_keys())
|
||||
c.user_keys = sorted(c.region.backend.list_keys(prefix=cache_namespace_uid))
|
||||
|
||||
return self._get_template_context(c)
|
||||
|
||||
|
|
@ -1241,14 +1233,9 @@ class UsersView(UserAppView):
|
|||
c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
|
||||
|
||||
cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
|
||||
c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
|
||||
del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid)
|
||||
|
||||
c.user_keys = self._get_user_cache_keys(
|
||||
cache_namespace_uid, c.region.backend.list_keys())
|
||||
for k in c.user_keys:
|
||||
c.region.delete(k)
|
||||
|
||||
h.flash(_("Deleted {} cache keys").format(len(c.user_keys)), category='success')
|
||||
h.flash(_("Deleted {} cache keys").format(del_keys), category='success')
|
||||
|
||||
return HTTPFound(h.route_path(
|
||||
'edit_user_caches', user_id=c.user.user_id))
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from pyramid.view import view_config
|
|||
from rhodecode.apps._base import RepoAppView
|
||||
from rhodecode.lib.auth import (
|
||||
LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
|
||||
from rhodecode.lib import helpers as h
|
||||
from rhodecode.lib import helpers as h, rc_cache
|
||||
from rhodecode.lib import system_info
|
||||
from rhodecode.model.meta import Session
|
||||
from rhodecode.model.scm import ScmModel
|
||||
|
|
@ -54,6 +54,12 @@ class RepoCachesView(RepoAppView):
|
|||
if os.path.isdir(cached_diffs_dir):
|
||||
c.cached_diff_size = system_info.get_storage_size(cached_diffs_dir)
|
||||
c.shadow_repos = c.rhodecode_db_repo.shadow_repos()
|
||||
|
||||
cache_namespace_uid = 'cache_repo.{}'.format(self.db_repo.repo_id)
|
||||
c.region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
|
||||
c.backend = c.region.backend
|
||||
c.repo_keys = sorted(c.region.backend.list_keys(prefix=cache_namespace_uid))
|
||||
|
||||
return self._get_template_context(c)
|
||||
|
||||
@LoginRequired()
|
||||
|
|
@ -68,7 +74,9 @@ class RepoCachesView(RepoAppView):
|
|||
|
||||
try:
|
||||
ScmModel().mark_for_invalidation(self.db_repo_name, delete=True)
|
||||
|
||||
Session().commit()
|
||||
|
||||
h.flash(_('Cache invalidation successful'),
|
||||
category='success')
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ from pyramid.response import Response
|
|||
from rhodecode.apps._base import RepoAppView
|
||||
|
||||
from rhodecode.controllers.utils import parse_path_ref
|
||||
from rhodecode.lib import diffs, helpers as h, caches
|
||||
from rhodecode.lib import diffs, helpers as h, caches, rc_cache
|
||||
from rhodecode.lib import audit_logger
|
||||
from rhodecode.lib.exceptions import NonRelativePathError
|
||||
from rhodecode.lib.codeblocks import (
|
||||
|
|
@ -187,32 +187,25 @@ class RepoFilesView(RepoAppView):
|
|||
# check if commit is a branch name or branch hash
|
||||
return commit_id in valid_heads
|
||||
|
||||
def _get_tree_cache_manager(self, namespace_type):
|
||||
_namespace = caches.get_repo_namespace_key(
|
||||
namespace_type, self.db_repo_name)
|
||||
return caches.get_cache_manager('repo_cache_long', _namespace)
|
||||
|
||||
def _get_tree_at_commit(
|
||||
self, c, commit_id, f_path, full_load=False, force=False):
|
||||
def _cached_tree():
|
||||
log.debug('Generating cached file tree for %s, %s, %s',
|
||||
self.db_repo_name, commit_id, f_path)
|
||||
self, c, commit_id, f_path, full_load=False):
|
||||
|
||||
repo_id = self.db_repo.repo_id
|
||||
|
||||
cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
|
||||
region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
|
||||
|
||||
@region.cache_on_arguments(namespace=cache_namespace_uid)
|
||||
def compute_file_tree(repo_id, commit_id, f_path, full_load):
|
||||
log.debug('Generating cached file tree for repo_id: %s, %s, %s',
|
||||
repo_id, commit_id, f_path)
|
||||
|
||||
c.full_load = full_load
|
||||
return render(
|
||||
'rhodecode:templates/files/files_browser_tree.mako',
|
||||
self._get_template_context(c), self.request)
|
||||
|
||||
cache_manager = self._get_tree_cache_manager(caches.FILE_TREE)
|
||||
|
||||
cache_key = caches.compute_key_from_params(
|
||||
self.db_repo_name, commit_id, f_path)
|
||||
|
||||
if force:
|
||||
# we want to force recompute of caches
|
||||
cache_manager.remove_value(cache_key)
|
||||
|
||||
return cache_manager.get(cache_key, createfunc=_cached_tree)
|
||||
return compute_file_tree(self.db_repo.repo_id, commit_id, f_path, full_load)
|
||||
|
||||
def _get_archive_spec(self, fname):
|
||||
log.debug('Detecting archive spec for: `%s`', fname)
|
||||
|
|
@ -664,12 +657,8 @@ class RepoFilesView(RepoAppView):
|
|||
c.file = dir_node
|
||||
c.commit = commit
|
||||
|
||||
# using force=True here, make a little trick. We flush the cache and
|
||||
# compute it using the same key as without previous full_load, so now
|
||||
# the fully loaded tree is now returned instead of partial,
|
||||
# and we store this in caches
|
||||
html = self._get_tree_at_commit(
|
||||
c, commit.raw_id, dir_node.path, full_load=True, force=True)
|
||||
c, commit.raw_id, dir_node.path, full_load=True)
|
||||
|
||||
return Response(html)
|
||||
|
||||
|
|
@ -784,10 +773,15 @@ class RepoFilesView(RepoAppView):
|
|||
|
||||
return response
|
||||
|
||||
def _get_nodelist_at_commit(self, repo_name, commit_id, f_path):
|
||||
def _cached_nodes():
|
||||
log.debug('Generating cached nodelist for %s, %s, %s',
|
||||
repo_name, commit_id, f_path)
|
||||
def _get_nodelist_at_commit(self, repo_name, repo_id, commit_id, f_path):
|
||||
|
||||
cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
|
||||
region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
|
||||
|
||||
@region.cache_on_arguments(namespace=cache_namespace_uid)
|
||||
def compute_file_search(repo_id, commit_id, f_path):
|
||||
log.debug('Generating cached nodelist for repo_id:%s, %s, %s',
|
||||
repo_id, commit_id, f_path)
|
||||
try:
|
||||
_d, _f = ScmModel().get_nodes(
|
||||
repo_name, commit_id, f_path, flat=False)
|
||||
|
|
@ -799,12 +793,7 @@ class RepoFilesView(RepoAppView):
|
|||
commit_id='tip', f_path='/'))
|
||||
return _d + _f
|
||||
|
||||
cache_manager = self._get_tree_cache_manager(
|
||||
caches.FILE_SEARCH_TREE_META)
|
||||
|
||||
cache_key = caches.compute_key_from_params(
|
||||
repo_name, commit_id, f_path)
|
||||
return cache_manager.get(cache_key, createfunc=_cached_nodes)
|
||||
return compute_file_search(self.db_repo.repo_id, commit_id, f_path)
|
||||
|
||||
@LoginRequired()
|
||||
@HasRepoPermissionAnyDecorator(
|
||||
|
|
@ -819,7 +808,7 @@ class RepoFilesView(RepoAppView):
|
|||
commit = self._get_commit_or_redirect(commit_id)
|
||||
|
||||
metadata = self._get_nodelist_at_commit(
|
||||
self.db_repo_name, commit.raw_id, f_path)
|
||||
self.db_repo_name, self.db_repo.repo_id, commit.raw_id, f_path)
|
||||
return {'nodes': metadata}
|
||||
|
||||
def _create_references(
|
||||
|
|
|
|||
|
|
@ -27,15 +27,14 @@ from beaker.cache import cache_region
|
|||
from rhodecode.controllers import utils
|
||||
from rhodecode.apps._base import RepoAppView
|
||||
from rhodecode.config.conf import (LANGUAGES_EXTENSIONS_MAP)
|
||||
from rhodecode.lib import caches, helpers as h
|
||||
from rhodecode.lib.helpers import RepoPage
|
||||
from rhodecode.lib import helpers as h, rc_cache
|
||||
from rhodecode.lib.utils2 import safe_str, safe_int
|
||||
from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
|
||||
from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
|
||||
from rhodecode.lib.ext_json import json
|
||||
from rhodecode.lib.vcs.backends.base import EmptyCommit
|
||||
from rhodecode.lib.vcs.exceptions import CommitError, EmptyRepositoryError, \
|
||||
CommitDoesNotExistError
|
||||
from rhodecode.lib.vcs.exceptions import (
|
||||
CommitError, EmptyRepositoryError, CommitDoesNotExistError)
|
||||
from rhodecode.model.db import Statistics, CacheKey, User
|
||||
from rhodecode.model.meta import Session
|
||||
from rhodecode.model.repo import ReadmeFinder
|
||||
|
|
@ -134,7 +133,7 @@ class RepoSummaryView(RepoAppView):
|
|||
except EmptyRepositoryError:
|
||||
collection = self.rhodecode_vcs_repo
|
||||
|
||||
c.repo_commits = RepoPage(
|
||||
c.repo_commits = h.RepoPage(
|
||||
collection, page=p, items_per_page=size, url=url_generator)
|
||||
page_ids = [x.raw_id for x in c.repo_commits]
|
||||
c.comments = self.db_repo.get_comments(page_ids)
|
||||
|
|
@ -247,16 +246,14 @@ class RepoSummaryView(RepoAppView):
|
|||
renderer='json_ext')
|
||||
def repo_stats(self):
|
||||
commit_id = self.get_request_commit_id()
|
||||
|
||||
_namespace = caches.get_repo_namespace_key(
|
||||
caches.SUMMARY_STATS, self.db_repo_name)
|
||||
show_stats = bool(self.db_repo.enable_statistics)
|
||||
cache_manager = caches.get_cache_manager(
|
||||
'repo_cache_long', _namespace)
|
||||
_cache_key = caches.compute_key_from_params(
|
||||
self.db_repo_name, commit_id, show_stats)
|
||||
repo_id = self.db_repo.repo_id
|
||||
|
||||
def compute_stats():
|
||||
cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
|
||||
region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
|
||||
|
||||
@region.cache_on_arguments(namespace=cache_namespace_uid)
|
||||
def compute_stats(repo_id, commit_id, show_stats):
|
||||
code_stats = {}
|
||||
size = 0
|
||||
try:
|
||||
|
|
@ -279,7 +276,7 @@ class RepoSummaryView(RepoAppView):
|
|||
return {'size': h.format_byte_size_binary(size),
|
||||
'code_stats': code_stats}
|
||||
|
||||
stats = cache_manager.get(_cache_key, createfunc=compute_stats)
|
||||
stats = compute_stats(self.db_repo.repo_id, commit_id, show_stats)
|
||||
return stats
|
||||
|
||||
@LoginRequired()
|
||||
|
|
|
|||
|
|
@ -435,6 +435,13 @@ def _sanitize_cache_settings(settings):
|
|||
_string_setting(settings, 'rc_cache.cache_perms.arguments.filename',
|
||||
os.path.join(tempfile.gettempdir(), 'rc_cache_1'))
|
||||
|
||||
_string_setting(settings, 'rc_cache.cache_repo.backend',
|
||||
'dogpile.cache.rc.file_namespace')
|
||||
_int_setting(settings, 'rc_cache.cache_repo.expiration_time',
|
||||
60)
|
||||
_string_setting(settings, 'rc_cache.cache_repo.arguments.filename',
|
||||
os.path.join(tempfile.gettempdir(), 'rc_cache_2'))
|
||||
|
||||
|
||||
def _int_setting(settings, name, default):
|
||||
settings[name] = int(settings.get(name, default))
|
||||
|
|
|
|||
|
|
@ -31,13 +31,6 @@ from rhodecode.model.db import Session, CacheKey
|
|||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
FILE_TREE = 'cache_file_tree'
|
||||
FILE_TREE_META = 'cache_file_tree_metadata'
|
||||
FILE_SEARCH_TREE_META = 'cache_file_search_metadata'
|
||||
SUMMARY_STATS = 'cache_summary_stats'
|
||||
|
||||
# This list of caches gets purged when invalidation happens
|
||||
USED_REPO_CACHES = (FILE_TREE, FILE_SEARCH_TREE_META)
|
||||
|
||||
DEFAULT_CACHE_MANAGER_CONFIG = {
|
||||
'type': 'memorylru_base',
|
||||
|
|
@ -129,14 +122,6 @@ def clear_cache_manager(cache_manager):
|
|||
cache_manager.clear()
|
||||
|
||||
|
||||
def clear_repo_caches(repo_name):
|
||||
# invalidate cache manager for this repo
|
||||
for prefix in USED_REPO_CACHES:
|
||||
namespace = get_repo_namespace_key(prefix, repo_name)
|
||||
cache_manager = get_cache_manager('repo_cache_long', namespace)
|
||||
clear_cache_manager(cache_manager)
|
||||
|
||||
|
||||
def compute_key_from_params(*args):
|
||||
"""
|
||||
Helper to compute key from given params to be used in cache manager
|
||||
|
|
@ -148,60 +133,6 @@ def get_repo_namespace_key(prefix, repo_name):
|
|||
return '{0}_{1}'.format(prefix, compute_key_from_params(repo_name))
|
||||
|
||||
|
||||
def conditional_cache(region, cache_namespace, condition, func):
|
||||
"""
|
||||
Conditional caching function use like::
|
||||
def _c(arg):
|
||||
# heavy computation function
|
||||
return data
|
||||
|
||||
# depending on the condition the compute is wrapped in cache or not
|
||||
compute = conditional_cache('short_term', 'cache_namespace_id',
|
||||
condition=True, func=func)
|
||||
return compute(arg)
|
||||
|
||||
:param region: name of cache region
|
||||
:param cache_namespace: cache namespace
|
||||
:param condition: condition for cache to be triggered, and
|
||||
return data cached
|
||||
:param func: wrapped heavy function to compute
|
||||
|
||||
"""
|
||||
wrapped = func
|
||||
if condition:
|
||||
log.debug('conditional_cache: True, wrapping call of '
|
||||
'func: %s into %s region cache', region, func)
|
||||
|
||||
def _cache_wrap(region_name, cache_namespace):
|
||||
"""Return a caching wrapper"""
|
||||
|
||||
def decorate(func):
|
||||
@functools.wraps(func)
|
||||
def cached(*args, **kwargs):
|
||||
if kwargs:
|
||||
raise AttributeError(
|
||||
'Usage of kwargs is not allowed. '
|
||||
'Use only positional arguments in wrapped function')
|
||||
manager = get_cache_manager(region_name, cache_namespace)
|
||||
cache_key = compute_key_from_params(*args)
|
||||
|
||||
def go():
|
||||
return func(*args, **kwargs)
|
||||
|
||||
# save org function name
|
||||
go.__name__ = '_cached_%s' % (func.__name__,)
|
||||
|
||||
return manager.get(cache_key, createfunc=go)
|
||||
return cached
|
||||
|
||||
return decorate
|
||||
|
||||
cached_region = _cache_wrap(region, cache_namespace)
|
||||
wrapped = cached_region(func)
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
class ActiveRegionCache(object):
|
||||
def __init__(self, context):
|
||||
self.context = context
|
||||
|
|
|
|||
|
|
@ -35,7 +35,9 @@ register_backend(
|
|||
|
||||
|
||||
from . import region_meta
|
||||
from .utils import get_default_cache_settings, key_generator, get_or_create_region
|
||||
from .utils import (
|
||||
get_default_cache_settings, key_generator, get_or_create_region,
|
||||
clear_cache_namespace)
|
||||
|
||||
|
||||
def configure_dogpile_cache(settings):
|
||||
|
|
|
|||
|
|
@ -49,9 +49,18 @@ class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
|
|||
def __init__(self, arguments):
|
||||
super(FileNamespaceBackend, self).__init__(arguments)
|
||||
|
||||
def list_keys(self):
|
||||
def list_keys(self, prefix=''):
|
||||
def cond(v):
|
||||
if not prefix:
|
||||
return True
|
||||
|
||||
if v.startswith(prefix):
|
||||
return True
|
||||
return False
|
||||
|
||||
with self._dbm_file(True) as dbm:
|
||||
return dbm.keys()
|
||||
|
||||
return filter(cond, dbm.keys())
|
||||
|
||||
def get_store(self):
|
||||
return self.filename
|
||||
|
|
@ -81,8 +90,10 @@ class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
|
|||
|
||||
|
||||
class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
|
||||
def list_keys(self):
|
||||
return self.client.keys()
|
||||
def list_keys(self, prefix=''):
|
||||
if prefix:
|
||||
prefix = prefix + '*'
|
||||
return self.client.keys(prefix)
|
||||
|
||||
def get_store(self):
|
||||
return self.client.connection_pool
|
||||
|
|
|
|||
|
|
@ -97,3 +97,11 @@ def get_or_create_region(region_name, region_namespace=None):
|
|||
region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
|
||||
|
||||
return region_obj
|
||||
|
||||
|
||||
def clear_cache_namespace(cache_region, cache_namespace_uid):
|
||||
region = get_or_create_region(cache_region, cache_namespace_uid)
|
||||
cache_keys = region.backend.list_keys(prefix=cache_namespace_uid)
|
||||
for k in cache_keys:
|
||||
region.delete(k)
|
||||
return len(cache_keys)
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ from rhodecode.lib.vcs import get_backend
|
|||
from rhodecode.lib.vcs.exceptions import RepositoryError, NodeNotChangedError
|
||||
from rhodecode.lib.vcs.nodes import FileNode
|
||||
from rhodecode.lib.vcs.backends.base import EmptyCommit
|
||||
from rhodecode.lib import helpers as h
|
||||
from rhodecode.lib import helpers as h, rc_cache
|
||||
from rhodecode.lib.auth import (
|
||||
HasRepoPermissionAny, HasRepoGroupPermissionAny,
|
||||
HasUserGroupPermissionAny)
|
||||
|
|
@ -267,16 +267,19 @@ class ScmModel(BaseModel):
|
|||
:param repo_name: the repo_name for which caches should be marked
|
||||
invalid, or deleted
|
||||
:param delete: delete the entry keys instead of setting bool
|
||||
flag on them
|
||||
flag on them, and also purge caches used by the dogpile
|
||||
"""
|
||||
CacheKey.set_invalidate(repo_name, delete=delete)
|
||||
repo = Repository.get_by_repo_name(repo_name)
|
||||
|
||||
if repo:
|
||||
repo_id = repo.repo_id
|
||||
config = repo._config
|
||||
config.set('extensions', 'largefiles', '')
|
||||
repo.update_commit_cache(config=config, cs_cache=None)
|
||||
caches.clear_repo_caches(repo_name)
|
||||
if delete:
|
||||
cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
|
||||
rc_cache.clear_cache_namespace('cache_repo', cache_namespace_uid)
|
||||
|
||||
def toggle_following_repo(self, follow_repo_id, user_id):
|
||||
|
||||
|
|
|
|||
|
|
@ -52,6 +52,41 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<h3 class="panel-title">
|
||||
${_('Cache keys')}
|
||||
</h3>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<p>
|
||||
Cache keys used for storing cached values of repository stats,
|
||||
file tree history and file tree search.
|
||||
Invalidating the cache will remove those entries.
|
||||
</p>
|
||||
<pre>
|
||||
region: ${c.region.name}
|
||||
backend: ${c.region.actual_backend.__class__}
|
||||
store: ${c.region.actual_backend.get_store()}
|
||||
|
||||
|
||||
% if c.repo_keys:
|
||||
${len(c.repo_keys)} <a href="#showKeys" onclick="$('#show-keys').toggle()">${_('Show all')}</a>
|
||||
<span id="show-keys" style="display: none">
|
||||
% for k in c.repo_keys:
|
||||
- ${k}
|
||||
% endfor
|
||||
</span>
|
||||
% else:
|
||||
NO KEYS FOUND
|
||||
% endif
|
||||
|
||||
</pre>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<h3 class="panel-title">${_('Shadow Repositories')}</h3>
|
||||
|
|
|
|||
|
|
@ -5,16 +5,28 @@
|
|||
<h3 class="panel-title">${_('Caches')}</h3>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<pre>
|
||||
<p>
|
||||
Cache keys used for storing cached values of user permissions and authentication plugin cache.
|
||||
Invalidating the cache will remove those entries.
|
||||
</p>
|
||||
|
||||
<pre>
|
||||
region: ${c.region.name}
|
||||
backend: ${c.region.actual_backend.__class__}
|
||||
store: ${c.region.actual_backend.get_store()}
|
||||
|
||||
% for k in c.user_keys:
|
||||
% if c.user_keys:
|
||||
${len(c.user_keys)} <a href="#showKeys" onclick="$('#show-keys').toggle()">${_('Show all')}</a>
|
||||
<span id="show-keys" style="display: none">
|
||||
% for k in c.user_keys:
|
||||
- ${k}
|
||||
% endfor
|
||||
</pre>
|
||||
|
||||
% endfor
|
||||
</span>
|
||||
% else:
|
||||
NO KEYS FOUND
|
||||
% endif
|
||||
</pre>
|
||||
<p></p>
|
||||
${h.secure_form(h.route_path('edit_user_caches_update', user_id=c.user.user_id), request=request)}
|
||||
<div class="form">
|
||||
<div class="fields">
|
||||
|
|
|
|||
|
|
@ -292,7 +292,7 @@ cache_dir = %(here)s/data
|
|||
beaker.cache.data_dir = %(here)s/rc/data/cache/beaker_data
|
||||
beaker.cache.lock_dir = %(here)s/rc/data/cache/beaker_lock
|
||||
|
||||
beaker.cache.regions = long_term, sql_cache_short, repo_cache_long
|
||||
beaker.cache.regions = long_term, sql_cache_short
|
||||
|
||||
beaker.cache.long_term.type = memory
|
||||
beaker.cache.long_term.expire = 36000
|
||||
|
|
@ -302,18 +302,6 @@ beaker.cache.sql_cache_short.type = memory
|
|||
beaker.cache.sql_cache_short.expire = 1
|
||||
beaker.cache.sql_cache_short.key_length = 256
|
||||
|
||||
beaker.cache.repo_cache_long.type = memorylru_base
|
||||
beaker.cache.repo_cache_long.max_items = 4096
|
||||
beaker.cache.repo_cache_long.expire = 2592000
|
||||
|
||||
## default is memorylru_base cache, configure only if required
|
||||
## using multi-node or multi-worker setup
|
||||
#beaker.cache.repo_cache_long.type = ext:memcached
|
||||
#beaker.cache.repo_cache_long.url = localhost:11211
|
||||
#beaker.cache.repo_cache_long.expire = 1209600
|
||||
#beaker.cache.repo_cache_long.key_length = 256
|
||||
|
||||
|
||||
#####################################
|
||||
### DOGPILE CACHE ####
|
||||
#####################################
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue