engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

caches: make sure the global cache namespace prefixes are used.

Browse source 
- fixes problems when porting from old RC to new python3 based and different serialization mech
This commit is contained in:
RhodeCode Admin 2023-07-26 16:15:52 +02:00
parent 3082b74880
commit 8ae0e8de44
13 changed files with 31 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

6
rhodecode/api/views/repo_api.py
View file

@ -624,10 +624,10 @@ def get_repo_fts_tree(request, apiuser, repoid, commit_id, root_path):
cache_seconds = rhodecode.ConfigGet().get_int('rc_cache.cache_repo.expiration_time')
cache_on = cache_seconds > 0
cache_namespace_uid = f'repo.{repo_id}'
cache_namespace_uid = f'repo.{rc_cache.FILE_TREE_CACHE_VER}.{repo_id}'
rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
def compute_fts_tree(cache_ver, repo_id, commit_id, root_path):
def compute_fts_tree(repo_id, commit_id, root_path):
return ScmModel().get_fts_data(repo_id, commit_id, root_path)
try:
@ -648,7 +648,7 @@ def get_repo_fts_tree(request, apiuser, repoid, commit_id, root_path):
'with caching: %s[TTL: %ss]' % (
repo_id, commit_id, cache_on, cache_seconds or 0))
tree_files = compute_fts_tree(rc_cache.FILE_TREE_CACHE_VER, repo_id, commit_id, root_path)
tree_files = compute_fts_tree(repo_id, commit_id, root_path)
return tree_files

8
rhodecode/apps/_base/subscribers.py
View file

@ -26,10 +26,10 @@ log = logging.getLogger(__name__)
# names of namespaces used for different permission related cached
# during flush operation we need to take care of all those
cache_namespaces = [
'cache_user_auth.{}',
'cache_user_repo_acl_ids.{}',
'cache_user_user_group_acl_ids.{}',
'cache_user_repo_group_acl_ids.{}'
f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{{}}',
f'cache_user_repo_acl_ids.{rc_cache.PERMISSIONS_CACHE_VER}.{{}}',
f'cache_user_user_group_acl_ids.{rc_cache.PERMISSIONS_CACHE_VER}.{{}}',
f'cache_user_repo_group_acl_ids.{rc_cache.PERMISSIONS_CACHE_VER}.{{}}'
]

4
rhodecode/apps/admin/views/users.py
View file

@ -1294,7 +1294,7 @@ class UsersView(UserAppView):
c.active = 'caches'
c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
cache_namespace_uid = f'cache_user_auth.{self.db_user.user_id}'
cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{self.db_user.user_id}'
c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
c.backend = c.region.backend
c.user_keys = sorted(c.region.backend.list_keys(prefix=cache_namespace_uid))
@ -1312,7 +1312,7 @@ class UsersView(UserAppView):
c.active = 'caches'
c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
cache_namespace_uid = f'cache_user_auth.{self.db_user.user_id}'
cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{self.db_user.user_id}'
del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid)
h.flash(_("Deleted {} cache keys").format(del_keys), category='success')

9
rhodecode/apps/repository/views/repo_files.py
View file

@ -310,13 +310,13 @@ class RepoFilesView(RepoAppView):
'with caching: %s[TTL: %ss]' % (
repo_id, commit_id, f_path, cache_on, cache_seconds or 0))
cache_namespace_uid = f'repo.{repo_id}'
cache_namespace_uid = f'repo.{rc_cache.FILE_TREE_CACHE_VER}.{repo_id}'
region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
@region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on)
def compute_file_tree(ver, _name_hash, _repo_id, _commit_id, _f_path, _full_load, _at_rev):
def compute_file_tree(_name_hash, _repo_id, _commit_id, _f_path, _full_load, _at_rev):
log.debug('Generating cached file tree at ver:%s for repo_id: %s, %s, %s',
ver, _repo_id, _commit_id, _f_path)
_repo_id, _commit_id, _f_path)
c.full_load = _full_load
return render(
@ -324,8 +324,7 @@ class RepoFilesView(RepoAppView):
self._get_template_context(c), self.request, _at_rev)
return compute_file_tree(
rc_cache.FILE_TREE_CACHE_VER, self.db_repo.repo_name_hash,
self.db_repo.repo_id, commit_id, f_path, full_load, at_rev)
self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path, full_load, at_rev)
def create_pure_path(self, *parts):
# Split paths and sanitize them, removing any ../ etc

2
rhodecode/authentication/base.py
View file

@ -750,7 +750,7 @@ def authenticate(username, password, environ=None, auth_type=None,
user_id = user.user_id if user else 'no-user'
# don't cache for empty users
plugin_cache_active = plugin_cache_active and user_id
cache_namespace_uid = f'cache_user_auth.{user_id}'
cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}'
region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
@region.conditional_cache_on_arguments(namespace=cache_namespace_uid,

2
rhodecode/authentication/registry.py
View file

@ -125,7 +125,7 @@ class AuthenticationPluginRegistry(object):
@classmethod
def get_cache_region(cls):
cache_namespace_uid = 'auth_plugins'
cache_namespace_uid = 'auth_plugins.v1'
region = rc_cache.get_or_create_region('cache_general', cache_namespace_uid)
return region, cache_namespace_uid

8
rhodecode/lib/auth.py
View file

@ -1255,7 +1255,7 @@ class AuthUser(object):
'Computing PERMISSION tree for user %s scope `%s` '
'with caching: %s[TTL: %ss]', user, scope, cache_on, cache_seconds or 0)
cache_namespace_uid = f'cache_user_auth.{user_id}'
cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}'
region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
@region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
@ -1358,7 +1358,7 @@ class AuthUser(object):
log.debug('Computing REPO ACL IDS user %s', self)
cache_namespace_uid = 'cache_user_repo_acl_ids.{}'.format(self.user_id)
cache_namespace_uid = f'cache_user_repo_acl_ids.{rc_cache.PERMISSIONS_CACHE_VER}.{self.user_id}'
region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
@region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache)
@ -1409,7 +1409,7 @@ class AuthUser(object):
log.debug('Computing REPO GROUP ACL IDS user %s', self)
cache_namespace_uid = 'cache_user_repo_group_acl_ids.{}'.format(self.user_id)
cache_namespace_uid = f'cache_user_repo_group_acl_ids.{rc_cache.PERMISSIONS_CACHE_VER}.{self.user_id}'
region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
@region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache)
@ -1458,7 +1458,7 @@ class AuthUser(object):
log.debug('Computing USER GROUP ACL IDS user %s', self)
cache_namespace_uid = 'cache_user_user_group_acl_ids.{}'.format(self.user_id)
cache_namespace_uid = f'cache_user_user_group_acl_ids.{rc_cache.PERMISSIONS_CACHE_VER}.{self.user_id}'
region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
@region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache)

2
rhodecode/lib/middleware/simplevcs.py
View file

@ -314,7 +314,7 @@ class SimpleVCS(object):
plugin_id, plugin_cache_active, cache_ttl)
user_id = user.user_id
cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}'
region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
@region.conditional_cache_on_arguments(namespace=cache_namespace_uid,

8
rhodecode/lib/rc_cache/__init__.py
View file

@ -56,9 +56,9 @@ register_backend(
log = logging.getLogger(__name__)
FILE_TREE_CACHE_VER = 'v4'
LICENSE_CACHE_VER = 'v2'
FILE_TREE_CACHE_VER = 'v5'
LICENSE_CACHE_VER = 'v3'
PERMISSIONS_CACHE_VER = 'v2'
CLEAR_DELETE = 'delete'
CLEAR_INVALIDATE = 'invalidate'
@ -111,7 +111,7 @@ def configure_dogpile_cache(settings):
if log.isEnabledFor(logging.DEBUG):
region_args = dict(backend=new_region.actual_backend,
region_invalidator=new_region.region_invalidator.__class__)
log.debug('dogpile: registering a new region `%s` %s', namespace_name, region_args)
log.debug('dogpile: registering a new region key=`%s` args=%s', namespace_name, region_args)
region_meta.dogpile_cache_regions[namespace_name] = new_region

7
rhodecode/lib/rc_cache/utils.py
View file

@ -28,7 +28,7 @@ from dogpile.cache import CacheRegion
import rhodecode
from rhodecode.lib.hash_utils import sha1
from rhodecode.lib.str_utils import safe_bytes
from rhodecode.lib.type_utils import str2bool
from rhodecode.lib.type_utils import str2bool # noqa :required by imports from .utils
from . import region_meta, cache_key_meta
@ -185,6 +185,7 @@ def get_or_create_region(region_name, region_namespace: str = None, use_async_ru
region_uid_name = f'{region_name}:{region_namespace}'
# Special case for ONLY the FileNamespaceBackend backend. We register one-file-per-region
if isinstance(region_obj.actual_backend, FileNamespaceBackend):
if not region_namespace:
raise ValueError(f'{FileNamespaceBackend} used requires to specify region_namespace param')
@ -200,7 +201,7 @@ def get_or_create_region(region_name, region_namespace: str = None, use_async_ru
namespace_cache_dir = cache_dir
# we default the namespace_cache_dir to our default cache dir.
# however if this backend is configured with filename= param, we prioritize that
# however, if this backend is configured with filename= param, we prioritize that
# so all caches within that particular region, even those namespaced end up in the same path
if region_obj.actual_backend.filename:
namespace_cache_dir = os.path.dirname(region_obj.actual_backend.filename)
@ -280,7 +281,7 @@ class InvalidationContext(object):
from rhodecode.lib import rc_cache
cache_namespace_uid = CacheKey.SOME_NAMESPACE.format(1)
region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
region = rc_cache.get_or_create_region('some_region', cache_namespace_uid)
@region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=True)
def heavy_compute(cache_name, param1, param2):

2
rhodecode/model/db.py
View file

@ -3675,7 +3675,7 @@ class CacheKey(Base, BaseModel):
CACHE_TYPE_FEED = 'FEED'
# namespaces used to register process/thread aware caches
REPO_INVALIDATION_NAMESPACE = 'repo_cache:{repo_id}'
REPO_INVALIDATION_NAMESPACE = 'repo_cache.v1:{repo_id}'
cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
cache_key = Column("cache_key", String(255), nullable=True, unique=None, default=None)

2
rhodecode/model/settings.py
View file

@ -210,7 +210,7 @@ class SettingsModel(BaseModel):
def get_cache_region(self):
repo = self._get_repo(self.repo) if self.repo else None
cache_key = f"repo.{repo.repo_id}" if repo else "repo.ALL"
cache_key = f"repo.v1.{repo.repo_id}" if repo else "repo.v1.ALL"
cache_namespace_uid = f'cache_settings.{cache_key}'
region = rc_cache.get_or_create_region('cache_general', cache_namespace_uid)
return region, cache_namespace_uid

3
rhodecode/tests/lib/test_caches.py
View file

@ -37,8 +37,7 @@ class TestCaches(object):
])
def test_cache_decorator_init(self, region_name):
namespace = region_name
cache_region = rc_cache.get_or_create_region(
region_name, region_namespace=namespace)
cache_region = rc_cache.get_or_create_region(region_name, region_namespace=namespace)
assert cache_region
@pytest.mark.parametrize('example_input', [