fix(permission-flush): use delete method for permission cache invalidation as it's multi-process safe.

Fixes RCCE-23

Makefile

@@ -145,7 +145,7 @@ dev-srv:
 .PHONY: dev-srv-g
 ## run gunicorn multi process workers
 dev-srv-g:
-	gunicorn --workers=2 --paste .dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py
+	gunicorn --paste .dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py --timeout=120
 
 
 # Default command on calling make

rhodecode/apps/_base/subscribers.py

@@ -44,7 +44,7 @@ def trigger_user_permission_flush(event):
     for user_id in affected_user_ids:
         for cache_namespace_uid_tmpl in cache_namespaces:
             cache_namespace_uid = cache_namespace_uid_tmpl.format(user_id)
-            del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid, method=rc_cache.CLEAR_INVALIDATE)
+            del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid, method=rc_cache.CLEAR_DELETE)
             log.debug('Invalidated %s cache keys for user_id: %s and namespace %s',
                       del_keys, user_id, cache_namespace_uid)
 

rhodecode/apps/admin/views/users.py

@@ -1313,7 +1313,7 @@ class UsersView(UserAppView):
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{self.db_user.user_id}'
-        del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid)
+        del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid, method=rc_cache.CLEAR_DELETE)
 
         h.flash(_("Deleted {} cache keys").format(del_keys), category='success')
 

rhodecode/lib/rc_cache/backends.py

@@ -74,10 +74,20 @@ class LRUMemoryBackend(memory_backend.MemoryBackend):
             # we don't care if key isn't there at deletion
             pass
 
+    def list_keys(self, prefix):
+        return list(self._cache.keys())
+
     def delete_multi(self, keys):
         for key in keys:
             self.delete(key)
 
+    def delete_multi_by_prefix(self, prefix):
+        cache_keys = self.list_keys(prefix=prefix)
+        num_affected_keys = len(cache_keys)
+        if num_affected_keys:
+            self.delete_multi(cache_keys)
+        return num_affected_keys
+
 
 class PickleSerializer:
     serializer: None | Serializer = staticmethod(  # type: ignore
@@ -178,6 +188,13 @@ class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
                 log.error('Failed to fetch DBM keys from DB: %s', self.get_store())
                 raise
 
+    def delete_multi_by_prefix(self, prefix):
+        cache_keys = self.list_keys(prefix=prefix)
+        num_affected_keys = len(cache_keys)
+        if num_affected_keys:
+            self.delete_multi(cache_keys)
+        return num_affected_keys
+
     def get_store(self):
         return self.filename
 
@@ -227,6 +244,25 @@ class BaseRedisBackend(redis_backend.RedisBackend):
         prefix = self._get_keys_pattern(prefix)
         return self.reader_client.keys(prefix)
 
+    def delete_multi_by_prefix(self, prefix, use_lua=False):
+        if use_lua:
+            # high efficient LUA script to delete ALL keys by prefix...
+            lua = """local keys = redis.call('keys', ARGV[1])
+                     for i=1,#keys,5000 do
+                     redis.call('del', unpack(keys, i, math.min(i+(5000-1), #keys)))
+                     end
+                     return #keys"""
+            num_affected_keys = self.writer_client.eval(
+                lua,
+                0,
+                f"{prefix}*")
+        else:
+            cache_keys = self.list_keys(prefix=prefix)
+            num_affected_keys = len(cache_keys)
+            if num_affected_keys:
+                self.delete_multi(cache_keys)
+        return num_affected_keys
+
     def get_store(self):
         return self.reader_client.connection_pool
 

rhodecode/lib/rc_cache/utils.py

@@ -232,14 +232,15 @@ def get_or_create_region(region_name, region_namespace: str = None, use_async_ru
     return region_obj
 
 
-def clear_cache_namespace(cache_region: str | RhodeCodeCacheRegion, cache_namespace_uid: str, method: str):
+def clear_cache_namespace(cache_region: str | RhodeCodeCacheRegion, cache_namespace_uid: str, method: str) -> int:
     from . import CLEAR_DELETE, CLEAR_INVALIDATE
 
     if not isinstance(cache_region, RhodeCodeCacheRegion):
         cache_region = get_or_create_region(cache_region, cache_namespace_uid)
-    log.debug('clearing cache region: %s with method=%s', cache_region, method)
+    log.debug('clearing cache region: %s [prefix:%s] with method=%s',
+              cache_region, cache_namespace_uid, method)
 
-    num_affected_keys = None
+    num_affected_keys = 0
 
     if method == CLEAR_INVALIDATE:
         # NOTE: The CacheRegion.invalidate() method’s default mode of
@@ -248,10 +249,7 @@ def clear_cache_namespace(cache_region: str | RhodeCodeCacheRegion, cache_namesp
         cache_region.invalidate(hard=True)
 
     if method == CLEAR_DELETE:
-        cache_keys = cache_region.backend.list_keys(prefix=cache_namespace_uid)
-        num_affected_keys = len(cache_keys)
-        if num_affected_keys:
-            cache_region.delete_multi(cache_keys)
+        num_affected_keys = cache_region.backend.delete_multi_by_prefix(prefix=cache_namespace_uid)
 
     return num_affected_keys