caches: synced cache logic with vcsserver.

rhodecode/lib/rc_cache/__init__.py

@@ -33,12 +33,16 @@ register_backend(
     "dogpile.cache.rc.redis", "rhodecode.lib.rc_cache.backends",
     "RedisPickleBackend")
 
+register_backend(
+    "dogpile.cache.rc.redis_msgpack", "rhodecode.lib.rc_cache.backends",
+    "RedisMsgPackBackend")
+
 
 log = logging.getLogger(__name__)
 
 from . import region_meta
 from .utils import (
-    get_default_cache_settings, key_generator, get_or_create_region,
+    get_default_cache_settings, backend_key_generator, get_or_create_region,
     clear_cache_namespace, make_region, InvalidationContext,
     FreshRegionCache, ActiveRegionCache)
 
@@ -61,13 +65,12 @@ def configure_dogpile_cache(settings):
     for region_name in avail_regions:
         new_region = make_region(
             name=region_name,
-            function_key_generator=key_generator
+            function_key_generator=None
         )
 
         new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
-
-        log.debug('dogpile: registering a new region %s[%s]',
-                  region_name, new_region.__dict__)
+        new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
+        log.debug('dogpile: registering a new region %s[%s]', region_name, new_region.__dict__)
         region_meta.dogpile_cache_regions[region_name] = new_region
 
 

rhodecode/lib/rc_cache/backends.py

@@ -22,8 +22,10 @@ import time
 import errno
 import logging
 
+import msgpack
 import gevent
 
+from dogpile.cache.api import CachedValue
 from dogpile.cache.backends import memory as memory_backend
 from dogpile.cache.backends import file as file_backend
 from dogpile.cache.backends import redis as redis_backend
@@ -39,6 +41,7 @@ log = logging.getLogger(__name__)
 
 
 class LRUMemoryBackend(memory_backend.MemoryBackend):
+    key_prefix = 'lru_mem_backend'
     pickle_values = False
 
     def __init__(self, arguments):
@@ -63,7 +66,8 @@ class LRUMemoryBackend(memory_backend.MemoryBackend):
             self.delete(key)
 
 
-class Serializer(object):
+class PickleSerializer(object):
+
     def _dumps(self, value, safe=False):
         try:
             return compat.pickle.dumps(value)
@@ -83,6 +87,32 @@ class Serializer(object):
                 raise
 
 
+class MsgPackSerializer(object):
+
+    def _dumps(self, value, safe=False):
+        try:
+            return msgpack.packb(value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+    def _loads(self, value, safe=True):
+        """
+        pickle maintained the `CachedValue` wrapper of the tuple
+        msgpack does not, so it must be added back in.
+       """
+        try:
+            value = msgpack.unpackb(value, use_list=False)
+            return CachedValue(*value)
+        except Exception:
+            if safe:
+                return NO_VALUE
+            else:
+                raise
+
+
 import fcntl
 flock_org = fcntl.flock
 
@@ -123,13 +153,16 @@ class CustomLockFactory(FileLock):
         return fcntl
 
 
-class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
+class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
+    key_prefix = 'file_backend'
 
     def __init__(self, arguments):
         arguments['lock_factory'] = CustomLockFactory
         super(FileNamespaceBackend, self).__init__(arguments)
 
     def list_keys(self, prefix=''):
+        prefix = '{}:{}'.format(self.key_prefix, prefix)
+
         def cond(v):
             if not prefix:
                 return True
@@ -169,10 +202,9 @@ class FileNamespaceBackend(Serializer, file_backend.DBMBackend):
                 dbm[key] = self._dumps(value)
 
 
-class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
+class BaseRedisBackend(redis_backend.RedisBackend):
     def list_keys(self, prefix=''):
-        if prefix:
-            prefix = prefix + '*'
+        prefix = '{}:{}*'.format(self.key_prefix, prefix)
         return self.client.keys(prefix)
 
     def get_store(self):
@@ -184,6 +216,15 @@ class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
             return NO_VALUE
         return self._loads(value)
 
+    def get_multi(self, keys):
+        if not keys:
+            return []
+        values = self.client.mget(keys)
+        loads = self._loads
+        return [
+            loads(v) if v is not None else NO_VALUE
+            for v in values]
+
     def set(self, key, value):
         if self.redis_expiration_time:
             self.client.setex(key, self.redis_expiration_time,
@@ -192,8 +233,9 @@ class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
             self.client.set(key, self._dumps(value))
 
     def set_multi(self, mapping):
+        dumps = self._dumps
         mapping = dict(
-            (k, self._dumps(v))
+            (k, dumps(v))
             for k, v in mapping.items()
         )
 
@@ -213,3 +255,13 @@ class RedisPickleBackend(Serializer, redis_backend.RedisBackend):
             return self.client.lock(lock_key, self.lock_timeout, self.lock_sleep)
         else:
             return None
+
+
+class RedisPickleBackend(PickleSerializer, BaseRedisBackend):
+    key_prefix = 'redis_pickle_backend'
+    pass
+
+
+class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
+    key_prefix = 'redis_msgpack_backend'
+    pass

rhodecode/lib/rc_cache/utils.py

@@ -134,13 +134,23 @@ def compute_key_from_params(*args):
     return sha1("_".join(map(safe_str, args)))
 
 
-def key_generator(namespace, fn):
+def backend_key_generator(backend):
+    """
+    Special wrapper that also sends over the backend to the key generator
+    """
+    def wrapper(namespace, fn):
+        return key_generator(backend, namespace, fn)
+    return wrapper
+
+
+def key_generator(backend, namespace, fn):
     fname = fn.__name__
 
     def generate_key(*args):
-        namespace_pref = namespace or 'default'
+        backend_prefix = getattr(backend, 'key_prefix', None) or 'backend_prefix'
+        namespace_pref = namespace or 'default_namespace'
         arg_key = compute_key_from_params(*args)
-        final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
+        final_key = "{}:{}:{}_{}".format(backend_prefix, namespace_pref, fname, arg_key)
 
         return final_key
 
@@ -167,7 +177,8 @@ def get_or_create_region(region_name, region_namespace=None):
         if not os.path.isdir(cache_dir):
             os.makedirs(cache_dir)
         new_region = make_region(
-            name=region_uid_name, function_key_generator=key_generator
+            name=region_uid_name,
+            function_key_generator=backend_key_generator(region_obj.actual_backend)
         )
         namespace_filename = os.path.join(
             cache_dir, "{}.cache.dbm".format(region_namespace))
@@ -179,7 +190,7 @@ def get_or_create_region(region_name, region_namespace=None):
         )
 
         # create and save in region caches
-        log.debug('configuring new region: %s',region_uid_name)
+        log.debug('configuring new region: %s', region_uid_name)
         region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
 
     return region_obj