cache-keys: register and self cleanup cache keys used for invalidation to prevent leaking lot of them into DB on worker recycle

rhodecode/lib/rc_cache/cache_key_meta.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2015-2019 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import atexit
+import logging
+
+log = logging.getLogger(__name__)
+
+cache_keys_by_pid = []
+
+
+@atexit.register
+def free_cache_keys():
+    from rhodecode.model.db import Session, CacheKey
+    log.info('Clearing %s cache keys', len(cache_keys_by_pid))
+    for cache_key in cache_keys_by_pid:
+        CacheKey.query().filter(CacheKey.cache_key == cache_key).delete()
+    Session().commit()

rhodecode/lib/rc_cache/utils.py

@@ -32,7 +32,8 @@ from rhodecode.lib.utils import safe_str, sha1
 from rhodecode.lib.utils2 import safe_unicode, str2bool
 from rhodecode.model.db import Session, CacheKey, IntegrityError
 
-from . import region_meta
+from rhodecode.lib.rc_cache import cache_key_meta
+from rhodecode.lib.rc_cache import region_meta
 
 log = logging.getLogger(__name__)
 
@@ -304,6 +305,8 @@ class InvalidationContext(object):
                 cache_state_uid = first_cache_obj.cache_state_uid
             cache_obj = CacheKey(self.cache_key, cache_args=new_cache_args,
                                  cache_state_uid=cache_state_uid)
+            cache_key_meta.cache_keys_by_pid.append(self.cache_key)
+
         return cache_obj
 
     def __enter__(self):