archive-cache: make it more resilient to errors and not break if single metadata is broken

docs/release-notes/release-notes-5.5.0.rst

@@ -11,12 +11,13 @@ New Features
 
 - Added automatic PR merging on approved code.
 
+
 General
 ^^^^^^^
 
 - Added check to avoid deleting(closing) landing branches during PR merge
 - Several Library fixes and updates.
-
+- Archive cache cleanup is more resilient to errors
 
 Security
 ^^^^^^^^

rhodecode/lib/archive_cache/backends/base.py

@@ -308,7 +308,11 @@ class BaseCache:
 
         for shard, key_file in self.iter_keys():
             with shard.fs.open(os.path.join(shard.storage_medium, key_file), "rb") as f:
-                metadata = json.loads(f.read())
+                try:
+                    metadata = json.loads(f.read())
+                except Exception as e:
+                    log.exception("Failed to decode metadata from %s", key_file)
+                    continue
 
             key_file_path = os.path.join(shard.storage_medium, key_file)