engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(archive-cache): added option to define a configurable top-level bucket for all shards

Browse source
This commit is contained in:
RhodeCode Admin 2024-06-18 14:03:06 +02:00
parent f4323f22c5
commit eed5751983
3 changed files with 13 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

1
rhodecode/config/config_maker.py
View file

@ -204,6 +204,7 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
settings_maker.make_setting('archive_cache.objectstore.key', '')
settings_maker.make_setting('archive_cache.objectstore.secret', '')
settings_maker.make_setting('archive_cache.objectstore.bucket_root', 'rhodecode-archive-cache')
settings_maker.make_setting('archive_cache.objectstore.bucket_shards', 8, parser='int')
settings_maker.make_setting('archive_cache.objectstore.cache_size_gb', 10, parser='float')

3
rhodecode/lib/archive_cache/backends/fanout_cache.py
View file

@ -115,6 +115,7 @@ class FileSystemShard(BaseShard):
class FileSystemFanoutCache(BaseCache):
shard_name = 'shard_%03d'
def __init__(self, locking_url, **settings):
"""
@ -150,7 +151,7 @@ class FileSystemFanoutCache(BaseCache):
self._shards = tuple(
FileSystemShard(
index=num,
directory=os.path.join(directory, 'shard_%03d' % num),
directory=os.path.join(directory, self.shard_name % num),
**settings,
)
for num in range(self._count)

12
rhodecode/lib/archive_cache/backends/objectstore_cache.py
View file

@ -41,12 +41,15 @@ class S3Shard(BaseShard):
key = settings.pop('archive_cache.objectstore.key')
secret = settings.pop('archive_cache.objectstore.secret')
# TODO: Add it all over the place...
self._bucket_root = settings.pop('archive_cache.objectstore.bucket_root')
self.fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret)
@property
def bucket(self):
"""Cache bucket."""
return self._bucket
return os.path.join(self._bucket_root, self._bucket)
def _get_keyfile(self, archive_key) -> tuple[str, str]:
key_file = f'{archive_key}-{self.key_suffix}'
@ -56,6 +59,10 @@ class S3Shard(BaseShard):
return self.fs.open(path, 'wb')
def _write_file(self, full_path, iterator, mode):
if self._bucket_root:
if not self.fs.exists(self._bucket_root):
self.fs.mkdir(self._bucket_root)
# ensure bucket exists
destination = self.bucket
if not self.fs.exists(destination):
@ -106,6 +113,7 @@ class S3Shard(BaseShard):
class ObjectStoreCache(BaseCache):
shard_name = 'shard-bucket-%03d'
def __init__(self, locking_url, **settings):
"""
@ -134,7 +142,7 @@ class ObjectStoreCache(BaseCache):
self._shards = tuple(
S3Shard(
index=num,
bucket='rhodecode-archivecache-%03d' % num,
bucket=self.shard_name % num,
**settings,
)
for num in range(self._count)