feat(artifacts): new artifact storage engines allowing an s3 based uploads

configs/development.ini

@@ -281,15 +281,56 @@ labs_settings_active = true
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
-; File store configuration. This is used to store and serve uploaded files
-file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store backend is always enabled
+#file_store.enabled = true
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
 ; Storage backend, available options are: local
-file_store.backend = local
+#file_store.backend = local
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
 ; path to store the uploaded binaries and artifacts
-file_store.storage_path = /var/opt/rhodecode_data/file_store
+#file_store.storage_path = /var/opt/rhodecode_data/file_store
 
+; Artifacts file-store, is used to store comment attachments and artifacts uploads.
+; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
+; filesystem_v1 is backwards compat with pre 5.1 storage changes
+; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
+; previous installations to keep the artifacts without a need of migration
+#file_store.backend.type = filesystem_v2
+
+; filesystem options...
+#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+
+; filesystem_v2 options...
+#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+#file_store.filesystem_v2.shards = 8
+
+; objectstore options...
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+#file_store.backend.type = objectstore
+#file_store.objectstore.url = http://s3-minio:9000
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
+#file_store.objectstore.bucket = rhodecode-file-store
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+#file_store.objectstore.bucket_shards = 8
+
+; key for s3 auth
+#file_store.objectstore.key = s3admin
+
+; secret for s3 auth
+#file_store.objectstore.secret = s3secret4
+
+;region for s3 storage
+#file_store.objectstore.region = eu-central-1
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1

configs/production.ini

@@ -249,15 +249,56 @@ labs_settings_active = true
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
-; File store configuration. This is used to store and serve uploaded files
-file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store backend is always enabled
+#file_store.enabled = true
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
 ; Storage backend, available options are: local
-file_store.backend = local
+#file_store.backend = local
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
 ; path to store the uploaded binaries and artifacts
-file_store.storage_path = /var/opt/rhodecode_data/file_store
+#file_store.storage_path = /var/opt/rhodecode_data/file_store
 
+; Artifacts file-store, is used to store comment attachments and artifacts uploads.
+; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
+; filesystem_v1 is backwards compat with pre 5.1 storage changes
+; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
+; previous installations to keep the artifacts without a need of migration
+#file_store.backend.type = filesystem_v2
+
+; filesystem options...
+#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+
+; filesystem_v2 options...
+#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+#file_store.filesystem_v2.shards = 8
+
+; objectstore options...
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+#file_store.backend.type = objectstore
+#file_store.objectstore.url = http://s3-minio:9000
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
+#file_store.objectstore.bucket = rhodecode-file-store
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+#file_store.objectstore.bucket_shards = 8
+
+; key for s3 auth
+#file_store.objectstore.key = s3admin
+
+; secret for s3 auth
+#file_store.objectstore.secret = s3secret4
+
+;region for s3 storage
+#file_store.objectstore.region = eu-central-1
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1

rhodecode/api/views/server_api.py

@@ -33,7 +33,7 @@ from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.model.db import UserIpMap
 from rhodecode.model.scm import ScmModel
-from rhodecode.apps.file_store import utils
+from rhodecode.apps.file_store import utils as store_utils
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
     FileOverSizeException
 

rhodecode/apps/admin/views/system_info.py

@@ -171,11 +171,17 @@ class AdminSystemInfoSettingsView(BaseAppView):
             (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
             ('', '', ''),  # spacer
 
-            (_('Archive cache storage type'), val('storage_archive')['type'], state('storage_archive')),
+            (_('Artifacts storage backend'), val('storage_artifacts')['type'], state('storage_artifacts')),
+            (_('Artifacts storage location'), val('storage_artifacts')['path'], state('storage_artifacts')),
+            (_('Artifacts info'), val('storage_artifacts')['text'], state('storage_artifacts')),
+            ('', '', ''),  # spacer
+
+            (_('Archive cache storage backend'), val('storage_archive')['type'], state('storage_archive')),
             (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
             (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
             ('', '', ''),  # spacer
 
+
             (_('Temp storage location'), val('storage_temp')['path'], state('storage_temp')),
             (_('Temp storage info'), val('storage_temp')['text'], state('storage_temp')),
             ('', '', ''),  # spacer

rhodecode/apps/file_store/__init__.py

@@ -16,7 +16,8 @@
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
-from rhodecode.apps.file_store import config_keys
+
+
 from rhodecode.config.settings_maker import SettingsMaker
 
 
@@ -24,18 +25,48 @@ def _sanitize_settings_and_apply_defaults(settings):
     """
     Set defaults, convert to python types and validate settings.
     """
+    from rhodecode.apps.file_store import config_keys
+
+    # translate "legacy" params into new config
+    settings.pop(config_keys.deprecated_enabled, True)
+    if config_keys.deprecated_backend in settings:
+        # if legacy backend key is detected we use "legacy" backward compat setting
+        settings.pop(config_keys.deprecated_backend)
+        settings[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+
+    if config_keys.deprecated_store_path in settings:
+        store_path = settings.pop(config_keys.deprecated_store_path)
+        settings[config_keys.legacy_filesystem_storage_path] = store_path
+
     settings_maker = SettingsMaker(settings)
 
-    settings_maker.make_setting(config_keys.enabled, True, parser='bool')
-    settings_maker.make_setting(config_keys.backend, 'local')
+    default_cache_dir = settings['cache_dir']
+    default_store_dir = os.path.join(default_cache_dir, 'artifacts_filestore')
 
-    default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store')
-    settings_maker.make_setting(config_keys.store_path, default_store)
+    # set default backend
+    settings_maker.make_setting(config_keys.backend_type, config_keys.backend_legacy_filesystem)
+
+    # legacy filesystem defaults
+    settings_maker.make_setting(config_keys.legacy_filesystem_storage_path, default_store_dir, default_when_empty=True, )
+
+    # filesystem defaults
+    settings_maker.make_setting(config_keys.filesystem_storage_path, default_store_dir, default_when_empty=True,)
+    settings_maker.make_setting(config_keys.filesystem_shards, 8, parser='int')
+
+    # objectstore defaults
+    settings_maker.make_setting(config_keys.objectstore_url, 'http://s3-minio:9000')
+    settings_maker.make_setting(config_keys.objectstore_bucket, 'rhodecode-artifacts-filestore')
+    settings_maker.make_setting(config_keys.objectstore_bucket_shards, 8, parser='int')
+
+    settings_maker.make_setting(config_keys.objectstore_region, '')
+    settings_maker.make_setting(config_keys.objectstore_key, '')
+    settings_maker.make_setting(config_keys.objectstore_secret, '')
 
     settings_maker.env_expand()
 
 
 def includeme(config):
+
     from rhodecode.apps.file_store.views import FileStoreView
 
     settings = config.registry.settings

rhodecode/apps/file_store/backends/base.py

@@ -0,0 +1,269 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import fsspec  # noqa
+import logging
+
+from rhodecode.lib.ext_json import json
+
+from rhodecode.apps.file_store.utils import sha256_safe, ShardFileReader, get_uid_filename
+from rhodecode.apps.file_store.extensions import resolve_extensions
+from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException  # noqa: F401
+
+log = logging.getLogger(__name__)
+
+
+class BaseShard:
+
+    metadata_suffix: str = '.metadata'
+    storage_type: str = ''
+    fs = None
+
+    @property
+    def storage_medium(self):
+        if not self.storage_type:
+            raise ValueError('No storage type set for this shard storage_type=""')
+        return getattr(self, self.storage_type)
+
+    def __contains__(self, key):
+        full_path = self.store_path(key)
+        return self.fs.exists(full_path)
+
+    def metadata_convert(self, uid_filename, metadata):
+        return metadata
+
+    def get_metadata_filename(self, uid_filename) -> tuple[str, str]:
+        metadata_file: str = f'{uid_filename}{self.metadata_suffix}'
+        return metadata_file, self.store_path(metadata_file)
+
+    def get_metadata(self, uid_filename, ignore_missing=False) -> dict:
+        _metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename)
+        if ignore_missing and not self.fs.exists(metadata_file_path):
+            return {}
+
+        with self.fs.open(metadata_file_path, 'rb') as f:
+            metadata = json.loads(f.read())
+
+        metadata = self.metadata_convert(uid_filename, metadata)
+        return metadata
+
+    def _store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+        raise NotImplementedError
+
+    def store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+        return self._store(key, uid_key, value_reader, max_filesize, metadata, **kwargs)
+
+    def _fetch(self, key, presigned_url_expires: int = 0):
+        raise NotImplementedError
+
+    def fetch(self, key, **kwargs) -> tuple[ShardFileReader, dict]:
+        return self._fetch(key)
+
+    def _delete(self, key):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata = self.get_metadata(key)
+        _metadata_file, metadata_file_path = self.get_metadata_filename(key)
+        artifact_file_path = metadata['filename_uid_path']
+        self.fs.rm(artifact_file_path)
+        self.fs.rm(metadata_file_path)
+
+        return 1
+
+    def delete(self, key):
+        raise NotImplementedError
+
+    def store_path(self, uid_filename):
+        raise NotImplementedError
+
+
+class BaseFileStoreBackend:
+    _shards = tuple()
+    _shard_cls = BaseShard
+    _config: dict | None = None
+    _storage_path: str = ''
+
+    def __init__(self, settings, extension_groups=None):
+        self._config = settings
+        extension_groups = extension_groups or ['any']
+        self.extensions = resolve_extensions([], groups=extension_groups)
+
+    def __contains__(self, key):
+        return self.filename_exists(key)
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__}(storage={self.storage_path})>'
+
+    @property
+    def storage_path(self):
+        return self._storage_path
+
+    @classmethod
+    def get_shard_index(cls, filename: str, num_shards) -> int:
+        # Generate a hash value from the filename
+        hash_value = sha256_safe(filename)
+
+        # Convert the hash value to an integer
+        hash_int = int(hash_value, 16)
+
+        # Map the hash integer to a shard number between 1 and num_shards
+        shard_number = (hash_int % num_shards)
+
+        return shard_number
+
+    @classmethod
+    def apply_counter(cls, counter: int, filename: str) -> str:
+        """
+        Apply a counter to the filename.
+
+        :param counter: The counter value to apply.
+        :param filename: The original filename.
+        :return: The modified filename with the counter.
+        """
+        name_counted = f'{counter:d}-{filename}'
+        return name_counted
+
+    def _get_shard(self, key) -> _shard_cls:
+        index = self.get_shard_index(key, len(self._shards))
+        shard = self._shards[index]
+        return shard
+
+    def get_conf(self, key, pop=False):
+        if key not in self._config:
+            raise ValueError(
+                f"No configuration key '{key}', please make sure it exists in filestore config")
+        val = self._config[key]
+        if pop:
+            del self._config[key]
+        return val
+
+    def filename_allowed(self, filename, extensions=None):
+        """Checks if a filename has an allowed extension
+
+        :param filename: base name of file
+        :param extensions: iterable of extensions (or self.extensions)
+        """
+        _, ext = os.path.splitext(filename)
+        return self.extension_allowed(ext, extensions)
+
+    def extension_allowed(self, ext, extensions=None):
+        """
+        Checks if an extension is permitted. Both e.g. ".jpg" and
+        "jpg" can be passed in. Extension lookup is case-insensitive.
+
+        :param ext: extension to check
+        :param extensions: iterable of extensions to validate against (or self.extensions)
+        """
+        def normalize_ext(_ext):
+            if _ext.startswith('.'):
+                _ext = _ext[1:]
+            return _ext.lower()
+
+        extensions = extensions or self.extensions
+        if not extensions:
+            return True
+
+        ext = normalize_ext(ext)
+
+        return ext in [normalize_ext(x) for x in extensions]
+
+    def filename_exists(self, uid_filename):
+        shard = self._get_shard(uid_filename)
+        return uid_filename in shard
+
+    def store_path(self, uid_filename):
+        """
+        Returns absolute file path of the uid_filename
+        """
+        shard = self._get_shard(uid_filename)
+        return shard.store_path(uid_filename)
+
+    def store_metadata(self, uid_filename):
+        shard = self._get_shard(uid_filename)
+        return shard.get_metadata_filename(uid_filename)
+
+    def store(self, filename, value_reader, extensions=None, metadata=None, max_filesize=None, randomized_name=True, **kwargs):
+        extensions = extensions or self.extensions
+
+        if not self.filename_allowed(filename, extensions):
+            msg = f'filename {filename} does not allow extensions {extensions}'
+            raise FileNotAllowedException(msg)
+
+        # # TODO: check why we need this setting ? it looks stupid...
+        # no_body_seek is used in stream mode importer somehow
+        # no_body_seek = kwargs.pop('no_body_seek', False)
+        # if no_body_seek:
+        #     pass
+        # else:
+        #     value_reader.seek(0)
+
+        uid_filename = kwargs.pop('uid_filename', None)
+        if uid_filename is None:
+            uid_filename = get_uid_filename(filename, randomized=randomized_name)
+
+        shard = self._get_shard(uid_filename)
+
+        return shard.store(filename, uid_filename, value_reader, max_filesize, metadata, **kwargs)
+
+    def import_to_store(self, value_reader, org_filename, uid_filename, metadata, **kwargs):
+        shard = self._get_shard(uid_filename)
+        max_filesize = None
+        return shard.store(org_filename, uid_filename, value_reader, max_filesize, metadata, import_mode=True)
+
+    def delete(self, uid_filename):
+        shard = self._get_shard(uid_filename)
+        return shard.delete(uid_filename)
+
+    def fetch(self, uid_filename) -> tuple[ShardFileReader, dict]:
+        shard = self._get_shard(uid_filename)
+        return shard.fetch(uid_filename)
+
+    def get_metadata(self, uid_filename, ignore_missing=False) -> dict:
+        shard = self._get_shard(uid_filename)
+        return shard.get_metadata(uid_filename, ignore_missing=ignore_missing)
+
+    def iter_keys(self):
+        for shard in self._shards:
+            if shard.fs.exists(shard.storage_medium):
+                for path, _dirs, _files in shard.fs.walk(shard.storage_medium):
+                    for key_file_path in _files:
+                        if key_file_path.endswith(shard.metadata_suffix):
+                            yield shard, key_file_path
+
+    def iter_artifacts(self):
+        for shard, key_file in self.iter_keys():
+            json_key = f"{shard.storage_medium}/{key_file}"
+            with shard.fs.open(json_key, 'rb') as f:
+                yield shard, json.loads(f.read())['filename_uid']
+
+    def get_statistics(self):
+        total_files = 0
+        total_size = 0
+        meta = {}
+
+        for shard, key_file in self.iter_keys():
+            json_key = f"{shard.storage_medium}/{key_file}"
+            with shard.fs.open(json_key, 'rb') as f:
+                total_files += 1
+                metadata = json.loads(f.read())
+                total_size += metadata['size']
+
+        return total_files, total_size, meta

rhodecode/apps/file_store/backends/filesystem.py

@@ -0,0 +1,183 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import hashlib
+import functools
+import time
+import logging
+
+from .. import config_keys
+from ..exceptions import FileOverSizeException
+from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
+
+from ....lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class FileSystemShard(BaseShard):
+    METADATA_VER = 'v2'
+    BACKEND_TYPE = config_keys.backend_filesystem
+    storage_type: str = 'directory'
+
+    def __init__(self, index, directory, directory_folder, fs, **settings):
+        self._index: int = index
+        self._directory: str = directory
+        self._directory_folder: str = directory_folder
+        self.fs = fs
+
+    @property
+    def directory(self) -> str:
+        """Cache directory final path."""
+        return os.path.join(self._directory, self._directory_folder)
+
+    def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
+
+        # ensure dir exists
+        destination, _ = os.path.split(full_path)
+        if not self.fs.exists(destination):
+            self.fs.makedirs(destination)
+
+        writer = self.fs.open(full_path, mode)
+
+        digest = hashlib.sha256()
+        oversize_cleanup = False
+        with writer:
+            size = 0
+            for chunk in iterator:
+                size += len(chunk)
+                digest.update(chunk)
+                writer.write(chunk)
+
+                if max_filesize and size > max_filesize:
+                    oversize_cleanup = True
+                    # free up the copied file, and raise exc
+                    break
+
+            writer.flush()
+            # Get the file descriptor
+            fd = writer.fileno()
+
+            # Sync the file descriptor to disk, helps with NFS cases...
+            os.fsync(fd)
+
+        if oversize_cleanup:
+            self.fs.rm(full_path)
+            raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
+
+        sha256 = digest.hexdigest()
+        log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
+        return size, sha256
+
+    def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+
+        filename = key
+        uid_filename = uid_key
+        full_path = self.store_path(uid_filename)
+
+        # STORE METADATA
+        _metadata = {
+            "version": self.METADATA_VER,
+            "store_type": self.BACKEND_TYPE,
+
+            "filename": filename,
+            "filename_uid_path": full_path,
+            "filename_uid": uid_filename,
+            "sha256": "",  # NOTE: filled in by reader iteration
+
+            "store_time": time.time(),
+
+            "size": 0
+        }
+
+        if metadata:
+            if kwargs.pop('import_mode', False):
+                # in import mode, we don't need to compute metadata, we just take the old version
+                _metadata["import_mode"] = True
+            else:
+                _metadata.update(metadata)
+
+        read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+        size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
+        _metadata['size'] = size
+        _metadata['sha256'] = sha256
+
+        # after storing the artifacts, we write the metadata present
+        _metadata_file, metadata_file_path = self.get_metadata_filename(uid_key)
+
+        with self.fs.open(metadata_file_path, 'wb') as f:
+            f.write(json.dumps(_metadata))
+
+        return uid_filename, _metadata
+
+    def store_path(self, uid_filename):
+        """
+        Returns absolute file path of the uid_filename
+        """
+        return os.path.join(self._directory, self._directory_folder, uid_filename)
+
+    def _fetch(self, key, presigned_url_expires: int = 0):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata = self.get_metadata(key)
+
+        file_path = metadata['filename_uid_path']
+        if presigned_url_expires and presigned_url_expires > 0:
+            metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
+
+        return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
+
+    def delete(self, key):
+        return self._delete(key)
+
+
+class FileSystemBackend(BaseFileStoreBackend):
+    shard_name: str = 'shard_{:03d}'
+    _shard_cls = FileSystemShard
+
+    def __init__(self, settings):
+        super().__init__(settings)
+
+        store_dir = self.get_conf(config_keys.filesystem_storage_path)
+        directory = os.path.expanduser(store_dir)
+
+        self._directory = directory
+        self._storage_path = directory  # common path for all from BaseCache
+        self._shard_count = int(self.get_conf(config_keys.filesystem_shards, pop=True))
+        if self._shard_count < 1:
+            raise ValueError(f'{config_keys.filesystem_shards} must be 1 or more')
+
+        log.debug('Initializing %s file_store instance', self)
+        fs = fsspec.filesystem('file')
+
+        if not fs.exists(self._directory):
+            fs.makedirs(self._directory, exist_ok=True)
+
+        self._shards = tuple(
+            self._shard_cls(
+                index=num,
+                directory=directory,
+                directory_folder=self.shard_name.format(num),
+                fs=fs,
+                **settings,
+            )
+            for num in range(self._shard_count)
+        )

rhodecode/apps/file_store/backends/filesystem_legacy.py

@@ -0,0 +1,278 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import errno
+import os
+import hashlib
+import functools
+import time
+import logging
+
+from .. import config_keys
+from ..exceptions import FileOverSizeException
+from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
+
+from ....lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class LegacyFileSystemShard(BaseShard):
+    # legacy ver
+    METADATA_VER = 'v2'
+    BACKEND_TYPE = config_keys.backend_legacy_filesystem
+    storage_type: str = 'dir_struct'
+
+    # legacy suffix
+    metadata_suffix: str = '.meta'
+
+    @classmethod
+    def _sub_store_from_filename(cls, filename):
+        return filename[:2]
+
+    @classmethod
+    def apply_counter(cls, counter, filename):
+        name_counted = '%d-%s' % (counter, filename)
+        return name_counted
+
+    @classmethod
+    def safe_make_dirs(cls, dir_path):
+        if not os.path.exists(dir_path):
+            try:
+                os.makedirs(dir_path)
+            except OSError as e:
+                if e.errno != errno.EEXIST:
+                    raise
+                return
+
+    @classmethod
+    def resolve_name(cls, name, directory):
+        """
+        Resolves a unique name and the correct path. If a filename
+        for that path already exists then a numeric prefix with values > 0 will be
+        added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
+
+        :param name: base name of file
+        :param directory: absolute directory path
+        """
+
+        counter = 0
+        while True:
+            name_counted = cls.apply_counter(counter, name)
+
+            # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
+            sub_store: str = cls._sub_store_from_filename(name_counted)
+            sub_store_path: str = os.path.join(directory, sub_store)
+            cls.safe_make_dirs(sub_store_path)
+
+            path = os.path.join(sub_store_path, name_counted)
+            if not os.path.exists(path):
+                return name_counted, path
+            counter += 1
+
+    def __init__(self, index, directory, directory_folder, fs, **settings):
+        self._index: int = index
+        self._directory: str = directory
+        self._directory_folder: str = directory_folder
+        self.fs = fs
+
+    @property
+    def dir_struct(self) -> str:
+        """Cache directory final path."""
+        return os.path.join(self._directory, '0-')
+
+    def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
+
+        # ensure dir exists
+        destination, _ = os.path.split(full_path)
+        if not self.fs.exists(destination):
+            self.fs.makedirs(destination)
+
+        writer = self.fs.open(full_path, mode)
+
+        digest = hashlib.sha256()
+        oversize_cleanup = False
+        with writer:
+            size = 0
+            for chunk in iterator:
+                size += len(chunk)
+                digest.update(chunk)
+                writer.write(chunk)
+
+                if max_filesize and size > max_filesize:
+                    # free up the copied file, and raise exc
+                    oversize_cleanup = True
+                    break
+
+            writer.flush()
+            # Get the file descriptor
+            fd = writer.fileno()
+
+            # Sync the file descriptor to disk, helps with NFS cases...
+            os.fsync(fd)
+
+        if oversize_cleanup:
+            self.fs.rm(full_path)
+            raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
+
+        sha256 = digest.hexdigest()
+        log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
+        return size, sha256
+
+    def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+
+        filename = key
+        uid_filename = uid_key
+
+        # NOTE:, also apply N- Counter...
+        uid_filename, full_path = self.resolve_name(uid_filename, self._directory)
+
+        # STORE METADATA
+        # TODO: make it compatible, and backward proof
+        _metadata = {
+            "version": self.METADATA_VER,
+
+            "filename": filename,
+            "filename_uid_path": full_path,
+            "filename_uid": uid_filename,
+            "sha256": "",  # NOTE: filled in by reader iteration
+
+            "store_time": time.time(),
+
+            "size": 0
+        }
+        if metadata:
+            _metadata.update(metadata)
+
+        read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+        size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
+        _metadata['size'] = size
+        _metadata['sha256'] = sha256
+
+        # after storing the artifacts, we write the metadata present
+        _metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename)
+
+        with self.fs.open(metadata_file_path, 'wb') as f:
+            f.write(json.dumps(_metadata))
+
+        return uid_filename, _metadata
+
+    def store_path(self, uid_filename):
+        """
+        Returns absolute file path of the uid_filename
+        """
+        prefix_dir = ''
+        if '/' in uid_filename:
+            prefix_dir, filename = uid_filename.split('/')
+            sub_store = self._sub_store_from_filename(filename)
+        else:
+            sub_store = self._sub_store_from_filename(uid_filename)
+
+        return os.path.join(self._directory, prefix_dir, sub_store, uid_filename)
+
+    def metadata_convert(self, uid_filename, metadata):
+        # NOTE: backward compat mode here... this is for file created PRE 5.2 system
+        if 'meta_ver' in metadata:
+            full_path = self.store_path(uid_filename)
+            metadata = {
+                "_converted": True,
+                "_org": metadata,
+                "version": self.METADATA_VER,
+                "store_type": self.BACKEND_TYPE,
+
+                "filename": metadata['filename'],
+                "filename_uid_path": full_path,
+                "filename_uid": uid_filename,
+                "sha256": metadata['sha256'],
+
+                "store_time": metadata['time'],
+
+                "size": metadata['size']
+            }
+        return metadata
+
+    def _fetch(self, key, presigned_url_expires: int = 0):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata = self.get_metadata(key)
+
+        file_path = metadata['filename_uid_path']
+        if presigned_url_expires and presigned_url_expires > 0:
+            metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
+
+        return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
+
+    def delete(self, key):
+        return self._delete(key)
+
+    def _delete(self, key):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata = self.get_metadata(key)
+        metadata_file, metadata_file_path = self.get_metadata_filename(key)
+        artifact_file_path = metadata['filename_uid_path']
+        self.fs.rm(artifact_file_path)
+        self.fs.rm(metadata_file_path)
+
+    def get_metadata_filename(self, uid_filename) -> tuple[str, str]:
+
+        metadata_file: str = f'{uid_filename}{self.metadata_suffix}'
+        uid_path_in_store = self.store_path(uid_filename)
+
+        metadata_file_path = f'{uid_path_in_store}{self.metadata_suffix}'
+        return metadata_file, metadata_file_path
+
+
+class LegacyFileSystemBackend(BaseFileStoreBackend):
+    _shard_cls = LegacyFileSystemShard
+
+    def __init__(self, settings):
+        super().__init__(settings)
+
+        store_dir = self.get_conf(config_keys.legacy_filesystem_storage_path)
+        directory = os.path.expanduser(store_dir)
+
+        self._directory = directory
+        self._storage_path = directory  # common path for all from BaseCache
+
+        log.debug('Initializing %s file_store instance', self)
+        fs = fsspec.filesystem('file')
+
+        if not fs.exists(self._directory):
+            fs.makedirs(self._directory, exist_ok=True)
+
+        # legacy system uses single shard
+        self._shards = tuple(
+            [
+                self._shard_cls(
+                    index=0,
+                    directory=directory,
+                    directory_folder='',
+                    fs=fs,
+                    **settings,
+                )
+            ]
+        )
+
+    @classmethod
+    def get_shard_index(cls, filename: str, num_shards) -> int:
+        # legacy filesystem doesn't use shards, and always uses single shard
+        return 0

rhodecode/apps/file_store/backends/local_store.py

@@ -1,268 +0,0 @@
-# Copyright (C) 2016-2023 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-import os
-import time
-import errno
-import hashlib
-
-from rhodecode.lib.ext_json import json
-from rhodecode.apps.file_store import utils
-from rhodecode.apps.file_store.extensions import resolve_extensions
-from rhodecode.apps.file_store.exceptions import (
-    FileNotAllowedException, FileOverSizeException)
-
-METADATA_VER = 'v1'
-
-
-def safe_make_dirs(dir_path):
-    if not os.path.exists(dir_path):
-        try:
-            os.makedirs(dir_path)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
-            return
-
-
-class LocalFileStorage(object):
-
-    @classmethod
-    def apply_counter(cls, counter, filename):
-        name_counted = '%d-%s' % (counter, filename)
-        return name_counted
-
-    @classmethod
-    def resolve_name(cls, name, directory):
-        """
-        Resolves a unique name and the correct path. If a filename
-        for that path already exists then a numeric prefix with values > 0 will be
-        added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
-
-        :param name: base name of file
-        :param directory: absolute directory path
-        """
-
-        counter = 0
-        while True:
-            name_counted = cls.apply_counter(counter, name)
-
-            # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
-            sub_store = cls._sub_store_from_filename(name_counted)
-            sub_store_path = os.path.join(directory, sub_store)
-            safe_make_dirs(sub_store_path)
-
-            path = os.path.join(sub_store_path, name_counted)
-            if not os.path.exists(path):
-                return name_counted, path
-            counter += 1
-
-    @classmethod
-    def _sub_store_from_filename(cls, filename):
-        return filename[:2]
-
-    @classmethod
-    def calculate_path_hash(cls, file_path):
-        """
-        Efficient calculation of file_path sha256 sum
-
-        :param file_path:
-        :return: sha256sum
-        """
-        digest = hashlib.sha256()
-        with open(file_path, 'rb') as f:
-            for chunk in iter(lambda: f.read(1024 * 100), b""):
-                digest.update(chunk)
-
-        return digest.hexdigest()
-
-    def __init__(self, base_path, extension_groups=None):
-
-        """
-        Local file storage
-
-        :param base_path: the absolute base path where uploads are stored
-        :param extension_groups: extensions string
-        """
-
-        extension_groups = extension_groups or ['any']
-        self.base_path = base_path
-        self.extensions = resolve_extensions([], groups=extension_groups)
-
-    def __repr__(self):
-        return f'{self.__class__}@{self.base_path}'
-
-    def store_path(self, filename):
-        """
-        Returns absolute file path of the filename, joined to the
-        base_path.
-
-        :param filename: base name of file
-        """
-        prefix_dir = ''
-        if '/' in filename:
-            prefix_dir, filename = filename.split('/')
-            sub_store = self._sub_store_from_filename(filename)
-        else:
-            sub_store = self._sub_store_from_filename(filename)
-        return os.path.join(self.base_path, prefix_dir, sub_store, filename)
-
-    def delete(self, filename):
-        """
-        Deletes the filename. Filename is resolved with the
-        absolute path based on base_path. If file does not exist,
-        returns **False**, otherwise **True**
-
-        :param filename: base name of file
-        """
-        if self.exists(filename):
-            os.remove(self.store_path(filename))
-            return True
-        return False
-
-    def exists(self, filename):
-        """
-        Checks if file exists. Resolves filename's absolute
-        path based on base_path.
-
-        :param filename: file_uid name of file, e.g 0-f62b2b2d-9708-4079-a071-ec3f958448d4.svg
-        """
-        return os.path.exists(self.store_path(filename))
-
-    def filename_allowed(self, filename, extensions=None):
-        """Checks if a filename has an allowed extension
-
-        :param filename: base name of file
-        :param extensions: iterable of extensions (or self.extensions)
-        """
-        _, ext = os.path.splitext(filename)
-        return self.extension_allowed(ext, extensions)
-
-    def extension_allowed(self, ext, extensions=None):
-        """
-        Checks if an extension is permitted. Both e.g. ".jpg" and
-        "jpg" can be passed in. Extension lookup is case-insensitive.
-
-        :param ext: extension to check
-        :param extensions: iterable of extensions to validate against (or self.extensions)
-        """
-        def normalize_ext(_ext):
-            if _ext.startswith('.'):
-                _ext = _ext[1:]
-            return _ext.lower()
-
-        extensions = extensions or self.extensions
-        if not extensions:
-            return True
-
-        ext = normalize_ext(ext)
-
-        return ext in [normalize_ext(x) for x in extensions]
-
-    def save_file(self, file_obj, filename, directory=None, extensions=None,
-                  extra_metadata=None, max_filesize=None, randomized_name=True, **kwargs):
-        """
-        Saves a file object to the uploads location.
-        Returns the resolved filename, i.e. the directory +
-        the (randomized/incremented) base name.
-
-        :param file_obj: **cgi.FieldStorage** object (or similar)
-        :param filename: original filename
-        :param directory: relative path of sub-directory
-        :param extensions: iterable of allowed extensions, if not default
-        :param max_filesize: maximum size of file that should be allowed
-        :param randomized_name: generate random generated UID or fixed based on the filename
-        :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
-
-        """
-
-        extensions = extensions or self.extensions
-
-        if not self.filename_allowed(filename, extensions):
-            raise FileNotAllowedException()
-
-        if directory:
-            dest_directory = os.path.join(self.base_path, directory)
-        else:
-            dest_directory = self.base_path
-
-        safe_make_dirs(dest_directory)
-
-        uid_filename = utils.uid_filename(filename, randomized=randomized_name)
-
-        # resolve also produces special sub-dir for file optimized store
-        filename, path = self.resolve_name(uid_filename, dest_directory)
-        stored_file_dir = os.path.dirname(path)
-
-        no_body_seek = kwargs.pop('no_body_seek', False)
-        if no_body_seek:
-            pass
-        else:
-            file_obj.seek(0)
-
-        with open(path, "wb") as dest:
-            length = 256 * 1024
-            while 1:
-                buf = file_obj.read(length)
-                if not buf:
-                    break
-                dest.write(buf)
-
-        metadata = {}
-        if extra_metadata:
-            metadata = extra_metadata
-
-        size = os.stat(path).st_size
-
-        if max_filesize and size > max_filesize:
-            # free up the copied file, and raise exc
-            os.remove(path)
-            raise FileOverSizeException()
-
-        file_hash = self.calculate_path_hash(path)
-
-        metadata.update({
-            "filename": filename,
-             "size": size,
-             "time": time.time(),
-             "sha256": file_hash,
-             "meta_ver": METADATA_VER
-        })
-
-        filename_meta = filename + '.meta'
-        with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
-            dest_meta.write(json.dumps(metadata))
-
-        if directory:
-            filename = os.path.join(directory, filename)
-
-        return filename, metadata
-
-    def get_metadata(self, filename, ignore_missing=False):
-        """
-        Reads JSON stored metadata for a file
-
-        :param filename:
-        :return:
-        """
-        filename = self.store_path(filename)
-        filename_meta = filename + '.meta'
-        if ignore_missing and not os.path.isfile(filename_meta):
-            return {}
-        with open(filename_meta, "rb") as source_meta:
-            return json.loads(source_meta.read())

rhodecode/apps/file_store/backends/objectstore.py

@@ -0,0 +1,184 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import hashlib
+import functools
+import time
+import logging
+
+from .. import config_keys
+from ..exceptions import FileOverSizeException
+from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
+
+from ....lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class S3Shard(BaseShard):
+    METADATA_VER = 'v2'
+    BACKEND_TYPE = config_keys.backend_objectstore
+    storage_type: str = 'bucket'
+
+    def __init__(self, index, bucket, bucket_folder, fs, **settings):
+        self._index: int = index
+        self._bucket_main: str = bucket
+        self._bucket_folder: str = bucket_folder
+
+        self.fs = fs
+
+    @property
+    def bucket(self) -> str:
+        """Cache bucket final path."""
+        return os.path.join(self._bucket_main, self._bucket_folder)
+
+    def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
+
+        # ensure dir exists
+        destination, _ = os.path.split(full_path)
+        if not self.fs.exists(destination):
+            self.fs.makedirs(destination)
+
+        writer = self.fs.open(full_path, mode)
+
+        digest = hashlib.sha256()
+        oversize_cleanup = False
+        with writer:
+            size = 0
+            for chunk in iterator:
+                size += len(chunk)
+                digest.update(chunk)
+                writer.write(chunk)
+
+                if max_filesize and size > max_filesize:
+                    oversize_cleanup = True
+                    # free up the copied file, and raise exc
+                    break
+
+        if oversize_cleanup:
+            self.fs.rm(full_path)
+            raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
+
+        sha256 = digest.hexdigest()
+        log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
+        return size, sha256
+
+    def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+
+        filename = key
+        uid_filename = uid_key
+        full_path = self.store_path(uid_filename)
+
+        # STORE METADATA
+        _metadata = {
+            "version": self.METADATA_VER,
+            "store_type": self.BACKEND_TYPE,
+
+            "filename": filename,
+            "filename_uid_path": full_path,
+            "filename_uid": uid_filename,
+            "sha256": "",  # NOTE: filled in by reader iteration
+
+            "store_time": time.time(),
+
+            "size": 0
+        }
+
+        if metadata:
+            if kwargs.pop('import_mode', False):
+                # in import mode, we don't need to compute metadata, we just take the old version
+                _metadata["import_mode"] = True
+            else:
+                _metadata.update(metadata)
+
+        read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+        size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
+        _metadata['size'] = size
+        _metadata['sha256'] = sha256
+
+        # after storing the artifacts, we write the metadata present
+        metadata_file, metadata_file_path = self.get_metadata_filename(uid_key)
+
+        with self.fs.open(metadata_file_path, 'wb') as f:
+            f.write(json.dumps(_metadata))
+
+        return uid_filename, _metadata
+
+    def store_path(self, uid_filename):
+        """
+        Returns absolute file path of the uid_filename
+        """
+        return os.path.join(self._bucket_main, self._bucket_folder, uid_filename)
+
+    def _fetch(self, key, presigned_url_expires: int = 0):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata_file, metadata_file_path = self.get_metadata_filename(key)
+        with self.fs.open(metadata_file_path, 'rb') as f:
+            metadata = json.loads(f.read())
+
+        file_path = metadata['filename_uid_path']
+        if presigned_url_expires and presigned_url_expires > 0:
+            metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
+
+        return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
+
+    def delete(self, key):
+        return self._delete(key)
+
+
+class ObjectStoreBackend(BaseFileStoreBackend):
+    shard_name: str = 'shard-{:03d}'
+    _shard_cls = S3Shard
+
+    def __init__(self, settings):
+        super().__init__(settings)
+
+        self._shard_count = int(self.get_conf(config_keys.objectstore_bucket_shards, pop=True))
+        if self._shard_count < 1:
+            raise ValueError('cache_shards must be 1 or more')
+
+        self._bucket = settings.pop(config_keys.objectstore_bucket)
+        if not self._bucket:
+            raise ValueError(f'{config_keys.objectstore_bucket} needs to have a value')
+
+        objectstore_url = self.get_conf(config_keys.objectstore_url)
+        key = settings.pop(config_keys.objectstore_key)
+        secret = settings.pop(config_keys.objectstore_secret)
+
+        self._storage_path = objectstore_url  # common path for all from BaseCache
+        log.debug('Initializing %s file_store instance', self)
+        fs = fsspec.filesystem('s3', anon=False, endpoint_url=objectstore_url, key=key, secret=secret)
+
+        # init main bucket
+        if not fs.exists(self._bucket):
+            fs.mkdir(self._bucket)
+
+        self._shards = tuple(
+            self._shard_cls(
+                index=num,
+                bucket=self._bucket,
+                bucket_folder=self.shard_name.format(num),
+                fs=fs,
+                **settings,
+            )
+            for num in range(self._shard_count)
+        )

rhodecode/apps/file_store/config_keys.py

@@ -20,6 +20,38 @@
 # Definition of setting keys used to configure this module. Defined here to
 # avoid repetition of keys throughout the module.
 
-enabled = 'file_store.enabled'
-backend = 'file_store.backend'
-store_path = 'file_store.storage_path'
+# OLD and deprecated keys not used anymore
+deprecated_enabled = 'file_store.enabled'
+deprecated_backend = 'file_store.backend'
+deprecated_store_path = 'file_store.storage_path'
+
+
+backend_type = 'file_store.backend.type'
+
+backend_legacy_filesystem = 'filesystem_v1'
+backend_filesystem = 'filesystem_v2'
+backend_objectstore = 'objectstore'
+
+backend_types = [
+    backend_legacy_filesystem,
+    backend_filesystem,
+    backend_objectstore,
+]
+
+# filesystem_v1 legacy
+legacy_filesystem_storage_path = 'file_store.filesystem_v1.storage_path'
+
+
+# filesystem_v2 new option
+filesystem_storage_path = 'file_store.filesystem_v2.storage_path'
+filesystem_shards = 'file_store.filesystem_v2.shards'
+
+# objectstore
+objectstore_url = 'file_store.objectstore.url'
+objectstore_bucket = 'file_store.objectstore.bucket'
+objectstore_bucket_shards = 'file_store.objectstore.bucket_shards'
+
+objectstore_region = 'file_store.objectstore.region'
+objectstore_key = 'file_store.objectstore.key'
+objectstore_secret = 'file_store.objectstore.secret'
+

rhodecode/apps/file_store/tests/__init__.py

@@ -16,3 +16,42 @@
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
+import os
+import random
+import tempfile
+import string
+
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+
+
+@pytest.fixture()
+def file_store_instance(ini_settings):
+    config = ini_settings
+    f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+    return f_store
+
+
+@pytest.fixture
+def random_binary_file():
+    # Generate random binary data
+    data = bytearray(random.getrandbits(8) for _ in range(1024 * 512))  # 512 KB of random data
+
+    # Create a temporary file
+    temp_file = tempfile.NamedTemporaryFile(delete=False)
+    filename = temp_file.name
+
+    try:
+        # Write the random binary data to the file
+        temp_file.write(data)
+        temp_file.seek(0)  # Rewind the file pointer to the beginning
+        yield filename, temp_file
+    finally:
+        # Close and delete the temporary file after the test
+        temp_file.close()
+        os.remove(filename)
+
+
+def generate_random_filename(length=10):
+    return ''.join(random.choices(string.ascii_letters + string.digits, k=length))

rhodecode/apps/file_store/tests/test_filestore_backends.py

@@ -0,0 +1,128 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps import file_store
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend
+from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend
+from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend
+from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store.tests import random_binary_file, file_store_instance
+
+
+class TestFileStoreBackends:
+
+    @pytest.mark.parametrize('backend_type, expected_instance', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+        (config_keys.backend_filesystem, FileSystemBackend),
+        (config_keys.backend_objectstore, ObjectStoreBackend),
+    ])
+    def test_get_backend(self, backend_type, expected_instance, ini_settings):
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+        assert isinstance(f_store, expected_instance)
+
+    @pytest.mark.parametrize('backend_type, expected_instance', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+        (config_keys.backend_filesystem, FileSystemBackend),
+        (config_keys.backend_objectstore, ObjectStoreBackend),
+    ])
+    def test_store_and_read(self, backend_type, expected_instance, ini_settings, random_binary_file):
+        filename, temp_file = random_binary_file
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+        metadata = {
+            'user_uploaded': {
+                'username': 'user1',
+                'user_id': 10,
+                'ip': '10.20.30.40'
+            }
+        }
+        store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata)
+        assert store_fid
+        assert metadata
+
+        # read-after write
+        reader, metadata2 = f_store.fetch(store_fid)
+        assert reader
+        assert metadata2['filename'] == filename
+
+    @pytest.mark.parametrize('backend_type, expected_instance', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+        (config_keys.backend_filesystem, FileSystemBackend),
+        (config_keys.backend_objectstore, ObjectStoreBackend),
+    ])
+    def test_store_file_not_allowed(self, backend_type, expected_instance, ini_settings, random_binary_file):
+        filename, temp_file = random_binary_file
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+        with pytest.raises(FileNotAllowedException):
+            f_store.store('notallowed.exe', temp_file, extensions=['.txt'])
+
+    @pytest.mark.parametrize('backend_type, expected_instance', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+        (config_keys.backend_filesystem, FileSystemBackend),
+        (config_keys.backend_objectstore, ObjectStoreBackend),
+    ])
+    def test_store_file_over_size(self, backend_type, expected_instance, ini_settings, random_binary_file):
+        filename, temp_file = random_binary_file
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+        with pytest.raises(FileOverSizeException):
+            f_store.store('toobig.exe', temp_file, extensions=['.exe'], max_filesize=124)
+
+    @pytest.mark.parametrize('backend_type, expected_instance, extra_conf', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend, {}),
+        (config_keys.backend_filesystem, FileSystemBackend, {config_keys.filesystem_storage_path: '/tmp/test-fs-store'}),
+        (config_keys.backend_objectstore, ObjectStoreBackend, {config_keys.objectstore_bucket: 'test-bucket'}),
+    ])
+    def test_store_stats_and_keys(self, backend_type, expected_instance, extra_conf, ini_settings, random_binary_file):
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        config.update(extra_conf)
+
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+
+        # purge storage before running
+        for shard, k in f_store.iter_artifacts():
+            f_store.delete(k)
+
+        for i in range(10):
+            filename, temp_file = random_binary_file
+
+            metadata = {
+                'user_uploaded': {
+                    'username': 'user1',
+                    'user_id': 10,
+                    'ip': '10.20.30.40'
+                }
+            }
+            store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata)
+            assert store_fid
+            assert metadata
+
+        cnt, size, meta = f_store.get_statistics()
+        assert cnt == 10
+        assert 10 == len(list(f_store.iter_keys()))

rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py

@@ -0,0 +1,52 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.tests import generate_random_filename
+
+
+@pytest.fixture()
+def file_store_filesystem_instance(ini_settings):
+    config = ini_settings
+    config[config_keys.backend_type] = config_keys.backend_filesystem
+    f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+    return f_store
+
+
+class TestFileStoreFileSystemBackend:
+
+    @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
+    def test_get_shard_number(self, filename, file_store_filesystem_instance):
+        shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards))
+        # Check that the shard number is between 0 and max-shards
+        assert 0 <= shard_number <= len(file_store_filesystem_instance._shards)
+
+    @pytest.mark.parametrize('filename, expected_shard_num', [
+        ('my-name-1', 3),
+        ('my-name-2', 2),
+        ('my-name-3', 4),
+        ('my-name-4', 1),
+
+        ('rhodecode-enterprise-ce', 5),
+        ('rhodecode-enterprise-ee', 6),
+    ])
+    def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_filesystem_instance):
+        shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards))
+        assert expected_shard_num == shard_number

rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py

@@ -0,0 +1,17 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/

rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py

@@ -0,0 +1,52 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.tests import generate_random_filename
+
+
+@pytest.fixture()
+def file_store_legacy_instance(ini_settings):
+    config = ini_settings
+    config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+    f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+    return f_store
+
+
+class TestFileStoreLegacyBackend:
+
+    @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
+    def test_get_shard_number(self, filename, file_store_legacy_instance):
+        shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards))
+        # Check that the shard number is 0 for legacy filesystem store we don't use shards
+        assert shard_number == 0
+
+    @pytest.mark.parametrize('filename, expected_shard_num', [
+        ('my-name-1', 0),
+        ('my-name-2', 0),
+        ('my-name-3', 0),
+        ('my-name-4', 0),
+
+        ('rhodecode-enterprise-ce', 0),
+        ('rhodecode-enterprise-ee', 0),
+    ])
+    def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_legacy_instance):
+        shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards))
+        assert expected_shard_num == shard_number

rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py

@@ -0,0 +1,52 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.tests import generate_random_filename
+
+
+@pytest.fixture()
+def file_store_objectstore_instance(ini_settings):
+    config = ini_settings
+    config[config_keys.backend_type] = config_keys.backend_objectstore
+    f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+    return f_store
+
+
+class TestFileStoreObjectStoreBackend:
+
+    @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
+    def test_get_shard_number(self, filename, file_store_objectstore_instance):
+        shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards))
+        # Check that the shard number is between 0 and shards
+        assert 0 <= shard_number <= len(file_store_objectstore_instance._shards)
+
+    @pytest.mark.parametrize('filename, expected_shard_num', [
+        ('my-name-1', 3),
+        ('my-name-2', 2),
+        ('my-name-3', 4),
+        ('my-name-4', 1),
+
+        ('rhodecode-enterprise-ce', 5),
+        ('rhodecode-enterprise-ee', 6),
+    ])
+    def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_objectstore_instance):
+        shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards))
+        assert expected_shard_num == shard_number

rhodecode/apps/file_store/tests/test_upload_file.py

@@ -15,13 +15,16 @@
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
+
 import os
+
 import pytest
 
 from rhodecode.lib.ext_json import json
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.db import Session, FileStore, Repository, User
-from rhodecode.apps.file_store import utils, config_keys
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
 
 from rhodecode.tests import TestController
 from rhodecode.tests.routes import route_path
@@ -29,27 +32,61 @@ from rhodecode.tests.routes import route_path
 
 class TestFileStoreViews(TestController):
 
+    @pytest.fixture()
+    def create_artifact_factory(self, tmpdir, ini_settings):
+
+        def factory(user_id, content, f_name='example.txt'):
+
+            config = ini_settings
+            config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+
+            f_store = store_utils.get_filestore_backend(config)
+
+            filesystem_file = os.path.join(str(tmpdir), f_name)
+            with open(filesystem_file, 'wt') as f:
+                f.write(content)
+
+            with open(filesystem_file, 'rb') as f:
+                store_uid, metadata = f_store.store(f_name, f, metadata={'filename': f_name})
+            os.remove(filesystem_file)
+
+            entry = FileStore.create(
+                file_uid=store_uid, filename=metadata["filename"],
+                file_hash=metadata["sha256"], file_size=metadata["size"],
+                file_display_name='file_display_name',
+                file_description='repo artifact `{}`'.format(metadata["filename"]),
+                check_acl=True, user_id=user_id,
+            )
+            Session().add(entry)
+            Session().commit()
+            return entry
+        return factory
+
     @pytest.mark.parametrize("fid, content, exists", [
         ('abcde-0.jpg', "xxxxx", True),
         ('abcde-0.exe', "1234567", True),
         ('abcde-0.jpg', "xxxxx", False),
     ])
-    def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util):
+    def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util, ini_settings):
         user = self.log_user()
         user_id = user['user_id']
         repo_id = user_util.create_repo().repo_id
-        store_path = self.app._pyramid_settings[config_keys.store_path]
+
+        config = ini_settings
+        config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+
         store_uid = fid
 
         if exists:
             status = 200
-            store = utils.get_file_storage({config_keys.store_path: store_path})
+            f_store = store_utils.get_filestore_backend(config)
             filesystem_file = os.path.join(str(tmpdir), fid)
             with open(filesystem_file, 'wt') as f:
                 f.write(content)
 
             with open(filesystem_file, 'rb') as f:
-                store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
+                store_uid, metadata = f_store.store(fid, f, metadata={'filename': fid})
+            os.remove(filesystem_file)
 
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
@@ -69,14 +106,10 @@ class TestFileStoreViews(TestController):
 
         if exists:
             assert response.text == content
-            file_store_path = os.path.dirname(store.resolve_name(store_uid, store_path)[1])
-            metadata_file = os.path.join(file_store_path, store_uid + '.meta')
-            assert os.path.exists(metadata_file)
-            with open(metadata_file, 'rb') as f:
-                json_data = json.loads(f.read())
 
-            assert json_data
-            assert 'size' in json_data
+            metadata = f_store.get_metadata(store_uid)
+
+            assert 'size' in metadata
 
     def test_upload_files_without_content_to_store(self):
         self.log_user()
@@ -112,32 +145,6 @@ class TestFileStoreViews(TestController):
 
         assert response.json['store_fid']
 
-    @pytest.fixture()
-    def create_artifact_factory(self, tmpdir):
-        def factory(user_id, content):
-            store_path = self.app._pyramid_settings[config_keys.store_path]
-            store = utils.get_file_storage({config_keys.store_path: store_path})
-            fid = 'example.txt'
-
-            filesystem_file = os.path.join(str(tmpdir), fid)
-            with open(filesystem_file, 'wt') as f:
-                f.write(content)
-
-            with open(filesystem_file, 'rb') as f:
-                store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
-
-            entry = FileStore.create(
-                file_uid=store_uid, filename=metadata["filename"],
-                file_hash=metadata["sha256"], file_size=metadata["size"],
-                file_display_name='file_display_name',
-                file_description='repo artifact `{}`'.format(metadata["filename"]),
-                check_acl=True, user_id=user_id,
-            )
-            Session().add(entry)
-            Session().commit()
-            return entry
-        return factory
-
     def test_download_file_non_scoped(self, user_util, create_artifact_factory):
         user = self.log_user()
         user_id = user['user_id']

rhodecode/apps/file_store/utils.py

@@ -19,21 +19,84 @@
 import io
 import uuid
 import pathlib
+import s3fs
+
+from rhodecode.lib.hash_utils import sha256_safe
+from rhodecode.apps.file_store import config_keys
 
 
-def get_file_storage(settings):
-    from rhodecode.apps.file_store.backends.local_store import LocalFileStorage
-    from rhodecode.apps.file_store import config_keys
-    store_path = settings.get(config_keys.store_path)
-    return LocalFileStorage(base_path=store_path)
+file_store_meta = None
+
+
+def get_filestore_config(config) -> dict:
+
+    final_config = {}
+
+    for k, v in config.items():
+        if k.startswith('file_store'):
+            final_config[k] = v
+
+    return final_config
+
+
+def get_filestore_backend(config, always_init=False):
+    """
+
+    usage::
+        from rhodecode.apps.file_store import get_filestore_backend
+        f_store = get_filestore_backend(config=CONFIG)
+
+    :param config:
+    :param always_init:
+    :return:
+    """
+
+    global file_store_meta
+    if file_store_meta is not None and not always_init:
+        return file_store_meta
+
+    config = get_filestore_config(config)
+    backend = config[config_keys.backend_type]
+
+    match backend:
+        case config_keys.backend_legacy_filesystem:
+            # Legacy backward compatible storage
+            from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend
+            d_cache = LegacyFileSystemBackend(
+                settings=config
+            )
+        case config_keys.backend_filesystem:
+            from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend
+            d_cache = FileSystemBackend(
+                settings=config
+            )
+        case config_keys.backend_objectstore:
+            from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend
+            d_cache = ObjectStoreBackend(
+                settings=config
+            )
+        case _:
+            raise ValueError(
+                f'file_store.backend.type only supports "{config_keys.backend_types}" got {backend}'
+            )
+
+    cache_meta = d_cache
+    return cache_meta
 
 
 def splitext(filename):
-    ext = ''.join(pathlib.Path(filename).suffixes)
+    final_ext = []
+    for suffix in pathlib.Path(filename).suffixes:
+        if not suffix.isascii():
+            continue
+
+        suffix = " ".join(suffix.split()).replace(" ", "")
+        final_ext.append(suffix)
+    ext = ''.join(final_ext)
     return filename, ext
 
 
-def uid_filename(filename, randomized=True):
+def get_uid_filename(filename, randomized=True):
     """
     Generates a randomized or stable (uuid) filename,
     preserving the original extension.
@@ -46,10 +109,37 @@ def uid_filename(filename, randomized=True):
     if randomized:
         uid = uuid.uuid4()
     else:
-        hash_key = '{}.{}'.format(filename, 'store')
+        store_suffix = "store"
+        hash_key = f'{filename}.{store_suffix}'
         uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
     return str(uid) + ext.lower()
 
 
 def bytes_to_file_obj(bytes_data):
-    return io.StringIO(bytes_data)
+    return io.BytesIO(bytes_data)
+
+
+class ShardFileReader:
+
+    def __init__(self, file_like_reader):
+        self._file_like_reader = file_like_reader
+
+    def __getattr__(self, item):
+        if isinstance(self._file_like_reader, s3fs.core.S3File):
+            match item:
+                case 'name':
+                    # S3 FileWrapper doesn't support name attribute, and we use it
+                    return self._file_like_reader.full_name
+                case _:
+                    return getattr(self._file_like_reader, item)
+        else:
+            return getattr(self._file_like_reader, item)
+
+
+def archive_iterator(_reader, block_size: int = 4096 * 512):
+    # 4096 * 64 = 64KB
+    while 1:
+        data = _reader.read(block_size)
+        if not data:
+            break
+        yield data

rhodecode/apps/file_store/views.py

@@ -17,12 +17,11 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import logging
 
-
-from pyramid.response import FileResponse
+from pyramid.response import Response
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 
 from rhodecode.apps._base import BaseAppView
-from rhodecode.apps.file_store import utils
+from rhodecode.apps.file_store import utils as store_utils
 from rhodecode.apps.file_store.exceptions import (
     FileNotAllowedException, FileOverSizeException)
 
@@ -31,6 +30,7 @@ from rhodecode.lib import audit_logger
 from rhodecode.lib.auth import (
     CSRFRequired, NotAnonymous, HasRepoPermissionAny, HasRepoGroupPermissionAny,
     LoginRequired)
+from rhodecode.lib.str_utils import header_safe_str
 from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db
 from rhodecode.model.db import Session, FileStore, UserApiKeys
 
@@ -42,7 +42,7 @@ class FileStoreView(BaseAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-        self.storage = utils.get_file_storage(self.request.registry.settings)
+        self.f_store = store_utils.get_filestore_backend(self.request.registry.settings)
         return c
 
     def _guess_type(self, file_name):
@@ -55,8 +55,8 @@ class FileStoreView(BaseAppView):
         return _content_type, _encoding
 
     def _serve_file(self, file_uid):
-        if not self.storage.exists(file_uid):
-            store_path = self.storage.store_path(file_uid)
+        if not self.f_store.filename_exists(file_uid):
+            store_path = self.f_store.store_path(file_uid)
             log.warning('File with FID:%s not found in the store under `%s`',
                         file_uid, store_path)
             raise HTTPNotFound()
@@ -98,28 +98,25 @@ class FileStoreView(BaseAppView):
 
         FileStore.bump_access_counter(file_uid)
 
-        file_path = self.storage.store_path(file_uid)
+        file_name = db_obj.file_display_name
         content_type = 'application/octet-stream'
-        content_encoding = None
 
-        _content_type, _encoding = self._guess_type(file_path)
+        _content_type, _encoding = self._guess_type(file_name)
         if _content_type:
             content_type = _content_type
 
         # For file store we don't submit any session data, this logic tells the
         # Session lib to skip it
         setattr(self.request, '_file_response', True)
-        response = FileResponse(
-            file_path, request=self.request,
-            content_type=content_type, content_encoding=content_encoding)
+        reader, _meta = self.f_store.fetch(file_uid)
 
-        file_name = db_obj.file_display_name
+        response = Response(app_iter=store_utils.archive_iterator(reader))
+
+        response.content_type = str(content_type)
+        response.content_disposition = f'attachment; filename="{header_safe_str(file_name)}"'
 
-        response.headers["Content-Disposition"] = (
-            f'attachment; filename="{str(file_name)}"'
-        )
         response.headers["X-RC-Artifact-Id"] = str(db_obj.file_store_id)
-        response.headers["X-RC-Artifact-Desc"] = str(db_obj.file_description)
+        response.headers["X-RC-Artifact-Desc"] = header_safe_str(db_obj.file_description)
         response.headers["X-RC-Artifact-Sha256"] = str(db_obj.file_hash)
         return response
 
@@ -147,8 +144,8 @@ class FileStoreView(BaseAppView):
                               'user_id': self._rhodecode_user.user_id,
                               'ip': self._rhodecode_user.ip_addr}}
         try:
-            store_uid, metadata = self.storage.save_file(
-                file_obj.file, filename, extra_metadata=metadata)
+            store_uid, metadata = self.f_store.store(
+                filename, file_obj.file, extra_metadata=metadata)
         except FileNotAllowedException:
             return {'store_fid': None,
                     'access_path': None,
@@ -182,7 +179,7 @@ class FileStoreView(BaseAppView):
     def download_file(self):
         self.load_default_context()
         file_uid = self.request.matchdict['fid']
-        log.debug('Requesting FID:%s from store %s', file_uid, self.storage)
+        log.debug('Requesting FID:%s from store %s', file_uid, self.f_store)
         return self._serve_file(file_uid)
 
     # in addition to @LoginRequired ACL is checked by scopes

rhodecode/apps/repository/views/repo_commits.py

@@ -601,26 +601,26 @@ class RepoCommitsView(RepoAppView):
         max_file_size = 10 * 1024 * 1024  # 10MB, also validated via dropzone.js
 
         try:
-            storage = store_utils.get_file_storage(self.request.registry.settings)
-            store_uid, metadata = storage.save_file(
-                file_obj.file, filename, extra_metadata=metadata,
+            f_store = store_utils.get_filestore_backend(self.request.registry.settings)
+            store_uid, metadata = f_store.store(
+                filename, file_obj.file, metadata=metadata,
                 extensions=allowed_extensions, max_filesize=max_file_size)
         except FileNotAllowedException:
             self.request.response.status = 400
             permitted_extensions = ', '.join(allowed_extensions)
-            error_msg = 'File `{}` is not allowed. ' \
-                        'Only following extensions are permitted: {}'.format(
-                            filename, permitted_extensions)
+            error_msg = f'File `{filename}` is not allowed. ' \
+                        f'Only following extensions are permitted: {permitted_extensions}'
+
             return {'store_fid': None,
                     'access_path': None,
                     'error': error_msg}
         except FileOverSizeException:
             self.request.response.status = 400
             limit_mb = h.format_byte_size_binary(max_file_size)
+            error_msg = f'File {filename} is exceeding allowed limit of {limit_mb}.'
             return {'store_fid': None,
                     'access_path': None,
-                    'error': 'File {} is exceeding allowed limit of {}.'.format(
-                        filename, limit_mb)}
+                    'error': error_msg}
 
         try:
             entry = FileStore.create(

rhodecode/apps/repository/views/repo_files.py

@@ -48,7 +48,7 @@ from rhodecode.lib.codeblocks import (
     filenode_as_lines_tokens, filenode_as_annotated_lines_tokens)
 from rhodecode.lib.utils2 import convert_line_endings, detect_mode
 from rhodecode.lib.type_utils import str2bool
-from rhodecode.lib.str_utils import safe_str, safe_int
+from rhodecode.lib.str_utils import safe_str, safe_int, header_safe_str
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
 from rhodecode.lib.vcs import path as vcspath
@@ -820,7 +820,7 @@ class RepoFilesView(RepoAppView):
                   "filename=\"{}\"; " \
                   "filename*=UTF-8\'\'{}".format(safe_path, encoded_path)
 
-        return safe_bytes(headers).decode('latin-1', errors='replace')
+        return header_safe_str(headers)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(

rhodecode/apps/repository/views/repo_settings_advanced.py

@@ -29,7 +29,7 @@ from rhodecode.lib import audit_logger
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired,
     HasRepoPermissionAny)
-from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError
+from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.lib.vcs import RepositoryError
 from rhodecode.model.db import Session, UserFollowing, User, Repository
@@ -136,6 +136,9 @@ class RepoSettingsAdvancedView(RepoAppView):
         elif handle_forks == 'delete_forks':
             handle_forks = 'delete'
 
+        repo_advanced_url = h.route_path(
+            'edit_repo_advanced', repo_name=self.db_repo_name,
+            _anchor='advanced-delete')
         try:
             old_data = self.db_repo.get_api_data()
             RepoModel().delete(self.db_repo, forks=handle_forks)
@@ -158,9 +161,6 @@ class RepoSettingsAdvancedView(RepoAppView):
                 category='success')
             Session().commit()
         except AttachedForksError:
-            repo_advanced_url = h.route_path(
-                'edit_repo_advanced', repo_name=self.db_repo_name,
-                _anchor='advanced-delete')
             delete_anchor = h.link_to(_('detach or delete'), repo_advanced_url)
             h.flash(_('Cannot delete `{repo}` it still contains attached forks. '
                       'Try using {delete_or_detach} option.')
@@ -171,9 +171,6 @@ class RepoSettingsAdvancedView(RepoAppView):
             raise HTTPFound(repo_advanced_url)
 
         except AttachedPullRequestsError:
-            repo_advanced_url = h.route_path(
-                'edit_repo_advanced', repo_name=self.db_repo_name,
-                _anchor='advanced-delete')
             attached_prs = len(self.db_repo.pull_requests_source +
                                self.db_repo.pull_requests_target)
             h.flash(
@@ -184,6 +181,16 @@ class RepoSettingsAdvancedView(RepoAppView):
             # redirect to advanced for forks handle action ?
             raise HTTPFound(repo_advanced_url)
 
+        except AttachedArtifactsError:
+
+            attached_artifacts = len(self.db_repo.artifacts)
+            h.flash(
+                _('Cannot delete `{repo}` it still contains {num} attached artifacts. '
+                  'Consider archiving the repository instead.').format(
+                    repo=self.db_repo_name, num=attached_artifacts), category='warning')
+
+            # redirect to advanced for forks handle action ?
+            raise HTTPFound(repo_advanced_url)
         except Exception:
             log.exception("Exception during deletion of repository")
             h.flash(_('An error occurred during deletion of `%s`')

rhodecode/config/config_maker.py

@@ -206,7 +206,7 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
     settings_maker.make_setting('archive_cache.filesystem.retry_backoff', 1, parser='int')
     settings_maker.make_setting('archive_cache.filesystem.retry_attempts', 10, parser='int')
 
-    settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
+    settings_maker.make_setting('archive_cache.objectstore.url', 'http://s3-minio:9000', default_when_empty=True,)
     settings_maker.make_setting('archive_cache.objectstore.key', '')
     settings_maker.make_setting('archive_cache.objectstore.secret', '')
     settings_maker.make_setting('archive_cache.objectstore.region', 'eu-central-1')

rhodecode/config/environment.py

@@ -16,7 +16,6 @@
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-import os
 import logging
 import rhodecode
 import collections
@@ -30,6 +29,21 @@ from rhodecode.lib.vcs import connect_vcs
 log = logging.getLogger(__name__)
 
 
+def propagate_rhodecode_config(global_config, settings, config):
+    # Store the settings to make them available to other modules.
+    settings_merged = global_config.copy()
+    settings_merged.update(settings)
+    if config:
+        settings_merged.update(config)
+
+    rhodecode.PYRAMID_SETTINGS = settings_merged
+    rhodecode.CONFIG = settings_merged
+
+    if 'default_user_id' not in rhodecode.CONFIG:
+        rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
+    log.debug('set rhodecode.CONFIG data')
+
+
 def load_pyramid_environment(global_config, settings):
     # Some parts of the code expect a merge of global and app settings.
     settings_merged = global_config.copy()
@@ -75,11 +89,8 @@ def load_pyramid_environment(global_config, settings):
 
     utils.configure_vcs(settings)
 
-    # Store the settings to make them available to other modules.
-
-    rhodecode.PYRAMID_SETTINGS = settings_merged
-    rhodecode.CONFIG = settings_merged
-    rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
+    # first run, to store data...
+    propagate_rhodecode_config(global_config, settings, {})
 
     if vcs_server_enabled:
         connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))

rhodecode/config/middleware.py

@@ -115,6 +115,9 @@ def make_pyramid_app(global_config, **settings):
     celery_settings = get_celery_config(settings)
     config.configure_celery(celery_settings)
 
+    # final config set...
+    propagate_rhodecode_config(global_config, settings, config.registry.settings)
+
     # creating the app uses a connection - return it after we are done
     meta.Session.remove()
 

rhodecode/lib/exceptions.py

@@ -80,6 +80,10 @@ class AttachedPullRequestsError(Exception):
     pass
 
 
+class AttachedArtifactsError(Exception):
+    pass
+
+
 class RepoGroupAssignmentError(Exception):
     pass
 

rhodecode/lib/helpers.py

@@ -81,7 +81,7 @@ from rhodecode.lib.action_parser import action_parser
 from rhodecode.lib.html_filters import sanitize_html
 from rhodecode.lib.pagination import Page, RepoPage, SqlPage
 from rhodecode.lib import ext_json
-from rhodecode.lib.ext_json import json
+from rhodecode.lib.ext_json import json, formatted_str_json
 from rhodecode.lib.str_utils import safe_bytes, convert_special_chars, base64_to_str
 from rhodecode.lib.utils import repo_name_slug, get_custom_lexer
 from rhodecode.lib.str_utils import safe_str
@@ -1416,62 +1416,14 @@ class InitialsGravatar(object):
         return "data:image/svg+xml;base64,{}".format(img_data)
 
 
-def initials_gravatar(request, email_address, first_name, last_name, size=30, store_on_disk=False):
+def initials_gravatar(request, email_address, first_name, last_name, size=30):
 
     svg_type = None
     if email_address == User.DEFAULT_USER_EMAIL:
         svg_type = 'default_user'
 
     klass = InitialsGravatar(email_address, first_name, last_name, size)
-
-    if store_on_disk:
-        from rhodecode.apps.file_store import utils as store_utils
-        from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
-            FileOverSizeException
-        from rhodecode.model.db import Session
-
-        image_key = md5_safe(email_address.lower()
-                             + first_name.lower() + last_name.lower())
-
-        storage = store_utils.get_file_storage(request.registry.settings)
-        filename = '{}.svg'.format(image_key)
-        subdir = 'gravatars'
-        # since final name has a counter, we apply the 0
-        uid = storage.apply_counter(0, store_utils.uid_filename(filename, randomized=False))
-        store_uid = os.path.join(subdir, uid)
-
-        db_entry = FileStore.get_by_store_uid(store_uid)
-        if db_entry:
-            return request.route_path('download_file', fid=store_uid)
-
-        img_data = klass.get_img_data(svg_type=svg_type)
-        img_file = store_utils.bytes_to_file_obj(img_data)
-
-        try:
-            store_uid, metadata = storage.save_file(
-                img_file, filename, directory=subdir,
-                extensions=['.svg'], randomized_name=False)
-        except (FileNotAllowedException, FileOverSizeException):
-            raise
-
-        try:
-            entry = FileStore.create(
-                file_uid=store_uid, filename=metadata["filename"],
-                file_hash=metadata["sha256"], file_size=metadata["size"],
-                file_display_name=filename,
-                file_description=f'user gravatar `{safe_str(filename)}`',
-                hidden=True, check_acl=False, user_id=1
-            )
-            Session().add(entry)
-            Session().commit()
-            log.debug('Stored upload in DB as %s', entry)
-        except Exception:
-            raise
-
-        return request.route_path('download_file', fid=store_uid)
-
-    else:
-        return klass.generate_svg(svg_type=svg_type)
+    return klass.generate_svg(svg_type=svg_type)
 
 
 def gravatar_external(request, gravatar_url_tmpl, email_address, size=30):

rhodecode/lib/rc_commands/add_artifact.py

@@ -91,15 +91,14 @@ def command(ini_path, filename, file_path, repo_id, user_id, description):
 
         auth_user = db_user.AuthUser(ip_addr='127.0.0.1')
 
-        storage = store_utils.get_file_storage(request.registry.settings)
+        f_store = store_utils.get_filestore_backend(request.registry.settings)
 
         with open(file_path, 'rb') as f:
             click.secho(f'Adding new artifact from path: `{file_path}`',
                         fg='green')
 
             file_data = _store_file(
-                storage, auth_user, filename, content=None, check_acl=True,
+                f_store, auth_user, filename, content=None, check_acl=True,
                 file_obj=f, description=description,
                 scope_repo_id=repo.repo_id)
-            click.secho(f'File Data: {file_data}',
-                        fg='green')
+            click.secho(f'File Data: {file_data}', fg='green')

rhodecode/lib/rc_commands/migrate_artifact.py

@@ -0,0 +1,122 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import sys
+import logging
+
+import click
+
+from rhodecode.lib.pyramid_utils import bootstrap
+from rhodecode.lib.ext_json import json
+from rhodecode.model.db import FileStore
+from rhodecode.apps.file_store import utils as store_utils
+
+log = logging.getLogger(__name__)
+
+
+@click.command()
+@click.argument('ini_path', type=click.Path(exists=True))
+@click.argument('file_uid')
+@click.option(
+    '--source-backend-conf',
+    type=click.Path(exists=True, dir_okay=False, readable=True),
+    help='Source backend config file path in a json format'
+)
+@click.option(
+    '--dest-backend-conf',
+    type=click.Path(exists=True, dir_okay=False, readable=True),
+    help='Source backend config file path in a json format'
+)
+def main(ini_path, file_uid, source_backend_conf, dest_backend_conf):
+    return command(ini_path, file_uid, source_backend_conf, dest_backend_conf)
+
+
+_source_settings = {}
+
+_dest_settings = {}
+
+
+def command(ini_path, file_uid, source_backend_conf, dest_backend_conf):
+    with bootstrap(ini_path, env={'RC_CMD_SETUP_RC': '1'}) as env:
+        migrate_func(env, file_uid, source_backend_conf, dest_backend_conf)
+
+
+def migrate_func(env, file_uid, source_backend_conf=None, dest_backend_conf=None):
+    """
+
+    Example usage::
+
+        from rhodecode.lib.rc_commands import migrate_artifact
+        migrate_artifact._source_settings = {
+            'file_store.backend.type': 'filesystem_v1',
+            'file_store.filesystem_v1.storage_path': '/var/opt/rhodecode_data/file_store',
+        }
+        migrate_artifact._dest_settings = {
+            'file_store.backend.type': 'objectstore',
+            'file_store.objectstore.url': 'http://s3-minio:9000',
+            'file_store.objectstore.bucket': 'rhodecode-file-store',
+            'file_store.objectstore.key': 's3admin',
+            'file_store.objectstore.secret': 's3secret4',
+            'file_store.objectstore.region': 'eu-central-1',
+        }
+        for db_obj in FileStore.query().all():
+            migrate_artifact.migrate_func({}, db_obj.file_uid)
+
+    """
+
+    try:
+        from rc_ee.api.views.store_api import _store_file
+    except ImportError:
+        click.secho('ERROR: Unable to import store_api. '
+                    'store_api is only available in EE edition of RhodeCode',
+                    fg='red')
+        sys.exit(-1)
+
+    source_settings = _source_settings
+    if source_backend_conf:
+        source_settings = json.loads(open(source_backend_conf).read())
+    dest_settings = _dest_settings
+    if dest_backend_conf:
+        dest_settings = json.loads(open(dest_backend_conf).read())
+
+    if file_uid.isnumeric():
+        file_store_db_obj = FileStore().query() \
+            .filter(FileStore.file_store_id == file_uid) \
+            .scalar()
+    else:
+        file_store_db_obj = FileStore().query() \
+            .filter(FileStore.file_uid == file_uid) \
+            .scalar()
+    if not file_store_db_obj:
+        click.secho(f'ERROR: Unable to fetch artifact from database file_uid={file_uid}',
+                    fg='red')
+        sys.exit(-1)
+
+    uid_filename = file_store_db_obj.file_uid
+    org_filename = file_store_db_obj.file_display_name
+    click.secho(f'Attempting to migrate artifact {uid_filename}, filename: {org_filename}', fg='green')
+
+    # get old version of f_store based on the data.
+
+    origin_f_store = store_utils.get_filestore_backend(source_settings, always_init=True)
+    reader, metadata = origin_f_store.fetch(uid_filename)
+
+    target_f_store = store_utils.get_filestore_backend(dest_settings, always_init=True)
+    target_f_store.import_to_store(reader, org_filename, uid_filename, metadata)
+
+    click.secho(f'Migrated artifact {uid_filename}, filename: {org_filename} into {target_f_store} storage', fg='green')

rhodecode/lib/str_utils.py

@@ -181,3 +181,7 @@ def splitnewlines(text: bytes):
         else:
             lines[-1] = lines[-1][:-1]
     return lines
+
+
+def header_safe_str(val):
+    return safe_bytes(val).decode('latin-1', errors='replace')

rhodecode/lib/system_info.py

@@ -396,17 +396,18 @@ def storage_inodes():
 
 
 @register_sysinfo
-def storage_archives():
+def storage_artifacts():
     import rhodecode
     from rhodecode.lib.helpers import format_byte_size_binary
     from rhodecode.lib.archive_cache import get_archival_cache_store
 
-    storage_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
+    backend_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
 
-    value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=storage_type)
+    value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type)
     state = STATE_OK_DEFAULT
     try:
         d_cache = get_archival_cache_store(config=rhodecode.CONFIG)
+        backend_type = str(d_cache)
 
         total_files, total_size, _directory_stats = d_cache.get_statistics()
 
@@ -415,7 +416,8 @@ def storage_archives():
             'used': total_size,
             'total': total_size,
             'items': total_files,
-            'path': d_cache.storage_path
+            'path': d_cache.storage_path,
+            'type': backend_type
         })
 
     except Exception as e:
@@ -425,8 +427,44 @@ def storage_archives():
     human_value = value.copy()
     human_value['used'] = format_byte_size_binary(value['used'])
     human_value['total'] = format_byte_size_binary(value['total'])
-    human_value['text'] = "{} ({} items)".format(
-        human_value['used'], value['items'])
+    human_value['text'] = f"{human_value['used']} ({value['items']} items)"
+
+    return SysInfoRes(value=value, state=state, human_value=human_value)
+
+
+@register_sysinfo
+def storage_archives():
+    import rhodecode
+    from rhodecode.lib.helpers import format_byte_size_binary
+    import rhodecode.apps.file_store.utils as store_utils
+    from rhodecode import CONFIG
+
+    backend_type = rhodecode.ConfigGet().get_str(store_utils.config_keys.backend_type)
+
+    value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type)
+    state = STATE_OK_DEFAULT
+    try:
+        f_store = store_utils.get_filestore_backend(config=CONFIG)
+        backend_type = str(f_store)
+        total_files, total_size, _directory_stats = f_store.get_statistics()
+
+        value.update({
+            'percent': 100,
+            'used': total_size,
+            'total': total_size,
+            'items': total_files,
+            'path': f_store.storage_path,
+            'type': backend_type
+        })
+
+    except Exception as e:
+        log.exception('failed to fetch archive cache storage')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = value.copy()
+    human_value['used'] = format_byte_size_binary(value['used'])
+    human_value['total'] = format_byte_size_binary(value['total'])
+    human_value['text'] = f"{human_value['used']} ({value['items']} items)"
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
@@ -798,6 +836,7 @@ def get_system_info(environ):
         'storage': SysInfo(storage)(),
         'storage_inodes': SysInfo(storage_inodes)(),
         'storage_archive': SysInfo(storage_archives)(),
+        'storage_artifacts': SysInfo(storage_artifacts)(),
         'storage_gist': SysInfo(storage_gist)(),
         'storage_temp': SysInfo(storage_temp)(),
 

rhodecode/model/db.py

@@ -5849,8 +5849,7 @@ class FileStore(Base, BaseModel):
                 .filter(FileStoreMetadata.file_store_meta_key == key) \
                 .scalar()
             if has_key:
-                msg = 'key `{}` already defined under section `{}` for this file.'\
-                    .format(key, section)
+                msg = f'key `{key}` already defined under section `{section}` for this file.'
                 raise ArtifactMetadataDuplicate(msg, err_section=section, err_key=key)
 
             # NOTE(marcink): raises ArtifactMetadataBadValueType
@@ -5949,7 +5948,7 @@ class FileStoreMetadata(Base, BaseModel):
     def valid_value_type(cls, value):
         if value.split('.')[0] not in cls.SETTINGS_TYPES:
             raise ArtifactMetadataBadValueType(
-                'value_type must be one of %s got %s' % (cls.SETTINGS_TYPES.keys(), value))
+                f'value_type must be one of {cls.SETTINGS_TYPES.keys()} got {value}')
 
     @hybrid_property
     def file_store_meta_section(self):

rhodecode/model/repo.py

@@ -31,7 +31,7 @@ from zope.cachedescriptors.property import Lazy as LazyProperty
 from rhodecode import events
 from rhodecode.lib.auth import HasUserGroupPermissionAny
 from rhodecode.lib.caching_query import FromCache
-from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError
+from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError
 from rhodecode.lib import hooks_base
 from rhodecode.lib.user_log_filter import user_log_filter
 from rhodecode.lib.utils import make_db_config
@@ -736,7 +736,7 @@ class RepoModel(BaseModel):
                 log.error(traceback.format_exc())
                 raise
 
-    def delete(self, repo, forks=None, pull_requests=None, fs_remove=True, cur_user=None):
+    def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None):
         """
         Delete given repository, forks parameter defines what do do with
         attached forks. Throws AttachedForksError if deleted repo has attached
@@ -745,6 +745,7 @@ class RepoModel(BaseModel):
         :param repo:
         :param forks: str 'delete' or 'detach'
         :param pull_requests: str 'delete' or None
+        :param artifacts: str 'delete' or None
         :param fs_remove: remove(archive) repo from filesystem
         """
         if not cur_user:
@@ -767,6 +768,13 @@ class RepoModel(BaseModel):
             if pull_requests != 'delete' and (pr_sources or pr_targets):
                 raise AttachedPullRequestsError()
 
+            artifacts_objs = repo.artifacts
+            if artifacts == 'delete':
+                for a in artifacts_objs:
+                    self.sa.delete(a)
+            elif [a for a in artifacts_objs]:
+                raise AttachedArtifactsError()
+
             old_repo_dict = repo.get_dict()
             events.trigger(events.RepoPreDeleteEvent(repo))
             try:

rhodecode/model/user.py

@@ -557,10 +557,10 @@ class UserModel(BaseModel):
         elif handle_mode == 'delete':
             from rhodecode.apps.file_store import utils as store_utils
             request = get_current_request()
-            storage = store_utils.get_file_storage(request.registry.settings)
+            f_store = store_utils.get_filestore_backend(request.registry.settings)
             for a in artifacts:
                 file_uid = a.file_uid
-                storage.delete(file_uid)
+                f_store.delete(file_uid)
                 self.sa.delete(a)
 
             left_overs = False

rhodecode/templates/admin/repos/repo_edit_advanced.mako

@@ -215,18 +215,35 @@
                     %endif
                 </td>
             </tr>
+
             <% attached_prs = len(c.rhodecode_db_repo.pull_requests_source + c.rhodecode_db_repo.pull_requests_target) %>
             % if c.rhodecode_db_repo.pull_requests_source or c.rhodecode_db_repo.pull_requests_target:
             <tr>
                 <td>
                     ${_ungettext('This repository has %s attached pull request.', 'This repository has %s attached pull requests.', attached_prs) % attached_prs}
                     <br/>
-                    ${_('Consider to archive this repository instead.')}
+                    <br/>
+                    <strong>${_('Consider to archive this repository instead.')}</strong>
                 </td>
                 <td></td>
                 <td></td>
             </tr>
             % endif
+
+            <% attached_artifacts = len(c.rhodecode_db_repo.artifacts) %>
+            % if attached_artifacts:
+            <tr>
+                <td>
+                    ${_ungettext('This repository has %s attached artifact.', 'This repository has %s attached artifacts.', attached_artifacts) % attached_artifacts}
+                    <br/>
+                    <br/>
+                    <strong>${_('Consider to archive this repository instead.')}</strong>
+                </td>
+                <td></td>
+                <td></td>
+            </tr>
+            % endif
+
         </table>
         <div style="margin: 0 0 20px 0" class="fake-space"></div>
 

rhodecode/tests/fixture.py

@@ -305,7 +305,7 @@ class Fixture(object):
         return r
 
     def destroy_repo(self, repo_name, **kwargs):
-        RepoModel().delete(repo_name, pull_requests='delete', **kwargs)
+        RepoModel().delete(repo_name, pull_requests='delete', artifacts='delete', **kwargs)
         Session().commit()
 
     def destroy_repo_on_filesystem(self, repo_name):

rhodecode/tests/rhodecode.ini

@@ -36,7 +36,7 @@ port = 10020
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
+; run with gunicorn  --config gunicorn_conf.py --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
@@ -249,15 +249,56 @@ labs_settings_active = true
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
-; File store configuration. This is used to store and serve uploaded files
-file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store backend is always enabled
+#file_store.enabled = true
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
 ; Storage backend, available options are: local
-file_store.backend = local
+#file_store.backend = local
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
 ; path to store the uploaded binaries and artifacts
-file_store.storage_path = /var/opt/rhodecode_data/file_store
+#file_store.storage_path = /var/opt/rhodecode_data/file_store
 
+; Artifacts file-store, is used to store comment attachments and artifacts uploads.
+; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
+; filesystem_v1 is backwards compat with pre 5.1 storage changes
+; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
+; previous installations to keep the artifacts without a need of migration
+file_store.backend.type = filesystem_v1
+
+; filesystem options...
+file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/test_artifacts_file_store
+
+; filesystem_v2 options...
+file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/test_artifacts_file_store_2
+file_store.filesystem_v2.shards = 8
+
+; objectstore options...
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+#file_store.backend.type = objectstore
+file_store.objectstore.url = http://s3-minio:9000
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
+file_store.objectstore.bucket = rhodecode-file-store-tests
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+file_store.objectstore.bucket_shards = 8
+
+; key for s3 auth
+file_store.objectstore.key = s3admin
+
+; secret for s3 auth
+file_store.objectstore.secret = s3secret4
+
+;region for s3 storage
+file_store.objectstore.region = eu-central-1
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1
@@ -593,6 +634,7 @@ vcs.scm_app_implementation = http
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 ; `celery` - use celery based hooks
+#DEPRECATED:vcs.hooks.protocol = http
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
@@ -626,6 +668,10 @@ vcs.methods.cache = false
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
+; Redis connection settings for svn integrations logic
+; This connection string needs to be the same on ce and vcsserver
+vcs.svn.redis_conn = redis://redis:6379/0
+
 ; Enable SVN proxy of requests over HTTP
 vcs.svn.proxy.enabled = true
 
@@ -681,7 +727,8 @@ ssh.authorized_keys_file_path = %(here)s/rc-tests/authorized_keys_rhodecode
 ; RhodeCode installation directory.
 ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
-ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false

setup.py

@@ -189,6 +189,7 @@ setup(
             'rc-upgrade-db=rhodecode.lib.rc_commands.upgrade_db:main',
             'rc-ishell=rhodecode.lib.rc_commands.ishell:main',
             'rc-add-artifact=rhodecode.lib.rc_commands.add_artifact:main',
+            'rc-migrate-artifact=rhodecode.lib.rc_commands.migrate_artifact:main',
             'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper_v1:main',
             'rc-ssh-wrapper-v2=rhodecode.apps.ssh_support.lib.ssh_wrapper_v2:main',
         ],