From 5ec57ac2bcced8279782c7e30ced4f1fe956777a Mon Sep 17 00:00:00 2001 From: Marcin Kuzminski Date: Mon, 24 Aug 2020 15:01:08 +0200 Subject: [PATCH] artifacts: few fixes for handling cases of using sub path to store artifacts. - fixed some bugs when using the sub path artifacts and generataed sub-directory store --- .../nginx/nginx-config-example.rst | 5 +- rhodecode/apps/file_store/__init__.py | 4 +- .../apps/file_store/backends/local_store.py | 55 +++++++++++++------ rhodecode/apps/file_store/utils.py | 6 +- rhodecode/apps/file_store/views.py | 2 +- rhodecode/model/db.py | 7 ++- 6 files changed, 54 insertions(+), 25 deletions(-) diff --git a/docs/admin/system_admin/nginx/nginx-config-example.rst b/docs/admin/system_admin/nginx/nginx-config-example.rst index ad98f305..9f5630e9 100644 --- a/docs/admin/system_admin/nginx/nginx-config-example.rst +++ b/docs/admin/system_admin/nginx/nginx-config-example.rst @@ -147,12 +147,13 @@ Use the following example to configure Nginx as a your web server. ## Special Cache for file store, make sure you enable this intentionally as ## it could bypass upload files permissions - # location /_file_store/download { + # location /_file_store/download/gravatars { # # proxy_cache cache_zone; # # ignore Set-Cookie # proxy_ignore_headers Set-Cookie; - # proxy_ignore_headers Cookie; + # # ignore cache-control + # proxy_ignore_headers Cache-Control; # # proxy_cache_key $host$uri$is_args$args; # proxy_cache_methods GET; diff --git a/rhodecode/apps/file_store/__init__.py b/rhodecode/apps/file_store/__init__.py index 05f36467..ab10bff0 100755 --- a/rhodecode/apps/file_store/__init__.py +++ b/rhodecode/apps/file_store/__init__.py @@ -43,10 +43,10 @@ def includeme(config): pattern='/_file_store/upload') config.add_route( name='download_file', - pattern='/_file_store/download/{fid}') + pattern='/_file_store/download/{fid:.*}') config.add_route( name='download_file_by_token', - pattern='/_file_store/token-download/{_auth_token}/{fid}') + pattern='/_file_store/token-download/{_auth_token}/{fid:.*}') # Scan module for configuration decorators. config.scan('.views', ignore='.tests') diff --git a/rhodecode/apps/file_store/backends/local_store.py b/rhodecode/apps/file_store/backends/local_store.py index 410d13e0..a0262aab 100755 --- a/rhodecode/apps/file_store/backends/local_store.py +++ b/rhodecode/apps/file_store/backends/local_store.py @@ -20,6 +20,7 @@ import os import time +import errno import shutil import hashlib @@ -32,8 +33,23 @@ from rhodecode.apps.file_store.exceptions import ( METADATA_VER = 'v1' +def safe_make_dirs(dir_path): + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + return + + class LocalFileStorage(object): + @classmethod + def apply_counter(cls, counter, filename): + name_counted = '%d-%s' % (counter, filename) + return name_counted + @classmethod def resolve_name(cls, name, directory): """ @@ -47,17 +63,16 @@ class LocalFileStorage(object): counter = 0 while True: - name = '%d-%s' % (counter, name) + name_counted = cls.apply_counter(counter, name) # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file - sub_store = cls._sub_store_from_filename(name) + sub_store = cls._sub_store_from_filename(name_counted) sub_store_path = os.path.join(directory, sub_store) - if not os.path.exists(sub_store_path): - os.makedirs(sub_store_path) + safe_make_dirs(sub_store_path) - path = os.path.join(sub_store_path, name) + path = os.path.join(sub_store_path, name_counted) if not os.path.exists(path): - return name, path + return name_counted, path counter += 1 @classmethod @@ -102,8 +117,13 @@ class LocalFileStorage(object): :param filename: base name of file """ - sub_store = self._sub_store_from_filename(filename) - return os.path.join(self.base_path, sub_store, filename) + prefix_dir = '' + if '/' in filename: + prefix_dir, filename = filename.split('/') + sub_store = self._sub_store_from_filename(filename) + else: + sub_store = self._sub_store_from_filename(filename) + return os.path.join(self.base_path, prefix_dir, sub_store, filename) def delete(self, filename): """ @@ -123,7 +143,7 @@ class LocalFileStorage(object): Checks if file exists. Resolves filename's absolute path based on base_path. - :param filename: base name of file + :param filename: file_uid name of file, e.g 0-f62b2b2d-9708-4079-a071-ec3f958448d4.svg """ return os.path.exists(self.store_path(filename)) @@ -158,7 +178,7 @@ class LocalFileStorage(object): return ext in [normalize_ext(x) for x in extensions] def save_file(self, file_obj, filename, directory=None, extensions=None, - extra_metadata=None, max_filesize=None, **kwargs): + extra_metadata=None, max_filesize=None, randomized_name=True, **kwargs): """ Saves a file object to the uploads location. Returns the resolved filename, i.e. the directory + @@ -169,6 +189,7 @@ class LocalFileStorage(object): :param directory: relative path of sub-directory :param extensions: iterable of allowed extensions, if not default :param max_filesize: maximum size of file that should be allowed + :param randomized_name: generate random generated UID or fixed based on the filename :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix """ @@ -183,13 +204,12 @@ class LocalFileStorage(object): else: dest_directory = self.base_path - if not os.path.exists(dest_directory): - os.makedirs(dest_directory) + safe_make_dirs(dest_directory) - filename = utils.uid_filename(filename) + uid_filename = utils.uid_filename(filename, randomized=randomized_name) # resolve also produces special sub-dir for file optimized store - filename, path = self.resolve_name(filename, dest_directory) + filename, path = self.resolve_name(uid_filename, dest_directory) stored_file_dir = os.path.dirname(path) file_obj.seek(0) @@ -210,12 +230,13 @@ class LocalFileStorage(object): file_hash = self.calculate_path_hash(path) - metadata.update( - {"filename": filename, + metadata.update({ + "filename": filename, "size": size, "time": time.time(), "sha256": file_hash, - "meta_ver": METADATA_VER}) + "meta_ver": METADATA_VER + }) filename_meta = filename + '.meta' with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta: diff --git a/rhodecode/apps/file_store/utils.py b/rhodecode/apps/file_store/utils.py index d20f2009..6b9e3740 100755 --- a/rhodecode/apps/file_store/utils.py +++ b/rhodecode/apps/file_store/utils.py @@ -20,7 +20,7 @@ import uuid - +import StringIO import pathlib2 @@ -52,3 +52,7 @@ def uid_filename(filename, randomized=True): hash_key = '{}.{}'.format(filename, 'store') uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key) return str(uid) + ext.lower() + + +def bytes_to_file_obj(bytes_data): + return StringIO.StringIO(bytes_data) diff --git a/rhodecode/apps/file_store/views.py b/rhodecode/apps/file_store/views.py index 65a013e8..cfd1d503 100644 --- a/rhodecode/apps/file_store/views.py +++ b/rhodecode/apps/file_store/views.py @@ -64,7 +64,7 @@ class FileStoreView(BaseAppView): file_uid, store_path) raise HTTPNotFound() - db_obj = FileStore().query().filter(FileStore.file_uid == file_uid).scalar() + db_obj = FileStore.get_by_store_uid(file_uid, safe=True) if not db_obj: raise HTTPNotFound() diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py index aa061263..577910e8 100644 --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -5442,8 +5442,11 @@ class FileStore(Base, BaseModel): repo_group = relationship('RepoGroup', lazy='joined') @classmethod - def get_by_store_uid(cls, file_store_uid): - return FileStore.query().filter(FileStore.file_uid == file_store_uid).scalar() + def get_by_store_uid(cls, file_store_uid, safe=False): + if safe: + return FileStore.query().filter(FileStore.file_uid == file_store_uid).first() + else: + return FileStore.query().filter(FileStore.file_uid == file_store_uid).scalar() @classmethod def create(cls, file_uid, filename, file_hash, file_size, file_display_name='',