artifacts: few fixes for handling cases of using sub path to store artifacts.
- fixed some bugs when using the sub path artifacts and generataed sub-directory store
This commit is contained in:
Marcin Kuzminski
parent
a849ac31f3
commit
5ec57ac2bc
6 changed files with 54 additions and 25 deletions
|
|
@ -147,12 +147,13 @@ Use the following example to configure Nginx as a your web server.
|
|||
|
||||
## Special Cache for file store, make sure you enable this intentionally as
|
||||
## it could bypass upload files permissions
|
||||
# location /_file_store/download {
|
||||
# location /_file_store/download/gravatars {
|
||||
#
|
||||
# proxy_cache cache_zone;
|
||||
# # ignore Set-Cookie
|
||||
# proxy_ignore_headers Set-Cookie;
|
||||
# proxy_ignore_headers Cookie;
|
||||
# # ignore cache-control
|
||||
# proxy_ignore_headers Cache-Control;
|
||||
#
|
||||
# proxy_cache_key $host$uri$is_args$args;
|
||||
# proxy_cache_methods GET;
|
||||
|
|
|
|||
|
|
@ -43,10 +43,10 @@ def includeme(config):
|
|||
pattern='/_file_store/upload')
|
||||
config.add_route(
|
||||
name='download_file',
|
||||
pattern='/_file_store/download/{fid}')
|
||||
pattern='/_file_store/download/{fid:.*}')
|
||||
config.add_route(
|
||||
name='download_file_by_token',
|
||||
pattern='/_file_store/token-download/{_auth_token}/{fid}')
|
||||
pattern='/_file_store/token-download/{_auth_token}/{fid:.*}')
|
||||
|
||||
# Scan module for configuration decorators.
|
||||
config.scan('.views', ignore='.tests')
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
|
||||
import os
|
||||
import time
|
||||
import errno
|
||||
import shutil
|
||||
import hashlib
|
||||
|
||||
|
|
@ -32,8 +33,23 @@ from rhodecode.apps.file_store.exceptions import (
|
|||
METADATA_VER = 'v1'
|
||||
|
||||
|
||||
def safe_make_dirs(dir_path):
|
||||
if not os.path.exists(dir_path):
|
||||
try:
|
||||
os.makedirs(dir_path)
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
return
|
||||
|
||||
|
||||
class LocalFileStorage(object):
|
||||
|
||||
@classmethod
|
||||
def apply_counter(cls, counter, filename):
|
||||
name_counted = '%d-%s' % (counter, filename)
|
||||
return name_counted
|
||||
|
||||
@classmethod
|
||||
def resolve_name(cls, name, directory):
|
||||
"""
|
||||
|
|
@ -47,17 +63,16 @@ class LocalFileStorage(object):
|
|||
|
||||
counter = 0
|
||||
while True:
|
||||
name = '%d-%s' % (counter, name)
|
||||
name_counted = cls.apply_counter(counter, name)
|
||||
|
||||
# sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
|
||||
sub_store = cls._sub_store_from_filename(name)
|
||||
sub_store = cls._sub_store_from_filename(name_counted)
|
||||
sub_store_path = os.path.join(directory, sub_store)
|
||||
if not os.path.exists(sub_store_path):
|
||||
os.makedirs(sub_store_path)
|
||||
safe_make_dirs(sub_store_path)
|
||||
|
||||
path = os.path.join(sub_store_path, name)
|
||||
path = os.path.join(sub_store_path, name_counted)
|
||||
if not os.path.exists(path):
|
||||
return name, path
|
||||
return name_counted, path
|
||||
counter += 1
|
||||
|
||||
@classmethod
|
||||
|
|
@ -102,8 +117,13 @@ class LocalFileStorage(object):
|
|||
|
||||
:param filename: base name of file
|
||||
"""
|
||||
sub_store = self._sub_store_from_filename(filename)
|
||||
return os.path.join(self.base_path, sub_store, filename)
|
||||
prefix_dir = ''
|
||||
if '/' in filename:
|
||||
prefix_dir, filename = filename.split('/')
|
||||
sub_store = self._sub_store_from_filename(filename)
|
||||
else:
|
||||
sub_store = self._sub_store_from_filename(filename)
|
||||
return os.path.join(self.base_path, prefix_dir, sub_store, filename)
|
||||
|
||||
def delete(self, filename):
|
||||
"""
|
||||
|
|
@ -123,7 +143,7 @@ class LocalFileStorage(object):
|
|||
Checks if file exists. Resolves filename's absolute
|
||||
path based on base_path.
|
||||
|
||||
:param filename: base name of file
|
||||
:param filename: file_uid name of file, e.g 0-f62b2b2d-9708-4079-a071-ec3f958448d4.svg
|
||||
"""
|
||||
return os.path.exists(self.store_path(filename))
|
||||
|
||||
|
|
@ -158,7 +178,7 @@ class LocalFileStorage(object):
|
|||
return ext in [normalize_ext(x) for x in extensions]
|
||||
|
||||
def save_file(self, file_obj, filename, directory=None, extensions=None,
|
||||
extra_metadata=None, max_filesize=None, **kwargs):
|
||||
extra_metadata=None, max_filesize=None, randomized_name=True, **kwargs):
|
||||
"""
|
||||
Saves a file object to the uploads location.
|
||||
Returns the resolved filename, i.e. the directory +
|
||||
|
|
@ -169,6 +189,7 @@ class LocalFileStorage(object):
|
|||
:param directory: relative path of sub-directory
|
||||
:param extensions: iterable of allowed extensions, if not default
|
||||
:param max_filesize: maximum size of file that should be allowed
|
||||
:param randomized_name: generate random generated UID or fixed based on the filename
|
||||
:param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
|
||||
|
||||
"""
|
||||
|
|
@ -183,13 +204,12 @@ class LocalFileStorage(object):
|
|||
else:
|
||||
dest_directory = self.base_path
|
||||
|
||||
if not os.path.exists(dest_directory):
|
||||
os.makedirs(dest_directory)
|
||||
safe_make_dirs(dest_directory)
|
||||
|
||||
filename = utils.uid_filename(filename)
|
||||
uid_filename = utils.uid_filename(filename, randomized=randomized_name)
|
||||
|
||||
# resolve also produces special sub-dir for file optimized store
|
||||
filename, path = self.resolve_name(filename, dest_directory)
|
||||
filename, path = self.resolve_name(uid_filename, dest_directory)
|
||||
stored_file_dir = os.path.dirname(path)
|
||||
|
||||
file_obj.seek(0)
|
||||
|
|
@ -210,12 +230,13 @@ class LocalFileStorage(object):
|
|||
|
||||
file_hash = self.calculate_path_hash(path)
|
||||
|
||||
metadata.update(
|
||||
{"filename": filename,
|
||||
metadata.update({
|
||||
"filename": filename,
|
||||
"size": size,
|
||||
"time": time.time(),
|
||||
"sha256": file_hash,
|
||||
"meta_ver": METADATA_VER})
|
||||
"meta_ver": METADATA_VER
|
||||
})
|
||||
|
||||
filename_meta = filename + '.meta'
|
||||
with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
|
||||
|
||||
import uuid
|
||||
|
||||
import StringIO
|
||||
import pathlib2
|
||||
|
||||
|
||||
|
|
@ -52,3 +52,7 @@ def uid_filename(filename, randomized=True):
|
|||
hash_key = '{}.{}'.format(filename, 'store')
|
||||
uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
|
||||
return str(uid) + ext.lower()
|
||||
|
||||
|
||||
def bytes_to_file_obj(bytes_data):
|
||||
return StringIO.StringIO(bytes_data)
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ class FileStoreView(BaseAppView):
|
|||
file_uid, store_path)
|
||||
raise HTTPNotFound()
|
||||
|
||||
db_obj = FileStore().query().filter(FileStore.file_uid == file_uid).scalar()
|
||||
db_obj = FileStore.get_by_store_uid(file_uid, safe=True)
|
||||
if not db_obj:
|
||||
raise HTTPNotFound()
|
||||
|
||||
|
|
|
|||
|
|
@ -5442,8 +5442,11 @@ class FileStore(Base, BaseModel):
|
|||
repo_group = relationship('RepoGroup', lazy='joined')
|
||||
|
||||
@classmethod
|
||||
def get_by_store_uid(cls, file_store_uid):
|
||||
return FileStore.query().filter(FileStore.file_uid == file_store_uid).scalar()
|
||||
def get_by_store_uid(cls, file_store_uid, safe=False):
|
||||
if safe:
|
||||
return FileStore.query().filter(FileStore.file_uid == file_store_uid).first()
|
||||
else:
|
||||
return FileStore.query().filter(FileStore.file_uid == file_store_uid).scalar()
|
||||
|
||||
@classmethod
|
||||
def create(cls, file_uid, filename, file_hash, file_size, file_display_name='',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue