file-uploads: created simple upload capabilities.

- This initially will be used for comments /ticket uploads of binarry attachements
This commit is contained in:
Marcin Lulek 2019-01-25 12:51:26 +01:00
parent 59bd82f930
commit f94d402566
14 changed files with 640 additions and 2 deletions

View file

@ -284,6 +284,13 @@ labs_settings_active = true
## This is used to store exception from RhodeCode in shared directory
#exception_tracker.store_path =
## File store configuration. This is used to store and serve uploaded files
file_store.enabled = true
## backend, only available one is local
file_store.backend = local
## path to store the uploaded binaries
file_store.storage_path = %(here)s/data/file_store
####################################
### CELERY CONFIG ####

View file

@ -259,6 +259,13 @@ labs_settings_active = true
## This is used to store exception from RhodeCode in shared directory
#exception_tracker.store_path =
## File store configuration. This is used to store and serve uploaded files
file_store.enabled = true
## backend, only available one is local
file_store.backend = local
## path to store the uploaded binaries
file_store.storage_path = %(here)s/data/file_store
####################################
### CELERY CONFIG ####

View file

@ -0,0 +1,49 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2019 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
from rhodecode.apps.upload_store import config_keys
from rhodecode.config.middleware import _bool_setting, _string_setting
def _sanitize_settings_and_apply_defaults(settings):
"""
Set defaults, convert to python types and validate settings.
"""
_bool_setting(settings, config_keys.enabled, 'true')
_string_setting(settings, config_keys.backend, 'local')
default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store')
_string_setting(settings, config_keys.store_path, default_store)
def includeme(config):
settings = config.registry.settings
_sanitize_settings_and_apply_defaults(settings)
config.add_route(
name='upload_file',
pattern='/_file_store/upload')
config.add_route(
name='download_file',
pattern='/_file_store/download/{fid}')
# Scan module for configuration decorators.
config.scan('.views', ignore='.tests')

View file

@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2019 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
# Definition of setting keys used to configure this module. Defined here to
# avoid repetition of keys throughout the module.
enabled = 'file_store.enabled'
backend = 'file_store.backend'
store_path = 'file_store.storage_path'

View file

@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2019 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
class FileNotAllowedException(Exception):
"""
Thrown if file does not have an allowed extension.
"""
class FileOverSizeException(Exception):
"""
Thrown if file is over the set limit.
"""

View file

@ -0,0 +1,66 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2019 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
ANY = []
TEXT_EXT = ['txt', 'md', 'rst', 'log']
DOCUMENTS_EXT = ['pdf', 'rtf', 'odf', 'ods', 'gnumeric', 'abw', 'doc', 'docx', 'xls', 'xlsx']
IMAGES_EXT = ['jpg', 'jpe', 'jpeg', 'png', 'gif', 'svg', 'bmp', 'tiff']
AUDIO_EXT = ['wav', 'mp3', 'aac', 'ogg', 'oga', 'flac']
VIDEO_EXT = ['mpeg', '3gp', 'avi', 'divx', 'dvr', 'flv', 'mp4', 'wmv']
DATA_EXT = ['csv', 'ini', 'json', 'plist', 'xml', 'yaml', 'yml']
SCRIPTS_EXT = ['js', 'php', 'pl', 'py', 'rb', 'sh', 'go', 'c', 'h']
ARCHIVES_EXT = ['gz', 'bz2', 'zip', 'tar', 'tgz', 'txz', '7z']
EXECUTABLES_EXT = ['so', 'exe', 'dll']
DEFAULT = DOCUMENTS_EXT + TEXT_EXT + IMAGES_EXT + DATA_EXT
GROUPS = dict((
('any', ANY),
('text', TEXT_EXT),
('documents', DOCUMENTS_EXT),
('images', IMAGES_EXT),
('audio', AUDIO_EXT),
('video', VIDEO_EXT),
('data', DATA_EXT),
('scripts', SCRIPTS_EXT),
('archives', ARCHIVES_EXT),
('executables', EXECUTABLES_EXT),
('default', DEFAULT),
))
def resolve_extensions(extensions, groups=None):
"""
Calculate allowed extensions based on a list of extensions provided, and optional
groups of extensions from the available lists.
:param extensions: a list of extensions e.g ['py', 'txt']
:param groups: additionally groups to extend the extensions.
"""
groups = groups or []
valid_exts = set([x.lower() for x in extensions])
for group in groups:
if group in GROUPS:
valid_exts.update(GROUPS[group])
return valid_exts

View file

@ -0,0 +1,167 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2019 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import shutil
from rhodecode.lib.ext_json import json
from rhodecode.apps.upload_store import utils
from rhodecode.apps.upload_store.extensions import resolve_extensions
from rhodecode.apps.upload_store.exceptions import FileNotAllowedException
class LocalFileStorage(object):
@classmethod
def resolve_name(cls, name, directory):
"""
Resolves a unique name and the correct path. If a filename
for that path already exists then a numeric prefix with values > 0 will be
added, for example test.jpg -> test-1.jpg etc. initially file would have 0 prefix.
:param name: base name of file
:param directory: absolute directory path
"""
basename, ext = os.path.splitext(name)
counter = 0
while True:
name = '%s-%d%s' % (basename, counter, ext)
path = os.path.join(directory, name)
if not os.path.exists(path):
return name, path
counter += 1
def __init__(self, base_path, extension_groups=None):
"""
Local file storage
:param base_path: the absolute base path where uploads are stored
:param extension_groups: extensions string
"""
extension_groups = extension_groups or ['any']
self.base_path = base_path
self.extensions = resolve_extensions([], groups=extension_groups)
def store_path(self, filename):
"""
Returns absolute file path of the filename, joined to the
base_path.
:param filename: base name of file
"""
return os.path.join(self.base_path, filename)
def delete(self, filename):
"""
Deletes the filename. Filename is resolved with the
absolute path based on base_path. If file does not exist,
returns **False**, otherwise **True**
:param filename: base name of file
"""
if self.exists(filename):
os.remove(self.store_path(filename))
return True
return False
def exists(self, filename):
"""
Checks if file exists. Resolves filename's absolute
path based on base_path.
:param filename: base name of file
"""
return os.path.exists(self.store_path(filename))
def filename_allowed(self, filename, extensions=None):
"""Checks if a filename has an allowed extension
:param filename: base name of file
:param extensions: iterable of extensions (or self.extensions)
"""
_, ext = os.path.splitext(filename)
return self.extension_allowed(ext, extensions)
def extension_allowed(self, ext, extensions=None):
"""
Checks if an extension is permitted. Both e.g. ".jpg" and
"jpg" can be passed in. Extension lookup is case-insensitive.
:param extensions: iterable of extensions (or self.extensions)
"""
extensions = extensions or self.extensions
if not extensions:
return True
if ext.startswith('.'):
ext = ext[1:]
return ext.lower() in extensions
def save_file(self, file_obj, filename, directory=None, extensions=None,
metadata=None, **kwargs):
"""
Saves a file object to the uploads location.
Returns the resolved filename, i.e. the directory +
the (randomized/incremented) base name.
:param file_obj: **cgi.FieldStorage** object (or similar)
:param filename: original filename
:param directory: relative path of sub-directory
:param extensions: iterable of allowed extensions, if not default
:param metadata: JSON metadata to store next to the file with .meta suffix
:returns: modified filename
"""
extensions = extensions or self.extensions
if not self.filename_allowed(filename, extensions):
raise FileNotAllowedException()
if directory:
dest_directory = os.path.join(self.base_path, directory)
else:
dest_directory = self.base_path
if not os.path.exists(dest_directory):
os.makedirs(dest_directory)
filename = utils.uid_filename(filename)
filename, path = self.resolve_name(filename, dest_directory)
filename_meta = filename + '.meta'
file_obj.seek(0)
with open(path, "wb") as dest:
shutil.copyfileobj(file_obj, dest)
if metadata:
size = os.stat(path).st_size
metadata.update({'size': size})
with open(os.path.join(dest_directory, filename_meta), "wb") as dest_meta:
dest_meta.write(json.dumps(metadata))
if directory:
filename = os.path.join(directory, filename)
return filename

View file

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2019 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/

View file

@ -0,0 +1,110 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2010-2019 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import pytest
from rhodecode.lib.ext_json import json
from rhodecode.tests import TestController
from rhodecode.apps.upload_store import utils, config_keys
def route_path(name, params=None, **kwargs):
import urllib
base_url = {
'upload_file': '/_file_store/upload',
'download_file': '/_file_store/download/{fid}',
}[name].format(**kwargs)
if params:
base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
return base_url
class TestFileStoreViews(TestController):
@pytest.mark.parametrize("fid, content, exists", [
('abcde-0.jpg', "xxxxx", True),
('abcde-0.exe', "1234567", True),
('abcde-0.jpg', "xxxxx", False),
])
def test_get_files_from_store(self, fid, content, exists, tmpdir):
self.log_user()
store_path = self.app._pyramid_settings[config_keys.store_path]
if exists:
status = 200
store = utils.get_file_storage({config_keys.store_path: store_path})
filesystem_file = os.path.join(str(tmpdir), fid)
with open(filesystem_file, 'wb') as f:
f.write(content)
with open(filesystem_file, 'rb') as f:
fid = store.save_file(f, fid, metadata={'filename': fid})
else:
status = 404
response = self.app.get(route_path('download_file', fid=fid), status=status)
if exists:
assert response.text == content
metadata = os.path.join(store_path, fid + '.meta')
assert os.path.exists(metadata)
with open(metadata, 'rb') as f:
json_data = json.loads(f.read())
assert json_data
assert 'size' in json_data
def test_upload_files_without_content_to_store(self):
self.log_user()
response = self.app.post(
route_path('upload_file'),
params={'csrf_token': self.csrf_token},
status=200)
assert response.json == {
u'error': u'store_file data field is missing',
u'access_path': None,
u'store_fid': None}
def test_upload_files_bogus_content_to_store(self):
self.log_user()
response = self.app.post(
route_path('upload_file'),
params={'csrf_token': self.csrf_token, 'store_file': 'bogus'},
status=200)
assert response.json == {
u'error': u'filename cannot be read from the data field',
u'access_path': None,
u'store_fid': None}
def test_upload_content_to_store(self):
self.log_user()
response = self.app.post(
route_path('upload_file'),
upload_files=[('store_file', 'myfile.txt', 'SOME CONTENT')],
params={'csrf_token': self.csrf_token},
status=200)
assert response.json['store_fid']

View file

@ -0,0 +1,47 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2019 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import uuid
def get_file_storage(settings):
from rhodecode.apps.upload_store.local_store import LocalFileStorage
from rhodecode.apps.upload_store import config_keys
store_path = settings.get(config_keys.store_path)
return LocalFileStorage(base_path=store_path)
def uid_filename(filename, randomized=True):
"""
Generates a randomized or stable (uuid) filename,
preserving the original extension.
:param filename: the original filename
:param randomized: define if filename should be stable (sha1 based) or randomized
"""
_, ext = os.path.splitext(filename)
if randomized:
uid = uuid.uuid4()
else:
hash_key = '{}.{}'.format(filename, 'store')
uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
return str(uid) + ext.lower()

View file

@ -0,0 +1,97 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2019 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
from pyramid.view import view_config
from pyramid.response import FileResponse
from pyramid.httpexceptions import HTTPFound, HTTPNotFound
from rhodecode.apps._base import BaseAppView
from rhodecode.apps.upload_store import utils
from rhodecode.apps.upload_store.exceptions import (
FileNotAllowedException,FileOverSizeException)
from rhodecode.lib import helpers as h
from rhodecode.lib import audit_logger
from rhodecode.lib.auth import (CSRFRequired, NotAnonymous)
log = logging.getLogger(__name__)
class FileStoreView(BaseAppView):
upload_key = 'store_file'
def load_default_context(self):
c = self._get_local_tmpl_context()
self.storage = utils.get_file_storage(self.request.registry.settings)
return c
@NotAnonymous()
@CSRFRequired()
@view_config(route_name='upload_file', request_method='POST', renderer='json_ext')
def upload_file(self):
self.load_default_context()
file_obj = self.request.POST.get(self.upload_key)
if file_obj is None:
return {'store_fid': None,
'access_path': None,
'error': '{} data field is missing'.format(self.upload_key)}
if not hasattr(file_obj, 'filename'):
return {'store_fid': None,
'access_path': None,
'error': 'filename cannot be read from the data field'}
filename = file_obj.filename
metadata = {
'filename': filename,
'size': '', # filled by save_file
'user_uploaded': {'username': self._rhodecode_user.username,
'user_id': self._rhodecode_user.user_id,
'ip': self._rhodecode_user.ip_addr}}
try:
store_fid = self.storage.save_file(file_obj.file, filename,
metadata=metadata)
except FileNotAllowedException:
return {'store_fid': None,
'access_path': None,
'error': 'File {} is not allowed.'.format(filename)}
except FileOverSizeException:
return {'store_fid': None,
'access_path': None,
'error': 'File {} is exceeding allowed limit.'.format(filename)}
return {'store_fid': store_fid,
'access_path': h.route_path('download_file', fid=store_fid)}
@view_config(route_name='download_file')
def download_file(self):
self.load_default_context()
file_uid = self.request.matchdict['fid']
log.debug('Requesting FID:%s from store %s', file_uid, self.storage)
if not self.storage.exists(file_uid):
log.debug('File with FID:%s not found in the store', file_uid)
raise HTTPNotFound()
file_path = self.storage.store_path(file_uid)
return FileResponse(file_path)

View file

@ -281,6 +281,7 @@ def includeme(config):
config.include('rhodecode.apps.ops')
config.include('rhodecode.apps.admin')
config.include('rhodecode.apps.channelstream')
config.include('rhodecode.apps.upload_store')
config.include('rhodecode.apps.login')
config.include('rhodecode.apps.home')
config.include('rhodecode.apps.journal')

View file

@ -136,6 +136,8 @@ function registerRCRoutes() {
pyroutes.register('channelstream_connect', '/_admin/channelstream/connect', []);
pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []);
pyroutes.register('channelstream_proxy', '/_channelstream', []);
pyroutes.register('upload_file', '/_file_store/upload', []);
pyroutes.register('download_file', '/_file_store/download/%(fid)s', ['fid']);
pyroutes.register('logout', '/_admin/logout', []);
pyroutes.register('reset_password', '/_admin/password_reset', []);
pyroutes.register('reset_password_confirmation', '/_admin/password_reset_confirmation', []);

View file

@ -101,8 +101,7 @@ class CustomTestResponse(TestResponse):
"""
from pyramid_beaker import session_factory_from_settings
session = session_factory_from_settings(
self.test_app.app.config.get_settings())
session = session_factory_from_settings(self.test_app._pyramid_settings)
return session(self.request)
@ -140,6 +139,14 @@ class CustomTestApp(TestApp):
def csrf_token(self):
return self.rc_login_data['csrf_token']
@property
def _pyramid_registry(self):
return self.app.config.registry
@property
def _pyramid_settings(self):
return self._pyramid_registry.settings
def set_anonymous_access(enabled):
"""(Dis)allows anonymous access depending on parameter `enabled`"""