diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 5fbb56e9..f367797f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.1.2 +current_version = 5.2.0 message = release: Bump version {current_version} to {new_version} [bumpversion:file:rhodecode/VERSION] diff --git a/.hgignore b/.hgignore index f84ab689..375ba1b6 100644 --- a/.hgignore +++ b/.hgignore @@ -54,7 +54,7 @@ syntax: regexp ^rhodecode\.log$ ^rhodecode_dev\.log$ ^test\.db$ - +^venv/ # ac-tests ^acceptance_tests/\.cache.*$ diff --git a/Makefile b/Makefile index a5d526b2..9967d2ee 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,49 @@ +.DEFAULT_GOAL := help + +# Pretty print values cf. https://misc.flogisoft.com/bash/tip_colors_and_formatting +RESET := \033[0m # Reset all formatting +GREEN := \033[0;32m # Resets before setting 16b colour (32 -- green) +YELLOW := \033[0;33m +ORANGE := \033[0;38;5;208m # Reset then set 256b colour (208 -- orange) +PEACH := \033[0;38;5;216m + + +## ---------------------------------------------------------------------------------- ## +## ------------------------- Help usage builder ------------------------------------- ## +## ---------------------------------------------------------------------------------- ## +# use '# >>> Build commands' to create section +# use '# target: target description' to create help for target +.PHONY: help +help: + @echo "Usage:" + @cat $(MAKEFILE_LIST) | grep -E '^# >>>|^# [A-Za-z0-9_.-]+:' | sed -E 's/^# //' | awk ' \ + BEGIN { \ + green="\033[32m"; \ + yellow="\033[33m"; \ + reset="\033[0m"; \ + section=""; \ + } \ + /^>>>/ { \ + section=substr($$0, 5); \ + printf "\n" green ">>> %s" reset "\n", section; \ + next; \ + } \ + /^([A-Za-z0-9_.-]+):/ { \ + target=$$1; \ + gsub(/:$$/, "", target); \ + description=substr($$0, index($$0, ":") + 2); \ + if (description == "") { description="-"; } \ + printf " - " yellow "%-35s" reset " %s\n", target, description; \ + } \ + ' + # required for pushd to work.. SHELL = /bin/bash - -# set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py -OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES} +# >>> Tests commands .PHONY: clean -## Cleanup compiled and cache py files +# clean: Cleanup compiled and cache py files clean: make test-clean find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';' @@ -14,14 +51,14 @@ clean: .PHONY: test -## run test-clean and tests +# test: run test-clean and tests test: make test-clean - make test-only + unset RC_SQLALCHEMY_DB1_URL && unset RC_DB_URL && make test-only .PHONY: test-clean -## run test-clean and tests +# test-clean: run test-clean and tests test-clean: rm -rf coverage.xml htmlcov junit.xml pylint.log result find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';' @@ -29,34 +66,36 @@ test-clean: .PHONY: test-only -## Run tests only without cleanup +# test-only: Run tests only without cleanup test-only: PYTHONHASHSEED=random \ py.test -x -vv -r xw -p no:sugar \ --cov-report=term-missing --cov-report=html \ --cov=rhodecode rhodecode +# >>> Docs commands .PHONY: docs -## build docs +# docs: build docs docs: (cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make clean html SPHINXOPTS="-W") .PHONY: docs-clean -## Cleanup docs +# docs-clean: Cleanup docs docs-clean: (cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make clean) .PHONY: docs-cleanup -## Cleanup docs +# docs-cleanup: Cleanup docs docs-cleanup: (cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make cleanup) +# >>> Dev commands .PHONY: web-build -## Build JS packages static/js +# web-build: Build JS packages static/js web-build: rm -rf node_modules docker run -it --rm -v $(PWD):/project --workdir=/project rhodecode/static-files-build:16 -c "npm install && /project/node_modules/.bin/grunt" @@ -64,25 +103,9 @@ web-build: ./rhodecode/tests/scripts/static-file-check.sh rhodecode/public/ rm -rf node_modules -.PHONY: ruff-check -## run a ruff analysis -ruff-check: - ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs --exclude .dev . - -.PHONY: pip-packages -## Show outdated packages -pip-packages: - python ${OUTDATED_PACKAGES} - - -.PHONY: build -## Build sdist/egg -build: - python -m build - .PHONY: dev-sh -## make dev-sh +# dev-sh: make dev-sh dev-sh: sudo echo "deb [trusted=yes] https://apt.fury.io/rsteube/ /" | sudo tee -a "/etc/apt/sources.list.d/fury.list" sudo apt-get update @@ -95,14 +118,14 @@ dev-sh: .PHONY: dev-cleanup -## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y +# dev-cleanup: Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y dev-cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y rm -rf /tmp/* .PHONY: dev-env -## make dev-env based on the requirements files and install develop of packages +# dev-env: make dev-env based on the requirements files and install develop of packages ## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y dev-env: sudo -u root chown rhodecode:rhodecode /home/rhodecode/.cache/pip/ @@ -114,7 +137,7 @@ dev-env: .PHONY: sh -## shortcut for make dev-sh dev-env +# sh: shortcut for make dev-sh dev-env sh: make dev-env make dev-sh @@ -124,49 +147,12 @@ sh: workers?=1 .PHONY: dev-srv -## run gunicorn web server with reloader, use workers=N to set multiworker mode +# dev-srv: run gunicorn web server with reloader, use workers=N to set multiworker mode, workers=N allows changes of workers dev-srv: gunicorn --paste=.dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py --timeout=120 --reload --workers=$(workers) +.PHONY: ruff-check +# ruff-check: run a ruff analysis +ruff-check: + ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs --exclude .dev . -# Default command on calling make -.DEFAULT_GOAL := show-help - -.PHONY: show-help -show-help: - @echo "$$(tput bold)Available rules:$$(tput sgr0)" - @echo - @sed -n -e "/^## / { \ - h; \ - s/.*//; \ - :doc" \ - -e "H; \ - n; \ - s/^## //; \ - t doc" \ - -e "s/:.*//; \ - G; \ - s/\\n## /---/; \ - s/\\n/ /g; \ - p; \ - }" ${MAKEFILE_LIST} \ - | LC_ALL='C' sort --ignore-case \ - | awk -F '---' \ - -v ncol=$$(tput cols) \ - -v indent=19 \ - -v col_on="$$(tput setaf 6)" \ - -v col_off="$$(tput sgr0)" \ - '{ \ - printf "%s%*s%s ", col_on, -indent, $$1, col_off; \ - n = split($$2, words, " "); \ - line_length = ncol - indent; \ - for (i = 1; i <= n; i++) { \ - line_length -= length(words[i]) + 1; \ - if (line_length <= 0) { \ - line_length = ncol - indent - length(words[i]) - 1; \ - printf "\n%*s ", -indent, " "; \ - } \ - printf "%s ", words[i]; \ - } \ - printf "\n"; \ - }' diff --git a/configs/development.ini b/configs/development.ini index b518afca..77fd4cbf 100644 --- a/configs/development.ini +++ b/configs/development.ini @@ -257,6 +257,13 @@ license_token = ; This flag hides sensitive information on the license page such as token, and license data license.hide_license_info = false +; Import EE license from this license path +#license.import_path = %(here)s/rhodecode_enterprise.license + +; import license 'if-missing' or 'force' (always override) +; if-missing means apply license if it doesn't exist. 'force' option always overrides it +license.import_path_mode = if-missing + ; supervisor connection uri, for managing supervisor and logs. supervisor.uri = @@ -281,15 +288,56 @@ labs_settings_active = true ; optional prefix to Add to email Subject #exception_tracker.email_prefix = [RHODECODE ERROR] -; File store configuration. This is used to store and serve uploaded files -file_store.enabled = true +; NOTE: this setting IS DEPRECATED: +; file_store backend is always enabled +#file_store.enabled = true +; NOTE: this setting IS DEPRECATED: +; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead ; Storage backend, available options are: local -file_store.backend = local +#file_store.backend = local +; NOTE: this setting IS DEPRECATED: +; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead ; path to store the uploaded binaries and artifacts -file_store.storage_path = /var/opt/rhodecode_data/file_store +#file_store.storage_path = /var/opt/rhodecode_data/file_store +; Artifacts file-store, is used to store comment attachments and artifacts uploads. +; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options +; filesystem_v1 is backwards compat with pre 5.1 storage changes +; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from +; previous installations to keep the artifacts without a need of migration +#file_store.backend.type = filesystem_v2 + +; filesystem options... +#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store + +; filesystem_v2 options... +#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store +#file_store.filesystem_v2.shards = 8 + +; objectstore options... +; url for s3 compatible storage that allows to upload artifacts +; e.g http://minio:9000 +#file_store.backend.type = objectstore +#file_store.objectstore.url = http://s3-minio:9000 + +; a top-level bucket to put all other shards in +; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number +#file_store.objectstore.bucket = rhodecode-file-store + +; number of sharded buckets to create to distribute archives across +; default is 8 shards +#file_store.objectstore.bucket_shards = 8 + +; key for s3 auth +#file_store.objectstore.key = s3admin + +; secret for s3 auth +#file_store.objectstore.secret = s3secret4 + +;region for s3 storage +#file_store.objectstore.region = eu-central-1 ; Redis url to acquire/check generation of archives locks archive_cache.locking.url = redis://redis:6379/1 @@ -624,7 +672,8 @@ vcs.scm_app_implementation = http ; Push/Pull operations hooks protocol, available options are: ; `http` - use http-rpc backend (default) ; `celery` - use celery based hooks -vcs.hooks.protocol = http +#DEPRECATED:vcs.hooks.protocol = http +vcs.hooks.protocol.v2 = celery ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be ; accessible via network. @@ -647,6 +696,12 @@ vcs.connection_timeout = 3600 ; It uses cache_region `cache_repo` vcs.methods.cache = true +; Filesystem location where Git lfs objects should be stored +vcs.git.lfs.storage_location = /var/opt/rhodecode_repo_store/.cache/git_lfs_store + +; Filesystem location where Mercurial largefile objects should be stored +vcs.hg.largefiles.storage_location = /var/opt/rhodecode_repo_store/.cache/hg_largefiles_store + ; #################################################### ; Subversion proxy support (mod_dav_svn) ; Maps RhodeCode repo groups into SVN paths for Apache @@ -716,7 +771,8 @@ ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecod ; RhodeCode installation directory. ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2 -ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper +#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper +ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2 ; Allow shell when executing the ssh-wrapper command ssh.wrapper_cmd_allow_shell = false diff --git a/configs/gunicorn_config.py b/configs/gunicorn_config.py index d4d22078..b7578015 100644 --- a/configs/gunicorn_config.py +++ b/configs/gunicorn_config.py @@ -13,6 +13,7 @@ import traceback import random import socket import dataclasses +import json from gunicorn.glogging import Logger @@ -37,17 +38,41 @@ accesslog = '-' worker_tmp_dir = None tmp_upload_dir = None -# use re-use port logic -#reuse_port = True +# use re-use port logic to let linux internals load-balance the requests better. +reuse_port = True # Custom log format #access_log_format = ( # '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"') # loki format for easier parsing in grafana -access_log_format = ( +loki_access_log_format = ( 'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"') +# JSON format +json_access_log_format = json.dumps({ + 'time': r'%(t)s', + 'pid': r'%(p)s', + 'level': 'INFO', + 'ip': r'%(h)s', + 'request_time': r'%(L)s', + 'remote_address': r'%(h)s', + 'user_name': r'%(u)s', + 'status': r'%(s)s', + 'method': r'%(m)s', + 'url_path': r'%(U)s', + 'query_string': r'%(q)s', + 'protocol': r'%(H)s', + 'response_length': r'%(B)s', + 'referer': r'%(f)s', + 'user_agent': r'%(a)s', + +}) + +access_log_format = loki_access_log_format +if os.environ.get('RC_LOGGING_FORMATTER') == 'json': + access_log_format = json_access_log_format + # self adjust workers based on CPU count, to use maximum of CPU and not overquota the resources # workers = get_workers() diff --git a/configs/init.d/.readme.txt b/configs/init.d/.readme.txt deleted file mode 100644 index 415fdcfd..00000000 --- a/configs/init.d/.readme.txt +++ /dev/null @@ -1 +0,0 @@ -Example init scripts. \ No newline at end of file diff --git a/configs/init.d/supervisord.conf b/configs/init.d/supervisord.conf deleted file mode 100644 index a2f3bdf9..00000000 --- a/configs/init.d/supervisord.conf +++ /dev/null @@ -1,61 +0,0 @@ -; Sample supervisor RhodeCode config file. -; -; For more information on the config file, please see: -; http://supervisord.org/configuration.html -; -; Note: shell expansion ("~" or "$HOME") is not supported. Environment -; variables can be expanded using this syntax: "%(ENV_HOME)s". - -[unix_http_server] -file=/tmp/supervisor.sock ; (the path to the socket file) -;chmod=0700 ; socket file mode (default 0700) -;chown=nobody:nogroup ; socket file uid:gid owner -;username=user ; (default is no username (open server)) -;password=123 ; (default is no password (open server)) - -[supervisord] -logfile=/home/ubuntu/rhodecode/supervisord.log ; (main log file;default $CWD/supervisord.log) -logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) -logfile_backups=10 ; (num of main logfile rotation backups;default 10) -loglevel=info ; (log level;default info; others: debug,warn,trace) -pidfile=/home/ubuntu/rhodecode/supervisord.pid ; (supervisord pidfile;default supervisord.pid) -nodaemon=true ; (start in foreground if true;default false) -minfds=1024 ; (min. avail startup file descriptors;default 1024) -minprocs=200 ; (min. avail process descriptors;default 200) -;umask=022 ; (process file creation umask;default 022) -user=ubuntu ; (default is current user, required if root) -;identifier=supervisor ; (supervisord identifier, default is 'supervisor') -;directory=/tmp ; (default is not to cd during start) -;nocleanup=true ; (don't clean up tempfiles at start;default false) -;childlogdir=/tmp ; ('AUTO' child log dir, default $TEMP) -environment=HOME=/home/ubuntu,LANG=en_US.UTF-8 ; (key value pairs to add to environment) -;strip_ansi=false ; (strip ansi escape codes in logs; def. false) - -; the below section must remain in the config file for RPC -; (supervisorctl/web interface) to work, additional interfaces may be -; added by defining them in separate rpcinterface: sections -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket -;username=chris ; should be same as http_username if set -;password=123 ; should be same as http_password if set - - -; restart with supervisorctl restart rhodecode:* -[program:rhodecode] -numprocs = 1 -numprocs_start = 5000 -directory=/home/ubuntu/rhodecode/source -command = /home/ubuntu/rhodecode/venv/bin/paster serve /home/ubuntu/rhodecode/source/prod.ini -process_name = %(program_name)s_%(process_num)04d -redirect_stderr=true -stdout_logfile=/home/ubuntu/rhodecode/rhodecode.log - -[program:rhodecode_workers] -numproces = 1 -directory = /home/ubuntu/rhodecode/source -command = /home/ubuntu/rhodecode/venv/bin/paster celeryd /home/ubuntu/rhodecode/source/prod.ini --autoscale=10,2 -redirect_stderr=true -stdout_logfile=/%(here)s/rhodecode_workers.log diff --git a/configs/production.ini b/configs/production.ini index 948c7c0c..30840933 100644 --- a/configs/production.ini +++ b/configs/production.ini @@ -225,6 +225,13 @@ license_token = ; This flag hides sensitive information on the license page such as token, and license data license.hide_license_info = false +; Import EE license from this license path +#license.import_path = %(here)s/rhodecode_enterprise.license + +; import license 'if-missing' or 'force' (always override) +; if-missing means apply license if it doesn't exist. 'force' option always overrides it +license.import_path_mode = if-missing + ; supervisor connection uri, for managing supervisor and logs. supervisor.uri = @@ -249,15 +256,56 @@ labs_settings_active = true ; optional prefix to Add to email Subject #exception_tracker.email_prefix = [RHODECODE ERROR] -; File store configuration. This is used to store and serve uploaded files -file_store.enabled = true +; NOTE: this setting IS DEPRECATED: +; file_store backend is always enabled +#file_store.enabled = true +; NOTE: this setting IS DEPRECATED: +; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead ; Storage backend, available options are: local -file_store.backend = local +#file_store.backend = local +; NOTE: this setting IS DEPRECATED: +; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead ; path to store the uploaded binaries and artifacts -file_store.storage_path = /var/opt/rhodecode_data/file_store +#file_store.storage_path = /var/opt/rhodecode_data/file_store +; Artifacts file-store, is used to store comment attachments and artifacts uploads. +; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options +; filesystem_v1 is backwards compat with pre 5.1 storage changes +; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from +; previous installations to keep the artifacts without a need of migration +#file_store.backend.type = filesystem_v2 + +; filesystem options... +#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store + +; filesystem_v2 options... +#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store +#file_store.filesystem_v2.shards = 8 + +; objectstore options... +; url for s3 compatible storage that allows to upload artifacts +; e.g http://minio:9000 +#file_store.backend.type = objectstore +#file_store.objectstore.url = http://s3-minio:9000 + +; a top-level bucket to put all other shards in +; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number +#file_store.objectstore.bucket = rhodecode-file-store + +; number of sharded buckets to create to distribute archives across +; default is 8 shards +#file_store.objectstore.bucket_shards = 8 + +; key for s3 auth +#file_store.objectstore.key = s3admin + +; secret for s3 auth +#file_store.objectstore.secret = s3secret4 + +;region for s3 storage +#file_store.objectstore.region = eu-central-1 ; Redis url to acquire/check generation of archives locks archive_cache.locking.url = redis://redis:6379/1 @@ -592,7 +640,8 @@ vcs.scm_app_implementation = http ; Push/Pull operations hooks protocol, available options are: ; `http` - use http-rpc backend (default) ; `celery` - use celery based hooks -vcs.hooks.protocol = http +#DEPRECATED:vcs.hooks.protocol = http +vcs.hooks.protocol.v2 = celery ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be ; accessible via network. @@ -615,6 +664,12 @@ vcs.connection_timeout = 3600 ; It uses cache_region `cache_repo` vcs.methods.cache = true +; Filesystem location where Git lfs objects should be stored +vcs.git.lfs.storage_location = /var/opt/rhodecode_repo_store/.cache/git_lfs_store + +; Filesystem location where Mercurial largefile objects should be stored +vcs.hg.largefiles.storage_location = /var/opt/rhodecode_repo_store/.cache/hg_largefiles_store + ; #################################################### ; Subversion proxy support (mod_dav_svn) ; Maps RhodeCode repo groups into SVN paths for Apache @@ -684,7 +739,8 @@ ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecod ; RhodeCode installation directory. ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2 -ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper +#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper +ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2 ; Allow shell when executing the ssh-wrapper command ssh.wrapper_cmd_allow_shell = false diff --git a/docs/Dockerfile b/docs/Dockerfile index 330e8872..0c25861b 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -22,6 +22,12 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + ./aws/install && \ + rm -rf ./aws && \ + rm awscliv2.zip + RUN \ python3 -m pip install --no-cache-dir --upgrade pip && \ python3 -m pip install --no-cache-dir Sphinx Pillow diff --git a/docs/admin/system-overview.rst b/docs/admin/system-overview.rst index b4f670f0..443ed102 100644 --- a/docs/admin/system-overview.rst +++ b/docs/admin/system-overview.rst @@ -147,10 +147,6 @@ Peer-to-peer Failover Support * Yes -Additional Binaries -------------------- - -* Yes, see :ref:`rhodecode-nix-ref` for full details. Remote Connectivity ------------------- diff --git a/docs/auth/auth-saml-azure.rst b/docs/auth/auth-saml-azure.rst new file mode 100644 index 00000000..2df03ed0 --- /dev/null +++ b/docs/auth/auth-saml-azure.rst @@ -0,0 +1,161 @@ +.. _config-saml-azure-ref: + + +SAML 2.0 with Azure Entra ID +---------------------------- + +**This plugin is available only in EE Edition.** + +|RCE| supports SAML 2.0 Authentication with Azure Entra ID provider. This allows +users to log-in to RhodeCode via SSO mechanism of external identity provider +such as Azure AD. The login can be triggered either by the external IDP, or internally +by clicking specific authentication button on the log-in page. + + +Configuration steps +^^^^^^^^^^^^^^^^^^^ + +To configure Duo Security SAML authentication, use the following steps: + +1. From the |RCE| interface, select + :menuselection:`Admin --> Authentication` +2. Activate the `Azure Entra ID` plugin and select :guilabel:`Save` +3. Go to newly available menu option called `Azure Entra ID` on the left side. +4. Check the `enabled` check box in the plugin configuration section, + and fill in the required SAML information and :guilabel:`Save`, for more details, + see :ref:`config-saml-azure` + + +.. _config-saml-azure: + + +Example SAML Azure Entra ID configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example configuration for SAML 2.0 with Azure Entra ID provider + + +Enabled + `True`: + + .. note:: + Enable or disable this authentication plugin. + + +Auth Cache TTL + `30`: + + .. note:: + Amount of seconds to cache the authentication and permissions check response call for this plugin. + Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled). + +Debug + `True`: + + .. note:: + Enable or disable debug mode that shows SAML errors in the RhodeCode logs. + + +Auth button name + `Azure Entra ID`: + + .. note:: + Alternative authentication display name. E.g AzureAuth, CorporateID etc. + + +Entity ID + `https://sts.windows.net/APP_ID/`: + + .. note:: + Identity Provider entity/metadata URI. Known as "Microsoft Entra Identifier" + E.g. https://sts.windows.net/abcd-c655-dcee-aab7-abcd/ + +SSO URL + `https://login.microsoftonline.com/APP_ID/saml2`: + + .. note:: + SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login, Known also as Login URL + E.g. https://login.microsoftonline.com/abcd-c655-dcee-aab7-abcd/saml2 + +SLO URL + `https://login.microsoftonline.com/APP_ID/saml2`: + + .. note:: + SLO (SingleLogout) endpoint URL of the IdP. , Known also as Logout URL + E.g. https://login.microsoftonline.com/abcd-c655-dcee-aab7-abcd/saml2 + +x509cert + ``: + + .. note:: + Identity provider public x509 certificate. It will be converted to single-line format without headers. + Download the raw base64 encoded certificate from the Identity provider and paste it here. + +SAML Signature + `sha-256`: + + .. note:: + Type of Algorithm to use for verification of SAML signature on Identity provider side. + +SAML Digest + `sha-256`: + + .. note:: + Type of Algorithm to use for verification of SAML digest on Identity provider side. + +Service Provider Cert Dir + `/etc/rhodecode/conf/saml_ssl/`: + + .. note:: + Optional directory to store service provider certificate and private keys. + Expected certs for the SP should be stored in this folder as: + + * sp.key Private Key + * sp.crt Public cert + * sp_new.crt Future Public cert + + Also you can use other cert to sign the metadata of the SP using the: + + * metadata.key + * metadata.crt + +Expected NameID Format + `nameid-format:emailAddress`: + + .. note:: + The format that specifies how the NameID is sent to the service provider. + +User ID Attribute + `user.email`: + + .. note:: + User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id. + Ensure this is returned from DuoSecurity for example via duo_username. + +Username Attribute + `user.username`: + + .. note:: + Username Attribute name. This defines which attribute in SAML response will map to a username. + +Email Attribute + `user.email`: + + .. note:: + Email Attribute name. This defines which attribute in SAML response will map to an email address. + + + +Below is example setup from Azure Administration page that can be used with above config. + +.. image:: ../images/saml-azure-service-provider-example.png + :alt: Azure SAML setup example + :scale: 50 % + + +Below is an example attribute mapping set for IDP provider required by the above config. + + +.. image:: ../images/saml-azure-attributes-example.png + :alt: Azure SAML setup example + :scale: 50 % \ No newline at end of file diff --git a/docs/auth/auth-saml-bulk-enroll-users.rst b/docs/auth/auth-saml-bulk-enroll-users.rst index cc3bd511..edc6f29a 100644 --- a/docs/auth/auth-saml-bulk-enroll-users.rst +++ b/docs/auth/auth-saml-bulk-enroll-users.rst @@ -13,7 +13,7 @@ This method simply enables SAML authentication for many users at once. From the server RhodeCode Enterprise is running run ishell on the instance which we want to apply the SAML migration:: - rccontrol ishell enterprise-1 + ./rcstack cli ishell Follow these steps to enable SAML authentication for multiple users. @@ -46,6 +46,8 @@ From available options pick only one and run the `import` statement # for Duo Security In [2]: from rc_auth_plugins.auth_duo_security import RhodeCodeAuthPlugin + # for Azure Entra + In [2]: from rc_auth_plugins.auth_azure import RhodeCodeAuthPlugin # for OneLogin In [2]: from rc_auth_plugins.auth_onelogin import RhodeCodeAuthPlugin # generic SAML plugin @@ -62,13 +64,13 @@ Enter in the ishell prompt ...: attrs = saml2user.get(user.user_id) ...: provider = RhodeCodeAuthPlugin.uid ...: if existing_identity: - ...: print('Identity for user `{}` already exists, skipping'.format(user.username)) + ...: print(f'Identity for user `{user.username}` already exists, skipping') ...: continue ...: if attrs: ...: external_id = attrs['id'] ...: new_external_identity = ExternalIdentity() ...: new_external_identity.external_id = external_id - ...: new_external_identity.external_username = '{}-saml-{}'.format(user.username, user.user_id) + ...: new_external_identity.external_username = f'{user.username}-saml-{user.user_id}' ...: new_external_identity.provider_name = provider ...: new_external_identity.local_user_id = user.user_id ...: new_external_identity.access_token = '' @@ -76,7 +78,7 @@ Enter in the ishell prompt ...: new_external_identity.alt_token = '' ...: Session().add(ex_identity) ...: Session().commit() - ...: print('Set user `{}` external identity bound to ExternalID:{}'.format(user.username, external_id)) + ...: print(f'Set user `{user.username}` external identity bound to ExternalID:{external_id}') .. note:: diff --git a/docs/auth/auth-saml-duosecurity.rst b/docs/auth/auth-saml-duosecurity.rst index 1dadfdce..9a5ffd94 100644 --- a/docs/auth/auth-saml-duosecurity.rst +++ b/docs/auth/auth-saml-duosecurity.rst @@ -32,62 +32,118 @@ To configure Duo Security SAML authentication, use the following steps: Example SAML Duo Security configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Example configuration for SAML 2.0 with Duo Security provider:: +Example configuration for SAML 2.0 with Duo Security provider - *option*: `enabled` => `True` - # Enable or disable this authentication plugin. - *option*: `cache_ttl` => `0` - # Amount of seconds to cache the authentication and permissions check response call for this plugin. - # Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled). +Enabled + `True`: - *option*: `debug` => `True` - # Enable or disable debug mode that shows SAML errors in the RhodeCode logs. + .. note:: + Enable or disable this authentication plugin. - *option*: `entity_id` => `http://rc-app.com/dag/saml2/idp/metadata.php` - # Identity Provider entity/metadata URI. - # E.g. https://duo-gateway.com/dag/saml2/idp/metadata.php - *option*: `sso_service_url` => `http://rc-app.com/dag/saml2/idp/SSOService.php?spentityid=http://rc.local.pl/_admin/auth/duosecurity/saml-metadata` - # SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login - # E.g. https://duo-gateway.com/dag/saml2/idp/SSOService.php?spentityid= +Auth Cache TTL + `30`: - *option*: `slo_service_url` => `http://rc-app.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=http://rc-app.com/dag/module.php/duosecurity/logout.php` - # SLO (SingleLogout) endpoint URL of the IdP. - # E.g. https://duo-gateway.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=http://duo-gateway.com/_admin/saml/sign-out-endpoint + .. note:: + Amount of seconds to cache the authentication and permissions check response call for this plugin. + Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled). - *option*: `x509cert` => `` - # Identity provider public x509 certificate. It will be converted to single-line format without headers +Debug + `True`: - *option*: `name_id_format` => `sha-1` - # The format that specifies how the NameID is sent to the service provider. + .. note:: + Enable or disable debug mode that shows SAML errors in the RhodeCode logs. - *option*: `signature_algo` => `sha-256` - # Type of Algorithm to use for verification of SAML signature on Identity provider side - *option*: `digest_algo` => `sha-256` - # Type of Algorithm to use for verification of SAML digest on Identity provider side +Auth button name + `Azure Entra ID`: - *option*: `cert_dir` => `/etc/saml/` - # Optional directory to store service provider certificate and private keys. - # Expected certs for the SP should be stored in this folder as: - # * sp.key Private Key - # * sp.crt Public cert - # * sp_new.crt Future Public cert - # - # Also you can use other cert to sign the metadata of the SP using the: - # * metadata.key - # * metadata.crt + .. note:: + Alternative authentication display name. E.g AzureAuth, CorporateID etc. - *option*: `user_id_attribute` => `PersonImmutableID` - # User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id. - # Ensure this is returned from DuoSecurity for example via duo_username - *option*: `username_attribute` => `User.username` - # Username Attribute name. This defines which attribute in SAML response will map to an username. +Entity ID + `https://my-duo-gateway.com/dag/saml2/idp/metadata.php`: + + .. note:: + Identity Provider entity/metadata URI. + E.g. https://duo-gateway.com/dag/saml2/idp/metadata.php + +SSO URL + `https://duo-gateway.com/dag/saml2/idp/SSOService.php?spentityid=`: + + .. note:: + SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login, Known also as Login URL + E.g. http://rc-app.com/dag/saml2/idp/SSOService.php?spentityid=https://docker-dev/_admin/auth/duosecurity/saml-metadata + +SLO URL + `https://duo-gateway.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=`: + + .. note:: + SLO (SingleLogout) endpoint URL of the IdP. , Known also as Logout URL + E.g. http://rc-app.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=https://docker-dev/_admin/auth/duosecurity/saml-sign-out-endpoint + +x509cert + ``: + + .. note:: + Identity provider public x509 certificate. It will be converted to single-line format without headers. + Download the raw base64 encoded certificate from the Identity provider and paste it here. + +SAML Signature + `sha-256`: + + .. note:: + Type of Algorithm to use for verification of SAML signature on Identity provider side. + +SAML Digest + `sha-256`: + + .. note:: + Type of Algorithm to use for verification of SAML digest on Identity provider side. + +Service Provider Cert Dir + `/etc/rhodecode/conf/saml_ssl/`: + + .. note:: + Optional directory to store service provider certificate and private keys. + Expected certs for the SP should be stored in this folder as: + + * sp.key Private Key + * sp.crt Public cert + * sp_new.crt Future Public cert + + Also you can use other cert to sign the metadata of the SP using the: + + * metadata.key + * metadata.crt + +Expected NameID Format + `nameid-format:emailAddress`: + + .. note:: + The format that specifies how the NameID is sent to the service provider. + +User ID Attribute + `PersonImmutableID`: + + .. note:: + User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id. + Ensure this is returned from DuoSecurity for example via duo_username. + +Username Attribute + `User.username`: + + .. note:: + Username Attribute name. This defines which attribute in SAML response will map to a username. + +Email Attribute + `User.email`: + + .. note:: + Email Attribute name. This defines which attribute in SAML response will map to an email address. - *option*: `email_attribute` => `User.email` - # Email Attribute name. This defines which attribute in SAML response will map to an email address. Below is example setup from DUO Administration page that can be used with above config. diff --git a/docs/auth/auth-saml-generic.rst b/docs/auth/auth-saml-generic.rst index db6a6205..b5e528bc 100644 --- a/docs/auth/auth-saml-generic.rst +++ b/docs/auth/auth-saml-generic.rst @@ -15,5 +15,6 @@ Please check for reference two example providers: auth-saml-duosecurity auth-saml-onelogin + auth-saml-azure auth-saml-bulk-enroll-users diff --git a/docs/auth/auth-saml-onelogin.rst b/docs/auth/auth-saml-onelogin.rst index 69b2cc1d..31684cb8 100644 --- a/docs/auth/auth-saml-onelogin.rst +++ b/docs/auth/auth-saml-onelogin.rst @@ -32,62 +32,117 @@ To configure OneLogin SAML authentication, use the following steps: Example SAML OneLogin configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Example configuration for SAML 2.0 with OneLogin provider:: +Example configuration for SAML 2.0 with OneLogin provider - *option*: `enabled` => `True` - # Enable or disable this authentication plugin. - *option*: `cache_ttl` => `0` - # Amount of seconds to cache the authentication and permissions check response call for this plugin. - # Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled). +Enabled + `True`: - *option*: `debug` => `True` - # Enable or disable debug mode that shows SAML errors in the RhodeCode logs. + .. note:: + Enable or disable this authentication plugin. - *option*: `entity_id` => `https://app.onelogin.com/saml/metadata/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` - # Identity Provider entity/metadata URI. - # E.g. https://app.onelogin.com/saml/metadata/ - *option*: `sso_service_url` => `https://customer-domain.onelogin.com/trust/saml2/http-post/sso/xxxxxx` - # SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login - # E.g. https://app.onelogin.com/trust/saml2/http-post/sso/ +Auth Cache TTL + `30`: - *option*: `slo_service_url` => `https://customer-domain.onelogin.com/trust/saml2/http-redirect/slo/xxxxxx` - # SLO (SingleLogout) endpoint URL of the IdP. - # E.g. https://app.onelogin.com/trust/saml2/http-redirect/slo/ + .. note:: + Amount of seconds to cache the authentication and permissions check response call for this plugin. + Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled). - *option*: `x509cert` => `` - # Identity provider public x509 certificate. It will be converted to single-line format without headers +Debug + `True`: - *option*: `name_id_format` => `sha-1` - # The format that specifies how the NameID is sent to the service provider. + .. note:: + Enable or disable debug mode that shows SAML errors in the RhodeCode logs. - *option*: `signature_algo` => `sha-256` - # Type of Algorithm to use for verification of SAML signature on Identity provider side - *option*: `digest_algo` => `sha-256` - # Type of Algorithm to use for verification of SAML digest on Identity provider side +Auth button name + `Azure Entra ID`: - *option*: `cert_dir` => `/etc/saml/` - # Optional directory to store service provider certificate and private keys. - # Expected certs for the SP should be stored in this folder as: - # * sp.key Private Key - # * sp.crt Public cert - # * sp_new.crt Future Public cert - # - # Also you can use other cert to sign the metadata of the SP using the: - # * metadata.key - # * metadata.crt + .. note:: + Alternative authentication display name. E.g AzureAuth, CorporateID etc. - *option*: `user_id_attribute` => `PersonImmutableID` - # User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id. - # Ensure this is returned from OneLogin for example via Internal ID - *option*: `username_attribute` => `User.username` - # Username Attribute name. This defines which attribute in SAML response will map to an username. +Entity ID + `https://app.onelogin.com/saml/metadata/`: - *option*: `email_attribute` => `User.email` - # Email Attribute name. This defines which attribute in SAML response will map to an email address. + .. note:: + Identity Provider entity/metadata URI. + E.g. https://app.onelogin.com/saml/metadata/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + +SSO URL + `https://app.onelogin.com/trust/saml2/http-post/sso/`: + + .. note:: + SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login, Known also as Login URL + E.g. https://app.onelogin.com/trust/saml2/http-post/sso/ + +SLO URL + `https://app.onelogin.com/trust/saml2/http-redirect/slo/`: + + .. note:: + SLO (SingleLogout) endpoint URL of the IdP. , Known also as Logout URL + E.g. https://app.onelogin.com/trust/saml2/http-redirect/slo/ + +x509cert + ``: + + .. note:: + Identity provider public x509 certificate. It will be converted to single-line format without headers. + Download the raw base64 encoded certificate from the Identity provider and paste it here. + +SAML Signature + `sha-256`: + + .. note:: + Type of Algorithm to use for verification of SAML signature on Identity provider side. + +SAML Digest + `sha-256`: + + .. note:: + Type of Algorithm to use for verification of SAML digest on Identity provider side. + +Service Provider Cert Dir + `/etc/rhodecode/conf/saml_ssl/`: + + .. note:: + Optional directory to store service provider certificate and private keys. + Expected certs for the SP should be stored in this folder as: + + * sp.key Private Key + * sp.crt Public cert + * sp_new.crt Future Public cert + + Also you can use other cert to sign the metadata of the SP using the: + + * metadata.key + * metadata.crt + +Expected NameID Format + `nameid-format:emailAddress`: + + .. note:: + The format that specifies how the NameID is sent to the service provider. + +User ID Attribute + `PersonImmutableID`: + + .. note:: + User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id. + Ensure this is returned from DuoSecurity for example via duo_username. + +Username Attribute + `User.username`: + + .. note:: + Username Attribute name. This defines which attribute in SAML response will map to a username. + +Email Attribute + `User.email`: + + .. note:: + Email Attribute name. This defines which attribute in SAML response will map to an email address. diff --git a/docs/auth/auth.rst b/docs/auth/auth.rst index bdc7abdb..498e7178 100644 --- a/docs/auth/auth.rst +++ b/docs/auth/auth.rst @@ -29,6 +29,7 @@ administrator greater control over how users authenticate with the system. auth-saml-generic auth-saml-onelogin auth-saml-duosecurity + auth-saml-azure auth-crowd auth-pam ssh-connection diff --git a/docs/contributing/dev-setup.rst b/docs/contributing/dev-setup.rst index a2923473..4cb5ccb3 100644 --- a/docs/contributing/dev-setup.rst +++ b/docs/contributing/dev-setup.rst @@ -4,237 +4,8 @@ Development setup =================== - -RhodeCode Enterprise runs inside a Nix managed environment. This ensures build -environment dependencies are correctly declared and installed during setup. -It also enables atomic upgrades, rollbacks, and multiple instances of RhodeCode -Enterprise running with isolation. - -To set up RhodeCode Enterprise inside the Nix environment, use the following steps: - - - -Setup Nix Package Manager -------------------------- - -To install the Nix Package Manager, please run:: - - $ curl https://releases.nixos.org/nix/nix-2.3.4/install | sh - -or go to https://nixos.org/nix/ and follow the installation instructions. -Once this is correctly set up on your system, you should be able to use the -following commands: - -* `nix-env` - -* `nix-shell` - - -.. tip:: - - Update your channels frequently by running ``nix-channel --update``. - -.. note:: - - To uninstall nix run the following: - - remove the . "$HOME/.nix-profile/etc/profile.d/nix.sh" line in your ~/.profile or ~/.bash_profile - rm -rf $HOME/{.nix-channels,.nix-defexpr,.nix-profile,.config/nixpkgs} - sudo rm -rf /nix - -Switch nix to the latest STABLE channel ---------------------------------------- - -run:: - - nix-channel --add https://nixos.org/channels/nixos-20.03 nixpkgs - -Followed by:: - - nix-channel --update - nix-env -i nix-2.3.4 - - -Install required binaries -------------------------- - -We need some handy tools first. - -run:: - - nix-env -i nix-prefetch-hg - nix-env -i nix-prefetch-git - - -Speed up JS build by installing PhantomJS ------------------------------------------ - -PhantomJS will be downloaded each time nix-shell is invoked. To speed this by -setting already downloaded version do this:: - - nix-env -i phantomjs-2.1.1 - - # and set nix bin path - export PATH=$PATH:~/.nix-profile/bin - - -Clone the required repositories -------------------------------- - -After Nix is set up, clone the RhodeCode Enterprise Community Edition and -RhodeCode VCSServer repositories into the same directory. -RhodeCode currently is using Mercurial Version Control System, please make sure -you have it installed before continuing. - -To obtain the required sources, use the following commands:: - - mkdir rhodecode-develop && cd rhodecode-develop - hg clone -u default https://code.rhodecode.com/rhodecode-enterprise-ce - hg clone -u default https://code.rhodecode.com/rhodecode-vcsserver - -.. note:: - - If you cannot clone the repository, please contact us via support@rhodecode.com - - -Install some required libraries -------------------------------- - -There are some required drivers and dev libraries that we need to install to -test RhodeCode under different types of databases. For example in Ubuntu we -need to install the following. - -required libraries:: - - # svn related - sudo apt-get install libapr1-dev libaprutil1-dev - sudo apt-get install libsvn-dev - # libcurl required too - sudo apt-get install libcurl4-openssl-dev - # mysql/pg server for development, optional - sudo apt-get install mysql-server libmysqlclient-dev - sudo apt-get install postgresql postgresql-contrib libpq-dev - - - -Enter the Development Shell ---------------------------- - -The final step is to start the development shells. To do this, run the -following command from inside the cloned repository:: - - # first, the vcsserver - cd ~/rhodecode-vcsserver - nix-shell - - # then enterprise sources - cd ~/rhodecode-enterprise-ce - nix-shell - -.. note:: - - On the first run, this will take a while to download and optionally compile - a few things. The following runs will be faster. The development shell works - fine on both MacOS and Linux platforms. - - -Create config.nix for development ---------------------------------- - -In order to run proper tests and setup linking across projects, a config.nix -file needs to be setup:: - - # create config - mkdir -p ~/.nixpkgs - touch ~/.nixpkgs/config.nix - - # put the below content into the ~/.nixpkgs/config.nix file - # adjusts, the path to where you cloned your repositories. - - { - rc = { - sources = { - rhodecode-vcsserver = "/home/dev/rhodecode-vcsserver"; - rhodecode-enterprise-ce = "/home/dev/rhodecode-enterprise-ce"; - rhodecode-enterprise-ee = "/home/dev/rhodecode-enterprise-ee"; - }; - }; - } - - - -Creating a Development Configuration ------------------------------------- - -To create a development environment for RhodeCode Enterprise, -use the following steps: - -1. Create a copy of vcsserver config: - `cp ~/rhodecode-vcsserver/configs/development.ini ~/rhodecode-vcsserver/configs/dev.ini` -2. Create a copy of rhodocode config: - `cp ~/rhodecode-enterprise-ce/configs/development.ini ~/rhodecode-enterprise-ce/configs/dev.ini` -3. Adjust the configuration settings to your needs if needed. - -.. note:: - - It is recommended to use the name `dev.ini` since it's included in .hgignore file. - - -Setup the Development Database -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To create a development database, use the following example. This is a one -time operation executed from the nix-shell of rhodecode-enterprise-ce sources :: - - rc-setup-app dev.ini \ - --user=admin --password=secret \ - --email=admin@example.com \ - --repos=~/my_dev_repos - - -Compile CSS and JavaScript -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To use the application's frontend and prepare it for production deployment, -you will need to compile the CSS and JavaScript with Grunt. -This is easily done from within the nix-shell using the following command:: - - make web-build - -When developing new features you will need to recompile following any -changes made to the CSS or JavaScript files when developing the code:: - - grunt watch - -This prepares the development (with comments/whitespace) versions of files. - -Start the Development Servers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -From the rhodecode-vcsserver directory, start the development server in another -nix-shell, using the following command:: - - pserve configs/dev.ini - -In the adjacent nix-shell which you created for your development server, you may -now start CE with the following command:: - - - pserve --reload configs/dev.ini - -.. note:: - - `--reload` flag will automatically reload the server when source file changes. - - -Run the Environment Tests -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Please make sure that the tests are passing to verify that your environment is -set up correctly. RhodeCode uses py.test to run tests. -While your instance is running, start a new nix-shell and simply run -``make test`` to run the basic test suite. - +Please refer to RCstack installed documentation for instructions on setting up dev environment: +https://docs.rhodecode.com/rcstack/dev/dev-setup.html Need Help? ^^^^^^^^^^ diff --git a/docs/images/saml-azure-attributes-example.png b/docs/images/saml-azure-attributes-example.png new file mode 100644 index 00000000..97b50a84 Binary files /dev/null and b/docs/images/saml-azure-attributes-example.png differ diff --git a/docs/images/saml-azure-service-provider-example.png b/docs/images/saml-azure-service-provider-example.png new file mode 100644 index 00000000..0c2cb0e0 Binary files /dev/null and b/docs/images/saml-azure-service-provider-example.png differ diff --git a/docs/index.rst b/docs/index.rst index 114dff13..da3a5b33 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,6 +35,18 @@ and commit files and |repos| while managing their security permissions. Table of Contents ----------------- +.. toctree:: + :maxdepth: 1 + :caption: Documentation directory + + Back to documentation directory + +.. toctree:: + :maxdepth: 1 + :caption: RhodeCode RCstack Documentation + + RhodeCode RCstack Installer + .. toctree:: :maxdepth: 1 :caption: Admin Documentation diff --git a/docs/install/quick-start.rst b/docs/install/quick-start.rst index 382b5f78..27339400 100644 --- a/docs/install/quick-start.rst +++ b/docs/install/quick-start.rst @@ -66,6 +66,7 @@ Output should look similar to this: fb77fb6496c6 channelstream/channelstream:0.7.1 Up 2 hours (healthy) rc_cluster_services-channelstream-1 8000/tcp cb6c5c022f5b postgres:14.6 Up 2 hours (healthy) rc_cluster_services-database-1 5432/tcp + At this point you should be able to access: - RhodeCode instance at your domain entered, e.g http://rhodecode.local, the default access @@ -76,6 +77,7 @@ At this point you should be able to access: RHODECODE_USER_PASS=super-secret-password + .. note:: Recommended post quick start install instructions: @@ -85,7 +87,6 @@ At this point you should be able to access: * Set up :ref:`indexing-ref` * Familiarise yourself with the :ref:`rhodecode-admin-ref` section. -.. _rhodecode.com/download/: https://rhodecode.com/download/ .. _rhodecode.com: https://rhodecode.com/ .. _rhodecode.com/register: https://rhodecode.com/register/ .. _rhodecode.com/download: https://rhodecode.com/download/ diff --git a/docs/install/using-sqllite.rst b/docs/install/using-sqllite.rst index cf262faf..7754e8d1 100644 --- a/docs/install/using-sqllite.rst +++ b/docs/install/using-sqllite.rst @@ -1,23 +1,12 @@ .. _install-sqlite-database: -SQLite ------- +SQLite (Deprecated) +------------------- .. important:: - We do not recommend using SQLite in a large development environment - as it has an internal locking mechanism which can become a performance - bottleneck when there are more than 5 concurrent users. + As of 5.x, SQLite is no longer supported, we advise to migrate to MySQL or PostgreSQL. -|RCE| installs SQLite as the default database if you do not specify another -during installation. SQLite is suitable for small teams, -projects with a low load, and evaluation purposes since it is built into -|RCE| and does not require any additional database server. - -Using MySQL or PostgreSQL in an large setup gives you much greater -performance, and while migration tools exist to move from one database type -to another, it is better to get it right first time and to immediately use -MySQL or PostgreSQL when you deploy |RCE| in a production environment. Migrating From SQLite to PostgreSQL ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/known-issues/known-issues.rst b/docs/known-issues/known-issues.rst index 55f605d3..c34eb434 100644 --- a/docs/known-issues/known-issues.rst +++ b/docs/known-issues/known-issues.rst @@ -42,6 +42,7 @@ Newer Operating system locales the local-archive format, which is now incompatible with our used glibc 2.26. Mostly affected are: + - Fedora 23+ - Ubuntu 18.04 - CentOS / RHEL 8 @@ -93,3 +94,24 @@ example to pass the correct locale information on boot. [Install] WantedBy=multi-user.target + +Merge stucks in "merging" status +-------------------------------- + +Similar issues: + +- Pull Request duplicated and/or stucks in "creating" status. + +Mostly affected are: + +- Kubernetes AWS EKS setup with NFS as shared storage +- AWS EFS as shared storage + +Workaround: + +1. Manually clear the repo cache via UI: +:menuselection:`Repository Settings --> Caches --> Invalidate repository cache` + +1. Open problematic PR and reset status to "created" + +Now you can merge PR normally diff --git a/docs/release-notes/release-notes-5.1.0.rst b/docs/release-notes/release-notes-5.1.0.rst index 27999cbb..d33f5f06 100644 --- a/docs/release-notes/release-notes-5.1.0.rst +++ b/docs/release-notes/release-notes-5.1.0.rst @@ -10,20 +10,20 @@ Release Date New Features ^^^^^^^^^^^^ -- We've introduced 2FA for users. Now alongside the external auth 2fa support RhodeCode allows to enable 2FA for users +- We've introduced 2FA for users. Now alongside the external auth 2FA support RhodeCode allows to enable 2FA for users. 2FA options will be available for each user individually, or enforced via authentication plugins like ldap, or internal. - Email based log-in. RhodeCode now allows to log-in using email as well as username for main authentication type. - Ability to replace a file using web UI. Now one can replace an existing file from the web-ui. - GIT LFS Sync automation. Remote push/pull commands now can also sync GIT LFS objects. -- Added ability to remove or close branches from the web ui -- Added ability to delete a branch automatically after merging PR for git repositories -- Added support for S3 based archive_cache based that allows storing cached archives in S3 compatible object store. +- Added ability to remove or close branches from the web ui. +- Added ability to delete a branch automatically after merging PR for git repositories. +- Added support for S3 based archive_cache that allows storing cached archives in S3 compatible object store. General ^^^^^^^ -- Upgraded all dependency libraries to their latest available versions +- Upgraded all dependency libraries to their latest available versions. - Repository storage is no longer controlled via DB settings, but .ini file. This allows easier automated deployments. - Bumped mercurial to 6.7.4 - Mercurial: enable httppostarguments for better support of large repositories with lots of heads. @@ -39,21 +39,20 @@ Performance ^^^^^^^^^^^ - Introduced a full rewrite of ssh backend for performance. The result is 2-5x speed improvement for operation with ssh. - enable new ssh wrapper by setting: `ssh.wrapper_cmd = /home/rhodecode/venv/bin/rc-ssh-wrapper-v2` -- Introduced a new hooks subsystem that is more scalable and faster, enable it by settings: `vcs.hooks.protocol = celery` + Enable new ssh wrapper by setting: `ssh.wrapper_cmd = /home/rhodecode/venv/bin/rc-ssh-wrapper-v2` +- Introduced a new hooks subsystem that is more scalable and faster, enable it by setting: `vcs.hooks.protocol = celery` Fixes ^^^^^ -- Archives: Zip archive download breaks when a gitmodules file is present -- Branch permissions: fixed bug preventing to specify own rules from 4.X install -- SVN: refactored svn events, thus fixing support for it in dockerized env -- Fixed empty server url in PR link after push from cli +- Archives: Zip archive download breaks when a gitmodules file is present. +- Branch permissions: fixed bug preventing to specify own rules from 4.X install. +- SVN: refactored svn events, thus fixing support for it in dockerized environment. +- Fixed empty server url in PR link after push from cli. Upgrade notes ^^^^^^^^^^^^^ -- RhodeCode 5.1.0 is a mayor feature release after big 5.0.0 python3 migration. Happy to ship a first time feature - rich release +- RhodeCode 5.1.0 is a major feature release after big 5.0.0 python3 migration. Happy to ship a first time feature-rich release. diff --git a/docs/release-notes/release-notes-5.1.1.rst b/docs/release-notes/release-notes-5.1.1.rst new file mode 100644 index 00000000..cd03af11 --- /dev/null +++ b/docs/release-notes/release-notes-5.1.1.rst @@ -0,0 +1,40 @@ +|RCE| 5.1.1 |RNS| +----------------- + +Release Date +^^^^^^^^^^^^ + +- 2024-07-23 + + +New Features +^^^^^^^^^^^^ + + + +General +^^^^^^^ + + + +Security +^^^^^^^^ + + + +Performance +^^^^^^^^^^^ + + + + +Fixes +^^^^^ + +- Fixed problems with JS static files build + + +Upgrade notes +^^^^^^^^^^^^^ + +- RhodeCode 5.1.1 is unscheduled bugfix release to address some build issues with 5.1 images diff --git a/docs/release-notes/release-notes-5.1.2.rst b/docs/release-notes/release-notes-5.1.2.rst new file mode 100644 index 00000000..a3e3d03d --- /dev/null +++ b/docs/release-notes/release-notes-5.1.2.rst @@ -0,0 +1,41 @@ +|RCE| 5.1.2 |RNS| +----------------- + +Release Date +^^^^^^^^^^^^ + +- 2024-09-12 + + +New Features +^^^^^^^^^^^^ + + + +General +^^^^^^^ + + + +Security +^^^^^^^^ + + + +Performance +^^^^^^^^^^^ + + + + +Fixes +^^^^^ + +- Fixed problems with Mercurial authentication after enabling httppostargs. + Currently this protocol will be disabled until proper fix is in place + + +Upgrade notes +^^^^^^^^^^^^^ + +- RhodeCode 5.1.2 is unscheduled bugfix release to address some build issues with 5.1 images diff --git a/docs/release-notes/release-notes-5.2.0.rst b/docs/release-notes/release-notes-5.2.0.rst new file mode 100644 index 00000000..2aa43dbb --- /dev/null +++ b/docs/release-notes/release-notes-5.2.0.rst @@ -0,0 +1,55 @@ +|RCE| 5.2.0 |RNS| +----------------- + +Release Date +^^^^^^^^^^^^ + +- 2024-10-09 + + +New Features +^^^^^^^^^^^^ + +- New artifact storage engines allowing an s3 based uploads +- Enterprise version only: Added security tab to admin interface and possibility to whitelist specific vcs client versions. Some older versions clients have known security vulnerabilities, now you can disallow them. +- Enterprise version only: Implemented support for Azure SAML authentication + + +General +^^^^^^^ +- Bumped version of packaging, gunicorn, orjson, zope.interface and some other requirements +- Few tweaks and changes to saml plugins to allows easier setup +- Configs: allow json log format for gunicorn +- Configs: deprecated old ssh wrapper command and make the v2 the default one +- Make sure commit-caches propagate to parent repo groups +- Configs: Moved git lfs path and path of hg large files to ini file + +Security +^^^^^^^^ + + + +Performance +^^^^^^^^^^^ + +- description escaper for better performance + +Fixes +^^^^^ + +- Email notifications not working properly +- Removed waitress as a default runner +- Fixed issue with branch permissions +- Ldap: fixed nested groups extraction logic +- Fixed possible db corruption in case of filesystem problems +- Cleanup and improvements to documentation +- Added Kubernetes deployment section to the documentation +- Added default value to celery result and broker +- Fixed broken backends function after python3 migration +- Explicitly disable mercurial web_push ssl flag to prevent from errors about ssl required +- VCS: fixed problems with locked repos and with branch permissions reporting + +Upgrade notes +^^^^^^^^^^^^^ + +- RhodeCode 5.2.0 is a planned major release featuring Azure SAML, whitelist for client versions, s3 artifacts backend and more! diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst index 1f700624..711c584e 100644 --- a/docs/release-notes/release-notes.rst +++ b/docs/release-notes/release-notes.rst @@ -9,7 +9,9 @@ Release Notes .. toctree:: :maxdepth: 1 - + release-notes-5.2.0.rst + release-notes-5.1.2.rst + release-notes-5.1.1.rst release-notes-5.1.0.rst release-notes-5.0.3.rst release-notes-5.0.2.rst diff --git a/docs/requirements_docs.txt b/docs/requirements_docs.txt index fe173635..d20a433f 100644 --- a/docs/requirements_docs.txt +++ b/docs/requirements_docs.txt @@ -4,7 +4,7 @@ furo==2023.9.10 sphinx-press-theme==0.8.0 sphinx-rtd-theme==1.3.0 -pygments==2.16.1 +pygments==2.18.0 docutils<0.19 markupsafe==2.1.3 diff --git a/requirements.txt b/requirements.txt index a39720eb..d3c228b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ alembic==1.13.1 markupsafe==2.1.2 sqlalchemy==1.4.52 greenlet==3.0.3 - typing_extensions==4.9.0 + typing_extensions==4.12.2 async-timeout==4.0.3 babel==2.12.1 beaker==1.12.1 @@ -18,8 +18,8 @@ celery==5.3.6 click==8.1.3 click-repl==0.2.0 click==8.1.3 - prompt-toolkit==3.0.38 - wcwidth==0.2.6 + prompt_toolkit==3.0.47 + wcwidth==0.2.13 six==1.16.0 kombu==5.3.5 amqp==5.2.0 @@ -33,7 +33,7 @@ channelstream==0.7.1 gevent==24.2.1 greenlet==3.0.3 zope.event==5.0.0 - zope.interface==6.3.0 + zope.interface==7.0.3 itsdangerous==1.1.0 marshmallow==2.18.0 pyramid==2.0.2 @@ -46,7 +46,7 @@ channelstream==0.7.1 venusian==3.0.0 webob==1.8.7 zope.deprecation==5.0.0 - zope.interface==6.3.0 + zope.interface==7.0.3 pyramid-jinja2==2.10 jinja2==3.1.2 markupsafe==2.1.2 @@ -61,7 +61,7 @@ channelstream==0.7.1 venusian==3.0.0 webob==1.8.7 zope.deprecation==5.0.0 - zope.interface==6.3.0 + zope.interface==7.0.3 zope.deprecation==5.0.0 python-dateutil==2.8.2 six==1.16.0 @@ -87,32 +87,31 @@ dogpile.cache==1.3.3 pbr==5.11.1 formencode==2.1.0 six==1.16.0 -fsspec==2024.6.0 -gunicorn==21.2.0 - packaging==24.0 +fsspec==2024.9.0 +gunicorn==23.0.0 + packaging==24.1 gevent==24.2.1 greenlet==3.0.3 zope.event==5.0.0 - zope.interface==6.3.0 -ipython==8.14.0 - backcall==0.2.0 + zope.interface==7.0.3 +ipython==8.26.0 decorator==5.1.1 - jedi==0.19.0 - parso==0.8.3 - matplotlib-inline==0.1.6 - traitlets==5.9.0 - pexpect==4.8.0 + jedi==0.19.1 + parso==0.8.4 + matplotlib-inline==0.1.7 + traitlets==5.14.3 + pexpect==4.9.0 ptyprocess==0.7.0 - pickleshare==0.7.5 - prompt-toolkit==3.0.38 - wcwidth==0.2.6 - pygments==2.15.1 - stack-data==0.6.2 - asttokens==2.2.1 + prompt_toolkit==3.0.47 + wcwidth==0.2.13 + pygments==2.18.0 + stack-data==0.6.3 + asttokens==2.4.1 six==1.16.0 - executing==1.2.0 - pure-eval==0.2.2 - traitlets==5.9.0 + executing==2.0.1 + pure_eval==0.2.3 + traitlets==5.14.3 + typing_extensions==4.12.2 markdown==3.4.3 msgpack==1.0.8 mysqlclient==2.1.1 @@ -127,7 +126,7 @@ nbconvert==7.7.3 markupsafe==2.1.2 jupyter_core==5.3.1 platformdirs==3.10.0 - traitlets==5.9.0 + traitlets==5.14.3 jupyterlab-pygments==0.2.2 markupsafe==2.1.2 mistune==2.0.5 @@ -135,15 +134,15 @@ nbconvert==7.7.3 jupyter_client==8.3.0 jupyter_core==5.3.1 platformdirs==3.10.0 - traitlets==5.9.0 + traitlets==5.14.3 python-dateutil==2.8.2 six==1.16.0 pyzmq==25.0.0 tornado==6.2 - traitlets==5.9.0 + traitlets==5.14.3 jupyter_core==5.3.1 platformdirs==3.10.0 - traitlets==5.9.0 + traitlets==5.14.3 nbformat==5.9.2 fastjsonschema==2.18.0 jsonschema==4.18.6 @@ -151,9 +150,9 @@ nbconvert==7.7.3 pyrsistent==0.19.3 jupyter_core==5.3.1 platformdirs==3.10.0 - traitlets==5.9.0 - traitlets==5.9.0 - traitlets==5.9.0 + traitlets==5.14.3 + traitlets==5.14.3 + traitlets==5.14.3 nbformat==5.9.2 fastjsonschema==2.18.0 jsonschema==4.18.6 @@ -161,20 +160,20 @@ nbconvert==7.7.3 pyrsistent==0.19.3 jupyter_core==5.3.1 platformdirs==3.10.0 - traitlets==5.9.0 - traitlets==5.9.0 + traitlets==5.14.3 + traitlets==5.14.3 pandocfilters==1.5.0 - pygments==2.15.1 + pygments==2.18.0 tinycss2==1.2.1 webencodings==0.5.1 - traitlets==5.9.0 -orjson==3.10.3 + traitlets==5.14.3 +orjson==3.10.7 paste==3.10.1 premailer==3.10.0 cachetools==5.3.3 cssselect==1.2.0 cssutils==2.6.0 - lxml==4.9.3 + lxml==5.3.0 requests==2.28.2 certifi==2022.12.7 charset-normalizer==3.1.0 @@ -191,33 +190,6 @@ pycurl==7.45.3 pymysql==1.0.3 pyotp==2.8.0 pyparsing==3.1.1 -pyramid-debugtoolbar==4.12.1 - pygments==2.15.1 - pyramid==2.0.2 - hupper==1.12 - plaster==1.1.2 - plaster-pastedeploy==1.0.1 - pastedeploy==3.1.0 - plaster==1.1.2 - translationstring==1.4 - venusian==3.0.0 - webob==1.8.7 - zope.deprecation==5.0.0 - zope.interface==6.3.0 - pyramid-mako==1.1.0 - mako==1.2.4 - markupsafe==2.1.2 - pyramid==2.0.2 - hupper==1.12 - plaster==1.1.2 - plaster-pastedeploy==1.0.1 - pastedeploy==3.1.0 - plaster==1.1.2 - translationstring==1.4 - venusian==3.0.0 - webob==1.8.7 - zope.deprecation==5.0.0 - zope.interface==6.3.0 pyramid-mailer==0.15.1 pyramid==2.0.2 hupper==1.12 @@ -229,13 +201,27 @@ pyramid-mailer==0.15.1 venusian==3.0.0 webob==1.8.7 zope.deprecation==5.0.0 - zope.interface==6.3.0 + zope.interface==7.0.3 repoze.sendmail==4.4.1 - transaction==3.1.0 - zope.interface==6.3.0 - zope.interface==6.3.0 - transaction==3.1.0 - zope.interface==6.3.0 + transaction==5.0.0 + zope.interface==7.0.3 + zope.interface==7.0.3 + transaction==5.0.0 + zope.interface==7.0.3 +pyramid-mako==1.1.0 + mako==1.2.4 + markupsafe==2.1.2 + pyramid==2.0.2 + hupper==1.12 + plaster==1.1.2 + plaster-pastedeploy==1.0.1 + pastedeploy==3.1.0 + plaster==1.1.2 + translationstring==1.4 + venusian==3.0.0 + webob==1.8.7 + zope.deprecation==5.0.0 + zope.interface==7.0.3 python-ldap==3.4.3 pyasn1==0.4.8 pyasn1-modules==0.2.8 @@ -243,20 +229,20 @@ python-ldap==3.4.3 python-memcached==1.59 six==1.16.0 python-pam==2.0.2 -python3-saml==1.15.0 +python3-saml==1.16.0 isodate==0.6.1 six==1.16.0 - lxml==4.9.3 - xmlsec==1.3.13 - lxml==4.9.3 + lxml==5.3.0 + xmlsec==1.3.14 + lxml==5.3.0 pyyaml==6.0.1 -redis==5.0.4 +redis==5.1.0 async-timeout==4.0.3 regex==2022.10.31 routes==2.5.1 repoze.lru==0.7 six==1.16.0 -s3fs==2024.6.0 +s3fs==2024.9.0 aiobotocore==2.13.0 aiohttp==3.9.5 aiosignal==1.3.1 @@ -283,7 +269,7 @@ s3fs==2024.6.0 yarl==1.9.4 idna==3.4 multidict==6.0.5 - fsspec==2024.6.0 + fsspec==2024.9.0 simplejson==3.19.2 sshpubkeys==3.3.1 cryptography==40.0.2 @@ -293,7 +279,7 @@ sshpubkeys==3.3.1 six==1.16.0 sqlalchemy==1.4.52 greenlet==3.0.3 - typing_extensions==4.9.0 + typing_extensions==4.12.2 supervisor==4.2.5 tzlocal==4.3 pytz-deprecation-shim==0.1.0.post0 diff --git a/requirements_debug.txt b/requirements_debug.txt index 4051d106..a8d517a4 100644 --- a/requirements_debug.txt +++ b/requirements_debug.txt @@ -9,6 +9,7 @@ pympler ipdb ipython rich +pyramid-debugtoolbar # format flake8 diff --git a/requirements_test.txt b/requirements_test.txt index dafed0fa..1e7cbcc6 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -4,38 +4,38 @@ pytest-cov==4.1.0 coverage==7.4.3 pytest==8.1.1 iniconfig==2.0.0 - packaging==24.0 + packaging==24.1 pluggy==1.4.0 pytest-env==1.1.3 pytest==8.1.1 iniconfig==2.0.0 - packaging==24.0 + packaging==24.1 pluggy==1.4.0 pytest-profiling==1.7.0 gprof2dot==2022.7.29 pytest==8.1.1 iniconfig==2.0.0 - packaging==24.0 + packaging==24.1 pluggy==1.4.0 six==1.16.0 pytest-rerunfailures==13.0 - packaging==24.0 + packaging==24.1 pytest==8.1.1 iniconfig==2.0.0 - packaging==24.0 + packaging==24.1 pluggy==1.4.0 pytest-runner==6.0.1 pytest-sugar==1.0.0 - packaging==24.0 + packaging==24.1 pytest==8.1.1 iniconfig==2.0.0 - packaging==24.0 + packaging==24.1 pluggy==1.4.0 termcolor==2.4.0 pytest-timeout==2.3.1 pytest==8.1.1 iniconfig==2.0.0 - packaging==24.0 + packaging==24.1 pluggy==1.4.0 webtest==3.0.0 beautifulsoup4==4.12.3 diff --git a/rhodecode/VERSION b/rhodecode/VERSION index 1b47e8f3..91ff5727 100644 --- a/rhodecode/VERSION +++ b/rhodecode/VERSION @@ -1 +1 @@ -5.1.2 \ No newline at end of file +5.2.0 diff --git a/rhodecode/api/__init__.py b/rhodecode/api/__init__.py index 965583c3..60326ae0 100644 --- a/rhodecode/api/__init__.py +++ b/rhodecode/api/__init__.py @@ -40,6 +40,7 @@ from rhodecode.lib import ext_json from rhodecode.lib.utils2 import safe_str from rhodecode.lib.plugins.utils import get_plugin_settings from rhodecode.model.db import User, UserApiKeys +from rhodecode.config.patches import inspect_getargspec log = logging.getLogger(__name__) @@ -186,7 +187,6 @@ def request_view(request): exposed method """ # cython compatible inspect - from rhodecode.config.patches import inspect_getargspec inspect = inspect_getargspec() # check if we can find this session using api_key, get_by_auth_token diff --git a/rhodecode/api/views/server_api.py b/rhodecode/api/views/server_api.py index c93b55fe..1494dff1 100644 --- a/rhodecode/api/views/server_api.py +++ b/rhodecode/api/views/server_api.py @@ -33,7 +33,7 @@ from rhodecode.lib.ext_json import json from rhodecode.lib.utils2 import safe_int from rhodecode.model.db import UserIpMap from rhodecode.model.scm import ScmModel -from rhodecode.apps.file_store import utils +from rhodecode.apps.file_store import utils as store_utils from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \ FileOverSizeException @@ -328,8 +328,8 @@ def get_method(request, apiuser, pattern=Optional('*')): ] error : null """ - from rhodecode.config.patches import inspect_getargspec - inspect = inspect_getargspec() + from rhodecode.config import patches + inspect = patches.inspect_getargspec() if not has_superadmin_permission(apiuser): raise JSONRPCForbidden() diff --git a/rhodecode/apps/admin/__init__.py b/rhodecode/apps/admin/__init__.py index 4e9a5b39..f5e8cc9c 100644 --- a/rhodecode/apps/admin/__init__.py +++ b/rhodecode/apps/admin/__init__.py @@ -43,7 +43,38 @@ def admin_routes(config): from rhodecode.apps.admin.views.system_info import AdminSystemInfoSettingsView from rhodecode.apps.admin.views.user_groups import AdminUserGroupsView from rhodecode.apps.admin.views.users import AdminUsersView, UsersView - + from rhodecode.apps.admin.views.security import AdminSecurityView + + # Security EE feature + + config.add_route( + 'admin_security', + pattern='/security') + config.add_view( + AdminSecurityView, + attr='security', + route_name='admin_security', request_method='GET', + renderer='rhodecode:templates/admin/security/security.mako') + + config.add_route( + name='admin_security_update', + pattern='/security/update') + config.add_view( + AdminSecurityView, + attr='security_update', + route_name='admin_security_update', request_method='POST', + renderer='rhodecode:templates/admin/security/security.mako') + + config.add_route( + name='admin_security_modify_allowed_vcs_client_versions', + pattern=ADMIN_PREFIX + '/security/modify/allowed_vcs_client_versions') + config.add_view( + AdminSecurityView, + attr='vcs_whitelisted_client_versions_edit', + route_name='admin_security_modify_allowed_vcs_client_versions', request_method=('GET', 'POST'), + renderer='rhodecode:templates/admin/security/edit_allowed_vcs_client_versions.mako') + + config.add_route( name='admin_audit_logs', pattern='/audit_logs') diff --git a/rhodecode/apps/admin/views/security.py b/rhodecode/apps/admin/views/security.py new file mode 100644 index 00000000..c15294a6 --- /dev/null +++ b/rhodecode/apps/admin/views/security.py @@ -0,0 +1,46 @@ +# Copyright (C) 2010-2024 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import logging + +from rhodecode.apps._base import BaseAppView +from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator + +log = logging.getLogger(__name__) + + +class AdminSecurityView(BaseAppView): + + def load_default_context(self): + c = self._get_local_tmpl_context() + return c + + @LoginRequired() + @HasPermissionAllDecorator('hg.admin') + def security(self): + c = self.load_default_context() + c.active = 'security' + return self._get_template_context(c) + + + @LoginRequired() + @HasPermissionAllDecorator('hg.admin') + def admin_security_modify_allowed_vcs_client_versions(self): + c = self.load_default_context() + c.active = 'security' + return self._get_template_context(c) diff --git a/rhodecode/apps/admin/views/settings.py b/rhodecode/apps/admin/views/settings.py index 689ffa8a..4f226076 100644 --- a/rhodecode/apps/admin/views/settings.py +++ b/rhodecode/apps/admin/views/settings.py @@ -75,14 +75,21 @@ class AdminSettingsView(BaseAppView): if not ret: raise Exception('Could not get application ui settings !') - settings = {} + settings = { + # legacy param that needs to be kept + 'web_push_ssl': False + } for each in ret: k = each.ui_key v = each.ui_value + # skip some options if they are defined + if k in ['push_ssl']: + continue + if k == '/': k = 'root_path' - if k in ['push_ssl', 'publish', 'enabled']: + if k in ['publish', 'enabled']: v = str2bool(v) if k.find('.') != -1: @@ -92,6 +99,7 @@ class AdminSettingsView(BaseAppView): v = each.ui_active settings[each.ui_section + '_' + k] = v + return settings @classmethod @@ -164,7 +172,6 @@ class AdminSettingsView(BaseAppView): return Response(html) try: - model.update_global_ssl_setting(form_result['web_push_ssl']) model.update_global_hook_settings(form_result) model.create_or_update_global_svn_settings(form_result) diff --git a/rhodecode/apps/admin/views/system_info.py b/rhodecode/apps/admin/views/system_info.py index 84efabeb..c36b2e2a 100644 --- a/rhodecode/apps/admin/views/system_info.py +++ b/rhodecode/apps/admin/views/system_info.py @@ -171,11 +171,17 @@ class AdminSystemInfoSettingsView(BaseAppView): (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')), ('', '', ''), # spacer - (_('Archive cache storage type'), val('storage_archive')['type'], state('storage_archive')), + (_('Artifacts storage backend'), val('storage_artifacts')['type'], state('storage_artifacts')), + (_('Artifacts storage location'), val('storage_artifacts')['path'], state('storage_artifacts')), + (_('Artifacts info'), val('storage_artifacts')['text'], state('storage_artifacts')), + ('', '', ''), # spacer + + (_('Archive cache storage backend'), val('storage_archive')['type'], state('storage_archive')), (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')), (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')), ('', '', ''), # spacer + (_('Temp storage location'), val('storage_temp')['path'], state('storage_temp')), (_('Temp storage info'), val('storage_temp')['text'], state('storage_temp')), ('', '', ''), # spacer diff --git a/rhodecode/apps/file_store/__init__.py b/rhodecode/apps/file_store/__init__.py index 9bfd1615..a7254497 100755 --- a/rhodecode/apps/file_store/__init__.py +++ b/rhodecode/apps/file_store/__init__.py @@ -16,7 +16,8 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ import os -from rhodecode.apps.file_store import config_keys + + from rhodecode.config.settings_maker import SettingsMaker @@ -24,18 +25,48 @@ def _sanitize_settings_and_apply_defaults(settings): """ Set defaults, convert to python types and validate settings. """ + from rhodecode.apps.file_store import config_keys + + # translate "legacy" params into new config + settings.pop(config_keys.deprecated_enabled, True) + if config_keys.deprecated_backend in settings: + # if legacy backend key is detected we use "legacy" backward compat setting + settings.pop(config_keys.deprecated_backend) + settings[config_keys.backend_type] = config_keys.backend_legacy_filesystem + + if config_keys.deprecated_store_path in settings: + store_path = settings.pop(config_keys.deprecated_store_path) + settings[config_keys.legacy_filesystem_storage_path] = store_path + settings_maker = SettingsMaker(settings) - settings_maker.make_setting(config_keys.enabled, True, parser='bool') - settings_maker.make_setting(config_keys.backend, 'local') + default_cache_dir = settings['cache_dir'] + default_store_dir = os.path.join(default_cache_dir, 'artifacts_filestore') - default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store') - settings_maker.make_setting(config_keys.store_path, default_store) + # set default backend + settings_maker.make_setting(config_keys.backend_type, config_keys.backend_legacy_filesystem) + + # legacy filesystem defaults + settings_maker.make_setting(config_keys.legacy_filesystem_storage_path, default_store_dir, default_when_empty=True, ) + + # filesystem defaults + settings_maker.make_setting(config_keys.filesystem_storage_path, default_store_dir, default_when_empty=True,) + settings_maker.make_setting(config_keys.filesystem_shards, 8, parser='int') + + # objectstore defaults + settings_maker.make_setting(config_keys.objectstore_url, 'http://s3-minio:9000') + settings_maker.make_setting(config_keys.objectstore_bucket, 'rhodecode-artifacts-filestore') + settings_maker.make_setting(config_keys.objectstore_bucket_shards, 8, parser='int') + + settings_maker.make_setting(config_keys.objectstore_region, '') + settings_maker.make_setting(config_keys.objectstore_key, '') + settings_maker.make_setting(config_keys.objectstore_secret, '') settings_maker.env_expand() def includeme(config): + from rhodecode.apps.file_store.views import FileStoreView settings = config.registry.settings diff --git a/rhodecode/apps/file_store/backends/base.py b/rhodecode/apps/file_store/backends/base.py new file mode 100644 index 00000000..7093c6ad --- /dev/null +++ b/rhodecode/apps/file_store/backends/base.py @@ -0,0 +1,269 @@ +# Copyright (C) 2016-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import os +import fsspec # noqa +import logging + +from rhodecode.lib.ext_json import json + +from rhodecode.apps.file_store.utils import sha256_safe, ShardFileReader, get_uid_filename +from rhodecode.apps.file_store.extensions import resolve_extensions +from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException # noqa: F401 + +log = logging.getLogger(__name__) + + +class BaseShard: + + metadata_suffix: str = '.metadata' + storage_type: str = '' + fs = None + + @property + def storage_medium(self): + if not self.storage_type: + raise ValueError('No storage type set for this shard storage_type=""') + return getattr(self, self.storage_type) + + def __contains__(self, key): + full_path = self.store_path(key) + return self.fs.exists(full_path) + + def metadata_convert(self, uid_filename, metadata): + return metadata + + def get_metadata_filename(self, uid_filename) -> tuple[str, str]: + metadata_file: str = f'{uid_filename}{self.metadata_suffix}' + return metadata_file, self.store_path(metadata_file) + + def get_metadata(self, uid_filename, ignore_missing=False) -> dict: + _metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename) + if ignore_missing and not self.fs.exists(metadata_file_path): + return {} + + with self.fs.open(metadata_file_path, 'rb') as f: + metadata = json.loads(f.read()) + + metadata = self.metadata_convert(uid_filename, metadata) + return metadata + + def _store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs): + raise NotImplementedError + + def store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs): + return self._store(key, uid_key, value_reader, max_filesize, metadata, **kwargs) + + def _fetch(self, key, presigned_url_expires: int = 0): + raise NotImplementedError + + def fetch(self, key, **kwargs) -> tuple[ShardFileReader, dict]: + return self._fetch(key) + + def _delete(self, key): + if key not in self: + log.exception(f'requested key={key} not found in {self}') + raise KeyError(key) + + metadata = self.get_metadata(key) + _metadata_file, metadata_file_path = self.get_metadata_filename(key) + artifact_file_path = metadata['filename_uid_path'] + self.fs.rm(artifact_file_path) + self.fs.rm(metadata_file_path) + + return 1 + + def delete(self, key): + raise NotImplementedError + + def store_path(self, uid_filename): + raise NotImplementedError + + +class BaseFileStoreBackend: + _shards = tuple() + _shard_cls = BaseShard + _config: dict | None = None + _storage_path: str = '' + + def __init__(self, settings, extension_groups=None): + self._config = settings + extension_groups = extension_groups or ['any'] + self.extensions = resolve_extensions([], groups=extension_groups) + + def __contains__(self, key): + return self.filename_exists(key) + + def __repr__(self): + return f'<{self.__class__.__name__}(storage={self.storage_path})>' + + @property + def storage_path(self): + return self._storage_path + + @classmethod + def get_shard_index(cls, filename: str, num_shards) -> int: + # Generate a hash value from the filename + hash_value = sha256_safe(filename) + + # Convert the hash value to an integer + hash_int = int(hash_value, 16) + + # Map the hash integer to a shard number between 1 and num_shards + shard_number = (hash_int % num_shards) + + return shard_number + + @classmethod + def apply_counter(cls, counter: int, filename: str) -> str: + """ + Apply a counter to the filename. + + :param counter: The counter value to apply. + :param filename: The original filename. + :return: The modified filename with the counter. + """ + name_counted = f'{counter:d}-{filename}' + return name_counted + + def _get_shard(self, key) -> _shard_cls: + index = self.get_shard_index(key, len(self._shards)) + shard = self._shards[index] + return shard + + def get_conf(self, key, pop=False): + if key not in self._config: + raise ValueError( + f"No configuration key '{key}', please make sure it exists in filestore config") + val = self._config[key] + if pop: + del self._config[key] + return val + + def filename_allowed(self, filename, extensions=None): + """Checks if a filename has an allowed extension + + :param filename: base name of file + :param extensions: iterable of extensions (or self.extensions) + """ + _, ext = os.path.splitext(filename) + return self.extension_allowed(ext, extensions) + + def extension_allowed(self, ext, extensions=None): + """ + Checks if an extension is permitted. Both e.g. ".jpg" and + "jpg" can be passed in. Extension lookup is case-insensitive. + + :param ext: extension to check + :param extensions: iterable of extensions to validate against (or self.extensions) + """ + def normalize_ext(_ext): + if _ext.startswith('.'): + _ext = _ext[1:] + return _ext.lower() + + extensions = extensions or self.extensions + if not extensions: + return True + + ext = normalize_ext(ext) + + return ext in [normalize_ext(x) for x in extensions] + + def filename_exists(self, uid_filename): + shard = self._get_shard(uid_filename) + return uid_filename in shard + + def store_path(self, uid_filename): + """ + Returns absolute file path of the uid_filename + """ + shard = self._get_shard(uid_filename) + return shard.store_path(uid_filename) + + def store_metadata(self, uid_filename): + shard = self._get_shard(uid_filename) + return shard.get_metadata_filename(uid_filename) + + def store(self, filename, value_reader, extensions=None, metadata=None, max_filesize=None, randomized_name=True, **kwargs): + extensions = extensions or self.extensions + + if not self.filename_allowed(filename, extensions): + msg = f'filename {filename} does not allow extensions {extensions}' + raise FileNotAllowedException(msg) + + # # TODO: check why we need this setting ? it looks stupid... + # no_body_seek is used in stream mode importer somehow + # no_body_seek = kwargs.pop('no_body_seek', False) + # if no_body_seek: + # pass + # else: + # value_reader.seek(0) + + uid_filename = kwargs.pop('uid_filename', None) + if uid_filename is None: + uid_filename = get_uid_filename(filename, randomized=randomized_name) + + shard = self._get_shard(uid_filename) + + return shard.store(filename, uid_filename, value_reader, max_filesize, metadata, **kwargs) + + def import_to_store(self, value_reader, org_filename, uid_filename, metadata, **kwargs): + shard = self._get_shard(uid_filename) + max_filesize = None + return shard.store(org_filename, uid_filename, value_reader, max_filesize, metadata, import_mode=True) + + def delete(self, uid_filename): + shard = self._get_shard(uid_filename) + return shard.delete(uid_filename) + + def fetch(self, uid_filename) -> tuple[ShardFileReader, dict]: + shard = self._get_shard(uid_filename) + return shard.fetch(uid_filename) + + def get_metadata(self, uid_filename, ignore_missing=False) -> dict: + shard = self._get_shard(uid_filename) + return shard.get_metadata(uid_filename, ignore_missing=ignore_missing) + + def iter_keys(self): + for shard in self._shards: + if shard.fs.exists(shard.storage_medium): + for path, _dirs, _files in shard.fs.walk(shard.storage_medium): + for key_file_path in _files: + if key_file_path.endswith(shard.metadata_suffix): + yield shard, key_file_path + + def iter_artifacts(self): + for shard, key_file in self.iter_keys(): + json_key = f"{shard.storage_medium}/{key_file}" + with shard.fs.open(json_key, 'rb') as f: + yield shard, json.loads(f.read())['filename_uid'] + + def get_statistics(self): + total_files = 0 + total_size = 0 + meta = {} + + for shard, key_file in self.iter_keys(): + json_key = f"{shard.storage_medium}/{key_file}" + with shard.fs.open(json_key, 'rb') as f: + total_files += 1 + metadata = json.loads(f.read()) + total_size += metadata['size'] + + return total_files, total_size, meta diff --git a/rhodecode/apps/file_store/backends/filesystem.py b/rhodecode/apps/file_store/backends/filesystem.py new file mode 100644 index 00000000..bff1689c --- /dev/null +++ b/rhodecode/apps/file_store/backends/filesystem.py @@ -0,0 +1,183 @@ +# Copyright (C) 2016-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import os +import hashlib +import functools +import time +import logging + +from .. import config_keys +from ..exceptions import FileOverSizeException +from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader + +from ....lib.ext_json import json + +log = logging.getLogger(__name__) + + +class FileSystemShard(BaseShard): + METADATA_VER = 'v2' + BACKEND_TYPE = config_keys.backend_filesystem + storage_type: str = 'directory' + + def __init__(self, index, directory, directory_folder, fs, **settings): + self._index: int = index + self._directory: str = directory + self._directory_folder: str = directory_folder + self.fs = fs + + @property + def directory(self) -> str: + """Cache directory final path.""" + return os.path.join(self._directory, self._directory_folder) + + def _write_file(self, full_path, iterator, max_filesize, mode='wb'): + + # ensure dir exists + destination, _ = os.path.split(full_path) + if not self.fs.exists(destination): + self.fs.makedirs(destination) + + writer = self.fs.open(full_path, mode) + + digest = hashlib.sha256() + oversize_cleanup = False + with writer: + size = 0 + for chunk in iterator: + size += len(chunk) + digest.update(chunk) + writer.write(chunk) + + if max_filesize and size > max_filesize: + oversize_cleanup = True + # free up the copied file, and raise exc + break + + writer.flush() + # Get the file descriptor + fd = writer.fileno() + + # Sync the file descriptor to disk, helps with NFS cases... + os.fsync(fd) + + if oversize_cleanup: + self.fs.rm(full_path) + raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}') + + sha256 = digest.hexdigest() + log.debug('written new artifact under %s, sha256: %s', full_path, sha256) + return size, sha256 + + def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs): + + filename = key + uid_filename = uid_key + full_path = self.store_path(uid_filename) + + # STORE METADATA + _metadata = { + "version": self.METADATA_VER, + "store_type": self.BACKEND_TYPE, + + "filename": filename, + "filename_uid_path": full_path, + "filename_uid": uid_filename, + "sha256": "", # NOTE: filled in by reader iteration + + "store_time": time.time(), + + "size": 0 + } + + if metadata: + if kwargs.pop('import_mode', False): + # in import mode, we don't need to compute metadata, we just take the old version + _metadata["import_mode"] = True + else: + _metadata.update(metadata) + + read_iterator = iter(functools.partial(value_reader.read, 2**22), b'') + size, sha256 = self._write_file(full_path, read_iterator, max_filesize) + _metadata['size'] = size + _metadata['sha256'] = sha256 + + # after storing the artifacts, we write the metadata present + _metadata_file, metadata_file_path = self.get_metadata_filename(uid_key) + + with self.fs.open(metadata_file_path, 'wb') as f: + f.write(json.dumps(_metadata)) + + return uid_filename, _metadata + + def store_path(self, uid_filename): + """ + Returns absolute file path of the uid_filename + """ + return os.path.join(self._directory, self._directory_folder, uid_filename) + + def _fetch(self, key, presigned_url_expires: int = 0): + if key not in self: + log.exception(f'requested key={key} not found in {self}') + raise KeyError(key) + + metadata = self.get_metadata(key) + + file_path = metadata['filename_uid_path'] + if presigned_url_expires and presigned_url_expires > 0: + metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires) + + return ShardFileReader(self.fs.open(file_path, 'rb')), metadata + + def delete(self, key): + return self._delete(key) + + +class FileSystemBackend(BaseFileStoreBackend): + shard_name: str = 'shard_{:03d}' + _shard_cls = FileSystemShard + + def __init__(self, settings): + super().__init__(settings) + + store_dir = self.get_conf(config_keys.filesystem_storage_path) + directory = os.path.expanduser(store_dir) + + self._directory = directory + self._storage_path = directory # common path for all from BaseCache + self._shard_count = int(self.get_conf(config_keys.filesystem_shards, pop=True)) + if self._shard_count < 1: + raise ValueError(f'{config_keys.filesystem_shards} must be 1 or more') + + log.debug('Initializing %s file_store instance', self) + fs = fsspec.filesystem('file') + + if not fs.exists(self._directory): + fs.makedirs(self._directory, exist_ok=True) + + self._shards = tuple( + self._shard_cls( + index=num, + directory=directory, + directory_folder=self.shard_name.format(num), + fs=fs, + **settings, + ) + for num in range(self._shard_count) + ) diff --git a/rhodecode/apps/file_store/backends/filesystem_legacy.py b/rhodecode/apps/file_store/backends/filesystem_legacy.py new file mode 100644 index 00000000..a278e8ed --- /dev/null +++ b/rhodecode/apps/file_store/backends/filesystem_legacy.py @@ -0,0 +1,278 @@ +# Copyright (C) 2016-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import errno +import os +import hashlib +import functools +import time +import logging + +from .. import config_keys +from ..exceptions import FileOverSizeException +from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader + +from ....lib.ext_json import json + +log = logging.getLogger(__name__) + + +class LegacyFileSystemShard(BaseShard): + # legacy ver + METADATA_VER = 'v2' + BACKEND_TYPE = config_keys.backend_legacy_filesystem + storage_type: str = 'dir_struct' + + # legacy suffix + metadata_suffix: str = '.meta' + + @classmethod + def _sub_store_from_filename(cls, filename): + return filename[:2] + + @classmethod + def apply_counter(cls, counter, filename): + name_counted = '%d-%s' % (counter, filename) + return name_counted + + @classmethod + def safe_make_dirs(cls, dir_path): + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + return + + @classmethod + def resolve_name(cls, name, directory): + """ + Resolves a unique name and the correct path. If a filename + for that path already exists then a numeric prefix with values > 0 will be + added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix. + + :param name: base name of file + :param directory: absolute directory path + """ + + counter = 0 + while True: + name_counted = cls.apply_counter(counter, name) + + # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file + sub_store: str = cls._sub_store_from_filename(name_counted) + sub_store_path: str = os.path.join(directory, sub_store) + cls.safe_make_dirs(sub_store_path) + + path = os.path.join(sub_store_path, name_counted) + if not os.path.exists(path): + return name_counted, path + counter += 1 + + def __init__(self, index, directory, directory_folder, fs, **settings): + self._index: int = index + self._directory: str = directory + self._directory_folder: str = directory_folder + self.fs = fs + + @property + def dir_struct(self) -> str: + """Cache directory final path.""" + return os.path.join(self._directory, '0-') + + def _write_file(self, full_path, iterator, max_filesize, mode='wb'): + + # ensure dir exists + destination, _ = os.path.split(full_path) + if not self.fs.exists(destination): + self.fs.makedirs(destination) + + writer = self.fs.open(full_path, mode) + + digest = hashlib.sha256() + oversize_cleanup = False + with writer: + size = 0 + for chunk in iterator: + size += len(chunk) + digest.update(chunk) + writer.write(chunk) + + if max_filesize and size > max_filesize: + # free up the copied file, and raise exc + oversize_cleanup = True + break + + writer.flush() + # Get the file descriptor + fd = writer.fileno() + + # Sync the file descriptor to disk, helps with NFS cases... + os.fsync(fd) + + if oversize_cleanup: + self.fs.rm(full_path) + raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}') + + sha256 = digest.hexdigest() + log.debug('written new artifact under %s, sha256: %s', full_path, sha256) + return size, sha256 + + def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs): + + filename = key + uid_filename = uid_key + + # NOTE:, also apply N- Counter... + uid_filename, full_path = self.resolve_name(uid_filename, self._directory) + + # STORE METADATA + # TODO: make it compatible, and backward proof + _metadata = { + "version": self.METADATA_VER, + + "filename": filename, + "filename_uid_path": full_path, + "filename_uid": uid_filename, + "sha256": "", # NOTE: filled in by reader iteration + + "store_time": time.time(), + + "size": 0 + } + if metadata: + _metadata.update(metadata) + + read_iterator = iter(functools.partial(value_reader.read, 2**22), b'') + size, sha256 = self._write_file(full_path, read_iterator, max_filesize) + _metadata['size'] = size + _metadata['sha256'] = sha256 + + # after storing the artifacts, we write the metadata present + _metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename) + + with self.fs.open(metadata_file_path, 'wb') as f: + f.write(json.dumps(_metadata)) + + return uid_filename, _metadata + + def store_path(self, uid_filename): + """ + Returns absolute file path of the uid_filename + """ + prefix_dir = '' + if '/' in uid_filename: + prefix_dir, filename = uid_filename.split('/') + sub_store = self._sub_store_from_filename(filename) + else: + sub_store = self._sub_store_from_filename(uid_filename) + + return os.path.join(self._directory, prefix_dir, sub_store, uid_filename) + + def metadata_convert(self, uid_filename, metadata): + # NOTE: backward compat mode here... this is for file created PRE 5.2 system + if 'meta_ver' in metadata: + full_path = self.store_path(uid_filename) + metadata = { + "_converted": True, + "_org": metadata, + "version": self.METADATA_VER, + "store_type": self.BACKEND_TYPE, + + "filename": metadata['filename'], + "filename_uid_path": full_path, + "filename_uid": uid_filename, + "sha256": metadata['sha256'], + + "store_time": metadata['time'], + + "size": metadata['size'] + } + return metadata + + def _fetch(self, key, presigned_url_expires: int = 0): + if key not in self: + log.exception(f'requested key={key} not found in {self}') + raise KeyError(key) + + metadata = self.get_metadata(key) + + file_path = metadata['filename_uid_path'] + if presigned_url_expires and presigned_url_expires > 0: + metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires) + + return ShardFileReader(self.fs.open(file_path, 'rb')), metadata + + def delete(self, key): + return self._delete(key) + + def _delete(self, key): + if key not in self: + log.exception(f'requested key={key} not found in {self}') + raise KeyError(key) + + metadata = self.get_metadata(key) + metadata_file, metadata_file_path = self.get_metadata_filename(key) + artifact_file_path = metadata['filename_uid_path'] + self.fs.rm(artifact_file_path) + self.fs.rm(metadata_file_path) + + def get_metadata_filename(self, uid_filename) -> tuple[str, str]: + + metadata_file: str = f'{uid_filename}{self.metadata_suffix}' + uid_path_in_store = self.store_path(uid_filename) + + metadata_file_path = f'{uid_path_in_store}{self.metadata_suffix}' + return metadata_file, metadata_file_path + + +class LegacyFileSystemBackend(BaseFileStoreBackend): + _shard_cls = LegacyFileSystemShard + + def __init__(self, settings): + super().__init__(settings) + + store_dir = self.get_conf(config_keys.legacy_filesystem_storage_path) + directory = os.path.expanduser(store_dir) + + self._directory = directory + self._storage_path = directory # common path for all from BaseCache + + log.debug('Initializing %s file_store instance', self) + fs = fsspec.filesystem('file') + + if not fs.exists(self._directory): + fs.makedirs(self._directory, exist_ok=True) + + # legacy system uses single shard + self._shards = tuple( + [ + self._shard_cls( + index=0, + directory=directory, + directory_folder='', + fs=fs, + **settings, + ) + ] + ) + + @classmethod + def get_shard_index(cls, filename: str, num_shards) -> int: + # legacy filesystem doesn't use shards, and always uses single shard + return 0 diff --git a/rhodecode/apps/file_store/backends/local_store.py b/rhodecode/apps/file_store/backends/local_store.py deleted file mode 100755 index f86a01b6..00000000 --- a/rhodecode/apps/file_store/backends/local_store.py +++ /dev/null @@ -1,268 +0,0 @@ -# Copyright (C) 2016-2023 RhodeCode GmbH -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License, version 3 -# (only), as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# This program is dual-licensed. If you wish to learn more about the -# RhodeCode Enterprise Edition, including its added features, Support services, -# and proprietary license terms, please see https://rhodecode.com/licenses/ - -import os -import time -import errno -import hashlib - -from rhodecode.lib.ext_json import json -from rhodecode.apps.file_store import utils -from rhodecode.apps.file_store.extensions import resolve_extensions -from rhodecode.apps.file_store.exceptions import ( - FileNotAllowedException, FileOverSizeException) - -METADATA_VER = 'v1' - - -def safe_make_dirs(dir_path): - if not os.path.exists(dir_path): - try: - os.makedirs(dir_path) - except OSError as e: - if e.errno != errno.EEXIST: - raise - return - - -class LocalFileStorage(object): - - @classmethod - def apply_counter(cls, counter, filename): - name_counted = '%d-%s' % (counter, filename) - return name_counted - - @classmethod - def resolve_name(cls, name, directory): - """ - Resolves a unique name and the correct path. If a filename - for that path already exists then a numeric prefix with values > 0 will be - added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix. - - :param name: base name of file - :param directory: absolute directory path - """ - - counter = 0 - while True: - name_counted = cls.apply_counter(counter, name) - - # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file - sub_store = cls._sub_store_from_filename(name_counted) - sub_store_path = os.path.join(directory, sub_store) - safe_make_dirs(sub_store_path) - - path = os.path.join(sub_store_path, name_counted) - if not os.path.exists(path): - return name_counted, path - counter += 1 - - @classmethod - def _sub_store_from_filename(cls, filename): - return filename[:2] - - @classmethod - def calculate_path_hash(cls, file_path): - """ - Efficient calculation of file_path sha256 sum - - :param file_path: - :return: sha256sum - """ - digest = hashlib.sha256() - with open(file_path, 'rb') as f: - for chunk in iter(lambda: f.read(1024 * 100), b""): - digest.update(chunk) - - return digest.hexdigest() - - def __init__(self, base_path, extension_groups=None): - - """ - Local file storage - - :param base_path: the absolute base path where uploads are stored - :param extension_groups: extensions string - """ - - extension_groups = extension_groups or ['any'] - self.base_path = base_path - self.extensions = resolve_extensions([], groups=extension_groups) - - def __repr__(self): - return f'{self.__class__}@{self.base_path}' - - def store_path(self, filename): - """ - Returns absolute file path of the filename, joined to the - base_path. - - :param filename: base name of file - """ - prefix_dir = '' - if '/' in filename: - prefix_dir, filename = filename.split('/') - sub_store = self._sub_store_from_filename(filename) - else: - sub_store = self._sub_store_from_filename(filename) - return os.path.join(self.base_path, prefix_dir, sub_store, filename) - - def delete(self, filename): - """ - Deletes the filename. Filename is resolved with the - absolute path based on base_path. If file does not exist, - returns **False**, otherwise **True** - - :param filename: base name of file - """ - if self.exists(filename): - os.remove(self.store_path(filename)) - return True - return False - - def exists(self, filename): - """ - Checks if file exists. Resolves filename's absolute - path based on base_path. - - :param filename: file_uid name of file, e.g 0-f62b2b2d-9708-4079-a071-ec3f958448d4.svg - """ - return os.path.exists(self.store_path(filename)) - - def filename_allowed(self, filename, extensions=None): - """Checks if a filename has an allowed extension - - :param filename: base name of file - :param extensions: iterable of extensions (or self.extensions) - """ - _, ext = os.path.splitext(filename) - return self.extension_allowed(ext, extensions) - - def extension_allowed(self, ext, extensions=None): - """ - Checks if an extension is permitted. Both e.g. ".jpg" and - "jpg" can be passed in. Extension lookup is case-insensitive. - - :param ext: extension to check - :param extensions: iterable of extensions to validate against (or self.extensions) - """ - def normalize_ext(_ext): - if _ext.startswith('.'): - _ext = _ext[1:] - return _ext.lower() - - extensions = extensions or self.extensions - if not extensions: - return True - - ext = normalize_ext(ext) - - return ext in [normalize_ext(x) for x in extensions] - - def save_file(self, file_obj, filename, directory=None, extensions=None, - extra_metadata=None, max_filesize=None, randomized_name=True, **kwargs): - """ - Saves a file object to the uploads location. - Returns the resolved filename, i.e. the directory + - the (randomized/incremented) base name. - - :param file_obj: **cgi.FieldStorage** object (or similar) - :param filename: original filename - :param directory: relative path of sub-directory - :param extensions: iterable of allowed extensions, if not default - :param max_filesize: maximum size of file that should be allowed - :param randomized_name: generate random generated UID or fixed based on the filename - :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix - - """ - - extensions = extensions or self.extensions - - if not self.filename_allowed(filename, extensions): - raise FileNotAllowedException() - - if directory: - dest_directory = os.path.join(self.base_path, directory) - else: - dest_directory = self.base_path - - safe_make_dirs(dest_directory) - - uid_filename = utils.uid_filename(filename, randomized=randomized_name) - - # resolve also produces special sub-dir for file optimized store - filename, path = self.resolve_name(uid_filename, dest_directory) - stored_file_dir = os.path.dirname(path) - - no_body_seek = kwargs.pop('no_body_seek', False) - if no_body_seek: - pass - else: - file_obj.seek(0) - - with open(path, "wb") as dest: - length = 256 * 1024 - while 1: - buf = file_obj.read(length) - if not buf: - break - dest.write(buf) - - metadata = {} - if extra_metadata: - metadata = extra_metadata - - size = os.stat(path).st_size - - if max_filesize and size > max_filesize: - # free up the copied file, and raise exc - os.remove(path) - raise FileOverSizeException() - - file_hash = self.calculate_path_hash(path) - - metadata.update({ - "filename": filename, - "size": size, - "time": time.time(), - "sha256": file_hash, - "meta_ver": METADATA_VER - }) - - filename_meta = filename + '.meta' - with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta: - dest_meta.write(json.dumps(metadata)) - - if directory: - filename = os.path.join(directory, filename) - - return filename, metadata - - def get_metadata(self, filename, ignore_missing=False): - """ - Reads JSON stored metadata for a file - - :param filename: - :return: - """ - filename = self.store_path(filename) - filename_meta = filename + '.meta' - if ignore_missing and not os.path.isfile(filename_meta): - return {} - with open(filename_meta, "rb") as source_meta: - return json.loads(source_meta.read()) diff --git a/rhodecode/apps/file_store/backends/objectstore.py b/rhodecode/apps/file_store/backends/objectstore.py new file mode 100644 index 00000000..f239deb5 --- /dev/null +++ b/rhodecode/apps/file_store/backends/objectstore.py @@ -0,0 +1,184 @@ +# Copyright (C) 2016-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import os +import hashlib +import functools +import time +import logging + +from .. import config_keys +from ..exceptions import FileOverSizeException +from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader + +from ....lib.ext_json import json + +log = logging.getLogger(__name__) + + +class S3Shard(BaseShard): + METADATA_VER = 'v2' + BACKEND_TYPE = config_keys.backend_objectstore + storage_type: str = 'bucket' + + def __init__(self, index, bucket, bucket_folder, fs, **settings): + self._index: int = index + self._bucket_main: str = bucket + self._bucket_folder: str = bucket_folder + + self.fs = fs + + @property + def bucket(self) -> str: + """Cache bucket final path.""" + return os.path.join(self._bucket_main, self._bucket_folder) + + def _write_file(self, full_path, iterator, max_filesize, mode='wb'): + + # ensure dir exists + destination, _ = os.path.split(full_path) + if not self.fs.exists(destination): + self.fs.makedirs(destination) + + writer = self.fs.open(full_path, mode) + + digest = hashlib.sha256() + oversize_cleanup = False + with writer: + size = 0 + for chunk in iterator: + size += len(chunk) + digest.update(chunk) + writer.write(chunk) + + if max_filesize and size > max_filesize: + oversize_cleanup = True + # free up the copied file, and raise exc + break + + if oversize_cleanup: + self.fs.rm(full_path) + raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}') + + sha256 = digest.hexdigest() + log.debug('written new artifact under %s, sha256: %s', full_path, sha256) + return size, sha256 + + def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs): + + filename = key + uid_filename = uid_key + full_path = self.store_path(uid_filename) + + # STORE METADATA + _metadata = { + "version": self.METADATA_VER, + "store_type": self.BACKEND_TYPE, + + "filename": filename, + "filename_uid_path": full_path, + "filename_uid": uid_filename, + "sha256": "", # NOTE: filled in by reader iteration + + "store_time": time.time(), + + "size": 0 + } + + if metadata: + if kwargs.pop('import_mode', False): + # in import mode, we don't need to compute metadata, we just take the old version + _metadata["import_mode"] = True + else: + _metadata.update(metadata) + + read_iterator = iter(functools.partial(value_reader.read, 2**22), b'') + size, sha256 = self._write_file(full_path, read_iterator, max_filesize) + _metadata['size'] = size + _metadata['sha256'] = sha256 + + # after storing the artifacts, we write the metadata present + metadata_file, metadata_file_path = self.get_metadata_filename(uid_key) + + with self.fs.open(metadata_file_path, 'wb') as f: + f.write(json.dumps(_metadata)) + + return uid_filename, _metadata + + def store_path(self, uid_filename): + """ + Returns absolute file path of the uid_filename + """ + return os.path.join(self._bucket_main, self._bucket_folder, uid_filename) + + def _fetch(self, key, presigned_url_expires: int = 0): + if key not in self: + log.exception(f'requested key={key} not found in {self}') + raise KeyError(key) + + metadata_file, metadata_file_path = self.get_metadata_filename(key) + with self.fs.open(metadata_file_path, 'rb') as f: + metadata = json.loads(f.read()) + + file_path = metadata['filename_uid_path'] + if presigned_url_expires and presigned_url_expires > 0: + metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires) + + return ShardFileReader(self.fs.open(file_path, 'rb')), metadata + + def delete(self, key): + return self._delete(key) + + +class ObjectStoreBackend(BaseFileStoreBackend): + shard_name: str = 'shard-{:03d}' + _shard_cls = S3Shard + + def __init__(self, settings): + super().__init__(settings) + + self._shard_count = int(self.get_conf(config_keys.objectstore_bucket_shards, pop=True)) + if self._shard_count < 1: + raise ValueError('cache_shards must be 1 or more') + + self._bucket = settings.pop(config_keys.objectstore_bucket) + if not self._bucket: + raise ValueError(f'{config_keys.objectstore_bucket} needs to have a value') + + objectstore_url = self.get_conf(config_keys.objectstore_url) + key = settings.pop(config_keys.objectstore_key) + secret = settings.pop(config_keys.objectstore_secret) + + self._storage_path = objectstore_url # common path for all from BaseCache + log.debug('Initializing %s file_store instance', self) + fs = fsspec.filesystem('s3', anon=False, endpoint_url=objectstore_url, key=key, secret=secret) + + # init main bucket + if not fs.exists(self._bucket): + fs.mkdir(self._bucket) + + self._shards = tuple( + self._shard_cls( + index=num, + bucket=self._bucket, + bucket_folder=self.shard_name.format(num), + fs=fs, + **settings, + ) + for num in range(self._shard_count) + ) diff --git a/rhodecode/apps/file_store/config_keys.py b/rhodecode/apps/file_store/config_keys.py index 35449c34..5e14af82 100644 --- a/rhodecode/apps/file_store/config_keys.py +++ b/rhodecode/apps/file_store/config_keys.py @@ -20,6 +20,38 @@ # Definition of setting keys used to configure this module. Defined here to # avoid repetition of keys throughout the module. -enabled = 'file_store.enabled' -backend = 'file_store.backend' -store_path = 'file_store.storage_path' +# OLD and deprecated keys not used anymore +deprecated_enabled = 'file_store.enabled' +deprecated_backend = 'file_store.backend' +deprecated_store_path = 'file_store.storage_path' + + +backend_type = 'file_store.backend.type' + +backend_legacy_filesystem = 'filesystem_v1' +backend_filesystem = 'filesystem_v2' +backend_objectstore = 'objectstore' + +backend_types = [ + backend_legacy_filesystem, + backend_filesystem, + backend_objectstore, +] + +# filesystem_v1 legacy +legacy_filesystem_storage_path = 'file_store.filesystem_v1.storage_path' + + +# filesystem_v2 new option +filesystem_storage_path = 'file_store.filesystem_v2.storage_path' +filesystem_shards = 'file_store.filesystem_v2.shards' + +# objectstore +objectstore_url = 'file_store.objectstore.url' +objectstore_bucket = 'file_store.objectstore.bucket' +objectstore_bucket_shards = 'file_store.objectstore.bucket_shards' + +objectstore_region = 'file_store.objectstore.region' +objectstore_key = 'file_store.objectstore.key' +objectstore_secret = 'file_store.objectstore.secret' + diff --git a/rhodecode/apps/file_store/tests/__init__.py b/rhodecode/apps/file_store/tests/__init__.py index 380ec147..d8288e80 100644 --- a/rhodecode/apps/file_store/tests/__init__.py +++ b/rhodecode/apps/file_store/tests/__init__.py @@ -16,3 +16,42 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ +import os +import random +import tempfile +import string + +import pytest + +from rhodecode.apps.file_store import utils as store_utils + + +@pytest.fixture() +def file_store_instance(ini_settings): + config = ini_settings + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + return f_store + + +@pytest.fixture +def random_binary_file(): + # Generate random binary data + data = bytearray(random.getrandbits(8) for _ in range(1024 * 512)) # 512 KB of random data + + # Create a temporary file + temp_file = tempfile.NamedTemporaryFile(delete=False) + filename = temp_file.name + + try: + # Write the random binary data to the file + temp_file.write(data) + temp_file.seek(0) # Rewind the file pointer to the beginning + yield filename, temp_file + finally: + # Close and delete the temporary file after the test + temp_file.close() + os.remove(filename) + + +def generate_random_filename(length=10): + return ''.join(random.choices(string.ascii_letters + string.digits, k=length)) \ No newline at end of file diff --git a/rhodecode/apps/file_store/tests/test_filestore_backends.py b/rhodecode/apps/file_store/tests/test_filestore_backends.py new file mode 100644 index 00000000..54a50ca0 --- /dev/null +++ b/rhodecode/apps/file_store/tests/test_filestore_backends.py @@ -0,0 +1,128 @@ +# Copyright (C) 2010-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import pytest + +from rhodecode.apps import file_store +from rhodecode.apps.file_store import config_keys +from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend +from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend +from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend +from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException + +from rhodecode.apps.file_store import utils as store_utils +from rhodecode.apps.file_store.tests import random_binary_file, file_store_instance + + +class TestFileStoreBackends: + + @pytest.mark.parametrize('backend_type, expected_instance', [ + (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend), + (config_keys.backend_filesystem, FileSystemBackend), + (config_keys.backend_objectstore, ObjectStoreBackend), + ]) + def test_get_backend(self, backend_type, expected_instance, ini_settings): + config = ini_settings + config[config_keys.backend_type] = backend_type + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + assert isinstance(f_store, expected_instance) + + @pytest.mark.parametrize('backend_type, expected_instance', [ + (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend), + (config_keys.backend_filesystem, FileSystemBackend), + (config_keys.backend_objectstore, ObjectStoreBackend), + ]) + def test_store_and_read(self, backend_type, expected_instance, ini_settings, random_binary_file): + filename, temp_file = random_binary_file + config = ini_settings + config[config_keys.backend_type] = backend_type + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + metadata = { + 'user_uploaded': { + 'username': 'user1', + 'user_id': 10, + 'ip': '10.20.30.40' + } + } + store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata) + assert store_fid + assert metadata + + # read-after write + reader, metadata2 = f_store.fetch(store_fid) + assert reader + assert metadata2['filename'] == filename + + @pytest.mark.parametrize('backend_type, expected_instance', [ + (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend), + (config_keys.backend_filesystem, FileSystemBackend), + (config_keys.backend_objectstore, ObjectStoreBackend), + ]) + def test_store_file_not_allowed(self, backend_type, expected_instance, ini_settings, random_binary_file): + filename, temp_file = random_binary_file + config = ini_settings + config[config_keys.backend_type] = backend_type + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + with pytest.raises(FileNotAllowedException): + f_store.store('notallowed.exe', temp_file, extensions=['.txt']) + + @pytest.mark.parametrize('backend_type, expected_instance', [ + (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend), + (config_keys.backend_filesystem, FileSystemBackend), + (config_keys.backend_objectstore, ObjectStoreBackend), + ]) + def test_store_file_over_size(self, backend_type, expected_instance, ini_settings, random_binary_file): + filename, temp_file = random_binary_file + config = ini_settings + config[config_keys.backend_type] = backend_type + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + with pytest.raises(FileOverSizeException): + f_store.store('toobig.exe', temp_file, extensions=['.exe'], max_filesize=124) + + @pytest.mark.parametrize('backend_type, expected_instance, extra_conf', [ + (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend, {}), + (config_keys.backend_filesystem, FileSystemBackend, {config_keys.filesystem_storage_path: '/tmp/test-fs-store'}), + (config_keys.backend_objectstore, ObjectStoreBackend, {config_keys.objectstore_bucket: 'test-bucket'}), + ]) + def test_store_stats_and_keys(self, backend_type, expected_instance, extra_conf, ini_settings, random_binary_file): + config = ini_settings + config[config_keys.backend_type] = backend_type + config.update(extra_conf) + + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + + # purge storage before running + for shard, k in f_store.iter_artifacts(): + f_store.delete(k) + + for i in range(10): + filename, temp_file = random_binary_file + + metadata = { + 'user_uploaded': { + 'username': 'user1', + 'user_id': 10, + 'ip': '10.20.30.40' + } + } + store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata) + assert store_fid + assert metadata + + cnt, size, meta = f_store.get_statistics() + assert cnt == 10 + assert 10 == len(list(f_store.iter_keys())) diff --git a/rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py b/rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py new file mode 100644 index 00000000..496145ce --- /dev/null +++ b/rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py @@ -0,0 +1,52 @@ +# Copyright (C) 2010-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import pytest + +from rhodecode.apps.file_store import utils as store_utils +from rhodecode.apps.file_store import config_keys +from rhodecode.apps.file_store.tests import generate_random_filename + + +@pytest.fixture() +def file_store_filesystem_instance(ini_settings): + config = ini_settings + config[config_keys.backend_type] = config_keys.backend_filesystem + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + return f_store + + +class TestFileStoreFileSystemBackend: + + @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)]) + def test_get_shard_number(self, filename, file_store_filesystem_instance): + shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards)) + # Check that the shard number is between 0 and max-shards + assert 0 <= shard_number <= len(file_store_filesystem_instance._shards) + + @pytest.mark.parametrize('filename, expected_shard_num', [ + ('my-name-1', 3), + ('my-name-2', 2), + ('my-name-3', 4), + ('my-name-4', 1), + + ('rhodecode-enterprise-ce', 5), + ('rhodecode-enterprise-ee', 6), + ]) + def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_filesystem_instance): + shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards)) + assert expected_shard_num == shard_number diff --git a/rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py b/rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py new file mode 100644 index 00000000..6298bf53 --- /dev/null +++ b/rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py @@ -0,0 +1,17 @@ +# Copyright (C) 2010-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ \ No newline at end of file diff --git a/rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py b/rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py new file mode 100644 index 00000000..f9783e07 --- /dev/null +++ b/rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py @@ -0,0 +1,52 @@ +# Copyright (C) 2010-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import pytest + +from rhodecode.apps.file_store import utils as store_utils +from rhodecode.apps.file_store import config_keys +from rhodecode.apps.file_store.tests import generate_random_filename + + +@pytest.fixture() +def file_store_legacy_instance(ini_settings): + config = ini_settings + config[config_keys.backend_type] = config_keys.backend_legacy_filesystem + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + return f_store + + +class TestFileStoreLegacyBackend: + + @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)]) + def test_get_shard_number(self, filename, file_store_legacy_instance): + shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards)) + # Check that the shard number is 0 for legacy filesystem store we don't use shards + assert shard_number == 0 + + @pytest.mark.parametrize('filename, expected_shard_num', [ + ('my-name-1', 0), + ('my-name-2', 0), + ('my-name-3', 0), + ('my-name-4', 0), + + ('rhodecode-enterprise-ce', 0), + ('rhodecode-enterprise-ee', 0), + ]) + def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_legacy_instance): + shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards)) + assert expected_shard_num == shard_number diff --git a/rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py b/rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py new file mode 100644 index 00000000..b77fe865 --- /dev/null +++ b/rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py @@ -0,0 +1,52 @@ +# Copyright (C) 2010-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import pytest + +from rhodecode.apps.file_store import utils as store_utils +from rhodecode.apps.file_store import config_keys +from rhodecode.apps.file_store.tests import generate_random_filename + + +@pytest.fixture() +def file_store_objectstore_instance(ini_settings): + config = ini_settings + config[config_keys.backend_type] = config_keys.backend_objectstore + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + return f_store + + +class TestFileStoreObjectStoreBackend: + + @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)]) + def test_get_shard_number(self, filename, file_store_objectstore_instance): + shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards)) + # Check that the shard number is between 0 and shards + assert 0 <= shard_number <= len(file_store_objectstore_instance._shards) + + @pytest.mark.parametrize('filename, expected_shard_num', [ + ('my-name-1', 3), + ('my-name-2', 2), + ('my-name-3', 4), + ('my-name-4', 1), + + ('rhodecode-enterprise-ce', 5), + ('rhodecode-enterprise-ee', 6), + ]) + def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_objectstore_instance): + shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards)) + assert expected_shard_num == shard_number diff --git a/rhodecode/apps/file_store/tests/test_upload_file.py b/rhodecode/apps/file_store/tests/test_upload_file.py index f7c9b766..87e3da61 100644 --- a/rhodecode/apps/file_store/tests/test_upload_file.py +++ b/rhodecode/apps/file_store/tests/test_upload_file.py @@ -15,13 +15,16 @@ # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ + import os + import pytest from rhodecode.lib.ext_json import json from rhodecode.model.auth_token import AuthTokenModel from rhodecode.model.db import Session, FileStore, Repository, User -from rhodecode.apps.file_store import utils, config_keys +from rhodecode.apps.file_store import utils as store_utils +from rhodecode.apps.file_store import config_keys from rhodecode.tests import TestController from rhodecode.tests.routes import route_path @@ -29,27 +32,61 @@ from rhodecode.tests.routes import route_path class TestFileStoreViews(TestController): + @pytest.fixture() + def create_artifact_factory(self, tmpdir, ini_settings): + + def factory(user_id, content, f_name='example.txt'): + + config = ini_settings + config[config_keys.backend_type] = config_keys.backend_legacy_filesystem + + f_store = store_utils.get_filestore_backend(config) + + filesystem_file = os.path.join(str(tmpdir), f_name) + with open(filesystem_file, 'wt') as f: + f.write(content) + + with open(filesystem_file, 'rb') as f: + store_uid, metadata = f_store.store(f_name, f, metadata={'filename': f_name}) + os.remove(filesystem_file) + + entry = FileStore.create( + file_uid=store_uid, filename=metadata["filename"], + file_hash=metadata["sha256"], file_size=metadata["size"], + file_display_name='file_display_name', + file_description='repo artifact `{}`'.format(metadata["filename"]), + check_acl=True, user_id=user_id, + ) + Session().add(entry) + Session().commit() + return entry + return factory + @pytest.mark.parametrize("fid, content, exists", [ ('abcde-0.jpg', "xxxxx", True), ('abcde-0.exe', "1234567", True), ('abcde-0.jpg', "xxxxx", False), ]) - def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util): + def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util, ini_settings): user = self.log_user() user_id = user['user_id'] repo_id = user_util.create_repo().repo_id - store_path = self.app._pyramid_settings[config_keys.store_path] + + config = ini_settings + config[config_keys.backend_type] = config_keys.backend_legacy_filesystem + store_uid = fid if exists: status = 200 - store = utils.get_file_storage({config_keys.store_path: store_path}) + f_store = store_utils.get_filestore_backend(config) filesystem_file = os.path.join(str(tmpdir), fid) with open(filesystem_file, 'wt') as f: f.write(content) with open(filesystem_file, 'rb') as f: - store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid}) + store_uid, metadata = f_store.store(fid, f, metadata={'filename': fid}) + os.remove(filesystem_file) entry = FileStore.create( file_uid=store_uid, filename=metadata["filename"], @@ -69,14 +106,10 @@ class TestFileStoreViews(TestController): if exists: assert response.text == content - file_store_path = os.path.dirname(store.resolve_name(store_uid, store_path)[1]) - metadata_file = os.path.join(file_store_path, store_uid + '.meta') - assert os.path.exists(metadata_file) - with open(metadata_file, 'rb') as f: - json_data = json.loads(f.read()) - assert json_data - assert 'size' in json_data + metadata = f_store.get_metadata(store_uid) + + assert 'size' in metadata def test_upload_files_without_content_to_store(self): self.log_user() @@ -112,32 +145,6 @@ class TestFileStoreViews(TestController): assert response.json['store_fid'] - @pytest.fixture() - def create_artifact_factory(self, tmpdir): - def factory(user_id, content): - store_path = self.app._pyramid_settings[config_keys.store_path] - store = utils.get_file_storage({config_keys.store_path: store_path}) - fid = 'example.txt' - - filesystem_file = os.path.join(str(tmpdir), fid) - with open(filesystem_file, 'wt') as f: - f.write(content) - - with open(filesystem_file, 'rb') as f: - store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid}) - - entry = FileStore.create( - file_uid=store_uid, filename=metadata["filename"], - file_hash=metadata["sha256"], file_size=metadata["size"], - file_display_name='file_display_name', - file_description='repo artifact `{}`'.format(metadata["filename"]), - check_acl=True, user_id=user_id, - ) - Session().add(entry) - Session().commit() - return entry - return factory - def test_download_file_non_scoped(self, user_util, create_artifact_factory): user = self.log_user() user_id = user['user_id'] diff --git a/rhodecode/apps/file_store/utils.py b/rhodecode/apps/file_store/utils.py index f421a1a4..0c4ed472 100755 --- a/rhodecode/apps/file_store/utils.py +++ b/rhodecode/apps/file_store/utils.py @@ -19,21 +19,84 @@ import io import uuid import pathlib +import s3fs + +from rhodecode.lib.hash_utils import sha256_safe +from rhodecode.apps.file_store import config_keys -def get_file_storage(settings): - from rhodecode.apps.file_store.backends.local_store import LocalFileStorage - from rhodecode.apps.file_store import config_keys - store_path = settings.get(config_keys.store_path) - return LocalFileStorage(base_path=store_path) +file_store_meta = None + + +def get_filestore_config(config) -> dict: + + final_config = {} + + for k, v in config.items(): + if k.startswith('file_store'): + final_config[k] = v + + return final_config + + +def get_filestore_backend(config, always_init=False): + """ + + usage:: + from rhodecode.apps.file_store import get_filestore_backend + f_store = get_filestore_backend(config=CONFIG) + + :param config: + :param always_init: + :return: + """ + + global file_store_meta + if file_store_meta is not None and not always_init: + return file_store_meta + + config = get_filestore_config(config) + backend = config[config_keys.backend_type] + + match backend: + case config_keys.backend_legacy_filesystem: + # Legacy backward compatible storage + from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend + d_cache = LegacyFileSystemBackend( + settings=config + ) + case config_keys.backend_filesystem: + from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend + d_cache = FileSystemBackend( + settings=config + ) + case config_keys.backend_objectstore: + from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend + d_cache = ObjectStoreBackend( + settings=config + ) + case _: + raise ValueError( + f'file_store.backend.type only supports "{config_keys.backend_types}" got {backend}' + ) + + cache_meta = d_cache + return cache_meta def splitext(filename): - ext = ''.join(pathlib.Path(filename).suffixes) + final_ext = [] + for suffix in pathlib.Path(filename).suffixes: + if not suffix.isascii(): + continue + + suffix = " ".join(suffix.split()).replace(" ", "") + final_ext.append(suffix) + ext = ''.join(final_ext) return filename, ext -def uid_filename(filename, randomized=True): +def get_uid_filename(filename, randomized=True): """ Generates a randomized or stable (uuid) filename, preserving the original extension. @@ -46,10 +109,37 @@ def uid_filename(filename, randomized=True): if randomized: uid = uuid.uuid4() else: - hash_key = '{}.{}'.format(filename, 'store') + store_suffix = "store" + hash_key = f'{filename}.{store_suffix}' uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key) return str(uid) + ext.lower() def bytes_to_file_obj(bytes_data): - return io.StringIO(bytes_data) + return io.BytesIO(bytes_data) + + +class ShardFileReader: + + def __init__(self, file_like_reader): + self._file_like_reader = file_like_reader + + def __getattr__(self, item): + if isinstance(self._file_like_reader, s3fs.core.S3File): + match item: + case 'name': + # S3 FileWrapper doesn't support name attribute, and we use it + return self._file_like_reader.full_name + case _: + return getattr(self._file_like_reader, item) + else: + return getattr(self._file_like_reader, item) + + +def archive_iterator(_reader, block_size: int = 4096 * 512): + # 4096 * 64 = 64KB + while 1: + data = _reader.read(block_size) + if not data: + break + yield data diff --git a/rhodecode/apps/file_store/views.py b/rhodecode/apps/file_store/views.py index e43e4b57..8ddde743 100644 --- a/rhodecode/apps/file_store/views.py +++ b/rhodecode/apps/file_store/views.py @@ -17,12 +17,11 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ import logging - -from pyramid.response import FileResponse +from pyramid.response import Response from pyramid.httpexceptions import HTTPFound, HTTPNotFound from rhodecode.apps._base import BaseAppView -from rhodecode.apps.file_store import utils +from rhodecode.apps.file_store import utils as store_utils from rhodecode.apps.file_store.exceptions import ( FileNotAllowedException, FileOverSizeException) @@ -31,6 +30,7 @@ from rhodecode.lib import audit_logger from rhodecode.lib.auth import ( CSRFRequired, NotAnonymous, HasRepoPermissionAny, HasRepoGroupPermissionAny, LoginRequired) +from rhodecode.lib.str_utils import header_safe_str from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db from rhodecode.model.db import Session, FileStore, UserApiKeys @@ -42,7 +42,7 @@ class FileStoreView(BaseAppView): def load_default_context(self): c = self._get_local_tmpl_context() - self.storage = utils.get_file_storage(self.request.registry.settings) + self.f_store = store_utils.get_filestore_backend(self.request.registry.settings) return c def _guess_type(self, file_name): @@ -55,10 +55,10 @@ class FileStoreView(BaseAppView): return _content_type, _encoding def _serve_file(self, file_uid): - if not self.storage.exists(file_uid): - store_path = self.storage.store_path(file_uid) - log.debug('File with FID:%s not found in the store under `%s`', - file_uid, store_path) + if not self.f_store.filename_exists(file_uid): + store_path = self.f_store.store_path(file_uid) + log.warning('File with FID:%s not found in the store under `%s`', + file_uid, store_path) raise HTTPNotFound() db_obj = FileStore.get_by_store_uid(file_uid, safe=True) @@ -98,28 +98,25 @@ class FileStoreView(BaseAppView): FileStore.bump_access_counter(file_uid) - file_path = self.storage.store_path(file_uid) + file_name = db_obj.file_display_name content_type = 'application/octet-stream' - content_encoding = None - _content_type, _encoding = self._guess_type(file_path) + _content_type, _encoding = self._guess_type(file_name) if _content_type: content_type = _content_type # For file store we don't submit any session data, this logic tells the # Session lib to skip it setattr(self.request, '_file_response', True) - response = FileResponse( - file_path, request=self.request, - content_type=content_type, content_encoding=content_encoding) + reader, _meta = self.f_store.fetch(file_uid) - file_name = db_obj.file_display_name + response = Response(app_iter=store_utils.archive_iterator(reader)) + + response.content_type = str(content_type) + response.content_disposition = f'attachment; filename="{header_safe_str(file_name)}"' - response.headers["Content-Disposition"] = ( - f'attachment; filename="{str(file_name)}"' - ) response.headers["X-RC-Artifact-Id"] = str(db_obj.file_store_id) - response.headers["X-RC-Artifact-Desc"] = str(db_obj.file_description) + response.headers["X-RC-Artifact-Desc"] = header_safe_str(db_obj.file_description) response.headers["X-RC-Artifact-Sha256"] = str(db_obj.file_hash) return response @@ -147,8 +144,8 @@ class FileStoreView(BaseAppView): 'user_id': self._rhodecode_user.user_id, 'ip': self._rhodecode_user.ip_addr}} try: - store_uid, metadata = self.storage.save_file( - file_obj.file, filename, extra_metadata=metadata) + store_uid, metadata = self.f_store.store( + filename, file_obj.file, extra_metadata=metadata) except FileNotAllowedException: return {'store_fid': None, 'access_path': None, @@ -182,7 +179,7 @@ class FileStoreView(BaseAppView): def download_file(self): self.load_default_context() file_uid = self.request.matchdict['fid'] - log.debug('Requesting FID:%s from store %s', file_uid, self.storage) + log.debug('Requesting FID:%s from store %s', file_uid, self.f_store) return self._serve_file(file_uid) # in addition to @LoginRequired ACL is checked by scopes diff --git a/rhodecode/apps/repository/views/repo_commits.py b/rhodecode/apps/repository/views/repo_commits.py index 46263411..9b50b5bc 100644 --- a/rhodecode/apps/repository/views/repo_commits.py +++ b/rhodecode/apps/repository/views/repo_commits.py @@ -601,26 +601,26 @@ class RepoCommitsView(RepoAppView): max_file_size = 10 * 1024 * 1024 # 10MB, also validated via dropzone.js try: - storage = store_utils.get_file_storage(self.request.registry.settings) - store_uid, metadata = storage.save_file( - file_obj.file, filename, extra_metadata=metadata, + f_store = store_utils.get_filestore_backend(self.request.registry.settings) + store_uid, metadata = f_store.store( + filename, file_obj.file, metadata=metadata, extensions=allowed_extensions, max_filesize=max_file_size) except FileNotAllowedException: self.request.response.status = 400 permitted_extensions = ', '.join(allowed_extensions) - error_msg = 'File `{}` is not allowed. ' \ - 'Only following extensions are permitted: {}'.format( - filename, permitted_extensions) + error_msg = f'File `{filename}` is not allowed. ' \ + f'Only following extensions are permitted: {permitted_extensions}' + return {'store_fid': None, 'access_path': None, 'error': error_msg} except FileOverSizeException: self.request.response.status = 400 limit_mb = h.format_byte_size_binary(max_file_size) + error_msg = f'File {filename} is exceeding allowed limit of {limit_mb}.' return {'store_fid': None, 'access_path': None, - 'error': 'File {} is exceeding allowed limit of {}.'.format( - filename, limit_mb)} + 'error': error_msg} try: entry = FileStore.create( diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py index d6b7097e..fb2fad78 100644 --- a/rhodecode/apps/repository/views/repo_files.py +++ b/rhodecode/apps/repository/views/repo_files.py @@ -48,7 +48,7 @@ from rhodecode.lib.codeblocks import ( filenode_as_lines_tokens, filenode_as_annotated_lines_tokens) from rhodecode.lib.utils2 import convert_line_endings, detect_mode from rhodecode.lib.type_utils import str2bool -from rhodecode.lib.str_utils import safe_str, safe_int +from rhodecode.lib.str_utils import safe_str, safe_int, header_safe_str from rhodecode.lib.auth import ( LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired) from rhodecode.lib.vcs import path as vcspath @@ -820,7 +820,7 @@ class RepoFilesView(RepoAppView): "filename=\"{}\"; " \ "filename*=UTF-8\'\'{}".format(safe_path, encoded_path) - return safe_bytes(headers).decode('latin-1', errors='replace') + return header_safe_str(headers) @LoginRequired() @HasRepoPermissionAnyDecorator( diff --git a/rhodecode/apps/repository/views/repo_settings_advanced.py b/rhodecode/apps/repository/views/repo_settings_advanced.py index 1f92d2e8..2703be83 100644 --- a/rhodecode/apps/repository/views/repo_settings_advanced.py +++ b/rhodecode/apps/repository/views/repo_settings_advanced.py @@ -29,7 +29,7 @@ from rhodecode.lib import audit_logger from rhodecode.lib.auth import ( LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired, HasRepoPermissionAny) -from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError +from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError from rhodecode.lib.utils2 import safe_int from rhodecode.lib.vcs import RepositoryError from rhodecode.model.db import Session, UserFollowing, User, Repository @@ -136,6 +136,9 @@ class RepoSettingsAdvancedView(RepoAppView): elif handle_forks == 'delete_forks': handle_forks = 'delete' + repo_advanced_url = h.route_path( + 'edit_repo_advanced', repo_name=self.db_repo_name, + _anchor='advanced-delete') try: old_data = self.db_repo.get_api_data() RepoModel().delete(self.db_repo, forks=handle_forks) @@ -158,9 +161,6 @@ class RepoSettingsAdvancedView(RepoAppView): category='success') Session().commit() except AttachedForksError: - repo_advanced_url = h.route_path( - 'edit_repo_advanced', repo_name=self.db_repo_name, - _anchor='advanced-delete') delete_anchor = h.link_to(_('detach or delete'), repo_advanced_url) h.flash(_('Cannot delete `{repo}` it still contains attached forks. ' 'Try using {delete_or_detach} option.') @@ -171,9 +171,6 @@ class RepoSettingsAdvancedView(RepoAppView): raise HTTPFound(repo_advanced_url) except AttachedPullRequestsError: - repo_advanced_url = h.route_path( - 'edit_repo_advanced', repo_name=self.db_repo_name, - _anchor='advanced-delete') attached_prs = len(self.db_repo.pull_requests_source + self.db_repo.pull_requests_target) h.flash( @@ -184,6 +181,16 @@ class RepoSettingsAdvancedView(RepoAppView): # redirect to advanced for forks handle action ? raise HTTPFound(repo_advanced_url) + except AttachedArtifactsError: + + attached_artifacts = len(self.db_repo.artifacts) + h.flash( + _('Cannot delete `{repo}` it still contains {num} attached artifacts. ' + 'Consider archiving the repository instead.').format( + repo=self.db_repo_name, num=attached_artifacts), category='warning') + + # redirect to advanced for forks handle action ? + raise HTTPFound(repo_advanced_url) except Exception: log.exception("Exception during deletion of repository") h.flash(_('An error occurred during deletion of `%s`') diff --git a/rhodecode/apps/ssh_support/__init__.py b/rhodecode/apps/ssh_support/__init__.py index 65e330f5..1efad2c1 100644 --- a/rhodecode/apps/ssh_support/__init__.py +++ b/rhodecode/apps/ssh_support/__init__.py @@ -37,7 +37,7 @@ def _sanitize_settings_and_apply_defaults(settings): settings_maker.make_setting(config_keys.ssh_key_generator_enabled, True, parser='bool') settings_maker.make_setting(config_keys.authorized_keys_file_path, '~/.ssh/authorized_keys_rhodecode') - settings_maker.make_setting(config_keys.wrapper_cmd, '') + settings_maker.make_setting(config_keys.wrapper_cmd, '/usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2') settings_maker.make_setting(config_keys.authorized_keys_line_ssh_opts, '') settings_maker.make_setting(config_keys.ssh_hg_bin, '/usr/local/bin/rhodecode_bin/vcs_bin/hg') diff --git a/rhodecode/apps/ssh_support/config_keys.py b/rhodecode/apps/ssh_support/config_keys.py index 8d10dd61..490b7fc5 100644 --- a/rhodecode/apps/ssh_support/config_keys.py +++ b/rhodecode/apps/ssh_support/config_keys.py @@ -23,7 +23,7 @@ generate_authorized_keyfile = 'ssh.generate_authorized_keyfile' authorized_keys_file_path = 'ssh.authorized_keys_file_path' authorized_keys_line_ssh_opts = 'ssh.authorized_keys_ssh_opts' ssh_key_generator_enabled = 'ssh.enable_ui_key_generator' -wrapper_cmd = 'ssh.wrapper_cmd' +wrapper_cmd = 'ssh.wrapper_cmd.v2' wrapper_allow_shell = 'ssh.wrapper_cmd_allow_shell' enable_debug_logging = 'ssh.enable_debug_logging' diff --git a/rhodecode/apps/ssh_support/lib/backends/base.py b/rhodecode/apps/ssh_support/lib/backends/base.py index ee1e2a78..ede5ad3a 100644 --- a/rhodecode/apps/ssh_support/lib/backends/base.py +++ b/rhodecode/apps/ssh_support/lib/backends/base.py @@ -157,7 +157,7 @@ class SshVcsServer(object): return exit_code, action == "push" def run(self, tunnel_extras=None): - self.hooks_protocol = self.settings['vcs.hooks.protocol'] + self.hooks_protocol = self.settings['vcs.hooks.protocol.v2'] tunnel_extras = tunnel_extras or {} extras = {} extras.update(tunnel_extras) diff --git a/rhodecode/apps/ssh_support/tests/test_server_git.py b/rhodecode/apps/ssh_support/tests/test_server_git.py index 175163f9..843bec16 100644 --- a/rhodecode/apps/ssh_support/tests/test_server_git.py +++ b/rhodecode/apps/ssh_support/tests/test_server_git.py @@ -32,7 +32,7 @@ class GitServerCreator(object): config_data = { 'app:main': { 'ssh.executable.git': git_path, - 'vcs.hooks.protocol': 'http', + 'vcs.hooks.protocol.v2': 'celery', } } repo_name = 'test_git' diff --git a/rhodecode/apps/ssh_support/tests/test_server_hg.py b/rhodecode/apps/ssh_support/tests/test_server_hg.py index 031f932d..9765eb52 100644 --- a/rhodecode/apps/ssh_support/tests/test_server_hg.py +++ b/rhodecode/apps/ssh_support/tests/test_server_hg.py @@ -31,7 +31,7 @@ class MercurialServerCreator(object): config_data = { 'app:main': { 'ssh.executable.hg': hg_path, - 'vcs.hooks.protocol': 'http', + 'vcs.hooks.protocol.v2': 'celery', } } repo_name = 'test_hg' diff --git a/rhodecode/apps/ssh_support/tests/test_server_svn.py b/rhodecode/apps/ssh_support/tests/test_server_svn.py index 0bbc68a2..d8e1d1f0 100644 --- a/rhodecode/apps/ssh_support/tests/test_server_svn.py +++ b/rhodecode/apps/ssh_support/tests/test_server_svn.py @@ -29,7 +29,7 @@ class SubversionServerCreator(object): config_data = { 'app:main': { 'ssh.executable.svn': svn_path, - 'vcs.hooks.protocol': 'http', + 'vcs.hooks.protocol.v2': 'celery', } } repo_name = 'test-svn' diff --git a/rhodecode/authentication/routes.py b/rhodecode/authentication/routes.py index bd83ad71..e9e4b969 100644 --- a/rhodecode/authentication/routes.py +++ b/rhodecode/authentication/routes.py @@ -52,6 +52,7 @@ class AuthnRootResource(AuthnResourceBase): """ This is the root traversal resource object for the authentication settings. """ + is_root = True def __init__(self): self._store = collections.OrderedDict() diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py index 87b05fe8..64cd3a72 100644 --- a/rhodecode/config/config_maker.py +++ b/rhodecode/config/config_maker.py @@ -52,7 +52,8 @@ def sanitize_settings_and_apply_defaults(global_config, settings): default=False, parser='bool') - logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini') + ini_loc = os.path.dirname(global_config.get('__file__')) + logging_conf = jn(ini_loc, 'logging.ini') settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG') # Default includes, possible to change as a user @@ -95,6 +96,11 @@ def sanitize_settings_and_apply_defaults(global_config, settings): settings_maker.make_setting('gzip_responses', False, parser='bool') settings_maker.make_setting('startup.import_repos', 'false', parser='bool') + # License settings. + settings_maker.make_setting('license.hide_license_info', False, parser='bool') + settings_maker.make_setting('license.import_path', '') + settings_maker.make_setting('license.import_path_mode', 'if-missing') + # statsd settings_maker.make_setting('statsd.enabled', False, parser='bool') settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string') @@ -106,7 +112,7 @@ def sanitize_settings_and_apply_defaults(global_config, settings): settings_maker.make_setting('vcs.svn.redis_conn', 'redis://redis:6379/0') settings_maker.make_setting('vcs.svn.proxy.enabled', True, parser='bool') settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090', parser='string') - settings_maker.make_setting('vcs.hooks.protocol', 'http') + settings_maker.make_setting('vcs.hooks.protocol.v2', 'celery') settings_maker.make_setting('vcs.hooks.host', '*') settings_maker.make_setting('vcs.scm_app_implementation', 'http') settings_maker.make_setting('vcs.server', '') @@ -116,6 +122,9 @@ def sanitize_settings_and_apply_defaults(global_config, settings): settings_maker.make_setting('vcs.start_server', 'false', parser='bool') settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list') settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int') + settings_maker.make_setting('vcs.git.lfs.storage_location', '/var/opt/rhodecode_repo_store/.cache/git_lfs_store') + settings_maker.make_setting('vcs.hg.largefiles.storage_location', + '/var/opt/rhodecode_repo_store/.cache/hg_largefiles_store') settings_maker.make_setting('vcs.methods.cache', True, parser='bool') @@ -152,6 +161,10 @@ def sanitize_settings_and_apply_defaults(global_config, settings): parser='file:ensured' ) + # celery + broker_url = settings_maker.make_setting('celery.broker_url', 'redis://redis:6379/8') + settings_maker.make_setting('celery.result_backend', broker_url) + settings_maker.make_setting('exception_tracker.send_email', False, parser='bool') settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True) @@ -202,7 +215,7 @@ def sanitize_settings_and_apply_defaults(global_config, settings): settings_maker.make_setting('archive_cache.filesystem.retry_backoff', 1, parser='int') settings_maker.make_setting('archive_cache.filesystem.retry_attempts', 10, parser='int') - settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,) + settings_maker.make_setting('archive_cache.objectstore.url', 'http://s3-minio:9000', default_when_empty=True,) settings_maker.make_setting('archive_cache.objectstore.key', '') settings_maker.make_setting('archive_cache.objectstore.secret', '') settings_maker.make_setting('archive_cache.objectstore.region', 'eu-central-1') diff --git a/rhodecode/config/environment.py b/rhodecode/config/environment.py index 26ef21b3..1c699e65 100644 --- a/rhodecode/config/environment.py +++ b/rhodecode/config/environment.py @@ -16,8 +16,8 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import os import logging + import rhodecode import collections @@ -30,6 +30,21 @@ from rhodecode.lib.vcs import connect_vcs log = logging.getLogger(__name__) +def propagate_rhodecode_config(global_config, settings, config): + # Store the settings to make them available to other modules. + settings_merged = global_config.copy() + settings_merged.update(settings) + if config: + settings_merged.update(config) + + rhodecode.PYRAMID_SETTINGS = settings_merged + rhodecode.CONFIG = settings_merged + + if 'default_user_id' not in rhodecode.CONFIG: + rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id() + log.debug('set rhodecode.CONFIG data') + + def load_pyramid_environment(global_config, settings): # Some parts of the code expect a merge of global and app settings. settings_merged = global_config.copy() @@ -75,11 +90,8 @@ def load_pyramid_environment(global_config, settings): utils.configure_vcs(settings) - # Store the settings to make them available to other modules. - - rhodecode.PYRAMID_SETTINGS = settings_merged - rhodecode.CONFIG = settings_merged - rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id() + # first run, to store data... + propagate_rhodecode_config(global_config, settings, {}) if vcs_server_enabled: connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings)) diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py index 2f83e856..c82395cd 100644 --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -35,7 +35,7 @@ from pyramid.renderers import render_to_response from rhodecode.model import meta from rhodecode.config import patches -from rhodecode.config.environment import load_pyramid_environment +from rhodecode.config.environment import load_pyramid_environment, propagate_rhodecode_config import rhodecode.events from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults @@ -50,7 +50,7 @@ from rhodecode.lib.utils2 import AttributeDict from rhodecode.lib.exc_tracking import store_exception, format_exc from rhodecode.subscribers import ( scan_repositories_if_enabled, write_js_routes_if_enabled, - write_metadata_if_needed, write_usage_data) + write_metadata_if_needed, write_usage_data, import_license_if_present) from rhodecode.lib.statsd_client import StatsdClient log = logging.getLogger(__name__) @@ -99,6 +99,7 @@ def make_pyramid_app(global_config, **settings): # Apply compatibility patches patches.inspect_getargspec() + patches.repoze_sendmail_lf_fix() load_pyramid_environment(global_config, settings) @@ -114,6 +115,9 @@ def make_pyramid_app(global_config, **settings): celery_settings = get_celery_config(settings) config.configure_celery(celery_settings) + # final config set... + propagate_rhodecode_config(global_config, settings, config.registry.settings) + # creating the app uses a connection - return it after we are done meta.Session.remove() @@ -396,7 +400,8 @@ def includeme(config, auth_resources=None): pyramid.events.ApplicationCreated) config.add_subscriber(write_js_routes_if_enabled, pyramid.events.ApplicationCreated) - + config.add_subscriber(import_license_if_present, + pyramid.events.ApplicationCreated) # Set the default renderer for HTML templates to mako. config.add_mako_renderer('.html') diff --git a/rhodecode/config/patches.py b/rhodecode/config/patches.py index 85804020..04eb02aa 100644 --- a/rhodecode/config/patches.py +++ b/rhodecode/config/patches.py @@ -158,3 +158,10 @@ def inspect_getargspec(): inspect.getargspec = inspect.getfullargspec return inspect + + +def repoze_sendmail_lf_fix(): + from repoze.sendmail import encoding + from email.policy import SMTP + + encoding.encode_message = lambda message, *args, **kwargs: message.as_bytes(policy=SMTP) diff --git a/rhodecode/config/utils.py b/rhodecode/config/utils.py index aec2cfc2..17236b25 100644 --- a/rhodecode/config/utils.py +++ b/rhodecode/config/utils.py @@ -35,7 +35,7 @@ def configure_vcs(config): 'svn': 'rhodecode.lib.vcs.backends.svn.SubversionRepository', } - conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol'] + conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol.v2'] conf.settings.HOOKS_HOST = config['vcs.hooks.host'] conf.settings.DEFAULT_ENCODINGS = config['default_encoding'] conf.settings.ALIASES[:] = config['vcs.backends'] diff --git a/rhodecode/lib/archive_cache/__init__.py b/rhodecode/lib/archive_cache/__init__.py index 53741448..d0cd5541 100644 --- a/rhodecode/lib/archive_cache/__init__.py +++ b/rhodecode/lib/archive_cache/__init__.py @@ -31,9 +31,11 @@ cache_meta = None def includeme(config): + return # don't init cache currently for faster startup time + # init our cache at start - settings = config.get_settings() - get_archival_cache_store(settings) + # settings = config.get_settings() + # get_archival_cache_store(settings) def get_archival_config(config): diff --git a/rhodecode/lib/archive_cache/backends/objectstore_cache.py b/rhodecode/lib/archive_cache/backends/objectstore_cache.py index 07a028d7..dceac413 100644 --- a/rhodecode/lib/archive_cache/backends/objectstore_cache.py +++ b/rhodecode/lib/archive_cache/backends/objectstore_cache.py @@ -58,7 +58,7 @@ class S3Shard(BaseShard): # ensure folder in bucket exists destination = self.bucket if not self.fs.exists(destination): - self.fs.mkdir(destination, s3_additional_kwargs={}) + self.fs.mkdir(destination) writer = self._get_writer(full_path, mode) diff --git a/rhodecode/lib/celerylib/loader.py b/rhodecode/lib/celerylib/loader.py index d1f066b5..deac2bfa 100644 --- a/rhodecode/lib/celerylib/loader.py +++ b/rhodecode/lib/celerylib/loader.py @@ -27,10 +27,11 @@ Celery loader, run with:: --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler \ --loglevel DEBUG --ini=.dev/dev.ini """ -from rhodecode.config.patches import inspect_getargspec, inspect_formatargspec -inspect_getargspec() -inspect_formatargspec() +from rhodecode.config import patches +patches.inspect_getargspec() +patches.inspect_formatargspec() # python3.11 inspect patches for backward compat on `paste` code +patches.repoze_sendmail_lf_fix() import sys import logging diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py index 020e340a..9cb04363 100644 --- a/rhodecode/lib/celerylib/tasks.py +++ b/rhodecode/lib/celerylib/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2023 RhodeCode GmbH +# Copyright (C) 2012-2024 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 @@ -64,8 +64,9 @@ def send_email(recipients, subject, body='', html_body='', email_config=None, "Make sure that `smtp_server` variable is configured " "inside the .ini file") return False - - subject = "%s %s" % (email_config.get('email_prefix', ''), subject) + conf_prefix = email_config.get('email_prefix', None) + prefix = f'{conf_prefix} ' if conf_prefix else '' + subject = f"{prefix}{subject}" if recipients: if isinstance(recipients, str): @@ -86,8 +87,8 @@ def send_email(recipients, subject, body='', html_body='', email_config=None, email_conf = dict( host=mail_server, port=email_config.get('smtp_port', 25), - username=email_config.get('smtp_username'), - password=email_config.get('smtp_password'), + username=email_config.get('smtp_username', None), + password=email_config.get('smtp_password', None), tls=str2bool(email_config.get('smtp_use_tls')), ssl=str2bool(email_config.get('smtp_use_ssl')), @@ -207,7 +208,7 @@ def create_repo(form_data, cur_user): hooks_base.create_repository(created_by=owner.username, **repo.get_dict()) # update repo commit caches initially - repo.update_commit_cache() + repo.update_commit_cache(recursive=False) # set new created state repo.set_state(Repository.STATE_CREATED) @@ -298,7 +299,7 @@ def create_repo_fork(form_data, cur_user): # update repo commit caches initially config = repo._config config.set('extensions', 'largefiles', '') - repo.update_commit_cache(config=config) + repo.update_commit_cache(config=config, recursive=False) # set new created state repo.set_state(Repository.STATE_CREATED) @@ -390,7 +391,7 @@ def sync_last_update_for_objects(*args, **kwargs): .order_by(Repository.group_id.asc()) for repo in repos: - repo.update_commit_cache() + repo.update_commit_cache(recursive=False) skip_groups = kwargs.get('skip_groups') if not skip_groups: diff --git a/rhodecode/lib/db_manage.py b/rhodecode/lib/db_manage.py index e1d2a152..93c95ef1 100644 --- a/rhodecode/lib/db_manage.py +++ b/rhodecode/lib/db_manage.py @@ -570,7 +570,6 @@ class DbManage(object): self.create_ui_settings(path) ui_config = [ - ('web', 'push_ssl', 'False'), ('web', 'allow_archive', 'gz zip bz2'), ('web', 'allow_push', '*'), ('web', 'baseurl', '/'), diff --git a/rhodecode/lib/dbmigrate/versions/115_version_5_1_0.py b/rhodecode/lib/dbmigrate/versions/115_version_5_1_0.py index d1d05862..90de0691 100644 --- a/rhodecode/lib/dbmigrate/versions/115_version_5_1_0.py +++ b/rhodecode/lib/dbmigrate/versions/115_version_5_1_0.py @@ -35,16 +35,19 @@ def downgrade(migrate_engine): def fixups(models, _SESSION): + for db_repo in _SESSION.query(models.Repository).all(): - config = db_repo._config - config.set('extensions', 'largefiles', '') - try: - scm = db_repo.scm_instance(cache=False, config=config) + config = db_repo._config + config.set('extensions', 'largefiles', '') + + scm = db_repo.scm_instance(cache=False, config=config, vcs_full_cache=False) if scm: print(f'installing hook for repo: {db_repo}') scm.install_hooks(force=True) + del scm # force GC + del config except Exception as e: print(e) print('continue...') diff --git a/rhodecode/lib/exceptions.py b/rhodecode/lib/exceptions.py index 0b4a3ef2..d4cd6c17 100644 --- a/rhodecode/lib/exceptions.py +++ b/rhodecode/lib/exceptions.py @@ -80,6 +80,10 @@ class AttachedPullRequestsError(Exception): pass +class AttachedArtifactsError(Exception): + pass + + class RepoGroupAssignmentError(Exception): pass @@ -98,6 +102,11 @@ class HTTPRequirementError(HTTPClientError): self.args = (message, ) +class ClientNotSupportedError(HTTPRequirementError): + title = explanation = 'Client Not Supported' + reason = None + + class HTTPLockedRC(HTTPClientError): """ Special Exception For locked Repos in RhodeCode, the return code can diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py index 033e4736..bfee147b 100644 --- a/rhodecode/lib/helpers.py +++ b/rhodecode/lib/helpers.py @@ -81,7 +81,7 @@ from rhodecode.lib.action_parser import action_parser from rhodecode.lib.html_filters import sanitize_html from rhodecode.lib.pagination import Page, RepoPage, SqlPage from rhodecode.lib import ext_json -from rhodecode.lib.ext_json import json +from rhodecode.lib.ext_json import json, formatted_str_json from rhodecode.lib.str_utils import safe_bytes, convert_special_chars, base64_to_str from rhodecode.lib.utils import repo_name_slug, get_custom_lexer from rhodecode.lib.str_utils import safe_str @@ -1416,62 +1416,14 @@ class InitialsGravatar(object): return "data:image/svg+xml;base64,{}".format(img_data) -def initials_gravatar(request, email_address, first_name, last_name, size=30, store_on_disk=False): +def initials_gravatar(request, email_address, first_name, last_name, size=30): svg_type = None if email_address == User.DEFAULT_USER_EMAIL: svg_type = 'default_user' klass = InitialsGravatar(email_address, first_name, last_name, size) - - if store_on_disk: - from rhodecode.apps.file_store import utils as store_utils - from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \ - FileOverSizeException - from rhodecode.model.db import Session - - image_key = md5_safe(email_address.lower() - + first_name.lower() + last_name.lower()) - - storage = store_utils.get_file_storage(request.registry.settings) - filename = '{}.svg'.format(image_key) - subdir = 'gravatars' - # since final name has a counter, we apply the 0 - uid = storage.apply_counter(0, store_utils.uid_filename(filename, randomized=False)) - store_uid = os.path.join(subdir, uid) - - db_entry = FileStore.get_by_store_uid(store_uid) - if db_entry: - return request.route_path('download_file', fid=store_uid) - - img_data = klass.get_img_data(svg_type=svg_type) - img_file = store_utils.bytes_to_file_obj(img_data) - - try: - store_uid, metadata = storage.save_file( - img_file, filename, directory=subdir, - extensions=['.svg'], randomized_name=False) - except (FileNotAllowedException, FileOverSizeException): - raise - - try: - entry = FileStore.create( - file_uid=store_uid, filename=metadata["filename"], - file_hash=metadata["sha256"], file_size=metadata["size"], - file_display_name=filename, - file_description=f'user gravatar `{safe_str(filename)}`', - hidden=True, check_acl=False, user_id=1 - ) - Session().add(entry) - Session().commit() - log.debug('Stored upload in DB as %s', entry) - except Exception: - raise - - return request.route_path('download_file', fid=store_uid) - - else: - return klass.generate_svg(svg_type=svg_type) + return klass.generate_svg(svg_type=svg_type) def gravatar_external(request, gravatar_url_tmpl, email_address, size=30): diff --git a/rhodecode/lib/hook_daemon/hook_module.py b/rhodecode/lib/hook_daemon/hook_module.py index d719f88a..c6e88c0b 100644 --- a/rhodecode/lib/hook_daemon/hook_module.py +++ b/rhodecode/lib/hook_daemon/hook_module.py @@ -66,12 +66,12 @@ class Hooks(object): result = hook(extras) if result is None: raise Exception(f'Failed to obtain hook result from func: {hook}') - except HTTPBranchProtected as handled_error: + except HTTPBranchProtected as error: # Those special cases don't need error reporting. It's a case of # locked repo or protected branch result = AttributeDict({ - 'status': handled_error.code, - 'output': handled_error.explanation + 'status': error.code, + 'output': error.explanation }) except (HTTPLockedRC, Exception) as error: # locked needs different handling since we need to also diff --git a/rhodecode/lib/hooks_base.py b/rhodecode/lib/hooks_base.py index 04186ecf..795987cc 100644 --- a/rhodecode/lib/hooks_base.py +++ b/rhodecode/lib/hooks_base.py @@ -30,7 +30,7 @@ from rhodecode.lib import helpers as h from rhodecode.lib import audit_logger from rhodecode.lib.utils2 import safe_str, user_agent_normalizer from rhodecode.lib.exceptions import ( - HTTPLockedRC, HTTPBranchProtected, UserCreationError) + HTTPLockedRC, HTTPBranchProtected, UserCreationError, ClientNotSupportedError) from rhodecode.model.db import Repository, User from rhodecode.lib.statsd_client import StatsdClient @@ -64,6 +64,18 @@ def is_shadow_repo(extras): return extras['is_shadow_repo'] +def check_vcs_client(extras): + """ + Checks if vcs client is allowed (Only works in enterprise edition) + """ + try: + from rc_ee.lib.security.utils import is_vcs_client_whitelisted + except ModuleNotFoundError: + is_vcs_client_whitelisted = lambda *x: True + backend = extras.get('scm') + if not is_vcs_client_whitelisted(extras.get('user_agent'), backend): + raise ClientNotSupportedError(f"Your {backend} client is forbidden") + def _get_scm_size(alias, root_path): if not alias.startswith('.'): @@ -108,6 +120,7 @@ def pre_push(extras): It bans pushing when the repository is locked. """ + check_vcs_client(extras) user = User.get_by_username(extras.username) output = '' if extras.locked_by[0] and user.user_id != int(extras.locked_by[0]): @@ -129,6 +142,8 @@ def pre_push(extras): if extras.commit_ids and extras.check_branch_perms: auth_user = user.AuthUser() repo = Repository.get_by_repo_name(extras.repository) + if not repo: + raise ValueError(f'Repo for {extras.repository} not found') affected_branches = [] if repo.repo_type == 'hg': for entry in extras.commit_ids: @@ -180,6 +195,7 @@ def pre_pull(extras): It bans pulling when the repository is locked. """ + check_vcs_client(extras) output = '' if extras.locked_by[0]: locked_by = User.get(extras.locked_by[0]).username diff --git a/rhodecode/lib/middleware/request_wrapper.py b/rhodecode/lib/middleware/request_wrapper.py index 3d5d438e..68b21aba 100644 --- a/rhodecode/lib/middleware/request_wrapper.py +++ b/rhodecode/lib/middleware/request_wrapper.py @@ -46,7 +46,7 @@ class RequestWrapperTween(object): def __call__(self, request): start = time.time() - log.debug('Starting request time measurement') + log.debug('Starting request processing') response = None request.req_wrapper_start = start @@ -63,7 +63,7 @@ class RequestWrapperTween(object): total = time.time() - start log.info( - 'Req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s', + 'Finished request processing: req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s', count, _auth_user, request.environ.get('REQUEST_METHOD'), _path, total, get_user_agent(request. environ), _ver_, extra={"time": total, "ver": _ver_, "ip": ip, diff --git a/rhodecode/lib/middleware/simplehg.py b/rhodecode/lib/middleware/simplehg.py index 95bd4e6b..5100f589 100644 --- a/rhodecode/lib/middleware/simplehg.py +++ b/rhodecode/lib/middleware/simplehg.py @@ -53,25 +53,31 @@ class SimpleHg(simplevcs.SimpleVCS): return repo_name.rstrip('/') _ACTION_MAPPING = { + 'between': 'pull', + 'branches': 'pull', + 'branchmap': 'pull', + 'capabilities': 'pull', 'changegroup': 'pull', 'changegroupsubset': 'pull', - 'getbundle': 'pull', - 'stream_out': 'pull', - 'listkeys': 'pull', - 'between': 'pull', - 'branchmap': 'pull', - 'branches': 'pull', + 'changesetdata': 'pull', 'clonebundles': 'pull', - 'capabilities': 'pull', + 'clonebundles_manifest': 'pull', 'debugwireargs': 'pull', + 'filedata': 'pull', + 'getbundle': 'pull', 'heads': 'pull', - 'lookup': 'pull', 'hello': 'pull', 'known': 'pull', + 'listkeys': 'pull', + 'lookup': 'pull', + 'manifestdata': 'pull', + 'narrow_widen': 'pull', + 'protocaps': 'pull', + 'stream_out': 'pull', # largefiles - 'putlfile': 'push', 'getlfile': 'pull', + 'putlfile': 'push', 'statlfile': 'pull', 'lheads': 'pull', diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py index 67480355..b3f7c230 100644 --- a/rhodecode/lib/middleware/simplevcs.py +++ b/rhodecode/lib/middleware/simplevcs.py @@ -293,7 +293,7 @@ class SimpleVCS(object): def compute_perm_vcs( cache_name, plugin_id, action, user_id, repo_name, ip_addr): - log.debug('auth: calculating permission access now...') + log.debug('auth: calculating permission access now for vcs operation: %s', action) # check IP inherit = user.inherit_default_permissions ip_allowed = AuthUser.check_ip_allowed( @@ -339,21 +339,6 @@ class SimpleVCS(object): log.exception('Failed to read http scheme') return 'http' - def _check_ssl(self, environ, start_response): - """ - Checks the SSL check flag and returns False if SSL is not present - and required True otherwise - """ - org_proto = environ['wsgi._org_proto'] - # check if we have SSL required ! if not it's a bad request ! - require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl')) - if require_ssl and org_proto == 'http': - log.debug( - 'Bad request: detected protocol is `%s` and ' - 'SSL/HTTPS is required.', org_proto) - return False - return True - def _get_default_cache_ttl(self): # take AUTH_CACHE_TTL from the `rhodecode` auth plugin plugin = loadplugin('egg:rhodecode-enterprise-ce#rhodecode') @@ -373,12 +358,6 @@ class SimpleVCS(object): meta.Session.remove() def _handle_request(self, environ, start_response): - if not self._check_ssl(environ, start_response): - reason = ('SSL required, while RhodeCode was unable ' - 'to detect this as SSL request') - log.debug('User not allowed to proceed, %s', reason) - return HTTPNotAcceptable(reason)(environ, start_response) - if not self.url_repo_name: log.warning('Repository name is empty: %s', self.url_repo_name) # failed to get repo name, we fail now diff --git a/rhodecode/lib/middleware/vcs.py b/rhodecode/lib/middleware/vcs.py index 49c636f2..59b26c8f 100644 --- a/rhodecode/lib/middleware/vcs.py +++ b/rhodecode/lib/middleware/vcs.py @@ -159,11 +159,18 @@ def detect_vcs_request(environ, backends): # favicon often requested by browsers 'favicon.ico', + # static files no detection + '_static++', + + # debug-toolbar + '_debug_toolbar++', + # e.g /_file_store/download '_file_store++', # login - "_admin/login", + f"{ADMIN_PREFIX}/login", + f"{ADMIN_PREFIX}/logout", # 2fa f"{ADMIN_PREFIX}/check_2fa", @@ -178,12 +185,6 @@ def detect_vcs_request(environ, backends): # _admin/my_account is safe too f'{ADMIN_PREFIX}/my_account++', - # static files no detection - '_static++', - - # debug-toolbar - '_debug_toolbar++', - # skip ops ping, status f'{ADMIN_PREFIX}/ops/ping', f'{ADMIN_PREFIX}/ops/status', @@ -193,11 +194,14 @@ def detect_vcs_request(environ, backends): '++/repo_creating_check' ] + path_info = get_path_info(environ) path_url = path_info.lstrip('/') req_method = environ.get('REQUEST_METHOD') for item in white_list: + item = item.lstrip('/') + if item.endswith('++') and path_url.startswith(item[:-2]): log.debug('path `%s` in whitelist (match:%s), skipping...', path_url, item) return handler diff --git a/rhodecode/lib/rc_cache/backends.py b/rhodecode/lib/rc_cache/backends.py index ddf939ad..b4f64aa5 100644 --- a/rhodecode/lib/rc_cache/backends.py +++ b/rhodecode/lib/rc_cache/backends.py @@ -38,9 +38,9 @@ from dogpile.cache.backends import redis as redis_backend from dogpile.cache.backends.file import FileLock from dogpile.cache.util import memoized_property -from rhodecode.lib.memory_lru_dict import LRUDict, LRUDictDebug -from rhodecode.lib.str_utils import safe_bytes, safe_str -from rhodecode.lib.type_utils import str2bool +from ...lib.memory_lru_dict import LRUDict, LRUDictDebug +from ...lib.str_utils import safe_bytes, safe_str +from ...lib.type_utils import str2bool _default_max_size = 1024 @@ -198,6 +198,13 @@ class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend): def get_store(self): return self.filename + def cleanup_store(self): + for ext in ("db", "dat", "pag", "dir"): + final_filename = self.filename + os.extsep + ext + if os.path.exists(final_filename): + os.remove(final_filename) + log.warning('Removed dbm file %s', final_filename) + class BaseRedisBackend(redis_backend.RedisBackend): key_prefix = '' @@ -289,7 +296,7 @@ class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend): def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False): - from rhodecode.lib._vendor import redis_lock + from ...lib._vendor import redis_lock class _RedisLockWrapper: """LockWrapper for redis_lock""" diff --git a/rhodecode/lib/rc_cache/utils.py b/rhodecode/lib/rc_cache/utils.py index f59dce38..5d0c39f4 100644 --- a/rhodecode/lib/rc_cache/utils.py +++ b/rhodecode/lib/rc_cache/utils.py @@ -26,9 +26,9 @@ import decorator from dogpile.cache import CacheRegion import rhodecode -from rhodecode.lib.hash_utils import sha1 -from rhodecode.lib.str_utils import safe_bytes -from rhodecode.lib.type_utils import str2bool # noqa :required by imports from .utils +from ...lib.hash_utils import sha1 +from ...lib.str_utils import safe_bytes +from ...lib.type_utils import str2bool # noqa :required by imports from .utils from . import region_meta diff --git a/rhodecode/lib/rc_commands/add_artifact.py b/rhodecode/lib/rc_commands/add_artifact.py index e04b8a6d..5bbe6d7a 100644 --- a/rhodecode/lib/rc_commands/add_artifact.py +++ b/rhodecode/lib/rc_commands/add_artifact.py @@ -91,15 +91,14 @@ def command(ini_path, filename, file_path, repo_id, user_id, description): auth_user = db_user.AuthUser(ip_addr='127.0.0.1') - storage = store_utils.get_file_storage(request.registry.settings) + f_store = store_utils.get_filestore_backend(request.registry.settings) with open(file_path, 'rb') as f: click.secho(f'Adding new artifact from path: `{file_path}`', fg='green') file_data = _store_file( - storage, auth_user, filename, content=None, check_acl=True, + f_store, auth_user, filename, content=None, check_acl=True, file_obj=f, description=description, scope_repo_id=repo.repo_id) - click.secho(f'File Data: {file_data}', - fg='green') + click.secho(f'File Data: {file_data}', fg='green') diff --git a/rhodecode/lib/rc_commands/migrate_artifact.py b/rhodecode/lib/rc_commands/migrate_artifact.py new file mode 100644 index 00000000..9205daba --- /dev/null +++ b/rhodecode/lib/rc_commands/migrate_artifact.py @@ -0,0 +1,122 @@ +# Copyright (C) 2016-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import sys +import logging + +import click + +from rhodecode.lib.pyramid_utils import bootstrap +from rhodecode.lib.ext_json import json +from rhodecode.model.db import FileStore +from rhodecode.apps.file_store import utils as store_utils + +log = logging.getLogger(__name__) + + +@click.command() +@click.argument('ini_path', type=click.Path(exists=True)) +@click.argument('file_uid') +@click.option( + '--source-backend-conf', + type=click.Path(exists=True, dir_okay=False, readable=True), + help='Source backend config file path in a json format' +) +@click.option( + '--dest-backend-conf', + type=click.Path(exists=True, dir_okay=False, readable=True), + help='Source backend config file path in a json format' +) +def main(ini_path, file_uid, source_backend_conf, dest_backend_conf): + return command(ini_path, file_uid, source_backend_conf, dest_backend_conf) + + +_source_settings = {} + +_dest_settings = {} + + +def command(ini_path, file_uid, source_backend_conf, dest_backend_conf): + with bootstrap(ini_path, env={'RC_CMD_SETUP_RC': '1'}) as env: + migrate_func(env, file_uid, source_backend_conf, dest_backend_conf) + + +def migrate_func(env, file_uid, source_backend_conf=None, dest_backend_conf=None): + """ + + Example usage:: + + from rhodecode.lib.rc_commands import migrate_artifact + migrate_artifact._source_settings = { + 'file_store.backend.type': 'filesystem_v1', + 'file_store.filesystem_v1.storage_path': '/var/opt/rhodecode_data/file_store', + } + migrate_artifact._dest_settings = { + 'file_store.backend.type': 'objectstore', + 'file_store.objectstore.url': 'http://s3-minio:9000', + 'file_store.objectstore.bucket': 'rhodecode-file-store', + 'file_store.objectstore.key': 's3admin', + 'file_store.objectstore.secret': 's3secret4', + 'file_store.objectstore.region': 'eu-central-1', + } + for db_obj in FileStore.query().all(): + migrate_artifact.migrate_func({}, db_obj.file_uid) + + """ + + try: + from rc_ee.api.views.store_api import _store_file + except ImportError: + click.secho('ERROR: Unable to import store_api. ' + 'store_api is only available in EE edition of RhodeCode', + fg='red') + sys.exit(-1) + + source_settings = _source_settings + if source_backend_conf: + source_settings = json.loads(open(source_backend_conf).read()) + dest_settings = _dest_settings + if dest_backend_conf: + dest_settings = json.loads(open(dest_backend_conf).read()) + + if file_uid.isnumeric(): + file_store_db_obj = FileStore().query() \ + .filter(FileStore.file_store_id == file_uid) \ + .scalar() + else: + file_store_db_obj = FileStore().query() \ + .filter(FileStore.file_uid == file_uid) \ + .scalar() + if not file_store_db_obj: + click.secho(f'ERROR: Unable to fetch artifact from database file_uid={file_uid}', + fg='red') + sys.exit(-1) + + uid_filename = file_store_db_obj.file_uid + org_filename = file_store_db_obj.file_display_name + click.secho(f'Attempting to migrate artifact {uid_filename}, filename: {org_filename}', fg='green') + + # get old version of f_store based on the data. + + origin_f_store = store_utils.get_filestore_backend(source_settings, always_init=True) + reader, metadata = origin_f_store.fetch(uid_filename) + + target_f_store = store_utils.get_filestore_backend(dest_settings, always_init=True) + target_f_store.import_to_store(reader, org_filename, uid_filename, metadata) + + click.secho(f'Migrated artifact {uid_filename}, filename: {org_filename} into {target_f_store} storage', fg='green') diff --git a/rhodecode/lib/rc_commands/setup_rc.py b/rhodecode/lib/rc_commands/setup_rc.py index b5626f6f..18d133ac 100644 --- a/rhodecode/lib/rc_commands/setup_rc.py +++ b/rhodecode/lib/rc_commands/setup_rc.py @@ -108,11 +108,10 @@ def command(ini_path, force_yes, user, email, password, api_key, repos, dbmanage.create_permissions() dbmanage.populate_default_permissions() if apply_license_key: - try: - from rc_license.models import apply_trial_license_if_missing - apply_trial_license_if_missing(force=True) - except ImportError: - pass + from rhodecode.model.license import apply_license_from_file + license_file_path = config.get('license.import_path') + if license_file_path: + apply_license_from_file(license_file_path, force=True) Session().commit() diff --git a/rhodecode/lib/str_utils.py b/rhodecode/lib/str_utils.py index 78633d2d..437aabde 100644 --- a/rhodecode/lib/str_utils.py +++ b/rhodecode/lib/str_utils.py @@ -181,3 +181,7 @@ def splitnewlines(text: bytes): else: lines[-1] = lines[-1][:-1] return lines + + +def header_safe_str(val): + return safe_bytes(val).decode('latin-1', errors='replace') diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py index 157b631c..f8f8ea50 100644 --- a/rhodecode/lib/system_info.py +++ b/rhodecode/lib/system_info.py @@ -396,17 +396,18 @@ def storage_inodes(): @register_sysinfo -def storage_archives(): +def storage_artifacts(): import rhodecode from rhodecode.lib.helpers import format_byte_size_binary from rhodecode.lib.archive_cache import get_archival_cache_store - storage_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type') + backend_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type') - value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=storage_type) + value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type) state = STATE_OK_DEFAULT try: d_cache = get_archival_cache_store(config=rhodecode.CONFIG) + backend_type = str(d_cache) total_files, total_size, _directory_stats = d_cache.get_statistics() @@ -415,7 +416,8 @@ def storage_archives(): 'used': total_size, 'total': total_size, 'items': total_files, - 'path': d_cache.storage_path + 'path': d_cache.storage_path, + 'type': backend_type }) except Exception as e: @@ -425,8 +427,44 @@ def storage_archives(): human_value = value.copy() human_value['used'] = format_byte_size_binary(value['used']) human_value['total'] = format_byte_size_binary(value['total']) - human_value['text'] = "{} ({} items)".format( - human_value['used'], value['items']) + human_value['text'] = f"{human_value['used']} ({value['items']} items)" + + return SysInfoRes(value=value, state=state, human_value=human_value) + + +@register_sysinfo +def storage_archives(): + import rhodecode + from rhodecode.lib.helpers import format_byte_size_binary + import rhodecode.apps.file_store.utils as store_utils + from rhodecode import CONFIG + + backend_type = rhodecode.ConfigGet().get_str(store_utils.config_keys.backend_type) + + value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type) + state = STATE_OK_DEFAULT + try: + f_store = store_utils.get_filestore_backend(config=CONFIG) + backend_type = str(f_store) + total_files, total_size, _directory_stats = f_store.get_statistics() + + value.update({ + 'percent': 100, + 'used': total_size, + 'total': total_size, + 'items': total_files, + 'path': f_store.storage_path, + 'type': backend_type + }) + + except Exception as e: + log.exception('failed to fetch archive cache storage') + state = {'message': str(e), 'type': STATE_ERR} + + human_value = value.copy() + human_value['used'] = format_byte_size_binary(value['used']) + human_value['total'] = format_byte_size_binary(value['total']) + human_value['text'] = f"{human_value['used']} ({value['items']} items)" return SysInfoRes(value=value, state=state, human_value=human_value) @@ -798,6 +836,7 @@ def get_system_info(environ): 'storage': SysInfo(storage)(), 'storage_inodes': SysInfo(storage_inodes)(), 'storage_archive': SysInfo(storage_archives)(), + 'storage_artifacts': SysInfo(storage_artifacts)(), 'storage_gist': SysInfo(storage_gist)(), 'storage_temp': SysInfo(storage_temp)(), diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py index ac07c77e..11331c06 100644 --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -42,6 +42,8 @@ from webhelpers2.text import collapse, strip_tags, convert_accented_entities, co from mako import exceptions +from rhodecode import ConfigGet +from rhodecode.lib.exceptions import HTTPBranchProtected, HTTPLockedRC from rhodecode.lib.hash_utils import sha256_safe, md5, sha1 from rhodecode.lib.type_utils import AttributeDict from rhodecode.lib.str_utils import safe_bytes, safe_str @@ -84,8 +86,39 @@ def adopt_for_celery(func): @wraps(func) def wrapper(extras): extras = AttributeDict(extras) - # HooksResponse implements to_json method which must be used there. - return func(extras).to_json() + try: + # HooksResponse implements to_json method which must be used there. + return func(extras).to_json() + except HTTPBranchProtected as error: + # Those special cases don't need error reporting. It's a case of + # locked repo or protected branch + error_args = error.args + return { + 'status': error.code, + 'output': error.explanation, + 'exception': type(error).__name__, + 'exception_args': error_args, + 'exception_traceback': '', + } + except HTTPLockedRC as error: + # Those special cases don't need error reporting. It's a case of + # locked repo or protected branch + error_args = error.args + return { + 'status': error.code, + 'output': error.explanation, + 'exception': type(error).__name__, + 'exception_args': error_args, + 'exception_traceback': '', + } + except Exception as e: + return { + 'status': 128, + 'output': '', + 'exception': type(e).__name__, + 'exception_args': e.args, + 'exception_traceback': '', + } return wrapper @@ -361,32 +394,39 @@ ui_sections = [ 'ui', 'web', ] -def config_data_from_db(clear_session=True, repo=None): +def prepare_config_data(clear_session=True, repo=None): """ - Read the configuration data from the database and return configuration + Read the configuration data from the database, *.ini files and return configuration tuples. """ from rhodecode.model.settings import VcsSettingsModel - config = [] - sa = meta.Session() settings_model = VcsSettingsModel(repo=repo, sa=sa) ui_settings = settings_model.get_ui_settings() ui_data = [] + config = [ + ('web', 'push_ssl', 'false'), + ] for setting in ui_settings: + # Todo: remove this section once transition to *.ini files will be completed + if setting.section in ('largefiles', 'vcs_git_lfs'): + if setting.key != 'enabled': + continue if setting.active: ui_data.append((setting.section, setting.key, setting.value)) config.append(( safe_str(setting.section), safe_str(setting.key), safe_str(setting.value))) if setting.key == 'push_ssl': - # force set push_ssl requirement to False, rhodecode - # handles that + # force set push_ssl requirement to False this is deprecated, and we must force it to False config.append(( safe_str(setting.section), safe_str(setting.key), False)) + config_getter = ConfigGet() + config.append(('vcs_git_lfs', 'store_location', config_getter.get_str('vcs.git.lfs.storage_location'))) + config.append(('largefiles', 'usercache', config_getter.get_str('vcs.hg.largefiles.storage_location'))) log.debug( 'settings ui from db@repo[%s]: %s', repo, @@ -415,7 +455,7 @@ def make_db_config(clear_session=True, repo=None): Create a :class:`Config` instance based on the values in the database. """ config = Config() - config_data = config_data_from_db(clear_session=clear_session, repo=repo) + config_data = prepare_config_data(clear_session=clear_session, repo=repo) for section, option, value in config_data: config.set(section, option, value) return config @@ -582,7 +622,7 @@ def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild log.debug('Running update server info') git_repo._update_server_info(force=True) - db_repo.update_commit_cache() + db_repo.update_commit_cache(recursive=False) config = db_repo._config config.set('extensions', 'largefiles', '') diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py index 4801e70d..8c2466e3 100644 --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -2568,10 +2568,10 @@ class Repository(Base, BaseModel): return commit def flush_commit_cache(self): - self.update_commit_cache(cs_cache={'raw_id':'0'}) + self.update_commit_cache(cs_cache={'raw_id': '0'}) self.update_commit_cache() - def update_commit_cache(self, cs_cache=None, config=None): + def update_commit_cache(self, cs_cache=None, config=None, recursive=True): """ Update cache of last commit for repository cache_keys should be:: @@ -2610,6 +2610,14 @@ class Repository(Base, BaseModel): if isinstance(cs_cache, BaseCommit): cs_cache = cs_cache.__json__() + def maybe_update_recursive(instance, _config, _recursive, _cs_cache, _last_change): + if _recursive: + repo_id = instance.repo_id + _cs_cache['source_repo_id'] = repo_id + for gr in instance.groups_with_parents: + gr.changeset_cache = _cs_cache + gr.updated_on = _last_change + def is_outdated(new_cs_cache): if (new_cs_cache['raw_id'] != self.changeset_cache['raw_id'] or new_cs_cache['revision'] != self.changeset_cache['revision']): @@ -2636,6 +2644,7 @@ class Repository(Base, BaseModel): self.changeset_cache = cs_cache self.updated_on = last_change Session().add(self) + maybe_update_recursive(self, config, recursive, cs_cache, last_change) Session().commit() else: @@ -2650,6 +2659,7 @@ class Repository(Base, BaseModel): self.changeset_cache = cs_cache self.updated_on = _date_latest Session().add(self) + maybe_update_recursive(self, config, recursive, cs_cache, _date_latest) Session().commit() log.debug('updated repo `%s` with new commit cache %s, and last update_date: %s', @@ -5839,8 +5849,7 @@ class FileStore(Base, BaseModel): .filter(FileStoreMetadata.file_store_meta_key == key) \ .scalar() if has_key: - msg = 'key `{}` already defined under section `{}` for this file.'\ - .format(key, section) + msg = f'key `{key}` already defined under section `{section}` for this file.' raise ArtifactMetadataDuplicate(msg, err_section=section, err_key=key) # NOTE(marcink): raises ArtifactMetadataBadValueType @@ -5939,7 +5948,7 @@ class FileStoreMetadata(Base, BaseModel): def valid_value_type(cls, value): if value.split('.')[0] not in cls.SETTINGS_TYPES: raise ArtifactMetadataBadValueType( - 'value_type must be one of %s got %s' % (cls.SETTINGS_TYPES.keys(), value)) + f'value_type must be one of {cls.SETTINGS_TYPES.keys()} got {value}') @hybrid_property def file_store_meta_section(self): diff --git a/rhodecode/model/forms.py b/rhodecode/model/forms.py index 6ceffd53..bc82d930 100644 --- a/rhodecode/model/forms.py +++ b/rhodecode/model/forms.py @@ -129,6 +129,20 @@ def TOTPForm(localizer, user, allow_recovery_code_use=False): return _TOTPForm +def WhitelistedVcsClientsForm(localizer): + _ = localizer + + class _WhitelistedVcsClientsForm(formencode.Schema): + regexp = r'^(?:\s*[<>=~^!]*\s*\d{1,2}\.\d{1,2}(?:\.\d{1,2})?\s*|\*)\s*(?:,\s*[<>=~^!]*\s*\d{1,2}\.\d{1,2}(?:\.\d{1,2})?\s*|\s*\*\s*)*$' + allow_extra_fields = True + filter_extra_fields = True + git = v.Regex(regexp) + hg = v.Regex(regexp) + svn = v.Regex(regexp) + + return _WhitelistedVcsClientsForm + + def UserForm(localizer, edit=False, available_languages=None, old_data=None): old_data = old_data or {} available_languages = available_languages or [] @@ -454,13 +468,6 @@ def ApplicationUiSettingsForm(localizer): _ = localizer class _ApplicationUiSettingsForm(_BaseVcsSettingsForm): - web_push_ssl = v.StringBoolean(if_missing=False) - largefiles_usercache = All( - v.ValidPath(localizer), - v.UnicodeString(strip=True, min=2, not_empty=True)) - vcs_git_lfs_store_location = All( - v.ValidPath(localizer), - v.UnicodeString(strip=True, min=2, not_empty=True)) extensions_hggit = v.StringBoolean(if_missing=False) new_svn_branch = v.ValidSvnPattern(localizer, section='vcs_svn_branch') new_svn_tag = v.ValidSvnPattern(localizer, section='vcs_svn_tag') diff --git a/rhodecode/model/license.py b/rhodecode/model/license.py new file mode 100644 index 00000000..011bb31d --- /dev/null +++ b/rhodecode/model/license.py @@ -0,0 +1,17 @@ + +def apply_license(*args, **kwargs): + pass + +try: + from rc_license.models import apply_license +except ImportError: + pass + + +def apply_license_from_file(*args, **kwargs): + pass + +try: + from rc_license.models import apply_license_from_file +except ImportError: + pass diff --git a/rhodecode/model/notification.py b/rhodecode/model/notification.py index df3af388..acad4e6c 100644 --- a/rhodecode/model/notification.py +++ b/rhodecode/model/notification.py @@ -117,14 +117,16 @@ class NotificationModel(BaseModel): # add mentioned users into recipients final_recipients = set(recipients_objs).union(mention_recipients) - (subject, email_body, email_body_plaintext) = \ - EmailNotificationModel().render_email(notification_type, **email_kwargs) + # No need to render email if we are sending just notification + if with_email: + (subject, email_body, email_body_plaintext) = \ + EmailNotificationModel().render_email(notification_type, **email_kwargs) - if not notification_subject: - notification_subject = subject + if not notification_subject: + notification_subject = subject - if not notification_body: - notification_body = email_body_plaintext + if not notification_body: + notification_body = email_body_plaintext notification = Notification.create( created_by=created_by_obj, subject=notification_subject, diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py index 8b8df315..312b89f1 100644 --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -31,7 +31,7 @@ from zope.cachedescriptors.property import Lazy as LazyProperty from rhodecode import events from rhodecode.lib.auth import HasUserGroupPermissionAny from rhodecode.lib.caching_query import FromCache -from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError +from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError from rhodecode.lib import hooks_base from rhodecode.lib.user_log_filter import user_log_filter from rhodecode.lib.utils import make_db_config @@ -736,7 +736,7 @@ class RepoModel(BaseModel): log.error(traceback.format_exc()) raise - def delete(self, repo, forks=None, pull_requests=None, fs_remove=True, cur_user=None): + def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None): """ Delete given repository, forks parameter defines what do do with attached forks. Throws AttachedForksError if deleted repo has attached @@ -745,6 +745,7 @@ class RepoModel(BaseModel): :param repo: :param forks: str 'delete' or 'detach' :param pull_requests: str 'delete' or None + :param artifacts: str 'delete' or None :param fs_remove: remove(archive) repo from filesystem """ if not cur_user: @@ -767,6 +768,13 @@ class RepoModel(BaseModel): if pull_requests != 'delete' and (pr_sources or pr_targets): raise AttachedPullRequestsError() + artifacts_objs = repo.artifacts + if artifacts == 'delete': + for a in artifacts_objs: + self.sa.delete(a) + elif [a for a in artifacts_objs]: + raise AttachedArtifactsError() + old_repo_dict = repo.get_dict() events.trigger(events.RepoPreDeleteEvent(repo)) try: diff --git a/rhodecode/model/settings.py b/rhodecode/model/settings.py index a3421304..275cd868 100644 --- a/rhodecode/model/settings.py +++ b/rhodecode/model/settings.py @@ -486,7 +486,6 @@ class VcsSettingsModel(object): ) GLOBAL_HG_SETTINGS = ( ('extensions', 'largefiles'), - ('largefiles', 'usercache'), ('phases', 'publish'), ('extensions', 'evolve'), ('extensions', 'topic'), @@ -496,12 +495,10 @@ class VcsSettingsModel(object): GLOBAL_GIT_SETTINGS = ( ('vcs_git_lfs', 'enabled'), - ('vcs_git_lfs', 'store_location') ) SVN_BRANCH_SECTION = 'vcs_svn_branch' SVN_TAG_SECTION = 'vcs_svn_tag' - SSL_SETTING = ('web', 'push_ssl') PATH_SETTING = ('paths', '/') def __init__(self, sa=None, repo=None): @@ -666,17 +663,15 @@ class VcsSettingsModel(object): self.repo_settings, *phases, value=safe_str(data[phases_key])) def create_or_update_global_hg_settings(self, data): - opts_len = 4 - largefiles, largefiles_store, phases, evolve \ + opts_len = 3 + largefiles, phases, evolve \ = self.GLOBAL_HG_SETTINGS[:opts_len] - largefiles_key, largefiles_store_key, phases_key, evolve_key \ + largefiles_key, phases_key, evolve_key \ = self._get_settings_keys(self.GLOBAL_HG_SETTINGS[:opts_len], data) self._create_or_update_ui( self.global_settings, *largefiles, value='', active=data[largefiles_key]) - self._create_or_update_ui( - self.global_settings, *largefiles_store, value=data[largefiles_store_key]) self._create_or_update_ui( self.global_settings, *phases, value=safe_str(data[phases_key])) self._create_or_update_ui( @@ -697,26 +692,17 @@ class VcsSettingsModel(object): active=data[lfs_enabled_key]) def create_or_update_global_git_settings(self, data): - lfs_enabled, lfs_store_location \ - = self.GLOBAL_GIT_SETTINGS - lfs_enabled_key, lfs_store_location_key \ - = self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data) + lfs_enabled = self.GLOBAL_GIT_SETTINGS[0] + lfs_enabled_key = self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data)[0] self._create_or_update_ui( self.global_settings, *lfs_enabled, value=data[lfs_enabled_key], active=data[lfs_enabled_key]) - self._create_or_update_ui( - self.global_settings, *lfs_store_location, - value=data[lfs_store_location_key]) def create_or_update_global_svn_settings(self, data): # branch/tags patterns self._create_svn_settings(self.global_settings, data) - def update_global_ssl_setting(self, value): - self._create_or_update_ui( - self.global_settings, *self.SSL_SETTING, value=value) - @assert_repo_settings def delete_repo_svn_pattern(self, id_): ui = self.repo_settings.UiDbModel.get(id_) diff --git a/rhodecode/model/user.py b/rhodecode/model/user.py index 69af24c0..44ebe36b 100644 --- a/rhodecode/model/user.py +++ b/rhodecode/model/user.py @@ -557,10 +557,10 @@ class UserModel(BaseModel): elif handle_mode == 'delete': from rhodecode.apps.file_store import utils as store_utils request = get_current_request() - storage = store_utils.get_file_storage(request.registry.settings) + f_store = store_utils.get_filestore_backend(request.registry.settings) for a in artifacts: file_uid = a.file_uid - storage.delete(file_uid) + f_store.delete(file_uid) self.sa.delete(a) left_overs = False diff --git a/rhodecode/public/js/rhodecode/routes.js b/rhodecode/public/js/rhodecode/routes.js index e224b7c5..37a985a0 100644 --- a/rhodecode/public/js/rhodecode/routes.js +++ b/rhodecode/public/js/rhodecode/routes.js @@ -86,6 +86,7 @@ function registerRCRoutes() { pyroutes.register('admin_settings_vcs_update', '/_admin/settings/vcs/update', []); pyroutes.register('admin_settings_visual', '/_admin/settings/visual', []); pyroutes.register('admin_settings_visual_update', '/_admin/settings/visual/update', []); + pyroutes.register('admin_security_modify_allowed_vcs_client_versions', '/_admin/security/modify/allowed_vcs_client_versions', []); pyroutes.register('apiv2', '/_admin/api', []); pyroutes.register('atom_feed_home', '/%(repo_name)s/feed-atom', ['repo_name']); pyroutes.register('atom_feed_home_old', '/%(repo_name)s/feed/atom', ['repo_name']); diff --git a/rhodecode/subscribers.py b/rhodecode/subscribers.py index 07c04fd9..4bf7f2b8 100644 --- a/rhodecode/subscribers.py +++ b/rhodecode/subscribers.py @@ -111,9 +111,11 @@ def scan_repositories_if_enabled(event): This is subscribed to the `pyramid.events.ApplicationCreated` event. It does a repository scan if enabled in the settings. """ + settings = event.app.registry.settings vcs_server_enabled = settings['vcs.server.enable'] import_on_startup = settings['startup.import_repos'] + if vcs_server_enabled and import_on_startup: from rhodecode.model.scm import ScmModel from rhodecode.lib.utils import repo2db_mapper @@ -205,7 +207,7 @@ def write_usage_data(event): return def get_update_age(dest_file): - now = datetime.datetime.utcnow() + now = datetime.datetime.now(datetime.UTC) with open(dest_file, 'rb') as f: data = ext_json.json.loads(f.read()) @@ -216,10 +218,9 @@ def write_usage_data(event): return 0 - utc_date = datetime.datetime.utcnow() + utc_date = datetime.datetime.now(datetime.UTC) hour_quarter = int(math.ceil((utc_date.hour + utc_date.minute/60.0) / 6.)) - fname = '.rc_usage_{date.year}{date.month:02d}{date.day:02d}_{hour}.json'.format( - date=utc_date, hour=hour_quarter) + fname = f'.rc_usage_{utc_date.year}{utc_date.month:02d}{utc_date.day:02d}_{hour_quarter}.json' ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__')) usage_dir = os.path.join(ini_loc, '.rcusage') @@ -314,6 +315,28 @@ def write_js_routes_if_enabled(event): log.exception('Failed to write routes.js into %s', jsroutes_file_path) +def import_license_if_present(event): + """ + This is subscribed to the `pyramid.events.ApplicationCreated` event. It + does a import license key based on a presence of the file. + """ + settings = event.app.registry.settings + + rhodecode_edition_id = settings.get('rhodecode.edition_id') + license_file_path = settings.get('license.import_path') + force = settings.get('license.import_path_mode') == 'force' + + if license_file_path and rhodecode_edition_id == 'EE': + log.debug('license.import_path= is set importing license from %s', license_file_path) + from rhodecode.model.meta import Session + from rhodecode.model.license import apply_license_from_file + try: + apply_license_from_file(license_file_path, force=force) + Session().commit() + except OSError: + log.exception('Failed to import license from %s, make sure this file exists', license_file_path) + + class Subscriber(object): """ Base class for subscribers to the pyramid event system. diff --git a/rhodecode/templates/admin/auth/auth_settings.mako b/rhodecode/templates/admin/auth/auth_settings.mako index 92ae19f2..1e430b34 100644 --- a/rhodecode/templates/admin/auth/auth_settings.mako +++ b/rhodecode/templates/admin/auth/auth_settings.mako @@ -26,8 +26,13 @@
diff --git a/rhodecode/templates/admin/auth/plugin_settings.mako b/rhodecode/templates/admin/auth/plugin_settings.mako index 0f09b80c..07cba2f0 100644 --- a/rhodecode/templates/admin/auth/plugin_settings.mako +++ b/rhodecode/templates/admin/auth/plugin_settings.mako @@ -50,6 +50,13 @@ ${h.secure_form(request.resource_path(resource, route_name='auth_home'), request=request)}
+ ## Allow derived templates to add something above the form + ## input fields + %if hasattr(next, 'above_form_fields'): + ${next.above_form_fields()} + %endif + +

${_('Plugin Configuration')}

%for node in plugin.get_settings_schema(): <% label_to_type = {'label-checkbox': 'bool', 'label-textarea': 'textarea'} diff --git a/rhodecode/templates/admin/repos/repo_edit_advanced.mako b/rhodecode/templates/admin/repos/repo_edit_advanced.mako index cbf9d37b..ffdc45fe 100644 --- a/rhodecode/templates/admin/repos/repo_edit_advanced.mako +++ b/rhodecode/templates/admin/repos/repo_edit_advanced.mako @@ -215,18 +215,35 @@ %endif + <% attached_prs = len(c.rhodecode_db_repo.pull_requests_source + c.rhodecode_db_repo.pull_requests_target) %> % if c.rhodecode_db_repo.pull_requests_source or c.rhodecode_db_repo.pull_requests_target: ${_ungettext('This repository has %s attached pull request.', 'This repository has %s attached pull requests.', attached_prs) % attached_prs}
- ${_('Consider to archive this repository instead.')} +
+ ${_('Consider to archive this repository instead.')} % endif + + <% attached_artifacts = len(c.rhodecode_db_repo.artifacts) %> + % if attached_artifacts: + + + ${_ungettext('This repository has %s attached artifact.', 'This repository has %s attached artifacts.', attached_artifacts) % attached_artifacts} +
+
+ ${_('Consider to archive this repository instead.')} + + + + + % endif +
diff --git a/rhodecode/templates/admin/security/security.mako b/rhodecode/templates/admin/security/security.mako new file mode 100644 index 00000000..0ae98225 --- /dev/null +++ b/rhodecode/templates/admin/security/security.mako @@ -0,0 +1,50 @@ +<%inherit file="/base/base.mako"/> + +<%def name="title()"> + ${_('Security Admin')} + %if c.rhodecode_name: + · ${h.branding(c.rhodecode_name)} + %endif + + +<%def name="breadcrumbs_links()"> + +<%def name="menu_bar_nav()"> + ${self.menu_items(active='admin')} + + +<%def name="menu_bar_subnav()"> + ${self.admin_menu(active='security')} + + +<%def name="main()"> + +
+ + + + + + + + + + + + + +
+
+

${_('Allowed client versions')}

+
+
+

${_('This feature is available in RhodeCode EE edition only. Contact {sales_email} to obtain a trial license.').format(sales_email='sales@rhodecode.com')|n}

+

+ ${_('Some outdated client versions may have security vulnerabilities. This section have rules for whitelisting versions of clients for Git, Mercurial and SVN.')} +

+
+ + +
+ + diff --git a/rhodecode/templates/base/base.mako b/rhodecode/templates/base/base.mako index 51a7a0c7..7d0191cc 100644 --- a/rhodecode/templates/base/base.mako +++ b/rhodecode/templates/base/base.mako @@ -114,6 +114,7 @@
  • ${_('Repository groups')}
  • ${_('Users')}
  • ${_('User groups')}
    • +
  • ${_('Security')}
  • ${_('Artifacts')}
  • ${_('Automation')}
  • ${_('Scheduler')}
    • diff --git a/rhodecode/templates/base/vcs_settings.mako b/rhodecode/templates/base/vcs_settings.mako index dba915e8..33ca7e91 100644 --- a/rhodecode/templates/base/vcs_settings.mako +++ b/rhodecode/templates/base/vcs_settings.mako @@ -5,22 +5,7 @@ <%def name="vcs_settings_fields(suffix='', svn_branch_patterns=None, svn_tag_patterns=None, repo_type=None, display_globals=False, **kwargs)"> % if display_globals: -
    -
    -

    ${_('General')}

    -
    -
    -
    -
    - ${h.checkbox('web_push_ssl' + suffix, 'True')} - -
    -
    - ${_('Activate to set RhodeCode to require SSL for pushing or pulling. If SSL certificate is missing it will return a HTTP Error 406: Not Acceptable.')} -
    -
    -
    -
    + % endif % if display_globals or repo_type in ['git', 'hg']: @@ -75,17 +60,6 @@ % endif
    - % if display_globals: -
    -
    - ${h.text('largefiles_usercache' + suffix, size=59)} -
    -
    -
    - ${_('Filesystem location where Mercurial largefile objects should be stored.')} -
    - % endif -
    ${h.checkbox('phases_publish' + suffix, 'True', **kwargs)} @@ -127,17 +101,6 @@ ${_('Enable lfs extensions for this repository.')} % endif
    - - % if display_globals: -
    -
    - ${h.text('vcs_git_lfs_store_location' + suffix, size=59)} -
    -
    -
    - ${_('Filesystem location where Git lfs objects should be stored.')} -
    - % endif
    % endif diff --git a/rhodecode/tests/config/test_sanitize_settings.py b/rhodecode/tests/config/test_sanitize_settings.py index 3a9f338f..4161904d 100644 --- a/rhodecode/tests/config/test_sanitize_settings.py +++ b/rhodecode/tests/config/test_sanitize_settings.py @@ -117,7 +117,7 @@ class TestSanitizeVcsSettings(object): _string_funcs = [ ('vcs.svn.compatible_version', ''), - ('vcs.hooks.protocol', 'http'), + ('vcs.hooks.protocol.v2', 'celery'), ('vcs.hooks.host', '*'), ('vcs.scm_app_implementation', 'http'), ('vcs.server', ''), diff --git a/rhodecode/tests/fixture.py b/rhodecode/tests/fixture.py index a3def7a7..d7ca57bd 100644 --- a/rhodecode/tests/fixture.py +++ b/rhodecode/tests/fixture.py @@ -305,7 +305,7 @@ class Fixture(object): return r def destroy_repo(self, repo_name, **kwargs): - RepoModel().delete(repo_name, pull_requests='delete', **kwargs) + RepoModel().delete(repo_name, pull_requests='delete', artifacts='delete', **kwargs) Session().commit() def destroy_repo_on_filesystem(self, repo_name): diff --git a/rhodecode/tests/fixture_mods/fixture_pyramid.py b/rhodecode/tests/fixture_mods/fixture_pyramid.py index 415dfc9c..e90d7d5b 100644 --- a/rhodecode/tests/fixture_mods/fixture_pyramid.py +++ b/rhodecode/tests/fixture_mods/fixture_pyramid.py @@ -110,7 +110,7 @@ def ini_config(request, tmpdir_factory, rcserver_port, vcsserver_port): 'vcs.server.protocol': 'http', 'vcs.scm_app_implementation': 'http', 'vcs.svn.proxy.enabled': 'true', - 'vcs.hooks.protocol': 'http', + 'vcs.hooks.protocol.v2': 'celery', 'vcs.hooks.host': '*', 'repo_store.path': TESTS_TMP_PATH, 'app.service_api.token': 'service_secret_token', diff --git a/rhodecode/tests/lib/middleware/test_simplehg.py b/rhodecode/tests/lib/middleware/test_simplehg.py index 02534b8e..69bbf833 100644 --- a/rhodecode/tests/lib/middleware/test_simplehg.py +++ b/rhodecode/tests/lib/middleware/test_simplehg.py @@ -120,7 +120,6 @@ def test_get_config(user_util, baseapp, request_stub): expected_config = [ ('vcs_svn_tag', 'ff89f8c714d135d865f44b90e5413b88de19a55f', '/tags/*'), - ('web', 'push_ssl', 'False'), ('web', 'allow_push', '*'), ('web', 'allow_archive', 'gz zip bz2'), ('web', 'baseurl', '/'), diff --git a/rhodecode/tests/lib/middleware/test_simplevcs.py b/rhodecode/tests/lib/middleware/test_simplevcs.py index 0cd06196..c8129d24 100644 --- a/rhodecode/tests/lib/middleware/test_simplevcs.py +++ b/rhodecode/tests/lib/middleware/test_simplevcs.py @@ -239,7 +239,6 @@ class TestShadowRepoExposure(object): """ controller = StubVCSController( baseapp.config.get_settings(), request_stub.registry) - controller._check_ssl = mock.Mock() controller.is_shadow_repo = True controller._action = 'pull' controller._is_shadow_repo_dir = True @@ -267,7 +266,6 @@ class TestShadowRepoExposure(object): """ controller = StubVCSController( baseapp.config.get_settings(), request_stub.registry) - controller._check_ssl = mock.Mock() controller.is_shadow_repo = True controller._action = 'pull' controller._is_shadow_repo_dir = False @@ -291,7 +289,6 @@ class TestShadowRepoExposure(object): """ controller = StubVCSController( baseapp.config.get_settings(), request_stub.registry) - controller._check_ssl = mock.Mock() controller.is_shadow_repo = True controller._action = 'push' controller.stub_response_body = (b'dummy body value',) @@ -399,7 +396,7 @@ class TestGenerateVcsResponse(object): def call_controller_with_response_body(self, response_body): settings = { 'base_path': 'fake_base_path', - 'vcs.hooks.protocol': 'http', + 'vcs.hooks.protocol.v2': 'celery', 'vcs.hooks.direct_calls': False, } registry = AttributeDict() diff --git a/rhodecode/tests/lib/test_utils.py b/rhodecode/tests/lib/test_utils.py index ecfcbdf7..41a48c40 100644 --- a/rhodecode/tests/lib/test_utils.py +++ b/rhodecode/tests/lib/test_utils.py @@ -371,7 +371,7 @@ class TestMakeDbConfig(object): ('section2', 'option2', 'value2'), ('section3', 'option3', 'value3'), ] - with mock.patch.object(utils, 'config_data_from_db') as config_mock: + with mock.patch.object(utils, 'prepare_config_data') as config_mock: config_mock.return_value = test_data kwargs = {'clear_session': False, 'repo': 'test_repo'} result = utils.make_db_config(**kwargs) @@ -381,8 +381,8 @@ class TestMakeDbConfig(object): assert value == expected_value -class TestConfigDataFromDb(object): - def test_config_data_from_db_returns_active_settings(self): +class TestPrepareConfigData(object): + def test_prepare_config_data_returns_active_settings(self): test_data = [ UiSetting('section1', 'option1', 'value1', True), UiSetting('section2', 'option2', 'value2', True), @@ -398,7 +398,7 @@ class TestConfigDataFromDb(object): instance_mock = mock.Mock() model_mock.return_value = instance_mock instance_mock.get_ui_settings.return_value = test_data - result = utils.config_data_from_db( + result = utils.prepare_config_data( clear_session=False, repo=repo_name) self._assert_repo_name_passed(model_mock, repo_name) @@ -407,7 +407,8 @@ class TestConfigDataFromDb(object): ('section1', 'option1', 'value1'), ('section2', 'option2', 'value2'), ] - assert result == expected_result + # We have extra config items returned, so we're ignoring two last items + assert result[:2] == expected_result def _assert_repo_name_passed(self, model_mock, repo_name): assert model_mock.call_count == 1 diff --git a/rhodecode/tests/models/settings/test_vcs_settings.py b/rhodecode/tests/models/settings/test_vcs_settings.py index e8dcbcc9..9ca4653a 100644 --- a/rhodecode/tests/models/settings/test_vcs_settings.py +++ b/rhodecode/tests/models/settings/test_vcs_settings.py @@ -578,21 +578,9 @@ class TestCreateOrUpdateRepoHgSettings(object): assert str(exc_info.value) == 'Repository is not specified' -class TestUpdateGlobalSslSetting(object): - def test_updates_global_hg_settings(self): - model = VcsSettingsModel() - with mock.patch.object(model, '_create_or_update_ui') as create_mock: - model.update_global_ssl_setting('False') - Session().commit() - - create_mock.assert_called_once_with( - model.global_settings, 'web', 'push_ssl', value='False') - - class TestCreateOrUpdateGlobalHgSettings(object): FORM_DATA = { 'extensions_largefiles': False, - 'largefiles_usercache': '/example/largefiles-store', 'phases_publish': False, 'extensions_evolve': False } @@ -605,7 +593,6 @@ class TestCreateOrUpdateGlobalHgSettings(object): expected_calls = [ mock.call(model.global_settings, 'extensions', 'largefiles', active=False, value=''), - mock.call(model.global_settings, 'largefiles', 'usercache', value='/example/largefiles-store'), mock.call(model.global_settings, 'phases', 'publish', value='False'), mock.call(model.global_settings, 'extensions', 'evolve', active=False, value=''), mock.call(model.global_settings, 'experimental', 'evolution', active=False, value=''), @@ -632,7 +619,6 @@ class TestCreateOrUpdateGlobalHgSettings(object): class TestCreateOrUpdateGlobalGitSettings(object): FORM_DATA = { 'vcs_git_lfs_enabled': False, - 'vcs_git_lfs_store_location': '/example/lfs-store', } def test_creates_repo_hg_settings_when_data_is_correct(self): @@ -643,7 +629,6 @@ class TestCreateOrUpdateGlobalGitSettings(object): expected_calls = [ mock.call(model.global_settings, 'vcs_git_lfs', 'enabled', active=False, value=False), - mock.call(model.global_settings, 'vcs_git_lfs', 'store_location', value='/example/lfs-store'), ] assert expected_calls == create_mock.call_args_list @@ -1001,9 +986,7 @@ class TestCreateOrUpdateRepoSettings(object): 'hooks_outgoing_pull_logger': False, 'extensions_largefiles': False, 'extensions_evolve': False, - 'largefiles_usercache': '/example/largefiles-store', 'vcs_git_lfs_enabled': False, - 'vcs_git_lfs_store_location': '/', 'phases_publish': 'False', 'rhodecode_pr_merge_enabled': False, 'rhodecode_use_outdated_comments': False, diff --git a/rhodecode/tests/models/test_pullrequest.py b/rhodecode/tests/models/test_pullrequest.py index 5a6529cc..f1d07c3c 100644 --- a/rhodecode/tests/models/test_pullrequest.py +++ b/rhodecode/tests/models/test_pullrequest.py @@ -449,7 +449,7 @@ class TestPullRequestModel(object): @pytest.mark.usefixtures('config_stub') class TestIntegrationMerge(object): @pytest.mark.parametrize('extra_config', ( - {'vcs.hooks.protocol': 'http', 'vcs.hooks.direct_calls': False}, + {'vcs.hooks.protocol.v2': 'celery', 'vcs.hooks.direct_calls': False}, )) def test_merge_triggers_push_hooks( self, pr_util, user_admin, capture_rcextensions, merge_extras, diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini index 1b534b63..be936bb8 100644 --- a/rhodecode/tests/rhodecode.ini +++ b/rhodecode/tests/rhodecode.ini @@ -36,7 +36,7 @@ port = 10020 ; GUNICORN APPLICATION SERVER ; ########################### -; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py +; run with gunicorn --config gunicorn_conf.py --paste rhodecode.ini ; Module to use, this setting shouldn't be changed use = egg:gunicorn#main @@ -249,15 +249,56 @@ labs_settings_active = true ; optional prefix to Add to email Subject #exception_tracker.email_prefix = [RHODECODE ERROR] -; File store configuration. This is used to store and serve uploaded files -file_store.enabled = true +; NOTE: this setting IS DEPRECATED: +; file_store backend is always enabled +#file_store.enabled = true +; NOTE: this setting IS DEPRECATED: +; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead ; Storage backend, available options are: local -file_store.backend = local +#file_store.backend = local +; NOTE: this setting IS DEPRECATED: +; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead ; path to store the uploaded binaries and artifacts -file_store.storage_path = /var/opt/rhodecode_data/file_store +#file_store.storage_path = /var/opt/rhodecode_data/file_store +; Artifacts file-store, is used to store comment attachments and artifacts uploads. +; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options +; filesystem_v1 is backwards compat with pre 5.1 storage changes +; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from +; previous installations to keep the artifacts without a need of migration +file_store.backend.type = filesystem_v1 + +; filesystem options... +file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/test_artifacts_file_store + +; filesystem_v2 options... +file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/test_artifacts_file_store_2 +file_store.filesystem_v2.shards = 8 + +; objectstore options... +; url for s3 compatible storage that allows to upload artifacts +; e.g http://minio:9000 +#file_store.backend.type = objectstore +file_store.objectstore.url = http://s3-minio:9000 + +; a top-level bucket to put all other shards in +; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number +file_store.objectstore.bucket = rhodecode-file-store-tests + +; number of sharded buckets to create to distribute archives across +; default is 8 shards +file_store.objectstore.bucket_shards = 8 + +; key for s3 auth +file_store.objectstore.key = s3admin + +; secret for s3 auth +file_store.objectstore.secret = s3secret4 + +;region for s3 storage +file_store.objectstore.region = eu-central-1 ; Redis url to acquire/check generation of archives locks archive_cache.locking.url = redis://redis:6379/1 @@ -593,6 +634,7 @@ vcs.scm_app_implementation = http ; Push/Pull operations hooks protocol, available options are: ; `http` - use http-rpc backend (default) ; `celery` - use celery based hooks +#DEPRECATED:vcs.hooks.protocol = http vcs.hooks.protocol = http ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be @@ -626,6 +668,10 @@ vcs.methods.cache = false ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible #vcs.svn.compatible_version = 1.8 +; Redis connection settings for svn integrations logic +; This connection string needs to be the same on ce and vcsserver +vcs.svn.redis_conn = redis://redis:6379/0 + ; Enable SVN proxy of requests over HTTP vcs.svn.proxy.enabled = true @@ -681,7 +727,8 @@ ssh.authorized_keys_file_path = %(here)s/rc-tests/authorized_keys_rhodecode ; RhodeCode installation directory. ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2 -ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper +#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper +ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2 ; Allow shell when executing the ssh-wrapper command ssh.wrapper_cmd_allow_shell = false diff --git a/setup.py b/setup.py index 21b2be8c..b0165a4e 100644 --- a/setup.py +++ b/setup.py @@ -189,6 +189,7 @@ setup( 'rc-upgrade-db=rhodecode.lib.rc_commands.upgrade_db:main', 'rc-ishell=rhodecode.lib.rc_commands.ishell:main', 'rc-add-artifact=rhodecode.lib.rc_commands.add_artifact:main', + 'rc-migrate-artifact=rhodecode.lib.rc_commands.migrate_artifact:main', 'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper_v1:main', 'rc-ssh-wrapper-v2=rhodecode.apps.ssh_support.lib.ssh_wrapper_v2:main', ],