engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: fixed merge conflicts

Browse source
This commit is contained in:
Andrii Verbytskyi 2024-10-09 16:58:32 +02:00
commit 65ade690a3
117 changed files with 3095 additions and 1253 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.bumpversion.cfg
View file

@ -1,5 +1,5 @@
[bumpversion]
current_version = 5.1.2
current_version = 5.2.0
message = release: Bump version {current_version} to {new_version}
[bumpversion:file:rhodecode/VERSION]

2
.hgignore
View file

@ -54,7 +54,7 @@ syntax: regexp
^rhodecode\.log$
^rhodecode_dev\.log$
^test\.db$
^venv/
# ac-tests
^acceptance_tests/\.cache.*$

134
Makefile
View file

@ -1,12 +1,49 @@
.DEFAULT_GOAL := help
# Pretty print values cf. https://misc.flogisoft.com/bash/tip_colors_and_formatting
RESET := \033[0m # Reset all formatting
GREEN := \033[0;32m # Resets before setting 16b colour (32 -- green)
YELLOW := \033[0;33m
ORANGE := \033[0;38;5;208m # Reset then set 256b colour (208 -- orange)
PEACH := \033[0;38;5;216m
## ---------------------------------------------------------------------------------- ##
## ------------------------- Help usage builder ------------------------------------- ##
## ---------------------------------------------------------------------------------- ##
# use '# >>> Build commands' to create section
# use '# target: target description' to create help for target
.PHONY: help
help:
@echo "Usage:"
@cat $(MAKEFILE_LIST) | grep -E '^# >>>|^# [A-Za-z0-9_.-]+:' | sed -E 's/^# //' | awk ' \
BEGIN { \
green="\033[32m"; \
yellow="\033[33m"; \
reset="\033[0m"; \
section=""; \
} \
/^>>>/ { \
section=substr($$0, 5); \
printf "\n" green ">>> %s" reset "\n", section; \
next; \
} \
/^([A-Za-z0-9_.-]+):/ { \
target=$$1; \
gsub(/:$$/, "", target); \
description=substr($$0, index($$0, ":") + 2); \
if (description == "") { description="-"; } \
printf " - " yellow "%-35s" reset " %s\n", target, description; \
} \
'
# required for pushd to work..
SHELL = /bin/bash
# set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
# >>> Tests commands
.PHONY: clean
## Cleanup compiled and cache py files
# clean: Cleanup compiled and cache py files
clean:
make test-clean
find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
@ -14,14 +51,14 @@ clean:
.PHONY: test
## run test-clean and tests
# test: run test-clean and tests
test:
make test-clean
make test-only
unset RC_SQLALCHEMY_DB1_URL && unset RC_DB_URL && make test-only
.PHONY: test-clean
## run test-clean and tests
# test-clean: run test-clean and tests
test-clean:
rm -rf coverage.xml htmlcov junit.xml pylint.log result
find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
@ -29,34 +66,36 @@ test-clean:
.PHONY: test-only
## Run tests only without cleanup
# test-only: Run tests only without cleanup
test-only:
PYTHONHASHSEED=random \
py.test -x -vv -r xw -p no:sugar \
--cov-report=term-missing --cov-report=html \
--cov=rhodecode rhodecode
# >>> Docs commands
.PHONY: docs
## build docs
# docs: build docs
docs:
(cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make clean html SPHINXOPTS="-W")
.PHONY: docs-clean
## Cleanup docs
# docs-clean: Cleanup docs
docs-clean:
(cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make clean)
.PHONY: docs-cleanup
## Cleanup docs
# docs-cleanup: Cleanup docs
docs-cleanup:
(cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make cleanup)
# >>> Dev commands
.PHONY: web-build
## Build JS packages static/js
# web-build: Build JS packages static/js
web-build:
rm -rf node_modules
docker run -it --rm -v $(PWD):/project --workdir=/project rhodecode/static-files-build:16 -c "npm install && /project/node_modules/.bin/grunt"
@ -64,25 +103,9 @@ web-build:
./rhodecode/tests/scripts/static-file-check.sh rhodecode/public/
rm -rf node_modules
.PHONY: ruff-check
## run a ruff analysis
ruff-check:
ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs --exclude .dev .
.PHONY: pip-packages
## Show outdated packages
pip-packages:
python ${OUTDATED_PACKAGES}
.PHONY: build
## Build sdist/egg
build:
python -m build
.PHONY: dev-sh
## make dev-sh
# dev-sh: make dev-sh
dev-sh:
sudo echo "deb [trusted=yes] https://apt.fury.io/rsteube/ /" | sudo tee -a "/etc/apt/sources.list.d/fury.list"
sudo apt-get update
@ -95,14 +118,14 @@ dev-sh:
.PHONY: dev-cleanup
## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
# dev-cleanup: Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
dev-cleanup:
pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
rm -rf /tmp/*
.PHONY: dev-env
## make dev-env based on the requirements files and install develop of packages
# dev-env: make dev-env based on the requirements files and install develop of packages
## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
dev-env:
sudo -u root chown rhodecode:rhodecode /home/rhodecode/.cache/pip/
@ -114,7 +137,7 @@ dev-env:
.PHONY: sh
## shortcut for make dev-sh dev-env
# sh: shortcut for make dev-sh dev-env
sh:
make dev-env
make dev-sh
@ -124,49 +147,12 @@ sh:
workers?=1
.PHONY: dev-srv
## run gunicorn web server with reloader, use workers=N to set multiworker mode
# dev-srv: run gunicorn web server with reloader, use workers=N to set multiworker mode, workers=N allows changes of workers
dev-srv:
gunicorn --paste=.dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py --timeout=120 --reload --workers=$(workers)
.PHONY: ruff-check
# ruff-check: run a ruff analysis
ruff-check:
ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs --exclude .dev .
# Default command on calling make
.DEFAULT_GOAL := show-help
.PHONY: show-help
show-help:
@echo "$$(tput bold)Available rules:$$(tput sgr0)"
@echo
@sed -n -e "/^## / { \
h; \
s/.*//; \
:doc" \
-e "H; \
n; \
s/^## //; \
t doc" \
-e "s/:.*//; \
G; \
s/\\n## /---/; \
s/\\n/ /g; \
p; \
}" ${MAKEFILE_LIST} \
| LC_ALL='C' sort --ignore-case \
| awk -F '---' \
-v ncol=$$(tput cols) \
-v indent=19 \
-v col_on="$$(tput setaf 6)" \
-v col_off="$$(tput sgr0)" \
'{ \
printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
n = split($$2, words, " "); \
line_length = ncol - indent; \
for (i = 1; i <= n; i++) { \
line_length -= length(words[i]) + 1; \
if (line_length <= 0) { \
line_length = ncol - indent - length(words[i]) - 1; \
printf "\n%*s ", -indent, " "; \
} \
printf "%s ", words[i]; \
} \
printf "\n"; \
}'

68
configs/development.ini
View file

@ -257,6 +257,13 @@ license_token =
; This flag hides sensitive information on the license page such as token, and license data
license.hide_license_info = false
; Import EE license from this license path
#license.import_path = %(here)s/rhodecode_enterprise.license
; import license 'if-missing' or 'force' (always override)
; if-missing means apply license if it doesn't exist. 'force' option always overrides it
license.import_path_mode = if-missing
; supervisor connection uri, for managing supervisor and logs.
supervisor.uri =
@ -281,15 +288,56 @@ labs_settings_active = true
; optional prefix to Add to email Subject
#exception_tracker.email_prefix = [RHODECODE ERROR]
; File store configuration. This is used to store and serve uploaded files
file_store.enabled = true
; NOTE: this setting IS DEPRECATED:
; file_store backend is always enabled
#file_store.enabled = true
; NOTE: this setting IS DEPRECATED:
; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
; Storage backend, available options are: local
file_store.backend = local
#file_store.backend = local
; NOTE: this setting IS DEPRECATED:
; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
; path to store the uploaded binaries and artifacts
file_store.storage_path = /var/opt/rhodecode_data/file_store
#file_store.storage_path = /var/opt/rhodecode_data/file_store
; Artifacts file-store, is used to store comment attachments and artifacts uploads.
; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
; filesystem_v1 is backwards compat with pre 5.1 storage changes
; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
; previous installations to keep the artifacts without a need of migration
#file_store.backend.type = filesystem_v2
; filesystem options...
#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store
; filesystem_v2 options...
#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store
#file_store.filesystem_v2.shards = 8
; objectstore options...
; url for s3 compatible storage that allows to upload artifacts
; e.g http://minio:9000
#file_store.backend.type = objectstore
#file_store.objectstore.url = http://s3-minio:9000
; a top-level bucket to put all other shards in
; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
#file_store.objectstore.bucket = rhodecode-file-store
; number of sharded buckets to create to distribute archives across
; default is 8 shards
#file_store.objectstore.bucket_shards = 8
; key for s3 auth
#file_store.objectstore.key = s3admin
; secret for s3 auth
#file_store.objectstore.secret = s3secret4
;region for s3 storage
#file_store.objectstore.region = eu-central-1
; Redis url to acquire/check generation of archives locks
archive_cache.locking.url = redis://redis:6379/1
@ -624,7 +672,8 @@ vcs.scm_app_implementation = http
; Push/Pull operations hooks protocol, available options are:
; `http` - use http-rpc backend (default)
; `celery` - use celery based hooks
vcs.hooks.protocol = http
#DEPRECATED:vcs.hooks.protocol = http
vcs.hooks.protocol.v2 = celery
; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
; accessible via network.
@ -647,6 +696,12 @@ vcs.connection_timeout = 3600
; It uses cache_region `cache_repo`
vcs.methods.cache = true
; Filesystem location where Git lfs objects should be stored
vcs.git.lfs.storage_location = /var/opt/rhodecode_repo_store/.cache/git_lfs_store
; Filesystem location where Mercurial largefile objects should be stored
vcs.hg.largefiles.storage_location = /var/opt/rhodecode_repo_store/.cache/hg_largefiles_store
; ####################################################
; Subversion proxy support (mod_dav_svn)
; Maps RhodeCode repo groups into SVN paths for Apache
@ -716,7 +771,8 @@ ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecod
; RhodeCode installation directory.
; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
; Allow shell when executing the ssh-wrapper command
ssh.wrapper_cmd_allow_shell = false

31
configs/gunicorn_config.py
View file

@ -13,6 +13,7 @@ import traceback
import random
import socket
import dataclasses
import json
from gunicorn.glogging import Logger
@ -37,17 +38,41 @@ accesslog = '-'
worker_tmp_dir = None
tmp_upload_dir = None
# use re-use port logic
#reuse_port = True
# use re-use port logic to let linux internals load-balance the requests better.
reuse_port = True
# Custom log format
#access_log_format = (
# '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
# loki format for easier parsing in grafana
access_log_format = (
loki_access_log_format = (
'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
# JSON format
json_access_log_format = json.dumps({
'time': r'%(t)s',
'pid': r'%(p)s',
'level': 'INFO',
'ip': r'%(h)s',
'request_time': r'%(L)s',
'remote_address': r'%(h)s',
'user_name': r'%(u)s',
'status': r'%(s)s',
'method': r'%(m)s',
'url_path': r'%(U)s',
'query_string': r'%(q)s',
'protocol': r'%(H)s',
'response_length': r'%(B)s',
'referer': r'%(f)s',
'user_agent': r'%(a)s',
})
access_log_format = loki_access_log_format
if os.environ.get('RC_LOGGING_FORMATTER') == 'json':
access_log_format = json_access_log_format
# self adjust workers based on CPU count, to use maximum of CPU and not overquota the resources
# workers = get_workers()

1
configs/init.d/.readme.txt
View file

@ -1 +0,0 @@
Example init scripts.

61
configs/init.d/supervisord.conf
View file

@ -1,61 +0,0 @@
; Sample supervisor RhodeCode config file.
;
; For more information on the config file, please see:
; http://supervisord.org/configuration.html
;
; Note: shell expansion ("~" or "$HOME") is not supported. Environment
; variables can be expanded using this syntax: "%(ENV_HOME)s".
[unix_http_server]
file=/tmp/supervisor.sock ; (the path to the socket file)
;chmod=0700 ; socket file mode (default 0700)
;chown=nobody:nogroup ; socket file uid:gid owner
;username=user ; (default is no username (open server))
;password=123 ; (default is no password (open server))
[supervisord]
logfile=/home/ubuntu/rhodecode/supervisord.log ; (main log file;default $CWD/supervisord.log)
logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB)
logfile_backups=10 ; (num of main logfile rotation backups;default 10)
loglevel=info ; (log level;default info; others: debug,warn,trace)
pidfile=/home/ubuntu/rhodecode/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
nodaemon=true ; (start in foreground if true;default false)
minfds=1024 ; (min. avail startup file descriptors;default 1024)
minprocs=200 ; (min. avail process descriptors;default 200)
;umask=022 ; (process file creation umask;default 022)
user=ubuntu ; (default is current user, required if root)
;identifier=supervisor ; (supervisord identifier, default is 'supervisor')
;directory=/tmp ; (default is not to cd during start)
;nocleanup=true ; (don't clean up tempfiles at start;default false)
;childlogdir=/tmp ; ('AUTO' child log dir, default $TEMP)
environment=HOME=/home/ubuntu,LANG=en_US.UTF-8 ; (key value pairs to add to environment)
;strip_ansi=false ; (strip ansi escape codes in logs; def. false)
; the below section must remain in the config file for RPC
; (supervisorctl/web interface) to work, additional interfaces may be
; added by defining them in separate rpcinterface: sections
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket
;username=chris ; should be same as http_username if set
;password=123 ; should be same as http_password if set
; restart with supervisorctl restart rhodecode:*
[program:rhodecode]
numprocs = 1
numprocs_start = 5000
directory=/home/ubuntu/rhodecode/source
command = /home/ubuntu/rhodecode/venv/bin/paster serve /home/ubuntu/rhodecode/source/prod.ini
process_name = %(program_name)s_%(process_num)04d
redirect_stderr=true
stdout_logfile=/home/ubuntu/rhodecode/rhodecode.log
[program:rhodecode_workers]
numproces = 1
directory = /home/ubuntu/rhodecode/source
command = /home/ubuntu/rhodecode/venv/bin/paster celeryd /home/ubuntu/rhodecode/source/prod.ini --autoscale=10,2
redirect_stderr=true
stdout_logfile=/%(here)s/rhodecode_workers.log

68
configs/production.ini
View file

@ -225,6 +225,13 @@ license_token =
; This flag hides sensitive information on the license page such as token, and license data
license.hide_license_info = false
; Import EE license from this license path
#license.import_path = %(here)s/rhodecode_enterprise.license
; import license 'if-missing' or 'force' (always override)
; if-missing means apply license if it doesn't exist. 'force' option always overrides it
license.import_path_mode = if-missing
; supervisor connection uri, for managing supervisor and logs.
supervisor.uri =
@ -249,15 +256,56 @@ labs_settings_active = true
; optional prefix to Add to email Subject
#exception_tracker.email_prefix = [RHODECODE ERROR]
; File store configuration. This is used to store and serve uploaded files
file_store.enabled = true
; NOTE: this setting IS DEPRECATED:
; file_store backend is always enabled
#file_store.enabled = true
; NOTE: this setting IS DEPRECATED:
; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
; Storage backend, available options are: local
file_store.backend = local
#file_store.backend = local
; NOTE: this setting IS DEPRECATED:
; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
; path to store the uploaded binaries and artifacts
file_store.storage_path = /var/opt/rhodecode_data/file_store
#file_store.storage_path = /var/opt/rhodecode_data/file_store
; Artifacts file-store, is used to store comment attachments and artifacts uploads.
; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
; filesystem_v1 is backwards compat with pre 5.1 storage changes
; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
; previous installations to keep the artifacts without a need of migration
#file_store.backend.type = filesystem_v2
; filesystem options...
#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store
; filesystem_v2 options...
#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store
#file_store.filesystem_v2.shards = 8
; objectstore options...
; url for s3 compatible storage that allows to upload artifacts
; e.g http://minio:9000
#file_store.backend.type = objectstore
#file_store.objectstore.url = http://s3-minio:9000
; a top-level bucket to put all other shards in
; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
#file_store.objectstore.bucket = rhodecode-file-store
; number of sharded buckets to create to distribute archives across
; default is 8 shards
#file_store.objectstore.bucket_shards = 8
; key for s3 auth
#file_store.objectstore.key = s3admin
; secret for s3 auth
#file_store.objectstore.secret = s3secret4
;region for s3 storage
#file_store.objectstore.region = eu-central-1
; Redis url to acquire/check generation of archives locks
archive_cache.locking.url = redis://redis:6379/1
@ -592,7 +640,8 @@ vcs.scm_app_implementation = http
; Push/Pull operations hooks protocol, available options are:
; `http` - use http-rpc backend (default)
; `celery` - use celery based hooks
vcs.hooks.protocol = http
#DEPRECATED:vcs.hooks.protocol = http
vcs.hooks.protocol.v2 = celery
; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
; accessible via network.
@ -615,6 +664,12 @@ vcs.connection_timeout = 3600
; It uses cache_region `cache_repo`
vcs.methods.cache = true
; Filesystem location where Git lfs objects should be stored
vcs.git.lfs.storage_location = /var/opt/rhodecode_repo_store/.cache/git_lfs_store
; Filesystem location where Mercurial largefile objects should be stored
vcs.hg.largefiles.storage_location = /var/opt/rhodecode_repo_store/.cache/hg_largefiles_store
; ####################################################
; Subversion proxy support (mod_dav_svn)
; Maps RhodeCode repo groups into SVN paths for Apache
@ -684,7 +739,8 @@ ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecod
; RhodeCode installation directory.
; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
; Allow shell when executing the ssh-wrapper command
ssh.wrapper_cmd_allow_shell = false

6
docs/Dockerfile
View file

@ -22,6 +22,12 @@ RUN apt-get update \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
unzip awscliv2.zip && \
./aws/install && \
rm -rf ./aws && \
rm awscliv2.zip
RUN \
python3 -m pip install --no-cache-dir --upgrade pip && \
python3 -m pip install --no-cache-dir Sphinx Pillow

4
docs/admin/system-overview.rst
View file

@ -147,10 +147,6 @@ Peer-to-peer Failover Support
* Yes
Additional Binaries
-------------------
* Yes, see :ref:`rhodecode-nix-ref` for full details.
Remote Connectivity
-------------------

161
docs/auth/auth-saml-azure.rst Normal file
View file

@ -0,0 +1,161 @@
.. _config-saml-azure-ref:
SAML 2.0 with Azure Entra ID
----------------------------
**This plugin is available only in EE Edition.**
|RCE| supports SAML 2.0 Authentication with Azure Entra ID provider. This allows
users to log-in to RhodeCode via SSO mechanism of external identity provider
such as Azure AD. The login can be triggered either by the external IDP, or internally
by clicking specific authentication button on the log-in page.
Configuration steps
^^^^^^^^^^^^^^^^^^^
To configure Duo Security SAML authentication, use the following steps:
1. From the |RCE| interface, select
:menuselection:`Admin --> Authentication`
2. Activate the `Azure Entra ID` plugin and select :guilabel:`Save`
3. Go to newly available menu option called `Azure Entra ID` on the left side.
4. Check the `enabled` check box in the plugin configuration section,
and fill in the required SAML information and :guilabel:`Save`, for more details,
see :ref:`config-saml-azure`
.. _config-saml-azure:
Example SAML Azure Entra ID configuration
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Example configuration for SAML 2.0 with Azure Entra ID provider
Enabled
`True`:
.. note::
Enable or disable this authentication plugin.
Auth Cache TTL
`30`:
.. note::
Amount of seconds to cache the authentication and permissions check response call for this plugin.
Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
Debug
`True`:
.. note::
Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
Auth button name
`Azure Entra ID`:
.. note::
Alternative authentication display name. E.g AzureAuth, CorporateID etc.
Entity ID
`https://sts.windows.net/APP_ID/`:
.. note::
Identity Provider entity/metadata URI. Known as "Microsoft Entra Identifier"
E.g. https://sts.windows.net/abcd-c655-dcee-aab7-abcd/
SSO URL
`https://login.microsoftonline.com/APP_ID/saml2`:
.. note::
SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login, Known also as Login URL
E.g. https://login.microsoftonline.com/abcd-c655-dcee-aab7-abcd/saml2
SLO URL
`https://login.microsoftonline.com/APP_ID/saml2`:
.. note::
SLO (SingleLogout) endpoint URL of the IdP. , Known also as Logout URL
E.g. https://login.microsoftonline.com/abcd-c655-dcee-aab7-abcd/saml2
x509cert
`<CERTIFICATE_STRING>`:
.. note::
Identity provider public x509 certificate. It will be converted to single-line format without headers.
Download the raw base64 encoded certificate from the Identity provider and paste it here.
SAML Signature
`sha-256`:
.. note::
Type of Algorithm to use for verification of SAML signature on Identity provider side.
SAML Digest
`sha-256`:
.. note::
Type of Algorithm to use for verification of SAML digest on Identity provider side.
Service Provider Cert Dir
`/etc/rhodecode/conf/saml_ssl/`:
.. note::
Optional directory to store service provider certificate and private keys.
Expected certs for the SP should be stored in this folder as:
* sp.key Private Key
* sp.crt Public cert
* sp_new.crt Future Public cert
Also you can use other cert to sign the metadata of the SP using the:
* metadata.key
* metadata.crt
Expected NameID Format
`nameid-format:emailAddress`:
.. note::
The format that specifies how the NameID is sent to the service provider.
User ID Attribute
`user.email`:
.. note::
User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
Ensure this is returned from DuoSecurity for example via duo_username.
Username Attribute
`user.username`:
.. note::
Username Attribute name. This defines which attribute in SAML response will map to a username.
Email Attribute
`user.email`:
.. note::
Email Attribute name. This defines which attribute in SAML response will map to an email address.
Below is example setup from Azure Administration page that can be used with above config.
.. image:: ../images/saml-azure-service-provider-example.png
:alt: Azure SAML setup example
:scale: 50 %
Below is an example attribute mapping set for IDP provider required by the above config.
.. image:: ../images/saml-azure-attributes-example.png
:alt: Azure SAML setup example
:scale: 50 %

10
docs/auth/auth-saml-bulk-enroll-users.rst
View file

@ -13,7 +13,7 @@ This method simply enables SAML authentication for many users at once.
From the server RhodeCode Enterprise is running run ishell on the instance which we
want to apply the SAML migration::
rccontrol ishell enterprise-1
./rcstack cli ishell
Follow these steps to enable SAML authentication for multiple users.
@ -46,6 +46,8 @@ From available options pick only one and run the `import` statement
# for Duo Security
In [2]: from rc_auth_plugins.auth_duo_security import RhodeCodeAuthPlugin
# for Azure Entra
In [2]: from rc_auth_plugins.auth_azure import RhodeCodeAuthPlugin
# for OneLogin
In [2]: from rc_auth_plugins.auth_onelogin import RhodeCodeAuthPlugin
# generic SAML plugin
@ -62,13 +64,13 @@ Enter in the ishell prompt
...: attrs = saml2user.get(user.user_id)
...: provider = RhodeCodeAuthPlugin.uid
...: if existing_identity:
...: print('Identity for user `{}` already exists, skipping'.format(user.username))
...: print(f'Identity for user `{user.username}` already exists, skipping')
...: continue
...: if attrs:
...: external_id = attrs['id']
...: new_external_identity = ExternalIdentity()
...: new_external_identity.external_id = external_id
...: new_external_identity.external_username = '{}-saml-{}'.format(user.username, user.user_id)
...: new_external_identity.external_username = f'{user.username}-saml-{user.user_id}'
...: new_external_identity.provider_name = provider
...: new_external_identity.local_user_id = user.user_id
...: new_external_identity.access_token = ''
@ -76,7 +78,7 @@ Enter in the ishell prompt
...: new_external_identity.alt_token = ''
...: Session().add(ex_identity)
...: Session().commit()
...: print('Set user `{}` external identity bound to ExternalID:{}'.format(user.username, external_id))
...: print(f'Set user `{user.username}` external identity bound to ExternalID:{external_id}')
.. note::

140
docs/auth/auth-saml-duosecurity.rst
View file

@ -32,62 +32,118 @@ To configure Duo Security SAML authentication, use the following steps:
Example SAML Duo Security configuration
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Example configuration for SAML 2.0 with Duo Security provider::
Example configuration for SAML 2.0 with Duo Security provider
*option*: `enabled` => `True`
# Enable or disable this authentication plugin.
*option*: `cache_ttl` => `0`
# Amount of seconds to cache the authentication and permissions check response call for this plugin.
# Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
Enabled
`True`:
*option*: `debug` => `True`
# Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
.. note::
Enable or disable this authentication plugin.
*option*: `entity_id` => `http://rc-app.com/dag/saml2/idp/metadata.php`
# Identity Provider entity/metadata URI.
# E.g. https://duo-gateway.com/dag/saml2/idp/metadata.php
*option*: `sso_service_url` => `http://rc-app.com/dag/saml2/idp/SSOService.php?spentityid=http://rc.local.pl/_admin/auth/duosecurity/saml-metadata`
# SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login
# E.g. https://duo-gateway.com/dag/saml2/idp/SSOService.php?spentityid=<metadata_entity_id>
Auth Cache TTL
`30`:
*option*: `slo_service_url` => `http://rc-app.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=http://rc-app.com/dag/module.php/duosecurity/logout.php`
# SLO (SingleLogout) endpoint URL of the IdP.
# E.g. https://duo-gateway.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=http://duo-gateway.com/_admin/saml/sign-out-endpoint
.. note::
Amount of seconds to cache the authentication and permissions check response call for this plugin.
Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
*option*: `x509cert` => `<CERTIFICATE_STRING>`
# Identity provider public x509 certificate. It will be converted to single-line format without headers
Debug
`True`:
*option*: `name_id_format` => `sha-1`
# The format that specifies how the NameID is sent to the service provider.
.. note::
Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
*option*: `signature_algo` => `sha-256`
# Type of Algorithm to use for verification of SAML signature on Identity provider side
*option*: `digest_algo` => `sha-256`
# Type of Algorithm to use for verification of SAML digest on Identity provider side
Auth button name
`Azure Entra ID`:
*option*: `cert_dir` => `/etc/saml/`
# Optional directory to store service provider certificate and private keys.
# Expected certs for the SP should be stored in this folder as:
# * sp.key Private Key
# * sp.crt Public cert
# * sp_new.crt Future Public cert
#
# Also you can use other cert to sign the metadata of the SP using the:
# * metadata.key
# * metadata.crt
.. note::
Alternative authentication display name. E.g AzureAuth, CorporateID etc.
*option*: `user_id_attribute` => `PersonImmutableID`
# User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
# Ensure this is returned from DuoSecurity for example via duo_username
*option*: `username_attribute` => `User.username`
# Username Attribute name. This defines which attribute in SAML response will map to an username.
Entity ID
`https://my-duo-gateway.com/dag/saml2/idp/metadata.php`:
.. note::
Identity Provider entity/metadata URI.
E.g. https://duo-gateway.com/dag/saml2/idp/metadata.php
SSO URL
`https://duo-gateway.com/dag/saml2/idp/SSOService.php?spentityid=<metadata_entity_id>`:
.. note::
SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login, Known also as Login URL
E.g. http://rc-app.com/dag/saml2/idp/SSOService.php?spentityid=https://docker-dev/_admin/auth/duosecurity/saml-metadata
SLO URL
`https://duo-gateway.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=<return_url>`:
.. note::
SLO (SingleLogout) endpoint URL of the IdP. , Known also as Logout URL
E.g. http://rc-app.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=https://docker-dev/_admin/auth/duosecurity/saml-sign-out-endpoint
x509cert
`<CERTIFICATE_STRING>`:
.. note::
Identity provider public x509 certificate. It will be converted to single-line format without headers.
Download the raw base64 encoded certificate from the Identity provider and paste it here.
SAML Signature
`sha-256`:
.. note::
Type of Algorithm to use for verification of SAML signature on Identity provider side.
SAML Digest
`sha-256`:
.. note::
Type of Algorithm to use for verification of SAML digest on Identity provider side.
Service Provider Cert Dir
`/etc/rhodecode/conf/saml_ssl/`:
.. note::
Optional directory to store service provider certificate and private keys.
Expected certs for the SP should be stored in this folder as:
* sp.key Private Key
* sp.crt Public cert
* sp_new.crt Future Public cert
Also you can use other cert to sign the metadata of the SP using the:
* metadata.key
* metadata.crt
Expected NameID Format
`nameid-format:emailAddress`:
.. note::
The format that specifies how the NameID is sent to the service provider.
User ID Attribute
`PersonImmutableID`:
.. note::
User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
Ensure this is returned from DuoSecurity for example via duo_username.
Username Attribute
`User.username`:
.. note::
Username Attribute name. This defines which attribute in SAML response will map to a username.
Email Attribute
`User.email`:
.. note::
Email Attribute name. This defines which attribute in SAML response will map to an email address.
*option*: `email_attribute` => `User.email`
# Email Attribute name. This defines which attribute in SAML response will map to an email address.
Below is example setup from DUO Administration page that can be used with above config.

1
docs/auth/auth-saml-generic.rst
View file

@ -15,5 +15,6 @@ Please check for reference two example providers:
auth-saml-duosecurity
auth-saml-onelogin
auth-saml-azure
auth-saml-bulk-enroll-users

139
docs/auth/auth-saml-onelogin.rst
View file

@ -32,62 +32,117 @@ To configure OneLogin SAML authentication, use the following steps:
Example SAML OneLogin configuration
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Example configuration for SAML 2.0 with OneLogin provider::
Example configuration for SAML 2.0 with OneLogin provider
*option*: `enabled` => `True`
# Enable or disable this authentication plugin.
*option*: `cache_ttl` => `0`
# Amount of seconds to cache the authentication and permissions check response call for this plugin.
# Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
Enabled
`True`:
*option*: `debug` => `True`
# Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
.. note::
Enable or disable this authentication plugin.
*option*: `entity_id` => `https://app.onelogin.com/saml/metadata/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`
# Identity Provider entity/metadata URI.
# E.g. https://app.onelogin.com/saml/metadata/<onelogin_connector_id>
*option*: `sso_service_url` => `https://customer-domain.onelogin.com/trust/saml2/http-post/sso/xxxxxx`
# SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login
# E.g. https://app.onelogin.com/trust/saml2/http-post/sso/<onelogin_connector_id>
Auth Cache TTL
`30`:
*option*: `slo_service_url` => `https://customer-domain.onelogin.com/trust/saml2/http-redirect/slo/xxxxxx`
# SLO (SingleLogout) endpoint URL of the IdP.
# E.g. https://app.onelogin.com/trust/saml2/http-redirect/slo/<onelogin_connector_id>
.. note::
Amount of seconds to cache the authentication and permissions check response call for this plugin.
Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
*option*: `x509cert` => `<CERTIFICATE_STRING>`
# Identity provider public x509 certificate. It will be converted to single-line format without headers
Debug
`True`:
*option*: `name_id_format` => `sha-1`
# The format that specifies how the NameID is sent to the service provider.
.. note::
Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
*option*: `signature_algo` => `sha-256`
# Type of Algorithm to use for verification of SAML signature on Identity provider side
*option*: `digest_algo` => `sha-256`
# Type of Algorithm to use for verification of SAML digest on Identity provider side
Auth button name
`Azure Entra ID`:
*option*: `cert_dir` => `/etc/saml/`
# Optional directory to store service provider certificate and private keys.
# Expected certs for the SP should be stored in this folder as:
# * sp.key Private Key
# * sp.crt Public cert
# * sp_new.crt Future Public cert
#
# Also you can use other cert to sign the metadata of the SP using the:
# * metadata.key
# * metadata.crt
.. note::
Alternative authentication display name. E.g AzureAuth, CorporateID etc.
*option*: `user_id_attribute` => `PersonImmutableID`
# User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
# Ensure this is returned from OneLogin for example via Internal ID
*option*: `username_attribute` => `User.username`
# Username Attribute name. This defines which attribute in SAML response will map to an username.
Entity ID
`https://app.onelogin.com/saml/metadata/<onelogin_connector_id>`:
*option*: `email_attribute` => `User.email`
# Email Attribute name. This defines which attribute in SAML response will map to an email address.
.. note::
Identity Provider entity/metadata URI.
E.g. https://app.onelogin.com/saml/metadata/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
SSO URL
`https://app.onelogin.com/trust/saml2/http-post/sso/<onelogin_connector_id>`:
.. note::
SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login, Known also as Login URL
E.g. https://app.onelogin.com/trust/saml2/http-post/sso/<onelogin_connector_id>
SLO URL
`https://app.onelogin.com/trust/saml2/http-redirect/slo/<onelogin_connector_id>`:
.. note::
SLO (SingleLogout) endpoint URL of the IdP. , Known also as Logout URL
E.g. https://app.onelogin.com/trust/saml2/http-redirect/slo/<onelogin_connector_id>
x509cert
`<CERTIFICATE_STRING>`:
.. note::
Identity provider public x509 certificate. It will be converted to single-line format without headers.
Download the raw base64 encoded certificate from the Identity provider and paste it here.
SAML Signature
`sha-256`:
.. note::
Type of Algorithm to use for verification of SAML signature on Identity provider side.
SAML Digest
`sha-256`:
.. note::
Type of Algorithm to use for verification of SAML digest on Identity provider side.
Service Provider Cert Dir
`/etc/rhodecode/conf/saml_ssl/`:
.. note::
Optional directory to store service provider certificate and private keys.
Expected certs for the SP should be stored in this folder as:
* sp.key Private Key
* sp.crt Public cert
* sp_new.crt Future Public cert
Also you can use other cert to sign the metadata of the SP using the:
* metadata.key
* metadata.crt
Expected NameID Format
`nameid-format:emailAddress`:
.. note::
The format that specifies how the NameID is sent to the service provider.
User ID Attribute
`PersonImmutableID`:
.. note::
User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
Ensure this is returned from DuoSecurity for example via duo_username.
Username Attribute
`User.username`:
.. note::
Username Attribute name. This defines which attribute in SAML response will map to a username.
Email Attribute
`User.email`:
.. note::
Email Attribute name. This defines which attribute in SAML response will map to an email address.

1
docs/auth/auth.rst
View file

@ -29,6 +29,7 @@ administrator greater control over how users authenticate with the system.
auth-saml-generic
auth-saml-onelogin
auth-saml-duosecurity
auth-saml-azure
auth-crowd
auth-pam
ssh-connection

233
docs/contributing/dev-setup.rst
View file

@ -4,237 +4,8 @@
Development setup
===================
RhodeCode Enterprise runs inside a Nix managed environment. This ensures build
environment dependencies are correctly declared and installed during setup.
It also enables atomic upgrades, rollbacks, and multiple instances of RhodeCode
Enterprise running with isolation.
To set up RhodeCode Enterprise inside the Nix environment, use the following steps:
Setup Nix Package Manager
-------------------------
To install the Nix Package Manager, please run::
$ curl https://releases.nixos.org/nix/nix-2.3.4/install | sh
or go to https://nixos.org/nix/ and follow the installation instructions.
Once this is correctly set up on your system, you should be able to use the
following commands:
* `nix-env`
* `nix-shell`
.. tip::
Update your channels frequently by running ``nix-channel --update``.
.. note::
To uninstall nix run the following:
remove the . "$HOME/.nix-profile/etc/profile.d/nix.sh" line in your ~/.profile or ~/.bash_profile
rm -rf $HOME/{.nix-channels,.nix-defexpr,.nix-profile,.config/nixpkgs}
sudo rm -rf /nix
Switch nix to the latest STABLE channel
---------------------------------------
run::
nix-channel --add https://nixos.org/channels/nixos-20.03 nixpkgs
Followed by::
nix-channel --update
nix-env -i nix-2.3.4
Install required binaries
-------------------------
We need some handy tools first.
run::
nix-env -i nix-prefetch-hg
nix-env -i nix-prefetch-git
Speed up JS build by installing PhantomJS
-----------------------------------------
PhantomJS will be downloaded each time nix-shell is invoked. To speed this by
setting already downloaded version do this::
nix-env -i phantomjs-2.1.1
# and set nix bin path
export PATH=$PATH:~/.nix-profile/bin
Clone the required repositories
-------------------------------
After Nix is set up, clone the RhodeCode Enterprise Community Edition and
RhodeCode VCSServer repositories into the same directory.
RhodeCode currently is using Mercurial Version Control System, please make sure
you have it installed before continuing.
To obtain the required sources, use the following commands::
mkdir rhodecode-develop && cd rhodecode-develop
hg clone -u default https://code.rhodecode.com/rhodecode-enterprise-ce
hg clone -u default https://code.rhodecode.com/rhodecode-vcsserver
.. note::
If you cannot clone the repository, please contact us via support@rhodecode.com
Install some required libraries
-------------------------------
There are some required drivers and dev libraries that we need to install to
test RhodeCode under different types of databases. For example in Ubuntu we
need to install the following.
required libraries::
# svn related
sudo apt-get install libapr1-dev libaprutil1-dev
sudo apt-get install libsvn-dev
# libcurl required too
sudo apt-get install libcurl4-openssl-dev
# mysql/pg server for development, optional
sudo apt-get install mysql-server libmysqlclient-dev
sudo apt-get install postgresql postgresql-contrib libpq-dev
Enter the Development Shell
---------------------------
The final step is to start the development shells. To do this, run the
following command from inside the cloned repository::
# first, the vcsserver
cd ~/rhodecode-vcsserver
nix-shell
# then enterprise sources
cd ~/rhodecode-enterprise-ce
nix-shell
.. note::
On the first run, this will take a while to download and optionally compile
a few things. The following runs will be faster. The development shell works
fine on both MacOS and Linux platforms.
Create config.nix for development
---------------------------------
In order to run proper tests and setup linking across projects, a config.nix
file needs to be setup::
# create config
mkdir -p ~/.nixpkgs
touch ~/.nixpkgs/config.nix
# put the below content into the ~/.nixpkgs/config.nix file
# adjusts, the path to where you cloned your repositories.
{
rc = {
sources = {
rhodecode-vcsserver = "/home/dev/rhodecode-vcsserver";
rhodecode-enterprise-ce = "/home/dev/rhodecode-enterprise-ce";
rhodecode-enterprise-ee = "/home/dev/rhodecode-enterprise-ee";
};
};
}
Creating a Development Configuration
------------------------------------
To create a development environment for RhodeCode Enterprise,
use the following steps:
1. Create a copy of vcsserver config:
`cp ~/rhodecode-vcsserver/configs/development.ini ~/rhodecode-vcsserver/configs/dev.ini`
2. Create a copy of rhodocode config:
`cp ~/rhodecode-enterprise-ce/configs/development.ini ~/rhodecode-enterprise-ce/configs/dev.ini`
3. Adjust the configuration settings to your needs if needed.
.. note::
It is recommended to use the name `dev.ini` since it's included in .hgignore file.
Setup the Development Database
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
To create a development database, use the following example. This is a one
time operation executed from the nix-shell of rhodecode-enterprise-ce sources ::
rc-setup-app dev.ini \
--user=admin --password=secret \
--email=admin@example.com \
--repos=~/my_dev_repos
Compile CSS and JavaScript
^^^^^^^^^^^^^^^^^^^^^^^^^^
To use the application's frontend and prepare it for production deployment,
you will need to compile the CSS and JavaScript with Grunt.
This is easily done from within the nix-shell using the following command::
make web-build
When developing new features you will need to recompile following any
changes made to the CSS or JavaScript files when developing the code::
grunt watch
This prepares the development (with comments/whitespace) versions of files.
Start the Development Servers
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
From the rhodecode-vcsserver directory, start the development server in another
nix-shell, using the following command::
pserve configs/dev.ini
In the adjacent nix-shell which you created for your development server, you may
now start CE with the following command::
pserve --reload configs/dev.ini
.. note::
`--reload` flag will automatically reload the server when source file changes.
Run the Environment Tests
^^^^^^^^^^^^^^^^^^^^^^^^^
Please make sure that the tests are passing to verify that your environment is
set up correctly. RhodeCode uses py.test to run tests.
While your instance is running, start a new nix-shell and simply run
``make test`` to run the basic test suite.
Please refer to RCstack installed documentation for instructions on setting up dev environment:
https://docs.rhodecode.com/rcstack/dev/dev-setup.html
Need Help?
^^^^^^^^^^

BIN
docs/images/saml-azure-attributes-example.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 26 KiB

BIN
docs/images/saml-azure-service-provider-example.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 85 KiB

12
docs/index.rst
View file

@ -35,6 +35,18 @@ and commit files and |repos| while managing their security permissions.
Table of Contents
-----------------
.. toctree::
:maxdepth: 1
:caption: Documentation directory
Back to documentation directory <https://docs.rhodecode.com/>
.. toctree::
:maxdepth: 1
:caption: RhodeCode RCstack Documentation
RhodeCode RCstack Installer <https://docs.rhodecode.com/rcstack/>
.. toctree::
:maxdepth: 1
:caption: Admin Documentation

3
docs/install/quick-start.rst
View file

@ -66,6 +66,7 @@ Output should look similar to this:
fb77fb6496c6 channelstream/channelstream:0.7.1 Up 2 hours (healthy) rc_cluster_services-channelstream-1 8000/tcp
cb6c5c022f5b postgres:14.6 Up 2 hours (healthy) rc_cluster_services-database-1 5432/tcp
At this point you should be able to access:
- RhodeCode instance at your domain entered, e.g http://rhodecode.local, the default access
@ -76,6 +77,7 @@ At this point you should be able to access:
RHODECODE_USER_PASS=super-secret-password
.. note::
Recommended post quick start install instructions:
@ -85,7 +87,6 @@ At this point you should be able to access:
* Set up :ref:`indexing-ref`
* Familiarise yourself with the :ref:`rhodecode-admin-ref` section.
.. _rhodecode.com/download/: https://rhodecode.com/download/
.. _rhodecode.com: https://rhodecode.com/
.. _rhodecode.com/register: https://rhodecode.com/register/
.. _rhodecode.com/download: https://rhodecode.com/download/

17
docs/install/using-sqllite.rst
View file

@ -1,23 +1,12 @@
.. _install-sqlite-database:
SQLite
------
SQLite (Deprecated)
-------------------
.. important::
We do not recommend using SQLite in a large development environment
as it has an internal locking mechanism which can become a performance
bottleneck when there are more than 5 concurrent users.
As of 5.x, SQLite is no longer supported, we advise to migrate to MySQL or PostgreSQL.
|RCE| installs SQLite as the default database if you do not specify another
during installation. SQLite is suitable for small teams,
projects with a low load, and evaluation purposes since it is built into
|RCE| and does not require any additional database server.
Using MySQL or PostgreSQL in an large setup gives you much greater
performance, and while migration tools exist to move from one database type
to another, it is better to get it right first time and to immediately use
MySQL or PostgreSQL when you deploy |RCE| in a production environment.
Migrating From SQLite to PostgreSQL
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

22
docs/known-issues/known-issues.rst
View file

@ -42,6 +42,7 @@ Newer Operating system locales
the local-archive format, which is now incompatible with our used glibc 2.26.
Mostly affected are:
- Fedora 23+
- Ubuntu 18.04
- CentOS / RHEL 8
@ -93,3 +94,24 @@ example to pass the correct locale information on boot.
[Install]
WantedBy=multi-user.target
Merge stucks in "merging" status
--------------------------------
Similar issues:
- Pull Request duplicated and/or stucks in "creating" status.
Mostly affected are:
- Kubernetes AWS EKS setup with NFS as shared storage
- AWS EFS as shared storage
Workaround:
1. Manually clear the repo cache via UI:
:menuselection:`Repository Settings --> Caches --> Invalidate repository cache`
1. Open problematic PR and reset status to "created"
Now you can merge PR normally

25
docs/release-notes/release-notes-5.1.0.rst
View file

@ -10,20 +10,20 @@ Release Date
New Features
^^^^^^^^^^^^
- We've introduced 2FA for users. Now alongside the external auth 2fa support RhodeCode allows to enable 2FA for users
- We've introduced 2FA for users. Now alongside the external auth 2FA support RhodeCode allows to enable 2FA for users.
2FA options will be available for each user individually, or enforced via authentication plugins like ldap, or internal.
- Email based log-in. RhodeCode now allows to log-in using email as well as username for main authentication type.
- Ability to replace a file using web UI. Now one can replace an existing file from the web-ui.
- GIT LFS Sync automation. Remote push/pull commands now can also sync GIT LFS objects.
- Added ability to remove or close branches from the web ui
- Added ability to delete a branch automatically after merging PR for git repositories
- Added support for S3 based archive_cache based that allows storing cached archives in S3 compatible object store.
- Added ability to remove or close branches from the web ui.
- Added ability to delete a branch automatically after merging PR for git repositories.
- Added support for S3 based archive_cache that allows storing cached archives in S3 compatible object store.
General
^^^^^^^
- Upgraded all dependency libraries to their latest available versions
- Upgraded all dependency libraries to their latest available versions.
- Repository storage is no longer controlled via DB settings, but .ini file. This allows easier automated deployments.
- Bumped mercurial to 6.7.4
- Mercurial: enable httppostarguments for better support of large repositories with lots of heads.
@ -39,21 +39,20 @@ Performance
^^^^^^^^^^^
- Introduced a full rewrite of ssh backend for performance. The result is 2-5x speed improvement for operation with ssh.
enable new ssh wrapper by setting: `ssh.wrapper_cmd = /home/rhodecode/venv/bin/rc-ssh-wrapper-v2`
- Introduced a new hooks subsystem that is more scalable and faster, enable it by settings: `vcs.hooks.protocol = celery`
Enable new ssh wrapper by setting: `ssh.wrapper_cmd = /home/rhodecode/venv/bin/rc-ssh-wrapper-v2`
- Introduced a new hooks subsystem that is more scalable and faster, enable it by setting: `vcs.hooks.protocol = celery`
Fixes
^^^^^
- Archives: Zip archive download breaks when a gitmodules file is present
- Branch permissions: fixed bug preventing to specify own rules from 4.X install
- SVN: refactored svn events, thus fixing support for it in dockerized env
- Fixed empty server url in PR link after push from cli
- Archives: Zip archive download breaks when a gitmodules file is present.
- Branch permissions: fixed bug preventing to specify own rules from 4.X install.
- SVN: refactored svn events, thus fixing support for it in dockerized environment.
- Fixed empty server url in PR link after push from cli.
Upgrade notes
^^^^^^^^^^^^^
- RhodeCode 5.1.0 is a mayor feature release after big 5.0.0 python3 migration. Happy to ship a first time feature
rich release
- RhodeCode 5.1.0 is a major feature release after big 5.0.0 python3 migration. Happy to ship a first time feature-rich release.

40
docs/release-notes/release-notes-5.1.1.rst Normal file
View file

@ -0,0 +1,40 @@
|RCE| 5.1.1 |RNS|
-----------------
Release Date
^^^^^^^^^^^^
- 2024-07-23
New Features
^^^^^^^^^^^^
General
^^^^^^^
Security
^^^^^^^^
Performance
^^^^^^^^^^^
Fixes
^^^^^
- Fixed problems with JS static files build
Upgrade notes
^^^^^^^^^^^^^
- RhodeCode 5.1.1 is unscheduled bugfix release to address some build issues with 5.1 images

41
docs/release-notes/release-notes-5.1.2.rst Normal file
View file

@ -0,0 +1,41 @@
|RCE| 5.1.2 |RNS|
-----------------
Release Date
^^^^^^^^^^^^
- 2024-09-12
New Features
^^^^^^^^^^^^
General
^^^^^^^
Security
^^^^^^^^
Performance
^^^^^^^^^^^
Fixes
^^^^^
- Fixed problems with Mercurial authentication after enabling httppostargs.
Currently this protocol will be disabled until proper fix is in place
Upgrade notes
^^^^^^^^^^^^^
- RhodeCode 5.1.2 is unscheduled bugfix release to address some build issues with 5.1 images

55
docs/release-notes/release-notes-5.2.0.rst Normal file
View file

@ -0,0 +1,55 @@
|RCE| 5.2.0 |RNS|
-----------------
Release Date
^^^^^^^^^^^^
- 2024-10-09
New Features
^^^^^^^^^^^^
- New artifact storage engines allowing an s3 based uploads
- Enterprise version only: Added security tab to admin interface and possibility to whitelist specific vcs client versions. Some older versions clients have known security vulnerabilities, now you can disallow them.
- Enterprise version only: Implemented support for Azure SAML authentication
General
^^^^^^^
- Bumped version of packaging, gunicorn, orjson, zope.interface and some other requirements
- Few tweaks and changes to saml plugins to allows easier setup
- Configs: allow json log format for gunicorn
- Configs: deprecated old ssh wrapper command and make the v2 the default one
- Make sure commit-caches propagate to parent repo groups
- Configs: Moved git lfs path and path of hg large files to ini file
Security
^^^^^^^^
Performance
^^^^^^^^^^^
- description escaper for better performance
Fixes
^^^^^
- Email notifications not working properly
- Removed waitress as a default runner
- Fixed issue with branch permissions
- Ldap: fixed nested groups extraction logic
- Fixed possible db corruption in case of filesystem problems
- Cleanup and improvements to documentation
- Added Kubernetes deployment section to the documentation
- Added default value to celery result and broker
- Fixed broken backends function after python3 migration
- Explicitly disable mercurial web_push ssl flag to prevent from errors about ssl required
- VCS: fixed problems with locked repos and with branch permissions reporting
Upgrade notes
^^^^^^^^^^^^^
- RhodeCode 5.2.0 is a planned major release featuring Azure SAML, whitelist for client versions, s3 artifacts backend and more!

4
docs/release-notes/release-notes.rst
View file

@ -9,7 +9,9 @@ Release Notes
.. toctree::
:maxdepth: 1
release-notes-5.2.0.rst
release-notes-5.1.2.rst
release-notes-5.1.1.rst
release-notes-5.1.0.rst
release-notes-5.0.3.rst
release-notes-5.0.2.rst

2
docs/requirements_docs.txt
View file

@ -4,7 +4,7 @@ furo==2023.9.10
sphinx-press-theme==0.8.0
sphinx-rtd-theme==1.3.0
pygments==2.16.1
pygments==2.18.0
docutils<0.19
markupsafe==2.1.3

146
requirements.txt
View file

@ -5,7 +5,7 @@ alembic==1.13.1
markupsafe==2.1.2
sqlalchemy==1.4.52
greenlet==3.0.3
typing_extensions==4.9.0
typing_extensions==4.12.2
async-timeout==4.0.3
babel==2.12.1
beaker==1.12.1
@ -18,8 +18,8 @@ celery==5.3.6
click==8.1.3
click-repl==0.2.0
click==8.1.3
prompt-toolkit==3.0.38
wcwidth==0.2.6
prompt_toolkit==3.0.47
wcwidth==0.2.13
six==1.16.0
kombu==5.3.5
amqp==5.2.0
@ -33,7 +33,7 @@ channelstream==0.7.1
gevent==24.2.1
greenlet==3.0.3
zope.event==5.0.0
zope.interface==6.3.0
zope.interface==7.0.3
itsdangerous==1.1.0
marshmallow==2.18.0
pyramid==2.0.2
@ -46,7 +46,7 @@ channelstream==0.7.1
venusian==3.0.0
webob==1.8.7
zope.deprecation==5.0.0
zope.interface==6.3.0
zope.interface==7.0.3
pyramid-jinja2==2.10
jinja2==3.1.2
markupsafe==2.1.2
@ -61,7 +61,7 @@ channelstream==0.7.1
venusian==3.0.0
webob==1.8.7
zope.deprecation==5.0.0
zope.interface==6.3.0
zope.interface==7.0.3
zope.deprecation==5.0.0
python-dateutil==2.8.2
six==1.16.0
@ -87,32 +87,31 @@ dogpile.cache==1.3.3
pbr==5.11.1
formencode==2.1.0
six==1.16.0
fsspec==2024.6.0
gunicorn==21.2.0
packaging==24.0
fsspec==2024.9.0
gunicorn==23.0.0
packaging==24.1
gevent==24.2.1
greenlet==3.0.3
zope.event==5.0.0
zope.interface==6.3.0
ipython==8.14.0
backcall==0.2.0
zope.interface==7.0.3
ipython==8.26.0
decorator==5.1.1
jedi==0.19.0
parso==0.8.3
matplotlib-inline==0.1.6
traitlets==5.9.0
pexpect==4.8.0
jedi==0.19.1
parso==0.8.4
matplotlib-inline==0.1.7
traitlets==5.14.3
pexpect==4.9.0
ptyprocess==0.7.0
pickleshare==0.7.5
prompt-toolkit==3.0.38
wcwidth==0.2.6
pygments==2.15.1
stack-data==0.6.2
asttokens==2.2.1
prompt_toolkit==3.0.47
wcwidth==0.2.13
pygments==2.18.0
stack-data==0.6.3
asttokens==2.4.1
six==1.16.0
executing==1.2.0
pure-eval==0.2.2
traitlets==5.9.0
executing==2.0.1
pure_eval==0.2.3
traitlets==5.14.3
typing_extensions==4.12.2
markdown==3.4.3
msgpack==1.0.8
mysqlclient==2.1.1
@ -127,7 +126,7 @@ nbconvert==7.7.3
markupsafe==2.1.2
jupyter_core==5.3.1
platformdirs==3.10.0
traitlets==5.9.0
traitlets==5.14.3
jupyterlab-pygments==0.2.2
markupsafe==2.1.2
mistune==2.0.5
@ -135,15 +134,15 @@ nbconvert==7.7.3
jupyter_client==8.3.0
jupyter_core==5.3.1
platformdirs==3.10.0
traitlets==5.9.0
traitlets==5.14.3
python-dateutil==2.8.2
six==1.16.0
pyzmq==25.0.0
tornado==6.2
traitlets==5.9.0
traitlets==5.14.3
jupyter_core==5.3.1
platformdirs==3.10.0
traitlets==5.9.0
traitlets==5.14.3
nbformat==5.9.2
fastjsonschema==2.18.0
jsonschema==4.18.6
@ -151,9 +150,9 @@ nbconvert==7.7.3
pyrsistent==0.19.3
jupyter_core==5.3.1
platformdirs==3.10.0
traitlets==5.9.0
traitlets==5.9.0
traitlets==5.9.0
traitlets==5.14.3
traitlets==5.14.3
traitlets==5.14.3
nbformat==5.9.2
fastjsonschema==2.18.0
jsonschema==4.18.6
@ -161,20 +160,20 @@ nbconvert==7.7.3
pyrsistent==0.19.3
jupyter_core==5.3.1
platformdirs==3.10.0
traitlets==5.9.0
traitlets==5.9.0
traitlets==5.14.3
traitlets==5.14.3
pandocfilters==1.5.0
pygments==2.15.1
pygments==2.18.0
tinycss2==1.2.1
webencodings==0.5.1
traitlets==5.9.0
orjson==3.10.3
traitlets==5.14.3
orjson==3.10.7
paste==3.10.1
premailer==3.10.0
cachetools==5.3.3
cssselect==1.2.0
cssutils==2.6.0
lxml==4.9.3
lxml==5.3.0
requests==2.28.2
certifi==2022.12.7
charset-normalizer==3.1.0
@ -191,33 +190,6 @@ pycurl==7.45.3
pymysql==1.0.3
pyotp==2.8.0
pyparsing==3.1.1
pyramid-debugtoolbar==4.12.1
pygments==2.15.1
pyramid==2.0.2
hupper==1.12
plaster==1.1.2
plaster-pastedeploy==1.0.1
pastedeploy==3.1.0
plaster==1.1.2
translationstring==1.4
venusian==3.0.0
webob==1.8.7
zope.deprecation==5.0.0
zope.interface==6.3.0
pyramid-mako==1.1.0
mako==1.2.4
markupsafe==2.1.2
pyramid==2.0.2
hupper==1.12
plaster==1.1.2
plaster-pastedeploy==1.0.1
pastedeploy==3.1.0
plaster==1.1.2
translationstring==1.4
venusian==3.0.0
webob==1.8.7
zope.deprecation==5.0.0
zope.interface==6.3.0
pyramid-mailer==0.15.1
pyramid==2.0.2
hupper==1.12
@ -229,13 +201,27 @@ pyramid-mailer==0.15.1
venusian==3.0.0
webob==1.8.7
zope.deprecation==5.0.0
zope.interface==6.3.0
zope.interface==7.0.3
repoze.sendmail==4.4.1
transaction==3.1.0
zope.interface==6.3.0
zope.interface==6.3.0
transaction==3.1.0
zope.interface==6.3.0
transaction==5.0.0
zope.interface==7.0.3
zope.interface==7.0.3
transaction==5.0.0
zope.interface==7.0.3
pyramid-mako==1.1.0
mako==1.2.4
markupsafe==2.1.2
pyramid==2.0.2
hupper==1.12
plaster==1.1.2
plaster-pastedeploy==1.0.1
pastedeploy==3.1.0
plaster==1.1.2
translationstring==1.4
venusian==3.0.0
webob==1.8.7
zope.deprecation==5.0.0
zope.interface==7.0.3
python-ldap==3.4.3
pyasn1==0.4.8
pyasn1-modules==0.2.8
@ -243,20 +229,20 @@ python-ldap==3.4.3
python-memcached==1.59
six==1.16.0
python-pam==2.0.2
python3-saml==1.15.0
python3-saml==1.16.0
isodate==0.6.1
six==1.16.0
lxml==4.9.3
xmlsec==1.3.13
lxml==4.9.3
lxml==5.3.0
xmlsec==1.3.14
lxml==5.3.0
pyyaml==6.0.1
redis==5.0.4
redis==5.1.0
async-timeout==4.0.3
regex==2022.10.31
routes==2.5.1
repoze.lru==0.7
six==1.16.0
s3fs==2024.6.0
s3fs==2024.9.0
aiobotocore==2.13.0
aiohttp==3.9.5
aiosignal==1.3.1
@ -283,7 +269,7 @@ s3fs==2024.6.0
yarl==1.9.4
idna==3.4
multidict==6.0.5
fsspec==2024.6.0
fsspec==2024.9.0
simplejson==3.19.2
sshpubkeys==3.3.1
cryptography==40.0.2
@ -293,7 +279,7 @@ sshpubkeys==3.3.1
six==1.16.0
sqlalchemy==1.4.52
greenlet==3.0.3
typing_extensions==4.9.0
typing_extensions==4.12.2
supervisor==4.2.5
tzlocal==4.3
pytz-deprecation-shim==0.1.0.post0

1
requirements_debug.txt
View file

@ -9,6 +9,7 @@ pympler
ipdb
ipython
rich
pyramid-debugtoolbar
# format
flake8

16
requirements_test.txt
View file

@ -4,38 +4,38 @@ pytest-cov==4.1.0
coverage==7.4.3
pytest==8.1.1
iniconfig==2.0.0
packaging==24.0
packaging==24.1
pluggy==1.4.0
pytest-env==1.1.3
pytest==8.1.1
iniconfig==2.0.0
packaging==24.0
packaging==24.1
pluggy==1.4.0
pytest-profiling==1.7.0
gprof2dot==2022.7.29
pytest==8.1.1
iniconfig==2.0.0
packaging==24.0
packaging==24.1
pluggy==1.4.0
six==1.16.0
pytest-rerunfailures==13.0
packaging==24.0
packaging==24.1
pytest==8.1.1
iniconfig==2.0.0
packaging==24.0
packaging==24.1
pluggy==1.4.0
pytest-runner==6.0.1
pytest-sugar==1.0.0
packaging==24.0
packaging==24.1
pytest==8.1.1
iniconfig==2.0.0
packaging==24.0
packaging==24.1
pluggy==1.4.0
termcolor==2.4.0
pytest-timeout==2.3.1
pytest==8.1.1
iniconfig==2.0.0
packaging==24.0
packaging==24.1
pluggy==1.4.0
webtest==3.0.0
beautifulsoup4==4.12.3

2
rhodecode/VERSION
View file

@ -1 +1 @@
5.1.2
5.2.0

2
rhodecode/api/__init__.py
View file

@ -40,6 +40,7 @@ from rhodecode.lib import ext_json
from rhodecode.lib.utils2 import safe_str
from rhodecode.lib.plugins.utils import get_plugin_settings
from rhodecode.model.db import User, UserApiKeys
from rhodecode.config.patches import inspect_getargspec
log = logging.getLogger(__name__)
@ -186,7 +187,6 @@ def request_view(request):
exposed method
"""
# cython compatible inspect
from rhodecode.config.patches import inspect_getargspec
inspect = inspect_getargspec()
# check if we can find this session using api_key, get_by_auth_token

6
rhodecode/api/views/server_api.py
View file

@ -33,7 +33,7 @@ from rhodecode.lib.ext_json import json
from rhodecode.lib.utils2 import safe_int
from rhodecode.model.db import UserIpMap
from rhodecode.model.scm import ScmModel
from rhodecode.apps.file_store import utils
from rhodecode.apps.file_store import utils as store_utils
from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
FileOverSizeException
@ -328,8 +328,8 @@ def get_method(request, apiuser, pattern=Optional('*')):
]
error : null
"""
from rhodecode.config.patches import inspect_getargspec
inspect = inspect_getargspec()
from rhodecode.config import patches
inspect = patches.inspect_getargspec()
if not has_superadmin_permission(apiuser):
raise JSONRPCForbidden()

33
rhodecode/apps/admin/__init__.py
View file

@ -43,7 +43,38 @@ def admin_routes(config):
from rhodecode.apps.admin.views.system_info import AdminSystemInfoSettingsView
from rhodecode.apps.admin.views.user_groups import AdminUserGroupsView
from rhodecode.apps.admin.views.users import AdminUsersView, UsersView
from rhodecode.apps.admin.views.security import AdminSecurityView
# Security EE feature
config.add_route(
'admin_security',
pattern='/security')
config.add_view(
AdminSecurityView,
attr='security',
route_name='admin_security', request_method='GET',
renderer='rhodecode:templates/admin/security/security.mako')
config.add_route(
name='admin_security_update',
pattern='/security/update')
config.add_view(
AdminSecurityView,
attr='security_update',
route_name='admin_security_update', request_method='POST',
renderer='rhodecode:templates/admin/security/security.mako')
config.add_route(
name='admin_security_modify_allowed_vcs_client_versions',
pattern=ADMIN_PREFIX + '/security/modify/allowed_vcs_client_versions')
config.add_view(
AdminSecurityView,
attr='vcs_whitelisted_client_versions_edit',
route_name='admin_security_modify_allowed_vcs_client_versions', request_method=('GET', 'POST'),
renderer='rhodecode:templates/admin/security/edit_allowed_vcs_client_versions.mako')
config.add_route(
name='admin_audit_logs',
pattern='/audit_logs')

46
rhodecode/apps/admin/views/security.py Normal file
View file

@ -0,0 +1,46 @@
# Copyright (C) 2010-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
from rhodecode.apps._base import BaseAppView
from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
log = logging.getLogger(__name__)
class AdminSecurityView(BaseAppView):
def load_default_context(self):
c = self._get_local_tmpl_context()
return c
@LoginRequired()
@HasPermissionAllDecorator('hg.admin')
def security(self):
c = self.load_default_context()
c.active = 'security'
return self._get_template_context(c)
@LoginRequired()
@HasPermissionAllDecorator('hg.admin')
def admin_security_modify_allowed_vcs_client_versions(self):
c = self.load_default_context()
c.active = 'security'
return self._get_template_context(c)

13
rhodecode/apps/admin/views/settings.py
View file

@ -75,14 +75,21 @@ class AdminSettingsView(BaseAppView):
if not ret:
raise Exception('Could not get application ui settings !')
settings = {}
settings = {
# legacy param that needs to be kept
'web_push_ssl': False
}
for each in ret:
k = each.ui_key
v = each.ui_value
# skip some options if they are defined
if k in ['push_ssl']:
continue
if k == '/':
k = 'root_path'
if k in ['push_ssl', 'publish', 'enabled']:
if k in ['publish', 'enabled']:
v = str2bool(v)
if k.find('.') != -1:
@ -92,6 +99,7 @@ class AdminSettingsView(BaseAppView):
v = each.ui_active
settings[each.ui_section + '_' + k] = v
return settings
@classmethod
@ -164,7 +172,6 @@ class AdminSettingsView(BaseAppView):
return Response(html)
try:
model.update_global_ssl_setting(form_result['web_push_ssl'])
model.update_global_hook_settings(form_result)
model.create_or_update_global_svn_settings(form_result)

8
rhodecode/apps/admin/views/system_info.py
View file

@ -171,11 +171,17 @@ class AdminSystemInfoSettingsView(BaseAppView):
(_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
('', '', ''), # spacer
(_('Archive cache storage type'), val('storage_archive')['type'], state('storage_archive')),
(_('Artifacts storage backend'), val('storage_artifacts')['type'], state('storage_artifacts')),
(_('Artifacts storage location'), val('storage_artifacts')['path'], state('storage_artifacts')),
(_('Artifacts info'), val('storage_artifacts')['text'], state('storage_artifacts')),
('', '', ''), # spacer
(_('Archive cache storage backend'), val('storage_archive')['type'], state('storage_archive')),
(_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
(_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
('', '', ''), # spacer
(_('Temp storage location'), val('storage_temp')['path'], state('storage_temp')),
(_('Temp storage info'), val('storage_temp')['text'], state('storage_temp')),
('', '', ''), # spacer

41
rhodecode/apps/file_store/__init__.py
View file

@ -16,7 +16,8 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
from rhodecode.apps.file_store import config_keys
from rhodecode.config.settings_maker import SettingsMaker
@ -24,18 +25,48 @@ def _sanitize_settings_and_apply_defaults(settings):
"""
Set defaults, convert to python types and validate settings.
"""
from rhodecode.apps.file_store import config_keys
# translate "legacy" params into new config
settings.pop(config_keys.deprecated_enabled, True)
if config_keys.deprecated_backend in settings:
# if legacy backend key is detected we use "legacy" backward compat setting
settings.pop(config_keys.deprecated_backend)
settings[config_keys.backend_type] = config_keys.backend_legacy_filesystem
if config_keys.deprecated_store_path in settings:
store_path = settings.pop(config_keys.deprecated_store_path)
settings[config_keys.legacy_filesystem_storage_path] = store_path
settings_maker = SettingsMaker(settings)
settings_maker.make_setting(config_keys.enabled, True, parser='bool')
settings_maker.make_setting(config_keys.backend, 'local')
default_cache_dir = settings['cache_dir']
default_store_dir = os.path.join(default_cache_dir, 'artifacts_filestore')
default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store')
settings_maker.make_setting(config_keys.store_path, default_store)
# set default backend
settings_maker.make_setting(config_keys.backend_type, config_keys.backend_legacy_filesystem)
# legacy filesystem defaults
settings_maker.make_setting(config_keys.legacy_filesystem_storage_path, default_store_dir, default_when_empty=True, )
# filesystem defaults
settings_maker.make_setting(config_keys.filesystem_storage_path, default_store_dir, default_when_empty=True,)
settings_maker.make_setting(config_keys.filesystem_shards, 8, parser='int')
# objectstore defaults
settings_maker.make_setting(config_keys.objectstore_url, 'http://s3-minio:9000')
settings_maker.make_setting(config_keys.objectstore_bucket, 'rhodecode-artifacts-filestore')
settings_maker.make_setting(config_keys.objectstore_bucket_shards, 8, parser='int')
settings_maker.make_setting(config_keys.objectstore_region, '')
settings_maker.make_setting(config_keys.objectstore_key, '')
settings_maker.make_setting(config_keys.objectstore_secret, '')
settings_maker.env_expand()
def includeme(config):
from rhodecode.apps.file_store.views import FileStoreView
settings = config.registry.settings

269
rhodecode/apps/file_store/backends/base.py Normal file
View file

@ -0,0 +1,269 @@
# Copyright (C) 2016-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import fsspec # noqa
import logging
from rhodecode.lib.ext_json import json
from rhodecode.apps.file_store.utils import sha256_safe, ShardFileReader, get_uid_filename
from rhodecode.apps.file_store.extensions import resolve_extensions
from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException # noqa: F401
log = logging.getLogger(__name__)
class BaseShard:
metadata_suffix: str = '.metadata'
storage_type: str = ''
fs = None
@property
def storage_medium(self):
if not self.storage_type:
raise ValueError('No storage type set for this shard storage_type=""')
return getattr(self, self.storage_type)
def __contains__(self, key):
full_path = self.store_path(key)
return self.fs.exists(full_path)
def metadata_convert(self, uid_filename, metadata):
return metadata
def get_metadata_filename(self, uid_filename) -> tuple[str, str]:
metadata_file: str = f'{uid_filename}{self.metadata_suffix}'
return metadata_file, self.store_path(metadata_file)
def get_metadata(self, uid_filename, ignore_missing=False) -> dict:
_metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename)
if ignore_missing and not self.fs.exists(metadata_file_path):
return {}
with self.fs.open(metadata_file_path, 'rb') as f:
metadata = json.loads(f.read())
metadata = self.metadata_convert(uid_filename, metadata)
return metadata
def _store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
raise NotImplementedError
def store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
return self._store(key, uid_key, value_reader, max_filesize, metadata, **kwargs)
def _fetch(self, key, presigned_url_expires: int = 0):
raise NotImplementedError
def fetch(self, key, **kwargs) -> tuple[ShardFileReader, dict]:
return self._fetch(key)
def _delete(self, key):
if key not in self:
log.exception(f'requested key={key} not found in {self}')
raise KeyError(key)
metadata = self.get_metadata(key)
_metadata_file, metadata_file_path = self.get_metadata_filename(key)
artifact_file_path = metadata['filename_uid_path']
self.fs.rm(artifact_file_path)
self.fs.rm(metadata_file_path)
return 1
def delete(self, key):
raise NotImplementedError
def store_path(self, uid_filename):
raise NotImplementedError
class BaseFileStoreBackend:
_shards = tuple()
_shard_cls = BaseShard
_config: dict | None = None
_storage_path: str = ''
def __init__(self, settings, extension_groups=None):
self._config = settings
extension_groups = extension_groups or ['any']
self.extensions = resolve_extensions([], groups=extension_groups)
def __contains__(self, key):
return self.filename_exists(key)
def __repr__(self):
return f'<{self.__class__.__name__}(storage={self.storage_path})>'
@property
def storage_path(self):
return self._storage_path
@classmethod
def get_shard_index(cls, filename: str, num_shards) -> int:
# Generate a hash value from the filename
hash_value = sha256_safe(filename)
# Convert the hash value to an integer
hash_int = int(hash_value, 16)
# Map the hash integer to a shard number between 1 and num_shards
shard_number = (hash_int % num_shards)
return shard_number
@classmethod
def apply_counter(cls, counter: int, filename: str) -> str:
"""
Apply a counter to the filename.
:param counter: The counter value to apply.
:param filename: The original filename.
:return: The modified filename with the counter.
"""
name_counted = f'{counter:d}-{filename}'
return name_counted
def _get_shard(self, key) -> _shard_cls:
index = self.get_shard_index(key, len(self._shards))
shard = self._shards[index]
return shard
def get_conf(self, key, pop=False):
if key not in self._config:
raise ValueError(
f"No configuration key '{key}', please make sure it exists in filestore config")
val = self._config[key]
if pop:
del self._config[key]
return val
def filename_allowed(self, filename, extensions=None):
"""Checks if a filename has an allowed extension
:param filename: base name of file
:param extensions: iterable of extensions (or self.extensions)
"""
_, ext = os.path.splitext(filename)
return self.extension_allowed(ext, extensions)
def extension_allowed(self, ext, extensions=None):
"""
Checks if an extension is permitted. Both e.g. ".jpg" and
"jpg" can be passed in. Extension lookup is case-insensitive.
:param ext: extension to check
:param extensions: iterable of extensions to validate against (or self.extensions)
"""
def normalize_ext(_ext):
if _ext.startswith('.'):
_ext = _ext[1:]
return _ext.lower()
extensions = extensions or self.extensions
if not extensions:
return True
ext = normalize_ext(ext)
return ext in [normalize_ext(x) for x in extensions]
def filename_exists(self, uid_filename):
shard = self._get_shard(uid_filename)
return uid_filename in shard
def store_path(self, uid_filename):
"""
Returns absolute file path of the uid_filename
"""
shard = self._get_shard(uid_filename)
return shard.store_path(uid_filename)
def store_metadata(self, uid_filename):
shard = self._get_shard(uid_filename)
return shard.get_metadata_filename(uid_filename)
def store(self, filename, value_reader, extensions=None, metadata=None, max_filesize=None, randomized_name=True, **kwargs):
extensions = extensions or self.extensions
if not self.filename_allowed(filename, extensions):
msg = f'filename {filename} does not allow extensions {extensions}'
raise FileNotAllowedException(msg)
# # TODO: check why we need this setting ? it looks stupid...
# no_body_seek is used in stream mode importer somehow
# no_body_seek = kwargs.pop('no_body_seek', False)
# if no_body_seek:
# pass
# else:
# value_reader.seek(0)
uid_filename = kwargs.pop('uid_filename', None)
if uid_filename is None:
uid_filename = get_uid_filename(filename, randomized=randomized_name)
shard = self._get_shard(uid_filename)
return shard.store(filename, uid_filename, value_reader, max_filesize, metadata, **kwargs)
def import_to_store(self, value_reader, org_filename, uid_filename, metadata, **kwargs):
shard = self._get_shard(uid_filename)
max_filesize = None
return shard.store(org_filename, uid_filename, value_reader, max_filesize, metadata, import_mode=True)
def delete(self, uid_filename):
shard = self._get_shard(uid_filename)
return shard.delete(uid_filename)
def fetch(self, uid_filename) -> tuple[ShardFileReader, dict]:
shard = self._get_shard(uid_filename)
return shard.fetch(uid_filename)
def get_metadata(self, uid_filename, ignore_missing=False) -> dict:
shard = self._get_shard(uid_filename)
return shard.get_metadata(uid_filename, ignore_missing=ignore_missing)
def iter_keys(self):
for shard in self._shards:
if shard.fs.exists(shard.storage_medium):
for path, _dirs, _files in shard.fs.walk(shard.storage_medium):
for key_file_path in _files:
if key_file_path.endswith(shard.metadata_suffix):
yield shard, key_file_path
def iter_artifacts(self):
for shard, key_file in self.iter_keys():
json_key = f"{shard.storage_medium}/{key_file}"
with shard.fs.open(json_key, 'rb') as f:
yield shard, json.loads(f.read())['filename_uid']
def get_statistics(self):
total_files = 0
total_size = 0
meta = {}
for shard, key_file in self.iter_keys():
json_key = f"{shard.storage_medium}/{key_file}"
with shard.fs.open(json_key, 'rb') as f:
total_files += 1
metadata = json.loads(f.read())
total_size += metadata['size']
return total_files, total_size, meta

183
rhodecode/apps/file_store/backends/filesystem.py Normal file
View file

@ -0,0 +1,183 @@
# Copyright (C) 2016-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import hashlib
import functools
import time
import logging
from .. import config_keys
from ..exceptions import FileOverSizeException
from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
from ....lib.ext_json import json
log = logging.getLogger(__name__)
class FileSystemShard(BaseShard):
METADATA_VER = 'v2'
BACKEND_TYPE = config_keys.backend_filesystem
storage_type: str = 'directory'
def __init__(self, index, directory, directory_folder, fs, **settings):
self._index: int = index
self._directory: str = directory
self._directory_folder: str = directory_folder
self.fs = fs
@property
def directory(self) -> str:
"""Cache directory final path."""
return os.path.join(self._directory, self._directory_folder)
def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
# ensure dir exists
destination, _ = os.path.split(full_path)
if not self.fs.exists(destination):
self.fs.makedirs(destination)
writer = self.fs.open(full_path, mode)
digest = hashlib.sha256()
oversize_cleanup = False
with writer:
size = 0
for chunk in iterator:
size += len(chunk)
digest.update(chunk)
writer.write(chunk)
if max_filesize and size > max_filesize:
oversize_cleanup = True
# free up the copied file, and raise exc
break
writer.flush()
# Get the file descriptor
fd = writer.fileno()
# Sync the file descriptor to disk, helps with NFS cases...
os.fsync(fd)
if oversize_cleanup:
self.fs.rm(full_path)
raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
sha256 = digest.hexdigest()
log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
return size, sha256
def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
filename = key
uid_filename = uid_key
full_path = self.store_path(uid_filename)
# STORE METADATA
_metadata = {
"version": self.METADATA_VER,
"store_type": self.BACKEND_TYPE,
"filename": filename,
"filename_uid_path": full_path,
"filename_uid": uid_filename,
"sha256": "", # NOTE: filled in by reader iteration
"store_time": time.time(),
"size": 0
}
if metadata:
if kwargs.pop('import_mode', False):
# in import mode, we don't need to compute metadata, we just take the old version
_metadata["import_mode"] = True
else:
_metadata.update(metadata)
read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
_metadata['size'] = size
_metadata['sha256'] = sha256
# after storing the artifacts, we write the metadata present
_metadata_file, metadata_file_path = self.get_metadata_filename(uid_key)
with self.fs.open(metadata_file_path, 'wb') as f:
f.write(json.dumps(_metadata))
return uid_filename, _metadata
def store_path(self, uid_filename):
"""
Returns absolute file path of the uid_filename
"""
return os.path.join(self._directory, self._directory_folder, uid_filename)
def _fetch(self, key, presigned_url_expires: int = 0):
if key not in self:
log.exception(f'requested key={key} not found in {self}')
raise KeyError(key)
metadata = self.get_metadata(key)
file_path = metadata['filename_uid_path']
if presigned_url_expires and presigned_url_expires > 0:
metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
def delete(self, key):
return self._delete(key)
class FileSystemBackend(BaseFileStoreBackend):
shard_name: str = 'shard_{:03d}'
_shard_cls = FileSystemShard
def __init__(self, settings):
super().__init__(settings)
store_dir = self.get_conf(config_keys.filesystem_storage_path)
directory = os.path.expanduser(store_dir)
self._directory = directory
self._storage_path = directory # common path for all from BaseCache
self._shard_count = int(self.get_conf(config_keys.filesystem_shards, pop=True))
if self._shard_count < 1:
raise ValueError(f'{config_keys.filesystem_shards} must be 1 or more')
log.debug('Initializing %s file_store instance', self)
fs = fsspec.filesystem('file')
if not fs.exists(self._directory):
fs.makedirs(self._directory, exist_ok=True)
self._shards = tuple(
self._shard_cls(
index=num,
directory=directory,
directory_folder=self.shard_name.format(num),
fs=fs,
**settings,
)
for num in range(self._shard_count)
)

278
rhodecode/apps/file_store/backends/filesystem_legacy.py Normal file
View file

@ -0,0 +1,278 @@
# Copyright (C) 2016-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import errno
import os
import hashlib
import functools
import time
import logging
from .. import config_keys
from ..exceptions import FileOverSizeException
from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
from ....lib.ext_json import json
log = logging.getLogger(__name__)
class LegacyFileSystemShard(BaseShard):
# legacy ver
METADATA_VER = 'v2'
BACKEND_TYPE = config_keys.backend_legacy_filesystem
storage_type: str = 'dir_struct'
# legacy suffix
metadata_suffix: str = '.meta'
@classmethod
def _sub_store_from_filename(cls, filename):
return filename[:2]
@classmethod
def apply_counter(cls, counter, filename):
name_counted = '%d-%s' % (counter, filename)
return name_counted
@classmethod
def safe_make_dirs(cls, dir_path):
if not os.path.exists(dir_path):
try:
os.makedirs(dir_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
return
@classmethod
def resolve_name(cls, name, directory):
"""
Resolves a unique name and the correct path. If a filename
for that path already exists then a numeric prefix with values > 0 will be
added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
:param name: base name of file
:param directory: absolute directory path
"""
counter = 0
while True:
name_counted = cls.apply_counter(counter, name)
# sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
sub_store: str = cls._sub_store_from_filename(name_counted)
sub_store_path: str = os.path.join(directory, sub_store)
cls.safe_make_dirs(sub_store_path)
path = os.path.join(sub_store_path, name_counted)
if not os.path.exists(path):
return name_counted, path
counter += 1
def __init__(self, index, directory, directory_folder, fs, **settings):
self._index: int = index
self._directory: str = directory
self._directory_folder: str = directory_folder
self.fs = fs
@property
def dir_struct(self) -> str:
"""Cache directory final path."""
return os.path.join(self._directory, '0-')
def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
# ensure dir exists
destination, _ = os.path.split(full_path)
if not self.fs.exists(destination):
self.fs.makedirs(destination)
writer = self.fs.open(full_path, mode)
digest = hashlib.sha256()
oversize_cleanup = False
with writer:
size = 0
for chunk in iterator:
size += len(chunk)
digest.update(chunk)
writer.write(chunk)
if max_filesize and size > max_filesize:
# free up the copied file, and raise exc
oversize_cleanup = True
break
writer.flush()
# Get the file descriptor
fd = writer.fileno()
# Sync the file descriptor to disk, helps with NFS cases...
os.fsync(fd)
if oversize_cleanup:
self.fs.rm(full_path)
raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
sha256 = digest.hexdigest()
log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
return size, sha256
def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
filename = key
uid_filename = uid_key
# NOTE:, also apply N- Counter...
uid_filename, full_path = self.resolve_name(uid_filename, self._directory)
# STORE METADATA
# TODO: make it compatible, and backward proof
_metadata = {
"version": self.METADATA_VER,
"filename": filename,
"filename_uid_path": full_path,
"filename_uid": uid_filename,
"sha256": "", # NOTE: filled in by reader iteration
"store_time": time.time(),
"size": 0
}
if metadata:
_metadata.update(metadata)
read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
_metadata['size'] = size
_metadata['sha256'] = sha256
# after storing the artifacts, we write the metadata present
_metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename)
with self.fs.open(metadata_file_path, 'wb') as f:
f.write(json.dumps(_metadata))
return uid_filename, _metadata
def store_path(self, uid_filename):
"""
Returns absolute file path of the uid_filename
"""
prefix_dir = ''
if '/' in uid_filename:
prefix_dir, filename = uid_filename.split('/')
sub_store = self._sub_store_from_filename(filename)
else:
sub_store = self._sub_store_from_filename(uid_filename)
return os.path.join(self._directory, prefix_dir, sub_store, uid_filename)
def metadata_convert(self, uid_filename, metadata):
# NOTE: backward compat mode here... this is for file created PRE 5.2 system
if 'meta_ver' in metadata:
full_path = self.store_path(uid_filename)
metadata = {
"_converted": True,
"_org": metadata,
"version": self.METADATA_VER,
"store_type": self.BACKEND_TYPE,
"filename": metadata['filename'],
"filename_uid_path": full_path,
"filename_uid": uid_filename,
"sha256": metadata['sha256'],
"store_time": metadata['time'],
"size": metadata['size']
}
return metadata
def _fetch(self, key, presigned_url_expires: int = 0):
if key not in self:
log.exception(f'requested key={key} not found in {self}')
raise KeyError(key)
metadata = self.get_metadata(key)
file_path = metadata['filename_uid_path']
if presigned_url_expires and presigned_url_expires > 0:
metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
def delete(self, key):
return self._delete(key)
def _delete(self, key):
if key not in self:
log.exception(f'requested key={key} not found in {self}')
raise KeyError(key)
metadata = self.get_metadata(key)
metadata_file, metadata_file_path = self.get_metadata_filename(key)
artifact_file_path = metadata['filename_uid_path']
self.fs.rm(artifact_file_path)
self.fs.rm(metadata_file_path)
def get_metadata_filename(self, uid_filename) -> tuple[str, str]:
metadata_file: str = f'{uid_filename}{self.metadata_suffix}'
uid_path_in_store = self.store_path(uid_filename)
metadata_file_path = f'{uid_path_in_store}{self.metadata_suffix}'
return metadata_file, metadata_file_path
class LegacyFileSystemBackend(BaseFileStoreBackend):
_shard_cls = LegacyFileSystemShard
def __init__(self, settings):
super().__init__(settings)
store_dir = self.get_conf(config_keys.legacy_filesystem_storage_path)
directory = os.path.expanduser(store_dir)
self._directory = directory
self._storage_path = directory # common path for all from BaseCache
log.debug('Initializing %s file_store instance', self)
fs = fsspec.filesystem('file')
if not fs.exists(self._directory):
fs.makedirs(self._directory, exist_ok=True)
# legacy system uses single shard
self._shards = tuple(
[
self._shard_cls(
index=0,
directory=directory,
directory_folder='',
fs=fs,
**settings,
)
]
)
@classmethod
def get_shard_index(cls, filename: str, num_shards) -> int:
# legacy filesystem doesn't use shards, and always uses single shard
return 0

268
rhodecode/apps/file_store/backends/local_store.py
View file

@ -1,268 +0,0 @@
# Copyright (C) 2016-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import time
import errno
import hashlib
from rhodecode.lib.ext_json import json
from rhodecode.apps.file_store import utils
from rhodecode.apps.file_store.extensions import resolve_extensions
from rhodecode.apps.file_store.exceptions import (
FileNotAllowedException, FileOverSizeException)
METADATA_VER = 'v1'
def safe_make_dirs(dir_path):
if not os.path.exists(dir_path):
try:
os.makedirs(dir_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
return
class LocalFileStorage(object):
@classmethod
def apply_counter(cls, counter, filename):
name_counted = '%d-%s' % (counter, filename)
return name_counted
@classmethod
def resolve_name(cls, name, directory):
"""
Resolves a unique name and the correct path. If a filename
for that path already exists then a numeric prefix with values > 0 will be
added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
:param name: base name of file
:param directory: absolute directory path
"""
counter = 0
while True:
name_counted = cls.apply_counter(counter, name)
# sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
sub_store = cls._sub_store_from_filename(name_counted)
sub_store_path = os.path.join(directory, sub_store)
safe_make_dirs(sub_store_path)
path = os.path.join(sub_store_path, name_counted)
if not os.path.exists(path):
return name_counted, path
counter += 1
@classmethod
def _sub_store_from_filename(cls, filename):
return filename[:2]
@classmethod
def calculate_path_hash(cls, file_path):
"""
Efficient calculation of file_path sha256 sum
:param file_path:
:return: sha256sum
"""
digest = hashlib.sha256()
with open(file_path, 'rb') as f:
for chunk in iter(lambda: f.read(1024 * 100), b""):
digest.update(chunk)
return digest.hexdigest()
def __init__(self, base_path, extension_groups=None):
"""
Local file storage
:param base_path: the absolute base path where uploads are stored
:param extension_groups: extensions string
"""
extension_groups = extension_groups or ['any']
self.base_path = base_path
self.extensions = resolve_extensions([], groups=extension_groups)
def __repr__(self):
return f'{self.__class__}@{self.base_path}'
def store_path(self, filename):
"""
Returns absolute file path of the filename, joined to the
base_path.
:param filename: base name of file
"""
prefix_dir = ''
if '/' in filename:
prefix_dir, filename = filename.split('/')
sub_store = self._sub_store_from_filename(filename)
else:
sub_store = self._sub_store_from_filename(filename)
return os.path.join(self.base_path, prefix_dir, sub_store, filename)
def delete(self, filename):
"""
Deletes the filename. Filename is resolved with the
absolute path based on base_path. If file does not exist,
returns **False**, otherwise **True**
:param filename: base name of file
"""
if self.exists(filename):
os.remove(self.store_path(filename))
return True
return False
def exists(self, filename):
"""
Checks if file exists. Resolves filename's absolute
path based on base_path.
:param filename: file_uid name of file, e.g 0-f62b2b2d-9708-4079-a071-ec3f958448d4.svg
"""
return os.path.exists(self.store_path(filename))
def filename_allowed(self, filename, extensions=None):
"""Checks if a filename has an allowed extension
:param filename: base name of file
:param extensions: iterable of extensions (or self.extensions)
"""
_, ext = os.path.splitext(filename)
return self.extension_allowed(ext, extensions)
def extension_allowed(self, ext, extensions=None):
"""
Checks if an extension is permitted. Both e.g. ".jpg" and
"jpg" can be passed in. Extension lookup is case-insensitive.
:param ext: extension to check
:param extensions: iterable of extensions to validate against (or self.extensions)
"""
def normalize_ext(_ext):
if _ext.startswith('.'):
_ext = _ext[1:]
return _ext.lower()
extensions = extensions or self.extensions
if not extensions:
return True
ext = normalize_ext(ext)
return ext in [normalize_ext(x) for x in extensions]
def save_file(self, file_obj, filename, directory=None, extensions=None,
extra_metadata=None, max_filesize=None, randomized_name=True, **kwargs):
"""
Saves a file object to the uploads location.
Returns the resolved filename, i.e. the directory +
the (randomized/incremented) base name.
:param file_obj: **cgi.FieldStorage** object (or similar)
:param filename: original filename
:param directory: relative path of sub-directory
:param extensions: iterable of allowed extensions, if not default
:param max_filesize: maximum size of file that should be allowed
:param randomized_name: generate random generated UID or fixed based on the filename
:param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
"""
extensions = extensions or self.extensions
if not self.filename_allowed(filename, extensions):
raise FileNotAllowedException()
if directory:
dest_directory = os.path.join(self.base_path, directory)
else:
dest_directory = self.base_path
safe_make_dirs(dest_directory)
uid_filename = utils.uid_filename(filename, randomized=randomized_name)
# resolve also produces special sub-dir for file optimized store
filename, path = self.resolve_name(uid_filename, dest_directory)
stored_file_dir = os.path.dirname(path)
no_body_seek = kwargs.pop('no_body_seek', False)
if no_body_seek:
pass
else:
file_obj.seek(0)
with open(path, "wb") as dest:
length = 256 * 1024
while 1:
buf = file_obj.read(length)
if not buf:
break
dest.write(buf)
metadata = {}
if extra_metadata:
metadata = extra_metadata
size = os.stat(path).st_size
if max_filesize and size > max_filesize:
# free up the copied file, and raise exc
os.remove(path)
raise FileOverSizeException()
file_hash = self.calculate_path_hash(path)
metadata.update({
"filename": filename,
"size": size,
"time": time.time(),
"sha256": file_hash,
"meta_ver": METADATA_VER
})
filename_meta = filename + '.meta'
with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
dest_meta.write(json.dumps(metadata))
if directory:
filename = os.path.join(directory, filename)
return filename, metadata
def get_metadata(self, filename, ignore_missing=False):
"""
Reads JSON stored metadata for a file
:param filename:
:return:
"""
filename = self.store_path(filename)
filename_meta = filename + '.meta'
if ignore_missing and not os.path.isfile(filename_meta):
return {}
with open(filename_meta, "rb") as source_meta:
return json.loads(source_meta.read())

184
rhodecode/apps/file_store/backends/objectstore.py Normal file
View file

@ -0,0 +1,184 @@
# Copyright (C) 2016-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import hashlib
import functools
import time
import logging
from .. import config_keys
from ..exceptions import FileOverSizeException
from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
from ....lib.ext_json import json
log = logging.getLogger(__name__)
class S3Shard(BaseShard):
METADATA_VER = 'v2'
BACKEND_TYPE = config_keys.backend_objectstore
storage_type: str = 'bucket'
def __init__(self, index, bucket, bucket_folder, fs, **settings):
self._index: int = index
self._bucket_main: str = bucket
self._bucket_folder: str = bucket_folder
self.fs = fs
@property
def bucket(self) -> str:
"""Cache bucket final path."""
return os.path.join(self._bucket_main, self._bucket_folder)
def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
# ensure dir exists
destination, _ = os.path.split(full_path)
if not self.fs.exists(destination):
self.fs.makedirs(destination)
writer = self.fs.open(full_path, mode)
digest = hashlib.sha256()
oversize_cleanup = False
with writer:
size = 0
for chunk in iterator:
size += len(chunk)
digest.update(chunk)
writer.write(chunk)
if max_filesize and size > max_filesize:
oversize_cleanup = True
# free up the copied file, and raise exc
break
if oversize_cleanup:
self.fs.rm(full_path)
raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
sha256 = digest.hexdigest()
log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
return size, sha256
def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
filename = key
uid_filename = uid_key
full_path = self.store_path(uid_filename)
# STORE METADATA
_metadata = {
"version": self.METADATA_VER,
"store_type": self.BACKEND_TYPE,
"filename": filename,
"filename_uid_path": full_path,
"filename_uid": uid_filename,
"sha256": "", # NOTE: filled in by reader iteration
"store_time": time.time(),
"size": 0
}
if metadata:
if kwargs.pop('import_mode', False):
# in import mode, we don't need to compute metadata, we just take the old version
_metadata["import_mode"] = True
else:
_metadata.update(metadata)
read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
_metadata['size'] = size
_metadata['sha256'] = sha256
# after storing the artifacts, we write the metadata present
metadata_file, metadata_file_path = self.get_metadata_filename(uid_key)
with self.fs.open(metadata_file_path, 'wb') as f:
f.write(json.dumps(_metadata))
return uid_filename, _metadata
def store_path(self, uid_filename):
"""
Returns absolute file path of the uid_filename
"""
return os.path.join(self._bucket_main, self._bucket_folder, uid_filename)
def _fetch(self, key, presigned_url_expires: int = 0):
if key not in self:
log.exception(f'requested key={key} not found in {self}')
raise KeyError(key)
metadata_file, metadata_file_path = self.get_metadata_filename(key)
with self.fs.open(metadata_file_path, 'rb') as f:
metadata = json.loads(f.read())
file_path = metadata['filename_uid_path']
if presigned_url_expires and presigned_url_expires > 0:
metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
def delete(self, key):
return self._delete(key)
class ObjectStoreBackend(BaseFileStoreBackend):
shard_name: str = 'shard-{:03d}'
_shard_cls = S3Shard
def __init__(self, settings):
super().__init__(settings)
self._shard_count = int(self.get_conf(config_keys.objectstore_bucket_shards, pop=True))
if self._shard_count < 1:
raise ValueError('cache_shards must be 1 or more')
self._bucket = settings.pop(config_keys.objectstore_bucket)
if not self._bucket:
raise ValueError(f'{config_keys.objectstore_bucket} needs to have a value')
objectstore_url = self.get_conf(config_keys.objectstore_url)
key = settings.pop(config_keys.objectstore_key)
secret = settings.pop(config_keys.objectstore_secret)
self._storage_path = objectstore_url # common path for all from BaseCache
log.debug('Initializing %s file_store instance', self)
fs = fsspec.filesystem('s3', anon=False, endpoint_url=objectstore_url, key=key, secret=secret)
# init main bucket
if not fs.exists(self._bucket):
fs.mkdir(self._bucket)
self._shards = tuple(
self._shard_cls(
index=num,
bucket=self._bucket,
bucket_folder=self.shard_name.format(num),
fs=fs,
**settings,
)
for num in range(self._shard_count)
)

38
rhodecode/apps/file_store/config_keys.py
View file

@ -20,6 +20,38 @@
# Definition of setting keys used to configure this module. Defined here to
# avoid repetition of keys throughout the module.
enabled = 'file_store.enabled'
backend = 'file_store.backend'
store_path = 'file_store.storage_path'
# OLD and deprecated keys not used anymore
deprecated_enabled = 'file_store.enabled'
deprecated_backend = 'file_store.backend'
deprecated_store_path = 'file_store.storage_path'
backend_type = 'file_store.backend.type'
backend_legacy_filesystem = 'filesystem_v1'
backend_filesystem = 'filesystem_v2'
backend_objectstore = 'objectstore'
backend_types = [
backend_legacy_filesystem,
backend_filesystem,
backend_objectstore,
]
# filesystem_v1 legacy
legacy_filesystem_storage_path = 'file_store.filesystem_v1.storage_path'
# filesystem_v2 new option
filesystem_storage_path = 'file_store.filesystem_v2.storage_path'
filesystem_shards = 'file_store.filesystem_v2.shards'
# objectstore
objectstore_url = 'file_store.objectstore.url'
objectstore_bucket = 'file_store.objectstore.bucket'
objectstore_bucket_shards = 'file_store.objectstore.bucket_shards'
objectstore_region = 'file_store.objectstore.region'
objectstore_key = 'file_store.objectstore.key'
objectstore_secret = 'file_store.objectstore.secret'

39
rhodecode/apps/file_store/tests/__init__.py
View file

@ -16,3 +16,42 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import random
import tempfile
import string
import pytest
from rhodecode.apps.file_store import utils as store_utils
@pytest.fixture()
def file_store_instance(ini_settings):
config = ini_settings
f_store = store_utils.get_filestore_backend(config=config, always_init=True)
return f_store
@pytest.fixture
def random_binary_file():
# Generate random binary data
data = bytearray(random.getrandbits(8) for _ in range(1024 * 512)) # 512 KB of random data
# Create a temporary file
temp_file = tempfile.NamedTemporaryFile(delete=False)
filename = temp_file.name
try:
# Write the random binary data to the file
temp_file.write(data)
temp_file.seek(0) # Rewind the file pointer to the beginning
yield filename, temp_file
finally:
# Close and delete the temporary file after the test
temp_file.close()
os.remove(filename)
def generate_random_filename(length=10):
return ''.join(random.choices(string.ascii_letters + string.digits, k=length))

128
rhodecode/apps/file_store/tests/test_filestore_backends.py Normal file
View file

@ -0,0 +1,128 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import pytest
from rhodecode.apps import file_store
from rhodecode.apps.file_store import config_keys
from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend
from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend
from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend
from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException
from rhodecode.apps.file_store import utils as store_utils
from rhodecode.apps.file_store.tests import random_binary_file, file_store_instance
class TestFileStoreBackends:
@pytest.mark.parametrize('backend_type, expected_instance', [
(config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
(config_keys.backend_filesystem, FileSystemBackend),
(config_keys.backend_objectstore, ObjectStoreBackend),
])
def test_get_backend(self, backend_type, expected_instance, ini_settings):
config = ini_settings
config[config_keys.backend_type] = backend_type
f_store = store_utils.get_filestore_backend(config=config, always_init=True)
assert isinstance(f_store, expected_instance)
@pytest.mark.parametrize('backend_type, expected_instance', [
(config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
(config_keys.backend_filesystem, FileSystemBackend),
(config_keys.backend_objectstore, ObjectStoreBackend),
])
def test_store_and_read(self, backend_type, expected_instance, ini_settings, random_binary_file):
filename, temp_file = random_binary_file
config = ini_settings
config[config_keys.backend_type] = backend_type
f_store = store_utils.get_filestore_backend(config=config, always_init=True)
metadata = {
'user_uploaded': {
'username': 'user1',
'user_id': 10,
'ip': '10.20.30.40'
}
}
store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata)
assert store_fid
assert metadata
# read-after write
reader, metadata2 = f_store.fetch(store_fid)
assert reader
assert metadata2['filename'] == filename
@pytest.mark.parametrize('backend_type, expected_instance', [
(config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
(config_keys.backend_filesystem, FileSystemBackend),
(config_keys.backend_objectstore, ObjectStoreBackend),
])
def test_store_file_not_allowed(self, backend_type, expected_instance, ini_settings, random_binary_file):
filename, temp_file = random_binary_file
config = ini_settings
config[config_keys.backend_type] = backend_type
f_store = store_utils.get_filestore_backend(config=config, always_init=True)
with pytest.raises(FileNotAllowedException):
f_store.store('notallowed.exe', temp_file, extensions=['.txt'])
@pytest.mark.parametrize('backend_type, expected_instance', [
(config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
(config_keys.backend_filesystem, FileSystemBackend),
(config_keys.backend_objectstore, ObjectStoreBackend),
])
def test_store_file_over_size(self, backend_type, expected_instance, ini_settings, random_binary_file):
filename, temp_file = random_binary_file
config = ini_settings
config[config_keys.backend_type] = backend_type
f_store = store_utils.get_filestore_backend(config=config, always_init=True)
with pytest.raises(FileOverSizeException):
f_store.store('toobig.exe', temp_file, extensions=['.exe'], max_filesize=124)
@pytest.mark.parametrize('backend_type, expected_instance, extra_conf', [
(config_keys.backend_legacy_filesystem, LegacyFileSystemBackend, {}),
(config_keys.backend_filesystem, FileSystemBackend, {config_keys.filesystem_storage_path: '/tmp/test-fs-store'}),
(config_keys.backend_objectstore, ObjectStoreBackend, {config_keys.objectstore_bucket: 'test-bucket'}),
])
def test_store_stats_and_keys(self, backend_type, expected_instance, extra_conf, ini_settings, random_binary_file):
config = ini_settings
config[config_keys.backend_type] = backend_type
config.update(extra_conf)
f_store = store_utils.get_filestore_backend(config=config, always_init=True)
# purge storage before running
for shard, k in f_store.iter_artifacts():
f_store.delete(k)
for i in range(10):
filename, temp_file = random_binary_file
metadata = {
'user_uploaded': {
'username': 'user1',
'user_id': 10,
'ip': '10.20.30.40'
}
}
store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata)
assert store_fid
assert metadata
cnt, size, meta = f_store.get_statistics()
assert cnt == 10
assert 10 == len(list(f_store.iter_keys()))

52
rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py Normal file
View file

@ -0,0 +1,52 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import pytest
from rhodecode.apps.file_store import utils as store_utils
from rhodecode.apps.file_store import config_keys
from rhodecode.apps.file_store.tests import generate_random_filename
@pytest.fixture()
def file_store_filesystem_instance(ini_settings):
config = ini_settings
config[config_keys.backend_type] = config_keys.backend_filesystem
f_store = store_utils.get_filestore_backend(config=config, always_init=True)
return f_store
class TestFileStoreFileSystemBackend:
@pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
def test_get_shard_number(self, filename, file_store_filesystem_instance):
shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards))
# Check that the shard number is between 0 and max-shards
assert 0 <= shard_number <= len(file_store_filesystem_instance._shards)
@pytest.mark.parametrize('filename, expected_shard_num', [
('my-name-1', 3),
('my-name-2', 2),
('my-name-3', 4),
('my-name-4', 1),
('rhodecode-enterprise-ce', 5),
('rhodecode-enterprise-ee', 6),
])
def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_filesystem_instance):
shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards))
assert expected_shard_num == shard_number

17
rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py Normal file
View file

@ -0,0 +1,17 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/

52
rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py Normal file
View file

@ -0,0 +1,52 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import pytest
from rhodecode.apps.file_store import utils as store_utils
from rhodecode.apps.file_store import config_keys
from rhodecode.apps.file_store.tests import generate_random_filename
@pytest.fixture()
def file_store_legacy_instance(ini_settings):
config = ini_settings
config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
f_store = store_utils.get_filestore_backend(config=config, always_init=True)
return f_store
class TestFileStoreLegacyBackend:
@pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
def test_get_shard_number(self, filename, file_store_legacy_instance):
shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards))
# Check that the shard number is 0 for legacy filesystem store we don't use shards
assert shard_number == 0
@pytest.mark.parametrize('filename, expected_shard_num', [
('my-name-1', 0),
('my-name-2', 0),
('my-name-3', 0),
('my-name-4', 0),
('rhodecode-enterprise-ce', 0),
('rhodecode-enterprise-ee', 0),
])
def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_legacy_instance):
shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards))
assert expected_shard_num == shard_number

52
rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py Normal file
View file

@ -0,0 +1,52 @@
# Copyright (C) 2010-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import pytest
from rhodecode.apps.file_store import utils as store_utils
from rhodecode.apps.file_store import config_keys
from rhodecode.apps.file_store.tests import generate_random_filename
@pytest.fixture()
def file_store_objectstore_instance(ini_settings):
config = ini_settings
config[config_keys.backend_type] = config_keys.backend_objectstore
f_store = store_utils.get_filestore_backend(config=config, always_init=True)
return f_store
class TestFileStoreObjectStoreBackend:
@pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
def test_get_shard_number(self, filename, file_store_objectstore_instance):
shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards))
# Check that the shard number is between 0 and shards
assert 0 <= shard_number <= len(file_store_objectstore_instance._shards)
@pytest.mark.parametrize('filename, expected_shard_num', [
('my-name-1', 3),
('my-name-2', 2),
('my-name-3', 4),
('my-name-4', 1),
('rhodecode-enterprise-ce', 5),
('rhodecode-enterprise-ee', 6),
])
def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_objectstore_instance):
shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards))
assert expected_shard_num == shard_number

83
rhodecode/apps/file_store/tests/test_upload_file.py
View file

@ -15,13 +15,16 @@
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import pytest
from rhodecode.lib.ext_json import json
from rhodecode.model.auth_token import AuthTokenModel
from rhodecode.model.db import Session, FileStore, Repository, User
from rhodecode.apps.file_store import utils, config_keys
from rhodecode.apps.file_store import utils as store_utils
from rhodecode.apps.file_store import config_keys
from rhodecode.tests import TestController
from rhodecode.tests.routes import route_path
@ -29,27 +32,61 @@ from rhodecode.tests.routes import route_path
class TestFileStoreViews(TestController):
@pytest.fixture()
def create_artifact_factory(self, tmpdir, ini_settings):
def factory(user_id, content, f_name='example.txt'):
config = ini_settings
config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
f_store = store_utils.get_filestore_backend(config)
filesystem_file = os.path.join(str(tmpdir), f_name)
with open(filesystem_file, 'wt') as f:
f.write(content)
with open(filesystem_file, 'rb') as f:
store_uid, metadata = f_store.store(f_name, f, metadata={'filename': f_name})
os.remove(filesystem_file)
entry = FileStore.create(
file_uid=store_uid, filename=metadata["filename"],
file_hash=metadata["sha256"], file_size=metadata["size"],
file_display_name='file_display_name',
file_description='repo artifact `{}`'.format(metadata["filename"]),
check_acl=True, user_id=user_id,
)
Session().add(entry)
Session().commit()
return entry
return factory
@pytest.mark.parametrize("fid, content, exists", [
('abcde-0.jpg', "xxxxx", True),
('abcde-0.exe', "1234567", True),
('abcde-0.jpg', "xxxxx", False),
])
def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util):
def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util, ini_settings):
user = self.log_user()
user_id = user['user_id']
repo_id = user_util.create_repo().repo_id
store_path = self.app._pyramid_settings[config_keys.store_path]
config = ini_settings
config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
store_uid = fid
if exists:
status = 200
store = utils.get_file_storage({config_keys.store_path: store_path})
f_store = store_utils.get_filestore_backend(config)
filesystem_file = os.path.join(str(tmpdir), fid)
with open(filesystem_file, 'wt') as f:
f.write(content)
with open(filesystem_file, 'rb') as f:
store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
store_uid, metadata = f_store.store(fid, f, metadata={'filename': fid})
os.remove(filesystem_file)
entry = FileStore.create(
file_uid=store_uid, filename=metadata["filename"],
@ -69,14 +106,10 @@ class TestFileStoreViews(TestController):
if exists:
assert response.text == content
file_store_path = os.path.dirname(store.resolve_name(store_uid, store_path)[1])
metadata_file = os.path.join(file_store_path, store_uid + '.meta')
assert os.path.exists(metadata_file)
with open(metadata_file, 'rb') as f:
json_data = json.loads(f.read())
assert json_data
assert 'size' in json_data
metadata = f_store.get_metadata(store_uid)
assert 'size' in metadata
def test_upload_files_without_content_to_store(self):
self.log_user()
@ -112,32 +145,6 @@ class TestFileStoreViews(TestController):
assert response.json['store_fid']
@pytest.fixture()
def create_artifact_factory(self, tmpdir):
def factory(user_id, content):
store_path = self.app._pyramid_settings[config_keys.store_path]
store = utils.get_file_storage({config_keys.store_path: store_path})
fid = 'example.txt'
filesystem_file = os.path.join(str(tmpdir), fid)
with open(filesystem_file, 'wt') as f:
f.write(content)
with open(filesystem_file, 'rb') as f:
store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
entry = FileStore.create(
file_uid=store_uid, filename=metadata["filename"],
file_hash=metadata["sha256"], file_size=metadata["size"],
file_display_name='file_display_name',
file_description='repo artifact `{}`'.format(metadata["filename"]),
check_acl=True, user_id=user_id,
)
Session().add(entry)
Session().commit()
return entry
return factory
def test_download_file_non_scoped(self, user_util, create_artifact_factory):
user = self.log_user()
user_id = user['user_id']

108
rhodecode/apps/file_store/utils.py
View file

@ -19,21 +19,84 @@
import io
import uuid
import pathlib
import s3fs
from rhodecode.lib.hash_utils import sha256_safe
from rhodecode.apps.file_store import config_keys
def get_file_storage(settings):
from rhodecode.apps.file_store.backends.local_store import LocalFileStorage
from rhodecode.apps.file_store import config_keys
store_path = settings.get(config_keys.store_path)
return LocalFileStorage(base_path=store_path)
file_store_meta = None
def get_filestore_config(config) -> dict:
final_config = {}
for k, v in config.items():
if k.startswith('file_store'):
final_config[k] = v
return final_config
def get_filestore_backend(config, always_init=False):
"""
usage::
from rhodecode.apps.file_store import get_filestore_backend
f_store = get_filestore_backend(config=CONFIG)
:param config:
:param always_init:
:return:
"""
global file_store_meta
if file_store_meta is not None and not always_init:
return file_store_meta
config = get_filestore_config(config)
backend = config[config_keys.backend_type]
match backend:
case config_keys.backend_legacy_filesystem:
# Legacy backward compatible storage
from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend
d_cache = LegacyFileSystemBackend(
settings=config
)
case config_keys.backend_filesystem:
from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend
d_cache = FileSystemBackend(
settings=config
)
case config_keys.backend_objectstore:
from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend
d_cache = ObjectStoreBackend(
settings=config
)
case _:
raise ValueError(
f'file_store.backend.type only supports "{config_keys.backend_types}" got {backend}'
)
cache_meta = d_cache
return cache_meta
def splitext(filename):
ext = ''.join(pathlib.Path(filename).suffixes)
final_ext = []
for suffix in pathlib.Path(filename).suffixes:
if not suffix.isascii():
continue
suffix = " ".join(suffix.split()).replace(" ", "")
final_ext.append(suffix)
ext = ''.join(final_ext)
return filename, ext
def uid_filename(filename, randomized=True):
def get_uid_filename(filename, randomized=True):
"""
Generates a randomized or stable (uuid) filename,
preserving the original extension.
@ -46,10 +109,37 @@ def uid_filename(filename, randomized=True):
if randomized:
uid = uuid.uuid4()
else:
hash_key = '{}.{}'.format(filename, 'store')
store_suffix = "store"
hash_key = f'{filename}.{store_suffix}'
uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
return str(uid) + ext.lower()
def bytes_to_file_obj(bytes_data):
return io.StringIO(bytes_data)
return io.BytesIO(bytes_data)
class ShardFileReader:
def __init__(self, file_like_reader):
self._file_like_reader = file_like_reader
def __getattr__(self, item):
if isinstance(self._file_like_reader, s3fs.core.S3File):
match item:
case 'name':
# S3 FileWrapper doesn't support name attribute, and we use it
return self._file_like_reader.full_name
case _:
return getattr(self._file_like_reader, item)
else:
return getattr(self._file_like_reader, item)
def archive_iterator(_reader, block_size: int = 4096 * 512):
# 4096 * 64 = 64KB
while 1:
data = _reader.read(block_size)
if not data:
break
yield data

41
rhodecode/apps/file_store/views.py
View file

@ -17,12 +17,11 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
from pyramid.response import FileResponse
from pyramid.response import Response
from pyramid.httpexceptions import HTTPFound, HTTPNotFound
from rhodecode.apps._base import BaseAppView
from rhodecode.apps.file_store import utils
from rhodecode.apps.file_store import utils as store_utils
from rhodecode.apps.file_store.exceptions import (
FileNotAllowedException, FileOverSizeException)
@ -31,6 +30,7 @@ from rhodecode.lib import audit_logger
from rhodecode.lib.auth import (
CSRFRequired, NotAnonymous, HasRepoPermissionAny, HasRepoGroupPermissionAny,
LoginRequired)
from rhodecode.lib.str_utils import header_safe_str
from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db
from rhodecode.model.db import Session, FileStore, UserApiKeys
@ -42,7 +42,7 @@ class FileStoreView(BaseAppView):
def load_default_context(self):
c = self._get_local_tmpl_context()
self.storage = utils.get_file_storage(self.request.registry.settings)
self.f_store = store_utils.get_filestore_backend(self.request.registry.settings)
return c
def _guess_type(self, file_name):
@ -55,10 +55,10 @@ class FileStoreView(BaseAppView):
return _content_type, _encoding
def _serve_file(self, file_uid):
if not self.storage.exists(file_uid):
store_path = self.storage.store_path(file_uid)
log.debug('File with FID:%s not found in the store under `%s`',
file_uid, store_path)
if not self.f_store.filename_exists(file_uid):
store_path = self.f_store.store_path(file_uid)
log.warning('File with FID:%s not found in the store under `%s`',
file_uid, store_path)
raise HTTPNotFound()
db_obj = FileStore.get_by_store_uid(file_uid, safe=True)
@ -98,28 +98,25 @@ class FileStoreView(BaseAppView):
FileStore.bump_access_counter(file_uid)
file_path = self.storage.store_path(file_uid)
file_name = db_obj.file_display_name
content_type = 'application/octet-stream'
content_encoding = None
_content_type, _encoding = self._guess_type(file_path)
_content_type, _encoding = self._guess_type(file_name)
if _content_type:
content_type = _content_type
# For file store we don't submit any session data, this logic tells the
# Session lib to skip it
setattr(self.request, '_file_response', True)
response = FileResponse(
file_path, request=self.request,
content_type=content_type, content_encoding=content_encoding)
reader, _meta = self.f_store.fetch(file_uid)
file_name = db_obj.file_display_name
response = Response(app_iter=store_utils.archive_iterator(reader))
response.content_type = str(content_type)
response.content_disposition = f'attachment; filename="{header_safe_str(file_name)}"'
response.headers["Content-Disposition"] = (
f'attachment; filename="{str(file_name)}"'
)
response.headers["X-RC-Artifact-Id"] = str(db_obj.file_store_id)
response.headers["X-RC-Artifact-Desc"] = str(db_obj.file_description)
response.headers["X-RC-Artifact-Desc"] = header_safe_str(db_obj.file_description)
response.headers["X-RC-Artifact-Sha256"] = str(db_obj.file_hash)
return response
@ -147,8 +144,8 @@ class FileStoreView(BaseAppView):
'user_id': self._rhodecode_user.user_id,
'ip': self._rhodecode_user.ip_addr}}
try:
store_uid, metadata = self.storage.save_file(
file_obj.file, filename, extra_metadata=metadata)
store_uid, metadata = self.f_store.store(
filename, file_obj.file, extra_metadata=metadata)
except FileNotAllowedException:
return {'store_fid': None,
'access_path': None,
@ -182,7 +179,7 @@ class FileStoreView(BaseAppView):
def download_file(self):
self.load_default_context()
file_uid = self.request.matchdict['fid']
log.debug('Requesting FID:%s from store %s', file_uid, self.storage)
log.debug('Requesting FID:%s from store %s', file_uid, self.f_store)
return self._serve_file(file_uid)
# in addition to @LoginRequired ACL is checked by scopes

16
rhodecode/apps/repository/views/repo_commits.py
View file

@ -601,26 +601,26 @@ class RepoCommitsView(RepoAppView):
max_file_size = 10 * 1024 * 1024 # 10MB, also validated via dropzone.js
try:
storage = store_utils.get_file_storage(self.request.registry.settings)
store_uid, metadata = storage.save_file(
file_obj.file, filename, extra_metadata=metadata,
f_store = store_utils.get_filestore_backend(self.request.registry.settings)
store_uid, metadata = f_store.store(
filename, file_obj.file, metadata=metadata,
extensions=allowed_extensions, max_filesize=max_file_size)
except FileNotAllowedException:
self.request.response.status = 400
permitted_extensions = ', '.join(allowed_extensions)
error_msg = 'File `{}` is not allowed. ' \
'Only following extensions are permitted: {}'.format(
filename, permitted_extensions)
error_msg = f'File `{filename}` is not allowed. ' \
f'Only following extensions are permitted: {permitted_extensions}'
return {'store_fid': None,
'access_path': None,
'error': error_msg}
except FileOverSizeException:
self.request.response.status = 400
limit_mb = h.format_byte_size_binary(max_file_size)
error_msg = f'File {filename} is exceeding allowed limit of {limit_mb}.'
return {'store_fid': None,
'access_path': None,
'error': 'File {} is exceeding allowed limit of {}.'.format(
filename, limit_mb)}
'error': error_msg}
try:
entry = FileStore.create(

4
rhodecode/apps/repository/views/repo_files.py
View file

@ -48,7 +48,7 @@ from rhodecode.lib.codeblocks import (
filenode_as_lines_tokens, filenode_as_annotated_lines_tokens)
from rhodecode.lib.utils2 import convert_line_endings, detect_mode
from rhodecode.lib.type_utils import str2bool
from rhodecode.lib.str_utils import safe_str, safe_int
from rhodecode.lib.str_utils import safe_str, safe_int, header_safe_str
from rhodecode.lib.auth import (
LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
from rhodecode.lib.vcs import path as vcspath
@ -820,7 +820,7 @@ class RepoFilesView(RepoAppView):
"filename=\"{}\"; " \
"filename*=UTF-8\'\'{}".format(safe_path, encoded_path)
return safe_bytes(headers).decode('latin-1', errors='replace')
return header_safe_str(headers)
@LoginRequired()
@HasRepoPermissionAnyDecorator(

21
rhodecode/apps/repository/views/repo_settings_advanced.py
View file

@ -29,7 +29,7 @@ from rhodecode.lib import audit_logger
from rhodecode.lib.auth import (
LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired,
HasRepoPermissionAny)
from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError
from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError
from rhodecode.lib.utils2 import safe_int
from rhodecode.lib.vcs import RepositoryError
from rhodecode.model.db import Session, UserFollowing, User, Repository
@ -136,6 +136,9 @@ class RepoSettingsAdvancedView(RepoAppView):
elif handle_forks == 'delete_forks':
handle_forks = 'delete'
repo_advanced_url = h.route_path(
'edit_repo_advanced', repo_name=self.db_repo_name,
_anchor='advanced-delete')
try:
old_data = self.db_repo.get_api_data()
RepoModel().delete(self.db_repo, forks=handle_forks)
@ -158,9 +161,6 @@ class RepoSettingsAdvancedView(RepoAppView):
category='success')
Session().commit()
except AttachedForksError:
repo_advanced_url = h.route_path(
'edit_repo_advanced', repo_name=self.db_repo_name,
_anchor='advanced-delete')
delete_anchor = h.link_to(_('detach or delete'), repo_advanced_url)
h.flash(_('Cannot delete `{repo}` it still contains attached forks. '
'Try using {delete_or_detach} option.')
@ -171,9 +171,6 @@ class RepoSettingsAdvancedView(RepoAppView):
raise HTTPFound(repo_advanced_url)
except AttachedPullRequestsError:
repo_advanced_url = h.route_path(
'edit_repo_advanced', repo_name=self.db_repo_name,
_anchor='advanced-delete')
attached_prs = len(self.db_repo.pull_requests_source +
self.db_repo.pull_requests_target)
h.flash(
@ -184,6 +181,16 @@ class RepoSettingsAdvancedView(RepoAppView):
# redirect to advanced for forks handle action ?
raise HTTPFound(repo_advanced_url)
except AttachedArtifactsError:
attached_artifacts = len(self.db_repo.artifacts)
h.flash(
_('Cannot delete `{repo}` it still contains {num} attached artifacts. '
'Consider archiving the repository instead.').format(
repo=self.db_repo_name, num=attached_artifacts), category='warning')
# redirect to advanced for forks handle action ?
raise HTTPFound(repo_advanced_url)
except Exception:
log.exception("Exception during deletion of repository")
h.flash(_('An error occurred during deletion of `%s`')

2
rhodecode/apps/ssh_support/__init__.py
View file

@ -37,7 +37,7 @@ def _sanitize_settings_and_apply_defaults(settings):
settings_maker.make_setting(config_keys.ssh_key_generator_enabled, True, parser='bool')
settings_maker.make_setting(config_keys.authorized_keys_file_path, '~/.ssh/authorized_keys_rhodecode')
settings_maker.make_setting(config_keys.wrapper_cmd, '')
settings_maker.make_setting(config_keys.wrapper_cmd, '/usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2')
settings_maker.make_setting(config_keys.authorized_keys_line_ssh_opts, '')
settings_maker.make_setting(config_keys.ssh_hg_bin, '/usr/local/bin/rhodecode_bin/vcs_bin/hg')

2
rhodecode/apps/ssh_support/config_keys.py
View file

@ -23,7 +23,7 @@ generate_authorized_keyfile = 'ssh.generate_authorized_keyfile'
authorized_keys_file_path = 'ssh.authorized_keys_file_path'
authorized_keys_line_ssh_opts = 'ssh.authorized_keys_ssh_opts'
ssh_key_generator_enabled = 'ssh.enable_ui_key_generator'
wrapper_cmd = 'ssh.wrapper_cmd'
wrapper_cmd = 'ssh.wrapper_cmd.v2'
wrapper_allow_shell = 'ssh.wrapper_cmd_allow_shell'
enable_debug_logging = 'ssh.enable_debug_logging'

2
rhodecode/apps/ssh_support/lib/backends/base.py
View file

@ -157,7 +157,7 @@ class SshVcsServer(object):
return exit_code, action == "push"
def run(self, tunnel_extras=None):
self.hooks_protocol = self.settings['vcs.hooks.protocol']
self.hooks_protocol = self.settings['vcs.hooks.protocol.v2']
tunnel_extras = tunnel_extras or {}
extras = {}
extras.update(tunnel_extras)

2
rhodecode/apps/ssh_support/tests/test_server_git.py
View file

@ -32,7 +32,7 @@ class GitServerCreator(object):
config_data = {
'app:main': {
'ssh.executable.git': git_path,
'vcs.hooks.protocol': 'http',
'vcs.hooks.protocol.v2': 'celery',
}
}
repo_name = 'test_git'

2
rhodecode/apps/ssh_support/tests/test_server_hg.py
View file

@ -31,7 +31,7 @@ class MercurialServerCreator(object):
config_data = {
'app:main': {
'ssh.executable.hg': hg_path,
'vcs.hooks.protocol': 'http',
'vcs.hooks.protocol.v2': 'celery',
}
}
repo_name = 'test_hg'

2
rhodecode/apps/ssh_support/tests/test_server_svn.py
View file

@ -29,7 +29,7 @@ class SubversionServerCreator(object):
config_data = {
'app:main': {
'ssh.executable.svn': svn_path,
'vcs.hooks.protocol': 'http',
'vcs.hooks.protocol.v2': 'celery',
}
}
repo_name = 'test-svn'

1
rhodecode/authentication/routes.py
View file

@ -52,6 +52,7 @@ class AuthnRootResource(AuthnResourceBase):
"""
This is the root traversal resource object for the authentication settings.
"""
is_root = True
def __init__(self):
self._store = collections.OrderedDict()

19
rhodecode/config/config_maker.py
View file

@ -52,7 +52,8 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
default=False,
parser='bool')
logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
ini_loc = os.path.dirname(global_config.get('__file__'))
logging_conf = jn(ini_loc, 'logging.ini')
settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
# Default includes, possible to change as a user
@ -95,6 +96,11 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
settings_maker.make_setting('gzip_responses', False, parser='bool')
settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
# License settings.
settings_maker.make_setting('license.hide_license_info', False, parser='bool')
settings_maker.make_setting('license.import_path', '')
settings_maker.make_setting('license.import_path_mode', 'if-missing')
# statsd
settings_maker.make_setting('statsd.enabled', False, parser='bool')
settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
@ -106,7 +112,7 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
settings_maker.make_setting('vcs.svn.redis_conn', 'redis://redis:6379/0')
settings_maker.make_setting('vcs.svn.proxy.enabled', True, parser='bool')
settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090', parser='string')
settings_maker.make_setting('vcs.hooks.protocol', 'http')
settings_maker.make_setting('vcs.hooks.protocol.v2', 'celery')
settings_maker.make_setting('vcs.hooks.host', '*')
settings_maker.make_setting('vcs.scm_app_implementation', 'http')
settings_maker.make_setting('vcs.server', '')
@ -116,6 +122,9 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
settings_maker.make_setting('vcs.git.lfs.storage_location', '/var/opt/rhodecode_repo_store/.cache/git_lfs_store')
settings_maker.make_setting('vcs.hg.largefiles.storage_location',
'/var/opt/rhodecode_repo_store/.cache/hg_largefiles_store')
settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
@ -152,6 +161,10 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
parser='file:ensured'
)
# celery
broker_url = settings_maker.make_setting('celery.broker_url', 'redis://redis:6379/8')
settings_maker.make_setting('celery.result_backend', broker_url)
settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
@ -202,7 +215,7 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
settings_maker.make_setting('archive_cache.filesystem.retry_backoff', 1, parser='int')
settings_maker.make_setting('archive_cache.filesystem.retry_attempts', 10, parser='int')
settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
settings_maker.make_setting('archive_cache.objectstore.url', 'http://s3-minio:9000', default_when_empty=True,)
settings_maker.make_setting('archive_cache.objectstore.key', '')
settings_maker.make_setting('archive_cache.objectstore.secret', '')
settings_maker.make_setting('archive_cache.objectstore.region', 'eu-central-1')

24
rhodecode/config/environment.py
View file

@ -16,8 +16,8 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import logging
import rhodecode
import collections
@ -30,6 +30,21 @@ from rhodecode.lib.vcs import connect_vcs
log = logging.getLogger(__name__)
def propagate_rhodecode_config(global_config, settings, config):
# Store the settings to make them available to other modules.
settings_merged = global_config.copy()
settings_merged.update(settings)
if config:
settings_merged.update(config)
rhodecode.PYRAMID_SETTINGS = settings_merged
rhodecode.CONFIG = settings_merged
if 'default_user_id' not in rhodecode.CONFIG:
rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
log.debug('set rhodecode.CONFIG data')
def load_pyramid_environment(global_config, settings):
# Some parts of the code expect a merge of global and app settings.
settings_merged = global_config.copy()
@ -75,11 +90,8 @@ def load_pyramid_environment(global_config, settings):
utils.configure_vcs(settings)
# Store the settings to make them available to other modules.
rhodecode.PYRAMID_SETTINGS = settings_merged
rhodecode.CONFIG = settings_merged
rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
# first run, to store data...
propagate_rhodecode_config(global_config, settings, {})
if vcs_server_enabled:
connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))

11
rhodecode/config/middleware.py
View file

@ -35,7 +35,7 @@ from pyramid.renderers import render_to_response
from rhodecode.model import meta
from rhodecode.config import patches
from rhodecode.config.environment import load_pyramid_environment
from rhodecode.config.environment import load_pyramid_environment, propagate_rhodecode_config
import rhodecode.events
from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
@ -50,7 +50,7 @@ from rhodecode.lib.utils2 import AttributeDict
from rhodecode.lib.exc_tracking import store_exception, format_exc
from rhodecode.subscribers import (
scan_repositories_if_enabled, write_js_routes_if_enabled,
write_metadata_if_needed, write_usage_data)
write_metadata_if_needed, write_usage_data, import_license_if_present)
from rhodecode.lib.statsd_client import StatsdClient
log = logging.getLogger(__name__)
@ -99,6 +99,7 @@ def make_pyramid_app(global_config, **settings):
# Apply compatibility patches
patches.inspect_getargspec()
patches.repoze_sendmail_lf_fix()
load_pyramid_environment(global_config, settings)
@ -114,6 +115,9 @@ def make_pyramid_app(global_config, **settings):
celery_settings = get_celery_config(settings)
config.configure_celery(celery_settings)
# final config set...
propagate_rhodecode_config(global_config, settings, config.registry.settings)
# creating the app uses a connection - return it after we are done
meta.Session.remove()
@ -396,7 +400,8 @@ def includeme(config, auth_resources=None):
pyramid.events.ApplicationCreated)
config.add_subscriber(write_js_routes_if_enabled,
pyramid.events.ApplicationCreated)
config.add_subscriber(import_license_if_present,
pyramid.events.ApplicationCreated)
# Set the default renderer for HTML templates to mako.
config.add_mako_renderer('.html')

7
rhodecode/config/patches.py
View file

@ -158,3 +158,10 @@ def inspect_getargspec():
inspect.getargspec = inspect.getfullargspec
return inspect
def repoze_sendmail_lf_fix():
from repoze.sendmail import encoding
from email.policy import SMTP
encoding.encode_message = lambda message, *args, **kwargs: message.as_bytes(policy=SMTP)

2
rhodecode/config/utils.py
View file

@ -35,7 +35,7 @@ def configure_vcs(config):
'svn': 'rhodecode.lib.vcs.backends.svn.SubversionRepository',
}
conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol']
conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol.v2']
conf.settings.HOOKS_HOST = config['vcs.hooks.host']
conf.settings.DEFAULT_ENCODINGS = config['default_encoding']
conf.settings.ALIASES[:] = config['vcs.backends']

6
rhodecode/lib/archive_cache/__init__.py
View file

@ -31,9 +31,11 @@ cache_meta = None
def includeme(config):
return # don't init cache currently for faster startup time
# init our cache at start
settings = config.get_settings()
get_archival_cache_store(settings)
# settings = config.get_settings()
# get_archival_cache_store(settings)
def get_archival_config(config):

2
rhodecode/lib/archive_cache/backends/objectstore_cache.py
View file

@ -58,7 +58,7 @@ class S3Shard(BaseShard):
# ensure folder in bucket exists
destination = self.bucket
if not self.fs.exists(destination):
self.fs.mkdir(destination, s3_additional_kwargs={})
self.fs.mkdir(destination)
writer = self._get_writer(full_path, mode)

7
rhodecode/lib/celerylib/loader.py
View file

@ -27,10 +27,11 @@ Celery loader, run with::
--scheduler rhodecode.lib.celerylib.scheduler.RcScheduler \
--loglevel DEBUG --ini=.dev/dev.ini
"""
from rhodecode.config.patches import inspect_getargspec, inspect_formatargspec
inspect_getargspec()
inspect_formatargspec()
from rhodecode.config import patches
patches.inspect_getargspec()
patches.inspect_formatargspec()
# python3.11 inspect patches for backward compat on `paste` code
patches.repoze_sendmail_lf_fix()
import sys
import logging

17
rhodecode/lib/celerylib/tasks.py
View file

@ -1,4 +1,4 @@
# Copyright (C) 2012-2023 RhodeCode GmbH
# Copyright (C) 2012-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
@ -64,8 +64,9 @@ def send_email(recipients, subject, body='', html_body='', email_config=None,
"Make sure that `smtp_server` variable is configured "
"inside the .ini file")
return False
subject = "%s %s" % (email_config.get('email_prefix', ''), subject)
conf_prefix = email_config.get('email_prefix', None)
prefix = f'{conf_prefix} ' if conf_prefix else ''
subject = f"{prefix}{subject}"
if recipients:
if isinstance(recipients, str):
@ -86,8 +87,8 @@ def send_email(recipients, subject, body='', html_body='', email_config=None,
email_conf = dict(
host=mail_server,
port=email_config.get('smtp_port', 25),
username=email_config.get('smtp_username'),
password=email_config.get('smtp_password'),
username=email_config.get('smtp_username', None),
password=email_config.get('smtp_password', None),
tls=str2bool(email_config.get('smtp_use_tls')),
ssl=str2bool(email_config.get('smtp_use_ssl')),
@ -207,7 +208,7 @@ def create_repo(form_data, cur_user):
hooks_base.create_repository(created_by=owner.username, **repo.get_dict())
# update repo commit caches initially
repo.update_commit_cache()
repo.update_commit_cache(recursive=False)
# set new created state
repo.set_state(Repository.STATE_CREATED)
@ -298,7 +299,7 @@ def create_repo_fork(form_data, cur_user):
# update repo commit caches initially
config = repo._config
config.set('extensions', 'largefiles', '')
repo.update_commit_cache(config=config)
repo.update_commit_cache(config=config, recursive=False)
# set new created state
repo.set_state(Repository.STATE_CREATED)
@ -390,7 +391,7 @@ def sync_last_update_for_objects(*args, **kwargs):
.order_by(Repository.group_id.asc())
for repo in repos:
repo.update_commit_cache()
repo.update_commit_cache(recursive=False)
skip_groups = kwargs.get('skip_groups')
if not skip_groups:

1
rhodecode/lib/db_manage.py
View file

@ -570,7 +570,6 @@ class DbManage(object):
self.create_ui_settings(path)
ui_config = [
('web', 'push_ssl', 'False'),
('web', 'allow_archive', 'gz zip bz2'),
('web', 'allow_push', '*'),
('web', 'baseurl', '/'),

11
rhodecode/lib/dbmigrate/versions/115_version_5_1_0.py
View file

@ -35,16 +35,19 @@ def downgrade(migrate_engine):
def fixups(models, _SESSION):
for db_repo in _SESSION.query(models.Repository).all():
config = db_repo._config
config.set('extensions', 'largefiles', '')
try:
scm = db_repo.scm_instance(cache=False, config=config)
config = db_repo._config
config.set('extensions', 'largefiles', '')
scm = db_repo.scm_instance(cache=False, config=config, vcs_full_cache=False)
if scm:
print(f'installing hook for repo: {db_repo}')
scm.install_hooks(force=True)
del scm # force GC
del config
except Exception as e:
print(e)
print('continue...')

9
rhodecode/lib/exceptions.py
View file

@ -80,6 +80,10 @@ class AttachedPullRequestsError(Exception):
pass
class AttachedArtifactsError(Exception):
pass
class RepoGroupAssignmentError(Exception):
pass
@ -98,6 +102,11 @@ class HTTPRequirementError(HTTPClientError):
self.args = (message, )
class ClientNotSupportedError(HTTPRequirementError):
title = explanation = 'Client Not Supported'
reason = None
class HTTPLockedRC(HTTPClientError):
"""
Special Exception For locked Repos in RhodeCode, the return code can

54
rhodecode/lib/helpers.py
View file

@ -81,7 +81,7 @@ from rhodecode.lib.action_parser import action_parser
from rhodecode.lib.html_filters import sanitize_html
from rhodecode.lib.pagination import Page, RepoPage, SqlPage
from rhodecode.lib import ext_json
from rhodecode.lib.ext_json import json
from rhodecode.lib.ext_json import json, formatted_str_json
from rhodecode.lib.str_utils import safe_bytes, convert_special_chars, base64_to_str
from rhodecode.lib.utils import repo_name_slug, get_custom_lexer
from rhodecode.lib.str_utils import safe_str
@ -1416,62 +1416,14 @@ class InitialsGravatar(object):
return "data:image/svg+xml;base64,{}".format(img_data)
def initials_gravatar(request, email_address, first_name, last_name, size=30, store_on_disk=False):
def initials_gravatar(request, email_address, first_name, last_name, size=30):
svg_type = None
if email_address == User.DEFAULT_USER_EMAIL:
svg_type = 'default_user'
klass = InitialsGravatar(email_address, first_name, last_name, size)
if store_on_disk:
from rhodecode.apps.file_store import utils as store_utils
from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
FileOverSizeException
from rhodecode.model.db import Session
image_key = md5_safe(email_address.lower()
+ first_name.lower() + last_name.lower())
storage = store_utils.get_file_storage(request.registry.settings)
filename = '{}.svg'.format(image_key)
subdir = 'gravatars'
# since final name has a counter, we apply the 0
uid = storage.apply_counter(0, store_utils.uid_filename(filename, randomized=False))
store_uid = os.path.join(subdir, uid)
db_entry = FileStore.get_by_store_uid(store_uid)
if db_entry:
return request.route_path('download_file', fid=store_uid)
img_data = klass.get_img_data(svg_type=svg_type)
img_file = store_utils.bytes_to_file_obj(img_data)
try:
store_uid, metadata = storage.save_file(
img_file, filename, directory=subdir,
extensions=['.svg'], randomized_name=False)
except (FileNotAllowedException, FileOverSizeException):
raise
try:
entry = FileStore.create(
file_uid=store_uid, filename=metadata["filename"],
file_hash=metadata["sha256"], file_size=metadata["size"],
file_display_name=filename,
file_description=f'user gravatar `{safe_str(filename)}`',
hidden=True, check_acl=False, user_id=1
)
Session().add(entry)
Session().commit()
log.debug('Stored upload in DB as %s', entry)
except Exception:
raise
return request.route_path('download_file', fid=store_uid)
else:
return klass.generate_svg(svg_type=svg_type)
return klass.generate_svg(svg_type=svg_type)
def gravatar_external(request, gravatar_url_tmpl, email_address, size=30):

6
rhodecode/lib/hook_daemon/hook_module.py
View file

@ -66,12 +66,12 @@ class Hooks(object):
result = hook(extras)
if result is None:
raise Exception(f'Failed to obtain hook result from func: {hook}')
except HTTPBranchProtected as handled_error:
except HTTPBranchProtected as error:
# Those special cases don't need error reporting. It's a case of
# locked repo or protected branch
result = AttributeDict({
'status': handled_error.code,
'output': handled_error.explanation
'status': error.code,
'output': error.explanation
})
except (HTTPLockedRC, Exception) as error:
# locked needs different handling since we need to also

18
rhodecode/lib/hooks_base.py
View file

@ -30,7 +30,7 @@ from rhodecode.lib import helpers as h
from rhodecode.lib import audit_logger
from rhodecode.lib.utils2 import safe_str, user_agent_normalizer
from rhodecode.lib.exceptions import (
HTTPLockedRC, HTTPBranchProtected, UserCreationError)
HTTPLockedRC, HTTPBranchProtected, UserCreationError, ClientNotSupportedError)
from rhodecode.model.db import Repository, User
from rhodecode.lib.statsd_client import StatsdClient
@ -64,6 +64,18 @@ def is_shadow_repo(extras):
return extras['is_shadow_repo']
def check_vcs_client(extras):
"""
Checks if vcs client is allowed (Only works in enterprise edition)
"""
try:
from rc_ee.lib.security.utils import is_vcs_client_whitelisted
except ModuleNotFoundError:
is_vcs_client_whitelisted = lambda *x: True
backend = extras.get('scm')
if not is_vcs_client_whitelisted(extras.get('user_agent'), backend):
raise ClientNotSupportedError(f"Your {backend} client is forbidden")
def _get_scm_size(alias, root_path):
if not alias.startswith('.'):
@ -108,6 +120,7 @@ def pre_push(extras):
It bans pushing when the repository is locked.
"""
check_vcs_client(extras)
user = User.get_by_username(extras.username)
output = ''
if extras.locked_by[0] and user.user_id != int(extras.locked_by[0]):
@ -129,6 +142,8 @@ def pre_push(extras):
if extras.commit_ids and extras.check_branch_perms:
auth_user = user.AuthUser()
repo = Repository.get_by_repo_name(extras.repository)
if not repo:
raise ValueError(f'Repo for {extras.repository} not found')
affected_branches = []
if repo.repo_type == 'hg':
for entry in extras.commit_ids:
@ -180,6 +195,7 @@ def pre_pull(extras):
It bans pulling when the repository is locked.
"""
check_vcs_client(extras)
output = ''
if extras.locked_by[0]:
locked_by = User.get(extras.locked_by[0]).username

4
rhodecode/lib/middleware/request_wrapper.py
View file

@ -46,7 +46,7 @@ class RequestWrapperTween(object):
def __call__(self, request):
start = time.time()
log.debug('Starting request time measurement')
log.debug('Starting request processing')
response = None
request.req_wrapper_start = start
@ -63,7 +63,7 @@ class RequestWrapperTween(object):
total = time.time() - start
log.info(
'Req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s',
'Finished request processing: req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s',
count, _auth_user, request.environ.get('REQUEST_METHOD'),
_path, total, get_user_agent(request. environ), _ver_,
extra={"time": total, "ver": _ver_, "ip": ip,

24
rhodecode/lib/middleware/simplehg.py
View file

@ -53,25 +53,31 @@ class SimpleHg(simplevcs.SimpleVCS):
return repo_name.rstrip('/')
_ACTION_MAPPING = {
'between': 'pull',
'branches': 'pull',
'branchmap': 'pull',
'capabilities': 'pull',
'changegroup': 'pull',
'changegroupsubset': 'pull',
'getbundle': 'pull',
'stream_out': 'pull',
'listkeys': 'pull',
'between': 'pull',
'branchmap': 'pull',
'branches': 'pull',
'changesetdata': 'pull',
'clonebundles': 'pull',
'capabilities': 'pull',
'clonebundles_manifest': 'pull',
'debugwireargs': 'pull',
'filedata': 'pull',
'getbundle': 'pull',
'heads': 'pull',
'lookup': 'pull',
'hello': 'pull',
'known': 'pull',
'listkeys': 'pull',
'lookup': 'pull',
'manifestdata': 'pull',
'narrow_widen': 'pull',
'protocaps': 'pull',
'stream_out': 'pull',
# largefiles
'putlfile': 'push',
'getlfile': 'pull',
'putlfile': 'push',
'statlfile': 'pull',
'lheads': 'pull',

23
rhodecode/lib/middleware/simplevcs.py
View file

@ -293,7 +293,7 @@ class SimpleVCS(object):
def compute_perm_vcs(
cache_name, plugin_id, action, user_id, repo_name, ip_addr):
log.debug('auth: calculating permission access now...')
log.debug('auth: calculating permission access now for vcs operation: %s', action)
# check IP
inherit = user.inherit_default_permissions
ip_allowed = AuthUser.check_ip_allowed(
@ -339,21 +339,6 @@ class SimpleVCS(object):
log.exception('Failed to read http scheme')
return 'http'
def _check_ssl(self, environ, start_response):
"""
Checks the SSL check flag and returns False if SSL is not present
and required True otherwise
"""
org_proto = environ['wsgi._org_proto']
# check if we have SSL required ! if not it's a bad request !
require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
if require_ssl and org_proto == 'http':
log.debug(
'Bad request: detected protocol is `%s` and '
'SSL/HTTPS is required.', org_proto)
return False
return True
def _get_default_cache_ttl(self):
# take AUTH_CACHE_TTL from the `rhodecode` auth plugin
plugin = loadplugin('egg:rhodecode-enterprise-ce#rhodecode')
@ -373,12 +358,6 @@ class SimpleVCS(object):
meta.Session.remove()
def _handle_request(self, environ, start_response):
if not self._check_ssl(environ, start_response):
reason = ('SSL required, while RhodeCode was unable '
'to detect this as SSL request')
log.debug('User not allowed to proceed, %s', reason)
return HTTPNotAcceptable(reason)(environ, start_response)
if not self.url_repo_name:
log.warning('Repository name is empty: %s', self.url_repo_name)
# failed to get repo name, we fail now

18
rhodecode/lib/middleware/vcs.py
View file

@ -159,11 +159,18 @@ def detect_vcs_request(environ, backends):
# favicon often requested by browsers
'favicon.ico',
# static files no detection
'_static++',
# debug-toolbar
'_debug_toolbar++',
# e.g /_file_store/download
'_file_store++',
# login
"_admin/login",
f"{ADMIN_PREFIX}/login",
f"{ADMIN_PREFIX}/logout",
# 2fa
f"{ADMIN_PREFIX}/check_2fa",
@ -178,12 +185,6 @@ def detect_vcs_request(environ, backends):
# _admin/my_account is safe too
f'{ADMIN_PREFIX}/my_account++',
# static files no detection
'_static++',
# debug-toolbar
'_debug_toolbar++',
# skip ops ping, status
f'{ADMIN_PREFIX}/ops/ping',
f'{ADMIN_PREFIX}/ops/status',
@ -193,11 +194,14 @@ def detect_vcs_request(environ, backends):
'++/repo_creating_check'
]
path_info = get_path_info(environ)
path_url = path_info.lstrip('/')
req_method = environ.get('REQUEST_METHOD')
for item in white_list:
item = item.lstrip('/')
if item.endswith('++') and path_url.startswith(item[:-2]):
log.debug('path `%s` in whitelist (match:%s), skipping...', path_url, item)
return handler

15
rhodecode/lib/rc_cache/backends.py
View file

@ -38,9 +38,9 @@ from dogpile.cache.backends import redis as redis_backend
from dogpile.cache.backends.file import FileLock
from dogpile.cache.util import memoized_property
from rhodecode.lib.memory_lru_dict import LRUDict, LRUDictDebug
from rhodecode.lib.str_utils import safe_bytes, safe_str
from rhodecode.lib.type_utils import str2bool
from ...lib.memory_lru_dict import LRUDict, LRUDictDebug
from ...lib.str_utils import safe_bytes, safe_str
from ...lib.type_utils import str2bool
_default_max_size = 1024
@ -198,6 +198,13 @@ class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
def get_store(self):
return self.filename
def cleanup_store(self):
for ext in ("db", "dat", "pag", "dir"):
final_filename = self.filename + os.extsep + ext
if os.path.exists(final_filename):
os.remove(final_filename)
log.warning('Removed dbm file %s', final_filename)
class BaseRedisBackend(redis_backend.RedisBackend):
key_prefix = ''
@ -289,7 +296,7 @@ class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
from rhodecode.lib._vendor import redis_lock
from ...lib._vendor import redis_lock
class _RedisLockWrapper:
"""LockWrapper for redis_lock"""

6
rhodecode/lib/rc_cache/utils.py
View file

@ -26,9 +26,9 @@ import decorator
from dogpile.cache import CacheRegion
import rhodecode
from rhodecode.lib.hash_utils import sha1
from rhodecode.lib.str_utils import safe_bytes
from rhodecode.lib.type_utils import str2bool # noqa :required by imports from .utils
from ...lib.hash_utils import sha1
from ...lib.str_utils import safe_bytes
from ...lib.type_utils import str2bool # noqa :required by imports from .utils
from . import region_meta

7
rhodecode/lib/rc_commands/add_artifact.py
View file

@ -91,15 +91,14 @@ def command(ini_path, filename, file_path, repo_id, user_id, description):
auth_user = db_user.AuthUser(ip_addr='127.0.0.1')
storage = store_utils.get_file_storage(request.registry.settings)
f_store = store_utils.get_filestore_backend(request.registry.settings)
with open(file_path, 'rb') as f:
click.secho(f'Adding new artifact from path: `{file_path}`',
fg='green')
file_data = _store_file(
storage, auth_user, filename, content=None, check_acl=True,
f_store, auth_user, filename, content=None, check_acl=True,
file_obj=f, description=description,
scope_repo_id=repo.repo_id)
click.secho(f'File Data: {file_data}',
fg='green')
click.secho(f'File Data: {file_data}', fg='green')

122
rhodecode/lib/rc_commands/migrate_artifact.py Normal file
View file

@ -0,0 +1,122 @@
# Copyright (C) 2016-2023 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import sys
import logging
import click
from rhodecode.lib.pyramid_utils import bootstrap
from rhodecode.lib.ext_json import json
from rhodecode.model.db import FileStore
from rhodecode.apps.file_store import utils as store_utils
log = logging.getLogger(__name__)
@click.command()
@click.argument('ini_path', type=click.Path(exists=True))
@click.argument('file_uid')
@click.option(
'--source-backend-conf',
type=click.Path(exists=True, dir_okay=False, readable=True),
help='Source backend config file path in a json format'
)
@click.option(
'--dest-backend-conf',
type=click.Path(exists=True, dir_okay=False, readable=True),
help='Source backend config file path in a json format'
)
def main(ini_path, file_uid, source_backend_conf, dest_backend_conf):
return command(ini_path, file_uid, source_backend_conf, dest_backend_conf)
_source_settings = {}
_dest_settings = {}
def command(ini_path, file_uid, source_backend_conf, dest_backend_conf):
with bootstrap(ini_path, env={'RC_CMD_SETUP_RC': '1'}) as env:
migrate_func(env, file_uid, source_backend_conf, dest_backend_conf)
def migrate_func(env, file_uid, source_backend_conf=None, dest_backend_conf=None):
"""
Example usage::
from rhodecode.lib.rc_commands import migrate_artifact
migrate_artifact._source_settings = {
'file_store.backend.type': 'filesystem_v1',
'file_store.filesystem_v1.storage_path': '/var/opt/rhodecode_data/file_store',
}
migrate_artifact._dest_settings = {
'file_store.backend.type': 'objectstore',
'file_store.objectstore.url': 'http://s3-minio:9000',
'file_store.objectstore.bucket': 'rhodecode-file-store',
'file_store.objectstore.key': 's3admin',
'file_store.objectstore.secret': 's3secret4',
'file_store.objectstore.region': 'eu-central-1',
}
for db_obj in FileStore.query().all():
migrate_artifact.migrate_func({}, db_obj.file_uid)
"""
try:
from rc_ee.api.views.store_api import _store_file
except ImportError:
click.secho('ERROR: Unable to import store_api. '
'store_api is only available in EE edition of RhodeCode',
fg='red')
sys.exit(-1)
source_settings = _source_settings
if source_backend_conf:
source_settings = json.loads(open(source_backend_conf).read())
dest_settings = _dest_settings
if dest_backend_conf:
dest_settings = json.loads(open(dest_backend_conf).read())
if file_uid.isnumeric():
file_store_db_obj = FileStore().query() \
.filter(FileStore.file_store_id == file_uid) \
.scalar()
else:
file_store_db_obj = FileStore().query() \
.filter(FileStore.file_uid == file_uid) \
.scalar()
if not file_store_db_obj:
click.secho(f'ERROR: Unable to fetch artifact from database file_uid={file_uid}',
fg='red')
sys.exit(-1)
uid_filename = file_store_db_obj.file_uid
org_filename = file_store_db_obj.file_display_name
click.secho(f'Attempting to migrate artifact {uid_filename}, filename: {org_filename}', fg='green')
# get old version of f_store based on the data.
origin_f_store = store_utils.get_filestore_backend(source_settings, always_init=True)
reader, metadata = origin_f_store.fetch(uid_filename)
target_f_store = store_utils.get_filestore_backend(dest_settings, always_init=True)
target_f_store.import_to_store(reader, org_filename, uid_filename, metadata)
click.secho(f'Migrated artifact {uid_filename}, filename: {org_filename} into {target_f_store} storage', fg='green')

9
rhodecode/lib/rc_commands/setup_rc.py
View file

@ -108,11 +108,10 @@ def command(ini_path, force_yes, user, email, password, api_key, repos,
dbmanage.create_permissions()
dbmanage.populate_default_permissions()
if apply_license_key:
try:
from rc_license.models import apply_trial_license_if_missing
apply_trial_license_if_missing(force=True)
except ImportError:
pass
from rhodecode.model.license import apply_license_from_file
license_file_path = config.get('license.import_path')
if license_file_path:
apply_license_from_file(license_file_path, force=True)
Session().commit()

4
rhodecode/lib/str_utils.py
View file

@ -181,3 +181,7 @@ def splitnewlines(text: bytes):
else:
lines[-1] = lines[-1][:-1]
return lines
def header_safe_str(val):
return safe_bytes(val).decode('latin-1', errors='replace')

51
rhodecode/lib/system_info.py
View file

@ -396,17 +396,18 @@ def storage_inodes():
@register_sysinfo
def storage_archives():
def storage_artifacts():
import rhodecode
from rhodecode.lib.helpers import format_byte_size_binary
from rhodecode.lib.archive_cache import get_archival_cache_store
storage_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
backend_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=storage_type)
value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type)
state = STATE_OK_DEFAULT
try:
d_cache = get_archival_cache_store(config=rhodecode.CONFIG)
backend_type = str(d_cache)
total_files, total_size, _directory_stats = d_cache.get_statistics()
@ -415,7 +416,8 @@ def storage_archives():
'used': total_size,
'total': total_size,
'items': total_files,
'path': d_cache.storage_path
'path': d_cache.storage_path,
'type': backend_type
})
except Exception as e:
@ -425,8 +427,44 @@ def storage_archives():
human_value = value.copy()
human_value['used'] = format_byte_size_binary(value['used'])
human_value['total'] = format_byte_size_binary(value['total'])
human_value['text'] = "{} ({} items)".format(
human_value['used'], value['items'])
human_value['text'] = f"{human_value['used']} ({value['items']} items)"
return SysInfoRes(value=value, state=state, human_value=human_value)
@register_sysinfo
def storage_archives():
import rhodecode
from rhodecode.lib.helpers import format_byte_size_binary
import rhodecode.apps.file_store.utils as store_utils
from rhodecode import CONFIG
backend_type = rhodecode.ConfigGet().get_str(store_utils.config_keys.backend_type)
value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type)
state = STATE_OK_DEFAULT
try:
f_store = store_utils.get_filestore_backend(config=CONFIG)
backend_type = str(f_store)
total_files, total_size, _directory_stats = f_store.get_statistics()
value.update({
'percent': 100,
'used': total_size,
'total': total_size,
'items': total_files,
'path': f_store.storage_path,
'type': backend_type
})
except Exception as e:
log.exception('failed to fetch archive cache storage')
state = {'message': str(e), 'type': STATE_ERR}
human_value = value.copy()
human_value['used'] = format_byte_size_binary(value['used'])
human_value['total'] = format_byte_size_binary(value['total'])
human_value['text'] = f"{human_value['used']} ({value['items']} items)"
return SysInfoRes(value=value, state=state, human_value=human_value)
@ -798,6 +836,7 @@ def get_system_info(environ):
'storage': SysInfo(storage)(),
'storage_inodes': SysInfo(storage_inodes)(),
'storage_archive': SysInfo(storage_archives)(),
'storage_artifacts': SysInfo(storage_artifacts)(),
'storage_gist': SysInfo(storage_gist)(),
'storage_temp': SysInfo(storage_temp)(),

60
rhodecode/lib/utils.py
View file

@ -42,6 +42,8 @@ from webhelpers2.text import collapse, strip_tags, convert_accented_entities, co
from mako import exceptions
from rhodecode import ConfigGet
from rhodecode.lib.exceptions import HTTPBranchProtected, HTTPLockedRC
from rhodecode.lib.hash_utils import sha256_safe, md5, sha1
from rhodecode.lib.type_utils import AttributeDict
from rhodecode.lib.str_utils import safe_bytes, safe_str
@ -84,8 +86,39 @@ def adopt_for_celery(func):
@wraps(func)
def wrapper(extras):
extras = AttributeDict(extras)
# HooksResponse implements to_json method which must be used there.
return func(extras).to_json()
try:
# HooksResponse implements to_json method which must be used there.
return func(extras).to_json()
except HTTPBranchProtected as error:
# Those special cases don't need error reporting. It's a case of
# locked repo or protected branch
error_args = error.args
return {
'status': error.code,
'output': error.explanation,
'exception': type(error).__name__,
'exception_args': error_args,
'exception_traceback': '',
}
except HTTPLockedRC as error:
# Those special cases don't need error reporting. It's a case of
# locked repo or protected branch
error_args = error.args
return {
'status': error.code,
'output': error.explanation,
'exception': type(error).__name__,
'exception_args': error_args,
'exception_traceback': '',
}
except Exception as e:
return {
'status': 128,
'output': '',
'exception': type(e).__name__,
'exception_args': e.args,
'exception_traceback': '',
}
return wrapper
@ -361,32 +394,39 @@ ui_sections = [
'ui', 'web', ]
def config_data_from_db(clear_session=True, repo=None):
def prepare_config_data(clear_session=True, repo=None):
"""
Read the configuration data from the database and return configuration
Read the configuration data from the database, *.ini files and return configuration
tuples.
"""
from rhodecode.model.settings import VcsSettingsModel
config = []
sa = meta.Session()
settings_model = VcsSettingsModel(repo=repo, sa=sa)
ui_settings = settings_model.get_ui_settings()
ui_data = []
config = [
('web', 'push_ssl', 'false'),
]
for setting in ui_settings:
# Todo: remove this section once transition to *.ini files will be completed
if setting.section in ('largefiles', 'vcs_git_lfs'):
if setting.key != 'enabled':
continue
if setting.active:
ui_data.append((setting.section, setting.key, setting.value))
config.append((
safe_str(setting.section), safe_str(setting.key),
safe_str(setting.value)))
if setting.key == 'push_ssl':
# force set push_ssl requirement to False, rhodecode
# handles that
# force set push_ssl requirement to False this is deprecated, and we must force it to False
config.append((
safe_str(setting.section), safe_str(setting.key), False))
config_getter = ConfigGet()
config.append(('vcs_git_lfs', 'store_location', config_getter.get_str('vcs.git.lfs.storage_location')))
config.append(('largefiles', 'usercache', config_getter.get_str('vcs.hg.largefiles.storage_location')))
log.debug(
'settings ui from db@repo[%s]: %s',
repo,
@ -415,7 +455,7 @@ def make_db_config(clear_session=True, repo=None):
Create a :class:`Config` instance based on the values in the database.
"""
config = Config()
config_data = config_data_from_db(clear_session=clear_session, repo=repo)
config_data = prepare_config_data(clear_session=clear_session, repo=repo)
for section, option, value in config_data:
config.set(section, option, value)
return config
@ -582,7 +622,7 @@ def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild
log.debug('Running update server info')
git_repo._update_server_info(force=True)
db_repo.update_commit_cache()
db_repo.update_commit_cache(recursive=False)
config = db_repo._config
config.set('extensions', 'largefiles', '')

19
rhodecode/model/db.py
View file

@ -2568,10 +2568,10 @@ class Repository(Base, BaseModel):
return commit
def flush_commit_cache(self):
self.update_commit_cache(cs_cache={'raw_id':'0'})
self.update_commit_cache(cs_cache={'raw_id': '0'})
self.update_commit_cache()
def update_commit_cache(self, cs_cache=None, config=None):
def update_commit_cache(self, cs_cache=None, config=None, recursive=True):
"""
Update cache of last commit for repository
cache_keys should be::
@ -2610,6 +2610,14 @@ class Repository(Base, BaseModel):
if isinstance(cs_cache, BaseCommit):
cs_cache = cs_cache.__json__()
def maybe_update_recursive(instance, _config, _recursive, _cs_cache, _last_change):
if _recursive:
repo_id = instance.repo_id
_cs_cache['source_repo_id'] = repo_id
for gr in instance.groups_with_parents:
gr.changeset_cache = _cs_cache
gr.updated_on = _last_change
def is_outdated(new_cs_cache):
if (new_cs_cache['raw_id'] != self.changeset_cache['raw_id'] or
new_cs_cache['revision'] != self.changeset_cache['revision']):
@ -2636,6 +2644,7 @@ class Repository(Base, BaseModel):
self.changeset_cache = cs_cache
self.updated_on = last_change
Session().add(self)
maybe_update_recursive(self, config, recursive, cs_cache, last_change)
Session().commit()
else:
@ -2650,6 +2659,7 @@ class Repository(Base, BaseModel):
self.changeset_cache = cs_cache
self.updated_on = _date_latest
Session().add(self)
maybe_update_recursive(self, config, recursive, cs_cache, _date_latest)
Session().commit()
log.debug('updated repo `%s` with new commit cache %s, and last update_date: %s',
@ -5839,8 +5849,7 @@ class FileStore(Base, BaseModel):
.filter(FileStoreMetadata.file_store_meta_key == key) \
.scalar()
if has_key:
msg = 'key `{}` already defined under section `{}` for this file.'\
.format(key, section)
msg = f'key `{key}` already defined under section `{section}` for this file.'
raise ArtifactMetadataDuplicate(msg, err_section=section, err_key=key)
# NOTE(marcink): raises ArtifactMetadataBadValueType
@ -5939,7 +5948,7 @@ class FileStoreMetadata(Base, BaseModel):
def valid_value_type(cls, value):
if value.split('.')[0] not in cls.SETTINGS_TYPES:
raise ArtifactMetadataBadValueType(
'value_type must be one of %s got %s' % (cls.SETTINGS_TYPES.keys(), value))
f'value_type must be one of {cls.SETTINGS_TYPES.keys()} got {value}')
@hybrid_property
def file_store_meta_section(self):

21
rhodecode/model/forms.py
View file

@ -129,6 +129,20 @@ def TOTPForm(localizer, user, allow_recovery_code_use=False):
return _TOTPForm
def WhitelistedVcsClientsForm(localizer):
_ = localizer
class _WhitelistedVcsClientsForm(formencode.Schema):
regexp = r'^(?:\s*[<>=~^!]*\s*\d{1,2}\.\d{1,2}(?:\.\d{1,2})?\s*|\*)\s*(?:,\s*[<>=~^!]*\s*\d{1,2}\.\d{1,2}(?:\.\d{1,2})?\s*|\s*\*\s*)*$'
allow_extra_fields = True
filter_extra_fields = True
git = v.Regex(regexp)
hg = v.Regex(regexp)
svn = v.Regex(regexp)
return _WhitelistedVcsClientsForm
def UserForm(localizer, edit=False, available_languages=None, old_data=None):
old_data = old_data or {}
available_languages = available_languages or []
@ -454,13 +468,6 @@ def ApplicationUiSettingsForm(localizer):
_ = localizer
class _ApplicationUiSettingsForm(_BaseVcsSettingsForm):
web_push_ssl = v.StringBoolean(if_missing=False)
largefiles_usercache = All(
v.ValidPath(localizer),
v.UnicodeString(strip=True, min=2, not_empty=True))
vcs_git_lfs_store_location = All(
v.ValidPath(localizer),
v.UnicodeString(strip=True, min=2, not_empty=True))
extensions_hggit = v.StringBoolean(if_missing=False)
new_svn_branch = v.ValidSvnPattern(localizer, section='vcs_svn_branch')
new_svn_tag = v.ValidSvnPattern(localizer, section='vcs_svn_tag')

17
rhodecode/model/license.py Normal file
View file

@ -0,0 +1,17 @@
def apply_license(*args, **kwargs):
pass
try:
from rc_license.models import apply_license
except ImportError:
pass
def apply_license_from_file(*args, **kwargs):
pass
try:
from rc_license.models import apply_license_from_file
except ImportError:
pass

14
rhodecode/model/notification.py
View file

@ -117,14 +117,16 @@ class NotificationModel(BaseModel):
# add mentioned users into recipients
final_recipients = set(recipients_objs).union(mention_recipients)
(subject, email_body, email_body_plaintext) = \
EmailNotificationModel().render_email(notification_type, **email_kwargs)
# No need to render email if we are sending just notification
if with_email:
(subject, email_body, email_body_plaintext) = \
EmailNotificationModel().render_email(notification_type, **email_kwargs)
if not notification_subject:
notification_subject = subject
if not notification_subject:
notification_subject = subject
if not notification_body:
notification_body = email_body_plaintext
if not notification_body:
notification_body = email_body_plaintext
notification = Notification.create(
created_by=created_by_obj, subject=notification_subject,

12
rhodecode/model/repo.py
View file

@ -31,7 +31,7 @@ from zope.cachedescriptors.property import Lazy as LazyProperty
from rhodecode import events
from rhodecode.lib.auth import HasUserGroupPermissionAny
from rhodecode.lib.caching_query import FromCache
from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError
from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError
from rhodecode.lib import hooks_base
from rhodecode.lib.user_log_filter import user_log_filter
from rhodecode.lib.utils import make_db_config
@ -736,7 +736,7 @@ class RepoModel(BaseModel):
log.error(traceback.format_exc())
raise
def delete(self, repo, forks=None, pull_requests=None, fs_remove=True, cur_user=None):
def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None):
"""
Delete given repository, forks parameter defines what do do with
attached forks. Throws AttachedForksError if deleted repo has attached
@ -745,6 +745,7 @@ class RepoModel(BaseModel):
:param repo:
:param forks: str 'delete' or 'detach'
:param pull_requests: str 'delete' or None
:param artifacts: str 'delete' or None
:param fs_remove: remove(archive) repo from filesystem
"""
if not cur_user:
@ -767,6 +768,13 @@ class RepoModel(BaseModel):
if pull_requests != 'delete' and (pr_sources or pr_targets):
raise AttachedPullRequestsError()
artifacts_objs = repo.artifacts
if artifacts == 'delete':
for a in artifacts_objs:
self.sa.delete(a)
elif [a for a in artifacts_objs]:
raise AttachedArtifactsError()
old_repo_dict = repo.get_dict()
events.trigger(events.RepoPreDeleteEvent(repo))
try:

24
rhodecode/model/settings.py
View file

@ -486,7 +486,6 @@ class VcsSettingsModel(object):
)
GLOBAL_HG_SETTINGS = (
('extensions', 'largefiles'),
('largefiles', 'usercache'),
('phases', 'publish'),
('extensions', 'evolve'),
('extensions', 'topic'),
@ -496,12 +495,10 @@ class VcsSettingsModel(object):
GLOBAL_GIT_SETTINGS = (
('vcs_git_lfs', 'enabled'),
('vcs_git_lfs', 'store_location')
)
SVN_BRANCH_SECTION = 'vcs_svn_branch'
SVN_TAG_SECTION = 'vcs_svn_tag'
SSL_SETTING = ('web', 'push_ssl')
PATH_SETTING = ('paths', '/')
def __init__(self, sa=None, repo=None):
@ -666,17 +663,15 @@ class VcsSettingsModel(object):
self.repo_settings, *phases, value=safe_str(data[phases_key]))
def create_or_update_global_hg_settings(self, data):
opts_len = 4
largefiles, largefiles_store, phases, evolve \
opts_len = 3
largefiles, phases, evolve \
= self.GLOBAL_HG_SETTINGS[:opts_len]
largefiles_key, largefiles_store_key, phases_key, evolve_key \
largefiles_key, phases_key, evolve_key \
= self._get_settings_keys(self.GLOBAL_HG_SETTINGS[:opts_len], data)
self._create_or_update_ui(
self.global_settings, *largefiles, value='',
active=data[largefiles_key])
self._create_or_update_ui(
self.global_settings, *largefiles_store, value=data[largefiles_store_key])
self._create_or_update_ui(
self.global_settings, *phases, value=safe_str(data[phases_key]))
self._create_or_update_ui(
@ -697,26 +692,17 @@ class VcsSettingsModel(object):
active=data[lfs_enabled_key])
def create_or_update_global_git_settings(self, data):
lfs_enabled, lfs_store_location \
= self.GLOBAL_GIT_SETTINGS
lfs_enabled_key, lfs_store_location_key \
= self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data)
lfs_enabled = self.GLOBAL_GIT_SETTINGS[0]
lfs_enabled_key = self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data)[0]
self._create_or_update_ui(
self.global_settings, *lfs_enabled, value=data[lfs_enabled_key],
active=data[lfs_enabled_key])
self._create_or_update_ui(
self.global_settings, *lfs_store_location,
value=data[lfs_store_location_key])
def create_or_update_global_svn_settings(self, data):
# branch/tags patterns
self._create_svn_settings(self.global_settings, data)
def update_global_ssl_setting(self, value):
self._create_or_update_ui(
self.global_settings, *self.SSL_SETTING, value=value)
@assert_repo_settings
def delete_repo_svn_pattern(self, id_):
ui = self.repo_settings.UiDbModel.get(id_)

4
rhodecode/model/user.py
View file

@ -557,10 +557,10 @@ class UserModel(BaseModel):
elif handle_mode == 'delete':
from rhodecode.apps.file_store import utils as store_utils
request = get_current_request()
storage = store_utils.get_file_storage(request.registry.settings)
f_store = store_utils.get_filestore_backend(request.registry.settings)
for a in artifacts:
file_uid = a.file_uid
storage.delete(file_uid)
f_store.delete(file_uid)
self.sa.delete(a)
left_overs = False

1
rhodecode/public/js/rhodecode/routes.js
View file

@ -86,6 +86,7 @@ function registerRCRoutes() {
pyroutes.register('admin_settings_vcs_update', '/_admin/settings/vcs/update', []);
pyroutes.register('admin_settings_visual', '/_admin/settings/visual', []);
pyroutes.register('admin_settings_visual_update', '/_admin/settings/visual/update', []);
pyroutes.register('admin_security_modify_allowed_vcs_client_versions', '/_admin/security/modify/allowed_vcs_client_versions', []);
pyroutes.register('apiv2', '/_admin/api', []);
pyroutes.register('atom_feed_home', '/%(repo_name)s/feed-atom', ['repo_name']);
pyroutes.register('atom_feed_home_old', '/%(repo_name)s/feed/atom', ['repo_name']);

Some files were not shown because too many files have changed in this diff Show more