From ec1b6d985c4ca0f94145075d6b946c74495de54e Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 22 Feb 2024 13:11:18 +0100
Subject: [PATCH 001/150] chore(dev): print dev help cmd for nicer startup

---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index ac29ce52..3d59ea61 100644
--- a/Makefile
+++ b/Makefile
@@ -109,7 +109,8 @@ dev-sh:
 	sudo apt-get install -y zsh carapace-bin
 	rm -rf /home/rhodecode/.oh-my-zsh
 	curl https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh | sh
-	echo "source <(carapace _carapace)" > /home/rhodecode/.zsrc
+	@echo "source <(carapace _carapace)" > /home/rhodecode/.zsrc
+	@echo "${RC_DEV_CMD_HELP}"
 	PROMPT='%(?.%F{green}√.%F{red}?%?)%f %B%F{240}%1~%f%b %# ' zsh
 
 

From 41983731e4daa05d418cceaacd8cacff68be4a70 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 26 Feb 2024 15:02:27 +0100
Subject: [PATCH 002/150] chore(deps): bumped tzdata==2024.1

---
 requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 9b32150e..e2e5c6a0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -27,7 +27,7 @@ celery==5.3.6
     vine==5.1.0
   python-dateutil==2.8.2
     six==1.16.0
-  tzdata==2023.4
+  tzdata==2024.1
   vine==5.1.0
 channelstream==0.7.1
   gevent==24.2.1
@@ -275,7 +275,7 @@ sqlalchemy==1.4.51
 supervisor==4.2.5
 tzlocal==4.3
   pytz-deprecation-shim==0.1.0.post0
-    tzdata==2023.4
+    tzdata==2024.1
 unidecode==1.3.6
 urlobject==2.4.3
 waitress==3.0.0

From ca4a10b1d455e48235d0619a1505142bad36de78 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 26 Feb 2024 15:03:39 +0100
Subject: [PATCH 003/150] chore(deps): bumped zope.interface==6.2.0

---
 requirements.txt | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index e2e5c6a0..d2de6b7f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -33,7 +33,7 @@ channelstream==0.7.1
   gevent==24.2.1
     greenlet==3.0.3
     zope.event==5.0.0
-    zope.interface==6.1.0
+    zope.interface==6.2.0
   itsdangerous==1.1.0
   marshmallow==2.18.0
   pyramid==2.0.2
@@ -46,7 +46,7 @@ channelstream==0.7.1
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.1.0
+    zope.interface==6.2.0
   pyramid-apispec==0.3.3
     apispec==1.3.3
   pyramid-jinja2==2.10
@@ -63,7 +63,7 @@ channelstream==0.7.1
       venusian==3.0.0
       webob==1.8.7
       zope.deprecation==5.0.0
-      zope.interface==6.1.0
+      zope.interface==6.2.0
     zope.deprecation==5.0.0
   python-dateutil==2.8.2
     six==1.16.0
@@ -95,7 +95,7 @@ gunicorn==21.2.0
 gevent==24.2.1
   greenlet==3.0.3
   zope.event==5.0.0
-  zope.interface==6.1.0
+  zope.interface==6.2.0
 ipython==8.14.0
   backcall==0.2.0
   decorator==5.1.1
@@ -210,7 +210,7 @@ pyramid-debugtoolbar==4.11
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.1.0
+    zope.interface==6.2.0
   pyramid-mako==1.1.0
     mako==1.2.4
       markupsafe==2.1.2
@@ -224,7 +224,7 @@ pyramid-debugtoolbar==4.11
       venusian==3.0.0
       webob==1.8.7
       zope.deprecation==5.0.0
-      zope.interface==6.1.0
+      zope.interface==6.2.0
 pyramid-mailer==0.15.1
   pyramid==2.0.2
     hupper==1.12
@@ -236,13 +236,13 @@ pyramid-mailer==0.15.1
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.1.0
+    zope.interface==6.2.0
   repoze.sendmail==4.4.1
     transaction==3.1.0
-      zope.interface==6.1.0
-    zope.interface==6.1.0
+      zope.interface==6.2.0
+    zope.interface==6.2.0
   transaction==3.1.0
-    zope.interface==6.1.0
+    zope.interface==6.2.0
 python-ldap==3.4.3
   pyasn1==0.4.8
   pyasn1-modules==0.2.8

From 1ea821d801e7a27f01f7f93cb3e6ea1cf9a47e7a Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 26 Feb 2024 15:04:20 +0100
Subject: [PATCH 004/150] chore(deps): bumped webhelpers2==2.1

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index d2de6b7f..24aaf3a7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -285,7 +285,7 @@ weberror==0.13.1
   pygments==2.15.1
   tempita==0.5.2
   webob==1.8.7
-webhelpers2==2.0
+webhelpers2==2.1
   markupsafe==2.1.2
   six==1.16.0
 whoosh==2.7.4

From 6f000325684bf7c66355321bdcd7af7957fd2b73 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 26 Feb 2024 15:04:53 +0100
Subject: [PATCH 005/150] chore(deps): bumped simplejson==3.19.2

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 24aaf3a7..dcb2028c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -262,7 +262,7 @@ regex==2022.10.31
 routes==2.5.1
   repoze.lru==0.7
   six==1.16.0
-simplejson==3.19.1
+simplejson==3.19.2
 sshpubkeys==3.3.1
   cryptography==40.0.2
     cffi==1.16.0

From a7987e5b31d738b2f06f3e4ebd8f9eab950c7bc4 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 26 Feb 2024 15:05:18 +0100
Subject: [PATCH 006/150] chore(deps): bumped pycurl==7.45.3

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index dcb2028c..279375d6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -194,7 +194,7 @@ pycmarkgfm==1.2.0
   cffi==1.16.0
     pycparser==2.21
 pycryptodome==3.17
-pycurl==7.45.2
+pycurl==7.45.3
 pymysql==1.0.3
 pyotp==2.8.0
 pyparsing==3.1.1

From 2cfcfbda10aebb22f26cc4faeda078dcceb787a1 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 26 Feb 2024 15:06:20 +0100
Subject: [PATCH 007/150] chore(deps): bumped orjson==3.9.15

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 279375d6..eecf0875 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -171,7 +171,7 @@ nbconvert==7.7.3
   tinycss2==1.2.1
     webencodings==0.5.1
   traitlets==5.9.0
-orjson==3.9.13
+orjson==3.9.15
 pastescript==3.4.0
   paste==3.7.1
     six==1.16.0

From 4e9e770ea848b3ea27c055db84e5b626fea9cc46 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 26 Feb 2024 15:07:25 +0100
Subject: [PATCH 008/150] chore(release): bumped version to 5.1.0

---
 .bumpversion.cfg  | 2 +-
 rhodecode/VERSION | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index ca71cca8..c76f1bd2 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 5.0.0
+current_version = 5.1.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:rhodecode/VERSION]
diff --git a/rhodecode/VERSION b/rhodecode/VERSION
index 28cbf7c0..acf69b48 100644
--- a/rhodecode/VERSION
+++ b/rhodecode/VERSION
@@ -1 +1 @@
-5.0.0
\ No newline at end of file
+5.1.0
\ No newline at end of file

From 17631cb9fd15d5e58abcc82b9a0d9db958a0fe5d Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 26 Feb 2024 15:19:54 +0100
Subject: [PATCH 009/150] chore(deps): bumped dogpile.cache==1.3.2

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index eecf0875..55b03477 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -84,7 +84,7 @@ deform==2.0.15
   zope.deprecation==5.0.0
 diskcache==5.6.3
 docutils==0.19
-dogpile.cache==1.3.0
+dogpile.cache==1.3.2
   decorator==5.1.1
   stevedore==5.1.0
     pbr==5.11.1

From 16c169bec2cf50587662ef01d8e7a5936ef192eb Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Mon, 19 Feb 2024 13:49:51 +0200
Subject: [PATCH 010/150] fix(ssh): Added alternative SshWrapper and changes
 needed to support it + service api. Fixes: RCCE-6

---
 configs/development.ini                       |   6 +
 configs/production.ini                        |   6 +
 rhodecode/api/__init__.py                     |  79 ++++++-----
 rhodecode/api/tests/test_service_api.py       |  55 ++++++++
 rhodecode/api/views/service_api.py            | 125 +++++++++++++++++
 .../apps/ssh_support/lib/backends/__init__.py | 129 ++++++++++++++++++
 .../apps/ssh_support/lib/backends/base.py     |  33 +++--
 rhodecode/apps/ssh_support/lib/backends/hg.py |   9 ++
 .../lib/{ssh_wrapper.py => ssh_wrapper_v1.py} |   0
 .../apps/ssh_support/lib/ssh_wrapper_v2.py    |  72 ++++++++++
 rhodecode/apps/ssh_support/tests/conftest.py  |   2 +-
 rhodecode/lib/utils.py                        |  28 ++++
 rhodecode/lib/vcs/exceptions.py               |   4 +
 .../tests/fixture_mods/fixture_pyramid.py     |   1 +
 setup.py                                      |   3 +-
 15 files changed, 505 insertions(+), 47 deletions(-)
 create mode 100644 rhodecode/api/tests/test_service_api.py
 create mode 100644 rhodecode/api/views/service_api.py
 rename rhodecode/apps/ssh_support/lib/{ssh_wrapper.py => ssh_wrapper_v1.py} (100%)
 create mode 100644 rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py

diff --git a/configs/development.ini b/configs/development.ini
index feed95be..d7a492a2 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -153,6 +153,12 @@ startup.import_repos = false
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
+; Host at which the Service API is running.
+app.service_api.host = http://rhodecode.local:10020
+
+; Secret for Service API authentication.
+app.service_api.token =
+
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
diff --git a/configs/production.ini b/configs/production.ini
index f9c0708c..6ffeeb7d 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -104,6 +104,12 @@ startup.import_repos = false
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
+; Host at which the Service API is running.
+app.service_api.host= http://rhodecode.local:10020
+
+; Secret for Service API authentication.
+app.service_api.token =
+
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
diff --git a/rhodecode/api/__init__.py b/rhodecode/api/__init__.py
index 106ff435..9a2f63e5 100644
--- a/rhodecode/api/__init__.py
+++ b/rhodecode/api/__init__.py
@@ -46,6 +46,7 @@ log = logging.getLogger(__name__)
 
 DEFAULT_RENDERER = 'jsonrpc_renderer'
 DEFAULT_URL = '/_admin/apiv2'
+SERVICE_API_IDENTIFIER = 'service_'
 
 
 def find_methods(jsonrpc_methods, pattern):
@@ -54,7 +55,9 @@ def find_methods(jsonrpc_methods, pattern):
         pattern = [pattern]
 
     for single_pattern in pattern:
-        for method_name, method in jsonrpc_methods.items():
+        for method_name, method in filter(
+                lambda x: not x[0].startswith(SERVICE_API_IDENTIFIER), jsonrpc_methods.items()
+        ):
             if fnmatch.fnmatch(method_name, single_pattern):
                 matches[method_name] = method
     return matches
@@ -190,43 +193,48 @@ def request_view(request):
     # check if we can find this session using api_key, get_by_auth_token
     # search not expired tokens only
     try:
-        api_user = User.get_by_auth_token(request.rpc_api_key)
+        if not request.rpc_method.startswith(SERVICE_API_IDENTIFIER):
+            api_user = User.get_by_auth_token(request.rpc_api_key)
 
-        if api_user is None:
-            return jsonrpc_error(
-                request, retid=request.rpc_id, message='Invalid API KEY')
+            if api_user is None:
+                return jsonrpc_error(
+                    request, retid=request.rpc_id, message='Invalid API KEY')
 
-        if not api_user.active:
-            return jsonrpc_error(
-                request, retid=request.rpc_id,
-                message='Request from this user not allowed')
+            if not api_user.active:
+                return jsonrpc_error(
+                    request, retid=request.rpc_id,
+                    message='Request from this user not allowed')
 
-        # check if we are allowed to use this IP
-        auth_u = AuthUser(
-            api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr)
-        if not auth_u.ip_allowed:
-            return jsonrpc_error(
-                request, retid=request.rpc_id,
-                message='Request from IP:{} not allowed'.format(
-                    request.rpc_ip_addr))
+            # check if we are allowed to use this IP
+            auth_u = AuthUser(
+                api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr)
+            if not auth_u.ip_allowed:
+                return jsonrpc_error(
+                    request, retid=request.rpc_id,
+                    message='Request from IP:{} not allowed'.format(
+                        request.rpc_ip_addr))
+            else:
+                log.info('Access for IP:%s allowed', request.rpc_ip_addr)
+
+            # register our auth-user
+            request.rpc_user = auth_u
+            request.environ['rc_auth_user_id'] = str(auth_u.user_id)
+
+            # now check if token is valid for API
+            auth_token = request.rpc_api_key
+            token_match = api_user.authenticate_by_token(
+                auth_token, roles=[UserApiKeys.ROLE_API])
+            invalid_token = not token_match
+
+            log.debug('Checking if API KEY is valid with proper role')
+            if invalid_token:
+                return jsonrpc_error(
+                    request, retid=request.rpc_id,
+                    message='API KEY invalid or, has bad role for an API call')
         else:
-            log.info('Access for IP:%s allowed', request.rpc_ip_addr)
-
-        # register our auth-user
-        request.rpc_user = auth_u
-        request.environ['rc_auth_user_id'] = str(auth_u.user_id)
-
-        # now check if token is valid for API
-        auth_token = request.rpc_api_key
-        token_match = api_user.authenticate_by_token(
-            auth_token, roles=[UserApiKeys.ROLE_API])
-        invalid_token = not token_match
-
-        log.debug('Checking if API KEY is valid with proper role')
-        if invalid_token:
-            return jsonrpc_error(
-                request, retid=request.rpc_id,
-                message='API KEY invalid or, has bad role for an API call')
+            auth_u = 'service'
+            if request.rpc_api_key != request.registry.settings['app.service_api.token']:
+                raise Exception("Provided service secret is not recognized!")
 
     except Exception:
         log.exception('Error on API AUTH')
@@ -290,7 +298,8 @@ def request_view(request):
     })
 
     # register some common functions for usage
-    attach_context_attributes(TemplateArgs(), request, request.rpc_user.user_id)
+    rpc_user = request.rpc_user.user_id if hasattr(request, 'rpc_user') else None
+    attach_context_attributes(TemplateArgs(), request, rpc_user)
 
     statsd = request.registry.statsd
 
diff --git a/rhodecode/api/tests/test_service_api.py b/rhodecode/api/tests/test_service_api.py
new file mode 100644
index 00000000..b5780777
--- /dev/null
+++ b/rhodecode/api/tests/test_service_api.py
@@ -0,0 +1,55 @@
+
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import pytest
+
+from rhodecode.api.tests.utils import (
+    build_data, api_call)
+
+
+@pytest.mark.usefixtures("app")
+class TestServiceApi:
+
+    def test_service_api_with_wrong_secret(self):
+        id, payload = build_data("wrong_api_key", 'service_get_repo_name_by_id')
+        response = api_call(self.app, payload)
+
+        assert 'Invalid API KEY' == response.json['error']
+
+    def test_service_api_with_legit_secret(self):
+        id, payload = build_data(self.app.app.config.get_settings()['app.service_api.token'],
+                                 'service_get_repo_name_by_id', repo_id='1')
+        response = api_call(self.app, payload)
+        assert not response.json['error']
+
+    def test_service_api_not_a_part_of_public_api_suggestions(self):
+        id, payload = build_data("secret", 'some_random_guess_method')
+        response = api_call(self.app, payload)
+        assert 'service_' not in response.json['error']
+
+    def test_service_get_data_for_ssh_wrapper_output(self):
+        id, payload = build_data(
+            self.app.app.config.get_settings()['app.service_api.token'],
+            'service_get_data_for_ssh_wrapper',
+            user_id=1,
+            repo_name='vcs_test_git')
+        response = api_call(self.app, payload)
+
+        assert ['branch_permissions', 'repo_permissions', 'repos_path', 'user_id', 'username']\
+               == list(response.json['result'].keys())
diff --git a/rhodecode/api/views/service_api.py b/rhodecode/api/views/service_api.py
new file mode 100644
index 00000000..e2bfa45a
--- /dev/null
+++ b/rhodecode/api/views/service_api.py
@@ -0,0 +1,125 @@
+# Copyright (C) 2011-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+import datetime
+from collections import defaultdict
+
+from sqlalchemy import Table
+from rhodecode.api import jsonrpc_method, SERVICE_API_IDENTIFIER
+
+
+log = logging.getLogger(__name__)
+
+
+@jsonrpc_method()
+def service_get_data_for_ssh_wrapper(request, apiuser, user_id, repo_name, key_id=None):
+    from rhodecode.model.db import User
+    from rhodecode.model.scm import ScmModel
+    from rhodecode.model.meta import raw_query_executor, Base
+
+    if key_id:
+        table = Table('user_ssh_keys', Base.metadata, autoload=False)
+        atime = datetime.datetime.utcnow()
+        stmt = (
+            table.update()
+                .where(table.c.ssh_key_id == key_id)
+                .values(accessed_on=atime)
+        )
+
+        res_count = None
+        with raw_query_executor() as session:
+            result = session.execute(stmt)
+            if result.rowcount:
+                res_count = result.rowcount
+
+        if res_count:
+            log.debug(f'Update key id:{key_id} access time')
+    db_user = User.get(user_id)
+    if not db_user:
+        return None
+    auth_user = db_user.AuthUser()
+
+    return {
+        'user_id': db_user.user_id,
+        'username': db_user.username,
+        'repo_permissions': auth_user.permissions['repositories'],
+        "branch_permissions": auth_user.get_branch_permissions(repo_name),
+        "repos_path": ScmModel().repos_path
+    }
+
+
+@jsonrpc_method()
+def service_get_repo_name_by_id(request, apiuser, repo_id):
+    from rhodecode.model.repo import RepoModel
+    by_id_match = RepoModel().get_repo_by_id(repo_id)
+    if by_id_match:
+        repo_name = by_id_match.repo_name
+        return {
+            'repo_name': repo_name
+        }
+    return None
+
+
+@jsonrpc_method()
+def service_mark_for_invalidation(request, apiuser, repo_name):
+    from rhodecode.model.scm import ScmModel
+    ScmModel().mark_for_invalidation(repo_name)
+    return {'msg': "Applied"}
+
+
+@jsonrpc_method()
+def service_config_to_hgrc(request, apiuser, cli_flags, repo_name):
+    from rhodecode.model.db import RhodeCodeUi
+    from rhodecode.model.settings import VcsSettingsModel
+
+    ui_sections = defaultdict(list)
+    ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
+
+    default_hooks = [
+        ('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
+        ('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
+        ('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
+
+        ('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
+        ('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
+    ]
+
+    for k, v in default_hooks:
+        ui_sections['hooks'].append((k, v))
+
+    for entry in ui:
+        if not entry.active:
+            continue
+        sec = entry.section
+        key = entry.key
+
+        if sec in cli_flags:
+            # we want only custom hooks, so we skip builtins
+            if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
+                continue
+
+            ui_sections[sec].append([key, entry.value])
+
+    flags = []
+    for _sec, key_val in ui_sections.items():
+        flags.append(' ')
+        flags.append(f'[{_sec}]')
+        for key, val in key_val:
+            flags.append(f'{key}= {val}')
+    return {'flags': flags}
diff --git a/rhodecode/apps/ssh_support/lib/backends/__init__.py b/rhodecode/apps/ssh_support/lib/backends/__init__.py
index 8b1bfc76..d4479be6 100644
--- a/rhodecode/apps/ssh_support/lib/backends/__init__.py
+++ b/rhodecode/apps/ssh_support/lib/backends/__init__.py
@@ -23,6 +23,7 @@ import datetime
 import configparser
 from sqlalchemy import Table
 
+from rhodecode.lib.utils import call_service_api
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.model.scm import ScmModel
 
@@ -261,3 +262,131 @@ class SshWrapper(object):
             exit_code = -1
 
         return exit_code
+
+
+class SshWrapperStandalone(SshWrapper):
+    """
+    New version of SshWrapper designed to be depended only on service API
+    """
+    repos_path = None
+
+    @staticmethod
+    def parse_user_related_data(user_data):
+        user = AttributeDict()
+        user.user_id = user_data['user_id']
+        user.username = user_data['username']
+        user.repo_permissions = user_data['repo_permissions']
+        user.branch_permissions = user_data['branch_permissions']
+        return user
+
+    def wrap(self):
+        mode = self.mode
+        username = self.username
+        user_id = self.user_id
+        shell = self.shell
+
+        scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
+
+        log.debug(
+            'Mode: `%s` User: `name:%s : id:%s` Shell: `%s` SSH Command: `\"%s\"` '
+            'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
+            mode, username, user_id, shell, self.command,
+            scm_detected, scm_mode, scm_repo)
+
+        log.debug('SSH Connection info %s', self.get_connection_info())
+
+        if shell and self.command is None:
+            log.info('Dropping to shell, no command given and shell is allowed')
+            os.execl('/bin/bash', '-l')
+            exit_code = 1
+
+        elif scm_detected:
+            data = call_service_api(self.ini_path, {
+                "method": "service_get_data_for_ssh_wrapper",
+                "args": {"user_id": user_id, "repo_name": scm_repo, "key_id": self.key_id}
+            })
+            user = self.parse_user_related_data(data)
+            if not user:
+                log.warning('User with id %s not found', user_id)
+                exit_code = -1
+                return exit_code
+            self.repos_path = data['repos_path']
+            permissions = user.repo_permissions
+            repo_branch_permissions = user.branch_permissions
+            try:
+                exit_code, is_updated = self.serve(
+                    scm_detected, scm_repo, scm_mode, user, permissions,
+                    repo_branch_permissions)
+            except Exception:
+                log.exception('Error occurred during execution of SshWrapper')
+                exit_code = -1
+
+        elif self.command is None and shell is False:
+            log.error('No Command given.')
+            exit_code = -1
+
+        else:
+            log.error('Unhandled Command: "%s" Aborting.', self.command)
+            exit_code = -1
+
+        return exit_code
+
+    def maybe_translate_repo_uid(self, repo_name):
+        _org_name = repo_name
+        if _org_name.startswith('_'):
+            _org_name = _org_name.split('/', 1)[0]
+
+        if repo_name.startswith('_'):
+            org_repo_name = repo_name
+            log.debug('translating UID repo %s', org_repo_name)
+            by_id_match = call_service_api(self.ini_path, {
+                'method': 'service_get_repo_name_by_id',
+                "args": {"repo_id": repo_name}
+            })
+            if by_id_match:
+                repo_name = by_id_match['repo_name']
+                log.debug('translation of UID repo %s got `%s`', org_repo_name, repo_name)
+
+        return repo_name, _org_name
+
+    def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
+        store = self.repos_path
+
+        check_branch_perms = False
+        detect_force_push = False
+
+        if branch_permissions:
+            check_branch_perms = True
+            detect_force_push = True
+
+        log.debug(
+            'VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s',
+            vcs, mode, repo, check_branch_perms)
+
+        # detect if we have to check branch permissions
+        extras = {
+            'detect_force_push': detect_force_push,
+            'check_branch_perms': check_branch_perms,
+            'config': self.ini_path
+        }
+
+        match vcs:
+            case 'hg':
+                server = MercurialServer(
+                    store=store, ini_path=self.ini_path,
+                    repo_name=repo, user=user,
+                    user_permissions=permissions, config=self.config, env=self.env)
+            case 'git':
+                server = GitServer(
+                    store=store, ini_path=self.ini_path,
+                    repo_name=repo, repo_mode=mode, user=user,
+                    user_permissions=permissions, config=self.config, env=self.env)
+            case 'svn':
+                server = SubversionServer(
+                    store=store, ini_path=self.ini_path,
+                    repo_name=None, user=user,
+                    user_permissions=permissions, config=self.config, env=self.env)
+            case _:
+                raise Exception(f'Unrecognised VCS: {vcs}')
+        self.server_impl = server
+        return server.run(tunnel_extras=extras)
diff --git a/rhodecode/apps/ssh_support/lib/backends/base.py b/rhodecode/apps/ssh_support/lib/backends/base.py
index c320d736..620232d7 100644
--- a/rhodecode/apps/ssh_support/lib/backends/base.py
+++ b/rhodecode/apps/ssh_support/lib/backends/base.py
@@ -23,6 +23,7 @@ import logging
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.vcs.conf import settings as vcs_settings
+from rhodecode.lib.utils import call_service_api
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
@@ -47,6 +48,7 @@ class VcsServer(object):
         self.repo_mode = None
         self.store = ''
         self.ini_path = ''
+        self.hooks_protocol = None
 
     def _invalidate_cache(self, repo_name):
         """
@@ -54,7 +56,15 @@ class VcsServer(object):
 
         :param repo_name: full repo name, also a cache key
         """
-        ScmModel().mark_for_invalidation(repo_name)
+        # Todo: Leave only "celery" case after transition.
+        match self.hooks_protocol:
+            case 'http':
+                ScmModel().mark_for_invalidation(repo_name)
+            case 'celery':
+                call_service_api(self.ini_path, {
+                    "method": "service_mark_for_invalidation",
+                    "args": {"repo_name": repo_name}
+                })
 
     def has_write_perm(self):
         permission = self.user_permissions.get(self.repo_name)
@@ -65,30 +75,31 @@ class VcsServer(object):
 
     def _check_permissions(self, action):
         permission = self.user_permissions.get(self.repo_name)
+        user_info = f'{self.user["user_id"]}:{self.user["username"]}'
         log.debug('permission for %s on %s are: %s',
-                  self.user, self.repo_name, permission)
+                  user_info, self.repo_name, permission)
 
         if not permission:
             log.error('user `%s` permissions to repo:%s are empty. Forbidding access.',
-                      self.user, self.repo_name)
+                      user_info, self.repo_name)
             return -2
 
         if action == 'pull':
             if permission in self.read_perms:
                 log.info(
                     'READ Permissions for User "%s" detected to repo "%s"!',
-                    self.user, self.repo_name)
+                    user_info, self.repo_name)
                 return 0
         else:
             if permission in self.write_perms:
                 log.info(
                     'WRITE, or Higher Permissions for User "%s" detected to repo "%s"!',
-                    self.user, self.repo_name)
+                    user_info, self.repo_name)
                 return 0
 
         log.error('Cannot properly fetch or verify user `%s` permissions. '
                   'Permissions: %s, vcs action: %s',
-                  self.user, permission, action)
+                  user_info, permission, action)
         return -2
 
     def update_environment(self, action, extras=None):
@@ -134,9 +145,10 @@ class VcsServer(object):
         if exit_code:
             return exit_code, False
 
-        req = self.env['request']
-        server_url = req.host_url + req.script_name
-        extras['server_url'] = server_url
+        req = self.env.get('request')
+        if req:
+            server_url = req.host_url + req.script_name
+            extras['server_url'] = server_url
 
         log.debug('Using %s binaries from path %s', self.backend, self._path)
         exit_code = self.tunnel.run(extras)
@@ -144,12 +156,13 @@ class VcsServer(object):
         return exit_code, action == "push"
 
     def run(self, tunnel_extras=None):
+        self.hooks_protocol = self.config.get('app:main', 'vcs.hooks.protocol')
         tunnel_extras = tunnel_extras or {}
         extras = {}
         extras.update(tunnel_extras)
 
         callback_daemon, extras = prepare_callback_daemon(
-            extras, protocol=vcs_settings.HOOKS_PROTOCOL,
+            extras, protocol=self.hooks_protocol,
             host=vcs_settings.HOOKS_HOST)
 
         with callback_daemon:
diff --git a/rhodecode/apps/ssh_support/lib/backends/hg.py b/rhodecode/apps/ssh_support/lib/backends/hg.py
index 4378a60a..aff94301 100644
--- a/rhodecode/apps/ssh_support/lib/backends/hg.py
+++ b/rhodecode/apps/ssh_support/lib/backends/hg.py
@@ -23,6 +23,7 @@ import tempfile
 import textwrap
 import collections
 from .base import VcsServer
+from rhodecode.lib.utils import call_service_api
 from rhodecode.model.db import RhodeCodeUi
 from rhodecode.model.settings import VcsSettingsModel
 
@@ -108,6 +109,14 @@ class MercurialServer(VcsServer):
         self.tunnel = MercurialTunnelWrapper(server=self)
 
     def config_to_hgrc(self, repo_name):
+        # Todo: once transition is done only call to service api should exist
+        if self.hooks_protocol == 'celery':
+            data = call_service_api(self.ini_path, {
+                "method": "service_config_to_hgrc",
+                "args": {"cli_flags": self.cli_flags, "repo_name": repo_name}
+            })
+            return data['flags']
+
         ui_sections = collections.defaultdict(list)
         ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
 
diff --git a/rhodecode/apps/ssh_support/lib/ssh_wrapper.py b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py
similarity index 100%
rename from rhodecode/apps/ssh_support/lib/ssh_wrapper.py
rename to rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py
diff --git a/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
new file mode 100644
index 00000000..f9383f86
--- /dev/null
+++ b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
@@ -0,0 +1,72 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import sys
+import time
+import logging
+
+import click
+
+from pyramid.paster import setup_logging
+
+from rhodecode.lib.statsd_client import StatsdClient
+from .backends import SshWrapperStandalone
+from .ssh_wrapper_v1 import setup_custom_logging
+
+log = logging.getLogger(__name__)
+
+
+@click.command()
+@click.argument('ini_path', type=click.Path(exists=True))
+@click.option(
+    '--mode', '-m', required=False, default='auto',
+    type=click.Choice(['auto', 'vcs', 'git', 'hg', 'svn', 'test']),
+    help='mode of operation')
+@click.option('--user', help='Username for which the command will be executed')
+@click.option('--user-id', help='User ID for which the command will be executed')
+@click.option('--key-id', help='ID of the key from the database')
+@click.option('--shell', '-s', is_flag=True, help='Allow Shell')
+@click.option('--debug', is_flag=True, help='Enabled detailed output logging')
+def main(ini_path, mode, user, user_id, key_id, shell, debug):
+    setup_custom_logging(ini_path, debug)
+
+    command = os.environ.get('SSH_ORIGINAL_COMMAND', '')
+    if not command and mode not in ['test']:
+        raise ValueError(
+            'Unable to fetch SSH_ORIGINAL_COMMAND from environment.'
+            'Please make sure this is set and available during execution '
+            'of this script.')
+    connection_info = os.environ.get('SSH_CONNECTION', '')
+    time_start = time.time()
+    env = {'RC_CMD_SSH_WRAPPER': '1'}
+    statsd = StatsdClient.statsd
+    try:
+        ssh_wrapper = SshWrapperStandalone(
+            command, connection_info, mode,
+            user, user_id, key_id, shell, ini_path, env)
+    except Exception:
+        log.exception('Failed to execute SshWrapper')
+        sys.exit(-5)
+    return_code = ssh_wrapper.wrap()
+    operation_took = time.time() - time_start
+    if statsd:
+        operation_took_ms = round(1000.0 * operation_took)
+        statsd.timing("rhodecode_ssh_wrapper_timing.histogram", operation_took_ms,
+                      use_decimals=False)
+    sys.exit(return_code)
diff --git a/rhodecode/apps/ssh_support/tests/conftest.py b/rhodecode/apps/ssh_support/tests/conftest.py
index 88c0ca0f..3b581b47 100644
--- a/rhodecode/apps/ssh_support/tests/conftest.py
+++ b/rhodecode/apps/ssh_support/tests/conftest.py
@@ -20,7 +20,7 @@ import os
 import pytest
 import configparser
 
-from rhodecode.apps.ssh_support.lib.ssh_wrapper import SshWrapper
+from rhodecode.apps.ssh_support.lib.ssh_wrapper_v1 import SshWrapper
 from rhodecode.lib.utils2 import AttributeDict
 
 
diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py
index 6e15197c..a5f8b11a 100644
--- a/rhodecode/lib/utils.py
+++ b/rhodecode/lib/utils.py
@@ -34,6 +34,7 @@ import tarfile
 import warnings
 from functools import wraps
 from os.path import join as jn
+from configparser import NoOptionError
 
 import paste
 import pkg_resources
@@ -52,6 +53,9 @@ from rhodecode.model import meta
 from rhodecode.model.db import (
     Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
 from rhodecode.model.meta import Session
+from rhodecode.lib.pyramid_utils import get_config
+from rhodecode.lib.vcs import CurlSession
+from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
 
 
 log = logging.getLogger(__name__)
@@ -821,3 +825,27 @@ def send_test_email(recipients, email_body='TEST EMAIL'):
     email_body = email_body_plaintext = email_body
     subject = f'SUBJECT FROM: {socket.gethostname()}'
     tasks.send_email(recipients, subject, email_body_plaintext, email_body)
+
+
+def call_service_api(ini_path, payload):
+    config = get_config(ini_path)
+    try:
+        host = config.get('app:main', 'app.service_api.host')
+    except NoOptionError:
+        raise ImproperlyConfiguredError(
+            "app.service_api.host is missing. "
+            "Please ensure that app.service_api.host and app.service_api.token are "
+            "defined inside of .ini configuration file."
+        )
+    api_url = config.get('app:main', 'rhodecode.api.url')
+    payload.update({
+        'id': 'service',
+        'auth_token': config.get('app:main', 'app.service_api.token')
+    })
+
+    response = CurlSession().post(f'{host}{api_url}', json.dumps(payload))
+
+    if response.status_code != 200:
+        raise Exception("Service API responded with error")
+
+    return json.loads(response.content)['result']
diff --git a/rhodecode/lib/vcs/exceptions.py b/rhodecode/lib/vcs/exceptions.py
index b95ea5cf..55a1f838 100644
--- a/rhodecode/lib/vcs/exceptions.py
+++ b/rhodecode/lib/vcs/exceptions.py
@@ -146,6 +146,10 @@ class CommandError(VCSError):
     pass
 
 
+class ImproperlyConfiguredError(Exception):
+    pass
+
+
 class UnhandledException(VCSError):
     """
     Signals that something unexpected went wrong.
diff --git a/rhodecode/tests/fixture_mods/fixture_pyramid.py b/rhodecode/tests/fixture_mods/fixture_pyramid.py
index 3c5999b8..4c3450cd 100644
--- a/rhodecode/tests/fixture_mods/fixture_pyramid.py
+++ b/rhodecode/tests/fixture_mods/fixture_pyramid.py
@@ -110,6 +110,7 @@ def ini_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
             'vcs.scm_app_implementation': 'http',
             'vcs.hooks.protocol': 'http',
             'vcs.hooks.host': '*',
+            'app.service_api.token': 'service_secret_token',
         }},
 
         {'handler_console': {
diff --git a/setup.py b/setup.py
index 373cac0f..a5acd918 100644
--- a/setup.py
+++ b/setup.py
@@ -196,7 +196,8 @@ setup(
             'rc-upgrade-db=rhodecode.lib.rc_commands.upgrade_db:main',
             'rc-ishell=rhodecode.lib.rc_commands.ishell:main',
             'rc-add-artifact=rhodecode.lib.rc_commands.add_artifact:main',
-            'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper:main',
+            'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper_v1:main',
+            'rc-ssh-wrapper-v2=rhodecode.apps.ssh_support.lib.ssh_wrapper_v2:main',
         ],
         'beaker.backends': [
             'memorylru_base=rhodecode.lib.memory_lru_dict:MemoryLRUNamespaceManagerBase',

From 0b735351409b84b92686901a7c22a7aedcad9ec2 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 27 Feb 2024 09:44:28 +0100
Subject: [PATCH 011/150] settings: remove links to rccontrol from settings and
 default to the docker image paths

---
 rhodecode/apps/ssh_support/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/rhodecode/apps/ssh_support/__init__.py b/rhodecode/apps/ssh_support/__init__.py
index ab536301..fe522ec6 100644
--- a/rhodecode/apps/ssh_support/__init__.py
+++ b/rhodecode/apps/ssh_support/__init__.py
@@ -42,9 +42,9 @@ def _sanitize_settings_and_apply_defaults(settings):
     settings_maker.make_setting(config_keys.wrapper_cmd, '')
     settings_maker.make_setting(config_keys.authorized_keys_line_ssh_opts, '')
 
-    settings_maker.make_setting(config_keys.ssh_hg_bin, '~/.rccontrol/vcsserver-1/profile/bin/hg')
-    settings_maker.make_setting(config_keys.ssh_git_bin, '~/.rccontrol/vcsserver-1/profile/bin/git')
-    settings_maker.make_setting(config_keys.ssh_svn_bin, '~/.rccontrol/vcsserver-1/profile/bin/svnserve')
+    settings_maker.make_setting(config_keys.ssh_hg_bin, '/usr/local/bin/rhodecode_bin/vcs_bin/hg')
+    settings_maker.make_setting(config_keys.ssh_git_bin, '/usr/local/bin/rhodecode_bin/vcs_bin/git')
+    settings_maker.make_setting(config_keys.ssh_svn_bin, '/usr/local/bin/rhodecode_bin/vcs_bin/svnserve')
 
     settings_maker.env_expand()
 

From 2f216c3acf082ef660f4038de87021642b08c271 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 27 Feb 2024 09:48:58 +0100
Subject: [PATCH 012/150] feat(ssh-wrapeprs): moved the setup logging into
 common function to not relly on v1 codebase

---
 .../apps/ssh_support/lib/ssh_wrapper_v1.py    | 16 +--------
 .../apps/ssh_support/lib/ssh_wrapper_v2.py    |  4 +--
 rhodecode/apps/ssh_support/lib/utils.py       | 34 +++++++++++++++++++
 3 files changed, 36 insertions(+), 18 deletions(-)
 create mode 100644 rhodecode/apps/ssh_support/lib/utils.py

diff --git a/rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py
index bf05e723..34da4267 100644
--- a/rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py
+++ b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py
@@ -23,28 +23,14 @@ import logging
 
 import click
 
-from pyramid.paster import setup_logging
-
 from rhodecode.lib.pyramid_utils import bootstrap
 from rhodecode.lib.statsd_client import StatsdClient
 from .backends import SshWrapper
+from .utils import setup_custom_logging
 
 log = logging.getLogger(__name__)
 
 
-def setup_custom_logging(ini_path, debug):
-    if debug:
-        # enabled rhodecode.ini controlled logging setup
-        setup_logging(ini_path)
-    else:
-        # configure logging in a mode that doesn't print anything.
-        # in case of regularly configured logging it gets printed out back
-        # to the client doing an SSH command.
-        logger = logging.getLogger('')
-        null = logging.NullHandler()
-        # add the handler to the root logger
-        logger.handlers = [null]
-
 
 @click.command()
 @click.argument('ini_path', type=click.Path(exists=True))
diff --git a/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
index f9383f86..e76a88c5 100644
--- a/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
+++ b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
@@ -23,11 +23,9 @@ import logging
 
 import click
 
-from pyramid.paster import setup_logging
-
 from rhodecode.lib.statsd_client import StatsdClient
 from .backends import SshWrapperStandalone
-from .ssh_wrapper_v1 import setup_custom_logging
+from .utils import setup_custom_logging
 
 log = logging.getLogger(__name__)
 
diff --git a/rhodecode/apps/ssh_support/lib/utils.py b/rhodecode/apps/ssh_support/lib/utils.py
new file mode 100644
index 00000000..977e4eff
--- /dev/null
+++ b/rhodecode/apps/ssh_support/lib/utils.py
@@ -0,0 +1,34 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+from pyramid.paster import setup_logging
+
+
+def setup_custom_logging(ini_path, debug):
+    if debug:
+        # enabled rhodecode.ini controlled logging setup
+        setup_logging(ini_path)
+    else:
+        # configure logging in a mode that doesn't print anything.
+        # in case of regularly configured logging it gets printed out back
+        # to the client doing an SSH command.
+        logger = logging.getLogger('')
+        null = logging.NullHandler()
+        # add the handler to the root logger
+        logger.handlers = [null]

From 8de3597958480d7df61b7651b455cf232f093aa6 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 27 Feb 2024 09:50:56 +0100
Subject: [PATCH 013/150] fix(api.url): set default api.url and re-use defaults
 in ssh wrappers

---
 rhodecode/api/__init__.py      | 3 +--
 rhodecode/config/middleware.py | 4 ++--
 rhodecode/lib/utils.py         | 8 +++++++-
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/rhodecode/api/__init__.py b/rhodecode/api/__init__.py
index 9a2f63e5..965583c3 100644
--- a/rhodecode/api/__init__.py
+++ b/rhodecode/api/__init__.py
@@ -22,7 +22,6 @@ import sys
 import fnmatch
 
 import decorator
-import typing
 import venusian
 from collections import OrderedDict
 
@@ -45,7 +44,7 @@ from rhodecode.model.db import User, UserApiKeys
 log = logging.getLogger(__name__)
 
 DEFAULT_RENDERER = 'jsonrpc_renderer'
-DEFAULT_URL = '/_admin/apiv2'
+DEFAULT_URL = '/_admin/api'
 SERVICE_API_IDENTIFIER = 'service_'
 
 
diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py
index e21b0b01..aa78fcf7 100644
--- a/rhodecode/config/middleware.py
+++ b/rhodecode/config/middleware.py
@@ -26,13 +26,13 @@ import logging.config
 from paste.gzipper import make_gzip_middleware
 import pyramid.events
 from pyramid.wsgi import wsgiapp
-from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid.config import Configurator
 from pyramid.settings import asbool, aslist
 from pyramid.httpexceptions import (
     HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
 from pyramid.renderers import render_to_response
 
+from rhodecode import api
 from rhodecode.model import meta
 from rhodecode.config import patches
 from rhodecode.config import utils as config_utils
@@ -527,7 +527,7 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
-    settings_maker.make_setting('rhodecode.api.url', '/_admin/api')
+    settings_maker.make_setting('rhodecode.api.url', api.DEFAULT_URL)
 
     # Sanitize generic settings.
     settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py
index a5f8b11a..1f8c383f 100644
--- a/rhodecode/lib/utils.py
+++ b/rhodecode/lib/utils.py
@@ -837,7 +837,13 @@ def call_service_api(ini_path, payload):
             "Please ensure that app.service_api.host and app.service_api.token are "
             "defined inside of .ini configuration file."
         )
-    api_url = config.get('app:main', 'rhodecode.api.url')
+    try:
+        api_url = config.get('app:main', 'rhodecode.api.url')
+    except NoOptionError:
+        from rhodecode import api
+        log.debug('Cannot find rhodecode.api.url, setting API URL TO Default value')
+        api_url = api.DEFAULT_URL
+
     payload.update({
         'id': 'service',
         'auth_token': config.get('app:main', 'app.service_api.token')

From fb6b7a3692890a42c740217baf83f71c8f6154cd Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 27 Feb 2024 09:51:41 +0100
Subject: [PATCH 014/150] chore(imports): optimize imports for ssh events

---
 rhodecode/apps/admin/views/permissions.py              | 2 +-
 rhodecode/apps/admin/views/users.py                    | 2 +-
 rhodecode/apps/my_account/views/my_account_ssh_keys.py | 2 +-
 rhodecode/apps/ssh_support/__init__.py                 | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/rhodecode/apps/admin/views/permissions.py b/rhodecode/apps/admin/views/permissions.py
index ee13c1cc..14631216 100644
--- a/rhodecode/apps/admin/views/permissions.py
+++ b/rhodecode/apps/admin/views/permissions.py
@@ -28,7 +28,7 @@ from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import BaseAppView, DataGridAppView
-from rhodecode.apps.ssh_support import SshKeyFileChangeEvent
+from rhodecode.apps.ssh_support.events import SshKeyFileChangeEvent
 from rhodecode import events
 
 from rhodecode.lib import helpers as h
diff --git a/rhodecode/apps/admin/views/users.py b/rhodecode/apps/admin/views/users.py
index f58c94ed..f116c77b 100644
--- a/rhodecode/apps/admin/views/users.py
+++ b/rhodecode/apps/admin/views/users.py
@@ -27,7 +27,7 @@ from pyramid.response import Response
 
 from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView, UserAppView
-from rhodecode.apps.ssh_support import SshKeyFileChangeEvent
+from rhodecode.apps.ssh_support.events import SshKeyFileChangeEvent
 from rhodecode.authentication.base import get_authn_registry, RhodeCodeExternalAuthPlugin
 from rhodecode.authentication.plugins import auth_rhodecode
 from rhodecode.events import trigger
diff --git a/rhodecode/apps/my_account/views/my_account_ssh_keys.py b/rhodecode/apps/my_account/views/my_account_ssh_keys.py
index 22788aec..8887f280 100644
--- a/rhodecode/apps/my_account/views/my_account_ssh_keys.py
+++ b/rhodecode/apps/my_account/views/my_account_ssh_keys.py
@@ -21,7 +21,7 @@ import logging
 from pyramid.httpexceptions import HTTPFound
 
 from rhodecode.apps._base import BaseAppView, DataGridAppView
-from rhodecode.apps.ssh_support import SshKeyFileChangeEvent
+from rhodecode.apps.ssh_support.events import SshKeyFileChangeEvent
 from rhodecode.events import trigger
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
diff --git a/rhodecode/apps/ssh_support/__init__.py b/rhodecode/apps/ssh_support/__init__.py
index fe522ec6..4adcadbb 100644
--- a/rhodecode/apps/ssh_support/__init__.py
+++ b/rhodecode/apps/ssh_support/__init__.py
@@ -19,8 +19,6 @@
 import logging
 
 from . import config_keys
-from .events import SshKeyFileChangeEvent
-from .subscribers import generate_ssh_authorized_keys_file_subscriber
 
 from rhodecode.config.settings_maker import SettingsMaker
 
@@ -55,5 +53,7 @@ def includeme(config):
 
     # if we have enable generation of file, subscribe to event
     if settings[config_keys.generate_authorized_keyfile]:
+        # lazy import here for faster code reading... via sshwrapper-v2 mode
+        from .subscribers import generate_ssh_authorized_keys_file_subscriber
         config.add_subscriber(
             generate_ssh_authorized_keys_file_subscriber, SshKeyFileChangeEvent)

From 5d6400eec0692ba798ec42c0a6fec06253fb2a3d Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 27 Feb 2024 09:52:29 +0100
Subject: [PATCH 015/150] fix(service-api): rely on urljoin for constructing
 the call url

---
 rhodecode/lib/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py
index 1f8c383f..01902210 100644
--- a/rhodecode/lib/utils.py
+++ b/rhodecode/lib/utils.py
@@ -21,6 +21,7 @@ Utilities library for RhodeCode
 """
 
 import datetime
+
 import decorator
 import logging
 import os
@@ -31,6 +32,7 @@ import socket
 import tempfile
 import traceback
 import tarfile
+import urllib.parse
 import warnings
 from functools import wraps
 from os.path import join as jn
@@ -849,7 +851,7 @@ def call_service_api(ini_path, payload):
         'auth_token': config.get('app:main', 'app.service_api.token')
     })
 
-    response = CurlSession().post(f'{host}{api_url}', json.dumps(payload))
+    response = CurlSession().post(urllib.parse.urljoin(host, api_url), json.dumps(payload))
 
     if response.status_code != 200:
         raise Exception("Service API responded with error")

From 93aa6295389bea790f5732f2e992397487138c5f Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 27 Feb 2024 10:32:01 +0100
Subject: [PATCH 016/150] feat(statsd): added extra logging into statsd
 initalization

---
 rhodecode/lib/_vendor/statsd/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rhodecode/lib/_vendor/statsd/__init__.py b/rhodecode/lib/_vendor/statsd/__init__.py
index 84a54797..2ef9a87c 100644
--- a/rhodecode/lib/_vendor/statsd/__init__.py
+++ b/rhodecode/lib/_vendor/statsd/__init__.py
@@ -26,9 +26,10 @@ def client_from_config(configuration, prefix='statsd.', **kwargs):
     from pyramid.settings import asbool
 
     _config = statsd_config(configuration, prefix)
+    statsd_flag = _config.get('enabled')
     statsd_enabled = asbool(_config.pop('enabled', False))
     if not statsd_enabled:
-        log.debug('statsd client not enabled by statsd.enabled =  flag, skipping...')
+        log.debug('statsd client not enabled by statsd.enabled = %s flag, skipping...', statsd_flag)
         return
 
     host = _config.pop('statsd_host', HOST)

From aadbbc076c39500c4bb53a7844dfc0452b527467 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 27 Feb 2024 10:41:51 +0100
Subject: [PATCH 017/150] fix(ssh): added missing import

---
 rhodecode/apps/ssh_support/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/rhodecode/apps/ssh_support/__init__.py b/rhodecode/apps/ssh_support/__init__.py
index 4adcadbb..65e330f5 100644
--- a/rhodecode/apps/ssh_support/__init__.py
+++ b/rhodecode/apps/ssh_support/__init__.py
@@ -55,5 +55,6 @@ def includeme(config):
     if settings[config_keys.generate_authorized_keyfile]:
         # lazy import here for faster code reading... via sshwrapper-v2 mode
         from .subscribers import generate_ssh_authorized_keys_file_subscriber
+        from .events import SshKeyFileChangeEvent
         config.add_subscriber(
             generate_ssh_authorized_keys_file_subscriber, SshKeyFileChangeEvent)

From 3982ae9102cec1d23fa8abbb707d2355bcffcb76 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 27 Feb 2024 15:37:33 +0100
Subject: [PATCH 018/150] fix(pycurl): added missing __all__ imports

---
 rhodecode/lib/vcs/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rhodecode/lib/vcs/__init__.py b/rhodecode/lib/vcs/__init__.py
index 6b2b8e65..1202d514 100644
--- a/rhodecode/lib/vcs/__init__.py
+++ b/rhodecode/lib/vcs/__init__.py
@@ -33,7 +33,8 @@ from rhodecode.lib.vcs.exceptions import (
 
 __all__ = [
     'get_vcs_instance', 'get_backend',
-    'VCSError', 'RepositoryError', 'CommitError', 'VCSCommunicationError'
+    'VCSError', 'RepositoryError', 'CommitError', 'VCSCommunicationError',
+    'CurlSession', 'CurlResponse'
 ]
 
 log = logging.getLogger(__name__)

From 25f36e96543bc8cb1d7b289efeca756af72b1ad7 Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Wed, 28 Feb 2024 09:34:14 +0200
Subject: [PATCH 019/150] feat(svn-config): moved svn related config keys to
 *.ini file. Fixes: RCCE-60

---
 configs/development.ini                       |  6 +++
 configs/production.ini                        |  6 +++
 .../repository/tests/test_repo_summary.py     | 37 +++++++++++--------
 .../apps/ssh_support/lib/backends/svn.py      |  7 ++--
 rhodecode/config/middleware.py                |  2 +
 rhodecode/lib/helpers.py                      |  5 +--
 rhodecode/lib/middleware/simplesvn.py         |  7 ++--
 rhodecode/model/forms.py                      |  4 --
 rhodecode/model/settings.py                   | 16 --------
 rhodecode/templates/base/vcs_settings.mako    | 36 ------------------
 .../tests/fixture_mods/fixture_pyramid.py     |  1 +
 11 files changed, 45 insertions(+), 82 deletions(-)

diff --git a/configs/development.ini b/configs/development.ini
index d7a492a2..7b63786e 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -615,6 +615,12 @@ vcs.connection_timeout = 3600
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
+; Enable SVN proxy of requests over HTTP
+vcs.svn.proxy.enabled = true
+
+; host to connect to running SVN subsystem
+vcs.svn.proxy.host = http://svn:8090
+
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
diff --git a/configs/production.ini b/configs/production.ini
index 6ffeeb7d..75a8de1f 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -566,6 +566,12 @@ vcs.connection_timeout = 3600
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
+; Enable SVN proxy of requests over HTTP
+vcs.svn.proxy.enabled = true
+
+; host to connect to running SVN subsystem
+vcs.svn.proxy.host = http://svn:8090
+
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
diff --git a/rhodecode/apps/repository/tests/test_repo_summary.py b/rhodecode/apps/repository/tests/test_repo_summary.py
index deb90971..8fb75b34 100644
--- a/rhodecode/apps/repository/tests/test_repo_summary.py
+++ b/rhodecode/apps/repository/tests/test_repo_summary.py
@@ -1,4 +1,3 @@
-
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
@@ -52,13 +51,13 @@ def assert_clone_url(response, server, repo, disabled=False):
 
 @pytest.mark.usefixtures('app')
 class TestSummaryView(object):
+
     def test_index(self, autologin_user, backend, http_host_only_stub):
         repo_id = backend.repo.repo_id
         repo_name = backend.repo_name
-        with mock.patch('rhodecode.lib.helpers.is_svn_without_proxy',
-                        return_value=False):
-            response = self.app.get(
-                route_path('repo_summary', repo_name=repo_name))
+
+        response = self.app.get(
+            route_path('repo_summary', repo_name=repo_name))
 
         # repo type
         response.mustcontain(
@@ -71,37 +70,43 @@ class TestSummaryView(object):
 
         # clone url...
         assert_clone_url(response, http_host_only_stub, repo_name)
-        assert_clone_url(response, http_host_only_stub, '_{}'.format(repo_id))
+        assert_clone_url(response, http_host_only_stub, f'_{repo_id}')
 
     def test_index_svn_without_proxy(
             self, autologin_user, backend_svn, http_host_only_stub):
+
         repo_id = backend_svn.repo.repo_id
         repo_name = backend_svn.repo_name
-        response = self.app.get(route_path('repo_summary', repo_name=repo_name))
-        # clone url...
 
+        # by default the SVN is enabled now, this is how inputs look when it's disabled
+        with mock.patch('rhodecode.lib.helpers.is_svn_without_proxy', return_value=True):
+
+            response = self.app.get(
+                route_path('repo_summary', repo_name=repo_name),
+                status=200)
+
+        # clone url test...
         assert_clone_url(response, http_host_only_stub, repo_name, disabled=True)
-        assert_clone_url(response, http_host_only_stub, '_{}'.format(repo_id), disabled=True)
+        assert_clone_url(response, http_host_only_stub, f'_{repo_id}', disabled=True)
 
     def test_index_with_trailing_slash(
             self, autologin_user, backend, http_host_only_stub):
 
         repo_id = backend.repo.repo_id
         repo_name = backend.repo_name
-        with mock.patch('rhodecode.lib.helpers.is_svn_without_proxy',
-                        return_value=False):
-            response = self.app.get(
-                route_path('repo_summary', repo_name=repo_name) + '/',
-                status=200)
+        trailing_slash = '/'
+        response = self.app.get(
+            route_path('repo_summary', repo_name=repo_name) + trailing_slash,
+            status=200)
 
         # clone url...
         assert_clone_url(response, http_host_only_stub, repo_name)
-        assert_clone_url(response, http_host_only_stub, '_{}'.format(repo_id))
+        assert_clone_url(response, http_host_only_stub, f'_{repo_id}')
 
     def test_index_by_id(self, autologin_user, backend):
         repo_id = backend.repo.repo_id
         response = self.app.get(
-            route_path('repo_summary', repo_name='_%s' % (repo_id,)))
+            route_path('repo_summary', repo_name=f'_{repo_id}'))
 
         # repo type
         response.mustcontain(
diff --git a/rhodecode/apps/ssh_support/lib/backends/svn.py b/rhodecode/apps/ssh_support/lib/backends/svn.py
index 728ec8ec..6d4ea5de 100644
--- a/rhodecode/apps/ssh_support/lib/backends/svn.py
+++ b/rhodecode/apps/ssh_support/lib/backends/svn.py
@@ -244,9 +244,10 @@ class SubversionServer(VcsServer):
         # if exit_code:
         #     return exit_code, False
 
-        req = self.env['request']
-        server_url = req.host_url + req.script_name
-        extras['server_url'] = server_url
+        req = self.env.get('request')
+        if req:
+            server_url = req.host_url + req.script_name
+            extras['server_url'] = server_url
 
         log.debug('Using %s binaries from path %s', self.backend, self._path)
         exit_code = self.tunnel.run(extras)
diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py
index e21b0b01..2962adee 100644
--- a/rhodecode/config/middleware.py
+++ b/rhodecode/config/middleware.py
@@ -542,6 +542,8 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
     settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
 
     settings_maker.make_setting('vcs.svn.compatible_version', '')
+    settings_maker.make_setting('vcs.svn.proxy.enabled', 'true', parser='bool')
+    settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090')
     settings_maker.make_setting('vcs.hooks.protocol', 'http')
     settings_maker.make_setting('vcs.hooks.host', '*')
     settings_maker.make_setting('vcs.scm_app_implementation', 'http')
diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py
index d8fb96f8..62e05336 100644
--- a/rhodecode/lib/helpers.py
+++ b/rhodecode/lib/helpers.py
@@ -74,6 +74,7 @@ from webhelpers2.html.tags import (
 
 from webhelpers2.number import format_byte_size
 # python3.11 backport fixes for webhelpers2
+from rhodecode import ConfigGet
 from rhodecode.lib._vendor.webhelpers_backports import raw_select
 
 from rhodecode.lib.action_parser import action_parser
@@ -916,9 +917,7 @@ def get_repo_type_by_name(repo_name):
 
 def is_svn_without_proxy(repository):
     if is_svn(repository):
-        from rhodecode.model.settings import VcsSettingsModel
-        conf = VcsSettingsModel().get_ui_settings_as_config_obj()
-        return not str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
+        return not ConfigGet().get_bool('vcs.svn.proxy.enabled')
     return False
 
 
diff --git a/rhodecode/lib/middleware/simplesvn.py b/rhodecode/lib/middleware/simplesvn.py
index 451e1708..0d42e179 100644
--- a/rhodecode/lib/middleware/simplesvn.py
+++ b/rhodecode/lib/middleware/simplesvn.py
@@ -27,6 +27,7 @@ import urllib.parse
 import requests
 from pyramid.httpexceptions import HTTPNotAcceptable
 
+from rhodecode import ConfigGet
 from rhodecode.lib import rc_cache
 from rhodecode.lib.middleware import simplevcs
 from rhodecode.lib.middleware.utils import get_path_info
@@ -232,12 +233,10 @@ class SimpleSvn(simplevcs.SimpleVCS):
         return DisabledSimpleSvnApp(config)
 
     def _is_svn_enabled(self):
-        conf = self.repo_vcs_config
-        return str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
+        return ConfigGet().get_bool('vcs.svn.proxy.enabled')
 
     def _create_config(self, extras, repo_name, scheme='http'):
-        conf = self.repo_vcs_config
-        server_url = conf.get('vcs_svn_proxy', 'http_server_url')
+        server_url = ConfigGet().get_str('vcs.svn.proxy.host')
         server_url = server_url or self.DEFAULT_HTTP_SERVER
 
         extras['subversion_http_server_url'] = server_url
diff --git a/rhodecode/model/forms.py b/rhodecode/model/forms.py
index 39d3b21c..e4b75723 100644
--- a/rhodecode/model/forms.py
+++ b/rhodecode/model/forms.py
@@ -421,10 +421,6 @@ class _BaseVcsSettingsForm(formencode.Schema):
     rhodecode_git_use_rebase_for_merging = v.StringBoolean(if_missing=False)
     rhodecode_git_close_branch_before_merging = v.StringBoolean(if_missing=False)
 
-    # svn
-    vcs_svn_proxy_http_requests_enabled = v.StringBoolean(if_missing=False)
-    vcs_svn_proxy_http_server_url = v.UnicodeString(strip=True, if_missing=None)
-
     # cache
     rhodecode_diff_cache = v.StringBoolean(if_missing=False)
 
diff --git a/rhodecode/model/settings.py b/rhodecode/model/settings.py
index 0beff5fe..1879fa2a 100644
--- a/rhodecode/model/settings.py
+++ b/rhodecode/model/settings.py
@@ -499,11 +499,6 @@ class VcsSettingsModel(object):
         ('vcs_git_lfs', 'store_location')
     )
 
-    GLOBAL_SVN_SETTINGS = (
-        ('vcs_svn_proxy', 'http_requests_enabled'),
-        ('vcs_svn_proxy', 'http_server_url')
-    )
-
     SVN_BRANCH_SECTION = 'vcs_svn_branch'
     SVN_TAG_SECTION = 'vcs_svn_tag'
     SSL_SETTING = ('web', 'push_ssl')
@@ -718,17 +713,6 @@ class VcsSettingsModel(object):
         # branch/tags patterns
         self._create_svn_settings(self.global_settings, data)
 
-        http_requests_enabled, http_server_url = self.GLOBAL_SVN_SETTINGS
-        http_requests_enabled_key, http_server_url_key = self._get_settings_keys(
-            self.GLOBAL_SVN_SETTINGS, data)
-
-        self._create_or_update_ui(
-            self.global_settings, *http_requests_enabled,
-            value=safe_str(data[http_requests_enabled_key]))
-        self._create_or_update_ui(
-            self.global_settings, *http_server_url,
-            value=data[http_server_url_key])
-
     def update_global_ssl_setting(self, value):
         self._create_or_update_ui(
             self.global_settings, *self.SSL_SETTING, value=value)
diff --git a/rhodecode/templates/base/vcs_settings.mako b/rhodecode/templates/base/vcs_settings.mako
index d9b80493..3cfec674 100644
--- a/rhodecode/templates/base/vcs_settings.mako
+++ b/rhodecode/templates/base/vcs_settings.mako
@@ -170,42 +170,6 @@
         </div>
     % endif
 
-
-    % if display_globals:
-        <div class="panel panel-default">
-            <div class="panel-heading" id="vcs-global-svn-options">
-                <h3 class="panel-title">${_('Global Subversion Settings')}<a class="permalink" href="#vcs-global-svn-options"> ¶</a></h3>
-            </div>
-            <div class="panel-body">
-                <div class="field">
-                    <div class="checkbox">
-                        ${h.checkbox('vcs_svn_proxy_http_requests_enabled' + suffix, 'True', **kwargs)}
-                        <label for="vcs_svn_proxy_http_requests_enabled${suffix}">${_('Proxy subversion HTTP requests')}</label>
-                    </div>
-                    <div class="label">
-                        <span class="help-block">
-                            ${_('Subversion HTTP Support. Enables communication with SVN over HTTP protocol.')}
-                            <a href="${h.route_url('enterprise_svn_setup')}" target="_blank">${_('SVN Protocol setup Documentation')}</a>.
-                        </span>
-                    </div>
-                </div>
-                <div class="field">
-                    <div class="label">
-                        <label for="vcs_svn_proxy_http_server_url">${_('Subversion HTTP Server URL')}</label><br/>
-                    </div>
-                    <div class="input">
-                        ${h.text('vcs_svn_proxy_http_server_url',size=59)}
-                        % if c.svn_proxy_generate_config:
-                            <span class="buttons">
-                                <button class="btn btn-primary" id="vcs_svn_generate_cfg">${_('Generate Apache Config')}</button>
-                            </span>
-                        % endif
-                    </div>
-                </div>
-            </div>
-        </div>
-    % endif
-
     % if display_globals or repo_type in ['svn']:
         <div class="panel panel-default">
             <div class="panel-heading" id="vcs-svn-options">
diff --git a/rhodecode/tests/fixture_mods/fixture_pyramid.py b/rhodecode/tests/fixture_mods/fixture_pyramid.py
index 4c3450cd..b896f4f0 100644
--- a/rhodecode/tests/fixture_mods/fixture_pyramid.py
+++ b/rhodecode/tests/fixture_mods/fixture_pyramid.py
@@ -108,6 +108,7 @@ def ini_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
 
             'vcs.server.protocol': 'http',
             'vcs.scm_app_implementation': 'http',
+            'vcs.svn.proxy.enabled': 'true',
             'vcs.hooks.protocol': 'http',
             'vcs.hooks.host': '*',
             'app.service_api.token': 'service_secret_token',

From cdc40e48c7f75d774fd6a87dc71d3ee9d7858c97 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 28 Feb 2024 09:29:49 +0100
Subject: [PATCH 020/150] statsd: added special repr for statsdclient for nicer
 debug

---
 rhodecode/lib/statsd_client.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/rhodecode/lib/statsd_client.py b/rhodecode/lib/statsd_client.py
index 7cc0a96d..a5959f83 100644
--- a/rhodecode/lib/statsd_client.py
+++ b/rhodecode/lib/statsd_client.py
@@ -25,6 +25,9 @@ class StatsdClientClass(Singleton):
     statsd_client = None
     statsd = None
 
+    def __repr__(self):
+        return f"{self.__class__}(statsd={self.statsd})"
+
     def __getattribute__(self, name):
 
         if name.startswith("statsd"):

From d57482be6221b6c89b3aa0462457a3c802cf6c31 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 28 Feb 2024 09:30:11 +0100
Subject: [PATCH 021/150] fix(pycurl): report nicer error log on failed calls
 in pycurl

---
 rhodecode/lib/vcs/__init__.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/rhodecode/lib/vcs/__init__.py b/rhodecode/lib/vcs/__init__.py
index 1202d514..da7de79b 100644
--- a/rhodecode/lib/vcs/__init__.py
+++ b/rhodecode/lib/vcs/__init__.py
@@ -136,7 +136,12 @@ class CurlSession(object):
         curl.setopt(curl.FOLLOWLOCATION, allow_redirects)
         curl.setopt(curl.WRITEDATA, response_buffer)
         curl.setopt(curl.HTTPHEADER, headers_list)
-        curl.perform()
+
+        try:
+            curl.perform()
+        except pycurl.error as exc:
+            log.error('Failed to call endpoint url: {} using pycurl'.format(url))
+            raise
 
         status_code = curl.getinfo(pycurl.HTTP_CODE)
         content_type = curl.getinfo(pycurl.CONTENT_TYPE)

From fc0ee0e99bd416bba464f6e5809e605cb1341310 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 28 Feb 2024 09:30:58 +0100
Subject: [PATCH 022/150] feat(ssh-wrapper-speedup): major rewrite of code to
 address imports problem with ssh-wrapper-v2

- use bootstrapped settings rather than config
- use more code split to make sure we don't import heavy code
---
 .../apps/ssh_support/lib/backends/__init__.py |  63 ++++--
 .../apps/ssh_support/lib/backends/base.py     |  23 +-
 .../apps/ssh_support/lib/backends/git.py      |  12 +-
 rhodecode/apps/ssh_support/lib/backends/hg.py |  85 ++++----
 .../apps/ssh_support/lib/backends/svn.py      |  12 +-
 .../apps/ssh_support/lib/ssh_wrapper_v1.py    |   4 +-
 .../apps/ssh_support/lib/ssh_wrapper_v2.py    |  32 ++-
 rhodecode/apps/ssh_support/lib/utils.py       |   2 +-
 rhodecode/apps/ssh_support/tests/conftest.py  |   7 +-
 .../apps/ssh_support/tests/test_server_git.py |  10 +-
 .../apps/ssh_support/tests/test_server_hg.py  |   7 +-
 .../apps/ssh_support/tests/test_server_svn.py |   7 +-
 .../ssh_support/tests/test_ssh_wrapper.py     |   4 -
 rhodecode/config/config_maker.py              | 198 ++++++++++++++++++
 rhodecode/config/middleware.py                | 177 +---------------
 rhodecode/config/utils.py                     |   3 +-
 rhodecode/lib/api_utils.py                    |  38 ++++
 rhodecode/lib/base.py                         |   2 +-
 rhodecode/lib/config_utils.py                 |  40 ++++
 rhodecode/lib/hook_daemon/__init__.py         |  17 ++
 rhodecode/lib/hook_daemon/base.py             | 115 ++++++++++
 .../lib/hook_daemon/celery_hooks_deamon.py    |  30 +++
 rhodecode/lib/hook_daemon/hook_module.py      | 104 +++++++++
 .../http_hooks_deamon.py}                     | 177 +---------------
 rhodecode/lib/middleware/simplesvn.py         |   3 +-
 rhodecode/lib/middleware/simplevcs.py         |   2 +-
 rhodecode/lib/pyramid_utils.py                |  16 +-
 rhodecode/lib/rc_commands/setup_rc.py         |   3 +-
 rhodecode/lib/utils.py                        |  37 +---
 rhodecode/model/pull_request.py               |   2 +-
 .../tests/fixture_mods/fixture_pyramid.py     |   2 +-
 rhodecode/tests/fixture_mods/fixture_utils.py |   2 +-
 rhodecode/tests/lib/test_hooks_daemon.py      |  90 ++++----
 33 files changed, 764 insertions(+), 562 deletions(-)
 create mode 100644 rhodecode/config/config_maker.py
 create mode 100644 rhodecode/lib/api_utils.py
 create mode 100644 rhodecode/lib/config_utils.py
 create mode 100644 rhodecode/lib/hook_daemon/__init__.py
 create mode 100644 rhodecode/lib/hook_daemon/base.py
 create mode 100644 rhodecode/lib/hook_daemon/celery_hooks_deamon.py
 create mode 100644 rhodecode/lib/hook_daemon/hook_module.py
 rename rhodecode/lib/{hooks_daemon.py => hook_daemon/http_hooks_deamon.py} (61%)

diff --git a/rhodecode/apps/ssh_support/lib/backends/__init__.py b/rhodecode/apps/ssh_support/lib/backends/__init__.py
index d4479be6..f0a3e80e 100644
--- a/rhodecode/apps/ssh_support/lib/backends/__init__.py
+++ b/rhodecode/apps/ssh_support/lib/backends/__init__.py
@@ -20,12 +20,11 @@ import os
 import re
 import logging
 import datetime
-import configparser
 from sqlalchemy import Table
 
-from rhodecode.lib.utils import call_service_api
+from rhodecode.lib.api_utils import call_service_api
 from rhodecode.lib.utils2 import AttributeDict
-from rhodecode.model.scm import ScmModel
+from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
 
 from .hg import MercurialServer
 from .git import GitServer
@@ -39,7 +38,7 @@ class SshWrapper(object):
     svn_cmd_pat = re.compile(r'^svnserve -t')
 
     def __init__(self, command, connection_info, mode,
-                 user, user_id, key_id: int, shell, ini_path: str, env):
+                 user, user_id, key_id: int, shell, ini_path: str, settings, env):
         self.command = command
         self.connection_info = connection_info
         self.mode = mode
@@ -49,15 +48,9 @@ class SshWrapper(object):
         self.shell = shell
         self.ini_path = ini_path
         self.env = env
-
-        self.config = self.parse_config(ini_path)
+        self.settings = settings
         self.server_impl = None
 
-    def parse_config(self, config_path):
-        parser = configparser.ConfigParser()
-        parser.read(config_path)
-        return parser
-
     def update_key_access_time(self, key_id):
         from rhodecode.model.meta import raw_query_executor, Base
 
@@ -162,6 +155,9 @@ class SshWrapper(object):
         return vcs_type, repo_name, mode
 
     def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
+        # TODO: remove this once we have .ini defined access path...
+        from rhodecode.model.scm import ScmModel
+
         store = ScmModel().repos_path
 
         check_branch_perms = False
@@ -186,7 +182,7 @@ class SshWrapper(object):
             server = MercurialServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=repo, user=user,
-                user_permissions=permissions, config=self.config, env=self.env)
+                user_permissions=permissions, settings=self.settings, env=self.env)
             self.server_impl = server
             return server.run(tunnel_extras=extras)
 
@@ -194,7 +190,7 @@ class SshWrapper(object):
             server = GitServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=repo, repo_mode=mode, user=user,
-                user_permissions=permissions, config=self.config, env=self.env)
+                user_permissions=permissions, settings=self.settings, env=self.env)
             self.server_impl = server
             return server.run(tunnel_extras=extras)
 
@@ -202,7 +198,7 @@ class SshWrapper(object):
             server = SubversionServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=None, user=user,
-                user_permissions=permissions, config=self.config, env=self.env)
+                user_permissions=permissions, settings=self.settings, env=self.env)
             self.server_impl = server
             return server.run(tunnel_extras=extras)
 
@@ -269,6 +265,35 @@ class SshWrapperStandalone(SshWrapper):
     New version of SshWrapper designed to be depended only on service API
     """
     repos_path = None
+    service_api_host: str
+    service_api_token: str
+    api_url: str
+
+    def __init__(self, command, connection_info, mode,
+                 user, user_id, key_id: int, shell, ini_path: str, settings, env):
+
+        # validate our settings for making a standalone calls
+        try:
+            self.service_api_host = settings['app.service_api.host']
+            self.service_api_token = settings['app.service_api.token']
+        except KeyError:
+            raise ImproperlyConfiguredError(
+                "app.service_api.host or app.service_api.token are missing. "
+                "Please ensure that app.service_api.host and app.service_api.token are "
+                "defined inside of .ini configuration file."
+            )
+
+        try:
+            self.api_url = settings['rhodecode.api.url']
+        except KeyError:
+            raise ImproperlyConfiguredError(
+                "rhodecode.api.url is missing. "
+                "Please ensure that rhodecode.api.url is "
+                "defined inside of .ini configuration file."
+            )
+
+        super(SshWrapperStandalone, self).__init__(
+            command, connection_info, mode, user, user_id, key_id, shell, ini_path, settings, env)
 
     @staticmethod
     def parse_user_related_data(user_data):
@@ -301,7 +326,7 @@ class SshWrapperStandalone(SshWrapper):
             exit_code = 1
 
         elif scm_detected:
-            data = call_service_api(self.ini_path, {
+            data = call_service_api(self.service_api_host, self.service_api_token, self.api_url, {
                 "method": "service_get_data_for_ssh_wrapper",
                 "args": {"user_id": user_id, "repo_name": scm_repo, "key_id": self.key_id}
             })
@@ -339,7 +364,7 @@ class SshWrapperStandalone(SshWrapper):
         if repo_name.startswith('_'):
             org_repo_name = repo_name
             log.debug('translating UID repo %s', org_repo_name)
-            by_id_match = call_service_api(self.ini_path, {
+            by_id_match = call_service_api(self.service_api_host, self.service_api_token, self.api_url, {
                 'method': 'service_get_repo_name_by_id',
                 "args": {"repo_id": repo_name}
             })
@@ -375,17 +400,17 @@ class SshWrapperStandalone(SshWrapper):
                 server = MercurialServer(
                     store=store, ini_path=self.ini_path,
                     repo_name=repo, user=user,
-                    user_permissions=permissions, config=self.config, env=self.env)
+                    user_permissions=permissions, settings=self.settings, env=self.env)
             case 'git':
                 server = GitServer(
                     store=store, ini_path=self.ini_path,
                     repo_name=repo, repo_mode=mode, user=user,
-                    user_permissions=permissions, config=self.config, env=self.env)
+                    user_permissions=permissions, settings=self.settings, env=self.env)
             case 'svn':
                 server = SubversionServer(
                     store=store, ini_path=self.ini_path,
                     repo_name=None, user=user,
-                    user_permissions=permissions, config=self.config, env=self.env)
+                    user_permissions=permissions, settings=self.settings, env=self.env)
             case _:
                 raise Exception(f'Unrecognised VCS: {vcs}')
         self.server_impl = server
diff --git a/rhodecode/apps/ssh_support/lib/backends/base.py b/rhodecode/apps/ssh_support/lib/backends/base.py
index 620232d7..46ffe33d 100644
--- a/rhodecode/apps/ssh_support/lib/backends/base.py
+++ b/rhodecode/apps/ssh_support/lib/backends/base.py
@@ -20,27 +20,27 @@ import os
 import sys
 import logging
 
-from rhodecode.lib.hooks_daemon import prepare_callback_daemon
+from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.vcs.conf import settings as vcs_settings
-from rhodecode.lib.utils import call_service_api
-from rhodecode.model.scm import ScmModel
+from rhodecode.lib.api_utils import call_service_api
 
 log = logging.getLogger(__name__)
 
 
-class VcsServer(object):
+class SSHVcsServer(object):
     repo_user_agent = None  # set in child classes
     _path = None  # set executable path for hg/git/svn binary
     backend = None  # set in child classes
     tunnel = None  # subprocess handling tunnel
+    settings = None  # parsed settings module
     write_perms = ['repository.admin', 'repository.write']
     read_perms = ['repository.read', 'repository.admin', 'repository.write']
 
-    def __init__(self, user, user_permissions, config, env):
+    def __init__(self, user, user_permissions, settings, env):
         self.user = user
         self.user_permissions = user_permissions
-        self.config = config
+        self.settings = settings
         self.env = env
         self.stdin = sys.stdin
 
@@ -59,9 +59,14 @@ class VcsServer(object):
         # Todo: Leave only "celery" case after transition.
         match self.hooks_protocol:
             case 'http':
+                from rhodecode.model.scm import ScmModel
                 ScmModel().mark_for_invalidation(repo_name)
             case 'celery':
-                call_service_api(self.ini_path, {
+                service_api_host = self.settings['app.service_api.host']
+                service_api_token = self.settings['app.service_api.token']
+                api_url = self.settings['rhodecode.api.url']
+
+                call_service_api(service_api_host, service_api_token, api_url, {
                     "method": "service_mark_for_invalidation",
                     "args": {"repo_name": repo_name}
                 })
@@ -118,7 +123,7 @@ class VcsServer(object):
             'server_url': None,
             'user_agent': f'{self.repo_user_agent}/ssh-user-agent',
             'hooks': ['push', 'pull'],
-            'hooks_module': 'rhodecode.lib.hooks_daemon',
+            'hooks_module': 'rhodecode.lib.hook_daemon.hook_module',
             'is_shadow_repo': False,
             'detect_force_push': False,
             'check_branch_perms': False,
@@ -156,7 +161,7 @@ class VcsServer(object):
         return exit_code, action == "push"
 
     def run(self, tunnel_extras=None):
-        self.hooks_protocol = self.config.get('app:main', 'vcs.hooks.protocol')
+        self.hooks_protocol = self.settings['vcs.hooks.protocol']
         tunnel_extras = tunnel_extras or {}
         extras = {}
         extras.update(tunnel_extras)
diff --git a/rhodecode/apps/ssh_support/lib/backends/git.py b/rhodecode/apps/ssh_support/lib/backends/git.py
index 5ee88bab..ffedfb85 100644
--- a/rhodecode/apps/ssh_support/lib/backends/git.py
+++ b/rhodecode/apps/ssh_support/lib/backends/git.py
@@ -21,7 +21,7 @@ import logging
 import subprocess
 
 from vcsserver import hooks
-from .base import VcsServer
+from .base import SSHVcsServer
 
 log = logging.getLogger(__name__)
 
@@ -70,19 +70,17 @@ class GitTunnelWrapper(object):
         return result
 
 
-class GitServer(VcsServer):
+class GitServer(SSHVcsServer):
     backend = 'git'
     repo_user_agent = 'git'
 
-    def __init__(self, store, ini_path, repo_name, repo_mode,
-                 user, user_permissions, config, env):
-        super().\
-            __init__(user, user_permissions, config, env)
+    def __init__(self, store, ini_path, repo_name, repo_mode, user, user_permissions, settings, env):
+        super().__init__(user, user_permissions, settings, env)
 
         self.store = store
         self.ini_path = ini_path
         self.repo_name = repo_name
-        self._path = self.git_path = config.get('app:main', 'ssh.executable.git')
+        self._path = self.git_path = settings['ssh.executable.git']
 
         self.repo_mode = repo_mode
         self.tunnel = GitTunnelWrapper(server=self)
diff --git a/rhodecode/apps/ssh_support/lib/backends/hg.py b/rhodecode/apps/ssh_support/lib/backends/hg.py
index aff94301..e7f7ed80 100644
--- a/rhodecode/apps/ssh_support/lib/backends/hg.py
+++ b/rhodecode/apps/ssh_support/lib/backends/hg.py
@@ -22,10 +22,10 @@ import logging
 import tempfile
 import textwrap
 import collections
-from .base import VcsServer
-from rhodecode.lib.utils import call_service_api
-from rhodecode.model.db import RhodeCodeUi
-from rhodecode.model.settings import VcsSettingsModel
+
+from .base import SSHVcsServer
+
+from rhodecode.lib.api_utils import call_service_api
 
 log = logging.getLogger(__name__)
 
@@ -57,7 +57,7 @@ class MercurialTunnelWrapper(object):
         # cleanup custom hgrc file
         if os.path.isfile(hgrc_custom):
             with open(hgrc_custom, 'wb') as f:
-                f.write('')
+                f.write(b'')
             log.debug('Cleanup custom hgrc file under %s', hgrc_custom)
 
         # write temp
@@ -94,62 +94,67 @@ class MercurialTunnelWrapper(object):
             self.remove_configs()
 
 
-class MercurialServer(VcsServer):
+class MercurialServer(SSHVcsServer):
     backend = 'hg'
     repo_user_agent = 'mercurial'
     cli_flags = ['phases', 'largefiles', 'extensions', 'experimental', 'hooks']
 
-    def __init__(self, store, ini_path, repo_name, user, user_permissions, config, env):
-        super().__init__(user, user_permissions, config, env)
+    def __init__(self, store, ini_path, repo_name, user, user_permissions, settings, env):
+        super().__init__(user, user_permissions, settings, env)
 
         self.store = store
         self.ini_path = ini_path
         self.repo_name = repo_name
-        self._path = self.hg_path = config.get('app:main', 'ssh.executable.hg')
+        self._path = self.hg_path = settings['ssh.executable.hg']
         self.tunnel = MercurialTunnelWrapper(server=self)
 
     def config_to_hgrc(self, repo_name):
         # Todo: once transition is done only call to service api should exist
         if self.hooks_protocol == 'celery':
-            data = call_service_api(self.ini_path, {
+            service_api_host = self.settings['app.service_api.host']
+            service_api_token = self.settings['app.service_api.token']
+            api_url = self.settings['rhodecode.api.url']
+            data = call_service_api(service_api_host, service_api_token, api_url, {
                 "method": "service_config_to_hgrc",
                 "args": {"cli_flags": self.cli_flags, "repo_name": repo_name}
             })
             return data['flags']
+        else:
+            from rhodecode.model.db import RhodeCodeUi
+            from rhodecode.model.settings import VcsSettingsModel
+            ui_sections = collections.defaultdict(list)
+            ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
 
-        ui_sections = collections.defaultdict(list)
-        ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
+            # write default hooks
+            default_hooks = [
+                ('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
+                ('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
+                ('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
 
-        # write default hooks
-        default_hooks = [
-            ('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
-            ('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
-            ('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
+                ('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
+                ('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
+            ]
 
-            ('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
-            ('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
-        ]
+            for k, v in default_hooks:
+                ui_sections['hooks'].append((k, v))
 
-        for k, v in default_hooks:
-            ui_sections['hooks'].append((k, v))
-
-        for entry in ui:
-            if not entry.active:
-                continue
-            sec = entry.section
-            key = entry.key
-
-            if sec in self.cli_flags:
-                # we want only custom hooks, so we skip builtins
-                if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
+            for entry in ui:
+                if not entry.active:
                     continue
+                sec = entry.section
+                key = entry.key
 
-                ui_sections[sec].append([key, entry.value])
+                if sec in self.cli_flags:
+                    # we want only custom hooks, so we skip builtins
+                    if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
+                        continue
 
-        flags = []
-        for _sec, key_val in ui_sections.items():
-            flags.append(' ')
-            flags.append(f'[{_sec}]')
-            for key, val in key_val:
-                flags.append(f'{key}= {val}')
-        return flags
+                    ui_sections[sec].append([key, entry.value])
+
+            flags = []
+            for _sec, key_val in ui_sections.items():
+                flags.append(' ')
+                flags.append(f'[{_sec}]')
+                for key, val in key_val:
+                    flags.append(f'{key}= {val}')
+            return flags
diff --git a/rhodecode/apps/ssh_support/lib/backends/svn.py b/rhodecode/apps/ssh_support/lib/backends/svn.py
index 728ec8ec..e7fefc29 100644
--- a/rhodecode/apps/ssh_support/lib/backends/svn.py
+++ b/rhodecode/apps/ssh_support/lib/backends/svn.py
@@ -25,7 +25,7 @@ import tempfile
 from subprocess import Popen, PIPE
 import urllib.parse
 
-from .base import VcsServer
+from .base import SSHVcsServer
 
 log = logging.getLogger(__name__)
 
@@ -218,20 +218,18 @@ class SubversionTunnelWrapper(object):
         return self.return_code
 
 
-class SubversionServer(VcsServer):
+class SubversionServer(SSHVcsServer):
     backend = 'svn'
     repo_user_agent = 'svn'
 
-    def __init__(self, store, ini_path, repo_name,
-                 user, user_permissions, config, env):
-        super()\
-            .__init__(user, user_permissions, config, env)
+    def __init__(self, store, ini_path, repo_name, user, user_permissions, settings, env):
+        super().__init__(user, user_permissions, settings, env)
         self.store = store
         self.ini_path = ini_path
         # NOTE(dan): repo_name at this point is empty,
         # this is set later in .run() based from parsed input stream
         self.repo_name = repo_name
-        self._path = self.svn_path = config.get('app:main', 'ssh.executable.svn')
+        self._path = self.svn_path = settings['ssh.executable.svn']
 
         self.tunnel = SubversionTunnelWrapper(server=self)
 
diff --git a/rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py
index 34da4267..1fd567ed 100644
--- a/rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py
+++ b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v1.py
@@ -31,7 +31,6 @@ from .utils import setup_custom_logging
 log = logging.getLogger(__name__)
 
 
-
 @click.command()
 @click.argument('ini_path', type=click.Path(exists=True))
 @click.option(
@@ -55,11 +54,12 @@ def main(ini_path, mode, user, user_id, key_id, shell, debug):
     connection_info = os.environ.get('SSH_CONNECTION', '')
     time_start = time.time()
     with bootstrap(ini_path, env={'RC_CMD_SSH_WRAPPER': '1'}) as env:
+        settings = env['registry'].settings
         statsd = StatsdClient.statsd
         try:
             ssh_wrapper = SshWrapper(
                 command, connection_info, mode,
-                user, user_id, key_id, shell, ini_path, env)
+                user, user_id, key_id, shell, ini_path, settings, env)
         except Exception:
             log.exception('Failed to execute SshWrapper')
             sys.exit(-5)
diff --git a/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
index e76a88c5..fff01ca3 100644
--- a/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
+++ b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
@@ -16,6 +16,14 @@
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
+"""
+WARNING: be really carefully with changing ANY imports in this file
+# This script is to mean as really fast executable, doing some imports here that would yield an import chain change
+# can affect execution times...
+# This can be easily debugged using such command::
+# time PYTHONPROFILEIMPORTTIME=1 rc-ssh-wrapper-v2 --debug --mode=test .dev/dev.ini
+"""
+
 import os
 import sys
 import time
@@ -23,9 +31,12 @@ import logging
 
 import click
 
+from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
 from rhodecode.lib.statsd_client import StatsdClient
-from .backends import SshWrapperStandalone
+from rhodecode.lib.config_utils import get_app_config_lightweight
+
 from .utils import setup_custom_logging
+from .backends import SshWrapperStandalone
 
 log = logging.getLogger(__name__)
 
@@ -42,6 +53,8 @@ log = logging.getLogger(__name__)
 @click.option('--shell', '-s', is_flag=True, help='Allow Shell')
 @click.option('--debug', is_flag=True, help='Enabled detailed output logging')
 def main(ini_path, mode, user, user_id, key_id, shell, debug):
+
+    time_start = time.time()
     setup_custom_logging(ini_path, debug)
 
     command = os.environ.get('SSH_ORIGINAL_COMMAND', '')
@@ -50,21 +63,30 @@ def main(ini_path, mode, user, user_id, key_id, shell, debug):
             'Unable to fetch SSH_ORIGINAL_COMMAND from environment.'
             'Please make sure this is set and available during execution '
             'of this script.')
-    connection_info = os.environ.get('SSH_CONNECTION', '')
-    time_start = time.time()
-    env = {'RC_CMD_SSH_WRAPPER': '1'}
+
+    # initialize settings and get defaults
+    settings = get_app_config_lightweight(ini_path)
+    settings = sanitize_settings_and_apply_defaults({'__file__': ini_path}, settings)
+
+    # init and bootstrap StatsdClient
+    StatsdClient.setup(settings)
     statsd = StatsdClient.statsd
+
     try:
+        connection_info = os.environ.get('SSH_CONNECTION', '')
+        env = {'RC_CMD_SSH_WRAPPER': '1'}
         ssh_wrapper = SshWrapperStandalone(
             command, connection_info, mode,
-            user, user_id, key_id, shell, ini_path, env)
+            user, user_id, key_id, shell, ini_path, settings, env)
     except Exception:
         log.exception('Failed to execute SshWrapper')
         sys.exit(-5)
+
     return_code = ssh_wrapper.wrap()
     operation_took = time.time() - time_start
     if statsd:
         operation_took_ms = round(1000.0 * operation_took)
         statsd.timing("rhodecode_ssh_wrapper_timing.histogram", operation_took_ms,
                       use_decimals=False)
+
     sys.exit(return_code)
diff --git a/rhodecode/apps/ssh_support/lib/utils.py b/rhodecode/apps/ssh_support/lib/utils.py
index 977e4eff..bfe91082 100644
--- a/rhodecode/apps/ssh_support/lib/utils.py
+++ b/rhodecode/apps/ssh_support/lib/utils.py
@@ -17,11 +17,11 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
-from pyramid.paster import setup_logging
 
 
 def setup_custom_logging(ini_path, debug):
     if debug:
+        from pyramid.paster import setup_logging  # Lazy import
         # enabled rhodecode.ini controlled logging setup
         setup_logging(ini_path)
     else:
diff --git a/rhodecode/apps/ssh_support/tests/conftest.py b/rhodecode/apps/ssh_support/tests/conftest.py
index 3b581b47..7c5a90cf 100644
--- a/rhodecode/apps/ssh_support/tests/conftest.py
+++ b/rhodecode/apps/ssh_support/tests/conftest.py
@@ -52,7 +52,10 @@ def dummy_env():
 
 
 def plain_dummy_user():
-    return AttributeDict(username='test_user')
+    return AttributeDict(
+        user_id=1,
+        username='test_user'
+    )
 
 
 @pytest.fixture()
@@ -65,4 +68,4 @@ def ssh_wrapper(app, dummy_conf_file, dummy_env):
     conn_info = '127.0.0.1 22 10.0.0.1 443'
     return SshWrapper(
         'random command', conn_info, 'auto', 'admin', '1', key_id='1',
-        shell=False, ini_path=dummy_conf_file, env=dummy_env)
+        shell=False, ini_path=dummy_conf_file, settings={}, env=dummy_env)
diff --git a/rhodecode/apps/ssh_support/tests/test_server_git.py b/rhodecode/apps/ssh_support/tests/test_server_git.py
index bb8e2088..175163f9 100644
--- a/rhodecode/apps/ssh_support/tests/test_server_git.py
+++ b/rhodecode/apps/ssh_support/tests/test_server_git.py
@@ -25,6 +25,7 @@ from rhodecode.apps.ssh_support.lib.backends.git import GitServer
 from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dummy_user
 from rhodecode.lib.ext_json import json
 
+
 class GitServerCreator(object):
     root = '/tmp/repo/path/'
     git_path = '/usr/local/bin/git'
@@ -39,10 +40,7 @@ class GitServerCreator(object):
     user = plain_dummy_user()
 
     def __init__(self):
-        def config_get(part, key):
-            return self.config_data.get(part, {}).get(key)
-        self.config_mock = mock.Mock()
-        self.config_mock.get = mock.Mock(side_effect=config_get)
+        pass
 
     def create(self, **kwargs):
         parameters = {
@@ -54,7 +52,7 @@ class GitServerCreator(object):
             'user_permissions': {
                 self.repo_name: 'repository.admin'
             },
-            'config': self.config_mock,
+            'settings': self.config_data['app:main'],
             'env': plain_dummy_env()
         }
         parameters.update(kwargs)
@@ -142,7 +140,7 @@ class TestGitServer(object):
             'server_url': None,
             'hooks': ['push', 'pull'],
             'is_shadow_repo': False,
-            'hooks_module': 'rhodecode.lib.hooks_daemon',
+            'hooks_module': 'rhodecode.lib.hook_daemon.hook_module',
             'check_branch_perms': False,
             'detect_force_push': False,
             'user_agent': u'git/ssh-user-agent',
diff --git a/rhodecode/apps/ssh_support/tests/test_server_hg.py b/rhodecode/apps/ssh_support/tests/test_server_hg.py
index ee1b1af6..031f932d 100644
--- a/rhodecode/apps/ssh_support/tests/test_server_hg.py
+++ b/rhodecode/apps/ssh_support/tests/test_server_hg.py
@@ -38,10 +38,7 @@ class MercurialServerCreator(object):
     user = plain_dummy_user()
 
     def __init__(self):
-        def config_get(part, key):
-            return self.config_data.get(part, {}).get(key)
-        self.config_mock = mock.Mock()
-        self.config_mock.get = mock.Mock(side_effect=config_get)
+        pass
 
     def create(self, **kwargs):
         parameters = {
@@ -52,7 +49,7 @@ class MercurialServerCreator(object):
             'user_permissions': {
                 'test_hg': 'repository.admin'
             },
-            'config': self.config_mock,
+            'settings': self.config_data['app:main'],
             'env': plain_dummy_env()
         }
         parameters.update(kwargs)
diff --git a/rhodecode/apps/ssh_support/tests/test_server_svn.py b/rhodecode/apps/ssh_support/tests/test_server_svn.py
index 7102f801..52bd6ad1 100644
--- a/rhodecode/apps/ssh_support/tests/test_server_svn.py
+++ b/rhodecode/apps/ssh_support/tests/test_server_svn.py
@@ -36,10 +36,7 @@ class SubversionServerCreator(object):
     user = plain_dummy_user()
 
     def __init__(self):
-        def config_get(part, key):
-            return self.config_data.get(part, {}).get(key)
-        self.config_mock = mock.Mock()
-        self.config_mock.get = mock.Mock(side_effect=config_get)
+        pass
 
     def create(self, **kwargs):
         parameters = {
@@ -50,7 +47,7 @@ class SubversionServerCreator(object):
             'user_permissions': {
                 self.repo_name: 'repository.admin'
             },
-            'config': self.config_mock,
+            'settings': self.config_data['app:main'],
             'env': plain_dummy_env()
         }
 
diff --git a/rhodecode/apps/ssh_support/tests/test_ssh_wrapper.py b/rhodecode/apps/ssh_support/tests/test_ssh_wrapper.py
index b5b7435b..fcb5aa93 100644
--- a/rhodecode/apps/ssh_support/tests/test_ssh_wrapper.py
+++ b/rhodecode/apps/ssh_support/tests/test_ssh_wrapper.py
@@ -28,10 +28,6 @@ class TestSSHWrapper(object):
                 permissions={}, branch_permissions={})
         assert str(exc_info.value) == 'Unrecognised VCS: microsoft-tfs'
 
-    def test_parse_config(self, ssh_wrapper):
-        config = ssh_wrapper.parse_config(ssh_wrapper.ini_path)
-        assert config
-
     def test_get_connection_info(self, ssh_wrapper):
         conn_info = ssh_wrapper.get_connection_info()
         assert {'client_ip': '127.0.0.1',
diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py
new file mode 100644
index 00000000..f33fd929
--- /dev/null
+++ b/rhodecode/config/config_maker.py
@@ -0,0 +1,198 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import tempfile
+import logging
+
+from pyramid.settings import asbool
+
+from rhodecode.config.settings_maker import SettingsMaker
+from rhodecode.config import utils as config_utils
+
+log = logging.getLogger(__name__)
+
+
+def sanitize_settings_and_apply_defaults(global_config, settings):
+    """
+    Applies settings defaults and does all type conversion.
+
+    We would move all settings parsing and preparation into this place, so that
+    we have only one place left which deals with this part. The remaining parts
+    of the application would start to rely fully on well-prepared settings.
+
+    This piece would later be split up per topic to avoid a big fat monster
+    function.
+    """
+    jn = os.path.join
+
+    global_settings_maker = SettingsMaker(global_config)
+    global_settings_maker.make_setting('debug', default=False, parser='bool')
+    debug_enabled = asbool(global_config.get('debug'))
+
+    settings_maker = SettingsMaker(settings)
+
+    settings_maker.make_setting(
+        'logging.autoconfigure',
+        default=False,
+        parser='bool')
+
+    logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
+    settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
+
+    # Default includes, possible to change as a user
+    pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
+    log.debug(
+        "Using the following pyramid.includes: %s",
+        pyramid_includes)
+
+    settings_maker.make_setting('rhodecode.edition', 'Community Edition')
+    settings_maker.make_setting('rhodecode.edition_id', 'CE')
+
+    if 'mako.default_filters' not in settings:
+        # set custom default filters if we don't have it defined
+        settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
+        settings['mako.default_filters'] = 'h_filter'
+
+    if 'mako.directories' not in settings:
+        mako_directories = settings.setdefault('mako.directories', [
+            # Base templates of the original application
+            'rhodecode:templates',
+        ])
+        log.debug(
+            "Using the following Mako template directories: %s",
+            mako_directories)
+
+    # NOTE(marcink): fix redis requirement for schema of connection since 3.X
+    if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
+        raw_url = settings['beaker.session.url']
+        if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
+            settings['beaker.session.url'] = 'redis://' + raw_url
+
+    settings_maker.make_setting('__file__', global_config.get('__file__'))
+
+    # TODO: johbo: Re-think this, usually the call to config.include
+    # should allow to pass in a prefix.
+    settings_maker.make_setting('rhodecode.api.url', '/_admin/api')
+
+    # Sanitize generic settings.
+    settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
+    settings_maker.make_setting('is_test', False, parser='bool')
+    settings_maker.make_setting('gzip_responses', False, parser='bool')
+
+    # statsd
+    settings_maker.make_setting('statsd.enabled', False, parser='bool')
+    settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
+    settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
+    settings_maker.make_setting('statsd.statsd_prefix', '')
+    settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
+
+    settings_maker.make_setting('vcs.svn.compatible_version', '')
+    settings_maker.make_setting('vcs.hooks.protocol', 'http')
+    settings_maker.make_setting('vcs.hooks.host', '*')
+    settings_maker.make_setting('vcs.scm_app_implementation', 'http')
+    settings_maker.make_setting('vcs.server', '')
+    settings_maker.make_setting('vcs.server.protocol', 'http')
+    settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
+    settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
+    settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
+    settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
+    settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
+    settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
+
+    settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
+
+    # Support legacy values of vcs.scm_app_implementation. Legacy
+    # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
+    # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
+    scm_app_impl = settings['vcs.scm_app_implementation']
+    if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
+        settings['vcs.scm_app_implementation'] = 'http'
+
+    settings_maker.make_setting('appenlight', False, parser='bool')
+
+    temp_store = tempfile.gettempdir()
+    tmp_cache_dir = jn(temp_store, 'rc_cache')
+
+    # save default, cache dir, and use it for all backends later.
+    default_cache_dir = settings_maker.make_setting(
+        'cache_dir',
+        default=tmp_cache_dir, default_when_empty=True,
+        parser='dir:ensured')
+
+    # exception store cache
+    settings_maker.make_setting(
+        'exception_tracker.store_path',
+        default=jn(default_cache_dir, 'exc_store'), default_when_empty=True,
+        parser='dir:ensured'
+    )
+
+    settings_maker.make_setting(
+        'celerybeat-schedule.path',
+        default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
+        parser='file:ensured'
+    )
+
+    settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
+    settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
+
+    # sessions, ensure file since no-value is memory
+    settings_maker.make_setting('beaker.session.type', 'file')
+    settings_maker.make_setting('beaker.session.data_dir',  jn(default_cache_dir, 'session_data'))
+
+    # cache_general
+    settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
+    settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db'))
+
+    # cache_perms
+    settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db'))
+
+    # cache_repo
+    settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
+    settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db'))
+
+    # cache_license
+    settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
+    settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db'))
+
+    # cache_repo_longterm memory, 96H
+    settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
+    settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
+    settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
+
+    # sql_cache_short
+    settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
+    settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
+    settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
+
+    # archive_cache
+    settings_maker.make_setting('archive_cache.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
+    settings_maker.make_setting('archive_cache.cache_size_gb', 10, parser='float')
+    settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int')
+
+    settings_maker.env_expand()
+
+    # configure instance id
+    config_utils.set_instance_id(settings)
+
+    return settings
diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py
index aa78fcf7..dafee2dc 100644
--- a/rhodecode/config/middleware.py
+++ b/rhodecode/config/middleware.py
@@ -19,7 +19,7 @@
 import os
 import sys
 import collections
-import tempfile
+
 import time
 import logging.config
 
@@ -32,14 +32,13 @@ from pyramid.httpexceptions import (
     HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
 from pyramid.renderers import render_to_response
 
-from rhodecode import api
 from rhodecode.model import meta
 from rhodecode.config import patches
-from rhodecode.config import utils as config_utils
-from rhodecode.config.settings_maker import SettingsMaker
+
 from rhodecode.config.environment import load_pyramid_environment
 
 import rhodecode.events
+from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
 from rhodecode.lib.middleware.vcs import VCSMiddleware
 from rhodecode.lib.request import Request
 from rhodecode.lib.vcs import VCSCommunicationError
@@ -465,173 +464,3 @@ def wrap_app_in_wsgi_middlewares(pyramid_app, config):
             log.debug('Request processing finalized: %.4fs', total)
 
     return pyramid_app_with_cleanup
-
-
-def sanitize_settings_and_apply_defaults(global_config, settings):
-    """
-    Applies settings defaults and does all type conversion.
-
-    We would move all settings parsing and preparation into this place, so that
-    we have only one place left which deals with this part. The remaining parts
-    of the application would start to rely fully on well prepared settings.
-
-    This piece would later be split up per topic to avoid a big fat monster
-    function.
-    """
-    jn = os.path.join
-
-    global_settings_maker = SettingsMaker(global_config)
-    global_settings_maker.make_setting('debug', default=False, parser='bool')
-    debug_enabled = asbool(global_config.get('debug'))
-
-    settings_maker = SettingsMaker(settings)
-
-    settings_maker.make_setting(
-        'logging.autoconfigure',
-        default=False,
-        parser='bool')
-
-    logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
-    settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
-
-    # Default includes, possible to change as a user
-    pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
-    log.debug(
-        "Using the following pyramid.includes: %s",
-        pyramid_includes)
-
-    settings_maker.make_setting('rhodecode.edition', 'Community Edition')
-    settings_maker.make_setting('rhodecode.edition_id', 'CE')
-
-    if 'mako.default_filters' not in settings:
-        # set custom default filters if we don't have it defined
-        settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
-        settings['mako.default_filters'] = 'h_filter'
-
-    if 'mako.directories' not in settings:
-        mako_directories = settings.setdefault('mako.directories', [
-            # Base templates of the original application
-            'rhodecode:templates',
-        ])
-        log.debug(
-            "Using the following Mako template directories: %s",
-            mako_directories)
-
-    # NOTE(marcink): fix redis requirement for schema of connection since 3.X
-    if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
-        raw_url = settings['beaker.session.url']
-        if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
-            settings['beaker.session.url'] = 'redis://' + raw_url
-
-    settings_maker.make_setting('__file__', global_config.get('__file__'))
-
-    # TODO: johbo: Re-think this, usually the call to config.include
-    # should allow to pass in a prefix.
-    settings_maker.make_setting('rhodecode.api.url', api.DEFAULT_URL)
-
-    # Sanitize generic settings.
-    settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
-    settings_maker.make_setting('is_test', False, parser='bool')
-    settings_maker.make_setting('gzip_responses', False, parser='bool')
-
-    # statsd
-    settings_maker.make_setting('statsd.enabled', False, parser='bool')
-    settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
-    settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
-    settings_maker.make_setting('statsd.statsd_prefix', '')
-    settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
-
-    settings_maker.make_setting('vcs.svn.compatible_version', '')
-    settings_maker.make_setting('vcs.hooks.protocol', 'http')
-    settings_maker.make_setting('vcs.hooks.host', '*')
-    settings_maker.make_setting('vcs.scm_app_implementation', 'http')
-    settings_maker.make_setting('vcs.server', '')
-    settings_maker.make_setting('vcs.server.protocol', 'http')
-    settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
-    settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
-    settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
-    settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
-    settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
-    settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
-
-    settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
-
-    # Support legacy values of vcs.scm_app_implementation. Legacy
-    # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
-    # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
-    scm_app_impl = settings['vcs.scm_app_implementation']
-    if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
-        settings['vcs.scm_app_implementation'] = 'http'
-
-    settings_maker.make_setting('appenlight', False, parser='bool')
-
-    temp_store = tempfile.gettempdir()
-    tmp_cache_dir = jn(temp_store, 'rc_cache')
-
-    # save default, cache dir, and use it for all backends later.
-    default_cache_dir = settings_maker.make_setting(
-        'cache_dir',
-        default=tmp_cache_dir, default_when_empty=True,
-        parser='dir:ensured')
-
-    # exception store cache
-    settings_maker.make_setting(
-        'exception_tracker.store_path',
-        default=jn(default_cache_dir, 'exc_store'), default_when_empty=True,
-        parser='dir:ensured'
-    )
-
-    settings_maker.make_setting(
-        'celerybeat-schedule.path',
-        default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
-        parser='file:ensured'
-    )
-
-    settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
-    settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
-
-    # sessions, ensure file since no-value is memory
-    settings_maker.make_setting('beaker.session.type', 'file')
-    settings_maker.make_setting('beaker.session.data_dir',  jn(default_cache_dir, 'session_data'))
-
-    # cache_general
-    settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
-    settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db'))
-
-    # cache_perms
-    settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
-    settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db'))
-
-    # cache_repo
-    settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
-    settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db'))
-
-    # cache_license
-    settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
-    settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
-    settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db'))
-
-    # cache_repo_longterm memory, 96H
-    settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
-    settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
-    settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
-
-    # sql_cache_short
-    settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
-    settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
-    settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
-
-    # archive_cache
-    settings_maker.make_setting('archive_cache.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
-    settings_maker.make_setting('archive_cache.cache_size_gb', 10, parser='float')
-    settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int')
-
-    settings_maker.env_expand()
-
-    # configure instance id
-    config_utils.set_instance_id(settings)
-
-    return settings
diff --git a/rhodecode/config/utils.py b/rhodecode/config/utils.py
index 40c8d2f4..6f443954 100644
--- a/rhodecode/config/utils.py
+++ b/rhodecode/config/utils.py
@@ -19,8 +19,6 @@
 import os
 import platform
 
-from rhodecode.model import init_model
-
 
 def configure_vcs(config):
     """
@@ -44,6 +42,7 @@ def configure_vcs(config):
 
 def initialize_database(config):
     from rhodecode.lib.utils2 import engine_from_config, get_encryption_key
+    from rhodecode.model import init_model
     engine = engine_from_config(config, 'sqlalchemy.db1.')
     init_model(engine, encryption_key=get_encryption_key(config))
 
diff --git a/rhodecode/lib/api_utils.py b/rhodecode/lib/api_utils.py
new file mode 100644
index 00000000..6c8f11ad
--- /dev/null
+++ b/rhodecode/lib/api_utils.py
@@ -0,0 +1,38 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import urllib.parse
+
+from rhodecode.lib.vcs import CurlSession
+from rhodecode.lib.ext_json import json
+
+
+def call_service_api(service_api_host, service_api_token, api_url, payload):
+
+    payload.update({
+        'id': 'service',
+        'auth_token': service_api_token
+    })
+
+    service_api_url = urllib.parse.urljoin(service_api_host, api_url)
+    response = CurlSession().post(service_api_url, json.dumps(payload))
+
+    if response.status_code != 200:
+        raise Exception(f"Service API at {service_api_url} responded with error: {response.status_code}")
+
+    return json.loads(response.content)['result']
diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py
index 7bd1c7d6..da8104fe 100644
--- a/rhodecode/lib/base.py
+++ b/rhodecode/lib/base.py
@@ -567,7 +567,7 @@ def add_events_routes(config):
 
 
 def bootstrap_config(request, registry_name='RcTestRegistry'):
-    from rhodecode.config.middleware import sanitize_settings_and_apply_defaults
+    from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
     import pyramid.testing
     registry = pyramid.testing.Registry(registry_name)
 
diff --git a/rhodecode/lib/config_utils.py b/rhodecode/lib/config_utils.py
new file mode 100644
index 00000000..eeafef5e
--- /dev/null
+++ b/rhodecode/lib/config_utils.py
@@ -0,0 +1,40 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import os
+
+
+def get_config(ini_path, **kwargs):
+    import configparser
+    parser = configparser.ConfigParser(**kwargs)
+    parser.read(ini_path)
+    return parser
+
+
+def get_app_config_lightweight(ini_path):
+    parser = get_config(ini_path)
+    parser.set('app:main', 'here', os.getcwd())
+    parser.set('app:main', '__file__', ini_path)
+    return dict(parser.items('app:main'))
+
+
+def get_app_config(ini_path):
+    """
+    This loads the app context and provides a heavy type iniliaziation of config
+    """
+    from paste.deploy.loadwsgi import appconfig
+    return appconfig(f'config:{ini_path}', relative_to=os.getcwd())
diff --git a/rhodecode/lib/hook_daemon/__init__.py b/rhodecode/lib/hook_daemon/__init__.py
new file mode 100644
index 00000000..c4d86aeb
--- /dev/null
+++ b/rhodecode/lib/hook_daemon/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
diff --git a/rhodecode/lib/hook_daemon/base.py b/rhodecode/lib/hook_daemon/base.py
new file mode 100644
index 00000000..14faca6d
--- /dev/null
+++ b/rhodecode/lib/hook_daemon/base.py
@@ -0,0 +1,115 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import os
+import time
+import logging
+import tempfile
+
+from rhodecode.lib.config_utils import get_config
+from rhodecode.lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class BaseHooksCallbackDaemon:
+    """
+    Basic context manager for actions that don't require some extra
+    """
+    def __init__(self):
+        pass
+
+    def __enter__(self):
+        log.debug('Running `%s` callback daemon', self.__class__.__name__)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
+
+
+class HooksModuleCallbackDaemon(BaseHooksCallbackDaemon):
+
+    def __init__(self, module):
+        super().__init__()
+        self.hooks_module = module
+
+
+def get_txn_id_data_path(txn_id):
+    import rhodecode
+
+    root = rhodecode.CONFIG.get('cache_dir') or tempfile.gettempdir()
+    final_dir = os.path.join(root, 'svn_txn_id')
+
+    if not os.path.isdir(final_dir):
+        os.makedirs(final_dir)
+    return os.path.join(final_dir, 'rc_txn_id_{}'.format(txn_id))
+
+
+def store_txn_id_data(txn_id, data_dict):
+    if not txn_id:
+        log.warning('Cannot store txn_id because it is empty')
+        return
+
+    path = get_txn_id_data_path(txn_id)
+    try:
+        with open(path, 'wb') as f:
+            f.write(json.dumps(data_dict))
+    except Exception:
+        log.exception('Failed to write txn_id metadata')
+
+
+def get_txn_id_from_store(txn_id):
+    """
+    Reads txn_id from store and if present returns the data for callback manager
+    """
+    path = get_txn_id_data_path(txn_id)
+    try:
+        with open(path, 'rb') as f:
+            return json.loads(f.read())
+    except Exception:
+        return {}
+
+
+def prepare_callback_daemon(extras, protocol, host, txn_id=None):
+    txn_details = get_txn_id_from_store(txn_id)
+    port = txn_details.get('port', 0)
+    match protocol:
+        case 'http':
+            from rhodecode.lib.hook_daemon.http_hooks_deamon import HttpHooksCallbackDaemon
+            callback_daemon = HttpHooksCallbackDaemon(
+                txn_id=txn_id, host=host, port=port)
+        case 'celery':
+            from rhodecode.lib.hook_daemon.celery_hooks_deamon import CeleryHooksCallbackDaemon
+            callback_daemon = CeleryHooksCallbackDaemon(get_config(extras['config']))
+        case 'local':
+            from rhodecode.lib.hook_daemon.hook_module import Hooks
+            callback_daemon = HooksModuleCallbackDaemon(Hooks.__module__)
+        case _:
+            log.error('Unsupported callback daemon protocol "%s"', protocol)
+            raise Exception('Unsupported callback daemon protocol.')
+
+    extras['hooks_uri'] = getattr(callback_daemon, 'hooks_uri', '')
+    extras['task_queue'] = getattr(callback_daemon, 'task_queue', '')
+    extras['task_backend'] = getattr(callback_daemon, 'task_backend', '')
+    extras['hooks_protocol'] = protocol
+    extras['time'] = time.time()
+
+    # register txn_id
+    extras['txn_id'] = txn_id
+    log.debug('Prepared a callback daemon: %s',
+              callback_daemon.__class__.__name__)
+    return callback_daemon, extras
diff --git a/rhodecode/lib/hook_daemon/celery_hooks_deamon.py b/rhodecode/lib/hook_daemon/celery_hooks_deamon.py
new file mode 100644
index 00000000..fc9883f4
--- /dev/null
+++ b/rhodecode/lib/hook_daemon/celery_hooks_deamon.py
@@ -0,0 +1,30 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+from rhodecode.lib.hook_daemon.base import BaseHooksCallbackDaemon
+
+
+class CeleryHooksCallbackDaemon(BaseHooksCallbackDaemon):
+    """
+    Context manger for achieving a compatibility with celery backend
+    """
+
+    def __init__(self, config):
+        # TODO: replace this with settings bootstrapped...
+        self.task_queue = config.get('app:main', 'celery.broker_url')
+        self.task_backend = config.get('app:main', 'celery.result_backend')
diff --git a/rhodecode/lib/hook_daemon/hook_module.py b/rhodecode/lib/hook_daemon/hook_module.py
new file mode 100644
index 00000000..d719f88a
--- /dev/null
+++ b/rhodecode/lib/hook_daemon/hook_module.py
@@ -0,0 +1,104 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+import traceback
+
+from rhodecode.model import meta
+
+from rhodecode.lib import hooks_base
+from rhodecode.lib.exceptions import HTTPLockedRC, HTTPBranchProtected
+from rhodecode.lib.utils2 import AttributeDict
+
+log = logging.getLogger(__name__)
+
+
+class Hooks(object):
+    """
+    Exposes the hooks for remote callbacks
+    """
+    def __init__(self, request=None, log_prefix=''):
+        self.log_prefix = log_prefix
+        self.request = request
+
+    def repo_size(self, extras):
+        log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.repo_size, extras)
+
+    def pre_pull(self, extras):
+        log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.pre_pull, extras)
+
+    def post_pull(self, extras):
+        log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.post_pull, extras)
+
+    def pre_push(self, extras):
+        log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.pre_push, extras)
+
+    def post_push(self, extras):
+        log.debug("%sCalled post_push of %s object", self.log_prefix, self)
+        return self._call_hook(hooks_base.post_push, extras)
+
+    def _call_hook(self, hook, extras):
+        extras = AttributeDict(extras)
+        _server_url = extras['server_url']
+
+        extras.request = self.request
+
+        try:
+            result = hook(extras)
+            if result is None:
+                raise Exception(f'Failed to obtain hook result from func: {hook}')
+        except HTTPBranchProtected as handled_error:
+            # Those special cases don't need error reporting. It's a case of
+            # locked repo or protected branch
+            result = AttributeDict({
+                'status': handled_error.code,
+                'output': handled_error.explanation
+            })
+        except (HTTPLockedRC, Exception) as error:
+            # locked needs different handling since we need to also
+            # handle PULL operations
+            exc_tb = ''
+            if not isinstance(error, HTTPLockedRC):
+                exc_tb = traceback.format_exc()
+                log.exception('%sException when handling hook %s', self.log_prefix, hook)
+            error_args = error.args
+            return {
+                'status': 128,
+                'output': '',
+                'exception': type(error).__name__,
+                'exception_traceback': exc_tb,
+                'exception_args': error_args,
+            }
+        finally:
+            meta.Session.remove()
+
+        log.debug('%sGot hook call response %s', self.log_prefix, result)
+        return {
+            'status': result.status,
+            'output': result.output,
+        }
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
diff --git a/rhodecode/lib/hooks_daemon.py b/rhodecode/lib/hook_daemon/http_hooks_deamon.py
similarity index 61%
rename from rhodecode/lib/hooks_daemon.py
rename to rhodecode/lib/hook_daemon/http_hooks_deamon.py
index 2811e78f..73ecd0ff 100644
--- a/rhodecode/lib/hooks_daemon.py
+++ b/rhodecode/lib/hook_daemon/http_hooks_deamon.py
@@ -17,9 +17,7 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
-import time
 import logging
-import tempfile
 import traceback
 import threading
 import socket
@@ -29,14 +27,11 @@ import gevent
 from http.server import BaseHTTPRequestHandler
 from socketserver import TCPServer
 
-import rhodecode
-from rhodecode.lib.exceptions import HTTPLockedRC, HTTPBranchProtected
 from rhodecode.model import meta
-from rhodecode.lib import hooks_base
-from rhodecode.lib.utils2 import AttributeDict
-from rhodecode.lib.pyramid_utils import get_config
 from rhodecode.lib.ext_json import json
 from rhodecode.lib import rc_cache
+from rhodecode.lib.hook_daemon.base import get_txn_id_data_path
+from rhodecode.lib.hook_daemon.hook_module import Hooks
 
 log = logging.getLogger(__name__)
 
@@ -126,7 +121,7 @@ class HooksHttpHandler(BaseHTTPRequestHandler):
     def log_message(self, format, *args):
         """
         This is an overridden method of BaseHTTPRequestHandler which logs using
-        logging library instead of writing directly to stderr.
+        a logging library instead of writing directly to stderr.
         """
 
         message = format % args
@@ -136,31 +131,6 @@ class HooksHttpHandler(BaseHTTPRequestHandler):
             self.log_date_time_string(), message)
 
 
-class BaseHooksCallbackDaemon:
-    """
-    Basic context manager for actions that don't require some extra
-    """
-    def __init__(self):
-        self.hooks_module = Hooks.__module__
-
-    def __enter__(self):
-        log.debug('Running `%s` callback daemon', self.__class__.__name__)
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
-
-
-class CeleryHooksCallbackDaemon(BaseHooksCallbackDaemon):
-    """
-    Context manger for achieving a compatibility with celery backend
-    """
-
-    def __init__(self, config):
-        self.task_queue = config.get('app:main', 'celery.broker_url')
-        self.task_backend = config.get('app:main', 'celery.result_backend')
-
-
 class ThreadedHookCallbackDaemon(object):
 
     _callback_thread = None
@@ -308,144 +278,3 @@ class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
                 os.remove(txn_id_file)
 
         log.debug("Background greenlet done.")
-
-
-def get_txn_id_data_path(txn_id):
-    import rhodecode
-
-    root = rhodecode.CONFIG.get('cache_dir') or tempfile.gettempdir()
-    final_dir = os.path.join(root, 'svn_txn_id')
-
-    if not os.path.isdir(final_dir):
-        os.makedirs(final_dir)
-    return os.path.join(final_dir, 'rc_txn_id_{}'.format(txn_id))
-
-
-def store_txn_id_data(txn_id, data_dict):
-    if not txn_id:
-        log.warning('Cannot store txn_id because it is empty')
-        return
-
-    path = get_txn_id_data_path(txn_id)
-    try:
-        with open(path, 'wb') as f:
-            f.write(json.dumps(data_dict))
-    except Exception:
-        log.exception('Failed to write txn_id metadata')
-
-
-def get_txn_id_from_store(txn_id):
-    """
-    Reads txn_id from store and if present returns the data for callback manager
-    """
-    path = get_txn_id_data_path(txn_id)
-    try:
-        with open(path, 'rb') as f:
-            return json.loads(f.read())
-    except Exception:
-        return {}
-
-
-def prepare_callback_daemon(extras, protocol, host, txn_id=None):
-    txn_details = get_txn_id_from_store(txn_id)
-    port = txn_details.get('port', 0)
-    match protocol:
-        case 'http':
-            callback_daemon = HttpHooksCallbackDaemon(
-                txn_id=txn_id, host=host, port=port)
-        case 'celery':
-            callback_daemon = CeleryHooksCallbackDaemon(get_config(extras['config']))
-        case 'local':
-            callback_daemon = BaseHooksCallbackDaemon()
-        case _:
-            log.error('Unsupported callback daemon protocol "%s"', protocol)
-            raise Exception('Unsupported callback daemon protocol.')
-
-    extras['hooks_uri'] = getattr(callback_daemon, 'hooks_uri', '')
-    extras['task_queue'] = getattr(callback_daemon, 'task_queue', '')
-    extras['task_backend'] = getattr(callback_daemon, 'task_backend', '')
-    extras['hooks_protocol'] = protocol
-    extras['time'] = time.time()
-
-    # register txn_id
-    extras['txn_id'] = txn_id
-    log.debug('Prepared a callback daemon: %s',
-              callback_daemon.__class__.__name__)
-    return callback_daemon, extras
-
-
-class Hooks(object):
-    """
-    Exposes the hooks for remote call backs
-    """
-    def __init__(self, request=None, log_prefix=''):
-        self.log_prefix = log_prefix
-        self.request = request
-
-    def repo_size(self, extras):
-        log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
-        return self._call_hook(hooks_base.repo_size, extras)
-
-    def pre_pull(self, extras):
-        log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
-        return self._call_hook(hooks_base.pre_pull, extras)
-
-    def post_pull(self, extras):
-        log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
-        return self._call_hook(hooks_base.post_pull, extras)
-
-    def pre_push(self, extras):
-        log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
-        return self._call_hook(hooks_base.pre_push, extras)
-
-    def post_push(self, extras):
-        log.debug("%sCalled post_push of %s object", self.log_prefix, self)
-        return self._call_hook(hooks_base.post_push, extras)
-
-    def _call_hook(self, hook, extras):
-        extras = AttributeDict(extras)
-        server_url = extras['server_url']
-
-        extras.request = self.request
-
-        try:
-            result = hook(extras)
-            if result is None:
-                raise Exception(
-                    'Failed to obtain hook result from func: {}'.format(hook))
-        except HTTPBranchProtected as handled_error:
-            # Those special cases doesn't need error reporting. It's a case of
-            # locked repo or protected branch
-            result = AttributeDict({
-                'status': handled_error.code,
-                'output': handled_error.explanation
-            })
-        except (HTTPLockedRC, Exception) as error:
-            # locked needs different handling since we need to also
-            # handle PULL operations
-            exc_tb = ''
-            if not isinstance(error, HTTPLockedRC):
-                exc_tb = traceback.format_exc()
-                log.exception('%sException when handling hook %s', self.log_prefix, hook)
-            error_args = error.args
-            return {
-                'status': 128,
-                'output': '',
-                'exception': type(error).__name__,
-                'exception_traceback': exc_tb,
-                'exception_args': error_args,
-            }
-        finally:
-            meta.Session.remove()
-
-        log.debug('%sGot hook call response %s', self.log_prefix, result)
-        return {
-            'status': result.status,
-            'output': result.output,
-        }
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        pass
diff --git a/rhodecode/lib/middleware/simplesvn.py b/rhodecode/lib/middleware/simplesvn.py
index 451e1708..681992b0 100644
--- a/rhodecode/lib/middleware/simplesvn.py
+++ b/rhodecode/lib/middleware/simplesvn.py
@@ -34,8 +34,7 @@ from rhodecode.lib.utils import is_valid_repo
 from rhodecode.lib.str_utils import safe_str, safe_int, safe_bytes
 from rhodecode.lib.type_utils import str2bool
 from rhodecode.lib.ext_json import json
-from rhodecode.lib.hooks_daemon import store_txn_id_data
-
+from rhodecode.lib.hook_daemon.base import store_txn_id_data
 
 log = logging.getLogger(__name__)
 
diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py
index 23e47104..57415de9 100644
--- a/rhodecode/lib/middleware/simplevcs.py
+++ b/rhodecode/lib/middleware/simplevcs.py
@@ -45,7 +45,7 @@ from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
 from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
-from rhodecode.lib.hooks_daemon import prepare_callback_daemon
+from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.lib.str_utils import safe_bytes
diff --git a/rhodecode/lib/pyramid_utils.py b/rhodecode/lib/pyramid_utils.py
index 81808f21..51aa1816 100644
--- a/rhodecode/lib/pyramid_utils.py
+++ b/rhodecode/lib/pyramid_utils.py
@@ -20,24 +20,14 @@
 
 import os
 import configparser
+
+from rhodecode.lib.config_utils import get_config
 from pyramid.paster import bootstrap as pyramid_bootstrap, setup_logging  # pragma: no cover
 
-from rhodecode.lib.request import Request
-
-
-def get_config(ini_path, **kwargs):
-    parser = configparser.ConfigParser(**kwargs)
-    parser.read(ini_path)
-    return parser
-
-
-def get_app_config(ini_path):
-    from paste.deploy.loadwsgi import appconfig
-    return appconfig(f'config:{ini_path}', relative_to=os.getcwd())
-
 
 def bootstrap(config_uri, options=None, env=None):
     from rhodecode.lib.utils2 import AttributeDict
+    from rhodecode.lib.request import Request
 
     if env:
         os.environ.update(env)
diff --git a/rhodecode/lib/rc_commands/setup_rc.py b/rhodecode/lib/rc_commands/setup_rc.py
index 82427e44..b5626f6f 100644
--- a/rhodecode/lib/rc_commands/setup_rc.py
+++ b/rhodecode/lib/rc_commands/setup_rc.py
@@ -20,7 +20,8 @@ import logging
 import click
 import pyramid.paster
 
-from rhodecode.lib.pyramid_utils import bootstrap, get_app_config
+from rhodecode.lib.pyramid_utils import bootstrap
+from rhodecode.lib.config_utils import get_app_config
 from rhodecode.lib.db_manage import DbManage
 from rhodecode.lib.utils2 import get_encryption_key
 from rhodecode.model.db import Session
diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py
index 01902210..d331d7cd 100644
--- a/rhodecode/lib/utils.py
+++ b/rhodecode/lib/utils.py
@@ -32,11 +32,9 @@ import socket
 import tempfile
 import traceback
 import tarfile
-import urllib.parse
-import warnings
+
 from functools import wraps
 from os.path import join as jn
-from configparser import NoOptionError
 
 import paste
 import pkg_resources
@@ -55,9 +53,6 @@ from rhodecode.model import meta
 from rhodecode.model.db import (
     Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
 from rhodecode.model.meta import Session
-from rhodecode.lib.pyramid_utils import get_config
-from rhodecode.lib.vcs import CurlSession
-from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
 
 
 log = logging.getLogger(__name__)
@@ -827,33 +822,3 @@ def send_test_email(recipients, email_body='TEST EMAIL'):
     email_body = email_body_plaintext = email_body
     subject = f'SUBJECT FROM: {socket.gethostname()}'
     tasks.send_email(recipients, subject, email_body_plaintext, email_body)
-
-
-def call_service_api(ini_path, payload):
-    config = get_config(ini_path)
-    try:
-        host = config.get('app:main', 'app.service_api.host')
-    except NoOptionError:
-        raise ImproperlyConfiguredError(
-            "app.service_api.host is missing. "
-            "Please ensure that app.service_api.host and app.service_api.token are "
-            "defined inside of .ini configuration file."
-        )
-    try:
-        api_url = config.get('app:main', 'rhodecode.api.url')
-    except NoOptionError:
-        from rhodecode import api
-        log.debug('Cannot find rhodecode.api.url, setting API URL TO Default value')
-        api_url = api.DEFAULT_URL
-
-    payload.update({
-        'id': 'service',
-        'auth_token': config.get('app:main', 'app.service_api.token')
-    })
-
-    response = CurlSession().post(urllib.parse.urljoin(host, api_url), json.dumps(payload))
-
-    if response.status_code != 200:
-        raise Exception("Service API responded with error")
-
-    return json.loads(response.content)['result']
diff --git a/rhodecode/model/pull_request.py b/rhodecode/model/pull_request.py
index 420e00b6..0848a826 100644
--- a/rhodecode/model/pull_request.py
+++ b/rhodecode/model/pull_request.py
@@ -38,7 +38,7 @@ from rhodecode.translation import lazy_ugettext
 from rhodecode.lib import helpers as h, hooks_utils, diffs
 from rhodecode.lib import audit_logger
 from collections import OrderedDict
-from rhodecode.lib.hooks_daemon import prepare_callback_daemon
+from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.markup_renderer import (
     DEFAULT_COMMENTS_RENDERER, RstTemplateRenderer)
diff --git a/rhodecode/tests/fixture_mods/fixture_pyramid.py b/rhodecode/tests/fixture_mods/fixture_pyramid.py
index 4c3450cd..75a2fcea 100644
--- a/rhodecode/tests/fixture_mods/fixture_pyramid.py
+++ b/rhodecode/tests/fixture_mods/fixture_pyramid.py
@@ -19,7 +19,7 @@
 
 import pytest
 
-from rhodecode.lib.pyramid_utils import get_app_config
+from rhodecode.lib.config_utils import get_app_config
 from rhodecode.tests.fixture import TestINI
 from rhodecode.tests.server_utils import RcVCSServer
 
diff --git a/rhodecode/tests/fixture_mods/fixture_utils.py b/rhodecode/tests/fixture_mods/fixture_utils.py
index 8d43a457..3351c104 100644
--- a/rhodecode/tests/fixture_mods/fixture_utils.py
+++ b/rhodecode/tests/fixture_mods/fixture_utils.py
@@ -174,7 +174,7 @@ def http_environ():
 
 @pytest.fixture(scope='session')
 def baseapp(ini_config, vcsserver, http_environ_session):
-    from rhodecode.lib.pyramid_utils import get_app_config
+    from rhodecode.lib.config_utils import get_app_config
     from rhodecode.config.middleware import make_pyramid_app
 
     log.info("Using the RhodeCode configuration:{}".format(ini_config))
diff --git a/rhodecode/tests/lib/test_hooks_daemon.py b/rhodecode/tests/lib/test_hooks_daemon.py
index 95832249..95e05e49 100644
--- a/rhodecode/tests/lib/test_hooks_daemon.py
+++ b/rhodecode/tests/lib/test_hooks_daemon.py
@@ -25,17 +25,20 @@ import msgpack
 import pytest
 import tempfile
 
-from rhodecode.lib import hooks_daemon
+from rhodecode.lib.hook_daemon import http_hooks_deamon
+from rhodecode.lib.hook_daemon import celery_hooks_deamon
+from rhodecode.lib.hook_daemon import hook_module
+from rhodecode.lib.hook_daemon import base as hook_base
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.tests.utils import assert_message_in_log
 from rhodecode.lib.ext_json import json
 
-test_proto = hooks_daemon.HooksHttpHandler.MSGPACK_HOOKS_PROTO
+test_proto = http_hooks_deamon.HooksHttpHandler.MSGPACK_HOOKS_PROTO
 
 
 class TestHooks(object):
     def test_hooks_can_be_used_as_a_context_processor(self):
-        hooks = hooks_daemon.Hooks()
+        hooks = hook_module.Hooks()
         with hooks as return_value:
             pass
         assert hooks == return_value
@@ -52,10 +55,10 @@ class TestHooksHttpHandler(object):
         }
         request = self._generate_post_request(data)
         hooks_patcher = mock.patch.object(
-            hooks_daemon.Hooks, data['method'], create=True, return_value=1)
+            hook_module.Hooks, data['method'], create=True, return_value=1)
 
         with hooks_patcher as hooks_mock:
-            handler = hooks_daemon.HooksHttpHandler
+            handler = http_hooks_deamon.HooksHttpHandler
             handler.DEFAULT_HOOKS_PROTO = test_proto
             handler.wbufsize = 10240
             MockServer(handler, request)
@@ -73,21 +76,21 @@ class TestHooksHttpHandler(object):
 
         # patching our _read to return test method and proto used
         read_patcher = mock.patch.object(
-            hooks_daemon.HooksHttpHandler, '_read_request',
+            http_hooks_deamon.HooksHttpHandler, '_read_request',
             return_value=(test_proto, rpc_method, extras))
 
         # patch Hooks instance to return hook_result data on 'test' call
         hooks_patcher = mock.patch.object(
-            hooks_daemon.Hooks, rpc_method, create=True,
+            hook_module.Hooks, rpc_method, create=True,
             return_value=hook_result)
 
         with read_patcher, hooks_patcher:
-            handler = hooks_daemon.HooksHttpHandler
+            handler = http_hooks_deamon.HooksHttpHandler
             handler.DEFAULT_HOOKS_PROTO = test_proto
             handler.wbufsize = 10240
             server = MockServer(handler, request)
 
-        expected_result = hooks_daemon.HooksHttpHandler.serialize_data(hook_result)
+        expected_result = http_hooks_deamon.HooksHttpHandler.serialize_data(hook_result)
 
         server.request.output_stream.seek(0)
         assert server.request.output_stream.readlines()[-1] == expected_result
@@ -97,15 +100,15 @@ class TestHooksHttpHandler(object):
         rpc_method = 'test'
 
         read_patcher = mock.patch.object(
-            hooks_daemon.HooksHttpHandler, '_read_request',
+            http_hooks_deamon.HooksHttpHandler, '_read_request',
             return_value=(test_proto, rpc_method, {}))
 
         hooks_patcher = mock.patch.object(
-            hooks_daemon.Hooks, rpc_method, create=True,
+            hook_module.Hooks, rpc_method, create=True,
             side_effect=Exception('Test exception'))
 
         with read_patcher, hooks_patcher:
-            handler = hooks_daemon.HooksHttpHandler
+            handler = http_hooks_deamon.HooksHttpHandler
             handler.DEFAULT_HOOKS_PROTO = test_proto
             handler.wbufsize = 10240
             server = MockServer(handler, request)
@@ -113,7 +116,7 @@ class TestHooksHttpHandler(object):
         server.request.output_stream.seek(0)
         data = server.request.output_stream.readlines()
         msgpack_data = b''.join(data[5:])
-        org_exc = hooks_daemon.HooksHttpHandler.deserialize_data(msgpack_data)
+        org_exc = http_hooks_deamon.HooksHttpHandler.deserialize_data(msgpack_data)
         expected_result = {
             'exception': 'Exception',
             'exception_traceback': org_exc['exception_traceback'],
@@ -123,8 +126,7 @@ class TestHooksHttpHandler(object):
 
     def test_log_message_writes_to_debug_log(self, caplog):
         ip_port = ('0.0.0.0', 8888)
-        handler = hooks_daemon.HooksHttpHandler(
-            MockRequest('POST /'), ip_port, mock.Mock())
+        handler = http_hooks_deamon.HooksHttpHandler(MockRequest('POST /'), ip_port, mock.Mock())
         fake_date = '1/Nov/2015 00:00:00'
         date_patcher = mock.patch.object(
             handler, 'log_date_time_string', return_value=fake_date)
@@ -136,10 +138,10 @@ class TestHooksHttpHandler(object):
 
         assert_message_in_log(
             caplog.records, expected_message,
-            levelno=logging.DEBUG, module='hooks_daemon')
+            levelno=logging.DEBUG, module='http_hooks_deamon')
 
     def _generate_post_request(self, data, proto=test_proto):
-        if proto == hooks_daemon.HooksHttpHandler.MSGPACK_HOOKS_PROTO:
+        if proto == http_hooks_deamon.HooksHttpHandler.MSGPACK_HOOKS_PROTO:
             payload = msgpack.packb(data)
         else:
             payload = json.dumps(data)
@@ -151,18 +153,18 @@ class TestHooksHttpHandler(object):
 class ThreadedHookCallbackDaemon(object):
     def test_constructor_calls_prepare(self):
         prepare_daemon_patcher = mock.patch.object(
-            hooks_daemon.ThreadedHookCallbackDaemon, '_prepare')
+            http_hooks_deamon.ThreadedHookCallbackDaemon, '_prepare')
         with prepare_daemon_patcher as prepare_daemon_mock:
-            hooks_daemon.ThreadedHookCallbackDaemon()
+            http_hooks_deamon.ThreadedHookCallbackDaemon()
         prepare_daemon_mock.assert_called_once_with()
 
     def test_run_is_called_on_context_start(self):
         patchers = mock.patch.multiple(
-            hooks_daemon.ThreadedHookCallbackDaemon,
+            http_hooks_deamon.ThreadedHookCallbackDaemon,
             _run=mock.DEFAULT, _prepare=mock.DEFAULT, __exit__=mock.DEFAULT)
 
         with patchers as mocks:
-            daemon = hooks_daemon.ThreadedHookCallbackDaemon()
+            daemon = http_hooks_deamon.ThreadedHookCallbackDaemon()
             with daemon as daemon_context:
                 pass
             mocks['_run'].assert_called_once_with()
@@ -170,11 +172,11 @@ class ThreadedHookCallbackDaemon(object):
 
     def test_stop_is_called_on_context_exit(self):
         patchers = mock.patch.multiple(
-            hooks_daemon.ThreadedHookCallbackDaemon,
+            http_hooks_deamon.ThreadedHookCallbackDaemon,
             _run=mock.DEFAULT, _prepare=mock.DEFAULT, _stop=mock.DEFAULT)
 
         with patchers as mocks:
-            daemon = hooks_daemon.ThreadedHookCallbackDaemon()
+            daemon = http_hooks_deamon.ThreadedHookCallbackDaemon()
             with daemon as daemon_context:
                 assert mocks['_stop'].call_count == 0
 
@@ -185,46 +187,47 @@ class ThreadedHookCallbackDaemon(object):
 class TestHttpHooksCallbackDaemon(object):
     def test_hooks_callback_generates_new_port(self, caplog):
         with caplog.at_level(logging.DEBUG):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
         assert daemon._daemon.server_address == ('127.0.0.1', 8881)
 
         with caplog.at_level(logging.DEBUG):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon(host=None, port=None)
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host=None, port=None)
         assert daemon._daemon.server_address[1] in range(0, 66000)
         assert daemon._daemon.server_address[0] != '127.0.0.1'
 
     def test_prepare_inits_daemon_variable(self, tcp_server, caplog):
         with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
         assert daemon._daemon == tcp_server
 
         _, port = tcp_server.server_address
 
         msg = f"HOOKS: 127.0.0.1:{port}  Preparing HTTP callback daemon registering " \
-              f"hook object: <class 'rhodecode.lib.hooks_daemon.HooksHttpHandler'>"
+              f"hook object: <class 'rhodecode.lib.hook_daemon.http_hooks_deamon.HooksHttpHandler'>"
         assert_message_in_log(
-            caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon')
+            caplog.records, msg, levelno=logging.DEBUG, module='http_hooks_deamon')
 
     def test_prepare_inits_hooks_uri_and_logs_it(
             self, tcp_server, caplog):
         with self._tcp_patcher(tcp_server), caplog.at_level(logging.DEBUG):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon(host='127.0.0.1', port=8881)
 
         _, port = tcp_server.server_address
         expected_uri = '{}:{}'.format('127.0.0.1', port)
         assert daemon.hooks_uri == expected_uri
 
         msg = f"HOOKS: 127.0.0.1:{port}  Preparing HTTP callback daemon registering " \
-              f"hook object: <class 'rhodecode.lib.hooks_daemon.HooksHttpHandler'>"
+              f"hook object: <class 'rhodecode.lib.hook_daemon.http_hooks_deamon.HooksHttpHandler'>"
+
         assert_message_in_log(
             caplog.records, msg,
-            levelno=logging.DEBUG, module='hooks_daemon')
+            levelno=logging.DEBUG, module='http_hooks_deamon')
 
     def test_run_creates_a_thread(self, tcp_server):
         thread = mock.Mock()
 
         with self._tcp_patcher(tcp_server):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon()
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon()
 
         with self._thread_patcher(thread) as thread_mock:
             daemon._run()
@@ -238,7 +241,7 @@ class TestHttpHooksCallbackDaemon(object):
     def test_run_logs(self, tcp_server, caplog):
 
         with self._tcp_patcher(tcp_server):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon()
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon()
 
         with self._thread_patcher(mock.Mock()), caplog.at_level(logging.DEBUG):
             daemon._run()
@@ -246,13 +249,13 @@ class TestHttpHooksCallbackDaemon(object):
         assert_message_in_log(
             caplog.records,
             'Running thread-based loop of callback daemon in background',
-            levelno=logging.DEBUG, module='hooks_daemon')
+            levelno=logging.DEBUG, module='http_hooks_deamon')
 
     def test_stop_cleans_up_the_connection(self, tcp_server, caplog):
         thread = mock.Mock()
 
         with self._tcp_patcher(tcp_server):
-            daemon = hooks_daemon.HttpHooksCallbackDaemon()
+            daemon = http_hooks_deamon.HttpHooksCallbackDaemon()
 
         with self._thread_patcher(thread), caplog.at_level(logging.DEBUG):
             with daemon:
@@ -266,18 +269,19 @@ class TestHttpHooksCallbackDaemon(object):
 
         assert_message_in_log(
             caplog.records, 'Waiting for background thread to finish.',
-            levelno=logging.DEBUG, module='hooks_daemon')
+            levelno=logging.DEBUG, module='http_hooks_deamon')
 
     def _tcp_patcher(self, tcp_server):
         return mock.patch.object(
-            hooks_daemon, 'TCPServer', return_value=tcp_server)
+            http_hooks_deamon, 'TCPServer', return_value=tcp_server)
 
     def _thread_patcher(self, thread):
         return mock.patch.object(
-            hooks_daemon.threading, 'Thread', return_value=thread)
+            http_hooks_deamon.threading, 'Thread', return_value=thread)
 
 
 class TestPrepareHooksDaemon(object):
+
     @pytest.mark.parametrize('protocol', ('celery',))
     def test_returns_celery_hooks_callback_daemon_when_celery_protocol_specified(
             self, protocol):
@@ -286,12 +290,12 @@ class TestPrepareHooksDaemon(object):
                             "celery.result_backend = redis://redis/0")
             temp_file.flush()
             expected_extras = {'config': temp_file.name}
-            callback, extras = hooks_daemon.prepare_callback_daemon(
+            callback, extras = hook_base.prepare_callback_daemon(
                 expected_extras, protocol=protocol, host='')
-            assert isinstance(callback, hooks_daemon.CeleryHooksCallbackDaemon)
+            assert isinstance(callback, celery_hooks_deamon.CeleryHooksCallbackDaemon)
 
     @pytest.mark.parametrize('protocol, expected_class', (
-        ('http', hooks_daemon.HttpHooksCallbackDaemon),
+        ('http', http_hooks_deamon.HttpHooksCallbackDaemon),
     ))
     def test_returns_real_hooks_callback_daemon_when_protocol_is_specified(
             self, protocol, expected_class):
@@ -302,7 +306,7 @@ class TestPrepareHooksDaemon(object):
             'task_backend': '',
             'task_queue': ''
         }
-        callback, extras = hooks_daemon.prepare_callback_daemon(
+        callback, extras = hook_base.prepare_callback_daemon(
             expected_extras.copy(), protocol=protocol, host='127.0.0.1',
             txn_id='txnid2')
         assert isinstance(callback, expected_class)
@@ -321,7 +325,7 @@ class TestPrepareHooksDaemon(object):
             'hooks_protocol': protocol.lower()
         }
         with pytest.raises(Exception):
-            callback, extras = hooks_daemon.prepare_callback_daemon(
+            callback, extras = hook_base.prepare_callback_daemon(
                 expected_extras.copy(),
                 protocol=protocol, host='127.0.0.1')
 

From 51a688c0c018605940ac18317091b86432f141f3 Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Wed, 28 Feb 2024 11:52:11 +0200
Subject: [PATCH 023/150] refactor(ssh-wrapper): changed SSHVcsServer to
 SshVcsServer, updated call_service_api method.

---
 .../apps/ssh_support/lib/backends/__init__.py | 34 ++-----------------
 .../apps/ssh_support/lib/backends/base.py     |  8 ++---
 .../apps/ssh_support/lib/backends/git.py      |  4 +--
 rhodecode/apps/ssh_support/lib/backends/hg.py |  9 ++---
 .../apps/ssh_support/lib/backends/svn.py      |  4 +--
 rhodecode/lib/api_utils.py                    | 19 ++++++++---
 6 files changed, 25 insertions(+), 53 deletions(-)

diff --git a/rhodecode/apps/ssh_support/lib/backends/__init__.py b/rhodecode/apps/ssh_support/lib/backends/__init__.py
index f0a3e80e..01a7283e 100644
--- a/rhodecode/apps/ssh_support/lib/backends/__init__.py
+++ b/rhodecode/apps/ssh_support/lib/backends/__init__.py
@@ -24,7 +24,6 @@ from sqlalchemy import Table
 
 from rhodecode.lib.api_utils import call_service_api
 from rhodecode.lib.utils2 import AttributeDict
-from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
 
 from .hg import MercurialServer
 from .git import GitServer
@@ -265,35 +264,6 @@ class SshWrapperStandalone(SshWrapper):
     New version of SshWrapper designed to be depended only on service API
     """
     repos_path = None
-    service_api_host: str
-    service_api_token: str
-    api_url: str
-
-    def __init__(self, command, connection_info, mode,
-                 user, user_id, key_id: int, shell, ini_path: str, settings, env):
-
-        # validate our settings for making a standalone calls
-        try:
-            self.service_api_host = settings['app.service_api.host']
-            self.service_api_token = settings['app.service_api.token']
-        except KeyError:
-            raise ImproperlyConfiguredError(
-                "app.service_api.host or app.service_api.token are missing. "
-                "Please ensure that app.service_api.host and app.service_api.token are "
-                "defined inside of .ini configuration file."
-            )
-
-        try:
-            self.api_url = settings['rhodecode.api.url']
-        except KeyError:
-            raise ImproperlyConfiguredError(
-                "rhodecode.api.url is missing. "
-                "Please ensure that rhodecode.api.url is "
-                "defined inside of .ini configuration file."
-            )
-
-        super(SshWrapperStandalone, self).__init__(
-            command, connection_info, mode, user, user_id, key_id, shell, ini_path, settings, env)
 
     @staticmethod
     def parse_user_related_data(user_data):
@@ -326,7 +296,7 @@ class SshWrapperStandalone(SshWrapper):
             exit_code = 1
 
         elif scm_detected:
-            data = call_service_api(self.service_api_host, self.service_api_token, self.api_url, {
+            data = call_service_api(self.settings, {
                 "method": "service_get_data_for_ssh_wrapper",
                 "args": {"user_id": user_id, "repo_name": scm_repo, "key_id": self.key_id}
             })
@@ -364,7 +334,7 @@ class SshWrapperStandalone(SshWrapper):
         if repo_name.startswith('_'):
             org_repo_name = repo_name
             log.debug('translating UID repo %s', org_repo_name)
-            by_id_match = call_service_api(self.service_api_host, self.service_api_token, self.api_url, {
+            by_id_match = call_service_api(self.settings, {
                 'method': 'service_get_repo_name_by_id',
                 "args": {"repo_id": repo_name}
             })
diff --git a/rhodecode/apps/ssh_support/lib/backends/base.py b/rhodecode/apps/ssh_support/lib/backends/base.py
index 46ffe33d..ee1e2a78 100644
--- a/rhodecode/apps/ssh_support/lib/backends/base.py
+++ b/rhodecode/apps/ssh_support/lib/backends/base.py
@@ -28,7 +28,7 @@ from rhodecode.lib.api_utils import call_service_api
 log = logging.getLogger(__name__)
 
 
-class SSHVcsServer(object):
+class SshVcsServer(object):
     repo_user_agent = None  # set in child classes
     _path = None  # set executable path for hg/git/svn binary
     backend = None  # set in child classes
@@ -62,11 +62,7 @@ class SSHVcsServer(object):
                 from rhodecode.model.scm import ScmModel
                 ScmModel().mark_for_invalidation(repo_name)
             case 'celery':
-                service_api_host = self.settings['app.service_api.host']
-                service_api_token = self.settings['app.service_api.token']
-                api_url = self.settings['rhodecode.api.url']
-
-                call_service_api(service_api_host, service_api_token, api_url, {
+                call_service_api(self.settings, {
                     "method": "service_mark_for_invalidation",
                     "args": {"repo_name": repo_name}
                 })
diff --git a/rhodecode/apps/ssh_support/lib/backends/git.py b/rhodecode/apps/ssh_support/lib/backends/git.py
index ffedfb85..26dba531 100644
--- a/rhodecode/apps/ssh_support/lib/backends/git.py
+++ b/rhodecode/apps/ssh_support/lib/backends/git.py
@@ -21,7 +21,7 @@ import logging
 import subprocess
 
 from vcsserver import hooks
-from .base import SSHVcsServer
+from .base import SshVcsServer
 
 log = logging.getLogger(__name__)
 
@@ -70,7 +70,7 @@ class GitTunnelWrapper(object):
         return result
 
 
-class GitServer(SSHVcsServer):
+class GitServer(SshVcsServer):
     backend = 'git'
     repo_user_agent = 'git'
 
diff --git a/rhodecode/apps/ssh_support/lib/backends/hg.py b/rhodecode/apps/ssh_support/lib/backends/hg.py
index e7f7ed80..3be6000e 100644
--- a/rhodecode/apps/ssh_support/lib/backends/hg.py
+++ b/rhodecode/apps/ssh_support/lib/backends/hg.py
@@ -23,7 +23,7 @@ import tempfile
 import textwrap
 import collections
 
-from .base import SSHVcsServer
+from .base import SshVcsServer
 
 from rhodecode.lib.api_utils import call_service_api
 
@@ -94,7 +94,7 @@ class MercurialTunnelWrapper(object):
             self.remove_configs()
 
 
-class MercurialServer(SSHVcsServer):
+class MercurialServer(SshVcsServer):
     backend = 'hg'
     repo_user_agent = 'mercurial'
     cli_flags = ['phases', 'largefiles', 'extensions', 'experimental', 'hooks']
@@ -111,10 +111,7 @@ class MercurialServer(SSHVcsServer):
     def config_to_hgrc(self, repo_name):
         # Todo: once transition is done only call to service api should exist
         if self.hooks_protocol == 'celery':
-            service_api_host = self.settings['app.service_api.host']
-            service_api_token = self.settings['app.service_api.token']
-            api_url = self.settings['rhodecode.api.url']
-            data = call_service_api(service_api_host, service_api_token, api_url, {
+            data = call_service_api(self.settings, {
                 "method": "service_config_to_hgrc",
                 "args": {"cli_flags": self.cli_flags, "repo_name": repo_name}
             })
diff --git a/rhodecode/apps/ssh_support/lib/backends/svn.py b/rhodecode/apps/ssh_support/lib/backends/svn.py
index e7fefc29..031dabbd 100644
--- a/rhodecode/apps/ssh_support/lib/backends/svn.py
+++ b/rhodecode/apps/ssh_support/lib/backends/svn.py
@@ -25,7 +25,7 @@ import tempfile
 from subprocess import Popen, PIPE
 import urllib.parse
 
-from .base import SSHVcsServer
+from .base import SshVcsServer
 
 log = logging.getLogger(__name__)
 
@@ -218,7 +218,7 @@ class SubversionTunnelWrapper(object):
         return self.return_code
 
 
-class SubversionServer(SSHVcsServer):
+class SubversionServer(SshVcsServer):
     backend = 'svn'
     repo_user_agent = 'svn'
 
diff --git a/rhodecode/lib/api_utils.py b/rhodecode/lib/api_utils.py
index 6c8f11ad..00e18c9f 100644
--- a/rhodecode/lib/api_utils.py
+++ b/rhodecode/lib/api_utils.py
@@ -20,16 +20,25 @@ import urllib.parse
 
 from rhodecode.lib.vcs import CurlSession
 from rhodecode.lib.ext_json import json
+from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
 
 
-def call_service_api(service_api_host, service_api_token, api_url, payload):
-
+def call_service_api(settings, payload):
+    try:
+        api_host = settings['app.service_api.host']
+        api_token = settings['app.service_api.token']
+        api_url = settings['rhodecode.api.url']
+    except KeyError as exc:
+        raise ImproperlyConfiguredError(
+            f"{str(exc)} is missing. "
+            "Please ensure that app.service_api.host, app.service_api.token and rhodecode.api.url are "
+            "defined inside of .ini configuration file."
+        )
     payload.update({
         'id': 'service',
-        'auth_token': service_api_token
+        'auth_token': api_token
     })
-
-    service_api_url = urllib.parse.urljoin(service_api_host, api_url)
+    service_api_url = urllib.parse.urljoin(api_host, api_url)
     response = CurlSession().post(service_api_url, json.dumps(payload))
 
     if response.status_code != 200:

From 7e2e69d2ba09e8b2958d36efb293e553158e3df1 Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Fri, 1 Mar 2024 09:39:14 +0200
Subject: [PATCH 024/150] feat(repo_path-config): moved main storage location
 path into ini file. Fixes: RCCE-61

---
 configs/development.ini                       |  4 +--
 configs/production.ini                        |  4 +--
 rhodecode/api/views/server_api.py             |  5 ++--
 rhodecode/apps/admin/views/settings.py        |  7 ++---
 rhodecode/apps/svn_support/utils.py           |  4 +--
 rhodecode/config/config_maker.py              |  2 ++
 rhodecode/config/environment.py               |  1 -
 rhodecode/config/utils.py                     | 12 --------
 rhodecode/lib/base.py                         |  2 --
 rhodecode/lib/middleware/simplevcs.py         | 11 ++-----
 rhodecode/lib/system_info.py                  | 13 ++++----
 rhodecode/lib/utils.py                        |  4 +--
 rhodecode/model/__init__.py                   |  9 ++++++
 rhodecode/model/db.py                         | 11 +++----
 rhodecode/model/forms.py                      |  4 ---
 rhodecode/model/repo.py                       |  8 -----
 rhodecode/model/repo_group.py                 |  9 ------
 rhodecode/model/scm.py                        |  9 ------
 rhodecode/model/settings.py                   |  3 --
 rhodecode/subscribers.py                      |  5 ++--
 .../admin/settings/settings_vcs.mako          |  3 +-
 rhodecode/templates/base/vcs_settings.mako    | 30 +------------------
 rhodecode/tests/__init__.py                   |  8 +++--
 .../tests/lib/middleware/test_simplesvn.py    |  8 ++---
 .../models/settings/test_vcs_settings.py      | 15 ----------
 rhodecode/tests/models/test_repos.py          | 26 ++++++++--------
 rhodecode/tests/rhodecode.ini                 |  3 --
 27 files changed, 63 insertions(+), 157 deletions(-)

diff --git a/configs/development.ini b/configs/development.ini
index d7a492a2..efbeaa4b 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -261,8 +261,8 @@ auth_ret_code_detection = false
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
-; allows to change the repository location in settings page
-allow_repo_location_change = true
+; Filesystem location were repositories should be stored
+repo_store.path = /var/opt/rhodecode_repo_store
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
diff --git a/configs/production.ini b/configs/production.ini
index 6ffeeb7d..cfe98212 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -212,8 +212,8 @@ auth_ret_code_detection = false
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
-; allows to change the repository location in settings page
-allow_repo_location_change = true
+; Filesystem location were repositories should be stored
+repo_store.path = /var/opt/rhodecode_repo_store
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
diff --git a/rhodecode/api/views/server_api.py b/rhodecode/api/views/server_api.py
index e4881dac..c93b55fe 100644
--- a/rhodecode/api/views/server_api.py
+++ b/rhodecode/api/views/server_api.py
@@ -25,7 +25,7 @@ from rhodecode.api import (
 
 from rhodecode.api.utils import (
     Optional, OAttr, has_superadmin_permission, get_user_or_error)
-from rhodecode.lib.utils import repo2db_mapper
+from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path
 from rhodecode.lib import system_info
 from rhodecode.lib import user_sessions
 from rhodecode.lib import exc_tracking
@@ -33,7 +33,6 @@ from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.model.db import UserIpMap
 from rhodecode.model.scm import ScmModel
-from rhodecode.model.settings import VcsSettingsModel
 from rhodecode.apps.file_store import utils
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
     FileOverSizeException
@@ -103,7 +102,7 @@ def get_repo_store(request, apiuser):
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
-    path = VcsSettingsModel().get_repos_location()
+    path = get_rhodecode_repo_store_path()
     return {"path": path}
 
 
diff --git a/rhodecode/apps/admin/views/settings.py b/rhodecode/apps/admin/views/settings.py
index 754cfa0f..5443da02 100644
--- a/rhodecode/apps/admin/views/settings.py
+++ b/rhodecode/apps/admin/views/settings.py
@@ -38,7 +38,7 @@ from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib.celerylib import tasks, run_task
 from rhodecode.lib.str_utils import safe_str
-from rhodecode.lib.utils import repo2db_mapper
+from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path
 from rhodecode.lib.utils2 import str2bool, AttributeDict
 from rhodecode.lib.index import searcher_from_config
 
@@ -167,9 +167,6 @@ class AdminSettingsView(BaseAppView):
             return Response(html)
 
         try:
-            if c.visual.allow_repo_location_change:
-                model.update_global_path_setting(form_result['paths_root_path'])
-
             model.update_global_ssl_setting(form_result['web_push_ssl'])
             model.update_global_hook_settings(form_result)
 
@@ -217,7 +214,7 @@ class AdminSettingsView(BaseAppView):
     def settings_mapping(self):
         c = self.load_default_context()
         c.active = 'mapping'
-        c.storage_path = VcsSettingsModel().get_repos_location()
+        c.storage_path = get_rhodecode_repo_store_path()
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
diff --git a/rhodecode/apps/svn_support/utils.py b/rhodecode/apps/svn_support/utils.py
index 621e54ea..ec0d2a7f 100644
--- a/rhodecode/apps/svn_support/utils.py
+++ b/rhodecode/apps/svn_support/utils.py
@@ -22,7 +22,7 @@ import os
 from pyramid.renderers import render
 
 from rhodecode.events import trigger
-from rhodecode.lib.utils import get_rhodecode_realm, get_rhodecode_base_path
+from rhodecode.lib.utils import get_rhodecode_realm, get_rhodecode_repo_store_path
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.model.db import RepoGroup
 
@@ -38,7 +38,7 @@ def write_mod_dav_svn_config(settings):
     file_path = settings[config_keys.config_file_path]
     config = _render_mod_dav_svn_config(
         use_ssl=use_ssl,
-        parent_path_root=get_rhodecode_base_path(),
+        parent_path_root=get_rhodecode_repo_store_path(),
         list_parent_path=settings[config_keys.list_parent_path],
         location_root=settings[config_keys.location_root],
         repo_groups=RepoGroup.get_all_repo_groups(),
diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py
index f33fd929..0ac075d9 100644
--- a/rhodecode/config/config_maker.py
+++ b/rhodecode/config/config_maker.py
@@ -117,6 +117,8 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
 
     settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
 
+    # repo_store path
+    settings_maker.make_setting('repo_store.path', '/var/opt/rhodecode_repo_store')
     # Support legacy values of vcs.scm_app_implementation. Legacy
     # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
     # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
diff --git a/rhodecode/config/environment.py b/rhodecode/config/environment.py
index 376cad85..b67da521 100644
--- a/rhodecode/config/environment.py
+++ b/rhodecode/config/environment.py
@@ -81,7 +81,6 @@ def load_pyramid_environment(global_config, settings):
     rhodecode.PYRAMID_SETTINGS = settings_merged
     rhodecode.CONFIG = settings_merged
     rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
-    rhodecode.CONFIG['default_base_path'] = utils.get_default_base_path()
 
     if vcs_server_enabled:
         connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
diff --git a/rhodecode/config/utils.py b/rhodecode/config/utils.py
index 6f443954..41f8c9dc 100644
--- a/rhodecode/config/utils.py
+++ b/rhodecode/config/utils.py
@@ -102,15 +102,3 @@ def get_default_user_id():
         user_id = result.first()[0]
 
     return user_id
-
-
-def get_default_base_path():
-    from sqlalchemy import text
-    from rhodecode.model import meta
-
-    engine = meta.get_engine()
-    with meta.SA_Session(engine) as session:
-        result = session.execute(text("SELECT ui_value from rhodecode_ui where ui_key = '/'"))
-        base_path = result.first()[0]
-
-    return base_path
diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py
index da8104fe..c82ef08e 100644
--- a/rhodecode/lib/base.py
+++ b/rhodecode/lib/base.py
@@ -347,8 +347,6 @@ def attach_context_attributes(context, request, user_id=None, is_api=None):
     context.ssh_key_generator_enabled = str2bool(
         config.get('ssh.enable_ui_key_generator', 'true'))
 
-    context.visual.allow_repo_location_change = str2bool(
-        config.get('allow_repo_location_change', True))
     context.visual.allow_custom_hooks_settings = str2bool(
         config.get('allow_custom_hooks_settings', True))
     context.debug_style = str2bool(config.get('debug_style', False))
diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py
index 57415de9..7f7f2a1c 100644
--- a/rhodecode/lib/middleware/simplevcs.py
+++ b/rhodecode/lib/middleware/simplevcs.py
@@ -156,17 +156,10 @@ class SimpleVCS(object):
 
     @property
     def base_path(self):
-        settings_path = self.repo_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
+        settings_path = self.config.get('repo_store.path')
 
         if not settings_path:
-            settings_path = self.global_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
-
-        if not settings_path:
-            # try, maybe we passed in explicitly as config option
-            settings_path = self.config.get('base_path')
-
-        if not settings_path:
-            raise ValueError('FATAL: base_path is empty')
+            raise ValueError('FATAL: repo_store.path is empty')
         return settings_path
 
     def set_repo_names(self, environ):
diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py
index 43ad948e..a38a1890 100644
--- a/rhodecode/lib/system_info.py
+++ b/rhodecode/lib/system_info.py
@@ -331,8 +331,8 @@ def cpu():
 @register_sysinfo
 def storage():
     from rhodecode.lib.helpers import format_byte_size_binary
-    from rhodecode.model.settings import VcsSettingsModel
-    path = VcsSettingsModel().get_repos_location()
+    from rhodecode.lib.utils import get_rhodecode_repo_store_path
+    path = get_rhodecode_repo_store_path()
 
     value = dict(percent=0, used=0, total=0, path=path, text='')
     state = STATE_OK_DEFAULT
@@ -364,8 +364,8 @@ def storage():
 
 @register_sysinfo
 def storage_inodes():
-    from rhodecode.model.settings import VcsSettingsModel
-    path = VcsSettingsModel().get_repos_location()
+    from rhodecode.lib.utils import get_rhodecode_repo_store_path
+    path = get_rhodecode_repo_store_path()
 
     value = dict(percent=0.0, free=0, used=0, total=0, path=path, text='')
     state = STATE_OK_DEFAULT
@@ -442,11 +442,10 @@ def storage_archives():
 @register_sysinfo
 def storage_gist():
     from rhodecode.model.gist import GIST_STORE_LOC
-    from rhodecode.model.settings import VcsSettingsModel
-    from rhodecode.lib.utils import safe_str
+    from rhodecode.lib.utils import safe_str, get_rhodecode_repo_store_path
     from rhodecode.lib.helpers import format_byte_size_binary
     path = safe_str(os.path.join(
-        VcsSettingsModel().get_repos_location(), GIST_STORE_LOC))
+        get_rhodecode_repo_store_path(), GIST_STORE_LOC))
 
     # gist storage
     value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py
index d331d7cd..ac07c77e 100644
--- a/rhodecode/lib/utils.py
+++ b/rhodecode/lib/utils.py
@@ -472,14 +472,14 @@ def get_rhodecode_realm():
     return safe_str(realm.app_settings_value)
 
 
-def get_rhodecode_base_path():
+def get_rhodecode_repo_store_path():
     """
     Returns the base path. The base path is the filesystem path which points
     to the repository store.
     """
 
     import rhodecode
-    return rhodecode.CONFIG['default_base_path']
+    return rhodecode.CONFIG['repo_store.path']
 
 
 def map_groups(path):
diff --git a/rhodecode/model/__init__.py b/rhodecode/model/__init__.py
index 0805a867..2683acc7 100644
--- a/rhodecode/model/__init__.py
+++ b/rhodecode/model/__init__.py
@@ -21,6 +21,7 @@ import logging
 
 import rhodecode
 from rhodecode.model import meta, db
+from rhodecode.lib.utils import get_rhodecode_repo_store_path
 from rhodecode.lib.utils2 import obfuscate_url_pw, get_encryption_key
 
 log = logging.getLogger(__name__)
@@ -138,3 +139,11 @@ class BaseModel(object):
         Returns all instances of what is defined in `cls` class variable
         """
         return cls.cls.getAll()
+
+    @property
+    def repos_path(self):
+        """
+        Gets the repositories root path from *ini file
+        """
+
+        return get_rhodecode_repo_store_path()
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index 34d74e5f..63e095bd 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -1942,8 +1942,8 @@ class Repository(Base, BaseModel):
 
         :param cls:
         """
-        from rhodecode.lib.utils import get_rhodecode_base_path
-        return get_rhodecode_base_path()
+        from rhodecode.lib.utils import get_rhodecode_repo_store_path
+        return get_rhodecode_repo_store_path()
 
     @classmethod
     def get_all_repos(cls, user_id=Optional(None), group_id=Optional(None),
@@ -2009,16 +2009,13 @@ class Repository(Base, BaseModel):
     def groups_and_repo(self):
         return self.groups_with_parents, self
 
-    @LazyProperty
+    @property
     def repo_path(self):
         """
         Returns base full path for that repository means where it actually
         exists on a filesystem
         """
-        q = Session().query(RhodeCodeUi).filter(
-            RhodeCodeUi.ui_key == self.NAME_SEP)
-        q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
-        return q.one().ui_value
+        return self.base_path()
 
     @property
     def repo_full_path(self):
diff --git a/rhodecode/model/forms.py b/rhodecode/model/forms.py
index 39d3b21c..d05871ae 100644
--- a/rhodecode/model/forms.py
+++ b/rhodecode/model/forms.py
@@ -434,10 +434,6 @@ def ApplicationUiSettingsForm(localizer):
 
     class _ApplicationUiSettingsForm(_BaseVcsSettingsForm):
         web_push_ssl = v.StringBoolean(if_missing=False)
-        paths_root_path = All(
-            v.ValidPath(localizer),
-            v.UnicodeString(strip=True, min=1, not_empty=True)
-        )
         largefiles_usercache = All(
             v.ValidPath(localizer),
             v.UnicodeString(strip=True, min=2, not_empty=True))
diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py
index e784833c..12f37b9a 100644
--- a/rhodecode/model/repo.py
+++ b/rhodecode/model/repo.py
@@ -83,14 +83,6 @@ class RepoModel(BaseModel):
 
         return repo_to_perm
 
-    @LazyProperty
-    def repos_path(self):
-        """
-        Gets the repositories root path from database
-        """
-        settings_model = VcsSettingsModel(sa=self.sa)
-        return settings_model.get_repos_location()
-
     def get(self, repo_id):
         repo = self.sa.query(Repository) \
             .filter(Repository.repo_id == repo_id)
diff --git a/rhodecode/model/repo_group.py b/rhodecode/model/repo_group.py
index 2d754169..64f6db56 100644
--- a/rhodecode/model/repo_group.py
+++ b/rhodecode/model/repo_group.py
@@ -62,15 +62,6 @@ class RepoGroupModel(BaseModel):
     def get_repo_group(self, repo_group):
         return self._get_repo_group(repo_group)
 
-    @LazyProperty
-    def repos_path(self):
-        """
-        Gets the repositories root path from database
-        """
-
-        settings_model = VcsSettingsModel(sa=self.sa)
-        return settings_model.get_repos_location()
-
     def get_by_group_name(self, repo_group_name, cache=None):
         repo = self.sa.query(RepoGroup) \
             .filter(RepoGroup.group_name == repo_group_name)
diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py
index 560abe59..30228e6b 100644
--- a/rhodecode/model/scm.py
+++ b/rhodecode/model/scm.py
@@ -189,15 +189,6 @@ class ScmModel(BaseModel):
     Generic Scm Model
     """
 
-    @LazyProperty
-    def repos_path(self):
-        """
-        Gets the repositories root path from database
-        """
-
-        settings_model = VcsSettingsModel(sa=self.sa)
-        return settings_model.get_repos_location()
-
     def repo_scan(self, repos_path=None):
         """
         Listing of repositories in given path. This path should not be a
diff --git a/rhodecode/model/settings.py b/rhodecode/model/settings.py
index 0beff5fe..3b3f15dc 100644
--- a/rhodecode/model/settings.py
+++ b/rhodecode/model/settings.py
@@ -811,9 +811,6 @@ class VcsSettingsModel(object):
         else:
             return self.get_repo_general_settings()
 
-    def get_repos_location(self):
-        return self.global_settings.get_ui_by_key('/').ui_value
-
     def _filter_ui_settings(self, settings):
         filtered_settings = [
             s for s in settings if self._should_keep_setting(s)]
diff --git a/rhodecode/subscribers.py b/rhodecode/subscribers.py
index b217c3e5..07c04fd9 100644
--- a/rhodecode/subscribers.py
+++ b/rhodecode/subscribers.py
@@ -116,8 +116,9 @@ def scan_repositories_if_enabled(event):
     import_on_startup = settings['startup.import_repos']
     if vcs_server_enabled and import_on_startup:
         from rhodecode.model.scm import ScmModel
-        from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path
-        repositories = ScmModel().repo_scan(get_rhodecode_base_path())
+        from rhodecode.lib.utils import repo2db_mapper
+        scm = ScmModel()
+        repositories = scm.repo_scan(scm.repos_path)
         repo2db_mapper(repositories, remove_obsolete=False)
 
 
diff --git a/rhodecode/templates/admin/settings/settings_vcs.mako b/rhodecode/templates/admin/settings/settings_vcs.mako
index 4f37f110..4b6dd11a 100644
--- a/rhodecode/templates/admin/settings/settings_vcs.mako
+++ b/rhodecode/templates/admin/settings/settings_vcs.mako
@@ -6,8 +6,7 @@ ${h.secure_form(h.route_path('admin_settings_vcs_update'), request=request)}
             suffix='',
             svn_tag_patterns=c.svn_tag_patterns,
             svn_branch_patterns=c.svn_branch_patterns,
-            display_globals=True,
-            allow_repo_location_change=c.visual.allow_repo_location_change
+            display_globals=True
         )}
         <div class="buttons">
             ${h.submit('save',_('Save settings'),class_="btn")}
diff --git a/rhodecode/templates/base/vcs_settings.mako b/rhodecode/templates/base/vcs_settings.mako
index d9b80493..1b2feb17 100644
--- a/rhodecode/templates/base/vcs_settings.mako
+++ b/rhodecode/templates/base/vcs_settings.mako
@@ -3,7 +3,7 @@
 ##    <%namespace name="vcss" file="/base/vcssettings.mako"/>
 ##    ${vcss.vcs_settings_fields()}
 
-<%def name="vcs_settings_fields(suffix='', svn_branch_patterns=None, svn_tag_patterns=None, repo_type=None, display_globals=False, allow_repo_location_change=False, **kwargs)">
+<%def name="vcs_settings_fields(suffix='', svn_branch_patterns=None, svn_tag_patterns=None, repo_type=None, display_globals=False, **kwargs)">
     % if display_globals:
         <div class="panel panel-default">
             <div class="panel-heading" id="general">
@@ -23,34 +23,6 @@
         </div>
     % endif
 
-    % if display_globals:
-        <div class="panel panel-default">
-            <div class="panel-heading" id="vcs-storage-options">
-                <h3 class="panel-title">${_('Main Storage Location')}<a class="permalink" href="#vcs-storage-options"> ¶</a></h3>
-            </div>
-            <div class="panel-body">
-                <div class="field">
-                    <div class="inputx locked_input">
-                        %if allow_repo_location_change:
-                            ${h.text('paths_root_path',size=59,readonly="readonly", class_="disabled")}
-                            <span id="path_unlock" class="tooltip"
-                                    title="${h.tooltip(_('Click to unlock. You must restart RhodeCode in order to make this setting take effect.'))}">
-                                <div class="btn btn-default lock_input_button"><i id="path_unlock_icon" class="icon-lock"></i></div>
-                            </span>
-                        %else:
-                            ${_('Repository location change is disabled. You can enable this by changing the `allow_repo_location_change` inside .ini file.')}
-                            ## form still requires this but we cannot internally change it anyway
-                            ${h.hidden('paths_root_path',size=30,readonly="readonly", class_="disabled")}
-                        %endif
-                    </div>
-                </div>
-                <div class="label">
-                    <span class="help-block">${_('Filesystem location where repositories should be stored. After changing this value a restart and rescan of the repository folder are required.')}</span>
-                </div>
-            </div>
-        </div>
-    % endif
-
     % if display_globals or repo_type in ['git', 'hg']:
         <div class="panel panel-default">
             <div class="panel-heading" id="vcs-hooks-options">
diff --git a/rhodecode/tests/__init__.py b/rhodecode/tests/__init__.py
index e507f3c2..55591696 100644
--- a/rhodecode/tests/__init__.py
+++ b/rhodecode/tests/__init__.py
@@ -27,6 +27,7 @@ import urllib.parse
 
 import pytest
 
+import rhodecode
 from rhodecode.model.db import User
 from rhodecode.lib import auth
 from rhodecode.lib import helpers as h
@@ -53,8 +54,11 @@ __all__ = [
 
 # SOME GLOBALS FOR TESTS
 TEST_DIR = tempfile.gettempdir()
+if os.getenv("RC_TEST"):
+    rhodecode.CONFIG['repo_store.path'] = TEST_DIR
 
-TESTS_TMP_PATH = jn(TEST_DIR, 'rc_test_{}'.format(next(tempfile._RandomNameSequence())))
+# Once repo_store.path in test config will be changed update TEST_TMP_PATH accordingly
+TESTS_TMP_PATH = jn(TEST_DIR, 'rc_test_a0hmmrva')
 TEST_USER_ADMIN_LOGIN = 'test_admin'
 TEST_USER_ADMIN_PASS = 'test12'
 TEST_USER_ADMIN_EMAIL = 'test_admin@mail.com'
@@ -111,7 +115,7 @@ def get_new_dir(title):
     hex_str = sha1_safe(f'{os.getpid()} {time.time()}')
     name_parts.append(hex_str)
     name = '-'.join(name_parts)
-    path = os.path.join(TEST_DIR, name)
+    path = jn(TEST_DIR, name)
     return get_normalized_path(path)
 
 
diff --git a/rhodecode/tests/lib/middleware/test_simplesvn.py b/rhodecode/tests/lib/middleware/test_simplesvn.py
index 21a5a890..5e73634e 100644
--- a/rhodecode/tests/lib/middleware/test_simplesvn.py
+++ b/rhodecode/tests/lib/middleware/test_simplesvn.py
@@ -23,16 +23,16 @@ import pytest
 from unittest.mock import patch, Mock, MagicMock
 
 from rhodecode.lib.middleware.simplesvn import SimpleSvn, SimpleSvnApp
-from rhodecode.lib.utils import get_rhodecode_base_path
+from rhodecode.lib.utils import get_rhodecode_repo_store_path
 from rhodecode.tests import SVN_REPO, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS
 
 
 class TestSimpleSvn(object):
     @pytest.fixture(autouse=True)
     def simple_svn(self, baseapp, request_stub):
-        base_path = get_rhodecode_base_path()
+        base_path = get_rhodecode_repo_store_path()
         self.app = SimpleSvn(
-            config={'auth_ret_code': '', 'base_path': base_path},
+            config={'auth_ret_code': '', 'repo_store.path': base_path},
             registry=request_stub.registry)
 
     def test_get_config(self):
@@ -126,7 +126,7 @@ class TestSimpleSvnApp(object):
     def setup_method(self, method):
         # note(marcink): this is hostname from docker compose used for testing...
         self.host = 'http://svn:8090'
-        base_path = get_rhodecode_base_path()
+        base_path = get_rhodecode_repo_store_path()
         self.app = SimpleSvnApp(
             config={'subversion_http_server_url': self.host,
                     'base_path': base_path})
diff --git a/rhodecode/tests/models/settings/test_vcs_settings.py b/rhodecode/tests/models/settings/test_vcs_settings.py
index fe3dcc23..946ce127 100644
--- a/rhodecode/tests/models/settings/test_vcs_settings.py
+++ b/rhodecode/tests/models/settings/test_vcs_settings.py
@@ -1004,21 +1004,6 @@ class TestGetSvnPatterns(object):
         settings_mock.assert_called_once_with(*args)
 
 
-class TestGetReposLocation(object):
-    def test_returns_repos_location(self, repo_stub):
-        model = VcsSettingsModel()
-
-        result_mock = mock.Mock()
-        result_mock.ui_value = '/tmp'
-
-        with mock.patch.object(model, 'global_settings') as settings_mock:
-            settings_mock.get_ui_by_key.return_value = result_mock
-            result = model.get_repos_location()
-
-        settings_mock.get_ui_by_key.assert_called_once_with('/')
-        assert result == '/tmp'
-
-
 class TestCreateOrUpdateRepoSettings(object):
     FORM_DATA = {
         'inherit_global_settings': False,
diff --git a/rhodecode/tests/models/test_repos.py b/rhodecode/tests/models/test_repos.py
index cc0a768f..0ba88a4a 100644
--- a/rhodecode/tests/models/test_repos.py
+++ b/rhodecode/tests/models/test_repos.py
@@ -121,20 +121,20 @@ class TestRepoModel(object):
     def test_create_filesystem_repo_installs_hooks(self, tmpdir, backend):
         repo = backend.create_repo()
         repo_name = repo.repo_name
-        model = RepoModel()
-        repo_location = tempfile.mkdtemp()
-        model.repos_path = repo_location
-        repo = model._create_filesystem_repo(
-            repo_name, backend.alias, repo_group='', clone_uri=None)
+        with mock.patch('rhodecode.model.repo.RepoModel.repos_path',
+                        new_callable=mock.PropertyMock) as mocked_models_property:
+            mocked_models_property.return_value = tempfile.mkdtemp()
+            repo = RepoModel()._create_filesystem_repo(
+                repo_name, backend.alias, repo_group='', clone_uri=None)
 
-        hooks = {
-            'svn': ('pre-commit', 'post-commit'),
-            'git': ('pre-receive', 'post-receive'),
-        }
-        for hook in hooks[backend.alias]:
-            with open(os.path.join(repo.path, 'hooks', hook)) as f:
-                data = f.read()
-                assert 'RC_HOOK_VER' in data
+            hooks = {
+                'svn': ('pre-commit', 'post-commit'),
+                'git': ('pre-receive', 'post-receive'),
+            }
+            for hook in hooks[backend.alias]:
+                with open(os.path.join(repo.path, 'hooks', hook)) as f:
+                    data = f.read()
+                    assert 'RC_HOOK_VER' in data
 
     @pytest.mark.parametrize("use_global_config, repo_name_passed", [
         (True, False),
diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini
index f01777ef..72fb2c9e 100644
--- a/rhodecode/tests/rhodecode.ini
+++ b/rhodecode/tests/rhodecode.ini
@@ -207,9 +207,6 @@ auth_ret_code_detection = false
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
-; allows to change the repository location in settings page
-allow_repo_location_change = true
-
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 

From 216e86d96eee3f3c41af266534972fa2005e2ece Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Fri, 1 Mar 2024 11:43:04 +0100
Subject: [PATCH 025/150] dev(makefile): hide prompt info on dev sh env

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 3d59ea61..e7112cec 100644
--- a/Makefile
+++ b/Makefile
@@ -111,7 +111,7 @@ dev-sh:
 	curl https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh | sh
 	@echo "source <(carapace _carapace)" > /home/rhodecode/.zsrc
 	@echo "${RC_DEV_CMD_HELP}"
-	PROMPT='%(?.%F{green}√.%F{red}?%?)%f %B%F{240}%1~%f%b %# ' zsh
+	@PROMPT='%(?.%F{green}√.%F{red}?%?)%f %B%F{240}%1~%f%b %# ' zsh
 
 
 .PHONY: dev-cleanup

From 747534f76d14b0c1646203dca2c340191cc1358c Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Fri, 1 Mar 2024 13:40:11 +0100
Subject: [PATCH 026/150] fix(svn-ssh): fixed svn ssh wrapper

---
 .../apps/ssh_support/lib/backends/svn.py      | 42 +++++++++++--------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/rhodecode/apps/ssh_support/lib/backends/svn.py b/rhodecode/apps/ssh_support/lib/backends/svn.py
index d9f1f858..7b728d75 100644
--- a/rhodecode/apps/ssh_support/lib/backends/svn.py
+++ b/rhodecode/apps/ssh_support/lib/backends/svn.py
@@ -25,6 +25,8 @@ import tempfile
 from subprocess import Popen, PIPE
 import urllib.parse
 
+from rhodecode.lib.str_utils import safe_bytes
+from rhodecode_tools.lib.utils import safe_str
 from .base import SshVcsServer
 
 log = logging.getLogger(__name__)
@@ -81,7 +83,7 @@ class SubversionTunnelWrapper(object):
 
     def sync(self):
         while self.process.poll() is None:
-            next_byte = self.stdin.read(1)
+            next_byte = self.stdin.buffer.read(1)
             if not next_byte:
                 break
             self.process.stdin.write(next_byte)
@@ -106,7 +108,7 @@ class SubversionTunnelWrapper(object):
         data['url'] = self._svn_string(data['url'])
         data['ra_client'] = self._svn_string(data['ra_client'])
         data['client'] = data['client'] or ''
-        buffer_ = (
+        buffer_ = safe_bytes(
             "( {version} ( {capabilities} ) {url}{ra_client}"
             "( {client}) ) ".format(**data))
         self.process.stdin.write(buffer_)
@@ -127,21 +129,21 @@ class SubversionTunnelWrapper(object):
         return f'{len(str_)}:{str_} '
 
     def _read_first_client_response(self):
-        buffer_ = ""
+        buffer_ = b""
         brackets_stack = []
         while True:
-            next_byte = self.stdin.read(1)
+            next_byte = self.stdin.buffer.read(1)
             buffer_ += next_byte
-            if next_byte == "(":
+            if next_byte == b"(":
                 brackets_stack.append(next_byte)
-            elif next_byte == ")":
+            elif next_byte == b")":
                 brackets_stack.pop()
-            elif next_byte == " " and not brackets_stack:
+            elif next_byte == b" " and not brackets_stack:
                 break
 
         return buffer_
 
-    def _parse_first_client_response(self, buffer_):
+    def _parse_first_client_response(self, buffer_: bytes):
         """
         According to the Subversion RA protocol, the first request
         should look like:
@@ -151,16 +153,20 @@ class SubversionTunnelWrapper(object):
 
         Please check https://svn.apache.org/repos/asf/subversion/trunk/subversion/libsvn_ra_svn/protocol
         """
-        version_re = r'(?P<version>\d+)'
-        capabilities_re = r'\(\s(?P<capabilities>[\w\d\-\ ]+)\s\)'
-        url_re = r'\d+\:(?P<url>[\W\w]+)'
-        ra_client_re = r'(\d+\:(?P<ra_client>[\W\w]+)\s)'
-        client_re = r'(\d+\:(?P<client>[\W\w]+)\s)*'
+        version_re = br'(?P<version>\d+)'
+        capabilities_re = br'\(\s(?P<capabilities>[\w\d\-\ ]+)\s\)'
+        url_re = br'\d+\:(?P<url>[\W\w]+)'
+        ra_client_re = br'(\d+\:(?P<ra_client>[\W\w]+)\s)'
+        client_re = br'(\d+\:(?P<client>[\W\w]+)\s)*'
         regex = re.compile(
-            r'^\(\s{version}\s{capabilities}\s{url}\s{ra_client}'
-            r'\(\s{client}\)\s\)\s*$'.format(
-                version=version_re, capabilities=capabilities_re,
-                url=url_re, ra_client=ra_client_re, client=client_re))
+            br'^\(\s%b\s%b\s%b\s%b'
+            br'\(\s%b\)\s\)\s*$' % (
+                version_re,
+                capabilities_re,
+                url_re,
+                ra_client_re,
+                client_re)
+        )
         matcher = regex.match(buffer_)
 
         return matcher.groupdict() if matcher else None
@@ -202,7 +208,7 @@ class SubversionTunnelWrapper(object):
 
         url_parts = urllib.parse.urlparse(first_response['url'])
 
-        self.server.repo_name = self._match_repo_name(url_parts.path.strip('/'))
+        self.server.repo_name = self._match_repo_name(safe_str(url_parts.path).strip('/'))
 
         exit_code = self.server._check_permissions(action)
         if exit_code:

From 0a814bd6b8f3a80c34313fd642604cf642234149 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Fri, 1 Mar 2024 13:49:28 +0100
Subject: [PATCH 027/150] fix(svn): ensure we have defaults for old .ini files
 and upgrades

---
 rhodecode/config/config_maker.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py
index f33fd929..d27897fe 100644
--- a/rhodecode/config/config_maker.py
+++ b/rhodecode/config/config_maker.py
@@ -94,6 +94,7 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
     settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
     settings_maker.make_setting('is_test', False, parser='bool')
     settings_maker.make_setting('gzip_responses', False, parser='bool')
+    settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
 
     # statsd
     settings_maker.make_setting('statsd.enabled', False, parser='bool')
@@ -103,13 +104,14 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
     settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
 
     settings_maker.make_setting('vcs.svn.compatible_version', '')
+    settings_maker.make_setting('vcs.svn.proxy.enabled', True, parser='bool')
+    settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090', parser='string')
     settings_maker.make_setting('vcs.hooks.protocol', 'http')
     settings_maker.make_setting('vcs.hooks.host', '*')
     settings_maker.make_setting('vcs.scm_app_implementation', 'http')
     settings_maker.make_setting('vcs.server', '')
     settings_maker.make_setting('vcs.server.protocol', 'http')
     settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
-    settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
     settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
     settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
     settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')

From b79db3eb9cb2284bef9214a040bd95e24a72990c Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Fri, 1 Mar 2024 16:19:48 +0100
Subject: [PATCH 028/150] fix(tests): fixed bad value for default binary_dir
 value

---
 rhodecode/tests/vcsserver_http.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rhodecode/tests/vcsserver_http.ini b/rhodecode/tests/vcsserver_http.ini
index 36dd49f7..461cf14c 100644
--- a/rhodecode/tests/vcsserver_http.ini
+++ b/rhodecode/tests/vcsserver_http.ini
@@ -33,7 +33,7 @@ locale = en_US.UTF-8
 ; path to binaries for vcsserver, it should be set by the installer
 ; at installation time, e.g /home/user/vcsserver-1/profile/bin
 ; it can also be a path to nix-build output in case of development
-core.binary_dir = ""
+core.binary_dir =
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory

From 104352ccda7dd3ed1eae18ca649d611278f4e807 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Sun, 3 Mar 2024 08:40:12 +0100
Subject: [PATCH 029/150] fix(svn): more binary protocol svn ssh fixes

---
 .../apps/ssh_support/lib/backends/svn.py      | 38 +++++++++++--------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/rhodecode/apps/ssh_support/lib/backends/svn.py b/rhodecode/apps/ssh_support/lib/backends/svn.py
index 7b728d75..b9273fe8 100644
--- a/rhodecode/apps/ssh_support/lib/backends/svn.py
+++ b/rhodecode/apps/ssh_support/lib/backends/svn.py
@@ -25,7 +25,6 @@ import tempfile
 from subprocess import Popen, PIPE
 import urllib.parse
 
-from rhodecode.lib.str_utils import safe_bytes
 from rhodecode_tools.lib.utils import safe_str
 from .base import SshVcsServer
 
@@ -103,19 +102,27 @@ class SubversionTunnelWrapper(object):
 
     def patch_first_client_response(self, response, **kwargs):
         self.create_hooks_env()
-        data = response.copy()
-        data.update(kwargs)
-        data['url'] = self._svn_string(data['url'])
-        data['ra_client'] = self._svn_string(data['ra_client'])
-        data['client'] = data['client'] or ''
-        buffer_ = safe_bytes(
-            "( {version} ( {capabilities} ) {url}{ra_client}"
-            "( {client}) ) ".format(**data))
+
+        version = response['version']
+        capabilities = response['capabilities']
+        client = response['client'] or b''
+
+        url = self._svn_bytes(response['url'])
+        ra_client = self._svn_bytes(response['ra_client'])
+
+        buffer_ = b"( %b ( %b ) %b%b( %b) ) " % (
+                version,
+                capabilities,
+                url,
+                ra_client,
+                client
+            )
         self.process.stdin.write(buffer_)
 
     def fail(self, message):
-        print("( failure ( ( 210005 {message} 0: 0 ) ) )".format(
-            message=self._svn_string(message)))
+        fail_msg = b"( failure ( ( 210005 %b 0: 0 ) ) )" % self._svn_bytes(message)
+        sys.stdout.buffer.write(fail_msg)
+        sys.stdout.flush()
         self.remove_configs()
         self.process.kill()
         return 1
@@ -123,10 +130,11 @@ class SubversionTunnelWrapper(object):
     def interrupt(self, signum, frame):
         self.fail("Exited by timeout")
 
-    def _svn_string(self, str_):
-        if not str_:
-            return ''
-        return f'{len(str_)}:{str_} '
+    def _svn_bytes(self, bytes_: bytes) -> bytes:
+        if not bytes_:
+            return b''
+
+        return f'{len(bytes_)}:'.encode() + bytes_ + b' '
 
     def _read_first_client_response(self):
         buffer_ = b""

From f928ad9b22e546f83bdfd4c11443b28cd64010ed Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Sun, 3 Mar 2024 08:40:40 +0100
Subject: [PATCH 030/150] feat(mercurial): added close branch commit message
 template

---
 rhodecode/lib/vcs/backends/hg/repository.py | 8 +++++++-
 rhodecode/lib/vcs/conf/settings.py          | 3 +++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/rhodecode/lib/vcs/backends/hg/repository.py b/rhodecode/lib/vcs/backends/hg/repository.py
index 9d213316..8b2330e1 100644
--- a/rhodecode/lib/vcs/backends/hg/repository.py
+++ b/rhodecode/lib/vcs/backends/hg/repository.py
@@ -35,6 +35,7 @@ from rhodecode.lib.datelib import (
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.utils2 import CachedProperty
 from rhodecode.lib.vcs import connection, exceptions
+from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.backends.base import (
     BaseRepository, CollectionGenerator, Config, MergeResponse,
     MergeFailureReason, Reference, BasePathPermissionChecker)
@@ -722,7 +723,12 @@ class MercurialRepository(BaseRepository):
         commit needs to be pushed.
         """
         self._update(source_ref.commit_id)
-        message = close_message or f"Closing branch: `{source_ref.name}`"
+        message = (close_message or vcs_settings.HG_CLOSE_BRANCH_MESSAGE_TMPL).format(
+            user_name=user_name,
+            user_email=user_email,
+            target_ref_name=target_ref.name,
+            source_ref_name=source_ref.name
+        )
         try:
             self._remote.commit(
                 message=safe_str(message),
diff --git a/rhodecode/lib/vcs/conf/settings.py b/rhodecode/lib/vcs/conf/settings.py
index 93dacded..8a8640a5 100644
--- a/rhodecode/lib/vcs/conf/settings.py
+++ b/rhodecode/lib/vcs/conf/settings.py
@@ -58,6 +58,9 @@ MERGE_MESSAGE_TMPL = (
 MERGE_DRY_RUN_MESSAGE = 'dry_run_merge_message_from_rhodecode'
 MERGE_DRY_RUN_USER = 'Dry-Run User'
 MERGE_DRY_RUN_EMAIL = 'dry-run-merge@rhodecode.com'
+HG_CLOSE_BRANCH_MESSAGE_TMPL = (
+    'Closing branch: `{source_ref_name}`'
+)
 
 
 def available_aliases():

From 2dd2eea658ee6e3f06a1dc7177871911f52aead9 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 4 Mar 2024 11:34:15 +0100
Subject: [PATCH 031/150] chore(configs): optimize configs for docker env

---
 configs/development.ini | 123 ++++++++++-----------------------------
 configs/production.ini  | 125 +++++++++++-----------------------------
 2 files changed, 65 insertions(+), 183 deletions(-)

diff --git a/configs/development.ini b/configs/development.ini
index 7b63786e..0e5aea4f 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -305,7 +305,7 @@ file_store.enabled = true
 file_store.backend = local
 
 ; path to store the uploaded binaries
-file_store.storage_path = %(here)s/data/file_store
+file_store.storage_path = /var/opt/rhodecode_data/file_store
 
 ; Uncomment and set this path to control settings for archive download cache.
 ; Generated repo archives will be cached at this location
@@ -314,7 +314,7 @@ file_store.storage_path = %(here)s/data/file_store
 ; RhodeCode and vcsserver
 
 ; Default is $cache_dir/archive_cache if not set
-archive_cache.store_dir = %(here)s/data/archive_cache
+archive_cache.store_dir = /var/opt/rhodecode_data/tarballcache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
 archive_cache.cache_size_gb = 10
@@ -328,7 +328,7 @@ archive_cache.cache_shards = 10
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
-use_celery = false
+use_celery = true
 
 ; path to store schedule database
 #celerybeat-schedule.path =
@@ -354,7 +354,7 @@ celery.task_always_eager = false
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
-cache_dir = %(here)s/data
+cache_dir = /var/opt/rhodecode_data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
@@ -463,12 +463,12 @@ rc_cache.cache_repo.expiration_time = 2592000
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached
 ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
-beaker.session.type = file
-beaker.session.data_dir = %(here)s/data/sessions
+#beaker.session.type = file
+#beaker.session.data_dir = %(here)s/data/sessions
 
 ; Redis based sessions
-#beaker.session.type = ext:redis
-#beaker.session.url = redis://127.0.0.1:6379/2
+beaker.session.type = ext:redis
+beaker.session.url = redis://redis:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
@@ -480,7 +480,7 @@ beaker.session.data_dir = %(here)s/data/sessions
 
 beaker.session.key = rhodecode
 beaker.session.secret = develop-rc-uytcxaz
-beaker.session.lock_dir = %(here)s/data/sessions/lock
+beaker.session.lock_dir = /data_ramdisk/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
@@ -521,18 +521,18 @@ search.location = %(here)s/data/index
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
-channelstream.enabled = false
+channelstream.enabled = true
 
 ; server address for channelstream server on the backend
-channelstream.server = 127.0.0.1:9800
+channelstream.server = channelstream:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
-channelstream.secret = secret
-channelstream.history.location = %(here)s/channelstream_history
+channelstream.secret = ENV_GENERATED
+channelstream.history.location = /var/opt/rhodecode_data/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
@@ -578,7 +578,7 @@ sqlalchemy.db1.pool_recycle = 3600
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
-vcs.server = localhost:9900
+vcs.server = vcsserver:10010
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
@@ -610,6 +610,15 @@ vcs.backends = hg, git, svn
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
+; Cache flag to cache vcsserver remote calls locally
+; It uses cache_region `cache_repo`
+vcs.methods.cache = true
+
+; ####################################################
+; Subversion proxy support (mod_dav_svn)
+; Maps RhodeCode repo groups into SVN paths for Apache
+; ####################################################
+
 ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ; Set a numeric version for your current SVN e.g 1.8, or 1.12
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
@@ -621,23 +630,14 @@ vcs.svn.proxy.enabled = true
 ; host to connect to running SVN subsystem
 vcs.svn.proxy.host = http://svn:8090
 
-; Cache flag to cache vcsserver remote calls locally
-; It uses cache_region `cache_repo`
-vcs.methods.cache = true
-
-; ####################################################
-; Subversion proxy support (mod_dav_svn)
-; Maps RhodeCode repo groups into SVN paths for Apache
-; ####################################################
-
 ; Enable or disable the config file generation.
-svn.proxy.generate_config = false
+svn.proxy.generate_config = true
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
-svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
+svn.proxy.config_file_path = /etc/rhodecode/conf/svn/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
@@ -665,7 +665,7 @@ svn.proxy.location_root = /
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
-ssh.generate_authorized_keyfile = false
+ssh.generate_authorized_keyfile = true
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
@@ -673,12 +673,12 @@ ssh.generate_authorized_keyfile = false
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
-ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode
+ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
-; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
-ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper
+; e.g /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
@@ -689,73 +689,14 @@ ssh.enable_debug_logging = true
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
-ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
-ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
-ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
+ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg
+ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git
+ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
-
-; #################
-; APPENLIGHT CONFIG
-; #################
-
-; Appenlight is tailored to work with RhodeCode, see
-; http://appenlight.rhodecode.com for details how to obtain an account
-
-; Appenlight integration enabled
-#appenlight = false
-
-#appenlight.server_url = https://api.appenlight.com
-#appenlight.api_key = YOUR_API_KEY
-#appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
-
-; used for JS client
-#appenlight.api_public_key = YOUR_API_PUBLIC_KEY
-
-; TWEAK AMOUNT OF INFO SENT HERE
-
-; enables 404 error logging (default False)
-#appenlight.report_404 = false
-
-; time in seconds after request is considered being slow (default 1)
-#appenlight.slow_request_time = 1
-
-; record slow requests in application
-; (needs to be enabled for slow datastore recording and time tracking)
-#appenlight.slow_requests = true
-
-; enable hooking to application loggers
-#appenlight.logging = true
-
-; minimum log level for log capture
-#ppenlight.logging.level = WARNING
-
-; send logs only from erroneous/slow requests
-; (saves API quota for intensive logging)
-#appenlight.logging_on_error = false
-
-; list of additional keywords that should be grabbed from environ object
-; can be string with comma separated list of words in lowercase
-; (by default client will always send following info:
-; 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
-; start with HTTP* this list be extended with additional keywords here
-#appenlight.environ_keys_whitelist =
-
-; list of keywords that should be blanked from request object
-; can be string with comma separated list of words in lowercase
-; (by default client will always blank keys that contain following words
-; 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
-; this list be extended with additional keywords set here
-#appenlight.request_keys_blacklist =
-
-; list of namespaces that should be ignores when gathering log entries
-; can be string with comma separated list of namespaces
-; (by default the client ignores own entries: appenlight_client.client)
-#appenlight.log_namespace_blacklist =
-
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false
diff --git a/configs/production.ini b/configs/production.ini
index 75a8de1f..fe05826b 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -105,7 +105,7 @@ startup.import_repos = false
 app.base_url = http://rhodecode.local
 
 ; Host at which the Service API is running.
-app.service_api.host= http://rhodecode.local:10020
+app.service_api.host = http://rhodecode.local:10020
 
 ; Secret for Service API authentication.
 app.service_api.token =
@@ -256,7 +256,7 @@ file_store.enabled = true
 file_store.backend = local
 
 ; path to store the uploaded binaries
-file_store.storage_path = %(here)s/data/file_store
+file_store.storage_path = /var/opt/rhodecode_data/file_store
 
 ; Uncomment and set this path to control settings for archive download cache.
 ; Generated repo archives will be cached at this location
@@ -265,7 +265,7 @@ file_store.storage_path = %(here)s/data/file_store
 ; RhodeCode and vcsserver
 
 ; Default is $cache_dir/archive_cache if not set
-archive_cache.store_dir = %(here)s/data/archive_cache
+archive_cache.store_dir = /var/opt/rhodecode_data/tarballcache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
 archive_cache.cache_size_gb = 40
@@ -279,7 +279,7 @@ archive_cache.cache_shards = 4
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
-use_celery = false
+use_celery = true
 
 ; path to store schedule database
 #celerybeat-schedule.path =
@@ -305,7 +305,7 @@ celery.task_always_eager = false
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
-cache_dir = %(here)s/data
+cache_dir = /var/opt/rhodecode_data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
@@ -414,12 +414,12 @@ rc_cache.cache_repo.expiration_time = 2592000
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached
 ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
-beaker.session.type = file
-beaker.session.data_dir = %(here)s/data/sessions
+#beaker.session.type = file
+#beaker.session.data_dir = %(here)s/data/sessions
 
 ; Redis based sessions
-#beaker.session.type = ext:redis
-#beaker.session.url = redis://127.0.0.1:6379/2
+beaker.session.type = ext:redis
+beaker.session.url = redis://redis:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
@@ -431,7 +431,7 @@ beaker.session.data_dir = %(here)s/data/sessions
 
 beaker.session.key = rhodecode
 beaker.session.secret = production-rc-uytcxaz
-beaker.session.lock_dir = %(here)s/data/sessions/lock
+beaker.session.lock_dir = /data_ramdisk/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
@@ -472,18 +472,18 @@ search.location = %(here)s/data/index
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
-channelstream.enabled = false
+channelstream.enabled = true
 
 ; server address for channelstream server on the backend
-channelstream.server = 127.0.0.1:9800
+channelstream.server = channelstream:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
-channelstream.secret = secret
-channelstream.history.location = %(here)s/channelstream_history
+channelstream.secret = ENV_GENERATED
+channelstream.history.location = /var/opt/rhodecode_data/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
@@ -529,7 +529,7 @@ sqlalchemy.db1.pool_recycle = 3600
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
-vcs.server = localhost:9900
+vcs.server = vcsserver:10010
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
@@ -561,6 +561,15 @@ vcs.backends = hg, git, svn
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
+; Cache flag to cache vcsserver remote calls locally
+; It uses cache_region `cache_repo`
+vcs.methods.cache = true
+
+; ####################################################
+; Subversion proxy support (mod_dav_svn)
+; Maps RhodeCode repo groups into SVN paths for Apache
+; ####################################################
+
 ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ; Set a numeric version for your current SVN e.g 1.8, or 1.12
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
@@ -572,23 +581,14 @@ vcs.svn.proxy.enabled = true
 ; host to connect to running SVN subsystem
 vcs.svn.proxy.host = http://svn:8090
 
-; Cache flag to cache vcsserver remote calls locally
-; It uses cache_region `cache_repo`
-vcs.methods.cache = true
-
-; ####################################################
-; Subversion proxy support (mod_dav_svn)
-; Maps RhodeCode repo groups into SVN paths for Apache
-; ####################################################
-
 ; Enable or disable the config file generation.
-svn.proxy.generate_config = false
+svn.proxy.generate_config = true
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
-svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
+svn.proxy.config_file_path = /etc/rhodecode/conf/svn/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
@@ -616,7 +616,7 @@ svn.proxy.location_root = /
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
-ssh.generate_authorized_keyfile = false
+ssh.generate_authorized_keyfile = true
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
@@ -624,12 +624,12 @@ ssh.generate_authorized_keyfile = false
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
-ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode
+ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
-; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
-ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper
+; e.g /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
@@ -640,73 +640,14 @@ ssh.enable_debug_logging = false
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
-ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
-ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
-ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
+ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg
+ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git
+ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
-
-; #################
-; APPENLIGHT CONFIG
-; #################
-
-; Appenlight is tailored to work with RhodeCode, see
-; http://appenlight.rhodecode.com for details how to obtain an account
-
-; Appenlight integration enabled
-#appenlight = false
-
-#appenlight.server_url = https://api.appenlight.com
-#appenlight.api_key = YOUR_API_KEY
-#appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
-
-; used for JS client
-#appenlight.api_public_key = YOUR_API_PUBLIC_KEY
-
-; TWEAK AMOUNT OF INFO SENT HERE
-
-; enables 404 error logging (default False)
-#appenlight.report_404 = false
-
-; time in seconds after request is considered being slow (default 1)
-#appenlight.slow_request_time = 1
-
-; record slow requests in application
-; (needs to be enabled for slow datastore recording and time tracking)
-#appenlight.slow_requests = true
-
-; enable hooking to application loggers
-#appenlight.logging = true
-
-; minimum log level for log capture
-#ppenlight.logging.level = WARNING
-
-; send logs only from erroneous/slow requests
-; (saves API quota for intensive logging)
-#appenlight.logging_on_error = false
-
-; list of additional keywords that should be grabbed from environ object
-; can be string with comma separated list of words in lowercase
-; (by default client will always send following info:
-; 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
-; start with HTTP* this list be extended with additional keywords here
-#appenlight.environ_keys_whitelist =
-
-; list of keywords that should be blanked from request object
-; can be string with comma separated list of words in lowercase
-; (by default client will always blank keys that contain following words
-; 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
-; this list be extended with additional keywords set here
-#appenlight.request_keys_blacklist =
-
-; list of namespaces that should be ignores when gathering log entries
-; can be string with comma separated list of namespaces
-; (by default the client ignores own entries: appenlight_client.client)
-#appenlight.log_namespace_blacklist =
-
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false

From d00bf18b2dd16024818a601aa9ba421ae63a237b Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 4 Mar 2024 12:54:09 +0100
Subject: [PATCH 032/150] fix(tests): fixed svn tests

---
 rhodecode/apps/ssh_support/lib/backends/svn.py      | 2 +-
 rhodecode/apps/ssh_support/tests/test_server_svn.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/rhodecode/apps/ssh_support/lib/backends/svn.py b/rhodecode/apps/ssh_support/lib/backends/svn.py
index b9273fe8..ca27890f 100644
--- a/rhodecode/apps/ssh_support/lib/backends/svn.py
+++ b/rhodecode/apps/ssh_support/lib/backends/svn.py
@@ -212,7 +212,7 @@ class SubversionTunnelWrapper(object):
 
         first_response = self.get_first_client_response()
         if not first_response:
-            return self.fail("Repository name cannot be extracted")
+            return self.fail(b"Repository name cannot be extracted")
 
         url_parts = urllib.parse.urlparse(first_response['url'])
 
diff --git a/rhodecode/apps/ssh_support/tests/test_server_svn.py b/rhodecode/apps/ssh_support/tests/test_server_svn.py
index 52bd6ad1..0bbc68a2 100644
--- a/rhodecode/apps/ssh_support/tests/test_server_svn.py
+++ b/rhodecode/apps/ssh_support/tests/test_server_svn.py
@@ -62,6 +62,7 @@ def svn_server(app):
 
 
 class TestSubversionServer(object):
+
     def test_command(self, svn_server):
         server = svn_server.create()
         expected_command = [

From ca042238f99d4cc46773f248078817353b5ecfaf Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 5 Mar 2024 15:36:11 +0100
Subject: [PATCH 033/150] chore(sync-up): synced libs/settings maker from
 vcsserver

---
 rhodecode/config/settings_maker.py | 22 ++++++++++++++++++++--
 rhodecode/lib/str_utils.py         | 14 ++++++++++++++
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/rhodecode/config/settings_maker.py b/rhodecode/config/settings_maker.py
index 873b1e7c..a82308dd 100644
--- a/rhodecode/config/settings_maker.py
+++ b/rhodecode/config/settings_maker.py
@@ -23,6 +23,7 @@ import functools
 import logging
 import tempfile
 import logging.config
+
 from rhodecode.lib.type_utils import str2bool, aslist
 
 log = logging.getLogger(__name__)
@@ -34,13 +35,16 @@ set_keys = {
 }
 
 
-class SettingsMaker(object):
+class SettingsMaker:
 
     def __init__(self, app_settings):
         self.settings = app_settings
 
     @classmethod
     def _bool_func(cls, input_val):
+        if isinstance(input_val, bytes):
+            # decode to str
+            input_val = input_val.decode('utf8')
         return str2bool(input_val)
 
     @classmethod
@@ -61,12 +65,25 @@ class SettingsMaker(object):
             input_val = input_val.lower()
         return input_val
 
+    @classmethod
+    def _string_no_quote_func(cls, input_val, lower=True):
+        """
+        Special case string function that detects if value is set to empty quote string
+        e.g.
+
+        core.binar_dir = ""
+        """
+
+        input_val = cls._string_func(input_val, lower=lower)
+        if input_val in ['""', "''"]:
+            return ''
+
     @classmethod
     def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
 
         # ensure we have our dir created
         if not os.path.isdir(input_val) and ensure_dir:
-            os.makedirs(input_val, mode=mode)
+            os.makedirs(input_val, mode=mode, exist_ok=True)
 
         if not os.path.isdir(input_val):
             raise Exception(f'Dir at {input_val} does not exist')
@@ -150,6 +167,7 @@ class SettingsMaker(object):
             'list:newline': functools.partial(self._list_func, sep='/n'),
             'list:spacesep': functools.partial(self._list_func, sep=' '),
             'string': functools.partial(self._string_func, lower=lower),
+            'string:noquote': functools.partial(self._string_no_quote_func, lower=lower),
             'dir': self._dir_func,
             'dir:ensured': functools.partial(self._dir_func, ensure_dir=True),
             'file': self._file_path_func,
diff --git a/rhodecode/lib/str_utils.py b/rhodecode/lib/str_utils.py
index a20e427e..78633d2d 100644
--- a/rhodecode/lib/str_utils.py
+++ b/rhodecode/lib/str_utils.py
@@ -167,3 +167,17 @@ def convert_special_chars(str_) -> str:
     value = safe_str(str_)
     converted_value = unidecode(value)
     return converted_value
+
+
+def splitnewlines(text: bytes):
+    """
+    like splitlines, but only split on newlines.
+    """
+
+    lines = [_l + b'\n' for _l in text.split(b'\n')]
+    if lines:
+        if lines[-1] == b'\n':
+            lines.pop()
+        else:
+            lines[-1] = lines[-1][:-1]
+    return lines

From 7f994fc993f9fd49fdb1f060aa28f0a43f7c8a55 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 11 Mar 2024 09:56:10 +0100
Subject: [PATCH 034/150] tests(core): made the test flag setting more
 flexible.

We no longer use .ini setting, but relly on env variables, that is now set
via pytest env plugin
---
 pytest.ini                           | 4 ++++
 requirements_test.txt                | 6 ++++++
 rhodecode/__init__.py                | 2 +-
 rhodecode/config/config_maker.py     | 1 -
 rhodecode/config/environment.py      | 3 +--
 rhodecode/tests/database/conftest.py | 3 +--
 6 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index a5d7bff1..e1bd2486 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,3 +21,7 @@ markers =
   skip_backends: Mark tests as skipped for given backends.
   backends: Mark backends
   dbs: database markers for running tests for given DB
+
+env =
+    RC_TEST=1
+    RUN_ENV=test
diff --git a/requirements_test.txt b/requirements_test.txt
index 72f7f74a..6b09e542 100644
--- a/requirements_test.txt
+++ b/requirements_test.txt
@@ -11,6 +11,12 @@ pytest-cov==4.0.0
     iniconfig==2.0.0
     packaging==23.1
     pluggy==1.0.0
+pytest-env==1.1.3
+  pytest==7.3.1
+    attrs==22.2.0
+    iniconfig==2.0.0
+    packaging==23.1
+    pluggy==1.0.0
 pytest-rerunfailures==12.0
 pytest-profiling==1.7.0
   gprof2dot==2022.7.29
diff --git a/rhodecode/__init__.py b/rhodecode/__init__.py
index 541a3418..2d6446aa 100644
--- a/rhodecode/__init__.py
+++ b/rhodecode/__init__.py
@@ -87,5 +87,5 @@ __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'
 __url__ = 'https://code.rhodecode.com'
 
-is_test = False
+is_test = os.getenv('RC_TEST')
 disable_error_handler = False
diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py
index d27897fe..9587c0b8 100644
--- a/rhodecode/config/config_maker.py
+++ b/rhodecode/config/config_maker.py
@@ -92,7 +92,6 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
 
     # Sanitize generic settings.
     settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
-    settings_maker.make_setting('is_test', False, parser='bool')
     settings_maker.make_setting('gzip_responses', False, parser='bool')
     settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
 
diff --git a/rhodecode/config/environment.py b/rhodecode/config/environment.py
index 376cad85..20ee0f38 100644
--- a/rhodecode/config/environment.py
+++ b/rhodecode/config/environment.py
@@ -46,8 +46,7 @@ def load_pyramid_environment(global_config, settings):
     # If this is a test run we prepare the test environment like
     # creating a test database, test search index and test repositories.
     # This has to be done before the database connection is initialized.
-    if settings['is_test']:
-        rhodecode.is_test = True
+    if rhodecode.is_test:
         rhodecode.disable_error_handler = True
         from rhodecode import authentication
         authentication.plugin_default_auth_ttl = 0
diff --git a/rhodecode/tests/database/conftest.py b/rhodecode/tests/database/conftest.py
index 26e67da7..7799f23c 100644
--- a/rhodecode/tests/database/conftest.py
+++ b/rhodecode/tests/database/conftest.py
@@ -86,8 +86,7 @@ class DBBackend(object):
     _store = os.path.dirname(os.path.abspath(__file__))
     _type = None
     _base_ini_config = [{'app:main': {'vcs.start_server': 'false',
-                                      'startup.import_repos': 'false',
-                                      'is_test': 'False'}}]
+                                      'startup.import_repos': 'false'}}]
     _db_url = [{'app:main': {'sqlalchemy.db1.url': ''}}]
     _base_db_name = 'rhodecode_test_db_backend'
 

From d7fd83acb17c21674d573f64eca19c7fd8df47b4 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 11 Mar 2024 10:29:11 +0100
Subject: [PATCH 035/150] chore(deps): bumped pytest related tests

---
 requirements.txt      |  7 ++---
 requirements_test.txt | 66 ++++++++++++++++++++-----------------------
 2 files changed, 34 insertions(+), 39 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 55b03477..a47a61dd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -91,7 +91,7 @@ dogpile.cache==1.3.2
 formencode==2.1.0
   six==1.16.0
 gunicorn==21.2.0
-  packaging==23.1
+  packaging==24.0
 gevent==24.2.1
   greenlet==3.0.3
   zope.event==5.0.0
@@ -119,8 +119,8 @@ markdown==3.4.3
 msgpack==1.0.7
 mysqlclient==2.1.1
 nbconvert==7.7.3
-  beautifulsoup4==4.11.2
-    soupsieve==2.4
+  beautifulsoup4==4.12.3
+    soupsieve==2.5
   bleach==6.1.0
     six==1.16.0
     webencodings==0.5.1
@@ -165,7 +165,6 @@ nbconvert==7.7.3
       platformdirs==3.10.0
       traitlets==5.9.0
     traitlets==5.9.0
-  packaging==23.1
   pandocfilters==1.5.0
   pygments==2.15.1
   tinycss2==1.2.1
diff --git a/requirements_test.txt b/requirements_test.txt
index 6b09e542..dafed0fa 100644
--- a/requirements_test.txt
+++ b/requirements_test.txt
@@ -1,49 +1,45 @@
 # test related requirements
-
-cov-core==1.15.0
-  coverage==7.2.3
-mock==5.0.2
-py==1.11.0
-pytest-cov==4.0.0
-  coverage==7.2.3
-  pytest==7.3.1
-    attrs==22.2.0
+mock==5.1.0
+pytest-cov==4.1.0
+  coverage==7.4.3
+  pytest==8.1.1
     iniconfig==2.0.0
-    packaging==23.1
-    pluggy==1.0.0
+    packaging==24.0
+    pluggy==1.4.0
 pytest-env==1.1.3
-  pytest==7.3.1
-    attrs==22.2.0
+  pytest==8.1.1
     iniconfig==2.0.0
-    packaging==23.1
-    pluggy==1.0.0
-pytest-rerunfailures==12.0
+    packaging==24.0
+    pluggy==1.4.0
 pytest-profiling==1.7.0
   gprof2dot==2022.7.29
-  pytest==7.3.1
-    attrs==22.2.0
+  pytest==8.1.1
     iniconfig==2.0.0
-    packaging==23.1
-    pluggy==1.0.0
+    packaging==24.0
+    pluggy==1.4.0
   six==1.16.0
-pytest-runner==6.0.0
-pytest-sugar==0.9.7
-  packaging==23.1
-  pytest==7.3.1
-    attrs==22.2.0
+pytest-rerunfailures==13.0
+  packaging==24.0
+  pytest==8.1.1
     iniconfig==2.0.0
-    packaging==23.1
-    pluggy==1.0.0
-  termcolor==2.3.0
-pytest-timeout==2.1.0
-  pytest==7.3.1
-    attrs==22.2.0
+    packaging==24.0
+    pluggy==1.4.0
+pytest-runner==6.0.1
+pytest-sugar==1.0.0
+  packaging==24.0
+  pytest==8.1.1
     iniconfig==2.0.0
-    packaging==23.1
-    pluggy==1.0.0
+    packaging==24.0
+    pluggy==1.4.0
+  termcolor==2.4.0
+pytest-timeout==2.3.1
+  pytest==8.1.1
+    iniconfig==2.0.0
+    packaging==24.0
+    pluggy==1.4.0
 webtest==3.0.0
-  beautifulsoup4==4.11.2
-    soupsieve==2.4
+  beautifulsoup4==4.12.3
+    soupsieve==2.5
   waitress==3.0.0
   webob==1.8.7
 

From e0b1f9c055516c143118d48605d01c51002c08ef Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 11 Mar 2024 14:55:48 +0100
Subject: [PATCH 036/150] fix(events): fixed celery based events where it was
 missing request object

---
 rhodecode/events/base.py         | 21 +++++++++++++++------
 rhodecode/events/repo.py         |  4 ++--
 rhodecode/lib/auth.py            |  2 +-
 rhodecode/lib/celerylib/tasks.py |  1 +
 rhodecode/lib/hooks_utils.py     |  2 +-
 rhodecode/lib/pyramid_utils.py   |  9 +++++++++
 rhodecode/model/repo.py          |  2 +-
 7 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/rhodecode/events/base.py b/rhodecode/events/base.py
index ec19fdbb..314866b6 100644
--- a/rhodecode/events/base.py
+++ b/rhodecode/events/base.py
@@ -15,13 +15,14 @@
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
+
 import logging
 import datetime
-import typing
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
-from pyramid.threadlocal import get_current_request
 
+from rhodecode.lib.pyramid_utils import get_current_request
+from rhodecode.lib.auth import AuthUser
 from rhodecode.lib.utils2 import AttributeDict
 
 
@@ -41,8 +42,9 @@ class RhodecodeEvent(object):
     name = "RhodeCodeEvent"
     no_url_set = '<no server_url available>'
 
-    def __init__(self, request=None):
+    def __init__(self, request=None, actor=None):
         self._request = request
+        self._actor = actor
         self.utc_timestamp = datetime.datetime.utcnow()
 
     def __repr__(self):
@@ -72,16 +74,22 @@ class RhodecodeEvent(object):
 
     @property
     def actor(self):
+        # if an explicit actor is specified, use this
+        if self._actor:
+            return self._actor
+
         auth_user = self.auth_user
-        if auth_user:
+        log.debug('Got integration actor: %s', auth_user)
+        if isinstance(auth_user, AuthUser):
             instance = auth_user.get_instance()
+            # we can't find this DB user...
             if not instance:
                 return AttributeDict(dict(
                     username=auth_user.username,
                     user_id=auth_user.user_id,
                 ))
-            return instance
-
+        elif auth_user:
+            return auth_user
         return SYSTEM_USER
 
     @property
@@ -129,3 +137,4 @@ class FtsBuild(RhodecodeEvent):
     """
     name = 'fts-build'
     display_name = 'Start FTS Build'
+
diff --git a/rhodecode/events/repo.py b/rhodecode/events/repo.py
index 8d859439..3794dd45 100644
--- a/rhodecode/events/repo.py
+++ b/rhodecode/events/repo.py
@@ -156,11 +156,11 @@ class RepoEvent(RhodeCodeIntegrationEvent):
     Base class for events acting on a repository.
     """
 
-    def __init__(self, repo):
+    def __init__(self, repo, actor=None):
         """
         :param repo: a :class:`Repository` instance
         """
-        super().__init__()
+        super().__init__(actor=actor)
         self.repo = repo
 
     def as_dict(self):
diff --git a/rhodecode/lib/auth.py b/rhodecode/lib/auth.py
index b694db41..34ec7b40 100644
--- a/rhodecode/lib/auth.py
+++ b/rhodecode/lib/auth.py
@@ -1688,7 +1688,7 @@ def get_csrf_token(session, force_new=False, save_if_missing=True):
 
 
 def get_request(perm_class_instance):
-    from pyramid.threadlocal import get_current_request
+    from rhodecode.lib.pyramid_utils import get_current_request
     pyramid_request = get_current_request()
     return pyramid_request
 
diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py
index 1fe90dbf..e3117e80 100644
--- a/rhodecode/lib/celerylib/tasks.py
+++ b/rhodecode/lib/celerylib/tasks.py
@@ -193,6 +193,7 @@ def create_repo(form_data, cur_user):
             enable_downloads=enable_downloads,
             state=state
         )
+
         Session().commit()
 
         # now create this repo on Filesystem
diff --git a/rhodecode/lib/hooks_utils.py b/rhodecode/lib/hooks_utils.py
index 42a45a95..9aeaa5c6 100644
--- a/rhodecode/lib/hooks_utils.py
+++ b/rhodecode/lib/hooks_utils.py
@@ -17,7 +17,6 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import webob
-from pyramid.threadlocal import get_current_request
 
 from rhodecode import events
 from rhodecode.lib import hooks_base
@@ -33,6 +32,7 @@ def _supports_repo_type(repo_type):
 def _get_vcs_operation_context(username, repo_name, repo_type, action):
     # NOTE(dan): import loop
     from rhodecode.lib.base import vcs_operation_context
+    from rhodecode.lib.pyramid_utils import get_current_request
 
     check_locking = action in ('pull', 'push')
 
diff --git a/rhodecode/lib/pyramid_utils.py b/rhodecode/lib/pyramid_utils.py
index 51aa1816..a69d03ec 100644
--- a/rhodecode/lib/pyramid_utils.py
+++ b/rhodecode/lib/pyramid_utils.py
@@ -23,6 +23,7 @@ import configparser
 
 from rhodecode.lib.config_utils import get_config
 from pyramid.paster import bootstrap as pyramid_bootstrap, setup_logging  # pragma: no cover
+from pyramid.threadlocal import get_current_request as pyramid_current_request
 
 
 def bootstrap(config_uri, options=None, env=None):
@@ -46,3 +47,11 @@ def bootstrap(config_uri, options=None, env=None):
                                   'ip_addr': '127.0.0.1'})
     return pyramid_bootstrap(config_uri, request=request, options=options)
 
+
+def get_current_request():
+    pyramid_req = pyramid_current_request()
+    if not pyramid_req:
+        # maybe we're in celery context and need to get the PYRAMID_REQUEST
+        from rhodecode.lib.celerylib.loader import celery_app
+        pyramid_req = celery_app.conf['PYRAMID_REQUEST']
+    return pyramid_req
diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py
index e784833c..0d7bd6c1 100644
--- a/rhodecode/model/repo.py
+++ b/rhodecode/model/repo.py
@@ -608,7 +608,7 @@ class RepoModel(BaseModel):
             # we need to flush here, in order to check if database won't
             # throw any exceptions, create filesystem dirs at the very end
             self.sa.flush()
-            events.trigger(events.RepoCreateEvent(new_repo))
+            events.trigger(events.RepoCreateEvent(new_repo, actor=owner))
             return new_repo
 
         except Exception:

From 684e1aaeeb77f8dc44025d68c70aec5d62e7c85b Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 11 Mar 2024 15:08:11 +0100
Subject: [PATCH 037/150] fix(imports): fixed circular import problem

---
 rhodecode/events/base.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rhodecode/events/base.py b/rhodecode/events/base.py
index 314866b6..9496ed79 100644
--- a/rhodecode/events/base.py
+++ b/rhodecode/events/base.py
@@ -22,7 +22,6 @@ import datetime
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.lib.pyramid_utils import get_current_request
-from rhodecode.lib.auth import AuthUser
 from rhodecode.lib.utils2 import AttributeDict
 
 
@@ -74,6 +73,8 @@ class RhodecodeEvent(object):
 
     @property
     def actor(self):
+        from rhodecode.lib.auth import AuthUser
+
         # if an explicit actor is specified, use this
         if self._actor:
             return self._actor

From 0e4d7ebf22be6800a242b052fe6688b1033f7289 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 12 Mar 2024 08:02:48 +0100
Subject: [PATCH 038/150] chore(configs): remove is_test flag which is
 deprecated

---
 rhodecode/tests/rhodecode.ini | 1 -
 1 file changed, 1 deletion(-)

diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini
index f01777ef..931ec9c4 100644
--- a/rhodecode/tests/rhodecode.ini
+++ b/rhodecode/tests/rhodecode.ini
@@ -66,7 +66,6 @@ prefix = /
 ;can be overridden by
 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
 
-is_test = True
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above

From b34d42bb36b0a77687e4636e6858f187267ddc9f Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 12 Mar 2024 09:52:52 +0100
Subject: [PATCH 039/150] chore(deps): bumped redis==5.0.3

---
 requirements.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index a47a61dd..b0674ec1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -256,7 +256,8 @@ python3-saml==1.15.0
   xmlsec==1.3.13
     lxml==4.9.3
 pyyaml==6.0.1
-redis==5.0.1
+redis==5.0.3
+  async-timeout==4.0.3
 regex==2022.10.31
 routes==2.5.1
   repoze.lru==0.7

From eef381dc5377ee80ab119368546000a3182245e5 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 12 Mar 2024 09:53:09 +0100
Subject: [PATCH 040/150] chore(deps): bumped msgpack==1.0.8

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b0674ec1..2ec36024 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -116,7 +116,7 @@ ipython==8.14.0
     pure-eval==0.2.2
   traitlets==5.9.0
 markdown==3.4.3
-msgpack==1.0.7
+msgpack==1.0.8
 mysqlclient==2.1.1
 nbconvert==7.7.3
   beautifulsoup4==4.12.3

From a9c2bf237b9f7daafa821db310ca18fc7e005ca3 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 12 Mar 2024 09:53:25 +0100
Subject: [PATCH 041/150] chore(deps): bumped sqlalchemy==1.4.52

---
 requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 2ec36024..21393c0c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@
 alembic==1.12.1
   mako==1.2.4
     markupsafe==2.1.2
-  sqlalchemy==1.4.51
+  sqlalchemy==1.4.52
     greenlet==3.0.3
   typing_extensions==4.9.0
 async-timeout==4.0.3
@@ -269,7 +269,7 @@ sshpubkeys==3.3.1
       pycparser==2.21
   ecdsa==0.18.0
     six==1.16.0
-sqlalchemy==1.4.51
+sqlalchemy==1.4.52
   greenlet==3.0.3
   typing_extensions==4.9.0
 supervisor==4.2.5

From f8bdbc4c723d52112891de77aa04202fc550b65c Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 12 Mar 2024 09:53:36 +0100
Subject: [PATCH 042/150] chore(deps): bumped alembic==1.13.1

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 21393c0c..a5820049 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 # deps, generated via pipdeptree --exclude setuptools,wheel,pipdeptree,pip -f | tr '[:upper:]' '[:lower:]'
 
-alembic==1.12.1
+alembic==1.13.1
   mako==1.2.4
     markupsafe==2.1.2
   sqlalchemy==1.4.52

From 15fd720c8945b549bfa69be969a5715bd4a96604 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 12 Mar 2024 10:08:17 +0100
Subject: [PATCH 043/150] fix(tests): fixed db migration tests after
 introduction of RC_TEST flag

---
 rhodecode/__init__.py                | 2 +-
 rhodecode/tests/database/conftest.py | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/rhodecode/__init__.py b/rhodecode/__init__.py
index 2d6446aa..82a7f06a 100644
--- a/rhodecode/__init__.py
+++ b/rhodecode/__init__.py
@@ -87,5 +87,5 @@ __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'
 __url__ = 'https://code.rhodecode.com'
 
-is_test = os.getenv('RC_TEST')
+is_test = os.getenv('RC_TEST', '0') == '1'
 disable_error_handler = False
diff --git a/rhodecode/tests/database/conftest.py b/rhodecode/tests/database/conftest.py
index 7799f23c..0bd2c0b3 100644
--- a/rhodecode/tests/database/conftest.py
+++ b/rhodecode/tests/database/conftest.py
@@ -89,6 +89,7 @@ class DBBackend(object):
                                       'startup.import_repos': 'false'}}]
     _db_url = [{'app:main': {'sqlalchemy.db1.url': ''}}]
     _base_db_name = 'rhodecode_test_db_backend'
+    std_env = {'RC_TEST': '0'}
 
     def __init__(
             self, config_file, db_name=None, basetemp=None,
@@ -134,13 +135,15 @@ class DBBackend(object):
         """
 
         command = cmd + ' ' + ' '.join(args)
-        sys.stdout.write(command)
+        sys.stdout.write(f'CMD: {command}')
 
         # Tell Python to use UTF-8 encoding out stdout
         _env = os.environ.copy()
         _env['PYTHONIOENCODING'] = 'UTF-8'
+        _env.update(self.std_env)
         if env:
             _env.update(env)
+
         self.p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, env=_env)
         self.stdout, self.stderr = self.p.communicate()
         stdout_str = safe_str(self.stdout)

From 14e425bce62c39e182eea9a0703c08594c4537d9 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 13 Mar 2024 19:46:19 +0100
Subject: [PATCH 044/150] fix(user-models): added extra protection against
 model username changes that would create duplicates

---
 rhodecode/apps/my_account/views/my_account.py | 1 +
 rhodecode/lib/exceptions.py                   | 4 ++++
 rhodecode/model/user.py                       | 6 +++++-
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/rhodecode/apps/my_account/views/my_account.py b/rhodecode/apps/my_account/views/my_account.py
index 94865969..6b7f184f 100644
--- a/rhodecode/apps/my_account/views/my_account.py
+++ b/rhodecode/apps/my_account/views/my_account.py
@@ -136,6 +136,7 @@ class MyAccountView(BaseAppView, DataGridAppView):
         except forms.ValidationFailure as e:
             c.form = e
             return self._get_template_context(c)
+
         except Exception:
             log.exception("Exception updating user")
             h.flash(_('Error occurred during update of user'),
diff --git a/rhodecode/lib/exceptions.py b/rhodecode/lib/exceptions.py
index 5556b242..0b4a3ef2 100644
--- a/rhodecode/lib/exceptions.py
+++ b/rhodecode/lib/exceptions.py
@@ -144,6 +144,10 @@ class NotAllowedToCreateUserError(Exception):
     pass
 
 
+class DuplicateUpdateUserError(Exception):
+    pass
+
+
 class RepositoryCreationError(Exception):
     pass
 
diff --git a/rhodecode/model/user.py b/rhodecode/model/user.py
index 4ddb2292..18e33254 100644
--- a/rhodecode/model/user.py
+++ b/rhodecode/model/user.py
@@ -37,7 +37,7 @@ from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.exceptions import (
     DefaultUserException, UserOwnsReposException, UserOwnsRepoGroupsException,
     UserOwnsUserGroupsException, NotAllowedToCreateUserError,
-    UserOwnsPullRequestsException, UserOwnsArtifactsException)
+    UserOwnsPullRequestsException, UserOwnsArtifactsException, DuplicateUpdateUserError)
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
@@ -308,6 +308,10 @@ class UserModel(BaseModel):
             log.debug('Checking for existing account in RhodeCode '
                       'database with user_id `%s` ', updating_user_id)
             user = User.get(updating_user_id)
+            # now also validate if USERNAME belongs to potentially other user
+            maybe_other_user = User.get_by_username(username, case_insensitive=True)
+            if maybe_other_user and maybe_other_user.user_id != updating_user_id:
+                raise DuplicateUpdateUserError(f'different user exists with the {username} username')
         else:
             log.debug('Checking for existing account in RhodeCode '
                       'database with username `%s` ', username)

From d684cf1f5d9ef505abc7cc139f9d9bf44d1b3d76 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 13 Mar 2024 19:46:50 +0100
Subject: [PATCH 045/150] feat(forms): user profile form would detect nicely
 duplicates and show a form error.

---
 rhodecode/model/validation_schema/schemas/user_schema.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/rhodecode/model/validation_schema/schemas/user_schema.py b/rhodecode/model/validation_schema/schemas/user_schema.py
index 8ec241f1..0431e414 100644
--- a/rhodecode/model/validation_schema/schemas/user_schema.py
+++ b/rhodecode/model/validation_schema/schemas/user_schema.py
@@ -64,6 +64,7 @@ class ChangePasswordSchema(colander.Schema):
 
 @colander.deferred
 def deferred_username_validator(node, kw):
+    old_username = kw.get('username')
 
     def name_validator(node, value):
         msg = _(
@@ -74,6 +75,11 @@ def deferred_username_validator(node, kw):
         if not re.match(r'^[\w]{1}[\w\-\.]{0,254}$', value):
             raise colander.Invalid(node, msg)
 
+        if value != old_username:
+            existing_user = User.get_by_username(value, case_insensitive=True)
+            if existing_user:
+                raise colander.Invalid(node, 'Username is already taken')
+
     return name_validator
 
 

From 0f1fbf4ccc79eb699752df27001c9ec605664962 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Fri, 29 Mar 2024 10:43:02 +0100
Subject: [PATCH 046/150] docs: fixed config files path for editing settings...

---
 docs/admin/lab-settings.rst                               | 2 +-
 docs/admin/sec-x-frame.rst                                | 2 +-
 docs/admin/sec-your-server.rst                            | 2 +-
 docs/admin/system-overview.rst                            | 2 +-
 docs/admin/system_admin/admin-tricks.rst                  | 2 +-
 docs/admin/system_admin/config-files-overview.rst         | 2 +-
 docs/admin/system_admin/enable-debug.rst                  | 4 ++--
 docs/admin/system_admin/svn-http.rst                      | 2 +-
 docs/admin/system_admin/tuning/tuning-change-encoding.rst | 2 +-
 docs/admin/system_admin/tuning/tuning-hg-auth-loop.rst    | 2 +-
 .../tuning/tuning-scale-horizontally-cluster.rst          | 8 ++++----
 .../tuning/tuning-user-sessions-performance.rst           | 4 ++--
 docs/admin/system_admin/vcs-server.rst                    | 2 +-
 docs/api/api.rst                                          | 2 +-
 docs/auth/ssh-connection.rst                              | 4 ++--
 docs/install/setup-email.rst                              | 2 +-
 docs/tutorials/multi-instance-setup.rst                   | 2 +-
 17 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/docs/admin/lab-settings.rst b/docs/admin/lab-settings.rst
index 4f7e9010..e1eb14ac 100644
--- a/docs/admin/lab-settings.rst
+++ b/docs/admin/lab-settings.rst
@@ -8,7 +8,7 @@ level of support to optimize for production scenarios. To enable lab settings,
 use the following instructions:
 
 1. Open the |RCE| configuration file,
-   :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+   :file:`config/_shared/rhodecode.ini`
 
 2. Add the following configuration option in the ``[app:main]`` section.
 
diff --git a/docs/admin/sec-x-frame.rst b/docs/admin/sec-x-frame.rst
index e894e5c5..da6829e4 100644
--- a/docs/admin/sec-x-frame.rst
+++ b/docs/admin/sec-x-frame.rst
@@ -42,7 +42,7 @@ information see the :ref:`apache-ws-ref` section.
 
 |RCE| can also be configured to force strict *https* connections and Strict
 Transport Security. To set this, configure the following options to ``true``
-in the :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+in the :file:`config/_shared/rhodecode.ini` file.
 
 .. code-block:: ini
 
diff --git a/docs/admin/sec-your-server.rst b/docs/admin/sec-your-server.rst
index b8808c38..95a91d26 100644
--- a/docs/admin/sec-your-server.rst
+++ b/docs/admin/sec-your-server.rst
@@ -83,7 +83,7 @@ see the `OpenSSL PKI tutorial`_ site, or this `Cloudflare PKI post`_.
 
 If the network you are running is SSL/TLS encrypted, you can configure |RCE|
 to always use secure connections using the ``force_https`` and ``use_htsts``
-options in the :file:`/home/user/.rccontrol/instance-id/rhodecode.ini` file.
+options in the :file:`config/_shared/rhodecode.ini` file.
 For more details, see the :ref:`x-frame` section.
 
 FireWalls and Ports
diff --git a/docs/admin/system-overview.rst b/docs/admin/system-overview.rst
index d79310ef..b4f670f0 100644
--- a/docs/admin/system-overview.rst
+++ b/docs/admin/system-overview.rst
@@ -78,7 +78,7 @@ For example:
 Configuration Files
 -------------------
 
-* :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+* :file:`config/_shared/rhodecode.ini`
 * :file:`/home/{user}/.rccontrol/{instance-id}/search_mapping.ini`
 * :file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.ini`
 * :file:`/home/{user}/.rccontrol/supervisor/supervisord.ini`
diff --git a/docs/admin/system_admin/admin-tricks.rst b/docs/admin/system_admin/admin-tricks.rst
index 7c6bb986..bae08bfd 100644
--- a/docs/admin/system_admin/admin-tricks.rst
+++ b/docs/admin/system_admin/admin-tricks.rst
@@ -188,7 +188,7 @@ Changing Default Language
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 To change the default language of a |RCE| instance, change the language code
-in the :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file. To
+in the :file:`config/_shared/rhodecode.ini` file. To
 do this, use the following steps.
 
 1. Open the :file:`rhodecode.ini` file and set the required language code.
diff --git a/docs/admin/system_admin/config-files-overview.rst b/docs/admin/system_admin/config-files-overview.rst
index 3f9a20b7..b151c07a 100644
--- a/docs/admin/system_admin/config-files-overview.rst
+++ b/docs/admin/system_admin/config-files-overview.rst
@@ -11,7 +11,7 @@ sections.
 
     \- **rhodecode.ini**
         Default location:
-        :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+        :file:`config/_shared/rhodecode.ini`
 
         This is the main |RCE| configuration file and controls much of its
         default behaviour. It is also used to configure certain customer
diff --git a/docs/admin/system_admin/enable-debug.rst b/docs/admin/system_admin/enable-debug.rst
index f688b72d..07779d99 100644
--- a/docs/admin/system_admin/enable-debug.rst
+++ b/docs/admin/system_admin/enable-debug.rst
@@ -14,7 +14,7 @@ track particular user logs only, and exclude other concurrent ones
 simply grep by `req_id` uuid which you'll have to find for the individual request.
 
 To enable debug mode on a |RCE| instance you need to set the debug property
-in the :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file. To
+in the :file:`config/_shared/rhodecode.ini` file. To
 do this, use the following steps
 
 1. Open the file and set the ``debug`` line to ``true``
@@ -38,7 +38,7 @@ Debug and Logging Configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Further debugging and logging settings can also be set in the
-:file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+:file:`config/_shared/rhodecode.ini` file.
 
 In the logging section, the various packages that run with |RCE| can have
 different debug levels set. If you want to increase the logging level change
diff --git a/docs/admin/system_admin/svn-http.rst b/docs/admin/system_admin/svn-http.rst
index 1fbed908..cb9041bd 100644
--- a/docs/admin/system_admin/svn-http.rst
+++ b/docs/admin/system_admin/svn-http.rst
@@ -134,7 +134,7 @@ permission issues could occur. To do this edit the ``/etc/apache2/envvars``
    :guilabel:`Subversion HTTP Server URL`.
 
 3. Open the |RCE| configuration file,
-   :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+   :file:`config/_shared/rhodecode.ini`
 
 4. Add the following configuration option in the ``[app:main]``
    section if you don't have it yet.
diff --git a/docs/admin/system_admin/tuning/tuning-change-encoding.rst b/docs/admin/system_admin/tuning/tuning-change-encoding.rst
index f9804678..1409bc1a 100644
--- a/docs/admin/system_admin/tuning/tuning-change-encoding.rst
+++ b/docs/admin/system_admin/tuning/tuning-change-encoding.rst
@@ -4,7 +4,7 @@ Change Default Encoding
 -----------------------
 
 |RCE| uses ``utf8`` encoding by default. You can change the default encoding
-in the :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file. To
+in the :file:`config/_shared/rhodecode.ini` file. To
 change the default encoding used by |RCE|, set a new value for the
 ``default_encoding``.
 
diff --git a/docs/admin/system_admin/tuning/tuning-hg-auth-loop.rst b/docs/admin/system_admin/tuning/tuning-hg-auth-loop.rst
index bfd61d1c..be85331b 100644
--- a/docs/admin/system_admin/tuning/tuning-hg-auth-loop.rst
+++ b/docs/admin/system_admin/tuning/tuning-hg-auth-loop.rst
@@ -7,7 +7,7 @@ When using external authentication tools such as LDAP with |hg|, a
 password retry loop in |hg| can result in users being locked out due to too
 many failed password attempts. To prevent this from happening, add the
 following setting to your
-:file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file, in the
+:file:`config/_shared/rhodecode.ini` file, in the
 ``[app:main]`` section.
 
 
diff --git a/docs/admin/system_admin/tuning/tuning-scale-horizontally-cluster.rst b/docs/admin/system_admin/tuning/tuning-scale-horizontally-cluster.rst
index bf967b9f..4f6d55b4 100644
--- a/docs/admin/system_admin/tuning/tuning-scale-horizontally-cluster.rst
+++ b/docs/admin/system_admin/tuning/tuning-scale-horizontally-cluster.rst
@@ -100,7 +100,7 @@ Each one should already connect to shared database during installation.
 
 1) Assuming our final url will be http://rc-node-1, Configure `instances_id`, `app.base_url`
 
-a) On **rc-node-2** find the following settings and edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+a) On **rc-node-2** find the following settings and edit :file:`config/_shared/rhodecode.ini`
 
 .. code-block:: ini
 
@@ -109,7 +109,7 @@ a) On **rc-node-2** find the following settings and edit :file:`/home/{user}/.rc
     app.base_url = http://rc-node-1
 
 
-b) On **rc-node-3** find the following settings and edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+b) On **rc-node-3** find the following settings and edit :file:`config/_shared/rhodecode.ini`
 
 .. code-block:: ini
 
@@ -121,7 +121,7 @@ b) On **rc-node-3** find the following settings and edit :file:`/home/{user}/.rc
 
 2) Configure `User Session` to use a shared database. Example config that should be
    changed on both **rc-node-2** and **rc-node-3** .
-   Edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+   Edit :file:`config/_shared/rhodecode.ini`
 
 .. code-block:: ini
 
@@ -163,7 +163,7 @@ persistent sessions across nodes. Please generate other one then in this example
 
 4) Use shared exception store. Example config that should be
    changed on both **rc-node-2** and **rc-node-3**, and also for VCSServer.
-   Edit :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` and
+   Edit :file:`config/_shared/rhodecode.ini` and
    :file:`/home/{user}/.rccontrol/{vcsserver-instance-id}/vcsserver.ini`
    and add/change following setting.
 
diff --git a/docs/admin/system_admin/tuning/tuning-user-sessions-performance.rst b/docs/admin/system_admin/tuning/tuning-user-sessions-performance.rst
index de0036d7..dd3879e0 100644
--- a/docs/admin/system_admin/tuning/tuning-user-sessions-performance.rst
+++ b/docs/admin/system_admin/tuning/tuning-user-sessions-performance.rst
@@ -15,7 +15,7 @@ scalability, and maintainability we recommend switching from file-based
 sessions to database-based user sessions or Redis based sessions.
 
 To switch to database-based user sessions uncomment the following section in
-your :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+your :file:`config/_shared/rhodecode.ini` file.
 
 
 .. code-block:: ini
@@ -49,7 +49,7 @@ uses, or if required it can be a different one. We recommend to use the same dat
 
 
 To switch to redis-based user sessions uncomment the following section in
-your :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+your :file:`config/_shared/rhodecode.ini` file.
 
 .. code-block:: ini
 
diff --git a/docs/admin/system_admin/vcs-server.rst b/docs/admin/system_admin/vcs-server.rst
index b0a87068..0850d9f1 100644
--- a/docs/admin/system_admin/vcs-server.rst
+++ b/docs/admin/system_admin/vcs-server.rst
@@ -52,7 +52,7 @@ To configure a |RCE| instance to use a VCS server, see
 The following list shows the available options on the |RCE| side of the
 connection to the VCS Server. The settings are configured per
 instance in the
-:file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+:file:`config/_shared/rhodecode.ini` file.
 
 .. rst-class:: dl-horizontal
 
diff --git a/docs/api/api.rst b/docs/api/api.rst
index ae7c491e..5cb976c6 100644
--- a/docs/api/api.rst
+++ b/docs/api/api.rst
@@ -27,7 +27,7 @@ of views that have API access enabled by default. To enable these,
 edit the |RCE| configuration ``.ini`` file. The default location is:
 
 * |RCE| Pre-2.2.7 :file:`root/rhodecode/data/production.ini`
-* |RCE| 3.0 :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+* |RCE| 3.0 :file:`config/_shared/rhodecode.ini`
 
 To configure the white list, edit this section of the file. In this
 configuration example, API access is granted to the patch/diff raw file and
diff --git a/docs/auth/ssh-connection.rst b/docs/auth/ssh-connection.rst
index 7a53b281..721af64d 100644
--- a/docs/auth/ssh-connection.rst
+++ b/docs/auth/ssh-connection.rst
@@ -87,7 +87,7 @@ following instructions.
    Hostname is required for the integration to properly set the instance URL.
 
    When your hostname is known (e.g https://code.rhodecode.com) please set it
-   inside :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+   inside :file:`config/_shared/rhodecode.ini`
 
    add into `[app:main]` section the following configuration:
 
@@ -111,7 +111,7 @@ following instructions.
 
        In case of connection problems please set
        `ssh.enable_debug_logging = true` inside the SSH configuration of
-       :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+       :file:`config/_shared/rhodecode.ini`
        Then add, remove your SSH key and try connecting again.
        Debug logging will be printed to help find the problems on the server side.
 
diff --git a/docs/install/setup-email.rst b/docs/install/setup-email.rst
index ab0190d1..0a81d6e3 100644
--- a/docs/install/setup-email.rst
+++ b/docs/install/setup-email.rst
@@ -4,7 +4,7 @@ Set up Email
 ------------
 
 To setup email with your |RCE| instance, open the default
-:file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+:file:`config/_shared/rhodecode.ini`
 file and uncomment and configure the email section. If it is not there,
 use the below example to insert it.
 
diff --git a/docs/tutorials/multi-instance-setup.rst b/docs/tutorials/multi-instance-setup.rst
index c794031c..e229b7dc 100644
--- a/docs/tutorials/multi-instance-setup.rst
+++ b/docs/tutorials/multi-instance-setup.rst
@@ -27,7 +27,7 @@ from the same database using the following steps:
 
    Once the new instance is installed you need to update the licence token and
    database connection string in the
-   :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini` file.
+   :file:`config/_shared/rhodecode.ini` file.
 
 .. code-block:: bash
 

From b6eb01f47ba09f9dc0da2116116bde737e4efbf1 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Fri, 29 Mar 2024 10:54:39 +0100
Subject: [PATCH 047/150] deps(js): drop qr generation lib from JS deps as we
 don't need it.

---
 package.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/package.json b/package.json
index 2988f2c8..3c007365 100644
--- a/package.json
+++ b/package.json
@@ -47,7 +47,6 @@
     "moment": "^2.18.1",
     "mousetrap": "^1.6.1",
     "polymer-webpack-loader": "^2.0.1",
-    "qrious": "^4.0.2",
     "raw-loader": "1.0.0-beta.0",
     "sticky-sidebar": "3.3.1",
     "style-loader": "^0.21.0",

From d8281bb5c0bcb0e2caa8f4a04732a38acd94ca7c Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Thu, 14 Mar 2024 11:18:07 +0200
Subject: [PATCH 048/150] feat(login by email option): added ability to log in
 with user primary email. Fixes: RCCE-63

---
 rhodecode/apps/login/tests/test_login.py           | 12 ++++++++++++
 rhodecode/apps/login/views.py                      |  8 ++++----
 rhodecode/authentication/base.py                   |  6 +-----
 rhodecode/authentication/plugins/auth_rhodecode.py |  2 +-
 rhodecode/model/db.py                              |  8 +++++++-
 rhodecode/model/validators.py                      |  2 +-
 rhodecode/templates/login.mako                     |  4 ++--
 7 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/rhodecode/apps/login/tests/test_login.py b/rhodecode/apps/login/tests/test_login.py
index 94aca8df..739ccfb7 100644
--- a/rhodecode/apps/login/tests/test_login.py
+++ b/rhodecode/apps/login/tests/test_login.py
@@ -80,6 +80,18 @@ class TestLoginController(object):
         assert username == 'test_regular'
         response.mustcontain('logout')
 
+    def test_login_with_primary_email(self):
+        user_email = 'test_regular@mail.com'
+        response = self.app.post(route_path('login'),
+                                 {'username': user_email,
+                                  'password': 'test12'}, status=302)
+        response = response.follow()
+        session = response.get_session_from_response()
+        user = session['rhodecode_user']
+        assert user['username'] == user_email.split('@')[0]
+        assert user['is_authenticated']
+        response.mustcontain('logout')
+
     def test_login_regular_forbidden_when_super_admin_restriction(self):
         from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin
         with fixture.auth_restriction(self.app._pyramid_registry,
diff --git a/rhodecode/apps/login/views.py b/rhodecode/apps/login/views.py
index c1269508..8dd93d3b 100644
--- a/rhodecode/apps/login/views.py
+++ b/rhodecode/apps/login/views.py
@@ -54,8 +54,8 @@ CaptchaData = collections.namedtuple(
     'CaptchaData', 'active, private_key, public_key')
 
 
-def store_user_in_session(session, username, remember=False):
-    user = User.get_by_username(username, case_insensitive=True)
+def store_user_in_session(session, user_identifier, remember=False):
+    user = User.get_by_username_or_primary_email(user_identifier)
     auth_user = AuthUser(user.user_id)
     auth_user.set_authenticated()
     cs = auth_user.get_cookie_store()
@@ -74,7 +74,7 @@ def store_user_in_session(session, username, remember=False):
     safe_cs = cs.copy()
     safe_cs['password'] = '****'
     log.info('user %s is now authenticated and stored in '
-             'session, session attrs %s', username, safe_cs)
+             'session, session attrs %s', user_identifier, safe_cs)
 
     # dumps session attrs back to cookie
     session._update_cookie_out()
@@ -181,7 +181,7 @@ class LoginView(BaseAppView):
             # form checks for username/password, now we're authenticated
             headers = store_user_in_session(
                 self.session,
-                username=form_result['username'],
+                user_identifier=form_result['username'],
                 remember=form_result['remember'])
             log.debug('Redirecting to "%s" after login.', c.came_from)
 
diff --git a/rhodecode/authentication/base.py b/rhodecode/authentication/base.py
index e19800aa..29971248 100644
--- a/rhodecode/authentication/base.py
+++ b/rhodecode/authentication/base.py
@@ -389,11 +389,7 @@ class RhodeCodeAuthPluginBase(object):
         log.debug(
             'Trying to fetch user `%s` from RhodeCode database', username)
         if username:
-            user = User.get_by_username(username)
-            if not user:
-                log.debug('User not found, fallback to fetch user in '
-                          'case insensitive mode')
-                user = User.get_by_username(username, case_insensitive=True)
+            user = User.get_by_username_or_primary_email(username)
         else:
             log.debug('provided username:`%s` is empty skipping...', username)
         if not user:
diff --git a/rhodecode/authentication/plugins/auth_rhodecode.py b/rhodecode/authentication/plugins/auth_rhodecode.py
index 6a041a46..e819b965 100644
--- a/rhodecode/authentication/plugins/auth_rhodecode.py
+++ b/rhodecode/authentication/plugins/auth_rhodecode.py
@@ -169,7 +169,7 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthPluginBase):
                          extra={"action": "user_auth_ok", "auth_module": "auth_rhodecode_anon", "username": userobj.username})
                 return user_attrs
 
-            elif userobj.username == username and password_match:
+            elif (userobj.username == username or userobj.email == username) and password_match:
                 log.info('user `%s` authenticated correctly', userobj.username,
                          extra={"action": "user_auth_ok", "auth_module": "auth_rhodecode", "username": userobj.username})
                 return user_attrs
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index 63e095bd..bae7154f 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -35,7 +35,7 @@ import collections
 
 from sqlalchemy import (
     or_, and_, not_, func, cast, TypeDecorator, event, select,
-    true, false, null,
+    true, false, null, union_all,
     Index, Sequence, UniqueConstraint, ForeignKey, CheckConstraint, Column,
     Boolean, String, Unicode, UnicodeText, DateTime, Integer, LargeBinary,
     Text, Float, PickleType, BigInteger)
@@ -953,6 +953,12 @@ class User(Base, BaseModel):
 
         return cls.execute(q).scalar_one_or_none()
 
+    @classmethod
+    def get_by_username_or_primary_email(cls, user_identifier):
+        qs = union_all(cls.select().where(func.lower(cls.username) == func.lower(user_identifier)),
+                       cls.select().where(func.lower(cls.email) == func.lower(user_identifier)))
+        return cls.execute(cls.select(User).from_statement(qs)).scalar_one_or_none()
+
     @classmethod
     def get_by_auth_token(cls, auth_token, cache=False):
 
diff --git a/rhodecode/model/validators.py b/rhodecode/model/validators.py
index 67ff9d0b..5236f82b 100644
--- a/rhodecode/model/validators.py
+++ b/rhodecode/model/validators.py
@@ -432,7 +432,7 @@ def ValidAuth(localizer):
 
             if not authenticate(username, password, '', HTTP_TYPE,
                                 skip_missing=True):
-                user = User.get_by_username(username)
+                user = User.get_by_username_or_primary_email(username)
                 if user and not user.active:
                     log.warning('user %s is disabled', username)
                     msg = M(self, 'disabled_account', state)
diff --git a/rhodecode/templates/login.mako b/rhodecode/templates/login.mako
index 3c52b0ba..3f5efae3 100644
--- a/rhodecode/templates/login.mako
+++ b/rhodecode/templates/login.mako
@@ -35,12 +35,12 @@
             <%block name="above_login_button" />
             <!-- login -->
             <div class="sign-in-title">
-                <h1>${_('Sign In using username/password')}</h1>
+                <h1>${_('Sign In using credentials')}</h1>
             </div>
             <div class="inner form">
                 ${h.form(request.route_path('login', _query={'came_from': c.came_from}), needs_csrf_token=False)}
 
-                    <label for="username">${_('Username')}:</label>
+                    <label for="username">${_('Username or email address')}:</label>
                     ${h.text('username', class_='focus', value=defaults.get('username'))}
                     %if 'username' in errors:
                     <span class="error-message">${errors.get('username')}</span>

From 31e3c38a18b540f4695be9cf1c15652b1b1996f5 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 3 Apr 2024 13:52:52 +0200
Subject: [PATCH 049/150] chore(deps): bumped paste and pastescript libs for
 deprecation warnings

---
 requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index a5820049..83609b12 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -171,8 +171,8 @@ nbconvert==7.7.3
     webencodings==0.5.1
   traitlets==5.9.0
 orjson==3.9.15
-pastescript==3.4.0
-  paste==3.7.1
+pastescript==3.5.1
+  paste==3.8.0
     six==1.16.0
   pastedeploy==3.1.0
   six==1.16.0
@@ -280,7 +280,7 @@ unidecode==1.3.6
 urlobject==2.4.3
 waitress==3.0.0
 weberror==0.13.1
-  paste==3.7.1
+  paste==3.8.0
     six==1.16.0
   pygments==2.15.1
   tempita==0.5.2

From 9678665ecde6ff650cec6d798b6f49bfc3e1c106 Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Mon, 15 Apr 2024 12:13:03 +0300
Subject: [PATCH 050/150] feat(2fa): Added 2fa option. Fixes: RCCE-65

---
 requirements.txt                              |   1 +
 rhodecode/apps/_base/__init__.py              |  47 +++++-
 rhodecode/apps/login/__init__.py              |  24 +++
 rhodecode/apps/login/tests/test_2fa.py        |  67 ++++++++
 rhodecode/apps/login/views.py                 |  78 ++++++++-
 rhodecode/apps/my_account/__init__.py         |  28 ++++
 rhodecode/apps/my_account/views/my_account.py |  27 ++++
 .../authentication/plugins/auth_rhodecode.py  |   8 +
 rhodecode/model/db.py                         |  91 +++++++++++
 rhodecode/model/forms.py                      |  22 +++
 rhodecode/public/js/rhodecode/routes.js       |   3 +
 .../admin/my_account/my_account.mako          |   1 +
 .../admin/my_account/my_account_2fa.mako      | 140 ++++++++++++++++
 rhodecode/templates/configure_2fa.mako        | 153 ++++++++++++++++++
 rhodecode/templates/verify_2fa.mako           |  37 +++++
 rhodecode/tests/routes.py                     |   1 +
 16 files changed, 722 insertions(+), 6 deletions(-)
 create mode 100644 rhodecode/apps/login/tests/test_2fa.py
 create mode 100644 rhodecode/templates/admin/my_account/my_account_2fa.mako
 create mode 100644 rhodecode/templates/configure_2fa.mako
 create mode 100644 rhodecode/templates/verify_2fa.mako

diff --git a/requirements.txt b/requirements.txt
index 83609b12..cb04cfc3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -290,6 +290,7 @@ webhelpers2==2.1
   six==1.16.0
 whoosh==2.7.4
 zope.cachedescriptors==5.0.0
+qrcode==7.4.2
 
 ## uncomment to add the debug libraries
 #-r requirements_debug.txt
diff --git a/rhodecode/apps/_base/__init__.py b/rhodecode/apps/_base/__init__.py
index fc6064b8..e500e6eb 100644
--- a/rhodecode/apps/_base/__init__.py
+++ b/rhodecode/apps/_base/__init__.py
@@ -104,6 +104,11 @@ class TemplateArgs(StrictAttributeDict):
 
 
 class BaseAppView(object):
+    DONT_CHECKOUT_VIEWS = ["channelstream_connect", "ops_ping"]
+    EXTRA_VIEWS_TO_IGNORE = ['login', 'register', 'logout']
+    SETUP_2FA_VIEW = 'setup_2fa'
+    VERIFY_2FA_VIEW = 'check_2fa'
+
     def __init__(self, context, request):
         self.request = request
         self.context = context
@@ -117,13 +122,19 @@ class BaseAppView(object):
 
         self._rhodecode_user = request.user  # auth user
         self._rhodecode_db_user = self._rhodecode_user.get_instance()
+        self.user_data = self._rhodecode_db_user.user_data if self._rhodecode_db_user else {}
         self._maybe_needs_password_change(
             request.matched_route.name, self._rhodecode_db_user
         )
+        self._maybe_needs_2fa_configuration(
+            request.matched_route.name, self._rhodecode_db_user
+        )
+        self._maybe_needs_2fa_check(
+            request.matched_route.name, self._rhodecode_db_user
+        )
 
     def _maybe_needs_password_change(self, view_name, user_obj):
-        dont_check_views = ["channelstream_connect", "ops_ping"]
-        if view_name in dont_check_views:
+        if view_name in self.DONT_CHECKOUT_VIEWS:
             return
 
         log.debug(
@@ -144,7 +155,7 @@ class BaseAppView(object):
             return
 
         now = time.time()
-        should_change = user_obj.user_data.get("force_password_change")
+        should_change = self.user_data.get("force_password_change")
         change_after = safe_int(should_change) or 0
         if should_change and now > change_after:
             log.debug("User %s requires password change", user_obj)
@@ -157,6 +168,36 @@ class BaseAppView(object):
             if view_name not in skip_user_views:
                 raise HTTPFound(self.request.route_path("my_account_password"))
 
+    def _maybe_needs_2fa_configuration(self, view_name, user_obj):
+        if view_name in self.DONT_CHECKOUT_VIEWS + self.EXTRA_VIEWS_TO_IGNORE:
+            return
+
+        if not user_obj:
+            return
+
+        if user_obj.has_forced_2fa and user_obj.extern_type != 'rhodecode':
+            return
+
+        if (user_obj.has_enabled_2fa
+            and not self.user_data.get('secret_2fa')) \
+                and view_name != self.SETUP_2FA_VIEW:
+            h.flash(
+                "You are required to configure 2FA",
+                "warning",
+                ignore_duplicate=False,
+            )
+            raise HTTPFound(self.request.route_path(self.SETUP_2FA_VIEW))
+
+    def _maybe_needs_2fa_check(self, view_name, user_obj):
+        if view_name in self.DONT_CHECKOUT_VIEWS + self.EXTRA_VIEWS_TO_IGNORE:
+            return
+
+        if not user_obj:
+            return
+
+        if self.user_data.get('check_2fa') and view_name != self.VERIFY_2FA_VIEW:
+            raise HTTPFound(self.request.route_path(self.VERIFY_2FA_VIEW))
+
     def _log_creation_exception(self, e, repo_name):
         _ = self.request.translate
         reason = None
diff --git a/rhodecode/apps/login/__init__.py b/rhodecode/apps/login/__init__.py
index 44f974b8..eed380b1 100644
--- a/rhodecode/apps/login/__init__.py
+++ b/rhodecode/apps/login/__init__.py
@@ -75,3 +75,27 @@ def includeme(config):
         LoginView,
         attr='password_reset_confirmation',
         route_name='reset_password_confirmation', request_method='GET')
+
+    config.add_route(
+        name='setup_2fa',
+        pattern=ADMIN_PREFIX + '/setup_2fa')
+    config.add_view(
+        LoginView,
+        attr='setup_2fa',
+        route_name='setup_2fa', request_method=['GET', 'POST'],
+        renderer='rhodecode:templates/configure_2fa.mako')
+
+    config.add_route(
+        name='check_2fa',
+        pattern=ADMIN_PREFIX + '/check_2fa')
+    config.add_view(
+        LoginView,
+        attr='verify_2fa',
+        route_name='check_2fa', request_method='GET',
+        renderer='rhodecode:templates/verify_2fa.mako')
+    config.add_view(
+        LoginView,
+        attr='verify_2fa',
+        route_name='check_2fa', request_method='POST',
+        renderer='rhodecode:templates/verify_2fa.mako')
+
diff --git a/rhodecode/apps/login/tests/test_2fa.py b/rhodecode/apps/login/tests/test_2fa.py
new file mode 100644
index 00000000..7a2ef2f3
--- /dev/null
+++ b/rhodecode/apps/login/tests/test_2fa.py
@@ -0,0 +1,67 @@
+import pytest
+
+from rhodecode.model.meta import Session
+from rhodecode.tests.fixture import Fixture
+from rhodecode.tests.routes import route_path
+from rhodecode.model.settings import SettingsModel
+
+fixture = Fixture()
+
+
+@pytest.mark.usefixtures('app')
+class Test2FA(object):
+    @classmethod
+    def setup_class(cls):
+        cls.password = 'valid-one'
+
+    @classmethod
+    def teardown_class(cls):
+        SettingsModel().create_or_update_setting('auth_rhodecode_global_2fa', False)
+
+    def test_redirect_to_2fa_setup_if_enabled_for_user(self, user_util):
+        user = user_util.create_user(password=self.password)
+        user.has_enabled_2fa = True
+        self.app.post(
+            route_path('login'),
+            {'username': user.username,
+             'password': self.password})
+
+        response = self.app.get('/')
+        assert response.status_code == 302
+        assert response.location.endswith(route_path('setup_2fa'))
+
+    def test_redirect_to_2fa_check_if_2fa_configured(self, user_util):
+        user = user_util.create_user(password=self.password)
+        user.has_enabled_2fa = True
+        user.secret_2fa
+        Session().add(user)
+        Session().commit()
+        self.app.post(
+            route_path('login'),
+            {'username': user.username,
+             'password': self.password})
+        response = self.app.get('/')
+        assert response.status_code == 302
+        assert response.location.endswith(route_path('check_2fa'))
+
+    def test_2fa_recovery_codes_works_only_once(self, user_util):
+        user = user_util.create_user(password=self.password)
+        user.has_enabled_2fa = True
+        user.secret_2fa
+        recovery_cod_to_check = user.get_2fa_recovery_codes()[0]
+        Session().add(user)
+        Session().commit()
+        self.app.post(
+            route_path('login'),
+            {'username': user.username,
+             'password': self.password})
+        response = self.app.post(route_path('check_2fa'), {'totp': recovery_cod_to_check})
+        assert response.status_code == 302
+        response = self.app.post(route_path('check_2fa'), {'totp': recovery_cod_to_check})
+        response.mustcontain('Code is invalid. Try again!')
+
+    def test_2fa_state_when_forced_by_admin(self, user_util):
+        user = user_util.create_user(password=self.password)
+        user.has_enabled_2fa = False
+        SettingsModel().create_or_update_setting('auth_rhodecode_global_2fa', True)
+        assert user.has_enabled_2fa
diff --git a/rhodecode/apps/login/views.py b/rhodecode/apps/login/views.py
index 8dd93d3b..e99deeaf 100644
--- a/rhodecode/apps/login/views.py
+++ b/rhodecode/apps/login/views.py
@@ -17,6 +17,9 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import time
+import json
+import pyotp
+import qrcode
 import collections
 import datetime
 import formencode
@@ -24,7 +27,11 @@ import formencode.htmlfill
 import logging
 import urllib.parse
 import requests
+from io import BytesIO
+from base64 import b64encode
 
+from pyramid.renderers import render
+from pyramid.response import Response
 from pyramid.httpexceptions import HTTPFound
 
 
@@ -35,12 +42,12 @@ from rhodecode.events import UserRegistered, trigger
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.auth import (
-    AuthUser, HasPermissionAnyDecorator, CSRFRequired)
+    AuthUser, HasPermissionAnyDecorator, CSRFRequired, LoginRequired, NotAnonymous)
 from rhodecode.lib.base import get_ip_addr
 from rhodecode.lib.exceptions import UserCreationError
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.model.db import User, UserApiKeys
-from rhodecode.model.forms import LoginForm, RegisterForm, PasswordResetForm
+from rhodecode.model.forms import LoginForm, RegisterForm, PasswordResetForm, TOTPForm
 from rhodecode.model.meta import Session
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.settings import SettingsModel
@@ -179,9 +186,12 @@ class LoginView(BaseAppView):
             self.session.invalidate()
             form_result = login_form.to_python(self.request.POST)
             # form checks for username/password, now we're authenticated
+            username = form_result['username']
+            if (user := User.get_by_username_or_primary_email(username)).has_enabled_2fa:
+                user.update_userdata(check_2fa=True)
             headers = store_user_in_session(
                 self.session,
-                user_identifier=form_result['username'],
+                user_identifier=username,
                 remember=form_result['remember'])
             log.debug('Redirecting to "%s" after login.', c.came_from)
 
@@ -436,6 +446,8 @@ class LoginView(BaseAppView):
 
         return self._get_template_context(c, **template_context)
 
+    @LoginRequired()
+    @NotAnonymous()
     def password_reset_confirmation(self):
         self.load_default_context()
         if self.request.GET and self.request.GET.get('key'):
@@ -467,3 +479,63 @@ class LoginView(BaseAppView):
                 return HTTPFound(self.request.route_path('reset_password'))
 
         return HTTPFound(self.request.route_path('login'))
+
+    @LoginRequired()
+    @NotAnonymous()
+    def setup_2fa(self):
+        _ = self.request.translate
+        c = self.load_default_context()
+        user_instance = self._rhodecode_db_user
+        form = TOTPForm(_, user_instance)()
+        render_ctx = {}
+        if self.request.method == 'POST':
+            try:
+                form.to_python(dict(self.request.POST))
+                Session().commit()
+                raise HTTPFound(c.came_from)
+            except formencode.Invalid as errors:
+                defaults = errors.value
+                render_ctx = {
+                    'errors': errors.error_dict,
+                    'defaults': defaults,
+                }
+        qr = qrcode.QRCode(version=1, box_size=10, border=5)
+        secret = user_instance.secret_2fa
+        Session().flush()
+        recovery_codes = user_instance.get_2fa_recovery_codes()
+        Session().commit()
+        qr.add_data(pyotp.totp.TOTP(secret).provisioning_uri(
+            name=self.request.user.name))
+        qr.make(fit=True)
+        img = qr.make_image(fill_color='black', back_color='white')
+        buffered = BytesIO()
+        img.save(buffered)
+        return self._get_template_context(
+            c,
+            qr=b64encode(buffered.getvalue()).decode("utf-8"),
+            key=secret, recovery_codes=json.dumps(recovery_codes),
+            codes_viewed=not bool(recovery_codes),
+            ** render_ctx
+        )
+
+    @LoginRequired()
+    @NotAnonymous()
+    def verify_2fa(self):
+        _ = self.request.translate
+        c = self.load_default_context()
+        render_ctx = {}
+        user_instance = self._rhodecode_db_user
+        totp_form = TOTPForm(_, user_instance, allow_recovery_code_use=True)()
+        if self.request.method == 'POST':
+            try:
+                totp_form.to_python(dict(self.request.POST))
+                user_instance.update_userdata(check_2fa=False)
+                Session().commit()
+                raise HTTPFound(c.came_from)
+            except formencode.Invalid as errors:
+                defaults = errors.value
+                render_ctx = {
+                    'errors': errors.error_dict,
+                    'defaults': defaults,
+                }
+        return self._get_template_context(c, **render_ctx)
diff --git a/rhodecode/apps/my_account/__init__.py b/rhodecode/apps/my_account/__init__.py
index 5ad11ab5..e8dd97cf 100644
--- a/rhodecode/apps/my_account/__init__.py
+++ b/rhodecode/apps/my_account/__init__.py
@@ -74,6 +74,34 @@ def includeme(config):
         route_name='my_account_password_update', request_method='POST',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
 
+    # my account 2fa
+    config.add_route(
+        name='my_account_enable_2fa',
+        pattern=ADMIN_PREFIX + '/my_account/enable_2fa')
+    config.add_view(
+        MyAccountView,
+        attr='my_account_2fa',
+        route_name='my_account_enable_2fa', request_method='GET',
+        renderer='rhodecode:templates/admin/my_account/my_account.mako')
+
+    config.add_route(
+        name='my_account_configure_2fa',
+        pattern=ADMIN_PREFIX + '/my_account/configure_2fa')
+    config.add_view(
+        MyAccountView,
+        attr='my_account_2fa_configure',
+        route_name='my_account_configure_2fa', request_method='POST', xhr=True,
+        renderer='json_ext')
+
+    config.add_route(
+        name='my_account_regenerate_2fa_recovery_codes',
+        pattern=ADMIN_PREFIX + '/my_account/regenerate_recovery_codes')
+    config.add_view(
+        MyAccountView,
+        attr='my_account_2fa_regenerate_recovery_codes',
+        route_name='my_account_regenerate_2fa_recovery_codes', request_method='POST', xhr=True,
+        renderer='json_ext')
+
     # my account tokens
     config.add_route(
         name='my_account_auth_tokens',
diff --git a/rhodecode/apps/my_account/views/my_account.py b/rhodecode/apps/my_account/views/my_account.py
index 6b7f184f..f17088c9 100644
--- a/rhodecode/apps/my_account/views/my_account.py
+++ b/rhodecode/apps/my_account/views/my_account.py
@@ -202,6 +202,33 @@ class MyAccountView(BaseAppView, DataGridAppView):
 
         raise HTTPFound(self.request.route_path('my_account_password'))
 
+    @LoginRequired()
+    @NotAnonymous()
+    def my_account_2fa(self):
+        _ = self.request.translate
+        c = self.load_default_context()
+        c.active = '2fa'
+        from rhodecode.model.settings import SettingsModel
+        user_instance = self._rhodecode_db_user
+        locked_by_admin = user_instance.has_forced_2fa
+        c.state_of_2fa = user_instance.has_enabled_2fa
+        c.locked_2fa = str2bool(locked_by_admin)
+        return self._get_template_context(c)
+
+    @LoginRequired()
+    @NotAnonymous()
+    @CSRFRequired()
+    def my_account_2fa_configure(self):
+        state = self.request.POST.get('state')
+        self._rhodecode_db_user.has_enabled_2fa = state
+        return {'state_of_2fa': state}
+
+    @LoginRequired()
+    @NotAnonymous()
+    @CSRFRequired()
+    def my_account_2fa_regenerate_recovery_codes(self):
+        return {'recovery_codes': self._rhodecode_db_user.regenerate_2fa_recovery_codes()}
+
     @LoginRequired()
     @NotAnonymous()
     def my_account_auth_tokens(self):
diff --git a/rhodecode/authentication/plugins/auth_rhodecode.py b/rhodecode/authentication/plugins/auth_rhodecode.py
index e819b965..91a64377 100644
--- a/rhodecode/authentication/plugins/auth_rhodecode.py
+++ b/rhodecode/authentication/plugins/auth_rhodecode.py
@@ -183,6 +183,14 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthPluginBase):
 
 
 class RhodeCodeSettingsSchema(AuthnPluginSettingsSchemaBase):
+    global_2fa = colander.SchemaNode(
+        colander.Bool(),
+        default=False,
+        description=_('Force all users to use two factor authentication by enabling this.'),
+        missing=False,
+        title=_('Global 2FA'),
+        widget='bool',
+    )
 
     auth_restriction_choices = [
         (RhodeCodeAuthPlugin.AUTH_RESTRICTION_NONE, 'All users'),
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index bae7154f..c7330451 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -33,6 +33,7 @@ import functools
 import traceback
 import collections
 
+import pyotp
 from sqlalchemy import (
     or_, and_, not_, func, cast, TypeDecorator, event, select,
     true, false, null, union_all,
@@ -51,6 +52,7 @@ from zope.cachedescriptors.property import Lazy as LazyProperty
 from pyramid.threadlocal import get_current_request
 from webhelpers2.text import remove_formatting
 
+from rhodecode import ConfigGet
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.translation import _
 from rhodecode.lib.vcs import get_vcs_instance, VCSError
@@ -586,6 +588,7 @@ class User(Base, BaseModel):
     DEFAULT_USER = 'default'
     DEFAULT_USER_EMAIL = 'anonymous@rhodecode.org'
     DEFAULT_GRAVATAR_URL = 'https://secure.gravatar.com/avatar/{md5email}?d=identicon&s={size}'
+    RECOVERY_CODES_COUNT = 10
 
     user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     username = Column("username", String(255), nullable=True, unique=None, default=None)
@@ -793,6 +796,94 @@ class User(Base, BaseModel):
             Session.commit()
             return artifact_token.api_key
 
+    @hybrid_property
+    def secret_2fa(self):
+        if not self.user_data.get('secret_2fa'):
+            secret = pyotp.random_base32()
+            self.update_userdata(secret_2fa=safe_str(enc_utils.encrypt_value(secret, enc_key=ENCRYPTION_KEY)))
+            return secret
+        return safe_str(
+            enc_utils.decrypt_value(self.user_data['secret_2fa'],
+                                    enc_key=ENCRYPTION_KEY,
+                                    strict_mode=ConfigGet().get_bool('rhodecode.encrypted_values.strict',
+                                                                               missing=True)
+                                    )
+        )
+
+    def is_totp_valid(self, received_code):
+        totp = pyotp.TOTP(self.secret_2fa)
+        return totp.verify(received_code)
+
+    def is_2fa_recovery_code_valid(self, received_code):
+        encrypted_recovery_codes = self.user_data.get('recovery_codes_2fa', [])
+        recovery_codes = list(map(
+            lambda x: safe_str(
+                enc_utils.decrypt_value(
+                    x,
+                    enc_key=ENCRYPTION_KEY,
+                    strict_mode=ConfigGet().get_bool('rhodecode.encrypted_values.strict', missing=True)
+                )),
+            encrypted_recovery_codes))
+        if received_code in recovery_codes:
+            encrypted_recovery_codes.pop(recovery_codes.index(received_code))
+            self.update_userdata(recovery_codes_2fa=encrypted_recovery_codes)
+            return True
+        return False
+
+    @hybrid_property
+    def has_forced_2fa(self):
+        """
+        Checks if 2fa was forced for ALL users (including current one)
+        """
+        from rhodecode.model.settings import SettingsModel
+        # So now we're supporting only auth_rhodecode_global_2f
+        if value := SettingsModel().get_setting_by_name('auth_rhodecode_global_2fa'):
+            return value.app_settings_value
+        return False
+
+    @hybrid_property
+    def has_enabled_2fa(self):
+        """
+        Checks if 2fa was enabled by user
+        """
+        if value := self.has_forced_2fa:
+            return value
+        return self.user_data.get('enabled_2fa', False)
+
+    @has_enabled_2fa.setter
+    def has_enabled_2fa(self, val):
+        val = str2bool(val)
+        self.update_userdata(enabled_2fa=str2bool(val))
+        if not val:
+            self.update_userdata(secret_2fa=None, recovery_codes_2fa=[])
+        Session().commit()
+
+    def get_2fa_recovery_codes(self):
+        """
+        Creates 2fa recovery codes
+        """
+        recovery_codes = self.user_data.get('recovery_codes_2fa', [])
+        encrypted_codes = []
+        if not recovery_codes:
+            for _ in range(self.RECOVERY_CODES_COUNT):
+                recovery_code = pyotp.random_base32()
+                recovery_codes.append(recovery_code)
+                encrypted_codes.append(safe_str(enc_utils.encrypt_value(recovery_code, enc_key=ENCRYPTION_KEY)))
+            self.update_userdata(recovery_codes_2fa=encrypted_codes)
+            return recovery_codes
+        # User should not check the same recovery codes more than once
+        return []
+
+    def regenerate_2fa_recovery_codes(self):
+        """
+        Regenerates 2fa recovery codes upon request
+        """
+        self.update_userdata(recovery_codes_2fa=[])
+        Session().flush()
+        new_recovery_codes = self.get_2fa_recovery_codes()
+        Session().commit()
+        return new_recovery_codes
+
     @classmethod
     def get(cls, user_id, cache=False):
         if not user_id:
diff --git a/rhodecode/model/forms.py b/rhodecode/model/forms.py
index 96772799..9d625536 100644
--- a/rhodecode/model/forms.py
+++ b/rhodecode/model/forms.py
@@ -104,6 +104,28 @@ def LoginForm(localizer):
     return _LoginForm
 
 
+def TOTPForm(localizer, user, allow_recovery_code_use=False):
+    _ = localizer
+
+    class _TOTPForm(formencode.Schema):
+        allow_extra_fields = True
+        filter_extra_fields = False
+        totp = v.Regex(r'^(?:\d{6}|[A-Z0-9]{32})$')
+
+        def to_python(self, value, state=None):
+            validation_checks = [user.is_totp_valid]
+            if allow_recovery_code_use:
+                validation_checks.append(user.is_2fa_recovery_code_valid)
+            form_data = super().to_python(value, state)
+            received_code = form_data['totp']
+            if not any(map(lambda x: x(received_code), validation_checks)):
+                error_msg = _('Code is invalid. Try again!')
+                raise formencode.Invalid(error_msg, v, state, error_dict={'totp': error_msg})
+            return True
+
+    return _TOTPForm
+
+
 def UserForm(localizer, edit=False, available_languages=None, old_data=None):
     old_data = old_data or {}
     available_languages = available_languages or []
diff --git a/rhodecode/public/js/rhodecode/routes.js b/rhodecode/public/js/rhodecode/routes.js
index 03e43b21..fcb944d4 100644
--- a/rhodecode/public/js/rhodecode/routes.js
+++ b/rhodecode/public/js/rhodecode/routes.js
@@ -224,6 +224,9 @@ function registerRCRoutes() {
     pyroutes.register('my_account_notifications', '/_admin/my_account/notifications', []);
     pyroutes.register('my_account_notifications_test_channelstream', '/_admin/my_account/test_channelstream', []);
     pyroutes.register('my_account_notifications_toggle_visibility', '/_admin/my_account/toggle_visibility', []);
+    pyroutes.register('check_2fa', '/_admin/check_2fa', []);
+    pyroutes.register('my_account_configure_2fa', '/_admin/my_account/configure_2fa', []);
+    pyroutes.register('my_account_regenerate_2fa_recovery_codes', '/_admin/my_account/regenerate_recovery_codes', []);
     pyroutes.register('my_account_password', '/_admin/my_account/password', []);
     pyroutes.register('my_account_password_update', '/_admin/my_account/password/update', []);
     pyroutes.register('my_account_perms', '/_admin/my_account/perms', []);
diff --git a/rhodecode/templates/admin/my_account/my_account.mako b/rhodecode/templates/admin/my_account/my_account.mako
index 9ba03b23..8f45ffb8 100644
--- a/rhodecode/templates/admin/my_account/my_account.mako
+++ b/rhodecode/templates/admin/my_account/my_account.mako
@@ -28,6 +28,7 @@
           <li class="${h.is_active(['profile', 'profile_edit'], c.active)}"><a href="${h.route_path('my_account_profile')}">${_('Profile')}</a></li>
           <li class="${h.is_active('emails', c.active)}"><a href="${h.route_path('my_account_emails')}">${_('Emails')}</a></li>
           <li class="${h.is_active('password', c.active)}"><a href="${h.route_path('my_account_password')}">${_('Password')}</a></li>
+          <li class="${h.is_active('2FA', c.active)}"><a href="${h.route_path('my_account_enable_2fa')}">${_('2FA')}</a></li>
           <li class="${h.is_active('bookmarks', c.active)}"><a href="${h.route_path('my_account_bookmarks')}">${_('Bookmarks')}</a></li>
           <li class="${h.is_active('auth_tokens', c.active)}"><a href="${h.route_path('my_account_auth_tokens')}">${_('Auth Tokens')}</a></li>
           <li class="${h.is_active(['ssh_keys', 'ssh_keys_generate'], c.active)}"><a href="${h.route_path('my_account_ssh_keys')}">${_('SSH Keys')}</a></li>
diff --git a/rhodecode/templates/admin/my_account/my_account_2fa.mako b/rhodecode/templates/admin/my_account/my_account_2fa.mako
new file mode 100644
index 00000000..36c03027
--- /dev/null
+++ b/rhodecode/templates/admin/my_account/my_account_2fa.mako
@@ -0,0 +1,140 @@
+<%namespace name="base" file="/base/base.mako"/>
+
+<div class="panel panel-default">
+    <div class="panel-heading">
+        <h3 class="panel-title">${_('Enable/Disable 2FA for your account')}</h3>
+    </div>
+    <div class="panel-body">
+        <div class="form">
+            <div class="fields">
+                <div class="field">
+                    <div class="label">
+                        <label>${_('2FA status')}:</label>
+                    </div>
+                    <div class="checkboxes">
+
+                        <div class="form-check">
+                            <label class="form-check-label">
+                                <input type="radio" id="2faEnabled" value="1" ${'checked' if c.state_of_2fa else ''}>
+                                ${_('Enabled')}
+                            </label>
+                            <label class="form-check-label">
+                                <input type="radio" id="2faDisabled" value="0" ${'checked' if not c.state_of_2fa else ''}>
+                                ${_('Disabled')}
+                            </label>
+                        </div>
+                        % if c.locked_2fa:
+                        <span class="help-block">${_('2FA settings cannot be changed here, because 2FA was forced enabled by RhodeCode Administrator.')}</span>
+                        % endif
+                    </div>
+                </div>
+            </div>
+            <button id="saveBtn" class="btn btn-primary" ${'disabled' if c.locked_2fa else ''}>${_('Save')}</button>
+        </div>
+        <div id="codesPopup" class="modal">
+            <div class="modal-content">
+                <ul id="recoveryCodesList"></ul>
+                <button id="copyAllBtn" class="btn btn-primary">Copy All</button>
+            </div>
+        </div>
+    </div>
+</div>
+% if c.state_of_2fa:
+<div class="panel panel-default">
+    <div class="panel-heading">
+        <h3 class="panel-title">${_('Regenerate 2FA recovery codes for your account')}</h3>
+    </div>
+    <div class="panel-body">
+        <form id="2faForm">
+            <input type="text" name="totp" placeholder="${_('Verify the code from the app')}" pattern="\d{6}"
+                   style="width: 20%">
+            <button type="button" class="btn btn-primary" onclick="submitForm()">Verify</button>
+        </form>
+        <div id="result"></div>
+    </div>
+
+</div>
+% endif
+<script>
+    function submitForm() {
+        let formData = new FormData(document.getElementById("2faForm"));
+        let xhr = new XMLHttpRequest();
+        let success = function (response) {
+            let recovery_codes = response.recovery_codes;
+            const codesList = document.getElementById("recoveryCodesList");
+
+            codesList.innerHTML = "";
+            recovery_codes.forEach(code => {
+                const listItem = document.createElement("li");
+                listItem.textContent = code;
+                codesList.appendChild(listItem);
+            });
+        }
+        xhr.onreadystatechange = function () {
+            if (xhr.readyState == 4 && xhr.status == 200) {
+                let responseDoc = new DOMParser().parseFromString(xhr.responseText, "text/html");
+                let contentToDisplay = responseDoc.querySelector('#formErrors');
+                if (contentToDisplay) {
+                    document.getElementById("result").innerHTML = contentToDisplay.innerHTML;
+                } else {
+                    let regenerate_url = pyroutes.url('my_account_regenerate_2fa_recovery_codes');
+                    ajaxPOST(regenerate_url, {'csrf_token': CSRF_TOKEN}, success);
+                    showRecoveryCodesPopup();
+                }
+            }
+        };
+        let url = pyroutes.url('check_2fa');
+        xhr.open("POST", url, true);
+        xhr.send(formData);
+    }
+</script>
+<script>
+    document.getElementById('2faEnabled').addEventListener('click', function () {
+        document.getElementById('2faDisabled').checked = false;
+    });
+    document.getElementById('2faDisabled').addEventListener('click', function () {
+        document.getElementById('2faEnabled').checked = false;
+    });
+
+    function getStateValue() {
+        if (document.getElementById('2faEnabled').checked) {
+            return '1';
+        } else {
+            return '0';
+        }
+    };
+
+    function saveChanges(state) {
+
+        let post_data = {'state': state, 'csrf_token': CSRF_TOKEN};
+        let url = pyroutes.url('my_account_configure_2fa');
+
+        ajaxPOST(url, post_data, null)
+    };
+    document.getElementById('saveBtn').addEventListener('click', function () {
+        var state = getStateValue();
+        saveChanges(state);
+    });
+</script>
+<script>
+    function showRecoveryCodesPopup() {
+        const popup = document.getElementById("codesPopup");
+        popup.style.display = "block";
+    }
+
+    document.getElementById("copyAllBtn").addEventListener("click", function () {
+        const codesListItems = document.querySelectorAll("#recoveryCodesList li");
+        const allCodes = Array.from(codesListItems).map(item => item.textContent).join(", ");
+
+        const textarea = document.createElement('textarea');
+        textarea.value = allCodes;
+        document.body.appendChild(textarea);
+
+        textarea.select();
+        document.execCommand('copy');
+
+        document.body.removeChild(textarea);
+        const popup = document.getElementById("codesPopup");
+        popup.style.display = ""
+    });
+</script>
diff --git a/rhodecode/templates/configure_2fa.mako b/rhodecode/templates/configure_2fa.mako
new file mode 100644
index 00000000..96bb4c6e
--- /dev/null
+++ b/rhodecode/templates/configure_2fa.mako
@@ -0,0 +1,153 @@
+<%inherit file="base/root.mako"/>
+
+<%def name="title()">
+    ${_('Setup authenticator app')}
+    %if c.rhodecode_name:
+        &middot; ${h.branding(c.rhodecode_name)}
+    %endif
+</%def>
+<style>body{background-color:#eeeeee;}</style>
+
+<div class="loginbox">
+    <div class="header-account">
+        <div id="header-inner" class="title">
+            <div id="logo">
+                % if c.rhodecode_name:
+                <div class="branding">
+                   <a href="${h.route_path('home')}">${h.branding(c.rhodecode_name)}</a>
+                </div>
+                % endif
+            </div>
+        </div>
+    </div>
+
+    <div class="loginwrapper">
+        <h1>Setup the authenticator app</h1>
+        <p>Authenticator apps like <a href='https://play.google.com/store/apps/details?id=com.google.android.apps.authenticator2' target="_blank" rel="noopener noreferrer">Google Authenticator</a>, etc. generate one-time passwords that are used as a second factor to verify you identity.</p>
+        <rhodecode-toast id="notifications"></rhodecode-toast>
+
+        <div id="setup_2fa">
+            <div class="sign-in-title">
+                <h1>${_('Scan the QR code')}</h1>
+            </div>
+            <p>Use an authenticator app to scan.</p>
+            <img src="data:image/png;base64,   ${qr}"/>
+            <p>${_('Unable to scan?')} <a id="toggleLink">${_('Click here')}</a></p>
+            <div id="secretDiv" class="hidden">
+                <p>${_('Copy and use this code to manually setup an authenticator app')}</p>
+                <input type="text" id="secretField" value=${key}>
+                <i class="tooltip icon-clipboard clipboard-action" data-clipboard-text="" title="${_('Copy the secret key')}"></i>
+            </div>
+            <div id="codesPopup" class="modal">
+              <div class="modal-content">
+                <ul id="recoveryCodesList"></ul>
+                <button id="copyAllBtn" class="btn btn-primary">Copy All</button>
+              </div>
+            </div>
+            <br><br>
+            <div id="verify_2fa">
+                ${h.secure_form(h.route_path('setup_2fa'), request=request, id='totp_form')}
+                <div class="form mt-4">
+                    <div class="field">
+                        <p>
+                        <div class="label">
+                            <label for="totp" class="form-label text-dark font-weight-bold" style="text-align: left;">${_('Verify the code from the app')}:</label>
+                        </div>
+                        </p>
+                        <p>
+                        <div>
+                            <div class="input-group">
+                                ${h.text('totp', class_='form-control', style='width: 40%;')}
+                                <div id="formErrors">
+                                    %if 'totp' in errors:
+                                        <span class="error-message">${errors.get('totp')}</span>
+                                        <br />
+                                    %endif
+                                </div>
+                                <div class="input-group-append">
+                                    ${h.submit('save',_('Verify'),class_="btn btn-primary", style='width: 40%;', disabled=not codes_viewed)}
+                                </div>
+                            </div>
+                        </div>
+                        </p>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+<script>
+  document.addEventListener('DOMContentLoaded', function() {
+            let clipboardIcons = document.querySelectorAll('.clipboard-action');
+
+            clipboardIcons.forEach(function(icon) {
+                icon.addEventListener('click', function() {
+                    var inputField = document.getElementById('secretField');
+                    inputField.select();
+                    document.execCommand('copy');
+
+                });
+            });
+    });
+</script>
+<script>
+  document.getElementById('toggleLink').addEventListener('click', function() {
+    let hiddenField = document.getElementById('secretDiv');
+    if (hiddenField.classList.contains('hidden')) {
+      hiddenField.classList.remove('hidden');
+  }
+  });
+</script>
+<script>
+const recovery_codes_string = '${recovery_codes}';
+const cleaned_recovery_codes_string = recovery_codes_string
+  .replace(/&#34;/g, '"')
+  .replace(/&#39;/g, "'");
+
+const recovery_codes = JSON.parse(cleaned_recovery_codes_string);
+
+const cleaned_recovery_codes = recovery_codes.map(code => code.replace(/['"]/g, ''));
+
+function showRecoveryCodesPopup() {
+  const popup = document.getElementById("codesPopup");
+  const codesList = document.getElementById("recoveryCodesList");
+  const verify_btn = document.getElementById('save')
+
+  if (verify_btn.disabled) {
+     codesList.innerHTML = "";
+
+      cleaned_recovery_codes.forEach(code => {
+        const listItem = document.createElement("li");
+        listItem.textContent = code;
+        codesList.appendChild(listItem);
+      });
+
+      popup.style.display = "block";
+      verify_btn.disabled = false;
+    }
+  }
+
+document.getElementById("save").addEventListener("mouseover", showRecoveryCodesPopup);
+
+const popup = document.getElementById("codesPopup");
+const closeButton = document.querySelector(".close");
+window.onclick = function(event) {
+  if (event.target === popup || event.target === closeButton) {
+    popup.style.display = "none";
+  }
+}
+
+document.getElementById("copyAllBtn").addEventListener("click", function() {
+  const codesListItems = document.querySelectorAll("#recoveryCodesList li");
+  const allCodes = Array.from(codesListItems).map(item => item.textContent).join(", ");
+
+  const textarea = document.createElement('textarea');
+  textarea.value = allCodes;
+  document.body.appendChild(textarea);
+
+  textarea.select();
+  document.execCommand('copy');
+
+  document.body.removeChild(textarea);
+});
+</script>
diff --git a/rhodecode/templates/verify_2fa.mako b/rhodecode/templates/verify_2fa.mako
new file mode 100644
index 00000000..c5d2f3d3
--- /dev/null
+++ b/rhodecode/templates/verify_2fa.mako
@@ -0,0 +1,37 @@
+<%inherit file="/base/root.mako"/>
+<%def name="title()">
+    ${_('Check 2FA')}
+    %if c.rhodecode_name:
+        &middot; ${h.branding(c.rhodecode_name)}
+    %endif
+</%def>
+
+<div class="box">
+    <div class="verify2FA">
+        ${h.secure_form(h.route_path('check_2fa'), request=request, id='totp_form')}
+        <div class="form mt-4" style="position: relative; margin-left: 35%; margin-top: 20%;">
+            <div class="field">
+                <p>
+                <div class="label">
+                    <label for="totp" class="form-label text-dark font-weight-bold" style="text-align: left;">${_('Verify the code from the app')}:</label>
+                </div>
+                </p>
+                <p>
+                <div>
+                    <div class="input-group">
+                        ${h.text('totp', class_="form-control", style='width: 38%;')}
+                        <div id="formErrors">
+                            %if 'totp' in errors:
+                                <span class="error-message">${errors.get('totp')}</span>
+                                <br />
+                            %endif
+                        </div>
+                        <br />
+                        ${h.submit('save',_('Verify'),class_="btn btn-primary", style='width: 40%;')}
+                    </div>
+                </div>
+                </p>
+            </div>
+        </div>
+    </div>
+</div>
diff --git a/rhodecode/tests/routes.py b/rhodecode/tests/routes.py
index f74e45ea..a11a9291 100644
--- a/rhodecode/tests/routes.py
+++ b/rhodecode/tests/routes.py
@@ -106,6 +106,7 @@ def get_url_defs():
         + "/gists/{gist_id}/rev/{revision}/{format}/{f_path}",
         "login": ADMIN_PREFIX + "/login",
         "logout": ADMIN_PREFIX + "/logout",
+        "check_2fa": ADMIN_PREFIX + "/check_2fa",
         "register": ADMIN_PREFIX + "/register",
         "reset_password": ADMIN_PREFIX + "/password_reset",
         "reset_password_confirmation": ADMIN_PREFIX + "/password_reset_confirmation",

From 0986ef02a25061dda5d9706ecaeb16f32d8678e9 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 16 Apr 2024 13:14:47 +0200
Subject: [PATCH 051/150] 2fa: recovery codes copy

---
 .../admin/my_account/my_account_2fa.mako      | 61 +++++++------------
 1 file changed, 22 insertions(+), 39 deletions(-)

diff --git a/rhodecode/templates/admin/my_account/my_account_2fa.mako b/rhodecode/templates/admin/my_account/my_account_2fa.mako
index 36c03027..52848a97 100644
--- a/rhodecode/templates/admin/my_account/my_account_2fa.mako
+++ b/rhodecode/templates/admin/my_account/my_account_2fa.mako
@@ -31,14 +31,9 @@
             </div>
             <button id="saveBtn" class="btn btn-primary" ${'disabled' if c.locked_2fa else ''}>${_('Save')}</button>
         </div>
-        <div id="codesPopup" class="modal">
-            <div class="modal-content">
-                <ul id="recoveryCodesList"></ul>
-                <button id="copyAllBtn" class="btn btn-primary">Copy All</button>
-            </div>
-        </div>
     </div>
 </div>
+
 % if c.state_of_2fa:
 <div class="panel panel-default">
     <div class="panel-heading">
@@ -54,22 +49,19 @@
     </div>
 
 </div>
+
 % endif
+
 <script>
     function submitForm() {
         let formData = new FormData(document.getElementById("2faForm"));
         let xhr = new XMLHttpRequest();
+
         let success = function (response) {
             let recovery_codes = response.recovery_codes;
-            const codesList = document.getElementById("recoveryCodesList");
-
-            codesList.innerHTML = "";
-            recovery_codes.forEach(code => {
-                const listItem = document.createElement("li");
-                listItem.textContent = code;
-                codesList.appendChild(listItem);
-            });
+            showRecoveryCodesPopup(recovery_codes);
         }
+
         xhr.onreadystatechange = function () {
             if (xhr.readyState == 4 && xhr.status == 200) {
                 let responseDoc = new DOMParser().parseFromString(xhr.responseText, "text/html");
@@ -79,7 +71,6 @@
                 } else {
                     let regenerate_url = pyroutes.url('my_account_regenerate_2fa_recovery_codes');
                     ajaxPOST(regenerate_url, {'csrf_token': CSRF_TOKEN}, success);
-                    showRecoveryCodesPopup();
                 }
             }
         };
@@ -87,8 +78,7 @@
         xhr.open("POST", url, true);
         xhr.send(formData);
     }
-</script>
-<script>
+
     document.getElementById('2faEnabled').addEventListener('click', function () {
         document.getElementById('2faDisabled').checked = false;
     });
@@ -109,32 +99,25 @@
         let post_data = {'state': state, 'csrf_token': CSRF_TOKEN};
         let url = pyroutes.url('my_account_configure_2fa');
 
-        ajaxPOST(url, post_data, null)
-    };
+        ajaxPOST(url, post_data, function(){}, function(){})
+    }
+
     document.getElementById('saveBtn').addEventListener('click', function () {
         var state = getStateValue();
         saveChanges(state);
     });
-</script>
-<script>
-    function showRecoveryCodesPopup() {
-        const popup = document.getElementById("codesPopup");
-        popup.style.display = "block";
+
+    function showRecoveryCodesPopup(recoveryCodes) {
+        let funcData = {'recoveryCodes': recoveryCodes}
+        let recoveryCodesHtml = renderTemplate('recoveryCodes', funcData)
+
+        SwalNoAnimation.fire({
+            allowOutsideClick: false,
+            confirmButtonText: _gettext('I Copied the codes'),
+            title: _gettext('2FA Recovery Codes'),
+            html: recoveryCodesHtml
+        })
+
     }
 
-    document.getElementById("copyAllBtn").addEventListener("click", function () {
-        const codesListItems = document.querySelectorAll("#recoveryCodesList li");
-        const allCodes = Array.from(codesListItems).map(item => item.textContent).join(", ");
-
-        const textarea = document.createElement('textarea');
-        textarea.value = allCodes;
-        document.body.appendChild(textarea);
-
-        textarea.select();
-        document.execCommand('copy');
-
-        document.body.removeChild(textarea);
-        const popup = document.getElementById("codesPopup");
-        popup.style.display = ""
-    });
 </script>

From 518dd69772ef65c84e117d2728d9b2f78443b6d9 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 16 Apr 2024 14:30:49 +0200
Subject: [PATCH 052/150] fix: added missing templates

---
 rhodecode/templates/ejs_templates/templates.html | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/rhodecode/templates/ejs_templates/templates.html b/rhodecode/templates/ejs_templates/templates.html
index eb16eeb0..636ed6f6 100644
--- a/rhodecode/templates/ejs_templates/templates.html
+++ b/rhodecode/templates/ejs_templates/templates.html
@@ -269,6 +269,18 @@ They are permanent until deleted, or converted to regular comments.<br/>
     <pre><%= submodule_url %></pre>
 </script>
 
+<script id="ejs_recoveryCodes" type="text/template" class="ejsTemplate">
+<code>
+    <ol>
+          <% for (var i = 0; i < recoveryCodes.length; i++) { %>
+              <% var code = recoveryCodes[i] %>
+               <li><%= code %></li>
+          <% } %>
+    </ol>
+</code>
+<i class="icon-clipboard clipboard-action" data-clipboard-text="<%= recoveryCodes %>" >Copy All</i>
+</script>
+
 ##// END OF EJS Templates
 </div>
 

From 70a5f445784c25f32ab51775c34813227171ea6f Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 22 Apr 2024 11:32:01 +0200
Subject: [PATCH 053/150] fix(encryptor): use a failsafe mechanism of detecting
 old algo for encryption to NOT crash the app when switching to fernet

---
 rhodecode/lib/enc_utils.py |  6 +++---
 rhodecode/lib/encrypt2.py  | 13 +++++++++++++
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/rhodecode/lib/enc_utils.py b/rhodecode/lib/enc_utils.py
index c472f9ea..4280f507 100644
--- a/rhodecode/lib/enc_utils.py
+++ b/rhodecode/lib/enc_utils.py
@@ -30,15 +30,15 @@ def encrypt_value(value: bytes, enc_key: bytes, algo: str = ''):
 
 
 def decrypt_value(value: bytes, enc_key: bytes, algo: str = '', strict_mode: bool = False):
+    enc_key = safe_bytes(enc_key)
+    value = safe_bytes(value)
 
     if not algo:
         # not explicit algo, just use what's set by config
-        algo = get_default_algo()
+        algo = Encryptor.detect_enc_algo(value) or get_default_algo()
     if algo not in ALLOWED_ALGOS:
         ValueError(f'Bad encryption algorithm, should be {ALLOWED_ALGOS}, got: {algo}')
 
-    enc_key = safe_bytes(enc_key)
-    value = safe_bytes(value)
     safe = not strict_mode
 
     if algo == 'aes':
diff --git a/rhodecode/lib/encrypt2.py b/rhodecode/lib/encrypt2.py
index c1e7e963..1c6832b0 100644
--- a/rhodecode/lib/encrypt2.py
+++ b/rhodecode/lib/encrypt2.py
@@ -23,8 +23,21 @@ class InvalidDecryptedValue(str):
 
 class Encryptor(object):
     key_format = b'enc2$salt:{1}$data:{2}'
+
     pref_len = 5  # salt:, data:
 
+    @classmethod
+    def detect_enc_algo(cls, enc_data: bytes):
+        parts = enc_data.split(b'$', 3)
+        if len(parts) != 3:
+            raise ValueError(f'Encrypted Data has invalid format, expected {cls.key_format}, got {parts}')
+
+        if b'enc$aes_hmac$' in enc_data:
+            return 'aes'
+        elif b'enc2$salt' in enc_data:
+            return 'fernet'
+        return None
+
     def __init__(self, enc_key: bytes):
         self.enc_key = enc_key
 

From 12af6697b87c64b58d172b111afd892d3da37489 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 22 Apr 2024 14:54:33 +0200
Subject: [PATCH 054/150] shell: remove module level imports

---
 rhodecode/lib/paster_commands/ishell.py | 9 +++++++--
 rhodecode/lib/pyramid_shell/__init__.py | 8 +++++++-
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/rhodecode/lib/paster_commands/ishell.py b/rhodecode/lib/paster_commands/ishell.py
index c680736a..b1714074 100644
--- a/rhodecode/lib/paster_commands/ishell.py
+++ b/rhodecode/lib/paster_commands/ishell.py
@@ -54,8 +54,13 @@ class Command(BasePasterCommand):
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Interactive shell"
 
+    def import_all_from_module(self, module_name):
+        import importlib
+        module = importlib.import_module(module_name)
+        globals().update({k: v for k, v in module.__dict__.items() if not k.startswith('_')})
+
     def command(self):
-        #get SqlAlchemy session
+        # get SqlAlchemy session
         self._init_session()
 
         # imports, used in ipython shell
@@ -64,7 +69,7 @@ class Command(BasePasterCommand):
         import time
         import shutil
         import datetime
-        from rhodecode.model.db import *
+        self.import_all_from_module('rhodecode.model.db')
 
         try:
             from IPython import embed
diff --git a/rhodecode/lib/pyramid_shell/__init__.py b/rhodecode/lib/pyramid_shell/__init__.py
index e7ebac6e..6ab59ad0 100644
--- a/rhodecode/lib/pyramid_shell/__init__.py
+++ b/rhodecode/lib/pyramid_shell/__init__.py
@@ -41,6 +41,12 @@ or reset some user/system settings.
 """
 
 
+def import_all_from_module(module_name):
+    import importlib
+    module = importlib.import_module(module_name)
+    globals().update({k: v for k, v in module.__dict__.items() if not k.startswith('_')})
+
+
 def ipython_shell_runner(env, help):
 
     # imports, used in ipython shell
@@ -50,7 +56,7 @@ def ipython_shell_runner(env, help):
     import shutil
     import datetime
     from rhodecode.model import user, user_group, repo, repo_group
-    from rhodecode.model.db import *
+    import_all_from_module('rhodecode.model.db')
 
     try:
         import IPython

From cd9fe75554d06aeb9b16631d44ac3c7be89614b8 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 23 Apr 2024 15:25:21 +0200
Subject: [PATCH 055/150] fix(caching): fixed problems with Cache query for
 users.

The old way of querying caused the user get query to be always cached, and returning old results even in 2fa forms.
The new limited query doesn't cache the user object resolving issues
---
 rhodecode/apps/my_account/views/my_account.py | 13 ++++--
 rhodecode/model/db.py                         | 45 ++++++++++---------
 rhodecode/model/user.py                       | 11 +++--
 .../my_account/my_account_bookmarks.mako      |  4 +-
 rhodecode/templates/base/base.mako            | 14 +++---
 5 files changed, 52 insertions(+), 35 deletions(-)

diff --git a/rhodecode/apps/my_account/views/my_account.py b/rhodecode/apps/my_account/views/my_account.py
index f17088c9..254be2ba 100644
--- a/rhodecode/apps/my_account/views/my_account.py
+++ b/rhodecode/apps/my_account/views/my_account.py
@@ -40,7 +40,7 @@ from rhodecode.lib.utils2 import safe_int, md5, str2bool
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
-    IntegrityError, or_, in_filter_generator,
+    IntegrityError, or_, in_filter_generator, select,
     Repository, UserEmailMap, UserApiKeys, UserFollowing,
     PullRequest, UserBookmark, RepoGroup, ChangesetStatus)
 from rhodecode.model.meta import Session
@@ -511,8 +511,15 @@ class MyAccountView(BaseAppView, DataGridAppView):
     def my_account_bookmarks(self):
         c = self.load_default_context()
         c.active = 'bookmarks'
-        c.bookmark_items = UserBookmark.get_bookmarks_for_user(
-            self._rhodecode_db_user.user_id, cache=False)
+
+        user_bookmarks = \
+            select(UserBookmark, Repository, RepoGroup) \
+            .where(UserBookmark.user_id == self._rhodecode_user.user_id) \
+            .outerjoin(Repository, Repository.repo_id == UserBookmark.bookmark_repo_id) \
+            .outerjoin(RepoGroup, RepoGroup.group_id == UserBookmark.bookmark_repo_group_id) \
+            .order_by(UserBookmark.position.asc())
+
+        c.user_bookmark_items = Session().execute(user_bookmarks).all()
         return self._get_template_context(c)
 
     def _process_bookmark_entry(self, entry, user_id):
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index c7330451..e479f99a 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -884,17 +884,6 @@ class User(Base, BaseModel):
         Session().commit()
         return new_recovery_codes
 
-    @classmethod
-    def get(cls, user_id, cache=False):
-        if not user_id:
-            return
-
-        user = cls.query()
-        if cache:
-            user = user.options(
-                FromCache("sql_cache_short", f"get_users_{user_id}"))
-        return user.get(user_id)
-
     @classmethod
     def extra_valid_auth_tokens(cls, user, role=None):
         tokens = UserApiKeys.query().filter(UserApiKeys.user == user)\
@@ -1021,12 +1010,23 @@ class User(Base, BaseModel):
     @user_data.setter
     def user_data(self, val):
         if not isinstance(val, dict):
-            raise Exception('user_data must be dict, got %s' % type(val))
+            raise Exception(f'user_data must be dict, got {type(val)}')
         try:
             self._user_data = safe_bytes(json.dumps(val))
         except Exception:
             log.error(traceback.format_exc())
 
+    @classmethod
+    def get(cls, user_id, cache=False):
+        if not user_id:
+            return
+
+        user = cls.query()
+        if cache:
+            user = user.options(
+                FromCache("sql_cache_short", f"get_users_{user_id}"))
+        return user.get(user_id)
+
     @classmethod
     def get_by_username(cls, username, case_insensitive=False,
                         cache=False):
@@ -1611,7 +1611,7 @@ class UserGroup(Base, BaseModel):
         user_group = cls.query()
         if cache:
             user_group = user_group.options(
-                FromCache("sql_cache_short", "get_users_group_%s" % user_group_id))
+                FromCache("sql_cache_short", f"get_users_group_{user_group_id}"))
         return user_group.get(user_group_id)
 
     def permissions(self, with_admins=True, with_owner=True,
@@ -2005,7 +2005,7 @@ class Repository(Base, BaseModel):
                 if val:
                     return val
             else:
-                cache_key = "get_repo_by_name_%s" % _hash_key(repo_name)
+                cache_key = f"get_repo_by_name_{_hash_key(repo_name)}"
                 q = q.options(
                     FromCache("sql_cache_short", cache_key))
 
@@ -5643,18 +5643,23 @@ class UserBookmark(Base, BaseModel):
 
     @classmethod
     def get_bookmarks_for_user(cls, user_id, cache=True):
-        bookmarks = cls.query() \
-            .filter(UserBookmark.user_id == user_id) \
-            .options(joinedload(UserBookmark.repository)) \
-            .options(joinedload(UserBookmark.repository_group)) \
+        bookmarks = select(
+            UserBookmark.title,
+            UserBookmark.position,
+            ) \
+            .add_columns(Repository.repo_id, Repository.repo_type, Repository.repo_name) \
+            .add_columns(RepoGroup.group_id, RepoGroup.group_name) \
+            .where(UserBookmark.user_id == user_id) \
+            .outerjoin(Repository, Repository.repo_id == UserBookmark.bookmark_repo_id) \
+            .outerjoin(RepoGroup, RepoGroup.group_id == UserBookmark.bookmark_repo_group_id) \
             .order_by(UserBookmark.position.asc())
 
         if cache:
             bookmarks = bookmarks.options(
-                FromCache("sql_cache_short", "get_user_{}_bookmarks".format(user_id))
+                FromCache("sql_cache_short", f"get_user_{user_id}_bookmarks")
             )
 
-        return bookmarks.all()
+        return Session().execute(bookmarks).all()
 
     def __repr__(self):
         return f'<UserBookmark({self.position} @ {self.redirect_url!r})>'
diff --git a/rhodecode/model/user.py b/rhodecode/model/user.py
index 18e33254..69af24c0 100644
--- a/rhodecode/model/user.py
+++ b/rhodecode/model/user.py
@@ -114,6 +114,7 @@ class UserModel(BaseModel):
         else:
             user = self.sa.query(User)\
                 .filter(User.username == username)
+
         if cache:
             name_key = _hash_key(username)
             user = user.options(
@@ -765,25 +766,29 @@ class UserModel(BaseModel):
             'AuthUser: fill data execution based on: '
             'user_id:%s api_key:%s username:%s', user_id, api_key, username)
         try:
+            found_with = ''
             dbuser = None
             if user_id:
                 dbuser = self.get(user_id)
+                found_with = 'user_id'
             elif api_key:
                 dbuser = self.get_by_auth_token(api_key)
+                found_with = 'auth_token'
             elif username:
                 dbuser = self.get_by_username(username)
+                found_with = 'username'
 
             if not dbuser:
                 log.warning(
-                    'Unable to lookup user by id:%s api_key:%s username:%s',
-                    user_id, token_obfuscate(api_key), username)
+                    'Unable to lookup user by id:%s api_key:%s username:%s, found with: %s',
+                    user_id, token_obfuscate(api_key), username, found_with)
                 return False
             if not dbuser.active:
                 log.debug('User `%s:%s` is inactive, skipping fill data',
                           username, user_id)
                 return False
 
-            log.debug('AuthUser: filling found user:%s data', dbuser)
+            log.debug('AuthUser: filling found user:%s data, found with: %s', dbuser, found_with)
 
             attrs = {
                 'user_id': dbuser.user_id,
diff --git a/rhodecode/templates/admin/my_account/my_account_bookmarks.mako b/rhodecode/templates/admin/my_account/my_account_bookmarks.mako
index 5e0a6e8b..559d8593 100644
--- a/rhodecode/templates/admin/my_account/my_account_bookmarks.mako
+++ b/rhodecode/templates/admin/my_account/my_account_bookmarks.mako
@@ -83,14 +83,14 @@
         <table class="rctable">
         ## generate always 10 entries
         <input type="hidden" name="__start__" value="bookmarks:sequence"/>
-        % for item in (c.bookmark_items + [None for i in range(10)])[:10]:
+        % for item in (c.user_bookmark_items + [None for i in range(10)])[:10]:
             <input type="hidden" name="__start__" value="bookmark:mapping"/>
             % if item is None:
                 ## empty placehodlder
                 ${form_item()}
             % else:
                 ## actual entry
-                ${form_item(position=item.position, title=item.title, redirect_url=item.redirect_url, repo=item.repository, repo_group=item.repository_group)}
+                ${form_item(position=item[0].position, title=item[0].title, redirect_url=item[0].redirect_url, repo=item[1], repo_group=item[2])}
             % endif
             <input type="hidden" name="__end__" value="bookmark:mapping"/>
         % endfor
diff --git a/rhodecode/templates/base/base.mako b/rhodecode/templates/base/base.mako
index 3b81decc..51a7a0c7 100644
--- a/rhodecode/templates/base/base.mako
+++ b/rhodecode/templates/base/base.mako
@@ -651,26 +651,26 @@
                       % endif
                       % for item in c.bookmark_items:
                       <li>
-                          % if item.repository:
+                          % if item.repo_id:
                               <div>
                                 <a class="bookmark-item" href="${h.route_path('my_account_goto_bookmark', bookmark_id=item.position)}">
                                 <code>${item.position}</code>
-                                % if item.repository.repo_type == 'hg':
+                                % if item.repo_type == 'hg':
                                     <i class="icon-hg" title="${_('Repository')}" style="font-size: 16px"></i>
-                                % elif item.repository.repo_type == 'git':
+                                % elif item.repo_type == 'git':
                                     <i class="icon-git" title="${_('Repository')}" style="font-size: 16px"></i>
-                                % elif item.repository.repo_type == 'svn':
+                                % elif item.repo_type == 'svn':
                                     <i class="icon-svn" title="${_('Repository')}" style="font-size: 16px"></i>
                                 % endif
-                                ${(item.title or h.shorter(item.repository.repo_name, 30))}
+                                ${(item.title or h.shorter(item.repo_name, 30))}
                               </a>
                               </div>
-                          % elif item.repository_group:
+                          % elif item.group_id:
                               <div>
                                 <a class="bookmark-item" href="${h.route_path('my_account_goto_bookmark', bookmark_id=item.position)}">
                                 <code>${item.position}</code>
                                 <i class="icon-repo-group" title="${_('Repository group')}" style="font-size: 14px"></i>
-                                ${(item.title or h.shorter(item.repository_group.group_name, 30))}
+                                ${(item.title or h.shorter(item.group_name, 30))}
                               </a>
                               </div>
                           % else:

From 2fcf5e8c05ff6f3bbb74505d3630c44389e9071d Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 23 Apr 2024 15:26:01 +0200
Subject: [PATCH 056/150] fix(gists): fixed old way of querying for gist store

---
 rhodecode/model/db.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index e479f99a..a3bc23c0 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -4997,10 +4997,9 @@ class Gist(Base, BaseModel):
         :param cls:
         """
         from rhodecode.model.gist import GIST_STORE_LOC
-        q = Session().query(RhodeCodeUi)\
-            .filter(RhodeCodeUi.ui_key == URL_SEP)
-        q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
-        return os.path.join(q.one().ui_value, GIST_STORE_LOC)
+        from rhodecode.lib.utils import get_rhodecode_repo_store_path
+        repo_store_path = get_rhodecode_repo_store_path()
+        return os.path.join(repo_store_path, GIST_STORE_LOC)
 
     def get_api_data(self):
         """

From 337031c7a49c46f7bf0e948b0a087815f45e06d7 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 24 Apr 2024 09:45:36 +0200
Subject: [PATCH 057/150] feat(2fa): refactor logic arround
 validation/recoverycodes and workflows of configuration of 2fa

- recovery codes are shown in 1 place only
- save status about view of recovery codes
- made the logic of saving states into user_data more explicit and no longer relly on hacky DB transaction logic
- turn JS forms into a regular forms
---
 rhodecode/apps/_base/__init__.py              |   6 +-
 rhodecode/apps/login/tests/test_2fa.py        |   6 +-
 rhodecode/apps/login/views.py                 |  39 ++--
 rhodecode/apps/my_account/__init__.py         |  25 ++-
 rhodecode/apps/my_account/views/my_account.py |  56 +++++-
 rhodecode/model/db.py                         | 127 +++++++++----
 rhodecode/model/forms.py                      |   7 +-
 rhodecode/public/js/rhodecode/routes.js       |   9 +-
 .../admin/my_account/my_account_2fa.mako      | 167 ++++++++++--------
 rhodecode/templates/configure_2fa.mako        | 112 +++---------
 rhodecode/templates/verify_2fa.mako           |  69 +++++---
 11 files changed, 353 insertions(+), 270 deletions(-)

diff --git a/rhodecode/apps/_base/__init__.py b/rhodecode/apps/_base/__init__.py
index e500e6eb..2a4f7e68 100644
--- a/rhodecode/apps/_base/__init__.py
+++ b/rhodecode/apps/_base/__init__.py
@@ -178,9 +178,7 @@ class BaseAppView(object):
         if user_obj.has_forced_2fa and user_obj.extern_type != 'rhodecode':
             return
 
-        if (user_obj.has_enabled_2fa
-            and not self.user_data.get('secret_2fa')) \
-                and view_name != self.SETUP_2FA_VIEW:
+        if user_obj.needs_2fa_configure and view_name != self.SETUP_2FA_VIEW:
             h.flash(
                 "You are required to configure 2FA",
                 "warning",
@@ -195,7 +193,7 @@ class BaseAppView(object):
         if not user_obj:
             return
 
-        if self.user_data.get('check_2fa') and view_name != self.VERIFY_2FA_VIEW:
+        if user_obj.has_check_2fa_flag and view_name != self.VERIFY_2FA_VIEW:
             raise HTTPFound(self.request.route_path(self.VERIFY_2FA_VIEW))
 
     def _log_creation_exception(self, e, repo_name):
diff --git a/rhodecode/apps/login/tests/test_2fa.py b/rhodecode/apps/login/tests/test_2fa.py
index 7a2ef2f3..024a8148 100644
--- a/rhodecode/apps/login/tests/test_2fa.py
+++ b/rhodecode/apps/login/tests/test_2fa.py
@@ -33,7 +33,7 @@ class Test2FA(object):
     def test_redirect_to_2fa_check_if_2fa_configured(self, user_util):
         user = user_util.create_user(password=self.password)
         user.has_enabled_2fa = True
-        user.secret_2fa
+        user.init_secret_2fa()
         Session().add(user)
         Session().commit()
         self.app.post(
@@ -47,8 +47,8 @@ class Test2FA(object):
     def test_2fa_recovery_codes_works_only_once(self, user_util):
         user = user_util.create_user(password=self.password)
         user.has_enabled_2fa = True
-        user.secret_2fa
-        recovery_cod_to_check = user.get_2fa_recovery_codes()[0]
+        user.init_secret_2fa()
+        recovery_cod_to_check = user.init_2fa_recovery_codes()[0]
         Session().add(user)
         Session().commit()
         self.app.post(
diff --git a/rhodecode/apps/login/views.py b/rhodecode/apps/login/views.py
index e99deeaf..6d96a72b 100644
--- a/rhodecode/apps/login/views.py
+++ b/rhodecode/apps/login/views.py
@@ -188,7 +188,8 @@ class LoginView(BaseAppView):
             # form checks for username/password, now we're authenticated
             username = form_result['username']
             if (user := User.get_by_username_or_primary_email(username)).has_enabled_2fa:
-                user.update_userdata(check_2fa=True)
+                user.has_check_2fa_flag = True
+
             headers = store_user_in_session(
                 self.session,
                 user_identifier=username,
@@ -489,23 +490,32 @@ class LoginView(BaseAppView):
         form = TOTPForm(_, user_instance)()
         render_ctx = {}
         if self.request.method == 'POST':
+            post_items = dict(self.request.POST)
+
             try:
-                form.to_python(dict(self.request.POST))
+                form_details = form.to_python(post_items)
+                secret = form_details['secret_totp']
+
+                user_instance.init_2fa_recovery_codes(persist=True, force=True)
+                user_instance.set_2fa_secret(secret)
+
                 Session().commit()
-                raise HTTPFound(c.came_from)
+                raise HTTPFound(self.request.route_path('my_account_enable_2fa', _query={'show-recovery-codes': 1}))
             except formencode.Invalid as errors:
                 defaults = errors.value
                 render_ctx = {
                     'errors': errors.error_dict,
                     'defaults': defaults,
                 }
+
+        # NOTE: here we DO NOT persist the secret 2FA, since this is only for setup, once a setup is completed
+        # only then we should persist it
+        secret = user_instance.init_secret_2fa(persist=False)
+
+        totp_name = f'RhodeCode token ({self.request.user.username})'
+
         qr = qrcode.QRCode(version=1, box_size=10, border=5)
-        secret = user_instance.secret_2fa
-        Session().flush()
-        recovery_codes = user_instance.get_2fa_recovery_codes()
-        Session().commit()
-        qr.add_data(pyotp.totp.TOTP(secret).provisioning_uri(
-            name=self.request.user.name))
+        qr.add_data(pyotp.totp.TOTP(secret).provisioning_uri(name=totp_name))
         qr.make(fit=True)
         img = qr.make_image(fill_color='black', back_color='white')
         buffered = BytesIO()
@@ -513,8 +523,8 @@ class LoginView(BaseAppView):
         return self._get_template_context(
             c,
             qr=b64encode(buffered.getvalue()).decode("utf-8"),
-            key=secret, recovery_codes=json.dumps(recovery_codes),
-            codes_viewed=not bool(recovery_codes),
+            key=secret,
+            totp_name=totp_name,
             ** render_ctx
         )
 
@@ -527,9 +537,12 @@ class LoginView(BaseAppView):
         user_instance = self._rhodecode_db_user
         totp_form = TOTPForm(_, user_instance, allow_recovery_code_use=True)()
         if self.request.method == 'POST':
+            post_items = dict(self.request.POST)
+            # NOTE: inject secret, as it's a post configured saved item.
+            post_items['secret_totp'] = user_instance.get_secret_2fa()
             try:
-                totp_form.to_python(dict(self.request.POST))
-                user_instance.update_userdata(check_2fa=False)
+                totp_form.to_python(post_items)
+                user_instance.has_check_2fa_flag = False
                 Session().commit()
                 raise HTTPFound(c.came_from)
             except formencode.Invalid as errors:
diff --git a/rhodecode/apps/my_account/__init__.py b/rhodecode/apps/my_account/__init__.py
index e8dd97cf..982871a4 100644
--- a/rhodecode/apps/my_account/__init__.py
+++ b/rhodecode/apps/my_account/__init__.py
@@ -83,24 +83,35 @@ def includeme(config):
         attr='my_account_2fa',
         route_name='my_account_enable_2fa', request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
-
+    # my account 2fa save
     config.add_route(
-        name='my_account_configure_2fa',
-        pattern=ADMIN_PREFIX + '/my_account/configure_2fa')
+        name='my_account_enable_2fa_save',
+        pattern=ADMIN_PREFIX + '/my_account/enable_2fa_save')
     config.add_view(
         MyAccountView,
-        attr='my_account_2fa_configure',
-        route_name='my_account_configure_2fa', request_method='POST', xhr=True,
+        attr='my_account_2fa_update',
+        route_name='my_account_enable_2fa_save', request_method='POST',
+        renderer='rhodecode:templates/admin/my_account/my_account.mako')
+
+    # my account 2fa recovery code-reset
+    config.add_route(
+        name='my_account_show_2fa_recovery_codes',
+        pattern=ADMIN_PREFIX + '/my_account/recovery_codes')
+    config.add_view(
+        MyAccountView,
+        attr='my_account_2fa_show_recovery_codes',
+        route_name='my_account_show_2fa_recovery_codes', request_method='POST', xhr=True,
         renderer='json_ext')
 
+    # my account 2fa recovery code-reset
     config.add_route(
         name='my_account_regenerate_2fa_recovery_codes',
         pattern=ADMIN_PREFIX + '/my_account/regenerate_recovery_codes')
     config.add_view(
         MyAccountView,
         attr='my_account_2fa_regenerate_recovery_codes',
-        route_name='my_account_regenerate_2fa_recovery_codes', request_method='POST', xhr=True,
-        renderer='json_ext')
+        route_name='my_account_regenerate_2fa_recovery_codes', request_method='POST',
+        renderer='rhodecode:templates/admin/my_account/my_account.mako')
 
     # my account tokens
     config.add_route(
diff --git a/rhodecode/apps/my_account/views/my_account.py b/rhodecode/apps/my_account/views/my_account.py
index 254be2ba..ac3df097 100644
--- a/rhodecode/apps/my_account/views/my_account.py
+++ b/rhodecode/apps/my_account/views/my_account.py
@@ -16,6 +16,7 @@
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
+import time
 import logging
 import datetime
 import string
@@ -43,6 +44,7 @@ from rhodecode.model.db import (
     IntegrityError, or_, in_filter_generator, select,
     Repository, UserEmailMap, UserApiKeys, UserFollowing,
     PullRequest, UserBookmark, RepoGroup, ChangesetStatus)
+from rhodecode.model.forms import TOTPForm
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.user import UserModel
@@ -207,27 +209,65 @@ class MyAccountView(BaseAppView, DataGridAppView):
     def my_account_2fa(self):
         _ = self.request.translate
         c = self.load_default_context()
-        c.active = '2fa'
-        from rhodecode.model.settings import SettingsModel
-        user_instance = self._rhodecode_db_user
+        c.active = '2FA'
+        user_instance = c.auth_user.get_instance()
         locked_by_admin = user_instance.has_forced_2fa
         c.state_of_2fa = user_instance.has_enabled_2fa
+        c.user_seen_2fa_recovery_codes = user_instance.has_seen_2fa_codes
         c.locked_2fa = str2bool(locked_by_admin)
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
-    def my_account_2fa_configure(self):
-        state = self.request.POST.get('state')
-        self._rhodecode_db_user.has_enabled_2fa = state
-        return {'state_of_2fa': state}
+    def my_account_2fa_update(self):
+        _ = self.request.translate
+        c = self.load_default_context()
+        c.active = '2FA'
+        user_instance = c.auth_user.get_instance()
+
+        state = self.request.POST.get('2fa_status') == '1'
+        user_instance.has_enabled_2fa = state
+        user_instance.update_userdata(update_2fa=time.time())
+        Session().commit()
+        h.flash(_("Successfully saved 2FA settings"), category='success')
+        raise HTTPFound(self.request.route_path('my_account_enable_2fa'))
+
+    @LoginRequired()
+    @NotAnonymous()
+    @CSRFRequired()
+    def my_account_2fa_show_recovery_codes(self):
+        c = self.load_default_context()
+        user_instance = c.auth_user.get_instance()
+        user_instance.has_seen_2fa_codes = True
+        Session().commit()
+        return {'recovery_codes': user_instance.get_2fa_recovery_codes()}
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def my_account_2fa_regenerate_recovery_codes(self):
-        return {'recovery_codes': self._rhodecode_db_user.regenerate_2fa_recovery_codes()}
+        _ = self.request.translate
+        c = self.load_default_context()
+        user_instance = c.auth_user.get_instance()
+
+        totp_form = TOTPForm(_, user_instance, allow_recovery_code_use=True)()
+
+        post_items = dict(self.request.POST)
+        # NOTE: inject secret, as it's a post configured saved item.
+        post_items['secret_totp'] = user_instance.get_secret_2fa()
+        try:
+            totp_form.to_python(post_items)
+            user_instance.regenerate_2fa_recovery_codes()
+            Session().commit()
+        except formencode.Invalid as errors:
+            h.flash(_("Failed to generate new recovery codes: {}").format(errors), category='error')
+            raise HTTPFound(self.request.route_path('my_account_enable_2fa'))
+        except Exception as e:
+            h.flash(_("Failed to generate new recovery codes: {}").format(e), category='error')
+            raise HTTPFound(self.request.route_path('my_account_enable_2fa'))
+
+        raise HTTPFound(self.request.route_path('my_account_enable_2fa', _query={'show-recovery-codes': 1}))
 
     @LoginRequired()
     @NotAnonymous()
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index a3bc23c0..9ea58d58 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -796,34 +796,13 @@ class User(Base, BaseModel):
             Session.commit()
             return artifact_token.api_key
 
-    @hybrid_property
-    def secret_2fa(self):
-        if not self.user_data.get('secret_2fa'):
-            secret = pyotp.random_base32()
-            self.update_userdata(secret_2fa=safe_str(enc_utils.encrypt_value(secret, enc_key=ENCRYPTION_KEY)))
-            return secret
-        return safe_str(
-            enc_utils.decrypt_value(self.user_data['secret_2fa'],
-                                    enc_key=ENCRYPTION_KEY,
-                                    strict_mode=ConfigGet().get_bool('rhodecode.encrypted_values.strict',
-                                                                               missing=True)
-                                    )
-        )
-
-    def is_totp_valid(self, received_code):
-        totp = pyotp.TOTP(self.secret_2fa)
+    def is_totp_valid(self, received_code, secret):
+        totp = pyotp.TOTP(secret)
         return totp.verify(received_code)
 
-    def is_2fa_recovery_code_valid(self, received_code):
+    def is_2fa_recovery_code_valid(self, received_code, secret):
         encrypted_recovery_codes = self.user_data.get('recovery_codes_2fa', [])
-        recovery_codes = list(map(
-            lambda x: safe_str(
-                enc_utils.decrypt_value(
-                    x,
-                    enc_key=ENCRYPTION_KEY,
-                    strict_mode=ConfigGet().get_bool('rhodecode.encrypted_values.strict', missing=True)
-                )),
-            encrypted_recovery_codes))
+        recovery_codes = self.get_2fa_recovery_codes()
         if received_code in recovery_codes:
             encrypted_recovery_codes.pop(recovery_codes.index(received_code))
             self.update_userdata(recovery_codes_2fa=encrypted_recovery_codes)
@@ -844,7 +823,7 @@ class User(Base, BaseModel):
     @hybrid_property
     def has_enabled_2fa(self):
         """
-        Checks if 2fa was enabled by user
+        Checks if user enabled 2fa
         """
         if value := self.has_forced_2fa:
             return value
@@ -853,34 +832,109 @@ class User(Base, BaseModel):
     @has_enabled_2fa.setter
     def has_enabled_2fa(self, val):
         val = str2bool(val)
-        self.update_userdata(enabled_2fa=str2bool(val))
+        self.update_userdata(enabled_2fa=val)
         if not val:
-            self.update_userdata(secret_2fa=None, recovery_codes_2fa=[])
+            # NOTE: setting to false we clear the user_data to not store any 2fa artifacts
+            self.update_userdata(secret_2fa=None, recovery_codes_2fa=[], check_2fa=False)
         Session().commit()
 
-    def get_2fa_recovery_codes(self):
+    @hybrid_property
+    def has_check_2fa_flag(self):
+        """
+        Check if check 2fa flag is set for this user
+        """
+        value = self.user_data.get('check_2fa', False)
+        return value
+
+    @has_check_2fa_flag.setter
+    def has_check_2fa_flag(self, val):
+        val = str2bool(val)
+        self.update_userdata(check_2fa=val)
+        Session().commit()
+
+    @hybrid_property
+    def has_seen_2fa_codes(self):
+        """
+        get the flag about if user has seen 2fa recovery codes
+        """
+        value = self.user_data.get('recovery_codes_2fa_seen', False)
+        return value
+
+    @has_seen_2fa_codes.setter
+    def has_seen_2fa_codes(self, val):
+        val = str2bool(val)
+        self.update_userdata(recovery_codes_2fa_seen=val)
+        Session().commit()
+
+    @hybrid_property
+    def needs_2fa_configure(self):
+        """
+        Determines if setup2fa has completed for this user. Means he has all needed data for 2fa to work.
+
+        Currently this is 2fa enabled and secret exists
+        """
+        if self.has_enabled_2fa:
+            return not self.user_data.get('secret_2fa')
+        return False
+
+    def init_2fa_recovery_codes(self, persist=True, force=False):
         """
         Creates 2fa recovery codes
         """
         recovery_codes = self.user_data.get('recovery_codes_2fa', [])
         encrypted_codes = []
-        if not recovery_codes:
+        if not recovery_codes or force:
             for _ in range(self.RECOVERY_CODES_COUNT):
                 recovery_code = pyotp.random_base32()
                 recovery_codes.append(recovery_code)
-                encrypted_codes.append(safe_str(enc_utils.encrypt_value(recovery_code, enc_key=ENCRYPTION_KEY)))
-            self.update_userdata(recovery_codes_2fa=encrypted_codes)
+                encrypted_code = enc_utils.encrypt_value(safe_bytes(recovery_code), enc_key=ENCRYPTION_KEY)
+                encrypted_codes.append(safe_str(encrypted_code))
+            if persist:
+                self.update_userdata(recovery_codes_2fa=encrypted_codes, recovery_codes_2fa_seen=False)
             return recovery_codes
         # User should not check the same recovery codes more than once
         return []
 
+    def get_2fa_recovery_codes(self):
+        encrypted_recovery_codes = self.user_data.get('recovery_codes_2fa', [])
+        strict_mode = ConfigGet().get_bool('rhodecode.encrypted_values.strict', missing=True)
+
+        recovery_codes = list(map(
+            lambda val: safe_str(
+                enc_utils.decrypt_value(
+                    val,
+                    enc_key=ENCRYPTION_KEY,
+                    strict_mode=strict_mode
+                )),
+            encrypted_recovery_codes))
+        return recovery_codes
+
+    def init_secret_2fa(self, persist=True, force=False):
+        secret_2fa = self.user_data.get('secret_2fa')
+        if not secret_2fa or force:
+            secret = pyotp.random_base32()
+            if persist:
+                self.update_userdata(secret_2fa=safe_str(enc_utils.encrypt_value(safe_bytes(secret), enc_key=ENCRYPTION_KEY)))
+            return secret
+        return ''
+
+    def get_secret_2fa(self) -> str:
+        secret_2fa = self.user_data['secret_2fa']
+        if secret_2fa:
+            strict_mode = ConfigGet().get_bool('rhodecode.encrypted_values.strict', missing=True)
+            return safe_str(
+                enc_utils.decrypt_value(secret_2fa, enc_key=ENCRYPTION_KEY, strict_mode=strict_mode))
+        return ''
+
+    def set_2fa_secret(self, value):
+        encrypted_value = enc_utils.encrypt_value(safe_bytes(value), enc_key=ENCRYPTION_KEY)
+        self.update_userdata(secret_2fa=safe_str(encrypted_value))
+
     def regenerate_2fa_recovery_codes(self):
         """
         Regenerates 2fa recovery codes upon request
         """
-        self.update_userdata(recovery_codes_2fa=[])
-        Session().flush()
-        new_recovery_codes = self.get_2fa_recovery_codes()
+        new_recovery_codes = self.init_2fa_recovery_codes(force=True)
         Session().commit()
         return new_recovery_codes
 
@@ -5021,8 +5075,7 @@ class Gist(Base, BaseModel):
         return data
 
     def __json__(self):
-        data = dict(
-        )
+        data = dict()
         data.update(self.get_api_data())
         return data
     # SCM functions
diff --git a/rhodecode/model/forms.py b/rhodecode/model/forms.py
index 9d625536..6ceffd53 100644
--- a/rhodecode/model/forms.py
+++ b/rhodecode/model/forms.py
@@ -111,6 +111,7 @@ def TOTPForm(localizer, user, allow_recovery_code_use=False):
         allow_extra_fields = True
         filter_extra_fields = False
         totp = v.Regex(r'^(?:\d{6}|[A-Z0-9]{32})$')
+        secret_totp = v.String()
 
         def to_python(self, value, state=None):
             validation_checks = [user.is_totp_valid]
@@ -118,10 +119,12 @@ def TOTPForm(localizer, user, allow_recovery_code_use=False):
                 validation_checks.append(user.is_2fa_recovery_code_valid)
             form_data = super().to_python(value, state)
             received_code = form_data['totp']
-            if not any(map(lambda x: x(received_code), validation_checks)):
+            secret = form_data.get('secret_totp')
+
+            if not any(map(lambda func: func(received_code, secret), validation_checks)):
                 error_msg = _('Code is invalid. Try again!')
                 raise formencode.Invalid(error_msg, v, state, error_dict={'totp': error_msg})
-            return True
+            return form_data
 
     return _TOTPForm
 
diff --git a/rhodecode/public/js/rhodecode/routes.js b/rhodecode/public/js/rhodecode/routes.js
index fcb944d4..3b7c6926 100644
--- a/rhodecode/public/js/rhodecode/routes.js
+++ b/rhodecode/public/js/rhodecode/routes.js
@@ -95,6 +95,7 @@ function registerRCRoutes() {
     pyroutes.register('channelstream_connect', '/_admin/channelstream/connect', []);
     pyroutes.register('channelstream_proxy', '/_channelstream', []);
     pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []);
+    pyroutes.register('check_2fa', '/_admin/check_2fa', []);
     pyroutes.register('commit_draft_comments_submit', '/%(repo_name)s/changeset/%(commit_id)s/draft_comments_submit', ['repo_name', 'commit_id']);
     pyroutes.register('debug_style_email', '/_admin/debug_style/email/%(email_id)s', ['email_id']);
     pyroutes.register('debug_style_email_plain_rendered', '/_admin/debug_style/email-rendered/%(email_id)s', ['email_id']);
@@ -218,22 +219,23 @@ function registerRCRoutes() {
     pyroutes.register('my_account_emails', '/_admin/my_account/emails', []);
     pyroutes.register('my_account_emails_add', '/_admin/my_account/emails/new', []);
     pyroutes.register('my_account_emails_delete', '/_admin/my_account/emails/delete', []);
+    pyroutes.register('my_account_enable_2fa', '/_admin/my_account/enable_2fa', []);
+    pyroutes.register('my_account_enable_2fa_save', '/_admin/my_account/enable_2fa_save', []);
     pyroutes.register('my_account_external_identity', '/_admin/my_account/external-identity', []);
     pyroutes.register('my_account_external_identity_delete', '/_admin/my_account/external-identity/delete', []);
     pyroutes.register('my_account_goto_bookmark', '/_admin/my_account/bookmark/%(bookmark_id)s', ['bookmark_id']);
     pyroutes.register('my_account_notifications', '/_admin/my_account/notifications', []);
     pyroutes.register('my_account_notifications_test_channelstream', '/_admin/my_account/test_channelstream', []);
     pyroutes.register('my_account_notifications_toggle_visibility', '/_admin/my_account/toggle_visibility', []);
-    pyroutes.register('check_2fa', '/_admin/check_2fa', []);
-    pyroutes.register('my_account_configure_2fa', '/_admin/my_account/configure_2fa', []);
-    pyroutes.register('my_account_regenerate_2fa_recovery_codes', '/_admin/my_account/regenerate_recovery_codes', []);
     pyroutes.register('my_account_password', '/_admin/my_account/password', []);
     pyroutes.register('my_account_password_update', '/_admin/my_account/password/update', []);
     pyroutes.register('my_account_perms', '/_admin/my_account/perms', []);
     pyroutes.register('my_account_profile', '/_admin/my_account/profile', []);
     pyroutes.register('my_account_pullrequests', '/_admin/my_account/pull_requests', []);
     pyroutes.register('my_account_pullrequests_data', '/_admin/my_account/pull_requests/data', []);
+    pyroutes.register('my_account_regenerate_2fa_recovery_codes', '/_admin/my_account/regenerate_recovery_codes', []);
     pyroutes.register('my_account_repos', '/_admin/my_account/repos', []);
+    pyroutes.register('my_account_show_2fa_recovery_codes', '/_admin/my_account/recovery_codes', []);
     pyroutes.register('my_account_ssh_keys', '/_admin/my_account/ssh_keys', []);
     pyroutes.register('my_account_ssh_keys_add', '/_admin/my_account/ssh_keys/new', []);
     pyroutes.register('my_account_ssh_keys_delete', '/_admin/my_account/ssh_keys/delete', []);
@@ -382,6 +384,7 @@ function registerRCRoutes() {
     pyroutes.register('search_repo', '/%(repo_name)s/_search', ['repo_name']);
     pyroutes.register('search_repo_alt', '/%(repo_name)s/search', ['repo_name']);
     pyroutes.register('search_repo_group', '/%(repo_group_name)s/_search', ['repo_group_name']);
+    pyroutes.register('setup_2fa', '/_admin/setup_2fa', []);
     pyroutes.register('store_user_session_value', '/_store_session_attr', []);
     pyroutes.register('strip_check', '/%(repo_name)s/settings/strip_check', ['repo_name']);
     pyroutes.register('strip_execute', '/%(repo_name)s/settings/strip_execute', ['repo_name']);
diff --git a/rhodecode/templates/admin/my_account/my_account_2fa.mako b/rhodecode/templates/admin/my_account/my_account_2fa.mako
index 52848a97..41e433b5 100644
--- a/rhodecode/templates/admin/my_account/my_account_2fa.mako
+++ b/rhodecode/templates/admin/my_account/my_account_2fa.mako
@@ -4,6 +4,7 @@
     <div class="panel-heading">
         <h3 class="panel-title">${_('Enable/Disable 2FA for your account')}</h3>
     </div>
+    ${h.secure_form(h.route_path('my_account_enable_2fa_save'), request=request)}
     <div class="panel-body">
         <div class="form">
             <div class="fields">
@@ -12,112 +13,122 @@
                         <label>${_('2FA status')}:</label>
                     </div>
                     <div class="checkboxes">
-
-                        <div class="form-check">
-                            <label class="form-check-label">
-                                <input type="radio" id="2faEnabled" value="1" ${'checked' if c.state_of_2fa else ''}>
-                                ${_('Enabled')}
-                            </label>
-                            <label class="form-check-label">
-                                <input type="radio" id="2faDisabled" value="0" ${'checked' if not c.state_of_2fa else ''}>
-                                ${_('Disabled')}
-                            </label>
-                        </div>
                         % if c.locked_2fa:
                         <span class="help-block">${_('2FA settings cannot be changed here, because 2FA was forced enabled by RhodeCode Administrator.')}</span>
+
+                        % else:
+                        <div class="form-check">
+                            <input type="radio" id="2faEnabled" name="2fa_status" value="1" ${'checked=1' if c.state_of_2fa else ''}/>
+                            <label for="2faEnabled">${_('Enable 2FA')}</label>
+
+                            <input type="radio" id="2faDisabled" name="2fa_status" value="0" ${'checked=1' if not c.state_of_2fa else ''} />
+                            <label for="2faDisabled">${_('Disable 2FA')}</label>
+                        </div>
                         % endif
+
                     </div>
                 </div>
             </div>
             <button id="saveBtn" class="btn btn-primary" ${'disabled' if c.locked_2fa else ''}>${_('Save')}</button>
         </div>
     </div>
+    ${h.end_form()}
 </div>
 
 % if c.state_of_2fa:
+
+
+% if not c.user_seen_2fa_recovery_codes:
+
+<div class="panel panel-warning">
+    <div class="panel-heading" id="advanced-archive">
+        <h3 class="panel-title">${_('2FA Recovery codes')} <a class="permalink" href="#advanced-archive"> ¶</a></h3>
+    </div>
+    <div class="panel-body">
+        <p>
+            ${_('You have not seen your 2FA recovery codes yet.')}
+            ${_('Please save them in a safe place, or you will lose access to your account in case of lost access to authenticator app.')}
+        </p>
+        <br/>
+        <a href="${request.route_path('my_account_enable_2fa', _query={'show-recovery-codes': 1})}" class="btn btn-primary">${_('Show recovery codes')}</a>
+    </div>
+</div>
+% endif
+
+
+${h.secure_form(h.route_path('my_account_regenerate_2fa_recovery_codes'), request=request)}
 <div class="panel panel-default">
     <div class="panel-heading">
         <h3 class="panel-title">${_('Regenerate 2FA recovery codes for your account')}</h3>
     </div>
     <div class="panel-body">
         <form id="2faForm">
-            <input type="text" name="totp" placeholder="${_('Verify the code from the app')}" pattern="\d{6}"
-                   style="width: 20%">
-            <button type="button" class="btn btn-primary" onclick="submitForm()">Verify</button>
+            <input type="text" name="totp" placeholder="${_('Verify the code from the app')}" pattern="\d{6}" style="width: 20%">
+            <button type="submit" class="btn btn-primary">${_('Verify and generate new codes')}</button>
         </form>
-        <div id="result"></div>
     </div>
 
 </div>
-
+${h.end_form()}
 % endif
 
+
 <script>
-    function submitForm() {
-        let formData = new FormData(document.getElementById("2faForm"));
-        let xhr = new XMLHttpRequest();
 
-        let success = function (response) {
-            let recovery_codes = response.recovery_codes;
-            showRecoveryCodesPopup(recovery_codes);
-        }
-
-        xhr.onreadystatechange = function () {
-            if (xhr.readyState == 4 && xhr.status == 200) {
-                let responseDoc = new DOMParser().parseFromString(xhr.responseText, "text/html");
-                let contentToDisplay = responseDoc.querySelector('#formErrors');
-                if (contentToDisplay) {
-                    document.getElementById("result").innerHTML = contentToDisplay.innerHTML;
-                } else {
-                    let regenerate_url = pyroutes.url('my_account_regenerate_2fa_recovery_codes');
-                    ajaxPOST(regenerate_url, {'csrf_token': CSRF_TOKEN}, success);
-                }
-            }
-        };
-        let url = pyroutes.url('check_2fa');
-        xhr.open("POST", url, true);
-        xhr.send(formData);
-    }
-
-    document.getElementById('2faEnabled').addEventListener('click', function () {
-        document.getElementById('2faDisabled').checked = false;
-    });
-    document.getElementById('2faDisabled').addEventListener('click', function () {
-        document.getElementById('2faEnabled').checked = false;
-    });
-
-    function getStateValue() {
-        if (document.getElementById('2faEnabled').checked) {
-            return '1';
-        } else {
-            return '0';
-        }
-    };
-
-    function saveChanges(state) {
-
-        let post_data = {'state': state, 'csrf_token': CSRF_TOKEN};
-        let url = pyroutes.url('my_account_configure_2fa');
-
-        ajaxPOST(url, post_data, function(){}, function(){})
-    }
-
-    document.getElementById('saveBtn').addEventListener('click', function () {
-        var state = getStateValue();
-        saveChanges(state);
-    });
-
-    function showRecoveryCodesPopup(recoveryCodes) {
-        let funcData = {'recoveryCodes': recoveryCodes}
-        let recoveryCodesHtml = renderTemplate('recoveryCodes', funcData)
+    function showRecoveryCodesPopup() {
 
         SwalNoAnimation.fire({
-            allowOutsideClick: false,
-            confirmButtonText: _gettext('I Copied the codes'),
-            title: _gettext('2FA Recovery Codes'),
-            html: recoveryCodesHtml
+            title: _gettext('2FA recovery codes'),
+            html: '<span>Should you ever lose your phone or access to your one time password secret, each of these recovery codes can be used one time each to regain access to your account. Please save them in a safe place, or you will lose access to your account.</span>',
+            showCancelButton: false,
+            showConfirmButton: true,
+            showLoaderOnConfirm: true,
+            confirmButtonText: _gettext('Show now'),
+            allowOutsideClick: function () {
+                !Swal.isLoading()
+            },
+
+            preConfirm: function () {
+
+                var postData = {
+                    'csrf_token': CSRF_TOKEN
+                };
+                return new Promise(function (resolve, reject) {
+                    $.ajax({
+                        type: 'POST',
+                        data: postData,
+                        url: pyroutes.url('my_account_show_2fa_recovery_codes'),
+                        headers: {'X-PARTIAL-XHR': true}
+                    })
+                        .done(function (data) {
+                            resolve(data);
+                        })
+                        .fail(function (jqXHR, textStatus, errorThrown) {
+                            var message = formatErrorMessage(jqXHR, textStatus, errorThrown);
+                            ajaxErrorSwal(message);
+                        });
+                })
+            }
+
         })
-
+            .then(function (result) {
+                if (result.value) {
+                    let funcData = {'recoveryCodes': result.value.recovery_codes}
+                    let recoveryCodesHtml = renderTemplate('recoveryCodes', funcData);
+                    SwalNoAnimation.fire({
+                        allowOutsideClick: false,
+                        confirmButtonText: _gettext('I Copied the codes'),
+                        title: _gettext('2FA Recovery Codes'),
+                        html: recoveryCodesHtml
+                    }).then(function (result) {
+                        if (result.isConfirmed) {
+                            window.location.reload()
+                        }
+                    })
+                }
+            })
     }
-
+    % if request.GET.get('show-recovery-codes') == '1' and not c.user_seen_2fa_recovery_codes:
+        showRecoveryCodesPopup();
+    % endif
 </script>
diff --git a/rhodecode/templates/configure_2fa.mako b/rhodecode/templates/configure_2fa.mako
index 96bb4c6e..6ee98c93 100644
--- a/rhodecode/templates/configure_2fa.mako
+++ b/rhodecode/templates/configure_2fa.mako
@@ -1,7 +1,7 @@
 <%inherit file="base/root.mako"/>
 
 <%def name="title()">
-    ${_('Setup authenticator app')}
+    ${_('Setup 2FA')}
     %if c.rhodecode_name:
         &middot; ${h.branding(c.rhodecode_name)}
     %endif
@@ -22,31 +22,28 @@
     </div>
 
     <div class="loginwrapper">
-        <h1>Setup the authenticator app</h1>
+        <h1>${_('Setup the authenticator app')}</h1>
+
         <p>Authenticator apps like <a href='https://play.google.com/store/apps/details?id=com.google.android.apps.authenticator2' target="_blank" rel="noopener noreferrer">Google Authenticator</a>, etc. generate one-time passwords that are used as a second factor to verify you identity.</p>
         <rhodecode-toast id="notifications"></rhodecode-toast>
 
         <div id="setup_2fa">
+            ${h.secure_form(h.route_path('setup_2fa'), request=request, id='totp_form')}
             <div class="sign-in-title">
-                <h1>${_('Scan the QR code')}</h1>
+                <h1>${_('Scan the QR code')}: "${totp_name}"</h1>
             </div>
-            <p>Use an authenticator app to scan.</p>
-            <img src="data:image/png;base64,   ${qr}"/>
+            <p>${_('Use an authenticator app to scan.')}</p>
+            <img alt="qr-code" src="data:image/png;base64, ${qr}"/>
+
             <p>${_('Unable to scan?')} <a id="toggleLink">${_('Click here')}</a></p>
             <div id="secretDiv" class="hidden">
-                <p>${_('Copy and use this code to manually setup an authenticator app')}</p>
-                <input type="text" id="secretField" value=${key}>
-                <i class="tooltip icon-clipboard clipboard-action" data-clipboard-text="" title="${_('Copy the secret key')}"></i>
+                <p>${_('Copy and use this code to manually set up an authenticator app')}</p>
+                <input type="text" class="input-monospace" value="${key}" id="secret_totp" name="secret_totp" style="width: 400px"/>
+                <i class="tooltip icon-clipboard clipboard-action" data-clipboard-text="${key}" title="${_('Copy the secret key')}"></i>
             </div>
-            <div id="codesPopup" class="modal">
-              <div class="modal-content">
-                <ul id="recoveryCodesList"></ul>
-                <button id="copyAllBtn" class="btn btn-primary">Copy All</button>
-              </div>
-            </div>
-            <br><br>
+
             <div id="verify_2fa">
-                ${h.secure_form(h.route_path('setup_2fa'), request=request, id='totp_form')}
+
                 <div class="form mt-4">
                     <div class="field">
                         <p>
@@ -59,13 +56,13 @@
                             <div class="input-group">
                                 ${h.text('totp', class_='form-control', style='width: 40%;')}
                                 <div id="formErrors">
-                                    %if 'totp' in errors:
+                                    % if 'totp' in errors:
                                         <span class="error-message">${errors.get('totp')}</span>
                                         <br />
-                                    %endif
+                                    % endif
                                 </div>
                                 <div class="input-group-append">
-                                    ${h.submit('save',_('Verify'),class_="btn btn-primary", style='width: 40%;', disabled=not codes_viewed)}
+                                    ${h.submit('verify_2fa',_('Verify'),class_="btn btn-primary", style='width: 40%;')}
                                 </div>
                             </div>
                         </div>
@@ -73,81 +70,18 @@
                     </div>
                 </div>
             </div>
+            ${h.end_form()}
         </div>
     </div>
 </div>
+
 <script>
-  document.addEventListener('DOMContentLoaded', function() {
-            let clipboardIcons = document.querySelectorAll('.clipboard-action');
 
-            clipboardIcons.forEach(function(icon) {
-                icon.addEventListener('click', function() {
-                    var inputField = document.getElementById('secretField');
-                    inputField.select();
-                    document.execCommand('copy');
-
-                });
-            });
-    });
-</script>
-<script>
-  document.getElementById('toggleLink').addEventListener('click', function() {
-    let hiddenField = document.getElementById('secretDiv');
-    if (hiddenField.classList.contains('hidden')) {
-      hiddenField.classList.remove('hidden');
-  }
-  });
-</script>
-<script>
-const recovery_codes_string = '${recovery_codes}';
-const cleaned_recovery_codes_string = recovery_codes_string
-  .replace(/&#34;/g, '"')
-  .replace(/&#39;/g, "'");
-
-const recovery_codes = JSON.parse(cleaned_recovery_codes_string);
-
-const cleaned_recovery_codes = recovery_codes.map(code => code.replace(/['"]/g, ''));
-
-function showRecoveryCodesPopup() {
-  const popup = document.getElementById("codesPopup");
-  const codesList = document.getElementById("recoveryCodesList");
-  const verify_btn = document.getElementById('save')
-
-  if (verify_btn.disabled) {
-     codesList.innerHTML = "";
-
-      cleaned_recovery_codes.forEach(code => {
-        const listItem = document.createElement("li");
-        listItem.textContent = code;
-        codesList.appendChild(listItem);
-      });
-
-      popup.style.display = "block";
-      verify_btn.disabled = false;
-    }
-  }
-
-document.getElementById("save").addEventListener("mouseover", showRecoveryCodesPopup);
-
-const popup = document.getElementById("codesPopup");
-const closeButton = document.querySelector(".close");
-window.onclick = function(event) {
-  if (event.target === popup || event.target === closeButton) {
-    popup.style.display = "none";
-  }
+document.getElementById('toggleLink').addEventListener('click', function() {
+let hiddenField = document.getElementById('secretDiv');
+if (hiddenField.classList.contains('hidden')) {
+  hiddenField.classList.remove('hidden');
 }
-
-document.getElementById("copyAllBtn").addEventListener("click", function() {
-  const codesListItems = document.querySelectorAll("#recoveryCodesList li");
-  const allCodes = Array.from(codesListItems).map(item => item.textContent).join(", ");
-
-  const textarea = document.createElement('textarea');
-  textarea.value = allCodes;
-  document.body.appendChild(textarea);
-
-  textarea.select();
-  document.execCommand('copy');
-
-  document.body.removeChild(textarea);
 });
+
 </script>
diff --git a/rhodecode/templates/verify_2fa.mako b/rhodecode/templates/verify_2fa.mako
index c5d2f3d3..fa235391 100644
--- a/rhodecode/templates/verify_2fa.mako
+++ b/rhodecode/templates/verify_2fa.mako
@@ -1,37 +1,54 @@
-<%inherit file="/base/root.mako"/>
+<%inherit file="base/root.mako"/>
+
 <%def name="title()">
-    ${_('Check 2FA')}
+    ${_('Verify 2FA')}
     %if c.rhodecode_name:
         &middot; ${h.branding(c.rhodecode_name)}
     %endif
 </%def>
+<style>body{background-color:#eeeeee;}</style>
 
-<div class="box">
-    <div class="verify2FA">
-        ${h.secure_form(h.route_path('check_2fa'), request=request, id='totp_form')}
-        <div class="form mt-4" style="position: relative; margin-left: 35%; margin-top: 20%;">
-            <div class="field">
-                <p>
-                <div class="label">
-                    <label for="totp" class="form-label text-dark font-weight-bold" style="text-align: left;">${_('Verify the code from the app')}:</label>
+<div class="loginbox">
+    <div class="header-account">
+        <div id="header-inner" class="title">
+            <div id="logo">
+                % if c.rhodecode_name:
+                <div class="branding">
+                   <a href="${h.route_path('home')}">${h.branding(c.rhodecode_name)}</a>
                 </div>
-                </p>
-                <p>
-                <div>
-                    <div class="input-group">
-                        ${h.text('totp', class_="form-control", style='width: 38%;')}
-                        <div id="formErrors">
-                            %if 'totp' in errors:
-                                <span class="error-message">${errors.get('totp')}</span>
-                                <br />
-                            %endif
-                        </div>
-                        <br />
-                        ${h.submit('save',_('Verify'),class_="btn btn-primary", style='width: 40%;')}
-                    </div>
-                </div>
-                </p>
+                % endif
             </div>
         </div>
     </div>
+
+    <div class="loginwrapper">
+        <rhodecode-toast id="notifications"></rhodecode-toast>
+
+        <div id="register">
+                <div class="sign-in-title">
+                    <h1>${_('Verify the code from the app')}</h1>
+                </div>
+                <div class="inner form">
+                    ${h.secure_form(h.route_path('check_2fa'), request=request, id='totp_form')}
+                        <label for="totp">${_('Verification code')}:</label>
+                        ${h.text('totp', class_="form-control")}
+                        %if 'totp' in errors:
+                            <span class="error-message">${errors.get('totp')}</span>
+                            <br />
+                        %endif
+                        <p class="help-block">${_('Enter the code from your two-factor authenticator app. If you\'ve lost your device, you can enter one of your recovery codes.')}</p>
+
+                        ${h.submit('send', _('Verify'), class_="btn sign-in")}
+                        <p class="help-block pull-right">
+                            RhodeCode ${c.rhodecode_edition}
+                        </p>
+                    ${h.end_form()}
+                </div>
+            </div>
+
+    </div>
 </div>
+
+
+
+

From e20aa86d2d7e2db911113a406026722e92dd19a8 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 24 Apr 2024 09:49:39 +0200
Subject: [PATCH 058/150] feat(2fa): updates routes names and urls to better
 reflect actions

---
 rhodecode/apps/login/views.py                        |  2 +-
 rhodecode/apps/my_account/__init__.py                | 12 ++++++------
 rhodecode/apps/my_account/views/my_account.py        |  8 ++++----
 rhodecode/public/js/rhodecode/routes.js              |  4 ++--
 rhodecode/templates/admin/my_account/my_account.mako |  2 +-
 .../templates/admin/my_account/my_account_2fa.mako   |  4 ++--
 6 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/rhodecode/apps/login/views.py b/rhodecode/apps/login/views.py
index 6d96a72b..e25112fe 100644
--- a/rhodecode/apps/login/views.py
+++ b/rhodecode/apps/login/views.py
@@ -500,7 +500,7 @@ class LoginView(BaseAppView):
                 user_instance.set_2fa_secret(secret)
 
                 Session().commit()
-                raise HTTPFound(self.request.route_path('my_account_enable_2fa', _query={'show-recovery-codes': 1}))
+                raise HTTPFound(self.request.route_path('my_account_configure_2fa', _query={'show-recovery-codes': 1}))
             except formencode.Invalid as errors:
                 defaults = errors.value
                 render_ctx = {
diff --git a/rhodecode/apps/my_account/__init__.py b/rhodecode/apps/my_account/__init__.py
index 982871a4..754d2677 100644
--- a/rhodecode/apps/my_account/__init__.py
+++ b/rhodecode/apps/my_account/__init__.py
@@ -76,21 +76,21 @@ def includeme(config):
 
     # my account 2fa
     config.add_route(
-        name='my_account_enable_2fa',
-        pattern=ADMIN_PREFIX + '/my_account/enable_2fa')
+        name='my_account_configure_2fa',
+        pattern=ADMIN_PREFIX + '/my_account/configure_2fa')
     config.add_view(
         MyAccountView,
         attr='my_account_2fa',
-        route_name='my_account_enable_2fa', request_method='GET',
+        route_name='my_account_configure_2fa', request_method='GET',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
     # my account 2fa save
     config.add_route(
-        name='my_account_enable_2fa_save',
-        pattern=ADMIN_PREFIX + '/my_account/enable_2fa_save')
+        name='my_account_configure_2fa_update',
+        pattern=ADMIN_PREFIX + '/my_account/configure_2fa_update')
     config.add_view(
         MyAccountView,
         attr='my_account_2fa_update',
-        route_name='my_account_enable_2fa_save', request_method='POST',
+        route_name='my_account_configure_2fa_update', request_method='POST',
         renderer='rhodecode:templates/admin/my_account/my_account.mako')
 
     # my account 2fa recovery code-reset
diff --git a/rhodecode/apps/my_account/views/my_account.py b/rhodecode/apps/my_account/views/my_account.py
index ac3df097..37a39b4a 100644
--- a/rhodecode/apps/my_account/views/my_account.py
+++ b/rhodecode/apps/my_account/views/my_account.py
@@ -231,7 +231,7 @@ class MyAccountView(BaseAppView, DataGridAppView):
         user_instance.update_userdata(update_2fa=time.time())
         Session().commit()
         h.flash(_("Successfully saved 2FA settings"), category='success')
-        raise HTTPFound(self.request.route_path('my_account_enable_2fa'))
+        raise HTTPFound(self.request.route_path('my_account_configure_2fa'))
 
     @LoginRequired()
     @NotAnonymous()
@@ -262,12 +262,12 @@ class MyAccountView(BaseAppView, DataGridAppView):
             Session().commit()
         except formencode.Invalid as errors:
             h.flash(_("Failed to generate new recovery codes: {}").format(errors), category='error')
-            raise HTTPFound(self.request.route_path('my_account_enable_2fa'))
+            raise HTTPFound(self.request.route_path('my_account_configure_2fa'))
         except Exception as e:
             h.flash(_("Failed to generate new recovery codes: {}").format(e), category='error')
-            raise HTTPFound(self.request.route_path('my_account_enable_2fa'))
+            raise HTTPFound(self.request.route_path('my_account_configure_2fa'))
 
-        raise HTTPFound(self.request.route_path('my_account_enable_2fa', _query={'show-recovery-codes': 1}))
+        raise HTTPFound(self.request.route_path('my_account_configure_2fa', _query={'show-recovery-codes': 1}))
 
     @LoginRequired()
     @NotAnonymous()
diff --git a/rhodecode/public/js/rhodecode/routes.js b/rhodecode/public/js/rhodecode/routes.js
index 3b7c6926..37ac3ed9 100644
--- a/rhodecode/public/js/rhodecode/routes.js
+++ b/rhodecode/public/js/rhodecode/routes.js
@@ -215,12 +215,12 @@ function registerRCRoutes() {
     pyroutes.register('my_account_auth_tokens_view', '/_admin/my_account/auth_tokens/view', []);
     pyroutes.register('my_account_bookmarks', '/_admin/my_account/bookmarks', []);
     pyroutes.register('my_account_bookmarks_update', '/_admin/my_account/bookmarks/update', []);
+    pyroutes.register('my_account_configure_2fa', '/_admin/my_account/configure_2fa', []);
+    pyroutes.register('my_account_configure_2fa_update', '/_admin/my_account/configure_2fa_update', []);
     pyroutes.register('my_account_edit', '/_admin/my_account/edit', []);
     pyroutes.register('my_account_emails', '/_admin/my_account/emails', []);
     pyroutes.register('my_account_emails_add', '/_admin/my_account/emails/new', []);
     pyroutes.register('my_account_emails_delete', '/_admin/my_account/emails/delete', []);
-    pyroutes.register('my_account_enable_2fa', '/_admin/my_account/enable_2fa', []);
-    pyroutes.register('my_account_enable_2fa_save', '/_admin/my_account/enable_2fa_save', []);
     pyroutes.register('my_account_external_identity', '/_admin/my_account/external-identity', []);
     pyroutes.register('my_account_external_identity_delete', '/_admin/my_account/external-identity/delete', []);
     pyroutes.register('my_account_goto_bookmark', '/_admin/my_account/bookmark/%(bookmark_id)s', ['bookmark_id']);
diff --git a/rhodecode/templates/admin/my_account/my_account.mako b/rhodecode/templates/admin/my_account/my_account.mako
index 8f45ffb8..6bd7b1f8 100644
--- a/rhodecode/templates/admin/my_account/my_account.mako
+++ b/rhodecode/templates/admin/my_account/my_account.mako
@@ -28,7 +28,7 @@
           <li class="${h.is_active(['profile', 'profile_edit'], c.active)}"><a href="${h.route_path('my_account_profile')}">${_('Profile')}</a></li>
           <li class="${h.is_active('emails', c.active)}"><a href="${h.route_path('my_account_emails')}">${_('Emails')}</a></li>
           <li class="${h.is_active('password', c.active)}"><a href="${h.route_path('my_account_password')}">${_('Password')}</a></li>
-          <li class="${h.is_active('2FA', c.active)}"><a href="${h.route_path('my_account_enable_2fa')}">${_('2FA')}</a></li>
+          <li class="${h.is_active('2FA', c.active)}"><a href="${h.route_path('my_account_configure_2fa')}">${_('2FA')}</a></li>
           <li class="${h.is_active('bookmarks', c.active)}"><a href="${h.route_path('my_account_bookmarks')}">${_('Bookmarks')}</a></li>
           <li class="${h.is_active('auth_tokens', c.active)}"><a href="${h.route_path('my_account_auth_tokens')}">${_('Auth Tokens')}</a></li>
           <li class="${h.is_active(['ssh_keys', 'ssh_keys_generate'], c.active)}"><a href="${h.route_path('my_account_ssh_keys')}">${_('SSH Keys')}</a></li>
diff --git a/rhodecode/templates/admin/my_account/my_account_2fa.mako b/rhodecode/templates/admin/my_account/my_account_2fa.mako
index 41e433b5..55ae4eb9 100644
--- a/rhodecode/templates/admin/my_account/my_account_2fa.mako
+++ b/rhodecode/templates/admin/my_account/my_account_2fa.mako
@@ -4,7 +4,7 @@
     <div class="panel-heading">
         <h3 class="panel-title">${_('Enable/Disable 2FA for your account')}</h3>
     </div>
-    ${h.secure_form(h.route_path('my_account_enable_2fa_save'), request=request)}
+    ${h.secure_form(h.route_path('my_account_configure_2fa_update'), request=request)}
     <div class="panel-body">
         <div class="form">
             <div class="fields">
@@ -50,7 +50,7 @@
             ${_('Please save them in a safe place, or you will lose access to your account in case of lost access to authenticator app.')}
         </p>
         <br/>
-        <a href="${request.route_path('my_account_enable_2fa', _query={'show-recovery-codes': 1})}" class="btn btn-primary">${_('Show recovery codes')}</a>
+        <a href="${request.route_path('my_account_configure_2fa', _query={'show-recovery-codes': 1})}" class="btn btn-primary">${_('Show recovery codes')}</a>
     </div>
 </div>
 % endif

From 566e572fffd9f8735568c8798aa2b506e8812a58 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 24 Apr 2024 10:38:07 +0200
Subject: [PATCH 059/150] fix(tests): fixed 2fa tests and password reset broken
 by accident

---
 rhodecode/apps/login/tests/test_login.py | 2 +-
 rhodecode/apps/login/views.py            | 8 +++-----
 rhodecode/tests/routes.py                | 1 +
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/rhodecode/apps/login/tests/test_login.py b/rhodecode/apps/login/tests/test_login.py
index 739ccfb7..88ac3877 100644
--- a/rhodecode/apps/login/tests/test_login.py
+++ b/rhodecode/apps/login/tests/test_login.py
@@ -435,7 +435,7 @@ class TestLoginController(object):
             'If such email exists, a password reset link was sent to it.')
 
         # BAD KEY
-        confirm_url = '{}?key={}'.format(route_path('reset_password_confirmation'), 'badkey')
+        confirm_url = route_path('reset_password_confirmation', params={'key': 'badkey'})
         response = self.app.get(confirm_url, status=302)
         assert response.location.endswith(route_path('reset_password'))
         assert_session_flash(response, 'Given reset token is invalid')
diff --git a/rhodecode/apps/login/views.py b/rhodecode/apps/login/views.py
index e25112fe..debd9e8a 100644
--- a/rhodecode/apps/login/views.py
+++ b/rhodecode/apps/login/views.py
@@ -447,16 +447,14 @@ class LoginView(BaseAppView):
 
         return self._get_template_context(c, **template_context)
 
-    @LoginRequired()
-    @NotAnonymous()
     def password_reset_confirmation(self):
         self.load_default_context()
-        if self.request.GET and self.request.GET.get('key'):
+
+        if key := self.request.GET.get('key'):
             # make this take 2s, to prevent brute forcing.
             time.sleep(2)
 
-            token = AuthTokenModel().get_auth_token(
-                self.request.GET.get('key'))
+            token = AuthTokenModel().get_auth_token(key)
 
             # verify token is the correct role
             if token is None or token.role != UserApiKeys.ROLE_PASSWORD_RESET:
diff --git a/rhodecode/tests/routes.py b/rhodecode/tests/routes.py
index a11a9291..0d120419 100644
--- a/rhodecode/tests/routes.py
+++ b/rhodecode/tests/routes.py
@@ -106,6 +106,7 @@ def get_url_defs():
         + "/gists/{gist_id}/rev/{revision}/{format}/{f_path}",
         "login": ADMIN_PREFIX + "/login",
         "logout": ADMIN_PREFIX + "/logout",
+        "setup_2fa": ADMIN_PREFIX + "/setup_2fa",
         "check_2fa": ADMIN_PREFIX + "/check_2fa",
         "register": ADMIN_PREFIX + "/register",
         "reset_password": ADMIN_PREFIX + "/password_reset",

From a36b846e92ef3190976c6ca5c34607e28547f433 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 24 Apr 2024 11:13:35 +0200
Subject: [PATCH 060/150] fix(2fa): fixed redirect loop in workflow when
 password reset was done.

---
 rhodecode/apps/_base/__init__.py |  1 +
 rhodecode/lib/middleware/vcs.py  | 17 +++++++++++------
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/rhodecode/apps/_base/__init__.py b/rhodecode/apps/_base/__init__.py
index 2a4f7e68..8f0324ab 100644
--- a/rhodecode/apps/_base/__init__.py
+++ b/rhodecode/apps/_base/__init__.py
@@ -144,6 +144,7 @@ class BaseAppView(object):
         skip_user_views = [
             "logout",
             "login",
+            "check_2fa",
             "my_account_password",
             "my_account_password_update",
         ]
diff --git a/rhodecode/lib/middleware/vcs.py b/rhodecode/lib/middleware/vcs.py
index 8796bd49..49c636f2 100644
--- a/rhodecode/lib/middleware/vcs.py
+++ b/rhodecode/lib/middleware/vcs.py
@@ -26,6 +26,7 @@ import urllib.parse
 from webob.exc import HTTPNotFound
 
 import rhodecode
+from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.lib.middleware.utils import get_path_info
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.simplegit import SimpleGit, GIT_PROTO_PAT
@@ -164,14 +165,18 @@ def detect_vcs_request(environ, backends):
         # login
         "_admin/login",
 
+        # 2fa
+        f"{ADMIN_PREFIX}/check_2fa",
+        f"{ADMIN_PREFIX}/setup_2fa",
+
         # _admin/api is safe too
-        '_admin/api',
+        f'{ADMIN_PREFIX}/api',
 
         # _admin/gist is safe too
-        '_admin/gists++',
+        f'{ADMIN_PREFIX}/gists++',
 
         # _admin/my_account is safe too
-        '_admin/my_account++',
+        f'{ADMIN_PREFIX}/my_account++',
 
         # static files no detection
         '_static++',
@@ -180,11 +185,11 @@ def detect_vcs_request(environ, backends):
         '_debug_toolbar++',
 
         # skip ops ping, status
-        '_admin/ops/ping',
-        '_admin/ops/status',
+        f'{ADMIN_PREFIX}/ops/ping',
+        f'{ADMIN_PREFIX}/ops/status',
 
         # full channelstream connect should be VCS skipped
-        '_admin/channelstream/connect',
+        f'{ADMIN_PREFIX}/channelstream/connect',
 
         '++/repo_creating_check'
     ]

From d80f6eb5a89c7c6212bd78d0db8271854f0ed42c Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 24 Apr 2024 11:29:09 +0200
Subject: [PATCH 061/150] fix(2fa): fixed case of imports for templates.

---
 rhodecode/apps/my_account/views/my_account.py        | 4 ++--
 rhodecode/templates/admin/my_account/my_account.mako | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/rhodecode/apps/my_account/views/my_account.py b/rhodecode/apps/my_account/views/my_account.py
index 37a39b4a..60edf563 100644
--- a/rhodecode/apps/my_account/views/my_account.py
+++ b/rhodecode/apps/my_account/views/my_account.py
@@ -209,7 +209,7 @@ class MyAccountView(BaseAppView, DataGridAppView):
     def my_account_2fa(self):
         _ = self.request.translate
         c = self.load_default_context()
-        c.active = '2FA'
+        c.active = '2fa'
         user_instance = c.auth_user.get_instance()
         locked_by_admin = user_instance.has_forced_2fa
         c.state_of_2fa = user_instance.has_enabled_2fa
@@ -223,7 +223,7 @@ class MyAccountView(BaseAppView, DataGridAppView):
     def my_account_2fa_update(self):
         _ = self.request.translate
         c = self.load_default_context()
-        c.active = '2FA'
+        c.active = '2fa'
         user_instance = c.auth_user.get_instance()
 
         state = self.request.POST.get('2fa_status') == '1'
diff --git a/rhodecode/templates/admin/my_account/my_account.mako b/rhodecode/templates/admin/my_account/my_account.mako
index 6bd7b1f8..ad0fd217 100644
--- a/rhodecode/templates/admin/my_account/my_account.mako
+++ b/rhodecode/templates/admin/my_account/my_account.mako
@@ -28,7 +28,7 @@
           <li class="${h.is_active(['profile', 'profile_edit'], c.active)}"><a href="${h.route_path('my_account_profile')}">${_('Profile')}</a></li>
           <li class="${h.is_active('emails', c.active)}"><a href="${h.route_path('my_account_emails')}">${_('Emails')}</a></li>
           <li class="${h.is_active('password', c.active)}"><a href="${h.route_path('my_account_password')}">${_('Password')}</a></li>
-          <li class="${h.is_active('2FA', c.active)}"><a href="${h.route_path('my_account_configure_2fa')}">${_('2FA')}</a></li>
+          <li class="${h.is_active('2fa', c.active)}"><a href="${h.route_path('my_account_configure_2fa')}">${_('2FA')}</a></li>
           <li class="${h.is_active('bookmarks', c.active)}"><a href="${h.route_path('my_account_bookmarks')}">${_('Bookmarks')}</a></li>
           <li class="${h.is_active('auth_tokens', c.active)}"><a href="${h.route_path('my_account_auth_tokens')}">${_('Auth Tokens')}</a></li>
           <li class="${h.is_active(['ssh_keys', 'ssh_keys_generate'], c.active)}"><a href="${h.route_path('my_account_ssh_keys')}">${_('SSH Keys')}</a></li>

From 0628f37e75d51ec74d8e7ad7b84d37512b4c07da Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 24 Apr 2024 11:57:40 +0200
Subject: [PATCH 062/150] feat(2fa): improve flash messages on 2fa settings
 page

---
 rhodecode/apps/my_account/views/my_account.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/rhodecode/apps/my_account/views/my_account.py b/rhodecode/apps/my_account/views/my_account.py
index 60edf563..c9d3d171 100644
--- a/rhodecode/apps/my_account/views/my_account.py
+++ b/rhodecode/apps/my_account/views/my_account.py
@@ -230,7 +230,10 @@ class MyAccountView(BaseAppView, DataGridAppView):
         user_instance.has_enabled_2fa = state
         user_instance.update_userdata(update_2fa=time.time())
         Session().commit()
-        h.flash(_("Successfully saved 2FA settings"), category='success')
+        if state:
+            h.flash(_("2FA has been successfully enabled"), category='success')
+        else:
+            h.flash(_("2FA has been successfully disabled"), category='success')
         raise HTTPFound(self.request.route_path('my_account_configure_2fa'))
 
     @LoginRequired()

From aad99df52134085b97b8cdb66f4d3fa775f1ea89 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 24 Apr 2024 15:22:16 +0200
Subject: [PATCH 063/150] feat(2fa): updated and UI fixes

- configure 2fa
- setup 2fa
- various small improvements
---
 rhodecode/apps/login/views.py          |  7 +-
 rhodecode/model/db.py                  |  2 +-
 rhodecode/templates/configure_2fa.mako | 97 +++++++++++++-------------
 rhodecode/templates/verify_2fa.mako    |  2 +-
 4 files changed, 56 insertions(+), 52 deletions(-)

diff --git a/rhodecode/apps/login/views.py b/rhodecode/apps/login/views.py
index debd9e8a..7140baa6 100644
--- a/rhodecode/apps/login/views.py
+++ b/rhodecode/apps/login/views.py
@@ -34,7 +34,7 @@ from pyramid.renderers import render
 from pyramid.response import Response
 from pyramid.httpexceptions import HTTPFound
 
-
+import rhodecode
 from rhodecode.apps._base import BaseAppView
 from rhodecode.authentication.base import authenticate, HTTP_TYPE
 from rhodecode.authentication.plugins import auth_rhodecode
@@ -510,9 +510,10 @@ class LoginView(BaseAppView):
         # only then we should persist it
         secret = user_instance.init_secret_2fa(persist=False)
 
-        totp_name = f'RhodeCode token ({self.request.user.username})'
+        instance_name = rhodecode.ConfigGet().get_str('app.base_url', 'rhodecode')
+        totp_name = f'{instance_name}:{self.request.user.username}'
 
-        qr = qrcode.QRCode(version=1, box_size=10, border=5)
+        qr = qrcode.QRCode(version=1, box_size=5, border=4)
         qr.add_data(pyotp.totp.TOTP(secret).provisioning_uri(name=totp_name))
         qr.make(fit=True)
         img = qr.make_image(fill_color='black', back_color='white')
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index 9ea58d58..4e26633b 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -919,7 +919,7 @@ class User(Base, BaseModel):
         return ''
 
     def get_secret_2fa(self) -> str:
-        secret_2fa = self.user_data['secret_2fa']
+        secret_2fa = self.user_data.get('secret_2fa')
         if secret_2fa:
             strict_mode = ConfigGet().get_bool('rhodecode.encrypted_values.strict', missing=True)
             return safe_str(
diff --git a/rhodecode/templates/configure_2fa.mako b/rhodecode/templates/configure_2fa.mako
index 6ee98c93..e538545c 100644
--- a/rhodecode/templates/configure_2fa.mako
+++ b/rhodecode/templates/configure_2fa.mako
@@ -6,9 +6,11 @@
         &middot; ${h.branding(c.rhodecode_name)}
     %endif
 </%def>
+
 <style>body{background-color:#eeeeee;}</style>
 
-<div class="loginbox">
+<div class="loginbox" style="width: 600px">
+
     <div class="header-account">
         <div id="header-inner" class="title">
             <div id="logo">
@@ -22,66 +24,67 @@
     </div>
 
     <div class="loginwrapper">
-        <h1>${_('Setup the authenticator app')}</h1>
-
-        <p>Authenticator apps like <a href='https://play.google.com/store/apps/details?id=com.google.android.apps.authenticator2' target="_blank" rel="noopener noreferrer">Google Authenticator</a>, etc. generate one-time passwords that are used as a second factor to verify you identity.</p>
         <rhodecode-toast id="notifications"></rhodecode-toast>
 
-        <div id="setup_2fa">
+        <div class="sign-in-title">
+            <h1>${_('Set up the authenticator app')} - ${_('scan the QR code')}</h1>
+        </div>
+        <div class="inner form">
             ${h.secure_form(h.route_path('setup_2fa'), request=request, id='totp_form')}
-            <div class="sign-in-title">
-                <h1>${_('Scan the QR code')}: "${totp_name}"</h1>
-            </div>
-            <p>${_('Use an authenticator app to scan.')}</p>
-            <img alt="qr-code" src="data:image/png;base64, ${qr}"/>
+            <strong>${_('Use an authenticator app to scan.')}</strong><br/>
 
-            <p>${_('Unable to scan?')} <a id="toggleLink">${_('Click here')}</a></p>
-            <div id="secretDiv" class="hidden">
-                <p>${_('Copy and use this code to manually set up an authenticator app')}</p>
-                <input type="text" class="input-monospace" value="${key}" id="secret_totp" name="secret_totp" style="width: 400px"/>
-                <i class="tooltip icon-clipboard clipboard-action" data-clipboard-text="${key}" title="${_('Copy the secret key')}"></i>
+             ## QR CODE
+            <code>${_('Account')}: ${totp_name}</code><br/>
+            <div class="qr-code-container">
+                <img alt="qr-code" src="data:image/png;base64, ${qr}"/>
             </div>
 
-            <div id="verify_2fa">
+            <div id="alternativeCode" style="margin: -10px 0 5px 0">${_('Unable to scan?')} <a id="toggleLink">${_('Click here')}</a></div>
 
-                <div class="form mt-4">
-                    <div class="field">
-                        <p>
-                        <div class="label">
-                            <label for="totp" class="form-label text-dark font-weight-bold" style="text-align: left;">${_('Verify the code from the app')}:</label>
-                        </div>
-                        </p>
-                        <p>
-                        <div>
-                            <div class="input-group">
-                                ${h.text('totp', class_='form-control', style='width: 40%;')}
-                                <div id="formErrors">
-                                    % if 'totp' in errors:
-                                        <span class="error-message">${errors.get('totp')}</span>
-                                        <br />
-                                    % endif
-                                </div>
-                                <div class="input-group-append">
-                                    ${h.submit('verify_2fa',_('Verify'),class_="btn btn-primary", style='width: 40%;')}
-                                </div>
-                            </div>
-                        </div>
-                        </p>
-                    </div>
+            ## Secret alternative code
+            <div id="secretDiv" style="display: none">
+
+                <div style="padding: 10px 0">
+                  <strong style="padding: 4px 0">${_('Copy and use this code to manually set up an authenticator app')}</strong>
+                  <code>${key}</code><i class="tooltip icon-clipboard clipboard-action" data-clipboard-text="${key}" title="${_('Copy the secret key')}" ></i><br/>
+                  <code>${_('type')}: time-based</code>
                 </div>
+
             </div>
+
+            <label for="totp">${_('Verify the code from the app')}:</label>
+            ${h.text('totp', class_='form-control', )}
+            <div id="formErrors">
+                % if 'totp' in errors:
+                    <span class="error-message">${errors.get('totp')}</span>
+                    <br />
+                % endif
+                % if 'secret_totp' in errors:
+                    <span class="error-message">SECRET:${errors.get('secret_totp')}</span>
+                    <br />
+                % endif
+            </div>
+            ${h.hidden('secret_totp', key)}
+            ${h.submit('verify_2fa',_('Verify'), class_="btn sign-in")}
+
             ${h.end_form()}
         </div>
+
     </div>
+
 </div>
 
-<script>
+<script type="text/javascript">
 
-document.getElementById('toggleLink').addEventListener('click', function() {
-let hiddenField = document.getElementById('secretDiv');
-if (hiddenField.classList.contains('hidden')) {
-  hiddenField.classList.remove('hidden');
-}
-});
+$(document).ready(function() {
+
+    $( "#toggleLink" ).on("click", function() {
+      $( "#secretDiv" ).toggle();
+      $( "#alternativeCode").hide();
+      $('#totp').focus();
+    });
+
+    $('#totp').focus();
+})
 
 </script>
diff --git a/rhodecode/templates/verify_2fa.mako b/rhodecode/templates/verify_2fa.mako
index fa235391..581adbca 100644
--- a/rhodecode/templates/verify_2fa.mako
+++ b/rhodecode/templates/verify_2fa.mako
@@ -8,7 +8,7 @@
 </%def>
 <style>body{background-color:#eeeeee;}</style>
 
-<div class="loginbox">
+<div class="loginbox" style="width: 600px">
     <div class="header-account">
         <div id="header-inner" class="title">
             <div id="logo">

From 922adcf2e7c5c2731e7206f7ca2d14a557692dea Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 24 Apr 2024 18:06:20 +0200
Subject: [PATCH 064/150] chore(2fa): refactor some attributes for users

---
 rhodecode/apps/_base/__init__.py              |  2 +-
 rhodecode/apps/login/views.py                 |  8 ++++----
 rhodecode/apps/my_account/views/my_account.py |  2 +-
 rhodecode/model/db.py                         | 15 ++++++++++-----
 4 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/rhodecode/apps/_base/__init__.py b/rhodecode/apps/_base/__init__.py
index 8f0324ab..c98789d7 100644
--- a/rhodecode/apps/_base/__init__.py
+++ b/rhodecode/apps/_base/__init__.py
@@ -194,7 +194,7 @@ class BaseAppView(object):
         if not user_obj:
             return
 
-        if user_obj.has_check_2fa_flag and view_name != self.VERIFY_2FA_VIEW:
+        if user_obj.check_2fa_required and view_name != self.VERIFY_2FA_VIEW:
             raise HTTPFound(self.request.route_path(self.VERIFY_2FA_VIEW))
 
     def _log_creation_exception(self, e, repo_name):
diff --git a/rhodecode/apps/login/views.py b/rhodecode/apps/login/views.py
index 7140baa6..995dc9ea 100644
--- a/rhodecode/apps/login/views.py
+++ b/rhodecode/apps/login/views.py
@@ -188,7 +188,7 @@ class LoginView(BaseAppView):
             # form checks for username/password, now we're authenticated
             username = form_result['username']
             if (user := User.get_by_username_or_primary_email(username)).has_enabled_2fa:
-                user.has_check_2fa_flag = True
+                user.check_2fa_required = True
 
             headers = store_user_in_session(
                 self.session,
@@ -495,7 +495,7 @@ class LoginView(BaseAppView):
                 secret = form_details['secret_totp']
 
                 user_instance.init_2fa_recovery_codes(persist=True, force=True)
-                user_instance.set_2fa_secret(secret)
+                user_instance.2fa_secret = secret
 
                 Session().commit()
                 raise HTTPFound(self.request.route_path('my_account_configure_2fa', _query={'show-recovery-codes': 1}))
@@ -538,10 +538,10 @@ class LoginView(BaseAppView):
         if self.request.method == 'POST':
             post_items = dict(self.request.POST)
             # NOTE: inject secret, as it's a post configured saved item.
-            post_items['secret_totp'] = user_instance.get_secret_2fa()
+            post_items['secret_totp'] = user_instance.secret_2fa
             try:
                 totp_form.to_python(post_items)
-                user_instance.has_check_2fa_flag = False
+                user_instance.check_2fa_required = False
                 Session().commit()
                 raise HTTPFound(c.came_from)
             except formencode.Invalid as errors:
diff --git a/rhodecode/apps/my_account/views/my_account.py b/rhodecode/apps/my_account/views/my_account.py
index c9d3d171..0a3a800a 100644
--- a/rhodecode/apps/my_account/views/my_account.py
+++ b/rhodecode/apps/my_account/views/my_account.py
@@ -258,7 +258,7 @@ class MyAccountView(BaseAppView, DataGridAppView):
 
         post_items = dict(self.request.POST)
         # NOTE: inject secret, as it's a post configured saved item.
-        post_items['secret_totp'] = user_instance.get_secret_2fa()
+        post_items['secret_totp'] = user_instance.secret_2fa
         try:
             totp_form.to_python(post_items)
             user_instance.regenerate_2fa_recovery_codes()
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index 4e26633b..6b054eea 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -839,15 +839,15 @@ class User(Base, BaseModel):
         Session().commit()
 
     @hybrid_property
-    def has_check_2fa_flag(self):
+    def check_2fa_required(self):
         """
         Check if check 2fa flag is set for this user
         """
         value = self.user_data.get('check_2fa', False)
         return value
 
-    @has_check_2fa_flag.setter
-    def has_check_2fa_flag(self, val):
+    @check_2fa_required.setter
+    def check_2fa_required(self, val):
         val = str2bool(val)
         self.update_userdata(check_2fa=val)
         Session().commit()
@@ -918,7 +918,11 @@ class User(Base, BaseModel):
             return secret
         return ''
 
-    def get_secret_2fa(self) -> str:
+    @hybrid_property
+    def secret_2fa(self) -> str:
+        """
+        get stored secret for 2fa
+        """
         secret_2fa = self.user_data.get('secret_2fa')
         if secret_2fa:
             strict_mode = ConfigGet().get_bool('rhodecode.encrypted_values.strict', missing=True)
@@ -926,7 +930,8 @@ class User(Base, BaseModel):
                 enc_utils.decrypt_value(secret_2fa, enc_key=ENCRYPTION_KEY, strict_mode=strict_mode))
         return ''
 
-    def set_2fa_secret(self, value):
+    @secret_2fa.setter
+    def secret_2fa(self, value: str) -> None:
         encrypted_value = enc_utils.encrypt_value(safe_bytes(value), enc_key=ENCRYPTION_KEY)
         self.update_userdata(secret_2fa=safe_str(encrypted_value))
 

From 6b1730f325e626afb003a4a2ba8492caba49ad3a Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 24 Apr 2024 18:07:14 +0200
Subject: [PATCH 065/150] fix(2fa): fixed typo from refactor

---
 rhodecode/apps/login/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rhodecode/apps/login/views.py b/rhodecode/apps/login/views.py
index 995dc9ea..f13660d3 100644
--- a/rhodecode/apps/login/views.py
+++ b/rhodecode/apps/login/views.py
@@ -495,7 +495,7 @@ class LoginView(BaseAppView):
                 secret = form_details['secret_totp']
 
                 user_instance.init_2fa_recovery_codes(persist=True, force=True)
-                user_instance.2fa_secret = secret
+                user_instance.secret_2fa = secret
 
                 Session().commit()
                 raise HTTPFound(self.request.route_path('my_account_configure_2fa', _query={'show-recovery-codes': 1}))

From 6b466e15b133bf43976679c98705210e9f4032ac Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 25 Apr 2024 10:56:23 +0200
Subject: [PATCH 066/150] fix(encryption): rely on default config based strict
 mode if not explicitly given into function params

---
 rhodecode/lib/enc_utils.py | 29 ++++++++++++++++++++++++++++-
 rhodecode/model/db.py      | 11 +++--------
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/rhodecode/lib/enc_utils.py b/rhodecode/lib/enc_utils.py
index 4280f507..9a882dc1 100644
--- a/rhodecode/lib/enc_utils.py
+++ b/rhodecode/lib/enc_utils.py
@@ -1,3 +1,21 @@
+# Copyright (C) 2011-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.lib.encrypt import encrypt_data, validate_and_decrypt_data
 from rhodecode.lib.encrypt2 import Encryptor
@@ -9,6 +27,10 @@ def get_default_algo():
     import rhodecode
     return rhodecode.CONFIG.get('rhodecode.encrypted_values.algorithm') or 'aes'
 
+def get_strict_mode():
+    import rhodecode
+    return rhodecode.ConfigGet().get_bool('rhodecode.encrypted_values.strict') or False
+
 
 def encrypt_value(value: bytes, enc_key: bytes, algo: str = ''):
     if not algo:
@@ -29,7 +51,12 @@ def encrypt_value(value: bytes, enc_key: bytes, algo: str = ''):
     return value
 
 
-def decrypt_value(value: bytes, enc_key: bytes, algo: str = '', strict_mode: bool = False):
+def decrypt_value(value: bytes, enc_key: bytes, algo: str = '', strict_mode: bool | None = None):
+
+    if strict_mode is None:
+        # we use config value rather then explicit True/False
+        strict_mode = get_strict_mode()
+
     enc_key = safe_bytes(enc_key)
     value = safe_bytes(value)
 
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index 6b054eea..8a8f332b 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -199,9 +199,7 @@ class EncryptedTextValue(TypeDecorator):
         if not value:
             return value
 
-        enc_strict_mode = rhodecode.ConfigGet().get_bool('rhodecode.encrypted_values.strict', missing=True)
-
-        bytes_val = enc_utils.decrypt_value(value, enc_key=ENCRYPTION_KEY, strict_mode=enc_strict_mode)
+        bytes_val = enc_utils.decrypt_value(value, enc_key=ENCRYPTION_KEY)
 
         return safe_str(bytes_val)
 
@@ -897,14 +895,12 @@ class User(Base, BaseModel):
 
     def get_2fa_recovery_codes(self):
         encrypted_recovery_codes = self.user_data.get('recovery_codes_2fa', [])
-        strict_mode = ConfigGet().get_bool('rhodecode.encrypted_values.strict', missing=True)
 
         recovery_codes = list(map(
             lambda val: safe_str(
                 enc_utils.decrypt_value(
                     val,
-                    enc_key=ENCRYPTION_KEY,
-                    strict_mode=strict_mode
+                    enc_key=ENCRYPTION_KEY
                 )),
             encrypted_recovery_codes))
         return recovery_codes
@@ -925,9 +921,8 @@ class User(Base, BaseModel):
         """
         secret_2fa = self.user_data.get('secret_2fa')
         if secret_2fa:
-            strict_mode = ConfigGet().get_bool('rhodecode.encrypted_values.strict', missing=True)
             return safe_str(
-                enc_utils.decrypt_value(secret_2fa, enc_key=ENCRYPTION_KEY, strict_mode=strict_mode))
+                enc_utils.decrypt_value(secret_2fa, enc_key=ENCRYPTION_KEY))
         return ''
 
     @secret_2fa.setter

From 45e456bdf089f507b3f512f41c7cba70009c0e28 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 25 Apr 2024 10:58:47 +0200
Subject: [PATCH 067/150] fix(2fa): fixed various typos

---
 rhodecode/apps/login/tests/test_2fa.py        | 6 +++---
 rhodecode/apps/my_account/views/my_account.py | 2 +-
 rhodecode/model/db.py                         | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/rhodecode/apps/login/tests/test_2fa.py b/rhodecode/apps/login/tests/test_2fa.py
index 024a8148..d065e83a 100644
--- a/rhodecode/apps/login/tests/test_2fa.py
+++ b/rhodecode/apps/login/tests/test_2fa.py
@@ -48,16 +48,16 @@ class Test2FA(object):
         user = user_util.create_user(password=self.password)
         user.has_enabled_2fa = True
         user.init_secret_2fa()
-        recovery_cod_to_check = user.init_2fa_recovery_codes()[0]
+        recovery_code_to_check = user.init_2fa_recovery_codes()[0]
         Session().add(user)
         Session().commit()
         self.app.post(
             route_path('login'),
             {'username': user.username,
              'password': self.password})
-        response = self.app.post(route_path('check_2fa'), {'totp': recovery_cod_to_check})
+        response = self.app.post(route_path('check_2fa'), {'totp': recovery_code_to_check})
         assert response.status_code == 302
-        response = self.app.post(route_path('check_2fa'), {'totp': recovery_cod_to_check})
+        response = self.app.post(route_path('check_2fa'), {'totp': recovery_code_to_check})
         response.mustcontain('Code is invalid. Try again!')
 
     def test_2fa_state_when_forced_by_admin(self, user_util):
diff --git a/rhodecode/apps/my_account/views/my_account.py b/rhodecode/apps/my_account/views/my_account.py
index 0a3a800a..098655b5 100644
--- a/rhodecode/apps/my_account/views/my_account.py
+++ b/rhodecode/apps/my_account/views/my_account.py
@@ -226,7 +226,7 @@ class MyAccountView(BaseAppView, DataGridAppView):
         c.active = '2fa'
         user_instance = c.auth_user.get_instance()
 
-        state = self.request.POST.get('2fa_status') == '1'
+        state = str2bool(self.request.POST.get('2fa_status'))
         user_instance.has_enabled_2fa = state
         user_instance.update_userdata(update_2fa=time.time())
         Session().commit()
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index 8a8f332b..848c7a88 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -813,7 +813,7 @@ class User(Base, BaseModel):
         Checks if 2fa was forced for ALL users (including current one)
         """
         from rhodecode.model.settings import SettingsModel
-        # So now we're supporting only auth_rhodecode_global_2f
+        # So now we're supporting only auth_rhodecode_global_2fa
         if value := SettingsModel().get_setting_by_name('auth_rhodecode_global_2fa'):
             return value.app_settings_value
         return False

From 85ed19f363bb6061bc44e3f9c24529c372c7379a Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 25 Apr 2024 11:40:13 +0200
Subject: [PATCH 068/150] fix(2fa): fixed tests to use mock instead of global
 value which causes some other tests to fail

---
 rhodecode/apps/login/tests/test_2fa.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/rhodecode/apps/login/tests/test_2fa.py b/rhodecode/apps/login/tests/test_2fa.py
index d065e83a..3c3ec102 100644
--- a/rhodecode/apps/login/tests/test_2fa.py
+++ b/rhodecode/apps/login/tests/test_2fa.py
@@ -1,5 +1,7 @@
 import pytest
+import mock
 
+from rhodecode.lib.type_utils import AttributeDict
 from rhodecode.model.meta import Session
 from rhodecode.tests.fixture import Fixture
 from rhodecode.tests.routes import route_path
@@ -14,10 +16,6 @@ class Test2FA(object):
     def setup_class(cls):
         cls.password = 'valid-one'
 
-    @classmethod
-    def teardown_class(cls):
-        SettingsModel().create_or_update_setting('auth_rhodecode_global_2fa', False)
-
     def test_redirect_to_2fa_setup_if_enabled_for_user(self, user_util):
         user = user_util.create_user(password=self.password)
         user.has_enabled_2fa = True
@@ -63,5 +61,7 @@ class Test2FA(object):
     def test_2fa_state_when_forced_by_admin(self, user_util):
         user = user_util.create_user(password=self.password)
         user.has_enabled_2fa = False
-        SettingsModel().create_or_update_setting('auth_rhodecode_global_2fa', True)
-        assert user.has_enabled_2fa
+        with mock.patch.object(
+                SettingsModel, 'get_setting_by_name', lambda *a, **kw: AttributeDict(app_settings_value=True)):
+
+            assert user.has_enabled_2fa

From 695fc88813eb716018a71f801c111de540c3d42f Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Fri, 26 Apr 2024 12:06:48 +0200
Subject: [PATCH 069/150] feat(cache): disable cache before using d_cache call
 to skip init of dcache on no-cached calls

---
 rhodecode/apps/repository/views/repo_files.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py
index c55782a0..f8919565 100644
--- a/rhodecode/apps/repository/views/repo_files.py
+++ b/rhodecode/apps/repository/views/repo_files.py
@@ -421,7 +421,7 @@ class RepoFilesView(RepoAppView):
         with ReentrantLock(d_cache, reentrant_lock_key):
             # This is also a cache key
             use_cached_archive = False
-            if archive_name_key in d_cache and not archive_cache_disable:
+            if not archive_cache_disable and archive_name_key in d_cache:
                 reader, tag = d_cache.get(archive_name_key, read=True, tag=True, retry=True)
                 use_cached_archive = True
                 log.debug('Found cached archive as key=%s tag=%s, serving archive from cache reader=%s',

From a970700222376286d08ca5e48ad2ac8487ab1c2e Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Fri, 26 Apr 2024 12:07:06 +0200
Subject: [PATCH 070/150] feat(locks): added no-blocking option for redis
 blocks

---
 rhodecode/lib/_vendor/redis_lock/__init__.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/rhodecode/lib/_vendor/redis_lock/__init__.py b/rhodecode/lib/_vendor/redis_lock/__init__.py
index 2ee12897..7ebb403e 100644
--- a/rhodecode/lib/_vendor/redis_lock/__init__.py
+++ b/rhodecode/lib/_vendor/redis_lock/__init__.py
@@ -111,11 +111,12 @@ class Lock(object):
     extend_script = None
     reset_script = None
     reset_all_script = None
+    blocking = None
 
     _lock_renewal_interval: float
     _lock_renewal_thread: Union[threading.Thread, None]
 
-    def __init__(self, redis_client, name, expire=None, id=None, auto_renewal=False, strict=True, signal_expire=1000):
+    def __init__(self, redis_client, name, expire=None, id=None, auto_renewal=False, strict=True, signal_expire=1000, blocking=True):
         """
         :param redis_client:
             An instance of :class:`~StrictRedis`.
@@ -143,6 +144,9 @@ class Lock(object):
             If set ``True`` then the ``redis_client`` needs to be an instance of ``redis.StrictRedis``.
         :param signal_expire:
             Advanced option to override signal list expiration in milliseconds. Increase it for very slow clients. Default: ``1000``.
+        :param blocking:
+            Boolean value specifying whether lock should be blocking or not.
+            Used in `__enter__` method.
         """
         if strict and not isinstance(redis_client, StrictRedis):
             raise ValueError("redis_client must be instance of StrictRedis. "
@@ -179,6 +183,8 @@ class Lock(object):
                                        else None)
         self._lock_renewal_thread = None
 
+        self.blocking = blocking
+
         self.register_scripts(redis_client)
 
     @classmethod
@@ -342,9 +348,11 @@ class Lock(object):
         loggers["refresh.exit"].debug("Renewal thread for Lock(%r) exited.", self._name)
 
     def __enter__(self):
-        acquired = self.acquire(blocking=True)
+        acquired = self.acquire(blocking=self.blocking)
         if not acquired:
-            raise AssertionError(f"Lock({self._name}) wasn't acquired, but blocking=True was used!")
+            if self.blocking:
+                raise AssertionError(f"Lock({self._name}) wasn't acquired, but blocking=True was used!")
+            raise NotAcquired(f"Lock({self._name}) is not acquired or it already expired.")
         return self
 
     def __exit__(self, exc_type=None, exc_value=None, traceback=None):

From 960d6c52185735065f83f4a6fda710ed066b6471 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 6 May 2024 14:54:21 +0200
Subject: [PATCH 071/150] fix(encryption): don't be strict on enc format when
 no enc headers are missing.

This fixes problem with migration of OLD unencrypted DBs into a encryption format
---
 rhodecode/lib/encrypt.py  |  1 +
 rhodecode/lib/encrypt2.py | 10 +++++++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/rhodecode/lib/encrypt.py b/rhodecode/lib/encrypt.py
index 34ec7e8b..bdf859ce 100644
--- a/rhodecode/lib/encrypt.py
+++ b/rhodecode/lib/encrypt.py
@@ -43,6 +43,7 @@ class InvalidDecryptedValue(str):
         content = f'<{cls.__name__}({content[:16]}...)>'
         return str.__new__(cls, content)
 
+
 KEY_FORMAT = b'enc$aes_hmac${1}'
 
 
diff --git a/rhodecode/lib/encrypt2.py b/rhodecode/lib/encrypt2.py
index 1c6832b0..bbd4cc46 100644
--- a/rhodecode/lib/encrypt2.py
+++ b/rhodecode/lib/encrypt2.py
@@ -29,12 +29,16 @@ class Encryptor(object):
     @classmethod
     def detect_enc_algo(cls, enc_data: bytes):
         parts = enc_data.split(b'$', 3)
-        if len(parts) != 3:
-            raise ValueError(f'Encrypted Data has invalid format, expected {cls.key_format}, got {parts}')
 
         if b'enc$aes_hmac$' in enc_data:
+            # we expect this data is encrypted, so validate the header
+            if len(parts) != 3:
+                raise ValueError(f'Encrypted Data has invalid format, expected {cls.key_format}, got `{parts}`')
             return 'aes'
         elif b'enc2$salt' in enc_data:
+            # we expect this data is encrypted, so validate the header
+            if len(parts) != 3:
+                raise ValueError(f'Encrypted Data has invalid format, expected {cls.key_format}, got `{parts}`')
             return 'fernet'
         return None
 
@@ -65,7 +69,7 @@ class Encryptor(object):
     def _get_parts(self, enc_data):
         parts = enc_data.split(b'$', 3)
         if len(parts) != 3:
-            raise ValueError(f'Encrypted Data has invalid format, expected {self.key_format}, got {parts}')
+            raise ValueError(f'Encrypted Data has invalid format, expected {self.key_format}, got `{parts}`')
         prefix, salt, enc_data = parts
 
         try:

From 3e3feaa08fcca8764ec20984b8606b06325467b2 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 7 May 2024 00:34:18 +0200
Subject: [PATCH 072/150] core(deps): bumped dogpile.cache to 1.3.3

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index cb04cfc3..157aa3a4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -84,7 +84,7 @@ deform==2.0.15
   zope.deprecation==5.0.0
 diskcache==5.6.3
 docutils==0.19
-dogpile.cache==1.3.2
+dogpile.cache==1.3.3
   decorator==5.1.1
   stevedore==5.1.0
     pbr==5.11.1

From a8ed69b19eb0531d08a83e9f379c37934a2d7113 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 7 May 2024 00:34:32 +0200
Subject: [PATCH 073/150] core(deps): bumped paste deps

---
 requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 157aa3a4..a1ebabaf 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -171,8 +171,8 @@ nbconvert==7.7.3
     webencodings==0.5.1
   traitlets==5.9.0
 orjson==3.9.15
-pastescript==3.5.1
-  paste==3.8.0
+pastescript==3.6.0
+  paste==3.10.1
     six==1.16.0
   pastedeploy==3.1.0
   six==1.16.0
@@ -280,7 +280,7 @@ unidecode==1.3.6
 urlobject==2.4.3
 waitress==3.0.0
 weberror==0.13.1
-  paste==3.8.0
+  paste==3.10.1
     six==1.16.0
   pygments==2.15.1
   tempita==0.5.2

From 6fdf9016889b90bf7c8d2c2dd9c70c4eeddd942d Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 7 May 2024 00:40:08 +0200
Subject: [PATCH 074/150] core(deps): bumped orjson to 3.10.3

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index a1ebabaf..36bf79c4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -170,7 +170,7 @@ nbconvert==7.7.3
   tinycss2==1.2.1
     webencodings==0.5.1
   traitlets==5.9.0
-orjson==3.9.15
+orjson==3.10.3
 pastescript==3.6.0
   paste==3.10.1
     six==1.16.0

From 338376372c67d20527d4213939cf17f47332f618 Mon Sep 17 00:00:00 2001
From: Andrii Verbytskyi <andverb@rhodecode.com>
Date: Wed, 8 May 2024 13:08:59 +0200
Subject: [PATCH 075/150] fixed permissions for pip cache folder

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index e7112cec..f604cfd1 100644
--- a/Makefile
+++ b/Makefile
@@ -124,6 +124,7 @@ dev-cleanup:
 .PHONY: dev-env
 ## make dev-env based on the requirements files and install develop of packages
 dev-env:
+	sudo -u root chown rhodecode:rhodecode /home/rhodecode/.cache/pip/
 	pip install build virtualenv
 	pushd ../rhodecode-vcsserver/ && make dev-env && popd
 	pip wheel --wheel-dir=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_rc_tools.txt -r requirements_test.txt -r requirements_debug.txt

From 6db6ef44a639a4ebe255303bbb45e63c9d37e9b9 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 7 May 2024 12:32:09 +0200
Subject: [PATCH 076/150] feat(deps): remove deprecated and unmaintained paste
 and pastescripts

---
 requirements.txt                              | 11 ---
 rhodecode/lib/paster_commands/__init__.py     | 89 -------------------
 .../paster_commands/deprecated/__init__.py    |  0
 .../lib/paster_commands/deprecated/celeryd.py | 44 ---------
 .../deprecated/setup_rhodecode.py             | 42 ---------
 rhodecode/lib/paster_commands/ishell.py       | 85 ------------------
 rhodecode/lib/paster_commands/upgrade_db.py   | 63 -------------
 setup.py                                      |  9 +-
 8 files changed, 1 insertion(+), 342 deletions(-)
 delete mode 100644 rhodecode/lib/paster_commands/__init__.py
 delete mode 100644 rhodecode/lib/paster_commands/deprecated/__init__.py
 delete mode 100644 rhodecode/lib/paster_commands/deprecated/celeryd.py
 delete mode 100644 rhodecode/lib/paster_commands/deprecated/setup_rhodecode.py
 delete mode 100644 rhodecode/lib/paster_commands/ishell.py
 delete mode 100644 rhodecode/lib/paster_commands/upgrade_db.py

diff --git a/requirements.txt b/requirements.txt
index 36bf79c4..1d39c5ed 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -171,11 +171,6 @@ nbconvert==7.7.3
     webencodings==0.5.1
   traitlets==5.9.0
 orjson==3.10.3
-pastescript==3.6.0
-  paste==3.10.1
-    six==1.16.0
-  pastedeploy==3.1.0
-  six==1.16.0
 premailer==3.10.0
   cachetools==5.3.2
   cssselect==1.2.0
@@ -279,12 +274,6 @@ tzlocal==4.3
 unidecode==1.3.6
 urlobject==2.4.3
 waitress==3.0.0
-weberror==0.13.1
-  paste==3.10.1
-    six==1.16.0
-  pygments==2.15.1
-  tempita==0.5.2
-  webob==1.8.7
 webhelpers2==2.1
   markupsafe==2.1.2
   six==1.16.0
diff --git a/rhodecode/lib/paster_commands/__init__.py b/rhodecode/lib/paster_commands/__init__.py
deleted file mode 100644
index 8648297c..00000000
--- a/rhodecode/lib/paster_commands/__init__.py
+++ /dev/null
@@ -1,89 +0,0 @@
-
-# Copyright (C) 2010-2023 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-import os
-import logging
-
-from paste.script.command import Command, BadCommand
-
-
-class BasePasterCommand(Command):
-    """
-    Abstract Base Class for paster commands.
-
-    The celery commands are somewhat aggressive about loading
-    celery.conf, and since our module sets the `CELERY_LOADER`
-    environment variable to our loader, we have to bootstrap a bit and
-    make sure we've had a chance to load the pylons config off of the
-    command line, otherwise everything fails.
-    """
-    min_args = 1
-    min_args_error = "Please provide a paster config file as an argument."
-    takes_config_file = 1
-    requires_config_file = True
-
-    def notify_msg(self, msg, log=False):
-        """Make a notification to user, additionally if logger is passed
-        it logs this action using given logger
-
-        :param msg: message that will be printed to user
-        :param log: logging instance, to use to additionally log this message
-
-        """
-        if log and isinstance(log, logging):
-            log(msg)
-
-    def run(self, args):
-        """
-        Overrides Command.run
-
-        Checks for a config file argument and loads it.
-        """
-        if len(args) < self.min_args:
-            raise BadCommand(
-                self.min_args_error % {'min_args': self.min_args,
-                                       'actual_args': len(args)})
-
-        # Decrement because we're going to lob off the first argument.
-        # @@ This is hacky
-        self.min_args -= 1
-        self.bootstrap_config(args[0])
-        self.update_parser()
-        return super(BasePasterCommand, self).run(args[1:])
-
-    def update_parser(self):
-        """
-        Abstract method.  Allows for the class' parser to be updated
-        before the superclass' `run` method is called.  Necessary to
-        allow options/arguments to be passed through to the underlying
-        celery command.
-        """
-        raise NotImplementedError("Abstract Method.")
-
-    def bootstrap_config(self, conf):
-        """
-        Loads the pylons configuration.
-        """
-        self.path_to_ini_file = os.path.realpath(conf)
-
-    def _init_session(self):
-        """
-        Inits SqlAlchemy Session
-        """
-        logging.config.fileConfig(self.path_to_ini_file)
-
diff --git a/rhodecode/lib/paster_commands/deprecated/__init__.py b/rhodecode/lib/paster_commands/deprecated/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/rhodecode/lib/paster_commands/deprecated/celeryd.py b/rhodecode/lib/paster_commands/deprecated/celeryd.py
deleted file mode 100644
index 616d2301..00000000
--- a/rhodecode/lib/paster_commands/deprecated/celeryd.py
+++ /dev/null
@@ -1,44 +0,0 @@
-
-
-# Copyright (C) 2013-2023 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-
-from rhodecode.lib.paster_commands import BasePasterCommand
-
-
-class Command(BasePasterCommand):
-    """
-    Start the celery worker
-
-    Starts the celery worker that uses a paste.deploy configuration
-    file.
-    """
-    usage = 'CONFIG_FILE'
-    summary = __doc__.splitlines()[0]
-    description = "".join(__doc__.splitlines()[2:])
-
-    parser = BasePasterCommand.standard_parser(quiet=True)
-
-    def update_parser(self):
-        pass
-
-    def command(self):
-        cmd = 'celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --loglevel DEBUG --ini=%s' % self.path_to_ini_file
-        raise Exception('This Command is deprecated please run: %s' % cmd)
-
diff --git a/rhodecode/lib/paster_commands/deprecated/setup_rhodecode.py b/rhodecode/lib/paster_commands/deprecated/setup_rhodecode.py
deleted file mode 100644
index 1040ede1..00000000
--- a/rhodecode/lib/paster_commands/deprecated/setup_rhodecode.py
+++ /dev/null
@@ -1,42 +0,0 @@
-
-# Copyright (C) 2010-2023 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-
-from rhodecode.lib.paster_commands import BasePasterCommand
-
-
-class Command(BasePasterCommand):
-    """
-    Start the celery worker
-
-    Starts the celery worker that uses a paste.deploy configuration
-    file.
-    """
-    usage = 'CONFIG_FILE [celeryd options...]'
-    summary = __doc__.splitlines()[0]
-    description = "".join(__doc__.splitlines()[2:])
-
-    parser = BasePasterCommand.standard_parser(quiet=True)
-
-    def update_parser(self):
-        pass
-
-    def command(self):
-        cmd = 'rc-setup-app %s' % self.path_to_ini_file
-        raise Exception('This Command is deprecated please run: %s' % cmd)
diff --git a/rhodecode/lib/paster_commands/ishell.py b/rhodecode/lib/paster_commands/ishell.py
deleted file mode 100644
index b1714074..00000000
--- a/rhodecode/lib/paster_commands/ishell.py
+++ /dev/null
@@ -1,85 +0,0 @@
-
-
-# Copyright (C) 2013-2023 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-"""
-interactive shell paster command for RhodeCode
-"""
-
-import os
-import sys
-import logging
-
-from rhodecode.lib.paster_commands import BasePasterCommand
-
-# fix rhodecode import
-from os.path import dirname as dn
-rc_path = dn(dn(dn(os.path.realpath(__file__))))
-sys.path.append(rc_path)
-
-log = logging.getLogger(__name__)
-
-welcome_banner = """Welcome to RhodeCode iShell.
-Type `exit` to exit the shell.
-iShell is interactive shell to interact directly with the
-internal RhodeCode APIs. You can rescue your lost password,
-or reset some user/system settings.
-"""
-
-
-class Command(BasePasterCommand):
-
-    max_args = 1
-    min_args = 1
-
-    usage = "CONFIG_FILE"
-    group_name = "RhodeCode"
-    takes_config_file = -1
-    parser = BasePasterCommand.standard_parser(verbose=True)
-    summary = "Interactive shell"
-
-    def import_all_from_module(self, module_name):
-        import importlib
-        module = importlib.import_module(module_name)
-        globals().update({k: v for k, v in module.__dict__.items() if not k.startswith('_')})
-
-    def command(self):
-        # get SqlAlchemy session
-        self._init_session()
-
-        # imports, used in ipython shell
-        import os
-        import sys
-        import time
-        import shutil
-        import datetime
-        self.import_all_from_module('rhodecode.model.db')
-
-        try:
-            from IPython import embed
-            from traitlets.config import Config
-            cfg = Config()
-            cfg.InteractiveShellEmbed.confirm_exit = False
-            embed(config=cfg, banner1=welcome_banner)
-        except ImportError:
-            print('ipython installation required for ishell')
-            sys.exit(-1)
-
-    def update_parser(self):
-        pass
diff --git a/rhodecode/lib/paster_commands/upgrade_db.py b/rhodecode/lib/paster_commands/upgrade_db.py
deleted file mode 100644
index dbc18225..00000000
--- a/rhodecode/lib/paster_commands/upgrade_db.py
+++ /dev/null
@@ -1,63 +0,0 @@
-
-# Copyright (C) 2010-2023 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-import logging
-
-from rhodecode.lib.paster_commands import BasePasterCommand, Command
-
-log = logging.getLogger(__name__)
-
-
-class UpgradeDb(BasePasterCommand):
-    """
-    Command used for paster to upgrade our database to newer version
-    """
-
-    max_args = 1
-    min_args = 1
-
-    usage = "CONFIG_FILE"
-    summary = "Upgrades current db to newer version"
-    group_name = "RhodeCode"
-
-    parser = Command.standard_parser(verbose=True)
-
-    def command(self):
-        from rhodecode.lib.rc_commands import upgrade_db
-        upgrade_db.command(
-            self.path_to_ini_file, self.options.__dict__.get('force_ask'), None)
-
-    def update_parser(self):
-        self.parser.add_option('--sql',
-                      action='store_true',
-                      dest='just_sql',
-                      help="Prints upgrade sql for further investigation",
-                      default=False)
-
-        self.parser.add_option('--force-yes',
-                           action='store_true',
-                           dest='force_ask',
-                           default=None,
-                           help='Force yes to every question')
-        self.parser.add_option('--force-no',
-                           action='store_false',
-                           dest='force_ask',
-                           default=None,
-                           help='Force no to every question')
-
diff --git a/setup.py b/setup.py
index a5acd918..21b2be8c 100644
--- a/setup.py
+++ b/setup.py
@@ -176,18 +176,11 @@ setup(
             ('public/**', 'ignore', None),
         ]
     },
-    paster_plugins=['PasteScript'],
+
     entry_points={
         'paste.app_factory': [
             'main=rhodecode.config.middleware:make_pyramid_app',
         ],
-        'paste.global_paster_command': [
-            'ishell=rhodecode.lib.paster_commands.ishell:Command',
-            'upgrade-db=rhodecode.lib.paster_commands.upgrade_db:UpgradeDb',
-
-            'setup-rhodecode=rhodecode.lib.paster_commands.deprecated.setup_rhodecode:Command',
-            'celeryd=rhodecode.lib.paster_commands.deprecated.celeryd:Command',
-        ],
         'pyramid.pshell_runner': [
             'ipython = rhodecode.lib.pyramid_shell:ipython_shell_runner',
         ],

From 9bd31d36b1c7195dffccd9a817a475eeca77d697 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 7 May 2024 15:52:54 +0200
Subject: [PATCH 077/150] chore(deps): bumped pyramid-debugtoolbar

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 1d39c5ed..524f050e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -192,7 +192,7 @@ pycurl==7.45.3
 pymysql==1.0.3
 pyotp==2.8.0
 pyparsing==3.1.1
-pyramid-debugtoolbar==4.11
+pyramid-debugtoolbar==4.12.1
   pygments==2.15.1
   pyramid==2.0.2
     hupper==1.12

From 96822f93f128ed8f6d0cab7ca64edd8eba3fe2b6 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 7 May 2024 15:53:20 +0200
Subject: [PATCH 078/150] chore(deps): bumped cachetools

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 524f050e..a0761b27 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -172,7 +172,7 @@ nbconvert==7.7.3
   traitlets==5.9.0
 orjson==3.10.3
 premailer==3.10.0
-  cachetools==5.3.2
+  cachetools==5.3.3
   cssselect==1.2.0
   cssutils==2.6.0
   lxml==4.9.3

From e24f08dba327e256c3f51bb7774f6fba28fe61dd Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 7 May 2024 15:53:39 +0200
Subject: [PATCH 079/150] chore(deps): bumped redis lib

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index a0761b27..91574c99 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -251,7 +251,7 @@ python3-saml==1.15.0
   xmlsec==1.3.13
     lxml==4.9.3
 pyyaml==6.0.1
-redis==5.0.3
+redis==5.0.4
   async-timeout==4.0.3
 regex==2022.10.31
 routes==2.5.1

From bfae657d278332dcb1b7b113e1519fe192b70428 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 7 May 2024 15:53:57 +0200
Subject: [PATCH 080/150] chore(deps): bumped zope.interface lib

---
 requirements.txt | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 91574c99..03c9ef62 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -33,7 +33,7 @@ channelstream==0.7.1
   gevent==24.2.1
     greenlet==3.0.3
     zope.event==5.0.0
-    zope.interface==6.2.0
+    zope.interface==6.3.0
   itsdangerous==1.1.0
   marshmallow==2.18.0
   pyramid==2.0.2
@@ -46,7 +46,7 @@ channelstream==0.7.1
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.2.0
+    zope.interface==6.3.0
   pyramid-apispec==0.3.3
     apispec==1.3.3
   pyramid-jinja2==2.10
@@ -63,7 +63,7 @@ channelstream==0.7.1
       venusian==3.0.0
       webob==1.8.7
       zope.deprecation==5.0.0
-      zope.interface==6.2.0
+      zope.interface==6.3.0
     zope.deprecation==5.0.0
   python-dateutil==2.8.2
     six==1.16.0
@@ -95,7 +95,7 @@ gunicorn==21.2.0
 gevent==24.2.1
   greenlet==3.0.3
   zope.event==5.0.0
-  zope.interface==6.2.0
+  zope.interface==6.3.0
 ipython==8.14.0
   backcall==0.2.0
   decorator==5.1.1
@@ -204,7 +204,7 @@ pyramid-debugtoolbar==4.12.1
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.2.0
+    zope.interface==6.3.0
   pyramid-mako==1.1.0
     mako==1.2.4
       markupsafe==2.1.2
@@ -218,7 +218,7 @@ pyramid-debugtoolbar==4.12.1
       venusian==3.0.0
       webob==1.8.7
       zope.deprecation==5.0.0
-      zope.interface==6.2.0
+      zope.interface==6.3.0
 pyramid-mailer==0.15.1
   pyramid==2.0.2
     hupper==1.12
@@ -230,13 +230,13 @@ pyramid-mailer==0.15.1
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.2.0
+    zope.interface==6.3.0
   repoze.sendmail==4.4.1
     transaction==3.1.0
-      zope.interface==6.2.0
-    zope.interface==6.2.0
+      zope.interface==6.3.0
+    zope.interface==6.3.0
   transaction==3.1.0
-    zope.interface==6.2.0
+    zope.interface==6.3.0
 python-ldap==3.4.3
   pyasn1==0.4.8
   pyasn1-modules==0.2.8

From 590bbea4e30736058552192bd006268d4469fdc3 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 8 May 2024 13:27:19 +0200
Subject: [PATCH 081/150] tests: updated test ini and configs for setting
 custom ini files

---
 configs/production.ini             |  2 +-
 rhodecode/tests/conftest_common.py |  7 ++--
 rhodecode/tests/rhodecode.ini      | 60 +++++++++++++++++++-----------
 3 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/configs/production.ini b/configs/production.ini
index f8103bce..e18f6a04 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -255,7 +255,7 @@ file_store.enabled = true
 ; Storage backend, available options are: local
 file_store.backend = local
 
-; path to store the uploaded binaries
+; path to store the uploaded binaries and artifacts
 file_store.storage_path = /var/opt/rhodecode_data/file_store
 
 ; Uncomment and set this path to control settings for archive download cache.
diff --git a/rhodecode/tests/conftest_common.py b/rhodecode/tests/conftest_common.py
index 01bdbff4..a773c0fb 100644
--- a/rhodecode/tests/conftest_common.py
+++ b/rhodecode/tests/conftest_common.py
@@ -82,10 +82,11 @@ def pytest_addoption(parser):
     parser.addoption(
         '--test-loglevel', dest='test_loglevel',
         help="Set default Logging level for tests, critical(default), error, warn , info, debug")
-    group = parser.getgroup('pylons')
+
+    group = parser.getgroup('pyramid')
     group.addoption(
-        '--with-pylons', dest='pyramid_config',
-        help="Set up a Pylons environment with the specified config file.")
+        '--pyramid-config', dest='pyramid_config',
+        help="Set up a pyramid with the specified ini config file.")
     group.addoption(
         '--ini-config-override', action='store', type=_parse_json,
         default=None, dest='pyramid_config_override', help=(
diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini
index a58a0a0c..d458d349 100644
--- a/rhodecode/tests/rhodecode.ini
+++ b/rhodecode/tests/rhodecode.ini
@@ -104,6 +104,12 @@ startup.import_repos = true
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
+; Host at which the Service API is running.
+app.service_api.host = http://rhodecode.local:10020
+
+; Secret for Service API authentication.
+app.service_api.token =
+
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
@@ -206,6 +212,9 @@ auth_ret_code_detection = false
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
+; Filesystem location were repositories should be stored
+repo_store.path = /var/opt/rhodecode_repo_store
+
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
@@ -246,8 +255,8 @@ file_store.enabled = true
 ; Storage backend, available options are: local
 file_store.backend = local
 
-; path to store the uploaded binaries
-file_store.storage_path = %(here)s/data/file_store
+; path to store the uploaded binaries and artifacts
+file_store.storage_path = /var/opt/rhodecode_data/file_store
 
 ; Uncomment and set this path to control settings for archive download cache.
 ; Generated repo archives will be cached at this location
@@ -256,7 +265,7 @@ file_store.storage_path = %(here)s/data/file_store
 ; RhodeCode and vcsserver
 
 ; Default is $cache_dir/archive_cache if not set
-archive_cache.store_dir = /tmp/rc-test-data/archive_cache
+archive_cache.store_dir = /var/opt/rhodecode_data/tarballcache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
 archive_cache.cache_size_gb = 10
@@ -276,7 +285,10 @@ use_celery = false
 #celerybeat-schedule.path =
 
 ; connection url to the message broker (default redis)
-celery.broker_url = redis://localhost:6379/8
+celery.broker_url = redis://redis:6379/8
+
+; results backend to get results for (default redis)
+celery.result_backend = redis://redis:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
@@ -285,7 +297,8 @@ celery.broker_url = redis://localhost:6379/8
 celery.max_tasks_per_child = 20
 
 ; tasks will never be sent to the queue, but executed locally instead.
-celery.task_always_eager = false
+celery.task_always_eager = true
+celery.task_store_eager_result = true
 
 ; #############
 ; DOGPILE CACHE
@@ -293,7 +306,7 @@ celery.task_always_eager = false
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
-cache_dir = %(here)s/rc-test-data
+cache_dir = /var/opt/rhodecode_data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
@@ -400,14 +413,14 @@ rc_cache.cache_repo.arguments.filename = %(here)s/cache-backend/cache_repo_db
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
-; types are file, ext:redis, ext:database, ext:memcached, and memory (default if not specified).
-; Fastest ones are Redis and ext:database
+; types are file, ext:redis, ext:database, ext:memcached
+; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/rc-tests/data/sessions
 
 ; Redis based sessions
 #beaker.session.type = ext:redis
-#beaker.session.url = redis://127.0.0.1:6379/2
+#beaker.session.url = redis://redis:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
@@ -437,9 +450,6 @@ beaker.session.httponly = true
 ; Set https secure cookie
 beaker.session.secure = false
 
-## auto save the session to not to use .save()
-beaker.session.auto = false
-
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
@@ -552,11 +562,6 @@ vcs.backends = hg, git, svn
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
-; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
-; Set a numeric version for your current SVN e.g 1.8, or 1.12
-; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
-#vcs.svn.compatible_version = 1.8
-
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = false
@@ -566,6 +571,17 @@ vcs.methods.cache = false
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
+; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
+; Set a numeric version for your current SVN e.g 1.8, or 1.12
+; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
+#vcs.svn.compatible_version = 1.8
+
+; Enable SVN proxy of requests over HTTP
+vcs.svn.proxy.enabled = true
+
+; host to connect to running SVN subsystem
+vcs.svn.proxy.host = http://svn:8090
+
 ; Enable or disable the config file generation.
 svn.proxy.generate_config = false
 
@@ -613,8 +629,8 @@ ssh.authorized_keys_file_path = %(here)s/rc/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
-; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
-ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper
+; e.g /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
@@ -625,9 +641,9 @@ ssh.enable_debug_logging = false
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
-ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
-ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
-ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
+ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg
+ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git
+ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.

From 8c10af342e97a710d99a2f214d4d5fd187158f9c Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 9 May 2024 08:34:33 +0200
Subject: [PATCH 082/150] fix(permissions): fixed missing tempita lib

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 03c9ef62..104910f8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -271,6 +271,7 @@ supervisor==4.2.5
 tzlocal==4.3
   pytz-deprecation-shim==0.1.0.post0
     tzdata==2024.1
+tempita==0.5.2
 unidecode==1.3.6
 urlobject==2.4.3
 waitress==3.0.0

From d05187a20c8b2b7f5f6f528e6c083f657db2534b Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 9 May 2024 11:34:26 +0200
Subject: [PATCH 083/150] fix(deps): fixed paste removal, we'\''re not ready to
 do that yet...

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 104910f8..00929d76 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -171,6 +171,7 @@ nbconvert==7.7.3
     webencodings==0.5.1
   traitlets==5.9.0
 orjson==3.10.3
+paste==3.10.1
 premailer==3.10.0
   cachetools==5.3.3
   cssselect==1.2.0

From 32f96b1ae9162bc696784202bcf40a241d78096b Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 13 May 2024 08:24:49 +0200
Subject: [PATCH 084/150] feat(svn): re-enable manual generation of SVN config
 that was removed when we moved svn path config to .ini files

- This still allows to trigger the config creation
- covers some edge cases on a need ro rre-create the svn configs
---
 rhodecode/apps/admin/views/settings.py     |  8 ++++----
 rhodecode/templates/base/vcs_settings.mako | 17 +++++++++++++++++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/rhodecode/apps/admin/views/settings.py b/rhodecode/apps/admin/views/settings.py
index 5443da02..3927d437 100644
--- a/rhodecode/apps/admin/views/settings.py
+++ b/rhodecode/apps/admin/views/settings.py
@@ -115,8 +115,8 @@ class AdminSettingsView(BaseAppView):
         c.svn_tag_patterns = model.get_global_svn_tag_patterns()
 
         settings = self.request.registry.settings
-        c.svn_proxy_generate_config = settings[generate_config]
-
+        c.svn_generate_config = settings[generate_config]
+        c.svn_config_path = rhodecode.ConfigGet().get_str('svn.proxy.config_file_path')
         defaults = self._form_defaults()
 
         model.create_largeobjects_dirs_if_needed(defaults['paths_root_path'])
@@ -144,8 +144,8 @@ class AdminSettingsView(BaseAppView):
         c.svn_tag_patterns = model.get_global_svn_tag_patterns()
 
         settings = self.request.registry.settings
-        c.svn_proxy_generate_config = settings[generate_config]
-
+        c.svn_generate_config = settings[generate_config]
+        c.svn_config_path = rhodecode.ConfigGet().get_str('svn.proxy.config_file_path')
         application_form = ApplicationUiSettingsForm(self.request.translate)()
 
         try:
diff --git a/rhodecode/templates/base/vcs_settings.mako b/rhodecode/templates/base/vcs_settings.mako
index 4a42fafc..2b1c01a6 100644
--- a/rhodecode/templates/base/vcs_settings.mako
+++ b/rhodecode/templates/base/vcs_settings.mako
@@ -148,6 +148,23 @@
                 <h3 class="panel-title">${_('Subversion Settings')}<a class="permalink" href="#vcs-svn-options"> ¶</a></h3>
             </div>
             <div class="panel-body">
+                <div class="field">
+                    <div class="content" >
+                        <label>${_('mod_dav config')}</label><br/>
+                        <code>path: ${c.svn_config_path}</code>
+                    </div>
+                    <br/>
+
+                    <div>
+
+                    % if c.svn_generate_config:
+                        <span class="buttons">
+                            <button class="btn btn-primary" id="vcs_svn_generate_cfg">${_('Re-generate Apache Config')}</button>
+                        </span>
+                    % endif
+                    </div>
+                </div>
+
                 <div class="field">
                     <div class="content" >
                         <label>${_('Repository patterns')}</label><br/>

From 16f04fd3ed6e8f94c30d3b04056dfec982fc9f65 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 13 May 2024 12:32:37 +0200
Subject: [PATCH 085/150] fix(svn-configs): cleanup code and fixed generate
 config on repo settings page which should not be visible

---
 rhodecode/apps/admin/views/settings.py        | 13 +++++--------
 .../repository/views/repo_settings_vcs.py     |  9 +++++++--
 .../admin/settings/settings_vcs.mako          | 11 -----------
 rhodecode/templates/base/vcs_settings.mako    | 19 +++++++++++++++++++
 4 files changed, 31 insertions(+), 21 deletions(-)

diff --git a/rhodecode/apps/admin/views/settings.py b/rhodecode/apps/admin/views/settings.py
index 3927d437..689ffa8a 100644
--- a/rhodecode/apps/admin/views/settings.py
+++ b/rhodecode/apps/admin/views/settings.py
@@ -32,7 +32,7 @@ from pyramid.response import Response
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.apps._base.navigation import navigation_list
-from rhodecode.apps.svn_support.config_keys import generate_config
+from rhodecode.apps.svn_support import config_keys
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
@@ -113,10 +113,8 @@ class AdminSettingsView(BaseAppView):
         model = VcsSettingsModel()
         c.svn_branch_patterns = model.get_global_svn_branch_patterns()
         c.svn_tag_patterns = model.get_global_svn_tag_patterns()
-
-        settings = self.request.registry.settings
-        c.svn_generate_config = settings[generate_config]
-        c.svn_config_path = rhodecode.ConfigGet().get_str('svn.proxy.config_file_path')
+        c.svn_generate_config = rhodecode.ConfigGet().get_bool(config_keys.generate_config)
+        c.svn_config_path = rhodecode.ConfigGet().get_str(config_keys.config_file_path)
         defaults = self._form_defaults()
 
         model.create_largeobjects_dirs_if_needed(defaults['paths_root_path'])
@@ -143,9 +141,8 @@ class AdminSettingsView(BaseAppView):
         c.svn_branch_patterns = model.get_global_svn_branch_patterns()
         c.svn_tag_patterns = model.get_global_svn_tag_patterns()
 
-        settings = self.request.registry.settings
-        c.svn_generate_config = settings[generate_config]
-        c.svn_config_path = rhodecode.ConfigGet().get_str('svn.proxy.config_file_path')
+        c.svn_generate_config = rhodecode.ConfigGet().get_bool(config_keys.generate_config)
+        c.svn_config_path = rhodecode.ConfigGet().get_str(config_keys.config_file_path)
         application_form = ApplicationUiSettingsForm(self.request.translate)()
 
         try:
diff --git a/rhodecode/apps/repository/views/repo_settings_vcs.py b/rhodecode/apps/repository/views/repo_settings_vcs.py
index 162da830..b8d8e0a1 100644
--- a/rhodecode/apps/repository/views/repo_settings_vcs.py
+++ b/rhodecode/apps/repository/views/repo_settings_vcs.py
@@ -24,7 +24,9 @@ from pyramid.httpexceptions import HTTPFound, HTTPBadRequest
 from pyramid.response import Response
 from pyramid.renderers import render
 
+import rhodecode
 from rhodecode.apps._base import RepoAppView
+from rhodecode.apps.svn_support import config_keys
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
@@ -38,8 +40,6 @@ log = logging.getLogger(__name__)
 class RepoSettingsVcsView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-
-
         return c
 
     def _vcs_form_defaults(self, repo_name):
@@ -77,6 +77,9 @@ class RepoSettingsVcsView(RepoAppView):
         c.svn_branch_patterns = model.get_repo_svn_branch_patterns()
         c.svn_tag_patterns = model.get_repo_svn_tag_patterns()
 
+        c.svn_generate_config = rhodecode.ConfigGet().get_bool(config_keys.generate_config)
+        c.svn_config_path = rhodecode.ConfigGet().get_str(config_keys.config_file_path)
+
         defaults = self._vcs_form_defaults(self.db_repo_name)
         c.inherit_global_settings = defaults['inherit_global_settings']
 
@@ -103,6 +106,8 @@ class RepoSettingsVcsView(RepoAppView):
         c.global_svn_tag_patterns = model.get_global_svn_tag_patterns()
         c.svn_branch_patterns = model.get_repo_svn_branch_patterns()
         c.svn_tag_patterns = model.get_repo_svn_tag_patterns()
+        c.svn_generate_config = rhodecode.ConfigGet().get_bool(config_keys.generate_config)
+        c.svn_config_path = rhodecode.ConfigGet().get_str(config_keys.config_file_path)
 
         defaults = self._vcs_form_defaults(self.db_repo_name)
         c.inherit_global_settings = defaults['inherit_global_settings']
diff --git a/rhodecode/templates/admin/settings/settings_vcs.mako b/rhodecode/templates/admin/settings/settings_vcs.mako
index 4b6dd11a..b49ad2e9 100644
--- a/rhodecode/templates/admin/settings/settings_vcs.mako
+++ b/rhodecode/templates/admin/settings/settings_vcs.mako
@@ -49,16 +49,5 @@ ${h.end_form()}
             unlockpath();
         }
 
-        /* On click handler for the `Generate Apache Config` button. It sends a
-        POST request to trigger the (re)generation of the mod_dav_svn config. */
-        $('#vcs_svn_generate_cfg').on('click', function(event) {
-            event.preventDefault();
-            var url = "${h.route_path('admin_settings_vcs_svn_generate_cfg')}";
-            var jqxhr = $.post(url, {'csrf_token': CSRF_TOKEN});
-            jqxhr.done(function(data) {
-                $.Topic('/notifications').publish(data);
-            });
-        });
-
     });
 </script>
diff --git a/rhodecode/templates/base/vcs_settings.mako b/rhodecode/templates/base/vcs_settings.mako
index 2b1c01a6..934e5bbb 100644
--- a/rhodecode/templates/base/vcs_settings.mako
+++ b/rhodecode/templates/base/vcs_settings.mako
@@ -148,6 +148,7 @@
                 <h3 class="panel-title">${_('Subversion Settings')}<a class="permalink" href="#vcs-svn-options"> ¶</a></h3>
             </div>
             <div class="panel-body">
+                % if display_globals:
                 <div class="field">
                     <div class="content" >
                         <label>${_('mod_dav config')}</label><br/>
@@ -164,6 +165,7 @@
                     % endif
                     </div>
                 </div>
+                % endif
 
                 <div class="field">
                     <div class="content" >
@@ -324,5 +326,22 @@
 ##         </div>
 ##     % endif
 
+<script type="text/javascript">
 
+$(document).ready(function() {
+    /* On click handler for the `Generate Apache Config` button. It sends a
+    POST request to trigger the (re)generation of the mod_dav_svn config. */
+    $('#vcs_svn_generate_cfg').on('click', function(event) {
+        event.preventDefault();
+        alert('i cliked it !!')
+        var url = "${h.route_path('admin_settings_vcs_svn_generate_cfg')}";
+        var jqxhr = $.post(url, {'csrf_token': CSRF_TOKEN});
+        jqxhr.done(function(data) {
+            $.Topic('/notifications').publish(data);
+        });
+    });
+});
+
+</script>
 </%def>
+

From aea08fa1b2116e101b91962d421f0dfdac439137 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 15 May 2024 10:23:32 +0200
Subject: [PATCH 086/150] docs: added missing release notes files

---
 docs/release-notes/release-notes-5.0.0.rst | 51 ++++++++++++++++++++++
 docs/release-notes/release-notes-5.1.0.rst | 49 +++++++++++++++++++++
 docs/release-notes/release-notes.rst       | 11 +++++
 3 files changed, 111 insertions(+)
 create mode 100644 docs/release-notes/release-notes-5.0.0.rst
 create mode 100644 docs/release-notes/release-notes-5.1.0.rst

diff --git a/docs/release-notes/release-notes-5.0.0.rst b/docs/release-notes/release-notes-5.0.0.rst
new file mode 100644
index 00000000..8a214018
--- /dev/null
+++ b/docs/release-notes/release-notes-5.0.0.rst
@@ -0,0 +1,51 @@
+|RCE| 5.0.0 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-05-14
+
+
+New Features
+^^^^^^^^^^^^
+
+- Full support of Python3 and Python3.11
+- Git repositories with LFS object are now pushing and pulling the LFS objects when remote sync is enabled.
+- Archive generation: implemented a new system for caching generated archive files that allows setting cache size limit see: archive_cache.cache_size_gb= option.
+- Introduced statsd metrics in various places for new monitoring stack to provide useful details on traffic and usage.
+
+
+General
+^^^^^^^
+
+- Upgraded all dependency libraries to their latest available versions for python3 compatability
+
+
+Security
+^^^^^^^^
+
+- fixed few edge cases of permission invalidation on change of permissions
+
+
+Performance
+^^^^^^^^^^^
+
+- Moved lot of dulwich based code into libgit2 implementation for better performance
+
+
+Fixes
+^^^^^
+
+- Various small fixes and improvements found during python3 migration
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.0.0 is a mayor upgrade that support latest Python versions 3.11,
+  and also comes with lots of internal changes and performance improvements.
+- RhodeCode 5.0.0 is only supported via our new docker rcstack installer or helm/k8s stack
+- Please see migration guide :ref:`rcstack:migration-to-rcstack`
+- RhodeCode 5.0.0 also requires a new license for EE instances, please reach out to us via support@rhodecode.com to
+  convert your license key.
diff --git a/docs/release-notes/release-notes-5.1.0.rst b/docs/release-notes/release-notes-5.1.0.rst
new file mode 100644
index 00000000..bbb4fa41
--- /dev/null
+++ b/docs/release-notes/release-notes-5.1.0.rst
@@ -0,0 +1,49 @@
+|RCE| 5.1.0 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- TBA
+- available as docker "beta" tag rhodecode/rhodecode-ce:beta, rhodecode/rhodecode-ee:beta
+
+
+New Features
+^^^^^^^^^^^^
+
+- We've introduced 2FA for users. Now alongside the external auth 2fa support RhodeCode allows to enable 2FA for users
+  2FA options will be available for each user individually, or enforced via authentication plugins like ldap, or internal.
+- Email based log-in. RhodeCode now allows to log-in using email as well as username for main authentication type.
+- Ability to replace a file using web UI. Now one can replace an existing file from the web-ui.
+- GIT LFS Sync automation. Remote push/pull commands now can also sync GIT LFS objects.
+
+General
+^^^^^^^
+
+- Upgraded all dependency libraries to their latest available versions
+- Repository storage is no longer controlled via DB settings, but .ini file. This allows easier automated deployments.
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+- Introduced a full rewrite of ssh backend for performance. The result is 2-5x speed improvement for operation with ssh.
+  enable new ssh wrapper by setting: `ssh.wrapper_cmd = /home/rhodecode/venv/bin/rc-ssh-wrapper-v2`
+
+
+Fixes
+^^^^^
+
+- Zip archive download breaks when a gitmodules file is present
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.1.0 is a mayor feature release after big 5.0.0 python3 migration. Happy to ship a first time feature
+  rich release
diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst
index 7474efcf..c7d8e52c 100644
--- a/docs/release-notes/release-notes.rst
+++ b/docs/release-notes/release-notes.rst
@@ -3,6 +3,17 @@
 Release Notes
 =============
 
+
+|RCE| 5.x Versions
+------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   release-notes-5.0.0.rst
+   release-notes-5.1.0.rst
+
+
 |RCE| 4.x Versions
 ------------------
 

From d069311e52eb319b821d89e8521c104580e44b4f Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Mon, 13 May 2024 16:24:31 +0300
Subject: [PATCH 087/150] feat(2fa): added 2fa for more auth plugins and moved
 2fa forced functionality to ee edition. Fixes: RCCE-68

---
 rhodecode/apps/_base/__init__.py                   |  3 ---
 rhodecode/authentication/plugins/auth_crowd.py     |  4 ++--
 rhodecode/authentication/plugins/auth_jasig_cas.py |  4 ++--
 rhodecode/authentication/plugins/auth_ldap.py      |  4 ++--
 rhodecode/authentication/plugins/auth_pam.py       |  4 ++--
 rhodecode/authentication/plugins/auth_rhodecode.py | 13 ++-----------
 rhodecode/authentication/schema.py                 | 14 ++++++++++++++
 rhodecode/model/db.py                              |  5 ++---
 .../templates/admin/auth/plugin_settings.mako      |  9 +++++++--
 9 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/rhodecode/apps/_base/__init__.py b/rhodecode/apps/_base/__init__.py
index c98789d7..1303d984 100644
--- a/rhodecode/apps/_base/__init__.py
+++ b/rhodecode/apps/_base/__init__.py
@@ -176,9 +176,6 @@ class BaseAppView(object):
         if not user_obj:
             return
 
-        if user_obj.has_forced_2fa and user_obj.extern_type != 'rhodecode':
-            return
-
         if user_obj.needs_2fa_configure and view_name != self.SETUP_2FA_VIEW:
             h.flash(
                 "You are required to configure 2FA",
diff --git a/rhodecode/authentication/plugins/auth_crowd.py b/rhodecode/authentication/plugins/auth_crowd.py
index 6b625452..613cc487 100644
--- a/rhodecode/authentication/plugins/auth_crowd.py
+++ b/rhodecode/authentication/plugins/auth_crowd.py
@@ -31,7 +31,7 @@ import urllib.parse
 from rhodecode.translation import _
 from rhodecode.authentication.base import (
     RhodeCodeExternalAuthPlugin, hybrid_property)
-from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
+from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase, TwoFactorAuthnPluginSettingsSchemaMixin
 from rhodecode.authentication.routes import AuthnPluginResourceBase
 from rhodecode.lib.colander_utils import strip_whitespace
 from rhodecode.lib.ext_json import json, formatted_json
@@ -53,7 +53,7 @@ class CrowdAuthnResource(AuthnPluginResourceBase):
     pass
 
 
-class CrowdSettingsSchema(AuthnPluginSettingsSchemaBase):
+class CrowdSettingsSchema(TwoFactorAuthnPluginSettingsSchemaMixin, AuthnPluginSettingsSchemaBase):
     host = colander.SchemaNode(
         colander.String(),
         default='127.0.0.1',
diff --git a/rhodecode/authentication/plugins/auth_jasig_cas.py b/rhodecode/authentication/plugins/auth_jasig_cas.py
index 43dd8603..ec7b67c6 100644
--- a/rhodecode/authentication/plugins/auth_jasig_cas.py
+++ b/rhodecode/authentication/plugins/auth_jasig_cas.py
@@ -33,7 +33,7 @@ import urllib.error
 from rhodecode.translation import _
 from rhodecode.authentication.base import (
     RhodeCodeExternalAuthPlugin, hybrid_property)
-from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
+from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase, TwoFactorAuthnPluginSettingsSchemaMixin
 from rhodecode.authentication.routes import AuthnPluginResourceBase
 from rhodecode.lib.colander_utils import strip_whitespace
 from rhodecode.model.db import User
@@ -55,7 +55,7 @@ class JasigCasAuthnResource(AuthnPluginResourceBase):
     pass
 
 
-class JasigCasSettingsSchema(AuthnPluginSettingsSchemaBase):
+class JasigCasSettingsSchema(TwoFactorAuthnPluginSettingsSchemaMixin, AuthnPluginSettingsSchemaBase):
     service_url = colander.SchemaNode(
         colander.String(),
         default='https://domain.com/cas/v1/tickets',
diff --git a/rhodecode/authentication/plugins/auth_ldap.py b/rhodecode/authentication/plugins/auth_ldap.py
index 793836fa..c4e72b4e 100644
--- a/rhodecode/authentication/plugins/auth_ldap.py
+++ b/rhodecode/authentication/plugins/auth_ldap.py
@@ -27,7 +27,7 @@ import colander
 from rhodecode.translation import _
 from rhodecode.authentication.base import (
     RhodeCodeExternalAuthPlugin, AuthLdapBase, hybrid_property)
-from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
+from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase, TwoFactorAuthnPluginSettingsSchemaMixin
 from rhodecode.authentication.routes import AuthnPluginResourceBase
 from rhodecode.lib.colander_utils import strip_whitespace
 from rhodecode.lib.exceptions import (
@@ -245,7 +245,7 @@ class AuthLdap(AuthLdapBase):
         return dn, user_attrs
 
 
-class LdapSettingsSchema(AuthnPluginSettingsSchemaBase):
+class LdapSettingsSchema(TwoFactorAuthnPluginSettingsSchemaMixin, AuthnPluginSettingsSchemaBase):
     tls_kind_choices = ['PLAIN', 'LDAPS', 'START_TLS']
     tls_reqcert_choices = ['NEVER', 'ALLOW', 'TRY', 'DEMAND', 'HARD']
     search_scope_choices = ['BASE', 'ONELEVEL', 'SUBTREE']
diff --git a/rhodecode/authentication/plugins/auth_pam.py b/rhodecode/authentication/plugins/auth_pam.py
index 43cf77bc..52c18fb2 100644
--- a/rhodecode/authentication/plugins/auth_pam.py
+++ b/rhodecode/authentication/plugins/auth_pam.py
@@ -31,7 +31,7 @@ import socket
 from rhodecode.translation import _
 from rhodecode.authentication.base import (
     RhodeCodeExternalAuthPlugin, hybrid_property)
-from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
+from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase, TwoFactorAuthnPluginSettingsSchemaMixin
 from rhodecode.authentication.routes import AuthnPluginResourceBase
 from rhodecode.lib.colander_utils import strip_whitespace
 
@@ -51,7 +51,7 @@ class PamAuthnResource(AuthnPluginResourceBase):
     pass
 
 
-class PamSettingsSchema(AuthnPluginSettingsSchemaBase):
+class PamSettingsSchema(TwoFactorAuthnPluginSettingsSchemaMixin, AuthnPluginSettingsSchemaBase):
     service = colander.SchemaNode(
         colander.String(),
         default='login',
diff --git a/rhodecode/authentication/plugins/auth_rhodecode.py b/rhodecode/authentication/plugins/auth_rhodecode.py
index 91a64377..ffa355d6 100644
--- a/rhodecode/authentication/plugins/auth_rhodecode.py
+++ b/rhodecode/authentication/plugins/auth_rhodecode.py
@@ -27,7 +27,7 @@ import colander
 from rhodecode.translation import _
 from rhodecode.lib.utils2 import safe_bytes
 from rhodecode.model.db import User
-from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
+from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase, TwoFactorAuthnPluginSettingsSchemaMixin
 from rhodecode.authentication.base import (
     RhodeCodeAuthPluginBase, hybrid_property, HTTP_TYPE, VCS_TYPE)
 from rhodecode.authentication.routes import AuthnPluginResourceBase
@@ -182,16 +182,7 @@ class RhodeCodeAuthPlugin(RhodeCodeAuthPluginBase):
             return None
 
 
-class RhodeCodeSettingsSchema(AuthnPluginSettingsSchemaBase):
-    global_2fa = colander.SchemaNode(
-        colander.Bool(),
-        default=False,
-        description=_('Force all users to use two factor authentication by enabling this.'),
-        missing=False,
-        title=_('Global 2FA'),
-        widget='bool',
-    )
-
+class RhodeCodeSettingsSchema(TwoFactorAuthnPluginSettingsSchemaMixin, AuthnPluginSettingsSchemaBase):
     auth_restriction_choices = [
         (RhodeCodeAuthPlugin.AUTH_RESTRICTION_NONE, 'All users'),
         (RhodeCodeAuthPlugin.AUTH_RESTRICTION_SUPER_ADMIN, 'Super admins only'),
diff --git a/rhodecode/authentication/schema.py b/rhodecode/authentication/schema.py
index e20fe235..49371064 100644
--- a/rhodecode/authentication/schema.py
+++ b/rhodecode/authentication/schema.py
@@ -48,3 +48,17 @@ class AuthnPluginSettingsSchemaBase(colander.MappingSchema):
         validator=colander.Range(min=0, max=None),
         widget='int',
     )
+
+
+class TwoFactorAuthnPluginSettingsSchemaMixin(colander.MappingSchema):
+    """
+    Mixin for extending plugins with two-factor authentication option.
+    """
+    global_2fa = colander.SchemaNode(
+        colander.Bool(),
+        default=False,
+        description=_('Force all users to use two factor authentication with this plugin.'),
+        missing=False,
+        title=_('enforce 2FA for users'),
+        widget='bool',
+    )
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index 848c7a88..8a9b200a 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -810,11 +810,10 @@ class User(Base, BaseModel):
     @hybrid_property
     def has_forced_2fa(self):
         """
-        Checks if 2fa was forced for ALL users (including current one)
+        Checks if 2fa was forced for current user
         """
         from rhodecode.model.settings import SettingsModel
-        # So now we're supporting only auth_rhodecode_global_2fa
-        if value := SettingsModel().get_setting_by_name('auth_rhodecode_global_2fa'):
+        if value := SettingsModel().get_setting_by_name(f'{self.extern_type}_global_2fa'):
             return value.app_settings_value
         return False
 
diff --git a/rhodecode/templates/admin/auth/plugin_settings.mako b/rhodecode/templates/admin/auth/plugin_settings.mako
index 07fbac41..0f09b80c 100644
--- a/rhodecode/templates/admin/auth/plugin_settings.mako
+++ b/rhodecode/templates/admin/auth/plugin_settings.mako
@@ -63,7 +63,12 @@
                         %elif node.widget == "password":
                           ${h.password(node.name, defaults.get(node.name), class_="large")}
                         %elif node.widget == "bool":
-                          <div class="checkbox">${h.checkbox(node.name, True, checked=defaults.get(node.name))}</div>
+                            %if node.name == "global_2fa" and c.rhodecode_edition_id != "EE":
+                                <input type="checkbox" disabled/>
+                                <%node.description = _('This feature is available in RhodeCode EE edition only. Contact {sales_email} to obtain a trial license.').format(sales_email='<a href="mailto:sales@rhodecode.com">sales@rhodecode.com</a>')%>
+                            %else:
+                                <div class="checkbox" >${h.checkbox(node.name, True, checked=defaults.get(node.name))}</div>
+                            %endif
                         %elif node.widget == "select":
                           ${h.select(node.name, defaults.get(node.name), node.validator.choices, class_="select2AuthSetting")}
                         %elif node.widget == "select_with_labels":
@@ -80,7 +85,7 @@
                           <span class="error-message">${errors.get(node.name)}</span>
                           <br />
                         %endif
-                        <p class="help-block pre-formatting">${node.description}</p>
+                        <p class="help-block pre-formatting">${node.description | n}</p>
                       </div>
                     </div>
                   %endfor

From f0066be1ff165404dc3e68024b08f74993f43290 Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Thu, 16 May 2024 16:01:22 +0300
Subject: [PATCH 088/150] fix(forced 2fa): fixed forced 2fa functionality.
 Fixes: RCCE-68

---
 rhodecode/apps/_base/__init__.py | 2 ++
 rhodecode/model/db.py            | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/rhodecode/apps/_base/__init__.py b/rhodecode/apps/_base/__init__.py
index 1303d984..290a903e 100644
--- a/rhodecode/apps/_base/__init__.py
+++ b/rhodecode/apps/_base/__init__.py
@@ -182,6 +182,8 @@ class BaseAppView(object):
                 "warning",
                 ignore_duplicate=False,
             )
+            # Special case for users created "on the fly" (ldap case for new user)
+            user_obj.check_2fa_required = False
             raise HTTPFound(self.request.route_path(self.SETUP_2FA_VIEW))
 
     def _maybe_needs_2fa_check(self, view_name, user_obj):
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index 8a9b200a..b2af4559 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -813,7 +813,7 @@ class User(Base, BaseModel):
         Checks if 2fa was forced for current user
         """
         from rhodecode.model.settings import SettingsModel
-        if value := SettingsModel().get_setting_by_name(f'{self.extern_type}_global_2fa'):
+        if value := SettingsModel().get_setting_by_name(f'auth_{self.extern_type}_global_2fa'):
             return value.app_settings_value
         return False
 

From 8c207d7cde209a7d43261777f54b5f9363f575a3 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 20 May 2024 09:00:12 +0200
Subject: [PATCH 089/150] chore(debug): improved logging for simplesvn

---
 rhodecode/lib/middleware/simplesvn.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rhodecode/lib/middleware/simplesvn.py b/rhodecode/lib/middleware/simplesvn.py
index 1faec9e7..12b12c8d 100644
--- a/rhodecode/lib/middleware/simplesvn.py
+++ b/rhodecode/lib/middleware/simplesvn.py
@@ -54,12 +54,12 @@ class SimpleSvnApp(object):
         request_headers = self._get_request_headers(environ)
         data_io = environ['wsgi.input']
         req_method: str = environ['REQUEST_METHOD']
-        has_content_length = 'CONTENT_LENGTH' in environ
+        has_content_length: bool = 'CONTENT_LENGTH' in environ
 
         path_info = self._get_url(
             self.config.get('subversion_http_server_url', ''), get_path_info(environ))
         transfer_encoding = environ.get('HTTP_TRANSFER_ENCODING', '')
-        log.debug('Handling: %s method via `%s`', req_method, path_info)
+        log.debug('Handling: %s method via `%s` has_content_length:%s', req_method, path_info, has_content_length)
 
         # stream control flag, based on request and content type...
         stream = False

From 0f0d045b9bca892cf8c451a172b6484685859275 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 20 May 2024 10:01:35 +0200
Subject: [PATCH 090/150] configs: update .ini files

---
 configs/development.ini | 8 +++++---
 configs/production.ini  | 4 +++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/configs/development.ini b/configs/development.ini
index 3e7fd702..cc3954d8 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -304,7 +304,7 @@ file_store.enabled = true
 ; Storage backend, available options are: local
 file_store.backend = local
 
-; path to store the uploaded binaries
+; path to store the uploaded binaries and artifacts
 file_store.storage_path = /var/opt/rhodecode_data/file_store
 
 ; Uncomment and set this path to control settings for archive download cache.
@@ -320,7 +320,7 @@ archive_cache.store_dir = /var/opt/rhodecode_data/tarballcache
 archive_cache.cache_size_gb = 10
 
 ; By default cache uses sharding technique, this specifies how many shards are there
-archive_cache.cache_shards = 10
+archive_cache.cache_shards = 4
 
 ; #############
 ; CELERY CONFIG
@@ -591,6 +591,7 @@ vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
+; `celery` - use celery based hooks
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
@@ -677,7 +678,8 @@ ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecod
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
-; e.g /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
diff --git a/configs/production.ini b/configs/production.ini
index e18f6a04..e64e3538 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -542,6 +542,7 @@ vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
+; `celery` - use celery based hooks
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
@@ -628,7 +629,8 @@ ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecod
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
-; e.g /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command

From f3e09a59959135b3e1b96fb6fe9b7dcb41de86c6 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 20 May 2024 11:14:50 +0200
Subject: [PATCH 091/150] docs(changelog): added 5.0.1 changelog items

---
 docs/release-notes/release-notes-5.0.1.rst | 42 ++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 docs/release-notes/release-notes-5.0.1.rst

diff --git a/docs/release-notes/release-notes-5.0.1.rst b/docs/release-notes/release-notes-5.0.1.rst
new file mode 100644
index 00000000..5cc5c495
--- /dev/null
+++ b/docs/release-notes/release-notes-5.0.1.rst
@@ -0,0 +1,42 @@
+|RCE| 5.0.0 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-05-20
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+
+
+
+Fixes
+^^^^^
+
+- Fixed Celery serialization issues
+- Fixed SVN hooks binary dir paths which in certain scenarios resulted in empty values forbidding hooks to execute
+- Fixed annotation bug for files without new lines or mixed newlines
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.0.1 is unscheduled bugfix release to address some of the issues found during 4.X -> 5.X migration

From 29de931a2ee05cb0ba91bd7d98b24cf6365e7da8 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 22 May 2024 15:35:24 +0200
Subject: [PATCH 092/150] fix(celery): signal startup failure with sys.exit to
 recycle celery worker on errors

---
 rhodecode/lib/celerylib/loader.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/rhodecode/lib/celerylib/loader.py b/rhodecode/lib/celerylib/loader.py
index a8d496b6..9fdf4533 100644
--- a/rhodecode/lib/celerylib/loader.py
+++ b/rhodecode/lib/celerylib/loader.py
@@ -32,6 +32,7 @@ inspect_getargspec()
 inspect_formatargspec()
 # python3.11 inspect patches for backward compat on `paste` code
 
+import sys
 import logging
 import importlib
 
@@ -160,12 +161,12 @@ def _init_celery(app_type=''):
     try:
         env = bootstrap(ini_file, options=options)
     except Exception:
-        log.exception('Failed to bootstrap RhodeCode APP')
+        log.exception('Failed to bootstrap RhodeCode APP. '
+                      'Probably there is another error present that prevents from running pyramid app')
 
     if not env:
-        raise EnvironmentError(
-            'Failed to load pyramid ENV. '
-            'Probably there is another error present that prevents from running pyramid app')
+        # we use sys.exit here since we need to signal app startup failure for docker to restart the container and re-try
+        sys.exit(1)
 
     log.debug('Got Pyramid ENV: %s', env)
 

From 112de0ec6da966e1f5d4106fb250da4051f587f5 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 22 May 2024 15:36:20 +0200
Subject: [PATCH 093/150] makefile: allow starting gunicorn with workers=N
 command

---
 Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index f604cfd1..18979009 100644
--- a/Makefile
+++ b/Makefile
@@ -144,11 +144,13 @@ sh:
 dev-srv:
 	pserve --reload .dev/dev.ini
 
+## Allows changes of workers e.g make dev-srv-g workers=2
+workers?=1
 
 .PHONY: dev-srv-g
 ## run gunicorn multi process workers
 dev-srv-g:
-	gunicorn --paste .dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py --timeout=120 --reload
+	gunicorn --paste=.dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py --timeout=120 --reload --workers=$(workers)
 
 
 # Default command on calling make

From 414221293768d710f3dd1569ea99bf942309f625 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 22 May 2024 15:37:11 +0200
Subject: [PATCH 094/150] docs: added 5.0.1 entry

---
 docs/release-notes/release-notes-5.0.1.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/release-notes/release-notes-5.0.1.rst b/docs/release-notes/release-notes-5.0.1.rst
index 5cc5c495..d4265214 100644
--- a/docs/release-notes/release-notes-5.0.1.rst
+++ b/docs/release-notes/release-notes-5.0.1.rst
@@ -1,4 +1,4 @@
-|RCE| 5.0.0 |RNS|
+|RCE| 5.0.1 |RNS|
 -----------------
 
 Release Date
@@ -32,6 +32,7 @@ Fixes
 ^^^^^
 
 - Fixed Celery serialization issues
+- Fixed Celery startup problems signaling
 - Fixed SVN hooks binary dir paths which in certain scenarios resulted in empty values forbidding hooks to execute
 - Fixed annotation bug for files without new lines or mixed newlines
 

From 793ec386f8906e44cf63a15bcfc22d16f1ec7814 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 20 May 2024 11:03:42 +0200
Subject: [PATCH 095/150] fix(celery): don't use msgpack serializer due to
 problems with certain data types

---
 rhodecode/lib/celerylib/loader.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rhodecode/lib/celerylib/loader.py b/rhodecode/lib/celerylib/loader.py
index 9fdf4533..d1f066b5 100644
--- a/rhodecode/lib/celerylib/loader.py
+++ b/rhodecode/lib/celerylib/loader.py
@@ -72,9 +72,9 @@ base_celery_config = {
     'worker_max_tasks_per_child': 100,
     'worker_hijack_root_logger': False,
     'worker_prefetch_multiplier': 1,
-    'task_serializer': 'msgpack',
+    'task_serializer': 'json',
     'accept_content': ['json', 'msgpack'],
-    'result_serializer': 'msgpack',
+    'result_serializer': 'json',
     'result_accept_content': ['json', 'msgpack'],
 
     'broker_connection_retry_on_startup': True,

From 83ac36f14a1225c04d92a4c8c6100ba992995137 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 23 May 2024 13:50:38 +0200
Subject: [PATCH 096/150] docs: fixed missing toctree for 5.0.1

---
 docs/release-notes/release-notes.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst
index c7d8e52c..c907c338 100644
--- a/docs/release-notes/release-notes.rst
+++ b/docs/release-notes/release-notes.rst
@@ -10,6 +10,7 @@ Release Notes
 .. toctree::
    :maxdepth: 1
 
+   release-notes-5.0.1.rst
    release-notes-5.0.0.rst
    release-notes-5.1.0.rst
 

From a3c59849da9bc03465491e2b9d2b9a5e75e69e34 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 23 May 2024 15:08:19 +0200
Subject: [PATCH 097/150] config: synced test ini changes with latest configs

---
 rhodecode/tests/rhodecode.ini | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini
index d458d349..b401f0e4 100644
--- a/rhodecode/tests/rhodecode.ini
+++ b/rhodecode/tests/rhodecode.ini
@@ -476,14 +476,14 @@ search.location = %(here)s/data/index
 channelstream.enabled = false
 
 ; server address for channelstream server on the backend
-channelstream.server = 127.0.0.1:9800
+channelstream.server = channelstream:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
-channelstream.secret = secret
+channelstream.secret = ENV_GENERATED
 channelstream.history.location = %(here)s/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
@@ -543,6 +543,7 @@ vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
+; `celery` - use celery based hooks
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
@@ -629,7 +630,8 @@ ssh.authorized_keys_file_path = %(here)s/rc/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
-; e.g /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command

From 1f3e46703ea4c66254b9209c0ae825f7e6a229bc Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 29 May 2024 09:37:55 +0200
Subject: [PATCH 098/150] fix(docs): fixed order of release notes

---
 docs/release-notes/release-notes.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst
index c907c338..e4c27434 100644
--- a/docs/release-notes/release-notes.rst
+++ b/docs/release-notes/release-notes.rst
@@ -10,9 +10,9 @@ Release Notes
 .. toctree::
    :maxdepth: 1
 
+   release-notes-5.1.0.rst
    release-notes-5.0.1.rst
    release-notes-5.0.0.rst
-   release-notes-5.1.0.rst
 
 
 |RCE| 4.x Versions

From 4462f85332ac4d849ad4c2d6c7ade4e29ecfe871 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 29 May 2024 09:47:57 +0200
Subject: [PATCH 099/150] fix(configs): removed waitress as a default runner
 for web service on development.

One of the big issues discovered receantly was due to waitress vs gunicorn differences on handling
requests. Since gunicorn has a nice dev mode with code reloader we should use it as a default runner so
we can avoid such issues in the past where dev setup runned different workers than production.
---
 Makefile                |  7 +------
 configs/development.ini | 19 +------------------
 2 files changed, 2 insertions(+), 24 deletions(-)

diff --git a/Makefile b/Makefile
index 18979009..cdc83ba9 100644
--- a/Makefile
+++ b/Makefile
@@ -139,15 +139,10 @@ sh:
 	make dev-sh
 
 
-.PHONY: dev-srv
-## run develop server instance, docker exec -it $(docker ps -q --filter 'name=dev-enterprise-ce') /bin/bash
-dev-srv:
-	pserve --reload .dev/dev.ini
-
 ## Allows changes of workers e.g make dev-srv-g workers=2
 workers?=1
 
-.PHONY: dev-srv-g
+.PHONY: dev-srv
 ## run gunicorn multi process workers
 dev-srv-g:
 	gunicorn --paste=.dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py --timeout=120 --reload --workers=$(workers)
diff --git a/configs/development.ini b/configs/development.ini
index cc3954d8..86b270c2 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -31,23 +31,6 @@ debug = true
 host = 127.0.0.1
 port = 10020
 
-; ##################################################
-; WAITRESS WSGI SERVER - Recommended for Development
-; ##################################################
-
-; use server type
-use = egg:waitress#main
-
-; number of worker threads
-threads = 5
-
-; MAX BODY SIZE 100GB
-max_request_body_size = 107374182400
-
-; Use poll instead of select, fixes file descriptors limits problems.
-; May not work on old windows systems.
-asyncore_use_poll = true
-
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
@@ -56,7 +39,7 @@ asyncore_use_poll = true
 ; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
 
 ; Module to use, this setting shouldn't be changed
-#use = egg:gunicorn#main
+use = egg:gunicorn#main
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.

From 64f674caf5d1e70b43ecf21dc8a0d3a2014079b5 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 29 May 2024 09:54:27 +0200
Subject: [PATCH 100/150] make: synced makefiles

---
 Makefile | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index cdc83ba9..27369d9f 100644
--- a/Makefile
+++ b/Makefile
@@ -88,6 +88,10 @@ web-build:
 	./rhodecode/tests/scripts/static-file-check.sh rhodecode/public/
 	rm -rf node_modules
 
+.PHONY: ruff-check
+## run a ruff analysis
+ruff-check:
+	ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs  --exclude .dev .
 
 .PHONY: pip-packages
 ## Show outdated packages
@@ -123,6 +127,7 @@ dev-cleanup:
 
 .PHONY: dev-env
 ## make dev-env based on the requirements files and install develop of packages
+## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 dev-env:
 	sudo -u root chown rhodecode:rhodecode /home/rhodecode/.cache/pip/
 	pip install build virtualenv
@@ -143,8 +148,8 @@ sh:
 workers?=1
 
 .PHONY: dev-srv
-## run gunicorn multi process workers
-dev-srv-g:
+## run gunicorn web server with reloader, use workers=N to set multiworker mode
+dev-srv:
 	gunicorn --paste=.dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py --timeout=120 --reload --workers=$(workers)
 
 

From 3aec75e4bf3c36c09a07d07fe7a79313fa7e92f2 Mon Sep 17 00:00:00 2001
From: Andrii Verbytskyi <andrii@rhodecode.com>
Date: Wed, 29 May 2024 12:21:43 +0200
Subject: [PATCH 101/150] chore: release notes for 5.0.2

---
 docs/release-notes/release-notes.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst
index e4c27434..5d03853e 100644
--- a/docs/release-notes/release-notes.rst
+++ b/docs/release-notes/release-notes.rst
@@ -10,7 +10,9 @@ Release Notes
 .. toctree::
    :maxdepth: 1
 
+
    release-notes-5.1.0.rst
+   release-notes-5.0.2.rst
    release-notes-5.0.1.rst
    release-notes-5.0.0.rst
 

From 5617fc51434845f6b84f15f79c1266ba25a3c0d6 Mon Sep 17 00:00:00 2001
From: Andrii Verbytskyi <andrii@rhodecode.com>
Date: Wed, 29 May 2024 12:22:40 +0200
Subject: [PATCH 102/150] chore: release notes for 5.0.2

---
 docs/release-notes/release-notes-5.0.2.rst | 39 ++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 docs/release-notes/release-notes-5.0.2.rst

diff --git a/docs/release-notes/release-notes-5.0.2.rst b/docs/release-notes/release-notes-5.0.2.rst
new file mode 100644
index 00000000..7f084247
--- /dev/null
+++ b/docs/release-notes/release-notes-5.0.2.rst
@@ -0,0 +1,39 @@
+|RCE| 5.0.2 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-05-29
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+
+
+
+Fixes
+^^^^^
+
+- Fixed problems with saving branch permissions
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.0.2 is unscheduled bugfix release to address some of the issues found during 4.X -> 5.X migration

From 0eaacd34a08575ba0636daaf25f264ceb51c06cb Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 4 Jun 2024 17:19:57 +0200
Subject: [PATCH 103/150] makefiles: synced makefiles

---
 Makefile | 25 -------------------------
 1 file changed, 25 deletions(-)

diff --git a/Makefile b/Makefile
index 27369d9f..705e115f 100644
--- a/Makefile
+++ b/Makefile
@@ -37,31 +37,6 @@ test-only:
 	--cov=rhodecode rhodecode
 
 
-.PHONY: test-only-mysql
-## run tests against mysql
-test-only-mysql:
-	PYTHONHASHSEED=random \
-	py.test -x -vv -r xw -p no:sugar \
-	--cov-report=term-missing --cov-report=html \
-    --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "mysql://root:qweqwe@localhost/rhodecode_test?charset=utf8"}}' \
-    --cov=rhodecode rhodecode
-
-
-.PHONY: test-only-postgres
-## run tests against postgres
-test-only-postgres:
-	PYTHONHASHSEED=random \
-	py.test -x -vv -r xw -p no:sugar \
-	--cov-report=term-missing --cov-report=html \
-    --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "postgresql://postgres:qweqwe@localhost/rhodecode_test"}}' \
-    --cov=rhodecode rhodecode
-
-.PHONY: ruff-check
-## run a ruff analysis
-ruff-check:
-	ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs  --exclude .dev .
-
-
 .PHONY: docs
 ## build docs
 docs:

From 39bed20bcb226501147f58ba8ffef54cf66ef2d8 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 5 Jun 2024 12:32:12 +0200
Subject: [PATCH 104/150] feat(disk-cache): rewrite diskcache backend to be k8s
 and NFS safe.

- no longer depend on diskcache
- use simpler version with Redis lock, and filesystem storage
- fixes RCCE-78
---
 configs/development.ini                       |  16 +-
 configs/production.ini                        |  16 +-
 requirements.txt                              |   1 -
 rhodecode/apps/repository/views/repo_files.py |  67 +++--
 rhodecode/config/config_maker.py              |  10 +-
 rhodecode/lib/rc_cache/archive_cache.py       |  88 ------
 .../lib/rc_cache/archive_cache/__init__.py    |  29 ++
 .../rc_cache/archive_cache/fanout_cache.py    | 264 ++++++++++++++++++
 rhodecode/lib/rc_cache/archive_cache/lock.py  |  60 ++++
 rhodecode/lib/rc_cache/archive_cache/utils.py |  30 ++
 rhodecode/lib/system_info.py                  |  13 +-
 11 files changed, 463 insertions(+), 131 deletions(-)
 delete mode 100644 rhodecode/lib/rc_cache/archive_cache.py
 create mode 100644 rhodecode/lib/rc_cache/archive_cache/__init__.py
 create mode 100644 rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
 create mode 100644 rhodecode/lib/rc_cache/archive_cache/lock.py
 create mode 100644 rhodecode/lib/rc_cache/archive_cache/utils.py

diff --git a/configs/development.ini b/configs/development.ini
index 86b270c2..09a90ef3 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -296,14 +296,24 @@ file_store.storage_path = /var/opt/rhodecode_data/file_store
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
 
+; Redis url to acquire/check generation of archives locks
+archive_cache.locking.url = redis://redis:6379/1
+
+; Storage backend, only 'filesystem' is available now
+archive_cache.backend.type = filesystem
+
 ; Default is $cache_dir/archive_cache if not set
-archive_cache.store_dir = /var/opt/rhodecode_data/tarballcache
+archive_cache.filesystem.store_dir = /var/opt/rhodecode_data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
-archive_cache.cache_size_gb = 10
+archive_cache.filesystem.cache_size_gb = 1
+
+; Eviction policy used to clear out after cache_size_gb limit is reached
+archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
-archive_cache.cache_shards = 4
+archive_cache.filesystem.cache_shards = 8
+
 
 ; #############
 ; CELERY CONFIG
diff --git a/configs/production.ini b/configs/production.ini
index e64e3538..b5f1212c 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -264,14 +264,24 @@ file_store.storage_path = /var/opt/rhodecode_data/file_store
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
 
+; Redis url to acquire/check generation of archives locks
+archive_cache.locking.url = redis://redis:6379/1
+
+; Storage backend, only 'filesystem' is available now
+archive_cache.backend.type = filesystem
+
 ; Default is $cache_dir/archive_cache if not set
-archive_cache.store_dir = /var/opt/rhodecode_data/tarballcache
+archive_cache.filesystem.store_dir = /var/opt/rhodecode_data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
-archive_cache.cache_size_gb = 40
+archive_cache.filesystem.cache_size_gb = 40
+
+; Eviction policy used to clear out after cache_size_gb limit is reached
+archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
-archive_cache.cache_shards = 4
+archive_cache.filesystem.cache_shards = 8
+
 
 ; #############
 ; CELERY CONFIG
diff --git a/requirements.txt b/requirements.txt
index 00929d76..43a16bfb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -82,7 +82,6 @@ deform==2.0.15
   peppercorn==0.6
   translationstring==1.4
   zope.deprecation==5.0.0
-diskcache==5.6.3
 docutils==0.19
 dogpile.cache==1.3.3
   decorator==5.1.1
diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py
index f8919565..018577ea 100644
--- a/rhodecode/apps/repository/views/repo_files.py
+++ b/rhodecode/apps/repository/views/repo_files.py
@@ -24,6 +24,8 @@ import urllib.request
 import urllib.parse
 import urllib.error
 import pathlib
+import time
+import random
 
 from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest, HTTPFound
 
@@ -37,7 +39,8 @@ from rhodecode.apps._base import RepoAppView
 from rhodecode.lib import diffs, helpers as h, rc_cache
 from rhodecode.lib import audit_logger
 from rhodecode.lib.hash_utils import sha1_safe
-from rhodecode.lib.rc_cache.archive_cache import get_archival_cache_store, get_archival_config, ReentrantLock
+from rhodecode.lib.rc_cache.archive_cache import (
+    get_archival_cache_store, get_archival_config, ArchiveCacheLock, archive_iterator)
 from rhodecode.lib.str_utils import safe_bytes, convert_special_chars
 from rhodecode.lib.view_utils import parse_path_ref
 from rhodecode.lib.exceptions import NonRelativePathError
@@ -417,41 +420,45 @@ class RepoFilesView(RepoAppView):
         # NOTE: we get the config to pass to a call to lazy-init the SAME type of cache on vcsserver
         d_cache_conf = get_archival_config(config=CONFIG)
 
+        # This is also a cache key, and lock key
         reentrant_lock_key = archive_name_key + '.lock'
-        with ReentrantLock(d_cache, reentrant_lock_key):
-            # This is also a cache key
-            use_cached_archive = False
-            if not archive_cache_disable and archive_name_key in d_cache:
-                reader, tag = d_cache.get(archive_name_key, read=True, tag=True, retry=True)
-                use_cached_archive = True
-                log.debug('Found cached archive as key=%s tag=%s, serving archive from cache reader=%s',
-                          archive_name_key, tag, reader.name)
-            else:
-                reader = None
-                log.debug('Archive with key=%s is not yet cached, creating one now...', archive_name_key)
 
-            # generate new archive, as previous was not found in the cache
-            if not reader:
+        use_cached_archive = False
+        if not archive_cache_disable and archive_name_key in d_cache:
+            reader, metadata = d_cache.fetch(archive_name_key)
 
-                try:
-                    commit.archive_repo(archive_name_key, archive_dir_name=archive_dir_name,
-                                        kind=fileformat, subrepos=subrepos,
-                                        archive_at_path=at_path, cache_config=d_cache_conf)
-                except ImproperArchiveTypeError:
-                    return _('Unknown archive type')
-
-            reader, tag = d_cache.get(archive_name_key, read=True, tag=True, retry=True)
+            use_cached_archive = True
+            log.debug('Found cached archive as key=%s tag=%s, serving archive from cache reader=%s',
+                      archive_name_key, metadata, reader.name)
+        else:
+            reader = None
+            log.debug('Archive with key=%s is not yet cached, creating one now...', archive_name_key)
 
         if not reader:
-            raise ValueError('archive cache reader is empty, failed to fetch file from distributed archive cache')
+            # generate new archive, as previous was not found in the cache
+            try:
+                with d_cache.get_lock(reentrant_lock_key):
+                    try:
+                        commit.archive_repo(archive_name_key, archive_dir_name=archive_dir_name,
+                                            kind=fileformat, subrepos=subrepos,
+                                            archive_at_path=at_path, cache_config=d_cache_conf)
+                    except ImproperArchiveTypeError:
+                        return _('Unknown archive type')
+            except ArchiveCacheLock:
+                retry_after = round(random.uniform(0.3, 3.0), 1)
+                time.sleep(retry_after)
 
-        def archive_iterator(_reader, block_size: int = 4096*512):
-            # 4096 * 64 = 64KB
-            while 1:
-                data = _reader.read(block_size)
-                if not data:
-                    break
-                yield data
+                location = self.request.url
+                response = Response(
+                    f"archive {archive_name_key} generation in progress, Retry-After={retry_after}, Location={location}"
+                )
+                response.headers["Retry-After"] = str(retry_after)
+                response.status_code = 307 # temporary redirect
+
+                response.location = location
+                return response
+
+            reader, metadata = d_cache.fetch(archive_name_key)
 
         response = Response(app_iter=archive_iterator(reader))
         response.content_disposition = f'attachment; filename={response_archive_name}'
diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py
index 362087e7..550ccc3b 100644
--- a/rhodecode/config/config_maker.py
+++ b/rhodecode/config/config_maker.py
@@ -189,9 +189,13 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
     settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
 
     # archive_cache
-    settings_maker.make_setting('archive_cache.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
-    settings_maker.make_setting('archive_cache.cache_size_gb', 10, parser='float')
-    settings_maker.make_setting('archive_cache.cache_shards', 10, parser='int')
+    settings_maker.make_setting('archive_cache.locking.url', 'redis://redis:6379/1')
+    settings_maker.make_setting('archive_cache.backend.type', 'filesystem')
+
+    settings_maker.make_setting('archive_cache.filesystem.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
+    settings_maker.make_setting('archive_cache.filesystem.cache_size_gb', 10, parser='float')
+    settings_maker.make_setting('archive_cache.filesystem.cache_shards', 8, parser='int')
+    settings_maker.make_setting('archive_cache.filesystem.eviction_policy', 'least-recently-stored')
 
     settings_maker.env_expand()
 
diff --git a/rhodecode/lib/rc_cache/archive_cache.py b/rhodecode/lib/rc_cache/archive_cache.py
deleted file mode 100644
index a52830c3..00000000
--- a/rhodecode/lib/rc_cache/archive_cache.py
+++ /dev/null
@@ -1,88 +0,0 @@
-# Copyright (C) 2015-2023 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-import logging
-import os
-import diskcache
-from diskcache import RLock
-
-log = logging.getLogger(__name__)
-
-cache_meta = None
-
-
-class ReentrantLock(RLock):
-    def __enter__(self):
-        reentrant_lock_key = self._key
-
-        log.debug('Acquire ReentrantLock(key=%s) for archive cache generation...', reentrant_lock_key)
-        #self.acquire()
-        log.debug('Lock for key=%s acquired', reentrant_lock_key)
-
-    def __exit__(self, *exc_info):
-        #self.release()
-        pass
-
-
-def get_archival_config(config):
-
-    final_config = {
-        'archive_cache.eviction_policy': 'least-frequently-used'
-    }
-
-    for k, v in config.items():
-        if k.startswith('archive_cache'):
-            final_config[k] = v
-
-    return final_config
-
-
-def get_archival_cache_store(config):
-
-    global cache_meta
-    if cache_meta is not None:
-        return cache_meta
-
-    config = get_archival_config(config)
-
-    archive_cache_dir = config['archive_cache.store_dir']
-    archive_cache_size_gb = config['archive_cache.cache_size_gb']
-    archive_cache_shards = config['archive_cache.cache_shards']
-    archive_cache_eviction_policy = config['archive_cache.eviction_policy']
-
-    log.debug('Initializing archival cache instance under %s', archive_cache_dir)
-
-    # check if it's ok to write, and re-create the archive cache
-    if not os.path.isdir(archive_cache_dir):
-        os.makedirs(archive_cache_dir, exist_ok=True)
-
-    d_cache = diskcache.FanoutCache(
-        archive_cache_dir, shards=archive_cache_shards,
-        cull_limit=0,  # manual eviction required
-        size_limit=archive_cache_size_gb * 1024 * 1024 * 1024,
-        eviction_policy=archive_cache_eviction_policy,
-        timeout=30
-    )
-    cache_meta = d_cache
-    return cache_meta
-
-
-def includeme(config):
-    # init our cache at start
-    settings = config.get_settings()
-    get_archival_cache_store(settings)
diff --git a/rhodecode/lib/rc_cache/archive_cache/__init__.py b/rhodecode/lib/rc_cache/archive_cache/__init__.py
new file mode 100644
index 00000000..5ba824ca
--- /dev/null
+++ b/rhodecode/lib/rc_cache/archive_cache/__init__.py
@@ -0,0 +1,29 @@
+# Copyright (C) 2015-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+from .fanout_cache import get_archival_cache_store
+from .fanout_cache import get_archival_config
+
+from .utils import archive_iterator
+from .utils import ArchiveCacheLock
+
+
+def includeme(config):
+    # init our cache at start
+    settings = config.get_settings()
+    get_archival_cache_store(settings)
diff --git a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
new file mode 100644
index 00000000..50966384
--- /dev/null
+++ b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
@@ -0,0 +1,264 @@
+# Copyright (C) 2015-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import codecs
+import contextlib
+import functools
+import os
+import logging
+import time
+import typing
+import zlib
+
+from rhodecode.lib.ext_json import json
+from .lock import GenerationLock
+
+log = logging.getLogger(__name__)
+
+cache_meta = None
+
+UNKNOWN = -241
+NO_VAL = -917
+
+MODE_BINARY = 'BINARY'
+
+
+class FileSystemCache:
+
+    def __init__(self, index, directory, **settings):
+        self._index = index
+        self._directory = directory
+
+    def _write_file(self, full_path, iterator, mode, encoding=None):
+        full_dir, _ = os.path.split(full_path)
+
+        for count in range(1, 11):
+            with contextlib.suppress(OSError):
+                os.makedirs(full_dir)
+
+            try:
+                # Another cache may have deleted the directory before
+                # the file could be opened.
+                writer = open(full_path, mode, encoding=encoding)
+            except OSError:
+                if count == 10:
+                    # Give up after 10 tries to open the file.
+                    raise
+                continue
+
+            with writer:
+                size = 0
+                for chunk in iterator:
+                    size += len(chunk)
+                    writer.write(chunk)
+                return size
+
+    def _get_keyfile(self, key):
+        return os.path.join(self._directory, f'{key}.key')
+
+    def store(self, key, value_reader, metadata):
+        filename, full_path = self.random_filename()
+        key_file = self._get_keyfile(key)
+
+        # STORE METADATA
+        _metadata = {
+            "version": "v1",
+            "timestamp": time.time(),
+            "filename": filename,
+            "full_path": full_path,
+            "key_file": key_file,
+        }
+        if metadata:
+            _metadata.update(metadata)
+
+        reader = functools.partial(value_reader.read, 2**22)
+
+        iterator = iter(reader, b'')
+        size = self._write_file(full_path, iterator, 'xb')
+
+        # after archive is finished, we create a key to save the presence of the binary file
+        with open(key_file, 'wb') as f:
+            f.write(json.dumps(_metadata))
+
+        return key, size, MODE_BINARY, filename, _metadata
+
+    def fetch(self, key) -> tuple[typing.BinaryIO, dict]:
+        if key not in self:
+            raise KeyError(key)
+
+        key_file = self._get_keyfile(key)
+        with open(key_file, 'rb') as f:
+            metadata = json.loads(f.read())
+
+        filename = metadata['filename']
+
+        return open(os.path.join(self._directory, filename), 'rb'), metadata
+
+    def random_filename(self):
+        """Return filename and full-path tuple for file storage.
+
+        Filename will be a randomly generated 28 character hexadecimal string
+        with ".archive_cache" suffixed. Two levels of sub-directories will be used to
+        reduce the size of directories. On older filesystems, lookups in
+        directories with many files may be slow.
+        """
+
+        hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8')
+        sub_dir = os.path.join(hex_name[:2], hex_name[2:4])
+        name = hex_name[4:] + '.archive_cache'
+        filename = os.path.join(sub_dir, name)
+        full_path = os.path.join(self._directory, filename)
+        return filename, full_path
+
+    def hash(self, key):
+        """Compute portable hash for `key`.
+
+        :param key: key to hash
+        :return: hash value
+
+        """
+        mask = 0xFFFFFFFF
+        return zlib.adler32(key.encode('utf-8')) & mask  # noqa
+
+    def __contains__(self, key):
+        """Return `True` if `key` matching item is found in cache.
+
+        :param key: key matching item
+        :return: True if key matching item
+
+        """
+        key_file = self._get_keyfile(key)
+        return os.path.exists(key_file)
+
+
+class FanoutCache:
+    """Cache that shards keys and values."""
+
+    def __init__(
+        self, directory=None, **settings
+    ):
+        """Initialize cache instance.
+
+        :param str directory: cache directory
+        :param settings: settings dict
+
+        """
+        if directory is None:
+            raise ValueError('directory cannot be None')
+
+        directory = str(directory)
+        directory = os.path.expanduser(directory)
+        directory = os.path.expandvars(directory)
+        self._directory = directory
+
+        self._count = settings.pop('cache_shards')
+        self._locking_url = settings.pop('locking_url')
+
+        self._shards = tuple(
+            FileSystemCache(
+                index=num,
+                directory=os.path.join(directory, 'shard_%03d' % num),
+                **settings,
+            )
+            for num in range(self._count)
+        )
+        self._hash = self._shards[0].hash
+
+    def get_lock(self, lock_key):
+        return GenerationLock(lock_key, self._locking_url)
+
+    def _get_shard(self, key) -> FileSystemCache:
+        index = self._hash(key) % self._count
+        shard = self._shards[index]
+        return shard
+
+    def store(self, key, value_reader, metadata=None):
+        shard = self._get_shard(key)
+        return shard.store(key, value_reader, metadata)
+
+    def fetch(self, key):
+        """Return file handle corresponding to `key` from cache.
+        """
+        shard = self._get_shard(key)
+        return shard.fetch(key)
+
+    def has_key(self, key):
+        """Return `True` if `key` matching item is found in cache.
+
+        :param key: key for item
+        :return: True if key is found
+
+        """
+        shard = self._get_shard(key)
+        return key in shard
+
+    def __contains__(self, item):
+        return self.has_key(item)
+
+    def evict(self):
+        """Remove old items based on the conditions"""
+        # TODO: Implement this...
+        return
+
+
+def get_archival_config(config):
+
+    final_config = {
+
+    }
+
+    for k, v in config.items():
+        if k.startswith('archive_cache'):
+            final_config[k] = v
+
+    return final_config
+
+
+def get_archival_cache_store(config):
+
+    global cache_meta
+    if cache_meta is not None:
+        return cache_meta
+
+    config = get_archival_config(config)
+    backend = config['archive_cache.backend.type']
+    if backend != 'filesystem':
+        raise ValueError('archive_cache.backend.type only supports "filesystem"')
+
+    archive_cache_locking_url = config['archive_cache.locking.url']
+    archive_cache_dir = config['archive_cache.filesystem.store_dir']
+    archive_cache_size_gb = config['archive_cache.filesystem.cache_size_gb']
+    archive_cache_shards = config['archive_cache.filesystem.cache_shards']
+    archive_cache_eviction_policy = config['archive_cache.filesystem.eviction_policy']
+
+    log.debug('Initializing archival cache instance under %s', archive_cache_dir)
+
+    # check if it's ok to write, and re-create the archive cache
+    if not os.path.isdir(archive_cache_dir):
+        os.makedirs(archive_cache_dir, exist_ok=True)
+
+    d_cache = FanoutCache(
+        archive_cache_dir,
+        locking_url=archive_cache_locking_url,
+        cache_shards=archive_cache_shards,
+        cache_size_limit=archive_cache_size_gb * 1024 * 1024 * 1024,
+        cache_eviction_policy=archive_cache_eviction_policy
+    )
+    cache_meta = d_cache
+    return cache_meta
+
diff --git a/rhodecode/lib/rc_cache/archive_cache/lock.py b/rhodecode/lib/rc_cache/archive_cache/lock.py
new file mode 100644
index 00000000..166fba8c
--- /dev/null
+++ b/rhodecode/lib/rc_cache/archive_cache/lock.py
@@ -0,0 +1,60 @@
+# Copyright (C) 2015-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import redis
+from rhodecode.lib._vendor import redis_lock
+
+from .utils import ArchiveCacheLock
+
+
+class GenerationLock:
+    """
+    Locking mechanism that detects if a lock is acquired
+
+    with GenerationLock(lock_key):
+        compute_archive()
+    """
+    lock_timeout = 7200
+
+    def __init__(self, lock_key, url):
+        self.lock_key = lock_key
+        self._create_client(url)
+        self.lock = self.get_lock()
+
+    def _create_client(self, url):
+        connection_pool = redis.ConnectionPool.from_url(url)
+        self.writer_client = redis.StrictRedis(
+            connection_pool=connection_pool
+        )
+        self.reader_client = self.writer_client
+
+    def get_lock(self):
+        return redis_lock.Lock(
+            redis_client=self.writer_client,
+            name=self.lock_key,
+            expire=self.lock_timeout,
+            strict=True
+        )
+
+    def __enter__(self):
+        acquired = self.lock.acquire(blocking=False)
+        if not acquired:
+            raise ArchiveCacheLock('Failed to create a lock')
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.lock.release()
diff --git a/rhodecode/lib/rc_cache/archive_cache/utils.py b/rhodecode/lib/rc_cache/archive_cache/utils.py
new file mode 100644
index 00000000..343c4cdc
--- /dev/null
+++ b/rhodecode/lib/rc_cache/archive_cache/utils.py
@@ -0,0 +1,30 @@
+# Copyright (C) 2015-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+
+class ArchiveCacheLock(Exception):
+    pass
+
+
+def archive_iterator(_reader, block_size: int = 4096 * 512):
+    # 4096 * 64 = 64KB
+    while 1:
+        data = _reader.read(block_size)
+        if not data:
+            break
+        yield data
diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py
index a38a1890..effbdadc 100644
--- a/rhodecode/lib/system_info.py
+++ b/rhodecode/lib/system_info.py
@@ -401,13 +401,20 @@ def storage_archives():
     from rhodecode.lib.utils import safe_str
     from rhodecode.lib.helpers import format_byte_size_binary
 
-    msg = 'Archive cache storage is controlled by ' \
-          'archive_cache.store_dir=/path/to/cache option in the .ini file'
-    path = safe_str(rhodecode.CONFIG.get('archive_cache.store_dir', msg))
+    storage_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
+    storage_key = 'archive_cache.filesystem.store_dir'
+
+    default_msg = 'Archive cache storage is controlled by '\
+                  f'{storage_key}=/path/to/cache option in the .ini file'
+    path = rhodecode.ConfigGet().get_str(storage_key, missing=default_msg)
 
     value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
     state = STATE_OK_DEFAULT
     try:
+        if storage_type != 'filesystem':
+            # raise Exc to stop reporting on different type
+            raise ValueError('Storage type must be "filesystem"')
+
         items_count = 0
         used = 0
         for root, dirs, files in os.walk(path):

From a98ae47e79516a54252edde8c06ef2b894f1974c Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 5 Jun 2024 16:27:54 +0200
Subject: [PATCH 105/150] feat(system-info): properly report storage stats
 after diskcache removal

---
 rhodecode/apps/admin/views/system_info.py     |  1 +
 rhodecode/lib/rc_cache/archive_cache/utils.py | 41 +++++++++++++++++
 rhodecode/lib/system_info.py                  | 45 ++++++-------------
 3 files changed, 56 insertions(+), 31 deletions(-)

diff --git a/rhodecode/apps/admin/views/system_info.py b/rhodecode/apps/admin/views/system_info.py
index 4ccd863b..97a82db3 100644
--- a/rhodecode/apps/admin/views/system_info.py
+++ b/rhodecode/apps/admin/views/system_info.py
@@ -169,6 +169,7 @@ class AdminSystemInfoSettingsView(BaseAppView):
             (_('Gist storage location'), val('storage_gist')['path'], state('storage_gist')),
             (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
 
+            (_('Archive cache storage type'), val('storage_archive')['type'], state('storage_archive')),
             (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
             (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
 
diff --git a/rhodecode/lib/rc_cache/archive_cache/utils.py b/rhodecode/lib/rc_cache/archive_cache/utils.py
index 343c4cdc..f6228a99 100644
--- a/rhodecode/lib/rc_cache/archive_cache/utils.py
+++ b/rhodecode/lib/rc_cache/archive_cache/utils.py
@@ -15,6 +15,7 @@
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
+import os
 
 
 class ArchiveCacheLock(Exception):
@@ -28,3 +29,43 @@ def archive_iterator(_reader, block_size: int = 4096 * 512):
         if not data:
             break
         yield data
+
+
+def get_directory_statistics(start_path):
+    """
+    total_files, total_size, directory_stats = get_directory_statistics(start_path)
+
+    print(f"Directory statistics for: {start_path}\n")
+    print(f"Total files: {total_files}")
+    print(f"Total size: {format_size(total_size)}\n")
+
+    :param start_path:
+    :return:
+    """
+
+    total_files = 0
+    total_size = 0
+    directory_stats = {}
+
+    for dir_path, dir_names, file_names in os.walk(start_path):
+        dir_size = 0
+        file_count = len(file_names)
+
+        for file in file_names:
+            filepath = os.path.join(dir_path, file)
+            file_size = os.path.getsize(filepath)
+            dir_size += file_size
+
+        directory_stats[dir_path] = {'file_count': file_count, 'size': dir_size}
+        total_files += file_count
+        total_size += dir_size
+
+    return total_files, total_size, directory_stats
+
+
+def format_size(size):
+    # Convert size in bytes to a human-readable format (e.g., KB, MB, GB)
+    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
+        if size < 1024:
+            return f"{size:.2f} {unit}"
+        size /= 1024
diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py
index effbdadc..7a831a52 100644
--- a/rhodecode/lib/system_info.py
+++ b/rhodecode/lib/system_info.py
@@ -100,7 +100,7 @@ def get_cert_path(ini_path):
     default = '/etc/ssl/certs/ca-certificates.crt'
     control_ca_bundle = os.path.join(
         os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(ini_path)))),
-        '.rccontrol-profile/etc/ca-bundle.crt')
+        '/etc/ssl/certs/ca-certificates.crt')
     if os.path.isfile(control_ca_bundle):
         default = control_ca_bundle
 
@@ -323,7 +323,7 @@ def cpu():
     value['cpu_count'] = psutil.cpu_count()
 
     human_value = value.copy()
-    human_value['text'] = '{} cores at {} %'.format(value['cpu_count'], value['cpu'])
+    human_value['text'] = f'{value["cpu_count"]} cores at {value["cpu"]} %'
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
@@ -398,8 +398,8 @@ def storage_inodes():
 @register_sysinfo
 def storage_archives():
     import rhodecode
-    from rhodecode.lib.utils import safe_str
     from rhodecode.lib.helpers import format_byte_size_binary
+    from rhodecode.lib.rc_cache.archive_cache.utils import get_directory_statistics
 
     storage_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
     storage_key = 'archive_cache.filesystem.store_dir'
@@ -408,29 +408,20 @@ def storage_archives():
                   f'{storage_key}=/path/to/cache option in the .ini file'
     path = rhodecode.ConfigGet().get_str(storage_key, missing=default_msg)
 
-    value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
+    value = dict(percent=0, used=0, total=0, items=0, path=path, text='', type=storage_type)
     state = STATE_OK_DEFAULT
     try:
         if storage_type != 'filesystem':
             # raise Exc to stop reporting on different type
             raise ValueError('Storage type must be "filesystem"')
 
-        items_count = 0
-        used = 0
-        for root, dirs, files in os.walk(path):
-            if root == path:
-                items_count = len(dirs)
+        total_files, total_size, _directory_stats = get_directory_statistics(path)
 
-            for f in files:
-                try:
-                    used += os.path.getsize(os.path.join(root, f))
-                except OSError:
-                    pass
         value.update({
             'percent': 100,
-            'used': used,
-            'total': used,
-            'items': items_count
+            'used': total_size,
+            'total': total_size,
+            'items': total_files
         })
 
     except Exception as e:
@@ -451,6 +442,8 @@ def storage_gist():
     from rhodecode.model.gist import GIST_STORE_LOC
     from rhodecode.lib.utils import safe_str, get_rhodecode_repo_store_path
     from rhodecode.lib.helpers import format_byte_size_binary
+    from rhodecode.lib.rc_cache.archive_cache.utils import get_directory_statistics
+
     path = safe_str(os.path.join(
         get_rhodecode_repo_store_path(), GIST_STORE_LOC))
 
@@ -459,22 +452,12 @@ def storage_gist():
     state = STATE_OK_DEFAULT
 
     try:
-        items_count = 0
-        used = 0
-        for root, dirs, files in os.walk(path):
-            if root == path:
-                items_count = len(dirs)
-
-            for f in files:
-                try:
-                    used += os.path.getsize(os.path.join(root, f))
-                except OSError:
-                    pass
+        total_files, total_size, _directory_stats = get_directory_statistics(path)
         value.update({
             'percent': 100,
-            'used': used,
-            'total': used,
-            'items': items_count
+            'used': total_size,
+            'total': total_size,
+            'items': total_files
         })
     except Exception as e:
         log.exception('failed to fetch gist storage items')

From 308a5029a116b10b745afa5ac0c12d0b3372c977 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 6 Jun 2024 14:43:26 +0200
Subject: [PATCH 106/150] feat(archive-cache): implement more usage stats for
 later easier evictions

---
 .../rc_cache/archive_cache/fanout_cache.py    | 20 ++++++++++++++++---
 rhodecode/lib/rc_cache/archive_cache/utils.py |  1 +
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
index 50966384..d3a545a0 100644
--- a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
+++ b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
@@ -78,10 +78,13 @@ class FileSystemCache:
         # STORE METADATA
         _metadata = {
             "version": "v1",
-            "timestamp": time.time(),
             "filename": filename,
             "full_path": full_path,
             "key_file": key_file,
+            "store_time": time.time(),
+            "access_count": 1,
+            "access_time": 0,
+            "size": 0
         }
         if metadata:
             _metadata.update(metadata)
@@ -90,6 +93,7 @@ class FileSystemCache:
 
         iterator = iter(reader, b'')
         size = self._write_file(full_path, iterator, 'xb')
+        metadata['size'] = size
 
         # after archive is finished, we create a key to save the presence of the binary file
         with open(key_file, 'wb') as f:
@@ -107,7 +111,15 @@ class FileSystemCache:
 
         filename = metadata['filename']
 
-        return open(os.path.join(self._directory, filename), 'rb'), metadata
+        try:
+            return open(os.path.join(self._directory, filename), 'rb'), metadata
+        finally:
+            # update usage stats, count and accessed
+            metadata["access_count"] = metadata.get("access_count", 0) + 1
+            metadata["access_time"] = time.time()
+
+            with open(key_file, 'wb') as f:
+                f.write(json.dumps(metadata))
 
     def random_filename(self):
         """Return filename and full-path tuple for file storage.
@@ -169,6 +181,9 @@ class FanoutCache:
         self._count = settings.pop('cache_shards')
         self._locking_url = settings.pop('locking_url')
 
+        self._eviction_policy = settings['cache_eviction_policy']
+        self._cache_size_limit = settings['cache_size_limit']
+
         self._shards = tuple(
             FileSystemCache(
                 index=num,
@@ -261,4 +276,3 @@ def get_archival_cache_store(config):
     )
     cache_meta = d_cache
     return cache_meta
-
diff --git a/rhodecode/lib/rc_cache/archive_cache/utils.py b/rhodecode/lib/rc_cache/archive_cache/utils.py
index f6228a99..9d2af2ab 100644
--- a/rhodecode/lib/rc_cache/archive_cache/utils.py
+++ b/rhodecode/lib/rc_cache/archive_cache/utils.py
@@ -15,6 +15,7 @@
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
+
 import os
 
 

From fd5f035c74dfaf4fd0cc7b8741ad96e43829d0a9 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 6 Jun 2024 14:48:07 +0200
Subject: [PATCH 107/150] feat(archive-cache): implemented eviction policy
 logic

---
 .../rc_cache/archive_cache/fanout_cache.py    | 140 +++++++++++++++++-
 1 file changed, 137 insertions(+), 3 deletions(-)

diff --git a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
index d3a545a0..ddc6d398 100644
--- a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
+++ b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
@@ -24,6 +24,7 @@ import logging
 import time
 import typing
 import zlib
+import sqlite3
 
 from rhodecode.lib.ext_json import json
 from .lock import GenerationLock
@@ -38,6 +39,72 @@ NO_VAL = -917
 MODE_BINARY = 'BINARY'
 
 
+EVICTION_POLICY = {
+    'none': {
+        'evict': None,
+    },
+    'least-recently-stored': {
+        'evict': 'SELECT {fields} FROM archive_cache ORDER BY store_time',
+    },
+    'least-recently-used': {
+        'evict': 'SELECT {fields} FROM archive_cache ORDER BY access_time',
+    },
+    'least-frequently-used': {
+        'evict': 'SELECT {fields} FROM archive_cache ORDER BY access_count',
+    },
+}
+
+
+class DB:
+
+    def __init__(self):
+        self.connection = sqlite3.connect(':memory:')
+        self._init_db()
+
+    def _init_db(self):
+        qry = '''
+            CREATE TABLE IF NOT EXISTS archive_cache (
+             rowid INTEGER PRIMARY KEY,
+             key_file TEXT,
+             key_file_path TEXT,
+             filename TEXT,
+             full_path TEXT,
+             store_time REAL,
+             access_time REAL,
+             access_count INTEGER DEFAULT 0,
+             size INTEGER DEFAULT 0
+             )
+        '''
+
+        self.sql(qry)
+        self.connection.commit()
+
+    @property
+    def sql(self):
+        return self.connection.execute
+
+    def bulk_insert(self, rows):
+        qry = '''
+            INSERT INTO archive_cache (
+             rowid,
+             key_file,
+             key_file_path,
+             filename,
+             full_path,
+             store_time,
+             access_time,
+             access_count,
+             size
+            )
+            VALUES (
+            ?, ?, ?, ?, ?, ?, ?, ?, ?
+            )
+        '''
+        cursor = self.connection.cursor()
+        cursor.executemany(qry, rows)
+        self.connection.commit()
+
+
 class FileSystemCache:
 
     def __init__(self, index, directory, **settings):
@@ -225,9 +292,76 @@ class FanoutCache:
     def __contains__(self, item):
         return self.has_key(item)
 
-    def evict(self):
-        """Remove old items based on the conditions"""
-        # TODO: Implement this...
+    def evict(self, policy=None, size_limit=None):
+        """
+        Remove old items based on the conditions
+
+
+        explanation of this algo:
+        iterate over each shard, then for each shard iterate over the .key files
+        read the key files metadata stored. This gives us a full list of keys, cached_archived, their size and
+        access data, time creation, and access counts.
+
+        Store that into a memory DB so we can run different sorting strategies easily.
+        Summing the size is a sum sql query.
+
+        Then we run a sorting strategy based on eviction policy.
+        We iterate over sorted keys, and remove each checking if we hit the overall limit.
+        """
+
+        policy = policy or self._eviction_policy
+        size_limit = size_limit or self._cache_size_limit
+
+        select_policy = EVICTION_POLICY[policy]['evict']
+
+        if select_policy is None:
+            return 0
+
+        db = DB()
+
+        data = []
+        cnt = 1
+        for shard in self._shards:
+            for key_file in os.listdir(shard._directory):
+                if key_file.endswith('.key'):
+                    key_file_path = os.path.join(shard._directory, key_file)
+                    with open(key_file_path, 'rb') as f:
+                        metadata = json.loads(f.read())
+                        # in case we don't have size re-calc it...
+                        if not metadata.get('size'):
+                            fn = metadata.get('full_path')
+                            size = os.stat(fn).st_size
+
+                        data.append([
+                          cnt,
+                          key_file,
+                          key_file_path,
+                          metadata.get('filename'),
+                          metadata.get('full_path'),
+                          metadata.get('store_time', 0),
+                          metadata.get('access_time', 0),
+                          metadata.get('access_count', 0),
+                          metadata.get('size', size),
+                        ])
+                        cnt += 1
+
+        # Insert bulk data using executemany
+        db.bulk_insert(data)
+
+        ((total_size,),) = db.sql('SELECT COALESCE(SUM(size), 0) FROM archive_cache').fetchall()
+
+        select_policy_qry = select_policy.format(fields='key_file_path, full_path, size')
+        sorted_keys = db.sql(select_policy_qry).fetchall()
+
+        for key, cached_file, size in sorted_keys:
+            # simulate removal impact BEFORE removal
+            total_size -= size
+            if total_size <= size_limit:
+                # we obtained what we wanted...
+                break
+
+            os.remove(cached_file)
+            os.remove(key)
         return
 
 

From 4ce101104bd6747a3cc1c437776c4edb371df33a Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 6 Jun 2024 16:54:19 +0200
Subject: [PATCH 108/150] feat(archive-cache): added logging and relative
 imports

---
 .../lib/rc_cache/archive_cache/fanout_cache.py  | 17 ++++++++++++++---
 rhodecode/lib/rc_cache/archive_cache/lock.py    |  3 +--
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
index ddc6d398..53f50d78 100644
--- a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
+++ b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
@@ -26,8 +26,9 @@ import typing
 import zlib
 import sqlite3
 
-from rhodecode.lib.ext_json import json
+from ...ext_json import json
 from .lock import GenerationLock
+from .utils import format_size
 
 log = logging.getLogger(__name__)
 
@@ -314,6 +315,9 @@ class FanoutCache:
 
         select_policy = EVICTION_POLICY[policy]['evict']
 
+        log.debug('Running eviction policy \'%s\', and checking for size limit: %s',
+                  policy, format_size(size_limit))
+
         if select_policy is None:
             return 0
 
@@ -349,20 +353,27 @@ class FanoutCache:
         db.bulk_insert(data)
 
         ((total_size,),) = db.sql('SELECT COALESCE(SUM(size), 0) FROM archive_cache').fetchall()
-
+        log.debug('Analyzed %s keys, occupied: %s', len(data), format_size(total_size))
         select_policy_qry = select_policy.format(fields='key_file_path, full_path, size')
         sorted_keys = db.sql(select_policy_qry).fetchall()
 
+        removed_items = 0
+        removed_size = 0
         for key, cached_file, size in sorted_keys:
             # simulate removal impact BEFORE removal
             total_size -= size
+
             if total_size <= size_limit:
                 # we obtained what we wanted...
                 break
 
             os.remove(cached_file)
             os.remove(key)
-        return
+            removed_items += 1
+            removed_size += size
+
+        log.debug('Removed %s cache archives, and reduced size: %s', removed_items, format_size(removed_size))
+        return removed_items
 
 
 def get_archival_config(config):
diff --git a/rhodecode/lib/rc_cache/archive_cache/lock.py b/rhodecode/lib/rc_cache/archive_cache/lock.py
index 166fba8c..ec703bbf 100644
--- a/rhodecode/lib/rc_cache/archive_cache/lock.py
+++ b/rhodecode/lib/rc_cache/archive_cache/lock.py
@@ -17,8 +17,7 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import redis
-from rhodecode.lib._vendor import redis_lock
-
+from ..._vendor import redis_lock
 from .utils import ArchiveCacheLock
 
 

From f642c7aebf843c34237319626f55741b2bd89188 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 6 Jun 2024 19:50:31 +0200
Subject: [PATCH 109/150] feat(archive-cache): re-calculate size better if we
 miss it in stats

---
 .../lib/rc_cache/archive_cache/fanout_cache.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
index 53f50d78..9201151d 100644
--- a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
+++ b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
@@ -331,21 +331,25 @@ class FanoutCache:
                     key_file_path = os.path.join(shard._directory, key_file)
                     with open(key_file_path, 'rb') as f:
                         metadata = json.loads(f.read())
-                        # in case we don't have size re-calc it...
-                        if not metadata.get('size'):
-                            fn = metadata.get('full_path')
-                            size = os.stat(fn).st_size
+
+                        size = metadata.get('size')
+                        filename = metadata.get('filename')
+                        full_path = metadata.get('full_path')
+
+                        if not size:
+                            # in case we don't have size re-calc it...
+                            size = os.stat(full_path).st_size
 
                         data.append([
                           cnt,
                           key_file,
                           key_file_path,
-                          metadata.get('filename'),
-                          metadata.get('full_path'),
+                          filename,
+                          full_path,
                           metadata.get('store_time', 0),
                           metadata.get('access_time', 0),
                           metadata.get('access_count', 0),
-                          metadata.get('size', size),
+                          size,
                         ])
                         cnt += 1
 

From 164d313bea57baf35060e098e9427d5dcb823694 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Sun, 9 Jun 2024 22:01:19 +0200
Subject: [PATCH 110/150] feat(archive-cache): added retry mechanism, and some
 code cleanups

---
 rhodecode/apps/repository/views/repo_files.py |  7 ++--
 .../lib/rc_cache/archive_cache/__init__.py    |  2 +-
 .../rc_cache/archive_cache/fanout_cache.py    | 36 +++++++++++++++----
 rhodecode/lib/rc_cache/archive_cache/lock.py  |  4 +--
 rhodecode/lib/rc_cache/archive_cache/utils.py |  2 +-
 5 files changed, 37 insertions(+), 14 deletions(-)

diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py
index 018577ea..407d420c 100644
--- a/rhodecode/apps/repository/views/repo_files.py
+++ b/rhodecode/apps/repository/views/repo_files.py
@@ -40,7 +40,7 @@ from rhodecode.lib import diffs, helpers as h, rc_cache
 from rhodecode.lib import audit_logger
 from rhodecode.lib.hash_utils import sha1_safe
 from rhodecode.lib.rc_cache.archive_cache import (
-    get_archival_cache_store, get_archival_config, ArchiveCacheLock, archive_iterator)
+    get_archival_cache_store, get_archival_config, ArchiveCacheGenerationLock, archive_iterator)
 from rhodecode.lib.str_utils import safe_bytes, convert_special_chars
 from rhodecode.lib.view_utils import parse_path_ref
 from rhodecode.lib.exceptions import NonRelativePathError
@@ -444,7 +444,8 @@ class RepoFilesView(RepoAppView):
                                             archive_at_path=at_path, cache_config=d_cache_conf)
                     except ImproperArchiveTypeError:
                         return _('Unknown archive type')
-            except ArchiveCacheLock:
+
+            except ArchiveCacheGenerationLock:
                 retry_after = round(random.uniform(0.3, 3.0), 1)
                 time.sleep(retry_after)
 
@@ -453,7 +454,7 @@ class RepoFilesView(RepoAppView):
                     f"archive {archive_name_key} generation in progress, Retry-After={retry_after}, Location={location}"
                 )
                 response.headers["Retry-After"] = str(retry_after)
-                response.status_code = 307 # temporary redirect
+                response.status_code = 307  # temporary redirect
 
                 response.location = location
                 return response
diff --git a/rhodecode/lib/rc_cache/archive_cache/__init__.py b/rhodecode/lib/rc_cache/archive_cache/__init__.py
index 5ba824ca..bf13977f 100644
--- a/rhodecode/lib/rc_cache/archive_cache/__init__.py
+++ b/rhodecode/lib/rc_cache/archive_cache/__init__.py
@@ -20,7 +20,7 @@ from .fanout_cache import get_archival_cache_store
 from .fanout_cache import get_archival_config
 
 from .utils import archive_iterator
-from .utils import ArchiveCacheLock
+from .utils import ArchiveCacheGenerationLock
 
 
 def includeme(config):
diff --git a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
index 9201151d..32b27f11 100644
--- a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
+++ b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
@@ -112,6 +112,11 @@ class FileSystemCache:
         self._index = index
         self._directory = directory
 
+    @property
+    def directory(self):
+        """Cache directory."""
+        return self._directory
+
     def _write_file(self, full_path, iterator, mode, encoding=None):
         full_dir, _ = os.path.split(full_path)
 
@@ -169,8 +174,17 @@ class FileSystemCache:
 
         return key, size, MODE_BINARY, filename, _metadata
 
-    def fetch(self, key) -> tuple[typing.BinaryIO, dict]:
+    def fetch(self, key, retry=False, retry_attempts=10) -> tuple[typing.BinaryIO, dict]:
+
+        if retry:
+            for attempt in range(retry_attempts):
+                if key in self:
+                    break
+                # we dind't find the key, wait 1s, and re-check
+                time.sleep(1)
+
         if key not in self:
+            log.exception('requested {key} not found in {self}', key, self)
             raise KeyError(key)
 
         key_file = self._get_keyfile(key)
@@ -180,7 +194,7 @@ class FileSystemCache:
         filename = metadata['filename']
 
         try:
-            return open(os.path.join(self._directory, filename), 'rb'), metadata
+            return open(os.path.join(self.directory, filename), 'rb'), metadata
         finally:
             # update usage stats, count and accessed
             metadata["access_count"] = metadata.get("access_count", 0) + 1
@@ -202,7 +216,7 @@ class FileSystemCache:
         sub_dir = os.path.join(hex_name[:2], hex_name[2:4])
         name = hex_name[4:] + '.archive_cache'
         filename = os.path.join(sub_dir, name)
-        full_path = os.path.join(self._directory, filename)
+        full_path = os.path.join(self.directory, filename)
         return filename, full_path
 
     def hash(self, key):
@@ -225,6 +239,9 @@ class FileSystemCache:
         key_file = self._get_keyfile(key)
         return os.path.exists(key_file)
 
+    def __repr__(self):
+        return f'FileSystemCache(index={self._index}, dir={self.directory})'
+
 
 class FanoutCache:
     """Cache that shards keys and values."""
@@ -262,6 +279,11 @@ class FanoutCache:
         )
         self._hash = self._shards[0].hash
 
+    @property
+    def directory(self):
+        """Cache directory."""
+        return self._directory
+
     def get_lock(self, lock_key):
         return GenerationLock(lock_key, self._locking_url)
 
@@ -274,11 +296,11 @@ class FanoutCache:
         shard = self._get_shard(key)
         return shard.store(key, value_reader, metadata)
 
-    def fetch(self, key):
+    def fetch(self, key, retry=False, retry_attempts=10):
         """Return file handle corresponding to `key` from cache.
         """
         shard = self._get_shard(key)
-        return shard.fetch(key)
+        return shard.fetch(key, retry=retry, retry_attempts=retry_attempts)
 
     def has_key(self, key):
         """Return `True` if `key` matching item is found in cache.
@@ -326,9 +348,9 @@ class FanoutCache:
         data = []
         cnt = 1
         for shard in self._shards:
-            for key_file in os.listdir(shard._directory):
+            for key_file in os.listdir(shard.directory):
                 if key_file.endswith('.key'):
-                    key_file_path = os.path.join(shard._directory, key_file)
+                    key_file_path = os.path.join(shard.directory, key_file)
                     with open(key_file_path, 'rb') as f:
                         metadata = json.loads(f.read())
 
diff --git a/rhodecode/lib/rc_cache/archive_cache/lock.py b/rhodecode/lib/rc_cache/archive_cache/lock.py
index ec703bbf..4652c0aa 100644
--- a/rhodecode/lib/rc_cache/archive_cache/lock.py
+++ b/rhodecode/lib/rc_cache/archive_cache/lock.py
@@ -18,7 +18,7 @@
 
 import redis
 from ..._vendor import redis_lock
-from .utils import ArchiveCacheLock
+from .utils import ArchiveCacheGenerationLock
 
 
 class GenerationLock:
@@ -53,7 +53,7 @@ class GenerationLock:
     def __enter__(self):
         acquired = self.lock.acquire(blocking=False)
         if not acquired:
-            raise ArchiveCacheLock('Failed to create a lock')
+            raise ArchiveCacheGenerationLock('Failed to create a lock')
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.lock.release()
diff --git a/rhodecode/lib/rc_cache/archive_cache/utils.py b/rhodecode/lib/rc_cache/archive_cache/utils.py
index 9d2af2ab..736cfdb7 100644
--- a/rhodecode/lib/rc_cache/archive_cache/utils.py
+++ b/rhodecode/lib/rc_cache/archive_cache/utils.py
@@ -19,7 +19,7 @@
 import os
 
 
-class ArchiveCacheLock(Exception):
+class ArchiveCacheGenerationLock(Exception):
     pass
 
 

From 3d0e81968b0ccd7f39be1650705cff5576dbc68c Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 10 Jun 2024 17:01:42 +0200
Subject: [PATCH 111/150] feat(disk-cache): use fsync to force flush changes on
 NFS, and use retry mechanism to archive caches...

---
 rhodecode/apps/repository/views/repo_files.py        | 2 +-
 rhodecode/lib/rc_cache/archive_cache/fanout_cache.py | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py
index 407d420c..b4b2f466 100644
--- a/rhodecode/apps/repository/views/repo_files.py
+++ b/rhodecode/apps/repository/views/repo_files.py
@@ -459,7 +459,7 @@ class RepoFilesView(RepoAppView):
                 response.location = location
                 return response
 
-            reader, metadata = d_cache.fetch(archive_name_key)
+            reader, metadata = d_cache.fetch(archive_name_key, retry=True, retry_attempts=30)
 
         response = Response(app_iter=archive_iterator(reader))
         response.content_disposition = f'attachment; filename={response_archive_name}'
diff --git a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
index 32b27f11..90530b8b 100644
--- a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
+++ b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
@@ -139,6 +139,13 @@ class FileSystemCache:
                 for chunk in iterator:
                     size += len(chunk)
                     writer.write(chunk)
+                writer.flush()
+                # Get the file descriptor
+                fd = writer.fileno()
+
+                # Sync the file descriptor to disk, helps with NFS cases...
+                os.fsync(fd)
+                log.debug('written new archive cache under %s', full_path)
                 return size
 
     def _get_keyfile(self, key):

From 59e99e4368658be7193c8ed2ed808da088615ebd Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Fri, 7 Jun 2024 12:28:04 +0300
Subject: [PATCH 112/150] feat(branch removal trough UI): Added ability to
 remove branches trough UI from git and hg repositories. Fixes: RCCE-75

---
 rhodecode/apps/_base/__init__.py              |  1 +
 rhodecode/apps/repository/__init__.py         |  9 +++
 .../repository/tests/test_repo_branches.py    | 48 ++++++++++++++
 .../apps/repository/views/repo_branches.py    | 66 ++++++++++++++++++-
 rhodecode/lib/vcs/backends/git/repository.py  |  3 +
 rhodecode/model/db.py                         |  6 ++
 rhodecode/public/js/rhodecode/routes.js       |  1 +
 rhodecode/templates/branches/branches.mako    | 26 +++++---
 .../templates/data_table/_dt_elements.mako    | 24 +++++++
 rhodecode/tests/routes.py                     |  1 +
 10 files changed, 174 insertions(+), 11 deletions(-)

diff --git a/rhodecode/apps/_base/__init__.py b/rhodecode/apps/_base/__init__.py
index 290a903e..cfb3157b 100644
--- a/rhodecode/apps/_base/__init__.py
+++ b/rhodecode/apps/_base/__init__.py
@@ -715,6 +715,7 @@ class BaseReferencesView(RepoAppView):
                 {
                     "name": _render("name", ref_name, files_url, closed),
                     "name_raw": ref_name,
+                    "closed": closed,
                     "date": _render("date", commit.date),
                     "date_raw": datetime_to_time(commit.date),
                     "author": _render("author", commit.author),
diff --git a/rhodecode/apps/repository/__init__.py b/rhodecode/apps/repository/__init__.py
index 58a55bfe..782fa882 100644
--- a/rhodecode/apps/repository/__init__.py
+++ b/rhodecode/apps/repository/__init__.py
@@ -591,6 +591,15 @@ def includeme(config):
         route_name='branches_home', request_method='GET',
         renderer='rhodecode:templates/branches/branches.mako')
 
+    config.add_route(
+        name='branch_remove',
+        pattern='/{repo_name:.*?[^/]}/{branch_name:.*?[^/]}/remove', repo_route=True, repo_accepted_types=['hg', 'git'])
+    config.add_view(
+        RepoBranchesView,
+        attr='remove_branch',
+        route_name='branch_remove', request_method='POST'
+    )
+
     # Bookmarks
     config.add_route(
         name='bookmarks_home',
diff --git a/rhodecode/apps/repository/tests/test_repo_branches.py b/rhodecode/apps/repository/tests/test_repo_branches.py
index 69994198..5dfc7019 100644
--- a/rhodecode/apps/repository/tests/test_repo_branches.py
+++ b/rhodecode/apps/repository/tests/test_repo_branches.py
@@ -19,6 +19,7 @@
 import pytest
 from rhodecode.model.db import Repository
 from rhodecode.tests.routes import route_path
+from rhodecode.tests import assert_session_flash
 
 
 @pytest.mark.usefixtures('autologin_user', 'app')
@@ -33,3 +34,50 @@ class TestBranchesController(object):
         for commit_id, obj_name in repo.scm_instance().branches.items():
             assert commit_id in response
             assert obj_name in response
+
+    def test_landing_branch_delete(self, backend, csrf_token):
+        if backend.alias == 'svn':
+            pytest.skip("Not supported yet")
+        branch_related_data_per_backend = {
+            'git': {'name': 'master'},
+            'hg': {'name': 'default'},
+        }
+        response = self.app.post(
+            route_path('branch_remove', repo_name=backend.repo_name,
+                       branch_name=branch_related_data_per_backend[backend.alias]['name']),
+            params={'csrf_token': csrf_token}, status=302)
+        assert_session_flash(
+            response,
+            f"This branch {branch_related_data_per_backend[backend.alias]['name']} cannot be removed as it's currently set as landing branch"
+        )
+
+    def test_delete_branch_by_repo_owner(self, backend, csrf_token):
+        if backend.alias in ('svn', 'hg'):
+            pytest.skip("Skipping for hg and svn")
+        branch_to_be_removed = 'remove_me'
+        repo = Repository.get_by_repo_name(backend.repo_name)
+        repo.scm_instance()._create_branch(branch_to_be_removed, repo.scm_instance().commit_ids[1])
+        response = self.app.post(
+            route_path('branch_remove', repo_name=backend.repo_name,
+                       branch_name=branch_to_be_removed),
+            params={'csrf_token': csrf_token}, status=302)
+        assert_session_flash(response, f"Branch {branch_to_be_removed} has been successfully deleted")
+
+    def test_delete_branch_by_not_repo_owner(self, backend, csrf_token):
+        username = 'test_regular'
+        pwd = 'test12'
+        branch_related_data_per_backend = {
+            'git': {'name': 'master', 'action': 'deleted'},
+            'hg': {'name': 'stable', 'action': 'closed'},
+        }
+        if backend.alias == 'svn':
+            pytest.skip("Not supported yet")
+        self.app.post(route_path('login'),
+                      {'username': username,
+                       'password': pwd})
+        selected_branch = branch_related_data_per_backend[backend.alias]['name']
+        response = self.app.post(
+            route_path('branch_remove', repo_name=backend.repo_name,
+                       branch_name=selected_branch),
+            params={'csrf_token': csrf_token, 'username': username, 'password': pwd}, status=404)
+        assert response.status_code == 404
diff --git a/rhodecode/apps/repository/views/repo_branches.py b/rhodecode/apps/repository/views/repo_branches.py
index 779b8262..5b20f5ba 100644
--- a/rhodecode/apps/repository/views/repo_branches.py
+++ b/rhodecode/apps/repository/views/repo_branches.py
@@ -18,11 +18,15 @@
 
 import logging
 
+from pyramid.httpexceptions import HTTPFound
 
 from rhodecode.apps._base import BaseReferencesView
 from rhodecode.lib import ext_json
-from rhodecode.lib.auth import (LoginRequired, HasRepoPermissionAnyDecorator)
+from rhodecode.lib import helpers as h
+from rhodecode.lib.auth import (LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
 from rhodecode.model.scm import ScmModel
+from rhodecode.model.meta import Session
+from rhodecode.model.db import PullRequest
 
 log = logging.getLogger(__name__)
 
@@ -33,15 +37,71 @@ class RepoBranchesView(BaseReferencesView):
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def branches(self):
+        partial_render = self.request.get_partial_renderer(
+            'rhodecode:templates/data_table/_dt_elements.mako')
+        repo_name = self.db_repo_name
         c = self.load_default_context()
         self._prepare_and_set_clone_url(c)
         c.rhodecode_repo = self.rhodecode_vcs_repo
         c.repository_forks = ScmModel().get_forks(self.db_repo)
-
         ref_items = self.rhodecode_vcs_repo.branches_all.items()
         data = self.load_refs_context(
             ref_items=ref_items, partials_template='branches/branches_data.mako')
-
+        data_with_actions = []
+        if self.db_repo.repo_type != 'svn':
+            for branch in data:
+                branch['action'] = partial_render(
+                    f"branch_actions_{self.db_repo.repo_type}", branch['name_raw'], repo_name, closed=branch['closed']
+                )
+                data_with_actions.append(branch)
+            data = data_with_actions
         c.has_references = bool(data)
         c.data = ext_json.str_json(data)
         return self._get_template_context(c)
+
+    @LoginRequired()
+    @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
+    @CSRFRequired()
+    def remove_branch(self):
+        _ = self.request.translate
+        self.load_default_context()
+        repo = self.db_repo
+        repo_name = self.db_repo_name
+        repo_type = repo.repo_type
+        action = _('deleted') if repo_type == 'git' else _('closed')
+        redirect = HTTPFound(location=self.request.route_path('branches_home', repo_name=repo_name))
+        branch_name = self.request.matchdict.get('branch_name')
+        if repo.landing_ref_name == branch_name:
+            h.flash(
+                _("This branch {} cannot be removed as it's currently set as landing branch").format(branch_name),
+                category='error'
+            )
+            return redirect
+        if prs_related_to := Session().query(PullRequest).filter(PullRequest.target_repo_id == repo.repo_id,
+                                                                 PullRequest.status != PullRequest.STATUS_CLOSED).filter(
+            (PullRequest.source_ref.like(f'branch:{branch_name}:%')) | (
+                    PullRequest.target_ref.like(f'branch:{branch_name}:%'))
+        ).all():
+            h.flash(_("Branch cannot be {} - it's used in following open Pull Request ids: {}").format(action, ','.join(
+                map(str, prs_related_to))), category='error')
+            return redirect
+
+        match repo_type:
+            case 'git':
+                self.rhodecode_vcs_repo.delete_branch(branch_name)
+            case 'hg':
+                from rhodecode.lib.vcs.backends.base import Reference
+                self.rhodecode_vcs_repo._local_close(
+                    source_ref=Reference(type='branch', name=branch_name,
+                                         commit_id=self.rhodecode_vcs_repo.branches[branch_name]),
+                    target_ref=Reference(type='branch', name='', commit_id=None),
+                    user_name=self.request.user.name,
+                    user_email=self.request.user.email)
+            case _:
+                raise NotImplementedError('Branch deleting functionality not yet implemented')
+        ScmModel().mark_for_invalidation(repo_name)
+        self.rhodecode_vcs_repo._invalidate_prop_cache('commit_ids')
+        self.rhodecode_vcs_repo._invalidate_prop_cache('_refs')
+        self.rhodecode_vcs_repo._invalidate_prop_cache('branches')
+        h.flash(_("Branch {} has been successfully {}").format(branch_name, action), category='success')
+        return redirect
diff --git a/rhodecode/lib/vcs/backends/git/repository.py b/rhodecode/lib/vcs/backends/git/repository.py
index 9a1afbb3..7aa28d40 100644
--- a/rhodecode/lib/vcs/backends/git/repository.py
+++ b/rhodecode/lib/vcs/backends/git/repository.py
@@ -326,6 +326,9 @@ class GitRepository(BaseRepository):
     def _get_branches(self):
         return self._get_refs_entries(prefix='refs/heads/', strip_prefix=True)
 
+    def delete_branch(self, branch_name):
+        return self._remote.delete_branch(branch_name)
+
     @CachedProperty
     def branches(self):
         return self._get_branches()
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index b2af4559..9474b93a 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -4585,6 +4585,12 @@ class PullRequest(Base, _PullRequestBase):
         else:
             return f'<DB:PullRequest at {id(self)!r}>'
 
+    def __str__(self):
+        if self.pull_request_id:
+            return f'#{self.pull_request_id}'
+        else:
+            return f'#{id(self)!r}'
+
     reviewers = relationship('PullRequestReviewers', cascade="all, delete-orphan", back_populates='pull_request')
     statuses = relationship('ChangesetStatus', cascade="all, delete-orphan", back_populates='pull_request')
     comments = relationship('ChangesetComment', cascade="all, delete-orphan", back_populates='pull_request')
diff --git a/rhodecode/public/js/rhodecode/routes.js b/rhodecode/public/js/rhodecode/routes.js
index 37ac3ed9..960ea4ec 100644
--- a/rhodecode/public/js/rhodecode/routes.js
+++ b/rhodecode/public/js/rhodecode/routes.js
@@ -92,6 +92,7 @@ function registerRCRoutes() {
     pyroutes.register('auth_home', '/_admin/auth*traverse', []);
     pyroutes.register('bookmarks_home', '/%(repo_name)s/bookmarks', ['repo_name']);
     pyroutes.register('branches_home', '/%(repo_name)s/branches', ['repo_name']);
+    pyroutes.register('branch_remove', '/%(repo_name)s/%(branch_name)s/remove', ['repo_name', 'branch_name']);
     pyroutes.register('channelstream_connect', '/_admin/channelstream/connect', []);
     pyroutes.register('channelstream_proxy', '/_channelstream', []);
     pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []);
diff --git a/rhodecode/templates/branches/branches.mako b/rhodecode/templates/branches/branches.mako
index 3966382c..80b14de1 100644
--- a/rhodecode/templates/branches/branches.mako
+++ b/rhodecode/templates/branches/branches.mako
@@ -62,13 +62,8 @@ $(document).ready(function() {
     };
 
     var branches_data = ${c.data|n};
-    // object list
-    $('#obj_list_table').DataTable({
-      data: branches_data,
-      dom: 'rtp',
-      pageLength: ${c.visual.dashboard_items},
-      order: [[ 0, "asc" ]],
-      columns: [
+    var repo_type = "${c.rhodecode_db_repo.repo_type}";
+    var columns = [
          { data: {"_": "name",
                   "sort": "name_raw"}, title: "${_('Name')}", className: "td-tags" },
          { data: {"_": "date",
@@ -80,7 +75,22 @@ $(document).ready(function() {
                   "type": Number}, title: "${_('Commit')}", className: "td-hash" },
          { data: {"_": "compare",
                   "sort": "compare"}, title: "${_('Compare')}", className: "td-compare" }
-      ],
+      ];
+      if (repo_type !== 'svn') {
+          columns.push({
+              data: { "_": "action", "sort": "action" },
+              title: `${_('Action')}`,
+              className: "td-action",
+              orderable: false
+          });
+      }
+
+    $('#obj_list_table').DataTable({
+      data: branches_data,
+      dom: 'rtp',
+      pageLength: ${c.visual.dashboard_items},
+      order: [[ 0, "asc" ]],
+      columns: columns,
       language: {
             paginate: DEFAULT_GRID_PAGINATION,
             emptyTable: _gettext("No branches available yet.")
diff --git a/rhodecode/templates/data_table/_dt_elements.mako b/rhodecode/templates/data_table/_dt_elements.mako
index 4b4dc244..d5835c06 100644
--- a/rhodecode/templates/data_table/_dt_elements.mako
+++ b/rhodecode/templates/data_table/_dt_elements.mako
@@ -277,6 +277,30 @@ ${h.style_metatag(tag_type, tag)|n,trim}
  </div>
 </%def>
 
+<%def name="branch_actions_git(branch_name, repo_name, **kwargs)">
+ <div class="grid_delete">
+  ${h.secure_form(h.route_path('branch_remove', repo_name=repo_name, branch_name=branch_name), request=request)}
+    <input class="btn btn-link btn-danger" id="remove_branch_${branch_name}" name="remove_branch_${branch_name}"
+           onclick="submitConfirm(event, this, _gettext('Confirm to delete this branch'), _gettext('Delete'), '${branch_name}')"
+           type="submit" value="Delete"
+    >
+  ${h.end_form()}
+ </div>
+</%def>
+
+<%def name="branch_actions_hg(branch_name, repo_name, **kwargs)">
+ <div class="grid_delete">
+  %if not kwargs['closed']:
+      ${h.secure_form(h.route_path('branch_remove', repo_name=repo_name, branch_name=branch_name), request=request)}
+        <input class="btn btn-link btn-danger" id="remove_branch_${branch_name}" name="remove_branch_${branch_name}"
+               onclick="submitConfirm(event, this, _gettext('Confirm to close this branch'), _gettext('Close'), '${branch_name}')"
+               type="submit" value="Close"
+        >
+      ${h.end_form()}
+ %endif
+ </div>
+</%def>
+
 <%def name="user_group_actions(user_group_id, user_group_name)">
  <div class="grid_edit">
     <a href="${h.route_path('edit_user_group', user_group_id=user_group_id)}" title="${_('Edit')}">Edit</a>
diff --git a/rhodecode/tests/routes.py b/rhodecode/tests/routes.py
index 0d120419..d6f01fc4 100644
--- a/rhodecode/tests/routes.py
+++ b/rhodecode/tests/routes.py
@@ -252,6 +252,7 @@ def get_url_defs():
         "pullrequest_show_all_data": "/{repo_name}/pull-request-data",
         "bookmarks_home": "/{repo_name}/bookmarks",
         "branches_home": "/{repo_name}/branches",
+        "branch_remove": "/{repo_name}/{branch_name}/remove",
         "tags_home": "/{repo_name}/tags",
         "repo_changelog": "/{repo_name}/changelog",
         "repo_commits": "/{repo_name}/commits",

From b549afbfee927f27155c93a3e6f94e2c26d003c4 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 13 Jun 2024 09:42:48 +0200
Subject: [PATCH 113/150] feat: system-upgrade improved message on new version
 check page

---
 .../apps/admin/tests/test_admin_settings.py   |  8 +++---
 .../admin/settings/settings_system.mako       |  2 +-
 .../settings/settings_system_update.mako      | 25 +++++++++++--------
 3 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/rhodecode/apps/admin/tests/test_admin_settings.py b/rhodecode/apps/admin/tests/test_admin_settings.py
index ce82a94d..11d43d89 100644
--- a/rhodecode/apps/admin/tests/test_admin_settings.py
+++ b/rhodecode/apps/admin/tests/test_admin_settings.py
@@ -485,7 +485,7 @@ class TestAdminSystemInfo(object):
         update_data = {
             'versions': [
                 {
-                    'version': '100.3.1415926535',
+                    'version': '100.0.0',
                     'general': 'The latest version we are ever going to ship'
                 },
                 {
@@ -502,15 +502,15 @@ class TestAdminSystemInfo(object):
         update_data = {
             'versions': [
                 {
-                    'version': '0.0.0',
+                    'version': '4.0.0',
                     'general': 'The first version we ever shipped'
                 }
             ]
         }
+        text = f"Your current version, {rhodecode.__version__}, is up-to-date as it is equal to or newer than the latest available version, 4.0.0."
         with mock.patch(UPDATE_DATA_QUALNAME, return_value=update_data):
             response = self.app.get(route_path('admin_settings_system_update'))
-            response.mustcontain(
-                'This instance is already running the <b>latest</b> stable version')
+            response.mustcontain(text)
 
     def test_system_update_bad_response(self, autologin_user):
         with mock.patch(UPDATE_DATA_QUALNAME, side_effect=ValueError('foo')):
diff --git a/rhodecode/templates/admin/settings/settings_system.mako b/rhodecode/templates/admin/settings/settings_system.mako
index 0bb12ff1..7c050aeb 100644
--- a/rhodecode/templates/admin/settings/settings_system.mako
+++ b/rhodecode/templates/admin/settings/settings_system.mako
@@ -84,6 +84,6 @@
 <script>
     $('#check_for_update').click(function(e){
         $('#update_notice').show();
-        $('#update_notice').load("${h.route_path('admin_settings_system_update')}");
+        $('#update_notice').load("${h.route_path('admin_settings_system_update', _query={'ver': request.GET.get('ver')})}");
     })
 </script>
diff --git a/rhodecode/templates/admin/settings/settings_system_update.mako b/rhodecode/templates/admin/settings/settings_system_update.mako
index 4a67545a..fda9ea51 100644
--- a/rhodecode/templates/admin/settings/settings_system_update.mako
+++ b/rhodecode/templates/admin/settings/settings_system_update.mako
@@ -1,25 +1,30 @@
 ## upgrade block rendered afte on-click check
 
 <div class="alert ${'alert-warning' if c.should_upgrade else 'alert-success'}">
-<p>
+
 %if c.should_upgrade:
-    A <b>new version</b> is available:
+    <span style="font-size: 130%">A <b>new version</b> is available !</span>
+    <br/>
+    <br/>
+
     %if c.latest_data.get('title'):
-        <b>${h.literal(c.latest_data['title'])}</b>
+        RhodeCode <b>${c.latest_ver}</b> - ${h.literal(c.latest_data['title'])}
     %else:
-        <b>${c.latest_ver}</b>
+        RhodeCode <b>${c.latest_ver}</b>
     %endif
 %else:
-    This instance is already running the <b>latest</b> stable version ${c.latest_ver}.
+    Your current version, ${c.cur_ver}, is up-to-date as it is equal to or newer than the latest available version, ${c.latest_ver}.
 %endif
-</p>
+
 
 % if c.should_upgrade and c.important_notices:
-<div>Important notes for this release:</div>
-    <ul>
+<br/>
+<br/>
+<div>Summary:</div>
+<br/>
     % for notice in c.important_notices:
-        <li>- ${notice}</li>
+    - ${notice}<br/>
     % endfor
-    </ul>
 % endif
+
 </div>

From 80f7b38d7d93fc92cfb1c9765217502e7976205c Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 13 Jun 2024 10:43:17 +0200
Subject: [PATCH 114/150] fix(bootstrap): fixed issues that lead to leaving
 rhodecode.local as a default setting which is now explicitly read from .ini
 config

---
 rhodecode/config/utils.py      |  7 +++++--
 rhodecode/lib/pyramid_utils.py | 14 +++++---------
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/rhodecode/config/utils.py b/rhodecode/config/utils.py
index 41f8c9dc..cdb42632 100644
--- a/rhodecode/config/utils.py
+++ b/rhodecode/config/utils.py
@@ -19,6 +19,8 @@
 import os
 import platform
 
+DEFAULT_USER = 'default'
+
 
 def configure_vcs(config):
     """
@@ -92,13 +94,14 @@ def set_instance_id(config):
 
 
 def get_default_user_id():
-    DEFAULT_USER = 'default'
     from sqlalchemy import text
     from rhodecode.model import meta
 
     engine = meta.get_engine()
     with meta.SA_Session(engine) as session:
-        result = session.execute(text("SELECT user_id from users where username = :uname"), {'uname': DEFAULT_USER})
+        result = session.execute(text(
+            "SELECT user_id from users where username = :uname"
+        ), {'uname': DEFAULT_USER})
         user_id = result.first()[0]
 
     return user_id
diff --git a/rhodecode/lib/pyramid_utils.py b/rhodecode/lib/pyramid_utils.py
index a69d03ec..daaf32e3 100644
--- a/rhodecode/lib/pyramid_utils.py
+++ b/rhodecode/lib/pyramid_utils.py
@@ -19,30 +19,26 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
-import configparser
 
-from rhodecode.lib.config_utils import get_config
 from pyramid.paster import bootstrap as pyramid_bootstrap, setup_logging  # pragma: no cover
 from pyramid.threadlocal import get_current_request as pyramid_current_request
 
 
 def bootstrap(config_uri, options=None, env=None):
+    from rhodecode.config.utils import DEFAULT_USER
+    from rhodecode.lib.config_utils import get_app_config_lightweight
     from rhodecode.lib.utils2 import AttributeDict
     from rhodecode.lib.request import Request
 
     if env:
         os.environ.update(env)
 
-    config = get_config(config_uri)
-    base_url = 'http://rhodecode.local'
-    try:
-        base_url = config.get('app:main', 'app.base_url')
-    except (configparser.NoSectionError, configparser.NoOptionError):
-        pass
+    config = get_app_config_lightweight(config_uri)
+    base_url = config['app.base_url']
 
     request = Request.blank('/', base_url=base_url)
     # fake inject a running user for bootstrap request !
-    request.user = AttributeDict({'username': 'bootstrap-user',
+    request.user = AttributeDict({'username': DEFAULT_USER,
                                   'user_id': 1,
                                   'ip_addr': '127.0.0.1'})
     return pyramid_bootstrap(config_uri, request=request, options=options)

From b8bd0261717d12db9d95b52f43a10090713206d2 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 13 Jun 2024 21:30:11 +0200
Subject: [PATCH 115/150] core: refactor of vendor lib to easier sync with
 vcsserver

---
 rhodecode/lib/_vendor/jsonlogger/__init__.py |  8 ++++----
 rhodecode/lib/_vendor/redis_lock/__init__.py |  2 +-
 rhodecode/lib/_vendor/statsd/__init__.py     |  2 --
 rhodecode/lib/_vendor/statsd/base.py         | 18 ++++++++----------
 rhodecode/lib/_vendor/statsd/stream.py       |  2 --
 rhodecode/lib/_vendor/statsd/timer.py        |  4 +---
 rhodecode/lib/_vendor/statsd/udp.py          |  4 +---
 rhodecode/lib/logging_formatter.py           |  2 +-
 8 files changed, 16 insertions(+), 26 deletions(-)

diff --git a/rhodecode/lib/_vendor/jsonlogger/__init__.py b/rhodecode/lib/_vendor/jsonlogger/__init__.py
index 1ce022cc..10b5a0e0 100644
--- a/rhodecode/lib/_vendor/jsonlogger/__init__.py
+++ b/rhodecode/lib/_vendor/jsonlogger/__init__.py
@@ -11,9 +11,9 @@ import importlib
 from inspect import istraceback
 
 from collections import OrderedDict
-from rhodecode.lib.logging_formatter import _inject_req_id, ExceptionAwareFormatter
-from rhodecode.lib.ext_json import sjson as json
 
+from ...logging_formatter import _inject_req_id, ExceptionAwareFormatter
+from ...ext_json import sjson as json
 
 ZERO = timedelta(0)
 HOUR = timedelta(hours=1)
@@ -78,7 +78,7 @@ class JsonEncoder(json.JSONEncoder):
             return str(obj)
 
         try:
-            return super(JsonEncoder, self).default(obj)
+            return super().default(obj)
 
         except TypeError:
             try:
@@ -194,7 +194,7 @@ class JsonFormatter(ExceptionAwareFormatter):
 
     def serialize_log_record(self, log_record):
         """Returns the final representation of the log record."""
-        return "%s%s" % (self.prefix, self.jsonify_log_record(log_record))
+        return "{}{}".format(self.prefix, self.jsonify_log_record(log_record))
 
     def format(self, record):
         """Formats a log record and serializes to json"""
diff --git a/rhodecode/lib/_vendor/redis_lock/__init__.py b/rhodecode/lib/_vendor/redis_lock/__init__.py
index 7ebb403e..d1894c99 100644
--- a/rhodecode/lib/_vendor/redis_lock/__init__.py
+++ b/rhodecode/lib/_vendor/redis_lock/__init__.py
@@ -102,7 +102,7 @@ class NotExpirable(RuntimeError):
     pass
 
 
-class Lock(object):
+class Lock:
     """
     A Lock context manager implemented via redis SETNX/BLPOP.
     """
diff --git a/rhodecode/lib/_vendor/statsd/__init__.py b/rhodecode/lib/_vendor/statsd/__init__.py
index 2ef9a87c..f5fe7d0b 100644
--- a/rhodecode/lib/_vendor/statsd/__init__.py
+++ b/rhodecode/lib/_vendor/statsd/__init__.py
@@ -1,5 +1,3 @@
-
-
 import logging
 
 from .stream import TCPStatsClient, UnixSocketStatsClient  # noqa
diff --git a/rhodecode/lib/_vendor/statsd/base.py b/rhodecode/lib/_vendor/statsd/base.py
index 10193137..5bc2504d 100644
--- a/rhodecode/lib/_vendor/statsd/base.py
+++ b/rhodecode/lib/_vendor/statsd/base.py
@@ -1,5 +1,3 @@
-
-
 import re
 import random
 from collections import deque
@@ -31,7 +29,7 @@ def normalize_tags(tag_list):
     return _normalize_tags_with_cache(tuple(tag_list))
 
 
-class StatsClientBase(object):
+class StatsClientBase:
     """A Base class for various statsd clients."""
 
     def close(self):
@@ -73,7 +71,7 @@ class StatsClientBase(object):
 
     def incr(self, stat, count=1, rate=1, tags=None):
         """Increment a stat by `count`."""
-        self._send_stat(stat, '%s|c' % count, rate, tags)
+        self._send_stat(stat, f'{count}|c', rate, tags)
 
     def decr(self, stat, count=1, rate=1, tags=None):
         """Decrement a stat by `count`."""
@@ -87,18 +85,18 @@ class StatsClientBase(object):
                     return
             with self.pipeline() as pipe:
                 pipe._send_stat(stat, '0|g', 1)
-                pipe._send_stat(stat, '%s|g' % value, 1)
+                pipe._send_stat(stat, f'{value}|g', 1)
         else:
             prefix = '+' if delta and value >= 0 else ''
-            self._send_stat(stat, '%s%s|g' % (prefix, value), rate, tags)
+            self._send_stat(stat, f'{prefix}{value}|g', rate, tags)
 
     def set(self, stat, value, rate=1):
         """Set a set value."""
-        self._send_stat(stat, '%s|s' % value, rate)
+        self._send_stat(stat, f'{value}|s', rate)
 
     def histogram(self, stat, value, rate=1, tags=None):
         """Set a histogram"""
-        self._send_stat(stat, '%s|h' % value, rate, tags)
+        self._send_stat(stat, f'{value}|h', rate, tags)
 
     def _send_stat(self, stat, value, rate, tags=None):
         self._after(self._prepare(stat, value, rate, tags))
@@ -110,10 +108,10 @@ class StatsClientBase(object):
         if rate < 1:
             if random.random() > rate:
                 return
-            value = '%s|@%s' % (value, rate)
+            value = f'{value}|@{rate}'
 
         if self._prefix:
-            stat = '%s.%s' % (self._prefix, stat)
+            stat = f'{self._prefix}.{stat}'
 
         res = '%s:%s%s' % (
             stat,
diff --git a/rhodecode/lib/_vendor/statsd/stream.py b/rhodecode/lib/_vendor/statsd/stream.py
index 0979fcc7..79951d25 100644
--- a/rhodecode/lib/_vendor/statsd/stream.py
+++ b/rhodecode/lib/_vendor/statsd/stream.py
@@ -1,5 +1,3 @@
-
-
 import socket
 
 from .base import StatsClientBase, PipelineBase
diff --git a/rhodecode/lib/_vendor/statsd/timer.py b/rhodecode/lib/_vendor/statsd/timer.py
index 8fa64266..90ffd68e 100644
--- a/rhodecode/lib/_vendor/statsd/timer.py
+++ b/rhodecode/lib/_vendor/statsd/timer.py
@@ -1,5 +1,3 @@
-
-
 import functools
 from time import perf_counter as time_now
 
@@ -11,7 +9,7 @@ def safe_wraps(wrapper, *args, **kwargs):
     return functools.wraps(wrapper, *args, **kwargs)
 
 
-class Timer(object):
+class Timer:
     """A context manager/decorator for statsd.timing()."""
 
     def __init__(self, client, stat, rate=1, tags=None, use_decimals=True, auto_send=True):
diff --git a/rhodecode/lib/_vendor/statsd/udp.py b/rhodecode/lib/_vendor/statsd/udp.py
index aae39803..427481de 100644
--- a/rhodecode/lib/_vendor/statsd/udp.py
+++ b/rhodecode/lib/_vendor/statsd/udp.py
@@ -1,5 +1,3 @@
-
-
 import socket
 
 from .base import StatsClientBase, PipelineBase
@@ -8,7 +6,7 @@ from .base import StatsClientBase, PipelineBase
 class Pipeline(PipelineBase):
 
     def __init__(self, client):
-        super(Pipeline, self).__init__(client)
+        super().__init__(client)
         self._maxudpsize = client._maxudpsize
 
     def _send(self):
diff --git a/rhodecode/lib/logging_formatter.py b/rhodecode/lib/logging_formatter.py
index 3379d3a2..6bc7b6c4 100644
--- a/rhodecode/lib/logging_formatter.py
+++ b/rhodecode/lib/logging_formatter.py
@@ -141,7 +141,7 @@ class ColorFormatter(ExceptionAwareFormatter):
         """
         Changes record's levelname to use with COLORS enum
         """
-        def_record = super(ColorFormatter, self).format(record)
+        def_record = super().format(record)
 
         levelname = record.levelname
         start = COLOR_SEQ % (COLORS[levelname])

From b44b38740ba471cb1f0a80f964b2924e6d5c7911 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 13 Jun 2024 10:44:25 +0200
Subject: [PATCH 116/150] feat(system-info): update report and made a ver
 option be passed via GET param to test check for update logic

---
 rhodecode/apps/admin/views/system_info.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/rhodecode/apps/admin/views/system_info.py b/rhodecode/apps/admin/views/system_info.py
index 97a82db3..84efabeb 100644
--- a/rhodecode/apps/admin/views/system_info.py
+++ b/rhodecode/apps/admin/views/system_info.py
@@ -165,16 +165,20 @@ class AdminSystemInfoSettingsView(BaseAppView):
             (_('Storage location'), val('storage')['path'], state('storage')),
             (_('Storage info'), val('storage')['text'], state('storage')),
             (_('Storage inodes'), val('storage_inodes')['text'], state('storage_inodes')),
+            ('', '', ''),  # spacer
 
             (_('Gist storage location'), val('storage_gist')['path'], state('storage_gist')),
             (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
+            ('', '', ''),  # spacer
 
             (_('Archive cache storage type'), val('storage_archive')['type'], state('storage_archive')),
             (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
             (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
+            ('', '', ''),  # spacer
 
             (_('Temp storage location'), val('storage_temp')['path'], state('storage_temp')),
             (_('Temp storage info'), val('storage_temp')['text'], state('storage_temp')),
+            ('', '', ''),  # spacer
 
             (_('Search info'), val('search')['text'], state('search')),
             (_('Search location'), val('search')['location'], state('search')),
@@ -190,7 +194,7 @@ class AdminSystemInfoSettingsView(BaseAppView):
         ]
 
         c.vcsserver_data_items = [
-            (k, v) for k,v in (val('vcs_server_config') or {}).items()
+            (k, v) for k, v in (val('vcs_server_config') or {}).items()
         ]
 
         if snapshot:
@@ -210,7 +214,8 @@ class AdminSystemInfoSettingsView(BaseAppView):
         update_url = UpdateModel().get_update_url()
 
         def _err(s):
-            return '<div style="color:#ff8888; padding:4px 0px">{}</div>'.format(s)
+            return f'<div style="color:#ff8888; padding:4px 0px">{s}</div>'
+
         try:
             data = UpdateModel().get_update_data(update_url)
         except urllib.error.URLError as e:
@@ -226,12 +231,12 @@ class AdminSystemInfoSettingsView(BaseAppView):
 
         c.update_url = update_url
         c.latest_data = latest
-        c.latest_ver = latest['version']
-        c.cur_ver = rhodecode.__version__
+        c.latest_ver = (latest['version'] or '').strip()
+        c.cur_ver = self.request.GET.get('ver') or rhodecode.__version__
         c.should_upgrade = False
 
-        is_oudated = UpdateModel().is_outdated(c.cur_ver, c.latest_ver)
-        if is_oudated:
+        is_outdated = UpdateModel().is_outdated(c.cur_ver, c.latest_ver)
+        if is_outdated:
             c.should_upgrade = True
         c.important_notices = latest['general']
         UpdateModel().store_version(latest['version'])

From 984a9a564f965655f99125778e4fa3794797aab3 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 13 Jun 2024 21:29:37 +0200
Subject: [PATCH 117/150] feat(archive-cache): implemented s3 based backend for
 filecaches

---
 configs/development.ini                       |  44 +-
 configs/production.ini                        |  44 +-
 requirements.txt                              |  29 ++
 rhodecode/apps/repository/views/repo_files.py |   2 +-
 rhodecode/config/config_maker.py              |  18 +-
 rhodecode/config/middleware.py                |   2 +-
 rhodecode/lib/archive_cache/__init__.py       |  78 +++
 .../backends}/__init__.py                     |  12 -
 rhodecode/lib/archive_cache/backends/base.py  | 348 +++++++++++++
 .../archive_cache/backends/fanout_cache.py    | 166 +++++++
 .../backends/objectstore_cache.py             | 150 ++++++
 .../lib/{rc_cache => }/archive_cache/lock.py  |   7 +-
 rhodecode/lib/archive_cache/utils.py          | 134 +++++
 rhodecode/lib/base.py                         |   2 +-
 rhodecode/lib/helpers.py                      |  32 ++
 .../rc_cache/archive_cache/fanout_cache.py    | 456 ------------------
 rhodecode/lib/rc_cache/archive_cache/utils.py |  72 ---
 rhodecode/lib/system_info.py                  |  21 +-
 .../tests/fixture_mods/fixture_pyramid.py     |   2 +-
 rhodecode/tests/lib/test_archive_caches.py    | 105 ++++
 rhodecode/tests/rhodecode.ini                 |  72 ++-
 rhodecode/tests/vcs/test_archives.py          |   2 +-
 22 files changed, 1209 insertions(+), 589 deletions(-)
 create mode 100644 rhodecode/lib/archive_cache/__init__.py
 rename rhodecode/lib/{rc_cache/archive_cache => archive_cache/backends}/__init__.py (73%)
 create mode 100644 rhodecode/lib/archive_cache/backends/base.py
 create mode 100644 rhodecode/lib/archive_cache/backends/fanout_cache.py
 create mode 100644 rhodecode/lib/archive_cache/backends/objectstore_cache.py
 rename rhodecode/lib/{rc_cache => }/archive_cache/lock.py (95%)
 create mode 100644 rhodecode/lib/archive_cache/utils.py
 delete mode 100644 rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
 delete mode 100644 rhodecode/lib/rc_cache/archive_cache/utils.py
 create mode 100644 rhodecode/tests/lib/test_archive_caches.py

diff --git a/configs/development.ini b/configs/development.ini
index 09a90ef3..9ae745ec 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -290,19 +290,41 @@ file_store.backend = local
 ; path to store the uploaded binaries and artifacts
 file_store.storage_path = /var/opt/rhodecode_data/file_store
 
-; Uncomment and set this path to control settings for archive download cache.
-; Generated repo archives will be cached at this location
-; and served from the cache during subsequent requests for the same archive of
-; the repository. This path is important to be shared across filesystems and with
-; RhodeCode and vcsserver
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1
 
-; Storage backend, only 'filesystem' is available now
+; Storage backend, only 'filesystem' and 'objectstore' are available now
 archive_cache.backend.type = filesystem
 
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+archive_cache.objectstore.url = http://s3-minio:9000
+
+; key for s3 auth
+archive_cache.objectstore.key = key
+
+; secret for s3 auth
+archive_cache.objectstore.secret = secret
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+archive_cache.objectstore.bucket_shards = 8
+
+; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
+archive_cache.objectstore.retry = false
+
+; number of seconds to wait for next try using retry
+archive_cache.objectstore.retry_backoff = 1
+
+; how many tries do do a retry fetch from this backend
+archive_cache.objectstore.retry_attempts = 10
+
 ; Default is $cache_dir/archive_cache if not set
+; Generated repo archives will be cached at this location
+; and served from the cache during subsequent requests for the same archive of
+; the repository. This path is important to be shared across filesystems and with
+; RhodeCode and vcsserver
 archive_cache.filesystem.store_dir = /var/opt/rhodecode_data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
@@ -312,8 +334,18 @@ archive_cache.filesystem.cache_size_gb = 1
 archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
+; default is 8 shards
 archive_cache.filesystem.cache_shards = 8
 
+; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
+archive_cache.filesystem.retry = false
+
+; number of seconds to wait for next try using retry
+archive_cache.filesystem.retry_backoff = 1
+
+; how many tries do do a retry fetch from this backend
+archive_cache.filesystem.retry_attempts = 10
+
 
 ; #############
 ; CELERY CONFIG
diff --git a/configs/production.ini b/configs/production.ini
index b5f1212c..205a1884 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -258,19 +258,41 @@ file_store.backend = local
 ; path to store the uploaded binaries and artifacts
 file_store.storage_path = /var/opt/rhodecode_data/file_store
 
-; Uncomment and set this path to control settings for archive download cache.
-; Generated repo archives will be cached at this location
-; and served from the cache during subsequent requests for the same archive of
-; the repository. This path is important to be shared across filesystems and with
-; RhodeCode and vcsserver
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1
 
-; Storage backend, only 'filesystem' is available now
+; Storage backend, only 'filesystem' and 'objectstore' are available now
 archive_cache.backend.type = filesystem
 
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+archive_cache.objectstore.url = http://s3-minio:9000
+
+; key for s3 auth
+archive_cache.objectstore.key = key
+
+; secret for s3 auth
+archive_cache.objectstore.secret = secret
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+archive_cache.objectstore.bucket_shards = 8
+
+; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
+archive_cache.objectstore.retry = false
+
+; number of seconds to wait for next try using retry
+archive_cache.objectstore.retry_backoff = 1
+
+; how many tries do do a retry fetch from this backend
+archive_cache.objectstore.retry_attempts = 10
+
 ; Default is $cache_dir/archive_cache if not set
+; Generated repo archives will be cached at this location
+; and served from the cache during subsequent requests for the same archive of
+; the repository. This path is important to be shared across filesystems and with
+; RhodeCode and vcsserver
 archive_cache.filesystem.store_dir = /var/opt/rhodecode_data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
@@ -280,8 +302,18 @@ archive_cache.filesystem.cache_size_gb = 40
 archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
+; default is 8 shards
 archive_cache.filesystem.cache_shards = 8
 
+; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
+archive_cache.filesystem.retry = false
+
+; number of seconds to wait for next try using retry
+archive_cache.filesystem.retry_backoff = 1
+
+; how many tries do do a retry fetch from this backend
+archive_cache.filesystem.retry_attempts = 10
+
 
 ; #############
 ; CELERY CONFIG
diff --git a/requirements.txt b/requirements.txt
index 43a16bfb..677d9550 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -89,6 +89,7 @@ dogpile.cache==1.3.3
     pbr==5.11.1
 formencode==2.1.0
   six==1.16.0
+fsspec==2024.6.0
 gunicorn==21.2.0
   packaging==24.0
 gevent==24.2.1
@@ -257,6 +258,34 @@ regex==2022.10.31
 routes==2.5.1
   repoze.lru==0.7
   six==1.16.0
+s3fs==2024.6.0
+  aiobotocore==2.13.0
+    aiohttp==3.9.5
+      aiosignal==1.3.1
+        frozenlist==1.4.1
+      attrs==22.2.0
+      frozenlist==1.4.1
+      multidict==6.0.5
+      yarl==1.9.4
+        idna==3.4
+        multidict==6.0.5
+    aioitertools==0.11.0
+    botocore==1.34.106
+      jmespath==1.0.1
+      python-dateutil==2.8.2
+        six==1.16.0
+      urllib3==1.26.14
+    wrapt==1.16.0
+  aiohttp==3.9.5
+    aiosignal==1.3.1
+      frozenlist==1.4.1
+    attrs==22.2.0
+    frozenlist==1.4.1
+    multidict==6.0.5
+    yarl==1.9.4
+      idna==3.4
+      multidict==6.0.5
+  fsspec==2024.6.0
 simplejson==3.19.2
 sshpubkeys==3.3.1
   cryptography==40.0.2
diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py
index b4b2f466..d6b7097e 100644
--- a/rhodecode/apps/repository/views/repo_files.py
+++ b/rhodecode/apps/repository/views/repo_files.py
@@ -39,7 +39,7 @@ from rhodecode.apps._base import RepoAppView
 from rhodecode.lib import diffs, helpers as h, rc_cache
 from rhodecode.lib import audit_logger
 from rhodecode.lib.hash_utils import sha1_safe
-from rhodecode.lib.rc_cache.archive_cache import (
+from rhodecode.lib.archive_cache import (
     get_archival_cache_store, get_archival_config, ArchiveCacheGenerationLock, archive_iterator)
 from rhodecode.lib.str_utils import safe_bytes, convert_special_chars
 from rhodecode.lib.view_utils import parse_path_ref
diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py
index 550ccc3b..4e74457b 100644
--- a/rhodecode/config/config_maker.py
+++ b/rhodecode/config/config_maker.py
@@ -193,10 +193,26 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
     settings_maker.make_setting('archive_cache.backend.type', 'filesystem')
 
     settings_maker.make_setting('archive_cache.filesystem.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
-    settings_maker.make_setting('archive_cache.filesystem.cache_size_gb', 10, parser='float')
     settings_maker.make_setting('archive_cache.filesystem.cache_shards', 8, parser='int')
+    settings_maker.make_setting('archive_cache.filesystem.cache_size_gb', 10, parser='float')
     settings_maker.make_setting('archive_cache.filesystem.eviction_policy', 'least-recently-stored')
 
+    settings_maker.make_setting('archive_cache.filesystem.retry', False, parser='bool')
+    settings_maker.make_setting('archive_cache.filesystem.retry_backoff', 1, parser='int')
+    settings_maker.make_setting('archive_cache.filesystem.retry_attempts', 10, parser='int')
+
+    settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
+    settings_maker.make_setting('archive_cache.objectstore.key', '')
+    settings_maker.make_setting('archive_cache.objectstore.secret', '')
+    settings_maker.make_setting('archive_cache.objectstore.bucket_shards', 8, parser='int')
+
+    settings_maker.make_setting('archive_cache.objectstore.cache_size_gb', 10, parser='float')
+    settings_maker.make_setting('archive_cache.objectstore.eviction_policy', 'least-recently-stored')
+
+    settings_maker.make_setting('archive_cache.objectstore.retry', False, parser='bool')
+    settings_maker.make_setting('archive_cache.objectstore.retry_backoff', 1, parser='int')
+    settings_maker.make_setting('archive_cache.objectstore.retry_attempts', 10, parser='int')
+
     settings_maker.env_expand()
 
     # configure instance id
diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py
index dafee2dc..2f83e856 100644
--- a/rhodecode/config/middleware.py
+++ b/rhodecode/config/middleware.py
@@ -326,7 +326,7 @@ def includeme(config, auth_resources=None):
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
-    config.include('rhodecode.lib.rc_cache.archive_cache')
+    config.include('rhodecode.lib.archive_cache')
 
     config.include('rhodecode.apps._base.navigation')
     config.include('rhodecode.apps._base.subscribers')
diff --git a/rhodecode/lib/archive_cache/__init__.py b/rhodecode/lib/archive_cache/__init__.py
new file mode 100644
index 00000000..53741448
--- /dev/null
+++ b/rhodecode/lib/archive_cache/__init__.py
@@ -0,0 +1,78 @@
+# Copyright (C) 2015-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+
+from .backends.fanout_cache import FileSystemFanoutCache
+from .backends.objectstore_cache import ObjectStoreCache
+
+from .utils import archive_iterator  # noqa
+from .lock import ArchiveCacheGenerationLock # noqa
+
+log = logging.getLogger(__name__)
+
+
+cache_meta = None
+
+
+def includeme(config):
+    # init our cache at start
+    settings = config.get_settings()
+    get_archival_cache_store(settings)
+
+
+def get_archival_config(config):
+
+    final_config = {
+
+    }
+
+    for k, v in config.items():
+        if k.startswith('archive_cache'):
+            final_config[k] = v
+
+    return final_config
+
+
+def get_archival_cache_store(config, always_init=False):
+
+    global cache_meta
+    if cache_meta is not None and not always_init:
+        return cache_meta
+
+    config = get_archival_config(config)
+    backend = config['archive_cache.backend.type']
+
+    archive_cache_locking_url = config['archive_cache.locking.url']
+
+    match backend:
+        case 'filesystem':
+            d_cache = FileSystemFanoutCache(
+                locking_url=archive_cache_locking_url,
+                **config
+            )
+        case 'objectstore':
+            d_cache = ObjectStoreCache(
+                locking_url=archive_cache_locking_url,
+                **config
+            )
+        case _:
+            raise ValueError(f'archive_cache.backend.type only supports "filesystem" or "objectstore" got {backend} ')
+
+    cache_meta = d_cache
+    return cache_meta
diff --git a/rhodecode/lib/rc_cache/archive_cache/__init__.py b/rhodecode/lib/archive_cache/backends/__init__.py
similarity index 73%
rename from rhodecode/lib/rc_cache/archive_cache/__init__.py
rename to rhodecode/lib/archive_cache/backends/__init__.py
index bf13977f..677eee8c 100644
--- a/rhodecode/lib/rc_cache/archive_cache/__init__.py
+++ b/rhodecode/lib/archive_cache/backends/__init__.py
@@ -15,15 +15,3 @@
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
-
-from .fanout_cache import get_archival_cache_store
-from .fanout_cache import get_archival_config
-
-from .utils import archive_iterator
-from .utils import ArchiveCacheGenerationLock
-
-
-def includeme(config):
-    # init our cache at start
-    settings = config.get_settings()
-    get_archival_cache_store(settings)
diff --git a/rhodecode/lib/archive_cache/backends/base.py b/rhodecode/lib/archive_cache/backends/base.py
new file mode 100644
index 00000000..3d9d5637
--- /dev/null
+++ b/rhodecode/lib/archive_cache/backends/base.py
@@ -0,0 +1,348 @@
+# Copyright (C) 2015-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import functools
+import logging
+import typing
+import time
+import zlib
+
+from ...ext_json import json
+from ..utils import StatsDB, NOT_GIVEN, ShardFileReader, EVICTION_POLICY, format_size
+from ..lock import GenerationLock
+
+log = logging.getLogger(__name__)
+
+
+class BaseShard:
+    storage_type: str = ''
+    fs = None
+
+    @classmethod
+    def hash(cls, key):
+        """Compute portable hash for `key`.
+
+        :param key: key to hash
+        :return: hash value
+
+        """
+        mask = 0xFFFFFFFF
+        return zlib.adler32(key.encode('utf-8')) & mask  # noqa
+
+    def _write_file(self, full_path, read_iterator, mode):
+        raise NotImplementedError
+
+    def _get_keyfile(self, key):
+        raise NotImplementedError
+
+    def random_filename(self):
+        raise NotImplementedError
+
+    def _store(self, key, value_reader, metadata, mode):
+        (filename,  # hash-name
+         full_path  # full-path/hash-name
+         ) = self.random_filename()
+
+        key_file, key_file_path = self._get_keyfile(key)
+
+        # STORE METADATA
+        _metadata = {
+            "version": "v1",
+
+            "key_file": key_file,  # this is the .key.json file storing meta
+            "key_file_path": key_file_path,  # full path to key_file
+            "archive_key": key,  # original name we stored archive under, e.g my-archive.zip
+            "archive_filename": filename,  # the actual filename we stored that file under
+            "archive_full_path": full_path,
+
+            "store_time": time.time(),
+            "access_count": 0,
+            "access_time": 0,
+
+            "size": 0
+        }
+        if metadata:
+            _metadata.update(metadata)
+
+        read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+        size, sha256 = self._write_file(full_path, read_iterator, mode)
+        _metadata['size'] = size
+        _metadata['sha256'] = sha256
+
+        # after archive is finished, we create a key to save the presence of the binary file
+        with self.fs.open(key_file_path, 'wb') as f:
+            f.write(json.dumps(_metadata))
+
+        return key, filename, size, _metadata
+
+    def _fetch(self,  key, retry, retry_attempts, retry_backoff):
+        if retry is NOT_GIVEN:
+            retry = False
+        if retry_attempts is NOT_GIVEN:
+            retry_attempts = 0
+
+        if retry and retry_attempts > 0:
+            for attempt in range(1, retry_attempts + 1):
+                if key in self:
+                    break
+                # we didn't find the key, wait retry_backoff N seconds, and re-check
+                time.sleep(retry_backoff)
+
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self} retry={retry}, attempts={retry_attempts}')
+            raise KeyError(key)
+
+        key_file, key_file_path = self._get_keyfile(key)
+        with self.fs.open(key_file_path, 'rb') as f:
+            metadata = json.loads(f.read())
+
+        archive_path = metadata['archive_full_path']
+
+        try:
+            return ShardFileReader(self.fs.open(archive_path, 'rb')), metadata
+        finally:
+            # update usage stats, count and accessed
+            metadata["access_count"] = metadata.get("access_count", 0) + 1
+            metadata["access_time"] = time.time()
+            log.debug('Updated %s with access snapshot, access_count=%s access_time=%s',
+                      key_file, metadata['access_count'], metadata['access_time'])
+            with self.fs.open(key_file_path, 'wb') as f:
+                f.write(json.dumps(metadata))
+
+    def _remove(self, key):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        key_file, key_file_path = self._get_keyfile(key)
+        with self.fs.open(key_file_path, 'rb') as f:
+            metadata = json.loads(f.read())
+
+        archive_path = metadata['archive_full_path']
+        self.fs.rm(archive_path)
+        self.fs.rm(key_file_path)
+        return 1
+
+    @property
+    def storage_medium(self):
+        return getattr(self, self.storage_type)
+
+    @property
+    def key_suffix(self):
+        return 'key.json'
+
+    def __contains__(self, key):
+        """Return `True` if `key` matching item is found in cache.
+
+        :param key: key matching item
+        :return: True if key matching item
+
+        """
+        key_file, key_file_path = self._get_keyfile(key)
+        return self.fs.exists(key_file_path)
+
+
+class BaseCache:
+    _locking_url: str = ''
+    _storage_path: str = ''
+    _config = {}
+    retry = False
+    retry_attempts = 0
+    retry_backoff = 1
+    _shards = tuple()
+
+    def __contains__(self, key):
+        """Return `True` if `key` matching item is found in cache.
+
+        :param key: key matching item
+        :return: True if key matching item
+
+        """
+        return self.has_key(key)
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__}(storage={self._storage_path})>'
+
+    @classmethod
+    def gb_to_bytes(cls, gb):
+        return gb * (1024 ** 3)
+
+    @property
+    def storage_path(self):
+        return self._storage_path
+
+    @classmethod
+    def get_stats_db(cls):
+        return StatsDB()
+
+    def get_conf(self, key, pop=False):
+        if key not in self._config:
+            raise ValueError(f"No configuration key '{key}', please make sure it exists in archive_cache config")
+        val = self._config[key]
+        if pop:
+            del self._config[key]
+        return val
+
+    def _get_shard(self, key):
+        raise NotImplementedError
+
+    def _get_size(self, shard, archive_path):
+        raise NotImplementedError
+
+    def store(self, key, value_reader, metadata=None):
+        shard = self._get_shard(key)
+        return shard.store(key, value_reader, metadata)
+
+    def fetch(self, key, retry=NOT_GIVEN, retry_attempts=NOT_GIVEN) -> tuple[typing.BinaryIO, dict]:
+        """
+        Return file handle corresponding to `key` from specific shard cache.
+        """
+        if retry is NOT_GIVEN:
+            retry = self.retry
+        if retry_attempts is NOT_GIVEN:
+            retry_attempts = self.retry_attempts
+        retry_backoff = self.retry_backoff
+
+        shard = self._get_shard(key)
+        return shard.fetch(key, retry=retry, retry_attempts=retry_attempts, retry_backoff=retry_backoff)
+
+    def remove(self, key):
+        shard = self._get_shard(key)
+        return shard.remove(key)
+
+    def has_key(self, archive_key):
+        """Return `True` if `key` matching item is found in cache.
+
+        :param archive_key: key for item, this is a unique archive name we want to store data under. e.g my-archive-svn.zip
+        :return: True if key is found
+
+        """
+        shard = self._get_shard(archive_key)
+        return archive_key in shard
+
+    def iter_keys(self):
+        for shard in self._shards:
+            if shard.fs.exists(shard.storage_medium):
+                for path, _dirs, _files in shard.fs.walk(shard.storage_medium):
+                    for key_file_path in _files:
+                        if key_file_path.endswith(shard.key_suffix):
+                            yield shard, key_file_path
+
+    def get_lock(self, lock_key):
+        return GenerationLock(lock_key, self._locking_url)
+
+    def evict(self, policy=None, size_limit=None) -> int:
+        """
+        Remove old items based on the conditions
+
+
+        explanation of this algo:
+        iterate over each shard, then for each shard iterate over the .key files
+        read the key files metadata stored. This gives us a full list of keys, cached_archived, their size and
+        access data, time creation, and access counts.
+
+        Store that into a memory DB so we can run different sorting strategies easily.
+        Summing the size is a sum sql query.
+
+        Then we run a sorting strategy based on eviction policy.
+        We iterate over sorted keys, and remove each checking if we hit the overall limit.
+        """
+
+        policy = policy or self._eviction_policy
+        size_limit = size_limit or self._cache_size_limit
+
+        select_policy = EVICTION_POLICY[policy]['evict']
+
+        log.debug('Running eviction policy \'%s\', and checking for size limit: %s',
+                  policy, format_size(size_limit))
+
+        if select_policy is None:
+            return 0
+
+        db = self.get_stats_db()
+
+        data = []
+        cnt = 1
+
+        for shard, key_file in self.iter_keys():
+            with shard.fs.open(os.path.join(shard.storage_medium, key_file), 'rb') as f:
+                metadata = json.loads(f.read())
+
+            key_file_path = os.path.join(shard.storage_medium, key_file)
+
+            archive_key = metadata['archive_key']
+            archive_path = metadata['archive_full_path']
+
+            size = metadata.get('size')
+            if not size:
+                # in case we don't have size re-calc it...
+                size = self._get_size(shard, archive_path)
+
+            data.append([
+                cnt,
+                key_file,
+                key_file_path,
+                archive_key,
+                archive_path,
+                metadata.get('store_time', 0),
+                metadata.get('access_time', 0),
+                metadata.get('access_count', 0),
+                size,
+            ])
+            cnt += 1
+
+        # Insert bulk data using executemany
+        db.bulk_insert(data)
+
+        total_size = db.get_total_size()
+        log.debug('Analyzed %s keys, occupying: %s, running eviction to match %s',
+                  len(data), format_size(total_size), format_size(size_limit))
+
+        removed_items = 0
+        removed_size = 0
+        for key_file, archive_key, size in db.get_sorted_keys(select_policy):
+            # simulate removal impact BEFORE removal
+            total_size -= size
+
+            if total_size <= size_limit:
+                # we obtained what we wanted...
+                break
+
+            self.remove(archive_key)
+            removed_items += 1
+            removed_size += size
+
+        log.debug('Removed %s cache archives, and reduced size by: %s',
+                  removed_items, format_size(removed_size))
+        return removed_items
+
+    def get_statistics(self):
+        total_files = 0
+        total_size = 0
+        meta = {}
+
+        for shard, key_file in self.iter_keys():
+            json_key = f"{shard.storage_medium}/{key_file}"
+            with shard.fs.open(json_key, 'rb') as f:
+                total_files += 1
+                metadata = json.loads(f.read())
+                total_size += metadata['size']
+
+        return total_files, total_size, meta
+
diff --git a/rhodecode/lib/archive_cache/backends/fanout_cache.py b/rhodecode/lib/archive_cache/backends/fanout_cache.py
new file mode 100644
index 00000000..2d96a04a
--- /dev/null
+++ b/rhodecode/lib/archive_cache/backends/fanout_cache.py
@@ -0,0 +1,166 @@
+# Copyright (C) 2015-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import codecs
+import hashlib
+import logging
+import os
+
+import fsspec
+
+from .base import BaseCache, BaseShard
+from ..utils import ShardFileReader, NOT_GIVEN
+from ...type_utils import str2bool
+
+log = logging.getLogger(__name__)
+
+
+class FileSystemShard(BaseShard):
+
+    def __init__(self, index, directory, **settings):
+        self._index = index
+        self._directory = directory
+        self.storage_type = 'directory'
+        self.fs = fsspec.filesystem('file')
+
+    @property
+    def directory(self):
+        """Cache directory."""
+        return self._directory
+
+    def _get_keyfile(self, archive_key) -> tuple[str, str]:
+        key_file = f'{archive_key}.{self.key_suffix}'
+        return key_file, os.path.join(self.directory, key_file)
+
+    def _get_writer(self, path, mode):
+        for count in range(1, 11):
+            try:
+                # Another cache may have deleted the directory before
+                # the file could be opened.
+                return self.fs.open(path, mode)
+            except OSError:
+                if count == 10:
+                    # Give up after 10 tries to open the file.
+                    raise
+                continue
+
+    def _write_file(self, full_path, iterator, mode):
+        # ensure dir exists
+        destination, _ = os.path.split(full_path)
+        if not self.fs.exists(destination):
+            self.fs.makedirs(destination)
+
+        writer = self._get_writer(full_path, mode)
+
+        digest = hashlib.sha256()
+        with writer:
+            size = 0
+            for chunk in iterator:
+                size += len(chunk)
+                digest.update(chunk)
+                writer.write(chunk)
+            writer.flush()
+            # Get the file descriptor
+            fd = writer.fileno()
+
+            # Sync the file descriptor to disk, helps with NFS cases...
+            os.fsync(fd)
+            sha256 = digest.hexdigest()
+            log.debug('written new archive cache under %s, sha256: %s', full_path, sha256)
+            return size, sha256
+
+    def store(self, key, value_reader, metadata: dict | None = None):
+        return self._store(key, value_reader, metadata, mode='xb')
+
+    def fetch(self, key, retry=NOT_GIVEN, retry_attempts=NOT_GIVEN, retry_backoff=1) -> tuple[ShardFileReader, dict]:
+        return self._fetch(key, retry, retry_attempts, retry_backoff)
+
+    def remove(self, key):
+        return self._remove(key)
+
+    def random_filename(self):
+        """Return filename and full-path tuple for file storage.
+
+        Filename will be a randomly generated 28 character hexadecimal string
+        with ".archive_cache" suffixed. Two levels of sub-directories will be used to
+        reduce the size of directories. On older filesystems, lookups in
+        directories with many files may be slow.
+        """
+
+        hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8')
+
+        archive_name = hex_name[4:] + '.archive_cache'
+        filename = f"{hex_name[:2]}/{hex_name[2:4]}/{archive_name}"
+
+        full_path = os.path.join(self.directory, filename)
+        return archive_name, full_path
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}(index={self._index}, dir={self.directory})'
+
+
+class FileSystemFanoutCache(BaseCache):
+
+    def __init__(self, locking_url, **settings):
+        """
+        Initialize file system cache instance.
+
+        :param str locking_url: redis url for a lock
+        :param settings: settings dict
+
+        """
+        self._locking_url = locking_url
+        self._config = settings
+        cache_dir = self.get_conf('archive_cache.filesystem.store_dir')
+        directory = str(cache_dir)
+        directory = os.path.expanduser(directory)
+        directory = os.path.expandvars(directory)
+        self._directory = directory
+        self._storage_path = directory
+
+        # check if it's ok to write, and re-create the archive cache
+        if not os.path.isdir(self._directory):
+            os.makedirs(self._directory, exist_ok=True)
+
+        self._count = int(self.get_conf('archive_cache.filesystem.cache_shards', pop=True))
+
+        self._eviction_policy = self.get_conf('archive_cache.filesystem.eviction_policy', pop=True)
+        self._cache_size_limit = self.gb_to_bytes(int(self.get_conf('archive_cache.filesystem.cache_size_gb')))
+
+        self.retry = str2bool(self.get_conf('archive_cache.filesystem.retry', pop=True))
+        self.retry_attempts = int(self.get_conf('archive_cache.filesystem.retry_attempts', pop=True))
+        self.retry_backoff = int(self.get_conf('archive_cache.filesystem.retry_backoff', pop=True))
+
+        log.debug('Initializing archival cache instance under %s', self._directory)
+        self._shards = tuple(
+            FileSystemShard(
+                index=num,
+                directory=os.path.join(directory, 'shard_%03d' % num),
+                **settings,
+            )
+            for num in range(self._count)
+        )
+        self._hash = self._shards[0].hash
+
+    def _get_shard(self, key) -> FileSystemShard:
+        index = self._hash(key) % self._count
+        shard = self._shards[index]
+        return shard
+
+    def _get_size(self, shard, archive_path):
+        return os.stat(archive_path).st_size
diff --git a/rhodecode/lib/archive_cache/backends/objectstore_cache.py b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
new file mode 100644
index 00000000..e2d5ed2b
--- /dev/null
+++ b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
@@ -0,0 +1,150 @@
+# Copyright (C) 2015-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import codecs
+import hashlib
+import logging
+import os
+
+import fsspec
+
+from .base import BaseCache, BaseShard
+from ..utils import ShardFileReader, NOT_GIVEN
+from ...type_utils import str2bool
+
+log = logging.getLogger(__name__)
+
+
+class S3Shard(BaseShard):
+
+    def __init__(self, index, bucket, **settings):
+        self._index = index
+        self._bucket = bucket
+        self.storage_type = 'bucket'
+
+        endpoint_url = settings.pop('archive_cache.objectstore.url')
+        key = settings.pop('archive_cache.objectstore.key')
+        secret = settings.pop('archive_cache.objectstore.secret')
+
+        self.fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret)
+
+    @property
+    def bucket(self):
+        """Cache bucket."""
+        return self._bucket
+
+    def _get_keyfile(self, archive_key) -> tuple[str, str]:
+        key_file = f'{archive_key}-{self.key_suffix}'
+        return key_file, os.path.join(self.bucket, key_file)
+
+    def _get_writer(self, path, mode):
+        return self.fs.open(path, 'wb')
+
+    def _write_file(self, full_path, iterator, mode):
+        # ensure bucket exists
+        destination = self.bucket
+        if not self.fs.exists(destination):
+            self.fs.mkdir(destination, s3_additional_kwargs={})
+
+        writer = self._get_writer(full_path, mode)
+
+        digest = hashlib.sha256()
+        with writer:
+            size = 0
+            for chunk in iterator:
+                size += len(chunk)
+                digest.update(chunk)
+                writer.write(chunk)
+
+            sha256 = digest.hexdigest()
+            log.debug('written new archive cache under %s, sha256: %s', full_path, sha256)
+            return size, sha256
+
+    def store(self, key, value_reader, metadata: dict | None = None):
+        return self._store(key, value_reader, metadata, mode='wb')
+
+    def fetch(self, key, retry=NOT_GIVEN, retry_attempts=NOT_GIVEN, retry_backoff=1) -> tuple[ShardFileReader, dict]:
+        return self._fetch(key, retry, retry_attempts, retry_backoff)
+
+    def remove(self, key):
+        return self._remove(key)
+
+    def random_filename(self):
+        """Return filename and full-path tuple for file storage.
+
+        Filename will be a randomly generated 28 character hexadecimal string
+        with ".archive_cache" suffixed. Two levels of sub-directories will be used to
+        reduce the size of directories. On older filesystems, lookups in
+        directories with many files may be slow.
+        """
+
+        hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8')
+
+        archive_name = hex_name[4:] + '.archive_cache'
+        filename = f"{hex_name[:2]}-{hex_name[2:4]}-{archive_name}"
+
+        full_path = os.path.join(self.bucket, filename)
+        return archive_name, full_path
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}(index={self._index}, bucket={self.bucket})'
+
+
+class ObjectStoreCache(BaseCache):
+
+    def __init__(self, locking_url, **settings):
+        """
+        Initialize objectstore cache instance.
+
+        :param str locking_url: redis url for a lock
+        :param settings: settings dict
+
+        """
+        self._locking_url = locking_url
+        self._config = settings
+
+        objectstore_url = self.get_conf('archive_cache.objectstore.url')
+        self._storage_path = objectstore_url
+
+        self._count = int(self.get_conf('archive_cache.objectstore.bucket_shards', pop=True))
+
+        self._eviction_policy = self.get_conf('archive_cache.objectstore.eviction_policy', pop=True)
+        self._cache_size_limit = self.gb_to_bytes(int(self.get_conf('archive_cache.objectstore.cache_size_gb')))
+
+        self.retry = str2bool(self.get_conf('archive_cache.objectstore.retry', pop=True))
+        self.retry_attempts = int(self.get_conf('archive_cache.objectstore.retry_attempts', pop=True))
+        self.retry_backoff = int(self.get_conf('archive_cache.objectstore.retry_backoff', pop=True))
+
+        log.debug('Initializing archival cache instance under %s', objectstore_url)
+        self._shards = tuple(
+            S3Shard(
+                index=num,
+                bucket='rhodecode-archivecache-%03d' % num,
+                **settings,
+            )
+            for num in range(self._count)
+        )
+        self._hash = self._shards[0].hash
+
+    def _get_shard(self, key) -> S3Shard:
+        index = self._hash(key) % self._count
+        shard = self._shards[index]
+        return shard
+
+    def _get_size(self, shard, archive_path):
+        return shard.fs.info(archive_path)['size']
diff --git a/rhodecode/lib/rc_cache/archive_cache/lock.py b/rhodecode/lib/archive_cache/lock.py
similarity index 95%
rename from rhodecode/lib/rc_cache/archive_cache/lock.py
rename to rhodecode/lib/archive_cache/lock.py
index 4652c0aa..f79d8820 100644
--- a/rhodecode/lib/rc_cache/archive_cache/lock.py
+++ b/rhodecode/lib/archive_cache/lock.py
@@ -17,8 +17,11 @@
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import redis
-from ..._vendor import redis_lock
-from .utils import ArchiveCacheGenerationLock
+from .._vendor import redis_lock
+
+
+class ArchiveCacheGenerationLock(Exception):
+    pass
 
 
 class GenerationLock:
diff --git a/rhodecode/lib/archive_cache/utils.py b/rhodecode/lib/archive_cache/utils.py
new file mode 100644
index 00000000..8e46ec7a
--- /dev/null
+++ b/rhodecode/lib/archive_cache/utils.py
@@ -0,0 +1,134 @@
+# Copyright (C) 2015-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import sqlite3
+import s3fs.core
+
+NOT_GIVEN = -917
+
+
+EVICTION_POLICY = {
+    'none': {
+        'evict': None,
+    },
+    'least-recently-stored': {
+        'evict': 'SELECT {fields} FROM archive_cache ORDER BY store_time',
+    },
+    'least-recently-used': {
+        'evict': 'SELECT {fields} FROM archive_cache ORDER BY access_time',
+    },
+    'least-frequently-used': {
+        'evict': 'SELECT {fields} FROM archive_cache ORDER BY access_count',
+    },
+}
+
+
+def archive_iterator(_reader, block_size: int = 4096 * 512):
+    # 4096 * 64 = 64KB
+    while 1:
+        data = _reader.read(block_size)
+        if not data:
+            break
+        yield data
+
+
+def format_size(size):
+    # Convert size in bytes to a human-readable format (e.g., KB, MB, GB)
+    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
+        if size < 1024:
+            return f"{size:.2f} {unit}"
+        size /= 1024
+
+
+class StatsDB:
+
+    def __init__(self):
+        self.connection = sqlite3.connect(':memory:')
+        self._init_db()
+
+    def _init_db(self):
+        qry = '''
+            CREATE TABLE IF NOT EXISTS archive_cache (
+             rowid INTEGER PRIMARY KEY,
+             key_file TEXT,
+             key_file_path TEXT,
+             archive_key TEXT,
+             archive_path TEXT,
+             store_time REAL,
+             access_time REAL,
+             access_count INTEGER DEFAULT 0,
+             size INTEGER DEFAULT 0
+             )
+        '''
+
+        self.sql(qry)
+        self.connection.commit()
+
+    @property
+    def sql(self):
+        return self.connection.execute
+
+    def bulk_insert(self, rows):
+        qry = '''
+            INSERT INTO archive_cache (
+             rowid,
+             key_file,
+             key_file_path,
+             archive_key,
+             archive_path,
+             store_time,
+             access_time,
+             access_count,
+             size
+            )
+            VALUES (
+            ?, ?, ?, ?, ?, ?, ?, ?, ?
+            )
+        '''
+        cursor = self.connection.cursor()
+        cursor.executemany(qry, rows)
+        self.connection.commit()
+
+    def get_total_size(self):
+        qry = 'SELECT COALESCE(SUM(size), 0) FROM archive_cache'
+        ((total_size,),) = self.sql(qry).fetchall()
+        return total_size
+
+    def get_sorted_keys(self, select_policy):
+        select_policy_qry = select_policy.format(fields='key_file, archive_key, size')
+        return self.sql(select_policy_qry).fetchall()
+
+
+class ShardFileReader:
+
+    def __init__(self, file_like_reader):
+        self._file_like_reader = file_like_reader
+
+    def __getattr__(self, item):
+        if isinstance(self._file_like_reader, s3fs.core.S3File):
+            match item:
+                case 'name':
+                    # S3 FileWrapper doesn't support name attribute, and we use it
+                    return self._file_like_reader.full_name
+                case _:
+                    return getattr(self._file_like_reader, item)
+        else:
+            return getattr(self._file_like_reader, item)
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__}={self._file_like_reader}>'
diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py
index c82ef08e..45f74ae3 100644
--- a/rhodecode/lib/base.py
+++ b/rhodecode/lib/base.py
@@ -578,7 +578,7 @@ def bootstrap_config(request, registry_name='RcTestRegistry'):
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
-    config.include('rhodecode.lib.rc_cache.archive_cache')
+    config.include('rhodecode.lib.archive_cache')
     add_events_routes(config)
 
     return config
diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py
index 62e05336..033e4736 100644
--- a/rhodecode/lib/helpers.py
+++ b/rhodecode/lib/helpers.py
@@ -2196,3 +2196,35 @@ class IssuesRegistry(object):
     @property
     def issues_unique_count(self):
         return len(set(i['id'] for i in self.issues))
+
+
+def get_directory_statistics(start_path):
+    """
+    total_files, total_size, directory_stats = get_directory_statistics(start_path)
+
+    print(f"Directory statistics for: {start_path}\n")
+    print(f"Total files: {total_files}")
+    print(f"Total size: {format_size(total_size)}\n")
+
+    :param start_path:
+    :return:
+    """
+
+    total_files = 0
+    total_size = 0
+    directory_stats = {}
+
+    for dir_path, dir_names, file_names in os.walk(start_path):
+        dir_size = 0
+        file_count = len(file_names)
+
+        for fname in file_names:
+            filepath = os.path.join(dir_path, fname)
+            file_size = os.path.getsize(filepath)
+            dir_size += file_size
+
+        directory_stats[dir_path] = {'file_count': file_count, 'size': dir_size}
+        total_files += file_count
+        total_size += dir_size
+
+    return total_files, total_size, directory_stats
diff --git a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py b/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
deleted file mode 100644
index 90530b8b..00000000
--- a/rhodecode/lib/rc_cache/archive_cache/fanout_cache.py
+++ /dev/null
@@ -1,456 +0,0 @@
-# Copyright (C) 2015-2024 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-import codecs
-import contextlib
-import functools
-import os
-import logging
-import time
-import typing
-import zlib
-import sqlite3
-
-from ...ext_json import json
-from .lock import GenerationLock
-from .utils import format_size
-
-log = logging.getLogger(__name__)
-
-cache_meta = None
-
-UNKNOWN = -241
-NO_VAL = -917
-
-MODE_BINARY = 'BINARY'
-
-
-EVICTION_POLICY = {
-    'none': {
-        'evict': None,
-    },
-    'least-recently-stored': {
-        'evict': 'SELECT {fields} FROM archive_cache ORDER BY store_time',
-    },
-    'least-recently-used': {
-        'evict': 'SELECT {fields} FROM archive_cache ORDER BY access_time',
-    },
-    'least-frequently-used': {
-        'evict': 'SELECT {fields} FROM archive_cache ORDER BY access_count',
-    },
-}
-
-
-class DB:
-
-    def __init__(self):
-        self.connection = sqlite3.connect(':memory:')
-        self._init_db()
-
-    def _init_db(self):
-        qry = '''
-            CREATE TABLE IF NOT EXISTS archive_cache (
-             rowid INTEGER PRIMARY KEY,
-             key_file TEXT,
-             key_file_path TEXT,
-             filename TEXT,
-             full_path TEXT,
-             store_time REAL,
-             access_time REAL,
-             access_count INTEGER DEFAULT 0,
-             size INTEGER DEFAULT 0
-             )
-        '''
-
-        self.sql(qry)
-        self.connection.commit()
-
-    @property
-    def sql(self):
-        return self.connection.execute
-
-    def bulk_insert(self, rows):
-        qry = '''
-            INSERT INTO archive_cache (
-             rowid,
-             key_file,
-             key_file_path,
-             filename,
-             full_path,
-             store_time,
-             access_time,
-             access_count,
-             size
-            )
-            VALUES (
-            ?, ?, ?, ?, ?, ?, ?, ?, ?
-            )
-        '''
-        cursor = self.connection.cursor()
-        cursor.executemany(qry, rows)
-        self.connection.commit()
-
-
-class FileSystemCache:
-
-    def __init__(self, index, directory, **settings):
-        self._index = index
-        self._directory = directory
-
-    @property
-    def directory(self):
-        """Cache directory."""
-        return self._directory
-
-    def _write_file(self, full_path, iterator, mode, encoding=None):
-        full_dir, _ = os.path.split(full_path)
-
-        for count in range(1, 11):
-            with contextlib.suppress(OSError):
-                os.makedirs(full_dir)
-
-            try:
-                # Another cache may have deleted the directory before
-                # the file could be opened.
-                writer = open(full_path, mode, encoding=encoding)
-            except OSError:
-                if count == 10:
-                    # Give up after 10 tries to open the file.
-                    raise
-                continue
-
-            with writer:
-                size = 0
-                for chunk in iterator:
-                    size += len(chunk)
-                    writer.write(chunk)
-                writer.flush()
-                # Get the file descriptor
-                fd = writer.fileno()
-
-                # Sync the file descriptor to disk, helps with NFS cases...
-                os.fsync(fd)
-                log.debug('written new archive cache under %s', full_path)
-                return size
-
-    def _get_keyfile(self, key):
-        return os.path.join(self._directory, f'{key}.key')
-
-    def store(self, key, value_reader, metadata):
-        filename, full_path = self.random_filename()
-        key_file = self._get_keyfile(key)
-
-        # STORE METADATA
-        _metadata = {
-            "version": "v1",
-            "filename": filename,
-            "full_path": full_path,
-            "key_file": key_file,
-            "store_time": time.time(),
-            "access_count": 1,
-            "access_time": 0,
-            "size": 0
-        }
-        if metadata:
-            _metadata.update(metadata)
-
-        reader = functools.partial(value_reader.read, 2**22)
-
-        iterator = iter(reader, b'')
-        size = self._write_file(full_path, iterator, 'xb')
-        metadata['size'] = size
-
-        # after archive is finished, we create a key to save the presence of the binary file
-        with open(key_file, 'wb') as f:
-            f.write(json.dumps(_metadata))
-
-        return key, size, MODE_BINARY, filename, _metadata
-
-    def fetch(self, key, retry=False, retry_attempts=10) -> tuple[typing.BinaryIO, dict]:
-
-        if retry:
-            for attempt in range(retry_attempts):
-                if key in self:
-                    break
-                # we dind't find the key, wait 1s, and re-check
-                time.sleep(1)
-
-        if key not in self:
-            log.exception('requested {key} not found in {self}', key, self)
-            raise KeyError(key)
-
-        key_file = self._get_keyfile(key)
-        with open(key_file, 'rb') as f:
-            metadata = json.loads(f.read())
-
-        filename = metadata['filename']
-
-        try:
-            return open(os.path.join(self.directory, filename), 'rb'), metadata
-        finally:
-            # update usage stats, count and accessed
-            metadata["access_count"] = metadata.get("access_count", 0) + 1
-            metadata["access_time"] = time.time()
-
-            with open(key_file, 'wb') as f:
-                f.write(json.dumps(metadata))
-
-    def random_filename(self):
-        """Return filename and full-path tuple for file storage.
-
-        Filename will be a randomly generated 28 character hexadecimal string
-        with ".archive_cache" suffixed. Two levels of sub-directories will be used to
-        reduce the size of directories. On older filesystems, lookups in
-        directories with many files may be slow.
-        """
-
-        hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8')
-        sub_dir = os.path.join(hex_name[:2], hex_name[2:4])
-        name = hex_name[4:] + '.archive_cache'
-        filename = os.path.join(sub_dir, name)
-        full_path = os.path.join(self.directory, filename)
-        return filename, full_path
-
-    def hash(self, key):
-        """Compute portable hash for `key`.
-
-        :param key: key to hash
-        :return: hash value
-
-        """
-        mask = 0xFFFFFFFF
-        return zlib.adler32(key.encode('utf-8')) & mask  # noqa
-
-    def __contains__(self, key):
-        """Return `True` if `key` matching item is found in cache.
-
-        :param key: key matching item
-        :return: True if key matching item
-
-        """
-        key_file = self._get_keyfile(key)
-        return os.path.exists(key_file)
-
-    def __repr__(self):
-        return f'FileSystemCache(index={self._index}, dir={self.directory})'
-
-
-class FanoutCache:
-    """Cache that shards keys and values."""
-
-    def __init__(
-        self, directory=None, **settings
-    ):
-        """Initialize cache instance.
-
-        :param str directory: cache directory
-        :param settings: settings dict
-
-        """
-        if directory is None:
-            raise ValueError('directory cannot be None')
-
-        directory = str(directory)
-        directory = os.path.expanduser(directory)
-        directory = os.path.expandvars(directory)
-        self._directory = directory
-
-        self._count = settings.pop('cache_shards')
-        self._locking_url = settings.pop('locking_url')
-
-        self._eviction_policy = settings['cache_eviction_policy']
-        self._cache_size_limit = settings['cache_size_limit']
-
-        self._shards = tuple(
-            FileSystemCache(
-                index=num,
-                directory=os.path.join(directory, 'shard_%03d' % num),
-                **settings,
-            )
-            for num in range(self._count)
-        )
-        self._hash = self._shards[0].hash
-
-    @property
-    def directory(self):
-        """Cache directory."""
-        return self._directory
-
-    def get_lock(self, lock_key):
-        return GenerationLock(lock_key, self._locking_url)
-
-    def _get_shard(self, key) -> FileSystemCache:
-        index = self._hash(key) % self._count
-        shard = self._shards[index]
-        return shard
-
-    def store(self, key, value_reader, metadata=None):
-        shard = self._get_shard(key)
-        return shard.store(key, value_reader, metadata)
-
-    def fetch(self, key, retry=False, retry_attempts=10):
-        """Return file handle corresponding to `key` from cache.
-        """
-        shard = self._get_shard(key)
-        return shard.fetch(key, retry=retry, retry_attempts=retry_attempts)
-
-    def has_key(self, key):
-        """Return `True` if `key` matching item is found in cache.
-
-        :param key: key for item
-        :return: True if key is found
-
-        """
-        shard = self._get_shard(key)
-        return key in shard
-
-    def __contains__(self, item):
-        return self.has_key(item)
-
-    def evict(self, policy=None, size_limit=None):
-        """
-        Remove old items based on the conditions
-
-
-        explanation of this algo:
-        iterate over each shard, then for each shard iterate over the .key files
-        read the key files metadata stored. This gives us a full list of keys, cached_archived, their size and
-        access data, time creation, and access counts.
-
-        Store that into a memory DB so we can run different sorting strategies easily.
-        Summing the size is a sum sql query.
-
-        Then we run a sorting strategy based on eviction policy.
-        We iterate over sorted keys, and remove each checking if we hit the overall limit.
-        """
-
-        policy = policy or self._eviction_policy
-        size_limit = size_limit or self._cache_size_limit
-
-        select_policy = EVICTION_POLICY[policy]['evict']
-
-        log.debug('Running eviction policy \'%s\', and checking for size limit: %s',
-                  policy, format_size(size_limit))
-
-        if select_policy is None:
-            return 0
-
-        db = DB()
-
-        data = []
-        cnt = 1
-        for shard in self._shards:
-            for key_file in os.listdir(shard.directory):
-                if key_file.endswith('.key'):
-                    key_file_path = os.path.join(shard.directory, key_file)
-                    with open(key_file_path, 'rb') as f:
-                        metadata = json.loads(f.read())
-
-                        size = metadata.get('size')
-                        filename = metadata.get('filename')
-                        full_path = metadata.get('full_path')
-
-                        if not size:
-                            # in case we don't have size re-calc it...
-                            size = os.stat(full_path).st_size
-
-                        data.append([
-                          cnt,
-                          key_file,
-                          key_file_path,
-                          filename,
-                          full_path,
-                          metadata.get('store_time', 0),
-                          metadata.get('access_time', 0),
-                          metadata.get('access_count', 0),
-                          size,
-                        ])
-                        cnt += 1
-
-        # Insert bulk data using executemany
-        db.bulk_insert(data)
-
-        ((total_size,),) = db.sql('SELECT COALESCE(SUM(size), 0) FROM archive_cache').fetchall()
-        log.debug('Analyzed %s keys, occupied: %s', len(data), format_size(total_size))
-        select_policy_qry = select_policy.format(fields='key_file_path, full_path, size')
-        sorted_keys = db.sql(select_policy_qry).fetchall()
-
-        removed_items = 0
-        removed_size = 0
-        for key, cached_file, size in sorted_keys:
-            # simulate removal impact BEFORE removal
-            total_size -= size
-
-            if total_size <= size_limit:
-                # we obtained what we wanted...
-                break
-
-            os.remove(cached_file)
-            os.remove(key)
-            removed_items += 1
-            removed_size += size
-
-        log.debug('Removed %s cache archives, and reduced size: %s', removed_items, format_size(removed_size))
-        return removed_items
-
-
-def get_archival_config(config):
-
-    final_config = {
-
-    }
-
-    for k, v in config.items():
-        if k.startswith('archive_cache'):
-            final_config[k] = v
-
-    return final_config
-
-
-def get_archival_cache_store(config):
-
-    global cache_meta
-    if cache_meta is not None:
-        return cache_meta
-
-    config = get_archival_config(config)
-    backend = config['archive_cache.backend.type']
-    if backend != 'filesystem':
-        raise ValueError('archive_cache.backend.type only supports "filesystem"')
-
-    archive_cache_locking_url = config['archive_cache.locking.url']
-    archive_cache_dir = config['archive_cache.filesystem.store_dir']
-    archive_cache_size_gb = config['archive_cache.filesystem.cache_size_gb']
-    archive_cache_shards = config['archive_cache.filesystem.cache_shards']
-    archive_cache_eviction_policy = config['archive_cache.filesystem.eviction_policy']
-
-    log.debug('Initializing archival cache instance under %s', archive_cache_dir)
-
-    # check if it's ok to write, and re-create the archive cache
-    if not os.path.isdir(archive_cache_dir):
-        os.makedirs(archive_cache_dir, exist_ok=True)
-
-    d_cache = FanoutCache(
-        archive_cache_dir,
-        locking_url=archive_cache_locking_url,
-        cache_shards=archive_cache_shards,
-        cache_size_limit=archive_cache_size_gb * 1024 * 1024 * 1024,
-        cache_eviction_policy=archive_cache_eviction_policy
-    )
-    cache_meta = d_cache
-    return cache_meta
diff --git a/rhodecode/lib/rc_cache/archive_cache/utils.py b/rhodecode/lib/rc_cache/archive_cache/utils.py
deleted file mode 100644
index 736cfdb7..00000000
--- a/rhodecode/lib/rc_cache/archive_cache/utils.py
+++ /dev/null
@@ -1,72 +0,0 @@
-# Copyright (C) 2015-2024 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-import os
-
-
-class ArchiveCacheGenerationLock(Exception):
-    pass
-
-
-def archive_iterator(_reader, block_size: int = 4096 * 512):
-    # 4096 * 64 = 64KB
-    while 1:
-        data = _reader.read(block_size)
-        if not data:
-            break
-        yield data
-
-
-def get_directory_statistics(start_path):
-    """
-    total_files, total_size, directory_stats = get_directory_statistics(start_path)
-
-    print(f"Directory statistics for: {start_path}\n")
-    print(f"Total files: {total_files}")
-    print(f"Total size: {format_size(total_size)}\n")
-
-    :param start_path:
-    :return:
-    """
-
-    total_files = 0
-    total_size = 0
-    directory_stats = {}
-
-    for dir_path, dir_names, file_names in os.walk(start_path):
-        dir_size = 0
-        file_count = len(file_names)
-
-        for file in file_names:
-            filepath = os.path.join(dir_path, file)
-            file_size = os.path.getsize(filepath)
-            dir_size += file_size
-
-        directory_stats[dir_path] = {'file_count': file_count, 'size': dir_size}
-        total_files += file_count
-        total_size += dir_size
-
-    return total_files, total_size, directory_stats
-
-
-def format_size(size):
-    # Convert size in bytes to a human-readable format (e.g., KB, MB, GB)
-    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
-        if size < 1024:
-            return f"{size:.2f} {unit}"
-        size /= 1024
diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py
index 7a831a52..157b631c 100644
--- a/rhodecode/lib/system_info.py
+++ b/rhodecode/lib/system_info.py
@@ -399,29 +399,23 @@ def storage_inodes():
 def storage_archives():
     import rhodecode
     from rhodecode.lib.helpers import format_byte_size_binary
-    from rhodecode.lib.rc_cache.archive_cache.utils import get_directory_statistics
+    from rhodecode.lib.archive_cache import get_archival_cache_store
 
     storage_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
-    storage_key = 'archive_cache.filesystem.store_dir'
 
-    default_msg = 'Archive cache storage is controlled by '\
-                  f'{storage_key}=/path/to/cache option in the .ini file'
-    path = rhodecode.ConfigGet().get_str(storage_key, missing=default_msg)
-
-    value = dict(percent=0, used=0, total=0, items=0, path=path, text='', type=storage_type)
+    value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=storage_type)
     state = STATE_OK_DEFAULT
     try:
-        if storage_type != 'filesystem':
-            # raise Exc to stop reporting on different type
-            raise ValueError('Storage type must be "filesystem"')
+        d_cache = get_archival_cache_store(config=rhodecode.CONFIG)
 
-        total_files, total_size, _directory_stats = get_directory_statistics(path)
+        total_files, total_size, _directory_stats = d_cache.get_statistics()
 
         value.update({
             'percent': 100,
             'used': total_size,
             'total': total_size,
-            'items': total_files
+            'items': total_files,
+            'path': d_cache.storage_path
         })
 
     except Exception as e:
@@ -441,8 +435,7 @@ def storage_archives():
 def storage_gist():
     from rhodecode.model.gist import GIST_STORE_LOC
     from rhodecode.lib.utils import safe_str, get_rhodecode_repo_store_path
-    from rhodecode.lib.helpers import format_byte_size_binary
-    from rhodecode.lib.rc_cache.archive_cache.utils import get_directory_statistics
+    from rhodecode.lib.helpers import format_byte_size_binary, get_directory_statistics
 
     path = safe_str(os.path.join(
         get_rhodecode_repo_store_path(), GIST_STORE_LOC))
diff --git a/rhodecode/tests/fixture_mods/fixture_pyramid.py b/rhodecode/tests/fixture_mods/fixture_pyramid.py
index 8173595a..b5c80d88 100644
--- a/rhodecode/tests/fixture_mods/fixture_pyramid.py
+++ b/rhodecode/tests/fixture_mods/fixture_pyramid.py
@@ -100,7 +100,7 @@ def ini_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
     overrides = [
         {'server:main': {'port': rcserver_port}},
         {'app:main': {
-            'cache_dir': '%(here)s/rc_data',
+            'cache_dir': '%(here)s/rc-tests/rc_data',
             'vcs.server': f'localhost:{vcsserver_port}',
             # johbo: We will always start the VCSServer on our own based on the
             # fixtures of the test cases. For the test run it must always be
diff --git a/rhodecode/tests/lib/test_archive_caches.py b/rhodecode/tests/lib/test_archive_caches.py
new file mode 100644
index 00000000..91dd5a05
--- /dev/null
+++ b/rhodecode/tests/lib/test_archive_caches.py
@@ -0,0 +1,105 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import time
+import pytest
+import rhodecode
+import os
+import shutil
+from tempfile import mkdtemp
+
+from rhodecode.lib import archive_cache
+
+
+def file_reader(temp_store):
+    with open(temp_store, 'w') as f:
+        for cnt in range(10000):
+            f.write(str(cnt))
+    return open(temp_store, 'rb')
+
+
+@pytest.fixture()
+def d_cache_instance(ini_settings):
+    config = ini_settings
+    d_cache = archive_cache.get_archival_cache_store(config=config, always_init=True)
+    return d_cache
+
+
+@pytest.mark.usefixtures('app')
+class TestArchiveCaches(object):
+
+    def test_archivecache_empty_stats(self, d_cache_instance):
+        d_cache = d_cache_instance
+        shutil.rmtree(d_cache._directory)
+
+        stats = d_cache.get_statistics()
+        assert (0, 0, {}) == stats
+
+    def test_archivecache_store_keys(self, d_cache_instance, tmp_path):
+        d_cache = d_cache_instance
+        shutil.rmtree(d_cache._directory)
+
+        for n in range(100):
+
+            archive_name = f'my-archive-abc-{n}.zip'
+            temp_archive_path = os.path.join(tmp_path, archive_name)
+            d_cache.store(archive_name, file_reader(temp_archive_path ), {'foo': 'bar'})
+            reader, meta = d_cache.fetch(archive_name)
+            content = reader.read()
+            assert content == open(temp_archive_path, 'rb').read()
+
+        stats = d_cache.get_statistics()
+        assert (100, 3889000, {}) == stats
+
+    def test_archivecache_remove_keys(self, d_cache_instance, tmp_path):
+        d_cache = d_cache_instance
+        shutil.rmtree(d_cache._directory)
+
+        n = 1
+        archive_name = f'my-archive-abc-{n}.zip'
+        temp_archive_path = os.path.join(tmp_path, archive_name)
+
+        d_cache.store(archive_name, file_reader(temp_archive_path ), {'foo': 'bar'})
+        stats = d_cache.get_statistics()
+        assert (1, 38890, {}) == stats
+
+        assert 1 == d_cache.remove(archive_name)
+
+        stats = d_cache.get_statistics()
+        assert (0, 0, {}) == stats
+
+    def test_archivecache_evict_keys(self, d_cache_instance, tmp_path):
+        d_cache = d_cache_instance
+        shutil.rmtree(d_cache._directory)
+        tries = 500
+        for n in range(tries):
+
+            archive_name = f'my-archive-abc-{n}.zip'
+            temp_archive_path = os.path.join(tmp_path, archive_name)
+            d_cache.store(archive_name, file_reader(temp_archive_path ), {'foo': 'bar'})
+
+        stats = d_cache.get_statistics()
+        assert (tries, 19445000, {}) == stats
+        evict_to = 0.005  # around (5mb)
+        evicted_items = d_cache.evict(size_limit=d_cache.gb_to_bytes(evict_to))
+        evicted = 361
+        assert evicted == evicted_items
+
+        stats = d_cache.get_statistics()
+        assert (tries - evicted, 5405710, {}) == stats
+
diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini
index b401f0e4..8ce4c67d 100644
--- a/rhodecode/tests/rhodecode.ini
+++ b/rhodecode/tests/rhodecode.ini
@@ -258,20 +258,62 @@ file_store.backend = local
 ; path to store the uploaded binaries and artifacts
 file_store.storage_path = /var/opt/rhodecode_data/file_store
 
-; Uncomment and set this path to control settings for archive download cache.
+
+; Redis url to acquire/check generation of archives locks
+archive_cache.locking.url = redis://redis:6379/1
+
+; Storage backend, only 'filesystem' and 'objectstore' are available now
+archive_cache.backend.type = filesystem
+
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+archive_cache.objectstore.url = http://s3-minio:9000
+
+; key for s3 auth
+archive_cache.objectstore.key = key
+
+; secret for s3 auth
+archive_cache.objectstore.secret = secret
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+archive_cache.objectstore.bucket_shards = 8
+
+; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
+archive_cache.objectstore.retry = false
+
+; number of seconds to wait for next try using retry
+archive_cache.objectstore.retry_backoff = 1
+
+; how many tries do do a retry fetch from this backend
+archive_cache.objectstore.retry_attempts = 10
+
+; Default is $cache_dir/archive_cache if not set
 ; Generated repo archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
-
-; Default is $cache_dir/archive_cache if not set
-archive_cache.store_dir = /var/opt/rhodecode_data/tarballcache
+archive_cache.filesystem.store_dir = %(here)s/rc-tests/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
-archive_cache.cache_size_gb = 10
+archive_cache.filesystem.cache_size_gb = 2
+
+; Eviction policy used to clear out after cache_size_gb limit is reached
+archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
-archive_cache.cache_shards = 10
+; default is 8 shards
+archive_cache.filesystem.cache_shards = 8
+
+; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
+archive_cache.filesystem.retry = false
+
+; number of seconds to wait for next try using retry
+archive_cache.filesystem.retry_backoff = 1
+
+; how many tries do do a retry fetch from this backend
+archive_cache.filesystem.retry_attempts = 10
+
 
 ; #############
 ; CELERY CONFIG
@@ -335,7 +377,7 @@ rc_cache.cache_repo_longterm.max_size = 10000
 rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_general.expiration_time = 43200
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
-rc_cache.cache_general.arguments.filename = %(here)s/cache-backend/cache_general_db
+rc_cache.cache_general.arguments.filename = %(here)s/rc-tests/cache-backend/cache_general_db
 
 ; alternative `cache_general` redis backend with distributed lock
 #rc_cache.cache_general.backend = dogpile.cache.rc.redis
@@ -362,7 +404,7 @@ rc_cache.cache_general.arguments.filename = %(here)s/cache-backend/cache_general
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 0
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
-rc_cache.cache_perms.arguments.filename = %(here)s/cache-backend/cache_perms_db
+rc_cache.cache_perms.arguments.filename = %(here)s/rc-tests/cache-backend/cache_perms_db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
@@ -389,7 +431,7 @@ rc_cache.cache_perms.arguments.filename = %(here)s/cache-backend/cache_perms_db
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
-rc_cache.cache_repo.arguments.filename = %(here)s/cache-backend/cache_repo_db
+rc_cache.cache_repo.arguments.filename = %(here)s/rc-tests/cache-backend/cache_repo_db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
@@ -432,7 +474,7 @@ beaker.session.data_dir = %(here)s/rc-tests/data/sessions
 
 beaker.session.key = rhodecode
 beaker.session.secret = test-rc-uytcxaz
-beaker.session.lock_dir = %(here)s/data/sessions/lock
+beaker.session.lock_dir = %(here)s/rc-tests/data/sessions/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
@@ -464,7 +506,7 @@ beaker.session.secure = false
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
-search.location = %(here)s/data/index
+search.location = %(here)s/rc-tests/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
@@ -484,7 +526,7 @@ channelstream.server = channelstream:9800
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = ENV_GENERATED
-channelstream.history.location = %(here)s/channelstream_history
+channelstream.history.location = %(here)s/rc-tests/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
@@ -501,7 +543,7 @@ channelstream.proxy_path = /_channelstream
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
-sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode_test.db?timeout=30
+sqlalchemy.db1.url = sqlite:///%(here)s/rc-tests/rhodecode_test.db?timeout=30
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
@@ -590,7 +632,7 @@ svn.proxy.generate_config = false
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
-svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
+svn.proxy.config_file_path = %(here)s/rc-tests/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
@@ -626,7 +668,7 @@ ssh.generate_authorized_keyfile = true
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
-ssh.authorized_keys_file_path = %(here)s/rc/authorized_keys_rhodecode
+ssh.authorized_keys_file_path = %(here)s/rc-tests/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
diff --git a/rhodecode/tests/vcs/test_archives.py b/rhodecode/tests/vcs/test_archives.py
index 3f285346..b9be4745 100644
--- a/rhodecode/tests/vcs/test_archives.py
+++ b/rhodecode/tests/vcs/test_archives.py
@@ -28,7 +28,7 @@ import mock
 import pytest
 
 import rhodecode
-from rhodecode.lib.rc_cache.archive_cache import get_archival_config
+from rhodecode.lib.archive_cache import get_archival_config
 from rhodecode.lib.str_utils import ascii_bytes
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.vcs.exceptions import ImproperArchiveTypeError, VCSError

From bb46f148f2c0812b2b4f939f1592a5f149f2d79d Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Fri, 14 Jun 2024 08:57:51 +0200
Subject: [PATCH 118/150] feat(archive-cache): added extra info on number of
 evicted caches

---
 rhodecode/lib/archive_cache/backends/base.py | 16 ++++++++++------
 rhodecode/tests/lib/test_archive_caches.py   |  2 +-
 rhodecode/tests/rhodecode.ini                |  4 ++--
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/rhodecode/lib/archive_cache/backends/base.py b/rhodecode/lib/archive_cache/backends/base.py
index 3d9d5637..c47c6907 100644
--- a/rhodecode/lib/archive_cache/backends/base.py
+++ b/rhodecode/lib/archive_cache/backends/base.py
@@ -247,7 +247,7 @@ class BaseCache:
     def get_lock(self, lock_key):
         return GenerationLock(lock_key, self._locking_url)
 
-    def evict(self, policy=None, size_limit=None) -> int:
+    def evict(self, policy=None, size_limit=None) -> dict:
         """
         Remove old items based on the conditions
 
@@ -257,13 +257,16 @@ class BaseCache:
         read the key files metadata stored. This gives us a full list of keys, cached_archived, their size and
         access data, time creation, and access counts.
 
-        Store that into a memory DB so we can run different sorting strategies easily.
+        Store that into a memory DB in order we can run different sorting strategies easily.
         Summing the size is a sum sql query.
 
         Then we run a sorting strategy based on eviction policy.
         We iterate over sorted keys, and remove each checking if we hit the overall limit.
         """
-
+        removal_info = {
+            "removed_items": 0,
+            "removed_size": 0
+        }
         policy = policy or self._eviction_policy
         size_limit = size_limit or self._cache_size_limit
 
@@ -273,7 +276,7 @@ class BaseCache:
                   policy, format_size(size_limit))
 
         if select_policy is None:
-            return 0
+            return removal_info
 
         db = self.get_stats_db()
 
@@ -327,10 +330,11 @@ class BaseCache:
             self.remove(archive_key)
             removed_items += 1
             removed_size += size
-
+        removal_info['removed_items'] = removed_items
+        removal_info['removed_size'] = removed_size
         log.debug('Removed %s cache archives, and reduced size by: %s',
                   removed_items, format_size(removed_size))
-        return removed_items
+        return removal_info
 
     def get_statistics(self):
         total_files = 0
diff --git a/rhodecode/tests/lib/test_archive_caches.py b/rhodecode/tests/lib/test_archive_caches.py
index 91dd5a05..9459cabb 100644
--- a/rhodecode/tests/lib/test_archive_caches.py
+++ b/rhodecode/tests/lib/test_archive_caches.py
@@ -98,7 +98,7 @@ class TestArchiveCaches(object):
         evict_to = 0.005  # around (5mb)
         evicted_items = d_cache.evict(size_limit=d_cache.gb_to_bytes(evict_to))
         evicted = 361
-        assert evicted == evicted_items
+        assert {'removed_items': evicted, 'removed_size': 14039290} == evicted_items
 
         stats = d_cache.get_statistics()
         assert (tries - evicted, 5405710, {}) == stats
diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini
index 8ce4c67d..8af252b4 100644
--- a/rhodecode/tests/rhodecode.ini
+++ b/rhodecode/tests/rhodecode.ini
@@ -348,7 +348,7 @@ celery.task_store_eager_result = true
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
-cache_dir = /var/opt/rhodecode_data
+cache_dir = %(here)s/rc-test-data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
@@ -681,7 +681,7 @@ ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
-ssh.enable_debug_logging = false
+ssh.enable_debug_logging = true
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one

From c2075c12399c34deb299549a6298cca16d9622fc Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Fri, 14 Jun 2024 08:58:18 +0200
Subject: [PATCH 119/150] celery: added test view to test celery exception
 handling

---
 rhodecode/apps/ops/__init__.py   |  9 +++++++++
 rhodecode/apps/ops/views.py      | 14 ++++++++++++++
 rhodecode/lib/celerylib/tasks.py |  5 +++++
 3 files changed, 28 insertions(+)

diff --git a/rhodecode/apps/ops/__init__.py b/rhodecode/apps/ops/__init__.py
index d8a7a2f4..46566ba7 100644
--- a/rhodecode/apps/ops/__init__.py
+++ b/rhodecode/apps/ops/__init__.py
@@ -40,6 +40,15 @@ def admin_routes(config):
         route_name='ops_error_test', request_method='GET',
         renderer='json_ext')
 
+    config.add_route(
+        name='ops_celery_error_test',
+        pattern='/error-celery')
+    config.add_view(
+        OpsView,
+        attr='ops_celery_error_test',
+        route_name='ops_celery_error_test', request_method='GET',
+        renderer='json_ext')
+
     config.add_route(
         name='ops_redirect_test',
         pattern='/redirect')
diff --git a/rhodecode/apps/ops/views.py b/rhodecode/apps/ops/views.py
index 4c0e2045..7b179b54 100644
--- a/rhodecode/apps/ops/views.py
+++ b/rhodecode/apps/ops/views.py
@@ -66,6 +66,20 @@ class OpsView(BaseAppView):
                'Client:{}. Generation time: {}.'.format(self.request.user, time.time()))
         raise TestException(msg)
 
+    def ops_celery_error_test(self):
+        """
+        Test exception handling and emails on errors
+        """
+        from rhodecode.lib.celerylib import tasks, run_task
+
+        # add timeout so we add some sort of rate limiter
+        time.sleep(2)
+
+        msg = ('RhodeCode Enterprise test exception. '
+               'Client:{}. Generation time: {}.'.format(self.request.user, time.time()))
+        celery_task = run_task(tasks.test_celery_exception, msg)
+        return {'task': str(celery_task)}
+
     def ops_redirect_test(self):
         """
         Test redirect handling
diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py
index e3117e80..020e340a 100644
--- a/rhodecode/lib/celerylib/tasks.py
+++ b/rhodecode/lib/celerylib/tasks.py
@@ -402,6 +402,11 @@ def sync_last_update_for_objects(*args, **kwargs):
                 repo_gr.update_commit_cache()
 
 
+@async_task(ignore_result=True, base=RequestContextTask)
+def test_celery_exception(msg):
+    raise Exception(f'Test exception: {msg}')
+
+
 @async_task(ignore_result=True, base=RequestContextTask)
 def sync_last_update(*args, **kwargs):
     sync_last_update_for_objects(*args, **kwargs)

From 9db393bf1acbb6b0d92d184bae5bdff1ffb3c04d Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Fri, 14 Jun 2024 13:10:47 +0300
Subject: [PATCH 120/150] fix(branch removal trough UI): changed route path.
 Fixes: RCCE-75

---
 rhodecode/apps/repository/__init__.py   | 2 +-
 rhodecode/public/js/rhodecode/routes.js | 2 +-
 rhodecode/tests/routes.py               | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/rhodecode/apps/repository/__init__.py b/rhodecode/apps/repository/__init__.py
index 782fa882..b7d4de34 100644
--- a/rhodecode/apps/repository/__init__.py
+++ b/rhodecode/apps/repository/__init__.py
@@ -593,7 +593,7 @@ def includeme(config):
 
     config.add_route(
         name='branch_remove',
-        pattern='/{repo_name:.*?[^/]}/{branch_name:.*?[^/]}/remove', repo_route=True, repo_accepted_types=['hg', 'git'])
+        pattern='/{repo_name:.*?[^/]}/branches/{branch_name:.*?[^/]}/remove', repo_route=True, repo_accepted_types=['hg', 'git'])
     config.add_view(
         RepoBranchesView,
         attr='remove_branch',
diff --git a/rhodecode/public/js/rhodecode/routes.js b/rhodecode/public/js/rhodecode/routes.js
index 960ea4ec..28809228 100644
--- a/rhodecode/public/js/rhodecode/routes.js
+++ b/rhodecode/public/js/rhodecode/routes.js
@@ -92,7 +92,7 @@ function registerRCRoutes() {
     pyroutes.register('auth_home', '/_admin/auth*traverse', []);
     pyroutes.register('bookmarks_home', '/%(repo_name)s/bookmarks', ['repo_name']);
     pyroutes.register('branches_home', '/%(repo_name)s/branches', ['repo_name']);
-    pyroutes.register('branch_remove', '/%(repo_name)s/%(branch_name)s/remove', ['repo_name', 'branch_name']);
+    pyroutes.register('branch_remove', '/%(repo_name)s/branches/%(branch_name)s/remove', ['repo_name', 'branch_name']);
     pyroutes.register('channelstream_connect', '/_admin/channelstream/connect', []);
     pyroutes.register('channelstream_proxy', '/_channelstream', []);
     pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []);
diff --git a/rhodecode/tests/routes.py b/rhodecode/tests/routes.py
index d6f01fc4..f0176744 100644
--- a/rhodecode/tests/routes.py
+++ b/rhodecode/tests/routes.py
@@ -252,7 +252,7 @@ def get_url_defs():
         "pullrequest_show_all_data": "/{repo_name}/pull-request-data",
         "bookmarks_home": "/{repo_name}/bookmarks",
         "branches_home": "/{repo_name}/branches",
-        "branch_remove": "/{repo_name}/{branch_name}/remove",
+        "branch_remove": "/{repo_name}/branches/{branch_name}/remove",
         "tags_home": "/{repo_name}/tags",
         "repo_changelog": "/{repo_name}/changelog",
         "repo_commits": "/{repo_name}/commits",

From 08fbe9fb8f54b4d0faf521933c4b58929e25ac32 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Fri, 14 Jun 2024 19:26:13 +0200
Subject: [PATCH 121/150] docs: updated release notes for 5.1

---
 docs/release-notes/release-notes-5.1.0.rst | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/release-notes/release-notes-5.1.0.rst b/docs/release-notes/release-notes-5.1.0.rst
index bbb4fa41..9e7e5a4f 100644
--- a/docs/release-notes/release-notes-5.1.0.rst
+++ b/docs/release-notes/release-notes-5.1.0.rst
@@ -16,13 +16,16 @@ New Features
 - Email based log-in. RhodeCode now allows to log-in using email as well as username for main authentication type.
 - Ability to replace a file using web UI. Now one can replace an existing file from the web-ui.
 - GIT LFS Sync automation. Remote push/pull commands now can also sync GIT LFS objects.
+- Added ability to remove or close branches from the web ui
+- Added support for S3 based archive_cache based that allows storing cached archives in S3 compatible object store.
+
 
 General
 ^^^^^^^
 
 - Upgraded all dependency libraries to their latest available versions
 - Repository storage is no longer controlled via DB settings, but .ini file. This allows easier automated deployments.
-
+- Bumped mercurial to 6.7.4
 
 Security
 ^^^^^^^^
@@ -34,6 +37,7 @@ Performance
 
 - Introduced a full rewrite of ssh backend for performance. The result is 2-5x speed improvement for operation with ssh.
   enable new ssh wrapper by setting: `ssh.wrapper_cmd = /home/rhodecode/venv/bin/rc-ssh-wrapper-v2`
+- Introduced a new hooks subsystem that is more scalable and faster, enable it by settings: `vcs.hooks.protocol = celery`
 
 
 Fixes

From f7df41ee5fbf99341af5434a5764040b05bd95de Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 17 Jun 2024 11:03:36 +0200
Subject: [PATCH 122/150] docs: added 5.0.3 release notes

---
 docs/release-notes/release-notes-5.0.3.rst | 39 ++++++++++++++++++++++
 docs/release-notes/release-notes.rst       |  1 +
 2 files changed, 40 insertions(+)
 create mode 100644 docs/release-notes/release-notes-5.0.3.rst

diff --git a/docs/release-notes/release-notes-5.0.3.rst b/docs/release-notes/release-notes-5.0.3.rst
new file mode 100644
index 00000000..a48b409a
--- /dev/null
+++ b/docs/release-notes/release-notes-5.0.3.rst
@@ -0,0 +1,39 @@
+|RCE| 5.0.3 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-06-17
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+
+
+
+Fixes
+^^^^^
+
+- Fixed problems nested ldap groups
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.0.3 is unscheduled bugfix release to address some of the issues found during 4.X -> 5.X migration
diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst
index 5d03853e..3dd4e34c 100644
--- a/docs/release-notes/release-notes.rst
+++ b/docs/release-notes/release-notes.rst
@@ -13,6 +13,7 @@ Release Notes
 
    release-notes-5.1.0.rst
    release-notes-5.0.2.rst
+   release-notes-5.0.2.rst
    release-notes-5.0.1.rst
    release-notes-5.0.0.rst
 

From 7459e289a5cde2d838ffbc097586ea639a5d4c56 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 17 Jun 2024 11:03:57 +0200
Subject: [PATCH 123/150] routes: updated routes JS

---
 rhodecode/public/js/rhodecode/routes.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rhodecode/public/js/rhodecode/routes.js b/rhodecode/public/js/rhodecode/routes.js
index 28809228..e224b7c5 100644
--- a/rhodecode/public/js/rhodecode/routes.js
+++ b/rhodecode/public/js/rhodecode/routes.js
@@ -91,8 +91,8 @@ function registerRCRoutes() {
     pyroutes.register('atom_feed_home_old', '/%(repo_name)s/feed/atom', ['repo_name']);
     pyroutes.register('auth_home', '/_admin/auth*traverse', []);
     pyroutes.register('bookmarks_home', '/%(repo_name)s/bookmarks', ['repo_name']);
-    pyroutes.register('branches_home', '/%(repo_name)s/branches', ['repo_name']);
     pyroutes.register('branch_remove', '/%(repo_name)s/branches/%(branch_name)s/remove', ['repo_name', 'branch_name']);
+    pyroutes.register('branches_home', '/%(repo_name)s/branches', ['repo_name']);
     pyroutes.register('channelstream_connect', '/_admin/channelstream/connect', []);
     pyroutes.register('channelstream_proxy', '/_channelstream', []);
     pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []);
@@ -249,6 +249,7 @@ function registerRCRoutes() {
     pyroutes.register('notifications_show', '/_admin/notifications/%(notification_id)s', ['notification_id']);
     pyroutes.register('notifications_show_all', '/_admin/notifications', []);
     pyroutes.register('notifications_update', '/_admin/notifications/%(notification_id)s/update', ['notification_id']);
+    pyroutes.register('ops_celery_error_test', '/_admin/ops/error-celery', []);
     pyroutes.register('ops_error_test', '/_admin/ops/error', []);
     pyroutes.register('ops_healthcheck', '/_admin/ops/status', []);
     pyroutes.register('ops_ping', '/_admin/ops/ping', []);

From f4323f22c5fa4bb4f8ad18bbb7c66ac488d245fb Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 17 Jun 2024 13:01:04 +0200
Subject: [PATCH 124/150] docs: fixed wrong docs link

---
 docs/release-notes/release-notes.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst
index 3dd4e34c..71ee152a 100644
--- a/docs/release-notes/release-notes.rst
+++ b/docs/release-notes/release-notes.rst
@@ -12,7 +12,7 @@ Release Notes
 
 
    release-notes-5.1.0.rst
-   release-notes-5.0.2.rst
+   release-notes-5.0.3.rst
    release-notes-5.0.2.rst
    release-notes-5.0.1.rst
    release-notes-5.0.0.rst

From eed5751983e81fbb12cc106e84c1e1a71c43fb07 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 18 Jun 2024 14:03:06 +0200
Subject: [PATCH 125/150] feat(archive-cache): added option to define a
 configurable top-level bucket for all shards

---
 rhodecode/config/config_maker.py                     |  1 +
 rhodecode/lib/archive_cache/backends/fanout_cache.py |  3 ++-
 .../lib/archive_cache/backends/objectstore_cache.py  | 12 ++++++++++--
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py
index 4e74457b..950ad8aa 100644
--- a/rhodecode/config/config_maker.py
+++ b/rhodecode/config/config_maker.py
@@ -204,6 +204,7 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
     settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
     settings_maker.make_setting('archive_cache.objectstore.key', '')
     settings_maker.make_setting('archive_cache.objectstore.secret', '')
+    settings_maker.make_setting('archive_cache.objectstore.bucket_root', 'rhodecode-archive-cache')
     settings_maker.make_setting('archive_cache.objectstore.bucket_shards', 8, parser='int')
 
     settings_maker.make_setting('archive_cache.objectstore.cache_size_gb', 10, parser='float')
diff --git a/rhodecode/lib/archive_cache/backends/fanout_cache.py b/rhodecode/lib/archive_cache/backends/fanout_cache.py
index 2d96a04a..3c666139 100644
--- a/rhodecode/lib/archive_cache/backends/fanout_cache.py
+++ b/rhodecode/lib/archive_cache/backends/fanout_cache.py
@@ -115,6 +115,7 @@ class FileSystemShard(BaseShard):
 
 
 class FileSystemFanoutCache(BaseCache):
+    shard_name = 'shard_%03d'
 
     def __init__(self, locking_url, **settings):
         """
@@ -150,7 +151,7 @@ class FileSystemFanoutCache(BaseCache):
         self._shards = tuple(
             FileSystemShard(
                 index=num,
-                directory=os.path.join(directory, 'shard_%03d' % num),
+                directory=os.path.join(directory, self.shard_name % num),
                 **settings,
             )
             for num in range(self._count)
diff --git a/rhodecode/lib/archive_cache/backends/objectstore_cache.py b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
index e2d5ed2b..7ae972f1 100644
--- a/rhodecode/lib/archive_cache/backends/objectstore_cache.py
+++ b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
@@ -41,12 +41,15 @@ class S3Shard(BaseShard):
         key = settings.pop('archive_cache.objectstore.key')
         secret = settings.pop('archive_cache.objectstore.secret')
 
+        # TODO: Add it all over the place...
+        self._bucket_root = settings.pop('archive_cache.objectstore.bucket_root')
+
         self.fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret)
 
     @property
     def bucket(self):
         """Cache bucket."""
-        return self._bucket
+        return os.path.join(self._bucket_root, self._bucket)
 
     def _get_keyfile(self, archive_key) -> tuple[str, str]:
         key_file = f'{archive_key}-{self.key_suffix}'
@@ -56,6 +59,10 @@ class S3Shard(BaseShard):
         return self.fs.open(path, 'wb')
 
     def _write_file(self, full_path, iterator, mode):
+        if self._bucket_root:
+            if not self.fs.exists(self._bucket_root):
+                self.fs.mkdir(self._bucket_root)
+
         # ensure bucket exists
         destination = self.bucket
         if not self.fs.exists(destination):
@@ -106,6 +113,7 @@ class S3Shard(BaseShard):
 
 
 class ObjectStoreCache(BaseCache):
+    shard_name = 'shard-bucket-%03d'
 
     def __init__(self, locking_url, **settings):
         """
@@ -134,7 +142,7 @@ class ObjectStoreCache(BaseCache):
         self._shards = tuple(
             S3Shard(
                 index=num,
-                bucket='rhodecode-archivecache-%03d' % num,
+                bucket=self.shard_name % num,
                 **settings,
             )
             for num in range(self._count)

From 5206570f1b2bdc8ba34506b0a194a5f0dcc1dce6 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 18 Jun 2024 14:05:33 +0200
Subject: [PATCH 126/150] configs: added config for top-level bucket

---
 configs/development.ini       | 5 +++++
 configs/production.ini        | 5 +++++
 rhodecode/tests/rhodecode.ini | 5 +++++
 3 files changed, 15 insertions(+)

diff --git a/configs/development.ini b/configs/development.ini
index 9ae745ec..741136b6 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -311,6 +311,11 @@ archive_cache.objectstore.secret = secret
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
 
+; a top-level bucket to put all other sharded buckets in
+; in case it's empty all buckets will be created in top-level (not recommended)
+; objects will be stored in rhodecode-archive-cache/shard-bucket-N based on the bucket_shards number
+archive_cache.objectstore.bucket_root = rhodecode-archive-cache
+
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.objectstore.retry = false
 
diff --git a/configs/production.ini b/configs/production.ini
index 205a1884..e4da1b32 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -279,6 +279,11 @@ archive_cache.objectstore.secret = secret
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
 
+; a top-level bucket to put all other sharded buckets in
+; in case it's empty all buckets will be created in top-level (not recommended)
+; objects will be stored in rhodecode-archive-cache/shard-bucket-N based on the bucket_shards number
+archive_cache.objectstore.bucket_root = rhodecode-archive-cache
+
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.objectstore.retry = false
 
diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini
index 8af252b4..2795cc99 100644
--- a/rhodecode/tests/rhodecode.ini
+++ b/rhodecode/tests/rhodecode.ini
@@ -279,6 +279,11 @@ archive_cache.objectstore.secret = secret
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
 
+; a top-level bucket to put all other sharded buckets in
+; in case it's empty all buckets will be created in top-level (not recommended)
+; objects will be stored in rhodecode-archive-cache/shard-bucket-N based on the bucket_shards number
+archive_cache.objectstore.bucket_root = rhodecode-archive-cache
+
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.objectstore.retry = false
 

From eab897ac5c36bb4f71fc95c7ffa3648f6e54b1b3 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 19 Jun 2024 12:34:05 +0200
Subject: [PATCH 127/150] feat(archive-cache): objectstore now makes bucket
 required, and use shards as folders inside it

---
 configs/development.ini                       |  7 +--
 configs/production.ini                        |  7 +--
 rhodecode/config/config_maker.py              |  2 +-
 rhodecode/lib/archive_cache/backends/base.py  |  7 ++-
 .../archive_cache/backends/fanout_cache.py    | 45 ++++++++------
 .../backends/objectstore_cache.py             | 62 ++++++++++---------
 rhodecode/tests/rhodecode.ini                 |  7 +--
 7 files changed, 75 insertions(+), 62 deletions(-)

diff --git a/configs/development.ini b/configs/development.ini
index 741136b6..49ca314c 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -311,10 +311,9 @@ archive_cache.objectstore.secret = secret
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
 
-; a top-level bucket to put all other sharded buckets in
-; in case it's empty all buckets will be created in top-level (not recommended)
-; objects will be stored in rhodecode-archive-cache/shard-bucket-N based on the bucket_shards number
-archive_cache.objectstore.bucket_root = rhodecode-archive-cache
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-archive-cache/shard-N based on the bucket_shards number
+archive_cache.objectstore.bucket = rhodecode-archive-cache
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.objectstore.retry = false
diff --git a/configs/production.ini b/configs/production.ini
index e4da1b32..0b4540cf 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -279,10 +279,9 @@ archive_cache.objectstore.secret = secret
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
 
-; a top-level bucket to put all other sharded buckets in
-; in case it's empty all buckets will be created in top-level (not recommended)
-; objects will be stored in rhodecode-archive-cache/shard-bucket-N based on the bucket_shards number
-archive_cache.objectstore.bucket_root = rhodecode-archive-cache
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-archive-cache/shard-N based on the bucket_shards number
+archive_cache.objectstore.bucket = rhodecode-archive-cache
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.objectstore.retry = false
diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py
index 950ad8aa..b53de92e 100644
--- a/rhodecode/config/config_maker.py
+++ b/rhodecode/config/config_maker.py
@@ -204,7 +204,7 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
     settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
     settings_maker.make_setting('archive_cache.objectstore.key', '')
     settings_maker.make_setting('archive_cache.objectstore.secret', '')
-    settings_maker.make_setting('archive_cache.objectstore.bucket_root', 'rhodecode-archive-cache')
+    settings_maker.make_setting('archive_cache.objectstore.bucket', 'rhodecode-archive-cache', default_when_empty=True,)
     settings_maker.make_setting('archive_cache.objectstore.bucket_shards', 8, parser='int')
 
     settings_maker.make_setting('archive_cache.objectstore.cache_size_gb', 10, parser='float')
diff --git a/rhodecode/lib/archive_cache/backends/base.py b/rhodecode/lib/archive_cache/backends/base.py
index c47c6907..a0ed2959 100644
--- a/rhodecode/lib/archive_cache/backends/base.py
+++ b/rhodecode/lib/archive_cache/backends/base.py
@@ -166,6 +166,7 @@ class BaseCache:
     retry_attempts = 0
     retry_backoff = 1
     _shards = tuple()
+    shard_cls = BaseShard
 
     def __contains__(self, key):
         """Return `True` if `key` matching item is found in cache.
@@ -199,8 +200,10 @@ class BaseCache:
             del self._config[key]
         return val
 
-    def _get_shard(self, key):
-        raise NotImplementedError
+    def _get_shard(self, key) -> shard_cls:
+        index = self._hash(key) % self._shard_count
+        shard = self._shards[index]
+        return shard
 
     def _get_size(self, shard, archive_path):
         raise NotImplementedError
diff --git a/rhodecode/lib/archive_cache/backends/fanout_cache.py b/rhodecode/lib/archive_cache/backends/fanout_cache.py
index 3c666139..acd531b4 100644
--- a/rhodecode/lib/archive_cache/backends/fanout_cache.py
+++ b/rhodecode/lib/archive_cache/backends/fanout_cache.py
@@ -32,16 +32,18 @@ log = logging.getLogger(__name__)
 
 class FileSystemShard(BaseShard):
 
-    def __init__(self, index, directory, **settings):
+    def __init__(self, index, directory, directory_folder, fs, **settings):
         self._index = index
         self._directory = directory
+        self._directory_folder = directory_folder
         self.storage_type = 'directory'
-        self.fs = fsspec.filesystem('file')
+
+        self.fs = fs
 
     @property
     def directory(self):
-        """Cache directory."""
-        return self._directory
+        """Cache directory final path."""
+        return os.path.join(self._directory, self._directory_folder)
 
     def _get_keyfile(self, archive_key) -> tuple[str, str]:
         key_file = f'{archive_key}.{self.key_suffix}'
@@ -116,6 +118,7 @@ class FileSystemShard(BaseShard):
 
 class FileSystemFanoutCache(BaseCache):
     shard_name = 'shard_%03d'
+    shard_cls = FileSystemShard
 
     def __init__(self, locking_url, **settings):
         """
@@ -132,13 +135,11 @@ class FileSystemFanoutCache(BaseCache):
         directory = os.path.expanduser(directory)
         directory = os.path.expandvars(directory)
         self._directory = directory
-        self._storage_path = directory
+        self._storage_path = directory  # common path for all from BaseCache
 
-        # check if it's ok to write, and re-create the archive cache
-        if not os.path.isdir(self._directory):
-            os.makedirs(self._directory, exist_ok=True)
-
-        self._count = int(self.get_conf('archive_cache.filesystem.cache_shards', pop=True))
+        self._shard_count = int(self.get_conf('archive_cache.filesystem.cache_shards', pop=True))
+        if self._shard_count < 1:
+            raise ValueError('cache_shards must be 1 or more')
 
         self._eviction_policy = self.get_conf('archive_cache.filesystem.eviction_policy', pop=True)
         self._cache_size_limit = self.gb_to_bytes(int(self.get_conf('archive_cache.filesystem.cache_size_gb')))
@@ -147,21 +148,27 @@ class FileSystemFanoutCache(BaseCache):
         self.retry_attempts = int(self.get_conf('archive_cache.filesystem.retry_attempts', pop=True))
         self.retry_backoff = int(self.get_conf('archive_cache.filesystem.retry_backoff', pop=True))
 
-        log.debug('Initializing archival cache instance under %s', self._directory)
+        log.debug('Initializing %s archival cache instance under %s', self)
+        fs = fsspec.filesystem('file')
+        # check if it's ok to write, and re-create the archive cache main dir
+        # A directory is the virtual equivalent of a physical file cabinet.
+        # In other words, it's a container for organizing digital data.
+        # Unlike a folder, which can only store files, a directory can store files,
+        # subdirectories, and other directories.
+        if not fs.exists(self._directory):
+            fs.makedirs(self._directory, exist_ok=True)
+
         self._shards = tuple(
-            FileSystemShard(
+            self.shard_cls(
                 index=num,
-                directory=os.path.join(directory, self.shard_name % num),
+                directory=directory,
+                directory_folder=self.shard_name % num,
+                fs=fs,
                 **settings,
             )
-            for num in range(self._count)
+            for num in range(self._shard_count)
         )
         self._hash = self._shards[0].hash
 
-    def _get_shard(self, key) -> FileSystemShard:
-        index = self._hash(key) % self._count
-        shard = self._shards[index]
-        return shard
-
     def _get_size(self, shard, archive_path):
         return os.stat(archive_path).st_size
diff --git a/rhodecode/lib/archive_cache/backends/objectstore_cache.py b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
index 7ae972f1..66ee11ee 100644
--- a/rhodecode/lib/archive_cache/backends/objectstore_cache.py
+++ b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
@@ -32,24 +32,18 @@ log = logging.getLogger(__name__)
 
 class S3Shard(BaseShard):
 
-    def __init__(self, index, bucket, **settings):
+    def __init__(self, index, bucket, bucket_folder, fs, **settings):
         self._index = index
-        self._bucket = bucket
+        self._bucket_folder = bucket_folder
         self.storage_type = 'bucket'
+        self._bucket_main = bucket
 
-        endpoint_url = settings.pop('archive_cache.objectstore.url')
-        key = settings.pop('archive_cache.objectstore.key')
-        secret = settings.pop('archive_cache.objectstore.secret')
-
-        # TODO: Add it all over the place...
-        self._bucket_root = settings.pop('archive_cache.objectstore.bucket_root')
-
-        self.fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret)
+        self.fs = fs
 
     @property
     def bucket(self):
-        """Cache bucket."""
-        return os.path.join(self._bucket_root, self._bucket)
+        """Cache bucket final path."""
+        return os.path.join(self._bucket_main, self._bucket_folder)
 
     def _get_keyfile(self, archive_key) -> tuple[str, str]:
         key_file = f'{archive_key}-{self.key_suffix}'
@@ -59,11 +53,8 @@ class S3Shard(BaseShard):
         return self.fs.open(path, 'wb')
 
     def _write_file(self, full_path, iterator, mode):
-        if self._bucket_root:
-            if not self.fs.exists(self._bucket_root):
-                self.fs.mkdir(self._bucket_root)
 
-        # ensure bucket exists
+        # ensure folder in bucket exists
         destination = self.bucket
         if not self.fs.exists(destination):
             self.fs.mkdir(destination, s3_additional_kwargs={})
@@ -113,7 +104,8 @@ class S3Shard(BaseShard):
 
 
 class ObjectStoreCache(BaseCache):
-    shard_name = 'shard-bucket-%03d'
+    shard_name = 'shard-%03d'
+    shard_cls = S3Shard
 
     def __init__(self, locking_url, **settings):
         """
@@ -127,9 +119,15 @@ class ObjectStoreCache(BaseCache):
         self._config = settings
 
         objectstore_url = self.get_conf('archive_cache.objectstore.url')
-        self._storage_path = objectstore_url
+        self._storage_path = objectstore_url  # common path for all from BaseCache
 
-        self._count = int(self.get_conf('archive_cache.objectstore.bucket_shards', pop=True))
+        self._shard_count = int(self.get_conf('archive_cache.objectstore.bucket_shards', pop=True))
+        if self._shard_count < 1:
+            raise ValueError('cache_shards must be 1 or more')
+
+        self._bucket = settings.pop('archive_cache.objectstore.bucket')
+        if not self._bucket:
+            raise ValueError('archive_cache.objectstore.bucket needs to have a value')
 
         self._eviction_policy = self.get_conf('archive_cache.objectstore.eviction_policy', pop=True)
         self._cache_size_limit = self.gb_to_bytes(int(self.get_conf('archive_cache.objectstore.cache_size_gb')))
@@ -138,21 +136,29 @@ class ObjectStoreCache(BaseCache):
         self.retry_attempts = int(self.get_conf('archive_cache.objectstore.retry_attempts', pop=True))
         self.retry_backoff = int(self.get_conf('archive_cache.objectstore.retry_backoff', pop=True))
 
-        log.debug('Initializing archival cache instance under %s', objectstore_url)
+        endpoint_url = settings.pop('archive_cache.objectstore.url')
+        key = settings.pop('archive_cache.objectstore.key')
+        secret = settings.pop('archive_cache.objectstore.secret')
+
+        log.debug('Initializing %s archival cache instance under %s', self)
+
+        fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret)
+
+        # init main bucket
+        if not fs.exists(self._bucket):
+            fs.mkdir(self._bucket)
+
         self._shards = tuple(
-            S3Shard(
+            self.shard_cls(
                 index=num,
-                bucket=self.shard_name % num,
+                bucket=self._bucket,
+                bucket_folder=self.shard_name % num,
+                fs=fs,
                 **settings,
             )
-            for num in range(self._count)
+            for num in range(self._shard_count)
         )
         self._hash = self._shards[0].hash
 
-    def _get_shard(self, key) -> S3Shard:
-        index = self._hash(key) % self._count
-        shard = self._shards[index]
-        return shard
-
     def _get_size(self, shard, archive_path):
         return shard.fs.info(archive_path)['size']
diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini
index 2795cc99..1e64be75 100644
--- a/rhodecode/tests/rhodecode.ini
+++ b/rhodecode/tests/rhodecode.ini
@@ -279,10 +279,9 @@ archive_cache.objectstore.secret = secret
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
 
-; a top-level bucket to put all other sharded buckets in
-; in case it's empty all buckets will be created in top-level (not recommended)
-; objects will be stored in rhodecode-archive-cache/shard-bucket-N based on the bucket_shards number
-archive_cache.objectstore.bucket_root = rhodecode-archive-cache
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-archive-cache/shard-N based on the bucket_shards number
+archive_cache.objectstore.bucket = rhodecode-archive-cache
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.objectstore.retry = false

From b965526856f9a7d42cc40811f163b76f1f4d8128 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 19 Jun 2024 13:14:05 +0200
Subject: [PATCH 128/150] fix: fixed logging

---
 rhodecode/lib/archive_cache/backends/fanout_cache.py      | 2 +-
 rhodecode/lib/archive_cache/backends/objectstore_cache.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/rhodecode/lib/archive_cache/backends/fanout_cache.py b/rhodecode/lib/archive_cache/backends/fanout_cache.py
index acd531b4..ff023064 100644
--- a/rhodecode/lib/archive_cache/backends/fanout_cache.py
+++ b/rhodecode/lib/archive_cache/backends/fanout_cache.py
@@ -148,7 +148,7 @@ class FileSystemFanoutCache(BaseCache):
         self.retry_attempts = int(self.get_conf('archive_cache.filesystem.retry_attempts', pop=True))
         self.retry_backoff = int(self.get_conf('archive_cache.filesystem.retry_backoff', pop=True))
 
-        log.debug('Initializing %s archival cache instance under %s', self)
+        log.debug('Initializing %s archival cache instance', self)
         fs = fsspec.filesystem('file')
         # check if it's ok to write, and re-create the archive cache main dir
         # A directory is the virtual equivalent of a physical file cabinet.
diff --git a/rhodecode/lib/archive_cache/backends/objectstore_cache.py b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
index 66ee11ee..7543cbe2 100644
--- a/rhodecode/lib/archive_cache/backends/objectstore_cache.py
+++ b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
@@ -140,7 +140,7 @@ class ObjectStoreCache(BaseCache):
         key = settings.pop('archive_cache.objectstore.key')
         secret = settings.pop('archive_cache.objectstore.secret')
 
-        log.debug('Initializing %s archival cache instance under %s', self)
+        log.debug('Initializing %s archival cache instance', self)
 
         fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret)
 

From 7d866e5cad0629bd208720e1e10350e8b9814130 Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Mon, 17 Jun 2024 14:59:54 +0300
Subject: [PATCH 129/150] feat(delete-branch-after-merge-git): added ability to
 delete a branch automatically after merging PR (if it was configured). Fixes:
 RCCE-76

---
 rhodecode/lib/vcs/backends/git/repository.py |  2 +
 rhodecode/templates/base/vcs_settings.mako   | 49 ++++++++++----------
 2 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/rhodecode/lib/vcs/backends/git/repository.py b/rhodecode/lib/vcs/backends/git/repository.py
index 7aa28d40..73a1a060 100644
--- a/rhodecode/lib/vcs/backends/git/repository.py
+++ b/rhodecode/lib/vcs/backends/git/repository.py
@@ -1040,6 +1040,8 @@ class GitRepository(BaseRepository):
                     pr_branch, self.path, target_ref.name, enable_hooks=True,
                     rc_scm_data=self.config.get('rhodecode', 'RC_SCM_DATA'))
                 merge_succeeded = True
+                if close_branch and source_ref.name != target_ref.name and not dry_run and source_ref.type == 'branch':
+                    self.delete_branch(source_ref.name)
             except RepositoryError:
                 log.exception(
                     'Failure when doing local push from the shadow '
diff --git a/rhodecode/templates/base/vcs_settings.mako b/rhodecode/templates/base/vcs_settings.mako
index 934e5bbb..14b76db0 100644
--- a/rhodecode/templates/base/vcs_settings.mako
+++ b/rhodecode/templates/base/vcs_settings.mako
@@ -300,31 +300,30 @@
         </div>
     % endif
 
-## DISABLED FOR GIT FOR NOW as the rebase/close is not supported yet
-##     % if display_globals or repo_type in ['git']:
-##         <div class="panel panel-default">
-##             <div class="panel-heading" id="vcs-pull-requests-options">
-##                 <h3 class="panel-title">${_('Git Pull Request Settings')}<a class="permalink" href="#vcs-git-pull-requests-options"> ¶</a></h3>
-##             </div>
-##             <div class="panel-body">
-##                 <div class="checkbox">
-##                     ${h.checkbox('rhodecode_git_use_rebase_for_merging' + suffix, 'True', **kwargs)}
-##                     <label for="rhodecode_git_use_rebase_for_merging${suffix}">${_('Use rebase as merge strategy')}</label>
-##                 </div>
-##                 <div class="label">
-##                     <span class="help-block">${_('Use rebase instead of creating a merge commit when merging via web interface.')}</span>
-##                 </div>
-##
-##                 <div class="checkbox">
-##                     ${h.checkbox('rhodecode_git_close_branch_before_merging' + suffix, 'True', **kwargs)}
-##                     <label for="rhodecode_git_close_branch_before_merging{suffix}">${_('Delete branch after merging it')}</label>
-##                 </div>
-##                 <div class="label">
-##                     <span class="help-block">${_('Delete branch after merging it into destination branch. No effect when rebase strategy is use.')}</span>
-##                 </div>
-##             </div>
-##         </div>
-##     % endif
+% if display_globals or repo_type in ['git']:
+    <div class="panel panel-default">
+        <div class="panel-heading" id="vcs-pull-requests-options">
+            <h3 class="panel-title">${_('Git Pull Request Settings')}<a class="permalink" href="#vcs-git-pull-requests-options"> ¶</a></h3>
+        </div>
+        <div class="panel-body">
+##                <div class="checkbox">
+##                    ${h.checkbox('rhodecode_git_use_rebase_for_merging' + suffix, 'True', **kwargs)}
+##                    <label for="rhodecode_git_use_rebase_for_merging${suffix}">${_('Use rebase as merge strategy')}</label>
+##                </div>
+##                <div class="label">
+##                  <span class="help-block">${_('Use rebase instead of creating a merge commit when merging via web interface.')}</span>
+##               </div>
+
+            <div class="checkbox">
+                ${h.checkbox('rhodecode_git_close_branch_before_merging' + suffix, 'True', **kwargs)}
+                <label for="rhodecode_git_close_branch_before_merging{suffix}">${_('Delete branch after merging it')}</label>
+            </div>
+            <div class="label">
+                <span class="help-block">${_('Delete branch after merging it into destination branch.')}</span>
+            </div>
+        </div>
+    </div>
+% endif
 
 <script type="text/javascript">
 

From f9dff2a21f9ae92a50d2579587da2c4425041863 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Thu, 20 Jun 2024 10:57:56 +0200
Subject: [PATCH 130/150] feat(archive-cache): added presigned_url support for
 objectstore

---
 rhodecode/lib/archive_cache/backends/base.py  | 29 +++++++++++++++----
 .../archive_cache/backends/fanout_cache.py    | 21 ++++++++------
 .../backends/objectstore_cache.py             | 26 ++++++++++-------
 3 files changed, 51 insertions(+), 25 deletions(-)

diff --git a/rhodecode/lib/archive_cache/backends/base.py b/rhodecode/lib/archive_cache/backends/base.py
index a0ed2959..76c4688b 100644
--- a/rhodecode/lib/archive_cache/backends/base.py
+++ b/rhodecode/lib/archive_cache/backends/base.py
@@ -54,6 +54,9 @@ class BaseShard:
     def random_filename(self):
         raise NotImplementedError
 
+    def store(self, *args, **kwargs):
+        raise NotImplementedError
+
     def _store(self, key, value_reader, metadata, mode):
         (filename,  # hash-name
          full_path  # full-path/hash-name
@@ -91,7 +94,11 @@ class BaseShard:
 
         return key, filename, size, _metadata
 
-    def _fetch(self,  key, retry, retry_attempts, retry_backoff):
+    def fetch(self, *args, **kwargs):
+        raise NotImplementedError
+
+    def _fetch(self,  key, retry, retry_attempts, retry_backoff,
+               presigned_url_expires: int = 0) -> tuple[ShardFileReader, dict]:
         if retry is NOT_GIVEN:
             retry = False
         if retry_attempts is NOT_GIVEN:
@@ -113,6 +120,8 @@ class BaseShard:
             metadata = json.loads(f.read())
 
         archive_path = metadata['archive_full_path']
+        if presigned_url_expires and presigned_url_expires > 0:
+            metadata['url'] = self.fs.url(archive_path, expires=presigned_url_expires)
 
         try:
             return ShardFileReader(self.fs.open(archive_path, 'rb')), metadata
@@ -125,6 +134,9 @@ class BaseShard:
             with self.fs.open(key_file_path, 'wb') as f:
                 f.write(json.dumps(metadata))
 
+    def remove(self, *args, **kwargs):
+        raise NotImplementedError
+
     def _remove(self, key):
         if key not in self:
             log.exception(f'requested key={key} not found in {self}')
@@ -161,12 +173,14 @@ class BaseShard:
 class BaseCache:
     _locking_url: str = ''
     _storage_path: str = ''
-    _config = {}
+    _config: dict = {}
     retry = False
-    retry_attempts = 0
-    retry_backoff = 1
+    retry_attempts: int = 0
+    retry_backoff: int | float = 1
     _shards = tuple()
     shard_cls = BaseShard
+    # define the presigned url expiration, 0 == disabled
+    presigned_url_expires: int = 0
 
     def __contains__(self, key):
         """Return `True` if `key` matching item is found in cache.
@@ -221,9 +235,13 @@ class BaseCache:
         if retry_attempts is NOT_GIVEN:
             retry_attempts = self.retry_attempts
         retry_backoff = self.retry_backoff
+        presigned_url_expires = self.presigned_url_expires
 
         shard = self._get_shard(key)
-        return shard.fetch(key, retry=retry, retry_attempts=retry_attempts, retry_backoff=retry_backoff)
+        return shard.fetch(key, retry=retry,
+                           retry_attempts=retry_attempts,
+                           retry_backoff=retry_backoff,
+                           presigned_url_expires=presigned_url_expires)
 
     def remove(self, key):
         shard = self._get_shard(key)
@@ -352,4 +370,3 @@ class BaseCache:
                 total_size += metadata['size']
 
         return total_files, total_size, meta
-
diff --git a/rhodecode/lib/archive_cache/backends/fanout_cache.py b/rhodecode/lib/archive_cache/backends/fanout_cache.py
index ff023064..1a786f64 100644
--- a/rhodecode/lib/archive_cache/backends/fanout_cache.py
+++ b/rhodecode/lib/archive_cache/backends/fanout_cache.py
@@ -20,6 +20,7 @@ import codecs
 import hashlib
 import logging
 import os
+import typing
 
 import fsspec
 
@@ -33,20 +34,20 @@ log = logging.getLogger(__name__)
 class FileSystemShard(BaseShard):
 
     def __init__(self, index, directory, directory_folder, fs, **settings):
-        self._index = index
-        self._directory = directory
-        self._directory_folder = directory_folder
-        self.storage_type = 'directory'
+        self._index: int = index
+        self._directory: str = directory
+        self._directory_folder: str = directory_folder
+        self.storage_type: str = 'directory'
 
         self.fs = fs
 
     @property
-    def directory(self):
+    def directory(self) -> str:
         """Cache directory final path."""
         return os.path.join(self._directory, self._directory_folder)
 
     def _get_keyfile(self, archive_key) -> tuple[str, str]:
-        key_file = f'{archive_key}.{self.key_suffix}'
+        key_file: str = f'{archive_key}.{self.key_suffix}'
         return key_file, os.path.join(self.directory, key_file)
 
     def _get_writer(self, path, mode):
@@ -62,6 +63,7 @@ class FileSystemShard(BaseShard):
                 continue
 
     def _write_file(self, full_path, iterator, mode):
+
         # ensure dir exists
         destination, _ = os.path.split(full_path)
         if not self.fs.exists(destination):
@@ -89,7 +91,8 @@ class FileSystemShard(BaseShard):
     def store(self, key, value_reader, metadata: dict | None = None):
         return self._store(key, value_reader, metadata, mode='xb')
 
-    def fetch(self, key, retry=NOT_GIVEN, retry_attempts=NOT_GIVEN, retry_backoff=1) -> tuple[ShardFileReader, dict]:
+    def fetch(self, key, retry=NOT_GIVEN,
+              retry_attempts=NOT_GIVEN, retry_backoff=1, **kwargs) -> tuple[ShardFileReader, dict]:
         return self._fetch(key, retry, retry_attempts, retry_backoff)
 
     def remove(self, key):
@@ -117,7 +120,7 @@ class FileSystemShard(BaseShard):
 
 
 class FileSystemFanoutCache(BaseCache):
-    shard_name = 'shard_%03d'
+    shard_name: str = 'shard_{:03d}'
     shard_cls = FileSystemShard
 
     def __init__(self, locking_url, **settings):
@@ -162,7 +165,7 @@ class FileSystemFanoutCache(BaseCache):
             self.shard_cls(
                 index=num,
                 directory=directory,
-                directory_folder=self.shard_name % num,
+                directory_folder=self.shard_name.format(num),
                 fs=fs,
                 **settings,
             )
diff --git a/rhodecode/lib/archive_cache/backends/objectstore_cache.py b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
index 7543cbe2..aee44e63 100644
--- a/rhodecode/lib/archive_cache/backends/objectstore_cache.py
+++ b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
@@ -20,6 +20,7 @@ import codecs
 import hashlib
 import logging
 import os
+import typing
 
 import fsspec
 
@@ -33,20 +34,20 @@ log = logging.getLogger(__name__)
 class S3Shard(BaseShard):
 
     def __init__(self, index, bucket, bucket_folder, fs, **settings):
-        self._index = index
-        self._bucket_folder = bucket_folder
-        self.storage_type = 'bucket'
-        self._bucket_main = bucket
+        self._index: int = index
+        self._bucket_folder: str = bucket_folder
+        self.storage_type: str = 'bucket'
+        self._bucket_main: str = bucket
 
         self.fs = fs
 
     @property
-    def bucket(self):
+    def bucket(self) -> str:
         """Cache bucket final path."""
         return os.path.join(self._bucket_main, self._bucket_folder)
 
     def _get_keyfile(self, archive_key) -> tuple[str, str]:
-        key_file = f'{archive_key}-{self.key_suffix}'
+        key_file: str = f'{archive_key}-{self.key_suffix}'
         return key_file, os.path.join(self.bucket, key_file)
 
     def _get_writer(self, path, mode):
@@ -76,8 +77,10 @@ class S3Shard(BaseShard):
     def store(self, key, value_reader, metadata: dict | None = None):
         return self._store(key, value_reader, metadata, mode='wb')
 
-    def fetch(self, key, retry=NOT_GIVEN, retry_attempts=NOT_GIVEN, retry_backoff=1) -> tuple[ShardFileReader, dict]:
-        return self._fetch(key, retry, retry_attempts, retry_backoff)
+    def fetch(self, key, retry=NOT_GIVEN,
+              retry_attempts=NOT_GIVEN, retry_backoff=1,
+              presigned_url_expires: int = 0) -> tuple[ShardFileReader, dict]:
+        return self._fetch(key, retry, retry_attempts, retry_backoff, presigned_url_expires=presigned_url_expires)
 
     def remove(self, key):
         return self._remove(key)
@@ -104,7 +107,7 @@ class S3Shard(BaseShard):
 
 
 class ObjectStoreCache(BaseCache):
-    shard_name = 'shard-%03d'
+    shard_name: str = 'shard-{:03d}'
     shard_cls = S3Shard
 
     def __init__(self, locking_url, **settings):
@@ -152,7 +155,7 @@ class ObjectStoreCache(BaseCache):
             self.shard_cls(
                 index=num,
                 bucket=self._bucket,
-                bucket_folder=self.shard_name % num,
+                bucket_folder=self.shard_name.format(num),
                 fs=fs,
                 **settings,
             )
@@ -162,3 +165,6 @@ class ObjectStoreCache(BaseCache):
 
     def _get_size(self, shard, archive_path):
         return shard.fs.info(archive_path)['size']
+
+    def set_presigned_url_expiry(self, val: int) -> None:
+        self.presigned_url_expires = val

From 8ac5fafd3db04d9197c9d7c7c26f505a328dd855 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Sun, 23 Jun 2024 10:01:08 +0200
Subject: [PATCH 131/150] fix(users-creation): fixed reported error for emails
 as dict not label

---
 rhodecode/apps/admin/views/users.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rhodecode/apps/admin/views/users.py b/rhodecode/apps/admin/views/users.py
index f116c77b..a39ba0c3 100644
--- a/rhodecode/apps/admin/views/users.py
+++ b/rhodecode/apps/admin/views/users.py
@@ -214,7 +214,7 @@ class AdminUsersView(BaseAppView, DataGridAppView):
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
-                errors=errors.unpack_errors() or {},
+                errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False

From e013e4ca46738f84e559de56e17a4924b5f95846 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Sun, 23 Jun 2024 21:45:34 +0200
Subject: [PATCH 132/150] fix(scheduler): fixed scheduler task UID conflicts
 due to orjson not understanding JSONRAW and also bad sorting usage

fixes RCCE-92
---
 rhodecode/model/db.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index 9474b93a..7956e0bf 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -5485,8 +5485,7 @@ class ScheduleEntry(Base, BaseModel):
     @schedule_type.setter
     def schedule_type(self, val):
         if val not in self.schedule_types:
-            raise ValueError('Value must be on of `{}` and got `{}`'.format(
-                val, self.schedule_type))
+            raise ValueError(f'Value must be on of `{val}` and got `{self.schedule_type}`')
 
         self._schedule_type = val
 
@@ -5494,21 +5493,25 @@ class ScheduleEntry(Base, BaseModel):
     def get_uid(cls, obj):
         args = obj.task_args
         kwargs = obj.task_kwargs
+
         if isinstance(args, JsonRaw):
             try:
-                args = json.loads(args)
+                args = json.loads(str(args))
             except ValueError:
+                log.exception('json.loads of args failed...')
                 args = tuple()
 
         if isinstance(kwargs, JsonRaw):
             try:
-                kwargs = json.loads(kwargs)
+                kwargs = json.loads(str(kwargs))
             except ValueError:
+                log.exception('json.loads of kwargs failed...')
                 kwargs = dict()
 
         dot_notation = obj.task_dot_notation
-        val = '.'.join(map(safe_str, [
-            sorted(dot_notation), args, sorted(kwargs.items())]))
+        val = '.'.join(map(safe_str, [dot_notation, args, sorted(kwargs.items())]))
+        log.debug('calculating task uid using id:`%s`', val)
+
         return sha1(safe_bytes(val))
 
     @classmethod
@@ -5519,6 +5522,10 @@ class ScheduleEntry(Base, BaseModel):
     def get_by_schedule_id(cls, schedule_id):
         return cls.query().filter(cls.schedule_entry_id == schedule_id).scalar()
 
+    @classmethod
+    def get_by_task_uid(cls, task_uid):
+        return cls.query().filter(cls.task_uid == task_uid).scalar()
+
     @property
     def task(self):
         return self.task_dot_notation

From 8f1e706ad8fc74641c126171c441182d4708caf9 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Sun, 23 Jun 2024 21:48:10 +0200
Subject: [PATCH 133/150] chore(cleanups): cleanup unicode usage...

---
 rhodecode/api/tests/test_create_repo.py               | 2 +-
 rhodecode/api/tests/test_create_repo_group.py         | 8 ++++----
 rhodecode/api/tests/test_get_gist.py                  | 4 ++--
 rhodecode/apps/admin/tests/test_admin_repos.py        | 4 ++--
 rhodecode/apps/login/tests/test_login.py              | 4 ++--
 rhodecode/tests/auth_external_test.py                 | 2 +-
 rhodecode/tests/lib/auth_modules/test_auth_modules.py | 2 +-
 rhodecode/tests/vcs/test_git.py                       | 4 ++--
 rhodecode/tests/vcs/test_svn.py                       | 4 ++--
 9 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/rhodecode/api/tests/test_create_repo.py b/rhodecode/api/tests/test_create_repo.py
index a5649b8b..be373a42 100644
--- a/rhodecode/api/tests/test_create_repo.py
+++ b/rhodecode/api/tests/test_create_repo.py
@@ -41,7 +41,7 @@ class TestCreateRepo(object):
     @pytest.mark.parametrize('given, expected_name, expected_exc', [
         ('api repo-1', 'api-repo-1', False),
         ('api-repo 1-ąć', 'api-repo-1-ąć', False),
-        (u'unicode-ąć', u'unicode-ąć', False),
+        ('unicode-ąć', u'unicode-ąć', False),
         ('some repo v1.2', 'some-repo-v1.2', False),
         ('v2.0', 'v2.0', False),
     ])
diff --git a/rhodecode/api/tests/test_create_repo_group.py b/rhodecode/api/tests/test_create_repo_group.py
index 6c98d751..72e7ee56 100644
--- a/rhodecode/api/tests/test_create_repo_group.py
+++ b/rhodecode/api/tests/test_create_repo_group.py
@@ -211,8 +211,8 @@ class TestCreateRepoGroup(object):
 
         expected = {
             'repo_group':
-                u'You do not have the permission to store '
-                u'repository groups in the root location.'}
+                'You do not have the permission to store '
+                'repository groups in the root location.'}
         assert_error(id_, expected, given=response.body)
 
     def test_api_create_repo_group_regular_user_no_parent_group_perms(self):
@@ -232,8 +232,8 @@ class TestCreateRepoGroup(object):
 
         expected = {
             'repo_group':
-                u"You do not have the permissions to store "
-                u"repository groups inside repository group `{}`".format(repo_group_name)}
+                "You do not have the permissions to store "
+                "repository groups inside repository group `{}`".format(repo_group_name)}
         try:
             assert_error(id_, expected, given=response.body)
         finally:
diff --git a/rhodecode/api/tests/test_get_gist.py b/rhodecode/api/tests/test_get_gist.py
index bd007626..436ca06d 100644
--- a/rhodecode/api/tests/test_get_gist.py
+++ b/rhodecode/api/tests/test_get_gist.py
@@ -76,8 +76,8 @@ class TestApiGetGist(object):
             'url': 'http://%s/_admin/gists/%s' % (http_host_only_stub, gist_id,),
             'acl_level': Gist.ACL_LEVEL_PUBLIC,
             'content': {
-                u'filename1.txt': u'hello world',
-                u'filename1ą.txt': u'hello worldę'
+                'filename1.txt': 'hello world',
+                'filename1ą.txt': 'hello worldę'
             },
         }
 
diff --git a/rhodecode/apps/admin/tests/test_admin_repos.py b/rhodecode/apps/admin/tests/test_admin_repos.py
index a547b080..014a79b7 100644
--- a/rhodecode/apps/admin/tests/test_admin_repos.py
+++ b/rhodecode/apps/admin/tests/test_admin_repos.py
@@ -446,8 +446,8 @@ class TestAdminRepos(object):
                 csrf_token=csrf_token))
 
         response.mustcontain(
-            u"You do not have the permission to store repositories in "
-            u"the root location.")
+            "You do not have the permission to store repositories in "
+            "the root location.")
 
     @mock.patch.object(RepoModel, '_create_filesystem_repo', error_function)
     def test_create_repo_when_filesystem_op_fails(
diff --git a/rhodecode/apps/login/tests/test_login.py b/rhodecode/apps/login/tests/test_login.py
index 88ac3877..424cfabf 100644
--- a/rhodecode/apps/login/tests/test_login.py
+++ b/rhodecode/apps/login/tests/test_login.py
@@ -266,7 +266,7 @@ class TestLoginController(object):
         )
 
         assertr = response.assert_response()
-        msg = u'This e-mail address is already taken'
+        msg = 'This e-mail address is already taken'
         assertr.element_contains('#email+.error-message', msg)
 
     def test_register_err_same_email_case_sensitive(self):
@@ -282,7 +282,7 @@ class TestLoginController(object):
             }
         )
         assertr = response.assert_response()
-        msg = u'This e-mail address is already taken'
+        msg = 'This e-mail address is already taken'
         assertr.element_contains('#email+.error-message', msg)
 
     def test_register_err_wrong_data(self):
diff --git a/rhodecode/tests/auth_external_test.py b/rhodecode/tests/auth_external_test.py
index 62a03936..07010731 100644
--- a/rhodecode/tests/auth_external_test.py
+++ b/rhodecode/tests/auth_external_test.py
@@ -44,7 +44,7 @@ class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
 
     @hybrid_property
     def name(self):
-        return u"external_test"
+        return "external_test"
 
     def settings(self):
         settings = [
diff --git a/rhodecode/tests/lib/auth_modules/test_auth_modules.py b/rhodecode/tests/lib/auth_modules/test_auth_modules.py
index b35e9e28..b213c40a 100644
--- a/rhodecode/tests/lib/auth_modules/test_auth_modules.py
+++ b/rhodecode/tests/lib/auth_modules/test_auth_modules.py
@@ -29,7 +29,7 @@ from rhodecode.model import db
 class RcTestAuthPlugin(RhodeCodeAuthPluginBase):
 
     def name(self):
-        return u'stub_auth'
+        return 'stub_auth'
 
 
 def test_authenticate_returns_from_auth(stub_auth_data):
diff --git a/rhodecode/tests/vcs/test_git.py b/rhodecode/tests/vcs/test_git.py
index ab3d1417..3bdd6c41 100644
--- a/rhodecode/tests/vcs/test_git.py
+++ b/rhodecode/tests/vcs/test_git.py
@@ -969,7 +969,7 @@ class TestGitCommit(object):
             branches = self.repo.branches
 
         assert 'unicode' in branches
-        assert u'uniçö∂e' in branches
+        assert 'uniçö∂e' in branches
 
     def test_unicode_tag_refs(self):
         unicode_tags = {
@@ -983,7 +983,7 @@ class TestGitCommit(object):
             tags = self.repo.tags
 
         assert 'unicode' in tags
-        assert u'uniçö∂e' in tags
+        assert 'uniçö∂e' in tags
 
     def test_commit_message_is_unicode(self):
         for commit in self.repo:
diff --git a/rhodecode/tests/vcs/test_svn.py b/rhodecode/tests/vcs/test_svn.py
index fee1905d..f4510b35 100644
--- a/rhodecode/tests/vcs/test_svn.py
+++ b/rhodecode/tests/vcs/test_svn.py
@@ -145,14 +145,14 @@ def test_unicode_refs(vcsbackend, filename, content, branch, mime_type):
     with mock.patch(("rhodecode.lib.vcs.backends.svn.repository"
                     ".SubversionRepository._patterns_from_section"),
                     return_value=['branches/*']):
-        assert u'branches/{0}'.format(branch) in repo.branches
+        assert f'branches/{branch}' in repo.branches
 
 
 def test_compatible_version(monkeypatch, vcsbackend):
     monkeypatch.setattr(settings, 'SVN_COMPATIBLE_VERSION', 'pre-1.8-compatible')
     path = vcsbackend.new_repo_path()
     SubversionRepository(path, create=True)
-    with open('{}/db/format'.format(path)) as f:
+    with open(f'{path}/db/format') as f:
         first_line = f.readline().strip()
         assert first_line == '4'
 

From 4bd0fecc1878d11d792ee09de4a2db2329948c17 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Mon, 24 Jun 2024 09:08:12 +0200
Subject: [PATCH 134/150] deps: removed unused pyramid-apispec

---
 requirements.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 677d9550..a39720eb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -47,8 +47,6 @@ channelstream==0.7.1
     webob==1.8.7
     zope.deprecation==5.0.0
     zope.interface==6.3.0
-  pyramid-apispec==0.3.3
-    apispec==1.3.3
   pyramid-jinja2==2.10
     jinja2==3.1.2
       markupsafe==2.1.2

From 9019f7e9c877e39bc5ddc53f64b3f2b22cffbaf3 Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Fri, 21 Jun 2024 09:25:02 +0300
Subject: [PATCH 135/150] fix: fixed empty server url in PR link. Fixes:
 RCCE-77

---
 rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
index fff01ca3..94034cfc 100644
--- a/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
+++ b/rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py
@@ -32,6 +32,8 @@ import logging
 import click
 
 from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
+from rhodecode.lib.request import Request
+from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.lib.statsd_client import StatsdClient
 from rhodecode.lib.config_utils import get_app_config_lightweight
 
@@ -74,7 +76,11 @@ def main(ini_path, mode, user, user_id, key_id, shell, debug):
 
     try:
         connection_info = os.environ.get('SSH_CONNECTION', '')
-        env = {'RC_CMD_SSH_WRAPPER': '1'}
+        request = Request.blank('/', base_url=settings['app.base_url'])
+        request.user = AttributeDict({'username': user,
+                                      'user_id': user_id,
+                                      'ip_addr': connection_info.split(' ')[0] if connection_info else None})
+        env = {'RC_CMD_SSH_WRAPPER': '1', 'request': request}
         ssh_wrapper = SshWrapperStandalone(
             command, connection_info, mode,
             user, user_id, key_id, shell, ini_path, settings, env)

From 61c700577bfbbfe3c9c6768e45b677d74aceeb78 Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Wed, 3 Jul 2024 14:17:17 +0300
Subject: [PATCH 136/150] feat(region for s3 storage): added configurable
 region of s3 storage.

---
 configs/development.ini                                   | 3 +++
 configs/production.ini                                    | 3 +++
 rhodecode/config/config_maker.py                          | 1 +
 rhodecode/lib/archive_cache/backends/objectstore_cache.py | 3 ++-
 rhodecode/tests/rhodecode.ini                             | 3 +++
 5 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/configs/development.ini b/configs/development.ini
index 49ca314c..bf3a14d7 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -307,6 +307,9 @@ archive_cache.objectstore.key = key
 ; secret for s3 auth
 archive_cache.objectstore.secret = secret
 
+;region for s3 storage
+archive_cache.objectstore.region = eu-central-1
+
 ; number of sharded buckets to create to distribute archives across
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
diff --git a/configs/production.ini b/configs/production.ini
index 0b4540cf..bf325a15 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -275,6 +275,9 @@ archive_cache.objectstore.key = key
 ; secret for s3 auth
 archive_cache.objectstore.secret = secret
 
+;region for s3 storage
+archive_cache.objectstore.region = eu-central-1
+
 ; number of sharded buckets to create to distribute archives across
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py
index b53de92e..afd6916c 100644
--- a/rhodecode/config/config_maker.py
+++ b/rhodecode/config/config_maker.py
@@ -204,6 +204,7 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
     settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
     settings_maker.make_setting('archive_cache.objectstore.key', '')
     settings_maker.make_setting('archive_cache.objectstore.secret', '')
+    settings_maker.make_setting('archive_cache.objectstore.region', 'eu-central-1')
     settings_maker.make_setting('archive_cache.objectstore.bucket', 'rhodecode-archive-cache', default_when_empty=True,)
     settings_maker.make_setting('archive_cache.objectstore.bucket_shards', 8, parser='int')
 
diff --git a/rhodecode/lib/archive_cache/backends/objectstore_cache.py b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
index aee44e63..bd8bd6e8 100644
--- a/rhodecode/lib/archive_cache/backends/objectstore_cache.py
+++ b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
@@ -142,10 +142,11 @@ class ObjectStoreCache(BaseCache):
         endpoint_url = settings.pop('archive_cache.objectstore.url')
         key = settings.pop('archive_cache.objectstore.key')
         secret = settings.pop('archive_cache.objectstore.secret')
+        region = settings.pop('archive_cache.objectstore.region')
 
         log.debug('Initializing %s archival cache instance', self)
 
-        fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret)
+        fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret, region=region)
 
         # init main bucket
         if not fs.exists(self._bucket):
diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini
index 1e64be75..1b534b63 100644
--- a/rhodecode/tests/rhodecode.ini
+++ b/rhodecode/tests/rhodecode.ini
@@ -275,6 +275,9 @@ archive_cache.objectstore.key = key
 ; secret for s3 auth
 archive_cache.objectstore.secret = secret
 
+;region for s3 storage
+archive_cache.objectstore.region = eu-central-1
+
 ; number of sharded buckets to create to distribute archives across
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8

From 93f783912f027a46cf4defe2623a15a1d0b37ab0 Mon Sep 17 00:00:00 2001
From: Serhii Ilin <serhii@rhodecode.com>
Date: Thu, 11 Jul 2024 09:36:35 +0300
Subject: [PATCH 137/150] fix: fixed s3 region. Fixes: RCCE-98

---
 rhodecode/lib/archive_cache/backends/objectstore_cache.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/rhodecode/lib/archive_cache/backends/objectstore_cache.py b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
index bd8bd6e8..07a028d7 100644
--- a/rhodecode/lib/archive_cache/backends/objectstore_cache.py
+++ b/rhodecode/lib/archive_cache/backends/objectstore_cache.py
@@ -146,7 +146,9 @@ class ObjectStoreCache(BaseCache):
 
         log.debug('Initializing %s archival cache instance', self)
 
-        fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret, region=region)
+        fs = fsspec.filesystem(
+            's3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret, client_kwargs={'region_name': region}
+        )
 
         # init main bucket
         if not fs.exists(self._bucket):

From 34dcb83bfb36a26fec10ecf41ed989cfca69a29d Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 3 Jul 2024 09:50:40 +0200
Subject: [PATCH 138/150] chore: added excplicit db-migrate to update hooks for
 5.X rel

---
 rhodecode/__init__.py                         |  2 +-
 .../dbmigrate/versions/115_version_5_1_0.py   | 50 +++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 rhodecode/lib/dbmigrate/versions/115_version_5_1_0.py

diff --git a/rhodecode/__init__.py b/rhodecode/__init__.py
index 82a7f06a..881b10b9 100644
--- a/rhodecode/__init__.py
+++ b/rhodecode/__init__.py
@@ -82,7 +82,7 @@ PYRAMID_SETTINGS = {}
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 114  # defines current db version for migrations
+__dbversion__ = 115  # defines current db version for migrations
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'
 __url__ = 'https://code.rhodecode.com'
diff --git a/rhodecode/lib/dbmigrate/versions/115_version_5_1_0.py b/rhodecode/lib/dbmigrate/versions/115_version_5_1_0.py
new file mode 100644
index 00000000..d1d05862
--- /dev/null
+++ b/rhodecode/lib/dbmigrate/versions/115_version_5_1_0.py
@@ -0,0 +1,50 @@
+
+
+import logging
+from sqlalchemy import *
+from sqlalchemy.engine import reflection
+
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+
+from rhodecode.lib.dbmigrate.versions import _reset_base
+from rhodecode.model import meta, init_model_encryption
+
+
+log = logging.getLogger(__name__)
+
+
+def _get_indexes_list(migrate_engine, table_name):
+    inspector = reflection.Inspector.from_engine(migrate_engine)
+    return inspector.get_indexes(table_name)
+
+
+def upgrade(migrate_engine):
+    """
+    Upgrade operations go here.
+    Don't create your own engine; bind migrate_engine to your metadata
+    """
+    from rhodecode.model import db as db_5_1_0_0
+
+    # issue fixups
+    fixups(db_5_1_0_0, meta.Session)
+
+
+def downgrade(migrate_engine):
+    pass
+
+
+def fixups(models, _SESSION):
+    for db_repo in _SESSION.query(models.Repository).all():
+
+        config = db_repo._config
+        config.set('extensions', 'largefiles', '')
+
+        try:
+            scm = db_repo.scm_instance(cache=False, config=config)
+            if scm:
+                print(f'installing hook for repo: {db_repo}')
+                scm.install_hooks(force=True)
+        except Exception as e:
+            print(e)
+            print('continue...')

From 26d18ef0fec45c4fbdf638a688323e6362d490b9 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 4 Jun 2024 17:20:16 +0200
Subject: [PATCH 139/150] fix(svn): svn events fixes and change the way how we
 handle the events

---
 configs/development.ini                       |   4 +
 configs/production.ini                        |   4 +
 rhodecode/config/config_maker.py              |   1 +
 rhodecode/lib/action_parser.py                |   3 +-
 rhodecode/lib/hook_daemon/base.py             |  52 ++---
 .../lib/hook_daemon/celery_hooks_deamon.py    |   3 +
 .../lib/hook_daemon/http_hooks_deamon.py      |  17 +-
 rhodecode/lib/middleware/simplesvn.py         |  92 ++++----
 rhodecode/lib/middleware/simplevcs.py         |  83 ++++----
 rhodecode/lib/svn_txn_utils.py                | 132 ++++++++++++
 rhodecode/public/js/rhodecode/routes.js       |  44 +---
 .../base/issue_tracker_settings.mako          |   4 +-
 rhodecode/templates/base/vcs_settings.mako    |   1 -
 .../tests/fixture_mods/fixture_pyramid.py     |   2 +-
 rhodecode/tests/lib/test_hooks_daemon.py      |   3 +-
 rhodecode/tests/server_utils.py               |  12 +-
 rhodecode/tests/vcs_operations/__init__.py    |  49 +++--
 rhodecode/tests/vcs_operations/conftest.py    |  20 +-
 .../test_vcs_calls_small_post_buffer.py       |   2 +-
 .../vcs_operations/test_vcs_operations_git.py | 199 ++++++++++++++++++
 ...perations.py => test_vcs_operations_hg.py} | 152 +------------
 .../vcs_operations/test_vcs_operations_svn.py | 197 +++++++++++++++++
 .../test_vcs_operations_tag_push.py           |   3 +
 rhodecode/tests/vcsserver_http.ini            |  67 ++++--
 24 files changed, 779 insertions(+), 367 deletions(-)
 create mode 100644 rhodecode/lib/svn_txn_utils.py
 create mode 100644 rhodecode/tests/vcs_operations/test_vcs_operations_git.py
 rename rhodecode/tests/vcs_operations/{test_vcs_operations.py => test_vcs_operations_hg.py} (58%)
 create mode 100644 rhodecode/tests/vcs_operations/test_vcs_operations_svn.py

diff --git a/configs/development.ini b/configs/development.ini
index bf3a14d7..5231b101 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -657,6 +657,10 @@ vcs.methods.cache = true
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
+; Redis connection settings for svn integrations logic
+; This connection string needs to be the same on ce and vcsserver
+vcs.svn.redis_conn = redis://redis:6379/0
+
 ; Enable SVN proxy of requests over HTTP
 vcs.svn.proxy.enabled = true
 
diff --git a/configs/production.ini b/configs/production.ini
index bf325a15..aa053c96 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -625,6 +625,10 @@ vcs.methods.cache = true
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
+; Redis connection settings for svn integrations logic
+; This connection string needs to be the same on ce and vcsserver
+vcs.svn.redis_conn = redis://redis:6379/0
+
 ; Enable SVN proxy of requests over HTTP
 vcs.svn.proxy.enabled = true
 
diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py
index afd6916c..87b05fe8 100644
--- a/rhodecode/config/config_maker.py
+++ b/rhodecode/config/config_maker.py
@@ -103,6 +103,7 @@ def sanitize_settings_and_apply_defaults(global_config, settings):
     settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
 
     settings_maker.make_setting('vcs.svn.compatible_version', '')
+    settings_maker.make_setting('vcs.svn.redis_conn', 'redis://redis:6379/0')
     settings_maker.make_setting('vcs.svn.proxy.enabled', True, parser='bool')
     settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090', parser='string')
     settings_maker.make_setting('vcs.hooks.protocol', 'http')
diff --git a/rhodecode/lib/action_parser.py b/rhodecode/lib/action_parser.py
index 2002d6d0..6ff4efb2 100644
--- a/rhodecode/lib/action_parser.py
+++ b/rhodecode/lib/action_parser.py
@@ -258,8 +258,7 @@ class ActionParser(object):
                     commit = repo.get_commit(commit_id=commit_id)
                     commits.append(commit)
                 except CommitDoesNotExistError:
-                    log.error(
-                        'cannot find commit id %s in this repository',
+                    log.error('cannot find commit id %s in this repository',
                         commit_id)
                     commits.append(commit_id)
                     continue
diff --git a/rhodecode/lib/hook_daemon/base.py b/rhodecode/lib/hook_daemon/base.py
index 14faca6d..d29dbec4 100644
--- a/rhodecode/lib/hook_daemon/base.py
+++ b/rhodecode/lib/hook_daemon/base.py
@@ -15,13 +15,14 @@
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
+
 import os
 import time
 import logging
-import tempfile
 
 from rhodecode.lib.config_utils import get_config
-from rhodecode.lib.ext_json import json
+
+from rhodecode.lib.svn_txn_utils import get_txn_id_from_store
 
 log = logging.getLogger(__name__)
 
@@ -47,49 +48,22 @@ class HooksModuleCallbackDaemon(BaseHooksCallbackDaemon):
         super().__init__()
         self.hooks_module = module
 
-
-def get_txn_id_data_path(txn_id):
-    import rhodecode
-
-    root = rhodecode.CONFIG.get('cache_dir') or tempfile.gettempdir()
-    final_dir = os.path.join(root, 'svn_txn_id')
-
-    if not os.path.isdir(final_dir):
-        os.makedirs(final_dir)
-    return os.path.join(final_dir, 'rc_txn_id_{}'.format(txn_id))
-
-
-def store_txn_id_data(txn_id, data_dict):
-    if not txn_id:
-        log.warning('Cannot store txn_id because it is empty')
-        return
-
-    path = get_txn_id_data_path(txn_id)
-    try:
-        with open(path, 'wb') as f:
-            f.write(json.dumps(data_dict))
-    except Exception:
-        log.exception('Failed to write txn_id metadata')
-
-
-def get_txn_id_from_store(txn_id):
-    """
-    Reads txn_id from store and if present returns the data for callback manager
-    """
-    path = get_txn_id_data_path(txn_id)
-    try:
-        with open(path, 'rb') as f:
-            return json.loads(f.read())
-    except Exception:
-        return {}
+    def __repr__(self):
+        return f'HooksModuleCallbackDaemon(hooks_module={self.hooks_module})'
 
 
 def prepare_callback_daemon(extras, protocol, host, txn_id=None):
-    txn_details = get_txn_id_from_store(txn_id)
-    port = txn_details.get('port', 0)
+
     match protocol:
         case 'http':
             from rhodecode.lib.hook_daemon.http_hooks_deamon import HttpHooksCallbackDaemon
+            port = 0
+            if txn_id:
+                # read txn-id to re-use the PORT for callback daemon
+                repo_path = os.path.join(extras['repo_store'], extras['repository'])
+                txn_details = get_txn_id_from_store(repo_path, txn_id)
+                port = txn_details.get('port', 0)
+
             callback_daemon = HttpHooksCallbackDaemon(
                 txn_id=txn_id, host=host, port=port)
         case 'celery':
diff --git a/rhodecode/lib/hook_daemon/celery_hooks_deamon.py b/rhodecode/lib/hook_daemon/celery_hooks_deamon.py
index fc9883f4..636f27ad 100644
--- a/rhodecode/lib/hook_daemon/celery_hooks_deamon.py
+++ b/rhodecode/lib/hook_daemon/celery_hooks_deamon.py
@@ -28,3 +28,6 @@ class CeleryHooksCallbackDaemon(BaseHooksCallbackDaemon):
         # TODO: replace this with settings bootstrapped...
         self.task_queue = config.get('app:main', 'celery.broker_url')
         self.task_backend = config.get('app:main', 'celery.result_backend')
+
+    def __repr__(self):
+        return f'CeleryHooksCallbackDaemon(task_queue={self.task_queue}, task_backend={self.task_backend})'
diff --git a/rhodecode/lib/hook_daemon/http_hooks_deamon.py b/rhodecode/lib/hook_daemon/http_hooks_deamon.py
index 73ecd0ff..bc86d0cb 100644
--- a/rhodecode/lib/hook_daemon/http_hooks_deamon.py
+++ b/rhodecode/lib/hook_daemon/http_hooks_deamon.py
@@ -30,7 +30,7 @@ from socketserver import TCPServer
 from rhodecode.model import meta
 from rhodecode.lib.ext_json import json
 from rhodecode.lib import rc_cache
-from rhodecode.lib.hook_daemon.base import get_txn_id_data_path
+from rhodecode.lib.svn_txn_utils import get_txn_id_data_key
 from rhodecode.lib.hook_daemon.hook_module import Hooks
 
 log = logging.getLogger(__name__)
@@ -185,9 +185,12 @@ class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
 
     use_gevent = False
 
+    def __repr__(self):
+        return f'HttpHooksCallbackDaemon(hooks_uri={self.hooks_uri})'
+
     @property
     def _hook_prefix(self):
-        return 'HOOKS: {} '.format(self.hooks_uri)
+        return f'HOOKS: {self.hooks_uri} '
 
     def get_hostname(self):
         return socket.gethostname() or '127.0.0.1'
@@ -205,7 +208,7 @@ class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
             port = self.get_available_port()
 
         server_address = (host, port)
-        self.hooks_uri = '{}:{}'.format(host, port)
+        self.hooks_uri = f'{host}:{port}'
         self.txn_id = txn_id
         self._done = False
 
@@ -249,7 +252,9 @@ class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
         self._daemon = None
         self._callback_thread = None
         if self.txn_id:
-            txn_id_file = get_txn_id_data_path(self.txn_id)
+            #TODO: figure out the repo_path...
+            repo_path = ''
+            txn_id_file = get_txn_id_data_key(repo_path, self.txn_id)
             log.debug('Cleaning up TXN ID %s', txn_id_file)
             if os.path.isfile(txn_id_file):
                 os.remove(txn_id_file)
@@ -272,7 +277,9 @@ class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
         self._callback_greenlet = None
 
         if self.txn_id:
-            txn_id_file = get_txn_id_data_path(self.txn_id)
+            #TODO: figure out the repo_path...
+            repo_path = ''
+            txn_id_file = get_txn_id_data_key(repo_path, self.txn_id)
             log.debug('Cleaning up TXN ID %s', txn_id_file)
             if os.path.isfile(txn_id_file):
                 os.remove(txn_id_file)
diff --git a/rhodecode/lib/middleware/simplesvn.py b/rhodecode/lib/middleware/simplesvn.py
index 12b12c8d..67fe64bf 100644
--- a/rhodecode/lib/middleware/simplesvn.py
+++ b/rhodecode/lib/middleware/simplesvn.py
@@ -17,7 +17,8 @@
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-import base64
+import re
+import os
 import logging
 import urllib.request
 import urllib.parse
@@ -28,14 +29,10 @@ import requests
 from pyramid.httpexceptions import HTTPNotAcceptable
 
 from rhodecode import ConfigGet
-from rhodecode.lib import rc_cache
 from rhodecode.lib.middleware import simplevcs
 from rhodecode.lib.middleware.utils import get_path_info
 from rhodecode.lib.utils import is_valid_repo
-from rhodecode.lib.str_utils import safe_str, safe_int, safe_bytes
-from rhodecode.lib.type_utils import str2bool
-from rhodecode.lib.ext_json import json
-from rhodecode.lib.hook_daemon.base import store_txn_id_data
+from rhodecode.lib.str_utils import safe_str
 
 log = logging.getLogger(__name__)
 
@@ -63,28 +60,11 @@ class SimpleSvnApp(object):
 
         # stream control flag, based on request and content type...
         stream = False
-
         if req_method in ['MKCOL'] or has_content_length:
-            data_processed = False
-            # read chunk to check if we have txn-with-props
-            initial_data: bytes = data_io.read(1024)
-            if initial_data.startswith(b'(create-txn-with-props'):
-                data_io = initial_data + data_io.read()
-                # store on-the-fly our rc_extra using svn revision properties
-                # those can be read later on in hooks executed so we have a way
-                # to pass in the data into svn hooks
-                rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
-                rc_data_len = str(len(rc_data))
-                # header defines data length, and serialized data
-                skel = b' rc-scm-extras %b %b' % (safe_bytes(rc_data_len), safe_bytes(rc_data))
-                data_io = data_io[:-2] + skel + b'))'
-                data_processed = True
-
-            if not data_processed:
-                # NOTE(johbo): Avoid that we end up with sending the request in chunked
-                # transfer encoding (mainly on Gunicorn). If we know the content
-                # length, then we should transfer the payload in one request.
-                data_io = initial_data + data_io.read()
+            # NOTE(johbo): Avoid that we end up with sending the request in chunked
+            # transfer encoding (mainly on Gunicorn). If we know the content
+            # length, then we should transfer the payload in one request.
+            data_io = data_io.read()
 
         if req_method in ['GET', 'PUT'] or transfer_encoding == 'chunked':
             # NOTE(marcink): when getting/uploading files, we want to STREAM content
@@ -101,6 +81,7 @@ class SimpleSvnApp(object):
             stream=stream
         )
         if req_method in ['HEAD', 'DELETE']:
+            # NOTE(marcink): HEAD might be deprecated for SVN 1.14+ protocol
             del call_kwargs['data']
 
         try:
@@ -120,14 +101,6 @@ class SimpleSvnApp(object):
             log.debug('got response code: %s', response.status_code)
 
         response_headers = self._get_response_headers(response.headers)
-
-        if response.headers.get('SVN-Txn-name'):
-            svn_tx_id = response.headers.get('SVN-Txn-name')
-            txn_id = rc_cache.utils.compute_key_from_params(
-                self.config['repository'], svn_tx_id)
-            port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1])
-            store_txn_id_data(txn_id, {'port': port})
-
         start_response(f'{response.status_code} {response.reason}', response_headers)
         return response.iter_content(chunk_size=1024)
 
@@ -137,6 +110,20 @@ class SimpleSvnApp(object):
         url_path = urllib.parse.quote(url_path, safe="/:=~+!$,;'")
         return url_path
 
+    def _get_txn_id(self, environ):
+        url = environ['RAW_URI']
+
+        # Define the regex pattern
+        pattern = r'/txr/([^/]+)/'
+
+        # Search for the pattern in the URL
+        match = re.search(pattern, url)
+
+        # Check if a match is found and extract the captured group
+        if match:
+            txn_id = match.group(1)
+            return txn_id
+
     def _get_request_headers(self, environ):
         headers = {}
         whitelist = {
@@ -182,10 +169,39 @@ class DisabledSimpleSvnApp(object):
 
 
 class SimpleSvn(simplevcs.SimpleVCS):
+    """
+        details: https://svn.apache.org/repos/asf/subversion/trunk/notes/http-and-webdav/webdav-protocol
+
+        Read Commands  :  (OPTIONS, PROPFIND, GET, REPORT)
+
+        GET: fetch info about resources
+        PROPFIND: Used to retrieve properties of resources.
+        REPORT: Used for specialized queries to the repository. E.g History etc...
+        OPTIONS: request is sent to an SVN server, the server responds with information about the available HTTP
+            methods and other server capabilities.
+
+        Write Commands  :  (MKACTIVITY, PROPPATCH, PUT, CHECKOUT, MKCOL, MOVE,
+        --------------      COPY, DELETE, LOCK, UNLOCK, MERGE)
+
+        With the exception of LOCK/UNLOCK, every write command performs some
+        sort of DeltaV commit operation.  In DeltaV, a commit always starts
+        by creating a transaction (MKACTIVITY), applies a log message
+        (PROPPATCH), does some other write methods, and then ends by
+        committing the transaction (MERGE).  If the MERGE fails, the client
+        may try to remove the transaction with a DELETE.
+
+        PROPPATCH: Used to set and/or remove properties on resources.
+        MKCOL: Creates a new collection (directory).
+        DELETE: Removes a resource.
+        COPY and MOVE: Used for copying and moving resources.
+        MERGE: Used to merge changes from different branches.
+        CHECKOUT, CHECKIN, UNCHECKOUT: DeltaV methods for managing working resources and versions.
+    """
 
     SCM = 'svn'
     READ_ONLY_COMMANDS = ('OPTIONS', 'PROPFIND', 'GET', 'REPORT')
-    DEFAULT_HTTP_SERVER = 'http://localhost:8090'
+    WRITE_COMMANDS = ('MERGE', 'POST', 'PUT', 'COPY', 'MOVE', 'DELETE', 'MKCOL')
+    DEFAULT_HTTP_SERVER = 'http://svn:8090'
 
     def _get_repository_name(self, environ):
         """
@@ -218,10 +234,10 @@ class SimpleSvn(simplevcs.SimpleVCS):
             else 'push')
 
     def _should_use_callback_daemon(self, extras, environ, action):
-        # only MERGE command triggers hooks, so we don't want to start
+        # only PUT & MERGE command triggers hooks, so we don't want to start
         # hooks server too many times. POST however starts the svn transaction
         # so we also need to run the init of callback daemon of POST
-        if environ['REQUEST_METHOD'] in ['MERGE', 'POST']:
+        if environ['REQUEST_METHOD'] not in self.READ_ONLY_COMMANDS:
             return True
         return False
 
diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py
index 7f7f2a1c..67480355 100644
--- a/rhodecode/lib/middleware/simplevcs.py
+++ b/rhodecode/lib/middleware/simplevcs.py
@@ -25,11 +25,9 @@ It's implemented with basic auth function
 
 import os
 import re
-import io
 import logging
 import importlib
 from functools import wraps
-from lxml import etree
 
 import time
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
@@ -41,6 +39,7 @@ from zope.cachedescriptors.property import Lazy as LazyProperty
 import rhodecode
 from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
 from rhodecode.lib import rc_cache
+from rhodecode.lib.svn_txn_utils import store_txn_id_data
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
@@ -48,7 +47,7 @@ from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserE
 from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
-from rhodecode.lib.str_utils import safe_bytes
+from rhodecode.lib.str_utils import safe_bytes, safe_int
 from rhodecode.lib.utils import is_valid_repo, SLUG_RE
 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
 from rhodecode.lib.vcs.conf import settings as vcs_settings
@@ -63,29 +62,6 @@ from rhodecode.model.settings import SettingsModel, VcsSettingsModel
 log = logging.getLogger(__name__)
 
 
-def extract_svn_txn_id(acl_repo_name, data: bytes):
-    """
-    Helper method for extraction of svn txn_id from submitted XML data during
-    POST operations
-    """
-
-    try:
-        root = etree.fromstring(data)
-        pat = re.compile(r'/txn/(?P<txn_id>.*)')
-        for el in root:
-            if el.tag == '{DAV:}source':
-                for sub_el in el:
-                    if sub_el.tag == '{DAV:}href':
-                        match = pat.search(sub_el.text)
-                        if match:
-                            svn_tx_id = match.groupdict()['txn_id']
-                            txn_id = rc_cache.utils.compute_key_from_params(
-                                acl_repo_name, svn_tx_id)
-                            return txn_id
-    except Exception:
-        log.exception('Failed to extract txn_id')
-
-
 def initialize_generator(factory):
     """
     Initializes the returned generator by draining its first element.
@@ -468,7 +444,6 @@ class SimpleVCS(object):
                     log.debug('Not enough credentials to access repo: `%s` '
                               'repository as anonymous user', self.acl_repo_name)
 
-
                 username = None
                 # ==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
@@ -582,6 +557,24 @@ class SimpleVCS(object):
         return self._generate_vcs_response(
             environ, start_response, repo_path, extras, action)
 
+    def _get_txn_id(self, environ):
+
+        for k in ['RAW_URI', 'HTTP_DESTINATION']:
+            url = environ.get(k)
+            if not url:
+                continue
+
+            # regex to search for svn-txn-id
+            pattern = r'/!svn/txr/([^/]+)/'
+
+            # Search for the pattern in the URL
+            match = re.search(pattern, url)
+
+            # Check if a match is found and extract the captured group
+            if match:
+                txn_id = match.group(1)
+                return txn_id
+
     @initialize_generator
     def _generate_vcs_response(
             self, environ, start_response, repo_path, extras, action):
@@ -593,28 +586,23 @@ class SimpleVCS(object):
         also handles the locking exceptions which will be triggered when
         the first chunk is produced by the underlying WSGI application.
         """
-
-        txn_id = ''
-        if 'CONTENT_LENGTH' in environ and environ['REQUEST_METHOD'] == 'MERGE':
-            # case for SVN, we want to re-use the callback daemon port
-            # so we use the txn_id, for this we peek the body, and still save
-            # it as wsgi.input
-
-            stream = environ['wsgi.input']
-
-            if isinstance(stream, io.BytesIO):
-                data: bytes = stream.getvalue()
-            elif hasattr(stream, 'buf'):  # most likely gunicorn.http.body.Body
-                data: bytes = stream.buf.getvalue()
-            else:
-                # fallback to the crudest way, copy the iterator
-                data = safe_bytes(stream.read())
-                environ['wsgi.input'] = io.BytesIO(data)
-
-            txn_id = extract_svn_txn_id(self.acl_repo_name, data)
+        svn_txn_id = ''
+        if action == 'push':
+            svn_txn_id = self._get_txn_id(environ)
 
         callback_daemon, extras = self._prepare_callback_daemon(
-            extras, environ, action, txn_id=txn_id)
+            extras, environ, action, txn_id=svn_txn_id)
+
+        if svn_txn_id:
+
+            port = safe_int(extras['hooks_uri'].split(':')[-1])
+            txn_id_data = extras.copy()
+            txn_id_data.update({'port': port})
+            txn_id_data.update({'req_method': environ['REQUEST_METHOD']})
+
+            full_repo_path = repo_path
+            store_txn_id_data(full_repo_path, svn_txn_id, txn_id_data)
+
         log.debug('HOOKS extras is %s', extras)
 
         http_scheme = self._get_http_scheme(environ)
@@ -677,6 +665,7 @@ class SimpleVCS(object):
 
     def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
         protocol = vcs_settings.HOOKS_PROTOCOL
+
         if not self._should_use_callback_daemon(extras, environ, action):
             # disable callback daemon for actions that don't require it
             protocol = 'local'
diff --git a/rhodecode/lib/svn_txn_utils.py b/rhodecode/lib/svn_txn_utils.py
new file mode 100644
index 00000000..5463bbb2
--- /dev/null
+++ b/rhodecode/lib/svn_txn_utils.py
@@ -0,0 +1,132 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+import redis
+
+from ..lib import rc_cache
+from ..lib.ext_json import json
+
+
+log = logging.getLogger(__name__)
+
+redis_client = None
+
+
+class RedisTxnClient:
+
+    def __init__(self, url):
+        self.url = url
+        self._create_client(url)
+
+    def _create_client(self, url):
+        connection_pool = redis.ConnectionPool.from_url(url)
+        self.writer_client = redis.StrictRedis(
+            connection_pool=connection_pool
+        )
+        self.reader_client = self.writer_client
+
+    def set(self, key, value, expire=24 * 60000):
+        self.writer_client.set(key, value, ex=expire)
+
+    def get(self, key):
+        return self.reader_client.get(key)
+
+    def delete(self, key):
+        self.writer_client.delete(key)
+
+
+def get_redis_client(url=''):
+
+    global redis_client
+    if redis_client is not None:
+        return redis_client
+    if not url:
+        from rhodecode import CONFIG
+        url = CONFIG['vcs.svn.redis_conn']
+    redis_client = RedisTxnClient(url)
+    return redis_client
+
+
+def extract_svn_txn_id(data: bytes):
+    """
+    Helper method for extraction of svn txn_id from submitted XML data during
+    POST operations
+    """
+    import re
+    from lxml import etree
+
+    try:
+        root = etree.fromstring(data)
+        pat = re.compile(r'/txn/(?P<txn_id>.*)')
+        for el in root:
+            if el.tag == '{DAV:}source':
+                for sub_el in el:
+                    if sub_el.tag == '{DAV:}href':
+                        match = pat.search(sub_el.text)
+                        if match:
+                            svn_tx_id = match.groupdict()['txn_id']
+                            return svn_tx_id
+    except Exception:
+        log.exception('Failed to extract txn_id')
+
+
+def get_txn_id_data_key(repo_path, svn_txn_id):
+    log.debug('svn-txn-id: %s, obtaining data path', svn_txn_id)
+    repo_key = rc_cache.utils.compute_key_from_params(repo_path)
+    final_key = f'{repo_key}.{svn_txn_id}.svn_txn_id'
+    log.debug('computed final key: %s', final_key)
+
+    return final_key
+
+
+def store_txn_id_data(repo_path, svn_txn_id, data_dict):
+    log.debug('svn-txn-id: %s, storing data', svn_txn_id)
+
+    if not svn_txn_id:
+        log.warning('Cannot store txn_id because it is empty')
+        return
+
+    redis_conn = get_redis_client()
+
+    store_key = get_txn_id_data_key(repo_path, svn_txn_id)
+    store_data = json.dumps(data_dict)
+    redis_conn.set(store_key, store_data)
+
+
+def get_txn_id_from_store(repo_path, svn_txn_id, rm_on_read=False):
+    """
+    Reads txn_id from store and if present returns the data for callback manager
+    """
+    log.debug('svn-txn-id: %s, retrieving data', svn_txn_id)
+    redis_conn = get_redis_client()
+
+    store_key = get_txn_id_data_key(repo_path, svn_txn_id)
+    data = {}
+    redis_conn.get(store_key)
+    try:
+        raw_data = redis_conn.get(store_key)
+        data = json.loads(raw_data)
+    except Exception:
+        log.exception('Failed to get txn_id metadata')
+
+    if rm_on_read:
+        log.debug('Cleaning up txn_id at %s', store_key)
+        redis_conn.delete(store_key)
+
+    return data
diff --git a/rhodecode/public/js/rhodecode/routes.js b/rhodecode/public/js/rhodecode/routes.js
index e224b7c5..44d89623 100644
--- a/rhodecode/public/js/rhodecode/routes.js
+++ b/rhodecode/public/js/rhodecode/routes.js
@@ -12,16 +12,14 @@
  ******************************************************************************/
 function registerRCRoutes() {
     // routes registration
-    pyroutes.register('admin_artifacts', '/_admin/artifacts', []);
-    pyroutes.register('admin_artifacts_data', '/_admin/artifacts-data', []);
-    pyroutes.register('admin_artifacts_delete', '/_admin/artifacts/%(uid)s/delete', ['uid']);
-    pyroutes.register('admin_artifacts_show_all', '/_admin/artifacts', []);
-    pyroutes.register('admin_artifacts_show_info', '/_admin/artifacts/%(uid)s', ['uid']);
-    pyroutes.register('admin_artifacts_update', '/_admin/artifacts/%(uid)s/update', ['uid']);
+    pyroutes.register('admin_artifacts', '/_admin/_admin/artifacts', []);
+    pyroutes.register('admin_artifacts_delete', '/_admin/_admin/artifacts/%(uid)s/delete', ['uid']);
+    pyroutes.register('admin_artifacts_show_all', '/_admin/_admin/artifacts', []);
+    pyroutes.register('admin_artifacts_show_info', '/_admin/_admin/artifacts/%(uid)s', ['uid']);
+    pyroutes.register('admin_artifacts_update', '/_admin/_admin/artifacts/%(uid)s/update', ['uid']);
     pyroutes.register('admin_audit_log_entry', '/_admin/audit_logs/%(audit_log_id)s', ['audit_log_id']);
     pyroutes.register('admin_audit_logs', '/_admin/audit_logs', []);
-    pyroutes.register('admin_automation', '/_admin/automation', []);
-    pyroutes.register('admin_automation_update', '/_admin/automation/%(entry_id)s/update', ['entry_id']);
+    pyroutes.register('admin_automation', '/_admin/_admin/automation', []);
     pyroutes.register('admin_defaults_repositories', '/_admin/defaults/repositories', []);
     pyroutes.register('admin_defaults_repositories_update', '/_admin/defaults/repositories/update', []);
     pyroutes.register('admin_home', '/_admin', []);
@@ -29,7 +27,6 @@ function registerRCRoutes() {
     pyroutes.register('admin_permissions_application_update', '/_admin/permissions/application/update', []);
     pyroutes.register('admin_permissions_auth_token_access', '/_admin/permissions/auth_token_access', []);
     pyroutes.register('admin_permissions_branch', '/_admin/permissions/branch', []);
-    pyroutes.register('admin_permissions_branch_update', '/_admin/permissions/branch/update', []);
     pyroutes.register('admin_permissions_global', '/_admin/permissions/global', []);
     pyroutes.register('admin_permissions_global_update', '/_admin/permissions/global/update', []);
     pyroutes.register('admin_permissions_ips', '/_admin/permissions/ips', []);
@@ -39,8 +36,7 @@ function registerRCRoutes() {
     pyroutes.register('admin_permissions_ssh_keys', '/_admin/permissions/ssh_keys', []);
     pyroutes.register('admin_permissions_ssh_keys_data', '/_admin/permissions/ssh_keys/data', []);
     pyroutes.register('admin_permissions_ssh_keys_update', '/_admin/permissions/ssh_keys/update', []);
-    pyroutes.register('admin_scheduler', '/_admin/scheduler', []);
-    pyroutes.register('admin_scheduler_show_tasks', '/_admin/scheduler/_tasks', []);
+    pyroutes.register('admin_scheduler', '/_admin/_admin/scheduler', []);
     pyroutes.register('admin_settings', '/_admin/settings', []);
     pyroutes.register('admin_settings_email', '/_admin/settings/email', []);
     pyroutes.register('admin_settings_email_update', '/_admin/settings/email/update', []);
@@ -59,8 +55,6 @@ function registerRCRoutes() {
     pyroutes.register('admin_settings_issuetracker_update', '/_admin/settings/issue-tracker/update', []);
     pyroutes.register('admin_settings_labs', '/_admin/settings/labs', []);
     pyroutes.register('admin_settings_labs_update', '/_admin/settings/labs/update', []);
-    pyroutes.register('admin_settings_license', '/_admin/settings/license', []);
-    pyroutes.register('admin_settings_license_unlock', '/_admin/settings/license_unlock', []);
     pyroutes.register('admin_settings_mapping', '/_admin/settings/mapping', []);
     pyroutes.register('admin_settings_mapping_update', '/_admin/settings/mapping/update', []);
     pyroutes.register('admin_settings_open_source', '/_admin/settings/open_source', []);
@@ -68,12 +62,6 @@ function registerRCRoutes() {
     pyroutes.register('admin_settings_process_management_data', '/_admin/settings/process_management/data', []);
     pyroutes.register('admin_settings_process_management_master_signal', '/_admin/settings/process_management/master_signal', []);
     pyroutes.register('admin_settings_process_management_signal', '/_admin/settings/process_management/signal', []);
-    pyroutes.register('admin_settings_scheduler_create', '/_admin/scheduler/create', []);
-    pyroutes.register('admin_settings_scheduler_delete', '/_admin/scheduler/%(schedule_id)s/delete', ['schedule_id']);
-    pyroutes.register('admin_settings_scheduler_edit', '/_admin/scheduler/%(schedule_id)s', ['schedule_id']);
-    pyroutes.register('admin_settings_scheduler_execute', '/_admin/scheduler/%(schedule_id)s/execute', ['schedule_id']);
-    pyroutes.register('admin_settings_scheduler_new', '/_admin/scheduler/new', []);
-    pyroutes.register('admin_settings_scheduler_update', '/_admin/scheduler/%(schedule_id)s/update', ['schedule_id']);
     pyroutes.register('admin_settings_search', '/_admin/settings/search', []);
     pyroutes.register('admin_settings_sessions', '/_admin/settings/sessions', []);
     pyroutes.register('admin_settings_sessions_cleanup', '/_admin/settings/sessions/cleanup', []);
@@ -97,7 +85,6 @@ function registerRCRoutes() {
     pyroutes.register('channelstream_proxy', '/_channelstream', []);
     pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []);
     pyroutes.register('check_2fa', '/_admin/check_2fa', []);
-    pyroutes.register('commit_draft_comments_submit', '/%(repo_name)s/changeset/%(commit_id)s/draft_comments_submit', ['repo_name', 'commit_id']);
     pyroutes.register('debug_style_email', '/_admin/debug_style/email/%(email_id)s', ['email_id']);
     pyroutes.register('debug_style_email_plain_rendered', '/_admin/debug_style/email-rendered/%(email_id)s', ['email_id']);
     pyroutes.register('debug_style_home', '/_admin/debug_style', []);
@@ -222,8 +209,6 @@ function registerRCRoutes() {
     pyroutes.register('my_account_emails', '/_admin/my_account/emails', []);
     pyroutes.register('my_account_emails_add', '/_admin/my_account/emails/new', []);
     pyroutes.register('my_account_emails_delete', '/_admin/my_account/emails/delete', []);
-    pyroutes.register('my_account_external_identity', '/_admin/my_account/external-identity', []);
-    pyroutes.register('my_account_external_identity_delete', '/_admin/my_account/external-identity/delete', []);
     pyroutes.register('my_account_goto_bookmark', '/_admin/my_account/bookmark/%(bookmark_id)s', ['bookmark_id']);
     pyroutes.register('my_account_notifications', '/_admin/my_account/notifications', []);
     pyroutes.register('my_account_notifications_test_channelstream', '/_admin/my_account/test_channelstream', []);
@@ -254,7 +239,6 @@ function registerRCRoutes() {
     pyroutes.register('ops_healthcheck', '/_admin/ops/status', []);
     pyroutes.register('ops_ping', '/_admin/ops/ping', []);
     pyroutes.register('ops_redirect_test', '/_admin/ops/redirect', []);
-    pyroutes.register('plugin_admin_chat', '/_admin/plugin_admin_chat/%(action)s', ['action']);
     pyroutes.register('pull_requests_global', '/_admin/pull-request/%(pull_request_id)s', ['pull_request_id']);
     pyroutes.register('pull_requests_global_0', '/_admin/pull_requests/%(pull_request_id)s', ['pull_request_id']);
     pyroutes.register('pull_requests_global_1', '/_admin/pull-requests/%(pull_request_id)s', ['pull_request_id']);
@@ -264,7 +248,6 @@ function registerRCRoutes() {
     pyroutes.register('pullrequest_comments', '/%(repo_name)s/pull-request/%(pull_request_id)s/comments', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_create', '/%(repo_name)s/pull-request/create', ['repo_name']);
     pyroutes.register('pullrequest_delete', '/%(repo_name)s/pull-request/%(pull_request_id)s/delete', ['repo_name', 'pull_request_id']);
-    pyroutes.register('pullrequest_draft_comments_submit', '/%(repo_name)s/pull-request/%(pull_request_id)s/draft_comments_submit', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_drafts', '/%(repo_name)s/pull-request/%(pull_request_id)s/drafts', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_merge', '/%(repo_name)s/pull-request/%(pull_request_id)s/merge', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_new', '/%(repo_name)s/pull-request/new', ['repo_name']);
@@ -277,18 +260,8 @@ function registerRCRoutes() {
     pyroutes.register('pullrequest_update', '/%(repo_name)s/pull-request/%(pull_request_id)s/update', ['repo_name', 'pull_request_id']);
     pyroutes.register('register', '/_admin/register', []);
     pyroutes.register('repo_archivefile', '/%(repo_name)s/archive/%(fname)s', ['repo_name', 'fname']);
-    pyroutes.register('repo_artifacts_data', '/%(repo_name)s/artifacts_data', ['repo_name']);
-    pyroutes.register('repo_artifacts_delete', '/%(repo_name)s/artifacts/delete/%(uid)s', ['repo_name', 'uid']);
-    pyroutes.register('repo_artifacts_get', '/%(repo_name)s/artifacts/download/%(uid)s', ['repo_name', 'uid']);
-    pyroutes.register('repo_artifacts_info', '/%(repo_name)s/artifacts/info/%(uid)s', ['repo_name', 'uid']);
     pyroutes.register('repo_artifacts_list', '/%(repo_name)s/artifacts', ['repo_name']);
-    pyroutes.register('repo_artifacts_new', '/%(repo_name)s/artifacts/new', ['repo_name']);
-    pyroutes.register('repo_artifacts_store', '/%(repo_name)s/artifacts/store', ['repo_name']);
-    pyroutes.register('repo_artifacts_stream_script', '/_file_store/stream-upload-script', []);
-    pyroutes.register('repo_artifacts_stream_store', '/_file_store/stream-upload', []);
-    pyroutes.register('repo_artifacts_update', '/%(repo_name)s/artifacts/update/%(uid)s', ['repo_name', 'uid']);
     pyroutes.register('repo_automation', '/%(repo_name)s/settings/automation', ['repo_name']);
-    pyroutes.register('repo_automation_update', '/%(repo_name)s/settings/automation/%(entry_id)s/update', ['repo_name', 'entry_id']);
     pyroutes.register('repo_changelog', '/%(repo_name)s/changelog', ['repo_name']);
     pyroutes.register('repo_changelog_file', '/%(repo_name)s/changelog/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_commit', '/%(repo_name)s/changeset/%(commit_id)s', ['repo_name', 'commit_id']);
@@ -366,9 +339,6 @@ function registerRCRoutes() {
     pyroutes.register('repo_refs_changelog_data', '/%(repo_name)s/refs-data-changelog', ['repo_name']);
     pyroutes.register('repo_refs_data', '/%(repo_name)s/refs-data', ['repo_name']);
     pyroutes.register('repo_reviewers', '/%(repo_name)s/settings/review/rules', ['repo_name']);
-    pyroutes.register('repo_reviewers_review_rule_delete', '/%(repo_name)s/settings/review/rules/%(rule_id)s/delete', ['repo_name', 'rule_id']);
-    pyroutes.register('repo_reviewers_review_rule_edit', '/%(repo_name)s/settings/review/rules/%(rule_id)s', ['repo_name', 'rule_id']);
-    pyroutes.register('repo_reviewers_review_rule_new', '/%(repo_name)s/settings/review/rules/new', ['repo_name']);
     pyroutes.register('repo_settings_quick_actions', '/%(repo_name)s/settings/quick-action', ['repo_name']);
     pyroutes.register('repo_stats', '/%(repo_name)s/repo_stats/%(commit_id)s', ['repo_name', 'commit_id']);
     pyroutes.register('repo_summary', '/%(repo_name)s', ['repo_name']);
diff --git a/rhodecode/templates/base/issue_tracker_settings.mako b/rhodecode/templates/base/issue_tracker_settings.mako
index 581baf81..91b0e56c 100644
--- a/rhodecode/templates/base/issue_tracker_settings.mako
+++ b/rhodecode/templates/base/issue_tracker_settings.mako
@@ -17,7 +17,7 @@ examples = [
 
     (
     'Tickets with #123 (Redmine etc)',
-    '(?<![a-zA-Z0-9_/]{1,10}-?)(#)(?P<issue_id>\d+)',
+    '(?<![a-zA-Z0-9_/]{1,10}-?)(#)(?P<issue_id>[0-9]+)',
     'https://myissueserver.com/${repo}/issue/${issue_id}',
     ''
     ),
@@ -60,7 +60,7 @@ examples = [
 
     (
     'Pivotal Tracker',
-    '(?:pivot-)(?P<project_id>\d+)-(?P<story>\d+)',
+    '(?:pivot-)(?P<project_id>\d+)-(?P<story>[0-9]+)',
     'https://www.pivotaltracker.com/s/projects/${project_id}/stories/${story}',
     'PIV-',
     ),
diff --git a/rhodecode/templates/base/vcs_settings.mako b/rhodecode/templates/base/vcs_settings.mako
index 14b76db0..dba915e8 100644
--- a/rhodecode/templates/base/vcs_settings.mako
+++ b/rhodecode/templates/base/vcs_settings.mako
@@ -332,7 +332,6 @@ $(document).ready(function() {
     POST request to trigger the (re)generation of the mod_dav_svn config. */
     $('#vcs_svn_generate_cfg').on('click', function(event) {
         event.preventDefault();
-        alert('i cliked it !!')
         var url = "${h.route_path('admin_settings_vcs_svn_generate_cfg')}";
         var jqxhr = $.post(url, {'csrf_token': CSRF_TOKEN});
         jqxhr.done(function(data) {
diff --git a/rhodecode/tests/fixture_mods/fixture_pyramid.py b/rhodecode/tests/fixture_mods/fixture_pyramid.py
index b5c80d88..d94aaa5d 100644
--- a/rhodecode/tests/fixture_mods/fixture_pyramid.py
+++ b/rhodecode/tests/fixture_mods/fixture_pyramid.py
@@ -161,7 +161,7 @@ def vcsserver_port(request):
 
 
 @pytest.fixture(scope='session')
-def available_port_factory():
+def available_port_factory() -> get_available_port:
     """
     Returns a callable which returns free port numbers.
     """
diff --git a/rhodecode/tests/lib/test_hooks_daemon.py b/rhodecode/tests/lib/test_hooks_daemon.py
index 95e05e49..49952bb0 100644
--- a/rhodecode/tests/lib/test_hooks_daemon.py
+++ b/rhodecode/tests/lib/test_hooks_daemon.py
@@ -304,7 +304,8 @@ class TestPrepareHooksDaemon(object):
             'txn_id': 'txnid2',
             'hooks_protocol': protocol.lower(),
             'task_backend': '',
-            'task_queue': ''
+            'task_queue': '',
+            'repo_store': '/var/opt/rhodecode_repo_store'
         }
         callback, extras = hook_base.prepare_callback_daemon(
             expected_extras.copy(), protocol=protocol, host='127.0.0.1',
diff --git a/rhodecode/tests/server_utils.py b/rhodecode/tests/server_utils.py
index 94caee60..da82dbb5 100644
--- a/rhodecode/tests/server_utils.py
+++ b/rhodecode/tests/server_utils.py
@@ -148,7 +148,7 @@ class RcVCSServer(ServerBase):
         self._args = [
             'gunicorn',
             '--bind', self.bind_addr,
-            '--worker-class', 'gevent',
+            '--worker-class', 'gthread',
             '--backlog', '16',
             '--timeout', '300',
             '--workers', workers,
@@ -185,7 +185,7 @@ class RcWebServer(ServerBase):
         self._args = [
             'gunicorn',
             '--bind', self.bind_addr,
-            '--worker-class', 'gevent',
+            '--worker-class', 'gthread',
             '--backlog', '16',
             '--timeout', '300',
             '--workers', workers,
@@ -219,3 +219,11 @@ class RcWebServer(ServerBase):
         params.update(**kwargs)
         _url = f"http://{params['user']}:{params['passwd']}@{params['host']}/{params['cloned_repo']}"
         return _url
+
+    def repo_clone_credentials(self, **kwargs):
+        params = {
+            'user': TEST_USER_ADMIN_LOGIN,
+            'passwd': TEST_USER_ADMIN_PASS,
+        }
+        params.update(**kwargs)
+        return params['user'], params['passwd']
diff --git a/rhodecode/tests/vcs_operations/__init__.py b/rhodecode/tests/vcs_operations/__init__.py
index 11862847..098ebf11 100644
--- a/rhodecode/tests/vcs_operations/__init__.py
+++ b/rhodecode/tests/vcs_operations/__init__.py
@@ -26,20 +26,21 @@ Base for test suite for making push/pull operations.
    to redirect things to stderr instead of stdout.
 """
 
-from os.path import join as jn
-from subprocess import Popen, PIPE
+
 import logging
 import os
 import tempfile
+import subprocess
 
 from rhodecode.lib.str_utils import safe_str
-from rhodecode.tests import GIT_REPO, HG_REPO
+from rhodecode.tests import GIT_REPO, HG_REPO, SVN_REPO
 
 DEBUG = True
 RC_LOG = os.path.join(tempfile.gettempdir(), 'rc.log')
 REPO_GROUP = 'a_repo_group'
-HG_REPO_WITH_GROUP = '%s/%s' % (REPO_GROUP, HG_REPO)
-GIT_REPO_WITH_GROUP = '%s/%s' % (REPO_GROUP, GIT_REPO)
+HG_REPO_WITH_GROUP = f'{REPO_GROUP}/{HG_REPO}'
+GIT_REPO_WITH_GROUP = f'{REPO_GROUP}/{GIT_REPO}'
+SVN_REPO_WITH_GROUP = f'{REPO_GROUP}/{SVN_REPO}'
 
 log = logging.getLogger(__name__)
 
@@ -65,8 +66,9 @@ class Command(object):
             if key.startswith('COV_CORE_'):
                 del env[key]
 
-        self.process = Popen(command, shell=True, stdout=PIPE, stderr=PIPE,
-                             cwd=self.cwd, env=env)
+        self.process = subprocess.Popen(
+            command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+            cwd=self.cwd, env=env)
         stdout, stderr = self.process.communicate()
 
         stdout = safe_str(stdout)
@@ -85,12 +87,14 @@ def _add_files(vcs, dest, clone_url=None, tags=None, target_branch=None, new_bra
     full_name = 'Marcin Kuźminski'
     email = 'me@email.com'
     git_ident = f"git config user.name {full_name} && git config user.email {email}"
-    cwd = path = jn(dest)
+    cwd = path = os.path.join(dest)
 
     tags = tags or []
-    added_file = jn(path, '{}_setup.py'.format(next(tempfile._RandomNameSequence())))
-    Command(cwd).execute('touch %s' % added_file)
-    Command(cwd).execute('%s add %s' % (vcs, added_file))
+    name_sequence = next(tempfile._RandomNameSequence())
+    added_file = os.path.join(path, f'{name_sequence}_setup.py')
+
+    Command(cwd).execute(f'touch {added_file}')
+    Command(cwd).execute(f'{vcs} add {added_file}')
     author_str = 'Marcin Kuźminski <me@email.com>'
 
     for i in range(kwargs.get('files_no', 3)):
@@ -128,7 +132,7 @@ def _add_files_and_push(vcs, dest, clone_url=None, tags=None, target_branch=None
     vcs is git or hg and defines what VCS we want to make those files for
     """
     git_ident = "git config user.name Marcin Kuźminski && git config user.email me@email.com"
-    cwd = jn(dest)
+    cwd = os.path.join(dest)
 
     # commit some stuff into this repo
     _add_files(vcs, dest, clone_url, tags, target_branch, new_branch, **kwargs)
@@ -147,12 +151,15 @@ def _add_files_and_push(vcs, dest, clone_url=None, tags=None, target_branch=None
         if new_branch:
             maybe_new_branch = '--new-branch'
         stdout, stderr = Command(cwd).execute(
-            'hg push --traceback --verbose {} -r {} {}'.format(maybe_new_branch, target_branch, clone_url)
+            f'hg push --traceback --verbose {maybe_new_branch} -r {target_branch} {clone_url}'
         )
     elif vcs == 'git':
         stdout, stderr = Command(cwd).execute(
-            """{} && 
-            git push --verbose --tags {} {}""".format(git_ident, clone_url, target_branch)
+            f'{git_ident} && git push --verbose --tags {clone_url} {target_branch}'
+        )
+    elif vcs == 'svn':
+        stdout, stderr = Command(cwd).execute(
+            f'svn ci -m "pushing to {target_branch}"'
         )
 
     return stdout, stderr
@@ -179,6 +186,13 @@ def _check_proper_hg_push(stdout, stderr, branch='default'):
     assert 'abort:' not in stderr
 
 
+def _check_proper_svn_push(stdout, stderr):
+    assert 'pushing to' in stdout
+    assert 'searching for changes' in stdout
+
+    assert 'abort:' not in stderr
+
+
 def _check_proper_clone(stdout, stderr, vcs):
     if vcs == 'hg':
         assert 'requesting all changes' in stdout
@@ -193,3 +207,8 @@ def _check_proper_clone(stdout, stderr, vcs):
         assert 'Cloning into' in stderr
         assert 'abort:' not in stderr
         assert 'fatal:' not in stderr
+
+    if vcs == 'svn':
+        assert 'dupa' in stdout
+
+
diff --git a/rhodecode/tests/vcs_operations/conftest.py b/rhodecode/tests/vcs_operations/conftest.py
index 6975abfd..d70fb2cd 100644
--- a/rhodecode/tests/vcs_operations/conftest.py
+++ b/rhodecode/tests/vcs_operations/conftest.py
@@ -42,7 +42,7 @@ from rhodecode.model.db import Repository
 from rhodecode.model.meta import Session
 from rhodecode.integrations.types.webhook import WebhookIntegrationType
 
-from rhodecode.tests import GIT_REPO, HG_REPO
+from rhodecode.tests import GIT_REPO, HG_REPO, SVN_REPO
 from rhodecode.tests.conftest import HTTPBIN_DOMAIN, HTTPBIN_POST
 from rhodecode.tests.fixture import Fixture
 from rhodecode.tests.server_utils import RcWebServer
@@ -51,13 +51,15 @@ from rhodecode.tests.server_utils import RcWebServer
 REPO_GROUP = 'a_repo_group'
 HG_REPO_WITH_GROUP = f'{REPO_GROUP}/{HG_REPO}'
 GIT_REPO_WITH_GROUP = f'{REPO_GROUP}/{GIT_REPO}'
+SVN_REPO_WITH_GROUP = f'{REPO_GROUP}/{SVN_REPO}'
 
 log = logging.getLogger(__name__)
 
 
 def check_httpbin_connection():
+    log.debug('Checking if HTTPBIN_DOMAIN: %s is available', HTTPBIN_DOMAIN)
     try:
-        response = requests.get(HTTPBIN_DOMAIN)
+        response = requests.get(HTTPBIN_DOMAIN, timeout=5)
         return response.status_code == 200
     except Exception as e:
         print(e)
@@ -102,11 +104,15 @@ def repos(request, db_connection):
     fixture.create_fork(GIT_REPO, GIT_REPO,
                         repo_name_full=GIT_REPO_WITH_GROUP,
                         repo_group=repo_group_id)
+    fixture.create_fork(SVN_REPO, SVN_REPO,
+                        repo_name_full=SVN_REPO_WITH_GROUP,
+                        repo_group=repo_group_id)
 
     @request.addfinalizer
     def cleanup():
         fixture.destroy_repo(HG_REPO_WITH_GROUP)
         fixture.destroy_repo(GIT_REPO_WITH_GROUP)
+        fixture.destroy_repo(SVN_REPO_WITH_GROUP)
         fixture.destroy_repo_group(repo_group_id)
 
 
@@ -139,11 +145,11 @@ def rc_web_server(
     """
     Run the web server as a subprocess. with its own instance of vcsserver
     """
-    rcweb_port = available_port_factory()
-    log.info('Using rcweb ops test port {}'.format(rcweb_port))
+    rcweb_port: int = available_port_factory()
+    log.info('Using rcweb ops test port %s', rcweb_port)
 
-    vcsserver_port = available_port_factory()
-    log.info('Using vcsserver ops test port {}'.format(vcsserver_port))
+    vcsserver_port: int = available_port_factory()
+    log.info('Using vcsserver ops test port %s', vcsserver_port)
 
     vcs_log = os.path.join(tempfile.gettempdir(), 'rc_op_vcs.log')
     vcsserver_factory(
@@ -303,5 +309,3 @@ def branch_permission_setter(request):
             Session().commit()
 
     return _branch_permissions_setter
-
-
diff --git a/rhodecode/tests/vcs_operations/test_vcs_calls_small_post_buffer.py b/rhodecode/tests/vcs_operations/test_vcs_calls_small_post_buffer.py
index 32963f51..b419622d 100644
--- a/rhodecode/tests/vcs_operations/test_vcs_calls_small_post_buffer.py
+++ b/rhodecode/tests/vcs_operations/test_vcs_calls_small_post_buffer.py
@@ -32,7 +32,7 @@ from rhodecode.lib.vcs.backends.git.repository import GitRepository
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.tests import GIT_REPO
 from rhodecode.tests.vcs_operations import Command
-from .test_vcs_operations import _check_proper_clone, _check_proper_git_push
+from .test_vcs_operations_git import _check_proper_clone, _check_proper_git_push
 
 
 def test_git_clone_with_small_push_buffer(backend_git, rc_web_server, tmpdir):
diff --git a/rhodecode/tests/vcs_operations/test_vcs_operations_git.py b/rhodecode/tests/vcs_operations/test_vcs_operations_git.py
new file mode 100644
index 00000000..d8d85cda
--- /dev/null
+++ b/rhodecode/tests/vcs_operations/test_vcs_operations_git.py
@@ -0,0 +1,199 @@
+
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+"""
+Test suite for making push/pull operations, on specially modified INI files
+
+.. important::
+
+   You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
+   to redirect things to stderr instead of stdout.
+"""
+
+
+import time
+import pytest
+
+from rhodecode.model.db import Repository, UserIpMap
+from rhodecode.model.meta import Session
+from rhodecode.model.repo import RepoModel
+from rhodecode.model.user import UserModel
+from rhodecode.tests import (GIT_REPO, TEST_USER_ADMIN_LOGIN)
+
+
+from rhodecode.tests.vcs_operations import (
+    Command, _check_proper_clone, _check_proper_git_push,
+    _add_files_and_push, GIT_REPO_WITH_GROUP)
+
+
+@pytest.mark.usefixtures("disable_locking", "disable_anonymous_user")
+class TestVCSOperations(object):
+
+    def test_clone_git_repo_by_admin(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
+        cmd = Command('/tmp')
+        stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
+        _check_proper_clone(stdout, stderr, 'git')
+        cmd.assert_returncode_success()
+
+    def test_clone_git_repo_by_admin_with_git_suffix(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
+        cmd = Command('/tmp')
+        stdout, stderr = cmd.execute('git clone', clone_url+".git", tmpdir.strpath)
+        _check_proper_clone(stdout, stderr, 'git')
+        cmd.assert_returncode_success()
+
+    def test_clone_git_repo_by_id_by_admin(self, rc_web_server, tmpdir):
+        repo_id = Repository.get_by_repo_name(GIT_REPO).repo_id
+        clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
+        cmd = Command('/tmp')
+        stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
+        _check_proper_clone(stdout, stderr, 'git')
+        cmd.assert_returncode_success()
+
+    def test_clone_git_repo_with_group_by_admin(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(GIT_REPO_WITH_GROUP)
+        cmd = Command('/tmp')
+        stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
+        _check_proper_clone(stdout, stderr, 'git')
+        cmd.assert_returncode_success()
+
+    def test_clone_git_repo_shallow_by_admin(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
+        cmd = Command('/tmp')
+        stdout, stderr = cmd.execute(
+            'git clone --depth=1', clone_url, tmpdir.strpath)
+
+        assert '' == stdout
+        assert 'Cloning into' in stderr
+        cmd.assert_returncode_success()
+
+
+    def test_clone_wrong_credentials_git(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(GIT_REPO, passwd='bad!')
+        stdout, stderr = Command('/tmp').execute(
+            'git clone', clone_url, tmpdir.strpath)
+        assert 'fatal: Authentication failed' in stderr
+
+    def test_clone_git_dir_as_hg(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
+        stdout, stderr = Command('/tmp').execute(
+            'hg clone', clone_url, tmpdir.strpath)
+        assert 'HTTP Error 404: Not Found' in stderr
+
+    def test_clone_non_existing_path_hg(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url('trololo')
+        stdout, stderr = Command('/tmp').execute(
+            'hg clone', clone_url, tmpdir.strpath)
+        assert 'HTTP Error 404: Not Found' in stderr
+
+    def test_clone_non_existing_path_git(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url('trololo')
+        stdout, stderr = Command('/tmp').execute('git clone', clone_url)
+        assert 'not found' in stderr
+
+    def test_clone_git_with_slashes(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url('//' + GIT_REPO)
+        stdout, stderr = Command('/tmp').execute('git clone', clone_url)
+        assert 'not found' in stderr
+
+    def test_clone_existing_path_git_not_in_database(
+            self, rc_web_server, tmpdir, fs_repo_only):
+        db_name = fs_repo_only('not-in-db-git', repo_type='git')
+        clone_url = rc_web_server.repo_clone_url(db_name)
+        stdout, stderr = Command('/tmp').execute(
+            'git clone', clone_url, tmpdir.strpath)
+        assert 'not found' in stderr
+
+    def test_clone_existing_path_git_not_in_database_different_scm(
+            self, rc_web_server, tmpdir, fs_repo_only):
+        db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
+        clone_url = rc_web_server.repo_clone_url(db_name)
+        stdout, stderr = Command('/tmp').execute(
+            'git clone', clone_url, tmpdir.strpath)
+        assert 'not found' in stderr
+
+    def test_clone_non_existing_store_path_git(self, rc_web_server, tmpdir, user_util):
+        repo = user_util.create_repo(repo_type='git')
+        clone_url = rc_web_server.repo_clone_url(repo.repo_name)
+
+        # Damage repo by removing it's folder
+        RepoModel()._delete_filesystem_repo(repo)
+
+        stdout, stderr = Command('/tmp').execute(
+            'git clone', clone_url, tmpdir.strpath)
+        assert 'not found' in stderr
+
+    def test_push_new_file_git(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
+        stdout, stderr = Command('/tmp').execute(
+            'git clone', clone_url, tmpdir.strpath)
+
+        # commit some stuff into this repo
+        stdout, stderr = _add_files_and_push(
+            'git', tmpdir.strpath, clone_url=clone_url)
+
+        _check_proper_git_push(stdout, stderr)
+
+    def test_push_wrong_credentials_git(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
+        stdout, stderr = Command('/tmp').execute(
+            'git clone', clone_url, tmpdir.strpath)
+
+        push_url = rc_web_server.repo_clone_url(
+            GIT_REPO, user='bad', passwd='name')
+        stdout, stderr = _add_files_and_push(
+            'git', tmpdir.strpath, clone_url=push_url)
+
+        assert 'fatal: Authentication failed' in stderr
+
+    def test_push_back_to_wrong_url_git(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
+        stdout, stderr = Command('/tmp').execute(
+            'git clone', clone_url, tmpdir.strpath)
+
+        stdout, stderr = _add_files_and_push(
+            'git', tmpdir.strpath,
+            clone_url=rc_web_server.repo_clone_url('not-existing'))
+
+        assert 'not found' in stderr
+
+    def test_ip_restriction_git(self, rc_web_server, tmpdir):
+        user_model = UserModel()
+        try:
+            user_model.add_extra_ip(TEST_USER_ADMIN_LOGIN, '10.10.10.10/32')
+            Session().commit()
+            time.sleep(2)
+            clone_url = rc_web_server.repo_clone_url(GIT_REPO)
+            stdout, stderr = Command('/tmp').execute(
+                'git clone', clone_url, tmpdir.strpath)
+            msg = "The requested URL returned error: 403"
+            assert msg in stderr
+        finally:
+            # release IP restrictions
+            for ip in UserIpMap.getAll():
+                UserIpMap.delete(ip.ip_id)
+            Session().commit()
+
+        time.sleep(2)
+
+        cmd = Command('/tmp')
+        stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
+        cmd.assert_returncode_success()
+        _check_proper_clone(stdout, stderr, 'git')
diff --git a/rhodecode/tests/vcs_operations/test_vcs_operations.py b/rhodecode/tests/vcs_operations/test_vcs_operations_hg.py
similarity index 58%
rename from rhodecode/tests/vcs_operations/test_vcs_operations.py
rename to rhodecode/tests/vcs_operations/test_vcs_operations_hg.py
index bf164434..74d8e035 100644
--- a/rhodecode/tests/vcs_operations/test_vcs_operations.py
+++ b/rhodecode/tests/vcs_operations/test_vcs_operations_hg.py
@@ -28,22 +28,18 @@ Test suite for making push/pull operations, on specially modified INI files
 
 
 import time
-import logging
 
 import pytest
 
 from rhodecode.lib import rc_cache
-from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.db import Repository, UserIpMap, CacheKey
 from rhodecode.model.meta import Session
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.user import UserModel
 from rhodecode.tests import (GIT_REPO, HG_REPO, TEST_USER_ADMIN_LOGIN)
-from rhodecode.tests.utils import assert_message_in_log
 
 from rhodecode.tests.vcs_operations import (
-    Command, _check_proper_clone, _check_proper_git_push,
-    _add_files_and_push, HG_REPO_WITH_GROUP, GIT_REPO_WITH_GROUP)
+    Command, _check_proper_clone, _add_files_and_push, HG_REPO_WITH_GROUP)
 
 
 @pytest.mark.usefixtures("disable_locking", "disable_anonymous_user")
@@ -69,20 +65,6 @@ class TestVCSOperations(object):
         assert 'transferred 1.' in stdout
         assert '114 files updated,' in stdout
 
-    def test_clone_git_repo_by_admin(self, rc_web_server, tmpdir):
-        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
-        cmd = Command('/tmp')
-        stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
-        _check_proper_clone(stdout, stderr, 'git')
-        cmd.assert_returncode_success()
-
-    def test_clone_git_repo_by_admin_with_git_suffix(self, rc_web_server, tmpdir):
-        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
-        cmd = Command('/tmp')
-        stdout, stderr = cmd.execute('git clone', clone_url+".git", tmpdir.strpath)
-        _check_proper_clone(stdout, stderr, 'git')
-        cmd.assert_returncode_success()
-
     def test_clone_hg_repo_by_id_by_admin(self, rc_web_server, tmpdir):
         repo_id = Repository.get_by_repo_name(HG_REPO).repo_id
         clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
@@ -90,82 +72,35 @@ class TestVCSOperations(object):
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
-    def test_clone_git_repo_by_id_by_admin(self, rc_web_server, tmpdir):
-        repo_id = Repository.get_by_repo_name(GIT_REPO).repo_id
-        clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
-        cmd = Command('/tmp')
-        stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
-        _check_proper_clone(stdout, stderr, 'git')
-        cmd.assert_returncode_success()
-
     def test_clone_hg_repo_with_group_by_admin(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO_WITH_GROUP)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
 
-    def test_clone_git_repo_with_group_by_admin(self, rc_web_server, tmpdir):
-        clone_url = rc_web_server.repo_clone_url(GIT_REPO_WITH_GROUP)
-        cmd = Command('/tmp')
-        stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
-        _check_proper_clone(stdout, stderr, 'git')
-        cmd.assert_returncode_success()
-
-    def test_clone_git_repo_shallow_by_admin(self, rc_web_server, tmpdir):
-        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
-        cmd = Command('/tmp')
-        stdout, stderr = cmd.execute(
-            'git clone --depth=1', clone_url, tmpdir.strpath)
-
-        assert '' == stdout
-        assert 'Cloning into' in stderr
-        cmd.assert_returncode_success()
-
     def test_clone_wrong_credentials_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO, passwd='bad!')
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'abort: authorization failed' in stderr
 
-    def test_clone_wrong_credentials_git(self, rc_web_server, tmpdir):
-        clone_url = rc_web_server.repo_clone_url(GIT_REPO, passwd='bad!')
-        stdout, stderr = Command('/tmp').execute(
-            'git clone', clone_url, tmpdir.strpath)
-        assert 'fatal: Authentication failed' in stderr
-
     def test_clone_git_dir_as_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
-    def test_clone_hg_repo_as_git(self, rc_web_server, tmpdir):
-        clone_url = rc_web_server.repo_clone_url(HG_REPO)
-        stdout, stderr = Command('/tmp').execute(
-            'git clone', clone_url, tmpdir.strpath)
-        assert 'not found' in stderr
-
     def test_clone_non_existing_path_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url('trololo')
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
-    def test_clone_non_existing_path_git(self, rc_web_server, tmpdir):
-        clone_url = rc_web_server.repo_clone_url('trololo')
-        stdout, stderr = Command('/tmp').execute('git clone', clone_url)
-        assert 'not found' in stderr
-
     def test_clone_hg_with_slashes(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url('//' + HG_REPO)
         stdout, stderr = Command('/tmp').execute('hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
-    def test_clone_git_with_slashes(self, rc_web_server, tmpdir):
-        clone_url = rc_web_server.repo_clone_url('//' + GIT_REPO)
-        stdout, stderr = Command('/tmp').execute('git clone', clone_url)
-        assert 'not found' in stderr
-
     def test_clone_existing_path_hg_not_in_database(
             self, rc_web_server, tmpdir, fs_repo_only):
 
@@ -175,14 +110,6 @@ class TestVCSOperations(object):
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
-    def test_clone_existing_path_git_not_in_database(
-            self, rc_web_server, tmpdir, fs_repo_only):
-        db_name = fs_repo_only('not-in-db-git', repo_type='git')
-        clone_url = rc_web_server.repo_clone_url(db_name)
-        stdout, stderr = Command('/tmp').execute(
-            'git clone', clone_url, tmpdir.strpath)
-        assert 'not found' in stderr
-
     def test_clone_existing_path_hg_not_in_database_different_scm(
             self, rc_web_server, tmpdir, fs_repo_only):
         db_name = fs_repo_only('not-in-db-git', repo_type='git')
@@ -191,14 +118,6 @@ class TestVCSOperations(object):
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
-    def test_clone_existing_path_git_not_in_database_different_scm(
-            self, rc_web_server, tmpdir, fs_repo_only):
-        db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
-        clone_url = rc_web_server.repo_clone_url(db_name)
-        stdout, stderr = Command('/tmp').execute(
-            'git clone', clone_url, tmpdir.strpath)
-        assert 'not found' in stderr
-
     def test_clone_non_existing_store_path_hg(self, rc_web_server, tmpdir, user_util):
         repo = user_util.create_repo()
         clone_url = rc_web_server.repo_clone_url(repo.repo_name)
@@ -210,17 +129,6 @@ class TestVCSOperations(object):
             'hg clone', clone_url, tmpdir.strpath)
         assert 'HTTP Error 404: Not Found' in stderr
 
-    def test_clone_non_existing_store_path_git(self, rc_web_server, tmpdir, user_util):
-        repo = user_util.create_repo(repo_type='git')
-        clone_url = rc_web_server.repo_clone_url(repo.repo_name)
-
-        # Damage repo by removing it's folder
-        RepoModel()._delete_filesystem_repo(repo)
-
-        stdout, stderr = Command('/tmp').execute(
-            'git clone', clone_url, tmpdir.strpath)
-        assert 'not found' in stderr
-
     def test_push_new_file_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
@@ -232,17 +140,6 @@ class TestVCSOperations(object):
         assert 'pushing to' in stdout
         assert 'size summary' in stdout
 
-    def test_push_new_file_git(self, rc_web_server, tmpdir):
-        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
-        stdout, stderr = Command('/tmp').execute(
-            'git clone', clone_url, tmpdir.strpath)
-
-        # commit some stuff into this repo
-        stdout, stderr = _add_files_and_push(
-            'git', tmpdir.strpath, clone_url=clone_url)
-
-        _check_proper_git_push(stdout, stderr)
-
     def test_push_invalidates_cache(self, rc_web_server, tmpdir):
         hg_repo = Repository.get_by_repo_name(HG_REPO)
 
@@ -295,18 +192,6 @@ class TestVCSOperations(object):
 
         assert 'abort: authorization failed' in stderr
 
-    def test_push_wrong_credentials_git(self, rc_web_server, tmpdir):
-        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
-        stdout, stderr = Command('/tmp').execute(
-            'git clone', clone_url, tmpdir.strpath)
-
-        push_url = rc_web_server.repo_clone_url(
-            GIT_REPO, user='bad', passwd='name')
-        stdout, stderr = _add_files_and_push(
-            'git', tmpdir.strpath, clone_url=push_url)
-
-        assert 'fatal: Authentication failed' in stderr
-
     def test_push_back_to_wrong_url_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
@@ -318,17 +203,6 @@ class TestVCSOperations(object):
 
         assert 'HTTP Error 404: Not Found' in stderr
 
-    def test_push_back_to_wrong_url_git(self, rc_web_server, tmpdir):
-        clone_url = rc_web_server.repo_clone_url(GIT_REPO)
-        stdout, stderr = Command('/tmp').execute(
-            'git clone', clone_url, tmpdir.strpath)
-
-        stdout, stderr = _add_files_and_push(
-            'git', tmpdir.strpath,
-            clone_url=rc_web_server.repo_clone_url('not-existing'))
-
-        assert 'not found' in stderr
-
     def test_ip_restriction_hg(self, rc_web_server, tmpdir):
         user_model = UserModel()
         try:
@@ -350,27 +224,3 @@ class TestVCSOperations(object):
         stdout, stderr = Command('/tmp').execute(
             'hg clone', clone_url, tmpdir.strpath)
         _check_proper_clone(stdout, stderr, 'hg')
-
-    def test_ip_restriction_git(self, rc_web_server, tmpdir):
-        user_model = UserModel()
-        try:
-            user_model.add_extra_ip(TEST_USER_ADMIN_LOGIN, '10.10.10.10/32')
-            Session().commit()
-            time.sleep(2)
-            clone_url = rc_web_server.repo_clone_url(GIT_REPO)
-            stdout, stderr = Command('/tmp').execute(
-                'git clone', clone_url, tmpdir.strpath)
-            msg = "The requested URL returned error: 403"
-            assert msg in stderr
-        finally:
-            # release IP restrictions
-            for ip in UserIpMap.getAll():
-                UserIpMap.delete(ip.ip_id)
-            Session().commit()
-
-        time.sleep(2)
-
-        cmd = Command('/tmp')
-        stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath)
-        cmd.assert_returncode_success()
-        _check_proper_clone(stdout, stderr, 'git')
diff --git a/rhodecode/tests/vcs_operations/test_vcs_operations_svn.py b/rhodecode/tests/vcs_operations/test_vcs_operations_svn.py
new file mode 100644
index 00000000..0442084d
--- /dev/null
+++ b/rhodecode/tests/vcs_operations/test_vcs_operations_svn.py
@@ -0,0 +1,197 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+"""
+Test suite for making push/pull operations, on specially modified INI files
+
+.. important::
+
+   You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
+   to redirect things to stderr instead of stdout.
+"""
+
+
+import time
+import pytest
+
+from rhodecode.model.db import Repository, UserIpMap
+from rhodecode.model.meta import Session
+from rhodecode.model.repo import RepoModel
+from rhodecode.model.user import UserModel
+from rhodecode.tests import (SVN_REPO, TEST_USER_ADMIN_LOGIN)
+
+
+from rhodecode.tests.vcs_operations import (
+    Command, _check_proper_clone, _check_proper_svn_push,
+    _add_files_and_push, SVN_REPO_WITH_GROUP)
+
+
+@pytest.mark.usefixtures("disable_locking", "disable_anonymous_user")
+class TestVCSOperations(object):
+
+    def test_clone_svn_repo_by_admin(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(SVN_REPO)
+        username, password = rc_web_server.repo_clone_credentials()
+
+        cmd = Command('/tmp')
+
+        auth = f'--non-interactive --username={username} --password={password}'
+        stdout, stderr = cmd.execute(f'svn checkout {auth}', clone_url, tmpdir.strpath)
+        _check_proper_clone(stdout, stderr, 'svn')
+        cmd.assert_returncode_success()
+
+    def test_clone_svn_repo_by_id_by_admin(self, rc_web_server, tmpdir):
+        repo_id = Repository.get_by_repo_name(SVN_REPO).repo_id
+        username, password = rc_web_server.repo_clone_credentials()
+
+        clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
+        cmd = Command('/tmp')
+        auth = f'--non-interactive --username={username} --password={password}'
+        stdout, stderr = cmd.execute(f'svn checkout {auth}', clone_url, tmpdir.strpath)
+        _check_proper_clone(stdout, stderr, 'svn')
+        cmd.assert_returncode_success()
+
+    def test_clone_svn_repo_with_group_by_admin(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(SVN_REPO_WITH_GROUP)
+        username, password = rc_web_server.repo_clone_credentials()
+
+        cmd = Command('/tmp')
+        auth = f'--non-interactive --username={username} --password={password}'
+        stdout, stderr = cmd.execute(f'svn checkout {auth}', clone_url, tmpdir.strpath)
+        _check_proper_clone(stdout, stderr, 'svn')
+        cmd.assert_returncode_success()
+
+    def test_clone_wrong_credentials_svn(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(SVN_REPO)
+        username, password = rc_web_server.repo_clone_credentials()
+        password = 'bad-password'
+
+        auth = f'--non-interactive --username={username} --password={password}'
+        stdout, stderr = Command('/tmp').execute(
+            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+        assert 'fatal: Authentication failed' in stderr
+
+    def test_clone_svn_with_slashes(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url('//' + SVN_REPO)
+        stdout, stderr = Command('/tmp').execute('svn checkout', clone_url)
+        assert 'not found' in stderr
+
+    def test_clone_existing_path_svn_not_in_database(
+            self, rc_web_server, tmpdir, fs_repo_only):
+        db_name = fs_repo_only('not-in-db-git', repo_type='git')
+        clone_url = rc_web_server.repo_clone_url(db_name)
+        username, password = '', ''
+        auth = f'--non-interactive --username={username} --password={password}'
+
+        stdout, stderr = Command('/tmp').execute(
+            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+        assert 'not found' in stderr
+
+    def test_clone_existing_path_svn_not_in_database_different_scm(
+            self, rc_web_server, tmpdir, fs_repo_only):
+        db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
+        clone_url = rc_web_server.repo_clone_url(db_name)
+
+        username, password = '', ''
+        auth = f'--non-interactive --username={username} --password={password}'
+        stdout, stderr = Command('/tmp').execute(
+            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+        assert 'not found' in stderr
+
+    def test_clone_non_existing_store_path_svn(self, rc_web_server, tmpdir, user_util):
+        repo = user_util.create_repo(repo_type='git')
+        clone_url = rc_web_server.repo_clone_url(repo.repo_name)
+
+        # Damage repo by removing it's folder
+        RepoModel()._delete_filesystem_repo(repo)
+
+        username, password = '', ''
+        auth = f'--non-interactive --username={username} --password={password}'
+        stdout, stderr = Command('/tmp').execute(
+            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+        assert 'not found' in stderr
+
+    def test_push_new_file_svn(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(SVN_REPO)
+        username, password = '', ''
+        auth = f'--non-interactive --username={username} --password={password}'
+
+        stdout, stderr = Command('/tmp').execute(
+            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+
+        # commit some stuff into this repo
+        stdout, stderr = _add_files_and_push(
+            'svn', tmpdir.strpath, clone_url=clone_url)
+
+        _check_proper_svn_push(stdout, stderr)
+
+    def test_push_wrong_credentials_svn(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(SVN_REPO)
+
+        username, password = '', ''
+        auth = f'--non-interactive --username={username} --password={password}'
+        stdout, stderr = Command('/tmp').execute(
+            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+
+        push_url = rc_web_server.repo_clone_url(
+            SVN_REPO, user='bad', passwd='name')
+        stdout, stderr = _add_files_and_push(
+            'svn', tmpdir.strpath, clone_url=push_url)
+
+        assert 'fatal: Authentication failed' in stderr
+
+    def test_push_back_to_wrong_url_svn(self, rc_web_server, tmpdir):
+        clone_url = rc_web_server.repo_clone_url(SVN_REPO)
+        username, password = '', ''
+        auth = f'--non-interactive --username={username} --password={password}'
+        Command('/tmp').execute(
+            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+
+        stdout, stderr = _add_files_and_push(
+            'svn', tmpdir.strpath,
+            clone_url=rc_web_server.repo_clone_url('not-existing'))
+
+        assert 'not found' in stderr
+
+    def test_ip_restriction_svn(self, rc_web_server, tmpdir):
+        user_model = UserModel()
+        username, password = '', ''
+        auth = f'--non-interactive --username={username} --password={password}'
+
+        try:
+            user_model.add_extra_ip(TEST_USER_ADMIN_LOGIN, '10.10.10.10/32')
+            Session().commit()
+            time.sleep(2)
+            clone_url = rc_web_server.repo_clone_url(SVN_REPO)
+
+            stdout, stderr = Command('/tmp').execute(
+                f'svn checkout {auth}', clone_url, tmpdir.strpath)
+            msg = "The requested URL returned error: 403"
+            assert msg in stderr
+        finally:
+            # release IP restrictions
+            for ip in UserIpMap.getAll():
+                UserIpMap.delete(ip.ip_id)
+            Session().commit()
+
+        time.sleep(2)
+
+        cmd = Command('/tmp')
+        stdout, stderr = cmd.execute(f'svn checkout {auth}', clone_url, tmpdir.strpath)
+        cmd.assert_returncode_success()
+        _check_proper_clone(stdout, stderr, 'svn')
diff --git a/rhodecode/tests/vcs_operations/test_vcs_operations_tag_push.py b/rhodecode/tests/vcs_operations/test_vcs_operations_tag_push.py
index 8ce8d166..eed4aefd 100644
--- a/rhodecode/tests/vcs_operations/test_vcs_operations_tag_push.py
+++ b/rhodecode/tests/vcs_operations/test_vcs_operations_tag_push.py
@@ -42,6 +42,7 @@ connection_available = pytest.mark.skipif(
     "enable_webhook_push_integration")
 class TestVCSOperationsOnCustomIniConfig(object):
 
+    @connection_available
     def test_push_tag_with_commit_hg(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(HG_REPO)
         stdout, stderr = Command('/tmp').execute(
@@ -56,6 +57,7 @@ class TestVCSOperationsOnCustomIniConfig(object):
         assert 'ERROR' not in rc_log
         assert "{'name': 'v1.0.0'," in rc_log
 
+    @connection_available
     def test_push_tag_with_commit_git(
             self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
@@ -71,6 +73,7 @@ class TestVCSOperationsOnCustomIniConfig(object):
         assert 'ERROR' not in rc_log
         assert "{'name': 'v1.0.0'," in rc_log
 
+    @connection_available
     def test_push_tag_with_no_commit_git(
             self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(GIT_REPO)
diff --git a/rhodecode/tests/vcsserver_http.ini b/rhodecode/tests/vcsserver_http.ini
index 461cf14c..62c09696 100644
--- a/rhodecode/tests/vcsserver_http.ini
+++ b/rhodecode/tests/vcsserver_http.ini
@@ -7,7 +7,7 @@
 [server:main]
 ; COMMON HOST/IP CONFIG
 host = 127.0.0.1
-port = 9900
+port = 10010
 
 
 ; ###########################
@@ -22,6 +22,17 @@ use = egg:gunicorn#main
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
+; Each option in the app:main can be override by an environmental variable
+;
+;To override an option:
+;
+;RC_<KeyName>
+;Everything should be uppercase, . and - should be replaced by _.
+;For example, if you have these configuration settings:
+;rc_cache.repo_object.backend = foo
+;can be overridden by
+;export RC_CACHE_REPO_OBJECT_BACKEND=foo
+
 use = egg:rhodecode-vcsserver
 
 ; Pyramid default locales, we need this to be set
@@ -30,11 +41,15 @@ pyramid.default_locale_name = en
 ; default locale used by VCS systems
 locale = en_US.UTF-8
 
-; path to binaries for vcsserver, it should be set by the installer
-; at installation time, e.g /home/user/vcsserver-1/profile/bin
-; it can also be a path to nix-build output in case of development
+; path to binaries (hg,git,svn) for vcsserver, it should be set by the installer
+; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
+; or /usr/local/bin/rhodecode_bin/vcs_bin
 core.binary_dir =
 
+; Redis connection settings for svn integrations logic
+; This connection string needs to be the same on ce and vcsserver
+vcs.svn.redis_conn = redis://redis:6379/0
+
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
@@ -52,14 +67,14 @@ cache_dir = %(here)s/data
 ; ***************************************
 
 ; `repo_object` cache settings for vcs methods for repositories
-rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
+#rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
 
 ; cache auto-expires after N seconds
 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
-rc_cache.repo_object.expiration_time = 2592000
+#rc_cache.repo_object.expiration_time = 2592000
 
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
-#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
+#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
 
 ; ***********************************************************
 ; `repo_object` cache with redis backend
@@ -83,19 +98,32 @@ rc_cache.repo_object.expiration_time = 2592000
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.repo_object.arguments.distributed_lock = true
 
-# legacy cache regions, please don't change
-beaker.cache.regions = repo_object
-beaker.cache.repo_object.type = memorylru
-beaker.cache.repo_object.max_items = 100
-# cache auto-expires after N seconds
-beaker.cache.repo_object.expire = 300
-beaker.cache.repo_object.enabled = true
+; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
+#rc_cache.repo_object.arguments.lock_auto_renewal = true
 
+; Statsd client config, this is used to send metrics to statsd
+; We recommend setting statsd_exported and scrape them using Promethues
+#statsd.enabled = false
+#statsd.statsd_host = 0.0.0.0
+#statsd.statsd_port = 8125
+#statsd.statsd_prefix =
+#statsd.statsd_ipv6 = false
 
+; configure logging automatically at server startup set to false
+; to use the below custom logging config.
+; RC_LOGGING_FORMATTER
+; RC_LOGGING_LEVEL
+; env variables can control the settings for logging in case of autoconfigure
+
+#logging.autoconfigure = true
+
+; specify your own custom logging config file to configure logging
+#logging.logging_conf_file = /path/to/custom_logging.ini
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
+
 [loggers]
 keys = root, vcsserver
 
@@ -103,7 +131,7 @@ keys = root, vcsserver
 keys = console
 
 [formatters]
-keys = generic
+keys = generic, json
 
 ; #######
 ; LOGGERS
@@ -113,12 +141,11 @@ level = NOTSET
 handlers = console
 
 [logger_vcsserver]
-level = DEBUG
+level = INFO
 handlers =
 qualname = vcsserver
 propagate = 1
 
-
 ; ########
 ; HANDLERS
 ; ########
@@ -127,6 +154,8 @@ propagate = 1
 class = StreamHandler
 args = (sys.stderr, )
 level = DEBUG
+; To enable JSON formatted logs replace 'generic' with 'json'
+; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 ; ##########
@@ -136,3 +165,7 @@ formatter = generic
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_json]
+format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
+class = vcsserver.lib._vendor.jsonlogger.JsonFormatter

From 1790b4d3d2aba2e88e60ed390872e56c46170306 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcin=20Ku=C5=BAmi=C5=84ski?= <marcin@rhodecode.com>
Date: Fri, 12 Jul 2024 16:41:59 +0000
Subject: [PATCH 140/150] Fixed problem with step param not initializing for 0
 step migration

---
 rhodecode/lib/db_manage.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/rhodecode/lib/db_manage.py b/rhodecode/lib/db_manage.py
index f2085d2e..e1d2a152 100644
--- a/rhodecode/lib/db_manage.py
+++ b/rhodecode/lib/db_manage.py
@@ -201,7 +201,7 @@ class DbManage(object):
                f'version {curr_version} to version {__dbversion__}')
 
         # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
-        _step = None
+        final_step = 'latest'
         for step in upgrade_steps:
             notify(f'performing upgrade step {step}')
             time.sleep(0.5)
@@ -210,10 +210,10 @@ class DbManage(object):
             self.sa.rollback()
             notify(f'schema upgrade for step {step} completed')
 
-            _step = step
+            final_step = step
 
         self.run_post_migration_tasks()
-        notify(f'upgrade to version {step} successful')
+        notify(f'upgrade to version {final_step} successful')
 
     def fix_repo_paths(self):
         """

From 1d566329850bfe1613f658526231d6a40e5705c3 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 16 Jul 2024 14:02:38 +0200
Subject: [PATCH 141/150] ini: cleanup

---
 configs/development.ini | 2 +-
 configs/production.ini  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/configs/development.ini b/configs/development.ini
index 5231b101..b518afca 100644
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -36,7 +36,7 @@ port = 10020
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
+; run with gunicorn  --config gunicorn_conf.py --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
diff --git a/configs/production.ini b/configs/production.ini
index aa053c96..948c7c0c 100644
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -36,7 +36,7 @@ port = 10020
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
+; run with gunicorn  --config gunicorn_conf.py --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main

From 6f7cbc64a7f62d7f61eadd8379bc122eea8d3700 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 16 Jul 2024 21:53:10 +0200
Subject: [PATCH 142/150] fix: startup, report nicer error in case DB is
 damaged or empty when starting services

---
 rhodecode/config/utils.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/rhodecode/config/utils.py b/rhodecode/config/utils.py
index cdb42632..aec2cfc2 100644
--- a/rhodecode/config/utils.py
+++ b/rhodecode/config/utils.py
@@ -102,6 +102,9 @@ def get_default_user_id():
         result = session.execute(text(
             "SELECT user_id from users where username = :uname"
         ), {'uname': DEFAULT_USER})
-        user_id = result.first()[0]
+        user = result.first()
+        if not user:
+            raise ValueError('Unable to retrieve default user data from DB')
+        user_id = user[0]
 
     return user_id

From 47e6265c093165ca028516c8912048e6b18e541d Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Tue, 16 Jul 2024 21:53:36 +0200
Subject: [PATCH 143/150] chore(cleanups): use single consistent escape for
 descriptions

---
 rhodecode/model/db.py | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index 7956e0bf..40919ba2 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -128,6 +128,11 @@ def _hash_key(k):
     return sha1_safe(k)
 
 
+def description_escaper(desc):
+    from rhodecode.lib import helpers as h
+    return h.html_escape(desc)
+
+
 def in_filter_generator(qry, items, limit=500):
     """
     Splits IN() into multiple with OR
@@ -663,16 +668,14 @@ class User(Base, BaseModel):
 
     @hybrid_property
     def first_name(self):
-        from rhodecode.lib import helpers as h
         if self.name:
-            return h.escape(self.name)
+            return description_escaper(self.name)
         return self.name
 
     @hybrid_property
     def last_name(self):
-        from rhodecode.lib import helpers as h
         if self.lastname:
-            return h.escape(self.lastname)
+            return description_escaper(self.lastname)
         return self.lastname
 
     @hybrid_property
@@ -1368,8 +1371,7 @@ class UserApiKeys(Base, BaseModel):
 
     @hybrid_property
     def description_safe(self):
-        from rhodecode.lib import helpers as h
-        return h.escape(self.description)
+        return description_escaper(self.description)
 
     @property
     def expired(self):
@@ -1472,8 +1474,7 @@ class UserIpMap(Base, BaseModel):
 
     @hybrid_property
     def description_safe(self):
-        from rhodecode.lib import helpers as h
-        return h.escape(self.description)
+        return description_escaper(self.description)
 
     @classmethod
     def _get_ip_range(cls, ip_addr):
@@ -1611,8 +1612,7 @@ class UserGroup(Base, BaseModel):
 
     @hybrid_property
     def description_safe(self):
-        from rhodecode.lib import helpers as h
-        return h.escape(self.user_group_description)
+        return description_escaper(self.user_group_description)
 
     @hybrid_property
     def group_data(self):
@@ -1956,8 +1956,7 @@ class Repository(Base, BaseModel):
 
     @hybrid_property
     def description_safe(self):
-        from rhodecode.lib import helpers as h
-        return h.escape(self.description)
+        return description_escaper(self.description)
 
     @hybrid_property
     def landing_rev(self):
@@ -2915,8 +2914,7 @@ class RepoGroup(Base, BaseModel):
 
     @hybrid_property
     def description_safe(self):
-        from rhodecode.lib import helpers as h
-        return h.escape(self.group_description)
+        return description_escaper(self.group_description)
 
     @classmethod
     def hash_repo_group_name(cls, repo_group_name):
@@ -4418,8 +4416,7 @@ class _PullRequestBase(BaseModel):
 
     @hybrid_property
     def description_safe(self):
-        from rhodecode.lib import helpers as h
-        return h.escape(self.description)
+        return description_escaper(self.description)
 
     @hybrid_property
     def revisions(self):
@@ -5027,8 +5024,7 @@ class Gist(Base, BaseModel):
 
     @hybrid_property
     def description_safe(self):
-        from rhodecode.lib import helpers as h
-        return h.escape(self.gist_description)
+        return description_escaper(self.gist_description)
 
     @classmethod
     def get_or_404(cls, id_):

From f6cd5ffe899f31407726f2b497a2d725b31539d1 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 17 Jul 2024 09:12:41 +0200
Subject: [PATCH 144/150] docs: updated release notes

---
 docs/release-notes/release-notes-5.1.0.rst | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/release-notes/release-notes-5.1.0.rst b/docs/release-notes/release-notes-5.1.0.rst
index 9e7e5a4f..ea34f15a 100644
--- a/docs/release-notes/release-notes-5.1.0.rst
+++ b/docs/release-notes/release-notes-5.1.0.rst
@@ -17,6 +17,7 @@ New Features
 - Ability to replace a file using web UI. Now one can replace an existing file from the web-ui.
 - GIT LFS Sync automation. Remote push/pull commands now can also sync GIT LFS objects.
 - Added ability to remove or close branches from the web ui
+- Added ability to delete a branch automatically after merging PR for git repositories
 - Added support for S3 based archive_cache based that allows storing cached archives in S3 compatible object store.
 
 
@@ -26,6 +27,9 @@ General
 - Upgraded all dependency libraries to their latest available versions
 - Repository storage is no longer controlled via DB settings, but .ini file. This allows easier automated deployments.
 - Bumped mercurial to 6.7.4
+- Mercurial: enable httppostarguments for better support of large repositories with lots of heads.
+- Added explicit db-migrate step to update hooks for 5.X release.
+
 
 Security
 ^^^^^^^^
@@ -43,7 +47,10 @@ Performance
 Fixes
 ^^^^^
 
-- Zip archive download breaks when a gitmodules file is present
+- Archives: Zip archive download breaks when a gitmodules file is present
+- Branch permissions: fixed bug preventing to specify own rules from 4.X install
+- SVN: refactored svn events, thus fixing support for it in dockerized env
+- Fixed empty server url in PR link after push from cli
 
 
 Upgrade notes

From 53e0af18ff24664f8d59526e741d47cf7a55d170 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 17 Jul 2024 10:16:28 +0200
Subject: [PATCH 145/150] fix: escaper, use h.escape instead of html_escape
 since it's faster and correct

---
 rhodecode/model/db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
index 40919ba2..4801e70d 100644
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -130,7 +130,7 @@ def _hash_key(k):
 
 def description_escaper(desc):
     from rhodecode.lib import helpers as h
-    return h.html_escape(desc)
+    return h.escape(desc)
 
 
 def in_filter_generator(qry, items, limit=500):

From 50c8c98c7f9b7d66d33e6c6f7beabb442d4d8518 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 17 Jul 2024 10:19:00 +0200
Subject: [PATCH 146/150] fix(js-routes): fixed bad urls in jsroutes

---
 rhodecode/public/js/rhodecode/routes.js | 44 +++++++++++++++++++++----
 1 file changed, 37 insertions(+), 7 deletions(-)

diff --git a/rhodecode/public/js/rhodecode/routes.js b/rhodecode/public/js/rhodecode/routes.js
index 44d89623..e224b7c5 100644
--- a/rhodecode/public/js/rhodecode/routes.js
+++ b/rhodecode/public/js/rhodecode/routes.js
@@ -12,14 +12,16 @@
  ******************************************************************************/
 function registerRCRoutes() {
     // routes registration
-    pyroutes.register('admin_artifacts', '/_admin/_admin/artifacts', []);
-    pyroutes.register('admin_artifacts_delete', '/_admin/_admin/artifacts/%(uid)s/delete', ['uid']);
-    pyroutes.register('admin_artifacts_show_all', '/_admin/_admin/artifacts', []);
-    pyroutes.register('admin_artifacts_show_info', '/_admin/_admin/artifacts/%(uid)s', ['uid']);
-    pyroutes.register('admin_artifacts_update', '/_admin/_admin/artifacts/%(uid)s/update', ['uid']);
+    pyroutes.register('admin_artifacts', '/_admin/artifacts', []);
+    pyroutes.register('admin_artifacts_data', '/_admin/artifacts-data', []);
+    pyroutes.register('admin_artifacts_delete', '/_admin/artifacts/%(uid)s/delete', ['uid']);
+    pyroutes.register('admin_artifacts_show_all', '/_admin/artifacts', []);
+    pyroutes.register('admin_artifacts_show_info', '/_admin/artifacts/%(uid)s', ['uid']);
+    pyroutes.register('admin_artifacts_update', '/_admin/artifacts/%(uid)s/update', ['uid']);
     pyroutes.register('admin_audit_log_entry', '/_admin/audit_logs/%(audit_log_id)s', ['audit_log_id']);
     pyroutes.register('admin_audit_logs', '/_admin/audit_logs', []);
-    pyroutes.register('admin_automation', '/_admin/_admin/automation', []);
+    pyroutes.register('admin_automation', '/_admin/automation', []);
+    pyroutes.register('admin_automation_update', '/_admin/automation/%(entry_id)s/update', ['entry_id']);
     pyroutes.register('admin_defaults_repositories', '/_admin/defaults/repositories', []);
     pyroutes.register('admin_defaults_repositories_update', '/_admin/defaults/repositories/update', []);
     pyroutes.register('admin_home', '/_admin', []);
@@ -27,6 +29,7 @@ function registerRCRoutes() {
     pyroutes.register('admin_permissions_application_update', '/_admin/permissions/application/update', []);
     pyroutes.register('admin_permissions_auth_token_access', '/_admin/permissions/auth_token_access', []);
     pyroutes.register('admin_permissions_branch', '/_admin/permissions/branch', []);
+    pyroutes.register('admin_permissions_branch_update', '/_admin/permissions/branch/update', []);
     pyroutes.register('admin_permissions_global', '/_admin/permissions/global', []);
     pyroutes.register('admin_permissions_global_update', '/_admin/permissions/global/update', []);
     pyroutes.register('admin_permissions_ips', '/_admin/permissions/ips', []);
@@ -36,7 +39,8 @@ function registerRCRoutes() {
     pyroutes.register('admin_permissions_ssh_keys', '/_admin/permissions/ssh_keys', []);
     pyroutes.register('admin_permissions_ssh_keys_data', '/_admin/permissions/ssh_keys/data', []);
     pyroutes.register('admin_permissions_ssh_keys_update', '/_admin/permissions/ssh_keys/update', []);
-    pyroutes.register('admin_scheduler', '/_admin/_admin/scheduler', []);
+    pyroutes.register('admin_scheduler', '/_admin/scheduler', []);
+    pyroutes.register('admin_scheduler_show_tasks', '/_admin/scheduler/_tasks', []);
     pyroutes.register('admin_settings', '/_admin/settings', []);
     pyroutes.register('admin_settings_email', '/_admin/settings/email', []);
     pyroutes.register('admin_settings_email_update', '/_admin/settings/email/update', []);
@@ -55,6 +59,8 @@ function registerRCRoutes() {
     pyroutes.register('admin_settings_issuetracker_update', '/_admin/settings/issue-tracker/update', []);
     pyroutes.register('admin_settings_labs', '/_admin/settings/labs', []);
     pyroutes.register('admin_settings_labs_update', '/_admin/settings/labs/update', []);
+    pyroutes.register('admin_settings_license', '/_admin/settings/license', []);
+    pyroutes.register('admin_settings_license_unlock', '/_admin/settings/license_unlock', []);
     pyroutes.register('admin_settings_mapping', '/_admin/settings/mapping', []);
     pyroutes.register('admin_settings_mapping_update', '/_admin/settings/mapping/update', []);
     pyroutes.register('admin_settings_open_source', '/_admin/settings/open_source', []);
@@ -62,6 +68,12 @@ function registerRCRoutes() {
     pyroutes.register('admin_settings_process_management_data', '/_admin/settings/process_management/data', []);
     pyroutes.register('admin_settings_process_management_master_signal', '/_admin/settings/process_management/master_signal', []);
     pyroutes.register('admin_settings_process_management_signal', '/_admin/settings/process_management/signal', []);
+    pyroutes.register('admin_settings_scheduler_create', '/_admin/scheduler/create', []);
+    pyroutes.register('admin_settings_scheduler_delete', '/_admin/scheduler/%(schedule_id)s/delete', ['schedule_id']);
+    pyroutes.register('admin_settings_scheduler_edit', '/_admin/scheduler/%(schedule_id)s', ['schedule_id']);
+    pyroutes.register('admin_settings_scheduler_execute', '/_admin/scheduler/%(schedule_id)s/execute', ['schedule_id']);
+    pyroutes.register('admin_settings_scheduler_new', '/_admin/scheduler/new', []);
+    pyroutes.register('admin_settings_scheduler_update', '/_admin/scheduler/%(schedule_id)s/update', ['schedule_id']);
     pyroutes.register('admin_settings_search', '/_admin/settings/search', []);
     pyroutes.register('admin_settings_sessions', '/_admin/settings/sessions', []);
     pyroutes.register('admin_settings_sessions_cleanup', '/_admin/settings/sessions/cleanup', []);
@@ -85,6 +97,7 @@ function registerRCRoutes() {
     pyroutes.register('channelstream_proxy', '/_channelstream', []);
     pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []);
     pyroutes.register('check_2fa', '/_admin/check_2fa', []);
+    pyroutes.register('commit_draft_comments_submit', '/%(repo_name)s/changeset/%(commit_id)s/draft_comments_submit', ['repo_name', 'commit_id']);
     pyroutes.register('debug_style_email', '/_admin/debug_style/email/%(email_id)s', ['email_id']);
     pyroutes.register('debug_style_email_plain_rendered', '/_admin/debug_style/email-rendered/%(email_id)s', ['email_id']);
     pyroutes.register('debug_style_home', '/_admin/debug_style', []);
@@ -209,6 +222,8 @@ function registerRCRoutes() {
     pyroutes.register('my_account_emails', '/_admin/my_account/emails', []);
     pyroutes.register('my_account_emails_add', '/_admin/my_account/emails/new', []);
     pyroutes.register('my_account_emails_delete', '/_admin/my_account/emails/delete', []);
+    pyroutes.register('my_account_external_identity', '/_admin/my_account/external-identity', []);
+    pyroutes.register('my_account_external_identity_delete', '/_admin/my_account/external-identity/delete', []);
     pyroutes.register('my_account_goto_bookmark', '/_admin/my_account/bookmark/%(bookmark_id)s', ['bookmark_id']);
     pyroutes.register('my_account_notifications', '/_admin/my_account/notifications', []);
     pyroutes.register('my_account_notifications_test_channelstream', '/_admin/my_account/test_channelstream', []);
@@ -239,6 +254,7 @@ function registerRCRoutes() {
     pyroutes.register('ops_healthcheck', '/_admin/ops/status', []);
     pyroutes.register('ops_ping', '/_admin/ops/ping', []);
     pyroutes.register('ops_redirect_test', '/_admin/ops/redirect', []);
+    pyroutes.register('plugin_admin_chat', '/_admin/plugin_admin_chat/%(action)s', ['action']);
     pyroutes.register('pull_requests_global', '/_admin/pull-request/%(pull_request_id)s', ['pull_request_id']);
     pyroutes.register('pull_requests_global_0', '/_admin/pull_requests/%(pull_request_id)s', ['pull_request_id']);
     pyroutes.register('pull_requests_global_1', '/_admin/pull-requests/%(pull_request_id)s', ['pull_request_id']);
@@ -248,6 +264,7 @@ function registerRCRoutes() {
     pyroutes.register('pullrequest_comments', '/%(repo_name)s/pull-request/%(pull_request_id)s/comments', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_create', '/%(repo_name)s/pull-request/create', ['repo_name']);
     pyroutes.register('pullrequest_delete', '/%(repo_name)s/pull-request/%(pull_request_id)s/delete', ['repo_name', 'pull_request_id']);
+    pyroutes.register('pullrequest_draft_comments_submit', '/%(repo_name)s/pull-request/%(pull_request_id)s/draft_comments_submit', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_drafts', '/%(repo_name)s/pull-request/%(pull_request_id)s/drafts', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_merge', '/%(repo_name)s/pull-request/%(pull_request_id)s/merge', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_new', '/%(repo_name)s/pull-request/new', ['repo_name']);
@@ -260,8 +277,18 @@ function registerRCRoutes() {
     pyroutes.register('pullrequest_update', '/%(repo_name)s/pull-request/%(pull_request_id)s/update', ['repo_name', 'pull_request_id']);
     pyroutes.register('register', '/_admin/register', []);
     pyroutes.register('repo_archivefile', '/%(repo_name)s/archive/%(fname)s', ['repo_name', 'fname']);
+    pyroutes.register('repo_artifacts_data', '/%(repo_name)s/artifacts_data', ['repo_name']);
+    pyroutes.register('repo_artifacts_delete', '/%(repo_name)s/artifacts/delete/%(uid)s', ['repo_name', 'uid']);
+    pyroutes.register('repo_artifacts_get', '/%(repo_name)s/artifacts/download/%(uid)s', ['repo_name', 'uid']);
+    pyroutes.register('repo_artifacts_info', '/%(repo_name)s/artifacts/info/%(uid)s', ['repo_name', 'uid']);
     pyroutes.register('repo_artifacts_list', '/%(repo_name)s/artifacts', ['repo_name']);
+    pyroutes.register('repo_artifacts_new', '/%(repo_name)s/artifacts/new', ['repo_name']);
+    pyroutes.register('repo_artifacts_store', '/%(repo_name)s/artifacts/store', ['repo_name']);
+    pyroutes.register('repo_artifacts_stream_script', '/_file_store/stream-upload-script', []);
+    pyroutes.register('repo_artifacts_stream_store', '/_file_store/stream-upload', []);
+    pyroutes.register('repo_artifacts_update', '/%(repo_name)s/artifacts/update/%(uid)s', ['repo_name', 'uid']);
     pyroutes.register('repo_automation', '/%(repo_name)s/settings/automation', ['repo_name']);
+    pyroutes.register('repo_automation_update', '/%(repo_name)s/settings/automation/%(entry_id)s/update', ['repo_name', 'entry_id']);
     pyroutes.register('repo_changelog', '/%(repo_name)s/changelog', ['repo_name']);
     pyroutes.register('repo_changelog_file', '/%(repo_name)s/changelog/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_commit', '/%(repo_name)s/changeset/%(commit_id)s', ['repo_name', 'commit_id']);
@@ -339,6 +366,9 @@ function registerRCRoutes() {
     pyroutes.register('repo_refs_changelog_data', '/%(repo_name)s/refs-data-changelog', ['repo_name']);
     pyroutes.register('repo_refs_data', '/%(repo_name)s/refs-data', ['repo_name']);
     pyroutes.register('repo_reviewers', '/%(repo_name)s/settings/review/rules', ['repo_name']);
+    pyroutes.register('repo_reviewers_review_rule_delete', '/%(repo_name)s/settings/review/rules/%(rule_id)s/delete', ['repo_name', 'rule_id']);
+    pyroutes.register('repo_reviewers_review_rule_edit', '/%(repo_name)s/settings/review/rules/%(rule_id)s', ['repo_name', 'rule_id']);
+    pyroutes.register('repo_reviewers_review_rule_new', '/%(repo_name)s/settings/review/rules/new', ['repo_name']);
     pyroutes.register('repo_settings_quick_actions', '/%(repo_name)s/settings/quick-action', ['repo_name']);
     pyroutes.register('repo_stats', '/%(repo_name)s/repo_stats/%(commit_id)s', ['repo_name', 'commit_id']);
     pyroutes.register('repo_summary', '/%(repo_name)s', ['repo_name']);

From 2b13bc5a0aa1cb0a369454af62d5e26e1fad4a82 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 17 Jul 2024 11:41:30 +0200
Subject: [PATCH 147/150] fix(tests): fixed callback deamon tests

---
 rhodecode/tests/lib/test_hooks_daemon.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/rhodecode/tests/lib/test_hooks_daemon.py b/rhodecode/tests/lib/test_hooks_daemon.py
index 49952bb0..08f2738f 100644
--- a/rhodecode/tests/lib/test_hooks_daemon.py
+++ b/rhodecode/tests/lib/test_hooks_daemon.py
@@ -305,8 +305,11 @@ class TestPrepareHooksDaemon(object):
             'hooks_protocol': protocol.lower(),
             'task_backend': '',
             'task_queue': '',
-            'repo_store': '/var/opt/rhodecode_repo_store'
+            'repo_store': '/var/opt/rhodecode_repo_store',
+            'repository': 'rhodecode',
         }
+        from rhodecode import CONFIG
+        CONFIG['vcs.svn.redis_conn'] = 'redis://redis:6379/0'
         callback, extras = hook_base.prepare_callback_daemon(
             expected_extras.copy(), protocol=protocol, host='127.0.0.1',
             txn_id='txnid2')

From 45775abb8134e6cbc3956cc2885ccb9e9e0fbe81 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 17 Jul 2024 20:35:17 +0200
Subject: [PATCH 148/150] fix(tests): fixed potential freeze on vcs_operations
 tests

---
 rhodecode/tests/fixture_mods/fixture_pyramid.py |  2 +-
 rhodecode/tests/server_utils.py                 | 12 +++++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/rhodecode/tests/fixture_mods/fixture_pyramid.py b/rhodecode/tests/fixture_mods/fixture_pyramid.py
index d94aaa5d..415dfc9c 100644
--- a/rhodecode/tests/fixture_mods/fixture_pyramid.py
+++ b/rhodecode/tests/fixture_mods/fixture_pyramid.py
@@ -58,7 +58,7 @@ def vcsserver_factory(tmpdir_factory):
     """
 
     def factory(request, overrides=(), vcsserver_port=None,
-                log_file=None, workers='2'):
+                log_file=None, workers='3'):
 
         if vcsserver_port is None:
             vcsserver_port = get_available_port()
diff --git a/rhodecode/tests/server_utils.py b/rhodecode/tests/server_utils.py
index da82dbb5..4b062a47 100644
--- a/rhodecode/tests/server_utils.py
+++ b/rhodecode/tests/server_utils.py
@@ -143,13 +143,14 @@ class RcVCSServer(ServerBase):
     log_file_name = 'rc-vcsserver.log'
     status_url_tmpl = 'http://{host}:{port}/status'
 
-    def __init__(self, config_file, log_file=None, workers='2'):
+    def __init__(self, config_file, log_file=None, workers='3'):
         super(RcVCSServer, self).__init__(config_file, log_file)
         self._args = [
             'gunicorn',
             '--bind', self.bind_addr,
-            '--worker-class', 'gthread',
-            '--backlog', '16',
+            '--worker-class', 'sync',
+            '--threads', '1',
+            '--backlog', '8',
             '--timeout', '300',
             '--workers', workers,
             '--paste', self.config_file]
@@ -180,13 +181,14 @@ class RcWebServer(ServerBase):
     log_file_name = 'rc-web.log'
     status_url_tmpl = 'http://{host}:{port}/_admin/ops/ping'
 
-    def __init__(self, config_file, log_file=None, workers='1'):
+    def __init__(self, config_file, log_file=None, workers='2'):
         super(RcWebServer, self).__init__(config_file, log_file)
         self._args = [
             'gunicorn',
             '--bind', self.bind_addr,
             '--worker-class', 'gthread',
-            '--backlog', '16',
+            '--threads', '4',
+            '--backlog', '8',
             '--timeout', '300',
             '--workers', workers,
             '--paste', self.config_file]

From 06b3778ab4c41b45f8785ef9a86ffcf9fa731048 Mon Sep 17 00:00:00 2001
From: RhodeCode Admin <admin@rhodecode.com>
Date: Wed, 17 Jul 2024 21:10:12 +0200
Subject: [PATCH 149/150] fix(tests): fixed svn tests cases when interactive
 prompt was displayed

---
 rhodecode/tests/vcs_operations/__init__.py    |  8 +-
 .../vcs_operations/test_vcs_operations_svn.py | 87 ++++++++++++-------
 2 files changed, 64 insertions(+), 31 deletions(-)

diff --git a/rhodecode/tests/vcs_operations/__init__.py b/rhodecode/tests/vcs_operations/__init__.py
index 098ebf11..b3881ecb 100644
--- a/rhodecode/tests/vcs_operations/__init__.py
+++ b/rhodecode/tests/vcs_operations/__init__.py
@@ -158,8 +158,14 @@ def _add_files_and_push(vcs, dest, clone_url=None, tags=None, target_branch=None
             f'{git_ident} && git push --verbose --tags {clone_url} {target_branch}'
         )
     elif vcs == 'svn':
+        username = kwargs.pop('username', '')
+        password = kwargs.pop('password', '')
+        auth = ''
+        if username and password:
+            auth = f'--username {username} --password {password}'
+
         stdout, stderr = Command(cwd).execute(
-            f'svn ci -m "pushing to {target_branch}"'
+            f'svn commit --no-auth-cache --non-interactive {auth} -m "pushing to {target_branch}"'
         )
 
     return stdout, stderr
diff --git a/rhodecode/tests/vcs_operations/test_vcs_operations_svn.py b/rhodecode/tests/vcs_operations/test_vcs_operations_svn.py
index 0442084d..0b60ebfc 100644
--- a/rhodecode/tests/vcs_operations/test_vcs_operations_svn.py
+++ b/rhodecode/tests/vcs_operations/test_vcs_operations_svn.py
@@ -41,6 +41,14 @@ from rhodecode.tests.vcs_operations import (
     _add_files_and_push, SVN_REPO_WITH_GROUP)
 
 
+def get_cli_flags(username, password):
+    flags = '--no-auth-cache --non-interactive'
+    auth = ''
+    if username and password:
+        auth = f'--username {username} --password {password}'
+    return flags, auth
+
+
 @pytest.mark.usefixtures("disable_locking", "disable_anonymous_user")
 class TestVCSOperations(object):
 
@@ -50,8 +58,11 @@ class TestVCSOperations(object):
 
         cmd = Command('/tmp')
 
-        auth = f'--non-interactive --username={username} --password={password}'
-        stdout, stderr = cmd.execute(f'svn checkout {auth}', clone_url, tmpdir.strpath)
+        flags, auth = get_cli_flags(username, password)
+
+        stdout, stderr = Command('/tmp').execute(
+            f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
+
         _check_proper_clone(stdout, stderr, 'svn')
         cmd.assert_returncode_success()
 
@@ -61,8 +72,12 @@ class TestVCSOperations(object):
 
         clone_url = rc_web_server.repo_clone_url('_%s' % repo_id)
         cmd = Command('/tmp')
-        auth = f'--non-interactive --username={username} --password={password}'
-        stdout, stderr = cmd.execute(f'svn checkout {auth}', clone_url, tmpdir.strpath)
+
+        flags, auth = get_cli_flags(username, password)
+
+        stdout, stderr = Command('/tmp').execute(
+            f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
+
         _check_proper_clone(stdout, stderr, 'svn')
         cmd.assert_returncode_success()
 
@@ -70,9 +85,11 @@ class TestVCSOperations(object):
         clone_url = rc_web_server.repo_clone_url(SVN_REPO_WITH_GROUP)
         username, password = rc_web_server.repo_clone_credentials()
 
-        cmd = Command('/tmp')
-        auth = f'--non-interactive --username={username} --password={password}'
-        stdout, stderr = cmd.execute(f'svn checkout {auth}', clone_url, tmpdir.strpath)
+        flags, auth = get_cli_flags(username, password)
+
+        stdout, stderr = Command('/tmp').execute(
+            f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
+
         _check_proper_clone(stdout, stderr, 'svn')
         cmd.assert_returncode_success()
 
@@ -81,14 +98,20 @@ class TestVCSOperations(object):
         username, password = rc_web_server.repo_clone_credentials()
         password = 'bad-password'
 
-        auth = f'--non-interactive --username={username} --password={password}'
+        flags, auth = get_cli_flags(username, password)
+
         stdout, stderr = Command('/tmp').execute(
-            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+            f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
         assert 'fatal: Authentication failed' in stderr
 
     def test_clone_svn_with_slashes(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url('//' + SVN_REPO)
-        stdout, stderr = Command('/tmp').execute('svn checkout', clone_url)
+        username, password = '', ''
+        flags, auth = get_cli_flags(username, password)
+
+        stdout, stderr = Command('/tmp').execute(
+            f'svn checkout {flags} {auth}', clone_url)
+
         assert 'not found' in stderr
 
     def test_clone_existing_path_svn_not_in_database(
@@ -96,10 +119,10 @@ class TestVCSOperations(object):
         db_name = fs_repo_only('not-in-db-git', repo_type='git')
         clone_url = rc_web_server.repo_clone_url(db_name)
         username, password = '', ''
-        auth = f'--non-interactive --username={username} --password={password}'
+        flags, auth = get_cli_flags(username, password)
 
         stdout, stderr = Command('/tmp').execute(
-            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+            f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
         assert 'not found' in stderr
 
     def test_clone_existing_path_svn_not_in_database_different_scm(
@@ -108,9 +131,10 @@ class TestVCSOperations(object):
         clone_url = rc_web_server.repo_clone_url(db_name)
 
         username, password = '', ''
-        auth = f'--non-interactive --username={username} --password={password}'
+        flags, auth = get_cli_flags(username, password)
+
         stdout, stderr = Command('/tmp').execute(
-            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+            f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
         assert 'not found' in stderr
 
     def test_clone_non_existing_store_path_svn(self, rc_web_server, tmpdir, user_util):
@@ -121,57 +145,60 @@ class TestVCSOperations(object):
         RepoModel()._delete_filesystem_repo(repo)
 
         username, password = '', ''
-        auth = f'--non-interactive --username={username} --password={password}'
+        flags, auth = get_cli_flags(username, password)
+
         stdout, stderr = Command('/tmp').execute(
-            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+            f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
         assert 'not found' in stderr
 
     def test_push_new_file_svn(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(SVN_REPO)
         username, password = '', ''
-        auth = f'--non-interactive --username={username} --password={password}'
+        flags, auth = get_cli_flags(username, password)
 
         stdout, stderr = Command('/tmp').execute(
-            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+            f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
 
         # commit some stuff into this repo
         stdout, stderr = _add_files_and_push(
-            'svn', tmpdir.strpath, clone_url=clone_url)
+            'svn', tmpdir.strpath, clone_url=clone_url, username=username, password=password)
 
         _check_proper_svn_push(stdout, stderr)
 
     def test_push_wrong_credentials_svn(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(SVN_REPO)
 
-        username, password = '', ''
-        auth = f'--non-interactive --username={username} --password={password}'
+        username, password = rc_web_server.repo_clone_credentials()
+        flags, auth = get_cli_flags(username, password)
+
         stdout, stderr = Command('/tmp').execute(
-            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+            f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
 
         push_url = rc_web_server.repo_clone_url(
             SVN_REPO, user='bad', passwd='name')
         stdout, stderr = _add_files_and_push(
-            'svn', tmpdir.strpath, clone_url=push_url)
+            'svn', tmpdir.strpath, clone_url=push_url, username=username, password=password)
 
         assert 'fatal: Authentication failed' in stderr
 
     def test_push_back_to_wrong_url_svn(self, rc_web_server, tmpdir):
         clone_url = rc_web_server.repo_clone_url(SVN_REPO)
         username, password = '', ''
-        auth = f'--non-interactive --username={username} --password={password}'
-        Command('/tmp').execute(
-            f'svn checkout {auth}', clone_url, tmpdir.strpath)
+        flags, auth = get_cli_flags(username, password)
+
+        stdout, stderr = Command('/tmp').execute(
+            f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
 
         stdout, stderr = _add_files_and_push(
             'svn', tmpdir.strpath,
-            clone_url=rc_web_server.repo_clone_url('not-existing'))
+            clone_url=rc_web_server.repo_clone_url('not-existing'), username=username, password=password)
 
         assert 'not found' in stderr
 
     def test_ip_restriction_svn(self, rc_web_server, tmpdir):
         user_model = UserModel()
         username, password = '', ''
-        auth = f'--non-interactive --username={username} --password={password}'
+        flags, auth = get_cli_flags(username, password)
 
         try:
             user_model.add_extra_ip(TEST_USER_ADMIN_LOGIN, '10.10.10.10/32')
@@ -180,7 +207,7 @@ class TestVCSOperations(object):
             clone_url = rc_web_server.repo_clone_url(SVN_REPO)
 
             stdout, stderr = Command('/tmp').execute(
-                f'svn checkout {auth}', clone_url, tmpdir.strpath)
+                f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
             msg = "The requested URL returned error: 403"
             assert msg in stderr
         finally:
@@ -192,6 +219,6 @@ class TestVCSOperations(object):
         time.sleep(2)
 
         cmd = Command('/tmp')
-        stdout, stderr = cmd.execute(f'svn checkout {auth}', clone_url, tmpdir.strpath)
+        stdout, stderr = cmd.execute(f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
         cmd.assert_returncode_success()
         _check_proper_clone(stdout, stderr, 'svn')

From fd2519123b4608542bf24e4509f944d16d46e055 Mon Sep 17 00:00:00 2001
From: Andrii Verbytskyi <andrii@rhodecode.com>
Date: Thu, 18 Jul 2024 15:39:39 +0200
Subject: [PATCH 150/150] chore: updated release notes

---
 docs/release-notes/release-notes-5.1.0.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/release-notes/release-notes-5.1.0.rst b/docs/release-notes/release-notes-5.1.0.rst
index ea34f15a..27999cbb 100644
--- a/docs/release-notes/release-notes-5.1.0.rst
+++ b/docs/release-notes/release-notes-5.1.0.rst
@@ -4,8 +4,7 @@
 Release Date
 ^^^^^^^^^^^^
 
-- TBA
-- available as docker "beta" tag rhodecode/rhodecode-ce:beta, rhodecode/rhodecode-ee:beta
+- 2024-07-18
 
 
 New Features