russell@unturf.com
722c3bd369
Replace password reset with email OTP verification flow. Add auth_otp module, OTP templates, and email delivery. Expand styleguide CSS with full component library. Rework login, register, and admin views for cookie sessions. Remove legacy 2FA templates and password reset flow. Update SSH wrappers, forms, validators, and middleware.
476 lines
17 KiB
Python
476 lines
17 KiB
Python
# Copyright (C) 2010-2024 RhodeCode GmbH
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License, version 3
|
|
# (only), as published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
# This program is dual-licensed. If you wish to learn more about the
|
|
# RhodeCode Enterprise Edition, including its added features, Support services,
|
|
# and proprietary license terms, please see https://rhodecode.com/licenses/
|
|
|
|
import os
|
|
import sys
|
|
import collections
|
|
|
|
import time
|
|
import logging.config
|
|
|
|
from paste.gzipper import make_gzip_middleware
|
|
import pyramid.events
|
|
from pyramid.wsgi import wsgiapp
|
|
from pyramid.config import Configurator
|
|
from pyramid.settings import asbool, aslist
|
|
from pyramid.httpexceptions import HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound
|
|
from pyramid.renderers import render_to_response
|
|
|
|
from rhodecode.config.constants import PYCHARM_DEBUG
|
|
from rhodecode.lib.middleware.pycharm_remote_debugger import PyCharmDebugMiddleware
|
|
from rhodecode.model import meta
|
|
from rhodecode.config import patches
|
|
|
|
from rhodecode.config.environment import load_pyramid_environment, propagate_rhodecode_config
|
|
|
|
import rhodecode.events
|
|
from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
|
|
from rhodecode.lib.middleware.vcs import VCSMiddleware
|
|
from rhodecode.lib.request import Request
|
|
from rhodecode.lib.vcs import VCSCommunicationError
|
|
from rhodecode.lib.exceptions import VCSServerUnavailable
|
|
from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
|
|
from rhodecode.lib.middleware.https_fixup import HttpsFixup
|
|
from rhodecode.lib.plugins.utils import register_rhodecode_plugin
|
|
from rhodecode.lib.utils2 import AttributeDict
|
|
from rhodecode.lib.exc_tracking import store_exception, format_exc
|
|
from rhodecode.subscribers import (
|
|
auto_merge_pr_if_needed,
|
|
scan_repositories_if_enabled,
|
|
write_js_routes_if_enabled,
|
|
write_metadata_if_needed,
|
|
write_usage_data,
|
|
import_license_if_present,
|
|
)
|
|
from rhodecode.lib.statsd_client import StatsdClient
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def is_http_error(response):
|
|
# error which should have traceback
|
|
return response.status_code > 499
|
|
|
|
|
|
def should_load_all():
|
|
"""
|
|
Returns if all application components should be loaded. In some cases it's
|
|
desired to skip apps loading for faster shell script execution
|
|
"""
|
|
ssh_cmd = os.environ.get("RC_CMD_SSH_WRAPPER")
|
|
if ssh_cmd:
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def make_pyramid_app(global_config, **settings):
|
|
"""
|
|
Constructs the WSGI application based on Pyramid.
|
|
|
|
Specials:
|
|
|
|
* The application can also be integrated like a plugin via the call to
|
|
`includeme`. This is accompanied with the other utility functions which
|
|
are called. Changing this should be done with great care to not break
|
|
cases when these fragments are assembled from another place.
|
|
|
|
"""
|
|
start_time = time.time()
|
|
log.info("Pyramid app config starting")
|
|
|
|
sanitize_settings_and_apply_defaults(global_config, settings)
|
|
|
|
# init and bootstrap StatsdClient
|
|
StatsdClient.setup(settings)
|
|
|
|
config = Configurator(settings=settings)
|
|
# Init our statsd at very start
|
|
config.registry.statsd = StatsdClient.statsd
|
|
|
|
# Apply compatibility patches
|
|
patches.inspect_getargspec()
|
|
patches.repoze_sendmail_lf_fix()
|
|
|
|
|
|
# first init, so load_pyramid_enviroment, can access some critical data, like __file__
|
|
propagate_rhodecode_config(global_config, {}, {}, full=False)
|
|
|
|
load_pyramid_environment(global_config, settings)
|
|
|
|
# Static file view comes first
|
|
includeme_first(config)
|
|
|
|
includeme(config)
|
|
|
|
pyramid_app = config.make_wsgi_app()
|
|
pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
|
|
pyramid_app.config = config
|
|
|
|
celery_settings = get_celery_config(settings)
|
|
config.configure_celery(celery_settings)
|
|
|
|
# final config set...
|
|
propagate_rhodecode_config(global_config, settings, config.registry.settings)
|
|
|
|
# creating the app uses a connection - return it after we are done
|
|
meta.Session.remove()
|
|
|
|
total_time = time.time() - start_time
|
|
log.info("Pyramid app created and configured in %.2fs", total_time)
|
|
return pyramid_app
|
|
|
|
|
|
def get_celery_config(settings):
|
|
"""
|
|
Converts basic ini configuration into celery 4.X options
|
|
"""
|
|
|
|
def key_converter(key_name):
|
|
pref = "celery."
|
|
if key_name.startswith(pref):
|
|
return key_name[len(pref) :].replace(".", "_").lower()
|
|
|
|
def type_converter(parsed_key, value):
|
|
if not isinstance(value, str):
|
|
return value
|
|
|
|
# cast to int
|
|
if value.isdigit():
|
|
return int(value)
|
|
|
|
# cast to bool
|
|
if value.lower() in ["true", "false", "True", "False"]:
|
|
return value.lower() == "true"
|
|
return value
|
|
|
|
celery_config = {}
|
|
for k, v in settings.items():
|
|
pref = "celery."
|
|
if k.startswith(pref):
|
|
celery_config[key_converter(k)] = type_converter(key_converter(k), v)
|
|
|
|
# TODO:rethink if we want to support celerybeat based file config, probably NOT
|
|
# beat_config = {}
|
|
# for section in parser.sections():
|
|
# if section.startswith('celerybeat:'):
|
|
# name = section.split(':', 1)[1]
|
|
# beat_config[name] = get_beat_config(parser, section)
|
|
|
|
# final compose of settings
|
|
celery_settings = {}
|
|
|
|
if celery_config:
|
|
celery_settings.update(celery_config)
|
|
# if beat_config:
|
|
# celery_settings.update({'beat_schedule': beat_config})
|
|
|
|
return celery_settings
|
|
|
|
|
|
def not_found_view(request):
|
|
"""
|
|
This creates the view which should be registered as not-found-view to
|
|
pyramid.
|
|
"""
|
|
|
|
if not getattr(request, "vcs_call", None):
|
|
# handle like regular case with our error_handler
|
|
return error_handler(HTTPNotFound(), request)
|
|
|
|
# handle not found view as a vcs call
|
|
settings = request.registry.settings
|
|
ae_client = getattr(request, "ae_client", None)
|
|
vcs_app = VCSMiddleware(HTTPNotFound(), request.registry, settings, appenlight_client=ae_client)
|
|
|
|
return wsgiapp(vcs_app)(None, request)
|
|
|
|
|
|
def error_handler(exception, request):
|
|
import rhodecode
|
|
from rhodecode.lib import helpers
|
|
|
|
rhodecode_title = rhodecode.CONFIG.get("rhodecode_title") or "RhodeCode"
|
|
|
|
base_response = HTTPInternalServerError()
|
|
# prefer original exception for the response since it may have headers set
|
|
if isinstance(exception, HTTPException):
|
|
base_response = exception
|
|
elif isinstance(exception, VCSCommunicationError):
|
|
base_response = VCSServerUnavailable()
|
|
|
|
if is_http_error(base_response):
|
|
traceback_info = format_exc(request.exc_info)
|
|
log.error("error occurred handling this request for path: %s, \n%s", request.path, traceback_info)
|
|
|
|
error_explanation = base_response.explanation or str(base_response)
|
|
if base_response.status_code == 404:
|
|
error_explanation += " Optionally you don't have permission to access this page."
|
|
c = AttributeDict()
|
|
c.error_message = base_response.status
|
|
c.error_explanation = error_explanation
|
|
c.visual = AttributeDict()
|
|
|
|
c.visual.rhodecode_support_url = request.registry.settings.get("rhodecode_support_url") or request.route_url(
|
|
"rhodecode_support"
|
|
)
|
|
c.redirect_time = 0
|
|
c.rhodecode_name = rhodecode_title
|
|
if not c.rhodecode_name:
|
|
c.rhodecode_name = "Rhodecode"
|
|
|
|
c.causes = []
|
|
if is_http_error(base_response):
|
|
c.causes.append("Server is overloaded.")
|
|
c.causes.append("Server database connection is lost.")
|
|
c.causes.append("Server expected unhandled error.")
|
|
|
|
if hasattr(base_response, "causes"):
|
|
c.causes = base_response.causes
|
|
|
|
c.messages = helpers.flash.pop_messages(request=request)
|
|
exc_info = sys.exc_info()
|
|
c.exception_id = id(exc_info)
|
|
c.show_exception_id = isinstance(base_response, VCSServerUnavailable) or base_response.status_code > 499
|
|
c.exception_id_url = request.route_url("admin_settings_exception_tracker_show", exception_id=c.exception_id)
|
|
|
|
debug_mode = rhodecode.ConfigGet().get_bool("debug")
|
|
if c.show_exception_id:
|
|
store_exception(c.exception_id, exc_info)
|
|
c.exception_debug = debug_mode
|
|
c.exception_config_ini = rhodecode.CONFIG.get("__file__")
|
|
|
|
if debug_mode:
|
|
try:
|
|
from rich.traceback import install
|
|
|
|
install(show_locals=True)
|
|
log.debug("Installing rich tracebacks...")
|
|
except ImportError:
|
|
pass
|
|
|
|
response = render_to_response(
|
|
"/errors/error_document.mako", {"c": c, "h": helpers}, request=request, response=base_response
|
|
)
|
|
|
|
response.headers["X-RC-Exception-Id"] = str(c.exception_id)
|
|
|
|
statsd = request.registry.statsd
|
|
if statsd and base_response.status_code > 499:
|
|
exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
|
|
statsd.incr(
|
|
"rhodecode_exception_total",
|
|
tags=["exc_source:web", f"http_code:{base_response.status_code}", f"type:{exc_type}"],
|
|
)
|
|
|
|
return response
|
|
|
|
|
|
def includeme_first(config):
|
|
# redirect automatic browser favicon.ico requests to correct place
|
|
def favicon_redirect(context, request):
|
|
return HTTPFound(request.static_path("rhodecode:public/images/favicon.ico"))
|
|
|
|
config.add_view(favicon_redirect, route_name="favicon")
|
|
config.add_route("favicon", "/favicon.ico")
|
|
|
|
def robots_redirect(context, request):
|
|
return HTTPFound(request.static_path("rhodecode:public/robots.txt"))
|
|
|
|
config.add_view(robots_redirect, route_name="robots")
|
|
config.add_route("robots", "/robots.txt")
|
|
|
|
config.add_static_view("_static/deform", "deform:static")
|
|
config.add_static_view("_static/rhodecode", path="rhodecode:public", cache_max_age=3600 * 24)
|
|
|
|
|
|
ce_auth_resources = [
|
|
"rhodecode.authentication.plugins.auth_crowd",
|
|
"rhodecode.authentication.plugins.auth_headers",
|
|
"rhodecode.authentication.plugins.auth_jasig_cas",
|
|
"rhodecode.authentication.plugins.auth_ldap",
|
|
"rhodecode.authentication.plugins.auth_pam",
|
|
"rhodecode.authentication.plugins.auth_rhodecode",
|
|
"rhodecode.authentication.plugins.auth_token",
|
|
]
|
|
|
|
|
|
def includeme(config, auth_resources=None):
|
|
from rhodecode.lib.celerylib.loader import configure_celery
|
|
|
|
log.debug("Initializing main includeme from %s", os.path.basename(__file__))
|
|
settings = config.registry.settings
|
|
config.set_request_factory(Request)
|
|
|
|
# plugin information
|
|
config.registry.rhodecode_plugins = collections.OrderedDict()
|
|
|
|
config.add_directive("register_rhodecode_plugin", register_rhodecode_plugin)
|
|
|
|
config.add_directive("configure_celery", configure_celery)
|
|
|
|
if settings.get("appenlight", False):
|
|
config.include("appenlight_client.ext.pyramid_tween")
|
|
|
|
load_all = should_load_all()
|
|
|
|
# Includes which are required. The application would fail without them.
|
|
config.include("pyramid_mako")
|
|
config.include("rhodecode.lib.rc_beaker")
|
|
config.include("rhodecode.lib.rc_cache")
|
|
config.include("rhodecode.lib.archive_cache")
|
|
|
|
config.include("rhodecode.apps._base.navigation")
|
|
config.include("rhodecode.apps._base.subscribers")
|
|
config.include("rhodecode.tweens")
|
|
config.include("rhodecode.authentication")
|
|
|
|
if load_all:
|
|
# load CE authentication plugins
|
|
|
|
if auth_resources:
|
|
ce_auth_resources.extend(auth_resources)
|
|
|
|
for resource in ce_auth_resources:
|
|
config.include(resource)
|
|
|
|
# Auto discover authentication plugins and include their configuration.
|
|
if asbool(settings.get("auth_plugin.import_legacy_plugins", "true")):
|
|
from rhodecode.authentication import discover_legacy_plugins
|
|
|
|
discover_legacy_plugins(config)
|
|
|
|
# apps
|
|
if load_all:
|
|
log.debug("Starting config.include() calls")
|
|
config.include("rhodecode.api.includeme")
|
|
config.include("rhodecode.apps._base.includeme")
|
|
config.include("rhodecode.apps._base.navigation.includeme")
|
|
config.include("rhodecode.apps._base.subscribers.includeme")
|
|
config.include("rhodecode.apps.hovercards.includeme")
|
|
config.include("rhodecode.apps.ops.includeme")
|
|
config.include("rhodecode.apps.channelstream.includeme")
|
|
config.include("rhodecode.apps.file_store.includeme")
|
|
config.include("rhodecode.apps.admin.includeme")
|
|
config.include("rhodecode.apps.login.includeme")
|
|
config.include("rhodecode.apps.home.includeme")
|
|
config.include("rhodecode.apps.journal.includeme")
|
|
|
|
config.include("rhodecode.apps.repository.includeme")
|
|
config.include("rhodecode.apps.repo_group.includeme")
|
|
config.include("rhodecode.apps.user_group.includeme")
|
|
config.include("rhodecode.apps.search.includeme")
|
|
config.include("rhodecode.apps.user_profile.includeme")
|
|
config.include("rhodecode.apps.user_group_profile.includeme")
|
|
config.include("rhodecode.apps.my_account.includeme")
|
|
config.include("rhodecode.apps.gist.includeme")
|
|
|
|
config.include("rhodecode.apps.svn_support.includeme")
|
|
config.include("rhodecode.apps.ssh_support.includeme")
|
|
config.include("rhodecode.apps.debug_style")
|
|
config.include("rhodecode.apps.styleguide.includeme")
|
|
|
|
if load_all:
|
|
config.include("rhodecode.integrations.includeme")
|
|
config.include("rhodecode.integrations.routes.includeme")
|
|
|
|
config.add_route("rhodecode_support", "https://rhodecode.com/help/", static=True)
|
|
settings["default_locale_name"] = settings.get("lang", "en")
|
|
config.add_translation_dirs("rhodecode:i18n/")
|
|
|
|
# Add subscribers.
|
|
if load_all:
|
|
log.debug("Adding subscribers...")
|
|
config.add_subscriber(auto_merge_pr_if_needed, rhodecode.events.PullRequestReviewEvent)
|
|
config.add_subscriber(scan_repositories_if_enabled, pyramid.events.ApplicationCreated)
|
|
config.add_subscriber(write_metadata_if_needed, pyramid.events.ApplicationCreated)
|
|
config.add_subscriber(write_usage_data, pyramid.events.ApplicationCreated)
|
|
config.add_subscriber(write_js_routes_if_enabled, pyramid.events.ApplicationCreated)
|
|
config.add_subscriber(import_license_if_present, pyramid.events.ApplicationCreated)
|
|
|
|
# Set the default renderer for HTML templates to mako.
|
|
config.add_mako_renderer(".html")
|
|
|
|
config.add_renderer(name="json_ext", factory="rhodecode.lib.ext_json_renderer.pyramid_ext_json")
|
|
|
|
config.add_renderer(name="string_html", factory="rhodecode.lib.string_renderer.html")
|
|
|
|
# include RhodeCode plugins
|
|
includes = aslist(settings.get("rhodecode.includes", []))
|
|
log.debug("processing rhodecode.includes data...")
|
|
for inc in includes:
|
|
config.include(inc)
|
|
|
|
# custom not found view, if our pyramid app doesn't know how to handle
|
|
# the request pass it to potential VCS handling ap
|
|
config.add_notfound_view(not_found_view)
|
|
if not settings.get("debugtoolbar.enabled", False):
|
|
# disabled debugtoolbar handle all exceptions via the error_handlers
|
|
config.add_view(error_handler, context=Exception)
|
|
|
|
# all errors including 403/404/50X
|
|
config.add_view(error_handler, context=HTTPError)
|
|
|
|
|
|
def wrap_app_in_pycharm_remote_debugger_if_enabled(pyramid_app):
|
|
pycharm_debugger_enabled = os.getenv(PYCHARM_DEBUG, "0") == "1"
|
|
log.debug(f"Pycharm remote debugger enabled: {pycharm_debugger_enabled}")
|
|
|
|
if pycharm_debugger_enabled:
|
|
return PyCharmDebugMiddleware(pyramid_app)
|
|
|
|
return pyramid_app
|
|
|
|
|
|
def wrap_app_in_wsgi_middlewares(pyramid_app, config):
|
|
"""
|
|
Apply outer WSGI middlewares around the application.
|
|
"""
|
|
registry = config.registry
|
|
settings = registry.settings
|
|
|
|
# enable https redirects based on HTTP_X_URL_SCHEME set by proxy
|
|
pyramid_app = HttpsFixup(pyramid_app, settings)
|
|
pyramid_app = wrap_app_in_pycharm_remote_debugger_if_enabled(pyramid_app)
|
|
|
|
pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(pyramid_app, settings)
|
|
registry.ae_client = _ae_client
|
|
|
|
if settings["gzip_responses"]:
|
|
pyramid_app = make_gzip_middleware(pyramid_app, settings, compress_level=1)
|
|
|
|
# this should be the outer most middleware in the wsgi stack since
|
|
# middleware like Routes make database calls
|
|
def pyramid_app_with_cleanup(environ, start_response):
|
|
start = time.time()
|
|
try:
|
|
return pyramid_app(environ, start_response)
|
|
finally:
|
|
# Dispose current database session and rollback uncommitted
|
|
# transactions.
|
|
meta.Session.remove()
|
|
|
|
# In a single threaded mode server, on non sqlite db we should have
|
|
# '0 Current Checked out connections' at the end of a request,
|
|
# if not, then something, somewhere is leaving a connection open
|
|
pool = meta.get_engine().pool
|
|
log.debug("sa pool status: %s", pool.status())
|
|
total = time.time() - start
|
|
log.debug("Request processing finalized: %.4fs", total)
|
|
|
|
return pyramid_app_with_cleanup
|