engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

requests: cleaned / unified way of handling requests generation from non-web scope.

Browse source 
- cli
- celery workers
This commit is contained in:
RhodeCode Admin 2023-01-31 20:31:36 +01:00
parent 5b6732fae5
commit f4336fe56e
6 changed files with 99 additions and 97 deletions
Download patch file Download diff file Expand all files Collapse all files

9
rhodecode/config/middleware.py
View file

@ -381,15 +381,6 @@ def includeme(config, auth_resources=None):
config.add_subscriber(write_js_routes_if_enabled,
pyramid.events.ApplicationCreated)
# request custom methods
config.add_request_method(
'rhodecode.lib.partial_renderer.get_partial_renderer',
'get_partial_renderer')
config.add_request_method(
'rhodecode.lib.request_counter.get_request_counter',
'request_count')
# Set the authorization policy.
authz_policy = ACLAuthorizationPolicy()
config.set_authorization_policy(authz_policy)

50
rhodecode/lib/base.py
View file

@ -35,7 +35,6 @@ from paste.httpexceptions import HTTPUnauthorized, HTTPForbidden, get_exception
from paste.httpheaders import WWW_AUTHENTICATE, AUTHORIZATION
import rhodecode
from rhodecode.apps._base import TemplateArgs
from rhodecode.authentication.base import VCS_TYPE
from rhodecode.lib import auth, utils2
from rhodecode.lib import helpers as h
@ -460,12 +459,7 @@ def attach_context_attributes(context, request, user_id=None, is_api=None):
context.rhodecode_user = request.rpc_user
# attach the whole call context to the request
# use set_call_context method if request has it
# sometimes in Celery context requests is "different"
if hasattr(request, 'set_call_context'):
request.set_call_context(context)
else:
request.call_context = context
request.set_call_context(context)
def get_auth_user(request):
@ -577,9 +571,9 @@ def add_events_routes(config):
pattern='/_hovercard/commit/{repo_name}/{commit_id}')
def bootstrap_config(request):
def bootstrap_config(request, registry_name='RcTestRegistry'):
import pyramid.testing
registry = pyramid.testing.Registry('RcTestRegistry')
registry = pyramid.testing.Registry(registry_name)
config = pyramid.testing.setUp(registry=registry, request=request)
@ -594,42 +588,24 @@ def bootstrap_config(request):
def bootstrap_request(**kwargs):
import pyramid.testing
"""
Returns a thin version of Request Object that is used in non-web context like testing/celery
"""
class TestRequest(pyramid.testing.DummyRequest):
import pyramid.testing
from rhodecode.lib.request import ThinRequest as _ThinRequest
class ThinRequest(_ThinRequest):
application_url = kwargs.pop('application_url', 'http://example.com')
host = kwargs.pop('host', 'example.com:80')
domain = kwargs.pop('domain', 'example.com')
def translate(self, msg):
return msg
def plularize(self, singular, plural, n):
return singular
def get_partial_renderer(self, tmpl_name):
from rhodecode.lib.partial_renderer import get_partial_renderer
return get_partial_renderer(request=self, tmpl_name=tmpl_name)
_call_context = TemplateArgs()
_call_context.visual = TemplateArgs()
_call_context.visual.show_sha_length = 12
_call_context.visual.show_revision_number = True
@property
def call_context(self):
return self._call_context
def set_call_context(self, new_context):
self._call_context = new_context
class TestDummySession(pyramid.testing.DummySession):
class ThinSession(pyramid.testing.DummySession):
def save(*arg, **kw):
pass
request = TestRequest(**kwargs)
request.session = TestDummySession()
request = ThinRequest(**kwargs)
request.session = ThinSession()
return request

48
rhodecode/lib/celerylib/loader.py
View file

@ -45,7 +45,7 @@ import rhodecode
from rhodecode.lib.auth import AuthUser
from rhodecode.lib.celerylib.utils import parse_ini_vars, ping_db
from rhodecode.lib.ext_json import json
from rhodecode.lib.pyramid_utils import bootstrap, setup_logging, prepare_request
from rhodecode.lib.pyramid_utils import bootstrap, setup_logging
from rhodecode.lib.utils2 import str2bool
from rhodecode.model import meta
@ -271,10 +271,20 @@ class RequestContextTask(Task):
link=None, link_error=None, shadow=None, **options):
""" queue the job to run (we are in web request context here) """
req = get_current_request()
req = self.app.conf['PYRAMID_REQUEST'] or get_current_request()
log.debug('Running Task with class: %s. Request Class: %s',
self.__class__, req.__class__)
proxy_data = getattr(self.request, 'rhodecode_proxy_data', None)
log.debug('celery proxy data:%r', proxy_data)
user_id = None
ip_addr = None
if proxy_data:
user_id = proxy_data['auth_user']['user_id']
ip_addr = proxy_data['auth_user']['ip_addr']
# web case
if hasattr(req, 'user'):
ip_addr = req.user.ip_addr
@ -285,14 +295,18 @@ class RequestContextTask(Task):
ip_addr = req.rpc_user.ip_addr
user_id = req.rpc_user.user_id
else:
raise Exception(
'Unable to fetch required data from request: {}. \n'
'This task is required to be executed from context of '
'request in a webapp. Task: {}'.format(
repr(req),
self
if user_id and ip_addr:
log.debug('Using data from celery proxy user')
else:
raise Exception(
'Unable to fetch required data from request: {}. \n'
'This task is required to be executed from context of '
'request in a webapp. Task: {}'.format(
repr(req),
self.__class__
)
)
)
if req:
# we hook into kwargs since it is the only way to pass our data to
@ -311,19 +325,3 @@ class RequestContextTask(Task):
return super(RequestContextTask, self).apply_async(
args, kwargs, task_id, producer, link, link_error, shadow, **options)
def __call__(self, *args, **kwargs):
""" rebuild the context and then run task on celery worker """
proxy_data = getattr(self.request, 'rhodecode_proxy_data', None)
if not proxy_data:
return super(RequestContextTask, self).__call__(*args, **kwargs)
log.debug('using celery proxy data to run task: %r', proxy_data)
# re-inject and register threadlocals for proper routing support
request = prepare_request(proxy_data['environ'])
request.user = AuthUser(user_id=proxy_data['auth_user']['user_id'],
ip_addr=proxy_data['auth_user']['ip_addr'])
return super(RequestContextTask, self).__call__(*args, **kwargs)

26
rhodecode/lib/pyramid_utils.py
View file

@ -37,24 +37,7 @@ def get_app_config(ini_path):
return appconfig('config:{}'.format(ini_path), relative_to=os.getcwd())
class BootstrappedRequest(Request):
"""
Special version of Request Which has some available methods like in pyramid.
Some code (used for template rendering) requires this, and we unsure it's present.
"""
def translate(self, msg):
return msg
def plularize(self, singular, plural, n):
return singular
def get_partial_renderer(self, tmpl_name):
from rhodecode.lib.partial_renderer import get_partial_renderer
return get_partial_renderer(request=self, tmpl_name=tmpl_name)
def bootstrap(config_uri, request=None, options=None, env=None):
def bootstrap(config_uri, options=None, env=None):
if env:
os.environ.update(env)
@ -65,12 +48,7 @@ def bootstrap(config_uri, request=None, options=None, env=None):
except (configparser.NoSectionError, configparser.NoOptionError):
pass
request = request or BootstrappedRequest.blank('/', base_url=base_url)
request = Request.blank('/', base_url=base_url)
return pyramid_bootstrap(config_uri, request=request, options=options)
def prepare_request(environ):
request = Request.blank('/', environ=environ)
prepare(request) # set pyramid threadlocal request
return request

61
rhodecode/lib/request.py
View file

@ -19,13 +19,24 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
from uuid import uuid4
import pyramid.testing
from pyramid.decorator import reify
from pyramid.request import Request as _Request
from rhodecode.translation import _ as tsf
from rhodecode.lib.utils2 import StrictAttributeDict
class Request(_Request):
class TemplateArgs(StrictAttributeDict):
pass
# Base Class with DummyMethods, testing / CLI scripts
class RequestBase(object):
_req_id_bucket = list()
_call_context = TemplateArgs()
_call_context.visual = TemplateArgs()
_call_context.visual.show_sha_length = 12
_call_context.visual.show_revision_number = True
@reify
def req_id(self):
@ -38,6 +49,47 @@ class Request(_Request):
def req_id_records_init(self):
self._req_id_bucket = list()
def translate(self, *args, **kwargs):
raise NotImplementedError()
def plularize(self, *args, **kwargs):
raise NotImplementedError()
def get_partial_renderer(self, tmpl_name):
raise NotImplementedError()
@property
def call_context(self):
return self._call_context
def set_call_context(self, new_context):
self._call_context = new_context
# for thin non-web/cli etc
class ThinRequest(RequestBase, pyramid.testing.DummyRequest):
def translate(self, msg):
return msg
def plularize(self, singular, plural, n):
return singular
def get_partial_renderer(self, tmpl_name):
from rhodecode.lib.partial_renderer import get_partial_renderer
return get_partial_renderer(request=self, tmpl_name=tmpl_name)
# for real-web-based
class RealRequest(RequestBase, _Request):
def get_partial_renderer(self, tmpl_name):
from rhodecode.lib.partial_renderer import get_partial_renderer
return get_partial_renderer(request=self, tmpl_name=tmpl_name)
def request_count(self):
from rhodecode.lib.request_counter import get_request_counter
return get_request_counter()
def plularize(self, *args, **kwargs):
return self.localizer.pluralize(*args, **kwargs)
@ -48,3 +100,10 @@ class Request(_Request):
return localizer.translate(tsf(*_args, **_kwargs))
return auto_translate(*args, **kwargs)
class Request(RealRequest):
"""
This is the main request object used in web-context
"""
pass

2
rhodecode/lib/request_counter.py
View file

@ -21,7 +21,7 @@
counter = 0
def get_request_counter(request):
def get_request_counter():
global counter
counter += 1
return counter