engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

core: run ruff format

Browse source
This commit is contained in:
RhodeCode Admin 2025-01-13 17:47:37 +01:00
parent 7130effb58
commit 166db21812
821 changed files with 82905 additions and 84121 deletions
Download patch file Download diff file Expand all files Collapse all files

214
configs/gunicorn_config.py
View file

@ -19,6 +19,7 @@ from gunicorn.glogging import Logger
def get_workers():
import multiprocessing
return multiprocessing.cpu_count() * 2 + 1
@ -26,10 +27,10 @@ bind = "127.0.0.1:10020"
# Error logging output for gunicorn (-) is stdout
errorlog = '-'
errorlog = "-"
# Access logging output for gunicorn (-) is stdout
accesslog = '-'
accesslog = "-"
# SERVER MECHANICS
@ -42,45 +43,45 @@ tmp_upload_dir = None
reuse_port = True
# Custom log format
#access_log_format = (
# access_log_format = (
# '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
# loki format for easier parsing in grafana
loki_access_log_format = (
'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
loki_access_log_format = 'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"'
# JSON format
json_access_log_format = json.dumps({
'time': r'%(t)s',
'pid': r'%(p)s',
'level': 'INFO',
'ip': r'%(h)s',
'request_time': r'%(L)s',
'remote_address': r'%(h)s',
'user_name': r'%(u)s',
'status': r'%(s)s',
'method': r'%(m)s',
'url_path': r'%(U)s',
'query_string': r'%(q)s',
'protocol': r'%(H)s',
'response_length': r'%(B)s',
'referer': r'%(f)s',
'user_agent': r'%(a)s',
})
json_access_log_format = json.dumps(
{
"time": r"%(t)s",
"pid": r"%(p)s",
"level": "INFO",
"ip": r"%(h)s",
"request_time": r"%(L)s",
"remote_address": r"%(h)s",
"user_name": r"%(u)s",
"status": r"%(s)s",
"method": r"%(m)s",
"url_path": r"%(U)s",
"query_string": r"%(q)s",
"protocol": r"%(H)s",
"response_length": r"%(B)s",
"referer": r"%(f)s",
"user_agent": r"%(a)s",
}
)
access_log_format = loki_access_log_format
if os.environ.get('RC_LOGGING_FORMATTER') == 'json':
if os.environ.get("RC_LOGGING_FORMATTER") == "json":
access_log_format = json_access_log_format
# self adjust workers based on CPU count, to use maximum of CPU and not overquota the resources
# workers = get_workers()
# Gunicorn access log level
loglevel = 'info'
loglevel = "info"
# Process name visible in a process list
proc_name = 'rhodecode_enterprise'
proc_name = "rhodecode_enterprise"
# Type of worker class, one of `sync`, `gevent` or `gthread`
# currently `sync` is the only option allowed for vcsserver and for rhodecode all of 3 are allowed
@ -90,7 +91,7 @@ proc_name = 'rhodecode_enterprise'
# gthread:
# In this case, the maximum number of concurrent requests is (N workers * X threads)
# e.g. workers = 3 threads=3 = 3*3, 9 concurrent requests can be handled
worker_class = 'gthread'
worker_class = "gthread"
# Sets the number of process workers. More workers means more concurrent connections
# RhodeCode can handle at the same time. Each additional worker also it increases
@ -173,6 +174,7 @@ class MemoryCheckConfig:
def _get_process_rss(pid=None):
try:
import psutil
if pid:
proc = psutil.Process(pid)
else:
@ -203,30 +205,30 @@ def get_memory_usage_params(config=None):
ini_path = os.path.abspath(config)
conf = _get_config(ini_path)
section = 'server:main'
section = "server:main"
if conf and conf.has_section(section):
if conf.has_option(section, "memory_max_usage"):
_memory_max_usage = conf.getint(section, "memory_max_usage")
if conf.has_option(section, 'memory_max_usage'):
_memory_max_usage = conf.getint(section, 'memory_max_usage')
if conf.has_option(section, "memory_usage_check_interval"):
_memory_usage_check_interval = conf.getint(section, "memory_usage_check_interval")
if conf.has_option(section, 'memory_usage_check_interval'):
_memory_usage_check_interval = conf.getint(section, 'memory_usage_check_interval')
if conf.has_option(section, "memory_usage_recovery_threshold"):
_memory_usage_recovery_threshold = conf.getfloat(section, "memory_usage_recovery_threshold")
if conf.has_option(section, 'memory_usage_recovery_threshold'):
_memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
_memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
or _memory_max_usage)
_memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
or _memory_usage_check_interval)
_memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
or _memory_usage_recovery_threshold)
_memory_max_usage = int(os.environ.get("RC_GUNICORN_MEMORY_MAX_USAGE", "") or _memory_max_usage)
_memory_usage_check_interval = int(
os.environ.get("RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL", "") or _memory_usage_check_interval
)
_memory_usage_recovery_threshold = float(
os.environ.get("RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD", "") or _memory_usage_recovery_threshold
)
return MemoryCheckConfig(_memory_max_usage, _memory_usage_check_interval, _memory_usage_recovery_threshold)
def _time_with_offset(check_interval):
return time.time() - random.randint(0, check_interval/2.0)
return time.time() - random.randint(0, check_interval / 2.0)
def pre_fork(server, worker):
@ -234,26 +236,27 @@ def pre_fork(server, worker):
def post_fork(server, worker):
memory_conf = get_memory_usage_params()
_memory_max_usage = memory_conf.max_usage
_memory_usage_check_interval = memory_conf.check_interval
_memory_usage_recovery_threshold = memory_conf.recovery_threshold
worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
or _memory_max_usage)
worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
or _memory_usage_check_interval)
worker._memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
or _memory_usage_recovery_threshold)
worker._memory_max_usage = int(os.environ.get("RC_GUNICORN_MEMORY_MAX_USAGE", "") or _memory_max_usage)
worker._memory_usage_check_interval = int(
os.environ.get("RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL", "") or _memory_usage_check_interval
)
worker._memory_usage_recovery_threshold = float(
os.environ.get("RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD", "") or _memory_usage_recovery_threshold
)
# register memory last check time, with some random offset so we don't recycle all
# at once
worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
if _memory_max_usage:
server.log.info("pid=[%-10s] WORKER spawned with max memory set at %s", worker.pid,
_format_data_size(_memory_max_usage))
server.log.info(
"pid=[%-10s] WORKER spawned with max memory set at %s", worker.pid, _format_data_size(_memory_max_usage)
)
else:
server.log.info("pid=[%-10s] WORKER spawned", worker.pid)
@ -263,9 +266,9 @@ def pre_exec(server):
def on_starting(server):
server_lbl = '{} {}'.format(server.proc_name, server.address)
server_lbl = "{} {}".format(server.proc_name, server.address)
server.log.info("Server %s is starting.", server_lbl)
server.log.info('Config:')
server.log.info("Config:")
server.log.info(f"\n{server.cfg}")
server.log.info(get_memory_usage_params())
@ -299,10 +302,10 @@ def _format_data_size(size, unit="B", precision=1, binary=True):
if not binary:
base = 1000
multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
multiples = ("", "k", "M", "G", "T", "P", "E", "Z", "Y")
else:
base = 1024
multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
multiples = ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi")
sign = ""
if size > 0:
@ -317,13 +320,13 @@ def _format_data_size(size, unit="B", precision=1, binary=True):
m = 8
if m == 0:
precision = '%.0f'
precision = "%.0f"
else:
precision = '%%.%df' % precision
precision = "%%.%df" % precision
size = precision % (size / math.pow(base, m))
return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
return "%s%s %s%s" % (sign, size.strip(), multiples[m], unit)
def _check_memory_usage(worker):
@ -339,8 +342,8 @@ def _check_memory_usage(worker):
mem_usage = _get_process_rss()
if mem_usage and mem_usage > _memory_max_usage:
worker.log.info(
"memory usage %s > %s, forcing gc",
_format_data_size(mem_usage), _format_data_size(_memory_max_usage))
"memory usage %s > %s, forcing gc", _format_data_size(mem_usage), _format_data_size(_memory_max_usage)
)
# Try to clean it up by forcing a full collection.
gc.collect()
mem_usage = _get_process_rss()
@ -348,7 +351,9 @@ def _check_memory_usage(worker):
# Didn't clean up enough, we'll have to terminate.
worker.log.warning(
"memory usage %s > %s after gc, quitting",
_format_data_size(mem_usage), _format_data_size(_memory_max_usage))
_format_data_size(mem_usage),
_format_data_size(_memory_max_usage),
)
# This will cause worker to auto-restart itself
worker.alive = False
worker._last_memory_check_time = time.time()
@ -364,8 +369,7 @@ def worker_int(worker):
code = []
for thread_id, stack in sys._current_frames().items(): # noqa
code.append(
"\n# Thread: %s(%d)" % (get_thread_id(thread_id), thread_id))
code.append("\n# Thread: %s(%d)" % (get_thread_id(thread_id), thread_id))
for fname, lineno, name, line in traceback.extract_stack(stack):
code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
if line:
@ -387,17 +391,21 @@ def child_exit(server, worker):
def pre_request(worker, req):
worker.start_time = time.time()
worker.log.debug(
"GNCRN PRE WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
worker.log.debug("GNCRN PRE WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
def post_request(worker, req, environ, resp):
total_time = time.time() - worker.start_time
# Gunicorn sometimes has problems with reading the status_code
status_code = getattr(resp, 'status_code', '')
status_code = getattr(resp, "status_code", "")
worker.log.debug(
"GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
worker.nr, req.method, req.path, status_code, total_time)
worker.nr,
req.method,
req.path,
status_code,
total_time,
)
_check_memory_usage(worker)
@ -410,8 +418,8 @@ def _filter_proxy(ip):
:param ip: ip string from headers
"""
if ',' in ip:
_ips = ip.split(',')
if "," in ip:
_ips = ip.split(",")
_first_ip = _ips[0].strip()
return _first_ip
return ip
@ -427,8 +435,9 @@ def _filter_port(ip):
:param ip:
"""
def is_ipv6(ip_addr):
if hasattr(socket, 'inet_pton'):
if hasattr(socket, "inet_pton"):
try:
socket.inet_pton(socket.AF_INET6, ip_addr)
except socket.error:
@ -437,24 +446,24 @@ def _filter_port(ip):
return False
return True
if ':' not in ip: # must be ipv4 pure ip
if ":" not in ip: # must be ipv4 pure ip
return ip
if '[' in ip and ']' in ip: # ipv6 with port
return ip.split(']')[0][1:].lower()
if "[" in ip and "]" in ip: # ipv6 with port
return ip.split("]")[0][1:].lower()
# must be ipv6 or ipv4 with port
if is_ipv6(ip):
return ip
else:
ip, _port = ip.split(':')[:2] # means ipv4+port
ip, _port = ip.split(":")[:2] # means ipv4+port
return ip
def get_ip_addr(environ):
proxy_key = 'HTTP_X_REAL_IP'
proxy_key2 = 'HTTP_X_FORWARDED_FOR'
def_key = 'REMOTE_ADDR'
proxy_key = "HTTP_X_REAL_IP"
proxy_key2 = "HTTP_X_FORWARDED_FOR"
def_key = "REMOTE_ADDR"
def _filters(x):
return _filter_port(_filter_proxy(x))
@ -467,7 +476,7 @@ def get_ip_addr(environ):
if ip:
return _filters(ip)
ip = environ.get(def_key, '0.0.0.0')
ip = environ.get(def_key, "0.0.0.0")
return _filters(ip)
@ -482,43 +491,40 @@ class RhodeCodeLogger(Logger):
Logger.__init__(self, cfg)
def now(self):
""" return date in RhodeCode Log format """
"""return date in RhodeCode Log format"""
now = time.time()
msecs = int((now - int(now)) * 1000)
return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
return time.strftime(self.datefmt, time.localtime(now)) + ".{0:03d}".format(msecs)
def atoms(self, resp, req, environ, request_time):
""" Gets atoms for log formatting.
"""
"""Gets atoms for log formatting."""
status = resp.status
if isinstance(status, str):
status = status.split(None, 1)[0]
atoms = {
'h': get_ip_addr(environ),
'l': '-',
'u': self._get_user(environ) or '-',
't': self.now(),
'r': "%s %s %s" % (environ['REQUEST_METHOD'],
environ['RAW_URI'],
environ["SERVER_PROTOCOL"]),
's': status,
'm': environ.get('REQUEST_METHOD'),
'U': environ.get('PATH_INFO'),
'q': environ.get('QUERY_STRING'),
'H': environ.get('SERVER_PROTOCOL'),
'b': getattr(resp, 'sent', None) is not None and str(resp.sent) or '-',
'B': getattr(resp, 'sent', None),
'f': environ.get('HTTP_REFERER', '-'),
'a': environ.get('HTTP_USER_AGENT', '-'),
'T': request_time.seconds,
'D': (request_time.seconds * 1000000) + request_time.microseconds,
'M': (request_time.seconds * 1000) + int(request_time.microseconds/1000),
'L': "%d.%06d" % (request_time.seconds, request_time.microseconds),
'p': "<%s>" % os.getpid()
"h": get_ip_addr(environ),
"l": "-",
"u": self._get_user(environ) or "-",
"t": self.now(),
"r": "%s %s %s" % (environ["REQUEST_METHOD"], environ["RAW_URI"], environ["SERVER_PROTOCOL"]),
"s": status,
"m": environ.get("REQUEST_METHOD"),
"U": environ.get("PATH_INFO"),
"q": environ.get("QUERY_STRING"),
"H": environ.get("SERVER_PROTOCOL"),
"b": getattr(resp, "sent", None) is not None and str(resp.sent) or "-",
"B": getattr(resp, "sent", None),
"f": environ.get("HTTP_REFERER", "-"),
"a": environ.get("HTTP_USER_AGENT", "-"),
"T": request_time.seconds,
"D": (request_time.seconds * 1000000) + request_time.microseconds,
"M": (request_time.seconds * 1000) + int(request_time.microseconds / 1000),
"L": "%d.%06d" % (request_time.seconds, request_time.microseconds),
"p": "<%s>" % os.getpid(),
}
# add request headers
if hasattr(req, 'headers'):
if hasattr(req, "headers"):
req_headers = req.headers
else:
req_headers = req