core: multiple fixes to unicode vs str usage

- python3 fixes
- import fixes
- list vs iterators
This commit is contained in:
RhodeCode Admin 2023-07-18 11:17:17 +02:00
parent 1f5a8fe199
commit abbc465e47
42 changed files with 222 additions and 148 deletions

View file

@ -36,7 +36,7 @@ from rhodecode import events
from rhodecode.lib import helpers as h
from rhodecode.lib.auth import (
LoginRequired, HasPermissionAllDecorator, CSRFRequired)
from rhodecode.lib.utils2 import aslist, safe_unicode
from rhodecode.lib.utils2 import aslist, safe_str
from rhodecode.model.db import (
or_, coalesce, User, UserIpMap, UserSshKeys)
from rhodecode.model.forms import (
@ -410,7 +410,7 @@ class AdminPermissionsView(BaseAppView, DataGridAppView):
base_q = UserSshKeys.query().join(UserSshKeys.user)
if search_q:
like_expression = u'%{}%'.format(safe_unicode(search_q))
like_expression = u'%{}%'.format(safe_str(search_q))
base_q = base_q.filter(or_(
User.username.ilike(like_expression),
UserSshKeys.ssh_key_fingerprint.ilike(like_expression),

View file

@ -36,7 +36,7 @@ from rhodecode.lib.auth import (
LoginRequired, CSRFRequired, NotAnonymous,
HasPermissionAny, HasRepoGroupPermissionAny)
from rhodecode.lib import helpers as h, audit_logger
from rhodecode.lib.utils2 import safe_int, safe_unicode, datetime_to_time
from rhodecode.lib.str_utils import safe_int, safe_str
from rhodecode.model.forms import RepoGroupForm
from rhodecode.model.permission import PermissionModel
from rhodecode.model.repo_group import RepoGroupModel

View file

@ -36,7 +36,7 @@ from rhodecode.lib.auth import (
HasPermissionAny, HasRepoGroupPermissionAny)
from rhodecode.lib import helpers as h
from rhodecode.lib.utils import repo_name_slug
from rhodecode.lib.utils2 import safe_int, safe_unicode
from rhodecode.lib.utils2 import safe_int, safe_str
from rhodecode.model.forms import RepoForm
from rhodecode.model.permission import PermissionModel
from rhodecode.model.repo import RepoModel
@ -58,7 +58,7 @@ class AdminReposView(BaseAppView, DataGridAppView):
acl_groups = RepoGroupList(RepoGroup.query().all(),
perm_set=['group.write', 'group.admin'])
c.repo_groups = RepoGroup.groups_choices(groups=acl_groups)
c.repo_groups_choices = map(lambda k: safe_unicode(k[0]), c.repo_groups)
c.repo_groups_choices = list(map(lambda k: safe_str(k[0]), c.repo_groups))
c.personal_repo_group = self._rhodecode_user.personal_repo_group
@LoginRequired()
@ -114,7 +114,7 @@ class AdminReposView(BaseAppView, DataGridAppView):
.group_by(Repository, User)
if search_q:
like_expression = u'%{}%'.format(safe_unicode(search_q))
like_expression = u'%{}%'.format(safe_str(search_q))
base_q = base_q.filter(or_(
Repository.repo_name.ilike(like_expression),
))

View file

@ -38,8 +38,9 @@ from rhodecode.lib import helpers as h
from rhodecode.lib.auth import (
LoginRequired, HasPermissionAllDecorator, CSRFRequired)
from rhodecode.lib.celerylib import tasks, run_task
from rhodecode.lib.str_utils import safe_str
from rhodecode.lib.utils import repo2db_mapper
from rhodecode.lib.utils2 import str2bool, safe_unicode, AttributeDict
from rhodecode.lib.utils2 import str2bool, AttributeDict
from rhodecode.lib.index import searcher_from_config
from rhodecode.model.db import RhodeCodeUi, Repository
@ -248,7 +249,8 @@ class AdminSettingsView(BaseAppView):
added, removed = repo2db_mapper(filesystem_repos, rm_obsolete)
PermissionModel().trigger_permission_flush()
_repr = lambda l: ', '.join(map(safe_unicode, l)) or '-'
def _repr(l):
return ', '.join(map(safe_str, l)) or '-'
h.flash(_('Repositories successfully '
'rescanned added: %s ; removed: %s') %
(_repr(added), _repr(removed)),
@ -311,6 +313,7 @@ class AdminSettingsView(BaseAppView):
('create_personal_repo_group', 'rhodecode_create_personal_repo_group', 'bool'),
('personal_repo_group_pattern', 'rhodecode_personal_repo_group_pattern', 'unicode'),
]
try:
for setting, form_key, type_ in settings:
sett = SettingsModel().create_or_update_setting(

View file

@ -19,7 +19,9 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
import urllib.request, urllib.error, urllib.parse
import urllib.request
import urllib.error
import urllib.parse
import os
import rhodecode
@ -206,7 +208,8 @@ class AdminSystemInfoSettingsView(BaseAppView):
update_url = UpdateModel().get_update_url()
_err = lambda s: '<div style="color:#ff8888; padding:4px 0px">{}</div>'.format(s)
def _err(s):
return '<div style="color:#ff8888; padding:4px 0px">{}</div>'.format(s)
try:
data = UpdateModel().get_update_data(update_url)
except urllib.error.URLError as e:

View file

@ -33,11 +33,10 @@ from rhodecode.apps._base import BaseAppView, DataGridAppView
from rhodecode.lib.auth import (
LoginRequired, NotAnonymous, CSRFRequired, HasPermissionAnyDecorator)
from rhodecode.lib import helpers as h, audit_logger
from rhodecode.lib.utils2 import safe_unicode
from rhodecode.lib.str_utils import safe_str
from rhodecode.model.forms import UserGroupForm
from rhodecode.model.permission import PermissionModel
from rhodecode.model.scm import UserGroupList
from rhodecode.model.db import (
or_, count, User, UserGroup, UserGroupMember, in_filter_generator)
from rhodecode.model.meta import Session
@ -129,7 +128,7 @@ class AdminUserGroupsView(BaseAppView, DataGridAppView):
base_q_inactive = base_q.filter(UserGroup.users_group_active != true())
if search_q:
like_expression = u'%{}%'.format(safe_unicode(search_q))
like_expression = u'%{}%'.format(safe_str(search_q))
base_q = base_q.filter(or_(
UserGroup.users_group_name.ilike(like_expression),
))

View file

@ -45,7 +45,7 @@ from rhodecode.lib.auth import (
LoginRequired, HasPermissionAllDecorator, CSRFRequired)
from rhodecode.lib import helpers as h
from rhodecode.lib.helpers import SqlPage
from rhodecode.lib.utils2 import safe_int, safe_unicode, AttributeDict
from rhodecode.lib.utils2 import safe_int, safe_str, AttributeDict
from rhodecode.model.auth_token import AuthTokenModel
from rhodecode.model.forms import (
UserForm, UserIndividualPermissionsForm, UserPermissionsForm,
@ -106,7 +106,7 @@ class AdminUsersView(BaseAppView, DataGridAppView):
base_inactive_q = base_q.filter(User.active != true())
if search_q:
like_expression = '%{}%'.format(safe_unicode(search_q))
like_expression = '%{}%'.format(safe_str(search_q))
base_q = base_q.filter(or_(
User.username.ilike(like_expression),
User._email.ilike(like_expression),
@ -223,7 +223,7 @@ class AdminUsersView(BaseAppView, DataGridAppView):
)
return Response(html)
except UserCreationError as e:
h.flash(safe_unicode(e), 'error')
h.flash(safe_str(e), 'error')
except Exception:
log.exception("Exception creation of user")
h.flash(_('Error occurred during creation of user %s')
@ -347,7 +347,7 @@ class UsersView(UserAppView):
)
return Response(html)
except UserCreationError as e:
h.flash(safe_unicode(e), 'error')
h.flash(safe_str(e), 'error')
except Exception:
log.exception("Exception updating user")
h.flash(_('Error occurred during update of user %s')
@ -470,7 +470,8 @@ class UsersView(UserAppView):
except (UserOwnsReposException, UserOwnsRepoGroupsException,
UserOwnsUserGroupsException, UserOwnsPullRequestsException,
UserOwnsArtifactsException, DefaultUserException) as e:
h.flash(e, category='warning')
h.flash(safe_str(e), category='warning')
except Exception:
log.exception("Exception during deletion of user")
h.flash(_('An error occurred during deletion of user'),

View file

@ -23,8 +23,8 @@ from pyramid.events import ApplicationCreated
from pyramid.settings import asbool
from rhodecode.apps._base import ADMIN_PREFIX
from rhodecode.lib.ext_json import json
from rhodecode.lib.str_utils import safe_str
from rhodecode.lib.ext_json import str_json
def url_gen(request):
@ -38,7 +38,7 @@ def url_gen(request):
'longpoll': longpoll_url or proxy_url,
'ws': ws_url or proxy_url.replace('http', 'ws')
}
return safe_str(json.dumps(urls))
return str_json(urls)
PLUGIN_DEFINITION = {

View file

@ -25,7 +25,7 @@ class VCSCallPredicate(object):
self.val = val
def text(self):
return 'vcs_call route = %s' % self.val
return f'vcs_call route = {self.val}'
phash = text

View file

@ -31,7 +31,7 @@ from rhodecode.lib.auth import (
HasRepoGroupPermissionAny, AuthUser)
from rhodecode.lib.codeblocks import filenode_as_lines_tokens
from rhodecode.lib.index import searcher_from_config
from rhodecode.lib.utils2 import safe_unicode, str2bool, safe_int, safe_str
from rhodecode.lib.utils2 import str2bool, safe_int, safe_str
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.model.db import (
func, true, or_, case, cast, in_filter_generator, String, Session,
@ -129,7 +129,7 @@ class HomeView(BaseAppView, DataGridAppView):
query = query.filter(Repository.repo_type == repo_type)
if name_contains:
ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
ilike_expression = '%{}%'.format(safe_str(name_contains))
query = query.filter(
Repository.repo_name.ilike(ilike_expression))
query = query.limit(limit)
@ -174,7 +174,7 @@ class HomeView(BaseAppView, DataGridAppView):
query = query.order_by(RepoGroup.group_name)
if name_contains:
ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
ilike_expression = u'%{}%'.format(safe_str(name_contains))
query = query.filter(
RepoGroup.group_name.ilike(ilike_expression))
query = query.limit(limit)
@ -216,7 +216,7 @@ class HomeView(BaseAppView, DataGridAppView):
.filter(User.username != User.DEFAULT_USER)
if name_contains:
ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
ilike_expression = u'%{}%'.format(safe_str(name_contains))
query = query.filter(
User.username.ilike(ilike_expression))
query = query.limit(limit)
@ -256,7 +256,7 @@ class HomeView(BaseAppView, DataGridAppView):
.order_by(UserGroup.users_group_name)
if name_contains:
ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
ilike_expression = u'%{}%'.format(safe_str(name_contains))
query = query.filter(
UserGroup.users_group_name.ilike(ilike_expression))
query = query.limit(limit)
@ -308,7 +308,7 @@ class HomeView(BaseAppView, DataGridAppView):
query = query.order_by(PullRequest.pull_request_id)
if name_contains:
ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
ilike_expression = u'%{}%'.format(safe_str(name_contains))
query = query.filter(or_(
cast(PullRequest.pull_request_id, String).ilike(ilike_expression),
PullRequest.title.ilike(ilike_expression),

View file

@ -32,7 +32,7 @@ from rhodecode.lib.auth import (
HasRepoPermissionAnyDecorator)
from rhodecode.lib.codeblocks import filenode_as_lines_tokens
from rhodecode.lib.index import searcher_from_config
from rhodecode.lib.utils2 import safe_unicode, str2bool, safe_int
from rhodecode.lib.utils2 import str2bool, safe_int
from rhodecode.lib.ext_json import json
from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError, EmptyRepositoryError
from rhodecode.lib.vcs.nodes import FileNode

View file

@ -118,7 +118,7 @@ class JournalView(BaseAppView):
return journal
def feed_uid(self, entry_id):
return '{}:{}'.format('journal', md5_safe(entry_id))
return '{}:{}'.format('journal', md5_safe(str(entry_id)))
def _atom_feed(self, repos, search_term, public=True):
_ = self.request.translate

View file

@ -37,6 +37,7 @@ from rhodecode.lib.auth import (
HasRepoPermissionAny, HasRepoGroupPermissionAny, AuthUser)
from rhodecode.lib.channelstream import (
channelstream_request, ChannelstreamException)
from rhodecode.lib.hash_utils import md5_safe
from rhodecode.lib.utils2 import safe_int, md5, str2bool
from rhodecode.model.auth_token import AuthTokenModel
from rhodecode.model.comment import CommentsModel
@ -114,7 +115,7 @@ class MyAccountView(BaseAppView, DataGridAppView):
form = forms.RcForm(
schema, buttons=(forms.buttons.save, forms.buttons.reset))
controls = self.request.POST.items()
controls = list(self.request.POST.items())
try:
valid_data = form.validate(controls)
skip_attrs = ['admin', 'active', 'extern_type', 'extern_name',
@ -179,7 +180,7 @@ class MyAccountView(BaseAppView, DataGridAppView):
if c.extern_type != 'rhodecode':
raise HTTPFound(self.request.route_path('my_account_password'))
controls = self.request.POST.items()
controls = list(self.request.POST.items())
try:
valid_data = form.validate(controls)
UserModel().update_user(c.user.user_id, **valid_data)
@ -196,7 +197,7 @@ class MyAccountView(BaseAppView, DataGridAppView):
else:
instance = c.auth_user.get_instance()
self.session.setdefault('rhodecode_user', {}).update(
{'password': md5(instance.password)})
{'password': md5_safe(instance.password)})
self.session.save()
h.flash(_("Successfully updated password"), category='success')
@ -326,7 +327,7 @@ class MyAccountView(BaseAppView, DataGridAppView):
schema, action=h.route_path('my_account_emails_add'),
buttons=(forms.buttons.save, forms.buttons.reset))
controls = self.request.POST.items()
controls = list(self.request.POST.items())
try:
valid_data = form.validate(controls)
UserModel().add_extra_email(c.user.user_id, valid_data['email'])

View file

@ -52,7 +52,7 @@ class RepoCachesView(RepoAppView):
c.cached_diff_size = system_info.get_storage_size(cached_diffs_dir)
c.shadow_repos = c.rhodecode_db_repo.shadow_repos()
cache_namespace_uid = 'cache_repo.{}'.format(self.db_repo.repo_id)
cache_namespace_uid = 'repo.{}'.format(self.db_repo.repo_id)
c.region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
c.backend = c.region.backend
c.repo_keys = sorted(c.region.backend.list_keys(prefix=cache_namespace_uid))

View file

@ -31,10 +31,10 @@ from rhodecode.lib import ext_json
from rhodecode.lib.auth import (
LoginRequired, HasRepoPermissionAnyDecorator)
from rhodecode.lib.ext_json import json
from rhodecode.lib.graphmod import _colored, _dagwalker
from rhodecode.lib.helpers import RepoPage
from rhodecode.lib.utils2 import safe_int, safe_str, str2bool, safe_unicode
from rhodecode.lib.utils2 import str2bool
from rhodecode.lib.str_utils import safe_int, safe_str
from rhodecode.lib.vcs.exceptions import (
RepositoryError, CommitDoesNotExistError,
CommitError, NodeDoesNotExistError, EmptyRepositoryError)
@ -83,7 +83,7 @@ class RepoChangelogView(RepoAppView):
:param commits: list of commits
"""
if not commits:
return json.dumps([]), json.dumps([])
return ext_json.str_json([]), ext_json.str_json([])
def serialize(commit, parents=True):
data = dict(
@ -100,6 +100,7 @@ class RepoChangelogView(RepoAppView):
next_data = next_data or []
current = [serialize(x) for x in commits]
commits = prev_data + current + next_data
dag = _dagwalker(repo, commits)
@ -110,7 +111,7 @@ class RepoChangelogView(RepoAppView):
def _check_if_valid_branch(self, branch_name, repo_name, f_path):
if branch_name not in self.rhodecode_vcs_repo.branches_all:
h.flash(u'Branch {} is not found.'.format(h.escape(safe_unicode(branch_name))),
h.flash(u'Branch {} is not found.'.format(h.escape(safe_str(branch_name))),
category='warning')
redirect_url = h.route_path(
'repo_commits_file', repo_name=repo_name,
@ -334,8 +335,8 @@ class RepoChangelogView(RepoAppView):
next_data = None
try:
prev_graph = json.loads(self.request.POST.get('graph') or '{}')
except json.JSONDecodeError:
prev_graph = ext_json.json.loads(self.request.POST.get('graph') or '{}')
except ext_json.json.JSONDecodeError:
prev_graph = {}
if self.request.GET.get('chunk') == 'prev':

View file

@ -39,7 +39,7 @@ from rhodecode.lib.diffs import (
get_diff_whitespace_flag)
from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError, CommentVersionMismatch
import rhodecode.lib.helpers as h
from rhodecode.lib.utils2 import safe_unicode, str2bool, StrictAttributeDict, safe_str
from rhodecode.lib.utils2 import str2bool, StrictAttributeDict, safe_str
from rhodecode.lib.vcs.backends.base import EmptyCommit
from rhodecode.lib.vcs.exceptions import (
RepositoryError, CommitDoesNotExistError)
@ -308,7 +308,8 @@ class RepoCommitsView(RepoAppView):
'attachment; filename=%s.diff' % commit_id_range[:12])
return response
elif method == 'patch':
c.diff = safe_unicode(diff)
c.diff = safe_str(diff)
patch = render(
'rhodecode:templates/changeset/patch_changeset.mako',
self._get_template_context(c), self.request)
@ -381,7 +382,7 @@ class RepoCommitsView(RepoAppView):
resolves_comment_id = entry['resolves_comment_id']
f_path = entry['f_path']
line_no = entry['line']
target_elem_id = 'file-{}'.format(h.safeid(h.safe_unicode(f_path)))
target_elem_id = 'file-{}'.format(h.safeid(h.safe_str(f_path)))
if status:
text = text or (_('Status change %(transition_icon)s %(status)s')
@ -626,7 +627,7 @@ class RepoCommitsView(RepoAppView):
file_uid=store_uid, filename=metadata["filename"],
file_hash=metadata["sha256"], file_size=metadata["size"],
file_display_name=file_display_name,
file_description=u'comment attachment `{}`'.format(safe_unicode(filename)),
file_description=u'comment attachment `{}`'.format(safe_str(filename)),
hidden=True, check_acl=True, user_id=self._rhodecode_user.user_id,
scope_repo_id=self.db_repo.repo_id
)

View file

@ -32,7 +32,7 @@ from rhodecode.lib import helpers as h
from rhodecode.lib import diffs, codeblocks
from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
from rhodecode.lib.utils import safe_str
from rhodecode.lib.utils2 import safe_unicode, str2bool
from rhodecode.lib.utils2 import str2bool
from rhodecode.lib.view_utils import parse_path_ref, get_commit_from_ref_name
from rhodecode.lib.vcs.exceptions import (
EmptyRepositoryError, RepositoryError, RepositoryRequirementError,
@ -259,9 +259,10 @@ class RepoCompareView(RepoAppView):
log.debug('calculating diff between '
'source_ref:%s and target_ref:%s for repo `%s`',
source_commit, target_commit,
safe_unicode(source_repo.scm_instance().path))
safe_str(source_repo.scm_instance().path))
if source_commit.repository != target_commit.repository:
msg = _(
"Repositories unrelated. "
"Cannot compare commit %(commit1)s from repository %(repo1)s "

View file

@ -34,13 +34,13 @@ from rhodecode.lib.auth import (
LoginRequired, HasRepoPermissionAnyDecorator, NotAnonymous,
HasRepoPermissionAny, HasPermissionAnyDecorator, CSRFRequired)
import rhodecode.lib.helpers as h
from rhodecode.lib.str_utils import safe_str
from rhodecode.lib.celerylib.utils import get_task_id
from rhodecode.model.db import coalesce, or_, Repository, RepoGroup
from rhodecode.model.permission import PermissionModel
from rhodecode.model.repo import RepoModel
from rhodecode.model.forms import RepoForkForm
from rhodecode.model.scm import ScmModel, RepoGroupList
from rhodecode.lib.utils2 import safe_int, safe_unicode
log = logging.getLogger(__name__)
@ -55,7 +55,7 @@ class RepoForksView(RepoAppView, DataGridAppView):
RepoGroup.query().all(),
perm_set=['group.write', 'group.admin'])
c.repo_groups = RepoGroup.groups_choices(groups=acl_groups)
c.repo_groups_choices = map(lambda k: safe_unicode(k[0]), c.repo_groups)
c.repo_groups_choices = list(map(lambda k: safe_str(k[0]), c.repo_groups))
c.personal_repo_group = c.rhodecode_user.personal_repo_group
@ -102,7 +102,7 @@ class RepoForksView(RepoAppView, DataGridAppView):
.filter(Repository.repo_id.in_(allowed_ids))\
if search_q:
like_expression = u'%{}%'.format(safe_unicode(search_q))
like_expression = u'%{}%'.format(safe_str(search_q))
base_q = base_q.filter(or_(
Repository.repo_name.ilike(like_expression),
Repository.description.ilike(like_expression),

View file

@ -39,7 +39,7 @@ from rhodecode.lib import ext_json
from rhodecode.lib.auth import (
LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator,
NotAnonymous, CSRFRequired)
from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode, safe_int, aslist, retry
from rhodecode.lib.utils2 import str2bool, safe_str, safe_int, aslist, retry
from rhodecode.lib.vcs.backends.base import (
EmptyCommit, UpdateFailureReason, unicode_to_reference)
from rhodecode.lib.vcs.exceptions import (
@ -953,7 +953,7 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
.filter(~Repository.repo_id.in_([x.repo_id for x in parent_target_repos]))
if filter_query:
ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
ilike_expression = u'%{}%'.format(safe_str(filter_query))
query = query.filter(Repository.repo_name.ilike(ilike_expression))
limit = max(20 - len(parent_target_repos), 5) # not less then 5
@ -1120,7 +1120,7 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
_form = form.to_python(controls)
except formencode.Invalid as errors:
if errors.error_dict.get('revisions'):
msg = 'Revisions: %s' % errors.error_dict['revisions']
msg = 'Revisions: {}'.format(errors.error_dict['revisions'])
elif errors.error_dict.get('pullrequest_title'):
msg = errors.error_dict.get('pullrequest_title')
else:
@ -1567,7 +1567,7 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
close_pull_request = entry['close_pull_request']
f_path = entry['f_path']
line_no = entry['line']
target_elem_id = 'file-{}'.format(h.safeid(h.safe_unicode(f_path)))
target_elem_id = 'file-{}'.format(h.safeid(h.safe_str(f_path)))
# the logic here should work like following, if we submit close
# pr comment, use `close_pull_request_with_comment` function
@ -1867,7 +1867,7 @@ class RepoPullRequestsView(RepoAppView, DataGridAppView):
'comment_id': comment.comment_id,
'comment_version': comment_history.version,
'comment_author_username': comment_history.author.username,
'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16),
'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16, request=self.request),
'comment_created_on': h.age_component(comment_history.created_on,
time_is_local=True),
}

View file

@ -49,7 +49,7 @@ class RepoSettingsView(RepoAppView):
RepoGroup.query().all(),
perm_set=['group.write', 'group.admin'])
c.repo_groups = RepoGroup.groups_choices(groups=acl_groups)
c.repo_groups_choices = map(lambda k: k[0], c.repo_groups)
c.repo_groups_choices = list(map(lambda k: k[0], c.repo_groups))
# in case someone no longer have a group.write access to a repository
# pre fill the list with this entry, we don't care if this is the same
@ -114,7 +114,7 @@ class RepoSettingsView(RepoAppView):
schema = self._get_schema(c, old_values=old_values)
c.form = RcForm(schema)
pstruct = self.request.POST.items()
pstruct = list(self.request.POST.items())
pstruct.append(('repo_type', self.db_repo.repo_type))
try:
schema_data = c.form.validate(pstruct)

View file

@ -161,7 +161,7 @@ class RepoSummaryView(RepoAppView):
'with caching: %s[TTL: %ss]' % (
repo_id, landing_commit, cache_on, cache_seconds or 0))
cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
cache_namespace_uid = 'repo.{}'.format(repo_id)
region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
@region.conditional_cache_on_arguments(namespace=cache_namespace_uid,

View file

@ -19,7 +19,9 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
import urllib.request, urllib.parse, urllib.error
import urllib.request
import urllib.parse
import urllib.error
from webhelpers2.html.tools import update_params

View file

@ -20,10 +20,10 @@
import os
import sys
import json
import logging
from rhodecode.lib.hooks_daemon import prepare_callback_daemon
from rhodecode.lib.ext_json import sjson as json
from rhodecode.lib.vcs.conf import settings as vcs_settings
from rhodecode.model.scm import ScmModel

View file

@ -1,4 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2020 RhodeCode GmbH
#

View file

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2010-2020 RhodeCode GmbH
#

View file

@ -24,6 +24,7 @@ import datetime
from rhodecode.lib.jsonalchemy import JsonRaw
from rhodecode.model import meta
from rhodecode.model.db import User, UserLog, Repository
from rhodecode.lib.str_utils import safe_str
log = logging.getLogger(__name__)
@ -215,7 +216,6 @@ def store(action, user, action_data=None, user_data=None, ip_addr=None,
ip_addr=self.request.remote_addr))
"""
from rhodecode.lib.utils2 import safe_unicode
from rhodecode.lib.auth import AuthUser
action_spec = ACTIONS.get(action, None)
@ -257,8 +257,8 @@ def store(action, user, action_data=None, user_data=None, ip_addr=None,
repository_id = getattr(
Repository.get_by_repo_name(repository_name), 'repo_id', None)
action_name = safe_unicode(action)
ip_address = safe_unicode(ip_addr)
action_name = safe_str(action)
ip_address = safe_str(ip_addr)
with sa_session.no_autoflush:

View file

@ -17,8 +17,6 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
import json
import logging
import datetime
import time
@ -30,6 +28,8 @@ from celery.result import AsyncResult
import celery.loaders.base
import celery.schedules
from rhodecode.lib.ext_json import sjson as json
log = logging.getLogger(__name__)

View file

@ -64,7 +64,7 @@ def date_to_timestamp_plus_offset(value):
assert not is_aware(value), (
"This code is not prepared to handle aware datetime instances")
value = date_astimestamp(value)
return (value, time.timezone)
return value, time.timezone
def is_aware(value):

View file

@ -6,7 +6,7 @@ import six
from xml.sax.saxutils import XMLGenerator, quoteattr
from urllib.parse import quote
from rhodecode.lib.utils import safe_str, safe_unicode
from rhodecode.lib.str_utils import safe_str
class SimplerXMLGenerator(XMLGenerator):
@ -54,4 +54,4 @@ def iri_to_uri(iri):
def force_text(text, strings_only=False):
return safe_unicode(text)
return safe_str(text)

View file

@ -20,6 +20,6 @@
# base64 filter e.g ${ example | base64,n }
def base64(text):
import base64
from rhodecode.lib.helpers import safe_str, safe_bytes
return safe_str(base64.encodebytes(safe_bytes(text)))
from rhodecode.lib.str_utils import base64_to_str
return base64_to_str(text)

View file

@ -34,7 +34,7 @@ from whoosh.qparser import QueryParser, QueryParserError
import rhodecode.lib.helpers as h
from rhodecode.lib.index import BaseSearcher
from rhodecode.lib.utils2 import safe_unicode
from rhodecode.lib.str_utils import safe_str
log = logging.getLogger(__name__)
@ -144,7 +144,7 @@ class WhooshSearcher(BaseSearcher):
allowed_repos_filter = self._get_repo_filter(
search_user, repo_name)
try:
query = qp.parse(safe_unicode(query))
query = qp.parse(safe_str(query))
log.debug('query: %s (%s)', query, repr(query))
reverse, sorted_by = False, None

View file

@ -19,7 +19,9 @@
import base64
import logging
import urllib.request, urllib.parse, urllib.error
import urllib.request
import urllib.parse
import urllib.error
import urllib.parse
import requests
@ -29,7 +31,8 @@ from rhodecode.lib import rc_cache
from rhodecode.lib.middleware import simplevcs
from rhodecode.lib.middleware.utils import get_path_info
from rhodecode.lib.utils import is_valid_repo
from rhodecode.lib.utils2 import str2bool, safe_int, safe_str
from rhodecode.lib.str_utils import safe_str, safe_int
from rhodecode.lib.type_utils import str2bool
from rhodecode.lib.ext_json import json
from rhodecode.lib.hooks_daemon import store_txn_id_data
@ -99,7 +102,6 @@ class SimpleSvnApp(object):
raise
if response.status_code not in [200, 401]:
from rhodecode.lib.utils2 import safe_str
text = '\n{}'.format(safe_str(response.text)) if response.text else ''
if response.status_code >= 500:
log.error('Got SVN response:%s with text:`%s`', response, text)

View file

@ -50,7 +50,7 @@ from rhodecode.lib.middleware import appenlight
from rhodecode.lib.middleware.utils import scm_app_http
from rhodecode.lib.str_utils import safe_bytes
from rhodecode.lib.utils import is_valid_repo, SLUG_RE
from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
from rhodecode.lib.vcs.conf import settings as vcs_settings
from rhodecode.lib.vcs.backends import base
@ -194,7 +194,7 @@ class SimpleVCS(object):
match_dict = match.groupdict()
# Build acl repo name from regex match.
acl_repo_name = safe_unicode('{groups}{target}'.format(
acl_repo_name = safe_str('{groups}{target}'.format(
groups=match_dict['groups'] or '',
target=match_dict['target']))
@ -471,8 +471,9 @@ class SimpleVCS(object):
'authentication')
if not anonymous_perm:
log.debug('Not enough credentials to access this '
'repository as anonymous user')
log.debug('Not enough credentials to access repo: `%s` '
'repository as anonymous user', self.acl_repo_name)
username = None
# ==============================================================
@ -481,13 +482,16 @@ class SimpleVCS(object):
# ==============================================================
# try to auth based on environ, container auth methods
log.debug('Running PRE-AUTH for container based authentication')
log.debug('Running PRE-AUTH for container|headers based authentication')
# headers auth, by just reading special headers and bypass the auth with user/passwd
pre_auth = authenticate(
'', '', environ, VCS_TYPE, registry=self.registry,
acl_repo_name=self.acl_repo_name)
if pre_auth and pre_auth.get('username'):
username = pre_auth['username']
log.debug('PRE-AUTH got %s as username', username)
log.debug('PRE-AUTH got `%s` as username', username)
if pre_auth:
log.debug('PRE-AUTH successful from %s',
pre_auth.get('auth_data', {}).get('_plugin'))
@ -498,6 +502,8 @@ class SimpleVCS(object):
plugin_cache_active, cache_ttl = False, 0
plugin = None
# regular auth chain
if not username:
self.authenticate.realm = self.authenticate.get_rc_realm()
@ -546,7 +552,7 @@ class SimpleVCS(object):
plugin, plugin_cache_active, cache_ttl)
if not perm:
return HTTPForbidden()(environ, start_response)
environ['rc_auth_user_id'] = user_id
environ['rc_auth_user_id'] = str(user_id)
if action == 'push':
perms = auth_user.get_branch_permissions(self.acl_repo_name)

View file

@ -870,7 +870,6 @@ class _Page(list):
return make_html_tag("a", text=text, href=target_url, **item["attrs"])
# Below is RhodeCode custom code
# Copyright (C) 2010-2020 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify

View file

@ -1,4 +1,4 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2013-2020 RhodeCode GmbH
#

View file

@ -17,11 +17,15 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import typing
import base64
import logging
from unidecode import unidecode
import rhodecode
import unicodedata
from rhodecode.lib.type_utils import aslist
log = logging.getLogger(__name__)
@ -42,18 +46,40 @@ def safe_int(val, default=None) -> int:
return val
def get_default_encodings():
def safe_float(val, default=None) -> float:
"""
Returns float() of val if val is not convertable to float use default
instead
:param val:
:param default:
"""
try:
val = float(val)
except (ValueError, TypeError):
val = default
return val
def base64_to_str(text) -> str:
return safe_str(base64.encodebytes(safe_bytes(text))).strip()
def get_default_encodings() -> typing.List[str]:
return aslist(rhodecode.CONFIG.get('default_encoding', 'utf8'), sep=',')
DEFAULT_ENCODINGS = get_default_encodings()
def safe_str(str_, to_encoding=None) -> str:
"""
safe str function. Does few trick to turn unicode_ into string
:param str_: str to encode
:param to_encoding: encode to this type UTF8 default
:rtype: str
:returns: str object
"""
if isinstance(str_, str):
return str_
@ -62,7 +88,7 @@ def safe_str(str_, to_encoding=None) -> str:
if not isinstance(str_, bytes):
return str(str_)
to_encoding = to_encoding or get_default_encodings()
to_encoding = to_encoding or DEFAULT_ENCODINGS
if not isinstance(to_encoding, (list, tuple)):
to_encoding = [to_encoding]
@ -81,14 +107,12 @@ def safe_bytes(str_, from_encoding=None) -> bytes:
:param str_: string to decode
:param from_encoding: encode from this type UTF8 default
:rtype: unicode
:returns: unicode object
"""
if isinstance(str_, bytes):
return str_
if not isinstance(str_, str):
raise ValueError('safe_bytes cannot convert other types than str: got: {}'.format(type(str_)))
raise ValueError(f'safe_bytes cannot convert other types than str: got: {type(str_)}')
from_encoding = from_encoding or get_default_encodings()
if not isinstance(from_encoding, (list, tuple)):
@ -116,11 +140,11 @@ def ascii_bytes(str_, allow_bytes=False) -> bytes:
return str_
if not isinstance(str_, str):
raise ValueError('ascii_bytes cannot convert other types than str: got: {}'.format(type(str_)))
raise ValueError(f'ascii_bytes cannot convert other types than str: got: {type(str_)}')
return str_.encode('ascii')
def ascii_str(str_):
def ascii_str(str_) -> str:
"""
Simple conversion from bytes to str, with assumption that str_ is pure ASCII.
Fails with UnicodeError on invalid input.
@ -131,16 +155,16 @@ def ascii_str(str_):
"""
if not isinstance(str_, bytes):
raise ValueError('ascii_str cannot convert other types than bytes: got: {}'.format(type(str_)))
raise ValueError(f'ascii_str cannot convert other types than bytes: got: {type(str_)}')
return str_.decode('ascii')
def convert_special_chars(str_):
def convert_special_chars(str_) -> str:
"""
trie to replace non-ascii letters to their ascii representation eg::
`żołw` converts into `zolw`
"""
value = safe_str(str_)
converted_value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode()
converted_value = unidecode(value)
return converted_value

View file

@ -191,7 +191,7 @@ def locale_info():
return f'FAILED_LOCALE_GET:{locale_name}'
value = dict(
locale_default=locale.getdefaultlocale(),
locale_default=locale.getlocale(),
locale_lc_all=safe_get_locale(locale.LC_ALL),
locale_lc_ctype=safe_get_locale(locale.LC_CTYPE),
lang_env=os.environ.get('LANG'),
@ -404,9 +404,9 @@ def storage_archives():
from rhodecode.lib.utils import safe_str
from rhodecode.lib.helpers import format_byte_size_binary
msg = 'Enable this by setting ' \
'archive_cache_dir=/path/to/cache option in the .ini file'
path = safe_str(rhodecode.CONFIG.get('archive_cache_dir', msg))
msg = 'Archive cache storage is controlled by ' \
'archive_cache.store_dir=/path/to/cache option in the .ini file'
path = safe_str(rhodecode.CONFIG.get('archive_cache.store_dir', msg))
value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
state = STATE_OK_DEFAULT
@ -415,7 +415,7 @@ def storage_archives():
used = 0
for root, dirs, files in os.walk(path):
if root == path:
items_count = len(files)
items_count = len(dirs)
for f in files:
try:
@ -726,7 +726,8 @@ def rhodecode_config():
def database_info():
import rhodecode
from sqlalchemy.engine import url as engine_url
from rhodecode.model.meta import Base as sql_base, Session
from rhodecode.model import meta
from rhodecode.model.meta import Session
from rhodecode.model.db import DbMigrateVersion
state = STATE_OK_DEFAULT
@ -737,7 +738,7 @@ def database_info():
db_url_obj = engine_url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
try:
engine = sql_base.metadata.bind
engine = meta.get_engine()
db_server_info = engine.dialect._get_server_version_info(
Session.connection(bind=engine))
db_version = '.'.join(map(str, db_server_info))

View file

@ -40,7 +40,7 @@ def str2bool(str_):
return str_ in ('t', 'true', 'y', 'yes', 'on', '1')
def aslist(obj, sep=None, strip=True):
def aslist(obj, sep=None, strip=True) -> list:
"""
Returns given string separated by sep as list
@ -49,6 +49,9 @@ def aslist(obj, sep=None, strip=True):
:param strip:
"""
if isinstance(obj, str):
if obj in ['', ""]:
return []
lst = obj.split(sep)
if strip:
lst = [v.strip() for v in lst]
@ -59,3 +62,32 @@ def aslist(obj, sep=None, strip=True):
return []
else:
return [obj]
class AttributeDictBase(dict):
def __getstate__(self):
odict = self.__dict__ # get attribute dictionary
return odict
def __setstate__(self, dict):
self.__dict__ = dict
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
class StrictAttributeDict(AttributeDictBase):
"""
Strict Version of Attribute dict which raises an Attribute error when
requested attribute is not set
"""
def __getattr__(self, attr):
try:
return self[attr]
except KeyError:
raise AttributeError(f'{self.__class__} object has no attribute {attr}')
class AttributeDict(AttributeDictBase):
def __getattr__(self, attr):
return self.get(attr, None)

View file

@ -25,7 +25,9 @@ from whoosh.fields import (TEXT, Schema, DATETIME, KEYWORD)
from sqlalchemy.sql.expression import or_, and_, not_, func
from rhodecode.model.db import UserLog
from rhodecode.lib.utils2 import remove_prefix, remove_suffix, safe_unicode
from rhodecode.lib.utils2 import remove_prefix, remove_suffix
from rhodecode.lib.str_utils import safe_str
# JOURNAL SCHEMA used only to generate queries in journal. We use whoosh
# querylang to build sql queries and filter journals
@ -54,7 +56,7 @@ def user_log_filter(user_log, search_term):
if search_term:
qp = QueryParser('repository', schema=AUDIT_LOG_SCHEMA)
qp.add_plugin(DateParserPlugin())
qry = qp.parse(safe_unicode(search_term))
qry = qp.parse(safe_str(search_term))
log.debug('Filtering using parsed query %r', qry)
def wildcard_handler(col, wc_term):

View file

@ -38,7 +38,7 @@ class BaseAuthSessions(object):
def __init__(self, config):
session_conf = {}
for k, v in config.items():
for k, v in list(config.items()):
if k.startswith('beaker.session'):
session_conf[k] = v
self.config = session_conf

View file

@ -30,7 +30,9 @@ import logging
import re
import sys
import time
import urllib.request, urllib.parse, urllib.error
import urllib.request
import urllib.parse
import urllib.error
import urlobject
import uuid
import getpass

View file

@ -37,8 +37,7 @@ from zope.cachedescriptors.property import Lazy as LazyProperty
import rhodecode
from rhodecode.translation import lazy_ugettext
from rhodecode.lib.utils2 import safe_str, safe_unicode, CachedProperty
from rhodecode.lib.vcs import connection
from rhodecode.lib.utils2 import safe_str, CachedProperty
from rhodecode.lib.vcs.utils import author_name, author_email
from rhodecode.lib.vcs.conf import settings
from rhodecode.lib.vcs.exceptions import (
@ -72,7 +71,7 @@ class Reference(_Reference):
return self.name
@property
def to_unicode(self):
def to_str(self):
return reference_to_unicode(self)
@ -88,13 +87,13 @@ def unicode_to_reference(raw):
return None
def reference_to_unicode(ref):
def reference_to_unicode(ref: Reference):
"""
Convert a reference object to unicode.
If reference is None it returns None.
"""
if ref:
return u':'.join(ref)
return ':'.join(ref)
else:
return None
@ -187,37 +186,37 @@ class MergeResponse(object):
# uses .format(**metadata) for variables
MERGE_STATUS_MESSAGES = {
MergeFailureReason.NONE: lazy_ugettext(
u'This pull request can be automatically merged.'),
'This pull request can be automatically merged.'),
MergeFailureReason.UNKNOWN: lazy_ugettext(
u'This pull request cannot be merged because of an unhandled exception. '
u'{exception}'),
'This pull request cannot be merged because of an unhandled exception. '
'{exception}'),
MergeFailureReason.MERGE_FAILED: lazy_ugettext(
u'This pull request cannot be merged because of merge conflicts. {unresolved_files}'),
'This pull request cannot be merged because of merge conflicts. {unresolved_files}'),
MergeFailureReason.PUSH_FAILED: lazy_ugettext(
u'This pull request could not be merged because push to '
u'target:`{target}@{merge_commit}` failed.'),
'This pull request could not be merged because push to '
'target:`{target}@{merge_commit}` failed.'),
MergeFailureReason.TARGET_IS_NOT_HEAD: lazy_ugettext(
u'This pull request cannot be merged because the target '
u'`{target_ref.name}` is not a head.'),
'This pull request cannot be merged because the target '
'`{target_ref.name}` is not a head.'),
MergeFailureReason.HG_SOURCE_HAS_MORE_BRANCHES: lazy_ugettext(
u'This pull request cannot be merged because the source contains '
u'more branches than the target.'),
'This pull request cannot be merged because the source contains '
'more branches than the target.'),
MergeFailureReason.HG_TARGET_HAS_MULTIPLE_HEADS: lazy_ugettext(
u'This pull request cannot be merged because the target `{target_ref.name}` '
u'has multiple heads: `{heads}`.'),
'This pull request cannot be merged because the target `{target_ref.name}` '
'has multiple heads: `{heads}`.'),
MergeFailureReason.TARGET_IS_LOCKED: lazy_ugettext(
u'This pull request cannot be merged because the target repository is '
u'locked by {locked_by}.'),
'This pull request cannot be merged because the target repository is '
'locked by {locked_by}.'),
MergeFailureReason.MISSING_TARGET_REF: lazy_ugettext(
u'This pull request cannot be merged because the target '
u'reference `{target_ref.name}` is missing.'),
'This pull request cannot be merged because the target '
'reference `{target_ref.name}` is missing.'),
MergeFailureReason.MISSING_SOURCE_REF: lazy_ugettext(
u'This pull request cannot be merged because the source '
u'reference `{source_ref.name}` is missing.'),
'This pull request cannot be merged because the source '
'reference `{source_ref.name}` is missing.'),
MergeFailureReason.SUBREPO_MERGE_FAILED: lazy_ugettext(
u'This pull request cannot be merged because of conflicts related '
u'to sub repositories.'),
'This pull request cannot be merged because of conflicts related '
'to sub repositories.'),
# Deprecations
MergeFailureReason._DEPRECATED_MISSING_COMMIT: lazy_ugettext(
@ -254,7 +253,7 @@ class MergeResponse(object):
"""
Return a human friendly error message for the given merge status code.
"""
msg = safe_unicode(self.MERGE_STATUS_MESSAGES[self.failure_reason])
msg = safe_str(self.MERGE_STATUS_MESSAGES[self.failure_reason])
try:
return msg.format(**self.metadata)
@ -379,7 +378,7 @@ class BaseRepository(object):
@LazyProperty
def name(self):
return safe_unicode(os.path.basename(self.path))
return safe_str(os.path.basename(self.path))
@LazyProperty
def description(self):
@ -785,11 +784,11 @@ class BaseRepository(object):
def _validate_commit_id(self, commit_id):
if not isinstance(commit_id, str):
raise TypeError("commit_id must be a string value got {} instead".format(type(commit_id)))
raise TypeError(f"commit_id must be a string value got {type(commit_id)} instead")
def _validate_commit_idx(self, commit_idx):
if not isinstance(commit_idx, int):
raise TypeError("commit_idx must be a numeric value")
raise TypeError(f"commit_idx must be a numeric value, got {type(commit_idx)}")
def _validate_branch_name(self, branch_name):
if branch_name and branch_name not in self.branches_all:
@ -931,21 +930,17 @@ class BaseCommit(object):
value as ``None``.
"""
_ARCHIVE_PREFIX_TEMPLATE = b'{repo_name}-{short_id}'
_ARCHIVE_PREFIX_TEMPLATE = '{repo_name}-{short_id}'
"""
This template is used to generate a default prefix for repository archives
if no prefix has been specified.
"""
def __str__(self):
return '<%s at %s:%s>' % (
self.__class__.__name__, self.idx, self.short_id)
def __repr__(self):
return self.__str__()
def __unicode__(self):
return u'%s:%s' % (self.idx, self.short_id)
def __str__(self):
return f'<{self.__class__.__name__} at {self.idx}:{self.short_id}>'
def __eq__(self, other):
same_instance = isinstance(other, self.__class__)