engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(encoding for file): fixed support of non utf-8 files in all backends

Browse source
This commit is contained in:
RhodeCode Admin 2024-11-27 19:35:02 +01:00
parent f687bfc994
commit d08633d220
42 changed files with 2001 additions and 1973 deletions
Download patch file Download diff file Expand all files Collapse all files

19
rhodecode/__init__.py
View file

@ -45,13 +45,20 @@ CELERY_EAGER = False
# link to config for pyramid
CONFIG = {}
class NotGivenMeta:
def __repr__(self):
return 'NotGivenObject()'
__str__ = __repr__
NotGiven = NotGivenMeta()
class ConfigGet:
NotGiven = object()
def _get_val_or_missing(self, key, missing):
@classmethod
def _get_val_or_missing(cls, key, missing):
if key not in CONFIG:
if missing == self.NotGiven:
if missing != NotGiven:
return missing
# we don't get key, we don't get missing value, return nothing similar as config.get(key)
return None
@ -74,6 +81,12 @@ class ConfigGet:
val = self._get_val_or_missing(key, missing)
return str2bool(val)
def get_list(self, key, missing=NotGiven):
from rhodecode.lib.type_utils import aslist
val = self._get_val_or_missing(key, missing)
return aslist(val, sep=',')
# Populated with the settings dictionary from application init in
# rhodecode.conf.environment.load_pyramid_environment
PYRAMID_SETTINGS = {}

3
rhodecode/api/tests/test_get_repo_comments.py
View file

@ -19,6 +19,7 @@
import pytest
from rhodecode.lib.str_utils import safe_str
from rhodecode.model.db import User, ChangesetComment
from rhodecode.model.meta import Session
from rhodecode.model.comment import CommentsModel
@ -36,7 +37,7 @@ def make_repo_comments_factory(request):
commit = repo.scm_instance()[0]
commit_id = commit.raw_id
file_0 = commit.affected_files[0]
file_0 = safe_str(commit.affected_files[0])
comments = []
# general

6
rhodecode/api/views/repo_api.py
View file

@ -317,8 +317,7 @@ def get_repo_changeset(request, apiuser, repoid, revision,
','.join(_changes_details_types)))
vcs_repo = repo.scm_instance()
pre_load = ['author', 'branch', 'date', 'message', 'parents',
'status', '_commit', '_file_paths']
pre_load = ['author', 'branch', 'date', 'message', 'parents', 'status', '_commit']
try:
commit = repo.get_commit(commit_id=revision, pre_load=pre_load)
@ -376,8 +375,7 @@ def get_repo_changesets(request, apiuser, repoid, start_rev, limit,
','.join(_changes_details_types)))
limit = int(limit)
pre_load = ['author', 'branch', 'date', 'message', 'parents',
'status', '_commit', '_file_paths']
pre_load = ['author', 'branch', 'date', 'message', 'parents', 'status', '_commit']
vcs_repo = repo.scm_instance()
# SVN needs a special case to distinguish its index and commit id

14
rhodecode/apps/repository/views/repo_changelog.py
View file

@ -33,7 +33,7 @@ from rhodecode.lib.auth import (
from rhodecode.lib.graphmod import _colored, _dagwalker
from rhodecode.lib.helpers import RepoPage
from rhodecode.lib.utils2 import str2bool
from rhodecode.lib.str_utils import safe_int, safe_str
from rhodecode.lib.str_utils import safe_int, safe_str, safe_bytes
from rhodecode.lib.vcs.exceptions import (
RepositoryError, CommitDoesNotExistError,
CommitError, NodeDoesNotExistError, EmptyRepositoryError)
@ -204,10 +204,9 @@ class RepoChangelogView(RepoAppView):
log.debug('generating changelog for path %s', f_path)
# get the history for the file !
base_commit = self.rhodecode_vcs_repo.get_commit(commit_id)
bytes_path = safe_bytes(f_path)
try:
collection = base_commit.get_path_history(
f_path, limit=hist_limit, pre_load=pre_load)
collection = base_commit.get_path_history(bytes_path, limit=hist_limit, pre_load=pre_load)
if collection and partial_xhr:
# for ajax call we remove first one since we're looking
# at it right now in the context of a file commit
@ -216,7 +215,7 @@ class RepoChangelogView(RepoAppView):
# this node is not present at tip!
try:
commit = self._get_commit_or_redirect(commit_id)
collection = commit.get_path_history(f_path)
collection = commit.get_path_history(bytes_path)
except RepositoryError as e:
h.flash(safe_str(e), category='warning')
redirect_url = h.route_path(
@ -310,9 +309,8 @@ class RepoChangelogView(RepoAppView):
log.exception(safe_str(e))
raise HTTPFound(
h.route_path('repo_commits', repo_name=self.db_repo_name))
collection = base_commit.get_path_history(
f_path, limit=hist_limit, pre_load=pre_load)
bytes_path = safe_bytes(f_path)
collection = base_commit.get_path_history(bytes_path, limit=hist_limit, pre_load=pre_load)
collection = list(reversed(collection))
else:
collection = self.rhodecode_vcs_repo.get_commits(

6
rhodecode/apps/repository/views/repo_commits.py
View file

@ -89,8 +89,7 @@ class RepoCommitsView(RepoAppView):
commit_range = commit_id_range.split('...')[:2]
try:
pre_load = ['affected_files', 'author', 'branch', 'date',
'message', 'parents']
pre_load = ['author', 'branch', 'date', 'message', 'parents']
if self.rhodecode_vcs_repo.alias == 'hg':
pre_load += ['hidden', 'obsolete', 'phase']
@ -100,8 +99,7 @@ class RepoCommitsView(RepoAppView):
pre_load=pre_load, translate_tags=False)
commits = list(commits)
else:
commits = [self.rhodecode_vcs_repo.get_commit(
commit_id=commit_id_range, pre_load=pre_load)]
commits = [self.rhodecode_vcs_repo.get_commit(commit_id=commit_id_range, pre_load=pre_load)]
c.commit_ranges = commits
if not c.commit_ranges:

144
rhodecode/apps/repository/views/repo_files.py
View file

@ -187,12 +187,14 @@ class RepoFilesView(RepoAppView):
default_commit_id = self.db_repo.landing_ref_name
default_f_path = '/'
commit_id = self.request.matchdict.get(
'commit_id', default_commit_id)
commit_id = self.request.matchdict.get('commit_id', default_commit_id)
f_path = self._get_f_path(self.request.matchdict, default_f_path)
return commit_id, f_path
def _get_default_encoding(self, c):
bytes_path = safe_bytes(f_path)
return commit_id, f_path, bytes_path
@classmethod
def _get_default_encoding(cls, c):
enc_list = getattr(c, 'default_encodings', [])
return enc_list[0] if enc_list else 'UTF-8'
@ -361,21 +363,21 @@ class RepoFilesView(RepoAppView):
from rhodecode import CONFIG
_ = self.request.translate
self.load_default_context()
default_at_path = '/'
fname = self.request.matchdict['fname']
subrepos = self.request.GET.get('subrepos') == 'true'
with_hash = str2bool(self.request.GET.get('with_hash', '1'))
default_at_path = '/'
fname = self.request.matchdict['fname']
at_path = self.request.GET.get('at_path') or default_at_path
if not self.db_repo.enable_downloads:
return Response(_('Downloads disabled'))
try:
commit_id, ext, fileformat, content_type = \
_get_archive_spec(fname)
commit_id, ext, file_format, content_type = _get_archive_spec(fname)
except ValueError:
return Response(_('Unknown archive type for: `{}`').format(
h.escape(fname)))
return Response(_('Unknown archive type for: `{}`').format(h.escape(fname)))
try:
commit = self.rhodecode_vcs_repo.get_commit(commit_id)
@ -391,7 +393,7 @@ class RepoFilesView(RepoAppView):
raise HTTPFound(self.request.current_route_path(fname=fname))
try:
at_path = commit.get_node(at_path).path or default_at_path
at_path = commit.get_node(safe_bytes(at_path)).path or default_at_path
except Exception:
return Response(_('No node at path {} for this repository').format(h.escape(at_path)))
@ -440,7 +442,7 @@ class RepoFilesView(RepoAppView):
with d_cache.get_lock(reentrant_lock_key):
try:
commit.archive_repo(archive_name_key, archive_dir_name=archive_dir_name,
kind=fileformat, subrepos=subrepos,
kind=file_format, subrepos=subrepos,
archive_at_path=at_path, cache_config=d_cache_conf)
except ImproperArchiveTypeError:
return _('Unknown archive type')
@ -484,7 +486,7 @@ class RepoFilesView(RepoAppView):
if commit_id not in ['', None, 'None', '0' * 12, '0' * 40]:
commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
try:
node = commit.get_node(f_path)
node = commit.get_node(safe_bytes(f_path))
if node.is_dir():
raise NodeError(f'{node} path is a {type(node)} not a file')
except NodeDoesNotExistError:
@ -648,7 +650,7 @@ class RepoFilesView(RepoAppView):
# overwrite auto rendering by setting this GET flag
c.renderer = view_name == 'repo_files:rendered' or not self.request.GET.get('no-render', False)
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
c.commit = self._get_commit_or_redirect(commit_id)
c.branch = self.request.GET.get('branch', None)
@ -657,7 +659,8 @@ class RepoFilesView(RepoAppView):
# files or dirs
try:
c.file = c.commit.get_node(f_path, pre_load=['is_binary', 'size', 'data'])
c.file = c.commit.get_node(bytes_path, pre_load=['is_binary', 'size', 'data'])
c.file_author = True
c.file_tree = ''
@ -666,11 +669,9 @@ class RepoFilesView(RepoAppView):
try:
prev_commit = c.commit.prev(c.branch)
c.prev_commit = prev_commit
c.url_prev = h.route_path(
'repo_files', repo_name=self.db_repo_name,
commit_id=prev_commit.raw_id, f_path=f_path)
c.url_prev = h.route_path('repo_files', repo_name=self.db_repo_name, commit_id=prev_commit.raw_id, f_path=f_path)
if c.branch:
c.url_prev += '?branch=%s' % c.branch
c.url_prev += f'?branch={c.branch}'
except (CommitDoesNotExistError, VCSError):
c.url_prev = '#'
c.prev_commit = EmptyCommit()
@ -679,11 +680,9 @@ class RepoFilesView(RepoAppView):
try:
next_commit = c.commit.next(c.branch)
c.next_commit = next_commit
c.url_next = h.route_path(
'repo_files', repo_name=self.db_repo_name,
commit_id=next_commit.raw_id, f_path=f_path)
c.url_next = h.route_path('repo_files', repo_name=self.db_repo_name, commit_id=next_commit.raw_id, f_path=f_path)
if c.branch:
c.url_next += '?branch=%s' % c.branch
c.url_next += f'?branch={c.branch}'
except (CommitDoesNotExistError, VCSError):
c.url_next = '#'
c.next_commit = EmptyCommit()
@ -739,8 +738,7 @@ class RepoFilesView(RepoAppView):
c.file_tree = self._get_tree_at_commit(c, c.commit.raw_id, f_path, at_rev=at_rev)
c.readme_data, c.readme_file = \
self._get_readme_data(self.db_repo, c.visual.default_renderer,
c.commit.raw_id, f_path)
self._get_readme_data(self.db_repo, c.visual.default_renderer, c.commit.raw_id, bytes_path)
except RepositoryError as e:
h.flash(h.escape(safe_str(e)), category='error')
@ -759,24 +757,24 @@ class RepoFilesView(RepoAppView):
def repo_files_annotated_previous(self):
self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, bytes_path, bytes_path = self._get_commit_and_path()
commit = self._get_commit_or_redirect(commit_id)
prev_commit_id = commit.raw_id
line_anchor = self.request.GET.get('line_anchor')
is_file = False
try:
_file = commit.get_node(f_path)
_file = commit.get_node(bytes_path)
is_file = _file.is_file()
except (NodeDoesNotExistError, CommitDoesNotExistError, VCSError):
pass
if is_file:
history = commit.get_path_history(f_path)
history = commit.get_path_history(bytes_path)
prev_commit_id = history[1].raw_id \
if len(history) > 1 else prev_commit_id
prev_url = h.route_path(
'repo_files:annotated', repo_name=self.db_repo_name,
commit_id=prev_commit_id, f_path=f_path,
commit_id=prev_commit_id, f_path=bytes_path,
_anchor=f'L{line_anchor}')
raise HTTPFound(prev_url)
@ -792,10 +790,10 @@ class RepoFilesView(RepoAppView):
"""
c = self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
commit = self._get_commit_or_redirect(commit_id)
try:
dir_node = commit.get_node(f_path)
dir_node = commit.get_node(bytes_path)
except RepositoryError as e:
return Response(f'error: {h.escape(safe_str(e))}')
@ -816,9 +814,9 @@ class RepoFilesView(RepoAppView):
safe_path = f_name.replace('"', '\\"')
encoded_path = urllib.parse.quote(f_name)
headers = "attachment; " \
"filename=\"{}\"; " \
"filename*=UTF-8\'\'{}".format(safe_path, encoded_path)
headers = f"attachment; " \
f"filename=\"{safe_path}\"; " \
f"filename*=UTF-8\'\'{encoded_path}"
return header_safe_str(headers)
@ -832,9 +830,9 @@ class RepoFilesView(RepoAppView):
"""
c = self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
commit = self._get_commit_or_redirect(commit_id)
file_node = self._get_filenode_or_redirect(commit, f_path)
file_node = self._get_filenode_or_redirect(commit, bytes_path)
raw_mimetype_mapping = {
# map original mimetype to a mimetype used for "show as raw"
@ -892,9 +890,9 @@ class RepoFilesView(RepoAppView):
def repo_file_download(self):
c = self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
commit = self._get_commit_or_redirect(commit_id)
file_node = self._get_filenode_or_redirect(commit, f_path)
file_node = self._get_filenode_or_redirect(commit, bytes_path)
if self.request.GET.get('lf'):
# only if lf get flag is passed, we download this file
@ -920,8 +918,7 @@ class RepoFilesView(RepoAppView):
def _get_nodelist_at_commit(self, repo_name, repo_id, commit_id, f_path):
cache_seconds = safe_int(
rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
cache_seconds = rhodecode.ConfigGet().get_int('rc_cache.cache_repo.expiration_time')
cache_on = cache_seconds > 0
log.debug(
'Computing FILE SEARCH for repo_id %s commit_id `%s` and path `%s`'
@ -933,21 +930,17 @@ class RepoFilesView(RepoAppView):
@region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on)
def compute_file_search(_name_hash, _repo_id, _commit_id, _f_path):
log.debug('Generating cached nodelist for repo_id:%s, %s, %s',
_repo_id, commit_id, f_path)
log.debug('Generating cached nodelist for repo_id:%s, %s, %s', _repo_id, commit_id, f_path)
try:
_d, _f = ScmModel().get_quick_filter_nodes(repo_name, _commit_id, _f_path)
except (RepositoryError, CommitDoesNotExistError, Exception) as e:
log.exception(safe_str(e))
h.flash(h.escape(safe_str(e)), category='error')
raise HTTPFound(h.route_path(
'repo_files', repo_name=self.db_repo_name,
commit_id='tip', f_path='/'))
raise HTTPFound(h.route_path('repo_files', repo_name=self.db_repo_name, commit_id='tip', f_path='/'))
return _d + _f
result = compute_file_search(self.db_repo.repo_name_hash, self.db_repo.repo_id,
commit_id, f_path)
result = compute_file_search(self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path)
return filter(lambda n: self.path_filter.path_access_allowed(n['name']), result)
@LoginRequired()
@ -956,7 +949,7 @@ class RepoFilesView(RepoAppView):
def repo_nodelist(self):
self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
commit = self._get_commit_or_redirect(commit_id)
metadata = self._get_nodelist_at_commit(
@ -996,10 +989,10 @@ class RepoFilesView(RepoAppView):
if commits is None:
pre_load = ["author", "branch"]
try:
commits = tip.get_path_history(f_path, pre_load=pre_load)
commits = tip.get_path_history(safe_bytes(f_path), pre_load=pre_load)
except (NodeDoesNotExistError, CommitError):
# this node is not present at tip!
commits = commit_obj.get_path_history(f_path, pre_load=pre_load)
commits = commit_obj.get_path_history(safe_bytes(f_path), pre_load=pre_load)
history = []
commits_group = ([], _("Changesets"))
@ -1040,9 +1033,9 @@ class RepoFilesView(RepoAppView):
def repo_file_history(self):
self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
commit = self._get_commit_or_redirect(commit_id)
file_node = self._get_filenode_or_redirect(commit, f_path)
file_node = self._get_filenode_or_redirect(commit, bytes_path)
if file_node.is_file():
file_history, _hist = self._get_node_history(commit, f_path)
@ -1083,9 +1076,9 @@ class RepoFilesView(RepoAppView):
def repo_file_authors(self):
c = self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
commit = self._get_commit_or_redirect(commit_id)
file_node = self._get_filenode_or_redirect(commit, f_path)
file_node = self._get_filenode_or_redirect(commit, bytes_path)
if not file_node.is_file():
raise HTTPBadRequest()
@ -1124,10 +1117,9 @@ class RepoFilesView(RepoAppView):
def repo_files_check_head(self):
self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
_branch_name, _sha_commit_id, is_head = \
self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
landing_ref=self.db_repo.landing_ref_name)
self._is_valid_head(commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name)
new_path = self.request.POST.get('path')
operation = self.request.POST.get('operation')
@ -1138,12 +1130,10 @@ class RepoFilesView(RepoAppView):
try:
commit_obj = self.rhodecode_vcs_repo.get_commit(commit_id)
# NOTE(dan): construct whole path without leading /
file_node = commit_obj.get_node(new_f_path)
if file_node is not None:
file_node = commit_obj.get_node(safe_bytes(new_f_path))
if file_node:
path_exist = new_f_path
except EmptyRepositoryError:
pass
except Exception:
except (EmptyRepositoryError, NodeDoesNotExistError):
pass
return {
@ -1158,7 +1148,7 @@ class RepoFilesView(RepoAppView):
def repo_files_remove_file(self):
_ = self.request.translate
c = self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
self._ensure_not_locked()
_branch_name, _sha_commit_id, is_head = \
@ -1169,7 +1159,7 @@ class RepoFilesView(RepoAppView):
self.check_branch_permission(_branch_name)
c.commit = self._get_commit_or_redirect(commit_id)
c.file = self._get_filenode_or_redirect(c.commit, f_path)
c.file = self._get_filenode_or_redirect(c.commit, bytes_path)
c.default_message = _(
'Deleted file {} via RhodeCode Enterprise').format(f_path)
@ -1184,7 +1174,7 @@ class RepoFilesView(RepoAppView):
_ = self.request.translate
c = self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
self._ensure_not_locked()
_branch_name, _sha_commit_id, is_head = \
@ -1195,10 +1185,9 @@ class RepoFilesView(RepoAppView):
self.check_branch_permission(_branch_name)
c.commit = self._get_commit_or_redirect(commit_id)
c.file = self._get_filenode_or_redirect(c.commit, f_path)
c.file = self._get_filenode_or_redirect(c.commit, bytes_path)
c.default_message = _(
'Deleted file {} via RhodeCode Enterprise').format(f_path)
c.default_message = _('Deleted file {} via RhodeCode Enterprise').format(f_path)
c.f_path = f_path
node_path = f_path
author = self._rhodecode_db_user.full_contact
@ -1232,7 +1221,7 @@ class RepoFilesView(RepoAppView):
def repo_files_edit_file(self):
_ = self.request.translate
c = self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
self._ensure_not_locked()
_branch_name, _sha_commit_id, is_head = \
@ -1243,7 +1232,7 @@ class RepoFilesView(RepoAppView):
self.check_branch_permission(_branch_name, commit_id=commit_id)
c.commit = self._get_commit_or_redirect(commit_id)
c.file = self._get_filenode_or_redirect(c.commit, f_path)
c.file = self._get_filenode_or_redirect(c.commit, bytes_path)
if c.file.is_binary:
files_url = h.route_path(
@ -1263,12 +1252,12 @@ class RepoFilesView(RepoAppView):
def repo_files_update_file(self):
_ = self.request.translate
c = self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
self._ensure_not_locked()
c.commit = self._get_commit_or_redirect(commit_id)
c.file = self._get_filenode_or_redirect(c.commit, f_path)
c.file = self._get_filenode_or_redirect(c.commit, bytes_path)
if c.file.is_binary:
raise HTTPFound(h.route_path('repo_files', repo_name=self.db_repo_name,
@ -1345,7 +1334,7 @@ class RepoFilesView(RepoAppView):
def repo_files_add_file(self):
_ = self.request.translate
c = self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
self._ensure_not_locked()
@ -1381,7 +1370,7 @@ class RepoFilesView(RepoAppView):
def repo_files_create_file(self):
_ = self.request.translate
c = self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
self._ensure_not_locked()
@ -1450,8 +1439,7 @@ class RepoFilesView(RepoAppView):
author=author,
)
h.flash(_('Successfully committed new file `{}`').format(
h.escape(node_path)), category='success')
h.flash(_('Successfully committed new file `{}`').format(h.escape(node_path)), category='success')
default_redirect_url = h.route_path(
'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id)
@ -1475,7 +1463,7 @@ class RepoFilesView(RepoAppView):
def repo_files_upload_file(self):
_ = self.request.translate
c = self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
self._ensure_not_locked()
@ -1604,7 +1592,7 @@ class RepoFilesView(RepoAppView):
def repo_files_replace_file(self):
_ = self.request.translate
c = self.load_default_context()
commit_id, f_path = self._get_commit_and_path()
commit_id, f_path, bytes_path = self._get_commit_and_path()
self._ensure_not_locked()

1
rhodecode/config/utils.py
View file

@ -39,7 +39,6 @@ def configure_vcs(config):
conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol.v2']
conf.settings.HOOKS_HOST = config['vcs.hooks.host']
conf.settings.DEFAULT_ENCODINGS = config['default_encoding']
conf.settings.ALIASES[:] = config['vcs.backends']
conf.settings.SVN_COMPATIBLE_VERSION = config['vcs.svn.compatible_version']

10
rhodecode/lib/codeblocks.py
View file

@ -478,7 +478,9 @@ class DiffSet(object):
log.debug('rendering diff for %r', patch['filename'])
source_filename = patch['original_filename']
source_filename_bytes = patch['original_filename_bytes']
target_filename = patch['filename']
target_filename_bytes = patch['filename_bytes']
source_lexer = plain_text_lexer
target_lexer = plain_text_lexer
@ -491,12 +493,12 @@ class DiffSet(object):
if (source_filename and patch['operation'] in ('D', 'M')
and source_filename not in self.source_nodes):
self.source_nodes[source_filename] = (
self.source_node_getter(source_filename))
self.source_node_getter(source_filename_bytes))
if (target_filename and patch['operation'] in ('A', 'M')
and target_filename not in self.target_nodes):
self.target_nodes[target_filename] = (
self.target_node_getter(target_filename))
self.target_node_getter(target_filename_bytes))
elif hl_mode == self.HL_FAST:
source_lexer = self._get_lexer_for_filename(source_filename)
@ -558,6 +560,7 @@ class DiffSet(object):
})
file_chunks = patch['chunks'][1:]
for i, hunk in enumerate(file_chunks, 1):
hunkbit = self.parse_hunk(hunk, source_file, target_file)
hunkbit.source_file_path = source_file_path
@ -593,12 +596,13 @@ class DiffSet(object):
return filediff
def parse_hunk(self, hunk, source_file, target_file):
result = AttributeDict(dict(
source_start=hunk['source_start'],
source_length=hunk['source_length'],
target_start=hunk['target_start'],
target_length=hunk['target_length'],
section_header=hunk['section_header'],
section_header=safe_str(hunk['section_header']),
lines=[],
))
before, after = [], []

13
rhodecode/lib/diffs.py
View file

@ -455,7 +455,9 @@ class DiffProcessor(object):
return arg
for chunk in self._diff.chunks():
bytes_head = chunk.header
head = chunk.header_as_str
log.debug('parsing diff chunk %r', chunk)
raw_diff = chunk.raw
@ -598,10 +600,17 @@ class DiffProcessor(object):
chunks.insert(0, frag)
original_filename = safe_str(head['a_path'])
original_filename = head['a_path']
original_filename_bytes = bytes_head['a_path']
filename = head['b_path']
filename_bytes = bytes_head['b_path']
_files.append({
'original_filename': original_filename,
'filename': safe_str(head['b_path']),
'original_filename_bytes': original_filename_bytes,
'filename': filename,
'filename_bytes': filename_bytes,
'old_revision': head['a_blob_id'],
'new_revision': head['b_blob_id'],
'chunks': chunks,

1
rhodecode/lib/hooks_base.py
View file

@ -84,7 +84,6 @@ def check_locked_repo(extras, check_same_user=True):
user = User.get_by_username(extras.username)
output = ''
if extras.locked_by[0] and (not check_same_user or user.user_id != extras.locked_by[0]):
locked_by = User.get(extras.locked_by[0]).username
reason = extras.locked_by[2]
# this exception is interpreted in git/hg middlewares and based

9
rhodecode/lib/str_utils.py
View file

@ -67,10 +67,7 @@ def base64_to_str(text: str | bytes) -> str:
def get_default_encodings() -> list[str]:
return aslist(rhodecode.CONFIG.get('default_encoding', 'utf8'), sep=',')
DEFAULT_ENCODINGS = get_default_encodings()
return rhodecode.ConfigGet().get_list('default_encoding', missing='utf8')
def safe_str(str_, to_encoding=None) -> str:
@ -87,7 +84,7 @@ def safe_str(str_, to_encoding=None) -> str:
if not isinstance(str_, bytes):
return str(str_)
to_encoding = to_encoding or DEFAULT_ENCODINGS
to_encoding = to_encoding or get_default_encodings()
if not isinstance(to_encoding, (list, tuple)):
to_encoding = [to_encoding]
@ -120,7 +117,7 @@ def safe_bytes(str_, from_encoding=None) -> bytes:
for enc in from_encoding:
try:
return str_.encode(enc)
except UnicodeDecodeError:
except (UnicodeDecodeError, UnicodeEncodeError):
pass
return str_.encode(from_encoding[0], 'replace')

2
rhodecode/lib/vcs/__init__.py
View file

@ -139,7 +139,7 @@ class CurlSession(object):
try:
curl.perform()
except pycurl.error as exc:
except pycurl.error:
log.error('Failed to call endpoint url: %s using pycurl', url)
raise

1633
rhodecode/lib/vcs/backends/base.py
View file

File diff suppressed because it is too large Load diff

295
rhodecode/lib/vcs/backends/git/commit.py
View file

@ -21,8 +21,8 @@ GIT commit module
"""
import io
import stat
import configparser
import logging
from itertools import chain
from zope.cachedescriptors.property import Lazy as LazyProperty
@ -32,9 +32,16 @@ from rhodecode.lib.str_utils import safe_bytes, safe_str
from rhodecode.lib.vcs.backends import base
from rhodecode.lib.vcs.exceptions import CommitError, NodeDoesNotExistError
from rhodecode.lib.vcs.nodes import (
FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
ChangedFileNodesGenerator, AddedFileNodesGenerator,
RemovedFileNodesGenerator, LargeFileNode)
FileNode,
DirNode,
NodeKind,
RootNode,
SubModuleNode,
LargeFileNode,
)
from rhodecode.lib.vcs_common import FILEMODE_LINK
log = logging.getLogger(__name__)
class GitCommit(base.BaseCommit):
@ -50,13 +57,11 @@ class GitCommit(base.BaseCommit):
# done through a more complex tree walk on parents
"status",
# mercurial specific property not supported here
"_file_paths",
"obsolete",
# mercurial specific property not supported here
'obsolete',
"phase",
# mercurial specific property not supported here
'phase',
# mercurial specific property not supported here
'hidden'
"hidden",
]
def __init__(self, repository, raw_id, idx, pre_load=None):
@ -69,17 +74,16 @@ class GitCommit(base.BaseCommit):
self._set_bulk_properties(pre_load)
# caches
self._stat_modes = {} # stat info for paths
self._paths = {} # path processed with parse_tree
self.nodes = {}
self._path_mode_cache = {} # path stats cache, e.g filemode etc
self._path_type_cache = {} # path type dir/file/link etc cache
self._submodules = None
def _set_bulk_properties(self, pre_load):
if not pre_load:
return
pre_load = [entry for entry in pre_load
if entry not in self._filter_pre_load]
pre_load = [entry for entry in pre_load if entry not in self._filter_pre_load]
if not pre_load:
return
@ -102,7 +106,7 @@ class GitCommit(base.BaseCommit):
@LazyProperty
def _tree_id(self):
return self._remote[self._commit['tree']]['id']
return self._remote[self._commit["tree"]]["id"]
@LazyProperty
def id(self):
@ -134,13 +138,12 @@ class GitCommit(base.BaseCommit):
"""
Returns modified, added, removed, deleted files for current commit
"""
return self.changed, self.added, self.removed
added, modified, deleted = self._changes_cache
return list(modified), list(modified), list(deleted)
@LazyProperty
def tags(self):
tags = [safe_str(name) for name,
commit_id in self.repository.tags.items()
if commit_id == self.raw_id]
tags = [safe_str(name) for name, commit_id in self.repository.tags.items() if commit_id == self.raw_id]
return tags
@LazyProperty
@ -161,47 +164,33 @@ class GitCommit(base.BaseCommit):
branches = self._remote.branch(self.raw_id)
return self._set_branch(branches)
def _get_tree_id_for_path(self, path):
def _get_path_tree_id_and_type(self, path: bytes):
path = safe_str(path)
if path in self._paths:
return self._paths[path]
if path in self._path_type_cache:
return self._path_type_cache[path]
tree_id = self._tree_id
if path == b"":
self._path_type_cache[b""] = [self._tree_id, NodeKind.DIR]
return self._path_type_cache[path]
path = path.strip('/')
if path == '':
data = [tree_id, "tree"]
self._paths[''] = data
return data
tree_id, tree_type, tree_mode = \
self._remote.tree_and_type_for_path(self.raw_id, path)
tree_id, tree_type, tree_mode = self._remote.tree_and_type_for_path(self.raw_id, path)
if tree_id is None:
raise self.no_node_at_path(path)
self._paths[path] = [tree_id, tree_type]
self._stat_modes[path] = tree_mode
self._path_type_cache[path] = [tree_id, tree_type]
self._path_mode_cache[path] = tree_mode
if path not in self._paths:
raise self.no_node_at_path(path)
return self._paths[path]
return self._path_type_cache[path]
def _get_kind(self, path):
tree_id, type_ = self._get_tree_id_for_path(path)
if type_ == 'blob':
return NodeKind.FILE
elif type_ == 'tree':
return NodeKind.DIR
elif type_ == 'link':
return NodeKind.SUBMODULE
return None
path = self._fix_path(path)
_, path_type = self._get_path_tree_id_and_type(path)
return path_type
def _assert_is_path(self, path):
path = self._fix_path(path)
if self._get_kind(path) != NodeKind.FILE:
raise CommitError(f"File does not exist for commit {self.raw_id} at '{path}'")
raise CommitError(f"File at path={path} does not exist for commit {self.raw_id}")
return path
def _get_file_nodes(self):
@ -237,15 +226,19 @@ class GitCommit(base.BaseCommit):
path = self._assert_is_path(path)
# ensure path is traversed
self._get_tree_id_for_path(path)
self._get_path_tree_id_and_type(path)
return self._stat_modes[path]
return self._path_mode_cache[path]
def is_link(self, path):
return stat.S_ISLNK(self.get_file_mode(path))
def is_link(self, path: bytes):
path = self._assert_is_path(path)
if path not in self._path_mode_cache:
self._path_mode_cache[path] = self._remote.fctx_flags(self.raw_id, path)
return self._path_mode_cache[path] == FILEMODE_LINK
def is_node_binary(self, path):
tree_id, _ = self._get_tree_id_for_path(path)
tree_id, _ = self._get_path_tree_id_and_type(path)
return self._remote.is_binary(tree_id)
def node_md5_hash(self, path):
@ -256,19 +249,19 @@ class GitCommit(base.BaseCommit):
"""
Returns content of the file at given `path`.
"""
tree_id, _ = self._get_tree_id_for_path(path)
tree_id, _ = self._get_path_tree_id_and_type(path)
return self._remote.blob_as_pretty_string(tree_id)
def get_file_content_streamed(self, path):
tree_id, _ = self._get_tree_id_for_path(path)
stream_method = getattr(self._remote, 'stream:blob_as_pretty_string')
tree_id, _ = self._get_path_tree_id_and_type(path)
stream_method = getattr(self._remote, "stream:blob_as_pretty_string")
return stream_method(tree_id)
def get_file_size(self, path):
"""
Returns size of the file at given `path`.
"""
tree_id, _ = self._get_tree_id_for_path(path)
tree_id, _ = self._get_path_tree_id_and_type(path)
return self._remote.blob_raw_length(tree_id)
def get_path_history(self, path, limit=None, pre_load=None):
@ -276,12 +269,9 @@ class GitCommit(base.BaseCommit):
Returns history of file as reversed list of `GitCommit` objects for
which file at given `path` has been modified.
"""
path = self._assert_is_path(path)
hist = self._remote.node_history(self.raw_id, path, limit)
return [
self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
for commit_id in hist]
history = self._remote.node_history(self.raw_id, path, limit)
return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) for commit_id in history]
def get_file_annotate(self, path, pre_load=None):
"""
@ -293,95 +283,105 @@ class GitCommit(base.BaseCommit):
for ln_no, commit_id, content in result:
yield (
ln_no, commit_id,
ln_no,
commit_id,
lambda: self.repository.get_commit(commit_id=commit_id, pre_load=pre_load),
content)
content,
)
def get_nodes(self, path, pre_load=None):
def get_nodes(self, path: bytes, pre_load=None):
if self._get_kind(path) != NodeKind.DIR:
raise CommitError(
f"Directory does not exist for commit {self.raw_id} at '{path}'")
raise CommitError(f"Directory does not exist for commit {self.raw_id} at '{path}'")
path = self._fix_path(path)
tree_id, _ = self._get_tree_id_for_path(path)
# call and check tree_id for this path
tree_id, _ = self._get_path_tree_id_and_type(path)
dirnodes = []
filenodes = []
path_nodes = []
# extracted tree ID gives us our files...
str_path = safe_str(path) # libgit operates on bytes
for name, stat_, id_, type_ in self._remote.tree_items(tree_id):
if type_ == 'link':
url = self._get_submodule_url('/'.join((str_path, name)))
dirnodes.append(SubModuleNode(
name, url=url, commit=id_, alias=self.repository.alias))
continue
for bytes_name, stat_, tree_item_id, node_kind in self._remote.tree_items(tree_id):
if node_kind is None:
raise CommitError(f"Requested object type={node_kind} cannot be determined")
if str_path != '':
obj_path = '/'.join((str_path, name))
if path != b"":
obj_path = b"/".join((path, bytes_name))
else:
obj_path = name
if obj_path not in self._stat_modes:
self._stat_modes[obj_path] = stat_
obj_path = bytes_name
if type_ == 'tree':
dirnodes.append(DirNode(safe_bytes(obj_path), commit=self))
elif type_ == 'blob':
filenodes.append(FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load))
# cache file mode for git, since we have it already
if obj_path not in self._path_mode_cache:
self._path_mode_cache[obj_path] = stat_
# cache type
if node_kind not in self._path_type_cache:
self._path_type_cache[obj_path] = [tree_item_id, node_kind]
entry = None
if obj_path in self.nodes:
entry = self.nodes[obj_path]
else:
raise CommitError(f"Requested object should be Tree or Blob, is {type_}")
if node_kind == NodeKind.SUBMODULE:
url = self._get_submodule_url(b"/".join((path, bytes_name)))
entry= SubModuleNode(bytes_name, url=url, commit=tree_item_id, alias=self.repository.alias)
elif node_kind == NodeKind.DIR:
entry = DirNode(safe_bytes(obj_path), commit=self)
elif node_kind == NodeKind.FILE:
entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load)
nodes = dirnodes + filenodes
for node in nodes:
if node.path not in self.nodes:
self.nodes[node.path] = node
nodes.sort()
return nodes
if entry:
self.nodes[obj_path] = entry
path_nodes.append(entry)
def get_node(self, path, pre_load=None):
path_nodes.sort()
return path_nodes
def get_node(self, path: bytes, pre_load=None):
path = self._fix_path(path)
if path not in self.nodes:
try:
tree_id, type_ = self._get_tree_id_for_path(path)
except CommitError:
raise NodeDoesNotExistError(
f"Cannot find one of parents' directories for a given "
f"path: {path}")
if type_ in ['link', 'commit']:
# use cached, if we have one
if path in self.nodes:
return self.nodes[path]
try:
tree_id, path_type = self._get_path_tree_id_and_type(path)
except CommitError:
raise NodeDoesNotExistError(f"Cannot find one of parents' directories for a given path: {path}")
if path == b"":
node = RootNode(commit=self)
else:
if path_type == NodeKind.SUBMODULE:
url = self._get_submodule_url(path)
node = SubModuleNode(path, url=url, commit=tree_id,
alias=self.repository.alias)
elif type_ == 'tree':
if path == '':
node = RootNode(commit=self)
else:
node = DirNode(safe_bytes(path), commit=self)
elif type_ == 'blob':
node = SubModuleNode(path, url=url, commit=tree_id, alias=self.repository.alias)
elif path_type == NodeKind.DIR:
node = DirNode(safe_bytes(path), commit=self)
elif path_type == NodeKind.FILE:
node = FileNode(safe_bytes(path), commit=self, pre_load=pre_load)
self._stat_modes[path] = node.mode
self._path_mode_cache[path] = node.mode
else:
raise self.no_node_at_path(path)
# cache node
self.nodes[path] = node
# cache node
self.nodes[path] = node
return self.nodes[path]
def get_largefile_node(self, path):
tree_id, _ = self._get_tree_id_for_path(path)
def get_largefile_node(self, path: bytes):
tree_id, _ = self._get_path_tree_id_and_type(path)
pointer_spec = self._remote.is_large_file(tree_id)
if pointer_spec:
# content of that file regular FileNode is the hash of largefile
file_id = pointer_spec.get('oid_hash')
if self._remote.in_largefiles_store(file_id):
lf_path = self._remote.store_path(file_id)
return LargeFileNode(safe_bytes(lf_path), commit=self, org_path=path)
file_id = pointer_spec.get("oid_hash")
if not self._remote.in_largefiles_store(file_id):
log.warning(f'Largefile oid={file_id} not found in store')
return None
lf_path = self._remote.store_path(file_id)
return LargeFileNode(safe_bytes(lf_path), commit=self, org_path=path)
@LazyProperty
def affected_files(self):
def affected_files(self) -> list[bytes]:
"""
Gets a fast accessible file changes for given commit
"""
@ -389,7 +389,7 @@ class GitCommit(base.BaseCommit):
return list(added.union(modified).union(deleted))
@LazyProperty
def _changes_cache(self):
def _changes_cache(self) -> tuple[set, set, set]:
added = set()
modified = set()
deleted = set()
@ -416,53 +416,22 @@ class GitCommit(base.BaseCommit):
:param status: one of: *added*, *modified* or *deleted*
"""
added, modified, deleted = self._changes_cache
return sorted({
'added': list(added),
'modified': list(modified),
'deleted': list(deleted)}[status]
)
@LazyProperty
def added(self):
"""
Returns list of added ``FileNode`` objects.
"""
if not self.parents:
return list(self._get_file_nodes())
return AddedFileNodesGenerator(self.added_paths, self)
return sorted({"added": list(added), "modified": list(modified), "deleted": list(deleted)}[status])
@LazyProperty
def added_paths(self):
return [n for n in self._get_paths_for_status('added')]
@LazyProperty
def changed(self):
"""
Returns list of modified ``FileNode`` objects.
"""
if not self.parents:
return []
return ChangedFileNodesGenerator(self.changed_paths, self)
return [n for n in self._get_paths_for_status("added")]
@LazyProperty
def changed_paths(self):
return [n for n in self._get_paths_for_status('modified')]
@LazyProperty
def removed(self):
"""
Returns list of removed ``FileNode`` objects.
"""
if not self.parents:
return []
return RemovedFileNodesGenerator(self.removed_paths, self)
return [n for n in self._get_paths_for_status("modified")]
@LazyProperty
def removed_paths(self):
return [n for n in self._get_paths_for_status('deleted')]
return [n for n in self._get_paths_for_status("deleted")]
def _get_submodule_url(self, submodule_path):
git_modules_path = '.gitmodules'
def _get_submodule_url(self, submodule_path: bytes):
git_modules_path = b".gitmodules"
if self._submodules is None:
self._submodules = {}
@ -476,9 +445,9 @@ class GitCommit(base.BaseCommit):
parser.read_file(io.StringIO(submodules_node.str_content))
for section in parser.sections():
path = parser.get(section, 'path')
url = parser.get(section, 'url')
path = parser.get(section, "path")
url = parser.get(section, "url")
if path and url:
self._submodules[path.strip('/')] = url
self._submodules[safe_bytes(path).strip(b"/")] = url
return self._submodules.get(submodule_path.strip('/'))
return self._submodules.get(submodule_path.strip(b"/"))

2
rhodecode/lib/vcs/backends/git/repository.py
View file

@ -425,7 +425,7 @@ class GitRepository(BaseRepository):
return
def get_commit(self, commit_id=None, commit_idx=None, pre_load=None,
translate_tag=True, maybe_unreachable=False, reference_obj=None):
translate_tag=True, maybe_unreachable=False, reference_obj=None) -> GitCommit:
"""
Returns `GitCommit` object representing commit from git repository
at the given `commit_id` or head (most recent commit) if None given.

228
rhodecode/lib/vcs/backends/hg/commit.py
View file

@ -20,20 +20,25 @@
HG commit module
"""
import os
import logging
from zope.cachedescriptors.property import Lazy as LazyProperty
from rhodecode.lib.datelib import utcdate_fromtimestamp
from rhodecode.lib.str_utils import safe_bytes, safe_str
from rhodecode.lib.vcs import path as vcspath
from rhodecode.lib.vcs.backends import base
from rhodecode.lib.vcs.exceptions import CommitError
from rhodecode.lib.vcs.nodes import (
AddedFileNodesGenerator, ChangedFileNodesGenerator, DirNode, FileNode,
NodeKind, RemovedFileNodesGenerator, RootNode, SubModuleNode,
LargeFileNode)
from rhodecode.lib.vcs.utils.paths import get_dirs_for_path
DirNode,
FileNode,
NodeKind,
RootNode,
SubModuleNode,
LargeFileNode,
)
from rhodecode.lib.vcs_common import FILEMODE_LINK
log = logging.getLogger(__name__)
class MercurialCommit(base.BaseCommit):
@ -59,13 +64,13 @@ class MercurialCommit(base.BaseCommit):
# caches
self.nodes = {}
self._stat_modes = {} # stat info for paths
self._path_mode_cache = {} # path stats cache, e.g filemode etc
self._path_type_cache = {} # path type dir/file/link etc cache
def _set_bulk_properties(self, pre_load):
if not pre_load:
return
pre_load = [entry for entry in pre_load
if entry not in self._filter_pre_load]
pre_load = [entry for entry in pre_load if entry not in self._filter_pre_load]
if not pre_load:
return
@ -86,8 +91,7 @@ class MercurialCommit(base.BaseCommit):
@LazyProperty
def tags(self):
tags = [name for name, commit_id in self.repository.tags.items()
if commit_id == self.raw_id]
tags = [name for name, commit_id in self.repository.tags.items() if commit_id == self.raw_id]
return tags
@LazyProperty
@ -96,9 +100,7 @@ class MercurialCommit(base.BaseCommit):
@LazyProperty
def bookmarks(self):
bookmarks = [
name for name, commit_id in self.repository.bookmarks.items()
if commit_id == self.raw_id]
bookmarks = [name for name, commit_id in self.repository.bookmarks.items() if commit_id == self.raw_id]
return bookmarks
@LazyProperty
@ -122,27 +124,13 @@ class MercurialCommit(base.BaseCommit):
"""
Returns modified, added, removed, deleted files for current commit
"""
return self._remote.ctx_status(self.raw_id)
@LazyProperty
def _file_paths(self):
return self._remote.ctx_list(self.raw_id)
@LazyProperty
def _dir_paths(self):
dir_paths = ['']
dir_paths.extend(list(set(get_dirs_for_path(*self._file_paths))))
return dir_paths
@LazyProperty
def _paths(self):
return self._dir_paths + self._file_paths
modified, added, deleted, *_ = self._remote.ctx_status(self.raw_id)
return modified, added, deleted
@LazyProperty
def id(self):
if self.last:
return 'tip'
return "tip"
return self.short_id
@LazyProperty
@ -150,8 +138,7 @@ class MercurialCommit(base.BaseCommit):
return self.raw_id[:12]
def _make_commits(self, commit_ids, pre_load=None):
return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
for commit_id in commit_ids]
return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) for commit_id in commit_ids]
@LazyProperty
def parents(self):
@ -163,10 +150,10 @@ class MercurialCommit(base.BaseCommit):
def _get_phase_text(self, phase_id):
return {
0: 'public',
1: 'draft',
2: 'secret',
}.get(phase_id) or ''
0: "public",
1: "draft",
2: "secret",
}.get(phase_id) or ""
@LazyProperty
def phase(self):
@ -195,17 +182,14 @@ class MercurialCommit(base.BaseCommit):
def _get_kind(self, path):
path = self._fix_path(path)
if path in self._file_paths:
return NodeKind.FILE
elif path in self._dir_paths:
return NodeKind.DIR
else:
raise CommitError(f"Node does not exist at the given path '{path}'")
path_type = self._get_path_type(path)
return path_type
def _assert_is_path(self, path) -> str:
def _assert_is_path(self, path) -> str | bytes:
path = self._fix_path(path)
if self._get_kind(path) != NodeKind.FILE:
raise CommitError(f"File does not exist for commit {self.raw_id} at '{path}'")
raise CommitError(f"File at path={path} does not exist for commit {self.raw_id}")
return path
@ -214,20 +198,17 @@ class MercurialCommit(base.BaseCommit):
Returns stat mode of the file at the given ``path``.
"""
path = self._assert_is_path(path)
if path not in self._path_mode_cache:
self._path_mode_cache[path] = self._remote.fctx_flags(self.raw_id, path)
if path not in self._stat_modes:
self._stat_modes[path] = self._remote.fctx_flags(self.raw_id, path)
return self._path_mode_cache[path]
if 'x' in self._stat_modes[path]:
return base.FILEMODE_EXECUTABLE
return base.FILEMODE_DEFAULT
def is_link(self, path):
def is_link(self, path: bytes):
path = self._assert_is_path(path)
if path not in self._stat_modes:
self._stat_modes[path] = self._remote.fctx_flags(self.raw_id, path)
if path not in self._path_mode_cache:
self._path_mode_cache[path] = self._remote.fctx_flags(self.raw_id, path)
return 'l' in self._stat_modes[path]
return self._path_mode_cache[path] == FILEMODE_LINK
def is_node_binary(self, path):
path = self._assert_is_path(path)
@ -246,7 +227,7 @@ class MercurialCommit(base.BaseCommit):
def get_file_content_streamed(self, path):
path = self._assert_is_path(path)
stream_method = getattr(self._remote, 'stream:fctx_node_data')
stream_method = getattr(self._remote, "stream:fctx_node_data")
return stream_method(self.raw_id, path)
def get_file_size(self, path):
@ -262,10 +243,8 @@ class MercurialCommit(base.BaseCommit):
for which file at given ``path`` has been modified.
"""
path = self._assert_is_path(path)
hist = self._remote.node_history(self.raw_id, path, limit)
return [
self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
for commit_id in hist]
history = self._remote.node_history(self.raw_id, path, limit)
return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) for commit_id in history]
def get_file_annotate(self, path, pre_load=None):
"""
@ -276,11 +255,13 @@ class MercurialCommit(base.BaseCommit):
for ln_no, commit_id, content in result:
yield (
ln_no, commit_id,
ln_no,
commit_id,
lambda: self.repository.get_commit(commit_id=commit_id, pre_load=pre_load),
content)
content,
)
def get_nodes(self, path, pre_load=None):
def get_nodes(self, path: bytes, pre_load=None):
"""
Returns combined ``DirNode`` and ``FileNode`` objects list representing
state of commit at the given ``path``. If node at the given ``path``
@ -288,59 +269,86 @@ class MercurialCommit(base.BaseCommit):
"""
if self._get_kind(path) != NodeKind.DIR:
raise CommitError(
f"Directory does not exist for idx {self.raw_id} at '{path}'")
raise CommitError(f"Directory does not exist for idx {self.raw_id} at '{path}'")
path = self._fix_path(path)
filenodes = [
FileNode(safe_bytes(f), commit=self, pre_load=pre_load) for f in self._file_paths
if os.path.dirname(f) == path]
# TODO: johbo: Check if this can be done in a more obvious way
dirs = path == '' and '' or [
d for d in self._dir_paths
if d and vcspath.dirname(d) == path]
dirnodes = [
DirNode(safe_bytes(d), commit=self) for d in dirs
if os.path.dirname(d) == path]
path_nodes = []
alias = self.repository.alias
for k, vals in self._submodules.items():
if vcspath.dirname(k) == path:
loc = vals[0]
commit = vals[1]
dirnodes.append(SubModuleNode(k, url=loc, commit=commit, alias=alias))
for obj_path, node_kind in self._remote.dir_items(self.raw_id, path):
nodes = dirnodes + filenodes
for node in nodes:
if node.path not in self.nodes:
self.nodes[node.path] = node
nodes.sort()
if node_kind is None:
raise CommitError(f"Requested object type={node_kind} cannot be mapped to a proper type")
return nodes
# TODO: implement it ??
stat_ = None
# # cache file mode
# if obj_path not in self._path_mode_cache:
# self._path_mode_cache[obj_path] = stat_
def get_node(self, path, pre_load=None):
# cache type
if node_kind not in self._path_type_cache:
self._path_type_cache[obj_path] = node_kind
entry = None
if obj_path in self.nodes:
entry = self.nodes[obj_path]
else:
if node_kind == NodeKind.DIR:
entry = DirNode(safe_bytes(obj_path), commit=self)
elif node_kind == NodeKind.FILE:
entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load)
if entry:
self.nodes[obj_path] = entry
path_nodes.append(entry)
path_nodes.sort()
return path_nodes
def get_node(self, path: bytes, pre_load=None):
"""
Returns `Node` object from the given `path`. If there is no node at
the given `path`, `NodeDoesNotExistError` would be raised.
"""
path = self._fix_path(path)
if path not in self.nodes:
if path in self._file_paths:
# use cached, if we have one
if path in self.nodes:
return self.nodes[path]
path_type = self._get_path_type(path)
if path == b"":
node = RootNode(commit=self)
else:
if path_type == NodeKind.DIR:
node = DirNode(safe_bytes(path), commit=self)
elif path_type == NodeKind.FILE:
node = FileNode(safe_bytes(path), commit=self, pre_load=pre_load)
elif path in self._dir_paths:
if path == '':
node = RootNode(commit=self)
else:
node = DirNode(safe_bytes(path), commit=self)
self._path_mode_cache[path] = node.mode
else:
raise self.no_node_at_path(path)
# cache node
self.nodes[path] = node
# cache node
self.nodes[path] = node
return self.nodes[path]
def get_largefile_node(self, path):
def _get_path_type(self, path: bytes):
if path in self._path_type_cache:
return self._path_type_cache[path]
if path == b"":
self._path_type_cache[b""] = NodeKind.DIR
return NodeKind.DIR
path_type, flags = self._remote.get_path_type(self.raw_id, path)
if not path_type:
raise self.no_node_at_path(path)
self._path_type_cache[path] = path_type
self._path_mode_cache[path] = flags
return self._path_type_cache[path]
def get_largefile_node(self, path: bytes):
pointer_spec = self._remote.is_large_file(self.raw_id, path)
if pointer_spec:
# content of that file regular FileNode is the hash of largefile
@ -363,40 +371,20 @@ class MercurialCommit(base.BaseCommit):
return self._remote.ctx_substate(self.raw_id)
@LazyProperty
def affected_files(self):
def affected_files(self) -> list[bytes]:
"""
Gets a fast accessible file changes for given commit
"""
return self._remote.ctx_files(self.raw_id)
@property
def added(self):
"""
Returns list of added ``FileNode`` objects.
"""
return AddedFileNodesGenerator(self.added_paths, self)
@LazyProperty
def added_paths(self):
return [n for n in self.status[1]]
@property
def changed(self):
"""
Returns list of modified ``FileNode`` objects.
"""
return ChangedFileNodesGenerator(self.changed_paths, self)
@LazyProperty
def changed_paths(self):
return [n for n in self.status[0]]
@property
def removed(self):
"""
Returns list of removed ``FileNode`` objects.
"""
return RemovedFileNodesGenerator(self.removed_paths, self)
@LazyProperty
def removed_paths(self):

5
rhodecode/lib/vcs/backends/hg/repository.py
View file

@ -450,7 +450,7 @@ class MercurialRepository(BaseRepository):
return os.path.join(self.path, '.hg', '.hgrc')
def get_commit(self, commit_id=None, commit_idx=None, pre_load=None,
translate_tag=None, maybe_unreachable=False, reference_obj=None):
translate_tag=None, maybe_unreachable=False, reference_obj=None) -> MercurialCommit:
"""
Returns ``MercurialCommit`` object representing repository's
commit at the given `commit_id` or `commit_idx`.
@ -598,8 +598,7 @@ class MercurialRepository(BaseRepository):
"""
Create a local clone of the current repo.
"""
self._remote.clone(self.path, clone_path, update_after_clone=True,
hooks=False)
self._remote.clone(self.path, clone_path, update_after_clone=True, hooks=False)
def _update(self, revision, clean=False):
"""

182
rhodecode/lib/vcs/backends/svn/commit.py
View file

@ -19,8 +19,7 @@
"""
SVN commit module
"""
import logging
import dateutil.parser
from zope.cachedescriptors.property import Lazy as LazyProperty
@ -28,9 +27,10 @@ from rhodecode.lib.str_utils import safe_bytes, safe_str
from rhodecode.lib.vcs import nodes, path as vcspath
from rhodecode.lib.vcs.backends import base
from rhodecode.lib.vcs.exceptions import CommitError
from vcsserver.lib.vcs_common import NodeKind, FILEMODE_EXECUTABLE, FILEMODE_DEFAULT, FILEMODE_LINK
_SVN_PROP_TRUE = "*"
_SVN_PROP_TRUE = '*'
log = logging.getLogger(__name__)
class SubversionCommit(base.BaseCommit):
@ -53,15 +53,16 @@ class SubversionCommit(base.BaseCommit):
# which knows how to translate commit index and commit id
self.raw_id = commit_id
self.short_id = commit_id
self.id = f'r{commit_id}'
self.id = f"r{commit_id}"
# TODO: Implement the following placeholder attributes
self.nodes = {}
self._path_mode_cache = {} # path stats cache, e.g filemode etc
self._path_type_cache = {} # path type dir/file/link etc cache
self.tags = []
@property
def author(self):
return safe_str(self._properties.get('svn:author'))
return safe_str(self._properties.get("svn:author"))
@property
def date(self):
@ -69,7 +70,7 @@ class SubversionCommit(base.BaseCommit):
@property
def message(self):
return safe_str(self._properties.get('svn:log'))
return safe_str(self._properties.get("svn:log"))
@LazyProperty
def _properties(self):
@ -91,19 +92,46 @@ class SubversionCommit(base.BaseCommit):
return [child]
return []
def get_file_mode(self, path: bytes):
def _calculate_file_mode(self, path: bytes):
# Note: Subversion flags files which are executable with a special
# property `svn:executable` which is set to the value ``"*"``.
if self._get_file_property(path, 'svn:executable') == _SVN_PROP_TRUE:
return base.FILEMODE_EXECUTABLE
if self._get_file_property(path, "svn:executable") == _SVN_PROP_TRUE:
return FILEMODE_EXECUTABLE
else:
return base.FILEMODE_DEFAULT
return FILEMODE_DEFAULT
def is_link(self, path):
def get_file_mode(self, path: bytes):
path = self._fix_path(path)
if path not in self._path_mode_cache:
self._path_mode_cache[path] = self._calculate_file_mode(path)
return self._path_mode_cache[path]
def _get_path_type(self, path: bytes):
if path in self._path_type_cache:
return self._path_type_cache[path]
if path == b"":
self._path_type_cache[b""] = NodeKind.DIR
return NodeKind.DIR
path_type = self._remote.get_node_type(self._svn_rev, path)
if not path_type:
raise self.no_node_at_path(path)
#flags = None
self._path_type_cache[path] = path_type
#self._path_mode_cache[path] = flags
return self._path_type_cache[path]
def is_link(self, path: bytes):
# Note: Subversion has a flag for special files, the content of the
# file contains the type of that file.
if self._get_file_property(path, 'svn:special') == _SVN_PROP_TRUE:
return self.get_file_content(path).startswith(b'link')
if self._get_file_property(path, "svn:special") == _SVN_PROP_TRUE:
return self.get_file_content(path).startswith(b"link")
return False
def is_node_binary(self, path):
@ -115,8 +143,7 @@ class SubversionCommit(base.BaseCommit):
return self._remote.md5_hash(self._svn_rev, safe_str(path))
def _get_file_property(self, path, name):
file_properties = self._remote.node_properties(
safe_str(path), self._svn_rev)
file_properties = self._remote.node_properties(safe_str(path), self._svn_rev)
return file_properties.get(name)
def get_file_content(self, path):
@ -126,7 +153,7 @@ class SubversionCommit(base.BaseCommit):
def get_file_content_streamed(self, path):
path = self._fix_path(path)
stream_method = getattr(self._remote, 'stream:get_file_content')
stream_method = getattr(self._remote, "stream:get_file_content")
return stream_method(self._svn_rev, safe_str(path))
def get_file_size(self, path):
@ -134,11 +161,9 @@ class SubversionCommit(base.BaseCommit):
return self._remote.get_file_size(self._svn_rev, safe_str(path))
def get_path_history(self, path, limit=None, pre_load=None):
path = safe_str(self._fix_path(path))
history = self._remote.node_history(path, self._svn_rev, limit)
return [
self.repository.get_commit(commit_id=str(svn_rev))
for svn_rev in history]
path = self._fix_path(path)
history = self._remote.node_history(self._svn_rev, safe_str(path), limit)
return [self.repository.get_commit(commit_id=str(svn_rev)) for svn_rev in history]
def get_file_annotate(self, path, pre_load=None):
result = self._remote.file_annotate(safe_str(path), self._svn_rev)
@ -146,67 +171,78 @@ class SubversionCommit(base.BaseCommit):
for zero_based_line_no, svn_rev, content in result:
commit_id = str(svn_rev)
line_no = zero_based_line_no + 1
yield (
line_no,
commit_id,
lambda: self.repository.get_commit(commit_id=commit_id),
content)
yield line_no, commit_id, lambda: self.repository.get_commit(commit_id=commit_id), content
def get_node(self, path, pre_load=None):
def get_node(self, path: bytes, pre_load=None):
path = self._fix_path(path)
if path not in self.nodes:
if path == '':
node = nodes.RootNode(commit=self)
# use cached, if we have one
if path in self.nodes:
return self.nodes[path]
path_type = self._get_path_type(path)
if path == b"":
node = nodes.RootNode(commit=self)
else:
if path_type == NodeKind.DIR:
node = nodes.DirNode(safe_bytes(path), commit=self)
elif path_type == NodeKind.FILE:
node = nodes.FileNode(safe_bytes(path), commit=self, pre_load=pre_load)
self._path_mode_cache[path] = node.mode
else:
node_type = self._remote.get_node_type(self._svn_rev, safe_str(path))
if node_type == 'dir':
node = nodes.DirNode(safe_bytes(path), commit=self)
elif node_type == 'file':
node = nodes.FileNode(safe_bytes(path), commit=self, pre_load=pre_load)
else:
raise self.no_node_at_path(path)
raise self.no_node_at_path(path)
self.nodes[path] = node
self.nodes[path] = node
return self.nodes[path]
def get_nodes(self, path, pre_load=None):
def get_nodes(self, path: bytes, pre_load=None):
if self._get_kind(path) != nodes.NodeKind.DIR:
raise CommitError(
f"Directory does not exist for commit {self.raw_id} at '{path}'")
path = safe_str(self._fix_path(path))
raise CommitError(f"Directory does not exist for commit {self.raw_id} at '{path}'")
path = self._fix_path(path)
path_nodes = []
for name, kind in self._remote.get_nodes(self._svn_rev, path):
node_path = vcspath.join(path, name)
if kind == 'dir':
node = nodes.DirNode(safe_bytes(node_path), commit=self)
elif kind == 'file':
node = nodes.FileNode(safe_bytes(node_path), commit=self, pre_load=pre_load)
else:
raise ValueError(f"Node kind {kind} not supported.")
self.nodes[node_path] = node
path_nodes.append(node)
for name, node_kind in self._remote.get_nodes(self._svn_rev, path):
obj_path = vcspath.join(path, name)
if node_kind is None:
raise CommitError(f"Requested object type={node_kind} cannot be determined")
# TODO: implement it ??
stat_ = None
# # cache file mode
# if obj_path not in self._path_mode_cache:
# self._path_mode_cache[obj_path] = stat_
# cache type
if node_kind not in self._path_type_cache:
self._path_type_cache[obj_path] = node_kind
entry = None
if obj_path in self.nodes:
entry = self.nodes[obj_path]
else:
if node_kind == NodeKind.DIR:
entry = nodes.DirNode(safe_bytes(obj_path), commit=self)
elif node_kind == NodeKind.FILE:
entry = nodes.FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load)
if entry:
self.nodes[obj_path] = entry
path_nodes.append(entry)
path_nodes.sort()
return path_nodes
def _get_kind(self, path):
path = self._fix_path(path)
kind = self._remote.get_node_type(self._svn_rev, path)
if kind == 'file':
return nodes.NodeKind.FILE
elif kind == 'dir':
return nodes.NodeKind.DIR
else:
raise CommitError(
f"Node does not exist at the given path '{path}'")
path_type = self._get_path_type(path)
return path_type
@LazyProperty
def _changes_cache(self):
return self._remote.revision_changes(self._svn_rev)
@LazyProperty
def affected_files(self):
def affected_files(self) -> list[bytes]:
changed_files = set()
for files in self._changes_cache.values():
changed_files.update(files)
@ -216,29 +252,17 @@ class SubversionCommit(base.BaseCommit):
def id(self):
return self.raw_id
@property
def added(self):
return nodes.AddedFileNodesGenerator(self.added_paths, self)
@LazyProperty
def added_paths(self):
return [n for n in self._changes_cache['added']]
@property
def changed(self):
return nodes.ChangedFileNodesGenerator(self.changed_paths, self)
return [n for n in self._changes_cache["added"]]
@LazyProperty
def changed_paths(self):
return [n for n in self._changes_cache['changed']]
@property
def removed(self):
return nodes.RemovedFileNodesGenerator(self.removed_paths, self)
return [n for n in self._changes_cache["changed"]]
@LazyProperty
def removed_paths(self):
return [n for n in self._changes_cache['removed']]
return [n for n in self._changes_cache["removed"]]
def _date_from_svn_properties(properties):
@ -248,7 +272,7 @@ def _date_from_svn_properties(properties):
:return: :class:`datetime.datetime` instance. The object is naive.
"""
aware_date = dateutil.parser.parse(properties.get('svn:date'))
aware_date = dateutil.parser.parse(properties.get("svn:date"))
# final_date = aware_date.astimezone(dateutil.tz.tzlocal())
final_date = aware_date
return final_date.replace(tzinfo=None)

17
rhodecode/lib/vcs/backends/svn/repository.py
View file

@ -30,7 +30,7 @@ from zope.cachedescriptors.property import Lazy as LazyProperty
from collections import OrderedDict
from rhodecode.lib.datelib import date_astimestamp
from rhodecode.lib.str_utils import safe_str
from rhodecode.lib.str_utils import safe_str, safe_bytes
from rhodecode.lib.utils2 import CachedProperty
from rhodecode.lib.vcs import connection, path as vcspath
from rhodecode.lib.vcs.backends import base
@ -157,16 +157,18 @@ class SubversionRepository(base.BaseRepository):
for pattern in self._patterns_from_section(config_section):
pattern = vcspath.sanitize(pattern)
bytes_pattern = safe_bytes(pattern)
tip = self.get_commit()
try:
if pattern.endswith('*'):
basedir = tip.get_node(vcspath.dirname(pattern))
if bytes_pattern.endswith(b'*'):
basedir = tip.get_node(vcspath.dirname(bytes_pattern))
directories = basedir.dirs
else:
directories = (tip.get_node(pattern), )
directories = (tip.get_node(bytes_pattern), )
except NodeDoesNotExistError:
continue
found_items.update((safe_str(n.path), self.commit_ids[-1]) for n in directories)
found_items.update((dir_node.str_path, self.commit_ids[-1]) for dir_node in directories)
def get_name(item):
return item[0]
@ -216,7 +218,7 @@ class SubversionRepository(base.BaseRepository):
def _get_commit_idx(self, commit_id):
try:
svn_rev = int(commit_id)
except:
except Exception:
# TODO: johbo: this might be only one case, HEAD, check this
svn_rev = self._remote.lookup(commit_id)
commit_idx = svn_rev - 1
@ -321,8 +323,7 @@ class SubversionRepository(base.BaseRepository):
# TODO: johbo: Reconsider impact of DEFAULT_BRANCH_NAME here
if branch_name not in [None, self.DEFAULT_BRANCH_NAME]:
svn_rev = int(self.commit_ids[-1])
commit_ids = self._remote.node_history(
path=branch_name, revision=svn_rev, limit=None)
commit_ids = self._remote.node_history(svn_rev, branch_name, None)
commit_ids = [str(i) for i in reversed(commit_ids)]
if start_pos or end_pos:

5
rhodecode/lib/vcs/client_http.py
View file

@ -27,14 +27,13 @@ import time
import urllib.request
import urllib.error
import urllib.parse
import urllib.parse
import uuid
import traceback
import pycurl
import msgpack
import requests
from requests.packages.urllib3.util.retry import Retry
from urllib3.util.retry import Retry
import rhodecode
from rhodecode.lib import rc_cache
@ -287,7 +286,7 @@ class RemoteRepo(object):
'fctx_size', 'stream:fctx_node_data', 'blob_raw_length',
'node_history',
'revision', 'tree_items',
'ctx_list', 'ctx_branch', 'ctx_description',
'ctx_branch', 'ctx_description',
'bulk_request',
'assert_correct_path',
'is_path_valid_repository',

2
rhodecode/lib/vcs/compat.py
View file

@ -1 +1 @@
from pyramid.compat import configparser
from pyramid.compat import configparser

3
rhodecode/lib/vcs/conf/settings.py
View file

@ -20,9 +20,6 @@
Internal settings for vcs-lib
"""
# list of default encoding used in safe_str methods
DEFAULT_ENCODINGS = ['utf8']
# Compatibility version when creating SVN repositories. None means newest.
# Other available options are: pre-1.4-compatible, pre-1.5-compatible,

4
rhodecode/lib/vcs/exceptions.py
View file

@ -102,10 +102,6 @@ class NodeError(VCSError):
pass
class RemovedFileNodeError(NodeError):
pass
class NodeAlreadyExistsError(CommittingError):
pass

386
rhodecode/lib/vcs/nodes.py
View file

@ -19,6 +19,7 @@
"""
Module holding everything related to vcs nodes, with vcs2 architecture.
"""
import functools
import os
import stat
@ -29,84 +30,26 @@ from rhodecode.config.conf import LANGUAGES_EXTENSIONS_MAP
from rhodecode.lib.str_utils import safe_str, safe_bytes
from rhodecode.lib.hash_utils import md5
from rhodecode.lib.vcs import path as vcspath
from rhodecode.lib.vcs.backends.base import EmptyCommit, FILEMODE_DEFAULT
from rhodecode.lib.vcs.backends.base import EmptyCommit
from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db
from rhodecode.lib.vcs.exceptions import NodeError, RemovedFileNodeError
from rhodecode.lib.vcs.exceptions import NodeError
from rhodecode.lib.vcs_common import NodeKind, FILEMODE_DEFAULT
LARGEFILE_PREFIX = '.hglf'
class NodeKind:
SUBMODULE = -1
DIR = 1
FILE = 2
LARGEFILE = 3
LARGEFILE_PREFIX = ".hglf"
class NodeState:
ADDED = 'added'
CHANGED = 'changed'
NOT_CHANGED = 'not changed'
REMOVED = 'removed'
ADDED = "added"
CHANGED = "changed"
NOT_CHANGED = "not changed"
REMOVED = "removed"
#TODO: not sure if that should be bytes or str ?
# TODO: not sure if that should be bytes or str ?
# most probably bytes because content should be bytes and we check it
BIN_BYTE_MARKER = b'\0'
BIN_BYTE_MARKER = b"\0"
class NodeGeneratorBase(object):
"""
Base class for removed added and changed filenodes, it's a lazy generator
class that will create filenodes only on iteration or call
The len method doesn't need to create filenodes at all
"""
def __init__(self, current_paths, cs):
self.cs = cs
self.current_paths = current_paths
def __call__(self):
return [n for n in self]
def __getitem__(self, key):
if isinstance(key, slice):
for p in self.current_paths[key.start:key.stop]:
yield self.cs.get_node(p)
def __len__(self):
return len(self.current_paths)
def __iter__(self):
for p in self.current_paths:
yield self.cs.get_node(p)
class AddedFileNodesGenerator(NodeGeneratorBase):
"""
Class holding added files for current commit
"""
class ChangedFileNodesGenerator(NodeGeneratorBase):
"""
Class holding changed files for current commit
"""
class RemovedFileNodesGenerator(NodeGeneratorBase):
"""
Class holding removed files for current commit
"""
def __iter__(self):
for p in self.current_paths:
yield RemovedFileNode(path=safe_bytes(p))
def __getitem__(self, key):
if isinstance(key, slice):
for p in self.current_paths[key.start:key.stop]:
yield RemovedFileNode(path=safe_bytes(p))
@functools.total_ordering
class Node(object):
@ -119,21 +62,22 @@ class Node(object):
only. Moreover, every single node is identified by the ``path`` attribute,
so it cannot end with slash, too. Otherwise, path could lead to mistakes.
"""
# RTLO marker allows swapping text, and certain
# security attacks could be used with this
RTLO_MARKER = "\u202E"
RTLO_MARKER = "\u202e"
commit = None
def __init__(self, path: bytes, kind):
self._validate_path(path) # can throw exception if path is invalid
self.bytes_path = path.rstrip(b'/') # store for __repr__
self.path = safe_str(self.bytes_path) # we store paths as str
self.bytes_path: bytes = path.rstrip(b"/") # store for mixed encoding, and raw version
self.str_path: str = safe_str(self.bytes_path) # we store paths as str
self.path: str = self.str_path
if self.bytes_path == b'' and kind != NodeKind.DIR:
raise NodeError("Only DirNode and its subclasses may be "
"initialized with empty path")
if self.bytes_path == b"" and kind != NodeKind.DIR:
raise NodeError("Only DirNode and its subclasses may be initialized with empty path")
self.kind = kind
if self.is_root() and not self.is_dir():
@ -142,7 +86,7 @@ class Node(object):
def __eq__(self, other):
if type(self) is not type(other):
return False
for attr in ['name', 'path', 'kind']:
for attr in ["name", "path", "kind"]:
if getattr(self, attr) != getattr(other, attr):
return False
if self.is_file():
@ -166,22 +110,9 @@ class Node(object):
if self.path > other.path:
return False
# def __cmp__(self, other):
# """
# Comparator using name of the node, needed for quick list sorting.
# """
#
# kind_cmp = cmp(self.kind, other.kind)
# if kind_cmp:
# if isinstance(self, SubModuleNode):
# # we make submodules equal to dirnode for "sorting" purposes
# return NodeKind.DIR
# return kind_cmp
# return cmp(self.name, other.name)
def __repr__(self):
maybe_path = getattr(self, 'path', 'UNKNOWN_PATH')
return f'<{self.__class__.__name__} {maybe_path!r}>'
maybe_path = getattr(self, "path", "UNKNOWN_PATH")
return f"<{self.__class__.__name__} {maybe_path!r}>"
def __str__(self):
return self.name
@ -189,29 +120,27 @@ class Node(object):
def _validate_path(self, path: bytes):
self._assert_bytes(path)
if path.startswith(b'/'):
if path.startswith(b"/"):
raise NodeError(
f"Cannot initialize Node objects with slash at "
f"the beginning as only relative paths are supported. "
f"Got {path}")
f"Got {path}"
)
def _assert_bytes(self, value):
@classmethod
def _assert_bytes(cls, value):
if not isinstance(value, bytes):
raise TypeError(f"Bytes required as input, got {type(value)} of {value}.")
@LazyProperty
def parent(self):
parent_path = self.get_parent_path()
parent_path: bytes = self.get_parent_path()
if parent_path:
if self.commit:
return self.commit.get_node(parent_path)
return DirNode(parent_path)
return None
@LazyProperty
def str_path(self) -> str:
return safe_str(self.path)
@LazyProperty
def has_rtlo(self):
"""Detects if a path has right-to-left-override marker"""
@ -223,10 +152,10 @@ class Node(object):
Returns name of the directory from full path of this vcs node. Empty
string is returned if there's no directory in the path
"""
_parts = self.path.rstrip('/').rsplit('/', 1)
_parts = self.path.rstrip("/").rsplit("/", 1)
if len(_parts) == 2:
return _parts[0]
return ''
return ""
@LazyProperty
def name(self):
@ -234,7 +163,7 @@ class Node(object):
Returns name of the node so if its path
then only last part is returned.
"""
return self.path.rstrip('/').split('/')[-1]
return self.str_path.rstrip("/").split("/")[-1]
@property
def kind(self):
@ -242,12 +171,12 @@ class Node(object):
@kind.setter
def kind(self, kind):
if hasattr(self, '_kind'):
if hasattr(self, "_kind"):
raise NodeError("Cannot change node's kind")
else:
self._kind = kind
# Post setter check (path's trailing slash)
if self.path.endswith('/'):
if self.str_path.endswith("/"):
raise NodeError("Node's path cannot end with slash")
def get_parent_path(self) -> bytes:
@ -255,8 +184,8 @@ class Node(object):
Returns node's parent path or empty string if node is root.
"""
if self.is_root():
return b''
str_path = vcspath.dirname(self.path.rstrip('/')) + '/'
return b""
str_path = vcspath.dirname(self.bytes_path.rstrip(b"/")) + b"/"
return safe_bytes(str_path)
@ -278,7 +207,7 @@ class Node(object):
"""
Returns ``True`` if node is a root node and ``False`` otherwise.
"""
return self.kind == NodeKind.DIR and self.path == ''
return self.kind == NodeKind.DIR and self.path == ""
def is_submodule(self):
"""
@ -292,29 +221,13 @@ class Node(object):
Returns ``True`` if node's kind is ``NodeKind.LARGEFILE``, ``False``
otherwise
"""
return self.kind == NodeKind.LARGEFILE
return self.kind == NodeKind.LARGE_FILE
def is_link(self):
if self.commit:
return self.commit.is_link(self.path)
return self.commit.is_link(self.bytes_path)
return False
@LazyProperty
def added(self):
return self.state is NodeState.ADDED
@LazyProperty
def changed(self):
return self.state is NodeState.CHANGED
@LazyProperty
def not_changed(self):
return self.state is NodeState.NOT_CHANGED
@LazyProperty
def removed(self):
return self.state is NodeState.REMOVED
class FileNode(Node):
"""
@ -325,6 +238,7 @@ class FileNode(Node):
:attribute: commit: if given, first time content is accessed, callback
:attribute: mode: stat mode for a node. Default is `FILEMODE_DEFAULT`.
"""
_filter_pre_load = []
def __init__(self, path: bytes, content: bytes | None = None, commit=None, mode=None, pre_load=None):
@ -359,7 +273,7 @@ class FileNode(Node):
return self.content == other.content
def __hash__(self):
raw_id = getattr(self.commit, 'raw_id', '')
raw_id = getattr(self.commit, "raw_id", "")
return hash((self.path, raw_id))
def __lt__(self, other):
@ -369,33 +283,32 @@ class FileNode(Node):
return self.content < other.content
def __repr__(self):
short_id = getattr(self.commit, 'short_id', '')
return f'<{self.__class__.__name__} path={self.path!r}, short_id={short_id}>'
short_id = getattr(self.commit, "short_id", "")
return f"<{self.__class__.__name__} path={self.str_path!r}, short_id={short_id}>"
def _set_bulk_properties(self, pre_load):
if not pre_load:
return
pre_load = [entry for entry in pre_load
if entry not in self._filter_pre_load]
pre_load = [entry for entry in pre_load if entry not in self._filter_pre_load]
if not pre_load:
return
remote = self.commit.get_remote()
result = remote.bulk_file_request(self.commit.raw_id, self.path, pre_load)
result = remote.bulk_file_request(self.commit.raw_id, self.bytes_path, pre_load)
for attr, value in result.items():
if attr == "flags":
self.__dict__['mode'] = safe_str(value)
self.__dict__["mode"] = safe_str(value)
elif attr == "size":
self.__dict__['size'] = value
self.__dict__["size"] = value
elif attr == "data":
self.__dict__['_content'] = value
self.__dict__["_content"] = value
elif attr == "is_binary":
self.__dict__['is_binary'] = value
self.__dict__["is_binary"] = value
elif attr == "md5":
self.__dict__['md5'] = value
self.__dict__["md5"] = value
else:
raise ValueError(f'Unsupported attr in bulk_property: {attr}')
raise ValueError(f"Unsupported attr in bulk_property: {attr}")
@LazyProperty
def mode(self):
@ -404,7 +317,7 @@ class FileNode(Node):
use value given at initialization or `FILEMODE_DEFAULT` (default).
"""
if self.commit:
mode = self.commit.get_file_mode(self.path)
mode = self.commit.get_file_mode(self.bytes_path)
else:
mode = self._mode
return mode
@ -416,7 +329,7 @@ class FileNode(Node):
"""
if self.commit:
if self._content is None:
self._content = self.commit.get_file_content(self.path)
self._content = self.commit.get_file_content(self.bytes_path)
content = self._content
else:
content = self._content
@ -427,7 +340,7 @@ class FileNode(Node):
Returns lazily content of the FileNode.
"""
if self.commit:
content = self.commit.get_file_content(self.path)
content = self.commit.get_file_content(self.bytes_path)
else:
content = self._content
return content
@ -438,7 +351,7 @@ class FileNode(Node):
vcsserver without loading it to memory.
"""
if self.commit:
return self.commit.get_file_content_streamed(self.path)
return self.commit.get_file_content_streamed(self.bytes_path)
raise NodeError("Cannot retrieve stream_bytes without related commit attribute")
def metadata_uncached(self):
@ -462,7 +375,7 @@ class FileNode(Node):
"""
content = self.raw_bytes
if content and not isinstance(content, bytes):
raise ValueError(f'Content is of type {type(content)} instead of bytes')
raise ValueError(f"Content is of type {type(content)} instead of bytes")
return content
@LazyProperty
@ -472,27 +385,21 @@ class FileNode(Node):
@LazyProperty
def size(self):
if self.commit:
return self.commit.get_file_size(self.path)
raise NodeError(
"Cannot retrieve size of the file without related "
"commit attribute")
return self.commit.get_file_size(self.bytes_path)
raise NodeError("Cannot retrieve size of the file without related commit attribute")
@LazyProperty
def message(self):
if self.commit:
return self.last_commit.message
raise NodeError(
"Cannot retrieve message of the file without related "
"commit attribute")
raise NodeError("Cannot retrieve message of the file without related " "commit attribute")
@LazyProperty
def last_commit(self):
if self.commit:
pre_load = ["author", "date", "message", "parents"]
return self.commit.get_path_commit(self.path, pre_load=pre_load)
raise NodeError(
"Cannot retrieve last commit of the file without "
"related commit attribute")
return self.commit.get_path_commit(self.bytes_path, pre_load=pre_load)
raise NodeError("Cannot retrieve last commit of the file without related commit attribute")
def get_mimetype(self):
"""
@ -502,28 +409,27 @@ class FileNode(Node):
attribute to indicate that type should *NOT* be calculated).
"""
if hasattr(self, '_mimetype'):
if (isinstance(self._mimetype, (tuple, list)) and
len(self._mimetype) == 2):
if hasattr(self, "_mimetype"):
if isinstance(self._mimetype, (tuple, list)) and len(self._mimetype) == 2:
return self._mimetype
else:
raise NodeError('given _mimetype attribute must be an 2 '
'element list or tuple')
raise NodeError("given _mimetype attribute must be an 2 element list or tuple")
db = get_mimetypes_db()
mtype, encoding = db.guess_type(self.name)
if mtype is None:
if not self.is_largefile() and self.is_binary:
mtype = 'application/octet-stream'
mtype = "application/octet-stream"
encoding = None
else:
mtype = 'text/plain'
mtype = "text/plain"
encoding = None
# try with pygments
try:
from pygments.lexers import get_lexer_for_filename
mt = get_lexer_for_filename(self.name).mimetypes
except Exception:
mt = None
@ -544,18 +450,17 @@ class FileNode(Node):
@LazyProperty
def mimetype_main(self):
return self.mimetype.split('/')[0]
return self.mimetype.split("/")[0]
@classmethod
def get_lexer(cls, filename, content=None):
from pygments import lexers
extension = filename.split('.')[-1]
extension = filename.split(".")[-1]
lexer = None
try:
lexer = lexers.guess_lexer_for_filename(
filename, content, stripnl=False)
lexer = lexers.guess_lexer_for_filename(filename, content, stripnl=False)
except lexers.ClassNotFound:
pass
@ -580,7 +485,7 @@ class FileNode(Node):
content, name and mimetype.
"""
# TODO: this is more proper, but super heavy on investigating the type based on the content
#self.get_lexer(self.name, self.content)
# self.get_lexer(self.name, self.content)
return self.get_lexer(self.name)
@ -597,8 +502,8 @@ class FileNode(Node):
Returns a list of commit for this file in which the file was changed
"""
if self.commit is None:
raise NodeError('Unable to get commit for this FileNode')
return self.commit.get_path_history(self.path)
raise NodeError("Unable to get commit for this FileNode")
return self.commit.get_path_history(self.bytes_path)
@LazyProperty
def annotate(self):
@ -606,22 +511,9 @@ class FileNode(Node):
Returns a list of three element tuples with lineno, commit and line
"""
if self.commit is None:
raise NodeError('Unable to get commit for this FileNode')
raise NodeError("Unable to get commit for this FileNode")
pre_load = ["author", "date", "message", "parents"]
return self.commit.get_file_annotate(self.path, pre_load=pre_load)
@LazyProperty
def state(self):
if not self.commit:
raise NodeError(
"Cannot check state of the node if it's not "
"linked with commit")
elif self.path in (node.path for node in self.commit.added):
return NodeState.ADDED
elif self.path in (node.path for node in self.commit.changed):
return NodeState.CHANGED
else:
return NodeState.NOT_CHANGED
return self.commit.get_file_annotate(self.bytes_path, pre_load=pre_load)
@LazyProperty
def is_binary(self):
@ -629,7 +521,7 @@ class FileNode(Node):
Returns True if file has binary content.
"""
if self.commit:
return self.commit.is_node_binary(self.path)
return self.commit.is_node_binary(self.bytes_path)
else:
raw_bytes = self._content
return bool(raw_bytes and BIN_BYTE_MARKER in raw_bytes)
@ -641,7 +533,7 @@ class FileNode(Node):
"""
if self.commit:
return self.commit.node_md5_hash(self.path)
return self.commit.node_md5_hash(self.bytes_path)
else:
raw_bytes = self._content
# TODO: this sucks, we're computing md5 on potentially super big stream data...
@ -650,7 +542,7 @@ class FileNode(Node):
@LazyProperty
def extension(self):
"""Returns filenode extension"""
return self.name.split('.')[-1]
return self.name.split(".")[-1]
@property
def is_executable(self):
@ -667,15 +559,15 @@ class FileNode(Node):
LF store.
"""
if self.commit:
return self.commit.get_largefile_node(self.path)
return self.commit.get_largefile_node(self.bytes_path)
def count_lines(self, content: str | bytes, count_empty=False):
if isinstance(content, str):
newline_marker = '\n'
newline_marker = "\n"
elif isinstance(content, bytes):
newline_marker = b'\n'
newline_marker = b"\n"
else:
raise ValueError('content must be bytes or str got {type(content)} instead')
raise ValueError("content must be bytes or str got {type(content)} instead")
if count_empty:
all_lines = 0
@ -704,33 +596,6 @@ class FileNode(Node):
return all_lines, empty_lines
class RemovedFileNode(FileNode):
"""
Dummy FileNode class - trying to access any public attribute except path,
name, kind or state (or methods/attributes checking those two) would raise
RemovedFileNodeError.
"""
ALLOWED_ATTRIBUTES = [
'name', 'path', 'state', 'is_root', 'is_file', 'is_dir', 'kind',
'added', 'changed', 'not_changed', 'removed', 'bytes_path'
]
def __init__(self, path):
"""
:param path: relative path to the node
"""
super().__init__(path=path)
def __getattribute__(self, attr):
if attr.startswith('_') or attr in RemovedFileNode.ALLOWED_ATTRIBUTES:
return super().__getattribute__(attr)
raise RemovedFileNodeError(f"Cannot access attribute {attr} on RemovedFileNode. Not in allowed attributes")
@LazyProperty
def state(self):
return NodeState.REMOVED
class DirNode(Node):
"""
DirNode stores list of files and directories within this node.
@ -752,7 +617,7 @@ class DirNode(Node):
super().__init__(path, NodeKind.DIR)
self.commit = commit
self._nodes = nodes
self.default_pre_load = default_pre_load or ['is_binary', 'size']
self.default_pre_load = default_pre_load or ["is_binary", "size"]
def __iter__(self):
yield from self.nodes
@ -782,10 +647,9 @@ class DirNode(Node):
@LazyProperty
def nodes(self):
if self.commit:
nodes = self.commit.get_nodes(self.path, pre_load=self.default_pre_load)
nodes = self.commit.get_nodes(self.bytes_path, pre_load=self.default_pre_load)
else:
nodes = self._nodes
self._nodes_dict = {node.path: node for node in nodes}
return sorted(nodes)
@LazyProperty
@ -796,47 +660,6 @@ class DirNode(Node):
def dirs(self):
return sorted(node for node in self.nodes if node.is_dir())
def get_node(self, path):
"""
Returns node from within this particular ``DirNode``, so it is now
allowed to fetch, i.e. node located at 'docs/api/index.rst' from node
'docs'. In order to access deeper nodes one must fetch nodes between
them first - this would work::
docs = root.get_node('docs')
docs.get_node('api').get_node('index.rst')
:param: path - relative to the current node
.. note::
To access lazily (as in example above) node have to be initialized
with related commit object - without it node is out of
context and may know nothing about anything else than nearest
(located at same level) nodes.
"""
try:
path = path.rstrip('/')
if path == '':
raise NodeError("Cannot retrieve node without path")
self.nodes # access nodes first in order to set _nodes_dict
paths = path.split('/')
if len(paths) == 1:
if not self.is_root():
path = '/'.join((self.path, paths[0]))
else:
path = paths[0]
return self._nodes_dict[path]
elif len(paths) > 1:
if self.commit is None:
raise NodeError("Cannot access deeper nodes without commit")
else:
path1, path2 = paths[0], '/'.join(paths[1:])
return self.get_node(path1).get_node(path2)
else:
raise KeyError
except KeyError:
raise NodeError(f"Node does not exist at {path}")
@LazyProperty
def state(self):
raise NodeError("Cannot access state of DirNode")
@ -844,7 +667,7 @@ class DirNode(Node):
@LazyProperty
def size(self):
size = 0
for root, dirs, files in self.commit.walk(self.path):
for root, dirs, files in self.commit.walk(self.bytes_path):
for f in files:
size += f.size
@ -854,14 +677,12 @@ class DirNode(Node):
def last_commit(self):
if self.commit:
pre_load = ["author", "date", "message", "parents"]
return self.commit.get_path_commit(self.path, pre_load=pre_load)
raise NodeError(
"Cannot retrieve last commit of the file without "
"related commit attribute")
return self.commit.get_path_commit(self.bytes_path, pre_load=pre_load)
raise NodeError("Cannot retrieve last commit of the file without related commit attribute")
def __repr__(self):
short_id = getattr(self.commit, 'short_id', '')
return f'<{self.__class__.__name__} {self.path!r} @ {short_id}>'
short_id = getattr(self.commit, "short_id", "")
return f"<{self.__class__.__name__} path={self.str_path!r}, short_id={short_id}>"
class RootNode(DirNode):
@ -870,21 +691,24 @@ class RootNode(DirNode):
"""
def __init__(self, nodes=(), commit=None):
super().__init__(path=b'', nodes=nodes, commit=commit)
super().__init__(path=b"", nodes=nodes, commit=commit)
def __repr__(self):
return f'<{self.__class__.__name__}>'
short_id = getattr(self.commit, "short_id", "")
return f"<{self.__class__.__name__} path={self.str_path!r}, short_id={short_id}>"
class SubModuleNode(Node):
"""
represents a SubModule of Git or SubRepo of Mercurial
"""
is_binary = False
size = 0
def __init__(self, name, url=None, commit=None, alias=None):
self.path = name
self.str_path: str = safe_str(self.path) # we store paths as str
self.kind = NodeKind.SUBMODULE
self.alias = alias
@ -894,8 +718,8 @@ class SubModuleNode(Node):
self.url = url or self._extract_submodule_url()
def __repr__(self):
short_id = getattr(self.commit, 'short_id', '')
return f'<{self.__class__.__name__} {self.path!r} @ {short_id}>'
short_id = getattr(self.commit, "short_id", "")
return f"<{self.__class__.__name__} {self.str_path!r} @ {short_id}>"
def _extract_submodule_url(self):
# TODO: find a way to parse gits submodule file and extract the
@ -908,22 +732,22 @@ class SubModuleNode(Node):
Returns name of the node so if its path
then only last part is returned.
"""
org = safe_str(self.path.rstrip('/').split('/')[-1])
return f'{org} @ {self.commit.short_id}'
org = self.str_path.rstrip("/").split("/")[-1]
return f"{org} @ {self.commit.short_id}"
class LargeFileNode(FileNode):
def __init__(self, path, url=None, commit=None, alias=None, org_path=None):
self._validate_path(path) # can throw exception if path is invalid
self.org_path = org_path # as stored in VCS as LF pointer
self.bytes_path = path.rstrip(b'/') # store for __repr__
self.path = safe_str(self.bytes_path) # we store paths as str
self.bytes_path = path.rstrip(b"/") # store for __repr__
self.str_path = safe_str(self.bytes_path)
self.path = self.str_path
self.kind = NodeKind.LARGEFILE
self.kind = NodeKind.LARGE_FILE
self.alias = alias
self._content = b''
self._content = b""
def _validate_path(self, path: bytes):
"""
@ -932,7 +756,7 @@ class LargeFileNode(FileNode):
self._assert_bytes(path)
def __repr__(self):
return f'<{self.__class__.__name__} {self.org_path} -> {self.path!r}>'
return f"<{self.__class__.__name__} {self.org_path} -> {self.str_path!r}>"
@LazyProperty
def size(self):
@ -940,7 +764,7 @@ class LargeFileNode(FileNode):
@LazyProperty
def raw_bytes(self):
with open(self.path, 'rb') as f:
with open(self.path, "rb") as f:
content = f.read()
return content
@ -952,7 +776,7 @@ class LargeFileNode(FileNode):
return self.org_path
def stream_bytes(self):
with open(self.path, 'rb') as stream:
with open(self.path, "rb") as stream:
while True:
data = stream.read(16 * 1024)
if not data:

46
rhodecode/lib/vcs_common.py Normal file
View file

@ -0,0 +1,46 @@
# Copyright (C) 2014-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
"""
Common VCS module for rhodecode and vcsserver
"""
import enum
FILEMODE_DEFAULT = 0o100644
FILEMODE_EXECUTABLE = 0o100755
FILEMODE_LINK = 0o120000
class NodeKind(int, enum.Enum):
SUBMODULE = -1
DIR = 1
FILE = 2
LARGE_FILE = 3
def map_git_obj_type(obj_type):
if obj_type == "blob":
return NodeKind.FILE
elif obj_type == "tree":
return NodeKind.DIR
elif obj_type == "link":
return NodeKind.SUBMODULE
return None

2
rhodecode/model/gist.py
View file

@ -103,7 +103,7 @@ class GistModel(BaseModel):
raise VCSError(f'Failed to load gist repository for {repo}')
commit = vcs_repo.get_commit(commit_id=revision)
return commit, [n for n in commit.get_node('/')]
return commit, [n for n in commit.get_node(b'/')]
def create(self, description, owner, gist_mapping,
gist_type=Gist.GIST_PUBLIC, lifetime=-1, gist_id=None,

10
rhodecode/model/pull_request.py
View file

@ -178,6 +178,7 @@ def get_diff_info(
log.debug('Calculating authors of changed files')
target_commit = source_repo.get_commit(ancestor_id)
# TODO: change to operate in bytes..
for fname, lines in changed_lines.items():
try:
@ -2223,8 +2224,7 @@ class MergeCheck(object):
)
@classmethod
def validate(cls, pull_request, auth_user, translator, fail_early=False,
force_shadow_repo_refresh=False):
def validate(cls, pull_request, auth_user, translator, fail_early=False, force_shadow_repo_refresh=False):
_ = translator
merge_check = cls()
@ -2285,12 +2285,10 @@ class MergeCheck(object):
# left over TODOs
todos = CommentsModel().get_pull_request_unresolved_todos(pull_request)
if todos:
log.debug("MergeCheck: cannot merge, {} "
"unresolved TODOs left.".format(len(todos)))
log.debug("MergeCheck: cannot merge, %s unresolved TODOs left.", len(todos))
if len(todos) == 1:
msg = _('Cannot merge, {} TODO still not resolved.').format(
len(todos))
msg = _('Cannot merge, {} TODO still not resolved.').format(len(todos))
else:
msg = _('Cannot merge, {} TODOs still not resolved.').format(
len(todos))

61
rhodecode/model/repo.py
View file

@ -33,6 +33,7 @@ from rhodecode.lib.auth import HasUserGroupPermissionAny
from rhodecode.lib.caching_query import FromCache
from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError
from rhodecode.lib import hooks_base
from rhodecode.lib.str_utils import safe_bytes
from rhodecode.lib.user_log_filter import user_log_filter
from rhodecode.lib.utils import make_db_config
from rhodecode.lib.utils2 import (
@ -1109,47 +1110,47 @@ class ReadmeFinder:
different.
"""
readme_re = re.compile(r'^readme(\.[^\.]+)?$', re.IGNORECASE)
path_re = re.compile(r'^docs?', re.IGNORECASE)
readme_re = re.compile(br'^readme(\.[^.]+)?$', re.IGNORECASE)
path_re = re.compile(br'^docs?', re.IGNORECASE)
default_priorities = {
None: 0,
'.rst': 1,
'.md': 1,
'.rest': 2,
'.mkdn': 2,
'.text': 2,
'.txt': 3,
'.mdown': 3,
'.markdown': 4,
b'.rst': 1,
b'.md': 1,
b'.rest': 2,
b'.mkdn': 2,
b'.text': 2,
b'.txt': 3,
b'.mdown': 3,
b'.markdown': 4,
}
path_priority = {
'doc': 0,
'docs': 1,
b'doc': 0,
b'docs': 1,
}
FALLBACK_PRIORITY = 99
RENDERER_TO_EXTENSION = {
'rst': ['.rst', '.rest'],
'markdown': ['.md', 'mkdn', '.mdown', '.markdown'],
'rst': [b'.rst', b'.rest'],
'markdown': [b'.md', b'mkdn', b'.mdown', b'.markdown'],
}
def __init__(self, default_renderer=None):
self._default_renderer = default_renderer
self._renderer_extensions = self.RENDERER_TO_EXTENSION.get(
default_renderer, [])
self._renderer_extensions = self.RENDERER_TO_EXTENSION.get(default_renderer, [])
def search(self, commit, path='/'):
def search(self, commit, path=b'/'):
"""
Find a readme in the given `commit`.
"""
# firstly, check the PATH type if it is actually a DIR
if commit.get_node(path).kind != NodeKind.DIR:
bytes_path = safe_bytes(path)
if commit.get_node(bytes_path).kind != NodeKind.DIR:
return None
nodes = commit.get_nodes(path)
nodes = commit.get_nodes(bytes_path)
matches = self._match_readmes(nodes)
matches = self._sort_according_to_priority(matches)
if matches:
@ -1157,8 +1158,8 @@ class ReadmeFinder:
paths = self._match_paths(nodes)
paths = self._sort_paths_according_to_priority(paths)
for path in paths:
match = self.search(commit, path=path)
for bytes_path in paths:
match = self.search(commit, path=bytes_path)
if match:
return match
@ -1168,7 +1169,7 @@ class ReadmeFinder:
for node in nodes:
if not node.is_file():
continue
path = node.path.rsplit('/', 1)[-1]
path = node.bytes_path.rsplit(b'/', 1)[-1]
match = self.readme_re.match(path)
if match:
extension = match.group(1)
@ -1178,28 +1179,26 @@ class ReadmeFinder:
for node in nodes:
if not node.is_dir():
continue
match = self.path_re.match(node.path)
match = self.path_re.match(node.bytes_path)
if match:
yield node.path
yield node.bytes_path
def _priority(self, extension):
renderer_priority = (
0 if extension in self._renderer_extensions else 1)
extension_priority = self.default_priorities.get(
extension, self.FALLBACK_PRIORITY)
return (renderer_priority, extension_priority)
renderer_priority = 0 if extension in self._renderer_extensions else 1
extension_priority = self.default_priorities.get(extension, self.FALLBACK_PRIORITY)
return renderer_priority, extension_priority
def _sort_according_to_priority(self, matches):
def priority_and_path(match):
return (match.priority, match.path)
return match.priority, match.path
return sorted(matches, key=priority_and_path)
def _sort_paths_according_to_priority(self, paths):
def priority_and_path(path):
return (self.path_priority.get(path, self.FALLBACK_PRIORITY), path)
return self.path_priority.get(path, self.FALLBACK_PRIORITY), path
return sorted(paths, key=priority_and_path)

11
rhodecode/model/scm.py
View file

@ -543,7 +543,7 @@ class ScmModel(BaseModel):
root_path = root_path.lstrip('/')
# get RootNode, inject pre-load options before walking
top_node = commit.get_node(root_path)
top_node = commit.get_node(safe_bytes(root_path))
extended_info_pre_load = []
if extended_info:
extended_info_pre_load += ['md5']
@ -614,12 +614,13 @@ class ScmModel(BaseModel):
_files = list()
_dirs = list()
bytes_path = safe_bytes(root_path)
try:
_repo = self._get_repo(repo_name)
commit = _repo.scm_instance().get_commit(commit_id=commit_id)
root_path = root_path.lstrip('/')
root_path = bytes_path.lstrip(b'/')
top_node = commit.get_node(root_path)
top_node = commit.get_node(safe_bytes(root_path))
top_node.default_pre_load = []
for __, dirs, files in commit.walk(top_node):
@ -736,7 +737,7 @@ class ScmModel(BaseModel):
_repo = self._get_repo(repo_name)
commit = _repo.scm_instance().get_commit(commit_id=commit_id)
root_path = root_path.lstrip('/')
top_node = commit.get_node(root_path)
top_node = commit.get_node(safe_bytes(root_path))
top_node.default_pre_load = []
for __, dirs, files in commit.walk(top_node):
@ -774,7 +775,7 @@ class ScmModel(BaseModel):
only for git
:param trigger_push_hook: trigger push hooks
:returns: new committed commit
:returns: new commit
"""
user, scm_instance, message, commiter, author, imc = self.initialize_inmemory_vars(
user, repo, message, author)

2
rhodecode/tests/config/test_routing_links.py
View file

@ -36,8 +36,8 @@ connection_available = pytest.mark.skipif(
import requests
from urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
def requests_retry_session(

4
rhodecode/tests/models/test_repos.py
View file

@ -92,8 +92,8 @@ class TestRepoModel(object):
is not None)
@pytest.mark.parametrize("filename, expected", [
("README", True),
("README.rst", False),
(b"README", True),
(b"README.rst", False),
])
def test_filenode_is_link(self, vcsbackend, filename, expected):
repo = vcsbackend.repo

16
rhodecode/tests/vcs/test_archives.py
View file

@ -28,7 +28,7 @@ import pytest
import rhodecode
from rhodecode.lib.archive_cache import get_archival_config
from rhodecode.lib.str_utils import ascii_bytes
from rhodecode.lib.str_utils import ascii_bytes, safe_bytes, safe_str
from rhodecode.lib.vcs.backends import base
from rhodecode.lib.vcs.exceptions import ImproperArchiveTypeError, VCSError
from rhodecode.lib.vcs.nodes import FileNode
@ -80,8 +80,8 @@ class TestArchives(BackendTestMixin):
out_file.close()
for x in range(5):
node_path = "%d/file_%d.txt" % (x, x)
with open(os.path.join(out_dir, "repo/" + node_path), "rb") as f:
node_path = b"%d/file_%d.txt" % (x, x)
with open(os.path.join(safe_bytes(str(out_dir)), b"repo/" + node_path), "rb") as f:
file_content = f.read()
assert file_content == self.tip.get_node(node_path).content
@ -120,8 +120,9 @@ class TestArchives(BackendTestMixin):
zip_file = zipfile.ZipFile(str(archive_lnk))
for x in range(5):
node_path = "%d/file_%d.txt" % (x, x)
data = zip_file.read(f"repo/{node_path}")
node_path = b"%d/file_%d.txt" % (x, x)
# NOTE: zipfile operates only on strings inside the archive
data = zip_file.read(safe_str(b"repo/%s" % node_path))
decompressed = io.BytesIO()
decompressed.write(data)
@ -143,8 +144,9 @@ class TestArchives(BackendTestMixin):
assert b"commit_id:%b" % raw_id in metafile
for x in range(5):
node_path = "%d/file_%d.txt" % (x, x)
data = zip_file.read(f"repo/{node_path}")
node_path = b"%d/file_%d.txt" % (x, x)
# NOTE: zipfile operates only on strings inside the archive
data = zip_file.read(safe_str(b"repo/%s" % node_path))
decompressed = io.BytesIO()
decompressed.write(data)
assert decompressed.getvalue() == self.tip.get_node(node_path).content

54
rhodecode/tests/vcs/test_commits.py
View file

@ -22,21 +22,17 @@ import time
import pytest
from rhodecode.lib.str_utils import safe_bytes
from rhodecode.lib.vcs.backends.base import CollectionGenerator, FILEMODE_DEFAULT, EmptyCommit
from rhodecode.lib.vcs.backends.base import CollectionGenerator, EmptyCommit
from rhodecode.lib.vcs.exceptions import (
BranchDoesNotExistError,
CommitDoesNotExistError,
RepositoryError,
EmptyRepositoryError,
)
from rhodecode.lib.vcs.nodes import (
FileNode,
AddedFileNodesGenerator,
ChangedFileNodesGenerator,
RemovedFileNodesGenerator,
)
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.tests import get_new_dir
from rhodecode.tests.vcs.conftest import BackendTestMixin
from rhodecode.lib.vcs_common import NodeKind, FILEMODE_EXECUTABLE, FILEMODE_DEFAULT, FILEMODE_LINK
class TestBaseChangeset(object):
@ -70,7 +66,7 @@ class TestCommitsInNonEmptyRepo(BackendTestMixin):
}
def test_walk_returns_empty_list_in_case_of_file(self):
result = list(self.tip.walk("file_0.txt"))
result = list(self.tip.walk(b"file_0.txt"))
assert result == []
@pytest.mark.backends("git", "hg")
@ -319,7 +315,7 @@ class TestCommits(BackendTestMixin):
def test_get_path_commit(self):
commit = self.repo.get_commit()
commit.get_path_commit("file_4.txt")
commit.get_path_commit(b"file_4.txt")
assert commit.message == "Commit 4"
def test_get_filenodes_generator(self):
@ -500,8 +496,8 @@ class TestCommits(BackendTestMixin):
@pytest.mark.parametrize(
"filename, expected",
[
("README.rst", False),
("README", True),
(b"README.rst", False),
(b"README", True),
],
)
def test_commit_is_link(vcsbackend, filename, expected):
@ -543,49 +539,41 @@ class TestCommitsChanges(BackendTestMixin):
def test_initial_commit(self, local_dt_to_utc):
commit = self.repo.get_commit(commit_idx=0)
assert set(commit.added) == {
commit.get_node("foo/bar"),
commit.get_node("foo/bał"),
commit.get_node("foobar"),
commit.get_node("qwe"),
}
assert set(commit.changed) == set()
assert set(commit.removed) == set()
assert set(commit.affected_files) == {"foo/bar", "foo/bał", "foobar", "qwe"}
assert sorted(commit.added_paths) == sorted([b"foo/bar", b"foo/ba\xc5\x82", b"foobar", b"qwe"])
assert commit.changed_paths == []
assert commit.removed_paths == []
assert sorted(commit.affected_files) == sorted([b"foo/bar", b"foo/ba\xc5\x82", b"foobar", b"qwe"])
assert commit.date == local_dt_to_utc(datetime.datetime(2010, 1, 1, 20, 0))
def test_head_added(self):
commit = self.repo.get_commit()
assert isinstance(commit.added, AddedFileNodesGenerator)
assert set(commit.added) == {commit.get_node("fallout")}
assert isinstance(commit.changed, ChangedFileNodesGenerator)
assert set(commit.changed) == {commit.get_node("foo/bar"), commit.get_node("foobar")}
assert isinstance(commit.removed, RemovedFileNodesGenerator)
assert len(commit.removed) == 1
assert list(commit.removed)[0].path == "qwe"
assert commit.added_paths == [b"fallout"]
assert commit.changed_paths == [b"foo/bar", b"foobar"]
assert commit.removed_paths == [b"qwe"]
def test_get_filemode(self):
commit = self.repo.get_commit()
assert FILEMODE_DEFAULT == commit.get_file_mode("foo/bar")
assert FILEMODE_DEFAULT == commit.get_file_mode(b"foo/bar")
def test_get_filemode_non_ascii(self):
commit = self.repo.get_commit()
assert FILEMODE_DEFAULT == commit.get_file_mode("foo/bał")
assert FILEMODE_DEFAULT == commit.get_file_mode("foo/bał")
assert FILEMODE_DEFAULT == commit.get_file_mode(b"foo/ba\xc5\x82")
assert FILEMODE_DEFAULT == commit.get_file_mode(b"foo/ba\xc5\x82")
def test_get_path_history(self):
commit = self.repo.get_commit()
history = commit.get_path_history("foo/bar")
history = commit.get_path_history(b"foo/bar")
assert len(history) == 2
def test_get_path_history_with_limit(self):
commit = self.repo.get_commit()
history = commit.get_path_history("foo/bar", limit=1)
history = commit.get_path_history(b"foo/bar", limit=1)
assert len(history) == 1
def test_get_path_history_first_commit(self):
commit = self.repo[0]
history = commit.get_path_history("foo/bar")
history = commit.get_path_history(b"foo/bar")
assert len(history) == 1

227
rhodecode/tests/vcs/test_filenodes_listing_and_caches.py Normal file
View file

@ -0,0 +1,227 @@
# Copyright (C) 2010-2024 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import datetime
import pytest
from rhodecode.lib.str_utils import safe_bytes
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.lib.vcs_common import NodeKind
from rhodecode.tests.vcs.conftest import BackendTestMixin
class TestFileNodesListingAndCaches:
def test_filenode_get_root_node(self, vcsbackend):
repo = vcsbackend.repo
commit = repo.get_commit()
# check if we start with empty nodes cache
assert commit.nodes == {}
assert commit._path_mode_cache == {}
assert commit._path_type_cache == {}
top_dir = commit.get_node(b"")
assert top_dir.is_dir()
assert list(commit.nodes.keys()) == [b""]
assert list(commit._path_type_cache.keys()) == [b""]
assert commit._path_mode_cache == {}
def test_filenode_get_file_node(self, vcsbackend):
repo = vcsbackend.repo
commit = repo.get_commit()
# check if we start with empty nodes cache
assert commit.nodes == {}
assert commit._path_mode_cache == {}
assert commit._path_type_cache == {}
file_node = commit.get_node(b"README.rst")
assert file_node.is_file()
assert file_node.last_commit
assert list(commit.nodes.keys()) == [b"README.rst"]
if repo.alias == "hg":
assert commit._path_type_cache == {b"README.rst": NodeKind.FILE}
assert commit._path_mode_cache == {b"README.rst": 33188}
if repo.alias == "git":
assert commit._path_type_cache == {
b"README.rst": ["8f111ab5152b9fd34f52b0fb288e1216b5d2f55e", NodeKind.FILE]
}
assert commit._path_mode_cache == {b"README.rst": 33188}
if repo.alias == "svn":
assert commit._path_type_cache == {b"README.rst": NodeKind.FILE}
assert commit._path_mode_cache == {b"README.rst": 33188}
def test_filenode_get_nodes_from_top_dir(self, vcsbackend):
repo = vcsbackend.repo
commit = repo.get_commit()
# check if we start with empty nodes cache
assert commit.nodes == {}
assert commit._path_mode_cache == {}
assert commit._path_type_cache == {}
node_list = commit.get_nodes(b"")
for node in node_list:
assert node
if repo.alias == "svn":
assert list(commit.nodes.keys()) == [
b"README",
b".hgignore",
b"MANIFEST.in",
b".travis.yml",
b"vcs",
b"tox.ini",
b"setup.py",
b"setup.cfg",
b"docs",
b"fakefile",
b"examples",
b"test_and_report.sh",
b"bin",
b".gitignore",
b".hgtags",
b"README.rst",
]
assert commit._path_type_cache == {
b"": NodeKind.DIR,
b"README": 2,
b".hgignore": 2,
b"MANIFEST.in": 2,
b".travis.yml": 2,
b"vcs": 1,
b"tox.ini": 2,
b"setup.py": 2,
b"setup.cfg": 2,
b"docs": 1,
b"fakefile": 2,
b"examples": 1,
b"test_and_report.sh": 2,
b"bin": 1,
b".gitignore": 2,
b".hgtags": 2,
b"README.rst": 2,
}
assert commit._path_mode_cache == {}
if repo.alias == "hg":
assert list(commit.nodes.keys()) == [
b".gitignore",
b".hgignore",
b".hgtags",
b".travis.yml",
b"MANIFEST.in",
b"README",
b"README.rst",
b"docs",
b"run_test_and_report.sh",
b"setup.cfg",
b"setup.py",
b"test_and_report.sh",
b"tox.ini",
b"vcs",
]
assert commit._path_type_cache == {
b"": NodeKind.DIR,
b".gitignore": 2,
b".hgignore": 2,
b".hgtags": 2,
b".travis.yml": 2,
b"MANIFEST.in": 2,
b"README": 2,
b"README.rst": 2,
b"docs": 1,
b"run_test_and_report.sh": 2,
b"setup.cfg": 2,
b"setup.py": 2,
b"test_and_report.sh": 2,
b"tox.ini": 2,
b"vcs": 1,
}
assert commit._path_mode_cache == {}
if repo.alias == "git":
assert list(commit.nodes.keys()) == [
b".gitignore",
b".hgignore",
b".hgtags",
b"MANIFEST.in",
b"README",
b"README.rst",
b"docs",
b"run_test_and_report.sh",
b"setup.cfg",
b"setup.py",
b"test_and_report.sh",
b"tox.ini",
b"vcs",
]
assert commit._path_type_cache == {
b"": ["2dab7f7377e36948b5633ae0877310380fdf64ba", NodeKind.DIR],
b".gitignore": ["7f95b0917e31b0cce1ad96a033b1a814c420cde3", 2],
b".hgignore": ["9bdee5ed67ceed7c4b5589ba26dada65df151aed", 2],
b".hgtags": ["63aa2e892deebdfdef1845fe52e2418ade03557b", 2],
b"MANIFEST.in": ["6d65aca5321c26589ae197b81511445cfb353c95", 2],
b"README": ["92cacd285355271487b7e379dba6ca60f9a554a4", 2],
b"README.rst": ["8f111ab5152b9fd34f52b0fb288e1216b5d2f55e", 2],
b"docs": ["eaa8cef76ec4e1baf06f515ab03bfc01719a913d", 1],
b"run_test_and_report.sh": ["e17011bdddc8812fda96e891a1087c73054f1f25", 2],
b"setup.cfg": ["2e76b65bc9106ed466d71fb2cfe710352eadfb49", 2],
b"setup.py": ["ccda2bd076eca961059deba6e39fb591d014052a", 2],
b"test_and_report.sh": ["0b14056dd7dc759a9719adef3d7bb529141a740e", 2],
b"tox.ini": ["c1735231e7af32f089b95455162db092824715a4", 2],
b"vcs": ["5868c69a8d74ccf3fdc655af8f3f185bc0ff2ef6", 1],
}
assert commit._path_mode_cache == {
b".gitignore": 33188,
b".hgignore": 33188,
b".hgtags": 33188,
b"MANIFEST.in": 33188,
b"README": 40960,
b"README.rst": 33188,
b"docs": 16384,
b"run_test_and_report.sh": 33261,
b"setup.cfg": 33188,
b"setup.py": 33188,
b"test_and_report.sh": 33261,
b"tox.ini": 33188,
b"vcs": 16384,
}
def test_filenode_get_nodes_from_docs_dir(self, vcsbackend):
repo = vcsbackend.repo
commit = repo.get_commit()
# check if we start with empty nodes cache
assert commit.nodes == {}
assert commit._path_mode_cache == {}
assert commit._path_type_cache == {}
node_list = commit.get_nodes(b"docs")
for node in node_list:
assert node

179
rhodecode/tests/vcs/test_git.py
View file

@ -28,7 +28,7 @@ from rhodecode.lib.utils import make_db_config
from rhodecode.lib.vcs.backends.base import Reference
from rhodecode.lib.vcs.backends.git import GitRepository, GitCommit, discover_git_version
from rhodecode.lib.vcs.exceptions import RepositoryError, VCSError, NodeDoesNotExistError
from rhodecode.lib.vcs.nodes import NodeKind, FileNode, DirNode, NodeState, SubModuleNode
from rhodecode.lib.vcs.nodes import NodeKind, FileNode, DirNode, NodeState, SubModuleNode, RootNode
from rhodecode.tests import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, get_new_dir
from rhodecode.tests.vcs.conftest import BackendTestMixin
@ -219,23 +219,38 @@ class TestGitRepository(object):
assert init_commit.message == "initial import\n"
assert init_author == "Marcin Kuzminski <marcin@python-blog.com>"
assert init_author == init_commit.committer
for path in ("vcs/__init__.py", "vcs/backends/BaseRepository.py", "vcs/backends/__init__.py"):
assert sorted(init_commit.added_paths) == sorted(
[
b"vcs/__init__.py",
b"vcs/backends/BaseRepository.py",
b"vcs/backends/__init__.py",
]
)
assert sorted(init_commit.affected_files) == sorted(
[
b"vcs/__init__.py",
b"vcs/backends/BaseRepository.py",
b"vcs/backends/__init__.py",
]
)
for path in (b"vcs/__init__.py", b"vcs/backends/BaseRepository.py", b"vcs/backends/__init__.py"):
assert isinstance(init_commit.get_node(path), FileNode)
for path in ("", "vcs", "vcs/backends"):
for path in (b"", b"vcs", b"vcs/backends"):
assert isinstance(init_commit.get_node(path), DirNode)
with pytest.raises(NodeDoesNotExistError):
init_commit.get_node(path="foobar")
init_commit.get_node(path=b"foobar")
node = init_commit.get_node("vcs/")
node = init_commit.get_node(b"vcs/")
assert hasattr(node, "kind")
assert node.kind == NodeKind.DIR
node = init_commit.get_node("vcs")
node = init_commit.get_node(b"vcs")
assert hasattr(node, "kind")
assert node.kind == NodeKind.DIR
node = init_commit.get_node("vcs/__init__.py")
node = init_commit.get_node(b"vcs/__init__.py")
assert hasattr(node, "kind")
assert node.kind == NodeKind.FILE
@ -257,7 +272,7 @@ Introduction
TODO: To be written...
"""
node = commit10.get_node("README.rst")
node = commit10.get_node(b"README.rst")
assert node.kind == NodeKind.FILE
assert node.str_content == README
@ -615,7 +630,7 @@ class TestGitCommit(object):
def test_root_node(self):
tip = self.repo.get_commit()
assert tip.root is tip.get_node("")
assert tip.root is tip.get_node(b"")
def test_lazy_fetch(self):
"""
@ -633,29 +648,29 @@ class TestGitCommit(object):
# accessing root.nodes updates commit.nodes
assert len(commit.nodes) == 9
docs = root.get_node("docs")
docs = commit.get_node(b"docs")
# we haven't yet accessed anything new as docs dir was already cached
assert len(commit.nodes) == 9
assert len(docs.nodes) == 8
# accessing docs.nodes updates commit.nodes
assert len(commit.nodes) == 17
assert docs is commit.get_node("docs")
assert docs is commit.get_node(b"docs")
assert docs is root.nodes[0]
assert docs is root.dirs[0]
assert docs is commit.get_node("docs")
assert docs is commit.get_node(b"docs")
def test_nodes_with_commit(self):
commit_id = "2a13f185e4525f9d4b59882791a2d397b90d5ddc"
commit = self.repo.get_commit(commit_id)
root = commit.root
docs = root.get_node("docs")
assert docs is commit.get_node("docs")
api = docs.get_node("api")
assert api is commit.get_node("docs/api")
index = api.get_node("index.rst")
assert index is commit.get_node("docs/api/index.rst")
assert index is commit.get_node("docs").get_node("api").get_node("index.rst")
assert isinstance(root, RootNode)
docs = commit.get_node(b"docs")
assert docs is commit.get_node(b"docs")
api = commit.get_node(b"docs/api")
assert api is commit.get_node(b"docs/api")
index = commit.get_node(b"docs/api/index.rst")
assert index is commit.get_node(b"docs/api/index.rst")
def test_branch_and_tags(self):
"""
@ -682,12 +697,12 @@ class TestGitCommit(object):
def test_file_size(self):
to_check = (
("c1214f7e79e02fc37156ff215cd71275450cffc3", "vcs/backends/BaseRepository.py", 502),
("d7e0d30fbcae12c90680eb095a4f5f02505ce501", "vcs/backends/hg.py", 854),
("6e125e7c890379446e98980d8ed60fba87d0f6d1", "setup.py", 1068),
("d955cd312c17b02143c04fa1099a352b04368118", "vcs/backends/base.py", 2921),
("ca1eb7957a54bce53b12d1a51b13452f95bc7c7e", "vcs/backends/base.py", 3936),
("f50f42baeed5af6518ef4b0cb2f1423f3851a941", "vcs/backends/base.py", 6189),
("c1214f7e79e02fc37156ff215cd71275450cffc3", b"vcs/backends/BaseRepository.py", 502),
("d7e0d30fbcae12c90680eb095a4f5f02505ce501", b"vcs/backends/hg.py", 854),
("6e125e7c890379446e98980d8ed60fba87d0f6d1", b"setup.py", 1068),
("d955cd312c17b02143c04fa1099a352b04368118", b"vcs/backends/base.py", 2921),
("ca1eb7957a54bce53b12d1a51b13452f95bc7c7e", b"vcs/backends/base.py", 3936),
("f50f42baeed5af6518ef4b0cb2f1423f3851a941", b"vcs/backends/base.py", 6189),
)
for commit_id, path, size in to_check:
node = self.repo.get_commit(commit_id).get_node(path)
@ -695,17 +710,17 @@ class TestGitCommit(object):
assert node.size == size
def test_file_history_from_commits(self):
node = self.repo[10].get_node("setup.py")
node = self.repo[10].get_node(b"setup.py")
commit_ids = [commit.raw_id for commit in node.history]
assert ["ff7ca51e58c505fec0dd2491de52c622bb7a806b"] == commit_ids
node = self.repo[20].get_node("setup.py")
node = self.repo[20].get_node(b"setup.py")
node_ids = [commit.raw_id for commit in node.history]
assert ["191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e", "ff7ca51e58c505fec0dd2491de52c622bb7a806b"] == node_ids
# special case we check history from commit that has this particular
# file changed this means we check if it's included as well
node = self.repo.get_commit("191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e").get_node("setup.py")
node = self.repo.get_commit("191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e").get_node(b"setup.py")
node_ids = [commit.raw_id for commit in node.history]
assert ["191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e", "ff7ca51e58c505fec0dd2491de52c622bb7a806b"] == node_ids
@ -713,7 +728,7 @@ class TestGitCommit(object):
# we can only check if those commits are present in the history
# as we cannot update this test every time file is changed
files = {
"setup.py": [
b"setup.py": [
"54386793436c938cff89326944d4c2702340037d",
"51d254f0ecf5df2ce50c0b115741f4cf13985dab",
"998ed409c795fec2012b1c0ca054d99888b22090",
@ -724,7 +739,7 @@ class TestGitCommit(object):
"191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e",
"ff7ca51e58c505fec0dd2491de52c622bb7a806b",
],
"vcs/nodes.py": [
b"vcs/nodes.py": [
"33fa3223355104431402a888fa77a4e9956feb3e",
"fa014c12c26d10ba682fadb78f2a11c24c8118e1",
"e686b958768ee96af8029fe19c6050b1a8dd3b2b",
@ -757,7 +772,7 @@ class TestGitCommit(object):
"dd80b0f6cf5052f17cc738c2951c4f2070200d7f",
"ff7ca51e58c505fec0dd2491de52c622bb7a806b",
],
"vcs/backends/git.py": [
b"vcs/backends/git.py": [
"4cf116ad5a457530381135e2f4c453e68a1b0105",
"9a751d84d8e9408e736329767387f41b36935153",
"cb681fb539c3faaedbcdf5ca71ca413425c18f01",
@ -778,7 +793,7 @@ class TestGitCommit(object):
def test_file_annotate(self):
files = {
"vcs/backends/__init__.py": {
b"vcs/backends/__init__.py": {
"c1214f7e79e02fc37156ff215cd71275450cffc3": {
"lines_no": 1,
"commits": [
@ -870,39 +885,31 @@ class TestGitCommit(object):
"""
Tests state of FileNodes.
"""
node = self.repo.get_commit("e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0").get_node("vcs/utils/diffs.py")
assert node.state, NodeState.ADDED
assert node.added
assert not node.changed
assert not node.not_changed
assert not node.removed
commit = self.repo.get_commit("e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0")
node = commit.get_node(b"vcs/utils/diffs.py")
assert node.bytes_path in commit.added_paths
node = self.repo.get_commit("33fa3223355104431402a888fa77a4e9956feb3e").get_node(".hgignore")
assert node.state, NodeState.CHANGED
assert not node.added
assert node.changed
assert not node.not_changed
assert not node.removed
commit = self.repo.get_commit("33fa3223355104431402a888fa77a4e9956feb3e")
node = commit.get_node(b".hgignore")
assert node.bytes_path in commit.changed_paths
node = self.repo.get_commit("e29b67bd158580fc90fc5e9111240b90e6e86064").get_node("setup.py")
assert node.state, NodeState.NOT_CHANGED
assert not node.added
assert not node.changed
assert node.not_changed
assert not node.removed
commit = self.repo.get_commit("e29b67bd158580fc90fc5e9111240b90e6e86064")
node = commit.get_node(b"setup.py")
assert node.bytes_path not in commit.affected_files
# If node has REMOVED state then trying to fetch it would raise
# CommitError exception
commit = self.repo.get_commit("fa6600f6848800641328adbf7811fd2372c02ab2")
path = "vcs/backends/BaseRepository.py"
path = b"vcs/backends/BaseRepository.py"
with pytest.raises(NodeDoesNotExistError):
commit.get_node(path)
# but it would be one of ``removed`` (commit's attribute)
assert path in [rf.path for rf in commit.removed]
assert path in [rf for rf in commit.removed_paths]
commit = self.repo.get_commit("54386793436c938cff89326944d4c2702340037d")
changed = ["setup.py", "tests/test_nodes.py", "vcs/backends/hg.py", "vcs/nodes.py"]
assert set(changed) == set([f.path for f in commit.changed])
changed = [b"setup.py", b"tests/test_nodes.py", b"vcs/backends/hg.py", b"vcs/nodes.py"]
assert set(changed) == set([f for f in commit.changed_paths])
def test_unicode_branch_refs(self):
unicode_branches = {
@ -936,14 +943,14 @@ class TestGitCommit(object):
def test_repo_files_content_types(self):
commit = self.repo.get_commit()
for node in commit.get_node("/"):
for node in commit.get_node(b"/"):
if node.is_file():
assert type(node.content) == bytes
assert type(node.str_content) == str
def test_wrong_path(self):
# There is 'setup.py' in the root dir but not there:
path = "foo/bar/setup.py"
path = b"foo/bar/setup.py"
tip = self.repo.get_commit()
with pytest.raises(VCSError):
tip.get_node(path)
@ -981,8 +988,7 @@ class TestLargeFileRepo(object):
repo = backend_git.create_test_repo("largefiles", conf)
tip = repo.scm_instance().get_commit()
node = tip.get_node("1MB.zip")
node = tip.get_node(b"1MB.zip")
# extract stored LF node into the origin cache
repo_lfs_store: str = os.path.join(repo.repo_path, repo.repo_name, "lfs_store")
@ -1002,7 +1008,7 @@ class TestLargeFileRepo(object):
assert lf_node.is_largefile() is True
assert lf_node.size == 1024000
assert lf_node.name == "1MB.zip"
assert lf_node.name == b"1MB.zip"
@pytest.mark.usefixtures("vcs_repository_support")
@ -1032,14 +1038,11 @@ class TestGitSpecificWithRepo(BackendTestMixin):
def test_paths_slow_traversing(self):
commit = self.repo.get_commit()
assert (
commit.get_node("foobar").get_node("static").get_node("js").get_node("admin").get_node("base.js").content
== b"base"
)
assert commit.get_node(b"foobar/static/js/admin/base.js").content == b"base"
def test_paths_fast_traversing(self):
commit = self.repo.get_commit()
assert commit.get_node("foobar/static/js/admin/base.js").content == b"base"
assert commit.get_node(b"foobar/static/js/admin/base.js").content == b"base"
def test_get_diff_runs_git_command_with_hashes(self):
comm1 = self.repo[0]
@ -1110,12 +1113,15 @@ class TestGitRegression(BackendTestMixin):
@pytest.mark.parametrize(
"path, expected_paths",
[
("bot", ["bot/build", "bot/templates", "bot/__init__.py"]),
("bot/build", ["bot/build/migrations", "bot/build/static", "bot/build/templates"]),
("bot/build/static", ["bot/build/static/templates"]),
("bot/build/static/templates", ["bot/build/static/templates/f.html", "bot/build/static/templates/f1.html"]),
("bot/build/templates", ["bot/build/templates/err.html", "bot/build/templates/err2.html"]),
("bot/templates/", ["bot/templates/404.html", "bot/templates/500.html"]),
(b"bot", ["bot/build", "bot/templates", "bot/__init__.py"]),
(b"bot/build", ["bot/build/migrations", "bot/build/static", "bot/build/templates"]),
(b"bot/build/static", ["bot/build/static/templates"]),
(
b"bot/build/static/templates",
["bot/build/static/templates/f.html", "bot/build/static/templates/f1.html"],
),
(b"bot/build/templates", ["bot/build/templates/err.html", "bot/build/templates/err2.html"]),
(b"bot/templates/", ["bot/templates/404.html", "bot/templates/500.html"]),
],
)
def test_similar_paths(self, path, expected_paths):
@ -1146,8 +1152,8 @@ class TestGetSubmoduleUrl(object):
node.str_content = (
'[submodule "subrepo1"]\n' "\tpath = subrepo1\n" "\turl = https://code.rhodecode.com/dulwich\n"
)
result = commit._get_submodule_url("subrepo1")
get_node_mock.assert_called_once_with(".gitmodules")
result = commit._get_submodule_url(b"subrepo1")
get_node_mock.assert_called_once_with(b".gitmodules")
assert result == "https://code.rhodecode.com/dulwich"
def test_complex_submodule_path(self):
@ -1160,14 +1166,14 @@ class TestGetSubmoduleUrl(object):
"\tpath = complex/subrepo/path\n"
"\turl = https://code.rhodecode.com/dulwich\n"
)
result = commit._get_submodule_url("complex/subrepo/path")
get_node_mock.assert_called_once_with(".gitmodules")
result = commit._get_submodule_url(b"complex/subrepo/path")
get_node_mock.assert_called_once_with(b".gitmodules")
assert result == "https://code.rhodecode.com/dulwich"
def test_submodules_file_not_found(self):
commit = GitCommit(repository=mock.Mock(), raw_id="abcdef12", idx=1)
with mock.patch.object(commit, "get_node", side_effect=NodeDoesNotExistError):
result = commit._get_submodule_url("complex/subrepo/path")
result = commit._get_submodule_url(b"complex/subrepo/path")
assert result is None
def test_path_not_found(self):
@ -1178,8 +1184,8 @@ class TestGetSubmoduleUrl(object):
node.str_content = (
'[submodule "subrepo1"]\n' "\tpath = subrepo1\n" "\turl = https://code.rhodecode.com/dulwich\n"
)
result = commit._get_submodule_url("subrepo2")
get_node_mock.assert_called_once_with(".gitmodules")
result = commit._get_submodule_url(b"subrepo2")
get_node_mock.assert_called_once_with(b".gitmodules")
assert result is None
def test_returns_cached_values(self):
@ -1191,44 +1197,43 @@ class TestGetSubmoduleUrl(object):
'[submodule "subrepo1"]\n' "\tpath = subrepo1\n" "\turl = https://code.rhodecode.com/dulwich\n"
)
for _ in range(3):
commit._get_submodule_url("subrepo1")
get_node_mock.assert_called_once_with(".gitmodules")
commit._get_submodule_url(b"subrepo1")
get_node_mock.assert_called_once_with(b".gitmodules")
def test_get_node_returns_a_link(self):
repository = mock.Mock()
repository.alias = "git"
commit = GitCommit(repository=repository, raw_id="abcdef12", idx=1)
submodule_url = "https://code.rhodecode.com/dulwich"
get_id_patch = mock.patch.object(commit, "_get_tree_id_for_path", return_value=(1, "link"))
get_id_patch = mock.patch.object(commit, "_get_path_tree_id_and_type", return_value=(1, NodeKind.SUBMODULE))
get_submodule_patch = mock.patch.object(commit, "_get_submodule_url", return_value=submodule_url)
with get_id_patch, get_submodule_patch as submodule_mock:
node = commit.get_node("/abcde")
node = commit.get_node(b"/abcde")
submodule_mock.assert_called_once_with("/abcde")
submodule_mock.assert_called_once_with(b"/abcde")
assert type(node) == SubModuleNode
assert node.url == submodule_url
def test_get_nodes_returns_links(self):
repository = mock.MagicMock()
repository.alias = "git"
repository._remote.tree_items.return_value = [("subrepo", "stat", 1, "link")]
repository._remote.tree_items.return_value = [(b"subrepo", "stat", 1, NodeKind.SUBMODULE)]
commit = GitCommit(repository=repository, raw_id="abcdef12", idx=1)
submodule_url = "https://code.rhodecode.com/dulwich"
get_id_patch = mock.patch.object(commit, "_get_tree_id_for_path", return_value=(1, "tree"))
get_id_patch = mock.patch.object(commit, "_get_path_tree_id_and_type", return_value=(1, NodeKind.DIR))
get_submodule_patch = mock.patch.object(commit, "_get_submodule_url", return_value=submodule_url)
with get_id_patch, get_submodule_patch as submodule_mock:
nodes = commit.get_nodes("/abcde")
nodes = commit.get_nodes(b"/abcde")
submodule_mock.assert_called_once_with("/abcde/subrepo")
submodule_mock.assert_called_once_with(b"/abcde/subrepo")
assert len(nodes) == 1
assert type(nodes[0]) == SubModuleNode
assert nodes[0].url == submodule_url
class TestGetShadowInstance(object):
@pytest.fixture()
def repo(self, vcsbackend_git):
_git_repo = vcsbackend_git.repo

311
rhodecode/tests/vcs/test_hg.py
View file

@ -27,7 +27,7 @@ from rhodecode.lib.vcs import backends
from rhodecode.lib.vcs.backends.base import Reference, MergeResponse, MergeFailureReason
from rhodecode.lib.vcs.backends.hg import MercurialRepository, MercurialCommit
from rhodecode.lib.vcs.exceptions import RepositoryError, VCSError, NodeDoesNotExistError, CommitDoesNotExistError
from rhodecode.lib.vcs.nodes import FileNode, NodeKind, NodeState
from rhodecode.lib.vcs.nodes import FileNode, NodeKind, DirNode, RootNode
from rhodecode.tests import TEST_HG_REPO, TEST_HG_REPO_CLONE, repo_id_generator
@ -41,22 +41,17 @@ def repo_path_generator():
i = 0
while True:
i += 1
yield "%s-%d" % (TEST_HG_REPO_CLONE, i)
yield f"{TEST_HG_REPO_CLONE}-{i:d}"
REPO_PATH_GENERATOR = repo_path_generator()
@pytest.fixture(scope="class", autouse=True)
def repo(request, baseapp):
repo = MercurialRepository(TEST_HG_REPO)
if request.cls:
request.cls.repo = repo
return repo
class TestMercurialRepository(object):
class TestMercurialRepository:
# pylint: disable=protected-access
@pytest.fixture(autouse=True)
def prepare(self):
self.repo = MercurialRepository(TEST_HG_REPO)
def get_clone_repo(self):
"""
@ -100,9 +95,8 @@ class TestMercurialRepository(object):
def test_repo_clone(self):
if os.path.exists(TEST_HG_REPO_CLONE):
self.fail(
"Cannot test mercurial clone repo as location %s already "
"exists. You should manually remove it first." % TEST_HG_REPO_CLONE
pytest.fail(
f"Cannot test mercurial clone repo as location {TEST_HG_REPO_CLONE} already exists. You should manually remove it first."
)
repo = MercurialRepository(TEST_HG_REPO)
@ -217,8 +211,8 @@ class TestMercurialRepository(object):
assert "git" in self.repo._get_branches(closed=True)
assert "web" in self.repo._get_branches(closed=True)
for name, id in self.repo.branches.items():
assert isinstance(self.repo.get_commit(id), MercurialCommit)
for name, commit_id in self.repo.branches.items():
assert isinstance(self.repo.get_commit(commit_id), MercurialCommit)
def test_tip_in_tags(self):
# tip is always a tag
@ -235,29 +229,38 @@ class TestMercurialRepository(object):
assert init_commit.message == "initial import"
assert init_author == "Marcin Kuzminski <marcin@python-blog.com>"
assert init_author == init_commit.committer
assert sorted(init_commit._file_paths) == sorted(
assert sorted(init_commit.added_paths) == sorted(
[
"vcs/__init__.py",
"vcs/backends/BaseRepository.py",
"vcs/backends/__init__.py",
b"vcs/__init__.py",
b"vcs/backends/BaseRepository.py",
b"vcs/backends/__init__.py",
]
)
assert sorted(init_commit.affected_files) == sorted(
[
b"vcs/__init__.py",
b"vcs/backends/BaseRepository.py",
b"vcs/backends/__init__.py",
]
)
assert sorted(init_commit._dir_paths) == sorted(["", "vcs", "vcs/backends"])
assert init_commit._dir_paths + init_commit._file_paths == init_commit._paths
for path in (b"vcs/__init__.py", b"vcs/backends/BaseRepository.py", b"vcs/backends/__init__.py"):
assert isinstance(init_commit.get_node(path), FileNode)
for path in (b"", b"vcs", b"vcs/backends"):
assert isinstance(init_commit.get_node(path), DirNode)
with pytest.raises(NodeDoesNotExistError):
init_commit.get_node(path="foobar")
init_commit.get_node(path=b"foobar")
node = init_commit.get_node("vcs/")
node = init_commit.get_node(b"vcs/")
assert hasattr(node, "kind")
assert node.kind == NodeKind.DIR
node = init_commit.get_node("vcs")
node = init_commit.get_node(b"vcs")
assert hasattr(node, "kind")
assert node.kind == NodeKind.DIR
node = init_commit.get_node("vcs/__init__.py")
node = init_commit.get_node(b"vcs/__init__.py")
assert hasattr(node, "kind")
assert node.kind == NodeKind.FILE
@ -279,7 +282,7 @@ class TestMercurialRepository(object):
def test_commit10(self):
commit10 = self.repo.get_commit(commit_idx=10)
README = """===
readme = """===
VCS
===
@ -291,9 +294,9 @@ Introduction
TODO: To be written...
"""
node = commit10.get_node("README.rst")
node = commit10.get_node(b"README.rst")
assert node.kind == NodeKind.FILE
assert node.str_content == README
assert node.str_content == readme
def test_local_clone(self):
clone_path = next(REPO_PATH_GENERATOR)
@ -370,7 +373,7 @@ TODO: To be written...
assert target_repo.branches["default"] == commit_id
def test_local_pull_from_same_repo(self):
reference = Reference("branch", "default", None)
reference = Reference("branch", "default", "")
with pytest.raises(ValueError):
self.repo._local_pull(self.repo.path, reference)
@ -503,7 +506,7 @@ TODO: To be written...
# Check we are not left in an intermediate merge state
assert not os.path.exists(os.path.join(target_repo.path, ".hg", "merge", "state"))
def test_local_merge_of_two_branches_of_the_same_repo(self, backend_hg):
def test_local_merge_of_two_branches_of_the_same_repo(self, backend_hg, vcs_repo):
commits = [
{"message": "a"},
{"message": "b", "branch": "b"},
@ -639,7 +642,7 @@ TODO: To be written...
# add an extra head to the target repo
imc = target_repo.in_memory_commit
imc.add(FileNode(b"file_x", content="foo"))
imc.add(FileNode(b"file_x", content=b"foo"))
commits = list(target_repo.get_commits())
imc.commit(
message="Automatic commit from repo merge test",
@ -728,8 +731,7 @@ TODO: To be written...
assert len(target_repo.commit_ids) == 2 + 2
class TestGetShadowInstance(object):
class TestGetShadowInstance:
@pytest.fixture()
def repo(self, vcsbackend_hg):
_hg_repo = vcsbackend_hg.repo
@ -742,17 +744,21 @@ class TestGetShadowInstance(object):
assert shadow.config.serialize() == repo.config.serialize()
def test_disables_hooks_section(self, repo):
repo.config.set('hooks', 'foo', 'val')
repo.config.set("hooks", "foo", "val")
shadow = repo.get_shadow_instance(repo.path)
assert not shadow.config.items('hooks')
assert not shadow.config.items("hooks")
def test_allows_to_keep_hooks(self, repo):
repo.config.set('hooks', 'foo', 'val')
repo.config.set("hooks", "foo", "val")
shadow = repo.get_shadow_instance(repo.path, enable_hooks=True)
assert shadow.config.items('hooks')
assert shadow.config.items("hooks")
class TestMercurialCommit(object):
class TestMercurialCommit:
@pytest.fixture(autouse=True)
def prepare(self):
self.repo = MercurialRepository(TEST_HG_REPO)
def _test_equality(self, commit):
idx = commit.idx
assert commit == self.repo.get_commit(commit_idx=idx)
@ -772,7 +778,7 @@ class TestMercurialCommit(object):
def test_root_node(self):
tip = self.repo.get_commit("tip")
assert tip.root is tip.get_node("")
assert tip.root is tip.get_node(b"")
def test_lazy_fetch(self):
"""
@ -788,28 +794,28 @@ class TestMercurialCommit(object):
# accessing root.nodes updates commit.nodes
assert len(commit.nodes) == 9
docs = root.get_node("docs")
docs = commit.get_node(b"docs")
# we haven't yet accessed anything new as docs dir was already cached
assert len(commit.nodes) == 9
assert len(docs.nodes) == 8
# accessing docs.nodes updates commit.nodes
assert len(commit.nodes) == 17
assert docs is commit.get_node("docs")
assert docs is commit.get_node(b"docs")
assert docs is root.nodes[0]
assert docs is root.dirs[0]
assert docs is commit.get_node("docs")
assert docs is commit.get_node(b"docs")
def test_nodes_with_commit(self):
commit = self.repo.get_commit(commit_idx=45)
root = commit.root
docs = root.get_node("docs")
assert docs is commit.get_node("docs")
api = docs.get_node("api")
assert api is commit.get_node("docs/api")
index = api.get_node("index.rst")
assert index is commit.get_node("docs/api/index.rst")
assert index is commit.get_node("docs").get_node("api").get_node("index.rst")
assert isinstance(root, RootNode)
docs = commit.get_node(b"docs")
assert docs is commit.get_node(b"docs")
api = commit.get_node(b"docs/api")
assert api is commit.get_node(b"docs/api")
index = commit.get_node(b"docs/api/index.rst")
assert index is commit.get_node(b"docs/api/index.rst")
def test_branch_and_tags(self):
commit0 = self.repo.get_commit(commit_idx=0)
@ -837,28 +843,28 @@ class TestMercurialCommit(object):
def test_file_size(self):
to_check = (
(10, "setup.py", 1068),
(20, "setup.py", 1106),
(60, "setup.py", 1074),
(10, "vcs/backends/base.py", 2921),
(20, "vcs/backends/base.py", 3936),
(60, "vcs/backends/base.py", 6189),
(10, b"setup.py", 1068),
(20, b"setup.py", 1106),
(60, b"setup.py", 1074),
(10, b"vcs/backends/base.py", 2921),
(20, b"vcs/backends/base.py", 3936),
(60, b"vcs/backends/base.py", 6189),
)
for idx, path, size in to_check:
self._test_file_size(idx, path, size)
def test_file_history_from_commits(self):
node = self.repo[10].get_node("setup.py")
node = self.repo[10].get_node(b"setup.py")
commit_ids = [commit.raw_id for commit in node.history]
assert ["3803844fdbd3b711175fc3da9bdacfcd6d29a6fb"] == commit_ids
node = self.repo[20].get_node("setup.py")
node = self.repo[20].get_node(b"setup.py")
node_ids = [commit.raw_id for commit in node.history]
assert ["eada5a770da98ab0dd7325e29d00e0714f228d09", "3803844fdbd3b711175fc3da9bdacfcd6d29a6fb"] == node_ids
# special case we check history from commit that has this particular
# file changed this means we check if it's included as well
node = self.repo.get_commit("eada5a770da98ab0dd7325e29d00e0714f228d09").get_node("setup.py")
node = self.repo.get_commit("eada5a770da98ab0dd7325e29d00e0714f228d09").get_node(b"setup.py")
node_ids = [commit.raw_id for commit in node.history]
assert ["eada5a770da98ab0dd7325e29d00e0714f228d09", "3803844fdbd3b711175fc3da9bdacfcd6d29a6fb"] == node_ids
@ -866,9 +872,9 @@ class TestMercurialCommit(object):
# we can only check if those commits are present in the history
# as we cannot update this test every time file is changed
files = {
"setup.py": [7, 18, 45, 46, 47, 69, 77],
"vcs/nodes.py": [7, 8, 24, 26, 30, 45, 47, 49, 56, 57, 58, 59, 60, 61, 73, 76],
"vcs/backends/hg.py": [
b"setup.py": [7, 18, 45, 46, 47, 69, 77],
b"vcs/nodes.py": [7, 8, 24, 26, 30, 45, 47, 49, 56, 57, 58, 59, 60, 61, 73, 76],
b"vcs/backends/hg.py": [
4,
5,
6,
@ -927,7 +933,7 @@ class TestMercurialCommit(object):
def test_file_annotate(self):
files = {
"vcs/backends/__init__.py": {
b"vcs/backends/__init__.py": {
89: {
"lines_no": 31,
"commits": [
@ -1002,7 +1008,7 @@ class TestMercurialCommit(object):
],
},
},
"vcs/exceptions.py": {
b"vcs/exceptions.py": {
89: {
"lines_no": 18,
"commits": [16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 17, 16, 16, 18, 18, 18],
@ -1016,25 +1022,25 @@ class TestMercurialCommit(object):
"commits": [16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 17, 16, 16, 18, 18, 18],
},
},
"MANIFEST.in": {
b"MANIFEST.in": {
89: {"lines_no": 5, "commits": [7, 7, 7, 71, 71]},
20: {"lines_no": 3, "commits": [7, 7, 7]},
55: {"lines_no": 3, "commits": [7, 7, 7]},
},
}
for fname, commit_dict in files.items():
for file_name, commit_dict in files.items():
for idx, __ in commit_dict.items():
commit = self.repo.get_commit(commit_idx=idx)
l1_1 = [x[1] for x in commit.get_file_annotate(fname)]
l1_2 = [x[2]().raw_id for x in commit.get_file_annotate(fname)]
l1_1 = [x[1] for x in commit.get_file_annotate(file_name)]
l1_2 = [x[2]().raw_id for x in commit.get_file_annotate(file_name)]
assert l1_1 == l1_2
l1 = l1_2 = [x[2]().idx for x in commit.get_file_annotate(fname)]
l2 = files[fname][idx]["commits"]
l1 = l1_2 = [x[2]().idx for x in commit.get_file_annotate(file_name)]
l2 = files[file_name][idx]["commits"]
assert l1 == l2, (
"The lists of commit for %s@commit_id%s"
"from annotation list should match each other,"
"got \n%s \nvs \n%s " % (fname, idx, l1, l2)
"got \n%s \nvs \n%s " % (file_name, idx, l1, l2)
)
def test_commit_state(self):
@ -1047,55 +1053,51 @@ class TestMercurialCommit(object):
# changed: 13
# added: 20
# removed: 1
changed = set(
[
".hgignore",
"README.rst",
"docs/conf.py",
"docs/index.rst",
"setup.py",
"tests/test_hg.py",
"tests/test_nodes.py",
"vcs/__init__.py",
"vcs/backends/__init__.py",
"vcs/backends/base.py",
"vcs/backends/hg.py",
"vcs/nodes.py",
"vcs/utils/__init__.py",
]
)
changed = {
b".hgignore",
b"README.rst",
b"docs/conf.py",
b"docs/index.rst",
b"setup.py",
b"tests/test_hg.py",
b"tests/test_nodes.py",
b"vcs/__init__.py",
b"vcs/backends/__init__.py",
b"vcs/backends/base.py",
b"vcs/backends/hg.py",
b"vcs/nodes.py",
b"vcs/utils/__init__.py",
}
added = set(
[
"docs/api/backends/hg.rst",
"docs/api/backends/index.rst",
"docs/api/index.rst",
"docs/api/nodes.rst",
"docs/api/web/index.rst",
"docs/api/web/simplevcs.rst",
"docs/installation.rst",
"docs/quickstart.rst",
"setup.cfg",
"vcs/utils/baseui_config.py",
"vcs/utils/web.py",
"vcs/web/__init__.py",
"vcs/web/exceptions.py",
"vcs/web/simplevcs/__init__.py",
"vcs/web/simplevcs/exceptions.py",
"vcs/web/simplevcs/middleware.py",
"vcs/web/simplevcs/models.py",
"vcs/web/simplevcs/settings.py",
"vcs/web/simplevcs/utils.py",
"vcs/web/simplevcs/views.py",
]
)
added = {
b"docs/api/backends/hg.rst",
b"docs/api/backends/index.rst",
b"docs/api/index.rst",
b"docs/api/nodes.rst",
b"docs/api/web/index.rst",
b"docs/api/web/simplevcs.rst",
b"docs/installation.rst",
b"docs/quickstart.rst",
b"setup.cfg",
b"vcs/utils/baseui_config.py",
b"vcs/utils/web.py",
b"vcs/web/__init__.py",
b"vcs/web/exceptions.py",
b"vcs/web/simplevcs/__init__.py",
b"vcs/web/simplevcs/exceptions.py",
b"vcs/web/simplevcs/middleware.py",
b"vcs/web/simplevcs/models.py",
b"vcs/web/simplevcs/settings.py",
b"vcs/web/simplevcs/utils.py",
b"vcs/web/simplevcs/views.py",
}
removed = set(["docs/api.rst"])
removed = {b"docs/api.rst"}
commit64 = self.repo.get_commit("46ad32a4f974")
assert set((node.path for node in commit64.added)) == added
assert set((node.path for node in commit64.changed)) == changed
assert set((node.path for node in commit64.removed)) == removed
assert set((node for node in commit64.added_paths)) == added
assert set((node for node in commit64.changed_paths)) == changed
assert set((node for node in commit64.removed_paths)) == removed
# commit_id b090f22d27d6:
# hg st --rev b090f22d27d6
@ -1103,9 +1105,9 @@ class TestMercurialCommit(object):
# added: 20
# removed: 1
commit88 = self.repo.get_commit("b090f22d27d6")
assert set((node.path for node in commit88.added)) == set()
assert set((node.path for node in commit88.changed)) == set([".hgignore"])
assert set((node.path for node in commit88.removed)) == set()
assert set((node for node in commit88.added_paths)) == set()
assert set((node for node in commit88.changed_paths)) == {b".hgignore"}
assert set((node for node in commit88.removed_paths)) == set()
#
# 85:
@ -1114,55 +1116,40 @@ class TestMercurialCommit(object):
# changed: 4 ['vcs/web/simplevcs/models.py', ...]
# removed: 1 ['vcs/utils/web.py']
commit85 = self.repo.get_commit(commit_idx=85)
assert set((node.path for node in commit85.added)) == set(
["vcs/utils/diffs.py", "vcs/web/simplevcs/views/diffs.py"]
)
assert set((node.path for node in commit85.changed)) == set(
[
"vcs/web/simplevcs/models.py",
"vcs/web/simplevcs/utils.py",
"vcs/web/simplevcs/views/__init__.py",
"vcs/web/simplevcs/views/repository.py",
]
)
assert set((node.path for node in commit85.removed)) == set(["vcs/utils/web.py"])
assert set((node for node in commit85.added_paths)) == {b"vcs/utils/diffs.py", b"vcs/web/simplevcs/views/diffs.py"}
assert set((node for node in commit85.changed_paths)) == {
b"vcs/web/simplevcs/models.py",
b"vcs/web/simplevcs/utils.py",
b"vcs/web/simplevcs/views/__init__.py",
b"vcs/web/simplevcs/views/repository.py",
}
assert set((node for node in commit85.removed_paths)) == {b"vcs/utils/web.py"}
def test_files_state(self):
"""
Tests state of FileNodes.
"""
commit = self.repo.get_commit(commit_idx=85)
node = commit.get_node("vcs/utils/diffs.py")
assert node.state, NodeState.ADDED
assert node.added
assert not node.changed
assert not node.not_changed
assert not node.removed
node = commit.get_node(b"vcs/utils/diffs.py")
assert node.bytes_path in commit.added_paths
commit = self.repo.get_commit(commit_idx=88)
node = commit.get_node(".hgignore")
assert node.state, NodeState.CHANGED
assert not node.added
assert node.changed
assert not node.not_changed
assert not node.removed
node = commit.get_node(b".hgignore")
assert node.bytes_path in commit.changed_paths
commit = self.repo.get_commit(commit_idx=85)
node = commit.get_node("setup.py")
assert node.state, NodeState.NOT_CHANGED
assert not node.added
assert not node.changed
assert node.not_changed
assert not node.removed
node = commit.get_node(b"setup.py")
assert node.bytes_path not in commit.affected_files
# If node has REMOVED state then trying to fetch it would raise
# CommitError exception
commit = self.repo.get_commit(commit_idx=2)
path = "vcs/backends/BaseRepository.py"
path = b"vcs/backends/BaseRepository.py"
with pytest.raises(NodeDoesNotExistError):
commit.get_node(path)
# but it would be one of ``removed`` (commit's attribute)
assert path in [rf.path for rf in commit.removed]
assert path in [rf for rf in commit.removed_paths]
def test_commit_message_is_unicode(self):
for cm in self.repo:
@ -1174,14 +1161,14 @@ class TestMercurialCommit(object):
def test_repo_files_content_type(self):
test_commit = self.repo.get_commit(commit_idx=100)
for node in test_commit.get_node("/"):
for node in test_commit.get_node(b"/"):
if node.is_file():
assert type(node.content) == bytes
assert type(node.str_content) == str
def test_wrong_path(self):
# There is 'setup.py' in the root dir but not there:
path = "foo/bar/setup.py"
path = b"foo/bar/setup.py"
with pytest.raises(VCSError):
self.repo.get_commit().get_node(path)
@ -1196,23 +1183,27 @@ class TestMercurialCommit(object):
assert "marcink" == self.repo.get_commit("84478366594b").author_name
class TestLargeFileRepo(object):
class TestLargeFileRepo:
def test_large_file(self, backend_hg):
conf = make_db_config()
hg_largefiles_store = conf.get("largefiles", "usercache")
repo = backend_hg.create_test_repo("largefiles", conf)
tip = repo.scm_instance().get_commit()
node = tip.get_node(".hglf/thisfileislarge")
node = tip.get_node(b".hglf/thisfileislarge")
lf_node = node.get_largefile_node()
assert lf_node.is_largefile() is True
assert lf_node.size == 1024000
assert lf_node.name == ".hglf/thisfileislarge"
assert lf_node.name == b".hglf/thisfileislarge"
class TestGetBranchName(object):
class TestGetBranchName:
@pytest.fixture(autouse=True)
def prepare(self):
self.repo = MercurialRepository(TEST_HG_REPO)
def test_returns_ref_name_when_type_is_branch(self):
ref = self._create_ref("branch", "fake-name")
result = self.repo._get_branch_name(ref)
@ -1235,7 +1226,11 @@ class TestGetBranchName(object):
return ref
class TestIsTheSameBranch(object):
class TestIsTheSameBranch:
@pytest.fixture(autouse=True)
def prepare(self):
self.repo = MercurialRepository(TEST_HG_REPO)
def test_returns_true_when_branches_are_equal(self):
source_ref = mock.Mock(name="source-ref")
target_ref = mock.Mock(name="target-ref")

14
rhodecode/tests/vcs/test_inmemory.py
View file

@ -128,8 +128,8 @@ class TestInMemoryCommit(BackendTestMixin):
]
self.imc.add(*to_add)
commit = self.imc.commit("Initial", "joe doe <joe.doe@example.com>")
assert isinstance(commit.get_node("foo"), DirNode)
assert isinstance(commit.get_node("foo/bar"), DirNode)
assert isinstance(commit.get_node(b"foo"), DirNode)
assert isinstance(commit.get_node(b"foo/bar"), DirNode)
self.assert_nodes_in_commit(commit, to_add)
# commit some more files again
@ -244,7 +244,7 @@ class TestInMemoryCommit(BackendTestMixin):
tip = self.repo.get_commit()
node = nodes[0]
assert node.content == tip.get_node(node.path).content
assert node.content == tip.get_node(node.bytes_path).content
self.imc.remove(node)
self.imc.commit(message=f"Removed {node.path}", author="Some Name <foo@bar.com>")
@ -252,7 +252,7 @@ class TestInMemoryCommit(BackendTestMixin):
assert tip != newtip
assert tip.id != newtip.id
with pytest.raises(NodeDoesNotExistError):
newtip.get_node(node.path)
newtip.get_node(node.bytes_path)
def test_remove_last_file_from_directory(self):
node = FileNode(b"omg/qwe/foo/bar", content=b"foobar")
@ -262,7 +262,7 @@ class TestInMemoryCommit(BackendTestMixin):
self.imc.remove(node)
tip = self.imc.commit("removed", "joe doe <joe@doe.com>")
with pytest.raises(NodeDoesNotExistError):
tip.get_node("omg/qwe/foo/bar")
tip.get_node(b"omg/qwe/foo/bar")
def test_remove_raise_node_does_not_exist(self, nodes):
self.imc.remove(nodes[0])
@ -338,5 +338,5 @@ class TestInMemoryCommit(BackendTestMixin):
def assert_nodes_in_commit(self, commit, nodes):
for node in nodes:
assert commit.get_node(node.path).path == node.path
assert commit.get_node(node.path).content == node.content
assert commit.get_node(node.bytes_path).path == node.path
assert commit.get_node(node.bytes_path).content == node.content

26
rhodecode/tests/vcs/test_nodes.py
View file

@ -155,30 +155,6 @@ class TestNodeBasics:
with pytest.raises(NodeError):
node.content # noqa
def test_dir_node_iter(self):
nodes = [
DirNode(b"docs"),
DirNode(b"tests"),
FileNode(b"bar"),
FileNode(b"foo"),
FileNode(b"readme.txt"),
FileNode(b"setup.py"),
]
dirnode = DirNode(b"", nodes=nodes)
for node in dirnode:
assert node == dirnode.get_node(node.path)
def test_node_state(self):
"""
Without link to commit nodes should raise NodeError.
"""
node = FileNode(b"anything")
with pytest.raises(NodeError):
node.state # noqa
node = DirNode(b"anything")
with pytest.raises(NodeError):
node.state # noqa
def test_file_node_stat(self):
node = FileNode(b"foobar", b"empty... almost")
mode = node.mode # default should be 0100644
@ -272,5 +248,5 @@ class TestNodesCommits(BackendTestMixin):
last_commit = repo.get_commit()
for x in range(3):
node = last_commit.get_node(f"file_{x}.txt")
node = last_commit.get_node(b"file_%d.txt" % x)
assert node.last_commit == repo[x]

10
rhodecode/tests/vcs/test_svn.py
View file

@ -128,7 +128,7 @@ def test_read_full_file_tree(head):
def test_topnode_files_attribute(head):
topnode = head.get_node("")
topnode = head.get_node(b"")
topnode.files
@ -173,23 +173,23 @@ class TestSVNCommit(object):
self.repo = repo
def test_file_history_from_commits(self):
node = self.repo[10].get_node("setup.py")
node = self.repo[10].get_node(b"setup.py")
commit_ids = [commit.raw_id for commit in node.history]
assert ["8"] == commit_ids
node = self.repo[20].get_node("setup.py")
node = self.repo[20].get_node(b"setup.py")
node_ids = [commit.raw_id for commit in node.history]
assert ["18", "8"] == node_ids
# special case we check history from commit that has this particular
# file changed this means we check if it's included as well
node = self.repo.get_commit("18").get_node("setup.py")
node = self.repo.get_commit("18").get_node(b"setup.py")
node_ids = [commit.raw_id for commit in node.history]
assert ["18", "8"] == node_ids
def test_repo_files_content_type(self):
test_commit = self.repo.get_commit(commit_idx=100)
for node in test_commit.get_node("/"):
for node in test_commit.get_node(b"/"):
if node.is_file():
assert type(node.content) == bytes
assert type(node.str_content) == str

4
rhodecode/tests/vcs/test_tags.py
View file

@ -30,11 +30,11 @@ class TestTags(BackendTestMixin):
def test_new_tag(self):
tip = self.repo.get_commit()
tagsize = len(self.repo.tags)
tag = self.repo.tag("last-commit", "joe", tip.raw_id)
tag_commit = self.repo.tag("last-commit", "joe", tip.raw_id)
assert len(self.repo.tags) == tagsize + 1
for top, __, __ in tip.walk():
assert top == tag.get_node(top.path)
assert top == tag_commit.get_node(top.bytes_path)
def test_tag_already_exist(self):
tip = self.repo.get_commit()

4
rhodecode/tests/vcs_operations/test_vcs_operations_bad_client.py
View file

@ -65,7 +65,7 @@ class TestVCSOperationsOnUsingBadClient(object):
# push fails repo is locked by other user !
push_url = rcstack.repo_clone_url(HG_REPO)
stdout, stderr = _add_files_and_push("hg", tmpdir.strpath, clone_url=push_url)
msg = "Your hg client (ver=mercurial/proto-1.0 (Mercurial 6.7.4)) is forbidden by security rules"
msg = "Your hg client (version=mercurial/proto-1.0 (Mercurial 6.7.4)) is forbidden by security rules"
assert msg in stderr
def test_push_with_bad_client_repo_by_other_user_git(self, rcstack, tmpdir):
@ -81,7 +81,7 @@ class TestVCSOperationsOnUsingBadClient(object):
push_url = rcstack.repo_clone_url(GIT_REPO)
stdout, stderr = _add_files_and_push("git", tmpdir.strpath, clone_url=push_url)
err = "Your git client (ver=git/2.45.2) is forbidden by security rules"
err = "Your git client (version=git/2.45.2) is forbidden by security rules"
assert err in stderr
@pytest.mark.xfail(reason="Lack of proper SVN support of cloning")

3
rhodecode/tests/vcs_operations/test_vcs_operations_svn.py
View file

@ -124,7 +124,6 @@ class TestVCSOperationsSVN(object):
assert 'not found' in stderr
@pytest.mark.xfail(reason='Lack of proper SVN support of cloning')
def test_clone_existing_path_svn_not_in_database(
self, rcstack, tmpdir, fs_repo_only):
db_name = fs_repo_only('not-in-db-git', repo_type='git')
@ -136,7 +135,6 @@ class TestVCSOperationsSVN(object):
f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
assert 'not found' in stderr
@pytest.mark.xfail(reason='Lack of proper SVN support of cloning')
def test_clone_existing_path_svn_not_in_database_different_scm(
self, rcstack, tmpdir, fs_repo_only):
db_name = fs_repo_only('not-in-db-hg', repo_type='hg')
@ -149,7 +147,6 @@ class TestVCSOperationsSVN(object):
f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath)
assert 'not found' in stderr
@pytest.mark.xfail(reason='Lack of proper SVN support of cloning')
def test_clone_non_existing_store_path_svn(self, rcstack, tmpdir, user_util):
repo = user_util.create_repo(repo_type='git')
clone_url = rcstack.repo_clone_url(repo.repo_name)