diff --git a/rhodecode/__init__.py b/rhodecode/__init__.py index 4a06569f..732b082b 100644 --- a/rhodecode/__init__.py +++ b/rhodecode/__init__.py @@ -45,13 +45,20 @@ CELERY_EAGER = False # link to config for pyramid CONFIG = {} +class NotGivenMeta: + + def __repr__(self): + return 'NotGivenObject()' + __str__ = __repr__ + +NotGiven = NotGivenMeta() class ConfigGet: - NotGiven = object() - def _get_val_or_missing(self, key, missing): + @classmethod + def _get_val_or_missing(cls, key, missing): if key not in CONFIG: - if missing == self.NotGiven: + if missing != NotGiven: return missing # we don't get key, we don't get missing value, return nothing similar as config.get(key) return None @@ -74,6 +81,12 @@ class ConfigGet: val = self._get_val_or_missing(key, missing) return str2bool(val) + def get_list(self, key, missing=NotGiven): + from rhodecode.lib.type_utils import aslist + val = self._get_val_or_missing(key, missing) + return aslist(val, sep=',') + + # Populated with the settings dictionary from application init in # rhodecode.conf.environment.load_pyramid_environment PYRAMID_SETTINGS = {} diff --git a/rhodecode/api/tests/test_get_repo_comments.py b/rhodecode/api/tests/test_get_repo_comments.py index a5d5d44e..09375f4c 100644 --- a/rhodecode/api/tests/test_get_repo_comments.py +++ b/rhodecode/api/tests/test_get_repo_comments.py @@ -19,6 +19,7 @@ import pytest +from rhodecode.lib.str_utils import safe_str from rhodecode.model.db import User, ChangesetComment from rhodecode.model.meta import Session from rhodecode.model.comment import CommentsModel @@ -36,7 +37,7 @@ def make_repo_comments_factory(request): commit = repo.scm_instance()[0] commit_id = commit.raw_id - file_0 = commit.affected_files[0] + file_0 = safe_str(commit.affected_files[0]) comments = [] # general diff --git a/rhodecode/api/views/repo_api.py b/rhodecode/api/views/repo_api.py index f4fac27e..60ee6b64 100644 --- a/rhodecode/api/views/repo_api.py +++ b/rhodecode/api/views/repo_api.py @@ -317,8 +317,7 @@ def get_repo_changeset(request, apiuser, repoid, revision, ','.join(_changes_details_types))) vcs_repo = repo.scm_instance() - pre_load = ['author', 'branch', 'date', 'message', 'parents', - 'status', '_commit', '_file_paths'] + pre_load = ['author', 'branch', 'date', 'message', 'parents', 'status', '_commit'] try: commit = repo.get_commit(commit_id=revision, pre_load=pre_load) @@ -376,8 +375,7 @@ def get_repo_changesets(request, apiuser, repoid, start_rev, limit, ','.join(_changes_details_types))) limit = int(limit) - pre_load = ['author', 'branch', 'date', 'message', 'parents', - 'status', '_commit', '_file_paths'] + pre_load = ['author', 'branch', 'date', 'message', 'parents', 'status', '_commit'] vcs_repo = repo.scm_instance() # SVN needs a special case to distinguish its index and commit id diff --git a/rhodecode/apps/repository/views/repo_changelog.py b/rhodecode/apps/repository/views/repo_changelog.py index 3c82f5ad..d5878a25 100644 --- a/rhodecode/apps/repository/views/repo_changelog.py +++ b/rhodecode/apps/repository/views/repo_changelog.py @@ -33,7 +33,7 @@ from rhodecode.lib.auth import ( from rhodecode.lib.graphmod import _colored, _dagwalker from rhodecode.lib.helpers import RepoPage from rhodecode.lib.utils2 import str2bool -from rhodecode.lib.str_utils import safe_int, safe_str +from rhodecode.lib.str_utils import safe_int, safe_str, safe_bytes from rhodecode.lib.vcs.exceptions import ( RepositoryError, CommitDoesNotExistError, CommitError, NodeDoesNotExistError, EmptyRepositoryError) @@ -204,10 +204,9 @@ class RepoChangelogView(RepoAppView): log.debug('generating changelog for path %s', f_path) # get the history for the file ! base_commit = self.rhodecode_vcs_repo.get_commit(commit_id) - + bytes_path = safe_bytes(f_path) try: - collection = base_commit.get_path_history( - f_path, limit=hist_limit, pre_load=pre_load) + collection = base_commit.get_path_history(bytes_path, limit=hist_limit, pre_load=pre_load) if collection and partial_xhr: # for ajax call we remove first one since we're looking # at it right now in the context of a file commit @@ -216,7 +215,7 @@ class RepoChangelogView(RepoAppView): # this node is not present at tip! try: commit = self._get_commit_or_redirect(commit_id) - collection = commit.get_path_history(f_path) + collection = commit.get_path_history(bytes_path) except RepositoryError as e: h.flash(safe_str(e), category='warning') redirect_url = h.route_path( @@ -310,9 +309,8 @@ class RepoChangelogView(RepoAppView): log.exception(safe_str(e)) raise HTTPFound( h.route_path('repo_commits', repo_name=self.db_repo_name)) - - collection = base_commit.get_path_history( - f_path, limit=hist_limit, pre_load=pre_load) + bytes_path = safe_bytes(f_path) + collection = base_commit.get_path_history(bytes_path, limit=hist_limit, pre_load=pre_load) collection = list(reversed(collection)) else: collection = self.rhodecode_vcs_repo.get_commits( diff --git a/rhodecode/apps/repository/views/repo_commits.py b/rhodecode/apps/repository/views/repo_commits.py index aeb2e082..76acd50f 100644 --- a/rhodecode/apps/repository/views/repo_commits.py +++ b/rhodecode/apps/repository/views/repo_commits.py @@ -89,8 +89,7 @@ class RepoCommitsView(RepoAppView): commit_range = commit_id_range.split('...')[:2] try: - pre_load = ['affected_files', 'author', 'branch', 'date', - 'message', 'parents'] + pre_load = ['author', 'branch', 'date', 'message', 'parents'] if self.rhodecode_vcs_repo.alias == 'hg': pre_load += ['hidden', 'obsolete', 'phase'] @@ -100,8 +99,7 @@ class RepoCommitsView(RepoAppView): pre_load=pre_load, translate_tags=False) commits = list(commits) else: - commits = [self.rhodecode_vcs_repo.get_commit( - commit_id=commit_id_range, pre_load=pre_load)] + commits = [self.rhodecode_vcs_repo.get_commit(commit_id=commit_id_range, pre_load=pre_load)] c.commit_ranges = commits if not c.commit_ranges: diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py index ab46c1b5..e1ba3261 100644 --- a/rhodecode/apps/repository/views/repo_files.py +++ b/rhodecode/apps/repository/views/repo_files.py @@ -187,12 +187,14 @@ class RepoFilesView(RepoAppView): default_commit_id = self.db_repo.landing_ref_name default_f_path = '/' - commit_id = self.request.matchdict.get( - 'commit_id', default_commit_id) + commit_id = self.request.matchdict.get('commit_id', default_commit_id) f_path = self._get_f_path(self.request.matchdict, default_f_path) - return commit_id, f_path - def _get_default_encoding(self, c): + bytes_path = safe_bytes(f_path) + return commit_id, f_path, bytes_path + + @classmethod + def _get_default_encoding(cls, c): enc_list = getattr(c, 'default_encodings', []) return enc_list[0] if enc_list else 'UTF-8' @@ -361,21 +363,21 @@ class RepoFilesView(RepoAppView): from rhodecode import CONFIG _ = self.request.translate self.load_default_context() - default_at_path = '/' - fname = self.request.matchdict['fname'] + subrepos = self.request.GET.get('subrepos') == 'true' with_hash = str2bool(self.request.GET.get('with_hash', '1')) + + default_at_path = '/' + fname = self.request.matchdict['fname'] at_path = self.request.GET.get('at_path') or default_at_path if not self.db_repo.enable_downloads: return Response(_('Downloads disabled')) try: - commit_id, ext, fileformat, content_type = \ - _get_archive_spec(fname) + commit_id, ext, file_format, content_type = _get_archive_spec(fname) except ValueError: - return Response(_('Unknown archive type for: `{}`').format( - h.escape(fname))) + return Response(_('Unknown archive type for: `{}`').format(h.escape(fname))) try: commit = self.rhodecode_vcs_repo.get_commit(commit_id) @@ -391,7 +393,7 @@ class RepoFilesView(RepoAppView): raise HTTPFound(self.request.current_route_path(fname=fname)) try: - at_path = commit.get_node(at_path).path or default_at_path + at_path = commit.get_node(safe_bytes(at_path)).path or default_at_path except Exception: return Response(_('No node at path {} for this repository').format(h.escape(at_path))) @@ -440,7 +442,7 @@ class RepoFilesView(RepoAppView): with d_cache.get_lock(reentrant_lock_key): try: commit.archive_repo(archive_name_key, archive_dir_name=archive_dir_name, - kind=fileformat, subrepos=subrepos, + kind=file_format, subrepos=subrepos, archive_at_path=at_path, cache_config=d_cache_conf) except ImproperArchiveTypeError: return _('Unknown archive type') @@ -484,7 +486,7 @@ class RepoFilesView(RepoAppView): if commit_id not in ['', None, 'None', '0' * 12, '0' * 40]: commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id) try: - node = commit.get_node(f_path) + node = commit.get_node(safe_bytes(f_path)) if node.is_dir(): raise NodeError(f'{node} path is a {type(node)} not a file') except NodeDoesNotExistError: @@ -648,7 +650,7 @@ class RepoFilesView(RepoAppView): # overwrite auto rendering by setting this GET flag c.renderer = view_name == 'repo_files:rendered' or not self.request.GET.get('no-render', False) - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() c.commit = self._get_commit_or_redirect(commit_id) c.branch = self.request.GET.get('branch', None) @@ -657,7 +659,8 @@ class RepoFilesView(RepoAppView): # files or dirs try: - c.file = c.commit.get_node(f_path, pre_load=['is_binary', 'size', 'data']) + + c.file = c.commit.get_node(bytes_path, pre_load=['is_binary', 'size', 'data']) c.file_author = True c.file_tree = '' @@ -666,11 +669,9 @@ class RepoFilesView(RepoAppView): try: prev_commit = c.commit.prev(c.branch) c.prev_commit = prev_commit - c.url_prev = h.route_path( - 'repo_files', repo_name=self.db_repo_name, - commit_id=prev_commit.raw_id, f_path=f_path) + c.url_prev = h.route_path('repo_files', repo_name=self.db_repo_name, commit_id=prev_commit.raw_id, f_path=f_path) if c.branch: - c.url_prev += '?branch=%s' % c.branch + c.url_prev += f'?branch={c.branch}' except (CommitDoesNotExistError, VCSError): c.url_prev = '#' c.prev_commit = EmptyCommit() @@ -679,11 +680,9 @@ class RepoFilesView(RepoAppView): try: next_commit = c.commit.next(c.branch) c.next_commit = next_commit - c.url_next = h.route_path( - 'repo_files', repo_name=self.db_repo_name, - commit_id=next_commit.raw_id, f_path=f_path) + c.url_next = h.route_path('repo_files', repo_name=self.db_repo_name, commit_id=next_commit.raw_id, f_path=f_path) if c.branch: - c.url_next += '?branch=%s' % c.branch + c.url_next += f'?branch={c.branch}' except (CommitDoesNotExistError, VCSError): c.url_next = '#' c.next_commit = EmptyCommit() @@ -739,8 +738,7 @@ class RepoFilesView(RepoAppView): c.file_tree = self._get_tree_at_commit(c, c.commit.raw_id, f_path, at_rev=at_rev) c.readme_data, c.readme_file = \ - self._get_readme_data(self.db_repo, c.visual.default_renderer, - c.commit.raw_id, f_path) + self._get_readme_data(self.db_repo, c.visual.default_renderer, c.commit.raw_id, bytes_path) except RepositoryError as e: h.flash(h.escape(safe_str(e)), category='error') @@ -759,24 +757,24 @@ class RepoFilesView(RepoAppView): def repo_files_annotated_previous(self): self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, bytes_path, bytes_path = self._get_commit_and_path() commit = self._get_commit_or_redirect(commit_id) prev_commit_id = commit.raw_id line_anchor = self.request.GET.get('line_anchor') is_file = False try: - _file = commit.get_node(f_path) + _file = commit.get_node(bytes_path) is_file = _file.is_file() except (NodeDoesNotExistError, CommitDoesNotExistError, VCSError): pass if is_file: - history = commit.get_path_history(f_path) + history = commit.get_path_history(bytes_path) prev_commit_id = history[1].raw_id \ if len(history) > 1 else prev_commit_id prev_url = h.route_path( 'repo_files:annotated', repo_name=self.db_repo_name, - commit_id=prev_commit_id, f_path=f_path, + commit_id=prev_commit_id, f_path=bytes_path, _anchor=f'L{line_anchor}') raise HTTPFound(prev_url) @@ -792,10 +790,10 @@ class RepoFilesView(RepoAppView): """ c = self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() commit = self._get_commit_or_redirect(commit_id) try: - dir_node = commit.get_node(f_path) + dir_node = commit.get_node(bytes_path) except RepositoryError as e: return Response(f'error: {h.escape(safe_str(e))}') @@ -816,9 +814,9 @@ class RepoFilesView(RepoAppView): safe_path = f_name.replace('"', '\\"') encoded_path = urllib.parse.quote(f_name) - headers = "attachment; " \ - "filename=\"{}\"; " \ - "filename*=UTF-8\'\'{}".format(safe_path, encoded_path) + headers = f"attachment; " \ + f"filename=\"{safe_path}\"; " \ + f"filename*=UTF-8\'\'{encoded_path}" return header_safe_str(headers) @@ -832,9 +830,9 @@ class RepoFilesView(RepoAppView): """ c = self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() commit = self._get_commit_or_redirect(commit_id) - file_node = self._get_filenode_or_redirect(commit, f_path) + file_node = self._get_filenode_or_redirect(commit, bytes_path) raw_mimetype_mapping = { # map original mimetype to a mimetype used for "show as raw" @@ -892,9 +890,9 @@ class RepoFilesView(RepoAppView): def repo_file_download(self): c = self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() commit = self._get_commit_or_redirect(commit_id) - file_node = self._get_filenode_or_redirect(commit, f_path) + file_node = self._get_filenode_or_redirect(commit, bytes_path) if self.request.GET.get('lf'): # only if lf get flag is passed, we download this file @@ -920,8 +918,7 @@ class RepoFilesView(RepoAppView): def _get_nodelist_at_commit(self, repo_name, repo_id, commit_id, f_path): - cache_seconds = safe_int( - rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time')) + cache_seconds = rhodecode.ConfigGet().get_int('rc_cache.cache_repo.expiration_time') cache_on = cache_seconds > 0 log.debug( 'Computing FILE SEARCH for repo_id %s commit_id `%s` and path `%s`' @@ -933,21 +930,17 @@ class RepoFilesView(RepoAppView): @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on) def compute_file_search(_name_hash, _repo_id, _commit_id, _f_path): - log.debug('Generating cached nodelist for repo_id:%s, %s, %s', - _repo_id, commit_id, f_path) + log.debug('Generating cached nodelist for repo_id:%s, %s, %s', _repo_id, commit_id, f_path) try: _d, _f = ScmModel().get_quick_filter_nodes(repo_name, _commit_id, _f_path) except (RepositoryError, CommitDoesNotExistError, Exception) as e: log.exception(safe_str(e)) h.flash(h.escape(safe_str(e)), category='error') - raise HTTPFound(h.route_path( - 'repo_files', repo_name=self.db_repo_name, - commit_id='tip', f_path='/')) + raise HTTPFound(h.route_path('repo_files', repo_name=self.db_repo_name, commit_id='tip', f_path='/')) return _d + _f - result = compute_file_search(self.db_repo.repo_name_hash, self.db_repo.repo_id, - commit_id, f_path) + result = compute_file_search(self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path) return filter(lambda n: self.path_filter.path_access_allowed(n['name']), result) @LoginRequired() @@ -956,7 +949,7 @@ class RepoFilesView(RepoAppView): def repo_nodelist(self): self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() commit = self._get_commit_or_redirect(commit_id) metadata = self._get_nodelist_at_commit( @@ -996,10 +989,10 @@ class RepoFilesView(RepoAppView): if commits is None: pre_load = ["author", "branch"] try: - commits = tip.get_path_history(f_path, pre_load=pre_load) + commits = tip.get_path_history(safe_bytes(f_path), pre_load=pre_load) except (NodeDoesNotExistError, CommitError): # this node is not present at tip! - commits = commit_obj.get_path_history(f_path, pre_load=pre_load) + commits = commit_obj.get_path_history(safe_bytes(f_path), pre_load=pre_load) history = [] commits_group = ([], _("Changesets")) @@ -1040,9 +1033,9 @@ class RepoFilesView(RepoAppView): def repo_file_history(self): self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() commit = self._get_commit_or_redirect(commit_id) - file_node = self._get_filenode_or_redirect(commit, f_path) + file_node = self._get_filenode_or_redirect(commit, bytes_path) if file_node.is_file(): file_history, _hist = self._get_node_history(commit, f_path) @@ -1083,9 +1076,9 @@ class RepoFilesView(RepoAppView): def repo_file_authors(self): c = self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() commit = self._get_commit_or_redirect(commit_id) - file_node = self._get_filenode_or_redirect(commit, f_path) + file_node = self._get_filenode_or_redirect(commit, bytes_path) if not file_node.is_file(): raise HTTPBadRequest() @@ -1124,10 +1117,9 @@ class RepoFilesView(RepoAppView): def repo_files_check_head(self): self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() _branch_name, _sha_commit_id, is_head = \ - self._is_valid_head(commit_id, self.rhodecode_vcs_repo, - landing_ref=self.db_repo.landing_ref_name) + self._is_valid_head(commit_id, self.rhodecode_vcs_repo, landing_ref=self.db_repo.landing_ref_name) new_path = self.request.POST.get('path') operation = self.request.POST.get('operation') @@ -1138,12 +1130,10 @@ class RepoFilesView(RepoAppView): try: commit_obj = self.rhodecode_vcs_repo.get_commit(commit_id) # NOTE(dan): construct whole path without leading / - file_node = commit_obj.get_node(new_f_path) - if file_node is not None: + file_node = commit_obj.get_node(safe_bytes(new_f_path)) + if file_node: path_exist = new_f_path - except EmptyRepositoryError: - pass - except Exception: + except (EmptyRepositoryError, NodeDoesNotExistError): pass return { @@ -1158,7 +1148,7 @@ class RepoFilesView(RepoAppView): def repo_files_remove_file(self): _ = self.request.translate c = self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() self._ensure_not_locked() _branch_name, _sha_commit_id, is_head = \ @@ -1169,7 +1159,7 @@ class RepoFilesView(RepoAppView): self.check_branch_permission(_branch_name) c.commit = self._get_commit_or_redirect(commit_id) - c.file = self._get_filenode_or_redirect(c.commit, f_path) + c.file = self._get_filenode_or_redirect(c.commit, bytes_path) c.default_message = _( 'Deleted file {} via RhodeCode Enterprise').format(f_path) @@ -1184,7 +1174,7 @@ class RepoFilesView(RepoAppView): _ = self.request.translate c = self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() self._ensure_not_locked() _branch_name, _sha_commit_id, is_head = \ @@ -1195,10 +1185,9 @@ class RepoFilesView(RepoAppView): self.check_branch_permission(_branch_name) c.commit = self._get_commit_or_redirect(commit_id) - c.file = self._get_filenode_or_redirect(c.commit, f_path) + c.file = self._get_filenode_or_redirect(c.commit, bytes_path) - c.default_message = _( - 'Deleted file {} via RhodeCode Enterprise').format(f_path) + c.default_message = _('Deleted file {} via RhodeCode Enterprise').format(f_path) c.f_path = f_path node_path = f_path author = self._rhodecode_db_user.full_contact @@ -1232,7 +1221,7 @@ class RepoFilesView(RepoAppView): def repo_files_edit_file(self): _ = self.request.translate c = self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() self._ensure_not_locked() _branch_name, _sha_commit_id, is_head = \ @@ -1243,7 +1232,7 @@ class RepoFilesView(RepoAppView): self.check_branch_permission(_branch_name, commit_id=commit_id) c.commit = self._get_commit_or_redirect(commit_id) - c.file = self._get_filenode_or_redirect(c.commit, f_path) + c.file = self._get_filenode_or_redirect(c.commit, bytes_path) if c.file.is_binary: files_url = h.route_path( @@ -1263,12 +1252,12 @@ class RepoFilesView(RepoAppView): def repo_files_update_file(self): _ = self.request.translate c = self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() self._ensure_not_locked() c.commit = self._get_commit_or_redirect(commit_id) - c.file = self._get_filenode_or_redirect(c.commit, f_path) + c.file = self._get_filenode_or_redirect(c.commit, bytes_path) if c.file.is_binary: raise HTTPFound(h.route_path('repo_files', repo_name=self.db_repo_name, @@ -1345,7 +1334,7 @@ class RepoFilesView(RepoAppView): def repo_files_add_file(self): _ = self.request.translate c = self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() self._ensure_not_locked() @@ -1381,7 +1370,7 @@ class RepoFilesView(RepoAppView): def repo_files_create_file(self): _ = self.request.translate c = self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() self._ensure_not_locked() @@ -1450,8 +1439,7 @@ class RepoFilesView(RepoAppView): author=author, ) - h.flash(_('Successfully committed new file `{}`').format( - h.escape(node_path)), category='success') + h.flash(_('Successfully committed new file `{}`').format(h.escape(node_path)), category='success') default_redirect_url = h.route_path( 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id) @@ -1475,7 +1463,7 @@ class RepoFilesView(RepoAppView): def repo_files_upload_file(self): _ = self.request.translate c = self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() self._ensure_not_locked() @@ -1604,7 +1592,7 @@ class RepoFilesView(RepoAppView): def repo_files_replace_file(self): _ = self.request.translate c = self.load_default_context() - commit_id, f_path = self._get_commit_and_path() + commit_id, f_path, bytes_path = self._get_commit_and_path() self._ensure_not_locked() diff --git a/rhodecode/config/utils.py b/rhodecode/config/utils.py index 82bf62b2..c2d1fbf8 100644 --- a/rhodecode/config/utils.py +++ b/rhodecode/config/utils.py @@ -39,7 +39,6 @@ def configure_vcs(config): conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol.v2'] conf.settings.HOOKS_HOST = config['vcs.hooks.host'] - conf.settings.DEFAULT_ENCODINGS = config['default_encoding'] conf.settings.ALIASES[:] = config['vcs.backends'] conf.settings.SVN_COMPATIBLE_VERSION = config['vcs.svn.compatible_version'] diff --git a/rhodecode/lib/codeblocks.py b/rhodecode/lib/codeblocks.py index f0aaf0a8..0a475ae2 100644 --- a/rhodecode/lib/codeblocks.py +++ b/rhodecode/lib/codeblocks.py @@ -478,7 +478,9 @@ class DiffSet(object): log.debug('rendering diff for %r', patch['filename']) source_filename = patch['original_filename'] + source_filename_bytes = patch['original_filename_bytes'] target_filename = patch['filename'] + target_filename_bytes = patch['filename_bytes'] source_lexer = plain_text_lexer target_lexer = plain_text_lexer @@ -491,12 +493,12 @@ class DiffSet(object): if (source_filename and patch['operation'] in ('D', 'M') and source_filename not in self.source_nodes): self.source_nodes[source_filename] = ( - self.source_node_getter(source_filename)) + self.source_node_getter(source_filename_bytes)) if (target_filename and patch['operation'] in ('A', 'M') and target_filename not in self.target_nodes): self.target_nodes[target_filename] = ( - self.target_node_getter(target_filename)) + self.target_node_getter(target_filename_bytes)) elif hl_mode == self.HL_FAST: source_lexer = self._get_lexer_for_filename(source_filename) @@ -558,6 +560,7 @@ class DiffSet(object): }) file_chunks = patch['chunks'][1:] + for i, hunk in enumerate(file_chunks, 1): hunkbit = self.parse_hunk(hunk, source_file, target_file) hunkbit.source_file_path = source_file_path @@ -593,12 +596,13 @@ class DiffSet(object): return filediff def parse_hunk(self, hunk, source_file, target_file): + result = AttributeDict(dict( source_start=hunk['source_start'], source_length=hunk['source_length'], target_start=hunk['target_start'], target_length=hunk['target_length'], - section_header=hunk['section_header'], + section_header=safe_str(hunk['section_header']), lines=[], )) before, after = [], [] diff --git a/rhodecode/lib/diffs.py b/rhodecode/lib/diffs.py index f721cb05..ff310961 100644 --- a/rhodecode/lib/diffs.py +++ b/rhodecode/lib/diffs.py @@ -455,7 +455,9 @@ class DiffProcessor(object): return arg for chunk in self._diff.chunks(): + bytes_head = chunk.header head = chunk.header_as_str + log.debug('parsing diff chunk %r', chunk) raw_diff = chunk.raw @@ -598,10 +600,17 @@ class DiffProcessor(object): chunks.insert(0, frag) - original_filename = safe_str(head['a_path']) + original_filename = head['a_path'] + original_filename_bytes = bytes_head['a_path'] + + filename = head['b_path'] + filename_bytes = bytes_head['b_path'] + _files.append({ 'original_filename': original_filename, - 'filename': safe_str(head['b_path']), + 'original_filename_bytes': original_filename_bytes, + 'filename': filename, + 'filename_bytes': filename_bytes, 'old_revision': head['a_blob_id'], 'new_revision': head['b_blob_id'], 'chunks': chunks, diff --git a/rhodecode/lib/hooks_base.py b/rhodecode/lib/hooks_base.py index 8931ab71..bb10ee9f 100644 --- a/rhodecode/lib/hooks_base.py +++ b/rhodecode/lib/hooks_base.py @@ -84,7 +84,6 @@ def check_locked_repo(extras, check_same_user=True): user = User.get_by_username(extras.username) output = '' if extras.locked_by[0] and (not check_same_user or user.user_id != extras.locked_by[0]): - locked_by = User.get(extras.locked_by[0]).username reason = extras.locked_by[2] # this exception is interpreted in git/hg middlewares and based diff --git a/rhodecode/lib/str_utils.py b/rhodecode/lib/str_utils.py index 7f437786..97aa6bd5 100644 --- a/rhodecode/lib/str_utils.py +++ b/rhodecode/lib/str_utils.py @@ -67,10 +67,7 @@ def base64_to_str(text: str | bytes) -> str: def get_default_encodings() -> list[str]: - return aslist(rhodecode.CONFIG.get('default_encoding', 'utf8'), sep=',') - - -DEFAULT_ENCODINGS = get_default_encodings() + return rhodecode.ConfigGet().get_list('default_encoding', missing='utf8') def safe_str(str_, to_encoding=None) -> str: @@ -87,7 +84,7 @@ def safe_str(str_, to_encoding=None) -> str: if not isinstance(str_, bytes): return str(str_) - to_encoding = to_encoding or DEFAULT_ENCODINGS + to_encoding = to_encoding or get_default_encodings() if not isinstance(to_encoding, (list, tuple)): to_encoding = [to_encoding] @@ -120,7 +117,7 @@ def safe_bytes(str_, from_encoding=None) -> bytes: for enc in from_encoding: try: return str_.encode(enc) - except UnicodeDecodeError: + except (UnicodeDecodeError, UnicodeEncodeError): pass return str_.encode(from_encoding[0], 'replace') diff --git a/rhodecode/lib/vcs/__init__.py b/rhodecode/lib/vcs/__init__.py index e03c4031..5c3aabd3 100644 --- a/rhodecode/lib/vcs/__init__.py +++ b/rhodecode/lib/vcs/__init__.py @@ -139,7 +139,7 @@ class CurlSession(object): try: curl.perform() - except pycurl.error as exc: + except pycurl.error: log.error('Failed to call endpoint url: %s using pycurl', url) raise diff --git a/rhodecode/lib/vcs/backends/base.py b/rhodecode/lib/vcs/backends/base.py index 2d0ea95d..a44e7621 100644 --- a/rhodecode/lib/vcs/backends/base.py +++ b/rhodecode/lib/vcs/backends/base.py @@ -19,6 +19,7 @@ """ Base module for all VCS systems """ + import os import re import time @@ -39,19 +40,25 @@ from rhodecode.lib.utils2 import safe_str, CachedProperty from rhodecode.lib.vcs.utils import author_name, author_email from rhodecode.lib.vcs.conf import settings from rhodecode.lib.vcs.exceptions import ( - CommitError, EmptyRepositoryError, NodeAlreadyAddedError, - NodeAlreadyChangedError, NodeAlreadyExistsError, NodeAlreadyRemovedError, - NodeDoesNotExistError, NodeNotChangedError, VCSError, - ImproperArchiveTypeError, BranchDoesNotExistError, CommitDoesNotExistError, - RepositoryError) + CommitError, + EmptyRepositoryError, + NodeAlreadyAddedError, + NodeAlreadyChangedError, + NodeAlreadyExistsError, + NodeAlreadyRemovedError, + NodeDoesNotExistError, + NodeNotChangedError, + VCSError, + ImproperArchiveTypeError, + BranchDoesNotExistError, + CommitDoesNotExistError, + RepositoryError, +) log = logging.getLogger(__name__) - -FILEMODE_DEFAULT = 0o100644 -FILEMODE_EXECUTABLE = 0o100755 -EMPTY_COMMIT_ID = '0' * 40 +EMPTY_COMMIT_ID = "0" * 40 @dataclasses.dataclass @@ -67,12 +74,12 @@ class Reference: @property def branch(self): - if self.type == 'branch': + if self.type == "branch": return self.name @property def bookmark(self): - if self.type == 'book': + if self.type == "book": return self.name @property @@ -80,11 +87,7 @@ class Reference: return reference_to_unicode(self) def asdict(self): - return dict( - type=self.type, - name=self.name, - commit_id=self.commit_id - ) + return dict(type=self.type, name=self.name, commit_id=self.commit_id) def unicode_to_reference(raw: str): @@ -93,7 +96,7 @@ def unicode_to_reference(raw: str): If unicode evaluates to False it returns None. """ if raw: - refs = raw.split(':') + refs = raw.split(":") return Reference(*refs) else: return None @@ -105,7 +108,7 @@ def reference_to_unicode(ref: Reference): If reference is None it returns None. """ if ref: - return ':'.join(ref) + return ":".join(ref) else: return None @@ -194,47 +197,44 @@ class UpdateFailureReason(object): class MergeResponse(object): - # uses .format(**metadata) for variables MERGE_STATUS_MESSAGES = { - MergeFailureReason.NONE: lazy_ugettext( - 'This pull request can be automatically merged.'), + MergeFailureReason.NONE: lazy_ugettext("This pull request can be automatically merged."), MergeFailureReason.UNKNOWN: lazy_ugettext( - 'This pull request cannot be merged because of an unhandled exception. ' - '{exception}'), + "This pull request cannot be merged because of an unhandled exception. " "{exception}" + ), MergeFailureReason.MERGE_FAILED: lazy_ugettext( - 'This pull request cannot be merged because of merge conflicts. {unresolved_files}'), + "This pull request cannot be merged because of merge conflicts. {unresolved_files}" + ), MergeFailureReason.PUSH_FAILED: lazy_ugettext( - 'This pull request could not be merged because push to ' - 'target:`{target}@{merge_commit}` failed.'), + "This pull request could not be merged because push to " "target:`{target}@{merge_commit}` failed." + ), MergeFailureReason.TARGET_IS_NOT_HEAD: lazy_ugettext( - 'This pull request cannot be merged because the target ' - '`{target_ref.name}` is not a head.'), + "This pull request cannot be merged because the target " "`{target_ref.name}` is not a head." + ), MergeFailureReason.HG_SOURCE_HAS_MORE_BRANCHES: lazy_ugettext( - 'This pull request cannot be merged because the source contains ' - 'more branches than the target.'), + "This pull request cannot be merged because the source contains " "more branches than the target." + ), MergeFailureReason.HG_TARGET_HAS_MULTIPLE_HEADS: lazy_ugettext( - 'This pull request cannot be merged because the target `{target_ref.name}` ' - 'has multiple heads: `{heads}`.'), + "This pull request cannot be merged because the target `{target_ref.name}` " + "has multiple heads: `{heads}`." + ), MergeFailureReason.TARGET_IS_LOCKED: lazy_ugettext( - 'This pull request cannot be merged because the target repository is ' - 'locked by {locked_by}.'), - + "This pull request cannot be merged because the target repository is " "locked by {locked_by}." + ), MergeFailureReason.MISSING_TARGET_REF: lazy_ugettext( - 'This pull request cannot be merged because the target ' - 'reference `{target_ref.name}` is missing.'), + "This pull request cannot be merged because the target " "reference `{target_ref.name}` is missing." + ), MergeFailureReason.MISSING_SOURCE_REF: lazy_ugettext( - 'This pull request cannot be merged because the source ' - 'reference `{source_ref.name}` is missing.'), + "This pull request cannot be merged because the source " "reference `{source_ref.name}` is missing." + ), MergeFailureReason.SUBREPO_MERGE_FAILED: lazy_ugettext( - 'This pull request cannot be merged because of conflicts related ' - 'to sub repositories.'), - + "This pull request cannot be merged because of conflicts related " "to sub repositories." + ), # Deprecations MergeFailureReason._DEPRECATED_MISSING_COMMIT: lazy_ugettext( - 'This pull request cannot be merged because the target or the ' - 'source reference is missing.'), - + "This pull request cannot be merged because the target or the " "source reference is missing." + ), } def __init__(self, possible, executed, merge_ref: Reference, failure_reason, metadata=None): @@ -245,19 +245,20 @@ class MergeResponse(object): self.metadata = metadata or {} def __repr__(self): - return f'' + return f"" def __eq__(self, other): same_instance = isinstance(other, self.__class__) - return same_instance \ - and self.possible == other.possible \ - and self.executed == other.executed \ - and self.failure_reason == other.failure_reason + return ( + same_instance + and self.possible == other.possible + and self.executed == other.executed + and self.failure_reason == other.failure_reason + ) @property def label(self): - label_dict = dict((v, k) for k, v in MergeFailureReason.__dict__.items() if - not k.startswith('_')) + label_dict = dict((v, k) for k, v in MergeFailureReason.__dict__.items() if not k.startswith("_")) return label_dict.get(self.failure_reason) @property @@ -270,13 +271,12 @@ class MergeResponse(object): try: return msg.format(**self.metadata) except Exception: - log.exception('Failed to format %s message', self) + log.exception("Failed to format %s message", self) return msg def asdict(self): data = {} - for k in ['possible', 'executed', 'merge_ref', 'failure_reason', - 'merge_status_message']: + for k in ["possible", "executed", "merge_ref", "failure_reason", "merge_status_message"]: data[k] = getattr(self, k) return data @@ -289,607 +289,7 @@ class SourceRefMissing(ValueError): pass -class BaseRepository(object): - """ - Base Repository for final backends - - .. attribute:: DEFAULT_BRANCH_NAME - - name of default branch (i.e. "trunk" for svn, "master" for git etc. - - .. attribute:: commit_ids - - list of all available commit ids, in ascending order - - .. attribute:: path - - absolute path to the repository - - .. attribute:: bookmarks - - Mapping from name to :term:`Commit ID` of the bookmark. Empty in case - there are no bookmarks or the backend implementation does not support - bookmarks. - - .. attribute:: tags - - Mapping from name to :term:`Commit ID` of the tag. - - """ - - DEFAULT_BRANCH_NAME = None - DEFAULT_CONTACT = "Unknown" - DEFAULT_DESCRIPTION = "unknown" - EMPTY_COMMIT_ID = '0' * 40 - COMMIT_ID_PAT = re.compile(r'[0-9a-fA-F]{40}') - - path = None - - _is_empty = None - _commit_ids = {} - - def __init__(self, repo_path, config=None, create=False, **kwargs): - """ - Initializes repository. Raises RepositoryError if repository could - not be find at the given ``repo_path`` or directory at ``repo_path`` - exists and ``create`` is set to True. - - :param repo_path: local path of the repository - :param config: repository configuration - :param create=False: if set to True, would try to create repository. - :param src_url=None: if set, should be proper url from which repository - would be cloned; requires ``create`` parameter to be set to True - - raises RepositoryError if src_url is set and create evaluates to - False - """ - raise NotImplementedError - - def __repr__(self): - return f'<{self.__class__.__name__} at {self.path}>' - - def __len__(self): - return self.count() - - def __eq__(self, other): - same_instance = isinstance(other, self.__class__) - return same_instance and other.path == self.path - - def __ne__(self, other): - return not self.__eq__(other) - - def get_create_shadow_cache_pr_path(self, db_repo): - path = db_repo.cached_diffs_dir - if not os.path.exists(path): - os.makedirs(path, 0o755) - return path - - @classmethod - def get_default_config(cls, default=None): - config = Config() - if default and isinstance(default, list): - for section, key, val in default: - config.set(section, key, val) - return config - - @LazyProperty - def _remote(self): - raise NotImplementedError - - def _heads(self, branch=None): - return [] - - @LazyProperty - def EMPTY_COMMIT(self): - return EmptyCommit(self.EMPTY_COMMIT_ID) - - @LazyProperty - def alias(self): - for k, v in settings.BACKENDS.items(): - if v.split('.')[-1] == str(self.__class__.__name__): - return k - - @LazyProperty - def name(self): - return safe_str(os.path.basename(self.path)) - - @LazyProperty - def description(self): - raise NotImplementedError - - def refs(self): - """ - returns a `dict` with branches, bookmarks, tags, and closed_branches - for this repository - """ - return dict( - branches=self.branches, - branches_closed=self.branches_closed, - tags=self.tags, - bookmarks=self.bookmarks - ) - - @LazyProperty - def branches(self): - """ - A `dict` which maps branch names to commit ids. - """ - raise NotImplementedError - - @LazyProperty - def branches_closed(self): - """ - A `dict` which maps tags names to commit ids. - """ - raise NotImplementedError - - @LazyProperty - def bookmarks(self): - """ - A `dict` which maps tags names to commit ids. - """ - raise NotImplementedError - - @LazyProperty - def tags(self): - """ - A `dict` which maps tags names to commit ids. - """ - raise NotImplementedError - - @LazyProperty - def size(self): - """ - Returns combined size in bytes for all repository files - """ - tip = self.get_commit() - return tip.size - - def size_at_commit(self, commit_id): - commit = self.get_commit(commit_id) - return commit.size - - def _check_for_empty(self): - no_commits = len(self._commit_ids) == 0 - if no_commits: - # check on remote to be sure - return self._remote.is_empty() - else: - return False - - def is_empty(self): - if rhodecode.is_test: - return self._check_for_empty() - - if self._is_empty is None: - # cache empty for production, but not tests - self._is_empty = self._check_for_empty() - - return self._is_empty - - @staticmethod - def check_url(url, config): - """ - Function will check given url and try to verify if it's a valid - link. - """ - raise NotImplementedError - - @staticmethod - def is_valid_repository(path): - """ - Check if given `path` contains a valid repository of this backend - """ - raise NotImplementedError - - # ========================================================================== - # COMMITS - # ========================================================================== - - @CachedProperty - def commit_ids(self): - raise NotImplementedError - - def append_commit_id(self, commit_id): - if commit_id not in self.commit_ids: - self._rebuild_cache(self.commit_ids + [commit_id]) - - # clear cache - self._invalidate_prop_cache('commit_ids') - self._is_empty = False - - def get_commit(self, commit_id=None, commit_idx=None, pre_load=None, - translate_tag=None, maybe_unreachable=False, reference_obj=None): - """ - Returns instance of `BaseCommit` class. If `commit_id` and `commit_idx` - are both None, most recent commit is returned. - - :param pre_load: Optional. List of commit attributes to load. - - :raises ``EmptyRepositoryError``: if there are no commits - """ - raise NotImplementedError - - def __iter__(self): - for commit_id in self.commit_ids: - yield self.get_commit(commit_id=commit_id) - - def get_commits( - self, start_id=None, end_id=None, start_date=None, end_date=None, - branch_name=None, show_hidden=False, pre_load=None, translate_tags=None): - """ - Returns iterator of `BaseCommit` objects from start to end - not inclusive. This should behave just like a list, ie. end is not - inclusive. - - :param start_id: None or str, must be a valid commit id - :param end_id: None or str, must be a valid commit id - :param start_date: - :param end_date: - :param branch_name: - :param show_hidden: - :param pre_load: - :param translate_tags: - """ - raise NotImplementedError - - def __getitem__(self, key): - """ - Allows index based access to the commit objects of this repository. - """ - pre_load = ["author", "branch", "date", "message", "parents"] - if isinstance(key, slice): - return self._get_range(key, pre_load) - return self.get_commit(commit_idx=key, pre_load=pre_load) - - def _get_range(self, slice_obj, pre_load): - for commit_id in self.commit_ids.__getitem__(slice_obj): - yield self.get_commit(commit_id=commit_id, pre_load=pre_load) - - def count(self): - return len(self.commit_ids) - - def tag(self, name, user, commit_id=None, message=None, date=None, **opts): - """ - Creates and returns a tag for the given ``commit_id``. - - :param name: name for new tag - :param user: full username, i.e.: "Joe Doe " - :param commit_id: commit id for which new tag would be created - :param message: message of the tag's commit - :param date: date of tag's commit - - :raises TagAlreadyExistError: if tag with same name already exists - """ - raise NotImplementedError - - def remove_tag(self, name, user, message=None, date=None): - """ - Removes tag with the given ``name``. - - :param name: name of the tag to be removed - :param user: full username, i.e.: "Joe Doe " - :param message: message of the tag's removal commit - :param date: date of tag's removal commit - - :raises TagDoesNotExistError: if tag with given name does not exists - """ - raise NotImplementedError - - def get_diff( - self, commit1, commit2, path=None, ignore_whitespace=False, - context=3, path1=None): - """ - Returns (git like) *diff*, as plain text. Shows changes introduced by - `commit2` since `commit1`. - - :param commit1: Entry point from which diff is shown. Can be - ``self.EMPTY_COMMIT`` - in this case, patch showing all - the changes since empty state of the repository until `commit2` - :param commit2: Until which commit changes should be shown. - :param path: Can be set to a path of a file to create a diff of that - file. If `path1` is also set, this value is only associated to - `commit2`. - :param ignore_whitespace: If set to ``True``, would not show whitespace - changes. Defaults to ``False``. - :param context: How many lines before/after changed lines should be - shown. Defaults to ``3``. - :param path1: Can be set to a path to associate with `commit1`. This - parameter works only for backends which support diff generation for - different paths. Other backends will raise a `ValueError` if `path1` - is set and has a different value than `path`. - :param file_path: filter this diff by given path pattern - """ - raise NotImplementedError - - def strip(self, commit_id, branch=None): - """ - Strip given commit_id from the repository - """ - raise NotImplementedError - - def get_common_ancestor(self, commit_id1, commit_id2, repo2): - """ - Return a latest common ancestor commit if one exists for this repo - `commit_id1` vs `commit_id2` from `repo2`. - - :param commit_id1: Commit it from this repository to use as a - target for the comparison. - :param commit_id2: Source commit id to use for comparison. - :param repo2: Source repository to use for comparison. - """ - raise NotImplementedError - - def compare(self, commit_id1, commit_id2, repo2, merge, pre_load=None): - """ - Compare this repository's revision `commit_id1` with `commit_id2`. - - Returns a tuple(commits, ancestor) that would be merged from - `commit_id2`. Doing a normal compare (``merge=False``), ``None`` - will be returned as ancestor. - - :param commit_id1: Commit it from this repository to use as a - target for the comparison. - :param commit_id2: Source commit id to use for comparison. - :param repo2: Source repository to use for comparison. - :param merge: If set to ``True`` will do a merge compare which also - returns the common ancestor. - :param pre_load: Optional. List of commit attributes to load. - """ - raise NotImplementedError - - def merge(self, repo_id, workspace_id, target_ref, source_repo, source_ref, - user_name='', user_email='', message='', dry_run=False, - use_rebase=False, close_branch=False): - """ - Merge the revisions specified in `source_ref` from `source_repo` - onto the `target_ref` of this repository. - - `source_ref` and `target_ref` are named tupls with the following - fields `type`, `name` and `commit_id`. - - Returns a MergeResponse named tuple with the following fields - 'possible', 'executed', 'source_commit', 'target_commit', - 'merge_commit'. - - :param repo_id: `repo_id` target repo id. - :param workspace_id: `workspace_id` unique identifier. - :param target_ref: `target_ref` points to the commit on top of which - the `source_ref` should be merged. - :param source_repo: The repository that contains the commits to be - merged. - :param source_ref: `source_ref` points to the topmost commit from - the `source_repo` which should be merged. - :param user_name: Merge commit `user_name`. - :param user_email: Merge commit `user_email`. - :param message: Merge commit `message`. - :param dry_run: If `True` the merge will not take place. - :param use_rebase: If `True` commits from the source will be rebased - on top of the target instead of being merged. - :param close_branch: If `True` branch will be close before merging it - """ - if dry_run: - message = message or settings.MERGE_DRY_RUN_MESSAGE - user_email = user_email or settings.MERGE_DRY_RUN_EMAIL - user_name = user_name or settings.MERGE_DRY_RUN_USER - else: - if not user_name: - raise ValueError('user_name cannot be empty') - if not user_email: - raise ValueError('user_email cannot be empty') - if not message: - raise ValueError('message cannot be empty') - - try: - return self._merge_repo( - repo_id, workspace_id, target_ref, source_repo, - source_ref, message, user_name, user_email, dry_run=dry_run, - use_rebase=use_rebase, close_branch=close_branch) - except RepositoryError as exc: - log.exception('Unexpected failure when running merge, dry-run=%s', dry_run) - return MergeResponse( - False, False, None, MergeFailureReason.UNKNOWN, - metadata={'exception': str(exc)}) - - def _merge_repo(self, repo_id, workspace_id, target_ref, - source_repo, source_ref, merge_message, - merger_name, merger_email, dry_run=False, - use_rebase=False, close_branch=False): - """Internal implementation of merge.""" - raise NotImplementedError - - def _maybe_prepare_merge_workspace( - self, repo_id, workspace_id, target_ref, source_ref): - """ - Create the merge workspace. - - :param workspace_id: `workspace_id` unique identifier. - """ - raise NotImplementedError - - @classmethod - def _get_legacy_shadow_repository_path(cls, repo_path, workspace_id): - """ - Legacy version that was used before. We still need it for - backward compat - """ - return os.path.join( - os.path.dirname(repo_path), - f'.__shadow_{os.path.basename(repo_path)}_{workspace_id}') - - @classmethod - def _get_shadow_repository_path(cls, repo_path, repo_id, workspace_id): - # The name of the shadow repository must start with '.', so it is - # skipped by 'rhodecode.lib.utils.get_filesystem_repos'. - legacy_repository_path = cls._get_legacy_shadow_repository_path(repo_path, workspace_id) - if os.path.exists(legacy_repository_path): - return legacy_repository_path - else: - return os.path.join( - os.path.dirname(repo_path), - f'.__shadow_repo_{repo_id}_{workspace_id}') - - def cleanup_merge_workspace(self, repo_id, workspace_id): - """ - Remove merge workspace. - - This function MUST not fail in case there is no workspace associated to - the given `workspace_id`. - - :param workspace_id: `workspace_id` unique identifier. - """ - shadow_repository_path = self._get_shadow_repository_path( - self.path, repo_id, workspace_id) - shadow_repository_path_del = '{}.{}.delete'.format( - shadow_repository_path, time.time()) - - # move the shadow repo, so it never conflicts with the one used. - # we use this method because shutil.rmtree had some edge case problems - # removing symlinked repositories - if not os.path.isdir(shadow_repository_path): - return - - shutil.move(shadow_repository_path, shadow_repository_path_del) - try: - shutil.rmtree(shadow_repository_path_del, ignore_errors=False) - except Exception: - log.exception('Failed to gracefully remove shadow repo under %s', - shadow_repository_path_del) - shutil.rmtree(shadow_repository_path_del, ignore_errors=True) - - # ========== # - # COMMIT API # - # ========== # - - @LazyProperty - def in_memory_commit(self): - """ - Returns :class:`InMemoryCommit` object for this repository. - """ - raise NotImplementedError - - # ======================== # - # UTILITIES FOR SUBCLASSES # - # ======================== # - - def _validate_diff_commits(self, commit1, commit2): - """ - Validates that the given commits are related to this repository. - - Intended as a utility for sub classes to have a consistent validation - of input parameters in methods like :meth:`get_diff`. - """ - self._validate_commit(commit1) - self._validate_commit(commit2) - if (isinstance(commit1, EmptyCommit) and - isinstance(commit2, EmptyCommit)): - raise ValueError("Cannot compare two empty commits") - - def _validate_commit(self, commit): - if not isinstance(commit, BaseCommit): - raise TypeError( - "%s is not of type BaseCommit" % repr(commit)) - if commit.repository != self and not isinstance(commit, EmptyCommit): - raise ValueError( - "Commit %s must be a valid commit from this repository %s, " - "related to this repository instead %s." % - (commit, self, commit.repository)) - - def _validate_commit_id(self, commit_id): - if not isinstance(commit_id, str): - raise TypeError(f"commit_id must be a string value got {type(commit_id)} instead") - - def _validate_commit_idx(self, commit_idx): - if not isinstance(commit_idx, int): - raise TypeError(f"commit_idx must be a numeric value, got {type(commit_idx)}") - - def _validate_branch_name(self, branch_name): - if branch_name and branch_name not in self.branches_all: - msg = (f"Branch {branch_name} not found in {self}") - raise BranchDoesNotExistError(msg) - - # - # Supporting deprecated API parts - # TODO: johbo: consider to move this into a mixin - # - - @property - def EMPTY_CHANGESET(self): - warnings.warn( - "Use EMPTY_COMMIT or EMPTY_COMMIT_ID instead", DeprecationWarning) - return self.EMPTY_COMMIT_ID - - @property - def revisions(self): - warnings.warn("Use commits attribute instead", DeprecationWarning) - return self.commit_ids - - @revisions.setter - def revisions(self, value): - warnings.warn("Use commits attribute instead", DeprecationWarning) - self.commit_ids = value - - def get_changeset(self, revision=None, pre_load=None): - warnings.warn("Use get_commit instead", DeprecationWarning) - commit_id = None - commit_idx = None - if isinstance(revision, str): - commit_id = revision - else: - commit_idx = revision - return self.get_commit( - commit_id=commit_id, commit_idx=commit_idx, pre_load=pre_load) - - def get_changesets( - self, start=None, end=None, start_date=None, end_date=None, - branch_name=None, pre_load=None): - warnings.warn("Use get_commits instead", DeprecationWarning) - start_id = self._revision_to_commit(start) - end_id = self._revision_to_commit(end) - return self.get_commits( - start_id=start_id, end_id=end_id, start_date=start_date, - end_date=end_date, branch_name=branch_name, pre_load=pre_load) - - def _revision_to_commit(self, revision): - """ - Translates a revision to a commit_id - - Helps to support the old changeset based API which allows to use - commit ids and commit indices interchangeable. - """ - if revision is None: - return revision - - if isinstance(revision, str): - commit_id = revision - else: - commit_id = self.commit_ids[revision] - return commit_id - - @property - def in_memory_changeset(self): - warnings.warn("Use in_memory_commit instead", DeprecationWarning) - return self.in_memory_commit - - def get_path_permissions(self, username): - """ - Returns a path permission checker or None if not supported - - :param username: session user name - :return: an instance of BasePathPermissionChecker or None - """ - return None - - def install_hooks(self, force=False): - return self._remote.install_hooks(force) - - def get_hooks_info(self): - return self._remote.get_hooks_info() - - def vcsserver_invalidate_cache(self, delete=False): - return self._remote.vcsserver_invalidate_cache(delete) - - -class BaseCommit(object): +class BaseCommit: """ Each backend should implement it's commit representation. @@ -933,6 +333,7 @@ class BaseCommit(object): list of parent commits """ + repository = None branch = None @@ -942,7 +343,7 @@ class BaseCommit(object): value as ``None``. """ - _ARCHIVE_PREFIX_TEMPLATE = '{repo_name}-{short_id}' + _ARCHIVE_PREFIX_TEMPLATE = "{repo_name}-{short_id}" """ This template is used to generate a default prefix for repository archives if no prefix has been specified. @@ -952,7 +353,7 @@ class BaseCommit(object): return self.__str__() def __str__(self): - return f'<{self.__class__.__name__} at {self.idx}:{self.short_id}>' + return f"<{self.__class__.__name__} at {self.idx}:{self.short_id}>" def __eq__(self, other): same_instance = isinstance(other, self.__class__) @@ -962,26 +363,26 @@ class BaseCommit(object): parents = [] try: for parent in self.parents: - parents.append({'raw_id': parent.raw_id}) + parents.append({"raw_id": parent.raw_id}) except NotImplementedError: # empty commit doesn't have parents implemented pass return { - 'short_id': self.short_id, - 'raw_id': self.raw_id, - 'revision': self.idx, - 'message': self.message, - 'date': self.date, - 'author': self.author, - 'parents': parents, - 'branch': self.branch + "short_id": self.short_id, + "raw_id": self.raw_id, + "revision": self.idx, + "message": self.message, + "date": self.date, + "author": self.author, + "parents": parents, + "branch": self.branch, } def __getstate__(self): d = self.__dict__.copy() - d.pop('_remote', None) - d.pop('repository', None) + d.pop("_remote", None) + d.pop("repository", None) return d def get_remote(self): @@ -992,9 +393,9 @@ class BaseCommit(object): def _get_refs(self): return { - 'branches': [self.branch] if self.branch else [], - 'bookmarks': getattr(self, 'bookmarks', []), - 'tags': self.tags + "branches": [self.branch] if self.branch else [], + "bookmarks": getattr(self, "bookmarks", []), + "tags": self.tags, } @LazyProperty @@ -1118,7 +519,7 @@ class BaseCommit(object): """ raise NotImplementedError - def is_link(self, path): + def is_link(self, path: bytes): """ Returns ``True`` if given `path` is a symlink """ @@ -1154,25 +555,24 @@ class BaseCommit(object): """ raise NotImplementedError - def get_path_commit(self, path, pre_load=None): + def get_path_commit(self, path: bytes, pre_load=None): """ Returns last commit of the file at the given `path`. - :param pre_load: Optional. List of commit attributes to load. """ commits = self.get_path_history(path, limit=1, pre_load=pre_load) if not commits: raise RepositoryError( - 'Failed to fetch history for path {}. ' - 'Please check if such path exists in your repository'.format( - path)) + f"Failed to fetch history for path {path}. Please check if such path exists in your repository" + ) return commits[0] - def get_path_history(self, path, limit=None, pre_load=None): + def get_path_history(self, path: bytes, limit=None, pre_load=None): """ Returns history of file as reversed list of :class:`BaseCommit` objects for which file at given `path` has been modified. + :param path: file path or dir path :param limit: Optional. Allows to limit the size of the returned history. This is intended as a hint to the underlying backend, so that it can apply optimizations depending on the limit. @@ -1180,16 +580,17 @@ class BaseCommit(object): """ raise NotImplementedError - def get_file_annotate(self, path, pre_load=None): + def get_file_annotate(self, path: bytes, pre_load=None): """ Returns a generator of four element tuples with lineno, sha, commit lazy loader and line + :param path: file path :param pre_load: Optional. List of commit attributes to load. """ raise NotImplementedError - def get_nodes(self, path, pre_load=None): + def get_nodes(self, path: bytes, pre_load=None): """ Returns combined ``DirNode`` and ``FileNode`` objects list representing state of commit at the given ``path``. @@ -1199,7 +600,7 @@ class BaseCommit(object): """ raise NotImplementedError - def get_node(self, path): + def get_node(self, path: bytes, pre_load=None): """ Returns ``Node`` object from the given ``path``. @@ -1208,16 +609,24 @@ class BaseCommit(object): """ raise NotImplementedError - def get_largefile_node(self, path): + def get_largefile_node(self, path: bytes): """ Returns the path to largefile from Mercurial/Git-lfs storage. or None if it's not a largefile node """ return None - def archive_repo(self, archive_name_key, kind='tgz', subrepos=None, - archive_dir_name=None, write_metadata=False, mtime=None, - archive_at_path='/', cache_config=None): + def archive_repo( + self, + archive_name_key, + kind="tgz", + subrepos=None, + archive_dir_name=None, + write_metadata=False, + mtime=None, + archive_at_path="/", + cache_config=None, + ): """ Creates an archive containing the contents of the repository. @@ -1237,27 +646,26 @@ class BaseCommit(object): cache_config = cache_config or {} allowed_kinds = [x[0] for x in settings.ARCHIVE_SPECS] if kind not in allowed_kinds: - raise ImproperArchiveTypeError( - f'Archive kind ({kind}) not supported use one of {allowed_kinds}') + raise ImproperArchiveTypeError(f"Archive kind ({kind}) not supported use one of {allowed_kinds}") archive_dir_name = self._validate_archive_prefix(archive_dir_name) mtime = mtime is not None or time.mktime(self.date.timetuple()) commit_id = self.raw_id return self.repository._remote.archive_repo( - archive_name_key, kind, mtime, archive_at_path, - archive_dir_name, commit_id, cache_config) + archive_name_key, kind, mtime, archive_at_path, archive_dir_name, commit_id, cache_config + ) def _validate_archive_prefix(self, archive_dir_name): if archive_dir_name is None: archive_dir_name = self._ARCHIVE_PREFIX_TEMPLATE.format( - repo_name=safe_str(self.repository.name), - short_id=self.short_id) + repo_name=safe_str(self.repository.name), short_id=self.short_id + ) elif not isinstance(archive_dir_name, str): raise ValueError(f"archive_dir_name is not str object but: {type(archive_dir_name)}") - elif archive_dir_name.startswith('/'): + elif archive_dir_name.startswith("/"): raise VCSError("Prefix cannot start with leading slash") - elif archive_dir_name.strip() == '': + elif archive_dir_name.strip() == "": raise VCSError("Prefix cannot be empty") elif not archive_dir_name.isascii(): raise VCSError("Prefix cannot contain non ascii characters") @@ -1268,7 +676,7 @@ class BaseCommit(object): """ Returns ``RootNode`` object for this commit. """ - return self.get_node('') + return self.get_node(b"") def next(self, branch=None): """ @@ -1292,8 +700,7 @@ class BaseCommit(object): def _find_next(self, indexes, branch=None): if branch and self.branch != branch: - raise VCSError('Branch option used on commit not belonging ' - 'to that branch') + raise VCSError("Branch option used on commit not belonging " "to that branch") for next_idx in indexes: commit = self.repository.get_commit(commit_idx=next_idx) @@ -1307,33 +714,9 @@ class BaseCommit(object): Returns a `Diff` object representing the change made by this commit. """ parent = self.first_parent - diff = self.repository.get_diff( - parent, self, - ignore_whitespace=ignore_whitespace, - context=context) + diff = self.repository.get_diff(parent, self, ignore_whitespace=ignore_whitespace, context=context) return diff - @LazyProperty - def added(self): - """ - Returns list of added ``FileNode`` objects. - """ - raise NotImplementedError - - @LazyProperty - def changed(self): - """ - Returns list of modified ``FileNode`` objects. - """ - raise NotImplementedError - - @LazyProperty - def removed(self): - """ - Returns list of removed ``FileNode`` objects. - """ - raise NotImplementedError - @LazyProperty def size(self): """ @@ -1341,7 +724,7 @@ class BaseCommit(object): """ return sum(node.size for node in self.get_filenodes_generator()) - def walk(self, topurl=''): + def walk(self, top_url=b""): """ Similar to os.walk method. Insted of filesystem it walks through commit starting at given ``topurl``. Returns generator of tuples @@ -1349,10 +732,10 @@ class BaseCommit(object): """ from rhodecode.lib.vcs.nodes import DirNode - if isinstance(topurl, DirNode): - top_node = topurl + if isinstance(top_url, DirNode): + top_node = top_url else: - top_node = self.get_node(topurl) + top_node = self.get_node(top_url) has_default_pre_load = False if isinstance(top_node, DirNode): @@ -1381,15 +764,20 @@ class BaseCommit(object): def no_node_at_path(self, path): return NodeDoesNotExistError( - f"There is no file nor directory at the given path: " - f"`{safe_str(path)}` at commit {self.short_id}") + f"There is no file nor directory at the given path: " f"`{safe_str(path)}` at commit {self.short_id}" + ) - def _fix_path(self, path: str) -> str: + @classmethod + def _fix_path(cls, path: bytes) -> bytes: """ - Paths are stored without trailing slash so we need to get rid off it if - needed. + Paths are stored without trailing slash so we need to get rid off it if needed. + It also validates that path is a bytestring for support of mixed encodings... """ - return safe_str(path).rstrip('/') + + if not isinstance(path, bytes): + raise ValueError(f'path=`{safe_str(path)}` must be bytes') + + return path.rstrip(b"/") # # Deprecated API based on changesets @@ -1410,17 +798,644 @@ class BaseCommit(object): return self.get_path_commit(path) -class BaseChangesetClass(type): +class BaseRepository(object): + """ + Base Repository for final backends + .. attribute:: DEFAULT_BRANCH_NAME + + name of default branch (i.e. "trunk" for svn, "master" for git etc. + + .. attribute:: commit_ids + + list of all available commit ids, in ascending order + + .. attribute:: path + + absolute path to the repository + + .. attribute:: bookmarks + + Mapping from name to :term:`Commit ID` of the bookmark. Empty in case + there are no bookmarks or the backend implementation does not support + bookmarks. + + .. attribute:: tags + + Mapping from name to :term:`Commit ID` of the tag. + + """ + + DEFAULT_BRANCH_NAME = None + DEFAULT_CONTACT = "Unknown" + DEFAULT_DESCRIPTION = "unknown" + EMPTY_COMMIT_ID = "0" * 40 + COMMIT_ID_PAT = re.compile(r"[0-9a-fA-F]{40}") + + path = None + + _is_empty = None + _commit_ids = {} + + def __init__(self, repo_path, config=None, create=False, **kwargs): + """ + Initializes repository. Raises RepositoryError if repository could + not be find at the given ``repo_path`` or directory at ``repo_path`` + exists and ``create`` is set to True. + + :param repo_path: local path of the repository + :param config: repository configuration + :param create=False: if set to True, would try to create repository. + :param src_url=None: if set, should be proper url from which repository + would be cloned; requires ``create`` parameter to be set to True - + raises RepositoryError if src_url is set and create evaluates to + False + """ + raise NotImplementedError + + def __repr__(self): + return f"<{self.__class__.__name__} at {self.path}>" + + def __len__(self): + return self.count() + + def __eq__(self, other): + same_instance = isinstance(other, self.__class__) + return same_instance and other.path == self.path + + def __ne__(self, other): + return not self.__eq__(other) + + @classmethod + def get_create_shadow_cache_pr_path(cls, db_repo): + path = db_repo.cached_diffs_dir + if not os.path.exists(path): + os.makedirs(path, 0o755) + return path + + @classmethod + def get_default_config(cls, default=None): + config = Config() + if default and isinstance(default, list): + for section, key, val in default: + config.set(section, key, val) + return config + + @LazyProperty + def _remote(self): + raise NotImplementedError + + def _heads(self, branch=None): + return [] + + @LazyProperty + def EMPTY_COMMIT(self): + return EmptyCommit(self.EMPTY_COMMIT_ID) + + @LazyProperty + def alias(self): + for k, v in settings.BACKENDS.items(): + if v.split(".")[-1] == str(self.__class__.__name__): + return k + + @LazyProperty + def name(self): + return safe_str(os.path.basename(self.path)) + + @LazyProperty + def description(self): + raise NotImplementedError + + def refs(self): + """ + returns a `dict` with branches, bookmarks, tags, and closed_branches + for this repository + """ + return dict( + branches=self.branches, branches_closed=self.branches_closed, tags=self.tags, bookmarks=self.bookmarks + ) + + @LazyProperty + def branches(self): + """ + A `dict` which maps branch names to commit ids. + """ + raise NotImplementedError + + @LazyProperty + def branches_closed(self): + """ + A `dict` which maps tags names to commit ids. + """ + raise NotImplementedError + + @LazyProperty + def bookmarks(self): + """ + A `dict` which maps tags names to commit ids. + """ + raise NotImplementedError + + @LazyProperty + def tags(self): + """ + A `dict` which maps tags names to commit ids. + """ + raise NotImplementedError + + @LazyProperty + def size(self): + """ + Returns combined size in bytes for all repository files + """ + tip = self.get_commit() + return tip.size + + def size_at_commit(self, commit_id): + commit = self.get_commit(commit_id) + return commit.size + + def _check_for_empty(self): + no_commits = len(self._commit_ids) == 0 + if no_commits: + # check on remote to be sure + return self._remote.is_empty() + else: + return False + + def is_empty(self): + if rhodecode.is_test: + return self._check_for_empty() + + if self._is_empty is None: + # cache empty for production, but not tests + self._is_empty = self._check_for_empty() + + return self._is_empty + + @staticmethod + def check_url(url, config): + """ + Function will check given url and try to verify if it's a valid + link. + """ + raise NotImplementedError + + @staticmethod + def is_valid_repository(path): + """ + Check if given `path` contains a valid repository of this backend + """ + raise NotImplementedError + + # ========================================================================== + # COMMITS + # ========================================================================== + + @CachedProperty + def commit_ids(self): + raise NotImplementedError + + def append_commit_id(self, commit_id): + if commit_id not in self.commit_ids: + self._rebuild_cache(self.commit_ids + [commit_id]) + + # clear cache + self._invalidate_prop_cache("commit_ids") + self._is_empty = False + + def get_commit( + self, + commit_id=None, + commit_idx=None, + pre_load=None, + translate_tag=None, + maybe_unreachable=False, + reference_obj=None, + ) -> BaseCommit: + """ + Returns instance of `BaseCommit` class. If `commit_id` and `commit_idx` + are both None, most recent commit is returned. + + :param pre_load: Optional. List of commit attributes to load. + + :raises ``EmptyRepositoryError``: if there are no commits + """ + raise NotImplementedError + + def __iter__(self): + for commit_id in self.commit_ids: + yield self.get_commit(commit_id=commit_id) + + def get_commits( + self, + start_id=None, + end_id=None, + start_date=None, + end_date=None, + branch_name=None, + show_hidden=False, + pre_load=None, + translate_tags=None, + ): + """ + Returns iterator of `BaseCommit` objects from start to end + not inclusive. This should behave just like a list, ie. end is not + inclusive. + + :param start_id: None or str, must be a valid commit id + :param end_id: None or str, must be a valid commit id + :param start_date: + :param end_date: + :param branch_name: + :param show_hidden: + :param pre_load: + :param translate_tags: + """ + raise NotImplementedError + + def __getitem__(self, key): + """ + Allows index based access to the commit objects of this repository. + """ + pre_load = ["author", "branch", "date", "message", "parents"] + if isinstance(key, slice): + return self._get_range(key, pre_load) + return self.get_commit(commit_idx=key, pre_load=pre_load) + + def _get_range(self, slice_obj, pre_load): + for commit_id in self.commit_ids.__getitem__(slice_obj): + yield self.get_commit(commit_id=commit_id, pre_load=pre_load) + + def count(self): + return len(self.commit_ids) + + def tag(self, name, user, commit_id=None, message=None, date=None, **opts): + """ + Creates and returns a tag for the given ``commit_id``. + + :param name: name for new tag + :param user: full username, i.e.: "Joe Doe " + :param commit_id: commit id for which new tag would be created + :param message: message of the tag's commit + :param date: date of tag's commit + + :raises TagAlreadyExistError: if tag with same name already exists + """ + raise NotImplementedError + + def remove_tag(self, name, user, message=None, date=None): + """ + Removes tag with the given ``name``. + + :param name: name of the tag to be removed + :param user: full username, i.e.: "Joe Doe " + :param message: message of the tag's removal commit + :param date: date of tag's removal commit + + :raises TagDoesNotExistError: if tag with given name does not exists + """ + raise NotImplementedError + + def get_diff(self, commit1, commit2, path=None, ignore_whitespace=False, context=3, path1=None): + """ + Returns (git like) *diff*, as plain text. Shows changes introduced by + `commit2` since `commit1`. + + :param commit1: Entry point from which diff is shown. Can be + ``self.EMPTY_COMMIT`` - in this case, patch showing all + the changes since empty state of the repository until `commit2` + :param commit2: Until which commit changes should be shown. + :param path: Can be set to a path of a file to create a diff of that + file. If `path1` is also set, this value is only associated to + `commit2`. + :param ignore_whitespace: If set to ``True``, would not show whitespace + changes. Defaults to ``False``. + :param context: How many lines before/after changed lines should be + shown. Defaults to ``3``. + :param path1: Can be set to a path to associate with `commit1`. This + parameter works only for backends which support diff generation for + different paths. Other backends will raise a `ValueError` if `path1` + is set and has a different value than `path`. + :param file_path: filter this diff by given path pattern + """ + raise NotImplementedError + + def strip(self, commit_id, branch=None): + """ + Strip given commit_id from the repository + """ + raise NotImplementedError + + def get_common_ancestor(self, commit_id1, commit_id2, repo2): + """ + Return a latest common ancestor commit if one exists for this repo + `commit_id1` vs `commit_id2` from `repo2`. + + :param commit_id1: Commit it from this repository to use as a + target for the comparison. + :param commit_id2: Source commit id to use for comparison. + :param repo2: Source repository to use for comparison. + """ + raise NotImplementedError + + def compare(self, commit_id1, commit_id2, repo2, merge, pre_load=None): + """ + Compare this repository's revision `commit_id1` with `commit_id2`. + + Returns a tuple(commits, ancestor) that would be merged from + `commit_id2`. Doing a normal compare (``merge=False``), ``None`` + will be returned as ancestor. + + :param commit_id1: Commit it from this repository to use as a + target for the comparison. + :param commit_id2: Source commit id to use for comparison. + :param repo2: Source repository to use for comparison. + :param merge: If set to ``True`` will do a merge compare which also + returns the common ancestor. + :param pre_load: Optional. List of commit attributes to load. + """ + raise NotImplementedError + + def merge( + self, + repo_id, + workspace_id, + target_ref, + source_repo, + source_ref, + user_name="", + user_email="", + message="", + dry_run=False, + use_rebase=False, + close_branch=False, + ): + """ + Merge the revisions specified in `source_ref` from `source_repo` + onto the `target_ref` of this repository. + + `source_ref` and `target_ref` are named tupls with the following + fields `type`, `name` and `commit_id`. + + Returns a MergeResponse named tuple with the following fields + 'possible', 'executed', 'source_commit', 'target_commit', + 'merge_commit'. + + :param repo_id: `repo_id` target repo id. + :param workspace_id: `workspace_id` unique identifier. + :param target_ref: `target_ref` points to the commit on top of which + the `source_ref` should be merged. + :param source_repo: The repository that contains the commits to be + merged. + :param source_ref: `source_ref` points to the topmost commit from + the `source_repo` which should be merged. + :param user_name: Merge commit `user_name`. + :param user_email: Merge commit `user_email`. + :param message: Merge commit `message`. + :param dry_run: If `True` the merge will not take place. + :param use_rebase: If `True` commits from the source will be rebased + on top of the target instead of being merged. + :param close_branch: If `True` branch will be close before merging it + """ + if dry_run: + message = message or settings.MERGE_DRY_RUN_MESSAGE + user_email = user_email or settings.MERGE_DRY_RUN_EMAIL + user_name = user_name or settings.MERGE_DRY_RUN_USER + else: + if not user_name: + raise ValueError("user_name cannot be empty") + if not user_email: + raise ValueError("user_email cannot be empty") + if not message: + raise ValueError("message cannot be empty") + + try: + return self._merge_repo( + repo_id, + workspace_id, + target_ref, + source_repo, + source_ref, + message, + user_name, + user_email, + dry_run=dry_run, + use_rebase=use_rebase, + close_branch=close_branch, + ) + except RepositoryError as exc: + log.exception("Unexpected failure when running merge, dry-run=%s", dry_run) + return MergeResponse(False, False, None, MergeFailureReason.UNKNOWN, metadata={"exception": str(exc)}) + + def _merge_repo( + self, + repo_id, + workspace_id, + target_ref, + source_repo, + source_ref, + merge_message, + merger_name, + merger_email, + dry_run=False, + use_rebase=False, + close_branch=False, + ): + """Internal implementation of merge.""" + raise NotImplementedError + + def _maybe_prepare_merge_workspace(self, repo_id, workspace_id, target_ref, source_ref): + """ + Create the merge workspace. + + :param workspace_id: `workspace_id` unique identifier. + """ + raise NotImplementedError + + @classmethod + def _get_legacy_shadow_repository_path(cls, repo_path, workspace_id): + """ + Legacy version that was used before. We still need it for + backward compat + """ + return os.path.join(os.path.dirname(repo_path), f".__shadow_{os.path.basename(repo_path)}_{workspace_id}") + + @classmethod + def _get_shadow_repository_path(cls, repo_path, repo_id, workspace_id): + # The name of the shadow repository must start with '.', so it is + # skipped by 'rhodecode.lib.utils.get_filesystem_repos'. + legacy_repository_path = cls._get_legacy_shadow_repository_path(repo_path, workspace_id) + if os.path.exists(legacy_repository_path): + return legacy_repository_path + else: + return os.path.join(os.path.dirname(repo_path), f".__shadow_repo_{repo_id}_{workspace_id}") + + def cleanup_merge_workspace(self, repo_id, workspace_id): + """ + Remove merge workspace. + + This function MUST not fail in case there is no workspace associated to + the given `workspace_id`. + + :param workspace_id: `workspace_id` unique identifier. + """ + shadow_repository_path = self._get_shadow_repository_path(self.path, repo_id, workspace_id) + shadow_repository_path_del = "{}.{}.delete".format(shadow_repository_path, time.time()) + + # move the shadow repo, so it never conflicts with the one used. + # we use this method because shutil.rmtree had some edge case problems + # removing symlinked repositories + if not os.path.isdir(shadow_repository_path): + return + + shutil.move(shadow_repository_path, shadow_repository_path_del) + try: + shutil.rmtree(shadow_repository_path_del, ignore_errors=False) + except Exception: + log.exception("Failed to gracefully remove shadow repo under %s", shadow_repository_path_del) + shutil.rmtree(shadow_repository_path_del, ignore_errors=True) + + # ========== # + # COMMIT API # + # ========== # + + @LazyProperty + def in_memory_commit(self): + """ + Returns :class:`InMemoryCommit` object for this repository. + """ + raise NotImplementedError + + # ======================== # + # UTILITIES FOR SUBCLASSES # + # ======================== # + + def _validate_diff_commits(self, commit1, commit2): + """ + Validates that the given commits are related to this repository. + + Intended as a utility for sub classes to have a consistent validation + of input parameters in methods like :meth:`get_diff`. + """ + self._validate_commit(commit1) + self._validate_commit(commit2) + if isinstance(commit1, EmptyCommit) and isinstance(commit2, EmptyCommit): + raise ValueError("Cannot compare two empty commits") + + def _validate_commit(self, commit): + if not isinstance(commit, BaseCommit): + raise TypeError("%s is not of type BaseCommit" % repr(commit)) + if commit.repository != self and not isinstance(commit, EmptyCommit): + raise ValueError( + "Commit %s must be a valid commit from this repository %s, " + "related to this repository instead %s." % (commit, self, commit.repository) + ) + + def _validate_commit_id(self, commit_id): + if not isinstance(commit_id, str): + raise TypeError(f"commit_id must be a string value got {type(commit_id)} instead") + + def _validate_commit_idx(self, commit_idx): + if not isinstance(commit_idx, int): + raise TypeError(f"commit_idx must be a numeric value, got {type(commit_idx)}") + + def _validate_branch_name(self, branch_name): + if branch_name and branch_name not in self.branches_all: + msg = f"Branch {branch_name} not found in {self}" + raise BranchDoesNotExistError(msg) + + # + # Supporting deprecated API parts + # TODO: johbo: consider to move this into a mixin + # + + @property + def EMPTY_CHANGESET(self): + warnings.warn("Use EMPTY_COMMIT or EMPTY_COMMIT_ID instead", DeprecationWarning) + return self.EMPTY_COMMIT_ID + + @property + def revisions(self): + warnings.warn("Use commits attribute instead", DeprecationWarning) + return self.commit_ids + + @revisions.setter + def revisions(self, value): + warnings.warn("Use commits attribute instead", DeprecationWarning) + self.commit_ids = value + + def get_changeset(self, revision=None, pre_load=None): + warnings.warn("Use get_commit instead", DeprecationWarning) + commit_id = None + commit_idx = None + if isinstance(revision, str): + commit_id = revision + else: + commit_idx = revision + return self.get_commit(commit_id=commit_id, commit_idx=commit_idx, pre_load=pre_load) + + def get_changesets(self, start=None, end=None, start_date=None, end_date=None, branch_name=None, pre_load=None): + warnings.warn("Use get_commits instead", DeprecationWarning) + start_id = self._revision_to_commit(start) + end_id = self._revision_to_commit(end) + return self.get_commits( + start_id=start_id, + end_id=end_id, + start_date=start_date, + end_date=end_date, + branch_name=branch_name, + pre_load=pre_load, + ) + + def _revision_to_commit(self, revision): + """ + Translates a revision to a commit_id + + Helps to support the old changeset based API which allows to use + commit ids and commit indices interchangeable. + """ + if revision is None: + return revision + + if isinstance(revision, str): + commit_id = revision + else: + commit_id = self.commit_ids[revision] + return commit_id + + @property + def in_memory_changeset(self): + warnings.warn("Use in_memory_commit instead", DeprecationWarning) + return self.in_memory_commit + + def get_path_permissions(self, username): + """ + Returns a path permission checker or None if not supported + + :param username: session user name + :return: an instance of BasePathPermissionChecker or None + """ + return None + + def install_hooks(self, force=False): + return self._remote.install_hooks(force) + + def get_hooks_info(self): + return self._remote.get_hooks_info() + + def vcsserver_invalidate_cache(self, delete=False): + return self._remote.vcsserver_invalidate_cache(delete) + + +class BaseChangesetClass(type): def __instancecheck__(self, instance): return isinstance(instance, BaseCommit) class BaseChangeset(BaseCommit, metaclass=BaseChangesetClass): - def __new__(cls, *args, **kwargs): - warnings.warn( - "Use BaseCommit instead of BaseChangeset", DeprecationWarning) + warnings.warn("Use BaseCommit instead of BaseChangeset", DeprecationWarning) return super().__new__(cls, *args, **kwargs) @@ -1441,8 +1456,7 @@ class BaseInMemoryCommit(object): list of ``FileNode`` objects marked as *changed* ``removed`` - list of ``FileNode`` or ``RemovedFileNode`` objects marked to be - *removed* + list of ``FileNode`` objects marked to be *removed* ``parents`` list of :class:`BaseCommit` instances representing parents of @@ -1469,9 +1483,7 @@ class BaseInMemoryCommit(object): # Check if not already marked as *added* first for node in filenodes: if node.path in (n.path for n in self.added): - raise NodeAlreadyAddedError( - "Such FileNode %s is already marked for addition" - % node.path) + raise NodeAlreadyAddedError(f"Such FileNode {node.path} is already marked for addition") for node in filenodes: self.added.append(node) @@ -1490,24 +1502,19 @@ class BaseInMemoryCommit(object): """ for node in filenodes: if node.path in (n.path for n in self.removed): - raise NodeAlreadyRemovedError( - "Node at %s is already marked as removed" % node.path) + raise NodeAlreadyRemovedError("Node at %s is already marked as removed" % node.path) try: self.repository.get_commit() except EmptyRepositoryError: - raise EmptyRepositoryError( - "Nothing to change - try to *add* new nodes rather than " - "changing them") + raise EmptyRepositoryError("Nothing to change - try to *add* new nodes rather than " "changing them") for node in filenodes: if node.path in (n.path for n in self.changed): - raise NodeAlreadyChangedError( - "Node at '%s' is already marked as changed" % node.path) + raise NodeAlreadyChangedError("Node at '%s' is already marked as changed" % node.path) self.changed.append(node) def remove(self, *filenodes): """ - Marks given ``FileNode`` (or ``RemovedFileNode``) objects to be - *removed* in next commit. + Marks given ``FileNode`` objects to be *removed* in next commit. :raises ``NodeAlreadyRemovedError``: if node has been already marked to be *removed* @@ -1516,11 +1523,9 @@ class BaseInMemoryCommit(object): """ for node in filenodes: if node.path in (n.path for n in self.removed): - raise NodeAlreadyRemovedError( - "Node is already marked to for removal at %s" % node.path) + raise NodeAlreadyRemovedError("Node is already marked to for removal at %s" % node.path) if node.path in (n.path for n in self.changed): - raise NodeAlreadyChangedError( - "Node is already marked to be changed at %s" % node.path) + raise NodeAlreadyChangedError("Node is already marked to be changed at %s" % node.path) # We only mark node as *removed* - real removal is done by # commit method self.removed.append(node) @@ -1575,12 +1580,11 @@ class BaseInMemoryCommit(object): for p in parents: for node in self.added: try: - p.get_node(node.path) + p.get_node(node.bytes_path) except NodeDoesNotExistError: pass else: - raise NodeAlreadyExistsError( - f"Node `{node.path}` already exists at {p}") + raise NodeAlreadyExistsError(f"Node `{node.path}` already exists at {p}") # Check nodes marked as changed missing = set(self.changed) @@ -1590,7 +1594,7 @@ class BaseInMemoryCommit(object): for p in parents: for node in self.changed: try: - old = p.get_node(node.path) + old = p.get_node(node.bytes_path) missing.remove(node) # if content actually changed, remove node from not_changed if old.content != node.content: @@ -1598,24 +1602,23 @@ class BaseInMemoryCommit(object): except NodeDoesNotExistError: pass if self.changed and missing: - raise NodeDoesNotExistError( - f"Node `{node.path}` marked as modified but missing in parents: {parents}") + raise NodeDoesNotExistError(f"Node `{node.path}` marked as modified but missing in parents: {parents}") if self.changed and not_changed: raise NodeNotChangedError( - "Node `%s` wasn't actually changed (parents: %s)" - % (not_changed.pop().path, parents)) + "Node `%s` wasn't actually changed (parents: %s)" % (not_changed.pop().path, parents) + ) # Check nodes marked as removed if self.removed and not parents: raise NodeDoesNotExistError( - "Cannot remove node at %s as there " - "were no parents specified" % self.removed[0].path) + "Cannot remove node at %s as there " "were no parents specified" % self.removed[0].path + ) really_removed = set() for p in parents: for node in self.removed: try: - p.get_node(node.path) + p.get_node(node.bytes_path) really_removed.add(node) except CommitError: pass @@ -1623,8 +1626,8 @@ class BaseInMemoryCommit(object): if not_removed: # TODO: johbo: This code branch does not seem to be covered raise NodeDoesNotExistError( - "Cannot remove node at %s from " - "following parents: %s" % (not_removed, parents)) + "Cannot remove node at %s from " "following parents: %s" % (not_removed, parents) + ) def commit(self, message, author, parents=None, branch=None, date=None, **kwargs): """ @@ -1652,16 +1655,13 @@ class BaseInMemoryCommit(object): class BaseInMemoryChangesetClass(type): - def __instancecheck__(self, instance): return isinstance(instance, BaseInMemoryCommit) class BaseInMemoryChangeset(BaseInMemoryCommit, metaclass=BaseInMemoryChangesetClass): - def __new__(cls, *args, **kwargs): - warnings.warn( - "Use BaseCommit instead of BaseInMemoryCommit", DeprecationWarning) + warnings.warn("Use BaseCommit instead of BaseInMemoryCommit", DeprecationWarning) return super().__new__(cls, *args, **kwargs) @@ -1671,9 +1671,7 @@ class EmptyCommit(BaseCommit): an EmptyCommit """ - def __init__( - self, commit_id=EMPTY_COMMIT_ID, repo=None, alias=None, idx=-1, - message='', author='', date=None): + def __init__(self, commit_id=EMPTY_COMMIT_ID, repo=None, alias=None, idx=-1, message="", author="", date=None): self._empty_commit_id = commit_id # TODO: johbo: Solve idx parameter, default value does not make # too much sense @@ -1697,6 +1695,7 @@ class EmptyCommit(BaseCommit): def branch(self): if self.alias: from rhodecode.lib.vcs.backends import get_backend + return get_backend(self.alias).DEFAULT_BRANCH_NAME @LazyProperty @@ -1711,7 +1710,7 @@ class EmptyCommit(BaseCommit): return self def get_file_content(self, path) -> bytes: - return b'' + return b"" def get_file_content_streamed(self, path): yield self.get_file_content(path) @@ -1721,27 +1720,29 @@ class EmptyCommit(BaseCommit): class EmptyChangesetClass(type): - def __instancecheck__(self, instance): return isinstance(instance, EmptyCommit) class EmptyChangeset(EmptyCommit, metaclass=EmptyChangesetClass): - def __new__(cls, *args, **kwargs): - warnings.warn( - "Use EmptyCommit instead of EmptyChangeset", DeprecationWarning) + warnings.warn("Use EmptyCommit instead of EmptyChangeset", DeprecationWarning) return super(EmptyCommit, cls).__new__(cls, *args, **kwargs) - def __init__(self, cs=EMPTY_COMMIT_ID, repo=None, requested_revision=None, - alias=None, revision=-1, message='', author='', date=None): + def __init__( + self, + cs=EMPTY_COMMIT_ID, + repo=None, + requested_revision=None, + alias=None, + revision=-1, + message="", + author="", + date=None, + ): if requested_revision is not None: - warnings.warn( - "Parameter requested_revision not supported anymore", - DeprecationWarning) - super().__init__( - commit_id=cs, repo=repo, alias=alias, idx=revision, - message=message, author=author, date=date) + warnings.warn("Parameter requested_revision not supported anymore", DeprecationWarning) + super().__init__(commit_id=cs, repo=repo, alias=alias, idx=revision, message=message, author=author, date=date) @property def revision(self): @@ -1760,11 +1761,11 @@ class EmptyRepository(BaseRepository): def get_diff(self, *args, **kwargs): from rhodecode.lib.vcs.backends.git.diff import GitDiff - return GitDiff(b'') + + return GitDiff(b"") class CollectionGenerator(object): - def __init__(self, repo, commit_ids, collection_size=None, pre_load=None, translate_tag=None): self.repo = repo self.commit_ids = commit_ids @@ -1786,26 +1787,22 @@ class CollectionGenerator(object): """ Allows backends to override the way commits are generated. """ - return self.repo.get_commit( - commit_id=commit_id, pre_load=self.pre_load, - translate_tag=self.translate_tag) + return self.repo.get_commit(commit_id=commit_id, pre_load=self.pre_load, translate_tag=self.translate_tag) def __getitem__(self, key): """Return either a single element by index, or a sliced collection.""" if isinstance(key, slice): - commit_ids = self.commit_ids[key.start:key.stop] + commit_ids = self.commit_ids[key.start : key.stop] else: # single item commit_ids = self.commit_ids[key] - return self.__class__( - self.repo, commit_ids, pre_load=self.pre_load, - translate_tag=self.translate_tag) + return self.__class__(self.repo, commit_ids, pre_load=self.pre_load, translate_tag=self.translate_tag) def __repr__(self): - return '' % (self.__len__()) + return "" % (self.__len__()) class Config(object): @@ -1826,8 +1823,7 @@ class Config(object): return clone def __repr__(self): - return ''.format( - len(self._values), hex(id(self))) + return "".format(len(self._values), hex(id(self))) def items(self, section): return self._values.get(section, {}).items() @@ -1844,7 +1840,7 @@ class Config(object): def drop_option(self, section, option): if section not in self._values: - raise ValueError(f'Section {section} does not exist') + raise ValueError(f"Section {section} does not exist") del self._values[section][option] def serialize(self): @@ -1855,8 +1851,7 @@ class Config(object): items = [] for section in self._values: for option, value in self._values[section].items(): - items.append( - (safe_str(section), safe_str(option), safe_str(value))) + items.append((safe_str(section), safe_str(option), safe_str(value))) return items @@ -1867,12 +1862,13 @@ class Diff(object): Subclasses have to provide a backend specific value for :attr:`_header_re` and :attr:`_meta_re`. """ + _meta_re = None - _header_re: bytes = re.compile(br"") + _header_re: bytes = re.compile(rb"") def __init__(self, raw_diff: bytes): if not isinstance(raw_diff, bytes): - raise Exception(f'raw_diff must be bytes - got {type(raw_diff)}') + raise Exception(f"raw_diff must be bytes - got {type(raw_diff)}") self.raw = memoryview(raw_diff) @@ -1886,7 +1882,7 @@ class Diff(object): we can detect last chunk as this was also has special rule """ - diff_parts = (b'\n' + bytes(self.raw)).split(b'\ndiff --git') + diff_parts = (b"\n" + bytes(self.raw)).split(b"\ndiff --git") chunks = diff_parts[1:] total_chunks = len(chunks) @@ -1894,44 +1890,46 @@ class Diff(object): def diff_iter(_chunks): for cur_chunk, chunk in enumerate(_chunks, start=1): yield DiffChunk(chunk, self, cur_chunk == total_chunks) + return diff_iter(chunks) class DiffChunk(object): - def __init__(self, chunk: bytes, diff_obj: Diff, is_last_chunk: bool): self.diff_obj = diff_obj # since we split by \ndiff --git that part is lost from original diff # we need to re-apply it at the end, EXCEPT ! if it's last chunk if not is_last_chunk: - chunk += b'\n' + chunk += b"\n" header_re = self.diff_obj.get_header_re() + match = header_re.match(chunk) self.header = match.groupdict() - self.diff = chunk[match.end():] + self.diff = chunk[match.end() :] self.raw = chunk @property def header_as_str(self): if self.header: + def safe_str_on_bytes(val): if isinstance(val, bytes): return safe_str(val) return val + return {safe_str(k): safe_str_on_bytes(v) for k, v in self.header.items()} def __repr__(self): - return f'DiffChunk({self.header_as_str})' + return f"DiffChunk({self.header_as_str})" class BasePathPermissionChecker(object): - @staticmethod def create_from_patterns(includes, excludes): - if includes and '*' in includes and not excludes: + if includes and "*" in includes and not excludes: return AllPathPermissionChecker() - elif excludes and '*' in excludes: + elif excludes and "*" in excludes: return NonePathPermissionChecker() else: return PatternPathPermissionChecker(includes, excludes) @@ -1945,7 +1943,6 @@ class BasePathPermissionChecker(object): class AllPathPermissionChecker(BasePathPermissionChecker): - @property def has_full_access(self): return True @@ -1955,7 +1952,6 @@ class AllPathPermissionChecker(BasePathPermissionChecker): class NonePathPermissionChecker(BasePathPermissionChecker): - @property def has_full_access(self): return False @@ -1965,18 +1961,15 @@ class NonePathPermissionChecker(BasePathPermissionChecker): class PatternPathPermissionChecker(BasePathPermissionChecker): - def __init__(self, includes, excludes): self.includes = includes self.excludes = excludes - self.includes_re = [] if not includes else [ - re.compile(fnmatch.translate(pattern)) for pattern in includes] - self.excludes_re = [] if not excludes else [ - re.compile(fnmatch.translate(pattern)) for pattern in excludes] + self.includes_re = [] if not includes else [re.compile(fnmatch.translate(pattern)) for pattern in includes] + self.excludes_re = [] if not excludes else [re.compile(fnmatch.translate(pattern)) for pattern in excludes] @property def has_full_access(self): - return '*' in self.includes and not self.excludes + return "*" in self.includes and not self.excludes def has_access(self, path): for regex in self.excludes_re: diff --git a/rhodecode/lib/vcs/backends/git/commit.py b/rhodecode/lib/vcs/backends/git/commit.py index 89f90d98..2b658135 100644 --- a/rhodecode/lib/vcs/backends/git/commit.py +++ b/rhodecode/lib/vcs/backends/git/commit.py @@ -21,8 +21,8 @@ GIT commit module """ import io -import stat import configparser +import logging from itertools import chain from zope.cachedescriptors.property import Lazy as LazyProperty @@ -32,9 +32,16 @@ from rhodecode.lib.str_utils import safe_bytes, safe_str from rhodecode.lib.vcs.backends import base from rhodecode.lib.vcs.exceptions import CommitError, NodeDoesNotExistError from rhodecode.lib.vcs.nodes import ( - FileNode, DirNode, NodeKind, RootNode, SubModuleNode, - ChangedFileNodesGenerator, AddedFileNodesGenerator, - RemovedFileNodesGenerator, LargeFileNode) + FileNode, + DirNode, + NodeKind, + RootNode, + SubModuleNode, + LargeFileNode, +) +from rhodecode.lib.vcs_common import FILEMODE_LINK + +log = logging.getLogger(__name__) class GitCommit(base.BaseCommit): @@ -50,13 +57,11 @@ class GitCommit(base.BaseCommit): # done through a more complex tree walk on parents "status", # mercurial specific property not supported here - "_file_paths", + "obsolete", # mercurial specific property not supported here - 'obsolete', + "phase", # mercurial specific property not supported here - 'phase', - # mercurial specific property not supported here - 'hidden' + "hidden", ] def __init__(self, repository, raw_id, idx, pre_load=None): @@ -69,17 +74,16 @@ class GitCommit(base.BaseCommit): self._set_bulk_properties(pre_load) # caches - self._stat_modes = {} # stat info for paths - self._paths = {} # path processed with parse_tree self.nodes = {} + self._path_mode_cache = {} # path stats cache, e.g filemode etc + self._path_type_cache = {} # path type dir/file/link etc cache + self._submodules = None def _set_bulk_properties(self, pre_load): - if not pre_load: return - pre_load = [entry for entry in pre_load - if entry not in self._filter_pre_load] + pre_load = [entry for entry in pre_load if entry not in self._filter_pre_load] if not pre_load: return @@ -102,7 +106,7 @@ class GitCommit(base.BaseCommit): @LazyProperty def _tree_id(self): - return self._remote[self._commit['tree']]['id'] + return self._remote[self._commit["tree"]]["id"] @LazyProperty def id(self): @@ -134,13 +138,12 @@ class GitCommit(base.BaseCommit): """ Returns modified, added, removed, deleted files for current commit """ - return self.changed, self.added, self.removed + added, modified, deleted = self._changes_cache + return list(modified), list(modified), list(deleted) @LazyProperty def tags(self): - tags = [safe_str(name) for name, - commit_id in self.repository.tags.items() - if commit_id == self.raw_id] + tags = [safe_str(name) for name, commit_id in self.repository.tags.items() if commit_id == self.raw_id] return tags @LazyProperty @@ -161,47 +164,33 @@ class GitCommit(base.BaseCommit): branches = self._remote.branch(self.raw_id) return self._set_branch(branches) - def _get_tree_id_for_path(self, path): + def _get_path_tree_id_and_type(self, path: bytes): - path = safe_str(path) - if path in self._paths: - return self._paths[path] + if path in self._path_type_cache: + return self._path_type_cache[path] - tree_id = self._tree_id + if path == b"": + self._path_type_cache[b""] = [self._tree_id, NodeKind.DIR] + return self._path_type_cache[path] - path = path.strip('/') - if path == '': - data = [tree_id, "tree"] - self._paths[''] = data - return data - - tree_id, tree_type, tree_mode = \ - self._remote.tree_and_type_for_path(self.raw_id, path) + tree_id, tree_type, tree_mode = self._remote.tree_and_type_for_path(self.raw_id, path) if tree_id is None: raise self.no_node_at_path(path) - self._paths[path] = [tree_id, tree_type] - self._stat_modes[path] = tree_mode + self._path_type_cache[path] = [tree_id, tree_type] + self._path_mode_cache[path] = tree_mode - if path not in self._paths: - raise self.no_node_at_path(path) - - return self._paths[path] + return self._path_type_cache[path] def _get_kind(self, path): - tree_id, type_ = self._get_tree_id_for_path(path) - if type_ == 'blob': - return NodeKind.FILE - elif type_ == 'tree': - return NodeKind.DIR - elif type_ == 'link': - return NodeKind.SUBMODULE - return None + path = self._fix_path(path) + _, path_type = self._get_path_tree_id_and_type(path) + return path_type def _assert_is_path(self, path): path = self._fix_path(path) if self._get_kind(path) != NodeKind.FILE: - raise CommitError(f"File does not exist for commit {self.raw_id} at '{path}'") + raise CommitError(f"File at path={path} does not exist for commit {self.raw_id}") return path def _get_file_nodes(self): @@ -237,15 +226,19 @@ class GitCommit(base.BaseCommit): path = self._assert_is_path(path) # ensure path is traversed - self._get_tree_id_for_path(path) + self._get_path_tree_id_and_type(path) - return self._stat_modes[path] + return self._path_mode_cache[path] - def is_link(self, path): - return stat.S_ISLNK(self.get_file_mode(path)) + def is_link(self, path: bytes): + path = self._assert_is_path(path) + if path not in self._path_mode_cache: + self._path_mode_cache[path] = self._remote.fctx_flags(self.raw_id, path) + + return self._path_mode_cache[path] == FILEMODE_LINK def is_node_binary(self, path): - tree_id, _ = self._get_tree_id_for_path(path) + tree_id, _ = self._get_path_tree_id_and_type(path) return self._remote.is_binary(tree_id) def node_md5_hash(self, path): @@ -256,19 +249,19 @@ class GitCommit(base.BaseCommit): """ Returns content of the file at given `path`. """ - tree_id, _ = self._get_tree_id_for_path(path) + tree_id, _ = self._get_path_tree_id_and_type(path) return self._remote.blob_as_pretty_string(tree_id) def get_file_content_streamed(self, path): - tree_id, _ = self._get_tree_id_for_path(path) - stream_method = getattr(self._remote, 'stream:blob_as_pretty_string') + tree_id, _ = self._get_path_tree_id_and_type(path) + stream_method = getattr(self._remote, "stream:blob_as_pretty_string") return stream_method(tree_id) def get_file_size(self, path): """ Returns size of the file at given `path`. """ - tree_id, _ = self._get_tree_id_for_path(path) + tree_id, _ = self._get_path_tree_id_and_type(path) return self._remote.blob_raw_length(tree_id) def get_path_history(self, path, limit=None, pre_load=None): @@ -276,12 +269,9 @@ class GitCommit(base.BaseCommit): Returns history of file as reversed list of `GitCommit` objects for which file at given `path` has been modified. """ - path = self._assert_is_path(path) - hist = self._remote.node_history(self.raw_id, path, limit) - return [ - self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) - for commit_id in hist] + history = self._remote.node_history(self.raw_id, path, limit) + return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) for commit_id in history] def get_file_annotate(self, path, pre_load=None): """ @@ -293,95 +283,105 @@ class GitCommit(base.BaseCommit): for ln_no, commit_id, content in result: yield ( - ln_no, commit_id, + ln_no, + commit_id, lambda: self.repository.get_commit(commit_id=commit_id, pre_load=pre_load), - content) + content, + ) - def get_nodes(self, path, pre_load=None): + def get_nodes(self, path: bytes, pre_load=None): if self._get_kind(path) != NodeKind.DIR: - raise CommitError( - f"Directory does not exist for commit {self.raw_id} at '{path}'") + raise CommitError(f"Directory does not exist for commit {self.raw_id} at '{path}'") path = self._fix_path(path) - tree_id, _ = self._get_tree_id_for_path(path) + # call and check tree_id for this path + tree_id, _ = self._get_path_tree_id_and_type(path) - dirnodes = [] - filenodes = [] + path_nodes = [] - # extracted tree ID gives us our files... - str_path = safe_str(path) # libgit operates on bytes - for name, stat_, id_, type_ in self._remote.tree_items(tree_id): - if type_ == 'link': - url = self._get_submodule_url('/'.join((str_path, name))) - dirnodes.append(SubModuleNode( - name, url=url, commit=id_, alias=self.repository.alias)) - continue + for bytes_name, stat_, tree_item_id, node_kind in self._remote.tree_items(tree_id): + if node_kind is None: + raise CommitError(f"Requested object type={node_kind} cannot be determined") - if str_path != '': - obj_path = '/'.join((str_path, name)) + if path != b"": + obj_path = b"/".join((path, bytes_name)) else: - obj_path = name - if obj_path not in self._stat_modes: - self._stat_modes[obj_path] = stat_ + obj_path = bytes_name - if type_ == 'tree': - dirnodes.append(DirNode(safe_bytes(obj_path), commit=self)) - elif type_ == 'blob': - filenodes.append(FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load)) + # cache file mode for git, since we have it already + if obj_path not in self._path_mode_cache: + self._path_mode_cache[obj_path] = stat_ + + # cache type + if node_kind not in self._path_type_cache: + self._path_type_cache[obj_path] = [tree_item_id, node_kind] + + entry = None + if obj_path in self.nodes: + entry = self.nodes[obj_path] else: - raise CommitError(f"Requested object should be Tree or Blob, is {type_}") + if node_kind == NodeKind.SUBMODULE: + url = self._get_submodule_url(b"/".join((path, bytes_name))) + entry= SubModuleNode(bytes_name, url=url, commit=tree_item_id, alias=self.repository.alias) + elif node_kind == NodeKind.DIR: + entry = DirNode(safe_bytes(obj_path), commit=self) + elif node_kind == NodeKind.FILE: + entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load) - nodes = dirnodes + filenodes - for node in nodes: - if node.path not in self.nodes: - self.nodes[node.path] = node - nodes.sort() - return nodes + if entry: + self.nodes[obj_path] = entry + path_nodes.append(entry) - def get_node(self, path, pre_load=None): + path_nodes.sort() + return path_nodes + + def get_node(self, path: bytes, pre_load=None): path = self._fix_path(path) - if path not in self.nodes: - try: - tree_id, type_ = self._get_tree_id_for_path(path) - except CommitError: - raise NodeDoesNotExistError( - f"Cannot find one of parents' directories for a given " - f"path: {path}") - if type_ in ['link', 'commit']: + # use cached, if we have one + if path in self.nodes: + return self.nodes[path] + + try: + tree_id, path_type = self._get_path_tree_id_and_type(path) + except CommitError: + raise NodeDoesNotExistError(f"Cannot find one of parents' directories for a given path: {path}") + + if path == b"": + node = RootNode(commit=self) + else: + if path_type == NodeKind.SUBMODULE: url = self._get_submodule_url(path) - node = SubModuleNode(path, url=url, commit=tree_id, - alias=self.repository.alias) - elif type_ == 'tree': - if path == '': - node = RootNode(commit=self) - else: - node = DirNode(safe_bytes(path), commit=self) - elif type_ == 'blob': + node = SubModuleNode(path, url=url, commit=tree_id, alias=self.repository.alias) + elif path_type == NodeKind.DIR: + node = DirNode(safe_bytes(path), commit=self) + elif path_type == NodeKind.FILE: node = FileNode(safe_bytes(path), commit=self, pre_load=pre_load) - self._stat_modes[path] = node.mode + self._path_mode_cache[path] = node.mode else: raise self.no_node_at_path(path) - # cache node - self.nodes[path] = node - + # cache node + self.nodes[path] = node return self.nodes[path] - def get_largefile_node(self, path): - tree_id, _ = self._get_tree_id_for_path(path) + def get_largefile_node(self, path: bytes): + tree_id, _ = self._get_path_tree_id_and_type(path) pointer_spec = self._remote.is_large_file(tree_id) if pointer_spec: # content of that file regular FileNode is the hash of largefile - file_id = pointer_spec.get('oid_hash') - if self._remote.in_largefiles_store(file_id): - lf_path = self._remote.store_path(file_id) - return LargeFileNode(safe_bytes(lf_path), commit=self, org_path=path) + file_id = pointer_spec.get("oid_hash") + if not self._remote.in_largefiles_store(file_id): + log.warning(f'Largefile oid={file_id} not found in store') + return None + + lf_path = self._remote.store_path(file_id) + return LargeFileNode(safe_bytes(lf_path), commit=self, org_path=path) @LazyProperty - def affected_files(self): + def affected_files(self) -> list[bytes]: """ Gets a fast accessible file changes for given commit """ @@ -389,7 +389,7 @@ class GitCommit(base.BaseCommit): return list(added.union(modified).union(deleted)) @LazyProperty - def _changes_cache(self): + def _changes_cache(self) -> tuple[set, set, set]: added = set() modified = set() deleted = set() @@ -416,53 +416,22 @@ class GitCommit(base.BaseCommit): :param status: one of: *added*, *modified* or *deleted* """ added, modified, deleted = self._changes_cache - return sorted({ - 'added': list(added), - 'modified': list(modified), - 'deleted': list(deleted)}[status] - ) - - @LazyProperty - def added(self): - """ - Returns list of added ``FileNode`` objects. - """ - if not self.parents: - return list(self._get_file_nodes()) - return AddedFileNodesGenerator(self.added_paths, self) + return sorted({"added": list(added), "modified": list(modified), "deleted": list(deleted)}[status]) @LazyProperty def added_paths(self): - return [n for n in self._get_paths_for_status('added')] - - @LazyProperty - def changed(self): - """ - Returns list of modified ``FileNode`` objects. - """ - if not self.parents: - return [] - return ChangedFileNodesGenerator(self.changed_paths, self) + return [n for n in self._get_paths_for_status("added")] @LazyProperty def changed_paths(self): - return [n for n in self._get_paths_for_status('modified')] - - @LazyProperty - def removed(self): - """ - Returns list of removed ``FileNode`` objects. - """ - if not self.parents: - return [] - return RemovedFileNodesGenerator(self.removed_paths, self) + return [n for n in self._get_paths_for_status("modified")] @LazyProperty def removed_paths(self): - return [n for n in self._get_paths_for_status('deleted')] + return [n for n in self._get_paths_for_status("deleted")] - def _get_submodule_url(self, submodule_path): - git_modules_path = '.gitmodules' + def _get_submodule_url(self, submodule_path: bytes): + git_modules_path = b".gitmodules" if self._submodules is None: self._submodules = {} @@ -476,9 +445,9 @@ class GitCommit(base.BaseCommit): parser.read_file(io.StringIO(submodules_node.str_content)) for section in parser.sections(): - path = parser.get(section, 'path') - url = parser.get(section, 'url') + path = parser.get(section, "path") + url = parser.get(section, "url") if path and url: - self._submodules[path.strip('/')] = url + self._submodules[safe_bytes(path).strip(b"/")] = url - return self._submodules.get(submodule_path.strip('/')) + return self._submodules.get(submodule_path.strip(b"/")) diff --git a/rhodecode/lib/vcs/backends/git/repository.py b/rhodecode/lib/vcs/backends/git/repository.py index c633ccb7..f3c4c4d7 100644 --- a/rhodecode/lib/vcs/backends/git/repository.py +++ b/rhodecode/lib/vcs/backends/git/repository.py @@ -425,7 +425,7 @@ class GitRepository(BaseRepository): return def get_commit(self, commit_id=None, commit_idx=None, pre_load=None, - translate_tag=True, maybe_unreachable=False, reference_obj=None): + translate_tag=True, maybe_unreachable=False, reference_obj=None) -> GitCommit: """ Returns `GitCommit` object representing commit from git repository at the given `commit_id` or head (most recent commit) if None given. diff --git a/rhodecode/lib/vcs/backends/hg/commit.py b/rhodecode/lib/vcs/backends/hg/commit.py index d79339c2..03b5bfd9 100644 --- a/rhodecode/lib/vcs/backends/hg/commit.py +++ b/rhodecode/lib/vcs/backends/hg/commit.py @@ -20,20 +20,25 @@ HG commit module """ -import os +import logging from zope.cachedescriptors.property import Lazy as LazyProperty from rhodecode.lib.datelib import utcdate_fromtimestamp from rhodecode.lib.str_utils import safe_bytes, safe_str -from rhodecode.lib.vcs import path as vcspath from rhodecode.lib.vcs.backends import base from rhodecode.lib.vcs.exceptions import CommitError from rhodecode.lib.vcs.nodes import ( - AddedFileNodesGenerator, ChangedFileNodesGenerator, DirNode, FileNode, - NodeKind, RemovedFileNodesGenerator, RootNode, SubModuleNode, - LargeFileNode) -from rhodecode.lib.vcs.utils.paths import get_dirs_for_path + DirNode, + FileNode, + NodeKind, + RootNode, + SubModuleNode, + LargeFileNode, +) +from rhodecode.lib.vcs_common import FILEMODE_LINK + +log = logging.getLogger(__name__) class MercurialCommit(base.BaseCommit): @@ -59,13 +64,13 @@ class MercurialCommit(base.BaseCommit): # caches self.nodes = {} - self._stat_modes = {} # stat info for paths + self._path_mode_cache = {} # path stats cache, e.g filemode etc + self._path_type_cache = {} # path type dir/file/link etc cache def _set_bulk_properties(self, pre_load): if not pre_load: return - pre_load = [entry for entry in pre_load - if entry not in self._filter_pre_load] + pre_load = [entry for entry in pre_load if entry not in self._filter_pre_load] if not pre_load: return @@ -86,8 +91,7 @@ class MercurialCommit(base.BaseCommit): @LazyProperty def tags(self): - tags = [name for name, commit_id in self.repository.tags.items() - if commit_id == self.raw_id] + tags = [name for name, commit_id in self.repository.tags.items() if commit_id == self.raw_id] return tags @LazyProperty @@ -96,9 +100,7 @@ class MercurialCommit(base.BaseCommit): @LazyProperty def bookmarks(self): - bookmarks = [ - name for name, commit_id in self.repository.bookmarks.items() - if commit_id == self.raw_id] + bookmarks = [name for name, commit_id in self.repository.bookmarks.items() if commit_id == self.raw_id] return bookmarks @LazyProperty @@ -122,27 +124,13 @@ class MercurialCommit(base.BaseCommit): """ Returns modified, added, removed, deleted files for current commit """ - return self._remote.ctx_status(self.raw_id) - - @LazyProperty - def _file_paths(self): - return self._remote.ctx_list(self.raw_id) - - @LazyProperty - def _dir_paths(self): - dir_paths = [''] - dir_paths.extend(list(set(get_dirs_for_path(*self._file_paths)))) - - return dir_paths - - @LazyProperty - def _paths(self): - return self._dir_paths + self._file_paths + modified, added, deleted, *_ = self._remote.ctx_status(self.raw_id) + return modified, added, deleted @LazyProperty def id(self): if self.last: - return 'tip' + return "tip" return self.short_id @LazyProperty @@ -150,8 +138,7 @@ class MercurialCommit(base.BaseCommit): return self.raw_id[:12] def _make_commits(self, commit_ids, pre_load=None): - return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) - for commit_id in commit_ids] + return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) for commit_id in commit_ids] @LazyProperty def parents(self): @@ -163,10 +150,10 @@ class MercurialCommit(base.BaseCommit): def _get_phase_text(self, phase_id): return { - 0: 'public', - 1: 'draft', - 2: 'secret', - }.get(phase_id) or '' + 0: "public", + 1: "draft", + 2: "secret", + }.get(phase_id) or "" @LazyProperty def phase(self): @@ -195,17 +182,14 @@ class MercurialCommit(base.BaseCommit): def _get_kind(self, path): path = self._fix_path(path) - if path in self._file_paths: - return NodeKind.FILE - elif path in self._dir_paths: - return NodeKind.DIR - else: - raise CommitError(f"Node does not exist at the given path '{path}'") + path_type = self._get_path_type(path) + return path_type - def _assert_is_path(self, path) -> str: + def _assert_is_path(self, path) -> str | bytes: path = self._fix_path(path) + if self._get_kind(path) != NodeKind.FILE: - raise CommitError(f"File does not exist for commit {self.raw_id} at '{path}'") + raise CommitError(f"File at path={path} does not exist for commit {self.raw_id}") return path @@ -214,20 +198,17 @@ class MercurialCommit(base.BaseCommit): Returns stat mode of the file at the given ``path``. """ path = self._assert_is_path(path) + if path not in self._path_mode_cache: + self._path_mode_cache[path] = self._remote.fctx_flags(self.raw_id, path) - if path not in self._stat_modes: - self._stat_modes[path] = self._remote.fctx_flags(self.raw_id, path) + return self._path_mode_cache[path] - if 'x' in self._stat_modes[path]: - return base.FILEMODE_EXECUTABLE - return base.FILEMODE_DEFAULT - - def is_link(self, path): + def is_link(self, path: bytes): path = self._assert_is_path(path) - if path not in self._stat_modes: - self._stat_modes[path] = self._remote.fctx_flags(self.raw_id, path) + if path not in self._path_mode_cache: + self._path_mode_cache[path] = self._remote.fctx_flags(self.raw_id, path) - return 'l' in self._stat_modes[path] + return self._path_mode_cache[path] == FILEMODE_LINK def is_node_binary(self, path): path = self._assert_is_path(path) @@ -246,7 +227,7 @@ class MercurialCommit(base.BaseCommit): def get_file_content_streamed(self, path): path = self._assert_is_path(path) - stream_method = getattr(self._remote, 'stream:fctx_node_data') + stream_method = getattr(self._remote, "stream:fctx_node_data") return stream_method(self.raw_id, path) def get_file_size(self, path): @@ -262,10 +243,8 @@ class MercurialCommit(base.BaseCommit): for which file at given ``path`` has been modified. """ path = self._assert_is_path(path) - hist = self._remote.node_history(self.raw_id, path, limit) - return [ - self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) - for commit_id in hist] + history = self._remote.node_history(self.raw_id, path, limit) + return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) for commit_id in history] def get_file_annotate(self, path, pre_load=None): """ @@ -276,11 +255,13 @@ class MercurialCommit(base.BaseCommit): for ln_no, commit_id, content in result: yield ( - ln_no, commit_id, + ln_no, + commit_id, lambda: self.repository.get_commit(commit_id=commit_id, pre_load=pre_load), - content) + content, + ) - def get_nodes(self, path, pre_load=None): + def get_nodes(self, path: bytes, pre_load=None): """ Returns combined ``DirNode`` and ``FileNode`` objects list representing state of commit at the given ``path``. If node at the given ``path`` @@ -288,59 +269,86 @@ class MercurialCommit(base.BaseCommit): """ if self._get_kind(path) != NodeKind.DIR: - raise CommitError( - f"Directory does not exist for idx {self.raw_id} at '{path}'") + raise CommitError(f"Directory does not exist for idx {self.raw_id} at '{path}'") path = self._fix_path(path) - filenodes = [ - FileNode(safe_bytes(f), commit=self, pre_load=pre_load) for f in self._file_paths - if os.path.dirname(f) == path] - # TODO: johbo: Check if this can be done in a more obvious way - dirs = path == '' and '' or [ - d for d in self._dir_paths - if d and vcspath.dirname(d) == path] - dirnodes = [ - DirNode(safe_bytes(d), commit=self) for d in dirs - if os.path.dirname(d) == path] + path_nodes = [] - alias = self.repository.alias - for k, vals in self._submodules.items(): - if vcspath.dirname(k) == path: - loc = vals[0] - commit = vals[1] - dirnodes.append(SubModuleNode(k, url=loc, commit=commit, alias=alias)) + for obj_path, node_kind in self._remote.dir_items(self.raw_id, path): - nodes = dirnodes + filenodes - for node in nodes: - if node.path not in self.nodes: - self.nodes[node.path] = node - nodes.sort() + if node_kind is None: + raise CommitError(f"Requested object type={node_kind} cannot be mapped to a proper type") - return nodes + # TODO: implement it ?? + stat_ = None + # # cache file mode + # if obj_path not in self._path_mode_cache: + # self._path_mode_cache[obj_path] = stat_ - def get_node(self, path, pre_load=None): + # cache type + if node_kind not in self._path_type_cache: + self._path_type_cache[obj_path] = node_kind + + entry = None + if obj_path in self.nodes: + entry = self.nodes[obj_path] + else: + if node_kind == NodeKind.DIR: + entry = DirNode(safe_bytes(obj_path), commit=self) + elif node_kind == NodeKind.FILE: + entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load) + if entry: + self.nodes[obj_path] = entry + path_nodes.append(entry) + + path_nodes.sort() + return path_nodes + + def get_node(self, path: bytes, pre_load=None): """ Returns `Node` object from the given `path`. If there is no node at the given `path`, `NodeDoesNotExistError` would be raised. """ path = self._fix_path(path) - if path not in self.nodes: - if path in self._file_paths: + # use cached, if we have one + if path in self.nodes: + return self.nodes[path] + + path_type = self._get_path_type(path) + if path == b"": + node = RootNode(commit=self) + else: + if path_type == NodeKind.DIR: + node = DirNode(safe_bytes(path), commit=self) + elif path_type == NodeKind.FILE: node = FileNode(safe_bytes(path), commit=self, pre_load=pre_load) - elif path in self._dir_paths: - if path == '': - node = RootNode(commit=self) - else: - node = DirNode(safe_bytes(path), commit=self) + self._path_mode_cache[path] = node.mode else: raise self.no_node_at_path(path) - - # cache node - self.nodes[path] = node + # cache node + self.nodes[path] = node return self.nodes[path] - def get_largefile_node(self, path): + def _get_path_type(self, path: bytes): + if path in self._path_type_cache: + return self._path_type_cache[path] + + if path == b"": + self._path_type_cache[b""] = NodeKind.DIR + return NodeKind.DIR + + path_type, flags = self._remote.get_path_type(self.raw_id, path) + + if not path_type: + raise self.no_node_at_path(path) + + self._path_type_cache[path] = path_type + self._path_mode_cache[path] = flags + + return self._path_type_cache[path] + + def get_largefile_node(self, path: bytes): pointer_spec = self._remote.is_large_file(self.raw_id, path) if pointer_spec: # content of that file regular FileNode is the hash of largefile @@ -363,40 +371,20 @@ class MercurialCommit(base.BaseCommit): return self._remote.ctx_substate(self.raw_id) @LazyProperty - def affected_files(self): + def affected_files(self) -> list[bytes]: """ Gets a fast accessible file changes for given commit """ return self._remote.ctx_files(self.raw_id) - @property - def added(self): - """ - Returns list of added ``FileNode`` objects. - """ - return AddedFileNodesGenerator(self.added_paths, self) - @LazyProperty def added_paths(self): return [n for n in self.status[1]] - @property - def changed(self): - """ - Returns list of modified ``FileNode`` objects. - """ - return ChangedFileNodesGenerator(self.changed_paths, self) - @LazyProperty def changed_paths(self): return [n for n in self.status[0]] - @property - def removed(self): - """ - Returns list of removed ``FileNode`` objects. - """ - return RemovedFileNodesGenerator(self.removed_paths, self) @LazyProperty def removed_paths(self): diff --git a/rhodecode/lib/vcs/backends/hg/repository.py b/rhodecode/lib/vcs/backends/hg/repository.py index dbe4f612..ecd65ac2 100644 --- a/rhodecode/lib/vcs/backends/hg/repository.py +++ b/rhodecode/lib/vcs/backends/hg/repository.py @@ -450,7 +450,7 @@ class MercurialRepository(BaseRepository): return os.path.join(self.path, '.hg', '.hgrc') def get_commit(self, commit_id=None, commit_idx=None, pre_load=None, - translate_tag=None, maybe_unreachable=False, reference_obj=None): + translate_tag=None, maybe_unreachable=False, reference_obj=None) -> MercurialCommit: """ Returns ``MercurialCommit`` object representing repository's commit at the given `commit_id` or `commit_idx`. @@ -598,8 +598,7 @@ class MercurialRepository(BaseRepository): """ Create a local clone of the current repo. """ - self._remote.clone(self.path, clone_path, update_after_clone=True, - hooks=False) + self._remote.clone(self.path, clone_path, update_after_clone=True, hooks=False) def _update(self, revision, clean=False): """ diff --git a/rhodecode/lib/vcs/backends/svn/commit.py b/rhodecode/lib/vcs/backends/svn/commit.py index c7fbec84..4d26e731 100644 --- a/rhodecode/lib/vcs/backends/svn/commit.py +++ b/rhodecode/lib/vcs/backends/svn/commit.py @@ -19,8 +19,7 @@ """ SVN commit module """ - - +import logging import dateutil.parser from zope.cachedescriptors.property import Lazy as LazyProperty @@ -28,9 +27,10 @@ from rhodecode.lib.str_utils import safe_bytes, safe_str from rhodecode.lib.vcs import nodes, path as vcspath from rhodecode.lib.vcs.backends import base from rhodecode.lib.vcs.exceptions import CommitError +from vcsserver.lib.vcs_common import NodeKind, FILEMODE_EXECUTABLE, FILEMODE_DEFAULT, FILEMODE_LINK +_SVN_PROP_TRUE = "*" - -_SVN_PROP_TRUE = '*' +log = logging.getLogger(__name__) class SubversionCommit(base.BaseCommit): @@ -53,15 +53,16 @@ class SubversionCommit(base.BaseCommit): # which knows how to translate commit index and commit id self.raw_id = commit_id self.short_id = commit_id - self.id = f'r{commit_id}' + self.id = f"r{commit_id}" - # TODO: Implement the following placeholder attributes self.nodes = {} + self._path_mode_cache = {} # path stats cache, e.g filemode etc + self._path_type_cache = {} # path type dir/file/link etc cache self.tags = [] @property def author(self): - return safe_str(self._properties.get('svn:author')) + return safe_str(self._properties.get("svn:author")) @property def date(self): @@ -69,7 +70,7 @@ class SubversionCommit(base.BaseCommit): @property def message(self): - return safe_str(self._properties.get('svn:log')) + return safe_str(self._properties.get("svn:log")) @LazyProperty def _properties(self): @@ -91,19 +92,46 @@ class SubversionCommit(base.BaseCommit): return [child] return [] - def get_file_mode(self, path: bytes): + def _calculate_file_mode(self, path: bytes): # Note: Subversion flags files which are executable with a special # property `svn:executable` which is set to the value ``"*"``. - if self._get_file_property(path, 'svn:executable') == _SVN_PROP_TRUE: - return base.FILEMODE_EXECUTABLE + if self._get_file_property(path, "svn:executable") == _SVN_PROP_TRUE: + return FILEMODE_EXECUTABLE else: - return base.FILEMODE_DEFAULT + return FILEMODE_DEFAULT - def is_link(self, path): + def get_file_mode(self, path: bytes): + path = self._fix_path(path) + + if path not in self._path_mode_cache: + self._path_mode_cache[path] = self._calculate_file_mode(path) + + return self._path_mode_cache[path] + + def _get_path_type(self, path: bytes): + if path in self._path_type_cache: + return self._path_type_cache[path] + + if path == b"": + self._path_type_cache[b""] = NodeKind.DIR + return NodeKind.DIR + + path_type = self._remote.get_node_type(self._svn_rev, path) + + if not path_type: + raise self.no_node_at_path(path) + + #flags = None + self._path_type_cache[path] = path_type + #self._path_mode_cache[path] = flags + + return self._path_type_cache[path] + + def is_link(self, path: bytes): # Note: Subversion has a flag for special files, the content of the # file contains the type of that file. - if self._get_file_property(path, 'svn:special') == _SVN_PROP_TRUE: - return self.get_file_content(path).startswith(b'link') + if self._get_file_property(path, "svn:special") == _SVN_PROP_TRUE: + return self.get_file_content(path).startswith(b"link") return False def is_node_binary(self, path): @@ -115,8 +143,7 @@ class SubversionCommit(base.BaseCommit): return self._remote.md5_hash(self._svn_rev, safe_str(path)) def _get_file_property(self, path, name): - file_properties = self._remote.node_properties( - safe_str(path), self._svn_rev) + file_properties = self._remote.node_properties(safe_str(path), self._svn_rev) return file_properties.get(name) def get_file_content(self, path): @@ -126,7 +153,7 @@ class SubversionCommit(base.BaseCommit): def get_file_content_streamed(self, path): path = self._fix_path(path) - stream_method = getattr(self._remote, 'stream:get_file_content') + stream_method = getattr(self._remote, "stream:get_file_content") return stream_method(self._svn_rev, safe_str(path)) def get_file_size(self, path): @@ -134,11 +161,9 @@ class SubversionCommit(base.BaseCommit): return self._remote.get_file_size(self._svn_rev, safe_str(path)) def get_path_history(self, path, limit=None, pre_load=None): - path = safe_str(self._fix_path(path)) - history = self._remote.node_history(path, self._svn_rev, limit) - return [ - self.repository.get_commit(commit_id=str(svn_rev)) - for svn_rev in history] + path = self._fix_path(path) + history = self._remote.node_history(self._svn_rev, safe_str(path), limit) + return [self.repository.get_commit(commit_id=str(svn_rev)) for svn_rev in history] def get_file_annotate(self, path, pre_load=None): result = self._remote.file_annotate(safe_str(path), self._svn_rev) @@ -146,67 +171,78 @@ class SubversionCommit(base.BaseCommit): for zero_based_line_no, svn_rev, content in result: commit_id = str(svn_rev) line_no = zero_based_line_no + 1 - yield ( - line_no, - commit_id, - lambda: self.repository.get_commit(commit_id=commit_id), - content) + yield line_no, commit_id, lambda: self.repository.get_commit(commit_id=commit_id), content - def get_node(self, path, pre_load=None): + def get_node(self, path: bytes, pre_load=None): path = self._fix_path(path) - if path not in self.nodes: - if path == '': - node = nodes.RootNode(commit=self) + # use cached, if we have one + if path in self.nodes: + return self.nodes[path] + + path_type = self._get_path_type(path) + if path == b"": + node = nodes.RootNode(commit=self) + else: + if path_type == NodeKind.DIR: + node = nodes.DirNode(safe_bytes(path), commit=self) + elif path_type == NodeKind.FILE: + node = nodes.FileNode(safe_bytes(path), commit=self, pre_load=pre_load) + self._path_mode_cache[path] = node.mode else: - node_type = self._remote.get_node_type(self._svn_rev, safe_str(path)) - if node_type == 'dir': - node = nodes.DirNode(safe_bytes(path), commit=self) - elif node_type == 'file': - node = nodes.FileNode(safe_bytes(path), commit=self, pre_load=pre_load) - else: - raise self.no_node_at_path(path) + raise self.no_node_at_path(path) - self.nodes[path] = node + self.nodes[path] = node return self.nodes[path] - def get_nodes(self, path, pre_load=None): + def get_nodes(self, path: bytes, pre_load=None): if self._get_kind(path) != nodes.NodeKind.DIR: - raise CommitError( - f"Directory does not exist for commit {self.raw_id} at '{path}'") - path = safe_str(self._fix_path(path)) + raise CommitError(f"Directory does not exist for commit {self.raw_id} at '{path}'") + path = self._fix_path(path) path_nodes = [] - for name, kind in self._remote.get_nodes(self._svn_rev, path): - node_path = vcspath.join(path, name) - if kind == 'dir': - node = nodes.DirNode(safe_bytes(node_path), commit=self) - elif kind == 'file': - node = nodes.FileNode(safe_bytes(node_path), commit=self, pre_load=pre_load) - else: - raise ValueError(f"Node kind {kind} not supported.") - self.nodes[node_path] = node - path_nodes.append(node) + for name, node_kind in self._remote.get_nodes(self._svn_rev, path): + obj_path = vcspath.join(path, name) + if node_kind is None: + raise CommitError(f"Requested object type={node_kind} cannot be determined") + + # TODO: implement it ?? + stat_ = None + # # cache file mode + # if obj_path not in self._path_mode_cache: + # self._path_mode_cache[obj_path] = stat_ + + # cache type + if node_kind not in self._path_type_cache: + self._path_type_cache[obj_path] = node_kind + + entry = None + if obj_path in self.nodes: + entry = self.nodes[obj_path] + else: + if node_kind == NodeKind.DIR: + entry = nodes.DirNode(safe_bytes(obj_path), commit=self) + elif node_kind == NodeKind.FILE: + entry = nodes.FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load) + if entry: + self.nodes[obj_path] = entry + path_nodes.append(entry) + + path_nodes.sort() return path_nodes def _get_kind(self, path): path = self._fix_path(path) - kind = self._remote.get_node_type(self._svn_rev, path) - if kind == 'file': - return nodes.NodeKind.FILE - elif kind == 'dir': - return nodes.NodeKind.DIR - else: - raise CommitError( - f"Node does not exist at the given path '{path}'") + path_type = self._get_path_type(path) + return path_type @LazyProperty def _changes_cache(self): return self._remote.revision_changes(self._svn_rev) @LazyProperty - def affected_files(self): + def affected_files(self) -> list[bytes]: changed_files = set() for files in self._changes_cache.values(): changed_files.update(files) @@ -216,29 +252,17 @@ class SubversionCommit(base.BaseCommit): def id(self): return self.raw_id - @property - def added(self): - return nodes.AddedFileNodesGenerator(self.added_paths, self) - @LazyProperty def added_paths(self): - return [n for n in self._changes_cache['added']] - - @property - def changed(self): - return nodes.ChangedFileNodesGenerator(self.changed_paths, self) + return [n for n in self._changes_cache["added"]] @LazyProperty def changed_paths(self): - return [n for n in self._changes_cache['changed']] - - @property - def removed(self): - return nodes.RemovedFileNodesGenerator(self.removed_paths, self) + return [n for n in self._changes_cache["changed"]] @LazyProperty def removed_paths(self): - return [n for n in self._changes_cache['removed']] + return [n for n in self._changes_cache["removed"]] def _date_from_svn_properties(properties): @@ -248,7 +272,7 @@ def _date_from_svn_properties(properties): :return: :class:`datetime.datetime` instance. The object is naive. """ - aware_date = dateutil.parser.parse(properties.get('svn:date')) + aware_date = dateutil.parser.parse(properties.get("svn:date")) # final_date = aware_date.astimezone(dateutil.tz.tzlocal()) final_date = aware_date return final_date.replace(tzinfo=None) diff --git a/rhodecode/lib/vcs/backends/svn/repository.py b/rhodecode/lib/vcs/backends/svn/repository.py index 7ec4e9d3..97e91066 100644 --- a/rhodecode/lib/vcs/backends/svn/repository.py +++ b/rhodecode/lib/vcs/backends/svn/repository.py @@ -30,7 +30,7 @@ from zope.cachedescriptors.property import Lazy as LazyProperty from collections import OrderedDict from rhodecode.lib.datelib import date_astimestamp -from rhodecode.lib.str_utils import safe_str +from rhodecode.lib.str_utils import safe_str, safe_bytes from rhodecode.lib.utils2 import CachedProperty from rhodecode.lib.vcs import connection, path as vcspath from rhodecode.lib.vcs.backends import base @@ -157,16 +157,18 @@ class SubversionRepository(base.BaseRepository): for pattern in self._patterns_from_section(config_section): pattern = vcspath.sanitize(pattern) + bytes_pattern = safe_bytes(pattern) + tip = self.get_commit() try: - if pattern.endswith('*'): - basedir = tip.get_node(vcspath.dirname(pattern)) + if bytes_pattern.endswith(b'*'): + basedir = tip.get_node(vcspath.dirname(bytes_pattern)) directories = basedir.dirs else: - directories = (tip.get_node(pattern), ) + directories = (tip.get_node(bytes_pattern), ) except NodeDoesNotExistError: continue - found_items.update((safe_str(n.path), self.commit_ids[-1]) for n in directories) + found_items.update((dir_node.str_path, self.commit_ids[-1]) for dir_node in directories) def get_name(item): return item[0] @@ -216,7 +218,7 @@ class SubversionRepository(base.BaseRepository): def _get_commit_idx(self, commit_id): try: svn_rev = int(commit_id) - except: + except Exception: # TODO: johbo: this might be only one case, HEAD, check this svn_rev = self._remote.lookup(commit_id) commit_idx = svn_rev - 1 @@ -321,8 +323,7 @@ class SubversionRepository(base.BaseRepository): # TODO: johbo: Reconsider impact of DEFAULT_BRANCH_NAME here if branch_name not in [None, self.DEFAULT_BRANCH_NAME]: svn_rev = int(self.commit_ids[-1]) - commit_ids = self._remote.node_history( - path=branch_name, revision=svn_rev, limit=None) + commit_ids = self._remote.node_history(svn_rev, branch_name, None) commit_ids = [str(i) for i in reversed(commit_ids)] if start_pos or end_pos: diff --git a/rhodecode/lib/vcs/client_http.py b/rhodecode/lib/vcs/client_http.py index 75eb01cc..59ed0110 100644 --- a/rhodecode/lib/vcs/client_http.py +++ b/rhodecode/lib/vcs/client_http.py @@ -27,14 +27,13 @@ import time import urllib.request import urllib.error import urllib.parse -import urllib.parse import uuid import traceback import pycurl import msgpack import requests -from requests.packages.urllib3.util.retry import Retry +from urllib3.util.retry import Retry import rhodecode from rhodecode.lib import rc_cache @@ -287,7 +286,7 @@ class RemoteRepo(object): 'fctx_size', 'stream:fctx_node_data', 'blob_raw_length', 'node_history', 'revision', 'tree_items', - 'ctx_list', 'ctx_branch', 'ctx_description', + 'ctx_branch', 'ctx_description', 'bulk_request', 'assert_correct_path', 'is_path_valid_repository', diff --git a/rhodecode/lib/vcs/compat.py b/rhodecode/lib/vcs/compat.py index 8a6bbc41..82daea14 100644 --- a/rhodecode/lib/vcs/compat.py +++ b/rhodecode/lib/vcs/compat.py @@ -1 +1 @@ -from pyramid.compat import configparser \ No newline at end of file +from pyramid.compat import configparser diff --git a/rhodecode/lib/vcs/conf/settings.py b/rhodecode/lib/vcs/conf/settings.py index ca9d954b..6318cc9f 100644 --- a/rhodecode/lib/vcs/conf/settings.py +++ b/rhodecode/lib/vcs/conf/settings.py @@ -20,9 +20,6 @@ Internal settings for vcs-lib """ -# list of default encoding used in safe_str methods -DEFAULT_ENCODINGS = ['utf8'] - # Compatibility version when creating SVN repositories. None means newest. # Other available options are: pre-1.4-compatible, pre-1.5-compatible, diff --git a/rhodecode/lib/vcs/exceptions.py b/rhodecode/lib/vcs/exceptions.py index fe4547b5..5af82684 100644 --- a/rhodecode/lib/vcs/exceptions.py +++ b/rhodecode/lib/vcs/exceptions.py @@ -102,10 +102,6 @@ class NodeError(VCSError): pass -class RemovedFileNodeError(NodeError): - pass - - class NodeAlreadyExistsError(CommittingError): pass diff --git a/rhodecode/lib/vcs/nodes.py b/rhodecode/lib/vcs/nodes.py index 9ed11448..be70067c 100644 --- a/rhodecode/lib/vcs/nodes.py +++ b/rhodecode/lib/vcs/nodes.py @@ -19,6 +19,7 @@ """ Module holding everything related to vcs nodes, with vcs2 architecture. """ + import functools import os import stat @@ -29,84 +30,26 @@ from rhodecode.config.conf import LANGUAGES_EXTENSIONS_MAP from rhodecode.lib.str_utils import safe_str, safe_bytes from rhodecode.lib.hash_utils import md5 from rhodecode.lib.vcs import path as vcspath -from rhodecode.lib.vcs.backends.base import EmptyCommit, FILEMODE_DEFAULT +from rhodecode.lib.vcs.backends.base import EmptyCommit from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db -from rhodecode.lib.vcs.exceptions import NodeError, RemovedFileNodeError +from rhodecode.lib.vcs.exceptions import NodeError +from rhodecode.lib.vcs_common import NodeKind, FILEMODE_DEFAULT -LARGEFILE_PREFIX = '.hglf' - - -class NodeKind: - SUBMODULE = -1 - DIR = 1 - FILE = 2 - LARGEFILE = 3 +LARGEFILE_PREFIX = ".hglf" class NodeState: - ADDED = 'added' - CHANGED = 'changed' - NOT_CHANGED = 'not changed' - REMOVED = 'removed' + ADDED = "added" + CHANGED = "changed" + NOT_CHANGED = "not changed" + REMOVED = "removed" -#TODO: not sure if that should be bytes or str ? + +# TODO: not sure if that should be bytes or str ? # most probably bytes because content should be bytes and we check it -BIN_BYTE_MARKER = b'\0' +BIN_BYTE_MARKER = b"\0" -class NodeGeneratorBase(object): - """ - Base class for removed added and changed filenodes, it's a lazy generator - class that will create filenodes only on iteration or call - - The len method doesn't need to create filenodes at all - """ - - def __init__(self, current_paths, cs): - self.cs = cs - self.current_paths = current_paths - - def __call__(self): - return [n for n in self] - - def __getitem__(self, key): - if isinstance(key, slice): - for p in self.current_paths[key.start:key.stop]: - yield self.cs.get_node(p) - - def __len__(self): - return len(self.current_paths) - - def __iter__(self): - for p in self.current_paths: - yield self.cs.get_node(p) - - -class AddedFileNodesGenerator(NodeGeneratorBase): - """ - Class holding added files for current commit - """ - - -class ChangedFileNodesGenerator(NodeGeneratorBase): - """ - Class holding changed files for current commit - """ - - -class RemovedFileNodesGenerator(NodeGeneratorBase): - """ - Class holding removed files for current commit - """ - def __iter__(self): - for p in self.current_paths: - yield RemovedFileNode(path=safe_bytes(p)) - - def __getitem__(self, key): - if isinstance(key, slice): - for p in self.current_paths[key.start:key.stop]: - yield RemovedFileNode(path=safe_bytes(p)) - @functools.total_ordering class Node(object): @@ -119,21 +62,22 @@ class Node(object): only. Moreover, every single node is identified by the ``path`` attribute, so it cannot end with slash, too. Otherwise, path could lead to mistakes. """ + # RTLO marker allows swapping text, and certain # security attacks could be used with this - RTLO_MARKER = "\u202E" + RTLO_MARKER = "\u202e" commit = None def __init__(self, path: bytes, kind): self._validate_path(path) # can throw exception if path is invalid - self.bytes_path = path.rstrip(b'/') # store for __repr__ - self.path = safe_str(self.bytes_path) # we store paths as str + self.bytes_path: bytes = path.rstrip(b"/") # store for mixed encoding, and raw version + self.str_path: str = safe_str(self.bytes_path) # we store paths as str + self.path: str = self.str_path - if self.bytes_path == b'' and kind != NodeKind.DIR: - raise NodeError("Only DirNode and its subclasses may be " - "initialized with empty path") + if self.bytes_path == b"" and kind != NodeKind.DIR: + raise NodeError("Only DirNode and its subclasses may be initialized with empty path") self.kind = kind if self.is_root() and not self.is_dir(): @@ -142,7 +86,7 @@ class Node(object): def __eq__(self, other): if type(self) is not type(other): return False - for attr in ['name', 'path', 'kind']: + for attr in ["name", "path", "kind"]: if getattr(self, attr) != getattr(other, attr): return False if self.is_file(): @@ -166,22 +110,9 @@ class Node(object): if self.path > other.path: return False - # def __cmp__(self, other): - # """ - # Comparator using name of the node, needed for quick list sorting. - # """ - # - # kind_cmp = cmp(self.kind, other.kind) - # if kind_cmp: - # if isinstance(self, SubModuleNode): - # # we make submodules equal to dirnode for "sorting" purposes - # return NodeKind.DIR - # return kind_cmp - # return cmp(self.name, other.name) - def __repr__(self): - maybe_path = getattr(self, 'path', 'UNKNOWN_PATH') - return f'<{self.__class__.__name__} {maybe_path!r}>' + maybe_path = getattr(self, "path", "UNKNOWN_PATH") + return f"<{self.__class__.__name__} {maybe_path!r}>" def __str__(self): return self.name @@ -189,29 +120,27 @@ class Node(object): def _validate_path(self, path: bytes): self._assert_bytes(path) - if path.startswith(b'/'): + if path.startswith(b"/"): raise NodeError( f"Cannot initialize Node objects with slash at " f"the beginning as only relative paths are supported. " - f"Got {path}") + f"Got {path}" + ) - def _assert_bytes(self, value): + @classmethod + def _assert_bytes(cls, value): if not isinstance(value, bytes): raise TypeError(f"Bytes required as input, got {type(value)} of {value}.") @LazyProperty def parent(self): - parent_path = self.get_parent_path() + parent_path: bytes = self.get_parent_path() if parent_path: if self.commit: return self.commit.get_node(parent_path) return DirNode(parent_path) return None - @LazyProperty - def str_path(self) -> str: - return safe_str(self.path) - @LazyProperty def has_rtlo(self): """Detects if a path has right-to-left-override marker""" @@ -223,10 +152,10 @@ class Node(object): Returns name of the directory from full path of this vcs node. Empty string is returned if there's no directory in the path """ - _parts = self.path.rstrip('/').rsplit('/', 1) + _parts = self.path.rstrip("/").rsplit("/", 1) if len(_parts) == 2: return _parts[0] - return '' + return "" @LazyProperty def name(self): @@ -234,7 +163,7 @@ class Node(object): Returns name of the node so if its path then only last part is returned. """ - return self.path.rstrip('/').split('/')[-1] + return self.str_path.rstrip("/").split("/")[-1] @property def kind(self): @@ -242,12 +171,12 @@ class Node(object): @kind.setter def kind(self, kind): - if hasattr(self, '_kind'): + if hasattr(self, "_kind"): raise NodeError("Cannot change node's kind") else: self._kind = kind # Post setter check (path's trailing slash) - if self.path.endswith('/'): + if self.str_path.endswith("/"): raise NodeError("Node's path cannot end with slash") def get_parent_path(self) -> bytes: @@ -255,8 +184,8 @@ class Node(object): Returns node's parent path or empty string if node is root. """ if self.is_root(): - return b'' - str_path = vcspath.dirname(self.path.rstrip('/')) + '/' + return b"" + str_path = vcspath.dirname(self.bytes_path.rstrip(b"/")) + b"/" return safe_bytes(str_path) @@ -278,7 +207,7 @@ class Node(object): """ Returns ``True`` if node is a root node and ``False`` otherwise. """ - return self.kind == NodeKind.DIR and self.path == '' + return self.kind == NodeKind.DIR and self.path == "" def is_submodule(self): """ @@ -292,29 +221,13 @@ class Node(object): Returns ``True`` if node's kind is ``NodeKind.LARGEFILE``, ``False`` otherwise """ - return self.kind == NodeKind.LARGEFILE + return self.kind == NodeKind.LARGE_FILE def is_link(self): if self.commit: - return self.commit.is_link(self.path) + return self.commit.is_link(self.bytes_path) return False - @LazyProperty - def added(self): - return self.state is NodeState.ADDED - - @LazyProperty - def changed(self): - return self.state is NodeState.CHANGED - - @LazyProperty - def not_changed(self): - return self.state is NodeState.NOT_CHANGED - - @LazyProperty - def removed(self): - return self.state is NodeState.REMOVED - class FileNode(Node): """ @@ -325,6 +238,7 @@ class FileNode(Node): :attribute: commit: if given, first time content is accessed, callback :attribute: mode: stat mode for a node. Default is `FILEMODE_DEFAULT`. """ + _filter_pre_load = [] def __init__(self, path: bytes, content: bytes | None = None, commit=None, mode=None, pre_load=None): @@ -359,7 +273,7 @@ class FileNode(Node): return self.content == other.content def __hash__(self): - raw_id = getattr(self.commit, 'raw_id', '') + raw_id = getattr(self.commit, "raw_id", "") return hash((self.path, raw_id)) def __lt__(self, other): @@ -369,33 +283,32 @@ class FileNode(Node): return self.content < other.content def __repr__(self): - short_id = getattr(self.commit, 'short_id', '') - return f'<{self.__class__.__name__} path={self.path!r}, short_id={short_id}>' + short_id = getattr(self.commit, "short_id", "") + return f"<{self.__class__.__name__} path={self.str_path!r}, short_id={short_id}>" def _set_bulk_properties(self, pre_load): if not pre_load: return - pre_load = [entry for entry in pre_load - if entry not in self._filter_pre_load] + pre_load = [entry for entry in pre_load if entry not in self._filter_pre_load] if not pre_load: return remote = self.commit.get_remote() - result = remote.bulk_file_request(self.commit.raw_id, self.path, pre_load) + result = remote.bulk_file_request(self.commit.raw_id, self.bytes_path, pre_load) for attr, value in result.items(): if attr == "flags": - self.__dict__['mode'] = safe_str(value) + self.__dict__["mode"] = safe_str(value) elif attr == "size": - self.__dict__['size'] = value + self.__dict__["size"] = value elif attr == "data": - self.__dict__['_content'] = value + self.__dict__["_content"] = value elif attr == "is_binary": - self.__dict__['is_binary'] = value + self.__dict__["is_binary"] = value elif attr == "md5": - self.__dict__['md5'] = value + self.__dict__["md5"] = value else: - raise ValueError(f'Unsupported attr in bulk_property: {attr}') + raise ValueError(f"Unsupported attr in bulk_property: {attr}") @LazyProperty def mode(self): @@ -404,7 +317,7 @@ class FileNode(Node): use value given at initialization or `FILEMODE_DEFAULT` (default). """ if self.commit: - mode = self.commit.get_file_mode(self.path) + mode = self.commit.get_file_mode(self.bytes_path) else: mode = self._mode return mode @@ -416,7 +329,7 @@ class FileNode(Node): """ if self.commit: if self._content is None: - self._content = self.commit.get_file_content(self.path) + self._content = self.commit.get_file_content(self.bytes_path) content = self._content else: content = self._content @@ -427,7 +340,7 @@ class FileNode(Node): Returns lazily content of the FileNode. """ if self.commit: - content = self.commit.get_file_content(self.path) + content = self.commit.get_file_content(self.bytes_path) else: content = self._content return content @@ -438,7 +351,7 @@ class FileNode(Node): vcsserver without loading it to memory. """ if self.commit: - return self.commit.get_file_content_streamed(self.path) + return self.commit.get_file_content_streamed(self.bytes_path) raise NodeError("Cannot retrieve stream_bytes without related commit attribute") def metadata_uncached(self): @@ -462,7 +375,7 @@ class FileNode(Node): """ content = self.raw_bytes if content and not isinstance(content, bytes): - raise ValueError(f'Content is of type {type(content)} instead of bytes') + raise ValueError(f"Content is of type {type(content)} instead of bytes") return content @LazyProperty @@ -472,27 +385,21 @@ class FileNode(Node): @LazyProperty def size(self): if self.commit: - return self.commit.get_file_size(self.path) - raise NodeError( - "Cannot retrieve size of the file without related " - "commit attribute") + return self.commit.get_file_size(self.bytes_path) + raise NodeError("Cannot retrieve size of the file without related commit attribute") @LazyProperty def message(self): if self.commit: return self.last_commit.message - raise NodeError( - "Cannot retrieve message of the file without related " - "commit attribute") + raise NodeError("Cannot retrieve message of the file without related " "commit attribute") @LazyProperty def last_commit(self): if self.commit: pre_load = ["author", "date", "message", "parents"] - return self.commit.get_path_commit(self.path, pre_load=pre_load) - raise NodeError( - "Cannot retrieve last commit of the file without " - "related commit attribute") + return self.commit.get_path_commit(self.bytes_path, pre_load=pre_load) + raise NodeError("Cannot retrieve last commit of the file without related commit attribute") def get_mimetype(self): """ @@ -502,28 +409,27 @@ class FileNode(Node): attribute to indicate that type should *NOT* be calculated). """ - if hasattr(self, '_mimetype'): - if (isinstance(self._mimetype, (tuple, list)) and - len(self._mimetype) == 2): + if hasattr(self, "_mimetype"): + if isinstance(self._mimetype, (tuple, list)) and len(self._mimetype) == 2: return self._mimetype else: - raise NodeError('given _mimetype attribute must be an 2 ' - 'element list or tuple') + raise NodeError("given _mimetype attribute must be an 2 element list or tuple") db = get_mimetypes_db() mtype, encoding = db.guess_type(self.name) if mtype is None: if not self.is_largefile() and self.is_binary: - mtype = 'application/octet-stream' + mtype = "application/octet-stream" encoding = None else: - mtype = 'text/plain' + mtype = "text/plain" encoding = None # try with pygments try: from pygments.lexers import get_lexer_for_filename + mt = get_lexer_for_filename(self.name).mimetypes except Exception: mt = None @@ -544,18 +450,17 @@ class FileNode(Node): @LazyProperty def mimetype_main(self): - return self.mimetype.split('/')[0] + return self.mimetype.split("/")[0] @classmethod def get_lexer(cls, filename, content=None): from pygments import lexers - extension = filename.split('.')[-1] + extension = filename.split(".")[-1] lexer = None try: - lexer = lexers.guess_lexer_for_filename( - filename, content, stripnl=False) + lexer = lexers.guess_lexer_for_filename(filename, content, stripnl=False) except lexers.ClassNotFound: pass @@ -580,7 +485,7 @@ class FileNode(Node): content, name and mimetype. """ # TODO: this is more proper, but super heavy on investigating the type based on the content - #self.get_lexer(self.name, self.content) + # self.get_lexer(self.name, self.content) return self.get_lexer(self.name) @@ -597,8 +502,8 @@ class FileNode(Node): Returns a list of commit for this file in which the file was changed """ if self.commit is None: - raise NodeError('Unable to get commit for this FileNode') - return self.commit.get_path_history(self.path) + raise NodeError("Unable to get commit for this FileNode") + return self.commit.get_path_history(self.bytes_path) @LazyProperty def annotate(self): @@ -606,22 +511,9 @@ class FileNode(Node): Returns a list of three element tuples with lineno, commit and line """ if self.commit is None: - raise NodeError('Unable to get commit for this FileNode') + raise NodeError("Unable to get commit for this FileNode") pre_load = ["author", "date", "message", "parents"] - return self.commit.get_file_annotate(self.path, pre_load=pre_load) - - @LazyProperty - def state(self): - if not self.commit: - raise NodeError( - "Cannot check state of the node if it's not " - "linked with commit") - elif self.path in (node.path for node in self.commit.added): - return NodeState.ADDED - elif self.path in (node.path for node in self.commit.changed): - return NodeState.CHANGED - else: - return NodeState.NOT_CHANGED + return self.commit.get_file_annotate(self.bytes_path, pre_load=pre_load) @LazyProperty def is_binary(self): @@ -629,7 +521,7 @@ class FileNode(Node): Returns True if file has binary content. """ if self.commit: - return self.commit.is_node_binary(self.path) + return self.commit.is_node_binary(self.bytes_path) else: raw_bytes = self._content return bool(raw_bytes and BIN_BYTE_MARKER in raw_bytes) @@ -641,7 +533,7 @@ class FileNode(Node): """ if self.commit: - return self.commit.node_md5_hash(self.path) + return self.commit.node_md5_hash(self.bytes_path) else: raw_bytes = self._content # TODO: this sucks, we're computing md5 on potentially super big stream data... @@ -650,7 +542,7 @@ class FileNode(Node): @LazyProperty def extension(self): """Returns filenode extension""" - return self.name.split('.')[-1] + return self.name.split(".")[-1] @property def is_executable(self): @@ -667,15 +559,15 @@ class FileNode(Node): LF store. """ if self.commit: - return self.commit.get_largefile_node(self.path) + return self.commit.get_largefile_node(self.bytes_path) def count_lines(self, content: str | bytes, count_empty=False): if isinstance(content, str): - newline_marker = '\n' + newline_marker = "\n" elif isinstance(content, bytes): - newline_marker = b'\n' + newline_marker = b"\n" else: - raise ValueError('content must be bytes or str got {type(content)} instead') + raise ValueError("content must be bytes or str got {type(content)} instead") if count_empty: all_lines = 0 @@ -704,33 +596,6 @@ class FileNode(Node): return all_lines, empty_lines -class RemovedFileNode(FileNode): - """ - Dummy FileNode class - trying to access any public attribute except path, - name, kind or state (or methods/attributes checking those two) would raise - RemovedFileNodeError. - """ - ALLOWED_ATTRIBUTES = [ - 'name', 'path', 'state', 'is_root', 'is_file', 'is_dir', 'kind', - 'added', 'changed', 'not_changed', 'removed', 'bytes_path' - ] - - def __init__(self, path): - """ - :param path: relative path to the node - """ - super().__init__(path=path) - - def __getattribute__(self, attr): - if attr.startswith('_') or attr in RemovedFileNode.ALLOWED_ATTRIBUTES: - return super().__getattribute__(attr) - raise RemovedFileNodeError(f"Cannot access attribute {attr} on RemovedFileNode. Not in allowed attributes") - - @LazyProperty - def state(self): - return NodeState.REMOVED - - class DirNode(Node): """ DirNode stores list of files and directories within this node. @@ -752,7 +617,7 @@ class DirNode(Node): super().__init__(path, NodeKind.DIR) self.commit = commit self._nodes = nodes - self.default_pre_load = default_pre_load or ['is_binary', 'size'] + self.default_pre_load = default_pre_load or ["is_binary", "size"] def __iter__(self): yield from self.nodes @@ -782,10 +647,9 @@ class DirNode(Node): @LazyProperty def nodes(self): if self.commit: - nodes = self.commit.get_nodes(self.path, pre_load=self.default_pre_load) + nodes = self.commit.get_nodes(self.bytes_path, pre_load=self.default_pre_load) else: nodes = self._nodes - self._nodes_dict = {node.path: node for node in nodes} return sorted(nodes) @LazyProperty @@ -796,47 +660,6 @@ class DirNode(Node): def dirs(self): return sorted(node for node in self.nodes if node.is_dir()) - def get_node(self, path): - """ - Returns node from within this particular ``DirNode``, so it is now - allowed to fetch, i.e. node located at 'docs/api/index.rst' from node - 'docs'. In order to access deeper nodes one must fetch nodes between - them first - this would work:: - - docs = root.get_node('docs') - docs.get_node('api').get_node('index.rst') - - :param: path - relative to the current node - - .. note:: - To access lazily (as in example above) node have to be initialized - with related commit object - without it node is out of - context and may know nothing about anything else than nearest - (located at same level) nodes. - """ - try: - path = path.rstrip('/') - if path == '': - raise NodeError("Cannot retrieve node without path") - self.nodes # access nodes first in order to set _nodes_dict - paths = path.split('/') - if len(paths) == 1: - if not self.is_root(): - path = '/'.join((self.path, paths[0])) - else: - path = paths[0] - return self._nodes_dict[path] - elif len(paths) > 1: - if self.commit is None: - raise NodeError("Cannot access deeper nodes without commit") - else: - path1, path2 = paths[0], '/'.join(paths[1:]) - return self.get_node(path1).get_node(path2) - else: - raise KeyError - except KeyError: - raise NodeError(f"Node does not exist at {path}") - @LazyProperty def state(self): raise NodeError("Cannot access state of DirNode") @@ -844,7 +667,7 @@ class DirNode(Node): @LazyProperty def size(self): size = 0 - for root, dirs, files in self.commit.walk(self.path): + for root, dirs, files in self.commit.walk(self.bytes_path): for f in files: size += f.size @@ -854,14 +677,12 @@ class DirNode(Node): def last_commit(self): if self.commit: pre_load = ["author", "date", "message", "parents"] - return self.commit.get_path_commit(self.path, pre_load=pre_load) - raise NodeError( - "Cannot retrieve last commit of the file without " - "related commit attribute") + return self.commit.get_path_commit(self.bytes_path, pre_load=pre_load) + raise NodeError("Cannot retrieve last commit of the file without related commit attribute") def __repr__(self): - short_id = getattr(self.commit, 'short_id', '') - return f'<{self.__class__.__name__} {self.path!r} @ {short_id}>' + short_id = getattr(self.commit, "short_id", "") + return f"<{self.__class__.__name__} path={self.str_path!r}, short_id={short_id}>" class RootNode(DirNode): @@ -870,21 +691,24 @@ class RootNode(DirNode): """ def __init__(self, nodes=(), commit=None): - super().__init__(path=b'', nodes=nodes, commit=commit) + super().__init__(path=b"", nodes=nodes, commit=commit) def __repr__(self): - return f'<{self.__class__.__name__}>' + short_id = getattr(self.commit, "short_id", "") + return f"<{self.__class__.__name__} path={self.str_path!r}, short_id={short_id}>" class SubModuleNode(Node): """ represents a SubModule of Git or SubRepo of Mercurial """ + is_binary = False size = 0 def __init__(self, name, url=None, commit=None, alias=None): self.path = name + self.str_path: str = safe_str(self.path) # we store paths as str self.kind = NodeKind.SUBMODULE self.alias = alias @@ -894,8 +718,8 @@ class SubModuleNode(Node): self.url = url or self._extract_submodule_url() def __repr__(self): - short_id = getattr(self.commit, 'short_id', '') - return f'<{self.__class__.__name__} {self.path!r} @ {short_id}>' + short_id = getattr(self.commit, "short_id", "") + return f"<{self.__class__.__name__} {self.str_path!r} @ {short_id}>" def _extract_submodule_url(self): # TODO: find a way to parse gits submodule file and extract the @@ -908,22 +732,22 @@ class SubModuleNode(Node): Returns name of the node so if its path then only last part is returned. """ - org = safe_str(self.path.rstrip('/').split('/')[-1]) - return f'{org} @ {self.commit.short_id}' + org = self.str_path.rstrip("/").split("/")[-1] + return f"{org} @ {self.commit.short_id}" class LargeFileNode(FileNode): - def __init__(self, path, url=None, commit=None, alias=None, org_path=None): self._validate_path(path) # can throw exception if path is invalid self.org_path = org_path # as stored in VCS as LF pointer - self.bytes_path = path.rstrip(b'/') # store for __repr__ - self.path = safe_str(self.bytes_path) # we store paths as str + self.bytes_path = path.rstrip(b"/") # store for __repr__ + self.str_path = safe_str(self.bytes_path) + self.path = self.str_path - self.kind = NodeKind.LARGEFILE + self.kind = NodeKind.LARGE_FILE self.alias = alias - self._content = b'' + self._content = b"" def _validate_path(self, path: bytes): """ @@ -932,7 +756,7 @@ class LargeFileNode(FileNode): self._assert_bytes(path) def __repr__(self): - return f'<{self.__class__.__name__} {self.org_path} -> {self.path!r}>' + return f"<{self.__class__.__name__} {self.org_path} -> {self.str_path!r}>" @LazyProperty def size(self): @@ -940,7 +764,7 @@ class LargeFileNode(FileNode): @LazyProperty def raw_bytes(self): - with open(self.path, 'rb') as f: + with open(self.path, "rb") as f: content = f.read() return content @@ -952,7 +776,7 @@ class LargeFileNode(FileNode): return self.org_path def stream_bytes(self): - with open(self.path, 'rb') as stream: + with open(self.path, "rb") as stream: while True: data = stream.read(16 * 1024) if not data: diff --git a/rhodecode/lib/vcs_common.py b/rhodecode/lib/vcs_common.py new file mode 100644 index 00000000..6ae5a26c --- /dev/null +++ b/rhodecode/lib/vcs_common.py @@ -0,0 +1,46 @@ +# Copyright (C) 2014-2024 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + + +""" +Common VCS module for rhodecode and vcsserver +""" + + +import enum + +FILEMODE_DEFAULT = 0o100644 +FILEMODE_EXECUTABLE = 0o100755 +FILEMODE_LINK = 0o120000 + + +class NodeKind(int, enum.Enum): + SUBMODULE = -1 + DIR = 1 + FILE = 2 + LARGE_FILE = 3 + + +def map_git_obj_type(obj_type): + if obj_type == "blob": + return NodeKind.FILE + elif obj_type == "tree": + return NodeKind.DIR + elif obj_type == "link": + return NodeKind.SUBMODULE + return None diff --git a/rhodecode/model/gist.py b/rhodecode/model/gist.py index ce150fa1..8127d205 100644 --- a/rhodecode/model/gist.py +++ b/rhodecode/model/gist.py @@ -103,7 +103,7 @@ class GistModel(BaseModel): raise VCSError(f'Failed to load gist repository for {repo}') commit = vcs_repo.get_commit(commit_id=revision) - return commit, [n for n in commit.get_node('/')] + return commit, [n for n in commit.get_node(b'/')] def create(self, description, owner, gist_mapping, gist_type=Gist.GIST_PUBLIC, lifetime=-1, gist_id=None, diff --git a/rhodecode/model/pull_request.py b/rhodecode/model/pull_request.py index 22be808e..7d86790b 100644 --- a/rhodecode/model/pull_request.py +++ b/rhodecode/model/pull_request.py @@ -178,6 +178,7 @@ def get_diff_info( log.debug('Calculating authors of changed files') target_commit = source_repo.get_commit(ancestor_id) + # TODO: change to operate in bytes.. for fname, lines in changed_lines.items(): try: @@ -2223,8 +2224,7 @@ class MergeCheck(object): ) @classmethod - def validate(cls, pull_request, auth_user, translator, fail_early=False, - force_shadow_repo_refresh=False): + def validate(cls, pull_request, auth_user, translator, fail_early=False, force_shadow_repo_refresh=False): _ = translator merge_check = cls() @@ -2285,12 +2285,10 @@ class MergeCheck(object): # left over TODOs todos = CommentsModel().get_pull_request_unresolved_todos(pull_request) if todos: - log.debug("MergeCheck: cannot merge, {} " - "unresolved TODOs left.".format(len(todos))) + log.debug("MergeCheck: cannot merge, %s unresolved TODOs left.", len(todos)) if len(todos) == 1: - msg = _('Cannot merge, {} TODO still not resolved.').format( - len(todos)) + msg = _('Cannot merge, {} TODO still not resolved.').format(len(todos)) else: msg = _('Cannot merge, {} TODOs still not resolved.').format( len(todos)) diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py index b7b72d40..059c8795 100644 --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -33,6 +33,7 @@ from rhodecode.lib.auth import HasUserGroupPermissionAny from rhodecode.lib.caching_query import FromCache from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError from rhodecode.lib import hooks_base +from rhodecode.lib.str_utils import safe_bytes from rhodecode.lib.user_log_filter import user_log_filter from rhodecode.lib.utils import make_db_config from rhodecode.lib.utils2 import ( @@ -1109,47 +1110,47 @@ class ReadmeFinder: different. """ - readme_re = re.compile(r'^readme(\.[^\.]+)?$', re.IGNORECASE) - path_re = re.compile(r'^docs?', re.IGNORECASE) + readme_re = re.compile(br'^readme(\.[^.]+)?$', re.IGNORECASE) + path_re = re.compile(br'^docs?', re.IGNORECASE) default_priorities = { None: 0, - '.rst': 1, - '.md': 1, - '.rest': 2, - '.mkdn': 2, - '.text': 2, - '.txt': 3, - '.mdown': 3, - '.markdown': 4, + b'.rst': 1, + b'.md': 1, + b'.rest': 2, + b'.mkdn': 2, + b'.text': 2, + b'.txt': 3, + b'.mdown': 3, + b'.markdown': 4, } path_priority = { - 'doc': 0, - 'docs': 1, + b'doc': 0, + b'docs': 1, } FALLBACK_PRIORITY = 99 RENDERER_TO_EXTENSION = { - 'rst': ['.rst', '.rest'], - 'markdown': ['.md', 'mkdn', '.mdown', '.markdown'], + 'rst': [b'.rst', b'.rest'], + 'markdown': [b'.md', b'mkdn', b'.mdown', b'.markdown'], } def __init__(self, default_renderer=None): self._default_renderer = default_renderer - self._renderer_extensions = self.RENDERER_TO_EXTENSION.get( - default_renderer, []) + self._renderer_extensions = self.RENDERER_TO_EXTENSION.get(default_renderer, []) - def search(self, commit, path='/'): + def search(self, commit, path=b'/'): """ Find a readme in the given `commit`. """ # firstly, check the PATH type if it is actually a DIR - if commit.get_node(path).kind != NodeKind.DIR: + bytes_path = safe_bytes(path) + if commit.get_node(bytes_path).kind != NodeKind.DIR: return None - nodes = commit.get_nodes(path) + nodes = commit.get_nodes(bytes_path) matches = self._match_readmes(nodes) matches = self._sort_according_to_priority(matches) if matches: @@ -1157,8 +1158,8 @@ class ReadmeFinder: paths = self._match_paths(nodes) paths = self._sort_paths_according_to_priority(paths) - for path in paths: - match = self.search(commit, path=path) + for bytes_path in paths: + match = self.search(commit, path=bytes_path) if match: return match @@ -1168,7 +1169,7 @@ class ReadmeFinder: for node in nodes: if not node.is_file(): continue - path = node.path.rsplit('/', 1)[-1] + path = node.bytes_path.rsplit(b'/', 1)[-1] match = self.readme_re.match(path) if match: extension = match.group(1) @@ -1178,28 +1179,26 @@ class ReadmeFinder: for node in nodes: if not node.is_dir(): continue - match = self.path_re.match(node.path) + match = self.path_re.match(node.bytes_path) if match: - yield node.path + yield node.bytes_path def _priority(self, extension): - renderer_priority = ( - 0 if extension in self._renderer_extensions else 1) - extension_priority = self.default_priorities.get( - extension, self.FALLBACK_PRIORITY) - return (renderer_priority, extension_priority) + renderer_priority = 0 if extension in self._renderer_extensions else 1 + extension_priority = self.default_priorities.get(extension, self.FALLBACK_PRIORITY) + return renderer_priority, extension_priority def _sort_according_to_priority(self, matches): def priority_and_path(match): - return (match.priority, match.path) + return match.priority, match.path return sorted(matches, key=priority_and_path) def _sort_paths_according_to_priority(self, paths): def priority_and_path(path): - return (self.path_priority.get(path, self.FALLBACK_PRIORITY), path) + return self.path_priority.get(path, self.FALLBACK_PRIORITY), path return sorted(paths, key=priority_and_path) diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py index c53c729f..c2f37890 100644 --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -543,7 +543,7 @@ class ScmModel(BaseModel): root_path = root_path.lstrip('/') # get RootNode, inject pre-load options before walking - top_node = commit.get_node(root_path) + top_node = commit.get_node(safe_bytes(root_path)) extended_info_pre_load = [] if extended_info: extended_info_pre_load += ['md5'] @@ -614,12 +614,13 @@ class ScmModel(BaseModel): _files = list() _dirs = list() + bytes_path = safe_bytes(root_path) try: _repo = self._get_repo(repo_name) commit = _repo.scm_instance().get_commit(commit_id=commit_id) - root_path = root_path.lstrip('/') + root_path = bytes_path.lstrip(b'/') - top_node = commit.get_node(root_path) + top_node = commit.get_node(safe_bytes(root_path)) top_node.default_pre_load = [] for __, dirs, files in commit.walk(top_node): @@ -736,7 +737,7 @@ class ScmModel(BaseModel): _repo = self._get_repo(repo_name) commit = _repo.scm_instance().get_commit(commit_id=commit_id) root_path = root_path.lstrip('/') - top_node = commit.get_node(root_path) + top_node = commit.get_node(safe_bytes(root_path)) top_node.default_pre_load = [] for __, dirs, files in commit.walk(top_node): @@ -774,7 +775,7 @@ class ScmModel(BaseModel): only for git :param trigger_push_hook: trigger push hooks - :returns: new committed commit + :returns: new commit """ user, scm_instance, message, commiter, author, imc = self.initialize_inmemory_vars( user, repo, message, author) diff --git a/rhodecode/tests/config/test_routing_links.py b/rhodecode/tests/config/test_routing_links.py index aec9964b..3b6a8588 100644 --- a/rhodecode/tests/config/test_routing_links.py +++ b/rhodecode/tests/config/test_routing_links.py @@ -36,8 +36,8 @@ connection_available = pytest.mark.skipif( import requests +from urllib3.util.retry import Retry from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry def requests_retry_session( diff --git a/rhodecode/tests/models/test_repos.py b/rhodecode/tests/models/test_repos.py index 929ac100..e8ce4223 100644 --- a/rhodecode/tests/models/test_repos.py +++ b/rhodecode/tests/models/test_repos.py @@ -92,8 +92,8 @@ class TestRepoModel(object): is not None) @pytest.mark.parametrize("filename, expected", [ - ("README", True), - ("README.rst", False), + (b"README", True), + (b"README.rst", False), ]) def test_filenode_is_link(self, vcsbackend, filename, expected): repo = vcsbackend.repo diff --git a/rhodecode/tests/vcs/test_archives.py b/rhodecode/tests/vcs/test_archives.py index de0625b3..0650b336 100644 --- a/rhodecode/tests/vcs/test_archives.py +++ b/rhodecode/tests/vcs/test_archives.py @@ -28,7 +28,7 @@ import pytest import rhodecode from rhodecode.lib.archive_cache import get_archival_config -from rhodecode.lib.str_utils import ascii_bytes +from rhodecode.lib.str_utils import ascii_bytes, safe_bytes, safe_str from rhodecode.lib.vcs.backends import base from rhodecode.lib.vcs.exceptions import ImproperArchiveTypeError, VCSError from rhodecode.lib.vcs.nodes import FileNode @@ -80,8 +80,8 @@ class TestArchives(BackendTestMixin): out_file.close() for x in range(5): - node_path = "%d/file_%d.txt" % (x, x) - with open(os.path.join(out_dir, "repo/" + node_path), "rb") as f: + node_path = b"%d/file_%d.txt" % (x, x) + with open(os.path.join(safe_bytes(str(out_dir)), b"repo/" + node_path), "rb") as f: file_content = f.read() assert file_content == self.tip.get_node(node_path).content @@ -120,8 +120,9 @@ class TestArchives(BackendTestMixin): zip_file = zipfile.ZipFile(str(archive_lnk)) for x in range(5): - node_path = "%d/file_%d.txt" % (x, x) - data = zip_file.read(f"repo/{node_path}") + node_path = b"%d/file_%d.txt" % (x, x) + # NOTE: zipfile operates only on strings inside the archive + data = zip_file.read(safe_str(b"repo/%s" % node_path)) decompressed = io.BytesIO() decompressed.write(data) @@ -143,8 +144,9 @@ class TestArchives(BackendTestMixin): assert b"commit_id:%b" % raw_id in metafile for x in range(5): - node_path = "%d/file_%d.txt" % (x, x) - data = zip_file.read(f"repo/{node_path}") + node_path = b"%d/file_%d.txt" % (x, x) + # NOTE: zipfile operates only on strings inside the archive + data = zip_file.read(safe_str(b"repo/%s" % node_path)) decompressed = io.BytesIO() decompressed.write(data) assert decompressed.getvalue() == self.tip.get_node(node_path).content diff --git a/rhodecode/tests/vcs/test_commits.py b/rhodecode/tests/vcs/test_commits.py index 2be37a6a..93a5b823 100644 --- a/rhodecode/tests/vcs/test_commits.py +++ b/rhodecode/tests/vcs/test_commits.py @@ -22,21 +22,17 @@ import time import pytest from rhodecode.lib.str_utils import safe_bytes -from rhodecode.lib.vcs.backends.base import CollectionGenerator, FILEMODE_DEFAULT, EmptyCommit +from rhodecode.lib.vcs.backends.base import CollectionGenerator, EmptyCommit from rhodecode.lib.vcs.exceptions import ( BranchDoesNotExistError, CommitDoesNotExistError, RepositoryError, EmptyRepositoryError, ) -from rhodecode.lib.vcs.nodes import ( - FileNode, - AddedFileNodesGenerator, - ChangedFileNodesGenerator, - RemovedFileNodesGenerator, -) +from rhodecode.lib.vcs.nodes import FileNode from rhodecode.tests import get_new_dir from rhodecode.tests.vcs.conftest import BackendTestMixin +from rhodecode.lib.vcs_common import NodeKind, FILEMODE_EXECUTABLE, FILEMODE_DEFAULT, FILEMODE_LINK class TestBaseChangeset(object): @@ -70,7 +66,7 @@ class TestCommitsInNonEmptyRepo(BackendTestMixin): } def test_walk_returns_empty_list_in_case_of_file(self): - result = list(self.tip.walk("file_0.txt")) + result = list(self.tip.walk(b"file_0.txt")) assert result == [] @pytest.mark.backends("git", "hg") @@ -319,7 +315,7 @@ class TestCommits(BackendTestMixin): def test_get_path_commit(self): commit = self.repo.get_commit() - commit.get_path_commit("file_4.txt") + commit.get_path_commit(b"file_4.txt") assert commit.message == "Commit 4" def test_get_filenodes_generator(self): @@ -500,8 +496,8 @@ class TestCommits(BackendTestMixin): @pytest.mark.parametrize( "filename, expected", [ - ("README.rst", False), - ("README", True), + (b"README.rst", False), + (b"README", True), ], ) def test_commit_is_link(vcsbackend, filename, expected): @@ -543,49 +539,41 @@ class TestCommitsChanges(BackendTestMixin): def test_initial_commit(self, local_dt_to_utc): commit = self.repo.get_commit(commit_idx=0) - assert set(commit.added) == { - commit.get_node("foo/bar"), - commit.get_node("foo/bał"), - commit.get_node("foobar"), - commit.get_node("qwe"), - } - assert set(commit.changed) == set() - assert set(commit.removed) == set() - assert set(commit.affected_files) == {"foo/bar", "foo/bał", "foobar", "qwe"} + assert sorted(commit.added_paths) == sorted([b"foo/bar", b"foo/ba\xc5\x82", b"foobar", b"qwe"]) + assert commit.changed_paths == [] + assert commit.removed_paths == [] + assert sorted(commit.affected_files) == sorted([b"foo/bar", b"foo/ba\xc5\x82", b"foobar", b"qwe"]) assert commit.date == local_dt_to_utc(datetime.datetime(2010, 1, 1, 20, 0)) def test_head_added(self): commit = self.repo.get_commit() - assert isinstance(commit.added, AddedFileNodesGenerator) - assert set(commit.added) == {commit.get_node("fallout")} - assert isinstance(commit.changed, ChangedFileNodesGenerator) - assert set(commit.changed) == {commit.get_node("foo/bar"), commit.get_node("foobar")} - assert isinstance(commit.removed, RemovedFileNodesGenerator) - assert len(commit.removed) == 1 - assert list(commit.removed)[0].path == "qwe" + + assert commit.added_paths == [b"fallout"] + assert commit.changed_paths == [b"foo/bar", b"foobar"] + assert commit.removed_paths == [b"qwe"] def test_get_filemode(self): commit = self.repo.get_commit() - assert FILEMODE_DEFAULT == commit.get_file_mode("foo/bar") + assert FILEMODE_DEFAULT == commit.get_file_mode(b"foo/bar") def test_get_filemode_non_ascii(self): commit = self.repo.get_commit() - assert FILEMODE_DEFAULT == commit.get_file_mode("foo/bał") - assert FILEMODE_DEFAULT == commit.get_file_mode("foo/bał") + assert FILEMODE_DEFAULT == commit.get_file_mode(b"foo/ba\xc5\x82") + assert FILEMODE_DEFAULT == commit.get_file_mode(b"foo/ba\xc5\x82") def test_get_path_history(self): commit = self.repo.get_commit() - history = commit.get_path_history("foo/bar") + history = commit.get_path_history(b"foo/bar") assert len(history) == 2 def test_get_path_history_with_limit(self): commit = self.repo.get_commit() - history = commit.get_path_history("foo/bar", limit=1) + history = commit.get_path_history(b"foo/bar", limit=1) assert len(history) == 1 def test_get_path_history_first_commit(self): commit = self.repo[0] - history = commit.get_path_history("foo/bar") + history = commit.get_path_history(b"foo/bar") assert len(history) == 1 diff --git a/rhodecode/tests/vcs/test_filenodes_listing_and_caches.py b/rhodecode/tests/vcs/test_filenodes_listing_and_caches.py new file mode 100644 index 00000000..a826ecc9 --- /dev/null +++ b/rhodecode/tests/vcs/test_filenodes_listing_and_caches.py @@ -0,0 +1,227 @@ +# Copyright (C) 2010-2024 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import datetime + +import pytest + +from rhodecode.lib.str_utils import safe_bytes +from rhodecode.lib.vcs.nodes import FileNode +from rhodecode.lib.vcs_common import NodeKind +from rhodecode.tests.vcs.conftest import BackendTestMixin + + +class TestFileNodesListingAndCaches: + def test_filenode_get_root_node(self, vcsbackend): + repo = vcsbackend.repo + commit = repo.get_commit() + + # check if we start with empty nodes cache + assert commit.nodes == {} + assert commit._path_mode_cache == {} + assert commit._path_type_cache == {} + + top_dir = commit.get_node(b"") + + assert top_dir.is_dir() + + assert list(commit.nodes.keys()) == [b""] + assert list(commit._path_type_cache.keys()) == [b""] + assert commit._path_mode_cache == {} + + def test_filenode_get_file_node(self, vcsbackend): + repo = vcsbackend.repo + commit = repo.get_commit() + + # check if we start with empty nodes cache + assert commit.nodes == {} + assert commit._path_mode_cache == {} + assert commit._path_type_cache == {} + + file_node = commit.get_node(b"README.rst") + + assert file_node.is_file() + assert file_node.last_commit + + assert list(commit.nodes.keys()) == [b"README.rst"] + if repo.alias == "hg": + assert commit._path_type_cache == {b"README.rst": NodeKind.FILE} + assert commit._path_mode_cache == {b"README.rst": 33188} + + if repo.alias == "git": + assert commit._path_type_cache == { + b"README.rst": ["8f111ab5152b9fd34f52b0fb288e1216b5d2f55e", NodeKind.FILE] + } + assert commit._path_mode_cache == {b"README.rst": 33188} + + if repo.alias == "svn": + assert commit._path_type_cache == {b"README.rst": NodeKind.FILE} + assert commit._path_mode_cache == {b"README.rst": 33188} + + def test_filenode_get_nodes_from_top_dir(self, vcsbackend): + repo = vcsbackend.repo + commit = repo.get_commit() + + # check if we start with empty nodes cache + assert commit.nodes == {} + assert commit._path_mode_cache == {} + assert commit._path_type_cache == {} + + node_list = commit.get_nodes(b"") + + for node in node_list: + assert node + + if repo.alias == "svn": + assert list(commit.nodes.keys()) == [ + b"README", + b".hgignore", + b"MANIFEST.in", + b".travis.yml", + b"vcs", + b"tox.ini", + b"setup.py", + b"setup.cfg", + b"docs", + b"fakefile", + b"examples", + b"test_and_report.sh", + b"bin", + b".gitignore", + b".hgtags", + b"README.rst", + ] + assert commit._path_type_cache == { + b"": NodeKind.DIR, + b"README": 2, + b".hgignore": 2, + b"MANIFEST.in": 2, + b".travis.yml": 2, + b"vcs": 1, + b"tox.ini": 2, + b"setup.py": 2, + b"setup.cfg": 2, + b"docs": 1, + b"fakefile": 2, + b"examples": 1, + b"test_and_report.sh": 2, + b"bin": 1, + b".gitignore": 2, + b".hgtags": 2, + b"README.rst": 2, + } + assert commit._path_mode_cache == {} + + if repo.alias == "hg": + assert list(commit.nodes.keys()) == [ + b".gitignore", + b".hgignore", + b".hgtags", + b".travis.yml", + b"MANIFEST.in", + b"README", + b"README.rst", + b"docs", + b"run_test_and_report.sh", + b"setup.cfg", + b"setup.py", + b"test_and_report.sh", + b"tox.ini", + b"vcs", + ] + + assert commit._path_type_cache == { + b"": NodeKind.DIR, + b".gitignore": 2, + b".hgignore": 2, + b".hgtags": 2, + b".travis.yml": 2, + b"MANIFEST.in": 2, + b"README": 2, + b"README.rst": 2, + b"docs": 1, + b"run_test_and_report.sh": 2, + b"setup.cfg": 2, + b"setup.py": 2, + b"test_and_report.sh": 2, + b"tox.ini": 2, + b"vcs": 1, + } + assert commit._path_mode_cache == {} + + if repo.alias == "git": + assert list(commit.nodes.keys()) == [ + b".gitignore", + b".hgignore", + b".hgtags", + b"MANIFEST.in", + b"README", + b"README.rst", + b"docs", + b"run_test_and_report.sh", + b"setup.cfg", + b"setup.py", + b"test_and_report.sh", + b"tox.ini", + b"vcs", + ] + assert commit._path_type_cache == { + b"": ["2dab7f7377e36948b5633ae0877310380fdf64ba", NodeKind.DIR], + b".gitignore": ["7f95b0917e31b0cce1ad96a033b1a814c420cde3", 2], + b".hgignore": ["9bdee5ed67ceed7c4b5589ba26dada65df151aed", 2], + b".hgtags": ["63aa2e892deebdfdef1845fe52e2418ade03557b", 2], + b"MANIFEST.in": ["6d65aca5321c26589ae197b81511445cfb353c95", 2], + b"README": ["92cacd285355271487b7e379dba6ca60f9a554a4", 2], + b"README.rst": ["8f111ab5152b9fd34f52b0fb288e1216b5d2f55e", 2], + b"docs": ["eaa8cef76ec4e1baf06f515ab03bfc01719a913d", 1], + b"run_test_and_report.sh": ["e17011bdddc8812fda96e891a1087c73054f1f25", 2], + b"setup.cfg": ["2e76b65bc9106ed466d71fb2cfe710352eadfb49", 2], + b"setup.py": ["ccda2bd076eca961059deba6e39fb591d014052a", 2], + b"test_and_report.sh": ["0b14056dd7dc759a9719adef3d7bb529141a740e", 2], + b"tox.ini": ["c1735231e7af32f089b95455162db092824715a4", 2], + b"vcs": ["5868c69a8d74ccf3fdc655af8f3f185bc0ff2ef6", 1], + } + assert commit._path_mode_cache == { + b".gitignore": 33188, + b".hgignore": 33188, + b".hgtags": 33188, + b"MANIFEST.in": 33188, + b"README": 40960, + b"README.rst": 33188, + b"docs": 16384, + b"run_test_and_report.sh": 33261, + b"setup.cfg": 33188, + b"setup.py": 33188, + b"test_and_report.sh": 33261, + b"tox.ini": 33188, + b"vcs": 16384, + } + + def test_filenode_get_nodes_from_docs_dir(self, vcsbackend): + repo = vcsbackend.repo + commit = repo.get_commit() + + # check if we start with empty nodes cache + assert commit.nodes == {} + assert commit._path_mode_cache == {} + assert commit._path_type_cache == {} + + node_list = commit.get_nodes(b"docs") + + for node in node_list: + assert node diff --git a/rhodecode/tests/vcs/test_git.py b/rhodecode/tests/vcs/test_git.py index 684f7e8b..d9d40405 100644 --- a/rhodecode/tests/vcs/test_git.py +++ b/rhodecode/tests/vcs/test_git.py @@ -28,7 +28,7 @@ from rhodecode.lib.utils import make_db_config from rhodecode.lib.vcs.backends.base import Reference from rhodecode.lib.vcs.backends.git import GitRepository, GitCommit, discover_git_version from rhodecode.lib.vcs.exceptions import RepositoryError, VCSError, NodeDoesNotExistError -from rhodecode.lib.vcs.nodes import NodeKind, FileNode, DirNode, NodeState, SubModuleNode +from rhodecode.lib.vcs.nodes import NodeKind, FileNode, DirNode, NodeState, SubModuleNode, RootNode from rhodecode.tests import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, get_new_dir from rhodecode.tests.vcs.conftest import BackendTestMixin @@ -219,23 +219,38 @@ class TestGitRepository(object): assert init_commit.message == "initial import\n" assert init_author == "Marcin Kuzminski " assert init_author == init_commit.committer - for path in ("vcs/__init__.py", "vcs/backends/BaseRepository.py", "vcs/backends/__init__.py"): + assert sorted(init_commit.added_paths) == sorted( + [ + b"vcs/__init__.py", + b"vcs/backends/BaseRepository.py", + b"vcs/backends/__init__.py", + ] + ) + assert sorted(init_commit.affected_files) == sorted( + [ + b"vcs/__init__.py", + b"vcs/backends/BaseRepository.py", + b"vcs/backends/__init__.py", + ] + ) + + for path in (b"vcs/__init__.py", b"vcs/backends/BaseRepository.py", b"vcs/backends/__init__.py"): assert isinstance(init_commit.get_node(path), FileNode) - for path in ("", "vcs", "vcs/backends"): + for path in (b"", b"vcs", b"vcs/backends"): assert isinstance(init_commit.get_node(path), DirNode) with pytest.raises(NodeDoesNotExistError): - init_commit.get_node(path="foobar") + init_commit.get_node(path=b"foobar") - node = init_commit.get_node("vcs/") + node = init_commit.get_node(b"vcs/") assert hasattr(node, "kind") assert node.kind == NodeKind.DIR - node = init_commit.get_node("vcs") + node = init_commit.get_node(b"vcs") assert hasattr(node, "kind") assert node.kind == NodeKind.DIR - node = init_commit.get_node("vcs/__init__.py") + node = init_commit.get_node(b"vcs/__init__.py") assert hasattr(node, "kind") assert node.kind == NodeKind.FILE @@ -257,7 +272,7 @@ Introduction TODO: To be written... """ - node = commit10.get_node("README.rst") + node = commit10.get_node(b"README.rst") assert node.kind == NodeKind.FILE assert node.str_content == README @@ -615,7 +630,7 @@ class TestGitCommit(object): def test_root_node(self): tip = self.repo.get_commit() - assert tip.root is tip.get_node("") + assert tip.root is tip.get_node(b"") def test_lazy_fetch(self): """ @@ -633,29 +648,29 @@ class TestGitCommit(object): # accessing root.nodes updates commit.nodes assert len(commit.nodes) == 9 - docs = root.get_node("docs") + docs = commit.get_node(b"docs") # we haven't yet accessed anything new as docs dir was already cached assert len(commit.nodes) == 9 assert len(docs.nodes) == 8 # accessing docs.nodes updates commit.nodes assert len(commit.nodes) == 17 - assert docs is commit.get_node("docs") + assert docs is commit.get_node(b"docs") assert docs is root.nodes[0] assert docs is root.dirs[0] - assert docs is commit.get_node("docs") + assert docs is commit.get_node(b"docs") def test_nodes_with_commit(self): commit_id = "2a13f185e4525f9d4b59882791a2d397b90d5ddc" commit = self.repo.get_commit(commit_id) root = commit.root - docs = root.get_node("docs") - assert docs is commit.get_node("docs") - api = docs.get_node("api") - assert api is commit.get_node("docs/api") - index = api.get_node("index.rst") - assert index is commit.get_node("docs/api/index.rst") - assert index is commit.get_node("docs").get_node("api").get_node("index.rst") + assert isinstance(root, RootNode) + docs = commit.get_node(b"docs") + assert docs is commit.get_node(b"docs") + api = commit.get_node(b"docs/api") + assert api is commit.get_node(b"docs/api") + index = commit.get_node(b"docs/api/index.rst") + assert index is commit.get_node(b"docs/api/index.rst") def test_branch_and_tags(self): """ @@ -682,12 +697,12 @@ class TestGitCommit(object): def test_file_size(self): to_check = ( - ("c1214f7e79e02fc37156ff215cd71275450cffc3", "vcs/backends/BaseRepository.py", 502), - ("d7e0d30fbcae12c90680eb095a4f5f02505ce501", "vcs/backends/hg.py", 854), - ("6e125e7c890379446e98980d8ed60fba87d0f6d1", "setup.py", 1068), - ("d955cd312c17b02143c04fa1099a352b04368118", "vcs/backends/base.py", 2921), - ("ca1eb7957a54bce53b12d1a51b13452f95bc7c7e", "vcs/backends/base.py", 3936), - ("f50f42baeed5af6518ef4b0cb2f1423f3851a941", "vcs/backends/base.py", 6189), + ("c1214f7e79e02fc37156ff215cd71275450cffc3", b"vcs/backends/BaseRepository.py", 502), + ("d7e0d30fbcae12c90680eb095a4f5f02505ce501", b"vcs/backends/hg.py", 854), + ("6e125e7c890379446e98980d8ed60fba87d0f6d1", b"setup.py", 1068), + ("d955cd312c17b02143c04fa1099a352b04368118", b"vcs/backends/base.py", 2921), + ("ca1eb7957a54bce53b12d1a51b13452f95bc7c7e", b"vcs/backends/base.py", 3936), + ("f50f42baeed5af6518ef4b0cb2f1423f3851a941", b"vcs/backends/base.py", 6189), ) for commit_id, path, size in to_check: node = self.repo.get_commit(commit_id).get_node(path) @@ -695,17 +710,17 @@ class TestGitCommit(object): assert node.size == size def test_file_history_from_commits(self): - node = self.repo[10].get_node("setup.py") + node = self.repo[10].get_node(b"setup.py") commit_ids = [commit.raw_id for commit in node.history] assert ["ff7ca51e58c505fec0dd2491de52c622bb7a806b"] == commit_ids - node = self.repo[20].get_node("setup.py") + node = self.repo[20].get_node(b"setup.py") node_ids = [commit.raw_id for commit in node.history] assert ["191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e", "ff7ca51e58c505fec0dd2491de52c622bb7a806b"] == node_ids # special case we check history from commit that has this particular # file changed this means we check if it's included as well - node = self.repo.get_commit("191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e").get_node("setup.py") + node = self.repo.get_commit("191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e").get_node(b"setup.py") node_ids = [commit.raw_id for commit in node.history] assert ["191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e", "ff7ca51e58c505fec0dd2491de52c622bb7a806b"] == node_ids @@ -713,7 +728,7 @@ class TestGitCommit(object): # we can only check if those commits are present in the history # as we cannot update this test every time file is changed files = { - "setup.py": [ + b"setup.py": [ "54386793436c938cff89326944d4c2702340037d", "51d254f0ecf5df2ce50c0b115741f4cf13985dab", "998ed409c795fec2012b1c0ca054d99888b22090", @@ -724,7 +739,7 @@ class TestGitCommit(object): "191caa5b2c81ed17c0794bf7bb9958f4dcb0b87e", "ff7ca51e58c505fec0dd2491de52c622bb7a806b", ], - "vcs/nodes.py": [ + b"vcs/nodes.py": [ "33fa3223355104431402a888fa77a4e9956feb3e", "fa014c12c26d10ba682fadb78f2a11c24c8118e1", "e686b958768ee96af8029fe19c6050b1a8dd3b2b", @@ -757,7 +772,7 @@ class TestGitCommit(object): "dd80b0f6cf5052f17cc738c2951c4f2070200d7f", "ff7ca51e58c505fec0dd2491de52c622bb7a806b", ], - "vcs/backends/git.py": [ + b"vcs/backends/git.py": [ "4cf116ad5a457530381135e2f4c453e68a1b0105", "9a751d84d8e9408e736329767387f41b36935153", "cb681fb539c3faaedbcdf5ca71ca413425c18f01", @@ -778,7 +793,7 @@ class TestGitCommit(object): def test_file_annotate(self): files = { - "vcs/backends/__init__.py": { + b"vcs/backends/__init__.py": { "c1214f7e79e02fc37156ff215cd71275450cffc3": { "lines_no": 1, "commits": [ @@ -870,39 +885,31 @@ class TestGitCommit(object): """ Tests state of FileNodes. """ - node = self.repo.get_commit("e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0").get_node("vcs/utils/diffs.py") - assert node.state, NodeState.ADDED - assert node.added - assert not node.changed - assert not node.not_changed - assert not node.removed + commit = self.repo.get_commit("e6ea6d16e2f26250124a1f4b4fe37a912f9d86a0") + node = commit.get_node(b"vcs/utils/diffs.py") + assert node.bytes_path in commit.added_paths - node = self.repo.get_commit("33fa3223355104431402a888fa77a4e9956feb3e").get_node(".hgignore") - assert node.state, NodeState.CHANGED - assert not node.added - assert node.changed - assert not node.not_changed - assert not node.removed + commit = self.repo.get_commit("33fa3223355104431402a888fa77a4e9956feb3e") + node = commit.get_node(b".hgignore") + assert node.bytes_path in commit.changed_paths - node = self.repo.get_commit("e29b67bd158580fc90fc5e9111240b90e6e86064").get_node("setup.py") - assert node.state, NodeState.NOT_CHANGED - assert not node.added - assert not node.changed - assert node.not_changed - assert not node.removed + commit = self.repo.get_commit("e29b67bd158580fc90fc5e9111240b90e6e86064") + node = commit.get_node(b"setup.py") + assert node.bytes_path not in commit.affected_files # If node has REMOVED state then trying to fetch it would raise # CommitError exception commit = self.repo.get_commit("fa6600f6848800641328adbf7811fd2372c02ab2") - path = "vcs/backends/BaseRepository.py" + path = b"vcs/backends/BaseRepository.py" with pytest.raises(NodeDoesNotExistError): commit.get_node(path) + # but it would be one of ``removed`` (commit's attribute) - assert path in [rf.path for rf in commit.removed] + assert path in [rf for rf in commit.removed_paths] commit = self.repo.get_commit("54386793436c938cff89326944d4c2702340037d") - changed = ["setup.py", "tests/test_nodes.py", "vcs/backends/hg.py", "vcs/nodes.py"] - assert set(changed) == set([f.path for f in commit.changed]) + changed = [b"setup.py", b"tests/test_nodes.py", b"vcs/backends/hg.py", b"vcs/nodes.py"] + assert set(changed) == set([f for f in commit.changed_paths]) def test_unicode_branch_refs(self): unicode_branches = { @@ -936,14 +943,14 @@ class TestGitCommit(object): def test_repo_files_content_types(self): commit = self.repo.get_commit() - for node in commit.get_node("/"): + for node in commit.get_node(b"/"): if node.is_file(): assert type(node.content) == bytes assert type(node.str_content) == str def test_wrong_path(self): # There is 'setup.py' in the root dir but not there: - path = "foo/bar/setup.py" + path = b"foo/bar/setup.py" tip = self.repo.get_commit() with pytest.raises(VCSError): tip.get_node(path) @@ -981,8 +988,7 @@ class TestLargeFileRepo(object): repo = backend_git.create_test_repo("largefiles", conf) tip = repo.scm_instance().get_commit() - node = tip.get_node("1MB.zip") - + node = tip.get_node(b"1MB.zip") # extract stored LF node into the origin cache repo_lfs_store: str = os.path.join(repo.repo_path, repo.repo_name, "lfs_store") @@ -1002,7 +1008,7 @@ class TestLargeFileRepo(object): assert lf_node.is_largefile() is True assert lf_node.size == 1024000 - assert lf_node.name == "1MB.zip" + assert lf_node.name == b"1MB.zip" @pytest.mark.usefixtures("vcs_repository_support") @@ -1032,14 +1038,11 @@ class TestGitSpecificWithRepo(BackendTestMixin): def test_paths_slow_traversing(self): commit = self.repo.get_commit() - assert ( - commit.get_node("foobar").get_node("static").get_node("js").get_node("admin").get_node("base.js").content - == b"base" - ) + assert commit.get_node(b"foobar/static/js/admin/base.js").content == b"base" def test_paths_fast_traversing(self): commit = self.repo.get_commit() - assert commit.get_node("foobar/static/js/admin/base.js").content == b"base" + assert commit.get_node(b"foobar/static/js/admin/base.js").content == b"base" def test_get_diff_runs_git_command_with_hashes(self): comm1 = self.repo[0] @@ -1110,12 +1113,15 @@ class TestGitRegression(BackendTestMixin): @pytest.mark.parametrize( "path, expected_paths", [ - ("bot", ["bot/build", "bot/templates", "bot/__init__.py"]), - ("bot/build", ["bot/build/migrations", "bot/build/static", "bot/build/templates"]), - ("bot/build/static", ["bot/build/static/templates"]), - ("bot/build/static/templates", ["bot/build/static/templates/f.html", "bot/build/static/templates/f1.html"]), - ("bot/build/templates", ["bot/build/templates/err.html", "bot/build/templates/err2.html"]), - ("bot/templates/", ["bot/templates/404.html", "bot/templates/500.html"]), + (b"bot", ["bot/build", "bot/templates", "bot/__init__.py"]), + (b"bot/build", ["bot/build/migrations", "bot/build/static", "bot/build/templates"]), + (b"bot/build/static", ["bot/build/static/templates"]), + ( + b"bot/build/static/templates", + ["bot/build/static/templates/f.html", "bot/build/static/templates/f1.html"], + ), + (b"bot/build/templates", ["bot/build/templates/err.html", "bot/build/templates/err2.html"]), + (b"bot/templates/", ["bot/templates/404.html", "bot/templates/500.html"]), ], ) def test_similar_paths(self, path, expected_paths): @@ -1146,8 +1152,8 @@ class TestGetSubmoduleUrl(object): node.str_content = ( '[submodule "subrepo1"]\n' "\tpath = subrepo1\n" "\turl = https://code.rhodecode.com/dulwich\n" ) - result = commit._get_submodule_url("subrepo1") - get_node_mock.assert_called_once_with(".gitmodules") + result = commit._get_submodule_url(b"subrepo1") + get_node_mock.assert_called_once_with(b".gitmodules") assert result == "https://code.rhodecode.com/dulwich" def test_complex_submodule_path(self): @@ -1160,14 +1166,14 @@ class TestGetSubmoduleUrl(object): "\tpath = complex/subrepo/path\n" "\turl = https://code.rhodecode.com/dulwich\n" ) - result = commit._get_submodule_url("complex/subrepo/path") - get_node_mock.assert_called_once_with(".gitmodules") + result = commit._get_submodule_url(b"complex/subrepo/path") + get_node_mock.assert_called_once_with(b".gitmodules") assert result == "https://code.rhodecode.com/dulwich" def test_submodules_file_not_found(self): commit = GitCommit(repository=mock.Mock(), raw_id="abcdef12", idx=1) with mock.patch.object(commit, "get_node", side_effect=NodeDoesNotExistError): - result = commit._get_submodule_url("complex/subrepo/path") + result = commit._get_submodule_url(b"complex/subrepo/path") assert result is None def test_path_not_found(self): @@ -1178,8 +1184,8 @@ class TestGetSubmoduleUrl(object): node.str_content = ( '[submodule "subrepo1"]\n' "\tpath = subrepo1\n" "\turl = https://code.rhodecode.com/dulwich\n" ) - result = commit._get_submodule_url("subrepo2") - get_node_mock.assert_called_once_with(".gitmodules") + result = commit._get_submodule_url(b"subrepo2") + get_node_mock.assert_called_once_with(b".gitmodules") assert result is None def test_returns_cached_values(self): @@ -1191,44 +1197,43 @@ class TestGetSubmoduleUrl(object): '[submodule "subrepo1"]\n' "\tpath = subrepo1\n" "\turl = https://code.rhodecode.com/dulwich\n" ) for _ in range(3): - commit._get_submodule_url("subrepo1") - get_node_mock.assert_called_once_with(".gitmodules") + commit._get_submodule_url(b"subrepo1") + get_node_mock.assert_called_once_with(b".gitmodules") def test_get_node_returns_a_link(self): repository = mock.Mock() repository.alias = "git" commit = GitCommit(repository=repository, raw_id="abcdef12", idx=1) submodule_url = "https://code.rhodecode.com/dulwich" - get_id_patch = mock.patch.object(commit, "_get_tree_id_for_path", return_value=(1, "link")) + get_id_patch = mock.patch.object(commit, "_get_path_tree_id_and_type", return_value=(1, NodeKind.SUBMODULE)) get_submodule_patch = mock.patch.object(commit, "_get_submodule_url", return_value=submodule_url) with get_id_patch, get_submodule_patch as submodule_mock: - node = commit.get_node("/abcde") + node = commit.get_node(b"/abcde") - submodule_mock.assert_called_once_with("/abcde") + submodule_mock.assert_called_once_with(b"/abcde") assert type(node) == SubModuleNode assert node.url == submodule_url def test_get_nodes_returns_links(self): repository = mock.MagicMock() repository.alias = "git" - repository._remote.tree_items.return_value = [("subrepo", "stat", 1, "link")] + repository._remote.tree_items.return_value = [(b"subrepo", "stat", 1, NodeKind.SUBMODULE)] commit = GitCommit(repository=repository, raw_id="abcdef12", idx=1) submodule_url = "https://code.rhodecode.com/dulwich" - get_id_patch = mock.patch.object(commit, "_get_tree_id_for_path", return_value=(1, "tree")) + get_id_patch = mock.patch.object(commit, "_get_path_tree_id_and_type", return_value=(1, NodeKind.DIR)) get_submodule_patch = mock.patch.object(commit, "_get_submodule_url", return_value=submodule_url) with get_id_patch, get_submodule_patch as submodule_mock: - nodes = commit.get_nodes("/abcde") + nodes = commit.get_nodes(b"/abcde") - submodule_mock.assert_called_once_with("/abcde/subrepo") + submodule_mock.assert_called_once_with(b"/abcde/subrepo") assert len(nodes) == 1 assert type(nodes[0]) == SubModuleNode assert nodes[0].url == submodule_url class TestGetShadowInstance(object): - @pytest.fixture() def repo(self, vcsbackend_git): _git_repo = vcsbackend_git.repo diff --git a/rhodecode/tests/vcs/test_hg.py b/rhodecode/tests/vcs/test_hg.py index 49f90868..a768e55d 100644 --- a/rhodecode/tests/vcs/test_hg.py +++ b/rhodecode/tests/vcs/test_hg.py @@ -27,7 +27,7 @@ from rhodecode.lib.vcs import backends from rhodecode.lib.vcs.backends.base import Reference, MergeResponse, MergeFailureReason from rhodecode.lib.vcs.backends.hg import MercurialRepository, MercurialCommit from rhodecode.lib.vcs.exceptions import RepositoryError, VCSError, NodeDoesNotExistError, CommitDoesNotExistError -from rhodecode.lib.vcs.nodes import FileNode, NodeKind, NodeState +from rhodecode.lib.vcs.nodes import FileNode, NodeKind, DirNode, RootNode from rhodecode.tests import TEST_HG_REPO, TEST_HG_REPO_CLONE, repo_id_generator @@ -41,22 +41,17 @@ def repo_path_generator(): i = 0 while True: i += 1 - yield "%s-%d" % (TEST_HG_REPO_CLONE, i) + yield f"{TEST_HG_REPO_CLONE}-{i:d}" REPO_PATH_GENERATOR = repo_path_generator() -@pytest.fixture(scope="class", autouse=True) -def repo(request, baseapp): - repo = MercurialRepository(TEST_HG_REPO) - if request.cls: - request.cls.repo = repo - return repo - - -class TestMercurialRepository(object): +class TestMercurialRepository: # pylint: disable=protected-access + @pytest.fixture(autouse=True) + def prepare(self): + self.repo = MercurialRepository(TEST_HG_REPO) def get_clone_repo(self): """ @@ -100,9 +95,8 @@ class TestMercurialRepository(object): def test_repo_clone(self): if os.path.exists(TEST_HG_REPO_CLONE): - self.fail( - "Cannot test mercurial clone repo as location %s already " - "exists. You should manually remove it first." % TEST_HG_REPO_CLONE + pytest.fail( + f"Cannot test mercurial clone repo as location {TEST_HG_REPO_CLONE} already exists. You should manually remove it first." ) repo = MercurialRepository(TEST_HG_REPO) @@ -217,8 +211,8 @@ class TestMercurialRepository(object): assert "git" in self.repo._get_branches(closed=True) assert "web" in self.repo._get_branches(closed=True) - for name, id in self.repo.branches.items(): - assert isinstance(self.repo.get_commit(id), MercurialCommit) + for name, commit_id in self.repo.branches.items(): + assert isinstance(self.repo.get_commit(commit_id), MercurialCommit) def test_tip_in_tags(self): # tip is always a tag @@ -235,29 +229,38 @@ class TestMercurialRepository(object): assert init_commit.message == "initial import" assert init_author == "Marcin Kuzminski " assert init_author == init_commit.committer - assert sorted(init_commit._file_paths) == sorted( + assert sorted(init_commit.added_paths) == sorted( [ - "vcs/__init__.py", - "vcs/backends/BaseRepository.py", - "vcs/backends/__init__.py", + b"vcs/__init__.py", + b"vcs/backends/BaseRepository.py", + b"vcs/backends/__init__.py", + ] + ) + assert sorted(init_commit.affected_files) == sorted( + [ + b"vcs/__init__.py", + b"vcs/backends/BaseRepository.py", + b"vcs/backends/__init__.py", ] ) - assert sorted(init_commit._dir_paths) == sorted(["", "vcs", "vcs/backends"]) - assert init_commit._dir_paths + init_commit._file_paths == init_commit._paths + for path in (b"vcs/__init__.py", b"vcs/backends/BaseRepository.py", b"vcs/backends/__init__.py"): + assert isinstance(init_commit.get_node(path), FileNode) + for path in (b"", b"vcs", b"vcs/backends"): + assert isinstance(init_commit.get_node(path), DirNode) with pytest.raises(NodeDoesNotExistError): - init_commit.get_node(path="foobar") + init_commit.get_node(path=b"foobar") - node = init_commit.get_node("vcs/") + node = init_commit.get_node(b"vcs/") assert hasattr(node, "kind") assert node.kind == NodeKind.DIR - node = init_commit.get_node("vcs") + node = init_commit.get_node(b"vcs") assert hasattr(node, "kind") assert node.kind == NodeKind.DIR - node = init_commit.get_node("vcs/__init__.py") + node = init_commit.get_node(b"vcs/__init__.py") assert hasattr(node, "kind") assert node.kind == NodeKind.FILE @@ -279,7 +282,7 @@ class TestMercurialRepository(object): def test_commit10(self): commit10 = self.repo.get_commit(commit_idx=10) - README = """=== + readme = """=== VCS === @@ -291,9 +294,9 @@ Introduction TODO: To be written... """ - node = commit10.get_node("README.rst") + node = commit10.get_node(b"README.rst") assert node.kind == NodeKind.FILE - assert node.str_content == README + assert node.str_content == readme def test_local_clone(self): clone_path = next(REPO_PATH_GENERATOR) @@ -370,7 +373,7 @@ TODO: To be written... assert target_repo.branches["default"] == commit_id def test_local_pull_from_same_repo(self): - reference = Reference("branch", "default", None) + reference = Reference("branch", "default", "") with pytest.raises(ValueError): self.repo._local_pull(self.repo.path, reference) @@ -503,7 +506,7 @@ TODO: To be written... # Check we are not left in an intermediate merge state assert not os.path.exists(os.path.join(target_repo.path, ".hg", "merge", "state")) - def test_local_merge_of_two_branches_of_the_same_repo(self, backend_hg): + def test_local_merge_of_two_branches_of_the_same_repo(self, backend_hg, vcs_repo): commits = [ {"message": "a"}, {"message": "b", "branch": "b"}, @@ -639,7 +642,7 @@ TODO: To be written... # add an extra head to the target repo imc = target_repo.in_memory_commit - imc.add(FileNode(b"file_x", content="foo")) + imc.add(FileNode(b"file_x", content=b"foo")) commits = list(target_repo.get_commits()) imc.commit( message="Automatic commit from repo merge test", @@ -728,8 +731,7 @@ TODO: To be written... assert len(target_repo.commit_ids) == 2 + 2 -class TestGetShadowInstance(object): - +class TestGetShadowInstance: @pytest.fixture() def repo(self, vcsbackend_hg): _hg_repo = vcsbackend_hg.repo @@ -742,17 +744,21 @@ class TestGetShadowInstance(object): assert shadow.config.serialize() == repo.config.serialize() def test_disables_hooks_section(self, repo): - repo.config.set('hooks', 'foo', 'val') + repo.config.set("hooks", "foo", "val") shadow = repo.get_shadow_instance(repo.path) - assert not shadow.config.items('hooks') + assert not shadow.config.items("hooks") def test_allows_to_keep_hooks(self, repo): - repo.config.set('hooks', 'foo', 'val') + repo.config.set("hooks", "foo", "val") shadow = repo.get_shadow_instance(repo.path, enable_hooks=True) - assert shadow.config.items('hooks') + assert shadow.config.items("hooks") -class TestMercurialCommit(object): +class TestMercurialCommit: + @pytest.fixture(autouse=True) + def prepare(self): + self.repo = MercurialRepository(TEST_HG_REPO) + def _test_equality(self, commit): idx = commit.idx assert commit == self.repo.get_commit(commit_idx=idx) @@ -772,7 +778,7 @@ class TestMercurialCommit(object): def test_root_node(self): tip = self.repo.get_commit("tip") - assert tip.root is tip.get_node("") + assert tip.root is tip.get_node(b"") def test_lazy_fetch(self): """ @@ -788,28 +794,28 @@ class TestMercurialCommit(object): # accessing root.nodes updates commit.nodes assert len(commit.nodes) == 9 - docs = root.get_node("docs") + docs = commit.get_node(b"docs") # we haven't yet accessed anything new as docs dir was already cached assert len(commit.nodes) == 9 assert len(docs.nodes) == 8 # accessing docs.nodes updates commit.nodes assert len(commit.nodes) == 17 - assert docs is commit.get_node("docs") + assert docs is commit.get_node(b"docs") assert docs is root.nodes[0] assert docs is root.dirs[0] - assert docs is commit.get_node("docs") + assert docs is commit.get_node(b"docs") def test_nodes_with_commit(self): commit = self.repo.get_commit(commit_idx=45) root = commit.root - docs = root.get_node("docs") - assert docs is commit.get_node("docs") - api = docs.get_node("api") - assert api is commit.get_node("docs/api") - index = api.get_node("index.rst") - assert index is commit.get_node("docs/api/index.rst") - assert index is commit.get_node("docs").get_node("api").get_node("index.rst") + assert isinstance(root, RootNode) + docs = commit.get_node(b"docs") + assert docs is commit.get_node(b"docs") + api = commit.get_node(b"docs/api") + assert api is commit.get_node(b"docs/api") + index = commit.get_node(b"docs/api/index.rst") + assert index is commit.get_node(b"docs/api/index.rst") def test_branch_and_tags(self): commit0 = self.repo.get_commit(commit_idx=0) @@ -837,28 +843,28 @@ class TestMercurialCommit(object): def test_file_size(self): to_check = ( - (10, "setup.py", 1068), - (20, "setup.py", 1106), - (60, "setup.py", 1074), - (10, "vcs/backends/base.py", 2921), - (20, "vcs/backends/base.py", 3936), - (60, "vcs/backends/base.py", 6189), + (10, b"setup.py", 1068), + (20, b"setup.py", 1106), + (60, b"setup.py", 1074), + (10, b"vcs/backends/base.py", 2921), + (20, b"vcs/backends/base.py", 3936), + (60, b"vcs/backends/base.py", 6189), ) for idx, path, size in to_check: self._test_file_size(idx, path, size) def test_file_history_from_commits(self): - node = self.repo[10].get_node("setup.py") + node = self.repo[10].get_node(b"setup.py") commit_ids = [commit.raw_id for commit in node.history] assert ["3803844fdbd3b711175fc3da9bdacfcd6d29a6fb"] == commit_ids - node = self.repo[20].get_node("setup.py") + node = self.repo[20].get_node(b"setup.py") node_ids = [commit.raw_id for commit in node.history] assert ["eada5a770da98ab0dd7325e29d00e0714f228d09", "3803844fdbd3b711175fc3da9bdacfcd6d29a6fb"] == node_ids # special case we check history from commit that has this particular # file changed this means we check if it's included as well - node = self.repo.get_commit("eada5a770da98ab0dd7325e29d00e0714f228d09").get_node("setup.py") + node = self.repo.get_commit("eada5a770da98ab0dd7325e29d00e0714f228d09").get_node(b"setup.py") node_ids = [commit.raw_id for commit in node.history] assert ["eada5a770da98ab0dd7325e29d00e0714f228d09", "3803844fdbd3b711175fc3da9bdacfcd6d29a6fb"] == node_ids @@ -866,9 +872,9 @@ class TestMercurialCommit(object): # we can only check if those commits are present in the history # as we cannot update this test every time file is changed files = { - "setup.py": [7, 18, 45, 46, 47, 69, 77], - "vcs/nodes.py": [7, 8, 24, 26, 30, 45, 47, 49, 56, 57, 58, 59, 60, 61, 73, 76], - "vcs/backends/hg.py": [ + b"setup.py": [7, 18, 45, 46, 47, 69, 77], + b"vcs/nodes.py": [7, 8, 24, 26, 30, 45, 47, 49, 56, 57, 58, 59, 60, 61, 73, 76], + b"vcs/backends/hg.py": [ 4, 5, 6, @@ -927,7 +933,7 @@ class TestMercurialCommit(object): def test_file_annotate(self): files = { - "vcs/backends/__init__.py": { + b"vcs/backends/__init__.py": { 89: { "lines_no": 31, "commits": [ @@ -1002,7 +1008,7 @@ class TestMercurialCommit(object): ], }, }, - "vcs/exceptions.py": { + b"vcs/exceptions.py": { 89: { "lines_no": 18, "commits": [16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 17, 16, 16, 18, 18, 18], @@ -1016,25 +1022,25 @@ class TestMercurialCommit(object): "commits": [16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 17, 16, 16, 18, 18, 18], }, }, - "MANIFEST.in": { + b"MANIFEST.in": { 89: {"lines_no": 5, "commits": [7, 7, 7, 71, 71]}, 20: {"lines_no": 3, "commits": [7, 7, 7]}, 55: {"lines_no": 3, "commits": [7, 7, 7]}, }, } - for fname, commit_dict in files.items(): + for file_name, commit_dict in files.items(): for idx, __ in commit_dict.items(): commit = self.repo.get_commit(commit_idx=idx) - l1_1 = [x[1] for x in commit.get_file_annotate(fname)] - l1_2 = [x[2]().raw_id for x in commit.get_file_annotate(fname)] + l1_1 = [x[1] for x in commit.get_file_annotate(file_name)] + l1_2 = [x[2]().raw_id for x in commit.get_file_annotate(file_name)] assert l1_1 == l1_2 - l1 = l1_2 = [x[2]().idx for x in commit.get_file_annotate(fname)] - l2 = files[fname][idx]["commits"] + l1 = l1_2 = [x[2]().idx for x in commit.get_file_annotate(file_name)] + l2 = files[file_name][idx]["commits"] assert l1 == l2, ( "The lists of commit for %s@commit_id%s" "from annotation list should match each other," - "got \n%s \nvs \n%s " % (fname, idx, l1, l2) + "got \n%s \nvs \n%s " % (file_name, idx, l1, l2) ) def test_commit_state(self): @@ -1047,55 +1053,51 @@ class TestMercurialCommit(object): # changed: 13 # added: 20 # removed: 1 - changed = set( - [ - ".hgignore", - "README.rst", - "docs/conf.py", - "docs/index.rst", - "setup.py", - "tests/test_hg.py", - "tests/test_nodes.py", - "vcs/__init__.py", - "vcs/backends/__init__.py", - "vcs/backends/base.py", - "vcs/backends/hg.py", - "vcs/nodes.py", - "vcs/utils/__init__.py", - ] - ) + changed = { + b".hgignore", + b"README.rst", + b"docs/conf.py", + b"docs/index.rst", + b"setup.py", + b"tests/test_hg.py", + b"tests/test_nodes.py", + b"vcs/__init__.py", + b"vcs/backends/__init__.py", + b"vcs/backends/base.py", + b"vcs/backends/hg.py", + b"vcs/nodes.py", + b"vcs/utils/__init__.py", + } - added = set( - [ - "docs/api/backends/hg.rst", - "docs/api/backends/index.rst", - "docs/api/index.rst", - "docs/api/nodes.rst", - "docs/api/web/index.rst", - "docs/api/web/simplevcs.rst", - "docs/installation.rst", - "docs/quickstart.rst", - "setup.cfg", - "vcs/utils/baseui_config.py", - "vcs/utils/web.py", - "vcs/web/__init__.py", - "vcs/web/exceptions.py", - "vcs/web/simplevcs/__init__.py", - "vcs/web/simplevcs/exceptions.py", - "vcs/web/simplevcs/middleware.py", - "vcs/web/simplevcs/models.py", - "vcs/web/simplevcs/settings.py", - "vcs/web/simplevcs/utils.py", - "vcs/web/simplevcs/views.py", - ] - ) + added = { + b"docs/api/backends/hg.rst", + b"docs/api/backends/index.rst", + b"docs/api/index.rst", + b"docs/api/nodes.rst", + b"docs/api/web/index.rst", + b"docs/api/web/simplevcs.rst", + b"docs/installation.rst", + b"docs/quickstart.rst", + b"setup.cfg", + b"vcs/utils/baseui_config.py", + b"vcs/utils/web.py", + b"vcs/web/__init__.py", + b"vcs/web/exceptions.py", + b"vcs/web/simplevcs/__init__.py", + b"vcs/web/simplevcs/exceptions.py", + b"vcs/web/simplevcs/middleware.py", + b"vcs/web/simplevcs/models.py", + b"vcs/web/simplevcs/settings.py", + b"vcs/web/simplevcs/utils.py", + b"vcs/web/simplevcs/views.py", + } - removed = set(["docs/api.rst"]) + removed = {b"docs/api.rst"} commit64 = self.repo.get_commit("46ad32a4f974") - assert set((node.path for node in commit64.added)) == added - assert set((node.path for node in commit64.changed)) == changed - assert set((node.path for node in commit64.removed)) == removed + assert set((node for node in commit64.added_paths)) == added + assert set((node for node in commit64.changed_paths)) == changed + assert set((node for node in commit64.removed_paths)) == removed # commit_id b090f22d27d6: # hg st --rev b090f22d27d6 @@ -1103,9 +1105,9 @@ class TestMercurialCommit(object): # added: 20 # removed: 1 commit88 = self.repo.get_commit("b090f22d27d6") - assert set((node.path for node in commit88.added)) == set() - assert set((node.path for node in commit88.changed)) == set([".hgignore"]) - assert set((node.path for node in commit88.removed)) == set() + assert set((node for node in commit88.added_paths)) == set() + assert set((node for node in commit88.changed_paths)) == {b".hgignore"} + assert set((node for node in commit88.removed_paths)) == set() # # 85: @@ -1114,55 +1116,40 @@ class TestMercurialCommit(object): # changed: 4 ['vcs/web/simplevcs/models.py', ...] # removed: 1 ['vcs/utils/web.py'] commit85 = self.repo.get_commit(commit_idx=85) - assert set((node.path for node in commit85.added)) == set( - ["vcs/utils/diffs.py", "vcs/web/simplevcs/views/diffs.py"] - ) - assert set((node.path for node in commit85.changed)) == set( - [ - "vcs/web/simplevcs/models.py", - "vcs/web/simplevcs/utils.py", - "vcs/web/simplevcs/views/__init__.py", - "vcs/web/simplevcs/views/repository.py", - ] - ) - assert set((node.path for node in commit85.removed)) == set(["vcs/utils/web.py"]) + assert set((node for node in commit85.added_paths)) == {b"vcs/utils/diffs.py", b"vcs/web/simplevcs/views/diffs.py"} + assert set((node for node in commit85.changed_paths)) == { + b"vcs/web/simplevcs/models.py", + b"vcs/web/simplevcs/utils.py", + b"vcs/web/simplevcs/views/__init__.py", + b"vcs/web/simplevcs/views/repository.py", + } + assert set((node for node in commit85.removed_paths)) == {b"vcs/utils/web.py"} def test_files_state(self): """ Tests state of FileNodes. """ commit = self.repo.get_commit(commit_idx=85) - node = commit.get_node("vcs/utils/diffs.py") - assert node.state, NodeState.ADDED - assert node.added - assert not node.changed - assert not node.not_changed - assert not node.removed + node = commit.get_node(b"vcs/utils/diffs.py") + assert node.bytes_path in commit.added_paths commit = self.repo.get_commit(commit_idx=88) - node = commit.get_node(".hgignore") - assert node.state, NodeState.CHANGED - assert not node.added - assert node.changed - assert not node.not_changed - assert not node.removed + node = commit.get_node(b".hgignore") + assert node.bytes_path in commit.changed_paths commit = self.repo.get_commit(commit_idx=85) - node = commit.get_node("setup.py") - assert node.state, NodeState.NOT_CHANGED - assert not node.added - assert not node.changed - assert node.not_changed - assert not node.removed + node = commit.get_node(b"setup.py") + assert node.bytes_path not in commit.affected_files # If node has REMOVED state then trying to fetch it would raise # CommitError exception commit = self.repo.get_commit(commit_idx=2) - path = "vcs/backends/BaseRepository.py" + path = b"vcs/backends/BaseRepository.py" with pytest.raises(NodeDoesNotExistError): commit.get_node(path) + # but it would be one of ``removed`` (commit's attribute) - assert path in [rf.path for rf in commit.removed] + assert path in [rf for rf in commit.removed_paths] def test_commit_message_is_unicode(self): for cm in self.repo: @@ -1174,14 +1161,14 @@ class TestMercurialCommit(object): def test_repo_files_content_type(self): test_commit = self.repo.get_commit(commit_idx=100) - for node in test_commit.get_node("/"): + for node in test_commit.get_node(b"/"): if node.is_file(): assert type(node.content) == bytes assert type(node.str_content) == str def test_wrong_path(self): # There is 'setup.py' in the root dir but not there: - path = "foo/bar/setup.py" + path = b"foo/bar/setup.py" with pytest.raises(VCSError): self.repo.get_commit().get_node(path) @@ -1196,23 +1183,27 @@ class TestMercurialCommit(object): assert "marcink" == self.repo.get_commit("84478366594b").author_name -class TestLargeFileRepo(object): +class TestLargeFileRepo: def test_large_file(self, backend_hg): conf = make_db_config() hg_largefiles_store = conf.get("largefiles", "usercache") repo = backend_hg.create_test_repo("largefiles", conf) tip = repo.scm_instance().get_commit() - node = tip.get_node(".hglf/thisfileislarge") + node = tip.get_node(b".hglf/thisfileislarge") lf_node = node.get_largefile_node() assert lf_node.is_largefile() is True assert lf_node.size == 1024000 - assert lf_node.name == ".hglf/thisfileislarge" + assert lf_node.name == b".hglf/thisfileislarge" -class TestGetBranchName(object): +class TestGetBranchName: + @pytest.fixture(autouse=True) + def prepare(self): + self.repo = MercurialRepository(TEST_HG_REPO) + def test_returns_ref_name_when_type_is_branch(self): ref = self._create_ref("branch", "fake-name") result = self.repo._get_branch_name(ref) @@ -1235,7 +1226,11 @@ class TestGetBranchName(object): return ref -class TestIsTheSameBranch(object): +class TestIsTheSameBranch: + @pytest.fixture(autouse=True) + def prepare(self): + self.repo = MercurialRepository(TEST_HG_REPO) + def test_returns_true_when_branches_are_equal(self): source_ref = mock.Mock(name="source-ref") target_ref = mock.Mock(name="target-ref") diff --git a/rhodecode/tests/vcs/test_inmemory.py b/rhodecode/tests/vcs/test_inmemory.py index 2d7d49f1..8d52e43b 100644 --- a/rhodecode/tests/vcs/test_inmemory.py +++ b/rhodecode/tests/vcs/test_inmemory.py @@ -128,8 +128,8 @@ class TestInMemoryCommit(BackendTestMixin): ] self.imc.add(*to_add) commit = self.imc.commit("Initial", "joe doe ") - assert isinstance(commit.get_node("foo"), DirNode) - assert isinstance(commit.get_node("foo/bar"), DirNode) + assert isinstance(commit.get_node(b"foo"), DirNode) + assert isinstance(commit.get_node(b"foo/bar"), DirNode) self.assert_nodes_in_commit(commit, to_add) # commit some more files again @@ -244,7 +244,7 @@ class TestInMemoryCommit(BackendTestMixin): tip = self.repo.get_commit() node = nodes[0] - assert node.content == tip.get_node(node.path).content + assert node.content == tip.get_node(node.bytes_path).content self.imc.remove(node) self.imc.commit(message=f"Removed {node.path}", author="Some Name ") @@ -252,7 +252,7 @@ class TestInMemoryCommit(BackendTestMixin): assert tip != newtip assert tip.id != newtip.id with pytest.raises(NodeDoesNotExistError): - newtip.get_node(node.path) + newtip.get_node(node.bytes_path) def test_remove_last_file_from_directory(self): node = FileNode(b"omg/qwe/foo/bar", content=b"foobar") @@ -262,7 +262,7 @@ class TestInMemoryCommit(BackendTestMixin): self.imc.remove(node) tip = self.imc.commit("removed", "joe doe ") with pytest.raises(NodeDoesNotExistError): - tip.get_node("omg/qwe/foo/bar") + tip.get_node(b"omg/qwe/foo/bar") def test_remove_raise_node_does_not_exist(self, nodes): self.imc.remove(nodes[0]) @@ -338,5 +338,5 @@ class TestInMemoryCommit(BackendTestMixin): def assert_nodes_in_commit(self, commit, nodes): for node in nodes: - assert commit.get_node(node.path).path == node.path - assert commit.get_node(node.path).content == node.content + assert commit.get_node(node.bytes_path).path == node.path + assert commit.get_node(node.bytes_path).content == node.content diff --git a/rhodecode/tests/vcs/test_nodes.py b/rhodecode/tests/vcs/test_nodes.py index 77e23da5..0efe1821 100644 --- a/rhodecode/tests/vcs/test_nodes.py +++ b/rhodecode/tests/vcs/test_nodes.py @@ -155,30 +155,6 @@ class TestNodeBasics: with pytest.raises(NodeError): node.content # noqa - def test_dir_node_iter(self): - nodes = [ - DirNode(b"docs"), - DirNode(b"tests"), - FileNode(b"bar"), - FileNode(b"foo"), - FileNode(b"readme.txt"), - FileNode(b"setup.py"), - ] - dirnode = DirNode(b"", nodes=nodes) - for node in dirnode: - assert node == dirnode.get_node(node.path) - - def test_node_state(self): - """ - Without link to commit nodes should raise NodeError. - """ - node = FileNode(b"anything") - with pytest.raises(NodeError): - node.state # noqa - node = DirNode(b"anything") - with pytest.raises(NodeError): - node.state # noqa - def test_file_node_stat(self): node = FileNode(b"foobar", b"empty... almost") mode = node.mode # default should be 0100644 @@ -272,5 +248,5 @@ class TestNodesCommits(BackendTestMixin): last_commit = repo.get_commit() for x in range(3): - node = last_commit.get_node(f"file_{x}.txt") + node = last_commit.get_node(b"file_%d.txt" % x) assert node.last_commit == repo[x] diff --git a/rhodecode/tests/vcs/test_svn.py b/rhodecode/tests/vcs/test_svn.py index 9eb64898..bb333593 100644 --- a/rhodecode/tests/vcs/test_svn.py +++ b/rhodecode/tests/vcs/test_svn.py @@ -128,7 +128,7 @@ def test_read_full_file_tree(head): def test_topnode_files_attribute(head): - topnode = head.get_node("") + topnode = head.get_node(b"") topnode.files @@ -173,23 +173,23 @@ class TestSVNCommit(object): self.repo = repo def test_file_history_from_commits(self): - node = self.repo[10].get_node("setup.py") + node = self.repo[10].get_node(b"setup.py") commit_ids = [commit.raw_id for commit in node.history] assert ["8"] == commit_ids - node = self.repo[20].get_node("setup.py") + node = self.repo[20].get_node(b"setup.py") node_ids = [commit.raw_id for commit in node.history] assert ["18", "8"] == node_ids # special case we check history from commit that has this particular # file changed this means we check if it's included as well - node = self.repo.get_commit("18").get_node("setup.py") + node = self.repo.get_commit("18").get_node(b"setup.py") node_ids = [commit.raw_id for commit in node.history] assert ["18", "8"] == node_ids def test_repo_files_content_type(self): test_commit = self.repo.get_commit(commit_idx=100) - for node in test_commit.get_node("/"): + for node in test_commit.get_node(b"/"): if node.is_file(): assert type(node.content) == bytes assert type(node.str_content) == str diff --git a/rhodecode/tests/vcs/test_tags.py b/rhodecode/tests/vcs/test_tags.py index 306f0aca..d0504f53 100644 --- a/rhodecode/tests/vcs/test_tags.py +++ b/rhodecode/tests/vcs/test_tags.py @@ -30,11 +30,11 @@ class TestTags(BackendTestMixin): def test_new_tag(self): tip = self.repo.get_commit() tagsize = len(self.repo.tags) - tag = self.repo.tag("last-commit", "joe", tip.raw_id) + tag_commit = self.repo.tag("last-commit", "joe", tip.raw_id) assert len(self.repo.tags) == tagsize + 1 for top, __, __ in tip.walk(): - assert top == tag.get_node(top.path) + assert top == tag_commit.get_node(top.bytes_path) def test_tag_already_exist(self): tip = self.repo.get_commit() diff --git a/rhodecode/tests/vcs_operations/test_vcs_operations_bad_client.py b/rhodecode/tests/vcs_operations/test_vcs_operations_bad_client.py index 6e5159fb..960b2c9f 100644 --- a/rhodecode/tests/vcs_operations/test_vcs_operations_bad_client.py +++ b/rhodecode/tests/vcs_operations/test_vcs_operations_bad_client.py @@ -65,7 +65,7 @@ class TestVCSOperationsOnUsingBadClient(object): # push fails repo is locked by other user ! push_url = rcstack.repo_clone_url(HG_REPO) stdout, stderr = _add_files_and_push("hg", tmpdir.strpath, clone_url=push_url) - msg = "Your hg client (ver=mercurial/proto-1.0 (Mercurial 6.7.4)) is forbidden by security rules" + msg = "Your hg client (version=mercurial/proto-1.0 (Mercurial 6.7.4)) is forbidden by security rules" assert msg in stderr def test_push_with_bad_client_repo_by_other_user_git(self, rcstack, tmpdir): @@ -81,7 +81,7 @@ class TestVCSOperationsOnUsingBadClient(object): push_url = rcstack.repo_clone_url(GIT_REPO) stdout, stderr = _add_files_and_push("git", tmpdir.strpath, clone_url=push_url) - err = "Your git client (ver=git/2.45.2) is forbidden by security rules" + err = "Your git client (version=git/2.45.2) is forbidden by security rules" assert err in stderr @pytest.mark.xfail(reason="Lack of proper SVN support of cloning") diff --git a/rhodecode/tests/vcs_operations/test_vcs_operations_svn.py b/rhodecode/tests/vcs_operations/test_vcs_operations_svn.py index 6f4b8a9b..84ea96fa 100644 --- a/rhodecode/tests/vcs_operations/test_vcs_operations_svn.py +++ b/rhodecode/tests/vcs_operations/test_vcs_operations_svn.py @@ -124,7 +124,6 @@ class TestVCSOperationsSVN(object): assert 'not found' in stderr - @pytest.mark.xfail(reason='Lack of proper SVN support of cloning') def test_clone_existing_path_svn_not_in_database( self, rcstack, tmpdir, fs_repo_only): db_name = fs_repo_only('not-in-db-git', repo_type='git') @@ -136,7 +135,6 @@ class TestVCSOperationsSVN(object): f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath) assert 'not found' in stderr - @pytest.mark.xfail(reason='Lack of proper SVN support of cloning') def test_clone_existing_path_svn_not_in_database_different_scm( self, rcstack, tmpdir, fs_repo_only): db_name = fs_repo_only('not-in-db-hg', repo_type='hg') @@ -149,7 +147,6 @@ class TestVCSOperationsSVN(object): f'svn checkout {flags} {auth}', clone_url, tmpdir.strpath) assert 'not found' in stderr - @pytest.mark.xfail(reason='Lack of proper SVN support of cloning') def test_clone_non_existing_store_path_svn(self, rcstack, tmpdir, user_util): repo = user_util.create_repo(repo_type='git') clone_url = rcstack.repo_clone_url(repo.repo_name)