fix(encoding for file): fixed support of non utf-8 files in all backends
This commit is contained in:
RhodeCode Admin
parent
f687bfc994
commit
d08633d220
42 changed files with 2001 additions and 1973 deletions
|
|
@ -21,8 +21,8 @@ GIT commit module
|
|||
"""
|
||||
|
||||
import io
|
||||
import stat
|
||||
import configparser
|
||||
import logging
|
||||
from itertools import chain
|
||||
|
||||
from zope.cachedescriptors.property import Lazy as LazyProperty
|
||||
|
|
@ -32,9 +32,16 @@ from rhodecode.lib.str_utils import safe_bytes, safe_str
|
|||
from rhodecode.lib.vcs.backends import base
|
||||
from rhodecode.lib.vcs.exceptions import CommitError, NodeDoesNotExistError
|
||||
from rhodecode.lib.vcs.nodes import (
|
||||
FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
|
||||
ChangedFileNodesGenerator, AddedFileNodesGenerator,
|
||||
RemovedFileNodesGenerator, LargeFileNode)
|
||||
FileNode,
|
||||
DirNode,
|
||||
NodeKind,
|
||||
RootNode,
|
||||
SubModuleNode,
|
||||
LargeFileNode,
|
||||
)
|
||||
from rhodecode.lib.vcs_common import FILEMODE_LINK
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GitCommit(base.BaseCommit):
|
||||
|
|
@ -50,13 +57,11 @@ class GitCommit(base.BaseCommit):
|
|||
# done through a more complex tree walk on parents
|
||||
"status",
|
||||
# mercurial specific property not supported here
|
||||
"_file_paths",
|
||||
"obsolete",
|
||||
# mercurial specific property not supported here
|
||||
'obsolete',
|
||||
"phase",
|
||||
# mercurial specific property not supported here
|
||||
'phase',
|
||||
# mercurial specific property not supported here
|
||||
'hidden'
|
||||
"hidden",
|
||||
]
|
||||
|
||||
def __init__(self, repository, raw_id, idx, pre_load=None):
|
||||
|
|
@ -69,17 +74,16 @@ class GitCommit(base.BaseCommit):
|
|||
self._set_bulk_properties(pre_load)
|
||||
|
||||
# caches
|
||||
self._stat_modes = {} # stat info for paths
|
||||
self._paths = {} # path processed with parse_tree
|
||||
self.nodes = {}
|
||||
self._path_mode_cache = {} # path stats cache, e.g filemode etc
|
||||
self._path_type_cache = {} # path type dir/file/link etc cache
|
||||
|
||||
self._submodules = None
|
||||
|
||||
def _set_bulk_properties(self, pre_load):
|
||||
|
||||
if not pre_load:
|
||||
return
|
||||
pre_load = [entry for entry in pre_load
|
||||
if entry not in self._filter_pre_load]
|
||||
pre_load = [entry for entry in pre_load if entry not in self._filter_pre_load]
|
||||
if not pre_load:
|
||||
return
|
||||
|
||||
|
|
@ -102,7 +106,7 @@ class GitCommit(base.BaseCommit):
|
|||
|
||||
@LazyProperty
|
||||
def _tree_id(self):
|
||||
return self._remote[self._commit['tree']]['id']
|
||||
return self._remote[self._commit["tree"]]["id"]
|
||||
|
||||
@LazyProperty
|
||||
def id(self):
|
||||
|
|
@ -134,13 +138,12 @@ class GitCommit(base.BaseCommit):
|
|||
"""
|
||||
Returns modified, added, removed, deleted files for current commit
|
||||
"""
|
||||
return self.changed, self.added, self.removed
|
||||
added, modified, deleted = self._changes_cache
|
||||
return list(modified), list(modified), list(deleted)
|
||||
|
||||
@LazyProperty
|
||||
def tags(self):
|
||||
tags = [safe_str(name) for name,
|
||||
commit_id in self.repository.tags.items()
|
||||
if commit_id == self.raw_id]
|
||||
tags = [safe_str(name) for name, commit_id in self.repository.tags.items() if commit_id == self.raw_id]
|
||||
return tags
|
||||
|
||||
@LazyProperty
|
||||
|
|
@ -161,47 +164,33 @@ class GitCommit(base.BaseCommit):
|
|||
branches = self._remote.branch(self.raw_id)
|
||||
return self._set_branch(branches)
|
||||
|
||||
def _get_tree_id_for_path(self, path):
|
||||
def _get_path_tree_id_and_type(self, path: bytes):
|
||||
|
||||
path = safe_str(path)
|
||||
if path in self._paths:
|
||||
return self._paths[path]
|
||||
if path in self._path_type_cache:
|
||||
return self._path_type_cache[path]
|
||||
|
||||
tree_id = self._tree_id
|
||||
if path == b"":
|
||||
self._path_type_cache[b""] = [self._tree_id, NodeKind.DIR]
|
||||
return self._path_type_cache[path]
|
||||
|
||||
path = path.strip('/')
|
||||
if path == '':
|
||||
data = [tree_id, "tree"]
|
||||
self._paths[''] = data
|
||||
return data
|
||||
|
||||
tree_id, tree_type, tree_mode = \
|
||||
self._remote.tree_and_type_for_path(self.raw_id, path)
|
||||
tree_id, tree_type, tree_mode = self._remote.tree_and_type_for_path(self.raw_id, path)
|
||||
if tree_id is None:
|
||||
raise self.no_node_at_path(path)
|
||||
|
||||
self._paths[path] = [tree_id, tree_type]
|
||||
self._stat_modes[path] = tree_mode
|
||||
self._path_type_cache[path] = [tree_id, tree_type]
|
||||
self._path_mode_cache[path] = tree_mode
|
||||
|
||||
if path not in self._paths:
|
||||
raise self.no_node_at_path(path)
|
||||
|
||||
return self._paths[path]
|
||||
return self._path_type_cache[path]
|
||||
|
||||
def _get_kind(self, path):
|
||||
tree_id, type_ = self._get_tree_id_for_path(path)
|
||||
if type_ == 'blob':
|
||||
return NodeKind.FILE
|
||||
elif type_ == 'tree':
|
||||
return NodeKind.DIR
|
||||
elif type_ == 'link':
|
||||
return NodeKind.SUBMODULE
|
||||
return None
|
||||
path = self._fix_path(path)
|
||||
_, path_type = self._get_path_tree_id_and_type(path)
|
||||
return path_type
|
||||
|
||||
def _assert_is_path(self, path):
|
||||
path = self._fix_path(path)
|
||||
if self._get_kind(path) != NodeKind.FILE:
|
||||
raise CommitError(f"File does not exist for commit {self.raw_id} at '{path}'")
|
||||
raise CommitError(f"File at path={path} does not exist for commit {self.raw_id}")
|
||||
return path
|
||||
|
||||
def _get_file_nodes(self):
|
||||
|
|
@ -237,15 +226,19 @@ class GitCommit(base.BaseCommit):
|
|||
path = self._assert_is_path(path)
|
||||
|
||||
# ensure path is traversed
|
||||
self._get_tree_id_for_path(path)
|
||||
self._get_path_tree_id_and_type(path)
|
||||
|
||||
return self._stat_modes[path]
|
||||
return self._path_mode_cache[path]
|
||||
|
||||
def is_link(self, path):
|
||||
return stat.S_ISLNK(self.get_file_mode(path))
|
||||
def is_link(self, path: bytes):
|
||||
path = self._assert_is_path(path)
|
||||
if path not in self._path_mode_cache:
|
||||
self._path_mode_cache[path] = self._remote.fctx_flags(self.raw_id, path)
|
||||
|
||||
return self._path_mode_cache[path] == FILEMODE_LINK
|
||||
|
||||
def is_node_binary(self, path):
|
||||
tree_id, _ = self._get_tree_id_for_path(path)
|
||||
tree_id, _ = self._get_path_tree_id_and_type(path)
|
||||
return self._remote.is_binary(tree_id)
|
||||
|
||||
def node_md5_hash(self, path):
|
||||
|
|
@ -256,19 +249,19 @@ class GitCommit(base.BaseCommit):
|
|||
"""
|
||||
Returns content of the file at given `path`.
|
||||
"""
|
||||
tree_id, _ = self._get_tree_id_for_path(path)
|
||||
tree_id, _ = self._get_path_tree_id_and_type(path)
|
||||
return self._remote.blob_as_pretty_string(tree_id)
|
||||
|
||||
def get_file_content_streamed(self, path):
|
||||
tree_id, _ = self._get_tree_id_for_path(path)
|
||||
stream_method = getattr(self._remote, 'stream:blob_as_pretty_string')
|
||||
tree_id, _ = self._get_path_tree_id_and_type(path)
|
||||
stream_method = getattr(self._remote, "stream:blob_as_pretty_string")
|
||||
return stream_method(tree_id)
|
||||
|
||||
def get_file_size(self, path):
|
||||
"""
|
||||
Returns size of the file at given `path`.
|
||||
"""
|
||||
tree_id, _ = self._get_tree_id_for_path(path)
|
||||
tree_id, _ = self._get_path_tree_id_and_type(path)
|
||||
return self._remote.blob_raw_length(tree_id)
|
||||
|
||||
def get_path_history(self, path, limit=None, pre_load=None):
|
||||
|
|
@ -276,12 +269,9 @@ class GitCommit(base.BaseCommit):
|
|||
Returns history of file as reversed list of `GitCommit` objects for
|
||||
which file at given `path` has been modified.
|
||||
"""
|
||||
|
||||
path = self._assert_is_path(path)
|
||||
hist = self._remote.node_history(self.raw_id, path, limit)
|
||||
return [
|
||||
self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
|
||||
for commit_id in hist]
|
||||
history = self._remote.node_history(self.raw_id, path, limit)
|
||||
return [self.repository.get_commit(commit_id=commit_id, pre_load=pre_load) for commit_id in history]
|
||||
|
||||
def get_file_annotate(self, path, pre_load=None):
|
||||
"""
|
||||
|
|
@ -293,95 +283,105 @@ class GitCommit(base.BaseCommit):
|
|||
|
||||
for ln_no, commit_id, content in result:
|
||||
yield (
|
||||
ln_no, commit_id,
|
||||
ln_no,
|
||||
commit_id,
|
||||
lambda: self.repository.get_commit(commit_id=commit_id, pre_load=pre_load),
|
||||
content)
|
||||
content,
|
||||
)
|
||||
|
||||
def get_nodes(self, path, pre_load=None):
|
||||
def get_nodes(self, path: bytes, pre_load=None):
|
||||
|
||||
if self._get_kind(path) != NodeKind.DIR:
|
||||
raise CommitError(
|
||||
f"Directory does not exist for commit {self.raw_id} at '{path}'")
|
||||
raise CommitError(f"Directory does not exist for commit {self.raw_id} at '{path}'")
|
||||
path = self._fix_path(path)
|
||||
|
||||
tree_id, _ = self._get_tree_id_for_path(path)
|
||||
# call and check tree_id for this path
|
||||
tree_id, _ = self._get_path_tree_id_and_type(path)
|
||||
|
||||
dirnodes = []
|
||||
filenodes = []
|
||||
path_nodes = []
|
||||
|
||||
# extracted tree ID gives us our files...
|
||||
str_path = safe_str(path) # libgit operates on bytes
|
||||
for name, stat_, id_, type_ in self._remote.tree_items(tree_id):
|
||||
if type_ == 'link':
|
||||
url = self._get_submodule_url('/'.join((str_path, name)))
|
||||
dirnodes.append(SubModuleNode(
|
||||
name, url=url, commit=id_, alias=self.repository.alias))
|
||||
continue
|
||||
for bytes_name, stat_, tree_item_id, node_kind in self._remote.tree_items(tree_id):
|
||||
if node_kind is None:
|
||||
raise CommitError(f"Requested object type={node_kind} cannot be determined")
|
||||
|
||||
if str_path != '':
|
||||
obj_path = '/'.join((str_path, name))
|
||||
if path != b"":
|
||||
obj_path = b"/".join((path, bytes_name))
|
||||
else:
|
||||
obj_path = name
|
||||
if obj_path not in self._stat_modes:
|
||||
self._stat_modes[obj_path] = stat_
|
||||
obj_path = bytes_name
|
||||
|
||||
if type_ == 'tree':
|
||||
dirnodes.append(DirNode(safe_bytes(obj_path), commit=self))
|
||||
elif type_ == 'blob':
|
||||
filenodes.append(FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load))
|
||||
# cache file mode for git, since we have it already
|
||||
if obj_path not in self._path_mode_cache:
|
||||
self._path_mode_cache[obj_path] = stat_
|
||||
|
||||
# cache type
|
||||
if node_kind not in self._path_type_cache:
|
||||
self._path_type_cache[obj_path] = [tree_item_id, node_kind]
|
||||
|
||||
entry = None
|
||||
if obj_path in self.nodes:
|
||||
entry = self.nodes[obj_path]
|
||||
else:
|
||||
raise CommitError(f"Requested object should be Tree or Blob, is {type_}")
|
||||
if node_kind == NodeKind.SUBMODULE:
|
||||
url = self._get_submodule_url(b"/".join((path, bytes_name)))
|
||||
entry= SubModuleNode(bytes_name, url=url, commit=tree_item_id, alias=self.repository.alias)
|
||||
elif node_kind == NodeKind.DIR:
|
||||
entry = DirNode(safe_bytes(obj_path), commit=self)
|
||||
elif node_kind == NodeKind.FILE:
|
||||
entry = FileNode(safe_bytes(obj_path), commit=self, mode=stat_, pre_load=pre_load)
|
||||
|
||||
nodes = dirnodes + filenodes
|
||||
for node in nodes:
|
||||
if node.path not in self.nodes:
|
||||
self.nodes[node.path] = node
|
||||
nodes.sort()
|
||||
return nodes
|
||||
if entry:
|
||||
self.nodes[obj_path] = entry
|
||||
path_nodes.append(entry)
|
||||
|
||||
def get_node(self, path, pre_load=None):
|
||||
path_nodes.sort()
|
||||
return path_nodes
|
||||
|
||||
def get_node(self, path: bytes, pre_load=None):
|
||||
path = self._fix_path(path)
|
||||
if path not in self.nodes:
|
||||
try:
|
||||
tree_id, type_ = self._get_tree_id_for_path(path)
|
||||
except CommitError:
|
||||
raise NodeDoesNotExistError(
|
||||
f"Cannot find one of parents' directories for a given "
|
||||
f"path: {path}")
|
||||
|
||||
if type_ in ['link', 'commit']:
|
||||
# use cached, if we have one
|
||||
if path in self.nodes:
|
||||
return self.nodes[path]
|
||||
|
||||
try:
|
||||
tree_id, path_type = self._get_path_tree_id_and_type(path)
|
||||
except CommitError:
|
||||
raise NodeDoesNotExistError(f"Cannot find one of parents' directories for a given path: {path}")
|
||||
|
||||
if path == b"":
|
||||
node = RootNode(commit=self)
|
||||
else:
|
||||
if path_type == NodeKind.SUBMODULE:
|
||||
url = self._get_submodule_url(path)
|
||||
node = SubModuleNode(path, url=url, commit=tree_id,
|
||||
alias=self.repository.alias)
|
||||
elif type_ == 'tree':
|
||||
if path == '':
|
||||
node = RootNode(commit=self)
|
||||
else:
|
||||
node = DirNode(safe_bytes(path), commit=self)
|
||||
elif type_ == 'blob':
|
||||
node = SubModuleNode(path, url=url, commit=tree_id, alias=self.repository.alias)
|
||||
elif path_type == NodeKind.DIR:
|
||||
node = DirNode(safe_bytes(path), commit=self)
|
||||
elif path_type == NodeKind.FILE:
|
||||
node = FileNode(safe_bytes(path), commit=self, pre_load=pre_load)
|
||||
self._stat_modes[path] = node.mode
|
||||
self._path_mode_cache[path] = node.mode
|
||||
else:
|
||||
raise self.no_node_at_path(path)
|
||||
|
||||
# cache node
|
||||
self.nodes[path] = node
|
||||
|
||||
# cache node
|
||||
self.nodes[path] = node
|
||||
return self.nodes[path]
|
||||
|
||||
def get_largefile_node(self, path):
|
||||
tree_id, _ = self._get_tree_id_for_path(path)
|
||||
def get_largefile_node(self, path: bytes):
|
||||
tree_id, _ = self._get_path_tree_id_and_type(path)
|
||||
pointer_spec = self._remote.is_large_file(tree_id)
|
||||
|
||||
if pointer_spec:
|
||||
# content of that file regular FileNode is the hash of largefile
|
||||
file_id = pointer_spec.get('oid_hash')
|
||||
if self._remote.in_largefiles_store(file_id):
|
||||
lf_path = self._remote.store_path(file_id)
|
||||
return LargeFileNode(safe_bytes(lf_path), commit=self, org_path=path)
|
||||
file_id = pointer_spec.get("oid_hash")
|
||||
if not self._remote.in_largefiles_store(file_id):
|
||||
log.warning(f'Largefile oid={file_id} not found in store')
|
||||
return None
|
||||
|
||||
lf_path = self._remote.store_path(file_id)
|
||||
return LargeFileNode(safe_bytes(lf_path), commit=self, org_path=path)
|
||||
|
||||
@LazyProperty
|
||||
def affected_files(self):
|
||||
def affected_files(self) -> list[bytes]:
|
||||
"""
|
||||
Gets a fast accessible file changes for given commit
|
||||
"""
|
||||
|
|
@ -389,7 +389,7 @@ class GitCommit(base.BaseCommit):
|
|||
return list(added.union(modified).union(deleted))
|
||||
|
||||
@LazyProperty
|
||||
def _changes_cache(self):
|
||||
def _changes_cache(self) -> tuple[set, set, set]:
|
||||
added = set()
|
||||
modified = set()
|
||||
deleted = set()
|
||||
|
|
@ -416,53 +416,22 @@ class GitCommit(base.BaseCommit):
|
|||
:param status: one of: *added*, *modified* or *deleted*
|
||||
"""
|
||||
added, modified, deleted = self._changes_cache
|
||||
return sorted({
|
||||
'added': list(added),
|
||||
'modified': list(modified),
|
||||
'deleted': list(deleted)}[status]
|
||||
)
|
||||
|
||||
@LazyProperty
|
||||
def added(self):
|
||||
"""
|
||||
Returns list of added ``FileNode`` objects.
|
||||
"""
|
||||
if not self.parents:
|
||||
return list(self._get_file_nodes())
|
||||
return AddedFileNodesGenerator(self.added_paths, self)
|
||||
return sorted({"added": list(added), "modified": list(modified), "deleted": list(deleted)}[status])
|
||||
|
||||
@LazyProperty
|
||||
def added_paths(self):
|
||||
return [n for n in self._get_paths_for_status('added')]
|
||||
|
||||
@LazyProperty
|
||||
def changed(self):
|
||||
"""
|
||||
Returns list of modified ``FileNode`` objects.
|
||||
"""
|
||||
if not self.parents:
|
||||
return []
|
||||
return ChangedFileNodesGenerator(self.changed_paths, self)
|
||||
return [n for n in self._get_paths_for_status("added")]
|
||||
|
||||
@LazyProperty
|
||||
def changed_paths(self):
|
||||
return [n for n in self._get_paths_for_status('modified')]
|
||||
|
||||
@LazyProperty
|
||||
def removed(self):
|
||||
"""
|
||||
Returns list of removed ``FileNode`` objects.
|
||||
"""
|
||||
if not self.parents:
|
||||
return []
|
||||
return RemovedFileNodesGenerator(self.removed_paths, self)
|
||||
return [n for n in self._get_paths_for_status("modified")]
|
||||
|
||||
@LazyProperty
|
||||
def removed_paths(self):
|
||||
return [n for n in self._get_paths_for_status('deleted')]
|
||||
return [n for n in self._get_paths_for_status("deleted")]
|
||||
|
||||
def _get_submodule_url(self, submodule_path):
|
||||
git_modules_path = '.gitmodules'
|
||||
def _get_submodule_url(self, submodule_path: bytes):
|
||||
git_modules_path = b".gitmodules"
|
||||
|
||||
if self._submodules is None:
|
||||
self._submodules = {}
|
||||
|
|
@ -476,9 +445,9 @@ class GitCommit(base.BaseCommit):
|
|||
parser.read_file(io.StringIO(submodules_node.str_content))
|
||||
|
||||
for section in parser.sections():
|
||||
path = parser.get(section, 'path')
|
||||
url = parser.get(section, 'url')
|
||||
path = parser.get(section, "path")
|
||||
url = parser.get(section, "url")
|
||||
if path and url:
|
||||
self._submodules[path.strip('/')] = url
|
||||
self._submodules[safe_bytes(path).strip(b"/")] = url
|
||||
|
||||
return self._submodules.get(submodule_path.strip('/'))
|
||||
return self._submodules.get(submodule_path.strip(b"/"))
|
||||
|
|
|
|||
|
|
@ -425,7 +425,7 @@ class GitRepository(BaseRepository):
|
|||
return
|
||||
|
||||
def get_commit(self, commit_id=None, commit_idx=None, pre_load=None,
|
||||
translate_tag=True, maybe_unreachable=False, reference_obj=None):
|
||||
translate_tag=True, maybe_unreachable=False, reference_obj=None) -> GitCommit:
|
||||
"""
|
||||
Returns `GitCommit` object representing commit from git repository
|
||||
at the given `commit_id` or head (most recent commit) if None given.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue