largefiles: enabled download of largefiles for git and mercurial from web interface.
- fixes #2817
This commit is contained in:
Marcin Kuzminski
parent
f5efab079d
commit
91c104ae49
10 changed files with 129 additions and 37 deletions
|
|
@ -226,8 +226,8 @@ let
|
|||
rhodecode-testdata-src = sources.rhodecode-testdata or (
|
||||
pkgs.fetchhg {
|
||||
url = "https://code.rhodecode.com/upstream/rc_testdata";
|
||||
rev = "v0.9.0";
|
||||
sha256 = "0k0ccb7cncd6mmzwckfbr6l7fsymcympwcm948qc3i0f0m6bbg1y";
|
||||
rev = "v0.10.0";
|
||||
sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0";
|
||||
});
|
||||
|
||||
# Apply all overrides and fix the final package set
|
||||
|
|
|
|||
|
|
@ -223,6 +223,8 @@ class FilesController(BaseRepoController):
|
|||
c.file_author = True
|
||||
c.file_tree = ''
|
||||
if c.file.is_file():
|
||||
c.lf_node = c.file.get_largefile_node()
|
||||
|
||||
c.file_source_page = 'true'
|
||||
c.file_last_commit = c.file.last_commit
|
||||
if c.file.size < self.cut_off_limit_file:
|
||||
|
|
@ -343,6 +345,14 @@ class FilesController(BaseRepoController):
|
|||
commit = self.__get_commit_or_redirect(revision, repo_name)
|
||||
file_node = self.__get_filenode_or_redirect(repo_name, commit, f_path)
|
||||
|
||||
if request.GET.get('lf'):
|
||||
# only if lf get flag is passed, we download this file
|
||||
# as LFS/Largefile
|
||||
lf_node = file_node.get_largefile_node()
|
||||
if lf_node:
|
||||
# overwrite our pointer with the REAL large-file
|
||||
file_node = lf_node
|
||||
|
||||
response.content_disposition = 'attachment; filename=%s' % \
|
||||
safe_str(f_path.split(Repository.NAME_SEP)[-1])
|
||||
|
||||
|
|
|
|||
|
|
@ -893,9 +893,10 @@ class BaseCommit(object):
|
|||
|
||||
def get_largefile_node(self, path):
|
||||
"""
|
||||
Returns the path to largefile from Mercurial storage.
|
||||
Returns the path to largefile from Mercurial/Git-lfs storage.
|
||||
or None if it's not a largefile node
|
||||
"""
|
||||
raise NotImplementedError
|
||||
return None
|
||||
|
||||
def archive_repo(self, file_path, kind='tgz', subrepos=None,
|
||||
prefix=None, write_metadata=False, mtime=None):
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ from rhodecode.lib.vcs.exceptions import CommitError, NodeDoesNotExistError
|
|||
from rhodecode.lib.vcs.nodes import (
|
||||
FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
|
||||
ChangedFileNodesGenerator, AddedFileNodesGenerator,
|
||||
RemovedFileNodesGenerator)
|
||||
RemovedFileNodesGenerator, LargeFileNode)
|
||||
|
||||
|
||||
class GitCommit(base.BaseCommit):
|
||||
|
|
@ -423,6 +423,17 @@ class GitCommit(base.BaseCommit):
|
|||
self.nodes[path] = node
|
||||
return self.nodes[path]
|
||||
|
||||
def get_largefile_node(self, path):
|
||||
id_, _ = self._get_id_for_path(path)
|
||||
pointer_spec = self._remote.is_large_file(id_)
|
||||
|
||||
if pointer_spec:
|
||||
# content of that file regular FileNode is the hash of largefile
|
||||
file_id = pointer_spec.get('oid_hash')
|
||||
if self._remote.in_largefiles_store(file_id):
|
||||
lf_path = self._remote.store_path(file_id)
|
||||
return LargeFileNode(lf_path, commit=self, org_path=path)
|
||||
|
||||
@LazyProperty
|
||||
def affected_files(self):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -312,18 +312,18 @@ class MercurialCommit(base.BaseCommit):
|
|||
return self.nodes[path]
|
||||
|
||||
def get_largefile_node(self, path):
|
||||
path = os.path.join(LARGEFILE_PREFIX, path)
|
||||
|
||||
if self._remote.is_large_file(path):
|
||||
# content of that file regular FileNode is the hash of largefile
|
||||
file_id = self.get_file_content(path).strip()
|
||||
if self._remote.in_store(file_id):
|
||||
path = self._remote.store_path(file_id)
|
||||
return LargeFileNode(path, commit=self)
|
||||
|
||||
if self._remote.in_largefiles_store(file_id):
|
||||
lf_path = self._remote.store_path(file_id)
|
||||
return LargeFileNode(lf_path, commit=self, org_path=path)
|
||||
elif self._remote.in_user_cache(file_id):
|
||||
path = self._remote.store_path(file_id)
|
||||
lf_path = self._remote.store_path(file_id)
|
||||
self._remote.link(file_id, path)
|
||||
return LargeFileNode(path, commit=self)
|
||||
return LargeFileNode(lf_path, commit=self, org_path=path)
|
||||
|
||||
@LazyProperty
|
||||
def _submodules(self):
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
Module holding everything related to vcs nodes, with vcs2 architecture.
|
||||
"""
|
||||
|
||||
|
||||
import os
|
||||
import stat
|
||||
|
||||
from zope.cachedescriptors.property import Lazy as LazyProperty
|
||||
|
|
@ -547,9 +547,8 @@ class FileNode(Node):
|
|||
create special instance of LargeFileNode which can get content from
|
||||
LF store.
|
||||
"""
|
||||
if self.commit and self.path.startswith(LARGEFILE_PREFIX):
|
||||
largefile_path = self.path.split(LARGEFILE_PREFIX)[-1].lstrip('/')
|
||||
return self.commit.get_largefile_node(largefile_path)
|
||||
if self.commit:
|
||||
return self.commit.get_largefile_node(self.path)
|
||||
|
||||
def lines(self, count_empty=False):
|
||||
all_lines, empty_lines = 0, 0
|
||||
|
|
@ -765,15 +764,37 @@ class SubModuleNode(Node):
|
|||
|
||||
class LargeFileNode(FileNode):
|
||||
|
||||
def __init__(self, path, url=None, commit=None, alias=None, org_path=None):
|
||||
self.path = path
|
||||
self.org_path = org_path
|
||||
self.kind = NodeKind.LARGEFILE
|
||||
self.alias = alias
|
||||
|
||||
def _validate_path(self, path):
|
||||
"""
|
||||
we override check since the LargeFileNode path is system absolute
|
||||
"""
|
||||
pass
|
||||
|
||||
def __repr__(self):
|
||||
return '<%s %r>' % (self.__class__.__name__, self.path)
|
||||
|
||||
@LazyProperty
|
||||
def size(self):
|
||||
return os.stat(self.path).st_size
|
||||
|
||||
@LazyProperty
|
||||
def raw_bytes(self):
|
||||
if self.commit:
|
||||
with open(self.path, 'rb') as f:
|
||||
content = f.read()
|
||||
else:
|
||||
content = self._content
|
||||
return content
|
||||
return content
|
||||
|
||||
@LazyProperty
|
||||
def name(self):
|
||||
"""
|
||||
Overwrites name to be the org lf path
|
||||
"""
|
||||
return self.org_path
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@
|
|||
<div class="codeblock-header">
|
||||
<div class="stats">
|
||||
<span> <strong>${c.file}</strong></span>
|
||||
% if c.lf_node:
|
||||
<span title="${_('This file is a pointer to large binary file')}"> | ${_('LargeFile')} ${h.format_byte_size_binary(c.lf_node.size)} </span>
|
||||
% endif
|
||||
<span> | ${c.file.lines()[0]} ${ungettext('line', 'lines', c.file.lines()[0])}</span>
|
||||
<span> | ${h.format_byte_size_binary(c.file.size)}</span>
|
||||
<span> | ${c.file.mimetype} </span>
|
||||
|
|
@ -22,9 +25,16 @@
|
|||
${h.link_to(_('Annotation'), h.url('files_annotate_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
|
||||
%endif
|
||||
| ${h.link_to(_('Raw'), h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
|
||||
| <a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path)}">
|
||||
${_('Download')}
|
||||
</a>
|
||||
|
|
||||
% if c.lf_node:
|
||||
<a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path, lf=1)}">
|
||||
${_('Download largefile')}
|
||||
</a>
|
||||
% else:
|
||||
<a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path)}">
|
||||
${_('Download')}
|
||||
</a>
|
||||
% endif
|
||||
|
||||
%if h.HasRepoPermissionAny('repository.write','repository.admin')(c.repo_name):
|
||||
|
|
||||
|
|
@ -46,11 +56,11 @@
|
|||
</div>
|
||||
<div id="file_history_container"></div>
|
||||
<div class="code-body">
|
||||
%if c.file.is_binary:
|
||||
%if c.file.is_binary:
|
||||
<div>
|
||||
${_('Binary file (%s)') % c.file.mimetype}
|
||||
</div>
|
||||
%else:
|
||||
%else:
|
||||
% if c.file.size < c.cut_off_limit:
|
||||
%if c.renderer and not c.annotate:
|
||||
${h.render(c.file.content, renderer=c.renderer, relative_url=h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
|
||||
|
|
@ -67,7 +77,6 @@
|
|||
%endfor
|
||||
%endif
|
||||
</table>
|
||||
</div>
|
||||
%endif
|
||||
%else:
|
||||
${_('File is too big to display')} ${h.link_to(_('Show as raw'),
|
||||
|
|
|
|||
|
|
@ -387,20 +387,22 @@ class TestRepoContainer(object):
|
|||
self._fixture = Fixture()
|
||||
self._repos = {}
|
||||
|
||||
def __call__(self, dump_name, backend_alias):
|
||||
def __call__(self, dump_name, backend_alias, config=None):
|
||||
key = (dump_name, backend_alias)
|
||||
if key not in self._repos:
|
||||
repo = self._create_repo(dump_name, backend_alias)
|
||||
repo = self._create_repo(dump_name, backend_alias, config)
|
||||
self._repos[key] = repo.repo_id
|
||||
return Repository.get(self._repos[key])
|
||||
|
||||
def _create_repo(self, dump_name, backend_alias):
|
||||
def _create_repo(self, dump_name, backend_alias, config):
|
||||
repo_name = '%s-%s' % (backend_alias, dump_name)
|
||||
backend_class = get_backend(backend_alias)
|
||||
dump_extractor = self.dump_extractors[backend_alias]
|
||||
repo_path = dump_extractor(dump_name, repo_name)
|
||||
vcs_repo = backend_class(repo_path)
|
||||
|
||||
vcs_repo = backend_class(repo_path, config=config)
|
||||
repo2db_mapper({repo_name: vcs_repo})
|
||||
|
||||
repo = RepoModel().get_by_repo_name(repo_name)
|
||||
self._cleanup_repos.append(repo_name)
|
||||
return repo
|
||||
|
|
@ -515,6 +517,9 @@ class Backend(object):
|
|||
def __getitem__(self, key):
|
||||
return self._test_repo_container(key, self.alias)
|
||||
|
||||
def create_test_repo(self, key, config=None):
|
||||
return self._test_repo_container(key, self.alias, config)
|
||||
|
||||
@property
|
||||
def repo(self):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -22,15 +22,16 @@ import datetime
|
|||
import mock
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
from rhodecode.lib.utils import make_db_config
|
||||
from rhodecode.lib.vcs.backends.base import Reference
|
||||
from rhodecode.lib.vcs.backends.git import (
|
||||
GitRepository, GitCommit, discover_git_version)
|
||||
from rhodecode.lib.vcs.exceptions import (
|
||||
RepositoryError, VCSError, NodeDoesNotExistError
|
||||
)
|
||||
RepositoryError, VCSError, NodeDoesNotExistError)
|
||||
from rhodecode.lib.vcs.nodes import (
|
||||
NodeKind, FileNode, DirNode, NodeState, SubModuleNode)
|
||||
from rhodecode.tests import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, get_new_dir
|
||||
|
|
@ -1003,6 +1004,32 @@ class TestGitCommit(object):
|
|||
assert author == commit.author_name
|
||||
|
||||
|
||||
class TestLargeFileRepo(object):
|
||||
|
||||
def test_large_file(self, backend_git):
|
||||
conf = make_db_config()
|
||||
repo = backend_git.create_test_repo('largefiles', conf)
|
||||
|
||||
tip = repo.scm_instance().get_commit()
|
||||
|
||||
# extract stored LF node into the origin cache
|
||||
lfs_store = os.path.join(repo.repo_path, repo.repo_name, 'lfs_store')
|
||||
|
||||
oid = '7b331c02e313c7599d5a90212e17e6d3cb729bd2e1c9b873c302a63c95a2f9bf'
|
||||
oid_path = os.path.join(lfs_store, oid)
|
||||
oid_destination = os.path.join(
|
||||
conf.get('vcs_git_lfs', 'store_location'), oid)
|
||||
shutil.copy(oid_path, oid_destination)
|
||||
|
||||
node = tip.get_node('1MB.zip')
|
||||
|
||||
lf_node = node.get_largefile_node()
|
||||
|
||||
assert lf_node.is_largefile() is True
|
||||
assert lf_node.size == 1024000
|
||||
assert lf_node.name == '1MB.zip'
|
||||
|
||||
|
||||
class TestGitSpecificWithRepo(BackendTestMixin):
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -23,14 +23,13 @@ import os
|
|||
import mock
|
||||
import pytest
|
||||
|
||||
import rhodecode.lib.vcs.conf.settings
|
||||
from rhodecode.lib.utils import make_db_config
|
||||
from rhodecode.lib.vcs import backends
|
||||
from rhodecode.lib.vcs.backends.base import (
|
||||
Reference, MergeResponse, MergeFailureReason)
|
||||
from rhodecode.lib.vcs.backends.hg import MercurialRepository, MercurialCommit
|
||||
from rhodecode.lib.vcs.exceptions import (
|
||||
CommitError, RepositoryError, VCSError, NodeDoesNotExistError,
|
||||
CommitDoesNotExistError)
|
||||
RepositoryError, VCSError, NodeDoesNotExistError, CommitDoesNotExistError)
|
||||
from rhodecode.lib.vcs.nodes import FileNode, NodeKind, NodeState
|
||||
from rhodecode.tests import TEST_HG_REPO, TEST_HG_REPO_CLONE
|
||||
|
||||
|
|
@ -1095,12 +1094,6 @@ class TestMercurialCommit(object):
|
|||
with pytest.raises(VCSError):
|
||||
self.repo.get_commit().get_node(path)
|
||||
|
||||
def test_large_file(self):
|
||||
# TODO: valid large file
|
||||
tip = self.repo.get_commit()
|
||||
with pytest.raises(CommitError):
|
||||
tip.get_largefile_node("invalid")
|
||||
|
||||
def test_author_email(self):
|
||||
assert 'marcin@python-blog.com' == \
|
||||
self.repo.get_commit('b986218ba1c9').author_email
|
||||
|
|
@ -1117,6 +1110,21 @@ class TestMercurialCommit(object):
|
|||
self.repo.get_commit('84478366594b').author_name
|
||||
|
||||
|
||||
class TestLargeFileRepo(object):
|
||||
|
||||
def test_large_file(self, backend_hg):
|
||||
repo = backend_hg.create_test_repo('largefiles', make_db_config())
|
||||
|
||||
tip = repo.scm_instance().get_commit()
|
||||
node = tip.get_node('.hglf/thisfileislarge')
|
||||
|
||||
lf_node = node.get_largefile_node()
|
||||
|
||||
assert lf_node.is_largefile() is True
|
||||
assert lf_node.size == 1024000
|
||||
assert lf_node.name == '.hglf/thisfileislarge'
|
||||
|
||||
|
||||
class TestGetBranchName(object):
|
||||
def test_returns_ref_name_when_type_is_branch(self):
|
||||
ref = self._create_ref('branch', 'fake-name')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue