engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

archives: fixed bugs with serving archives from non-ascii repos, and also deliver archives at much bigger reading blocks for faster downloads

Browse source
This commit is contained in:
RhodeCode Admin 2023-08-09 14:33:20 +02:00
parent d73dfa02a2
commit 53292e1e37
2 changed files with 14 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

23
rhodecode/apps/repository/views/repo_files.py
View file

@ -38,7 +38,7 @@ from rhodecode.lib import diffs, helpers as h, rc_cache
from rhodecode.lib import audit_logger
from rhodecode.lib.hash_utils import sha1_safe
from rhodecode.lib.rc_cache.archive_cache import get_archival_cache_store, get_archival_config, ReentrantLock
from rhodecode.lib.str_utils import safe_bytes
from rhodecode.lib.str_utils import safe_bytes, convert_special_chars
from rhodecode.lib.view_utils import parse_path_ref
from rhodecode.lib.exceptions import NonRelativePathError
from rhodecode.lib.codeblocks import (
@ -63,17 +63,17 @@ from rhodecode.model.db import Repository
log = logging.getLogger(__name__)
def get_archive_name(db_repo_name, commit_sha, ext, subrepos=False, path_sha='', with_hash=True):
def get_archive_name(db_repo_id, db_repo_name, commit_sha, ext, subrepos=False, path_sha='', with_hash=True):
# original backward compat name of archive
clean_name = safe_str(db_repo_name.replace('/', '_'))
# e.g vcsserver-sub-1-abcfdef-archive-all.zip
# vcsserver-sub-0-abcfdef-COMMIT_SHA-PATH_SHA.zip
clean_name = safe_str(convert_special_chars(db_repo_name).replace('/', '_'))
# e.g vcsserver-id-abcd-sub-1-abcfdef-archive-all.zip
# vcsserver-id-abcd-sub-0-abcfdef-COMMIT_SHA-PATH_SHA.zip
id_sha = sha1_safe(str(db_repo_id))[:4]
sub_repo = 'sub-1' if subrepos else 'sub-0'
commit = commit_sha if with_hash else 'archive'
path_marker = (path_sha if with_hash else '') or 'all'
archive_name = f'{clean_name}-{sub_repo}-{commit}-{path_marker}{ext}'
archive_name = f'{clean_name}-id-{id_sha}-{sub_repo}-{commit}-{path_marker}{ext}'
return archive_name
@ -396,7 +396,7 @@ class RepoFilesView(RepoAppView):
# used for cache etc, consistent unique archive name
archive_name_key = get_archive_name(
self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos,
self.db_repo.repo_id, self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos,
path_sha=path_sha, with_hash=True)
if not with_hash:
@ -404,7 +404,7 @@ class RepoFilesView(RepoAppView):
# what end client gets served
response_archive_name = get_archive_name(
self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos,
self.db_repo.repo_id, self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos,
path_sha=path_sha, with_hash=with_hash)
# remove extension from our archive directory name
@ -444,9 +444,10 @@ class RepoFilesView(RepoAppView):
if not reader:
raise ValueError('archive cache reader is empty, failed to fetch file from distributed archive cache')
def archive_iterator(_reader):
def archive_iterator(_reader, block_size: int = 4096*512):
# 4096 * 64 = 64KB
while 1:
data = _reader.read(1024)
data = _reader.read(block_size)
if not data:
break
yield data

5
rhodecode/lib/vcs/backends/base.py
View file

@ -1222,7 +1222,7 @@ class BaseCommit(object):
Creates an archive containing the contents of the repository.
:param archive_name_key: unique key under this archive should be generated
:param kind: one of following: ``"tbz2"``, ``"tgz"``, ``"zip"``.
:param kind: one of the following: ``"tbz2"``, ``"tgz"``, ``"zip"``.
:param archive_dir_name: name of root directory in archive.
Default is repository name and commit's short_id joined with dash:
``"{repo_name}-{short_id}"``.
@ -1238,8 +1238,7 @@ class BaseCommit(object):
allowed_kinds = [x[0] for x in settings.ARCHIVE_SPECS]
if kind not in allowed_kinds:
raise ImproperArchiveTypeError(
'Archive kind (%s) not supported use one of %s' %
(kind, allowed_kinds))
f'Archive kind ({kind}) not supported use one of {allowed_kinds}')
archive_dir_name = self._validate_archive_prefix(archive_dir_name)
mtime = mtime is not None or time.mktime(self.date.timetuple())