engineering/unrhodecode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

api: expose new functions for FTS

Browse source 
- vcs: added special md5 calculation that doesn't use content cache for more memory efficient md5 calculation.
This commit is contained in:
Marcin Kuzminski 2019-02-07 11:31:30 +01:00
parent 4c846e8f05
commit 5b8a28d0eb
3 changed files with 225 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

138
rhodecode/api/views/repo_api.py
View file

@ -428,8 +428,8 @@ def get_repo_nodes(request, apiuser, repoid, revision, root_path,
``all`` (default), ``files`` and ``dirs``.
:type ret_type: Optional(str)
:param details: Returns extended information about nodes, such as
md5, binary, and or content. The valid options are ``basic`` and
``full``.
md5, binary, and or content.
The valid options are ``basic`` and ``full``.
:type details: Optional(str)
:param max_file_bytes: Only return file content under this file size bytes
:type details: Optional(int)
@ -440,12 +440,17 @@ def get_repo_nodes(request, apiuser, repoid, revision, root_path,
id : <id_given_in_input>
result: [
{
"name" : "<name>"
"type" : "<type>",
"binary": "<true|false>" (only in extended mode)
"md5" : "<md5 of file content>" (only in extended mode)
},
{
"binary": false,
"content": "File line\nLine2\n",
"extension": "md",
"lines": 2,
"md5": "059fa5d29b19c0657e384749480f6422",
"mimetype": "text/x-minidsrc",
"name": "file.md",
"size": 580,
"type": "file"
},
...
]
error: null
@ -453,16 +458,14 @@ def get_repo_nodes(request, apiuser, repoid, revision, root_path,
repo = get_repo_or_error(repoid)
if not has_superadmin_permission(apiuser):
_perms = (
'repository.admin', 'repository.write', 'repository.read',)
_perms = ('repository.admin', 'repository.write', 'repository.read',)
validate_repo_permissions(apiuser, repoid, repo, _perms)
ret_type = Optional.extract(ret_type)
details = Optional.extract(details)
_extended_types = ['basic', 'full']
if details not in _extended_types:
raise JSONRPCError(
'ret_type must be one of %s' % (','.join(_extended_types)))
raise JSONRPCError('ret_type must be one of %s' % (','.join(_extended_types)))
extended_info = False
content = False
if details == 'basic':
@ -498,6 +501,117 @@ def get_repo_nodes(request, apiuser, repoid, revision, root_path,
)
@jsonrpc_method()
def get_repo_file(request, apiuser, repoid, commit_id, file_path,
max_file_bytes=Optional(None), details=Optional('basic')):
"""
Returns a single file from repository at given revision.
This command can only be run using an |authtoken| with admin rights,
or users with at least read rights to |repos|.
:param apiuser: This is filled automatically from the |authtoken|.
:type apiuser: AuthUser
:param repoid: The repository name or repository ID.
:type repoid: str or int
:param commit_id: The revision for which listing should be done.
:type commit_id: str
:param file_path: The path from which to start displaying.
:type file_path: str
:param details: Returns different set of information about nodes.
The valid options are ``minimal`` ``basic`` and ``full``.
:type details: Optional(str)
:param max_file_bytes: Only return file content under this file size bytes
:type details: Optional(int)
Example output:
.. code-block:: bash
id : <id_given_in_input>
result: {
"binary": false,
"extension": "py",
"lines": 35,
"content": "....",
"md5": "76318336366b0f17ee249e11b0c99c41",
"mimetype": "text/x-python",
"name": "python.py",
"size": 817,
"type": "file",
}
error: null
"""
repo = get_repo_or_error(repoid)
if not has_superadmin_permission(apiuser):
_perms = ('repository.admin', 'repository.write', 'repository.read',)
validate_repo_permissions(apiuser, repoid, repo, _perms)
details = Optional.extract(details)
_extended_types = ['minimal', 'minimal+search', 'basic', 'full']
if details not in _extended_types:
raise JSONRPCError(
'ret_type must be one of %s, got %s' % (','.join(_extended_types)), details)
extended_info = False
content = False
if details == 'minimal':
extended_info = False
elif details == 'basic':
extended_info = True
elif details == 'full':
extended_info = content = True
try:
# check if repo is not empty by any chance, skip quicker if it is.
_scm = repo.scm_instance()
if _scm.is_empty():
return None
node = ScmModel().get_node(
repo, commit_id, file_path, extended_info=extended_info,
content=content, max_file_bytes=max_file_bytes)
except Exception:
log.exception("Exception occurred while trying to get repo node")
raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name)
return node
@jsonrpc_method()
def get_repo_fts_tree(request, apiuser, repoid, commit_id, root_path):
"""
Returns a list of tree nodes for path at given revision. This api is built
strictly for usage in full text search building, and shouldn't be consumed
This command can only be run using an |authtoken| with admin rights,
or users with at least read rights to |repos|.
"""
repo = get_repo_or_error(repoid)
if not has_superadmin_permission(apiuser):
_perms = ('repository.admin', 'repository.write', 'repository.read',)
validate_repo_permissions(apiuser, repoid, repo, _perms)
try:
# check if repo is not empty by any chance, skip quicker if it is.
_scm = repo.scm_instance()
if _scm.is_empty():
return []
tree_files = ScmModel().get_fts_data(repo, commit_id, root_path)
return tree_files
except Exception:
log.exception("Exception occurred while trying to get repo nodes")
raise JSONRPCError('failed to get repo: `%s` nodes' % repo.repo_name)
@jsonrpc_method()
def get_repo_refs(request, apiuser, repoid):
"""

16
rhodecode/lib/vcs/nodes.py
View file

@ -372,6 +372,22 @@ class FileNode(Node):
"""
return md5(self.raw_bytes)
def metadata_uncached(self):
"""
Returns md5, binary flag of the file node, without any cache usage.
"""
if self.commit:
content = self.commit.get_file_content(self.path)
else:
content = self._content
is_binary = content and '\0' in content
size = 0
if content:
size = len(content)
return is_binary, md5(content), size
@LazyProperty
def content(self):
"""

84
rhodecode/model/scm.py
View file

@ -573,11 +573,93 @@ class ScmModel(BaseModel):
})
_dirs.append(_data)
except RepositoryError:
log.debug("Exception in get_nodes", exc_info=True)
log.exception("Exception in get_nodes")
raise
return _dirs, _files
def get_node(self, repo_name, commit_id, file_path,
extended_info=False, content=False, max_file_bytes=None):
"""
retrieve single node from commit
"""
try:
_repo = self._get_repo(repo_name)
commit = _repo.scm_instance().get_commit(commit_id=commit_id)
file_node = commit.get_node(file_path)
if file_node.is_dir():
raise RepositoryError('The given path is a directory')
_content = None
f_name = file_node.unicode_path
file_data = {
"name": h.escape(f_name),
"type": "file",
}
if extended_info:
file_data.update({
"md5": file_node.md5,
"binary": file_node.is_binary,
"size": file_node.size,
"extension": file_node.extension,
"mimetype": file_node.mimetype,
"lines": file_node.lines()[0]
})
if content:
over_size_limit = (max_file_bytes is not None
and file_node.size > max_file_bytes)
full_content = None
if not file_node.is_binary and not over_size_limit:
full_content = safe_str(file_node.content)
file_data.update({
"content": full_content,
})
except RepositoryError:
log.exception("Exception in get_node")
raise
return file_data
def get_fts_data(self, repo_name, commit_id, root_path='/'):
"""
Fetch node tree for usage in full text search
"""
tree_info = list()
try:
_repo = self._get_repo(repo_name)
commit = _repo.scm_instance().get_commit(commit_id=commit_id)
root_path = root_path.lstrip('/')
for __, dirs, files in commit.walk(root_path):
for f in files:
_content = None
_data = f_name = f.unicode_path
is_binary, md5, size = f.metadata_uncached()
_data = {
"name": h.escape(f_name),
"md5": md5,
"extension": f.extension,
"binary": is_binary,
"size": size
}
tree_info.append(_data)
except RepositoryError:
log.exception("Exception in get_nodes")
raise
return tree_info
def create_nodes(self, user, repo, message, nodes, parent_commit=None,
author=None, trigger_push_hook=True):
"""