api: add a max_file_bytes parameter to get_nodes so that large

files can be skipped

docs/api/api.rst

@@ -1880,6 +1880,8 @@ get_repo_nodes
        md5, binary, and or content.  The valid options are ``basic`` and
        ``full``.
    :type details: Optional(str)
+   :param max_file_bytes: Only return file content under this file size bytes
+   :type details: Optional(int)
 
    Example output:
 
@@ -2821,4 +2823,3 @@ delete_gist
      error :  {
        "failed to delete gist ID:<gist_id>"
      }
-

rhodecode/api/tests/test_get_repo_nodes.py

@@ -73,6 +73,29 @@ class TestGetRepoNodes(object):
         expected = 'failed to get repo: `%s` nodes' % (backend.repo_name,)
         assert_error(id_, expected, given=response.body)
 
+    def test_api_get_repo_nodes_max_file_bytes(self, backend):
+        commit_id = 'tip'
+        path = '/'
+        max_file_bytes = 500
+
+        id_, params = build_data(
+            self.apikey, 'get_repo_nodes',
+            repoid=backend.repo_name, revision=commit_id, details='full',
+            root_path=path)
+        response = api_call(self.app, params)
+        assert any(file['content'] and len(file['content']) > max_file_bytes
+                   for file in response.json['result'])
+
+        id_, params = build_data(
+            self.apikey, 'get_repo_nodes',
+            repoid=backend.repo_name, revision=commit_id,
+            root_path=path, details='full',
+            max_file_bytes=max_file_bytes)
+        response = api_call(self.app, params)
+        assert all(
+            file['content'] is None if file['size'] > max_file_bytes else True
+            for file in response.json['result'])
+
     def test_api_get_repo_nodes_bad_ret_type(self, backend):
         commit_id = 'tip'
         path = '/'

rhodecode/api/views/repo_api.py

@@ -400,7 +400,8 @@ def get_repo_changesets(request, apiuser, repoid, start_rev, limit,
 
 @jsonrpc_method()
 def get_repo_nodes(request, apiuser, repoid, revision, root_path,
-                   ret_type=Optional('all'), details=Optional('basic')):
+                   ret_type=Optional('all'), details=Optional('basic'),
+                   max_file_bytes=Optional(None)):
     """
     Returns a list of nodes and children in a flat list for a given
     path at given revision.
@@ -425,6 +426,8 @@ def get_repo_nodes(request, apiuser, repoid, revision, root_path,
         md5, binary, and or content.  The valid options are ``basic`` and
         ``full``.
     :type details: Optional(str)
+    :param max_file_bytes: Only return file content under this file size bytes
+    :type details: Optional(int)
 
     Example output:
 
@@ -472,7 +475,8 @@ def get_repo_nodes(request, apiuser, repoid, revision, root_path,
 
         _d, _f = ScmModel().get_nodes(
             repo, revision, root_path, flat=False,
-            extended_info=extended_info, content=content)
+            extended_info=extended_info, content=content,
+            max_file_bytes=max_file_bytes)
         _map = {
             'all': _d + _f,
             'files': _f,

rhodecode/model/scm.py

@@ -478,7 +478,7 @@ class ScmModel(BaseModel):
         return data
 
     def get_nodes(self, repo_name, commit_id, root_path='/', flat=True,
-                  extended_info=False, content=False):
+                  extended_info=False, content=False, max_file_bytes=None):
         """
         recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
@@ -487,6 +487,7 @@ class ScmModel(BaseModel):
         :param commit_id: commit id for which to list nodes
         :param root_path: root path to list
         :param flat: return as a list, if False returns a dict with description
+        :param max_file_bytes: will not return file contents over this limit
 
         """
         _files = list()
@@ -499,6 +500,8 @@ class ScmModel(BaseModel):
                 for f in files:
                     _content = None
                     _data = f.unicode_path
+                    over_size_limit = (max_file_bytes is not None
+                                       and f.size > max_file_bytes)
 
                     if not flat:
                         _data = {
@@ -517,7 +520,7 @@ class ScmModel(BaseModel):
 
                         if content:
                             full_content = None
-                            if not f.is_binary:
+                            if not f.is_binary and not over_size_limit:
                                 full_content = safe_str(f.content)
 
                             _data.update({
@@ -1096,4 +1099,4 @@ def _check_rhodecode_hook(hook_path):
 def _read_hook(hook_path):
     with open(hook_path, 'rb') as f:
         content = f.read()
-    return content
+    return content

rhodecode/tests/models/test_scm.py

@@ -148,6 +148,23 @@ def test_get_nodes_returns_unicode_non_flat(backend_random):
     assert_contains_only_unicode([f['name'] for f in files])
 
 
+def test_get_nodes_max_file_bytes(backend_random):
+    repo = backend_random.repo
+    max_file_bytes = 10
+    directories, files = scm.ScmModel().get_nodes(
+        repo.repo_name, repo.get_commit(commit_idx=0).raw_id, content=True,
+        extended_info=True, flat=False)
+    assert any(file['content'] and len(file['content']) > max_file_bytes
+               for file in files)
+
+    directories, files = scm.ScmModel().get_nodes(
+        repo.repo_name, repo.get_commit(commit_idx=0).raw_id, content=True,
+        extended_info=True, flat=False, max_file_bytes=max_file_bytes)
+    assert all(
+        file['content'] is None if file['size'] > max_file_bytes else True
+        for file in files)
+
+
 def assert_contains_only_unicode(structure):
     assert structure
     for value in structure: