fix(filesystem): read all files from large directories (#1745)

Author: pynappo

lua/neo-tree/sources/filesystem/lib/fs_scan.lua

@@ -14,6 +14,9 @@ local async = require("plenary.async")
 
 local M = {}
 
+--- how many entries to load per readdir
+local ENTRIES_BATCH_SIZE = 1000
+
 local on_directory_loaded = function(context, dir_path)
   local state = context.state
   local scanned_folder = context.folders[dir_path]
@@ -198,23 +201,28 @@ end
 
 local function get_children_sync(path)
   local children = {}
-  local dir, err = uv.fs_opendir(path, nil, 1000)
-  if err then
+  local dir, err = uv.fs_opendir(path, nil, ENTRIES_BATCH_SIZE)
+  if not dir then
+    ---@cast err -nil
     if is_permission_error(err) then
       log.debug(err)
     else
       log.error(err)
     end
     return children
   end
-  ---@cast dir uv.luv_dir_t
-  local stats = uv.fs_readdir(dir)
-  if stats then
-    for _, stat in ipairs(stats) do
+  repeat
+    local stats = uv.fs_readdir(dir)
+    if not stats then
+      break
+    end
+    local more = false
+    for i, stat in ipairs(stats) do
+      more = i == ENTRIES_BATCH_SIZE
       local child_path = utils.path_join(path, stat.name)
       table.insert(children, { path = child_path, type = stat.type })
     end
-  end
+  until not more
   uv.fs_closedir(dir)
   return children
 end
@@ -231,17 +239,27 @@ local function get_children_async(path, callback)
       callback(children)
       return
     end
-    uv.fs_readdir(dir, function(_, stats)
+    local readdir_batch
+    ---@param _ string?
+    ---@param stats uv.fs_readdir.entry[]
+    readdir_batch = function(_, stats)
       if stats then
-        for _, stat in ipairs(stats) do
+        local more = false
+        for i, stat in ipairs(stats) do
+          more = i == ENTRIES_BATCH_SIZE
           local child_path = utils.path_join(path, stat.name)
           table.insert(children, { path = child_path, type = stat.type })
         end
+        if more then
+          return uv.fs_readdir(dir, readdir_batch)
+        end
       end
       uv.fs_closedir(dir)
       callback(children)
-    end)
-  end, 1000)
+    end
+
+    uv.fs_readdir(dir, readdir_batch)
+  end, ENTRIES_BATCH_SIZE)
 end
 
 local function scan_dir_sync(context, path)
@@ -399,13 +417,19 @@ local function sync_scan(context, path_to_scan)
     end
     job_complete(context)
   else -- scan_mode == "shallow"
-    local dir, err = uv.fs_opendir(path_to_scan, nil, 1000)
+    local dir, err = uv.fs_opendir(path_to_scan, nil, ENTRIES_BATCH_SIZE)
     if dir then
       local stats = uv.fs_readdir(dir)
-      if stats then
-        for _, stat in ipairs(stats) do
+      repeat
+        if not stats then
+          break
+        end
+
+        local more = false
+        for i, stat in ipairs(stats) do
+          more = i == ENTRIES_BATCH_SIZE
           local path = utils.path_join(path_to_scan, stat.name)
-          local success, item = pcall(file_items.create_item, context, path, stat.type)
+          local success, _ = pcall(file_items.create_item, context, path, stat.type)
           if success then
             if context.recursive and stat.type == "directory" then
               table.insert(context.paths_to_load, path)
@@ -414,7 +438,7 @@ local function sync_scan(context, path_to_scan)
             log.error("error creating item for ", path)
           end
         end
-      end
+      until not more
       uv.fs_closedir(dir)
     else
       log.error("Error opening dir:", err)