fix: prevent "too many open files" errors from git ignored check (#1127)

fixes #1125, fixes #997

Author: cseickel

lua/neo-tree/defaults.lua

@@ -444,6 +444,9 @@ local config = {
       sidebar = "tab",   -- sidebar is when position = left or right
       current = "window" -- current is when position = current
     },
+    check_gitignore_in_search = true, -- check gitignore status for files/directories when searching
+                                      -- setting this to false will speed up searches, but gitignored
+                                      -- items won't be marked if they are visible.
     -- The renderer section provides the renderers that will be used to render the tree.
     --   The first level is the node type.
     --   For each node type, you can specify a list of components to render.

lua/neo-tree/git/ignored.lua

@@ -104,7 +104,29 @@ M.mark_ignored = function(state, items, callback)
   local all_results = {}
   if type(callback) == "function" then
     local jobs = {}
-    local progress = 0
+    local running_jobs = 0
+    local job_count = 0
+    local completed_jobs = 0
+
+    -- This is called when a job completes, and starts the next job if there are any left
+    -- or calls the callback if all jobs are complete.
+    -- It is also called once at the start to start the first 50 jobs.
+    --
+    -- This is done to avoid running too many jobs at once, which can cause a crash from
+    -- having too many open files.
+    local run_more_jobs = function()
+      while #jobs > 0 and running_jobs < 50 and job_count > completed_jobs do
+        local next_job = table.remove(jobs, #jobs)
+        next_job:start()
+        running_jobs = running_jobs + 1
+      end
+
+      if completed_jobs == job_count then
+        finalize(all_results)
+        callback(all_results)
+      end
+    end
+
     for folder, folder_items in pairs(folders) do
       local args = { "-C", folder, "check-ignore", "--stdin" }
       local job = Job:new({
@@ -124,19 +146,17 @@ M.mark_ignored = function(state, items, callback)
             result = self:result()
           end
           vim.list_extend(all_results, process_result(result))
-          progress = progress + 1
-          if progress == #jobs then
-            finalize(all_results)
-            callback(all_results)
-          end
+
+          running_jobs = running_jobs - 1
+          completed_jobs = completed_jobs + 1
+          run_more_jobs()
         end,
       })
       table.insert(jobs, job)
+      job_count = job_count + 1
     end
 
-    for _, job in ipairs(jobs) do
-      job:start()
-    end
+    run_more_jobs()
   else
     for folder, folder_items in pairs(folders) do
       local cmd = { "git", "-C", folder, "check-ignore", unpack(folder_items) }

lua/neo-tree/sources/filesystem/lib/fs_scan.lua

@@ -103,12 +103,32 @@ local render_context = function(context)
   context = nil
 end
 
-local job_complete_async = function(context)
+local should_check_gitignore = function (context)
   local state = context.state
-  local parent_id = context.parent_id
   if #context.all_items == 0 then
     log.info("No items, skipping git ignored/status lookups")
-  elseif state.filtered_items.hide_gitignored or state.enable_git_status then
+    return false
+  end
+  if state.search_pattern and state.check_gitignore_in_search == false then
+    return false
+  end
+  if state.filtered_items.hide_gitignored then
+    return true
+  end
+  if not state.enable_git_status then
+    return true
+  end
+  return false
+end
+
+local job_complete_async = function(context)
+  local state = context.state
+  local parent_id = context.parent_id
+  -- if state.search_pattern and #context.all_items > 50 then
+  --   -- don't do git ignored/status lookups when searching unless we are down to a reasonable number of items
+  --   return context
+  -- end
+  if should_check_gitignore(context) then
     local mark_ignored_async = async.wrap(function (_state, _all_items, _callback)
       git.mark_ignored(_state, _all_items, _callback)
     end, 3)
@@ -126,12 +146,7 @@ end
 local job_complete = function(context)
   local state = context.state
   local parent_id = context.parent_id
-  if #context.all_items == 0 then
-    log.info("No items, skipping git ignored/status lookups")
-    render_context(context)
-    return
-  end
-  if state.filtered_items.hide_gitignored or state.enable_git_status then
+  if should_check_gitignore(context) then
     if require("neo-tree").config.git_status_async then
       git.mark_ignored(state, context.all_items, function(all_items)
         if parent_id then
@@ -153,6 +168,8 @@ local job_complete = function(context)
       end
     end
     render_context(context)
+  else
+    render_context(context)
   end
 end