[pob2-port] Speedup export of skill tree and regenerating files

src/Export/Classes/GGPKData.lua

@@ -44,12 +44,8 @@ local GGPKClass = newClass("GGPKData", function(self, path, datPath, reExport)
 	self.dat = { }
 	self.txt = { }
 	self.ot = { }
-
-	if USE_DAT64 then
-		self:AddDat64Files()
-	else
-		self:AddDatFiles()
-	end
+
+	self:AddDat64Files()
 end)
 
 function GGPKClass:CleanDir(reExport)

src/Export/Classes/GGPKData.lua.rej

@@ -0,0 +1,121 @@
+diff a/src/Export/Classes/GGPKData.lua b/src/Export/Classes/GGPKData.lua	(rejected hunks)
+@@ -67,48 +63,44 @@ function GGPKClass:ExtractFilesWithBun(fileListStr, useRegex)
+ 	os.execute(cmd)
+ end
+
++-- Use manifest files to avoid command line limit and reduce cmd calls
++function GGPKClass:ExtractFilesWithBunFromTable(fileTable, useRegex)
++	local useRegex = useRegex or false
++	local manifest = self.oozPath .. "extract_list.txt"
++	local f = assert(io.open(manifest, "w"))
++	for _, fname in ipairs(fileTable) do
++		f:write(string.lower(fname), "\n")
++	end
++	f:close()
++	local cmd = 'cd "' .. self.oozPath .. '" && bun_extract_file.exe extract-files ' .. (useRegex and '--regex "' or '"') .. self.path .. '" . < "' .. manifest .. '"'
++	ConPrintf(cmd)
++	os.execute(cmd)
++	os.remove(manifest)
++end
++
+ function GGPKClass:ExtractFiles(reExport)
+ 	if reExport then
+ 		local datList, csdList, otList, itList = self:GetNeededFiles()
+-		local sweetSpotCharacter = 6000
+-		local fileList = ''
+-
++		local datFiles = {}
+ 		for _, fname in ipairs(datList) do
+-			if USE_DAT64 then
+-				fileList = fileList .. '"' .. fname .. 'c64" '
+-			else
+-				fileList = fileList .. '"' .. fname .. '" '
+-			end
+-
+-			if fileList:len() > sweetSpotCharacter then
+-				self:ExtractFilesWithBun(fileList)
+-				fileList = ''
+-			end
+-		end
+-
+-		for _, fname in ipairs(otList) do
+-			self:ExtractFilesWithBun('"' .. fname .. '"', true)
++			datFiles[#datFiles + 1] = fname .. "c64"
+ 		end
+
+-		for _, fname in ipairs(itList) do
+-			fileList = fileList .. '"' .. fname .. '" '
+-
+-			if fileList:len() > sweetSpotCharacter then
+-				self:ExtractFilesWithBun(fileList)
+-				fileList = ''
+-			end
++		-- non-regex chunk: dat files + itList
++		for i = 1, #itList do
++			datFiles[#datFiles + 1] = itList[i]
+ 		end
++		self:ExtractFilesWithBunFromTable(datFiles, false)
+
+-		if (fileList:len() > 0) then
+-			self:ExtractFilesWithBun(fileList)
+-			fileList = ''
++		-- regex chunk: otList + csdList (stat descriptions)
++		local regexFiles = {}
++		for i = 1, #otList do
++			regexFiles[#regexFiles + 1] = otList[i]
+ 		end
+-
+-		-- Special handling for stat descriptions (CSD) as they
+-		-- are regex based
+-		for _, fname in ipairs(csdList) do
+-			self:ExtractFilesWithBun('"' .. fname .. '"', true)
++		for i = 1, #csdList do
++			regexFiles[#regexFiles + 1] = csdList[i]
+ 		end
++		self:ExtractFilesWithBunFromTable(regexFiles, true)
+ 	end
+
+ 	-- Overwrite Enums
+@@ -120,39 +112,17 @@ end
+
+ function GGPKClass:ExtractList(listToExtract, cache, useRegex)
+ 	useRegex = useRegex or false
+-	local sweetSpotCharacter = 6000
+ 	printf("Extracting ...")
+-	local fileList = ''
++	local fileTable = {}
+ 	for _, fname in ipairs(listToExtract) do
+ 		-- we are going to validate if the file is already extracted in this session
+ 		if not cache[fname] then
+ 			cache[fname] = true
+-			fileList = fileList .. '"' .. string.lower(fname) .. '" '
+-
+-			if fileList:len() > sweetSpotCharacter then
+-				self:ExtractFilesWithBun(fileList, useRegex)
+-				fileList = ''
+-			end
++			fileTable[#fileTable + 1] = fname
+ 		end
+ 	end
+
+-	if fileList:len() > 0 then
+-		self:ExtractFilesWithBun(fileList, useRegex)
+-		fileList = ''
+-	end
+-end
+-
+-function GGPKClass:AddDatFiles()
+-	local datFiles = scanDir(self.oozPath .. "Data\\Balance\\", '%w+%.dat$')
+-	for _, f in ipairs(datFiles) do
+-		local record = { }
+-		record.name = f
+-		local rawFile = io.open(self.oozPath .. "Data\\Balance\\" .. f, 'rb')
+-		record.data = rawFile:read("*all")
+-		rawFile:close()
+-		--ConPrintf("FILENAME: %s", fname)
+-		t_insert(self.dat, record)
+-	end
++	self:ExtractFilesWithBunFromTable(fileTable, useRegex)
+ end
+
+ function GGPKClass:AddDat64Files()