Git Progress (#3094)

Add a callback and delegate methods to report the progress of Git fetch operations in the SwiftPM CLI.

Co-authored-by: Anders Bertelrud <anders@apple.com>

Author: tgymnich

Sources/Commands/SwiftTool.swift

@@ -36,6 +36,9 @@ private class ToolWorkspaceDelegate: WorkspaceDelegate {
     /// The progress animation for downloads.
     private let downloadAnimation: NinjaProgressAnimation
 
+    /// The progress animation for repository fetches.
+    private let fetchAnimation: NinjaProgressAnimation
+
     /// Wether the tool is in a verbose mode.
     private let isVerbose: Bool
 
@@ -44,9 +47,17 @@ private class ToolWorkspaceDelegate: WorkspaceDelegate {
         let totalBytesToDownload: Int64
     }
 
+    private struct FetchProgress {
+        let objectsFetched: Int
+        let totalObjectsToFetch: Int
+    }
+
     /// The progress of each individual downloads.
     private var downloadProgress: [String: DownloadProgress] = [:]
 
+    /// The progress of each individual fetch operation
+    private var fetchProgress: [String: FetchProgress] = [:]
+
     private let queue = DispatchQueue(label: "org.swift.swiftpm.commands.tool-workspace-delegate")
     private let diagnostics: DiagnosticsEngine
 
@@ -55,6 +66,7 @@ private class ToolWorkspaceDelegate: WorkspaceDelegate {
         // https://forums.swift.org/t/allow-self-x-in-class-convenience-initializers/15924
         self.stdoutStream = stdoutStream as? ThreadSafeOutputByteStream ?? ThreadSafeOutputByteStream(stdoutStream)
         self.downloadAnimation = NinjaProgressAnimation(stream: self.stdoutStream)
+        self.fetchAnimation = NinjaProgressAnimation(stream: self.stdoutStream)
         self.isVerbose = isVerbose
         self.diagnostics = diagnostics
     }
@@ -74,6 +86,18 @@ private class ToolWorkspaceDelegate: WorkspaceDelegate {
 
     func fetchingDidFinish(repository: String, fetchDetails: RepositoryManager.FetchDetails?, diagnostic: Diagnostic?, duration: DispatchTimeInterval) {
         queue.async {
+            if self.diagnostics.hasErrors {
+                self.fetchAnimation.clear()
+            }
+
+            let step = self.fetchProgress.values.reduce(0) { $0 + $1.objectsFetched }
+            let total = self.fetchProgress.values.reduce(0) { $0 + $1.totalObjectsToFetch }
+
+            if step == total && !self.fetchProgress.isEmpty {
+                self.fetchAnimation.complete(success: true)
+                self.fetchProgress.removeAll()
+            }
+
             self.stdoutStream <<< "Fetched \(repository) (\(duration.descriptionInSeconds))"
             self.stdoutStream <<< "\n"
             self.stdoutStream.flush()
@@ -195,6 +219,18 @@ private class ToolWorkspaceDelegate: WorkspaceDelegate {
         }
     }
 
+    func fetchingRepository(from repository: String, objectsFetched: Int, totalObjectsToFetch: Int) {
+        queue.async {
+            self.fetchProgress[repository] = FetchProgress(
+                objectsFetched: objectsFetched,
+                totalObjectsToFetch: totalObjectsToFetch)
+
+            let step = self.fetchProgress.values.reduce(0) { $0 + $1.objectsFetched }
+            let total = self.fetchProgress.values.reduce(0) { $0 + $1.totalObjectsToFetch }
+            self.fetchAnimation.update(step: step, total: total, text: "Fetching objects")
+        }
+    }
+
     // noop
 
     func willLoadManifest(packagePath: AbsolutePath, url: String, version: Version?, packageKind: PackageReference.Kind) {}

Sources/SPMTestSupport/InMemoryGitRepository.swift

@@ -397,7 +397,7 @@ public final class InMemoryGitRepositoryProvider: RepositoryProvider {
     // MARK: - RepositoryProvider conformance
     // Note: These methods use force unwrap (instead of throwing) to honor their preconditions.
 
-    public func fetch(repository: RepositorySpecifier, to path: AbsolutePath) throws {
+    public func fetch(repository: RepositorySpecifier, to path: AbsolutePath, progressHandler: FetchProgress.Handler? = nil) throws {
         let repo = specifierMap[RepositorySpecifier(url: repository.url)]!
         fetchedMap[path] = try repo.copy()
         add(specifier: RepositorySpecifier(url: path.asURL.absoluteString), repository: repo)

Sources/SPMTestSupport/MockWorkspace.swift

@@ -646,6 +646,9 @@ public final class MockWorkspaceDelegate: WorkspaceDelegate {
         self.append("fetching repo: \(repository)")
     }
 
+    public func fetchingRepository(from repository: String, objectsFetched: Int, totalObjectsToFetch: Int) {
+    }
+    
     public func fetchingDidFinish(repository: String, fetchDetails: RepositoryManager.FetchDetails?, diagnostic: Diagnostic?, duration: DispatchTimeInterval) {
         self.append("finished fetching repo: \(repository)")
     }

Sources/SourceControl/GitRepository.swift

@@ -27,8 +27,8 @@ private struct GitShellHelper {
     /// Private function to invoke the Git tool with its default environment and given set of arguments.  The specified
     /// failure message is used only in case of error.  This function waits for the invocation to finish and returns the
     /// output as a string.
-    func run(_ args: [String], environment: [String: String] = Git.environment) throws -> String {
-        let process = Process(arguments: [Git.tool] + args, environment: environment, outputRedirection: .collect)
+    func run(_ args: [String], environment: [String: String] = Git.environment, outputRedirection: Process.OutputRedirection = .collect) throws -> String {
+        let process = Process(arguments: [Git.tool] + args, environment: environment, outputRedirection: outputRedirection)
         let result: ProcessResult
         do {
             try self.processSet?.add(process)
@@ -66,15 +66,33 @@ public struct GitRepositoryProvider: RepositoryProvider {
     private func callGit(_ args: String...,
                          environment: [String: String] = Git.environment,
                          repository: RepositorySpecifier,
-                         failureMessage: String = "") throws -> String {
-        do {
-            return try self.git.run(args, environment: environment)
-        } catch let error as GitShellError {
-            throw GitCloneError(repository: repository, message: failureMessage, result: error.result)
+                         failureMessage: String = "",
+                         progress: FetchProgress.Handler? = nil) throws -> String {
+        if let progress = progress {
+            var stdoutBytes: [UInt8] = [], stderrBytes: [UInt8] = []
+            do {
+                // Capture stdout and stderr from the Git subprocess invocation, but also pass along stderr to the handler. We count on it being line-buffered.
+                let outputHandler = Process.OutputRedirection.stream(stdout: { stdoutBytes += $0 }, stderr: {
+                    stderrBytes += $0
+                    gitFetchStatusFilter($0, progress: progress)
+                })
+                return try self.git.run(args + ["--progress"], environment: environment, outputRedirection: outputHandler)
+            }
+            catch let error as GitShellError {
+                let result = ProcessResult(arguments: error.result.arguments, environment: error.result.environment, exitStatus: error.result.exitStatus, output: .success(stdoutBytes), stderrOutput: .success(stderrBytes))
+                throw GitCloneError(repository: repository, message: failureMessage, result: result)
+            }
+        }
+        else {
+            do {
+                return try self.git.run(args, environment: environment)
+            } catch let error as GitShellError {
+                throw GitCloneError(repository: repository, message: failureMessage, result: error.result)
+            }
         }
     }
 
-    public func fetch(repository: RepositorySpecifier, to path: AbsolutePath) throws {
+    public func fetch(repository: RepositorySpecifier, to path: AbsolutePath, progressHandler: FetchProgress.Handler? = nil) throws {
         // Perform a bare clone.
         //
         // NOTE: We intentionally do not create a shallow clone here; the
@@ -87,7 +105,8 @@ public struct GitRepositoryProvider: RepositoryProvider {
         // NOTE: Explicitly set `core.symlinks=true` on `git clone` to ensure that symbolic links are correctly resolved.
         try self.callGit("clone", "-c", "core.symlinks=true", "--mirror", repository.url, path.pathString,
                          repository: repository,
-                         failureMessage: "Failed to clone repository \(repository.url)")
+                         failureMessage: "Failed to clone repository \(repository.url)",
+                         progress: progressHandler)
     }
 
     public func isValidDirectory(_ directory: String) -> Bool {
@@ -302,11 +321,29 @@ public final class GitRepository: Repository, WorkingCheckout {
     @discardableResult
     private func callGit(_ args: String...,
                          environment: [String: String] = Git.environment,
-                         failureMessage: String = "") throws -> String {
-        do {
-            return try self.git.run(["-C", self.path.pathString] + args, environment: environment)
-        } catch let error as GitShellError {
-            throw GitRepositoryError(path: self.path, message: failureMessage, result: error.result)
+                         failureMessage: String = "",
+                         progress: FetchProgress.Handler? = nil) throws -> String {
+        if let progress = progress {
+            var stdoutBytes: [UInt8] = [], stderrBytes: [UInt8] = []
+            do {
+                // Capture stdout and stderr from the Git subprocess invocation, but also pass along stderr to the handler. We count on it being line-buffered.
+                let outputHandler = Process.OutputRedirection.stream(stdout: { stdoutBytes += $0 }, stderr: {
+                    stderrBytes += $0
+                    gitFetchStatusFilter($0, progress: progress)
+                })
+                return try self.git.run(["-C", self.path.pathString] + args, environment: environment, outputRedirection: outputHandler)
+            }
+            catch let error as GitShellError {
+                let result = ProcessResult(arguments: error.result.arguments, environment: error.result.environment, exitStatus: error.result.exitStatus, output: .success(stdoutBytes), stderrOutput: .success(stderrBytes))
+                throw GitRepositoryError(path: self.path, message: failureMessage, result: result)
+            }
+        }
+        else {
+            do {
+                return try self.git.run(["-C", self.path.pathString] + args, environment: environment)
+            } catch let error as GitShellError {
+                throw GitRepositoryError(path: self.path, message: failureMessage, result: error.result)
+            }
         }
     }
 
@@ -365,10 +402,14 @@ public final class GitRepository: Repository, WorkingCheckout {
     }
 
     public func fetch() throws {
+       try fetch(progress: nil)
+    }
+
+    public func fetch(progress: FetchProgress.Handler? = nil) throws {
         // use barrier for write operations
         try self.lock.withLock {
-            try callGit("remote", "update", "-p",
-                        failureMessage: "Couldn’t fetch updates from remote repositories")
+            try callGit("remote", "-v", "update", "-p",
+                        failureMessage: "Couldn’t fetch updates from remote repositories", progress: progress)
             self.cachedTags.clear()
         }
     }
@@ -922,3 +963,153 @@ public struct GitCloneError: Error, CustomStringConvertible, DiagnosticLocationP
         return "\(self.message):\n\(output)"
     }
 }
+
+public enum GitProgressParser: FetchProgress {
+    case enumeratingObjects(currentObjects: Int)
+    case countingObjects(progress: Double, currentObjects: Int, totalObjects: Int)
+    case compressingObjects(progress: Double, currentObjects: Int, totalObjects: Int)
+    case receivingObjects(progress: Double, currentObjects: Int, totalObjects: Int, downloadProgress: String?, downloadSpeed: String?)
+    case resolvingDeltas(progress: Double, currentObjects: Int, totalObjects: Int)
+
+    /// The pattern used to match git output. Caputre groups are labled from ?<i0> to ?<i19>.
+    static let pattern = #"""
+(?xi)
+(?:
+    remote: \h+ (?<i0>Enumerating \h objects): \h+ (?<i1>[0-9]+)
+)|
+(?:
+    remote: \h+ (?<i2>Counting \h objects): \h+ (?<i3>[0-9]+)% \h+ \((?<i4>[0-9]+)\/(?<i5>[0-9]+)\)
+)|
+(?:
+    remote: \h+ (?<i6>Compressing \h objects): \h+ (?<i7>[0-9]+)% \h+ \((?<i8>[0-9]+)\/(?<i9>[0-9]+)\)
+)|
+(?:
+    (?<i10>Resolving \h deltas): \h+ (?<i11>[0-9]+)% \h+ \((?<i12>[0-9]+)\/(?<i13>[0-9]+)\)
+)|
+(?:
+    (?<i14>Receiving \h objects): \h+ (?<i15>[0-9]+)% \h+ \((?<i16>[0-9]+)\/(?<i17>[0-9]+)\)
+    (?:, \h+ (?<i18>[0-9]+.?[0-9]+ \h [A-Z]iB) \h+ \| \h+ (?<i19>[0-9]+.?[0-9]+ \h [A-Z]iB\/s))?
+)
+"""#
+    static let regex = try? RegEx(pattern: pattern)
+
+    init?(from string: String) {
+        guard let matches = GitProgressParser.regex?.matchGroups(in: string).first, matches.count == 20 else { return nil }
+
+        if matches[0] == "Enumerating objects" {
+            guard let currentObjects = Int(matches[1]) else { return nil }
+
+            self = .enumeratingObjects(currentObjects: currentObjects)
+        } else if matches[2] == "Counting objects" {
+            guard let progress = Double(matches[3]),
+                  let currentObjects = Int(matches[4]),
+                  let totalObjects = Int(matches[5]) else { return nil }
+
+            self = .countingObjects(progress: progress / 100, currentObjects: currentObjects, totalObjects: totalObjects)
+
+        } else if matches[6] == "Compressing objects" {
+            guard let progress = Double(matches[7]),
+                  let currentObjects = Int(matches[8]),
+                  let totalObjects = Int(matches[9]) else { return nil }
+
+            self = .compressingObjects(progress: progress / 100, currentObjects: currentObjects, totalObjects: totalObjects)
+
+        } else if matches[10] == "Resolving deltas" {
+            guard let progress = Double(matches[11]),
+                  let currentObjects = Int(matches[12]),
+                  let totalObjects = Int(matches[13]) else { return nil }
+
+            self = .resolvingDeltas(progress: progress / 100, currentObjects: currentObjects, totalObjects: totalObjects)
+
+        } else if matches[14] == "Receiving objects" {
+            guard let progress = Double(matches[15]),
+                  let currentObjects = Int(matches[16]),
+                  let totalObjects = Int(matches[17]) else { return nil }
+
+            let downloadProgress = matches[18]
+            let downloadSpeed = matches[19]
+
+            self = .receivingObjects(progress: progress / 100, currentObjects: currentObjects, totalObjects: totalObjects, downloadProgress: downloadProgress, downloadSpeed: downloadSpeed)
+
+        } else {
+            return nil
+        }
+    }
+
+    public var message: String {
+        switch self {
+        case .enumeratingObjects: return "Enumerating objects"
+        case .countingObjects: return "Counting objects"
+        case .compressingObjects: return "Compressing objects"
+        case .receivingObjects: return "Receiving objects"
+        case .resolvingDeltas: return "Resolving deltas"
+        }
+    }
+
+    public var step: Int {
+        switch self {
+        case .enumeratingObjects(let currentObjects):
+            return  currentObjects
+        case .countingObjects(_, let currentObjects, _):
+            return currentObjects
+        case .compressingObjects(_, let currentObjects, _):
+            return currentObjects
+        case .receivingObjects(_, let currentObjects, _, _, _):
+            return currentObjects
+        case .resolvingDeltas(_, let currentObjects, _):
+            return currentObjects
+        }
+    }
+
+    public var totalSteps: Int? {
+        switch self {
+        case .enumeratingObjects:
+            return 0
+        case .countingObjects(_, _, let totalObjects):
+            return totalObjects
+        case .compressingObjects(_, _, let totalObjects):
+            return totalObjects
+        case .receivingObjects(_, _, let totalObjects, _, _):
+            return totalObjects
+        case .resolvingDeltas(_, _, let totalObjects):
+            return totalObjects
+        }
+    }
+
+    public var downloadProgress: String? {
+        switch self {
+        case .receivingObjects(_, _, _, let downloadProgress, _):
+            return downloadProgress
+        default:
+            return nil
+        }
+    }
+
+    public var downloadSpeed: String? {
+        switch self {
+        case .receivingObjects(_, _, _, _, let downloadSpeed):
+            return downloadSpeed
+        default:
+            return nil
+        }
+    }
+}
+
+/// Processes stdout output and calls the progress callback with `GitStatus` objects.
+fileprivate func gitFetchStatusFilter(_ bytes: [UInt8], progress: FetchProgress.Handler) {
+    guard let string = String(bytes: bytes, encoding: .utf8) else { return }
+    let lines = string
+        .split { $0.isNewline }
+        .map { String($0) }
+
+    for line in lines {
+        if let status = GitProgressParser(from: line) {
+            switch status {
+            case .receivingObjects:
+                progress(status)
+            default:
+                continue
+            }
+        }
+    }
+}