From 73f87aeada347d6f6114d09f3e71e8aed571f90f Mon Sep 17 00:00:00 2001 From: tienquocbui Date: Thu, 13 Nov 2025 23:07:41 +0100 Subject: [PATCH 01/18] Add failure summary to ConsoleOutputRecorder. Implemented end-of-run failure summary using expandedDebugDescription for detailed output, with support for custom display names and parameterized tests. --- .../Event.ConsoleOutputRecorder.swift | 9 + .../Event.HumanReadableOutputRecorder.swift | 170 ++++++++++++++++++ 2 files changed, 179 insertions(+) diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index 80e68c609..3bf6dcbc5 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -306,6 +306,15 @@ extension Event.ConsoleOutputRecorder { /// destination. @discardableResult public func record(_ event: borrowing Event, in context: borrowing Event.Context) -> Bool { let messages = _humanReadableOutputRecorder.record(event, in: context) + + // Print failure summary when run ends + if case .runEnded = event.kind { + let summary = _humanReadableOutputRecorder.generateFailureSummary(options: options) + if !summary.isEmpty { + // Add blank line before summary and after summary for visual separation + write("\n\(summary)\n") + } + } // Padding to use in place of a symbol for messages that don't have one. var padding = " " diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 9f0ac21b1..93af8c9d8 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -64,6 +64,18 @@ extension Event { /// A type describing data tracked on a per-test basis. struct TestData { + /// A lightweight struct containing information about a single issue. + struct IssueInfo: Sendable { + /// The source location where the issue occurred. + var sourceLocation: SourceLocation? + + /// A detailed description of what failed (using expanded debug description). + var description: String + + /// Whether this issue is a known issue. + var isKnown: Bool + } + /// The instant at which the test started. var startInstant: Test.Clock.Instant @@ -76,6 +88,16 @@ extension Event { /// Information about the cancellation of this test or test case. var cancellationInfo: SkipInfo? + + /// Array of all issues recorded for this test (for failure summary). + /// Each issue is stored individually with its own source location. + var issues: [IssueInfo] = [] + + /// The test's display name, if any. + var testDisplayName: String? + + /// The test case arguments, formatted for display (for parameterized tests). + var testCaseArguments: String? } /// Data tracked on a per-test basis. @@ -317,6 +339,36 @@ extension Event.HumanReadableOutputRecorder { let issueCount = testData.issueCount[issue.severity] ?? 0 testData.issueCount[issue.severity] = issueCount + 1 } + + // Store individual issue information for failure summary (only for errors) + if issue.severity == .error { + // Extract detailed failure message using expandedDebugDescription + let description: String + if case let .expectationFailed(expectation) = issue.kind { + // Use expandedDebugDescription for full detail (variable values expanded) + description = expectation.evaluatedExpression.expandedDebugDescription() + } else if let comment = issue.comments.first { + description = comment.rawValue + } else { + description = "Test failed" + } + + let issueInfo = Context.TestData.IssueInfo( + sourceLocation: issue.sourceLocation, + description: description, + isKnown: issue.isKnown + ) + testData.issues.append(issueInfo) + + // Capture test display name and test case arguments once per test (not per issue) + if testData.testDisplayName == nil { + testData.testDisplayName = test?.displayName + } + if testData.testCaseArguments == nil { + testData.testCaseArguments = testCase?.labeledArguments() + } + } + context.testData[keyPath] = testData case .testCaseStarted: @@ -629,6 +681,124 @@ extension Event.HumanReadableOutputRecorder { return [] } + + /// Generate a failure summary string with all failed tests and their issues. + /// + /// This method traverses the test graph and formats a summary of all failures + /// that occurred during the test run. It includes the fully qualified test name + /// (with suite path), individual issues with their source locations, and uses + /// indentation to clearly delineate issue boundaries. + /// + /// - Parameters: + /// - options: Options for formatting (e.g., for ANSI colors and symbols). + /// + /// - Returns: A formatted string containing the failure summary, or an empty + /// string if there were no failures. + public func generateFailureSummary(options: Event.ConsoleOutputRecorder.Options) -> String { + let context = _context.rawValue + + // Collect all failed tests (tests with error issues) + struct FailedTestInfo { + var testPath: [String] // Full path including suite names + var testName: String + var issues: [Context.TestData.IssueInfo] + var testDisplayName: String? + var testCaseArguments: String? + } + + var failedTests: [FailedTestInfo] = [] + + // Traverse the graph to find all tests with failures + func traverse(graph: Graph, path: [String]) { + // Check if this node has test data with failures + if let testData = graph.value, !testData.issues.isEmpty { + let testName = path.last ?? "Unknown" + + failedTests.append(FailedTestInfo( + testPath: path, + testName: testName, + issues: testData.issues, + testDisplayName: testData.testDisplayName, + testCaseArguments: testData.testCaseArguments + )) + } + + // Recursively traverse children + for (key, childGraph) in graph.children { + let pathComponent: String? + switch key { + case let .string(s): + let parts = s.split(separator: ":") + if s.hasSuffix(".swift:") || (parts.count >= 2 && parts[0].hasSuffix(".swift")) { + pathComponent = nil + } else { + pathComponent = s + } + case let .testCaseID(id): + // Only include parameterized test case IDs in path, skip non-parameterized ones + if let argumentIDs = id.argumentIDs, let discriminator = id.discriminator { + pathComponent = "arguments: \(argumentIDs), discriminator: \(discriminator)" + } else { + // Non-parameterized test - don't add to path + pathComponent = nil + } + } + + let newPath = pathComponent.map { path + [$0] } ?? path + traverse(graph: childGraph, path: newPath) + } + } + + // Start traversal from root + traverse(graph: context.testData, path: []) + + // If no failures, return empty string + guard !failedTests.isEmpty else { + return "" + } + + var summary = "" + + // Header with failure count + let testWord = failedTests.count == 1 ? "test" : "tests" + let totalIssues = failedTests.reduce(0) { $0 + $1.issues.count } + let issueWord = totalIssues == 1 ? "issue" : "issues" + summary += "Test run had \(failedTests.count) failed \(testWord) with \(totalIssues) \(issueWord):\n" + + // Get the failure symbol + let failSymbol = Event.Symbol.fail.stringValue(options: options) + + // List each failed test + for failedTest in failedTests { + // Build fully qualified name with suite path + var fullyQualifiedName = failedTest.testPath.joined(separator: "/") + + // Use display name for the last component if available + if let displayName = failedTest.testDisplayName, + !failedTest.testPath.isEmpty { + // Replace the last component (test name) with display name + let pathWithoutLast = failedTest.testPath.dropLast() + fullyQualifiedName = (pathWithoutLast + [#""\#(displayName)""#]).joined(separator: "/") + } + + summary += "\(failSymbol) \(fullyQualifiedName)\n" + + // Show test case arguments for parameterized tests (once per test, not per issue) + if let arguments = failedTest.testCaseArguments, !arguments.isEmpty { + summary += " (\(arguments))\n" + } + + // List each issue for this test with indentation + for issue in failedTest.issues { + summary += " - \(issue.description)\n" + if let location = issue.sourceLocation { + summary += " at \(location.fileID):\(location.line)\n" + } + } + } + + return summary + } } extension Test.ID { From 29c448f78b5d1a724ee3e3c44af840b9083bcbfe Mon Sep 17 00:00:00 2001 From: tienquocbui Date: Tue, 18 Nov 2025 15:47:31 +0100 Subject: [PATCH 02/18] Add severity tracking to IssueInfo struct and factor failure summary logic into TestRunSummary type --- .../Event.ConsoleOutputRecorder.swift | 3 +- .../Event.HumanReadableOutputRecorder.swift | 343 ++++++++++++------ 2 files changed, 228 insertions(+), 118 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index 3bf6dcbc5..513b861c9 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -309,8 +309,7 @@ extension Event.ConsoleOutputRecorder { // Print failure summary when run ends if case .runEnded = event.kind { - let summary = _humanReadableOutputRecorder.generateFailureSummary(options: options) - if !summary.isEmpty { + if let summary = _humanReadableOutputRecorder.generateFailureSummary(options: options) { // Add blank line before summary and after summary for visual separation write("\n\(summary)\n") } diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 93af8c9d8..a1e05a17f 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -9,6 +9,211 @@ // extension Event { + /// A type that generates a failure summary from test run data. + /// + /// This type encapsulates the logic for collecting failed tests from a test + /// data graph and formatting them into a human-readable failure summary. + @_spi(ForToolsIntegrationOnly) + public struct TestRunSummary: Sendable { + /// Information about a single failed test. + struct FailedTest: Sendable { + /// The full hierarchical path to the test (e.g., suite names). + var path: [String] + + /// The test's simple name (last component of the path). + var name: String + + /// All issues recorded for this test. + var issues: [IssueInfo] + + /// The test's display name, if any. + var displayName: String? + + /// The test case arguments for parameterized tests, if any. + var testCaseArguments: String? + } + + /// Information about a single issue within a failed test. + struct IssueInfo: Sendable { + /// The source location where the issue occurred. + var sourceLocation: SourceLocation? + + /// A detailed description of what failed. + var description: String + + /// Whether this issue is a known issue. + var isKnown: Bool + + /// The severity of this issue. + var severity: Issue.Severity + } + + /// The list of failed tests collected from the test run. + private let failedTests: [FailedTest] + + /// Initialize a test run summary by collecting failures from a test data graph. + /// + /// - Parameters: + /// - testData: The root test data graph to traverse. + fileprivate init(from testData: Graph) { + var collected: [FailedTest] = [] + + // Traverse the graph to find all tests with failures + func traverse(graph: Graph, path: [String]) { + // Check if this node has test data with failures + if let testData = graph.value, !testData.issues.isEmpty { + let testName = path.last ?? "Unknown" + + // Convert Context.TestData.IssueInfo to TestRunSummary.IssueInfo + let issues = testData.issues.map { issue in + IssueInfo( + sourceLocation: issue.sourceLocation, + description: issue.description, + isKnown: issue.isKnown, + severity: issue.severity + ) + } + + collected.append(FailedTest( + path: path, + name: testName, + issues: issues, + displayName: testData.displayName, + testCaseArguments: testData.testCaseArguments + )) + } + + // Recursively traverse children + for (key, childGraph) in graph.children { + let pathComponent: String? + switch key { + case let .string(s): + let parts = s.split(separator: ":") + if s.hasSuffix(".swift:") || (parts.count >= 2 && parts[0].hasSuffix(".swift")) { + pathComponent = nil // Filter out source location strings + } else { + pathComponent = s + } + case let .testCaseID(id): + // Only include parameterized test case IDs in path + if let argumentIDs = id.argumentIDs, let discriminator = id.discriminator { + pathComponent = "arguments: \(argumentIDs), discriminator: \(discriminator)" + } else { + pathComponent = nil // Filter out non-parameterized test case IDs + } + } + + let newPath = pathComponent.map { path + [$0] } ?? path + traverse(graph: childGraph, path: newPath) + } + } + + // Start traversal from root + traverse(graph: testData, path: []) + + self.failedTests = collected + } + + /// Generate a formatted failure summary string. + /// + /// - Parameters: + /// - options: Options for formatting (e.g., for ANSI colors and symbols). + /// + /// - Returns: A formatted string containing the failure summary, or `nil` + /// if there were no failures. + public func formatted(with options: Event.ConsoleOutputRecorder.Options) -> String? { + // If no failures, return nil + guard !failedTests.isEmpty else { + return nil + } + + var summary = "" + + // Add header with failure count + summary += header() + + // Get the failure symbol + let failSymbol = Event.Symbol.fail.stringValue(options: options) + + // Format each failed test + for failedTest in failedTests { + summary += formatFailedTest(failedTest, withSymbol: failSymbol) + } + + return summary + } + + /// Generate the summary header with failure counts. + /// + /// - Returns: A string containing the header line. + private func header() -> String { + let testWord = failedTests.count == 1 ? "test" : "tests" + let totalIssues = failedTests.reduce(0) { $0 + $1.issues.count } + let issueWord = totalIssues == 1 ? "issue" : "issues" + return "Test run had \(failedTests.count) failed \(testWord) with \(totalIssues) \(issueWord):\n" + } + + /// Format a single failed test entry. + /// + /// - Parameters: + /// - failedTest: The failed test to format. + /// - symbol: The failure symbol string to use. + /// + /// - Returns: A formatted string representing the failed test and its issues. + private func formatFailedTest(_ failedTest: FailedTest, withSymbol symbol: String) -> String { + var result = "" + + // Build fully qualified name + var fullyQualifiedName = fullyQualifiedName(for: failedTest) + + result += "\(symbol) \(fullyQualifiedName)\n" + + // Show test case arguments for parameterized tests (once per test) + if let arguments = failedTest.testCaseArguments, !arguments.isEmpty { + result += " (\(arguments))\n" + } + + // List each issue for this test with indentation + for issue in failedTest.issues { + result += formatIssue(issue) + } + + return result + } + + /// Build the fully qualified name for a failed test. + /// + /// - Parameters: + /// - failedTest: The failed test. + /// + /// - Returns: The fully qualified name, with display name substituted if available. + private func fullyQualifiedName(for failedTest: FailedTest) -> String { + var name = failedTest.path.joined(separator: "/") + + // Use display name for the last component if available + if let displayName = failedTest.displayName, !failedTest.path.isEmpty { + let pathWithoutLast = failedTest.path.dropLast() + name = (pathWithoutLast + [#""\#(displayName)""#]).joined(separator: "/") + } + + return name + } + + /// Format a single issue entry. + /// + /// - Parameters: + /// - issue: The issue to format. + /// + /// - Returns: A formatted string representing the issue with indentation. + private func formatIssue(_ issue: IssueInfo) -> String { + var result = " - \(issue.description)\n" + if let location = issue.sourceLocation { + result += " at \(location.fileID):\(location.line)\n" + } + return result + } + } + /// A type which handles ``Event`` instances and outputs representations of /// them as human-readable messages. /// @@ -74,6 +279,9 @@ extension Event { /// Whether this issue is a known issue. var isKnown: Bool + + /// The severity of this issue. + var severity: Issue.Severity } /// The instant at which the test started. @@ -94,7 +302,7 @@ extension Event { var issues: [IssueInfo] = [] /// The test's display name, if any. - var testDisplayName: String? + var displayName: String? /// The test case arguments, formatted for display (for parameterized tests). var testCaseArguments: String? @@ -342,11 +550,15 @@ extension Event.HumanReadableOutputRecorder { // Store individual issue information for failure summary (only for errors) if issue.severity == .error { - // Extract detailed failure message using expandedDebugDescription + // Extract detailed failure message let description: String if case let .expectationFailed(expectation) = issue.kind { - // Use expandedDebugDescription for full detail (variable values expanded) - description = expectation.evaluatedExpression.expandedDebugDescription() + // Use expandedDebugDescription only when verbose, otherwise use expandedDescription + description = if verbosity > 0 { + expectation.evaluatedExpression.expandedDebugDescription() + } else { + expectation.evaluatedExpression.expandedDescription() + } } else if let comment = issue.comments.first { description = comment.rawValue } else { @@ -356,13 +568,14 @@ extension Event.HumanReadableOutputRecorder { let issueInfo = Context.TestData.IssueInfo( sourceLocation: issue.sourceLocation, description: description, - isKnown: issue.isKnown + isKnown: issue.isKnown, + severity: issue.severity ) testData.issues.append(issueInfo) // Capture test display name and test case arguments once per test (not per issue) - if testData.testDisplayName == nil { - testData.testDisplayName = test?.displayName + if testData.displayName == nil { + testData.displayName = test?.displayName } if testData.testCaseArguments == nil { testData.testCaseArguments = testCase?.labeledArguments() @@ -684,120 +897,18 @@ extension Event.HumanReadableOutputRecorder { /// Generate a failure summary string with all failed tests and their issues. /// - /// This method traverses the test graph and formats a summary of all failures - /// that occurred during the test run. It includes the fully qualified test name - /// (with suite path), individual issues with their source locations, and uses - /// indentation to clearly delineate issue boundaries. + /// This method creates a ``TestRunSummary`` from the test data graph and + /// formats it for display. /// /// - Parameters: /// - options: Options for formatting (e.g., for ANSI colors and symbols). /// - /// - Returns: A formatted string containing the failure summary, or an empty - /// string if there were no failures. - public func generateFailureSummary(options: Event.ConsoleOutputRecorder.Options) -> String { + /// - Returns: A formatted string containing the failure summary, or `nil` + /// if there were no failures. + public func generateFailureSummary(options: Event.ConsoleOutputRecorder.Options) -> String? { let context = _context.rawValue - - // Collect all failed tests (tests with error issues) - struct FailedTestInfo { - var testPath: [String] // Full path including suite names - var testName: String - var issues: [Context.TestData.IssueInfo] - var testDisplayName: String? - var testCaseArguments: String? - } - - var failedTests: [FailedTestInfo] = [] - - // Traverse the graph to find all tests with failures - func traverse(graph: Graph, path: [String]) { - // Check if this node has test data with failures - if let testData = graph.value, !testData.issues.isEmpty { - let testName = path.last ?? "Unknown" - - failedTests.append(FailedTestInfo( - testPath: path, - testName: testName, - issues: testData.issues, - testDisplayName: testData.testDisplayName, - testCaseArguments: testData.testCaseArguments - )) - } - - // Recursively traverse children - for (key, childGraph) in graph.children { - let pathComponent: String? - switch key { - case let .string(s): - let parts = s.split(separator: ":") - if s.hasSuffix(".swift:") || (parts.count >= 2 && parts[0].hasSuffix(".swift")) { - pathComponent = nil - } else { - pathComponent = s - } - case let .testCaseID(id): - // Only include parameterized test case IDs in path, skip non-parameterized ones - if let argumentIDs = id.argumentIDs, let discriminator = id.discriminator { - pathComponent = "arguments: \(argumentIDs), discriminator: \(discriminator)" - } else { - // Non-parameterized test - don't add to path - pathComponent = nil - } - } - - let newPath = pathComponent.map { path + [$0] } ?? path - traverse(graph: childGraph, path: newPath) - } - } - - // Start traversal from root - traverse(graph: context.testData, path: []) - - // If no failures, return empty string - guard !failedTests.isEmpty else { - return "" - } - - var summary = "" - - // Header with failure count - let testWord = failedTests.count == 1 ? "test" : "tests" - let totalIssues = failedTests.reduce(0) { $0 + $1.issues.count } - let issueWord = totalIssues == 1 ? "issue" : "issues" - summary += "Test run had \(failedTests.count) failed \(testWord) with \(totalIssues) \(issueWord):\n" - - // Get the failure symbol - let failSymbol = Event.Symbol.fail.stringValue(options: options) - - // List each failed test - for failedTest in failedTests { - // Build fully qualified name with suite path - var fullyQualifiedName = failedTest.testPath.joined(separator: "/") - - // Use display name for the last component if available - if let displayName = failedTest.testDisplayName, - !failedTest.testPath.isEmpty { - // Replace the last component (test name) with display name - let pathWithoutLast = failedTest.testPath.dropLast() - fullyQualifiedName = (pathWithoutLast + [#""\#(displayName)""#]).joined(separator: "/") - } - - summary += "\(failSymbol) \(fullyQualifiedName)\n" - - // Show test case arguments for parameterized tests (once per test, not per issue) - if let arguments = failedTest.testCaseArguments, !arguments.isEmpty { - summary += " (\(arguments))\n" - } - - // List each issue for this test with indentation - for issue in failedTest.issues { - summary += " - \(issue.description)\n" - if let location = issue.sourceLocation { - summary += " at \(location.fileID):\(location.line)\n" - } - } - } - - return summary + let summary = Event.TestRunSummary(from: context.testData) + return summary.formatted(with: options) } } From 893529db8f5a426ea70e95afbbb509f25e89f547 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 10:01:18 -0600 Subject: [PATCH 03/18] nit: whitespace --- .../Event.HumanReadableOutputRecorder.swift | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index a1e05a17f..6642a771d 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -19,51 +19,51 @@ extension Event { struct FailedTest: Sendable { /// The full hierarchical path to the test (e.g., suite names). var path: [String] - + /// The test's simple name (last component of the path). var name: String - + /// All issues recorded for this test. var issues: [IssueInfo] - + /// The test's display name, if any. var displayName: String? - + /// The test case arguments for parameterized tests, if any. var testCaseArguments: String? } - + /// Information about a single issue within a failed test. struct IssueInfo: Sendable { /// The source location where the issue occurred. var sourceLocation: SourceLocation? - + /// A detailed description of what failed. var description: String - + /// Whether this issue is a known issue. var isKnown: Bool - + /// The severity of this issue. var severity: Issue.Severity } - + /// The list of failed tests collected from the test run. private let failedTests: [FailedTest] - + /// Initialize a test run summary by collecting failures from a test data graph. /// /// - Parameters: /// - testData: The root test data graph to traverse. fileprivate init(from testData: Graph) { var collected: [FailedTest] = [] - + // Traverse the graph to find all tests with failures func traverse(graph: Graph, path: [String]) { // Check if this node has test data with failures if let testData = graph.value, !testData.issues.isEmpty { let testName = path.last ?? "Unknown" - + // Convert Context.TestData.IssueInfo to TestRunSummary.IssueInfo let issues = testData.issues.map { issue in IssueInfo( @@ -73,7 +73,7 @@ extension Event { severity: issue.severity ) } - + collected.append(FailedTest( path: path, name: testName, @@ -82,7 +82,7 @@ extension Event { testCaseArguments: testData.testCaseArguments )) } - + // Recursively traverse children for (key, childGraph) in graph.children { let pathComponent: String? @@ -102,18 +102,18 @@ extension Event { pathComponent = nil // Filter out non-parameterized test case IDs } } - + let newPath = pathComponent.map { path + [$0] } ?? path traverse(graph: childGraph, path: newPath) } } - + // Start traversal from root traverse(graph: testData, path: []) - + self.failedTests = collected } - + /// Generate a formatted failure summary string. /// /// - Parameters: @@ -126,23 +126,23 @@ extension Event { guard !failedTests.isEmpty else { return nil } - + var summary = "" - + // Add header with failure count summary += header() - + // Get the failure symbol let failSymbol = Event.Symbol.fail.stringValue(options: options) - + // Format each failed test for failedTest in failedTests { summary += formatFailedTest(failedTest, withSymbol: failSymbol) } - + return summary } - + /// Generate the summary header with failure counts. /// /// - Returns: A string containing the header line. @@ -152,7 +152,7 @@ extension Event { let issueWord = totalIssues == 1 ? "issue" : "issues" return "Test run had \(failedTests.count) failed \(testWord) with \(totalIssues) \(issueWord):\n" } - + /// Format a single failed test entry. /// /// - Parameters: @@ -162,25 +162,25 @@ extension Event { /// - Returns: A formatted string representing the failed test and its issues. private func formatFailedTest(_ failedTest: FailedTest, withSymbol symbol: String) -> String { var result = "" - + // Build fully qualified name var fullyQualifiedName = fullyQualifiedName(for: failedTest) - + result += "\(symbol) \(fullyQualifiedName)\n" - + // Show test case arguments for parameterized tests (once per test) if let arguments = failedTest.testCaseArguments, !arguments.isEmpty { result += " (\(arguments))\n" } - + // List each issue for this test with indentation for issue in failedTest.issues { result += formatIssue(issue) } - + return result } - + /// Build the fully qualified name for a failed test. /// /// - Parameters: @@ -189,16 +189,16 @@ extension Event { /// - Returns: The fully qualified name, with display name substituted if available. private func fullyQualifiedName(for failedTest: FailedTest) -> String { var name = failedTest.path.joined(separator: "/") - + // Use display name for the last component if available if let displayName = failedTest.displayName, !failedTest.path.isEmpty { let pathWithoutLast = failedTest.path.dropLast() name = (pathWithoutLast + [#""\#(displayName)""#]).joined(separator: "/") } - + return name } - + /// Format a single issue entry. /// /// - Parameters: @@ -213,7 +213,7 @@ extension Event { return result } } - + /// A type which handles ``Event`` instances and outputs representations of /// them as human-readable messages. /// @@ -273,17 +273,17 @@ extension Event { struct IssueInfo: Sendable { /// The source location where the issue occurred. var sourceLocation: SourceLocation? - + /// A detailed description of what failed (using expanded debug description). var description: String - + /// Whether this issue is a known issue. var isKnown: Bool - + /// The severity of this issue. var severity: Issue.Severity } - + /// The instant at which the test started. var startInstant: Test.Clock.Instant @@ -296,14 +296,14 @@ extension Event { /// Information about the cancellation of this test or test case. var cancellationInfo: SkipInfo? - + /// Array of all issues recorded for this test (for failure summary). /// Each issue is stored individually with its own source location. var issues: [IssueInfo] = [] - + /// The test's display name, if any. var displayName: String? - + /// The test case arguments, formatted for display (for parameterized tests). var testCaseArguments: String? } @@ -547,7 +547,7 @@ extension Event.HumanReadableOutputRecorder { let issueCount = testData.issueCount[issue.severity] ?? 0 testData.issueCount[issue.severity] = issueCount + 1 } - + // Store individual issue information for failure summary (only for errors) if issue.severity == .error { // Extract detailed failure message @@ -564,7 +564,7 @@ extension Event.HumanReadableOutputRecorder { } else { description = "Test failed" } - + let issueInfo = Context.TestData.IssueInfo( sourceLocation: issue.sourceLocation, description: description, @@ -572,7 +572,7 @@ extension Event.HumanReadableOutputRecorder { severity: issue.severity ) testData.issues.append(issueInfo) - + // Capture test display name and test case arguments once per test (not per issue) if testData.displayName == nil { testData.displayName = test?.displayName @@ -581,9 +581,9 @@ extension Event.HumanReadableOutputRecorder { testData.testCaseArguments = testCase?.labeledArguments() } } - + context.testData[keyPath] = testData - + case .testCaseStarted: context.testData[keyPath] = .init(startInstant: instant) @@ -894,7 +894,7 @@ extension Event.HumanReadableOutputRecorder { return [] } - + /// Generate a failure summary string with all failed tests and their issues. /// /// This method creates a ``TestRunSummary`` from the test data graph and From 416bdf42146a91dc976662ee5d3f547fb8f058e9 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 10:02:09 -0600 Subject: [PATCH 04/18] Fix warning about unnecessary var --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 6642a771d..aa2efd328 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -164,7 +164,7 @@ extension Event { var result = "" // Build fully qualified name - var fullyQualifiedName = fullyQualifiedName(for: failedTest) + let fullyQualifiedName = fullyQualifiedName(for: failedTest) result += "\(symbol) \(fullyQualifiedName)\n" From 42d582df9a9073fa7c2b90099d6c639d1d3520ca Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 10:17:19 -0600 Subject: [PATCH 05/18] Include an environment variable to control whether the failure summary is shown, with the default behavior being enabled --- .../Events/Recorder/Event.ConsoleOutputRecorder.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index 513b861c9..172c94d89 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -307,8 +307,9 @@ extension Event.ConsoleOutputRecorder { @discardableResult public func record(_ event: borrowing Event, in context: borrowing Event.Context) -> Bool { let messages = _humanReadableOutputRecorder.record(event, in: context) - // Print failure summary when run ends - if case .runEnded = event.kind { + // Print failure summary when run ends, unless an environment variable is + // set to explicitly disable it. + if case .runEnded = event.kind, Environment.flag(named: "SWT_FAILURE_SUMMARY_ENABLED") != false { if let summary = _humanReadableOutputRecorder.generateFailureSummary(options: options) { // Add blank line before summary and after summary for visual separation write("\n\(summary)\n") From de83b879464fa2c3a4e622e535479a80a64390ff Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 10:19:45 -0600 Subject: [PATCH 06/18] TestRunSummary doesn't need to be `@_spi public`, it can be `private` --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index aa2efd328..4a4c92577 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -13,8 +13,7 @@ extension Event { /// /// This type encapsulates the logic for collecting failed tests from a test /// data graph and formatting them into a human-readable failure summary. - @_spi(ForToolsIntegrationOnly) - public struct TestRunSummary: Sendable { + private struct TestRunSummary: Sendable { /// Information about a single failed test. struct FailedTest: Sendable { /// The full hierarchical path to the test (e.g., suite names). From 69ac56e09490c55d48e929901149deb634f9cbe2 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 10:20:27 -0600 Subject: [PATCH 07/18] Use the `counting(_:)` utility to pluralize nouns --- .../Recorder/Event.HumanReadableOutputRecorder.swift | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 4a4c92577..8c3baf7ed 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -146,10 +146,11 @@ extension Event { /// /// - Returns: A string containing the header line. private func header() -> String { - let testWord = failedTests.count == 1 ? "test" : "tests" - let totalIssues = failedTests.reduce(0) { $0 + $1.issues.count } - let issueWord = totalIssues == 1 ? "issue" : "issues" - return "Test run had \(failedTests.count) failed \(testWord) with \(totalIssues) \(issueWord):\n" + let failedTestsCount = failedTests.count + let testWord = failedTestsCount.counting("test") + let totalIssuesCount = failedTests.reduce(0) { $0 + $1.issues.count } + let issueWord = totalIssuesCount.counting("issue") + return "Test run had \(failedTestsCount) failed \(testWord) with \(totalIssuesCount) \(issueWord):\n" } /// Format a single failed test entry. From da879e8d29a7a63342333f3315d6e80442ebb14b Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 10:24:27 -0600 Subject: [PATCH 08/18] Correct comment about "debug" description, specifically --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 8c3baf7ed..e524b3252 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -274,7 +274,7 @@ extension Event { /// The source location where the issue occurred. var sourceLocation: SourceLocation? - /// A detailed description of what failed (using expanded debug description). + /// A detailed description of what failed (using expanded description). var description: String /// Whether this issue is a known issue. From 8bcee7a4b56c6bbe20e72e94632c5b34b0cfec71 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 11:09:44 -0600 Subject: [PATCH 09/18] Fix error in my earlier change which adopted 'counting', and rephrase message so that it uses the 'test(s) which recorded N issue(s)" terminology --- .../Recorder/Event.HumanReadableOutputRecorder.swift | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index e524b3252..47aff7705 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -146,11 +146,10 @@ extension Event { /// /// - Returns: A string containing the header line. private func header() -> String { - let failedTestsCount = failedTests.count - let testWord = failedTestsCount.counting("test") + let failedTestsPhrase = failedTests.count.counting("test") let totalIssuesCount = failedTests.reduce(0) { $0 + $1.issues.count } - let issueWord = totalIssuesCount.counting("issue") - return "Test run had \(failedTestsCount) failed \(testWord) with \(totalIssuesCount) \(issueWord):\n" + let issuesPhrase = totalIssuesCount.counting("issue") + return "Test run had \(failedTestsPhrase) which recorded \(issuesPhrase) total:\n" } /// Format a single failed test entry. From 21e91c59d7f90a018777886638b29f9ffd0de1ea Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 11:11:20 -0600 Subject: [PATCH 10/18] fixup: whitespace --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 47aff7705..2cb280efa 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -158,7 +158,8 @@ extension Event { /// - failedTest: The failed test to format. /// - symbol: The failure symbol string to use. /// - /// - Returns: A formatted string representing the failed test and its issues. + /// - Returns: A formatted string representing the failed test and its + /// issues. private func formatFailedTest(_ failedTest: FailedTest, withSymbol symbol: String) -> String { var result = "" @@ -185,7 +186,8 @@ extension Event { /// - Parameters: /// - failedTest: The failed test. /// - /// - Returns: The fully qualified name, with display name substituted if available. + /// - Returns: The fully qualified name, with display name substituted if + /// available. private func fullyQualifiedName(for failedTest: FailedTest) -> String { var name = failedTest.path.joined(separator: "/") From fe58065d50359382312ce19561204dab936374e4 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 11:11:35 -0600 Subject: [PATCH 11/18] Simplify by beginning variable with initial value --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 2cb280efa..d98122895 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -126,10 +126,8 @@ extension Event { return nil } - var summary = "" - - // Add header with failure count - summary += header() + // Begin with the summary header. + var summary = header() // Get the failure symbol let failSymbol = Event.Symbol.fail.stringValue(options: options) From c83ae9baffd3a872ac550e1e171932bfcf89d902 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 11:13:44 -0600 Subject: [PATCH 12/18] Omit the module name on complete paths for tests --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index d98122895..db4c8d496 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -187,11 +187,11 @@ extension Event { /// - Returns: The fully qualified name, with display name substituted if /// available. private func fullyQualifiedName(for failedTest: FailedTest) -> String { - var name = failedTest.path.joined(separator: "/") + var name = failedTest.path.dropFirst().joined(separator: "/") // Use display name for the last component if available if let displayName = failedTest.displayName, !failedTest.path.isEmpty { - let pathWithoutLast = failedTest.path.dropLast() + let pathWithoutLast = failedTest.path.dropFirst().dropLast() name = (pathWithoutLast + [#""\#(displayName)""#]).joined(separator: "/") } From a46b1c58ad99df5a898e333539ed6155cdab1e39 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 11:14:10 -0600 Subject: [PATCH 13/18] Indent source location of issues one level deeper and use the CustomStringConvertible conformance instead of fileID:line --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index db4c8d496..efc83d0a4 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -207,7 +207,7 @@ extension Event { private func formatIssue(_ issue: IssueInfo) -> String { var result = " - \(issue.description)\n" if let location = issue.sourceLocation { - result += " at \(location.fileID):\(location.line)\n" + result += " at \(location)\n" } return result } From e6558c50079b0d13b210430a9e16de19b1119df9 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 11:15:06 -0600 Subject: [PATCH 14/18] Use severity >= .error instead of == .error --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index efc83d0a4..b23a31360 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -547,8 +547,9 @@ extension Event.HumanReadableOutputRecorder { testData.issueCount[issue.severity] = issueCount + 1 } - // Store individual issue information for failure summary (only for errors) - if issue.severity == .error { + // Store individual issue information for failure summary, but only for + // issues whose severity is error or greater. + if issue.severity >= .error { // Extract detailed failure message let description: String if case let .expectationFailed(expectation) = issue.kind { From b3ba640a4814b205ecbd1226c401dfcdd4505114 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 11:27:11 -0600 Subject: [PATCH 15/18] Refactor to simplify and consolidate some logic in fullyQualifiedName() --- .../Event.HumanReadableOutputRecorder.swift | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index b23a31360..47e7e00e7 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -187,15 +187,17 @@ extension Event { /// - Returns: The fully qualified name, with display name substituted if /// available. private func fullyQualifiedName(for failedTest: FailedTest) -> String { - var name = failedTest.path.dropFirst().joined(separator: "/") - - // Use display name for the last component if available - if let displayName = failedTest.displayName, !failedTest.path.isEmpty { - let pathWithoutLast = failedTest.path.dropFirst().dropLast() - name = (pathWithoutLast + [#""\#(displayName)""#]).joined(separator: "/") + // Omit the leading path component representing the module name from the + // fully-qualified name of the test. + let path = failedTest.path.dropFirst() + + // Use display name for the last component if available. Otherwise, join + // the path components. + return if let displayName = failedTest.displayName, !failedTest.path.isEmpty { + (path.dropLast() + [#""\#(displayName)""#]).joined(separator: "/") + } else { + path.joined(separator: "/") } - - return name } /// Format a single issue entry. From 46c077bab0e94b0bfc9b5e0d61b22489dc38db40 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 11:31:15 -0600 Subject: [PATCH 16/18] whitespace --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 47e7e00e7..cf2d4e1ee 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -585,7 +585,7 @@ extension Event.HumanReadableOutputRecorder { } context.testData[keyPath] = testData - + case .testCaseStarted: context.testData[keyPath] = .init(startInstant: instant) From 3acb802ed1ef05112275f8acb80755f046f22707 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 2 Dec 2025 11:31:25 -0600 Subject: [PATCH 17/18] Remove unnecessary 'public' access level --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index cf2d4e1ee..19429f5b1 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -907,7 +907,7 @@ extension Event.HumanReadableOutputRecorder { /// /// - Returns: A formatted string containing the failure summary, or `nil` /// if there were no failures. - public func generateFailureSummary(options: Event.ConsoleOutputRecorder.Options) -> String? { + func generateFailureSummary(options: Event.ConsoleOutputRecorder.Options) -> String? { let context = _context.rawValue let summary = Event.TestRunSummary(from: context.testData) return summary.formatted(with: options) From 44e71d49a97fd4e2b3784199f34c09b9bcc75305 Mon Sep 17 00:00:00 2001 From: tienquocbui Date: Wed, 3 Dec 2025 17:39:07 +0100 Subject: [PATCH 18/18] Fix parameterized test display and group test cases under parent test with proper indentation hierarchy --- .../Event.HumanReadableOutputRecorder.swift | 164 +++++++++++------- 1 file changed, 106 insertions(+), 58 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 19429f5b1..3b81d12b7 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -14,6 +14,15 @@ extension Event { /// This type encapsulates the logic for collecting failed tests from a test /// data graph and formatting them into a human-readable failure summary. private struct TestRunSummary: Sendable { + /// Information about a single failed test case (for parameterized tests). + struct FailedTestCase: Sendable { + /// The test case arguments for this parameterized test case. + var arguments: String + + /// All issues recorded for this test case. + var issues: [HumanReadableOutputRecorder.Context.TestData.IssueInfo] + } + /// Information about a single failed test. struct FailedTest: Sendable { /// The full hierarchical path to the test (e.g., suite names). @@ -22,29 +31,14 @@ extension Event { /// The test's simple name (last component of the path). var name: String - /// All issues recorded for this test. - var issues: [IssueInfo] - /// The test's display name, if any. var displayName: String? - /// The test case arguments for parameterized tests, if any. - var testCaseArguments: String? - } - - /// Information about a single issue within a failed test. - struct IssueInfo: Sendable { - /// The source location where the issue occurred. - var sourceLocation: SourceLocation? + /// For non-parameterized tests: issues recorded directly on the test. + var issues: [HumanReadableOutputRecorder.Context.TestData.IssueInfo] - /// A detailed description of what failed. - var description: String - - /// Whether this issue is a known issue. - var isKnown: Bool - - /// The severity of this issue. - var severity: Issue.Severity + /// For parameterized tests: test cases with their issues. + var testCases: [FailedTestCase] } /// The list of failed tests collected from the test run. @@ -55,62 +49,95 @@ extension Event { /// - Parameters: /// - testData: The root test data graph to traverse. fileprivate init(from testData: Graph) { - var collected: [FailedTest] = [] + var testMap: [String: FailedTest] = [:] // Traverse the graph to find all tests with failures - func traverse(graph: Graph, path: [String]) { + func traverse(graph: Graph, path: [String], isTestCase: Bool = false) { // Check if this node has test data with failures if let testData = graph.value, !testData.issues.isEmpty { let testName = path.last ?? "Unknown" - // Convert Context.TestData.IssueInfo to TestRunSummary.IssueInfo - let issues = testData.issues.map { issue in - IssueInfo( - sourceLocation: issue.sourceLocation, - description: issue.description, - isKnown: issue.isKnown, - severity: issue.severity + // Use issues directly from testData + let issues = testData.issues + + if isTestCase { + // This is a test case node - add it to the parent test's testCases array + // The parent test path is the path without the test case ID component + let parentPath = path.filter { !$0.hasPrefix("arguments:") } + let parentPathKey = parentPath.joined(separator: "/") + + if var parentTest = testMap[parentPathKey] { + // Add this test case to the parent + if let arguments = testData.testCaseArguments, !arguments.isEmpty { + parentTest.testCases.append(FailedTestCase( + arguments: arguments, + issues: issues + )) + testMap[parentPathKey] = parentTest + } + } else { + // Parent test not found in map, but should exist - create it + let parentTest = FailedTest( + path: parentPath, + name: parentPath.last ?? "Unknown", + displayName: testData.displayName, + issues: [], + testCases: (testData.testCaseArguments?.isEmpty ?? true) ? [] : [FailedTestCase( + arguments: testData.testCaseArguments ?? "", + issues: issues + )] + ) + testMap[parentPathKey] = parentTest + } + } else { + // This is a test node (not a test case) + let pathKey = path.joined(separator: "/") + let failedTest = FailedTest( + path: path, + name: testName, + displayName: testData.displayName, + issues: issues, + testCases: [] ) + testMap[pathKey] = failedTest } - - collected.append(FailedTest( - path: path, - name: testName, - issues: issues, - displayName: testData.displayName, - testCaseArguments: testData.testCaseArguments - )) } // Recursively traverse children for (key, childGraph) in graph.children { let pathComponent: String? + let isChildTestCase: Bool switch key { case let .string(s): let parts = s.split(separator: ":") if s.hasSuffix(".swift:") || (parts.count >= 2 && parts[0].hasSuffix(".swift")) { pathComponent = nil // Filter out source location strings + isChildTestCase = false } else { pathComponent = s + isChildTestCase = false } case let .testCaseID(id): // Only include parameterized test case IDs in path if let argumentIDs = id.argumentIDs, let discriminator = id.discriminator { pathComponent = "arguments: \(argumentIDs), discriminator: \(discriminator)" + isChildTestCase = true } else { pathComponent = nil // Filter out non-parameterized test case IDs + isChildTestCase = false } } let newPath = pathComponent.map { path + [$0] } ?? path - traverse(graph: childGraph, path: newPath) + traverse(graph: childGraph, path: newPath, isTestCase: isChildTestCase) } } // Start traversal from root traverse(graph: testData, path: []) - self.failedTests = collected + // Convert map to array, ensuring we only include tests that have failures + self.failedTests = Array(testMap.values).filter { !$0.issues.isEmpty || !$0.testCases.isEmpty } } /// Generate a formatted failure summary string. @@ -145,7 +172,13 @@ extension Event { /// - Returns: A string containing the header line. private func header() -> String { let failedTestsPhrase = failedTests.count.counting("test") - let totalIssuesCount = failedTests.reduce(0) { $0 + $1.issues.count } + var totalIssuesCount = 0 + for test in failedTests { + totalIssuesCount += test.issues.count + for testCase in test.testCases { + totalIssuesCount += testCase.issues.count + } + } let issuesPhrase = totalIssuesCount.counting("issue") return "Test run had \(failedTestsPhrase) which recorded \(issuesPhrase) total:\n" } @@ -166,14 +199,21 @@ extension Event { result += "\(symbol) \(fullyQualifiedName)\n" - // Show test case arguments for parameterized tests (once per test) - if let arguments = failedTest.testCaseArguments, !arguments.isEmpty { - result += " (\(arguments))\n" - } - - // List each issue for this test with indentation - for issue in failedTest.issues { - result += formatIssue(issue) + // For parameterized tests: show test cases grouped under the parent test + if !failedTest.testCases.isEmpty { + for testCase in failedTest.testCases { + // Show test case arguments with additional indentation + result += " (\(testCase.arguments))\n" + // List each issue for this test case with additional indentation + for issue in testCase.issues { + result += formatIssue(issue, indentLevel: 2) + } + } + } else { + // For non-parameterized tests: show issues directly + for issue in failedTest.issues { + result += formatIssue(issue) + } } return result @@ -185,31 +225,39 @@ extension Event { /// - failedTest: The failed test. /// /// - Returns: The fully qualified name, with display name substituted if - /// available. + /// available. Test case ID components are filtered out since they're + /// shown separately. private func fullyQualifiedName(for failedTest: FailedTest) -> String { // Omit the leading path component representing the module name from the // fully-qualified name of the test. - let path = failedTest.path.dropFirst() + var path = Array(failedTest.path.dropFirst()) - // Use display name for the last component if available. Otherwise, join - // the path components. - return if let displayName = failedTest.displayName, !failedTest.path.isEmpty { - (path.dropLast() + [#""\#(displayName)""#]).joined(separator: "/") - } else { - path.joined(separator: "/") + // Filter out test case ID components (they're shown separately with arguments) + path = path.filter { !$0.hasPrefix("arguments:") } + + // If we have a display name, replace the function name component (which is + // now the last component after filtering) with the display name. This avoids + // showing both the function name and display name. + if let displayName = failedTest.displayName, !path.isEmpty { + path[path.count - 1] = #""\#(displayName)""# } + + return path.joined(separator: "/") } /// Format a single issue entry. /// /// - Parameters: /// - issue: The issue to format. + /// - indentLevel: The number of indentation levels (each level is 2 spaces). + /// Defaults to 1. /// /// - Returns: A formatted string representing the issue with indentation. - private func formatIssue(_ issue: IssueInfo) -> String { - var result = " - \(issue.description)\n" + private func formatIssue(_ issue: HumanReadableOutputRecorder.Context.TestData.IssueInfo, indentLevel: Int = 1) -> String { + let indent = String(repeating: " ", count: indentLevel) + var result = "\(indent)- \(issue.description)\n" if let location = issue.sourceLocation { - result += " at \(location)\n" + result += "\(indent) at \(location)\n" } return result }