From 73f87aeada347d6f6114d09f3e71e8aed571f90f Mon Sep 17 00:00:00 2001 From: tienquocbui Date: Thu, 13 Nov 2025 23:07:41 +0100 Subject: [PATCH 1/2] Add failure summary to ConsoleOutputRecorder. Implemented end-of-run failure summary using expandedDebugDescription for detailed output, with support for custom display names and parameterized tests. --- .../Event.ConsoleOutputRecorder.swift | 9 + .../Event.HumanReadableOutputRecorder.swift | 170 ++++++++++++++++++ 2 files changed, 179 insertions(+) diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index 80e68c609..3bf6dcbc5 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -306,6 +306,15 @@ extension Event.ConsoleOutputRecorder { /// destination. @discardableResult public func record(_ event: borrowing Event, in context: borrowing Event.Context) -> Bool { let messages = _humanReadableOutputRecorder.record(event, in: context) + + // Print failure summary when run ends + if case .runEnded = event.kind { + let summary = _humanReadableOutputRecorder.generateFailureSummary(options: options) + if !summary.isEmpty { + // Add blank line before summary and after summary for visual separation + write("\n\(summary)\n") + } + } // Padding to use in place of a symbol for messages that don't have one. var padding = " " diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 9f0ac21b1..93af8c9d8 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -64,6 +64,18 @@ extension Event { /// A type describing data tracked on a per-test basis. struct TestData { + /// A lightweight struct containing information about a single issue. + struct IssueInfo: Sendable { + /// The source location where the issue occurred. + var sourceLocation: SourceLocation? + + /// A detailed description of what failed (using expanded debug description). + var description: String + + /// Whether this issue is a known issue. + var isKnown: Bool + } + /// The instant at which the test started. var startInstant: Test.Clock.Instant @@ -76,6 +88,16 @@ extension Event { /// Information about the cancellation of this test or test case. var cancellationInfo: SkipInfo? + + /// Array of all issues recorded for this test (for failure summary). + /// Each issue is stored individually with its own source location. + var issues: [IssueInfo] = [] + + /// The test's display name, if any. + var testDisplayName: String? + + /// The test case arguments, formatted for display (for parameterized tests). + var testCaseArguments: String? } /// Data tracked on a per-test basis. @@ -317,6 +339,36 @@ extension Event.HumanReadableOutputRecorder { let issueCount = testData.issueCount[issue.severity] ?? 0 testData.issueCount[issue.severity] = issueCount + 1 } + + // Store individual issue information for failure summary (only for errors) + if issue.severity == .error { + // Extract detailed failure message using expandedDebugDescription + let description: String + if case let .expectationFailed(expectation) = issue.kind { + // Use expandedDebugDescription for full detail (variable values expanded) + description = expectation.evaluatedExpression.expandedDebugDescription() + } else if let comment = issue.comments.first { + description = comment.rawValue + } else { + description = "Test failed" + } + + let issueInfo = Context.TestData.IssueInfo( + sourceLocation: issue.sourceLocation, + description: description, + isKnown: issue.isKnown + ) + testData.issues.append(issueInfo) + + // Capture test display name and test case arguments once per test (not per issue) + if testData.testDisplayName == nil { + testData.testDisplayName = test?.displayName + } + if testData.testCaseArguments == nil { + testData.testCaseArguments = testCase?.labeledArguments() + } + } + context.testData[keyPath] = testData case .testCaseStarted: @@ -629,6 +681,124 @@ extension Event.HumanReadableOutputRecorder { return [] } + + /// Generate a failure summary string with all failed tests and their issues. + /// + /// This method traverses the test graph and formats a summary of all failures + /// that occurred during the test run. It includes the fully qualified test name + /// (with suite path), individual issues with their source locations, and uses + /// indentation to clearly delineate issue boundaries. + /// + /// - Parameters: + /// - options: Options for formatting (e.g., for ANSI colors and symbols). + /// + /// - Returns: A formatted string containing the failure summary, or an empty + /// string if there were no failures. + public func generateFailureSummary(options: Event.ConsoleOutputRecorder.Options) -> String { + let context = _context.rawValue + + // Collect all failed tests (tests with error issues) + struct FailedTestInfo { + var testPath: [String] // Full path including suite names + var testName: String + var issues: [Context.TestData.IssueInfo] + var testDisplayName: String? + var testCaseArguments: String? + } + + var failedTests: [FailedTestInfo] = [] + + // Traverse the graph to find all tests with failures + func traverse(graph: Graph, path: [String]) { + // Check if this node has test data with failures + if let testData = graph.value, !testData.issues.isEmpty { + let testName = path.last ?? "Unknown" + + failedTests.append(FailedTestInfo( + testPath: path, + testName: testName, + issues: testData.issues, + testDisplayName: testData.testDisplayName, + testCaseArguments: testData.testCaseArguments + )) + } + + // Recursively traverse children + for (key, childGraph) in graph.children { + let pathComponent: String? + switch key { + case let .string(s): + let parts = s.split(separator: ":") + if s.hasSuffix(".swift:") || (parts.count >= 2 && parts[0].hasSuffix(".swift")) { + pathComponent = nil + } else { + pathComponent = s + } + case let .testCaseID(id): + // Only include parameterized test case IDs in path, skip non-parameterized ones + if let argumentIDs = id.argumentIDs, let discriminator = id.discriminator { + pathComponent = "arguments: \(argumentIDs), discriminator: \(discriminator)" + } else { + // Non-parameterized test - don't add to path + pathComponent = nil + } + } + + let newPath = pathComponent.map { path + [$0] } ?? path + traverse(graph: childGraph, path: newPath) + } + } + + // Start traversal from root + traverse(graph: context.testData, path: []) + + // If no failures, return empty string + guard !failedTests.isEmpty else { + return "" + } + + var summary = "" + + // Header with failure count + let testWord = failedTests.count == 1 ? "test" : "tests" + let totalIssues = failedTests.reduce(0) { $0 + $1.issues.count } + let issueWord = totalIssues == 1 ? "issue" : "issues" + summary += "Test run had \(failedTests.count) failed \(testWord) with \(totalIssues) \(issueWord):\n" + + // Get the failure symbol + let failSymbol = Event.Symbol.fail.stringValue(options: options) + + // List each failed test + for failedTest in failedTests { + // Build fully qualified name with suite path + var fullyQualifiedName = failedTest.testPath.joined(separator: "/") + + // Use display name for the last component if available + if let displayName = failedTest.testDisplayName, + !failedTest.testPath.isEmpty { + // Replace the last component (test name) with display name + let pathWithoutLast = failedTest.testPath.dropLast() + fullyQualifiedName = (pathWithoutLast + [#""\#(displayName)""#]).joined(separator: "/") + } + + summary += "\(failSymbol) \(fullyQualifiedName)\n" + + // Show test case arguments for parameterized tests (once per test, not per issue) + if let arguments = failedTest.testCaseArguments, !arguments.isEmpty { + summary += " (\(arguments))\n" + } + + // List each issue for this test with indentation + for issue in failedTest.issues { + summary += " - \(issue.description)\n" + if let location = issue.sourceLocation { + summary += " at \(location.fileID):\(location.line)\n" + } + } + } + + return summary + } } extension Test.ID { From 29c448f78b5d1a724ee3e3c44af840b9083bcbfe Mon Sep 17 00:00:00 2001 From: tienquocbui Date: Tue, 18 Nov 2025 15:47:31 +0100 Subject: [PATCH 2/2] Add severity tracking to IssueInfo struct and factor failure summary logic into TestRunSummary type --- .../Event.ConsoleOutputRecorder.swift | 3 +- .../Event.HumanReadableOutputRecorder.swift | 343 ++++++++++++------ 2 files changed, 228 insertions(+), 118 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index 3bf6dcbc5..513b861c9 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -309,8 +309,7 @@ extension Event.ConsoleOutputRecorder { // Print failure summary when run ends if case .runEnded = event.kind { - let summary = _humanReadableOutputRecorder.generateFailureSummary(options: options) - if !summary.isEmpty { + if let summary = _humanReadableOutputRecorder.generateFailureSummary(options: options) { // Add blank line before summary and after summary for visual separation write("\n\(summary)\n") } diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 93af8c9d8..a1e05a17f 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -9,6 +9,211 @@ // extension Event { + /// A type that generates a failure summary from test run data. + /// + /// This type encapsulates the logic for collecting failed tests from a test + /// data graph and formatting them into a human-readable failure summary. + @_spi(ForToolsIntegrationOnly) + public struct TestRunSummary: Sendable { + /// Information about a single failed test. + struct FailedTest: Sendable { + /// The full hierarchical path to the test (e.g., suite names). + var path: [String] + + /// The test's simple name (last component of the path). + var name: String + + /// All issues recorded for this test. + var issues: [IssueInfo] + + /// The test's display name, if any. + var displayName: String? + + /// The test case arguments for parameterized tests, if any. + var testCaseArguments: String? + } + + /// Information about a single issue within a failed test. + struct IssueInfo: Sendable { + /// The source location where the issue occurred. + var sourceLocation: SourceLocation? + + /// A detailed description of what failed. + var description: String + + /// Whether this issue is a known issue. + var isKnown: Bool + + /// The severity of this issue. + var severity: Issue.Severity + } + + /// The list of failed tests collected from the test run. + private let failedTests: [FailedTest] + + /// Initialize a test run summary by collecting failures from a test data graph. + /// + /// - Parameters: + /// - testData: The root test data graph to traverse. + fileprivate init(from testData: Graph) { + var collected: [FailedTest] = [] + + // Traverse the graph to find all tests with failures + func traverse(graph: Graph, path: [String]) { + // Check if this node has test data with failures + if let testData = graph.value, !testData.issues.isEmpty { + let testName = path.last ?? "Unknown" + + // Convert Context.TestData.IssueInfo to TestRunSummary.IssueInfo + let issues = testData.issues.map { issue in + IssueInfo( + sourceLocation: issue.sourceLocation, + description: issue.description, + isKnown: issue.isKnown, + severity: issue.severity + ) + } + + collected.append(FailedTest( + path: path, + name: testName, + issues: issues, + displayName: testData.displayName, + testCaseArguments: testData.testCaseArguments + )) + } + + // Recursively traverse children + for (key, childGraph) in graph.children { + let pathComponent: String? + switch key { + case let .string(s): + let parts = s.split(separator: ":") + if s.hasSuffix(".swift:") || (parts.count >= 2 && parts[0].hasSuffix(".swift")) { + pathComponent = nil // Filter out source location strings + } else { + pathComponent = s + } + case let .testCaseID(id): + // Only include parameterized test case IDs in path + if let argumentIDs = id.argumentIDs, let discriminator = id.discriminator { + pathComponent = "arguments: \(argumentIDs), discriminator: \(discriminator)" + } else { + pathComponent = nil // Filter out non-parameterized test case IDs + } + } + + let newPath = pathComponent.map { path + [$0] } ?? path + traverse(graph: childGraph, path: newPath) + } + } + + // Start traversal from root + traverse(graph: testData, path: []) + + self.failedTests = collected + } + + /// Generate a formatted failure summary string. + /// + /// - Parameters: + /// - options: Options for formatting (e.g., for ANSI colors and symbols). + /// + /// - Returns: A formatted string containing the failure summary, or `nil` + /// if there were no failures. + public func formatted(with options: Event.ConsoleOutputRecorder.Options) -> String? { + // If no failures, return nil + guard !failedTests.isEmpty else { + return nil + } + + var summary = "" + + // Add header with failure count + summary += header() + + // Get the failure symbol + let failSymbol = Event.Symbol.fail.stringValue(options: options) + + // Format each failed test + for failedTest in failedTests { + summary += formatFailedTest(failedTest, withSymbol: failSymbol) + } + + return summary + } + + /// Generate the summary header with failure counts. + /// + /// - Returns: A string containing the header line. + private func header() -> String { + let testWord = failedTests.count == 1 ? "test" : "tests" + let totalIssues = failedTests.reduce(0) { $0 + $1.issues.count } + let issueWord = totalIssues == 1 ? "issue" : "issues" + return "Test run had \(failedTests.count) failed \(testWord) with \(totalIssues) \(issueWord):\n" + } + + /// Format a single failed test entry. + /// + /// - Parameters: + /// - failedTest: The failed test to format. + /// - symbol: The failure symbol string to use. + /// + /// - Returns: A formatted string representing the failed test and its issues. + private func formatFailedTest(_ failedTest: FailedTest, withSymbol symbol: String) -> String { + var result = "" + + // Build fully qualified name + var fullyQualifiedName = fullyQualifiedName(for: failedTest) + + result += "\(symbol) \(fullyQualifiedName)\n" + + // Show test case arguments for parameterized tests (once per test) + if let arguments = failedTest.testCaseArguments, !arguments.isEmpty { + result += " (\(arguments))\n" + } + + // List each issue for this test with indentation + for issue in failedTest.issues { + result += formatIssue(issue) + } + + return result + } + + /// Build the fully qualified name for a failed test. + /// + /// - Parameters: + /// - failedTest: The failed test. + /// + /// - Returns: The fully qualified name, with display name substituted if available. + private func fullyQualifiedName(for failedTest: FailedTest) -> String { + var name = failedTest.path.joined(separator: "/") + + // Use display name for the last component if available + if let displayName = failedTest.displayName, !failedTest.path.isEmpty { + let pathWithoutLast = failedTest.path.dropLast() + name = (pathWithoutLast + [#""\#(displayName)""#]).joined(separator: "/") + } + + return name + } + + /// Format a single issue entry. + /// + /// - Parameters: + /// - issue: The issue to format. + /// + /// - Returns: A formatted string representing the issue with indentation. + private func formatIssue(_ issue: IssueInfo) -> String { + var result = " - \(issue.description)\n" + if let location = issue.sourceLocation { + result += " at \(location.fileID):\(location.line)\n" + } + return result + } + } + /// A type which handles ``Event`` instances and outputs representations of /// them as human-readable messages. /// @@ -74,6 +279,9 @@ extension Event { /// Whether this issue is a known issue. var isKnown: Bool + + /// The severity of this issue. + var severity: Issue.Severity } /// The instant at which the test started. @@ -94,7 +302,7 @@ extension Event { var issues: [IssueInfo] = [] /// The test's display name, if any. - var testDisplayName: String? + var displayName: String? /// The test case arguments, formatted for display (for parameterized tests). var testCaseArguments: String? @@ -342,11 +550,15 @@ extension Event.HumanReadableOutputRecorder { // Store individual issue information for failure summary (only for errors) if issue.severity == .error { - // Extract detailed failure message using expandedDebugDescription + // Extract detailed failure message let description: String if case let .expectationFailed(expectation) = issue.kind { - // Use expandedDebugDescription for full detail (variable values expanded) - description = expectation.evaluatedExpression.expandedDebugDescription() + // Use expandedDebugDescription only when verbose, otherwise use expandedDescription + description = if verbosity > 0 { + expectation.evaluatedExpression.expandedDebugDescription() + } else { + expectation.evaluatedExpression.expandedDescription() + } } else if let comment = issue.comments.first { description = comment.rawValue } else { @@ -356,13 +568,14 @@ extension Event.HumanReadableOutputRecorder { let issueInfo = Context.TestData.IssueInfo( sourceLocation: issue.sourceLocation, description: description, - isKnown: issue.isKnown + isKnown: issue.isKnown, + severity: issue.severity ) testData.issues.append(issueInfo) // Capture test display name and test case arguments once per test (not per issue) - if testData.testDisplayName == nil { - testData.testDisplayName = test?.displayName + if testData.displayName == nil { + testData.displayName = test?.displayName } if testData.testCaseArguments == nil { testData.testCaseArguments = testCase?.labeledArguments() @@ -684,120 +897,18 @@ extension Event.HumanReadableOutputRecorder { /// Generate a failure summary string with all failed tests and their issues. /// - /// This method traverses the test graph and formats a summary of all failures - /// that occurred during the test run. It includes the fully qualified test name - /// (with suite path), individual issues with their source locations, and uses - /// indentation to clearly delineate issue boundaries. + /// This method creates a ``TestRunSummary`` from the test data graph and + /// formats it for display. /// /// - Parameters: /// - options: Options for formatting (e.g., for ANSI colors and symbols). /// - /// - Returns: A formatted string containing the failure summary, or an empty - /// string if there were no failures. - public func generateFailureSummary(options: Event.ConsoleOutputRecorder.Options) -> String { + /// - Returns: A formatted string containing the failure summary, or `nil` + /// if there were no failures. + public func generateFailureSummary(options: Event.ConsoleOutputRecorder.Options) -> String? { let context = _context.rawValue - - // Collect all failed tests (tests with error issues) - struct FailedTestInfo { - var testPath: [String] // Full path including suite names - var testName: String - var issues: [Context.TestData.IssueInfo] - var testDisplayName: String? - var testCaseArguments: String? - } - - var failedTests: [FailedTestInfo] = [] - - // Traverse the graph to find all tests with failures - func traverse(graph: Graph, path: [String]) { - // Check if this node has test data with failures - if let testData = graph.value, !testData.issues.isEmpty { - let testName = path.last ?? "Unknown" - - failedTests.append(FailedTestInfo( - testPath: path, - testName: testName, - issues: testData.issues, - testDisplayName: testData.testDisplayName, - testCaseArguments: testData.testCaseArguments - )) - } - - // Recursively traverse children - for (key, childGraph) in graph.children { - let pathComponent: String? - switch key { - case let .string(s): - let parts = s.split(separator: ":") - if s.hasSuffix(".swift:") || (parts.count >= 2 && parts[0].hasSuffix(".swift")) { - pathComponent = nil - } else { - pathComponent = s - } - case let .testCaseID(id): - // Only include parameterized test case IDs in path, skip non-parameterized ones - if let argumentIDs = id.argumentIDs, let discriminator = id.discriminator { - pathComponent = "arguments: \(argumentIDs), discriminator: \(discriminator)" - } else { - // Non-parameterized test - don't add to path - pathComponent = nil - } - } - - let newPath = pathComponent.map { path + [$0] } ?? path - traverse(graph: childGraph, path: newPath) - } - } - - // Start traversal from root - traverse(graph: context.testData, path: []) - - // If no failures, return empty string - guard !failedTests.isEmpty else { - return "" - } - - var summary = "" - - // Header with failure count - let testWord = failedTests.count == 1 ? "test" : "tests" - let totalIssues = failedTests.reduce(0) { $0 + $1.issues.count } - let issueWord = totalIssues == 1 ? "issue" : "issues" - summary += "Test run had \(failedTests.count) failed \(testWord) with \(totalIssues) \(issueWord):\n" - - // Get the failure symbol - let failSymbol = Event.Symbol.fail.stringValue(options: options) - - // List each failed test - for failedTest in failedTests { - // Build fully qualified name with suite path - var fullyQualifiedName = failedTest.testPath.joined(separator: "/") - - // Use display name for the last component if available - if let displayName = failedTest.testDisplayName, - !failedTest.testPath.isEmpty { - // Replace the last component (test name) with display name - let pathWithoutLast = failedTest.testPath.dropLast() - fullyQualifiedName = (pathWithoutLast + [#""\#(displayName)""#]).joined(separator: "/") - } - - summary += "\(failSymbol) \(fullyQualifiedName)\n" - - // Show test case arguments for parameterized tests (once per test, not per issue) - if let arguments = failedTest.testCaseArguments, !arguments.isEmpty { - summary += " (\(arguments))\n" - } - - // List each issue for this test with indentation - for issue in failedTest.issues { - summary += " - \(issue.description)\n" - if let location = issue.sourceLocation { - summary += " at \(location.fileID):\(location.line)\n" - } - } - } - - return summary + let summary = Event.TestRunSummary(from: context.testData) + return summary.formatted(with: options) } }