diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index 80e68c609..513b861c9 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -306,6 +306,14 @@ extension Event.ConsoleOutputRecorder { /// destination. @discardableResult public func record(_ event: borrowing Event, in context: borrowing Event.Context) -> Bool { let messages = _humanReadableOutputRecorder.record(event, in: context) + + // Print failure summary when run ends + if case .runEnded = event.kind { + if let summary = _humanReadableOutputRecorder.generateFailureSummary(options: options) { + // Add blank line before summary and after summary for visual separation + write("\n\(summary)\n") + } + } // Padding to use in place of a symbol for messages that don't have one. var padding = " " diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 9f0ac21b1..a1e05a17f 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -9,6 +9,211 @@ // extension Event { + /// A type that generates a failure summary from test run data. + /// + /// This type encapsulates the logic for collecting failed tests from a test + /// data graph and formatting them into a human-readable failure summary. + @_spi(ForToolsIntegrationOnly) + public struct TestRunSummary: Sendable { + /// Information about a single failed test. + struct FailedTest: Sendable { + /// The full hierarchical path to the test (e.g., suite names). + var path: [String] + + /// The test's simple name (last component of the path). + var name: String + + /// All issues recorded for this test. + var issues: [IssueInfo] + + /// The test's display name, if any. + var displayName: String? + + /// The test case arguments for parameterized tests, if any. + var testCaseArguments: String? + } + + /// Information about a single issue within a failed test. + struct IssueInfo: Sendable { + /// The source location where the issue occurred. + var sourceLocation: SourceLocation? + + /// A detailed description of what failed. + var description: String + + /// Whether this issue is a known issue. + var isKnown: Bool + + /// The severity of this issue. + var severity: Issue.Severity + } + + /// The list of failed tests collected from the test run. + private let failedTests: [FailedTest] + + /// Initialize a test run summary by collecting failures from a test data graph. + /// + /// - Parameters: + /// - testData: The root test data graph to traverse. + fileprivate init(from testData: Graph) { + var collected: [FailedTest] = [] + + // Traverse the graph to find all tests with failures + func traverse(graph: Graph, path: [String]) { + // Check if this node has test data with failures + if let testData = graph.value, !testData.issues.isEmpty { + let testName = path.last ?? "Unknown" + + // Convert Context.TestData.IssueInfo to TestRunSummary.IssueInfo + let issues = testData.issues.map { issue in + IssueInfo( + sourceLocation: issue.sourceLocation, + description: issue.description, + isKnown: issue.isKnown, + severity: issue.severity + ) + } + + collected.append(FailedTest( + path: path, + name: testName, + issues: issues, + displayName: testData.displayName, + testCaseArguments: testData.testCaseArguments + )) + } + + // Recursively traverse children + for (key, childGraph) in graph.children { + let pathComponent: String? + switch key { + case let .string(s): + let parts = s.split(separator: ":") + if s.hasSuffix(".swift:") || (parts.count >= 2 && parts[0].hasSuffix(".swift")) { + pathComponent = nil // Filter out source location strings + } else { + pathComponent = s + } + case let .testCaseID(id): + // Only include parameterized test case IDs in path + if let argumentIDs = id.argumentIDs, let discriminator = id.discriminator { + pathComponent = "arguments: \(argumentIDs), discriminator: \(discriminator)" + } else { + pathComponent = nil // Filter out non-parameterized test case IDs + } + } + + let newPath = pathComponent.map { path + [$0] } ?? path + traverse(graph: childGraph, path: newPath) + } + } + + // Start traversal from root + traverse(graph: testData, path: []) + + self.failedTests = collected + } + + /// Generate a formatted failure summary string. + /// + /// - Parameters: + /// - options: Options for formatting (e.g., for ANSI colors and symbols). + /// + /// - Returns: A formatted string containing the failure summary, or `nil` + /// if there were no failures. + public func formatted(with options: Event.ConsoleOutputRecorder.Options) -> String? { + // If no failures, return nil + guard !failedTests.isEmpty else { + return nil + } + + var summary = "" + + // Add header with failure count + summary += header() + + // Get the failure symbol + let failSymbol = Event.Symbol.fail.stringValue(options: options) + + // Format each failed test + for failedTest in failedTests { + summary += formatFailedTest(failedTest, withSymbol: failSymbol) + } + + return summary + } + + /// Generate the summary header with failure counts. + /// + /// - Returns: A string containing the header line. + private func header() -> String { + let testWord = failedTests.count == 1 ? "test" : "tests" + let totalIssues = failedTests.reduce(0) { $0 + $1.issues.count } + let issueWord = totalIssues == 1 ? "issue" : "issues" + return "Test run had \(failedTests.count) failed \(testWord) with \(totalIssues) \(issueWord):\n" + } + + /// Format a single failed test entry. + /// + /// - Parameters: + /// - failedTest: The failed test to format. + /// - symbol: The failure symbol string to use. + /// + /// - Returns: A formatted string representing the failed test and its issues. + private func formatFailedTest(_ failedTest: FailedTest, withSymbol symbol: String) -> String { + var result = "" + + // Build fully qualified name + var fullyQualifiedName = fullyQualifiedName(for: failedTest) + + result += "\(symbol) \(fullyQualifiedName)\n" + + // Show test case arguments for parameterized tests (once per test) + if let arguments = failedTest.testCaseArguments, !arguments.isEmpty { + result += " (\(arguments))\n" + } + + // List each issue for this test with indentation + for issue in failedTest.issues { + result += formatIssue(issue) + } + + return result + } + + /// Build the fully qualified name for a failed test. + /// + /// - Parameters: + /// - failedTest: The failed test. + /// + /// - Returns: The fully qualified name, with display name substituted if available. + private func fullyQualifiedName(for failedTest: FailedTest) -> String { + var name = failedTest.path.joined(separator: "/") + + // Use display name for the last component if available + if let displayName = failedTest.displayName, !failedTest.path.isEmpty { + let pathWithoutLast = failedTest.path.dropLast() + name = (pathWithoutLast + [#""\#(displayName)""#]).joined(separator: "/") + } + + return name + } + + /// Format a single issue entry. + /// + /// - Parameters: + /// - issue: The issue to format. + /// + /// - Returns: A formatted string representing the issue with indentation. + private func formatIssue(_ issue: IssueInfo) -> String { + var result = " - \(issue.description)\n" + if let location = issue.sourceLocation { + result += " at \(location.fileID):\(location.line)\n" + } + return result + } + } + /// A type which handles ``Event`` instances and outputs representations of /// them as human-readable messages. /// @@ -64,6 +269,21 @@ extension Event { /// A type describing data tracked on a per-test basis. struct TestData { + /// A lightweight struct containing information about a single issue. + struct IssueInfo: Sendable { + /// The source location where the issue occurred. + var sourceLocation: SourceLocation? + + /// A detailed description of what failed (using expanded debug description). + var description: String + + /// Whether this issue is a known issue. + var isKnown: Bool + + /// The severity of this issue. + var severity: Issue.Severity + } + /// The instant at which the test started. var startInstant: Test.Clock.Instant @@ -76,6 +296,16 @@ extension Event { /// Information about the cancellation of this test or test case. var cancellationInfo: SkipInfo? + + /// Array of all issues recorded for this test (for failure summary). + /// Each issue is stored individually with its own source location. + var issues: [IssueInfo] = [] + + /// The test's display name, if any. + var displayName: String? + + /// The test case arguments, formatted for display (for parameterized tests). + var testCaseArguments: String? } /// Data tracked on a per-test basis. @@ -317,6 +547,41 @@ extension Event.HumanReadableOutputRecorder { let issueCount = testData.issueCount[issue.severity] ?? 0 testData.issueCount[issue.severity] = issueCount + 1 } + + // Store individual issue information for failure summary (only for errors) + if issue.severity == .error { + // Extract detailed failure message + let description: String + if case let .expectationFailed(expectation) = issue.kind { + // Use expandedDebugDescription only when verbose, otherwise use expandedDescription + description = if verbosity > 0 { + expectation.evaluatedExpression.expandedDebugDescription() + } else { + expectation.evaluatedExpression.expandedDescription() + } + } else if let comment = issue.comments.first { + description = comment.rawValue + } else { + description = "Test failed" + } + + let issueInfo = Context.TestData.IssueInfo( + sourceLocation: issue.sourceLocation, + description: description, + isKnown: issue.isKnown, + severity: issue.severity + ) + testData.issues.append(issueInfo) + + // Capture test display name and test case arguments once per test (not per issue) + if testData.displayName == nil { + testData.displayName = test?.displayName + } + if testData.testCaseArguments == nil { + testData.testCaseArguments = testCase?.labeledArguments() + } + } + context.testData[keyPath] = testData case .testCaseStarted: @@ -629,6 +894,22 @@ extension Event.HumanReadableOutputRecorder { return [] } + + /// Generate a failure summary string with all failed tests and their issues. + /// + /// This method creates a ``TestRunSummary`` from the test data graph and + /// formats it for display. + /// + /// - Parameters: + /// - options: Options for formatting (e.g., for ANSI colors and symbols). + /// + /// - Returns: A formatted string containing the failure summary, or `nil` + /// if there were no failures. + public func generateFailureSummary(options: Event.ConsoleOutputRecorder.Options) -> String? { + let context = _context.rawValue + let summary = Event.TestRunSummary(from: context.testData) + return summary.formatted(with: options) + } } extension Test.ID {