Add an upper bound on the number of test cases we run in parallel. (#1390)

grynspan · web-flow · commit b0aef4867851 · 2025-11-11T12:43:47.000-05:00
This PR adds an upper bound, `NCORES * 2`, on the number of test cases we run in parallel. Depending on the exact nature of your tests, this can significantly reduce the maximum amount of dirty memory needed, but does not generally impact execution time. As an example, Swift Testing's own tests go from > 300MB max memory usage to around 60MB. You can configure a different upper bound by setting the `"SWT_EXPERIMENTAL_MAXIMUM_PARALLELIZATION_WIDTH"` environment variable (look, naming is hard okay?) ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated.
diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift
@@ -210,6 +210,9 @@ public struct __CommandLineArguments_v0: Sendable {
   /// The value of the `--parallel` or `--no-parallel` argument.
   public var parallel: Bool?
 
+  /// The maximum number of test tasks to run in parallel.
+  public var experimentalMaximumParallelizationWidth: Int?
+
   /// The value of the `--symbolicate-backtraces` argument.
   public var symbolicateBacktraces: String?
 
@@ -336,6 +339,7 @@ extension __CommandLineArguments_v0: Codable {
   enum CodingKeys: String, CodingKey {
     case listTests
     case parallel
+    case experimentalMaximumParallelizationWidth
     case symbolicateBacktraces
     case verbose
     case veryVerbose
@@ -485,6 +489,10 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum
   if args.contains("--no-parallel") {
     result.parallel = false
   }
+  if let maximumParallelizationWidth = args.argumentValue(forLabel: "--experimental-maximum-parallelization-width").flatMap(Int.init) {
+    // TODO: decide if we want to repurpose --num-workers for this use case?
+    result.experimentalMaximumParallelizationWidth = maximumParallelizationWidth
+  }
 
   // Whether or not to symbolicate backtraces in the event stream.
   if let symbolicateBacktraces = args.argumentValue(forLabel: "--symbolicate-backtraces") {
@@ -545,7 +553,22 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr
   var configuration = Configuration()
 
   // Parallelization (on by default)
-  configuration.isParallelizationEnabled = args.parallel ?? true
+  if let parallel = args.parallel, !parallel {
+    configuration.isParallelizationEnabled = parallel
+  } else {
+    var maximumParallelizationWidth = args.experimentalMaximumParallelizationWidth
+    if maximumParallelizationWidth == nil && Test.current == nil {
+      // Don't check the environment variable when a current test is set (which
+      // presumably means we're running our own unit tests).
+      maximumParallelizationWidth = Environment.variable(named: "SWT_EXPERIMENTAL_MAXIMUM_PARALLELIZATION_WIDTH").flatMap(Int.init)
+    }
+    if let maximumParallelizationWidth {
+      if maximumParallelizationWidth < 1 {
+        throw _EntryPointError.invalidArgument("--experimental-maximum-parallelization-width", value: String(describing: maximumParallelizationWidth))
+      }
+      configuration.maximumParallelizationWidth = maximumParallelizationWidth
+    }
+  }
 
   // Whether or not to symbolicate backtraces in the event stream.
   if let symbolicateBacktraces = args.symbolicateBacktraces {
diff --git a/Sources/Testing/CMakeLists.txt b/Sources/Testing/CMakeLists.txt
@@ -90,6 +90,7 @@ add_library(Testing
   Support/Graph.swift
   Support/JSON.swift
   Support/Locked.swift
+  Support/Serializer.swift
   Support/VersionNumber.swift
   Support/Versions.swift
   Discovery+Macro.swift
diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift
@@ -8,6 +8,8 @@
 // See https://swift.org/CONTRIBUTORS.txt for Swift project authors
 //
 
+private import _TestingInternals
+
 /// A type containing settings for preparing and running tests.
 @_spi(ForToolsIntegrationOnly)
 public struct Configuration: Sendable {
@@ -18,7 +20,33 @@ public struct Configuration: Sendable {
   // MARK: - Parallelization
 
   /// Whether or not to parallelize the execution of tests and test cases.
-  public var isParallelizationEnabled: Bool = true
+  ///
+  /// - Note: Setting the value of this property implicitly sets the value of
+  ///   the experimental ``maximumParallelizationWidth`` property.
+  public var isParallelizationEnabled: Bool {
+    get {
+      maximumParallelizationWidth > 1
+    }
+    set {
+      maximumParallelizationWidth = newValue ? defaultParallelizationWidth : 1
+    }
+  }
+
+  /// The maximum width of parallelization.
+  ///
+  /// The value of this property determines how many tests (or rather, test
+  /// cases) will run in parallel.
+  ///
+  /// @Comment {
+  ///   The default value of this property is equal to twice the number of CPU
+  ///   cores reported by the operating system, or `Int.max` if that value is
+  ///   not available.
+  /// }
+  ///
+  /// - Note: Setting the value of this property implicitly sets the value of
+  ///   the ``isParallelizationEnabled`` property.
+  @_spi(Experimental)
+  public var maximumParallelizationWidth: Int = defaultParallelizationWidth
 
   /// How to symbolicate backtraces captured during a test run.
   ///
diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift
@@ -66,6 +66,20 @@ extension Runner {
     .current ?? .init()
   }
 
+  /// Context to apply to a test run.
+  ///
+  /// Instances of this type are passed directly to the various functions in
+  /// this file and represent context for the run itself. As such, they are not
+  /// task-local nor are they meant to change as the test run progresses.
+  ///
+  /// This type is distinct from ``Configuration`` which _can_ change on a
+  /// per-test basis. If you find yourself wanting to modify a property of this
+  /// type at runtime, it may be better-suited for ``Configuration`` instead.
+  private struct _Context: Sendable {
+    /// A serializer used to reduce parallelism among test cases.
+    var testCaseSerializer: Serializer?
+  }
+
   /// Apply the custom scope for any test scope providers of the traits
   /// associated with a specified test by calling their
   /// ``TestScoping/provideScope(for:testCase:performing:)`` function.
@@ -179,6 +193,7 @@ extension Runner {
   ///
   /// - Parameters:
   ///   - stepGraph: The subgraph whose root value, a step, is to be run.
+  ///   - context: Context for the test run.
   ///
   /// - Throws: Whatever is thrown from the test body. Thrown errors are
   ///   normally reported as test failures.
@@ -193,7 +208,7 @@ extension Runner {
   /// ## See Also
   ///
   /// - ``Runner/run()``
-  private static func _runStep(atRootOf stepGraph: Graph<String, Plan.Step?>) async throws {
+  private static func _runStep(atRootOf stepGraph: Graph<String, Plan.Step?>, context: _Context) async throws {
     // Whether to send a `.testEnded` event at the end of running this step.
     // Some steps' actions may not require a final event to be sent — for
     // example, a skip event only sends `.testSkipped`.
@@ -250,18 +265,18 @@ extension Runner {
           try await _applyScopingTraits(for: step.test, testCase: nil) {
             // Run the test function at this step (if one is present.)
             if let testCases = step.test.testCases {
-              await _runTestCases(testCases, within: step)
+              await _runTestCases(testCases, within: step, context: context)
             }
 
             // Run the children of this test (i.e. the tests in this suite.)
-            try await _runChildren(of: stepGraph)
+            try await _runChildren(of: stepGraph, context: context)
           }
         }
       }
     } else {
       // There is no test at this node in the graph, so just skip down to the
       // child nodes.
-      try await _runChildren(of: stepGraph)
+      try await _runChildren(of: stepGraph, context: context)
     }
   }
 
@@ -286,10 +301,11 @@ extension Runner {
   /// - Parameters:
   ///   - stepGraph: The subgraph whose root value, a step, will be used to
   ///     find children to run.
+  ///   - context: Context for the test run.
   ///
   /// - Throws: Whatever is thrown from the test body. Thrown errors are
   ///   normally reported as test failures.
-  private static func _runChildren(of stepGraph: Graph<String, Plan.Step?>) async throws {
+  private static func _runChildren(of stepGraph: Graph<String, Plan.Step?>, context: _Context) async throws {
     let childGraphs = if _configuration.isParallelizationEnabled {
       // Explicitly shuffle the steps to help detect accidental dependencies
       // between tests due to their ordering.
@@ -331,7 +347,7 @@ extension Runner {
 
     // Run the child nodes.
     try await _forEach(in: childGraphs.lazy.map(\.value), namingTasksWith: taskNamer) { childGraph in
-      try await _runStep(atRootOf: childGraph)
+      try await _runStep(atRootOf: childGraph, context: context)
     }
   }
 
@@ -340,12 +356,15 @@ extension Runner {
   /// - Parameters:
   ///   - testCases: The test cases to be run.
   ///   - step: The runner plan step associated with this test case.
+  ///   - context: Context for the test run.
   ///
   /// If parallelization is supported and enabled, the generated test cases will
   /// be run in parallel using a task group.
-  private static func _runTestCases(_ testCases: some Sequence<Test.Case>, within step: Plan.Step) async {
+  private static func _runTestCases(_ testCases: some Sequence<Test.Case>, within step: Plan.Step, context: _Context) async {
+    let configuration = _configuration
+
     // Apply the configuration's test case filter.
-    let testCaseFilter = _configuration.testCaseFilter
+    let testCaseFilter = configuration.testCaseFilter
     let testCases = testCases.lazy.filter { testCase in
       testCaseFilter(testCase, step.test)
     }
@@ -359,7 +378,13 @@ extension Runner {
     }
 
     await _forEach(in: testCases.enumerated(), namingTasksWith: taskNamer) { _, testCase in
-      await _runTestCase(testCase, within: step)
+      if let testCaseSerializer = context.testCaseSerializer {
+        // Note that if .serialized is applied to an inner scope, we still use
+        // this serializer (if set) so that we don't overcommit.
+        await testCaseSerializer.run { await _runTestCase(testCase, within: step, context: context) }
+      } else {
+        await _runTestCase(testCase, within: step, context: context)
+      }
     }
   }
 
@@ -368,10 +393,11 @@ extension Runner {
   /// - Parameters:
   ///   - testCase: The test case to run.
   ///   - step: The runner plan step associated with this test case.
+  ///   - context: Context for the test run.
   ///
   /// This function sets ``Test/Case/current``, then invokes the test case's
   /// body closure.
-  private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step) async {
+  private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step, context: _Context) async {
     let configuration = _configuration
 
     Event.post(.testCaseStarted, for: (step.test, testCase), configuration: configuration)
@@ -431,6 +457,21 @@ extension Runner {
       eventHandler(event, context)
     }
 
+    // Context to pass into the test run. We intentionally don't pass the Runner
+    // itself (implicitly as `self` nor as an argument) because we don't want to
+    // accidentally depend on e.g. the `configuration` property rather than the
+    // current configuration.
+    let context: _Context = {
+      var context = _Context()
+
+      let maximumParallelizationWidth = runner.configuration.maximumParallelizationWidth
+      if maximumParallelizationWidth > 1 && maximumParallelizationWidth < .max {
+        context.testCaseSerializer = Serializer(maximumWidth: runner.configuration.maximumParallelizationWidth)
+      }
+
+      return context
+    }()
+
     await Configuration.withCurrent(runner.configuration) {
       // Post an event for every test in the test plan being run. These events
       // are turned into JSON objects if JSON output is enabled.
@@ -457,7 +498,7 @@ extension Runner {
             taskAction = "running iteration #\(iterationIndex + 1)"
           }
           _ = taskGroup.addTaskUnlessCancelled(name: decorateTaskName("test run", withAction: taskAction)) {
-            try? await _runStep(atRootOf: runner.plan.stepGraph)
+            try? await _runStep(atRootOf: runner.plan.stepGraph, context: context)
           }
           await taskGroup.waitForAll()
         }
diff --git a/Sources/Testing/Support/Serializer.swift b/Sources/Testing/Support/Serializer.swift
@@ -0,0 +1,93 @@
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2024–2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+//
+
+private import _TestingInternals
+
+/// The number of CPU cores on the current system, or `nil` if that
+/// information is not available.
+var cpuCoreCount: Int? {
+#if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android)
+  return Int(sysconf(Int32(_SC_NPROCESSORS_CONF)))
+#elseif os(Windows)
+  var siInfo = SYSTEM_INFO()
+  GetSystemInfo(&siInfo)
+  return Int(siInfo.dwNumberOfProcessors)
+#elseif os(WASI)
+  return 1
+#else
+#warning("Platform-specific implementation missing: CPU core count unavailable")
+  return nil
+#endif
+}
+
+/// The default parallelization width when parallelized testing is enabled.
+var defaultParallelizationWidth: Int {
+  // cpuCoreCount.map { max(1, $0) * 2 } ?? .max
+  .max
+}
+
+/// A type whose instances can run a series of work items in strict order.
+///
+/// When a work item is scheduled on an instance of this type, it runs after any
+/// previously-scheduled work items. If it suspends, subsequently-scheduled work
+/// items do not start running; they must wait until the suspended work item
+/// either returns or throws an error.
+///
+/// This type is not part of the public interface of the testing library.
+final actor Serializer {
+  /// The maximum number of work items that may run concurrently.
+  nonisolated let maximumWidth: Int
+
+  /// The number of scheduled work items, including any currently running.
+  private var _currentWidth = 0
+
+  /// Continuations for any scheduled work items that haven't started yet.
+  private var _continuations = [CheckedContinuation<Void, Never>]()
+
+  init(maximumWidth: Int = 1) {
+    precondition(maximumWidth >= 1, "Invalid serializer width \(maximumWidth).")
+    self.maximumWidth = maximumWidth
+  }
+
+  /// Run a work item serially after any previously-scheduled work items.
+  ///
+  /// - Parameters:
+  ///     - workItem: A closure to run.
+  ///
+  /// - Returns: Whatever is returned from `workItem`.
+  ///
+  /// - Throws: Whatever is thrown by `workItem`.
+  func run<R>(_ workItem: @isolated(any) @Sendable () async throws -> R) async rethrows -> R where R: Sendable {
+    _currentWidth += 1
+    defer {
+      // Resume the next scheduled closure.
+      if !_continuations.isEmpty {
+        let continuation = _continuations.removeFirst()
+        continuation.resume()
+      }
+
+      _currentWidth -= 1
+    }
+
+    await withCheckedContinuation { continuation in
+      if _currentWidth <= maximumWidth {
+        // Nothing else was scheduled, so we can resume immediately.
+        continuation.resume()
+      } else {
+        // Something was scheduled, so add the continuation to the
+        // list. When it resumes, we can run.
+        _continuations.append(continuation)
+      }
+    }
+
+    return try await workItem()
+  }
+}
+
diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift
@@ -59,6 +59,25 @@ struct SwiftPMTests {
     #expect(!configuration.isParallelizationEnabled)
   }
 
+  @Test("--experimental-maximum-parallelization-width argument")
+  func maximumParallelizationWidth() throws {
+    var configuration = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-maximum-parallelization-width", "12345"])
+    #expect(configuration.isParallelizationEnabled)
+    #expect(configuration.maximumParallelizationWidth == 12345)
+
+    configuration = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-maximum-parallelization-width", "1"])
+    #expect(!configuration.isParallelizationEnabled)
+    #expect(configuration.maximumParallelizationWidth == 1)
+
+    configuration = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-maximum-parallelization-width", "\(Int.max)"])
+    #expect(configuration.isParallelizationEnabled)
+    #expect(configuration.maximumParallelizationWidth == .max)
+
+    #expect(throws: (any Error).self) {
+      _ = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-maximum-parallelization-width", "0"])
+    }
+  }
+
   @Test("--symbolicate-backtraces argument",
     arguments: [
       (String?.none, Backtrace.SymbolicationMode?.none),
