diff --git a/CMakeLists.txt b/CMakeLists.txt index ed60c850..474ba8c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -81,6 +81,7 @@ add_compile_definitions(USE_STATIC_PLUGIN_INITIALIZATION) find_package(ArgumentParser) find_package(LLBuild) find_package(SwiftDriver) +find_package(SwiftSubprocess) find_package(SwiftSystem) find_package(TSC) # NOTE: these two are required for LLBuild dependencies diff --git a/Package.swift b/Package.swift index 9e498086..d2b39769 100644 --- a/Package.swift +++ b/Package.swift @@ -206,6 +206,7 @@ let package = Package( "SWBCSupport", "SWBLibc", .product(name: "ArgumentParser", package: "swift-argument-parser"), + .product(name: "Subprocess", package: "swift-subprocess", condition: .when(platforms: [.android, .custom("freebsd"), .linux, .macOS, .openbsd, .windows])), .product(name: "SystemPackage", package: "swift-system", condition: .when(platforms: [.linux, .openbsd, .android, .windows, .custom("freebsd")])), ], exclude: ["CMakeLists.txt"], @@ -458,6 +459,7 @@ if isStaticBuild { if useLocalDependencies { package.dependencies += [ .package(path: "../swift-driver"), + .package(path: "../swift-subprocess"), .package(path: "../swift-system"), .package(path: "../swift-argument-parser"), ] @@ -467,6 +469,7 @@ if useLocalDependencies { } else { package.dependencies += [ .package(url: "https://github.com/swiftlang/swift-driver.git", branch: "main"), + .package(url: "https://github.com/swiftlang/swift-subprocess.git", .upToNextMinor(from: "0.1.0")), .package(url: "https://github.com/apple/swift-system.git", .upToNextMajor(from: "1.5.0")), .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.0.3"), ] diff --git a/Plugins/cmake-smoke-test/cmake-smoke-test.swift b/Plugins/cmake-smoke-test/cmake-smoke-test.swift index fcba09f3..a35bb67a 100644 --- a/Plugins/cmake-smoke-test/cmake-smoke-test.swift +++ b/Plugins/cmake-smoke-test/cmake-smoke-test.swift @@ -46,6 +46,9 @@ struct CMakeSmokeTest: CommandPlugin { let swiftToolsSupportCoreURL = try findDependency("swift-tools-support-core", pluginContext: context) let swiftToolsSupportCoreBuildURL = context.pluginWorkDirectoryURL.appending(component: "swift-tools-support-core") + let swiftSubprocessURL = try findDependency("swift-subprocess", pluginContext: context) + let swiftSubprocessBuildURL = context.pluginWorkDirectoryURL.appending(component: "swift-subprocess") + let swiftSystemURL = try findDependency("swift-system", pluginContext: context) let swiftSystemBuildURL = context.pluginWorkDirectoryURL.appending(component: "swift-system") @@ -58,7 +61,7 @@ struct CMakeSmokeTest: CommandPlugin { let swiftDriverURL = try findDependency("swift-driver", pluginContext: context) let swiftDriverBuildURL = context.pluginWorkDirectoryURL.appending(component: "swift-driver") - for url in [swiftToolsSupportCoreBuildURL, swiftSystemBuildURL, llbuildBuildURL, swiftArgumentParserBuildURL, swiftDriverBuildURL, swiftBuildBuildURL] { + for url in [swiftToolsSupportCoreBuildURL, swiftSubprocessBuildURL, swiftSystemBuildURL, llbuildBuildURL, swiftArgumentParserBuildURL, swiftDriverBuildURL, swiftBuildBuildURL] { try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) } @@ -75,6 +78,7 @@ struct CMakeSmokeTest: CommandPlugin { "-DLLBuild_DIR=\(llbuildBuildURL.appending(components: "cmake", "modules").filePath)", "-DTSC_DIR=\(swiftToolsSupportCoreBuildURL.appending(components: "cmake", "modules").filePath)", "-DSwiftDriver_DIR=\(swiftDriverBuildURL.appending(components: "cmake", "modules").filePath)", + "-DSwiftSubprocess_DIR=\(swiftSubprocessBuildURL.appending(components: "cmake", "modules").filePath)", "-DSwiftSystem_DIR=\(swiftSystemBuildURL.appending(components: "cmake", "modules").filePath)" ] @@ -90,6 +94,11 @@ struct CMakeSmokeTest: CommandPlugin { try await Process.checkNonZeroExit(url: ninjaURL, arguments: [], workingDirectory: swiftToolsSupportCoreBuildURL) Diagnostics.progress("Built swift-tools-support-core") + Diagnostics.progress("Building swift-subprocess") + try await Process.checkNonZeroExit(url: cmakeURL, arguments: sharedCMakeArgs + [swiftSubprocessURL.filePath], workingDirectory: swiftSubprocessBuildURL) + try await Process.checkNonZeroExit(url: ninjaURL, arguments: [], workingDirectory: swiftSubprocessBuildURL) + Diagnostics.progress("Built swift-subprocess") + if hostOS != .macOS { Diagnostics.progress("Building swift-system") try await Process.checkNonZeroExit(url: cmakeURL, arguments: sharedCMakeArgs + [swiftSystemURL.filePath], workingDirectory: swiftSystemBuildURL) diff --git a/Sources/SWBCore/ProcessExecutionCache.swift b/Sources/SWBCore/ProcessExecutionCache.swift index 1030561c..20755e23 100644 --- a/Sources/SWBCore/ProcessExecutionCache.swift +++ b/Sources/SWBCore/ProcessExecutionCache.swift @@ -17,11 +17,6 @@ public final class ProcessExecutionCache: Sendable { private let workingDirectory: Path? public init(workingDirectory: Path? = .root) { - // FIXME: Work around lack of thread-safe working directory support in Foundation (Amazon Linux 2, OpenBSD). Executing processes in the current working directory is less deterministic, but all of the clients which use this class are generally not expected to be sensitive to the working directory anyways. This workaround can be removed once we drop support for Amazon Linux 2 and/or adopt swift-subprocess and/or Foundation.Process's working directory support is made thread safe. - if try! Process.hasUnsafeWorkingDirectorySupport { - self.workingDirectory = nil - return - } self.workingDirectory = workingDirectory } diff --git a/Sources/SWBTestSupport/Misc.swift b/Sources/SWBTestSupport/Misc.swift index 9e77ad92..eb0b4d41 100644 --- a/Sources/SWBTestSupport/Misc.swift +++ b/Sources/SWBTestSupport/Misc.swift @@ -75,7 +75,7 @@ package func runProcessWithDeveloperDirectory(_ args: [String], workingDirectory package func runHostProcess(_ args: [String], workingDirectory: Path? = nil, interruptible: Bool = true, redirectStderr: Bool = true) async throws -> String { switch try ProcessInfo.processInfo.hostOperatingSystem() { case .macOS: - return try await InstalledXcode.currentlySelected().xcrun(args, workingDirectory: workingDirectory, redirectStderr: redirectStderr) + return try await InstalledXcode.currentlySelected().xcrun(args, workingDirectory: workingDirectory, interruptible: interruptible, redirectStderr: redirectStderr) default: return try await runProcess(args, workingDirectory: workingDirectory, environment: .current, interruptible: interruptible, redirectStderr: redirectStderr) } diff --git a/Sources/SWBTestSupport/SkippedTestSupport.swift b/Sources/SWBTestSupport/SkippedTestSupport.swift index 8bab2de5..df3b9283 100644 --- a/Sources/SWBTestSupport/SkippedTestSupport.swift +++ b/Sources/SWBTestSupport/SkippedTestSupport.swift @@ -152,11 +152,6 @@ extension Trait where Self == Testing.ConditionTrait { }) } - /// Constructs a condition trait that causes a test to be disabled if the Foundation process spawning implementation is not thread-safe. - package static var requireThreadSafeWorkingDirectory: Self { - disabled(if: try Process.hasUnsafeWorkingDirectorySupport, "Foundation.Process working directory support is not thread-safe.") - } - /// Constructs a condition trait that causes a test to be disabled if the specified llbuild API version requirement is not met. package static func requireLLBuild(apiVersion version: Int32) -> Self { let llbuildVersion = llb_get_api_version() diff --git a/Sources/SWBTestSupport/Xcode.swift b/Sources/SWBTestSupport/Xcode.swift index 969fc923..bea30914 100644 --- a/Sources/SWBTestSupport/Xcode.swift +++ b/Sources/SWBTestSupport/Xcode.swift @@ -31,8 +31,8 @@ package struct InstalledXcode: Sendable { return try await Path(xcrun(["-f", tool] + toolchainArgs).trimmingCharacters(in: .whitespacesAndNewlines)) } - package func xcrun(_ args: [String], workingDirectory: Path? = nil, redirectStderr: Bool = true) async throws -> String { - return try await runProcessWithDeveloperDirectory(["/usr/bin/xcrun"] + args, workingDirectory: workingDirectory, overrideDeveloperDirectory: self.developerDirPath.str, redirectStderr: redirectStderr) + package func xcrun(_ args: [String], workingDirectory: Path? = nil, interruptible: Bool = true, redirectStderr: Bool = true) async throws -> String { + return try await runProcessWithDeveloperDirectory(["/usr/bin/xcrun"] + args, workingDirectory: workingDirectory, overrideDeveloperDirectory: self.developerDirPath.str, interruptible: interruptible, redirectStderr: redirectStderr) } package func productBuildVersion() throws -> ProductBuildVersion { diff --git a/Sources/SWBUtil/CMakeLists.txt b/Sources/SWBUtil/CMakeLists.txt index 681fc86e..908e4bfb 100644 --- a/Sources/SWBUtil/CMakeLists.txt +++ b/Sources/SWBUtil/CMakeLists.txt @@ -78,6 +78,7 @@ add_library(SWBUtil POSIX.swift Process+Async.swift Process.swift + ProcessController.swift ProcessInfo.swift Promise.swift PropertyList.swift @@ -114,6 +115,7 @@ target_link_libraries(SWBUtil PUBLIC SWBCSupport SWBLibc ArgumentParser + SwiftSubprocess::Subprocess $<$>:SwiftSystem::SystemPackage>) set_target_properties(SWBUtil PROPERTIES diff --git a/Sources/SWBUtil/Process+Async.swift b/Sources/SWBUtil/Process+Async.swift index 93cecc3e..90976001 100644 --- a/Sources/SWBUtil/Process+Async.swift +++ b/Sources/SWBUtil/Process+Async.swift @@ -90,7 +90,7 @@ extension Process { /// - note: This method sets the process's termination handler, if one is set. /// - throws: ``CancellationError`` if the task was cancelled. Applies only when `interruptible` is true. /// - throws: Rethrows the error from ``Process/run`` if the task could not be launched. - public func run(interruptible: Bool = true) async throws { + public func run(interruptible: Bool = true, onStarted: () -> () = { }) async throws { @Sendable func cancelIfRunning() { // Only send the termination signal if the process is already running. // We might have created the termination monitoring continuation at this @@ -115,6 +115,7 @@ extension Process { } try run() + onStarted() } catch { terminationHandler = nil diff --git a/Sources/SWBUtil/Process.swift b/Sources/SWBUtil/Process.swift index 7f119cc6..88bbf535 100644 --- a/Sources/SWBUtil/Process.swift +++ b/Sources/SWBUtil/Process.swift @@ -11,18 +11,21 @@ //===----------------------------------------------------------------------===// public import Foundation -import SWBLibc +public import SWBLibc +import Synchronization -#if os(Windows) -public typealias pid_t = Int32 +#if canImport(Subprocess) && (!canImport(Darwin) || os(macOS)) +import Subprocess #endif -#if !canImport(Darwin) -extension ProcessInfo { - public var isMacCatalystApp: Bool { - false - } -} +#if canImport(System) +public import System +#else +public import SystemPackage +#endif + +#if os(Windows) +public typealias pid_t = Int32 #endif #if (!canImport(Foundation.NSTask) || targetEnvironment(macCatalyst)) && canImport(Darwin) @@ -64,7 +67,7 @@ public typealias Process = Foundation.Process #endif extension Process { - public static var hasUnsafeWorkingDirectorySupport: Bool { + fileprivate static var hasUnsafeWorkingDirectorySupport: Bool { get throws { switch try ProcessInfo.processInfo.hostOperatingSystem() { case .linux: @@ -81,6 +84,30 @@ extension Process { extension Process { public static func getOutput(url: URL, arguments: [String], currentDirectoryURL: URL? = nil, environment: Environment? = nil, interruptible: Bool = true) async throws -> Processes.ExecutionResult { + #if canImport(Subprocess) + #if !canImport(Darwin) || os(macOS) + var platformOptions = PlatformOptions() + if interruptible { + platformOptions.teardownSequence = [.gracefulShutDown(allowedDurationToNextStep: .seconds(5))] + } + let configuration = try Subprocess.Configuration( + .path(FilePath(url.filePath.str)), + arguments: .init(arguments), + environment: environment.map { .custom(.init($0)) } ?? .inherit, + workingDirectory: (currentDirectoryURL?.filePath.str).map { FilePath($0) } ?? nil, + platformOptions: platformOptions + ) + let result = try await Subprocess.run(configuration, body: { execution, inputWriter, outputReader, errorReader in + async let stdoutBytes = outputReader.collect().flatMap { $0.withUnsafeBytes(Array.init) } + async let stderrBytes = errorReader.collect().flatMap { $0.withUnsafeBytes(Array.init) } + try await inputWriter.finish() + return try await (stdoutBytes, stderrBytes) + }) + return Processes.ExecutionResult(exitStatus: .init(result.terminationStatus), stdout: Data(result.value.0), stderr: Data(result.value.1)) + #else + throw StubError.error("Process spawning is unavailable") + #endif + #else if #available(macOS 15, iOS 18, tvOS 18, watchOS 11, visionOS 2, *) { let stdoutPipe = Pipe() let stderrPipe = Pipe() @@ -118,9 +145,40 @@ extension Process { } return Processes.ExecutionResult(exitStatus: exitStatus, stdout: Data(output.stdoutData), stderr: Data(output.stderrData)) } + #endif } public static func getMergedOutput(url: URL, arguments: [String], currentDirectoryURL: URL? = nil, environment: Environment? = nil, interruptible: Bool = true) async throws -> (exitStatus: Processes.ExitStatus, output: Data) { + #if canImport(Subprocess) + #if !canImport(Darwin) || os(macOS) + let (readEnd, writeEnd) = try FileDescriptor.pipe() + return try await readEnd.closeAfter { + // Direct both stdout and stderr to the same fd. Only set `closeAfterSpawningProcess` on one of the outputs so it isn't double-closed (similarly avoid using closeAfter for the same reason). + var platformOptions = PlatformOptions() + if interruptible { + platformOptions.teardownSequence = [.gracefulShutDown(allowedDurationToNextStep: .seconds(5))] + } + let configuration = try Subprocess.Configuration( + .path(FilePath(url.filePath.str)), + arguments: .init(arguments), + environment: environment.map { .custom(.init($0)) } ?? .inherit, + workingDirectory: (currentDirectoryURL?.filePath.str).map { FilePath($0) } ?? nil, + platformOptions: platformOptions + ) + // FIXME: Use new API from https://github.com/swiftlang/swift-subprocess/pull/180 + let result = try await Subprocess.run(configuration, output: .fileDescriptor(writeEnd, closeAfterSpawningProcess: true), error: .fileDescriptor(writeEnd, closeAfterSpawningProcess: false), body: { execution in + if #available(macOS 15, iOS 18, tvOS 18, watchOS 11, visionOS 2, *) { + try await Array(Data(DispatchFD(fileDescriptor: readEnd).dataStream().collect())) + } else { + try await Array(Data(DispatchFD(fileDescriptor: readEnd)._dataStream().collect())) + } + }) + return (.init(result.terminationStatus), Data(result.value)) + } + #else + throw StubError.error("Process spawning is unavailable") + #endif + #else if #available(macOS 15, iOS 18, tvOS 18, watchOS 11, visionOS 2, *) { let pipe = Pipe() @@ -150,6 +208,7 @@ extension Process { } return (exitStatus: exitStatus, output: Data(output)) } + #endif } private static func _getOutput(url: URL, arguments: [String], currentDirectoryURL: URL?, environment: Environment?, interruptible: Bool, setup: (Process) -> T, collect: @Sendable (T) async throws -> U) async throws -> (exitStatus: Processes.ExitStatus, output: U) { @@ -215,9 +274,8 @@ public enum Processes: Sendable { case exit(_ code: Int32) case uncaughtSignal(_ signal: Int32) - public init?(rawValue: Int32) { - #if os(Windows) - let dwExitCode = DWORD(bitPattern: rawValue) + #if os(Windows) + public init(dwExitCode: DWORD) { // Do the same thing as swift-corelibs-foundation (the input value is the GetExitCodeProcess return value) if (dwExitCode & 0xF0000000) == 0x80000000 // HRESULT || (dwExitCode & 0xF0000000) == 0xC0000000 // NTSTATUS @@ -227,6 +285,12 @@ public enum Processes: Sendable { } else { self = .exit(Int32(bitPattern: UInt32(dwExitCode))) } + } + #endif + + public init?(rawValue: Int32) { + #if os(Windows) + self = .init(dwExitCode: DWORD(bitPattern: rawValue)) #else func WSTOPSIG(_ status: Int32) -> Int32 { return status >> 8 @@ -306,6 +370,37 @@ public enum Processes: Sendable { } } +#if canImport(Subprocess) && (!canImport(Darwin) || os(macOS)) +extension Processes.ExitStatus { + init(_ terminationStatus: TerminationStatus) { + switch terminationStatus { + case let .exited(code): + self = .exit(numericCast(code)) + case let .unhandledException(code): + #if os(Windows) + // Currently swift-subprocess returns the original raw GetExitCodeProcess value as uncaughtSignal for all values other than zero. + // See also: https://github.com/swiftlang/swift-subprocess/issues/114 + self = .init(dwExitCode: code) + #else + self = .uncaughtSignal(code) + #endif + } + } +} + +extension [Subprocess.Environment.Key: String] { + fileprivate init(_ environment: Environment) { + self.init() + let sorted = environment.sorted { $0.key < $1.key } + for (key, value) in sorted { + if let typedKey = Subprocess.Environment.Key(rawValue: key.rawValue) { + self[typedKey] = value + } + } + } +} +#endif + extension Processes.ExitStatus { public init(_ process: Process) throws { assert(!process.isRunning) diff --git a/Sources/SWBUtil/ProcessController.swift b/Sources/SWBUtil/ProcessController.swift new file mode 100644 index 00000000..30bbf17a --- /dev/null +++ b/Sources/SWBUtil/ProcessController.swift @@ -0,0 +1,196 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +public import Foundation + +#if canImport(Subprocess) && (!canImport(Darwin) || os(macOS)) +import Subprocess +#endif + +import Synchronization + +#if canImport(System) +public import System +#else +public import SystemPackage +#endif + +public final class ProcessController: Sendable { + public let path: Path + public let arguments: [String] + public let environment: Environment? + public let workingDirectory: Path? + private let state = SWBMutex(.unstarted) + private let done = WaitCondition() + + private struct RunningState { + var task: Task + var pid: pid_t? + } + + private enum State { + case unstarted + case running(_ runningState: RunningState) + case exited(exitStatus: Result) + } + + public init(path: Path, arguments: [String], environment: Environment?, workingDirectory: Path?) { + self.path = path + self.arguments = arguments + self.environment = environment + self.workingDirectory = workingDirectory + } + + /// Starts the process in the background. + /// - parameter input: The file descriptor to read from for standard input. + /// - parameter output: The file descriptor to write to for standard output. + /// - parameter error: The file descriptor to write to for standard error. + /// - parameter highPriority: Whether to increase the process's priority in a platform-specific way (quality of service on Darwin). + /// - parameter onStarted: Called with the running process's PID once the process has started running, or `nil` if the process did not start successfully. Will be called exactly once. + public func start(input: FileDescriptor, output: FileDescriptor, error: FileDescriptor, highPriority: Bool, onStarted: @Sendable @escaping (_ pid: pid_t?) -> ()) { + state.withLock { state in + guard case .unstarted = state else { + fatalError("API misuse: process was already started") + } + + let task = Task.detached { [path, arguments, environment, workingDirectory, done] in + func updateState(processIdentifier pid: pid_t?) { + self.state.withLock { state in + guard case var .running(runningState) = state, runningState.pid == nil else { + preconditionFailure() // unreachable + } + runningState.pid = pid + state = .running(runningState) + } + onStarted(pid) + } + + defer { done.signal() } + let result = await Result.catching { + #if !canImport(Darwin) || os(macOS) + #if canImport(Subprocess) + var platformOptions = PlatformOptions() + platformOptions.teardownSequence = [.gracefulShutDown(allowedDurationToNextStep: .seconds(5))] + #if os(macOS) + if highPriority { + platformOptions.qualityOfService = .userInitiated + } + #endif + let configuration = Subprocess.Configuration( + .path(FilePath(path.str)), + arguments: .init(arguments), + environment: environment.map { .custom([String: String]($0)) } ?? .inherit, + workingDirectory: (workingDirectory?.str).map { FilePath($0) } ?? nil, + platformOptions: platformOptions + ) + return try await Processes.ExitStatus(Subprocess.run(configuration, input: .fileDescriptor(input, closeAfterSpawningProcess: false), output: .fileDescriptor(output, closeAfterSpawningProcess: false), error: .fileDescriptor(error, closeAfterSpawningProcess: false), body: { execution in + updateState(processIdentifier: numericCast(execution.processIdentifier.value)) + }).terminationStatus) + #else + let process = Process() + process.executableURL = URL(fileURLWithPath: path.str) + process.arguments = arguments + process.environment = environment.map { .init($0) } ?? nil + if let workingDirectory { + process.currentDirectoryURL = URL(fileURLWithPath: workingDirectory.str) + } + if highPriority { + process.qualityOfService = .userInitiated + } + process.standardInput = FileHandle(fileDescriptor: input.rawValue, closeOnDealloc: false) + process.standardOutput = FileHandle(fileDescriptor: output.rawValue, closeOnDealloc: false) + process.standardError = FileHandle(fileDescriptor: error.rawValue, closeOnDealloc: false) + try await process.run { + updateState(processIdentifier: process.processIdentifier) + } + return try Processes.ExitStatus(process) + #endif + #else + throw StubError.error("Process spawning is unavailable") + #endif + } + + let invokedOnStarted = self.state.withLock { state in + switch state { + case let .running(runningState): + state = .exited(exitStatus: result) + return runningState.pid != nil + case .unstarted, .exited: + preconditionFailure() // unreachable + } + } + + if !invokedOnStarted { + onStarted(nil) + } + } + + state = .running(.init(task: task)) + } + } + + public func waitUntilExit() async { + await done.wait() + } + + public func terminate() { + state.withLock { state in + if case let .running(state) = state { + state.task.cancel() + } + } + } + + public var processIdentifier: pid_t? { + get { + state.withLock { state in + switch state { + case .unstarted, .exited: + nil + case let .running(state): + state.pid + } + } + } + } + + public var exitStatus: Processes.ExitStatus? { + get throws { + try state.withLock { state in + switch state { + case .unstarted: + nil + case .running: + nil + case let .exited(exitStatus): + try exitStatus.get() + } + } + } + } +} + +extension RunProcessNonZeroExitError { + public init?(_ process: ProcessController) throws { + self.args = [process.path.str] + process.arguments + self.workingDirectory = process.workingDirectory + self.environment = process.environment ?? .init() + guard let exitStatus = try process.exitStatus else { + return nil + } + self.status = exitStatus + self.output = nil + if self.status.isSuccess { + return nil + } + } +} diff --git a/Sources/SWBUtil/ProcessInfo.swift b/Sources/SWBUtil/ProcessInfo.swift index c441c3df..cc07017a 100644 --- a/Sources/SWBUtil/ProcessInfo.swift +++ b/Sources/SWBUtil/ProcessInfo.swift @@ -125,6 +125,14 @@ extension ProcessInfo { } } +#if !canImport(Darwin) +extension ProcessInfo { + public var isMacCatalystApp: Bool { + false + } +} +#endif + public enum OperatingSystem: Hashable, Sendable { case macOS case iOS(simulator: Bool) diff --git a/Sources/SwiftBuild/SWBBuildServiceConnection.swift b/Sources/SwiftBuild/SWBBuildServiceConnection.swift index 19efb51c..b15422fd 100644 --- a/Sources/SwiftBuild/SWBBuildServiceConnection.swift +++ b/Sources/SwiftBuild/SWBBuildServiceConnection.swift @@ -129,7 +129,7 @@ typealias swb_build_service_connection_message_handler_t = @Sendable (UInt64, SW self.connectionTransport = try connectionMode.createTransport(variant: variant, serviceBundleURL: serviceBundleURL, stdinPipe: stdinPipe, stdoutPipe: stdoutPipe) do { - try self.connectionTransport.start { [weak self] error in + try await self.connectionTransport.start { [weak self] error in guard let strongSelf = self else { return } strongSelf.suspend() @@ -658,7 +658,7 @@ fileprivate protocol ConnectionTransport: AnyObject, Sendable { var state: SWBBuildServiceConnection.State { get } var subprocessPID: pid_t? { get } - func start(terminationHandler: (@Sendable ((any Error)?) -> Void)?) throws + func start(terminationHandler: (@Sendable ((any Error)?) -> Void)?) async throws func terminate() async func close() async } @@ -852,13 +852,12 @@ fileprivate final class InProcessConnection: ConnectionTransport { #if os(macOS) || targetEnvironment(macCatalyst) || !canImport(Darwin) fileprivate final class OutOfProcessConnection: ConnectionTransport { - private let task: SWBUtil.Process + private let task: ProcessController private let done = WaitCondition() + private let stdinPipe: IOPipe + private let stdoutputPipe: IOPipe init(variant: SWBBuildServiceVariant, serviceBundleURL: URL?, stdinPipe: IOPipe, stdoutPipe: IOPipe) throws { - /// Create and configure an NSTask for launching the Swift Build subprocess. - task = Process() - // Compute the launch path and environment. var updatedEnvironment = ProcessInfo.processInfo.environment // Add the contents of the SWBBuildServiceEnvironmentOverrides user default. @@ -893,32 +892,28 @@ fileprivate final class OutOfProcessConnection: ConnectionTransport { } #endif - task.executableURL = launchURL - task.currentDirectoryURL = launchURL.deletingLastPathComponent() - task.environment = environment - - // Similar to the rationale for giving 'userInitiated' QoS for the 'SWBBuildService.ServiceHostConnection.receiveQueue' queue (see comments for that). - // Start the service subprocess with the max QoS so it is setup to service 'userInitiated' requests if required. - task.qualityOfService = .userInitiated + self.stdinPipe = stdinPipe + self.stdoutputPipe = stdoutPipe - task.standardInput = FileHandle(fileDescriptor: stdinPipe.readEnd.rawValue) - task.standardOutput = FileHandle(fileDescriptor: stdoutPipe.writeEnd.rawValue) + task = try ProcessController( + path: launchURL.filePath, + arguments: [], + environment: .init(environment), + workingDirectory: launchURL.deletingLastPathComponent().filePath) } var state: SWBBuildServiceConnection.State { - if task.isRunning { - return .running - } else { - switch task.terminationReason { + do { + switch try task.exitStatus { case .exit: return .exited case .uncaughtSignal: return .crashed - #if canImport(Foundation.NSTask) || !canImport(Darwin) - @unknown default: - preconditionFailure() - #endif + case nil: + return .running } + } catch { + return .crashed } } @@ -926,50 +921,45 @@ fileprivate final class OutOfProcessConnection: ConnectionTransport { return task.processIdentifier } - func start(terminationHandler: (@Sendable ((any Error)?) -> Void)?) throws { - // Install a termination handler that suspends us if we detect the termination of the subprocess. - task.terminationHandler = { [self] task in - defer { done.signal() } - - do { - try terminationHandler?(RunProcessNonZeroExitError(task)) - } catch { - terminationHandler?(error) - } - } - - do { - // Launch the Swift Build subprocess. - try task.run() - } catch { - // terminationHandler isn't going to be called if `run()` throws. - done.signal() - throw error - } + func start(terminationHandler: (@Sendable ((any Error)?) -> Void)?) async throws { + await withCheckedContinuation { (continuation: CheckedContinuation) in + // Similar to the rationale for giving 'userInitiated' QoS for the 'SWBBuildService.ServiceHostConnection.receiveQueue' queue (see comments for that). + // Start the service subprocess with the max QoS so it is setup to service 'userInitiated' requests if required. + task.start(input: stdinPipe.readEnd, output: stdoutputPipe.writeEnd, error: .standardError, highPriority: true, onStarted: { [self] processIdentifier in + defer { continuation.resume() } + + #if os(macOS) + if let processIdentifier { + do { + // If IBAutoAttach is enabled, send the message so Xcode will attach to the inferior. + try Debugger.requestXcodeAutoAttachIfEnabled(processIdentifier) + } catch { + // Terminate the subprocess if start() is going to throw, so that close() will not get stuck. + task.terminate() + } + } + #endif - #if os(macOS) - do { - // If IBAutoAttach is enabled, send the message so Xcode will attach to the inferior. - try Debugger.requestXcodeAutoAttachIfEnabled(task.processIdentifier) - } catch { - // Terminate the subprocess if start() is going to throw, so that close() will not get stuck. - task.terminate() + Task { + await task.waitUntilExit() + do { + try terminationHandler?(RunProcessNonZeroExitError(task)) + } catch { + terminationHandler?(error) + } + } + }) } - #endif } func terminate() async { - assert(task.processIdentifier > 0) task.terminate() - await done.wait() - assert(!task.isRunning) + await task.waitUntilExit() } /// Wait for the subprocess to terminate. func close() async { - assert(task.processIdentifier > 0) - await done.wait() - assert(!task.isRunning) + await task.waitUntilExit() } } #endif diff --git a/Tests/SWBCoreTests/ClangSerializedDiagnosticsTests.swift b/Tests/SWBCoreTests/ClangSerializedDiagnosticsTests.swift index 3f37e39e..e9f8eaac 100644 --- a/Tests/SWBCoreTests/ClangSerializedDiagnosticsTests.swift +++ b/Tests/SWBCoreTests/ClangSerializedDiagnosticsTests.swift @@ -28,7 +28,6 @@ fileprivate struct ClangSerializedDiagnosticsTests: CoreBasedTests { } /// Test that Clang serialized diagnostics are supported. - @Test(.requireThreadSafeWorkingDirectory) func clangSerializedDiagnosticSupported() async throws { try await withTemporaryDirectory { tmpDir in let diagnosticsPath = tmpDir.join("foo.diag") @@ -42,7 +41,6 @@ fileprivate struct ClangSerializedDiagnosticsTests: CoreBasedTests { } /// Test that Clang serialized diagnostics handle relative paths. - @Test(.requireThreadSafeWorkingDirectory) func clangSerializedDiagnosticRelativePaths() async throws { try await withTemporaryDirectory { tmpDir in let diagnosticsPath = tmpDir.join("foo.diag") @@ -71,7 +69,6 @@ fileprivate struct ClangSerializedDiagnosticsTests: CoreBasedTests { } /// Test some of the the details serialized diagnostics from Clang. - @Test(.requireThreadSafeWorkingDirectory) func clangSerializedClangDiagnosticClangsDetails() async throws { try await withTemporaryDirectory { tmpDir in let diagnosticsPath = tmpDir.join("foo.diag") @@ -98,7 +95,6 @@ fileprivate struct ClangSerializedDiagnosticsTests: CoreBasedTests { } /// Test some of the the details serialized diagnostics from SwiftC. - @Test(.requireThreadSafeWorkingDirectory) func clangSerializedSwiftDiagnosticsDetails() async throws { try await withTemporaryDirectory { tmpDir in try localFS.createDirectory(tmpDir.join("dir"), recursive: true) diff --git a/Tests/SWBUtilTests/ProcessTests.swift b/Tests/SWBUtilTests/ProcessTests.swift index 9c7e837d..9a960dde 100644 --- a/Tests/SWBUtilTests/ProcessTests.swift +++ b/Tests/SWBUtilTests/ProcessTests.swift @@ -51,7 +51,6 @@ fileprivate struct ProcessTests { } } - @Test(.requireThreadSafeWorkingDirectory) func workingDirectory() async throws { let previous = Path.currentDirectory.str diff --git a/Tests/SwiftBuildTests/ConsoleCommands/CLIConnection.swift b/Tests/SwiftBuildTests/ConsoleCommands/CLIConnection.swift index 7de7fb62..3c032d18 100644 --- a/Tests/SwiftBuildTests/ConsoleCommands/CLIConnection.swift +++ b/Tests/SwiftBuildTests/ConsoleCommands/CLIConnection.swift @@ -19,19 +19,20 @@ import SwiftBuild #if os(Windows) import WinSDK +#endif + #if canImport(System) import System #else import SystemPackage #endif -#endif /// Helper class for talking to 'swbuild' the tool final class CLIConnection { - private let task: SWBUtil.Process - private let monitorHandle: FileHandle + private let task: ProcessController + private let monitorHandle: FileDescriptor + private let sessionHandle: FileDescriptor private let temporaryDirectory: NamedTemporaryDirectory - private let exitPromise: Promise private let outputStream: AsyncThrowingStream private var outputStreamIterator: AsyncCLIConnectionResponseSequence>>.AsyncIterator @@ -107,45 +108,59 @@ final class CLIConnection { #if os(Windows) throw StubError.error("PTY not supported on Windows") #else - temporaryDirectory = try NamedTemporaryDirectory() + let temporaryDirectory = try NamedTemporaryDirectory() + self.temporaryDirectory = temporaryDirectory // Allocate a PTY we can use to talk to the tool. - var monitorFD = Int32(0) - var sessionFD = Int32(0) - if openpty(&monitorFD, &sessionFD, nil, nil, nil) != 0 { - throw POSIXError(errno, context: "openpty") - } - _ = fcntl(monitorFD, F_SETFD, FD_CLOEXEC) - _ = fcntl(sessionFD, F_SETFD, FD_CLOEXEC) - - monitorHandle = FileHandle(fileDescriptor: monitorFD, closeOnDealloc: true) - let sessionHandle = FileHandle(fileDescriptor: sessionFD, closeOnDealloc: true) - - // Launch the tool. - task = Process() - task.executableURL = try CLIConnection.swiftbuildToolURL - task.currentDirectoryURL = URL(fileURLWithPath: (currentDirectory ?? temporaryDirectory.path).str) - task.standardInput = sessionHandle - task.standardOutput = sessionHandle - task.standardError = sessionHandle - task.environment = .init(Self.environment) + let (monitorHandle, sessionHandle) = try FileDescriptor.openPTY() + + self.monitorHandle = monitorHandle + self.sessionHandle = sessionHandle + do { - exitPromise = try task.launch() + try monitorHandle.setCloseOnExec() + try sessionHandle.setCloseOnExec() + + // Launch the tool. + self.task = try ProcessController( + path: CLIConnection.swiftbuildToolURL.filePath, + arguments: [], + environment: Self.environment, + workingDirectory: currentDirectory ?? temporaryDirectory.path) } catch { - throw StubError.error("Failed to launch the CLI connection: \(error)") + try? monitorHandle.close() + try? sessionHandle.close() + throw error } - // Close the session handle, so the FD will close once the service stops. - try sessionHandle.close() - - outputStream = monitorHandle._bytes() + let startCondition = WaitCondition() + task.start( + input: sessionHandle, + output: sessionHandle, + error: sessionHandle, + highPriority: false, + onStarted: { _ in + // Close the session handle, so the FD will close once the service stops. + try? sessionHandle.close() + startCondition.signal() + }) + + outputStream = DispatchFD(fileDescriptor: monitorHandle)._dataStream() outputStreamIterator = outputStream.flattened.cliResponses.makeAsyncIterator() + await startCondition.wait() #endif } + deinit { + try? monitorHandle.close() + if task.processIdentifier == nil { + try? sessionHandle.close() + } + } + func shutdown() async { // If the task is still running, ensure orderly shutdown. - if task.isRunning { + if (try? task.exitStatus) == nil { try? send(command: "quit") _ = try? await getResponse() _ = try? await exitStatus @@ -162,7 +177,8 @@ final class CLIConnection { try Self.terminate(processIdentifier: processIdentifier) } - static func terminate(processIdentifier: Int32) throws { + static func terminate(processIdentifier: Int32?) throws { + guard let processIdentifier else { return } #if os(Windows) guard let proc = OpenProcess(DWORD(PROCESS_TERMINATE), false, DWORD(processIdentifier)) else { throw Win32Error(GetLastError()) @@ -186,20 +202,24 @@ final class CLIConnection { // occasion, prompting test failures . usleep(10) #endif - try monitorHandle.write(contentsOf: Data(command.appending("\n").utf8)) + try monitorHandle.writeAll(command.appending("\n").utf8) } func getResponse() async throws -> String { try await outputStreamIterator.next() ?? "" } - var processIdentifier: Int32 { + var processIdentifier: Int32? { task.processIdentifier } var exitStatus: Processes.ExitStatus { get async throws { - try await exitPromise.value + await task.waitUntilExit() + guard let exitStatus = try task.exitStatus else { + throw StubError.error("Task is still running") + } + return exitStatus } } } @@ -339,3 +359,22 @@ fileprivate func systemRoot() throws -> Path? { return nil #endif } + +#if !os(Windows) +extension FileDescriptor { + fileprivate static func openPTY() throws -> (monitorFD: FileDescriptor, sessionFD: FileDescriptor) { + var monitorFD = Int32(-1) + var sessionFD = Int32(-1) + if openpty(&monitorFD, &sessionFD, nil, nil, nil) != 0 { + throw POSIXError(errno, context: "openpty") + } + return (monitorFD: FileDescriptor(rawValue: monitorFD), sessionFD: FileDescriptor(rawValue: sessionFD)) + } + + fileprivate func setCloseOnExec() throws { + if fcntl(rawValue, F_SETFD, FD_CLOEXEC) == -1 { + throw POSIXError(errno, context: "fcntl") + } + } +} +#endif diff --git a/Tests/SwiftBuildTests/ConsoleCommands/GeneralCommandsTests.swift b/Tests/SwiftBuildTests/ConsoleCommands/GeneralCommandsTests.swift index fe762297..465d8b90 100644 --- a/Tests/SwiftBuildTests/ConsoleCommands/GeneralCommandsTests.swift +++ b/Tests/SwiftBuildTests/ConsoleCommands/GeneralCommandsTests.swift @@ -60,7 +60,7 @@ fileprivate struct GeneralCommandsTests { XCTAssertMatch(reply, .contains("service pid = ")) // The service PID is NOT the swbuild PID - XCTAssertNoMatch(reply, .contains("service pid = \(cli.processIdentifier)")) + XCTAssertNoMatch(reply, .contains("service pid = \(cli.processIdentifier ?? -1)")) case "headermap", "serializedDiagnostics": XCTAssertMatch(reply, .suffix("\(command)\r\nerror: no path specified\r\n\r\nusage: \(command) --dump path\r\nswbuild> ")) case "help": diff --git a/Tests/SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift b/Tests/SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift index 467977df..048679b0 100644 --- a/Tests/SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift +++ b/Tests/SwiftBuildTests/ConsoleCommands/ServiceConsoleTests.swift @@ -30,24 +30,14 @@ fileprivate struct ServiceConsoleTests { @Test func emptyInput() async throws { // Test against a non-pty. - let task = SWBUtil.Process() - task.executableURL = try CLIConnection.swiftbuildToolURL - task.environment = .init(CLIConnection.environment) + let result = try await Process.getOutput(url: CLIConnection.swiftbuildToolURL, arguments: [], environment: CLIConnection.environment) + let output = String(decoding: result.stdout, as: UTF8.self) - task.standardInput = FileHandle.nullDevice - try await withExtendedLifetime(Pipe()) { outputPipe in - let standardOutput = task._makeStream(for: \.standardOutputPipe, using: outputPipe) - let promise: Promise = try task.launch() - - let data = try await standardOutput.reduce(into: [], { $0.append(contentsOf: $1) }) - let output = String(decoding: data, as: UTF8.self) - - // Verify there were no errors. - #expect(output == "swbuild> \(String.newline)") + // Verify there were no errors. + #expect(output == "swbuild> \(String.newline)") - // Assert the tool exited successfully. - await #expect(try promise.value == .exit(0)) - } + // Assert the tool exited successfully. + #expect(try result.exitStatus == .exit(0)) } @Test @@ -78,7 +68,7 @@ fileprivate struct ServiceConsoleTests { servicePID = try #require(pid_t(match.output.pid)) } } - #expect(servicePID != -1, "unable to find service PID") + try #require(servicePID != -1, "unable to find service PID") return servicePID }() #expect(servicePID != cli.processIdentifier, "service PID (\(servicePID)) must not match the CLI PID (\(cli.processIdentifier)) when running in out-of-process mode")