Merge pull request #847 from owenv/owenv/macos-escaping

Retain old-syle response file escaping on macOS for compatibility

Author: owenv

Sources/SWBApplePlatform/MetalCompiler.swift

@@ -33,7 +33,7 @@ struct MetalSourceFileIndexingInfo: SourceFileIndexingInfo {
 
     fileprivate init(task: any ExecutableTask, payload: MetalIndexingPayload) {
         self.outputFile = Path(task.commandLine[payload.outputFileIndex].asString)
-        self.commandLine = ClangSourceFileIndexingInfo.indexingCommandLine(from: task.commandLine.map(\.asByteString), workingDir: payload.workingDir, responseFileMapping: [:])
+        self.commandLine = ClangSourceFileIndexingInfo.indexingCommandLine(from: task.commandLine.map(\.asByteString), workingDir: payload.workingDir, responseFileMapping: [:], responseFileFormat: nil)
         self.builtProductsDir = payload.builtProductsDir
         self.toolchains = payload.toolchains
     }

Sources/SWBBuildSystem/BuildOperation.swift

@@ -983,6 +983,10 @@ extension BuildOperation: TaskExecutionDelegate {
         return core
     }
 
+    package var hostOperatingSystem: OperatingSystem {
+        core.hostOperatingSystem
+    }
+
     package var sdkRegistry: SDKRegistry {
         return core.sdkRegistry
     }

Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift

@@ -103,6 +103,7 @@ public struct ClangIndexingPayload: Serializable, Encodable, Sendable {
     let prefixInfo: ClangPrefixInfo?
     public let toolchains: [String]
     let responseFileAttachmentPaths: [Path: Path]
+    let responseFileFormat: ResponseFileFormat
 
     init(sourceFileIndex: Int,
          outputFileIndex: Int,
@@ -112,7 +113,8 @@ public struct ClangIndexingPayload: Serializable, Encodable, Sendable {
          workingDir: Path,
          prefixInfo: ClangPrefixInfo?,
          toolchains: [String],
-         responseFileAttachmentPaths: [Path: Path]) {
+         responseFileAttachmentPaths: [Path: Path],
+         responseFileFormat: ResponseFileFormat) {
         self.sourceFileIndex = sourceFileIndex
         self.outputFileIndex = outputFileIndex
         self.sourceLanguageIndex = sourceLanguageIndex
@@ -122,14 +124,15 @@ public struct ClangIndexingPayload: Serializable, Encodable, Sendable {
         self.prefixInfo = prefixInfo
         self.toolchains = toolchains
         self.responseFileAttachmentPaths = responseFileAttachmentPaths
+        self.responseFileFormat = responseFileFormat
     }
 
     func sourceFile(for task: any ExecutableTask) -> Path {
         return Path(task.commandLine[self.sourceFileIndex].asString)
     }
 
     public func serialize<T: Serializer>(to serializer: T) {
-        serializer.serializeAggregate(9) {
+        serializer.serializeAggregate(10) {
             serializer.serialize(sourceFileIndex)
             serializer.serialize(outputFileIndex)
             serializer.serialize(sourceLanguageIndex)
@@ -139,11 +142,12 @@ public struct ClangIndexingPayload: Serializable, Encodable, Sendable {
             serializer.serializeUniquely(prefixInfo)
             serializer.serialize(toolchains)
             serializer.serialize(responseFileAttachmentPaths)
+            serializer.serialize(responseFileFormat)
         }
     }
 
     public init(from deserializer: any Deserializer) throws {
-        try deserializer.beginAggregate(9)
+        try deserializer.beginAggregate(10)
         self.sourceFileIndex = try deserializer.deserialize()
         self.outputFileIndex = try deserializer.deserialize()
         self.sourceLanguageIndex = try deserializer.deserialize()
@@ -153,6 +157,7 @@ public struct ClangIndexingPayload: Serializable, Encodable, Sendable {
         self.prefixInfo = try deserializer.deserializeUniquely()
         self.toolchains = try deserializer.deserialize()
         self.responseFileAttachmentPaths = try deserializer.deserialize()
+        self.responseFileFormat = try deserializer.deserialize()
     }
 }
 
@@ -192,7 +197,7 @@ public struct ClangSourceFileIndexingInfo: SourceFileIndexingInfo {
     fileprivate init(task: any ExecutableTask, payload: ClangIndexingPayload, enableIndexBuildArena: Bool) {
         self.outputFile = Path(task.commandLine[payload.outputFileIndex].asString)
         self.sourceLanguage = task.commandLine[payload.sourceLanguageIndex].asByteString
-        self.commandLine = Self.indexingCommandLine(from: task.commandLine.map(\.asByteString), workingDir: payload.workingDir, prefixInfo: payload.prefixInfo, addSupplementary: !enableIndexBuildArena, responseFileMapping: payload.responseFileAttachmentPaths)
+        self.commandLine = Self.indexingCommandLine(from: task.commandLine.map(\.asByteString), workingDir: payload.workingDir, prefixInfo: payload.prefixInfo, addSupplementary: !enableIndexBuildArena, responseFileMapping: payload.responseFileAttachmentPaths, responseFileFormat: payload.responseFileFormat)
         self.builtProductsDir = payload.builtProductsDir
         self.assetSymbolIndexPath = payload.assetSymbolIndexPath
         self.prefixInfo = payload.prefixInfo
@@ -202,7 +207,7 @@ public struct ClangSourceFileIndexingInfo: SourceFileIndexingInfo {
     static let skippedArgsWithoutValues = Set<ByteString>(["-M", "-MD", "-MMD", "-MG", "-MJ", "-MM", "-MP", "-MV", "-fmodules-validate-once-per-build-session"])
     static let skippedArgsWithValues = Set<ByteString>(["-MT", "-MF", "-MQ", "--serialize-diagnostics"])
 
-    public static func indexingCommandLine(from commandLine: [ByteString], workingDir: Path, prefixInfo: ClangPrefixInfo? = nil, addSupplementary: Bool = true, replaceCompile: Bool = true, responseFileMapping: [Path: Path]) -> [ByteString] {
+    public static func indexingCommandLine(from commandLine: [ByteString], workingDir: Path, prefixInfo: ClangPrefixInfo? = nil, addSupplementary: Bool = true, replaceCompile: Bool = true, responseFileMapping: [Path: Path], responseFileFormat: ResponseFileFormat?) -> [ByteString] {
         var result = [ByteString]()
         var iterator = commandLine.makeIterator()
         let _ = iterator.next() // Skip compiler path
@@ -235,7 +240,8 @@ public struct ClangSourceFileIndexingInfo: SourceFileIndexingInfo {
                 // Skip
             } else if arg.starts(with: ByteString(unicodeScalarLiteral: "@")),
                       let attachmentPath = responseFileMapping[Path(arg.asString.dropFirst())],
-                      let responseFileArgs = try? ResponseFiles.expandResponseFiles(["@\(attachmentPath.str)"], fileSystem: localFS, relativeTo: workingDir, format: .llvmStyleEscaping) {
+                      let responseFileFormat,
+                      let responseFileArgs = try? ResponseFiles.expandResponseFiles(["@\(attachmentPath.str)"], fileSystem: localFS, relativeTo: workingDir, format: responseFileFormat) {
                 result.append(contentsOf: responseFileArgs.map { ByteString(encodingAsUTF8: $0) })
             } else {
                 result.append(arg)
@@ -741,7 +747,8 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
             ctx.add(string: self.identifier)
 
             let responseFilePath = scope.evaluate(BuiltinMacros.PER_ARCH_OBJECT_FILE_DIR).join("\(ctx.signature.asString)-common-args.resp")
-            let attachmentPath = producer.writeFileSpec.constructFileTasks(CommandBuildContext(producer: producer, scope: scope, inputs: [], output: responseFilePath), delegate, contents: ByteString(encodingAsUTF8: ResponseFiles.responseFileContents(args: responseFileCommandLine, format: .llvmStyleEscaping)), permissions: nil, logContents: true, preparesForIndexing: true, additionalTaskOrderingOptions: [.immediate, .ignorePhaseOrdering])
+            let responseFileFormat = Self.responseFileFormat(hostOS: producer.hostOperatingSystem)
+            let attachmentPath = producer.writeFileSpec.constructFileTasks(CommandBuildContext(producer: producer, scope: scope, inputs: [], output: responseFilePath), delegate, contents: ByteString(encodingAsUTF8: ResponseFiles.responseFileContents(args: responseFileCommandLine, format: responseFileFormat)), permissions: nil, logContents: true, preparesForIndexing: true, additionalTaskOrderingOptions: [.immediate, .ignorePhaseOrdering])
 
             return ConstantFlags(flags: regularCommandLine + ["@\(responseFilePath.str)"], headerSearchPaths: headerSearchPaths, inputs: [responseFilePath], responseFileMapping: [responseFilePath: attachmentPath])
         } else {
@@ -1307,7 +1314,8 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
                 workingDir: cbc.scope.evaluate(BuiltinMacros.PROJECT_DIR),
                 prefixInfo: prefixInfo,
                 toolchains: cbc.producer.toolchains.map{ $0.identifier },
-                responseFileAttachmentPaths: constantFlags.responseFileMapping
+                responseFileAttachmentPaths: constantFlags.responseFileMapping,
+                responseFileFormat: Self.responseFileFormat(hostOS: cbc.producer.hostOperatingSystem)
             )
         } else {
             indexingPayload  = nil
@@ -1794,7 +1802,7 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
         return ClangPrefixInfo.PCHInfo(
             output: precompPath,
             hashCriteria: nil, // rdar://problem/24469921
-            commandLine: ClangSourceFileIndexingInfo.indexingCommandLine(from: byteStringCommandLine, workingDir: cbc.scope.evaluate(BuiltinMacros.PROJECT_DIR), addSupplementary: !hasEnabledIndexBuildArena, replaceCompile: false, responseFileMapping: constantFlags.responseFileMapping)
+            commandLine: ClangSourceFileIndexingInfo.indexingCommandLine(from: byteStringCommandLine, workingDir: cbc.scope.evaluate(BuiltinMacros.PROJECT_DIR), addSupplementary: !hasEnabledIndexBuildArena, replaceCompile: false, responseFileMapping: constantFlags.responseFileMapping, responseFileFormat: Self.responseFileFormat(hostOS: cbc.producer.hostOperatingSystem))
         )
     }
 
@@ -1846,6 +1854,16 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
         return try await discoveredClangToolInfo(producer, delegate, toolPath: toolPath, arch: arch, sysroot: sdk?.path, language: fileType?.languageDialect?.dialectNameForCompilerCommandLineArgument ?? "c", blocklistsPathOverride: userSpecifiedBlocklists)
     }
 
+    package static func responseFileFormat(hostOS: OperatingSystem) -> ResponseFileFormat {
+        switch hostOS {
+        case .macOS:
+            // Maintained for compatibility with projects which do manual shell escaping of their own in build settings.
+            .unixShellQuotedSpaceSeparated
+        default:
+            .llvmStyleEscaping
+        }
+    }
+
     override public func discoveredCommandLineToolSpecInfo(_ producer: any CommandProducer, _ scope: MacroEvaluationScope, _ delegate: any CoreClientTargetDiagnosticProducingDelegate) async -> (any DiscoveredCommandLineToolSpecInfo)? {
         do {
             return try await discoveredCommandLineToolSpecInfo(producer, scope, delegate, forLanguageOfFileType: nil)

Sources/SWBTaskExecution/Task.swift

@@ -569,6 +569,8 @@ public protocol TaskExecutionDelegate
     /// User preferences
     var userPreferences: UserPreferences { get }
 
+    var hostOperatingSystem: OperatingSystem { get }
+
     var emitFrontendCommandLines: Bool { get }
 
     var infoLookup: any PlatformInfoLookup { get }

Sources/SWBTaskExecution/TaskActions/ClangScanTaskAction.swift

@@ -74,7 +74,7 @@ public final class ClangScanTaskAction: TaskAction, BuildValueValidatingTaskActi
             self.scanningOutput = parsedOutput
             if expandResponseFiles {
                 do {
-                    self.commandLine = try ResponseFiles.expandResponseFiles(cliArguments, fileSystem: executionDelegate.fs, relativeTo: workingDirectory, format: .llvmStyleEscaping)
+                    self.commandLine = try ResponseFiles.expandResponseFiles(cliArguments, fileSystem: executionDelegate.fs, relativeTo: workingDirectory, format: ClangCompilerSpec.responseFileFormat(hostOS: executionDelegate.hostOperatingSystem))
                 } catch {
                     outputDelegate.error(error.localizedDescription)
                     return nil

Tests/SWBTaskExecutionTests/InProcessTaskTestSupport.swift

@@ -36,6 +36,7 @@ struct MockExecutionDelegate: TaskExecutionDelegate {
     let environment: [String: String]?
     let userPreferences: UserPreferences
     let infoLookup: any PlatformInfoLookup
+    var hostOperatingSystem: OperatingSystem { core!.hostOperatingSystem }
     var sdkRegistry: SDKRegistry { core!.sdkRegistry }
     var specRegistry: SpecRegistry { core!.specRegistry }
     var platformRegistry: PlatformRegistry { core!.platformRegistry }