Take the host platform into account when writing clang response files

Author: owenv

Sources/SWBCore/Settings/BuiltinMacros.swift

@@ -1189,6 +1189,7 @@ public final class BuiltinMacros {
     public static let _WRAPPER_PARENT_PATH = BuiltinMacros.declareStringMacro("_WRAPPER_PARENT_PATH")
     public static let _WRAPPER_RESOURCES_DIR = BuiltinMacros.declareStringMacro("_WRAPPER_RESOURCES_DIR")
     public static let __INPUT_FILE_LIST_PATH__ = BuiltinMacros.declarePathMacro("__INPUT_FILE_LIST_PATH__")
+    public static let HOST_RESPONSE_FILE_FORMAT = BuiltinMacros.declareEnumMacro("HOST_RESPONSE_FILE_FORMAT") as EnumMacroDeclaration<ResponseFileFormat>
     public static let LINKER_FILE_LIST_FORMAT = BuiltinMacros.declareEnumMacro("LINKER_FILE_LIST_FORMAT") as EnumMacroDeclaration<ResponseFileFormat>
     public static let LINKER_RESPONSE_FILE_FORMAT = BuiltinMacros.declareEnumMacro("LINKER_RESPONSE_FILE_FORMAT") as EnumMacroDeclaration<ResponseFileFormat>
     public static let SWIFT_RESPONSE_FILE_PATH = BuiltinMacros.declarePathMacro("SWIFT_RESPONSE_FILE_PATH")
@@ -2443,6 +2444,7 @@ public final class BuiltinMacros {
         _WRAPPER_PARENT_PATH,
         _WRAPPER_RESOURCES_DIR,
         __INPUT_FILE_LIST_PATH__,
+        HOST_RESPONSE_FILE_FORMAT,
         LINKER_FILE_LIST_FORMAT,
         LINKER_RESPONSE_FILE_FORMAT,
         __ARCHS__,

Sources/SWBCore/Settings/Settings.swift

@@ -2350,6 +2350,8 @@ private class SettingsBuilder {
             platformTable.push(BuiltinMacros.NATIVE_ARCH, literal: Architecture.hostStringValue ?? fallbackArch)
         }
 
+        platformTable.push(BuiltinMacros.HOST_RESPONSE_FILE_FORMAT, literal: workspaceContext.core.hostOperatingSystem.defaultResponseFileFormat)
+
         // Add the platform deployment target defaults, for real platforms.
         //
         // FIXME: For legacy compatibility, we only do this when configuring settings for a target. This distinction most likely isn't important, and should just be eliminated.

Sources/SWBCore/SpecImplementations/Tools/CCompiler.swift

@@ -92,6 +92,11 @@ public struct ClangPrefixInfo: Serializable, Hashable, Encodable, Sendable {
     }
 }
 
+public struct ClangResponseFileInfo: SerializableCodable, Sendable {
+    var attachmentPath: Path
+    var format: ResponseFileFormat
+}
+
 /// The minimal data we need to serialize to reconstruct `ClangSourceFileIndexingInfo` from `generateIndexingInfo`
 public struct ClangIndexingPayload: Serializable, Encodable, Sendable {
     let sourceFileIndex: Int
@@ -102,7 +107,7 @@ public struct ClangIndexingPayload: Serializable, Encodable, Sendable {
     let workingDir: Path
     let prefixInfo: ClangPrefixInfo?
     public let toolchains: [String]
-    let responseFileAttachmentPaths: [Path: Path]
+    let responseFileAttachmentPaths: [Path: ClangResponseFileInfo]
 
     init(sourceFileIndex: Int,
          outputFileIndex: Int,
@@ -112,7 +117,7 @@ public struct ClangIndexingPayload: Serializable, Encodable, Sendable {
          workingDir: Path,
          prefixInfo: ClangPrefixInfo?,
          toolchains: [String],
-         responseFileAttachmentPaths: [Path: Path]) {
+         responseFileAttachmentPaths: [Path: ClangResponseFileInfo]) {
         self.sourceFileIndex = sourceFileIndex
         self.outputFileIndex = outputFileIndex
         self.sourceLanguageIndex = sourceLanguageIndex
@@ -202,7 +207,7 @@ public struct ClangSourceFileIndexingInfo: SourceFileIndexingInfo {
     static let skippedArgsWithoutValues = Set<ByteString>(["-M", "-MD", "-MMD", "-MG", "-MJ", "-MM", "-MP", "-MV", "-fmodules-validate-once-per-build-session"])
     static let skippedArgsWithValues = Set<ByteString>(["-MT", "-MF", "-MQ", "--serialize-diagnostics"])
 
-    public static func indexingCommandLine(from commandLine: [ByteString], workingDir: Path, prefixInfo: ClangPrefixInfo? = nil, addSupplementary: Bool = true, replaceCompile: Bool = true, responseFileMapping: [Path: Path]) -> [ByteString] {
+    public static func indexingCommandLine(from commandLine: [ByteString], workingDir: Path, prefixInfo: ClangPrefixInfo? = nil, addSupplementary: Bool = true, replaceCompile: Bool = true, responseFileMapping: [Path: ClangResponseFileInfo]) -> [ByteString] {
         var result = [ByteString]()
         var iterator = commandLine.makeIterator()
         let _ = iterator.next() // Skip compiler path
@@ -234,8 +239,8 @@ public struct ClangSourceFileIndexingInfo: SourceFileIndexingInfo {
             } else if arg.bytes.starts(with: ByteString(stringLiteral: "-fbuild-session-file=").bytes) {
                 // Skip
             } else if arg.starts(with: ByteString(unicodeScalarLiteral: "@")),
-                      let attachmentPath = responseFileMapping[Path(arg.asString.dropFirst())],
-                      let responseFileArgs = try? ResponseFiles.expandResponseFiles(["@\(attachmentPath.str)"], fileSystem: localFS, relativeTo: workingDir, format: .unixShellQuotedSpaceSeparated) {
+                      let attachment = responseFileMapping[Path(arg.asString.dropFirst())],
+                      let responseFileArgs = try? ResponseFiles.expandResponseFiles(["@\(attachment.attachmentPath.str)"], fileSystem: localFS, relativeTo: workingDir, format: attachment.format) {
                 result.append(contentsOf: responseFileArgs.map { ByteString(encodingAsUTF8: $0) })
             } else {
                 result.append(arg)
@@ -568,7 +573,7 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
             let inputs: [Path]
 
             /// Maps response files in `flags` to the corresponding recorded attachment in the build description.
-            let responseFileMapping: [Path: Path]
+            let responseFileMapping: [Path: ClangResponseFileInfo]
 
         }
 
@@ -719,9 +724,10 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible
             ctx.add(string: self.identifier)
 
             let responseFilePath = scope.evaluate(BuiltinMacros.PER_ARCH_OBJECT_FILE_DIR).join("\(ctx.signature.asString)-common-args.resp")
-            let attachmentPath = producer.writeFileSpec.constructFileTasks(CommandBuildContext(producer: producer, scope: scope, inputs: [], output: responseFilePath), delegate, contents: ByteString(encodingAsUTF8: ResponseFiles.responseFileContents(args: responseFileCommandLine, format: .unixShellQuotedSpaceSeparated)), permissions: nil, logContents: true, preparesForIndexing: true, additionalTaskOrderingOptions: [.immediate, .ignorePhaseOrdering])
+            let responseFileFormat = scope.evaluate(BuiltinMacros.HOST_RESPONSE_FILE_FORMAT)
+            let attachmentPath = producer.writeFileSpec.constructFileTasks(CommandBuildContext(producer: producer, scope: scope, inputs: [], output: responseFilePath), delegate, contents: ByteString(encodingAsUTF8: ResponseFiles.responseFileContents(args: responseFileCommandLine, format: responseFileFormat)), permissions: nil, logContents: true, preparesForIndexing: true, additionalTaskOrderingOptions: [.immediate, .ignorePhaseOrdering])
 
-            return ConstantFlags(flags: regularCommandLine + ["@\(responseFilePath.str)"], headerSearchPaths: headerSearchPaths, inputs: [responseFilePath], responseFileMapping: [responseFilePath: attachmentPath])
+            return ConstantFlags(flags: regularCommandLine + ["@\(responseFilePath.str)"], headerSearchPaths: headerSearchPaths, inputs: [responseFilePath], responseFileMapping: [responseFilePath: .init(attachmentPath: attachmentPath, format: responseFileFormat)])
         } else {
             return ConstantFlags(flags: commandLine, headerSearchPaths: headerSearchPaths, inputs: [], responseFileMapping: [:])
         }

Sources/SWBUtil/ProcessInfo.swift

@@ -167,6 +167,15 @@ public enum OperatingSystem: Hashable, Sendable {
             return .elf
         }
     }
+
+    public var defaultResponseFileFormat: ResponseFileFormat {
+        switch self {
+        case .windows:
+            .windowsShellQuotedNewlineSeparated
+        case .macOS, .iOS, .tvOS, .watchOS, .visionOS, .linux, .freebsd, .openbsd, .android, .unknown:
+            .unixShellQuotedSpaceSeparated
+        }
+    }
 }
 
 public enum ImageFormat {

Tests/SWBTaskConstructionTests/ClangResponseFileTaskConstructionTests.swift

@@ -62,11 +62,13 @@ fileprivate struct ClangResponseFileTaskConstructionTests: CoreBasedTests {
                     let responseFilePath = try #require(task.outputs.only)
 
                     // The command arguments in the response file vary vastly between different platforms, so just check for some basics present in the content.
-                    let contentAsString = contents.asString
-                    #expect(contentAsString.contains("-target "))
-                    #expect(contentAsString.contains("Test-generated-files.hmap"))
-                    #expect(contentAsString.contains("Test-own-target-headers.hmap"))
-                    #expect(contentAsString.contains("Test-all-target-headers.hmap"))
+                    if tester.core.hostOperatingSystem.defaultResponseFileFormat == .unixShellQuotedSpaceSeparated {
+                        let contentAsString = contents.asString
+                        #expect(contentAsString.contains("-target "))
+                        #expect(contentAsString.contains("Test-generated-files.hmap"))
+                        #expect(contentAsString.contains("Test-own-target-headers.hmap"))
+                        #expect(contentAsString.contains("Test-all-target-headers.hmap"))
+                    }
 
                     for name in ["a.c", "b.c", "c.c"] {
                         results.checkTask(.matchRuleType("CompileC"), .matchRuleItemPattern(.suffix(name))) { compileTask in
@@ -166,20 +168,22 @@ fileprivate struct ClangResponseFileTaskConstructionTests: CoreBasedTests {
             let tester = try TaskConstructionTester(core, testProject)
             await tester.checkBuild(runDestination: .host) { results in
                 results.checkWriteAuxiliaryFileTask(.matchRuleItemPattern(.suffix("-common-args.resp"))) { task, contents in
-                    let stringContents = contents.asString
-                    #expect(stringContents.contains("-target"))
-                    let blocksFlag = switch runDestination {
+                    if tester.core.hostOperatingSystem.defaultResponseFileFormat == .unixShellQuotedSpaceSeparated {
+                        let stringContents = contents.asString
+                        #expect(stringContents.contains("-target"))
+                        let blocksFlag = switch runDestination {
                         case .macOS:
                             "-fasm-blocks"
                         case .linux:
                             "-fblocks"
                         default:
                             " "
+                        }
+                        #expect(stringContents.contains(blocksFlag))
+                        #expect(!stringContents.contains("-MMD"))
+                        #expect(!stringContents.contains("-fcolor-diagnostics"))
+                        #expect(!stringContents.contains("-Wno-private-module"))
                     }
-                    #expect(stringContents.contains(blocksFlag))
-                    #expect(!stringContents.contains("-MMD"))
-                    #expect(!stringContents.contains("-fcolor-diagnostics"))
-                    #expect(!stringContents.contains("-Wno-private-module"))
                     for name in ["a", "b", "c"] {
                         results.checkTask(.matchRuleType("CompileC"), .matchRuleItemPattern(.suffix(name + ".c"))) { compileTask in
                             compileTask.checkCommandLineMatches(["-MMD", "-MT", "dependencies", "-MF", .suffix(name + ".d")])
@@ -349,8 +353,10 @@ fileprivate struct ClangResponseFileTaskConstructionTests: CoreBasedTests {
             let tester = try TaskConstructionTester(core, testProject)
             await tester.checkBuild(runDestination: .host) { results in
                 results.checkWriteAuxiliaryFileTask(.matchRuleItemPattern(.suffix("-common-args.resp"))) { task, contents in
-                    let stringContents = contents.asString
-                    #expect(stringContents.contains("-Xclang -Wno-shorten-64-to-32"))
+                    if tester.core.hostOperatingSystem.defaultResponseFileFormat == .unixShellQuotedSpaceSeparated {
+                        let stringContents = contents.asString
+                        #expect(stringContents.contains("-Xclang -Wno-shorten-64-to-32"))
+                    }
                     results.checkTask(.matchRuleType("CompileC"), .matchRuleItemPattern(.suffix("a.c"))) { compileTask in
                         compileTask.checkCommandLineDoesNotContain("-Xclang")
                     }