Merge pull request #67942 from cachemeifyoucan/eng/PR-swift-scan-plugin-cas-support

[SwiftScan][Caching] Add support for plugin CAS for caching

Author: cachemeifyoucan

include/swift-c/DependencyScan/DependencyScan.h

@@ -25,7 +25,7 @@
 /// SWIFTSCAN_VERSION_MINOR should increase when there are API additions.
 /// SWIFTSCAN_VERSION_MAJOR is intended for "major" source/ABI breaking changes.
 #define SWIFTSCAN_VERSION_MAJOR 0
-#define SWIFTSCAN_VERSION_MINOR 4
+#define SWIFTSCAN_VERSION_MINOR 5
 
 SWIFTSCAN_BEGIN_DECLS
 
@@ -430,6 +430,9 @@ SWIFTSCAN_PUBLIC int invoke_swift_compiler(int argc, const char **argv);
 
 //=== Scanner CAS Operations ----------------------------------------------===//
 
+/// Opaque container for a CASOptions that describe how CAS should be created.
+typedef struct swiftscan_cas_options_s *swiftscan_cas_options_t;
+
 /// Opaque container for a CAS instance that includes both ObjectStore and
 /// ActionCache.
 typedef struct swiftscan_cas_s *swiftscan_cas_t;
@@ -445,22 +448,54 @@ typedef enum {
   SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5
 } swiftscan_output_kind_t;
 
-/// Create a \c cas instance that points to path.
-SWIFTSCAN_PUBLIC swiftscan_cas_t swiftscan_cas_create(const char *path);
+/// Create a \c CASOptions for creating CAS inside scanner specified.
+SWIFTSCAN_PUBLIC swiftscan_cas_options_t swiftscan_cas_options_create(void);
+
+/// Dispose \c CASOptions.
+SWIFTSCAN_PUBLIC void
+swiftscan_cas_options_dispose(swiftscan_cas_options_t options);
+
+/// Set on-disk path for the \c cas.
+SWIFTSCAN_PUBLIC void
+swiftscan_cas_options_set_ondisk_path(swiftscan_cas_options_t options,
+                                      const char *path);
+
+/// Set plugin path for the \c cas.
+SWIFTSCAN_PUBLIC void
+swiftscan_cas_options_set_plugin_path(swiftscan_cas_options_t options,
+                                      const char *path);
+
+/// Set option using a name/value pair. Return true if error.
+/// If error happens, the error message is returned via `error` parameter, and
+/// caller needs to free the error message via `swiftscan_string_dispose`.
+SWIFTSCAN_PUBLIC bool
+swiftscan_cas_options_set_option(swiftscan_cas_options_t options,
+                                 const char *name, const char *value,
+                                 swiftscan_string_ref_t *error);
+
+/// Create a \c cas instance from plugin. Return NULL if error.
+/// If error happens, the error message is returned via `error` parameter, and
+/// caller needs to free the error message via `swiftscan_string_dispose`.
+SWIFTSCAN_PUBLIC swiftscan_cas_t swiftscan_cas_create_from_options(
+    swiftscan_cas_options_t options, swiftscan_string_ref_t *error);
 
 /// Dispose the \c cas instance.
 SWIFTSCAN_PUBLIC void swiftscan_cas_dispose(swiftscan_cas_t cas);
 
-/// Store content into CAS. Return \c CASID as string.
-SWIFTSCAN_PUBLIC swiftscan_string_ref_t swiftscan_cas_store(swiftscan_cas_t cas,
-                                                            uint8_t *data,
-                                                            unsigned size);
+/// Store content into CAS. Return \c CASID as string. Return NULL on error.
+/// If error happens, the error message is returned via `error` parameter, and
+/// caller needs to free the error message via `swiftscan_string_dispose`.
+SWIFTSCAN_PUBLIC swiftscan_string_ref_t
+swiftscan_cas_store(swiftscan_cas_t cas, uint8_t *data, unsigned size,
+                    swiftscan_string_ref_t *error);
 
 /// Compute \c CacheKey for output of \c kind from the compiler invocation \c
 /// argc and \c argv with \c input. Return \c CacheKey as string.
-SWIFTSCAN_PUBLIC swiftscan_string_ref_t
-swiftscan_compute_cache_key(swiftscan_cas_t cas, int argc, const char **argv,
-                            const char *input, swiftscan_output_kind_t kind);
+/// If error happens, the error message is returned via `error` parameter, and
+/// caller needs to free the error message via `swiftscan_string_dispose`.
+SWIFTSCAN_PUBLIC swiftscan_string_ref_t swiftscan_compute_cache_key(
+    swiftscan_cas_t cas, int argc, const char **argv, const char *input,
+    swiftscan_output_kind_t kind, swiftscan_string_ref_t *error);
 
 //===----------------------------------------------------------------------===//
 

include/swift/AST/DiagnosticsCommon.def

@@ -201,6 +201,9 @@ ERROR(scanner_find_cycle, none,
 ERROR(scanner_arguments_invalid, none,
       "dependencies scanner cannot be configured with arguments: '%0'", (StringRef))
 
+ERROR(error_scanner_extra, none,
+      "failed inside dependency scanner: '%0'", (StringRef))
+
 WARNING(warn_scanner_deserialize_failed, none,
         "Failed to load module scanning dependency cache from: '%0', re-building scanner cache from scratch.", (StringRef))
 

include/swift/Frontend/CachingUtils.h

@@ -15,10 +15,11 @@
 
 #include "swift/Frontend/CachedDiagnostics.h"
 #include "swift/Frontend/FrontendInputsAndOutputs.h"
+#include "clang/CAS/CASOptions.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/CAS/ActionCache.h"
-#include "llvm/CAS/ObjectStore.h"
 #include "llvm/CAS/CASReference.h"
+#include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/VirtualOutputBackend.h"
 #include <memory>
@@ -58,29 +59,6 @@ llvm::Error storeCachedCompilerOutput(llvm::cas::ObjectStore &CAS,
 llvm::Expected<llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>>
 createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef<std::string> FSRoots,
                     ArrayRef<std::string> IncludeTreeRoots);
-
-namespace cas {
-/// Helper class to manage CAS/Caching from libSwiftScan C APIs.
-class CachingTool {
-public:
-  // Create the tool with a list of arguments from compiler invocation.
-  CachingTool(llvm::StringRef Path);
-
-  // Compute the CASID for PCH output from invocation.
-  std::string computeCacheKey(llvm::ArrayRef<const char *> Args,
-                              StringRef InputPath, file_types::ID OutputKind);
-
-  // Store content into CAS.
-  std::string storeContent(llvm::StringRef Content);
-
-  // Check if the tool is correctly initialized.
-  bool isValid() const { return CAS && Cache; }
-
-private:
-  std::unique_ptr<llvm::cas::ObjectStore> CAS;
-  std::unique_ptr<llvm::cas::ActionCache> Cache;
-};
-} // namespace cas
 }
 
 #endif

lib/Frontend/CachingUtils.cpp

@@ -15,6 +15,7 @@
 #include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/Basic/FileTypes.h"
 #include "swift/Frontend/CompileJobCacheKey.h"
+#include "clang/CAS/CASOptions.h"
 #include "clang/CAS/IncludeTree.h"
 #include "clang/Frontend/CompileJobCacheResult.h"
 #include "llvm/ADT/STLExtras.h"
@@ -422,51 +423,4 @@ createCASFileSystem(ObjectStore &CAS, ArrayRef<std::string> FSRoots,
   return CASFS;
 }
 
-namespace cas {
-
-CachingTool::CachingTool(StringRef Path) {
-  auto DB = llvm::cas::createOnDiskUnifiedCASDatabases(Path);
-  if (!DB) {
-    llvm::errs() << "Failed to create CAS at " << Path << ": "
-                 << toString(DB.takeError()) << "\n";
-    return;
-  }
-
-  CAS = std::move(DB->first);
-  Cache = std::move(DB->second);
-}
-
-std::string CachingTool::computeCacheKey(ArrayRef<const char *> Args,
-                                         StringRef InputPath,
-                                         file_types::ID OutputKind) {
-  auto BaseKey = createCompileJobBaseCacheKey(*CAS, Args);
-  if (!BaseKey) {
-    llvm::errs() << "Failed to create cache key: "
-                 << toString(BaseKey.takeError()) << "\n";
-    return "";
-  }
-
-  auto Key =
-      createCompileJobCacheKeyForOutput(*CAS, *BaseKey, InputPath, OutputKind);
-  if (!Key) {
-    llvm::errs() << "Failed to create cache key: " << toString(Key.takeError())
-                 << "\n";
-    return "";
-  }
-
-  return CAS->getID(*Key).toString();
-}
-
-std::string CachingTool::storeContent(StringRef Content) {
-  auto Result = CAS->storeFromString({}, Content);
-  if (!Result) {
-    llvm::errs() << "Failed to store to CAS: " << toString(Result.takeError())
-                 << "\n";
-    return "";
-  }
-
-  return CAS->getID(*Result).toString();
-}
-
-} // namespace cas
 } // namespace swift

tools/libSwiftScan/libSwiftScan.cpp

@@ -14,21 +14,55 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "swift/Basic/LLVMInitialize.h"
 #include "swift/Basic/InitializeSwiftModules.h"
-#include "swift/DriverTool/DriverTool.h"
+#include "swift/Basic/LLVMInitialize.h"
 #include "swift/DependencyScan/DependencyScanImpl.h"
 #include "swift/DependencyScan/DependencyScanningTool.h"
 #include "swift/DependencyScan/StringUtils.h"
-#include "swift/Frontend/CachingUtils.h"
+#include "swift/DriverTool/DriverTool.h"
+#include "swift/Frontend/CompileJobCacheKey.h"
 #include "swift/Option/Options.h"
+#include "clang/CAS/CASOptions.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/BuiltinUnifiedCASDatabases.h"
 #include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/Error.h"
+#include <memory>
 
 using namespace swift::dependencies;
-using namespace swift::cas;
+
+namespace {
+/// Helper class to manage CAS/Caching from libSwiftScan C APIs.
+class SwiftScanCAS {
+public:
+  // Compute the CASID for PCH output from invocation.
+  llvm::Expected<std::string> computeCacheKey(llvm::ArrayRef<const char *> Args,
+                                              llvm::StringRef InputPath,
+                                              swift::file_types::ID OutputKind);
+
+  // Store content into CAS.
+  llvm::Expected<std::string> storeContent(llvm::StringRef Content);
+
+  // Construct SwiftScanCAS.
+  static llvm::Expected<SwiftScanCAS *>
+  createSwiftScanCAS(llvm::StringRef Path);
+
+  static llvm::Expected<SwiftScanCAS *>
+  createSwiftScanCAS(clang::CASOptions &CASOpts);
+
+private:
+  SwiftScanCAS(std::shared_ptr<llvm::cas::ObjectStore> CAS,
+               std::shared_ptr<llvm::cas::ActionCache> Cache)
+      : CAS(CAS), Cache(Cache) {}
+
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
+  std::shared_ptr<llvm::cas::ActionCache> Cache;
+};
+} // namespace
 
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningTool, swiftscan_scanner_t)
-DEFINE_SIMPLE_CONVERSION_FUNCTIONS(CachingTool, swiftscan_cas_t)
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(clang::CASOptions, swiftscan_cas_options_t)
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SwiftScanCAS, swiftscan_cas_t)
 
 //=== Private Cleanup Functions -------------------------------------------===//
 
@@ -671,26 +705,58 @@ swiftscan_diagnostics_set_dispose(swiftscan_diagnostic_set_t* diagnostics){
 
 //=== CAS Functions ----------------------------------------------------------//
 
-swiftscan_cas_t swiftscan_cas_create(const char *path) {
-  std::string CASPath(path);
-  if (CASPath.empty())
-    CASPath = llvm::cas::getDefaultOnDiskCASPath();
+swiftscan_cas_options_t swiftscan_cas_options_create() {
+  clang::CASOptions *CASOpts = new clang::CASOptions();
+  return wrap(CASOpts);
+}
+
+void swiftscan_cas_options_dispose(swiftscan_cas_options_t options) {
+  delete unwrap(options);
+}
+
+void swiftscan_cas_options_set_ondisk_path(swiftscan_cas_options_t options,
+                                           const char *path) {
+  unwrap(options)->CASPath = path;
+}
+
+void swiftscan_cas_options_set_plugin_path(swiftscan_cas_options_t options,
+                                           const char *path) {
+  unwrap(options)->PluginPath = path;
+}
+
+bool swiftscan_cas_options_set_option(swiftscan_cas_options_t options,
+                                      const char *name, const char *value,
+                                      swiftscan_string_ref_t *error) {
+  unwrap(options)->PluginOptions.emplace_back(name, value);
+  return false;
+}
 
-  CachingTool *tool = new CachingTool(CASPath);
-  if (!tool->isValid()) {
-    delete tool;
+swiftscan_cas_t
+swiftscan_cas_create_from_options(swiftscan_cas_options_t options,
+                                  swiftscan_string_ref_t *error) {
+  clang::CASOptions *opts = unwrap(options);
+  auto cas = SwiftScanCAS::createSwiftScanCAS(*opts);
+  if (!cas) {
+    *error =
+        swift::c_string_utils::create_clone(toString(cas.takeError()).c_str());
     return nullptr;
   }
-  return wrap(tool);
+  return wrap(*cas);
 }
 
 void swiftscan_cas_dispose(swiftscan_cas_t cas) { delete unwrap(cas); }
 
-swiftscan_string_ref_t
-swiftscan_cas_store(swiftscan_cas_t cas, uint8_t *data, unsigned size) {
+swiftscan_string_ref_t swiftscan_cas_store(swiftscan_cas_t cas, uint8_t *data,
+                                           unsigned size,
+                                           swiftscan_string_ref_t *error) {
   llvm::StringRef StrContent((char*)data, size);
   auto ID = unwrap(cas)->storeContent(StrContent);
-  return swift::c_string_utils::create_clone(ID.c_str());
+  if (!ID) {
+    *error =
+        swift::c_string_utils::create_clone(toString(ID.takeError()).c_str());
+    return swift::c_string_utils::create_null();
+  }
+  return swift::c_string_utils::create_clone(ID->c_str());
 }
 
 static swift::file_types::ID
@@ -713,14 +779,62 @@ getFileTypeFromScanOutputKind(swiftscan_output_kind_t kind) {
 
 swiftscan_string_ref_t
 swiftscan_compute_cache_key(swiftscan_cas_t cas, int argc, const char **argv,
-                            const char *input, swiftscan_output_kind_t kind) {
+                            const char *input, swiftscan_output_kind_t kind,
+                            swiftscan_string_ref_t *error) {
   std::vector<const char *> Compilation;
   for (int i = 0; i < argc; ++i)
     Compilation.push_back(argv[i]);
 
   auto ID = unwrap(cas)->computeCacheKey(Compilation, input,
                                          getFileTypeFromScanOutputKind(kind));
-  return swift::c_string_utils::create_clone(ID.c_str());
+  if (!ID) {
+    *error =
+        swift::c_string_utils::create_clone(toString(ID.takeError()).c_str());
+    return swift::c_string_utils::create_null();
+  }
+  return swift::c_string_utils::create_clone(ID->c_str());
+}
+
+llvm::Expected<SwiftScanCAS *>
+SwiftScanCAS::createSwiftScanCAS(llvm::StringRef Path) {
+  clang::CASOptions Opts;
+  Opts.CASPath = Path;
+
+  return createSwiftScanCAS(Opts);
+}
+
+llvm::Expected<SwiftScanCAS *>
+SwiftScanCAS::createSwiftScanCAS(clang::CASOptions &CASOpts) {
+  auto DB = CASOpts.getOrCreateDatabases();
+  if (!DB)
+    return DB.takeError();
+
+  return new SwiftScanCAS(std::move(DB->first), std::move(DB->second));
+}
+
+llvm::Expected<std::string>
+SwiftScanCAS::computeCacheKey(llvm::ArrayRef<const char *> Args,
+                              llvm::StringRef InputPath,
+                              swift::file_types::ID OutputKind) {
+  auto BaseKey = swift::createCompileJobBaseCacheKey(*CAS, Args);
+  if (!BaseKey)
+    return BaseKey.takeError();
+
+  auto Key = swift::createCompileJobCacheKeyForOutput(*CAS, *BaseKey, InputPath,
+                                                      OutputKind);
+  if (!Key)
+    return Key.takeError();
+
+  return CAS->getID(*Key).toString();
+}
+
+llvm::Expected<std::string>
+SwiftScanCAS::storeContent(llvm::StringRef Content) {
+  auto Result = CAS->storeFromString({}, Content);
+  if (!Result)
+    return Result.takeError();
+
+  return CAS->getID(*Result).toString();
 }
 
 //=== Experimental Compiler Invocation Functions ------------------------===//