Disable ability for OVEP to fall back to CPU inference if compilation fails for NPU

onnxruntime/core/providers/openvino/exported_symbols.lst

@@ -1 +1,3 @@
 _GetProvider
+_CreateEpFactories
+_ReleaseEpFactory

onnxruntime/core/providers/openvino/openvino_provider_factory.cc

@@ -403,6 +403,18 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
     return ov_ep;
   }
 
+  // This is called during session creation when AppendExecutionProvider_V2 is used.
+  // This one is called because ParseProviderInfo / ParseConfigOptions, etc. are already
+  // performed in CreateIExecutionProvider, and so provider_info_ has already been populated.
+  std::unique_ptr<IExecutionProvider> CreateProvider_V2(const OrtSessionOptions& /*session_options*/,
+                                                        const OrtLogger& session_logger) {
+    ProviderInfo provider_info = provider_info_;
+    auto ov_ep = std::make_unique<OpenVINOExecutionProvider>(provider_info, shared_context_);
+    ov_ep->SetLogger(reinterpret_cast<const logging::Logger*>(&session_logger));
+    return ov_ep;
+  }
+
+
  private:
   ProviderInfo provider_info_;
   std::shared_ptr<SharedContext> shared_context_;
@@ -433,6 +445,119 @@ struct OpenVINO_Provider : Provider {
     return std::make_shared<OpenVINOProviderFactory>(pi, SharedContext::Get());
   }
 
+  Status CreateIExecutionProvider(const OrtHardwareDevice* const* /*devices*/,
+                                  const OrtKeyValuePairs* const* ep_metadata,
+                                  size_t num_devices,
+                                  ProviderOptions& provider_options,
+                                  const OrtSessionOptions& session_options,
+                                  const OrtLogger& logger,
+                                  std::unique_ptr<IExecutionProvider>& ep) override {
+    // Check if no devices are provided
+    if (num_devices == 0) {
+      return Status(common::ONNXRUNTIME, ORT_EP_FAIL, "No devices provided to CreateIExecutionProvider");
+    }
+
+    // For provider options that we don't support directly but are still supported through load_config,
+    // give some specific guidance & example about how to make use of the option through load_config.
+    const std::vector<std::pair<std::string, std::string>> block_and_advise_entries = {
+      {"cache_dir", "\"CACHE_DIR\": \"<filesystem_path>\""},
+      {"precision", "\"INFERENCE_PRECISION_HINT\": \"F32\""},
+      {"num_of_threads", "\"INFERENCE_NUM_THREADS\": \"1\""},
+      {"num_streams", "\"NUM_STREAMS\": \"1\""},
+      {"model_priority", "\"MODEL_PRIORITY\": \"LOW\""},
+      {"enable_opencl_throttling", "\"GPU\": {\"PLUGIN_THROTTLE\": \"1\"}"},
+      {"enable_qdq_optimizer", "\"NPU\": {\"NPU_QDQ_OPTIMIZATION\": \"YES\"}"}
+    };
+
+    for (auto& block_and_advise_entry : block_and_advise_entries) {
+      if (provider_options.find(block_and_advise_entry.first) != provider_options.end()) {
+        std::string message = "OpenVINO EP: Option '" + block_and_advise_entry.first +
+                              "' cannot be set when using AppendExecutionProvider_V2. " +
+                              "It can instead be enabled by a load_config key / value pair. For example: " +
+                              block_and_advise_entry.second;
+        return Status(common::ONNXRUNTIME, ORT_INVALID_ARGUMENT, message);
+      }
+    }
+
+    // For the rest of the disallowed provider options, give a generic error message.
+    const std::vector<std::string> blocked_provider_keys = {
+      "device_type", "device_id", "device_luid", "context", "disable_dynamic_shapes"};
+
+    for (const auto& key : blocked_provider_keys) {
+      if (provider_options.find(key) != provider_options.end()) {
+        return Status(common::ONNXRUNTIME, ORT_INVALID_ARGUMENT,
+                      "OpenVINO EP: Option '" + key + "' cannot be set when using AppendExecutionProvider_V2.");
+      }
+    }
+
+    const char* ov_device_key = "ov_device";
+    const char* ov_meta_device_key = "ov_meta_device";
+
+    // Create a unique list of ov_devices that were passed in.
+    std::unordered_set<std::string_view> unique_ov_devices;
+    std::vector<std::string_view> ordered_unique_ov_devices;
+    for (size_t i = 0; i < num_devices; ++i) {
+      const auto& device_meta_data = ep_metadata[i];
+      auto ov_device_it = device_meta_data->Entries().find(ov_device_key);
+      if (ov_device_it == device_meta_data->Entries().end()) {
+        return Status(common::ONNXRUNTIME, ORT_INVALID_ARGUMENT, "OpenVINO EP device metadata not found.");
+      }
+      auto &ov_device = ov_device_it->second;
+
+      // Add to ordered_unique only if not already present
+      if (unique_ov_devices.insert(ov_device).second) {
+        ordered_unique_ov_devices.push_back(ov_device);
+      }
+    }
+
+    std::string ov_meta_device_type = "NONE";
+    {
+      auto ov_meta_device_it = ep_metadata[0]->Entries().find(ov_meta_device_key);
+      if (ov_meta_device_it != ep_metadata[0]->Entries().end()) {
+        ov_meta_device_type = ov_meta_device_it->second;
+      }
+    }
+
+    bool is_meta_device_factory = (ov_meta_device_type != "NONE");
+
+    if (ordered_unique_ov_devices.size() > 1 && !is_meta_device_factory) {
+      LOGS_DEFAULT(WARNING) << "[OpenVINO EP] Multiple devices were specified that are not OpenVINO meta devices. Using first ov_device only: " << ordered_unique_ov_devices.at(0);
+      ordered_unique_ov_devices.resize(1);  // Use only the first device if not a meta device factory
+    }
+
+    std::string ov_device_string;
+    if (is_meta_device_factory) {
+      // Build up a meta device string based on the devices that are passed in. E.g. AUTO:NPU,GPU.0,CPU
+      ov_device_string = ov_meta_device_type;
+      ov_device_string += ":";
+    }
+
+    bool prepend_comma = false;
+    for (const auto& ov_device : ordered_unique_ov_devices) {
+      if (prepend_comma) {
+        ov_device_string += ",";
+      }
+      ov_device_string += ov_device;
+      prepend_comma = true;
+    }
+
+    provider_options["device_type"] = ov_device_string;
+
+    // Parse provider info with the device type
+    ProviderInfo pi;
+    const auto& config_options = session_options.GetConfigOptions();
+    ParseProviderInfo(provider_options, &config_options, pi);
+    ParseConfigOptions(pi);
+
+    // Force-disable the ability for NPU to fall back to CPU.
+    pi.so_disable_cpu_ep_fallback = true;
+
+    // Create and return the execution provider
+    auto factory = std::make_unique<OpenVINOProviderFactory>(pi, SharedContext::Get());
+    ep = factory->CreateProvider_V2(session_options, logger);
+    return Status::OK();
+  }
+
   void Initialize() override {
   }
 

onnxruntime/core/providers/openvino/ov_factory.cc

@@ -0,0 +1,182 @@
+// Copyright (C) Intel Corporation
+// Licensed under the MIT License
+
+#include <memory>
+#include <map>
+#include <string>
+#include <algorithm>
+#include <vector>
+#include <ranges>
+#include <format>
+
+#define ORT_API_MANUAL_INIT
+#include "onnxruntime_cxx_api.h"
+#undef ORT_API_MANUAL_INIT
+
+#include "onnxruntime_c_api.h"
+#include "ov_factory.h"
+#include "openvino/openvino.hpp"
+#include "ov_interface.h"
+
+using namespace onnxruntime::openvino_ep;
+using ov_core_singleton = onnxruntime::openvino_ep::WeakSingleton<ov::Core>;
+
+static void InitCxxApi(const OrtApiBase& ort_api_base) {
+  static std::once_flag init_api;
+  std::call_once(init_api, [&]() {
+    const OrtApi* ort_api = ort_api_base.GetApi(ORT_API_VERSION);
+    Ort::InitApi(ort_api);
+  });
+}
+
+OpenVINOEpPluginFactory::OpenVINOEpPluginFactory(ApiPtrs apis, const std::string& ov_metadevice_name, std::shared_ptr<ov::Core> core)
+    : ApiPtrs{apis},
+      ep_name_(ov_metadevice_name.empty() ? provider_name_ : std::string(provider_name_) + "." + ov_metadevice_name),
+      device_type_(ov_metadevice_name),
+      ov_core_(std::move(core)) {
+  OrtEpFactory::GetName = GetNameImpl;
+  OrtEpFactory::GetVendor = GetVendorImpl;
+  OrtEpFactory::GetVendorId = GetVendorIdImpl;
+  OrtEpFactory::GetSupportedDevices = GetSupportedDevicesImpl;
+  OrtEpFactory::GetVersion = GetVersionImpl;
+  OrtEpFactory::CreateDataTransfer = CreateDataTransferImpl;
+
+  ort_version_supported = ORT_API_VERSION;  // Set to the ORT version we were compiled with.
+}
+
+const std::vector<std::string>& OpenVINOEpPluginFactory::GetOvDevices() {
+  static std::vector<std::string> devices = ov_core_singleton::Get()->get_available_devices();
+  return devices;
+}
+
+const std::vector<std::string>& OpenVINOEpPluginFactory::GetOvMetaDevices() {
+  static std::vector<std::string> virtual_devices = [ov_core = ov_core_singleton::Get()] {
+    std::vector<std::string> supported_virtual_devices{};
+    for (const auto& meta_device : known_meta_devices_) {
+      try {
+        ov_core->get_property(meta_device, ov::supported_properties);
+        supported_virtual_devices.push_back(meta_device);
+      } catch (ov::Exception&) {
+        // meta device isn't supported.
+      }
+    }
+    return supported_virtual_devices;
+  }();
+
+  return virtual_devices;
+}
+
+OrtStatus* OpenVINOEpPluginFactory::GetSupportedDevices(const OrtHardwareDevice* const* devices,
+                                                        size_t num_devices,
+                                                        OrtEpDevice** ep_devices,
+                                                        size_t max_ep_devices,
+                                                        size_t* p_num_ep_devices) {
+  size_t& num_ep_devices = *p_num_ep_devices;
+
+  // Create a map for device type mapping
+  static const std::map<OrtHardwareDeviceType, std::string> ort_to_ov_device_name = {
+      {OrtHardwareDeviceType::OrtHardwareDeviceType_CPU, "CPU"},
+      {OrtHardwareDeviceType::OrtHardwareDeviceType_GPU, "GPU"},
+      {OrtHardwareDeviceType::OrtHardwareDeviceType_NPU, "NPU"},
+  };
+
+  for (size_t i = 0; i < num_devices && num_ep_devices < max_ep_devices; ++i) {
+    const OrtHardwareDevice& device = *devices[i];
+    if (ort_api.HardwareDevice_VendorId(&device) != vendor_id_) {
+      // Not an Intel Device.
+      continue;
+    }
+
+    auto device_type = ort_api.HardwareDevice_Type(&device);
+    auto device_it = ort_to_ov_device_name.find(device_type);
+    if (device_it == ort_to_ov_device_name.end()) {
+      // We don't know about this device type
+      continue;
+    }
+
+    const auto& ov_device_type = device_it->second;
+    std::string ov_device_name;
+    auto get_pci_device_id = [&](const std::string& ov_device) {
+      try {
+        ov::device::PCIInfo pci_info = ov_core_->get_property(ov_device, ov::device::pci_info);
+        return pci_info.device;
+      } catch (ov::Exception&) {
+        return 0u;  // If we can't get the PCI info, we won't have a device ID.
+      }
+    };
+
+    auto filtered_devices = GetOvDevices(ov_device_type);
+    auto matched_device = filtered_devices.begin();
+    if (filtered_devices.size() > 1 && device_type == OrtHardwareDeviceType::OrtHardwareDeviceType_GPU) {
+      // If there are multiple devices of the same type, we need to match by device ID.
+      matched_device = std::find_if(filtered_devices.begin(), filtered_devices.end(), [&](const std::string& ov_device) {
+        uint32_t ort_device_id = ort_api.HardwareDevice_DeviceId(&device);
+        return ort_device_id == get_pci_device_id(ov_device);
+      });
+    }
+
+    if (matched_device == filtered_devices.end()) {
+      // We didn't find a matching OpenVINO device for the OrtHardwareDevice.
+      continue;
+    }
+
+    // these can be returned as nullptr if you have nothing to add.
+    OrtKeyValuePairs* ep_metadata = nullptr;
+    OrtKeyValuePairs* ep_options = nullptr;
+    ort_api.CreateKeyValuePairs(&ep_metadata);
+    ort_api.AddKeyValuePair(ep_metadata, ov_device_key_, matched_device->c_str());
+
+    if (IsMetaDeviceFactory()) {
+      ort_api.AddKeyValuePair(ep_metadata, ov_meta_device_key_, device_type_.c_str());
+    }
+
+    // Create EP device
+    auto* status = ort_api.GetEpApi()->CreateEpDevice(this, &device, ep_metadata, ep_options,
+                                                      &ep_devices[num_ep_devices++]);
+
+    ort_api.ReleaseKeyValuePairs(ep_metadata);
+    ort_api.ReleaseKeyValuePairs(ep_options);
+
+    if (status != nullptr) {
+      return status;
+    }
+  }
+
+  return nullptr;
+}
+
+extern "C" {
+//
+// Public symbols
+//
+OrtStatus* CreateEpFactories(const char* /*registration_name*/, const OrtApiBase* ort_api_base,
+                             OrtEpFactory** factories, size_t max_factories, size_t* num_factories) {
+  InitCxxApi(*ort_api_base);
+  const ApiPtrs api_ptrs{Ort::GetApi(), Ort::GetEpApi(), Ort::GetModelEditorApi()};
+
+  // Get available devices from OpenVINO
+  auto ov_core = ov_core_singleton::Get();
+  std::vector<std::string> supported_factories = {""};
+  const auto& meta_devices = OpenVINOEpPluginFactory::GetOvMetaDevices();
+  supported_factories.insert(supported_factories.end(), meta_devices.begin(), meta_devices.end());
+
+  const size_t required_factories = supported_factories.size();
+  if (max_factories < required_factories) {
+    return Ort::Status(std::format("Not enough space to return EP factories. Need at least {} factories.", required_factories).c_str(), ORT_INVALID_ARGUMENT);
+  }
+
+  size_t factory_index = 0;
+  for (const auto& device_name : supported_factories) {
+    // Create a factory for this specific device
+    factories[factory_index++] = new OpenVINOEpPluginFactory(api_ptrs, device_name, ov_core);
+  }
+
+  *num_factories = factory_index;
+  return nullptr;
+}
+
+OrtStatus* ReleaseEpFactory(OrtEpFactory* factory) {
+  delete static_cast<OpenVINOEpPluginFactory*>(factory);
+  return nullptr;
+}
+}