Refactor the code for PR

Author: preetha-intel

onnxruntime/core/providers/openvino/backend_manager.cc

@@ -77,8 +77,15 @@ BackendManager::BackendManager(SessionContext& session_context,
   ptr_stream_t model_stream;
   std::unique_ptr<onnx::ModelProto> model_proto;
   if (subgraph_context_.is_ep_ctx_graph) {
-    std::cout << " inside is_ep_ctx_graph " << std::endl;
-    std::string model_name = onnxruntime::openvino_ep::BackendManager::stripAfterFirstDot(session_context_.onnx_model_path_name.filename().string());
+    std::string filename;
+    if (!session_context_.so_context_file_path.empty()) {
+      filename = session_context_.so_context_file_path.filename().string();
+    } else if (!session_context_.onnx_model_path_name.empty()) {
+      filename = session_context_.onnx_model_path_name.filename().string();
+    } else {
+      ORT_THROW("Either Session_options ep.context_file_path or model path must be specified");
+    }
+    std::string model_name = onnxruntime::openvino_ep::BackendManager::stripAfterFirstDot(filename);
     auto subgraph_name = model_name + "_" + subgraph_context_.subgraph_name;
     model_stream = ep_ctx_handle_.GetModelBlobStream(shared_context_,
                                                      session_context_.so_context_file_path,
@@ -103,9 +110,9 @@ BackendManager::BackendManager(SessionContext& session_context,
       if (!sw.mapped_weights) {
         sw.mapped_weights = std::make_unique<SharedContext::SharedWeights::WeightsFile>(weight_filename);
       }
-      std::cout << " Call createOVTensors in backend_manager.cc" << std::endl;
       backend_utils::CreateOVTensors(session_context_.device_type, sw.metadata, *sw.mapped_weights);
-      std::cout << " create OVTensors successful " << std::endl;
+    } else {
+      ORT_THROW(" External weight file is not found ");
     }
   }
 
@@ -207,11 +214,16 @@ BackendManager::BackendManager(SessionContext& session_context,
 }
 
 std::string BackendManager::stripAfterFirstDot(std::string filename) {
-  size_t dotPos = filename.find('.');  // Find first dot
-  if (dotPos == std::string::npos) {
+  size_t dotPos = filename.find('.');     // Find first dot
+  size_t ctxPos = filename.find("_ctx");  // Find first dot
+  if (dotPos == std::string::npos && ctxPos == std::string::npos) {
     return filename;  // No dot found, return full filename
   }
-  return filename.substr(0, dotPos);  // Return everything before first dot
+  if (dotPos != std::string::npos)
+    filename = filename.substr(0, dotPos);  // strip everything after first dot
+  if (ctxPos != std::string::npos)
+    filename = filename.substr(0, ctxPos);  // strip everything after _ctx
+  return filename;
 }
 
 // Call EPContext model exporter here if the provider option for exporting
@@ -227,36 +239,33 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie
     ORT_THROW(exception_str);
   }
 
-  std::cout << " inside export compiled model " << std::endl;
-
   // If embed_mode, then pass on the serialized blob
   // If not embed_mode, dump the blob here and only pass on the path to the blob
   std::string model_blob_str;
   auto compiled_model = concrete_backend_->GetOVCompiledModel();
   if (session_context_.so_share_ep_contexts) {
     std::ostringstream model_blob_stream;
     compiled_model.export_model(model_blob_stream);
-    std::cout << " inside export compiled model - share ep contexts" << std::endl;
 
-    // std::ofstream file(metadata_filename, std::ios::app| std::ios::binary);
-    // std::cout << " write to metadata bin - " << metadata_filename << std::endl;
     auto& subgraph_metadata = shared_context_.shared_weights.subgraph_metadata;
-    std::string model_name = onnxruntime::openvino_ep::BackendManager::stripAfterFirstDot(session_context_.onnx_model_path_name.filename().string());
+    std::string filename = "";
+    if (!session_context_.so_context_file_path.empty()) {
+      filename = session_context_.so_context_file_path.filename().string();
+    } else if (!session_context_.onnx_model_path_name.empty()) {
+      filename = session_context_.onnx_model_path_name.filename().string();
+    } else {
+      ORT_THROW("Either Session_options ep.context_file_path or model path must be specified");
+    }
+    std::string model_name = onnxruntime::openvino_ep::BackendManager::stripAfterFirstDot(filename);
     auto subgraph_name = model_name + "_" + subgraph_context_.subgraph_name;
     sw::SubgraphMetadata::Map::key_type key{subgraph_name};
     sw::SubgraphMetadata::Map::mapped_type value{};
 
     auto& bin_file = shared_context_.shared_weights.shared_bin_file.bin_file_;
-    std::cout << " subgraph name " << subgraph_name << "key = " << key.name << " For bin write " << std::endl;
     if (!subgraph_metadata.contains(key) && bin_file.is_open()) {
-      // std::cout << "Current offset before "<< subgraph_context_.subgraph_name << "  = " << bin_file.tellp() << std::endl;
-      value.epctx_offset = bin_file.tellp();
-      std::cout << " bin file location for writing subgraph = " << bin_file.tellp() << std::endl;
+      value.epctx_offset = static_cast<uint64_t>(bin_file.tellp());
       bin_file << model_blob_stream.str();
-      // compiled_model.export_model(bin_file);
-      // std::cout << "Current offset after "<< subgraph_context_.subgraph_name << "  = " << bin_file.tellp() << std::endl;
-      value.epctx_length = static_cast<size_t>(static_cast<std::streamoff>(bin_file.tellp()) - value.epctx_offset);
-      // std::cout << "Key = " << key.name << " Offset = " << value.epctx_offset << " , length = " << value.epctx_length << std::endl;
+      value.epctx_length = static_cast<size_t>(static_cast<uint64_t>(bin_file.tellp()) - value.epctx_offset);
       subgraph_metadata.emplace(key, std::move(value));
     }
 

onnxruntime/core/providers/openvino/backend_utils.cc

@@ -303,33 +303,23 @@ void CreateOVTensors(const std::string& device_name,
                      SharedContext::SharedWeights::Metadata::Map& metadata_map,
                      SharedContext::SharedWeights::WeightsFile& weights) {
   for (auto& [key, value] : metadata_map) {
-    // std::cout << " Key = " << key.name << std::endl;
     if (value.tensor) {
-      // std::cout << " Value already present for key = " << key.name << std::endl;
       continue;
     }
 
     // Get element data type
-    // std::cout << " value element type = " << value.element_type << std::endl;
     auto onnx_element_type = (ONNX_NAMESPACE::TensorProto_DataType)value.element_type;
 
     ov::element::Type ov_elementType = GetOpenVINOElementType(onnx_element_type);  // Map to OpenVINO data type
-    // std::cout << "value dimensions = " << std::endl;
-    // for (auto dim:value.dimensions){
-    //   std::cout << dim << std::endl;
-    // }
+
     // Create OpenVINO Tensor
     if (device_name == "NPU") {
       // Use remote tensors
       auto npu_context = OVCore::Get()->core.get_default_context("NPU").as<ov::intel_npu::level_zero::ZeroContext>();
       auto&& remote_tensor = npu_context.create_l0_host_tensor(ov_elementType, value.dimensions, ov::intel_npu::TensorType::INPUT);
-      // std::cout << " Remote tensor created " << std::endl;
       // Copy data to remote tensor
-      // std::cout << " value size = " << value.size << std::endl;
       weights.load_weights(value.data_offset, remote_tensor.get(), value.size);
       value.tensor = std::make_shared<ov::Tensor>(remote_tensor);
-      // std::cout << " value tensor created " << std::endl;
-
     } else {
       // Use vanilla tensors
       value.tensor = std::make_shared<ov::Tensor>(ov_elementType, value.dimensions);

onnxruntime/core/providers/openvino/backends/basic_backend.cc

@@ -116,7 +116,6 @@ BasicBackend::BasicBackend(std::unique_ptr<ONNX_NAMESPACE::ModelProto>& model_pr
           ov_model, hw_target, device_config, subgraph_context_.subgraph_name);
     }
 #endif
-    std::cout << " loaded model to the plugin " << std::endl;
     LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin";
   } catch (const char* msg) {
     ORT_THROW(msg);
@@ -128,14 +127,11 @@ BasicBackend::BasicBackend(std::unique_ptr<ONNX_NAMESPACE::ModelProto>& model_pr
   if (session_context_.so_share_ep_contexts) {
     initializer = [&metadata](OVInferRequestPtr ir_ptr) {
       const auto input_count = ir_ptr->GetNumInputs();
-      std::cout << " ov ir input count = " << input_count << std::endl;
       for (auto i = 0u; i < input_count; i++) {
         using Key = SharedContext::SharedWeights::Metadata::Key;
         const auto tensor_key = Key{ir_ptr->GetInputTensorName(i)};
         if (metadata.contains(tensor_key)) {
           auto& value = metadata.at(tensor_key);
-          // ORT_ENFORCE(value.tensor->get_byte_size() == value.size, "Unexpected tensor size mismatch");
-          std::cout << " value tensor is set with shape = " << value.tensor->get_byte_size() << " input size from metadata = " << value.size << std::endl;
           ir_ptr->SetTensor(tensor_key.name, value.tensor);
         }
       }

onnxruntime/core/providers/openvino/contexts.h

@@ -11,6 +11,7 @@
 #include <filesystem>
 #include <memory>
 #include "core/common/common.h"
+#include "core/providers/shared_library/provider_api.h"
 #include "core/providers/openvino/ov_interface.h"
 
 namespace onnxruntime {
@@ -27,12 +28,12 @@ class SharedContext : public WeakSingleton<SharedContext> {
   struct SharedWeights {
     struct Header {
       uint32_t bin_version = 1;
-      long footer_offset = 0;
+      uint64_t footer_offset = 0;
     } header_;
     struct Footer {
-      long subgraph_offset;
+      uint64_t subgraph_offset;
       size_t subgraph_length;
-      long metadata_offset;
+      uint64_t metadata_offset;
       size_t metadata_length;
     } footer_;
 
@@ -57,7 +58,7 @@ class SharedContext : public WeakSingleton<SharedContext> {
       using Map = std::unordered_map<Key, Value, Hash>;
       void writeMetadataToBinaryFile(SharedContext& shared_context, const Metadata::Map& metadata);
       void readMetadataFromBinaryFile(SharedContext& shared_context, Metadata::Map& metadata);
-    };
+    } metadata_;
 
     struct SubgraphMetadata {
       struct Key {
@@ -70,15 +71,15 @@ class SharedContext : public WeakSingleton<SharedContext> {
         }
       };
       struct Value {
-        long epctx_offset;
+        uint64_t epctx_offset;
         size_t epctx_length;
       };
       using Map = std::unordered_map<Key, Value, Hash>;
       void writeSubgraphDataToBinaryFile(SharedContext& shared_context,
                                          const SubgraphMetadata::Map& subgraph_metadata);
       void readSubgraphDataFromBinaryFile(SharedContext& shared_context,
                                           SubgraphMetadata::Map& subgraph_metadata);
-    };
+    } subgraph_metadata_;
 
     struct WeightsFile {
       ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(WeightsFile);
@@ -138,9 +139,7 @@ class SharedContext : public WeakSingleton<SharedContext> {
 
     fs::path external_weight_filename;
     std::unique_ptr<WeightsFile> mapped_weights;
-    Metadata metadata_;
     Metadata::Map metadata;
-    SubgraphMetadata subgraph_metadata_;
     SubgraphMetadata::Map subgraph_metadata;
   } shared_weights;
 };

onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc

@@ -124,32 +124,24 @@ std::unique_ptr<std::istream> EPCtxHandler::GetModelBlobStream(SharedContext& sh
     }
     blob_filepath = blob_filepath.parent_path() / ep_cache_context;
     ORT_ENFORCE(std::filesystem::exists(blob_filepath), "Blob file not found: ", blob_filepath.string());
-    std::cout << " blob_filepath " << blob_filepath.filename().string() << std::endl;
-    std::cout << " shared bin filename = " << shared_context_.shared_weights.shared_bin_file.shared_bin_filename.filename().string() << std::endl;
     if (blob_filepath == shared_context_.shared_weights.shared_bin_file.shared_bin_filename) {
       LOGS_DEFAULT(VERBOSE) << "[OpenVINO EP] Read blob from Shared bin file - " << blob_filepath;
       auto& sb = shared_context_.shared_weights.shared_bin_file;
       // check if size of bin file is greater than the header as it gets written at the begining
       ORT_ENFORCE(sb.bin_size_ > 8, " Bin file is empty. Regenerate the epctx model. Bin file path : ", blob_filepath.string());
       auto subgraph_metadata = shared_context_.shared_weights.subgraph_metadata;
       using Key = SharedContext::SharedWeights::SubgraphMetadata::Key;
-      std::cout << " subgraph name = " << subgraph_name << std::endl;
       const auto subgraph_key = Key{subgraph_name};
       auto it = subgraph_metadata.find(subgraph_key);
       if (it != subgraph_metadata.end()) {
         auto& value = it->second;
-        std::cout << " value.epctx_offset =  " << value.epctx_offset << std::endl;
-        std::cout << " value.epctx_length =  " << value.epctx_length << std::endl;
-        std::cout << " sb.bin_size_ =  " << sb.bin_size_ << std::endl;
-
         if (value.epctx_offset < sb.bin_size_ && value.epctx_length <= sb.bin_size_ &&
             (value.epctx_offset <= sb.bin_size_ - value.epctx_length)) {
           sb.bin_file_.seekg(value.epctx_offset);             // Move to the specified offset
           std::string buffer(value.epctx_length, '\0');       // preallocate space
           sb.bin_file_.read(&buffer[0], value.epctx_length);  // Read the specified length
           // Adjust string size in case of a short read
           buffer.resize(sb.bin_file_.gcount());
-          std::cout << " Read epctx into stream " << std::endl;
           result.reset((std::istream*)new std::istringstream(buffer));
         }
       }

onnxruntime/core/providers/openvino/onnx_ctx_model_helper.h

@@ -44,27 +44,5 @@ class EPCtxHandler {
   std::unique_ptr<Model> epctx_model_;
   const logging::Logger& logger_;
 };
-
-// class LimitedFileStreambuf : public std::streambuf {
-// private:
-//     std::fstream& file; // Reference to the existing file stream
-//     long start, end; // Start and end positions
-
-// protected:
-//     int_type underflow() override {
-//         if (file.tellg() >= end || file.eof())
-//             return traits_type::eof(); // Stop reading if we reach the limit
-
-//         return file.get(); // Read next character directly from the file
-//     }
-
-// public:
-//     LimitedFileStreambuf(std::fstream& bin_file_, long start, long end)
-//         : file(bin_file_), start(start), end(end) {
-//         file.clear(); // Clear error flags in case of previous reads
-//         file.seekg(start); // Move file pointer to the start position
-//     }
-// };
-
 }  // namespace openvino_ep
 }  // namespace onnxruntime