Add CLI renderer, optimize onnxruntime

.gitignore

@@ -24,5 +24,7 @@ xcschememanagement.plist
 Makefile
 *.ort
 onnxruntime*.tar.gz
-ThirdParty/onnxruntime
+ThirdParty/onnxruntime*
 /Installers/Windows/Output/
+Lib/ModelData/features_model.required_operators.config
+Lib/ModelData/features_model.required_operators.with_runtime_opt.config

CMake/CPM.cmake

@@ -0,0 +1,21 @@
+set(CPM_DOWNLOAD_VERSION 0.35.5)
+
+if(CPM_SOURCE_CACHE)
+  # Expand relative path. This is important if the provided path contains a tilde (~)
+  get_filename_component(CPM_SOURCE_CACHE ${CPM_SOURCE_CACHE} ABSOLUTE)
+  set(CPM_DOWNLOAD_LOCATION "${CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+elseif(DEFINED ENV{CPM_SOURCE_CACHE})
+  set(CPM_DOWNLOAD_LOCATION "$ENV{CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+else()
+  set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+endif()
+
+if(NOT (EXISTS ${CPM_DOWNLOAD_LOCATION}))
+  message(STATUS "Downloading CPM.cmake to ${CPM_DOWNLOAD_LOCATION}")
+  file(DOWNLOAD
+       https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake
+       ${CPM_DOWNLOAD_LOCATION}
+  )
+endif()
+
+include(${CPM_DOWNLOAD_LOCATION})

CMake/FindJUCE.cmake

@@ -0,0 +1,6 @@
+include(CPM)
+
+CPMAddPackage(
+        NAME JUCE
+        GITHUB_REPOSITORY juce-framework/JUCE
+        GIT_TAG origin/master)

CMake/FindRTNeural.cmake

@@ -0,0 +1,6 @@
+include(CPM)
+
+CPMAddPackage(
+        NAME RTNeural
+        GITHUB_REPOSITORY jatinchowdhury18/RTNeural
+        GIT_TAG origin/main)

CMakeLists.txt

@@ -2,15 +2,16 @@ cmake_minimum_required(VERSION 3.16)
 project(NeuralNotePlugin VERSION 1.1.0)
 
 # C++ settings
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
 enable_language(CXX)
 
 #Compile commands, useful for some IDEs like VS-Code
 set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
 set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
 
 #Minimum MacOS target, set globally
-set(CMAKE_OSX_DEPLOYMENT_TARGET "10.11" CACHE STRING "Minimum OS X deployment version" FORCE)
+set(CMAKE_OSX_DEPLOYMENT_TARGET "10.15" CACHE STRING "Minimum OS X deployment version" FORCE)
 
 option(UniversalBinary "Build universal binary for mac" OFF)
 option(RTNeural_Release "When CMAKE_BUILD_TYPE=Debug, overwrite it to Release for RTNeural only" OFF)
@@ -24,10 +25,13 @@ endif ()
 #static linking in Windows
 set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 
-add_subdirectory(ThirdParty/JUCE)
-add_subdirectory(ThirdParty/RTNeural)
+include(CMake/CPM.cmake)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/CMake)
+find_package(JUCE)
+find_package(RTNeural)
 
 set(BaseTargetName NeuralNote)
+set(CLITargetName NeuralNoteCLI)
 
 # Set COPY_PLUGIN_AFTER_BUILD to TRUE on Mac and to FALSE on Windows due to permission issues
 if (APPLE)
@@ -115,14 +119,15 @@ target_compile_definitions(${BaseTargetName}
         JUCE_USE_CURL=0
         JUCE_VST3_CAN_REPLACE_VST2=0)
 
-add_library(onnxruntime STATIC IMPORTED)
+add_library(onnxruntime SHARED IMPORTED)
 
 if (APPLE)
-    set_property(TARGET onnxruntime PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/ThirdParty/onnxruntime/lib/libonnxruntime.a)
+    set_property(TARGET onnxruntime PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/ThirdParty/onnxruntime-osx-arm64-1.22.0/lib/libonnxruntime.dylib)
 
 elseif (WIN32)
     set_property(TARGET onnxruntime APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE)
 
+    #TODO: Fix Windows
     set_target_properties(onnxruntime PROPERTIES
             IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX"
             IMPORTED_LOCATION_RELEASE "${CMAKE_CURRENT_LIST_DIR}/ThirdParty/onnxruntime/lib/onnxruntime.lib"
@@ -136,10 +141,8 @@ elseif (UNIX)
     set_property(TARGET onnxruntime PROPERTY IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/ThirdParty/onnxruntime/lib/libonnxruntime.a)
 endif ()
 
-
 target_include_directories("${BaseTargetName}" PRIVATE ${CMAKE_CURRENT_LIST_DIR}/ThirdParty/onnxruntime/include)
 
-
 target_link_libraries(${BaseTargetName}
         PRIVATE
         juce::juce_audio_utils
@@ -170,3 +173,37 @@ if (WIN32)
     target_include_directories(${BaseTargetName} PRIVATE ThirdParty/ASIO/common)
 endif ()
 
+juce_add_console_app(${CLITargetName}
+        PRODUCT_NAME ${CLITargetName})
+
+juce_generate_juce_header(${CLITargetName})
+
+target_sources(${CLITargetName} PRIVATE
+        NeuralNote/Source/MidiFile/MidiFileWriter.cpp
+        Lib/DSP/Resampler.cpp
+        Lib/MidiPostProcessing/NoteOptions.cpp
+        Lib/Model/BasicPitch.cpp
+        Lib/Model/Features.cpp
+        Lib/Model/Notes.cpp
+        NeuralNoteCLI/main.cpp)
+
+target_include_directories("${CLITargetName}" PRIVATE
+        ${CMAKE_CURRENT_LIST_DIR}/ThirdParty/onnxruntime/include
+        NeuralNote/Source
+        NeuralNote/Source/MidiFile
+        Lib/Model
+        Lib/Utils
+        )
+
+target_compile_definitions(${CLITargetName} PRIVATE
+        JUCE_WEB_BROWSER=0
+        JUCE_USE_CURL=0)
+
+target_link_libraries(${CLITargetName} PRIVATE
+        juce_recommended_config_flags
+        juce_recommended_lto_flags
+        juce_recommended_warning_flags
+        juce::juce_audio_utils
+        juce::juce_dsp
+        BasicPitchCNN
+        onnxruntime)

Lib/Model/Features.cpp

@@ -10,8 +10,11 @@ Features::Features()
 {
     mMemoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
 
+    mSessionOptions.SetExecutionMode(ExecutionMode::ORT_SEQUENTIAL);
+    mSessionOptions.SetIntraOpNumThreads(0);
     mSessionOptions.SetInterOpNumThreads(1);
-    mSessionOptions.SetIntraOpNumThreads(1);
+
+    mSessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL);
 
     mSession = Ort::Session(mEnv, BinaryData::features_model_ort, BinaryData::features_model_ortSize, mSessionOptions);
 }

NeuralNoteCLI/main.cpp

@@ -0,0 +1,140 @@
+#include <juce_core/juce_core.h>
+#include <juce_gui_basics/juce_gui_basics.h>
+#include <juce_audio_formats/juce_audio_formats.h>
+
+#include "BasicPitch.h"
+#include "MidiFileWriter.h"
+#include "TimeQuantizeOptions.h"
+
+std::string convertMillis(long long milliseconds) {
+    auto duration = std::chrono::milliseconds(milliseconds);
+    auto hours = std::chrono::duration_cast<std::chrono::hours>(duration);
+    duration -= hours;
+    auto minutes = std::chrono::duration_cast<std::chrono::minutes>(duration);
+    duration -= minutes;
+    auto seconds = std::chrono::duration_cast<std::chrono::seconds>(duration);
+
+    if (hours.count() > 0)
+        return std::to_string(hours.count()) + ":" +
+               std::to_string(minutes.count()) + ":" +
+               std::to_string(seconds.count());
+    else
+        return std::to_string(minutes.count()) + ":" +
+               std::to_string(seconds.count());
+}
+
+struct App : juce::JUCEApplication
+{
+    const juce::String getApplicationName() override
+    {
+        return "NeuralNoteCLI";
+    }
+
+    const juce::String getApplicationVersion() override { return "0.1"; }
+    void initialise(const juce::String& inputArgs) override
+    {
+        juce::StringArray tokens;
+        tokens.addTokens (inputArgs, " ", "\"");
+
+        if (tokens.size() != 2)
+        {
+            std::cerr << "Usage: <wav file> <out dir>"
+                      << std::endl;
+            exit(1);
+        }
+
+        // load audio passed as argument
+        std::string wav_file = tokens[0].toStdString();
+        juce::File input { wav_file };
+
+        juce::AudioFormatManager formatManager;
+        formatManager.registerBasicFormats();
+
+        std::unique_ptr<juce::FileInputStream> inputStream = input.createInputStream();
+
+        std::unique_ptr<juce::AudioBuffer<float>> buffer = std::make_unique<juce::AudioBuffer<float>>();
+
+        juce::AudioFormatReader* reader = formatManager.createReaderFor (input);
+
+        if (reader != nullptr)
+        {
+            buffer->setSize(reader->numChannels, (int)reader->lengthInSamples);
+            reader->read(buffer.get(),
+                         0,
+                         (int)reader->lengthInSamples,
+                         0,
+                         true,
+                         true);
+        }
+
+        // strip extension to make prefix for output filenames
+        std::filesystem::path output_file_prefix = wav_file;
+        output_file_prefix.replace_extension();
+
+        // output dir passed as argument
+        std::string out_dir = tokens[1].toStdString();
+
+        // Check if the output directory exists, and create it if not
+        std::filesystem::path output_dir_path(out_dir);
+        if (!std::filesystem::exists(output_dir_path))
+        {
+            std::cerr << "Directory does not exist: " << out_dir << ". Creating it."
+                      << std::endl;
+            if (!std::filesystem::create_directories(output_dir_path))
+            {
+                std::cerr << "Error: Unable to create directory: " << out_dir
+                          << std::endl;
+                return;
+            }
+        }
+        else if (!std::filesystem::is_directory(output_dir_path))
+        {
+            std::cerr << "Error: " << out_dir << " exists but is not a directory!"
+                      << std::endl;
+            return;
+        }
+
+        BasicPitch basicPitch;
+
+        float inNoteSensitivity = 0.7;
+        float inPitchSensitivity = 0.5;
+        float inMinNoteDurationMs = 50;
+
+        basicPitch.setParameters(inNoteSensitivity, inPitchSensitivity, inMinNoteDurationMs);
+
+        auto start = std::chrono::high_resolution_clock::now();
+        basicPitch.transcribeToMIDI(buffer->getWritePointer(0),
+                                    buffer->getNumSamples());
+        auto stop = std::chrono::high_resolution_clock::now();
+        auto processingTime = std::chrono::duration_cast<std::chrono::milliseconds>(stop - start);
+        std::cout << "Audio converted to MIDI in " << convertMillis(processingTime.count()) << "s" << std::endl;
+
+        auto noteEvents = basicPitch.getNoteEvents();
+
+        auto outFileName = output_file_prefix.string() + ".mid";
+
+        auto fullOutputPath = output_dir_path / outFileName;
+
+        juce::File outFile { fullOutputPath.string() };
+
+        MidiFileWriter midiFileWriter;
+        TimeQuantizeOptions::TimeQuantizeInfo quantizeInfo;
+        double bpm = 120;
+        PitchBendModes pitchBendMode = PitchBendModes::NoPitchBend;
+
+        if (!midiFileWriter.writeMidiFile(noteEvents, outFile, quantizeInfo, bpm, pitchBendMode)) {
+            std::cout << "Writing MIDI file failed" << std::endl;
+        }
+
+        delete(reader);
+
+        quit();
+    }
+
+    void shutdown() override
+    {
+
+    }
+};
+
+START_JUCE_APPLICATION(App)

ThirdParty/onnxruntime/include/coreml_provider_factory.h

@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+#pragma once
+
+#include "onnxruntime_c_api.h"
+
+// COREMLFlags are bool options we want to set for CoreML EP
+// This enum is defined as bit flags, and cannot have negative value
+// To generate an uint32_t coreml_flags for using with OrtSessionOptionsAppendExecutionProvider_CoreML below,
+//   uint32_t coreml_flags = 0;
+//   coreml_flags |= COREML_FLAG_USE_CPU_ONLY;
+enum COREMLFlags {
+  COREML_FLAG_USE_NONE = 0x000,
+
+  // Using CPU only in CoreML EP, this may decrease the perf but will provide
+  // reference output value without precision loss, which is useful for validation
+  COREML_FLAG_USE_CPU_ONLY = 0x001,
+
+  // Enable CoreML EP on subgraph
+  COREML_FLAG_ENABLE_ON_SUBGRAPH = 0x002,
+
+  // By default CoreML Execution provider will be enabled for all compatible Apple devices
+  // Enable this option will only enable CoreML EP for Apple devices with ANE (Apple Neural Engine)
+  // Please note, enable this option does not guarantee the entire model to be executed using ANE only
+  COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE = 0x004,
+
+  // Keep COREML_FLAG_MAX at the end of the enum definition
+  // And assign the last COREMLFlag to it
+  COREML_FLAG_LAST = COREML_FLAG_ONLY_ENABLE_DEVICE_WITH_ANE,
+};
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ORT_EXPORT ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CoreML,
+                          _In_ OrtSessionOptions* options, uint32_t coreml_flags);
+
+#ifdef __cplusplus
+}
+#endif

ThirdParty/onnxruntime/include/cpu_provider_factory.h

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "onnxruntime_c_api.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \param use_arena zero: false. non-zero: true.
+ */
+ORT_EXPORT
+ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CPU, _In_ OrtSessionOptions* options, int use_arena)
+ORT_ALL_ARGS_NONNULL;
+
+#ifdef __cplusplus
+}
+#endif