Handle missing data types. (#2984)

Summary:
**Changes**
- The runtime was failing if it encountered a datatype not supported by CoreML framework. The changes add support for all the datatypes that are supported by coremltools basically if `CoreMLBackend` can export a model then runtime would execute it. Complex types are not supported because `coremltools` doesn't support it.

- Improves and cleans the multiarray copying code.

- Adds portable ops to CoreML executor so that it can run a partitioned model.

**Testing**
- Tested partitioned model `coreml_stories.pte`
- Added multiarray copying tests.

Pull Request resolved: #2984

Reviewed By: kirklandsign

Differential Revision: D56003795

Pulled By: shoumikhin

fbshipit-source-id: fa1c7846f9510d68c359aed6761aedb2d10c6f46
(cherry picked from commit d731866)

Author: cymbalrush

backends/apple/coreml/CMakeLists.txt

@@ -13,6 +13,8 @@ if(NOT EXECUTORCH_ROOT)
   set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
 endif()
 
+option(COREML_BUILD_EXECUTOR_RUNNER "Build CoreML executor runner." OFF)
+
 # inmemoryfs sources
 set(INMEMORYFS_SOURCES
   runtime/inmemoryfs/inmemory_filesystem.cpp
@@ -181,6 +183,14 @@ target_link_libraries(coremldelegate
   ${SQLITE_LIBRARY}
 )
 
+if(COREML_BUILD_EXECUTOR_RUNNER)
+target_link_libraries(coremldelegate
+  PRIVATE
+  portable_ops_lib
+  portable_kernels
+)
+endif()
+
 target_compile_options(coremldelegate PRIVATE "-fobjc-arc")
 target_compile_options(coremldelegate PRIVATE "-fno-exceptions")
 

backends/apple/coreml/runtime/delegate/ETCoreMLAssetManager.mm

@@ -630,7 +630,7 @@ - (NSUInteger)_compact:(NSUInteger)sizeInBytes error:(NSError * __autoreleasing
     }
 
     if (_estimatedSizeInBytes <= sizeInBytes) {
-        return YES;
+        return _estimatedSizeInBytes;
     }
 
     std::error_code ec;

backends/apple/coreml/runtime/delegate/ETCoreMLDefaultModelExecutor.h

@@ -27,6 +27,9 @@ __attribute__((objc_subclassing_restricted)) @interface ETCoreMLDefaultModelExec
 /// The model.
 @property (readonly, strong, nonatomic) ETCoreMLModel* model;
 
+/// If set to `YES` then output backing are ignored.
+@property (readwrite, atomic) BOOL ignoreOutputBackings;
+
 @end
 
 NS_ASSUME_NONNULL_END

backends/apple/coreml/runtime/delegate/ETCoreMLDefaultModelExecutor.mm

@@ -26,6 +26,9 @@ - (instancetype)initWithModel:(ETCoreMLModel *)model {
                                               loggingOptions:(const executorchcoreml::ModelLoggingOptions& __unused)loggingOptions
                                                  eventLogger:(const executorchcoreml::ModelEventLogger* _Nullable __unused)eventLogger
                                                        error:(NSError * __autoreleasing *)error {
+    if (self.ignoreOutputBackings) {
+        predictionOptions.outputBackings = @{};
+    }
     id<MLFeatureProvider> outputs = [self.model.mlModel predictionFromFeatures:inputs
                                                                        options:predictionOptions
                                                                          error:error];

backends/apple/coreml/runtime/delegate/ETCoreMLLogging.h

@@ -7,6 +7,7 @@
 
 #import <Foundation/Foundation.h>
 
+#import <executorch/runtime/platform/log.h>
 #import <os/log.h>
 
 NS_ASSUME_NONNULL_BEGIN
@@ -48,7 +49,11 @@ typedef NS_ERROR_ENUM(ETCoreMLErrorDomain, ETCoreMLError) {
 
 /// Record the error with `os_log_error` and fills `*errorOut` with `NSError`.
 #define ETCoreMLLogErrorAndSetNSError(errorOut, errorCode, formatString, ...)                                        \
-    os_log_error(ETCoreMLErrorUtils.loggingChannel, formatString, ##__VA_ARGS__);                                    \
+    if (ET_LOG_ENABLED) {                                                                                            \
+        ET_LOG(Error, "%s", [NSString stringWithFormat:@formatString, ##__VA_ARGS__].UTF8String);                    \
+    } else {                                                                                                         \
+        os_log_error(ETCoreMLErrorUtils.loggingChannel, formatString, ##__VA_ARGS__);                                \
+    }                                                                                                                \
     if (errorOut) {                                                                                                  \
         *errorOut =                                                                                                  \
             [NSError errorWithDomain:ETCoreMLErrorDomain                                                             \
@@ -58,24 +63,31 @@ typedef NS_ERROR_ENUM(ETCoreMLErrorDomain, ETCoreMLError) {
                             }];                                                                                      \
     }
 
-/// Record the error and its underlying error with `os_log_error` and fills
-/// `*errorOut` with NSError.
+/// Record the error and its underlying error with `os_log_error` and fills `*errorOut` with `NSError`.
 #define ETCoreMLLogUnderlyingErrorAndSetNSError(errorOut, errorCode, underlyingNSError, formatString, ...) \
-    os_log_error(ETCoreMLErrorUtils.loggingChannel,                                                        \
-                 formatString ", with underlying error= %@.",                                              \
-                 ##__VA_ARGS__,                                                                            \
-                 (underlyingNSError).localizedDescription);                                                \
+    if (ET_LOG_ENABLED) {                                                                                  \
+        ET_LOG(Error, "%s", [NSString stringWithFormat:@formatString, ##__VA_ARGS__].UTF8String);          \
+    } else {                                                                                               \
+        os_log_error(ETCoreMLErrorUtils.loggingChannel,                                                    \
+                     formatString ", with underlying error= %@.",                                          \
+                     ##__VA_ARGS__,                                                                        \
+                     (underlyingNSError).localizedDescription);                                            \
+    }                                                                                                      \
     if (errorOut) {                                                                                        \
         *errorOut = [ETCoreMLErrorUtils errorWithCode:errorCode                                            \
                                       underlyingError:underlyingNSError                                    \
                                                format:@formatString, ##__VA_ARGS__];                       \
     }
 
-#define ETCoreMLLogError(error, formatString, ...)  \
-    os_log_error(ETCoreMLErrorUtils.loggingChannel, \
-                 formatString ", with error= %@.",  \
-                 ##__VA_ARGS__,                     \
-                 (error).localizedDescription);
+#define ETCoreMLLogError(error, formatString, ...)                                                \
+    if (ET_LOG_ENABLED) {                                                                         \
+        ET_LOG(Error, "%s", [NSString stringWithFormat:@formatString, ##__VA_ARGS__].UTF8String); \
+    } else {                                                                                      \
+        os_log_error(ETCoreMLErrorUtils.loggingChannel,                                           \
+                     formatString ", with error= %@.",                                            \
+                     ##__VA_ARGS__,                                                               \
+                     (error).localizedDescription);                                               \
+    }
 
 
 #pragma clang diagnostic pop

backends/apple/coreml/runtime/delegate/ETCoreMLModel.h

@@ -6,13 +6,18 @@
 // Please refer to the license found in the LICENSE file in the root directory of the source tree.
 
 #import <CoreML/CoreML.h>
+#import <vector>
 
 NS_ASSUME_NONNULL_BEGIN
 
 @class ETCoreMLAsset;
 
+namespace executorchcoreml {
+class MultiArray;
+}
+
 /// Represents a ML model, the class is a thin wrapper over `MLModel` with additional properties.
-@interface ETCoreMLModel : NSObject
+__attribute__((objc_subclassing_restricted)) @interface ETCoreMLModel : NSObject
 
 - (instancetype)init NS_UNAVAILABLE;
 
@@ -31,6 +36,12 @@ NS_ASSUME_NONNULL_BEGIN
                     orderedOutputNames:(NSOrderedSet<NSString*>*)orderedOutputNames
                                  error:(NSError* __autoreleasing*)error NS_DESIGNATED_INITIALIZER;
 
+- (nullable NSArray<MLMultiArray*>*)prepareInputs:(const std::vector<executorchcoreml::MultiArray>&)inputs
+                                            error:(NSError* __autoreleasing*)error;
+
+- (nullable NSArray<MLMultiArray*>*)prepareOutputBackings:(const std::vector<executorchcoreml::MultiArray>&)outputs
+                                                    error:(NSError* __autoreleasing*)error;
+
 /// The underlying MLModel.
 @property (strong, readonly, nonatomic) MLModel* mlModel;