Input standardization 2 (#41)

* Upload Input Inspection and Standardization 2

Maintain handle to canonical input asset inside UploadInput

If input standardized, standardized input URL is passed to UploadInfo
instead of the original input URL used for initializer

Note: SDK probably needs to re-export a high quality asset anyway so
possibly need a bridging status

Add dedicated internal initializer for MuxUpload error with unknown error code

Store all upload-related options in UploadInfo

Disable input standardization when running tests

Add inspection logic

Request local and remote assets

Status -> TransportStatus

Rely on input status in MuxUpload computed property getters

Expose progress from internal upload state if available

Track transport status inside of UploadInput

Transport status start time optional

Add TransportStatus docs

Inspect and check video track frame rate

Remove unused file

Migrate tests for MuxUpload input result handler

Author: andrewjl-mux

Sources/MuxUploadSDK/InputInspection/UploadInputFormatInspectionResult.swift

@@ -0,0 +1,44 @@
+//
+//  UploadInputFormatInspectionResult.swift
+//
+
+import AVFoundation
+import Foundation
+
+enum UploadInputFormatInspectionResult {
+
+    enum NonstandardInputReason {
+        case videoCodec
+        case audioCodec
+        case videoGOPSize
+        case videoFrameRate
+        case videoResolution
+        case videoBitrate
+        case pixelAspectRatio
+        case videoEditList
+        case audioEditList
+        case unexpectedMediaFileParameters
+        case unsupportedPixelFormat
+    }
+
+    case inspectionFailure
+    case standard
+    case nonstandard([NonstandardInputReason])
+
+    var isStandard: Bool {
+        if case Self.standard = self {
+            return true
+        } else {
+            return false
+        }
+    }
+
+    var nonstandardInputReasons: [NonstandardInputReason]? {
+        if case Self.nonstandard(let nonstandardInputReasons) = self {
+            return nonstandardInputReasons
+        } else {
+            return nil
+        }
+    }
+
+}

Sources/MuxUploadSDK/InputInspection/UploadInputInspectionWorker.swift

@@ -1,12 +1,122 @@
 //
 //  UploadInputInspector.swift
-//
+//  
 
 import AVFoundation
+import CoreMedia
 import Foundation
 
-class UploadInputInspector {
-    
+protocol UploadInputInspector {
+    func performInspection(
+        sourceInput: AVAsset,
+        completionHandler: @escaping (UploadInputFormatInspectionResult) -> ()
+    )
 }
 
+class AVFoundationUploadInputInspector: UploadInputInspector {
+
+    static let shared = AVFoundationUploadInputInspector()
+
+    func performInspection(
+        sourceInput: AVAsset,
+        completionHandler: @escaping (UploadInputFormatInspectionResult) -> ()
+    ) {
+        // TODO: Eventually load audio tracks too
+        if #available(iOS 15, *) {
+            sourceInput.loadTracks(
+                withMediaType: .video
+            ) { tracks, error in
+                if error != nil {
+                    completionHandler(.inspectionFailure)
+                    return
+                }
+
+                if let tracks {
+                    self.inspect(
+                        tracks: tracks,
+                        completionHandler: completionHandler
+                    )
+                }
+            }
+        } else {
+            sourceInput.loadValuesAsynchronously(
+                forKeys: ["tracks"]
+            ) {
+                // Non-blocking if "tracks" is already loaded
+                let tracks = sourceInput.tracks(
+                    withMediaType: .video
+                )
+                self.inspect(
+                    tracks: tracks,
+                    completionHandler: completionHandler
+                )
+            }
+        }
+
+    }
+
+    func inspect(
+        tracks: [AVAssetTrack],
+        completionHandler: @escaping (UploadInputFormatInspectionResult) -> ()
+    ) {
+        switch tracks.count {
+        case 0:
+            // Nothing to inspect, therefore nothing to standardize
+            // declare as already standard
+            completionHandler(.standard)
+        case 1:
+            if let track = tracks.first {
+                track.loadValuesAsynchronously(
+                    forKeys: [
+                        "formatDescriptions",
+                        "nominalFrameRate"
+                    ]
+                ) {
+                    guard let formatDescriptions = track.formatDescriptions as? [CMFormatDescription] else {
+                        completionHandler(.inspectionFailure)
+                        return
+                    }
+
+                    guard let formatDescription = formatDescriptions.first else {
+                        completionHandler(.inspectionFailure)
+                        return
+                    }
+
+                    var nonStandardReasons: [UploadInputFormatInspectionResult.NonstandardInputReason] = []
 
+                    let videoDimensions = CMVideoFormatDescriptionGetDimensions(
+                        formatDescription
+                    )
+
+                    if max(videoDimensions.width, videoDimensions.height) > 1920 {
+                        nonStandardReasons.append(.videoResolution)
+                    }
+
+                    let videoCodecType = formatDescription.mediaSubType
+
+                    let standard = CMFormatDescription.MediaSubType.h264
+
+                    if videoCodecType != standard {
+                        nonStandardReasons.append(.videoCodec)
+                    }
+
+                    let frameRate = track.nominalFrameRate
+                    if frameRate > 120.0 {
+                        nonStandardReasons.append(.videoFrameRate)
+                    }
+
+                    if nonStandardReasons.isEmpty {
+                        completionHandler(.standard)
+                    } else {
+                        completionHandler(.nonstandard(nonStandardReasons))
+                    }
+
+                }
+            }
+        default:
+            // Inspection fails for multi-video track inputs
+            // for the time being
+            completionHandler(.inspectionFailure)
+        }
+    }
+}

Sources/MuxUploadSDK/InputInspection/UploadInputInspector.swift

@@ -14,13 +14,17 @@ struct UploadInput {
         case started(AVAsset, UploadInfo)
         case underInspection(AVAsset, UploadInfo)
         case standardizing(AVAsset, UploadInfo)
-        case standardizationSucceeded(AVAsset, UploadInfo)
+        case standardizationSucceeded(
+            source: AVAsset,
+            standardized: AVAsset?,
+            uploadInfo: UploadInfo
+        )
         case standardizationFailed(AVAsset, UploadInfo)
         case awaitingUploadConfirmation(UploadInfo)
-        case uploadInProgress(UploadInfo)
-        case uploadPaused(UploadInfo)
-        case uploadSucceeded(UploadInfo)
-        case uploadFailed(UploadInfo)
+        case uploadInProgress(UploadInfo, MuxUpload.TransportStatus)
+        case uploadPaused(UploadInfo, MuxUpload.TransportStatus)
+        case uploadSucceeded(UploadInfo, MuxUpload.TransportStatus)
+        case uploadFailed(UploadInfo, MuxUpload.TransportStatus)
     }
 
     var status: Status
@@ -35,19 +39,19 @@ struct UploadInput {
             return sourceAsset
         case .standardizing(let sourceAsset, _):
             return sourceAsset
-        case .standardizationSucceeded(let sourceAsset, _):
+        case .standardizationSucceeded(let sourceAsset, _, _):
             return sourceAsset
         case .standardizationFailed(let sourceAsset, _):
             return sourceAsset
         case .awaitingUploadConfirmation(let uploadInfo):
             return uploadInfo.sourceAsset()
-        case .uploadInProgress(let uploadInfo):
+        case .uploadInProgress(let uploadInfo, _):
             return uploadInfo.sourceAsset()
-        case .uploadSucceeded(let uploadInfo):
+        case .uploadSucceeded(let uploadInfo, _):
             return uploadInfo.sourceAsset()
-        case .uploadFailed(let uploadInfo):
+        case .uploadFailed(let uploadInfo, _):
             return uploadInfo.sourceAsset()
-        case .uploadPaused(let uploadInfo):
+        case .uploadPaused(let uploadInfo, _):
             return uploadInfo.sourceAsset()
         }
     }
@@ -62,22 +66,71 @@ struct UploadInput {
             return uploadInfo
         case .standardizing(_, let uploadInfo):
             return uploadInfo
-        case .standardizationSucceeded(_, let uploadInfo):
+        case .standardizationSucceeded(_, _, let uploadInfo):
             return uploadInfo
         case .standardizationFailed(_, let uploadInfo):
             return uploadInfo
         case .awaitingUploadConfirmation(let uploadInfo):
             return uploadInfo
-        case .uploadInProgress(let uploadInfo):
+        case .uploadInProgress(let uploadInfo, _):
             return uploadInfo
-        case .uploadPaused(let uploadInfo):
+        case .uploadPaused(let uploadInfo, _):
             return uploadInfo
-        case .uploadSucceeded(let uploadInfo):
+        case .uploadSucceeded(let uploadInfo, _):
             return uploadInfo
-        case .uploadFailed(let uploadInfo):
+        case .uploadFailed(let uploadInfo, _):
             return uploadInfo
         }
     }
+
+    var transportStatus: MuxUpload.TransportStatus? {
+        switch status {
+        case .ready:
+            return nil
+        case .started:
+            return nil
+        case .underInspection:
+            return nil
+        case .standardizing:
+            return nil
+        case .standardizationSucceeded:
+            return nil
+        case .standardizationFailed:
+            return nil
+        case .awaitingUploadConfirmation:
+            return nil
+        case .uploadInProgress(_, let transportStatus):
+            return transportStatus
+        case .uploadPaused(_, let transportStatus):
+            return transportStatus
+        case .uploadSucceeded(_, let transportStatus):
+            return transportStatus
+        case .uploadFailed(_, let transportStatus):
+            return transportStatus
+        }
+    }
+}
+
+extension UploadInput {
+
+    mutating func processUploadCancellation() {
+        if case UploadInput.Status.ready = status {
+            return
+        }
+
+        status = .ready(sourceAsset, uploadInfo)
+    }
+
+    mutating func processStartNetworkTransport(
+        startingTransportStatus: MuxUpload.TransportStatus
+    ) {
+        if case UploadInput.Status.underInspection = status {
+            status = .uploadInProgress(uploadInfo, startingTransportStatus)
+        } else {
+            return
+        }
+    }
+
 }
 
 extension UploadInput.Status: Equatable {

Sources/MuxUploadSDK/InternalUtilities/UploadInput.swift

@@ -30,7 +30,8 @@ extension MuxUpload {
                     options: options
                 )
             ),
-            uploadManager: .shared
+            uploadManager: .shared,
+            inputInspector: .shared
         )
     }