Blazor Hybrid iOS & Mac Catalyst Performance Profiling (#68685)

TanayParikh · akoeplinger · web-flow · commit 05c446760221 · 2022-07-05T11:08:06.000-07:00
* [MAUI][Blazor][PERF] Add APK file usage for Android Maui Blazor SOD * Update android_scenarios.proj * Call ReportFullyDrawn * Add --use-fully-drawn-time --fully-drawn-extra-delay * Add System.Console.WriteLine for non-android platforms * Add 6 second delay * Missing `;` * Add back Directory.Build.{props, targets} (cherry picked from commit 28d9ef3ab4ac53b947dd435e6494428238b4dea5) * Fixes based on CI 1729796 https://dev.azure.com/dnceng/internal/_build/results?buildId=1729796&view=logs&j=4d93e3ae-0b8e-5532-bed2-af5c8a7c58d3&t=b13dec14-3eb2-5f1a-4501-8a9fdadb0e3a (cherry picked from commit d470655576d3abe6a6fec478c221c4d9de698ba9) * [REVERT THIS] Temporarily skip other jobs (cherry picked from commit df51582df9e7bb9d1ca82759d664823c76823c29) * [MAUI][Blazor][PERF] Add IPA file usage for iOS/MacCatalyst Maui Blazor SOD (cherry picked from commit b1aaaf2) * Remove / restore nuget.config * Escape quotes * Fix typo * Replicate changes from #67670 * use-fully-drawn-time * Use akoeplinger/performance * Fix misplaced quote character * Revert "Use akoeplinger/performance" This reverts commit 12358b6. The change was merged to dotnet/performance. * Add ` --startup-iterations 7` * Revert "[REVERT THIS] Temporarily skip other jobs" This reverts commit 54fa30d. * Remove SOD Startup Iterations Co-authored-by: Alexander Köplinger <alex.koeplinger@outlook.com>
diff --git a/eng/pipelines/coreclr/templates/build-perf-maui-apps.yml b/eng/pipelines/coreclr/templates/build-perf-maui-apps.yml
@@ -236,7 +236,7 @@ steps:
                           var activity = MainActivity.Context as Activity;
                           activity.ReportFullyDrawn();
                         #else
-                          System.Console.WriteLine("__MAUI_Blazor_WebView_OnAfterRender__");
+                          System.Console.WriteLine(\"__MAUI_Blazor_WebView_OnAfterRender__\");
                         #endif
                     }
                 }
@@ -256,6 +256,40 @@ steps:
     displayName: Build MAUI Blazor Android
     workingDirectory: $(Build.SourcesDirectory)/MauiBlazorTesting
 
+  - script: |
+      chmod -R a+r .
+      # remove net6.0-maccatalyst to work around https://github.com/dotnet/sdk/issues/21877
+      cp MauiBlazorTesting.csproj MauiBlazorTesting.csproj.bak
+      sed -i'' -e 's/net6.0-ios;net6.0-maccatalyst/net6.0-ios/g' MauiBlazorTesting.csproj
+
+      # NuGet.config file cannot be in the build directory currently due to https://github.com/dotnet/aspnetcore/issues/41397
+      rm NuGet.config
+
+      ../dotnet publish -bl:MauiBlazoriOS.binlog -f net6.0-ios --self-contained -r ios-arm64 -c Release /p:_RequireCodeSigning=false /p:ApplicationId=net.dot.mauiblazortesting
+      mv ./bin/Release/net6.0-ios/ios-arm64/publish/MauiBlazorTesting.ipa ./MauiBlazoriOSDefault.ipa
+
+      # Restore NuGet.config
+      cp $(Build.SourcesDirectory)/NuGet.config ./NuGet.config
+
+      cp MauiBlazorTesting.csproj.bak MauiBlazorTesting.csproj
+    displayName: Build MAUI Blazor iOS
+    workingDirectory: $(Build.SourcesDirectory)/MauiBlazorTesting
+
+  - script: |
+      chmod -R a+r .
+
+      # NuGet.config file cannot be in the build directory currently due to https://github.com/dotnet/aspnetcore/issues/41397
+      rm NuGet.config
+
+      ../dotnet publish -bl:MauiBlazorMacCatalyst.binlog -f net6.0-maccatalyst -c Release
+
+      # Restore NuGet.config
+      cp $(Build.SourcesDirectory)/NuGet.config ./NuGet.config
+
+      mv ./bin/Release/net6.0-maccatalyst/maccatalyst-x64/MauiBlazorTesting.app ./MauiBlazorMacCatalystDefault.app
+    displayName: Build MAUI Blazor MacCatalyst
+    workingDirectory: $(Build.SourcesDirectory)/MauiBlazorTesting
+
   - task: PublishBuildArtifacts@1
     displayName: 'Publish MauiAndroid binlog'
     condition: always()
@@ -284,6 +318,27 @@ steps:
       pathtoPublish: $(Build.SourcesDirectory)/MauiTesting/MauiMacCatalyst.binlog
       artifactName:  ${{ parameters.artifactName }}
 
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish MauiBlazorAndroid binlog'
+    condition: always()
+    inputs:
+      pathtoPublish: $(Build.SourcesDirectory)/MauiBlazorTesting/MauiBlazorAndroid.binlog
+      artifactName:  ${{ parameters.artifactName }}
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish MauiBlazoriOS binlog'
+    condition: always()
+    inputs:
+      pathtoPublish: $(Build.SourcesDirectory)/MauiBlazorTesting/MauiBlazoriOS.binlog
+      artifactName:  ${{ parameters.artifactName }}
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish MauiBlazorMacCatalyst binlog'
+    condition: always()
+    inputs:
+      pathtoPublish: $(Build.SourcesDirectory)/MauiBlazorTesting/MauiBlazorMacCatalyst.binlog
+      artifactName:  ${{ parameters.artifactName }}
+
   - task: PublishBuildArtifacts@1
     displayName: 'Publish MauiiOSPodcast binlog'
     condition: always()
@@ -331,6 +386,16 @@ steps:
         archiveType: tar
         tarCompression: gz
 
+  - template: /eng/pipelines/common/upload-artifact-step.yml
+    parameters:
+        rootFolder: $(Build.SourcesDirectory)/MauiBlazorTesting/MauiBlazoriOSDefault.ipa
+        includeRootFolder: true
+        displayName: Maui Blazor iOS IPA
+        artifactName: MauiBlazoriOSDefaultIPA
+        archiveExtension: '.tar.gz'
+        archiveType: tar
+        tarCompression: gz
+
   - template: /eng/pipelines/common/upload-artifact-step.yml
     parameters:
         rootFolder: $(Build.SourcesDirectory)/dotnet-podcasts/src/Mobile/MauiiOSPodcast.ipa
@@ -351,6 +416,16 @@ steps:
         archiveType: tar
         tarCompression: gz
 
+  - template: /eng/pipelines/common/upload-artifact-step.yml
+    parameters:
+        rootFolder: $(Build.SourcesDirectory)/MauiBlazorTesting/MauiBlazorMacCatalystDefault.app
+        includeRootFolder: true
+        displayName: Maui Blazor MacCatalyst App
+        artifactName: MauiBlazorMacCatalystDefault
+        archiveExtension: '.tar.gz'
+        archiveType: tar
+        tarCompression: gz
+
   - script: rm -r -f ./bin
     workingDirectory: $(Build.SourcesDirectory)/MauiTesting
     displayName: Clean MauiTesting bin directory
diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml
@@ -255,6 +255,20 @@ jobs:
           artifactFileName: 'MauiiOSPodcastIPA.tar.gz'
           artifactName: 'MauiiOSPodcastIPA'
           displayName: 'Maui iOS Podcast IPA'
+      - template: /eng/pipelines/common/download-artifact-step.yml
+        parameters:
+          unpackFolder: $(Build.SourcesDirectory)/MauiBlazoriOSDefaultIPA
+          cleanUnpackFolder: false
+          artifactFileName: 'MauiBlazoriOSDefaultIPA.tar.gz'
+          artifactName: 'MauiBlazoriOSDefaultIPA'
+          displayName: 'Maui Blazor iOS IPA'
+      - template: /eng/pipelines/common/download-artifact-step.yml
+        parameters:
+          unpackFolder: $(Build.SourcesDirectory)/MauiBlazorMacCatalystDefault
+          cleanUnpackFolder: false
+          artifactFileName: 'MauiBlazorMacCatalystDefault.tar.gz'
+          artifactName: 'MauiBlazorMacCatalystDefault'
+          displayName: 'Maui Blazor MacCatalyst App'
 
     # Create Core_Root
     - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg)
diff --git a/eng/testing/performance/ios_scenarios.proj b/eng/testing/performance/ios_scenarios.proj
@@ -18,21 +18,21 @@
     <LlvmPath>nollvm</LlvmPath>
     <LlvmPath Condition="'$(iOSLlvmBuild)' == 'True'">llvm</LlvmPath>
   </PropertyGroup>
-  
+
   <PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
     <ScenarioDirectory>%HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\</ScenarioDirectory>
-    
+
   </PropertyGroup>
   <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
     <ScenarioDirectory>$HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/</ScenarioDirectory>
   </PropertyGroup>
 
   <ItemGroup>
     <HelixWorkItem Include="SOD - iOS HelloWorld .app Size">
-        <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
-        <PreCommands>cd $(ScenarioDirectory)helloios;cp -rf $HELIX_CORRELATION_PAYLOAD/iosHelloWorld/$(LlvmPath) ./app;$(Python) pre.py --name app</PreCommands>
-        <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
-        <PostCommands>$(Python) post.py</PostCommands>
+      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <PreCommands>cd $(ScenarioDirectory)helloios;cp -rf $HELIX_CORRELATION_PAYLOAD/iosHelloWorld/$(LlvmPath) ./app;$(Python) pre.py --name app</PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+      <PostCommands>$(Python) post.py</PostCommands>
     </HelixWorkItem>
     <HelixWorkItem Include="SOD - iOS HelloWorld Mono Zip Size $(LlvmPath)">
       <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
@@ -84,6 +84,24 @@
       <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
       <PostCommands>$(Python) post.py</PostCommands>
     </HelixWorkItem>
+    <HelixWorkItem Include="SOD - Maui Blazor iOS IPA Size" Condition="'$(iOSLlvmBuild)' == 'False'">
+      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <PreCommands>cd $(ScenarioDirectory)mauiblazorios;cp $HELIX_CORRELATION_PAYLOAD/MauiBlazoriOSDefault.ipa .;$(Python) pre.py --name MauiBlazoriOSDefault.ipa</PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+      <PostCommands>$(Python) post.py</PostCommands>
+    </HelixWorkItem>
+    <HelixWorkItem Include="SOD - Maui Blazor iOS IPA Size Unzipped" Condition="'$(iOSLlvmBuild)' == 'False'">
+      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <PreCommands>cd $(ScenarioDirectory)mauiblazorios;cp $HELIX_CORRELATION_PAYLOAD/MauiBlazoriOSDefault.ipa .;$(Python) pre.py --unzip --name MauiBlazoriOSDefault.ipa</PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+      <PostCommands>$(Python) post.py</PostCommands>
+    </HelixWorkItem>
+    <HelixWorkItem Include="SOD - Maui Blazor MacCatalyst .app Size" Condition="'$(iOSLlvmBuild)' == 'False'">
+      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <PreCommands>cd $(ScenarioDirectory)mauiblazorios;cp -rf $HELIX_CORRELATION_PAYLOAD/MauiBlazorMacCatalystDefault ./app;$(Python) pre.py --name app</PreCommands>
+      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+      <PostCommands>$(Python) post.py</PostCommands>
+    </HelixWorkItem>
     <HelixWorkItem Include="SOD - Maui iOS Podcast IPA Size" Condition="'$(iOSLlvmBuild)' == 'False'">
       <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
       <PreCommands>cd $(ScenarioDirectory)mauiios;cp $HELIX_CORRELATION_PAYLOAD/MauiiOSPodcast.ipa .;$(Python) pre.py --name MauiiOSPodcast.ipa</PreCommands>
@@ -96,38 +114,63 @@
       <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
       <PostCommands>$(Python) post.py</PostCommands>
     </HelixWorkItem>
-	<XHarnessAppBundleToTest Include="Device Startup - iOS Maui Default" Condition="'$(iOSLlvmBuild)' == 'False'">
-    <AppBundlePath>$(WorkItemDirectory).zip</AppBundlePath>
-    <WorkItemTimeout>00:15:00</WorkItemTimeout>
-	  <TestTarget>ios-device</TestTarget>
-	  <CustomCommands>
-	   	<![CDATA[
+    <XHarnessAppBundleToTest Include="Device Startup - iOS Maui Default" Condition="'$(iOSLlvmBuild)' == 'False'">
+      <AppBundlePath>$(WorkItemDirectory).zip</AppBundlePath>
+      <WorkItemTimeout>00:15:00</WorkItemTimeout>
+      <TestTarget>ios-device</TestTarget>
+      <CustomCommands>
+        <![CDATA[
+            # PreCommands
+            export XHARNESSPATH=$XHARNESS_CLI_PATH
+
+            cp -r $HELIX_CORRELATION_PAYLOAD/MauiTesting.app $(ScenarioDirectory)mauiios/MauiTesting.app
+            cp -f embedded.mobileprovision $(ScenarioDirectory)mauiios/MauiTesting.app
+            cd $(ScenarioDirectory)mauiios
+            sign MauiTesting.app
+
+            $(Python) pre.py --name MauiTesting.app
+
+            # Testing commands
+            $(Python) test.py devicestartup --device-type ios --package-path MauiTesting.app --package-name net.dot.mauitesting --scenario-name "%(Identity)"
+            ((result=$?))
+
+            # Post commands
+            $(Python) post.py
+            exit $result
+          ]]>
+        </CustomCommands>
+    </XHarnessAppBundleToTest>
+    <XHarnessAppBundleToTest Include="Device Startup - iOS Maui Blazor Default" Condition="'$(iOSLlvmBuild)' == 'False'">
+      <AppBundlePath>$(WorkItemDirectory).zip</AppBundlePath>
+      <WorkItemTimeout>00:15:00</WorkItemTimeout>
+      <TestTarget>ios-device</TestTarget>
+      <CustomCommands>
+        <![CDATA[
           # PreCommands
           export XHARNESSPATH=$XHARNESS_CLI_PATH
 
-          cp -r $HELIX_CORRELATION_PAYLOAD/MauiTesting.app $(ScenarioDirectory)mauiios/MauiTesting.app
-          cp -f embedded.mobileprovision $(ScenarioDirectory)mauiios/MauiTesting.app
-          cd $(ScenarioDirectory)mauiios
-          sign MauiTesting.app
+          cp -r $HELIX_CORRELATION_PAYLOAD/MauiBlazorTesting.app $(ScenarioDirectory)mauiblazorios/MauiBlazorTesting.app
+          cp -f embedded.mobileprovision $(ScenarioDirectory)mauiblazorios/MauiBlazorTesting.app
+          cd $(ScenarioDirectory)mauiblazorios
+          sign MauiBlazorTesting.app
 
-          $(Python) pre.py --name MauiTesting.app
+          $(Python) pre.py --name MauiBlazorTesting.app
 
           # Testing commands
-          $(Python) test.py devicestartup --device-type ios --package-path MauiTesting.app --package-name net.dot.mauitesting --scenario-name "%(Identity)"
-          ((result=$?))
-          
+          $(Python) test.py devicestartup --device-type ios --package-path MauiBlazorTesting.app --package-name net.dot.mauiblazortesting --scenario-name "%(Identity)" --use-fully-drawn-time --fully-drawn-magic-string __MAUI_Blazor_WebView_OnAfterRender__ --startup-iterations 7
+
           # Post commands
           $(Python) post.py
           exit $result
         ]]>
-	  </CustomCommands>
-	</XHarnessAppBundleToTest>
+      </CustomCommands>
+    </XHarnessAppBundleToTest>
     <XHarnessAppBundleToTest Include="Device Startup - iOS Maui Podcast" Condition="'$(iOSLlvmBuild)' == 'False'">
-    <AppBundlePath>$(WorkItemDirectory).zip</AppBundlePath>
-    <WorkItemTimeout>00:15:00</WorkItemTimeout>
-	  <TestTarget>ios-device</TestTarget>
-	  <CustomCommands>
-	   	<![CDATA[
+      <AppBundlePath>$(WorkItemDirectory).zip</AppBundlePath>
+      <WorkItemTimeout>00:15:00</WorkItemTimeout>
+      <TestTarget>ios-device</TestTarget>
+      <CustomCommands>
+        <![CDATA[
           # PreCommands
           export XHARNESSPATH=$XHARNESS_CLI_PATH
 
@@ -146,8 +189,8 @@
           $(Python) post.py
           exit $result
         ]]>
-	  </CustomCommands>
-	</XHarnessAppBundleToTest>
+      </CustomCommands>
+    </XHarnessAppBundleToTest>
   </ItemGroup>
   <!--
     This target is to work around the XHarness command that depend on scripts in ORIGPYPATH
diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1
@@ -174,6 +174,8 @@ if ($iOSMono) {
         Copy-Item -path "$SourceDirectory\iosHelloWorld\nollvm" $PayloadDirectory\iosHelloWorld\nollvm -Recurse
         Copy-Item -path "$SourceDirectory\MauiiOSDefaultIPA" $PayloadDirectory\MauiiOSDefaultIPA -Recurse
         Copy-Item -path "$SourceDirectory\MauiMacCatalystDefault\MauiMacCatalystDefault.app" $PayloadDirectory\MauiMacCatalystDefault -Recurse
+        Copy-Item -path "$SourceDirectory\MauiBlazoriOSDefaultIPA" $PayloadDirectory\MauiBlazoriOSDefaultIPA -Recurse
+        Copy-Item -path "$SourceDirectory\MauiBlazorMacCatalystDefault\MauiBlazorMacCatalystDefault.app" $PayloadDirectory\MauiBlazorMacCatalystDefault -Recurse
         Copy-Item -path "$SourceDirectory\MauiiOSPodcastIPA" $PayloadDirectory\MauiiOSPodcastIPA -Recurse
     }
 
diff --git a/eng/testing/performance/performance-setup.sh b/eng/testing/performance/performance-setup.sh
@@ -359,12 +359,21 @@ if [[ "$iosmono" == "true" ]]; then
         mkdir -p $payload_directory/iosHelloWorld && cp -rv $source_directory/iosHelloWorld/nollvm $payload_directory/iosHelloWorld
         mkdir -p $payload_directory/iosHelloWorldZip/nollvmzip && cp -rv $source_directory/iosHelloWorldZip/nollvmzip $payload_directory/iosHelloWorldZip
         mkdir -p $payload_directory/MauiMacCatalystDefault && cp -rv $source_directory/MauiMacCatalystDefault/MauiMacCatalystDefault.app $payload_directory/MauiMacCatalystDefault
+        mkdir -p $payload_directory/MauiBlazorMacCatalystDefault && cp -rv $source_directory/MauiBlazorMacCatalystDefault/MauiBlazorMacCatalystDefault.app $payload_directory/MauiBlazorMacCatalystDefault
         cp -v $source_directory/MauiiOSDefaultIPA/MauiiOSDefault.ipa $payload_directory/MauiiOSDefault.ipa
+        cp -v $source_directory/MauiBlazoriOSDefaultIPA/MauiBlazoriOSDefault.ipa $payload_directory/MauiBlazoriOSDefault.ipa
         cp -v $source_directory/MauiiOSPodcastIPA/MauiiOSPodcast.ipa $payload_directory/MauiiOSPodcast.ipa
+
         # Get the .app so we can resign in the xharness item
         cp -v $source_directory/MauiiOSDefaultIPA/MauiiOSDefault.ipa $source_directory/MauiiOSDefaultIPA/MauiiOSDefault.zip
         unzip -d $source_directory/MauiiOSDefaultIPA $source_directory/MauiiOSDefaultIPA/MauiiOSDefault.zip
         mv $source_directory/MauiiOSDefaultIPA/Payload/MauiTesting.app $payload_directory/
+
+        # Get the .app so we can resign in the xharness item for Maui Blazor
+        cp -v $source_directory/MauiBlazoriOSDefaultIPA/MauiBlazoriOSDefault.ipa $source_directory/MauiBlazoriOSDefaultIPA/MauiBlazoriOSDefault.zip
+        unzip -d $source_directory/MauiBlazoriOSDefaultIPA $source_directory/MauiBlazoriOSDefaultIPA/MauiBlazoriOSDefault.zip
+        mv $source_directory/MauiBlazoriOSDefaultIPA/Payload/MauiBlazorTesting.app $payload_directory/
+
         # Get the .app so we can resign in the xharness item for podcast
         cp -v $source_directory/MauiiOSPodcastIPA/MauiiOSPodcast.ipa $source_directory/MauiiOSPodcastIPA/MauiiOSPodcast.zip
         unzip -d $source_directory/MauiiOSPodcastIPA $source_directory/MauiiOSPodcastIPA/MauiiOSPodcast.zip

Original file line number	Diff line number	Diff line change
`@@ -174,6 +174,8 @@ if ($iOSMono) {`
`174`	`174`	`Copy-Item -path "$SourceDirectory\iosHelloWorld\nollvm" $PayloadDirectory\iosHelloWorld\nollvm -Recurse`
`175`	`175`	`Copy-Item -path "$SourceDirectory\MauiiOSDefaultIPA" $PayloadDirectory\MauiiOSDefaultIPA -Recurse`
`176`	`176`	`Copy-Item -path "$SourceDirectory\MauiMacCatalystDefault\MauiMacCatalystDefault.app" $PayloadDirectory\MauiMacCatalystDefault -Recurse`
	`177`	`+ Copy-Item -path "$SourceDirectory\MauiBlazoriOSDefaultIPA" $PayloadDirectory\MauiBlazoriOSDefaultIPA -Recurse`
	`178`	`+ Copy-Item -path "$SourceDirectory\MauiBlazorMacCatalystDefault\MauiBlazorMacCatalystDefault.app" $PayloadDirectory\MauiBlazorMacCatalystDefault -Recurse`
`177`	`179`	`Copy-Item -path "$SourceDirectory\MauiiOSPodcastIPA" $PayloadDirectory\MauiiOSPodcastIPA -Recurse`
`178`	`180`	`}`
`179`	`181`