Use XHarnessAppBundleToTest to create helix work items when runtime tests run on iOSSimulator

eng/pipelines/runtime-staging.yml

@@ -381,15 +381,12 @@ jobs:
           eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true),
           eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true),
           eq(variables['isFullMatrix'], true))
-      # Test execution is temporarily disabled because test apps no longer launch
-      # and the test suite times out after two hours, even if xharness cannot
-      # successfully launch any tests. Re-enable once these issues have been fixed.
-      #
-      # extra steps, run tests
-      # extraStepsTemplate: /eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml
-      # extraStepsParameters:
-      #   creator: dotnet-bot
-      #   testRunNamePrefixSuffix: Mono_$(_BuildConfig)
+      ${{ if eq(variables['isFullMatrix'], true) }}:
+        # extra steps, run tests
+        extraStepsTemplate: /eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml
+        extraStepsParameters:
+          creator: dotnet-bot
+          testRunNamePrefixSuffix: Mono_$(_BuildConfig)
 
 #
 # Build the whole product using Mono for Android and run runtime tests with Android devices

src/tests/Common/CLRTest.Execute.Bash.targets

@@ -416,12 +416,7 @@ else
 	HARNESS_RUNNER="xharness"
 fi
 
-xcode_path="%24(dirname "%24(dirname "%24(xcode-select -p)")")"
-simulator_app="$xcode_path/Contents/Developer/Applications/Simulator.app"
-open -a "$simulator_app"
-helix_runner_uid="%24(id -u)"
-
-sudo launchctl asuser "$helix_runner_uid" $__Command $HARNESS_RUNNER apple just-run %5c
+$__Command $HARNESS_RUNNER apple just-run %5c
   --app="net.dot.$__Category" %5c
   --output-directory="$__OutputDir" %5c
   --set-env="MONO_APPLE_APP_ENTRY_POINT_LIB_NAME=testdir-$(MsBuildProjectName)/$(MsBuildProjectName).dll" %5c

src/tests/Common/helixpublishwitharcade.proj

@@ -237,7 +237,12 @@
     <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFiles" StopOnFirstFailure="true" />
   </Target>
 
-  <Target Name="ZipPayloadDirectory" AfterTargets="PreparePayloadsDirectories" Condition="'$(TargetOS)' == 'Android'">
+  <PropertyGroup>
+    <IsRunningOnMobileTargets>false</IsRunningOnMobileTargets>
+    <IsRunningOnMobileTargets Condition="'$(TargetOS)' == 'Android' or '$(TargetOS)' == 'iOSSimulator' or '$(TargetOS)' == 'tvOSSimulator' ">true</IsRunningOnMobileTargets>
+  </PropertyGroup>
+
+  <Target Name="ZipPayloadDirectory" AfterTargets="PreparePayloadsDirectories" Condition="'$(IsRunningOnMobileTargets)' == 'true'">
     <ItemGroup>
       <Payloads Include="$([System.IO.Directory]::GetDirectories($(PayloadsRootDirectory)))" Condition="Exists('$(PayloadsRootDirectory)')" />
       <Payloads Update="@(Payloads)">
@@ -325,11 +330,6 @@
     <HelixPostCommands>@(HelixPostCommand)</HelixPostCommands>
   </PropertyGroup>
 
-  <PropertyGroup>
-    <IsRunningOnMobileTargets>false</IsRunningOnMobileTargets>
-    <IsRunningOnMobileTargets Condition="'$(TargetOS)' == 'Android' or '$(TargetOS)' == 'iOSSimulator' or '$(TargetOS)' == 'tvOSSimulator' ">true</IsRunningOnMobileTargets>
-  </PropertyGroup>
-
   <PropertyGroup Condition=" '$(TestWrapperTargetsWindows)' == 'true' ">
     <XUnitRunnerDll Condition=" '$(IsRunningOnMobileTargets)' == 'true' ">$Env:CORE_ROOT\xunit\xunit.console.dll</XUnitRunnerDll>
     <XUnitRunnerDll Condition=" '$(IsRunningOnMobileTargets)' != 'true' ">%CORE_ROOT%\xunit\xunit.console.dll</XUnitRunnerDll>
@@ -368,6 +368,14 @@
       <CustomCommands>dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs)</CustomCommands>
       <CustomCommands Condition=" '%(TestGroup)' != '' ">dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup)</CustomCommands>
     </XHarnessApkToTest>
+
+    <XHarnessAppBundleToTest Include="@(Payloads->Metadata('PayloadZipFile'))" Condition="'$(TargetOS)' == 'iOSSimulator' or '$(TargetOS)' == 'tvOSSimulator'">
+      <TestTarget Condition="'$(TargetArchitecture)' == 'arm64'">ios-simulator-64</TestTarget>
+      <TestTarget Condition="'$(TargetArchitecture)' == 'x64'">ios-simulator-64</TestTarget>
+      <TestTimeout Condition=" '$(TimeoutPerTestCollectionInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</TestTimeout>
+      <CustomCommands>dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs)</CustomCommands>
+      <CustomCommands Condition=" '%(TestGroup)' != '' ">dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup)</CustomCommands>
+    </XHarnessAppBundleToTest>
 
     <HelixWorkItem Condition="'$(PALTestsDir)' != '' and '$(TestWrapperTargetsWindows)' != 'true'" Include="PALTests">
       <PayloadArchive>$(PayloadsRootDirectory)paltests.tar.gz</PayloadArchive>

src/tests/Common/testgrouping.proj

@@ -89,6 +89,16 @@
       <XUnitWrapperDll>$(TestBinDir)JIT\Regression\JIT.Regression.XUnitWrapper.dll</XUnitWrapperDll>
     </TestGrouping>
 
+    <TestGrouping Include="$(TestBinDir)JIT\Regression\JitBlue\GitHub_*\**">
+      <TestGroup>JIT.Regression.JitBlue</TestGroup>
+      <XUnitWrapperDll>$(TestBinDir)JIT\Regression\JIT.Regression.XUnitWrapper.dll</XUnitWrapperDll>
+    </TestGrouping>
+
+    <TestGrouping Include="$(TestBinDir)JIT\HardwareIntrinsics\X86\S*\**">
+      <TestGroup>JIT.HardwareIntrinsics.S</TestGroup>
+      <XUnitWrapperDll>$(TestBinDir)JIT\HardwareIntrinsics\JIT.HardwareIntrinsics.XUnitWrapper.dll</XUnitWrapperDll>
+    </TestGrouping>
+
     <TestGrouping Include="$(TestBinDir)Loader\classloader\generics\**">
       <TestGroup>Loader.classloader.generics</TestGroup>
       <XUnitWrapperDll>$(TestBinDir)Loader\classloader\Loader.classloader.XUnitWrapper.dll</XUnitWrapperDll>
@@ -158,13 +168,15 @@
 
     <XUnitWrapperGrouping Include="$(TestBinDir)JIT\BBT\*.XUnitWrapper.dll;
                                    $(TestBinDir)JIT\CheckProjects\*.XUnitWrapper.dll;
-                                   $(TestBinDir)JIT\Intrinsics\*.XUnitWrapper.dll;
                                    $(TestBinDir)JIT\opt\*.XUnitWrapper.dll;
                                    $(TestBinDir)JIT\Performance\*.XUnitWrapper.dll;
                                    $(TestBinDir)JIT\RyuJIT\*.XUnitWrapper.dll;
-                                   $(TestBinDir)JIT\SIMD\*.XUnitWrapper.dll;
                                    $(TestBinDir)JIT\superpmi\*.XUnitWrapper.dll">
-      <PayloadGroup>JIT</PayloadGroup>
+      <PayloadGroup>JIT.1</PayloadGroup>
+    </XUnitWrapperGrouping>
+    <XUnitWrapperGrouping Include="$(TestBinDir)JIT\Intrinsics\*.XUnitWrapper.dll;
+                                   $(TestBinDir)JIT\SIMD\*.XUnitWrapper.dll">
+      <PayloadGroup>JIT.2</PayloadGroup>
     </XUnitWrapperGrouping>
   </ItemGroup>
 </Project>

src/tests/issues.targets

@@ -3191,5 +3191,48 @@
         <ExcludeList Include = "$(XunitTestBinBase)/JIT/Regression/JitBlue/GitHub_25468/GitHub_25468/**">
             <Issue>Could not load file or assembly 'System.Drawing.Common, Version=6.0.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51' or one of its dependencies.</Issue>
         </ExcludeList>
+
+        <ExcludeList Include = "$(XunitTestBinBase)/JIT/Regression/JitBlue/DevDiv_461649/DevDiv_461649/*">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/Interop/PInvoke/SetLastError/SetLastErrorTest/*">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/Interop/UnmanagedCallConv/UnmanagedCallConvTest/*">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/Interop/StructMarshalling/ReversePInvoke/MarshalSeqStruct/DelegatePInvoke/DelegatePInvokeTest/*">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/tracing/eventpipe/eventsourceerror/eventsourceerror/*">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/JIT/IL_Conformance/Old/Conformance_Base/mul_ovf_u2/*">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/JIT/Regression/CLR-x86-JIT/V1-M11-Beta1/b41126/b41126/*">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/GC/Scenarios/Dynamo/dynamo/*">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/baseservices/TieredCompilation/BasicTest_DefaultMode_R2r/*">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/baseservices/TieredCompilation/BasicTest_DefaultMode_R2r/*">
+            <Issue>missing assembly</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/baseservices/TieredCompilation/BasicTest_QuickJitForLoopsOff_R2r/*">
+            <Issue>missing assembly</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/baseservices/TieredCompilation/BasicTest_QuickJitForLoopsOn_R2r/*">
+            <Issue>missing assembly</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/baseservices/TieredCompilation/BasicTest_QuickJitOff_R2r/*">
+            <Issue>missing assembly</Issue>
+        </ExcludeList>
+        <ExcludeList Include = "$(XunitTestBinBase)/baseservices/TieredCompilation/BasicTest_QuickJitOn_R2r/*">
+            <Issue>missing assembly</Issue>
+        </ExcludeList>
     </ItemGroup>
 </Project>