RunCrossGen in ReadyToRun tests, set TimeoutPerTest, set FailOnWorkItemFailure (#22261)

* For ReadyToRun tests in AzDO set RunCrossGen=true during Helix test running

* Set __TestTimeout and introduce TimeoutPerTestInMinutes for test jobs

* Set FailOnWorkItemFailure - fail the AzDO step on Helix work item "catastrophic failure"

5 files changed, 68 insertions, 22 deletions

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index fd37cc5a93..b010e5cb36 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -138,6 +138,7 @@ jobs:
           - normal
           - no_tiered_compilation
         timeoutInMinutes: 240
+        timeoutPerTestInMinutes: 10
 
 # Pri1 (CI)
 - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
@@ -153,8 +154,9 @@ jobs:
           - normal
           - no_tiered_compilation
         timeoutInMinutes: 360
+        timeoutPerTestInMinutes: 10
 
-# Pri1 crossgen (CI)
+# Pri1 ReadyToRun (CI)
 - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
   - template: eng/platform-matrix.yml
     parameters:
@@ -162,13 +164,14 @@ jobs:
       buildConfig: checked
       jobParameters:
         priority: 1
-        crossgen: true
+        readyToRun: true
         scenarios:
           asString: 'normal,no_tiered_compilation'
           asArray:
           - normal
           - no_tiered_compilation
         timeoutInMinutes: 360
+        timeoutPerTestInMinutes: 30
 
 # Pri1 (Schedule, Manual)
 - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule', 'Manual')) }}:
@@ -179,7 +182,7 @@ jobs:
       jobParameters:
         priority: 1
         scenarios:
-          asString: 'normal,no_tiered_compilation,jitstress1,jitstress2,jitstress1_tiered,jitstress2_tiered'
+          asString: 'normal,no_tiered_compilation,jitstress1,jitstress2,jitstress1_tiered,jitstress2_tiered,jitstressregs1,jitstressregs2,jitstressregs3,jitstressregs4,jitstressregs8,jitstressregs0x10,jitstressregs0x80,jitstressregs0x1000,jitminopts'
           asArray:
           - normal
           - no_tiered_compilation
@@ -187,7 +190,18 @@ jobs:
           - jitstress2
           - jitstress1_tiered
           - jitstress2_tiered
+          - jitstressregs1
+          - jitstressregs2
+          - jitstressregs3
+          - jitstressregs4
+          - jitstressregs8
+          - jitstressregs0x10
+          - jitstressregs0x80
+          - jitstressregs0x1000
+          - jitminopts
         timeoutInMinutes: 480
+        timeoutPerTestInMinutes: 30
+
 
 #
 # Release test builds
@@ -207,8 +221,9 @@ jobs:
           - normal
           - no_tiered_compilation
         timeoutInMinutes: 360
+        timeoutPerTestInMinutes: 10
 
-# Pri1 crossgen (Official Build)
+# Pri1 ReadyToRun (Official Build)
 - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
   - template: eng/platform-matrix.yml
     parameters:
@@ -216,13 +231,14 @@ jobs:
       buildConfig: release
       jobParameters:
         priority: 1
-        crossgen: true
+        readyToRun: true
         scenarios:
           asString: 'normal,no_tiered_compilation'
           asArray:
           - normal
           - no_tiered_compilation
         timeoutInMinutes: 360
+        timeoutPerTestInMinutes: 30
 
 
 # Publish build information to Build Assets Registry
diff --git a/eng/send-to-helix-step.yml b/eng/send-to-helix-step.yml
index 32cae4c040..30526ca50f 100644
--- a/eng/send-to-helix-step.yml
+++ b/eng/send-to-helix-step.yml
@@ -13,6 +13,8 @@ parameters:
   helixType: ''
   scenarios: ''
   timeoutInMinutes: ''
+  timeoutPerTestInMinutes: ''
+  readyToRun: ''
 
 steps:
 - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
@@ -33,6 +35,8 @@ steps:
       _HelixType: ${{ parameters.helixType }}
       _Scenarios: ${{ parameters.scenarios }}
       _TimeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+      _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
+      _ReadyToRun: ${{ parameters.readyToRun }}
       ${{ if eq(parameters.publishTestResults, 'true') }}:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)
 
@@ -54,5 +58,7 @@ steps:
       _HelixType: ${{ parameters.helixType }}
       _Scenarios: ${{ parameters.scenarios }}
       _TimeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+      _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
+      _ReadyToRun: ${{ parameters.readyToRun }}
       ${{ if eq(parameters.publishTestResults, 'true') }}:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)
diff --git a/eng/test-job.yml b/eng/test-job.yml
index d5a0a7fad1..a44acca5cd 100644
--- a/eng/test-job.yml
+++ b/eng/test-job.yml
@@ -4,10 +4,11 @@ parameters:
   osGroup: ''
   osIdentifier: ''
   priority: 0
-  crossgen: false
+  readyToRun: false
   scenarios: ''
   helixQueues: ''
   timeoutInMinutes: ''
+  timeoutPerTestInMinutes: ''
   crossrootfsDir: ''
 
 ### Test job
@@ -24,18 +25,18 @@ jobs:
     osIdentifier: ${{ parameters.osIdentifier }}
 
     # Compute job name from template parameters
-    ${{ if eq(parameters.crossgen, 'false') }}:
+    ${{ if eq(parameters.readyToRun, false) }}:
       name: ${{ format('testbuild_pri{0}_{1}_{2}_{3}', parameters.priority, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
       displayName: ${{ format('Test Pri{0} {1} {2} {3}', parameters.priority, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
-    ${{ if eq(parameters.crossgen, 'true') }}:
+    ${{ if eq(parameters.readyToRun, true) }}:
       name: ${{ format('testbuild_pri{0}_r2r_{1}_{2}_{3}', parameters.priority, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
       displayName: ${{ format('Test Pri{0} R2R {1} {2} {3}', parameters.priority, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
 
     crossrootfsDir: ${{ parameters.crossrootfsDir }}
 
-    ${{ if eq(parameters.crossgen, 'false') }}:
+    ${{ if eq(parameters.readyToRun, false) }}:
       helixType: ${{ format('test/functional/cli/pri{0}', parameters.priority) }}
-    ${{ if eq(parameters.crossgen, 'true') }}:
+    ${{ if eq(parameters.readyToRun, true) }}:
       helixType: ${{ format('test/functional/r2r/cli/pri{0}', parameters.priority) }}
 
     variables:
@@ -52,17 +53,17 @@ jobs:
       - name: priorityArg
         value: ''
 
-    - ${{ if eq(parameters.crossgen, 'true') }}:
+    - ${{ if eq(parameters.readyToRun, true) }}:
       - name: crossgenArg
         value: 'crossgen'
-    - ${{ if eq(parameters.crossgen, 'false') }}:
+    - ${{ if eq(parameters.readyToRun, false) }}:
       - name: crossgenArg
         value: ''
 
     # TODO: Enable crossgen in build-test.sh. It currently doesn't
     # accept a crossgen arg, so disable the macos/linux crossgen test
     # build jobs.
-    ${{ if and(eq(parameters.crossgen, 'true'), in(parameters.osGroup, 'Linux', 'OSX')) }}:
+    ${{ if and(eq(parameters.readyToRun, true), in(parameters.osGroup, 'Linux', 'OSX')) }}:
       condition: false
 
     # FreeBSD test jobs are disabled since we don't have any FreeBSD helix queues.
@@ -128,9 +129,10 @@ jobs:
           condition: false
 
         publishTestResults: true
-        # TODO: see if this amount is enough for all individual jobs to finish
-        # TODO: consider passing this value as a parameter to a test job
-        timeoutInMinutes: 30
+        timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+        timeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
+
+        readyToRun: ${{ parameters.readyToRun }}
 
         ${{ if eq(variables['System.TeamProject'], 'internal') }}:
           # Access token variable for internal project from the
diff --git a/tests/helixpublishwitharcade.proj b/tests/helixpublishwitharcade.proj
index 39ee7fe2df..b15863d0f8 100644
--- a/tests/helixpublishwitharcade.proj
+++ b/tests/helixpublishwitharcade.proj
@@ -23,7 +23,10 @@
     <HelixType>$(_HelixType)</HelixType>
 
     <TimeoutInMinutes>$(_TimeoutInMinutes)</TimeoutInMinutes>
-    <TimeoutInMinutes Condition=" '$(TimeoutInMinutes)' == '' ">5</TimeoutInMinutes>
+    <TimeoutPerTestInMilliseconds Condition=" '$(_TimeoutPerTestInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(_TimeoutPerTestInMinutes)).TotalMilliseconds)</TimeoutPerTestInMilliseconds>
+
+    <ReadyToRun>$(_ReadyToRun)</ReadyToRun>
+    <ReadyToRun Condition=" '$(ReadyToRun)' == '' ">false</ReadyToRun>
 
     <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
     <CoreRootDirectory>$(TestWorkingDir)\Tests\Core_Root</CoreRootDirectory>
@@ -71,6 +74,7 @@
     <PropertyGroup>
       <EnableXUnitReporter>true</EnableXUnitReporter>
       <FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure>
+      <FailOnWorkItemFailure>true</FailOnWorkItemFailure>
       <TestRunNamePrefix Condition=" '$(Scenario)' == 'normal' ">$(BuildOS) $(BuildArch) $(BuildType) @ </TestRunNamePrefix>
       <TestRunNamePrefix Condition=" '$(Scenario)' != 'normal' ">$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ </TestRunNamePrefix>
       <_XUnitRunnerArgs>-parallel collections -nocolor -noshadow -xml testResults.xml -notrait category=outerloop -notrait category=failing</_XUnitRunnerArgs>
@@ -79,13 +83,31 @@
     <PropertyGroup Condition=" '$(TargetsWindows)' == 'true' ">
       <_CoreRun>%CORE_ROOT%\CoreRun.exe</_CoreRun>
       <_XUnitRunnerDll>%CORE_ROOT%\xunit.console.dll</_XUnitRunnerDll>
-      <_HelixPreCommands>set CORE_ROOT=%HELIX_CORRELATION_PAYLOAD%;set __TestEnv=%HELIX_WORKITEM_PAYLOAD%\SetStressModes_$(Scenario)$(TestEnvFileExtension);type %__TestEnv%</_HelixPreCommands>
     </PropertyGroup>
 
+    <ItemGroup Condition=" '$(TargetsWindows)' == 'true' ">
+      <_HelixPreCommands Include="set CORE_ROOT=%HELIX_CORRELATION_PAYLOAD%" />
+      <_HelixPreCommands Include="set RunCrossGen=true" Condition=" '$(ReadyToRun)' == 'true' " />
+      <_HelixPreCommands Include="set __TestEnv=%HELIX_WORKITEM_PAYLOAD%\SetStressModes_$(Scenario)$(TestEnvFileExtension)" />
+      <_HelixPreCommands Include="set __TestTimeout=$(TimeoutPerTestInMilliseconds)" Condition=" '$(TimeoutPerTestInMilliseconds)' != '' " />
+      <_HelixPreCommands Include="type %__TestEnv%" />
+    </ItemGroup>
+
     <PropertyGroup Condition=" '$(TargetsWindows)' != 'true' ">
       <_CoreRun>$CORE_ROOT/corerun</_CoreRun>
       <_XUnitRunnerDll>$CORE_ROOT/xunit.console.dll</_XUnitRunnerDll>
-      <_HelixPreCommands>export CORE_ROOT=$HELIX_CORRELATION_PAYLOAD;export __TestEnv=$HELIX_WORKITEM_PAYLOAD/SetStressModes_$(Scenario)$(TestEnvFileExtension);cat $__TestEnv</_HelixPreCommands>
+    </PropertyGroup>
+
+    <ItemGroup Condition=" '$(TargetsWindows)' != 'true' ">
+      <_HelixPreCommands Include="export CORE_ROOT=$HELIX_CORRELATION_PAYLOAD" />
+      <_HelixPreCommands Include="export RunCrossGen=true" Condition=" '$(ReadyToRun)' == 'true' " />
+      <_HelixPreCommands Include="export __TestEnv=$HELIX_WORKITEM_PAYLOAD/SetStressModes_$(Scenario)$(TestEnvFileExtension)" />
+      <_HelixPreCommands Include="export __TestTimeout=$(TimeoutPerTestInMilliseconds)" Condition=" '$(TimeoutPerTestInMilliseconds)' != '' " />
+      <_HelixPreCommands Include="cat $__TestEnv" />
+    </ItemGroup>
+
+    <PropertyGroup>
+      <_HelixPreCommands>@(_HelixPreCommands)</_HelixPreCommands>
     </PropertyGroup>
 
     <ItemGroup>
@@ -93,7 +115,7 @@
         <PayloadDirectory>%(RootDir)%(Directory)</PayloadDirectory>
         <PreCommands>$(_HelixPreCommands)</PreCommands>
         <Command>$(_CoreRun) $(_XUnitRunnerDll) %(FileName)%(Extension) $(_XUnitRunnerArgs)</Command>
-        <Timeout>$([System.TimeSpan]::FromMinutes($(TimeoutInMinutes)))</Timeout>
+        <Timeout Condition=" '$(TimeoutInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutInMinutes)))</Timeout>
       </HelixWorkItem>
     </ItemGroup>
   </Target>
diff --git a/tests/testenvironment.proj b/tests/testenvironment.proj
index f99209ba03..27320e3bce 100644
--- a/tests/testenvironment.proj
+++ b/tests/testenvironment.proj
@@ -55,7 +55,7 @@
     <!-- "normal" scenario doesn't define any COMPlus_* variables and uses the coreclr runtime default values
          while other scenarios use the default values of COMPlus_* variables defined in ItemDefinitionGroup above -->
     <TestEnvironment Include="normal" TieredCompilation="" />
-    <TestEnvironment Include="minopts" JITMinOpts="1" />
+    <TestEnvironment Include="jitminopts" JITMinOpts="1" />
     <TestEnvironment Include="no_tiered_compilation" TieredCompilation="0" />
     <TestEnvironment Include="forcerelocs" ForceRelocs="1" />
     <TestEnvironment Include="jitstress1" JitStress="1" />
@@ -98,7 +98,7 @@
     <TestEnvironment Include="gcstress0xc_zapdisable_heapverify1" GCStress="0xC" ZapDisable="1" ReadyToRun="0" HeapVerify="1" />
     <TestEnvironment Include="gcstress0xc_jitstress1" GCStress="0xC" JitStress="1" />
     <TestEnvironment Include="gcstress0xc_jitstress2" GCStress="0xC" JitStress="2" />
-    <TestEnvironment Include="gcstress0xc_minopts_heapverify1" GCStress="0xC" JITMinOpts="1" HeapVerify="1" />
+    <TestEnvironment Include="gcstress0xc_jitminopts_heapverify1" GCStress="0xC" JITMinOpts="1" HeapVerify="1" />
   </ItemGroup>
 
   <!-- We use target batching on the COMPlusVariable items to iterate over the all COMPlus_* environment variables