Add timeoutPerTestCollection and use this to control Helix workitem timeout (#22312)

4 files changed, 32 insertions, 16 deletions

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index b010e5cb36..4e9967fc07 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -137,7 +137,14 @@ jobs:
           asArray:
           - normal
           - no_tiered_compilation
-        timeoutInMinutes: 240
+        # "Total" includes building tests, waiting for a queue in Helix to become available, and running the tests.
+        # In case test-job.yml gets split into two separate jobs (e.g. build-test-job.yml and run-test.yml)
+        # this number should be adjusted accordingly.
+        timeoutTotalInMinutes: 240
+        # "PerCollection" is time needed for the "biggest" xUnit test collection to complete.
+        # In case xUnit test wrappers get refactored this number should also be adjusted.
+        timeoutPerTestCollectionInMinutes: 30
+        # "PerTest" corresponds to individual test running time (i.e. __TestTimeout).
         timeoutPerTestInMinutes: 10
 
 # Pri1 (CI)
@@ -153,7 +160,8 @@ jobs:
           asArray:
           - normal
           - no_tiered_compilation
-        timeoutInMinutes: 360
+        timeoutTotalInMinutes: 360
+        timeoutPerTestCollectionInMinutes: 60
         timeoutPerTestInMinutes: 10
 
 # Pri1 ReadyToRun (CI)
@@ -170,8 +178,9 @@ jobs:
           asArray:
           - normal
           - no_tiered_compilation
-        timeoutInMinutes: 360
-        timeoutPerTestInMinutes: 30
+        timeoutTotalInMinutes: 360
+        timeoutPerTestCollectionInMinutes: 60
+        timeoutPerTestInMinutes: 10
 
 # Pri1 (Schedule, Manual)
 - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule', 'Manual')) }}:
@@ -199,7 +208,9 @@ jobs:
           - jitstressregs0x80
           - jitstressregs0x1000
           - jitminopts
-        timeoutInMinutes: 480
+        timeoutTotalInMinutes: 480
+        # TODO: Adjust this number as soon as we have more data on how long it takes to run these jobs in Helix.
+        timeoutPerTestCollectionInMinutes: 120
         timeoutPerTestInMinutes: 30
 
 
@@ -220,7 +231,8 @@ jobs:
           asArray:
           - normal
           - no_tiered_compilation
-        timeoutInMinutes: 360
+        timeoutTotalInMinutes: 360
+        timeoutPerTestCollectionInMinutes: 60
         timeoutPerTestInMinutes: 10
 
 # Pri1 ReadyToRun (Official Build)
@@ -237,8 +249,9 @@ jobs:
           asArray:
           - normal
           - no_tiered_compilation
-        timeoutInMinutes: 360
-        timeoutPerTestInMinutes: 30
+        timeoutTotalInMinutes: 360
+        timeoutPerTestCollectionInMinutes: 60
+        timeoutPerTestInMinutes: 10
 
 
 # Publish build information to Build Assets Registry
diff --git a/eng/send-to-helix-step.yml b/eng/send-to-helix-step.yml
index 30526ca50f..73b989d696 100644
--- a/eng/send-to-helix-step.yml
+++ b/eng/send-to-helix-step.yml
@@ -12,7 +12,7 @@ parameters:
   helixQueues: ''
   helixType: ''
   scenarios: ''
-  timeoutInMinutes: ''
+  timeoutPerTestCollectionInMinutes: ''
   timeoutPerTestInMinutes: ''
   readyToRun: ''
 
@@ -34,7 +34,7 @@ steps:
       _HelixTargetQueues: ${{ parameters.helixQueues }}
       _HelixType: ${{ parameters.helixType }}
       _Scenarios: ${{ parameters.scenarios }}
-      _TimeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+      _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
       _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
       _ReadyToRun: ${{ parameters.readyToRun }}
       ${{ if eq(parameters.publishTestResults, 'true') }}:
@@ -57,7 +57,7 @@ steps:
       _HelixTargetQueues: ${{ parameters.helixQueues }}
       _HelixType: ${{ parameters.helixType }}
       _Scenarios: ${{ parameters.scenarios }}
-      _TimeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+      _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
       _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
       _ReadyToRun: ${{ parameters.readyToRun }}
       ${{ if eq(parameters.publishTestResults, 'true') }}:
diff --git a/eng/test-job.yml b/eng/test-job.yml
index a44acca5cd..c52a93d614 100644
--- a/eng/test-job.yml
+++ b/eng/test-job.yml
@@ -7,7 +7,8 @@ parameters:
   readyToRun: false
   scenarios: ''
   helixQueues: ''
-  timeoutInMinutes: ''
+  timeoutTotalInMinutes: ''
+  timeoutPerTestCollectionInMinutes: ''
   timeoutPerTestInMinutes: ''
   crossrootfsDir: ''
 
@@ -77,7 +78,7 @@ jobs:
     # Note that the containers are resources defined in azure-pipelines.yml
     containerName: ${{ parameters.containerName }}
 
-    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+    timeoutInMinutes: ${{ parameters.timeoutTotalInMinutes }}
 
     steps:
 
@@ -129,7 +130,7 @@ jobs:
           condition: false
 
         publishTestResults: true
-        timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+        timeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
         timeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
 
         readyToRun: ${{ parameters.readyToRun }}
diff --git a/tests/helixpublishwitharcade.proj b/tests/helixpublishwitharcade.proj
index b15863d0f8..ac41ffcc5b 100644
--- a/tests/helixpublishwitharcade.proj
+++ b/tests/helixpublishwitharcade.proj
@@ -22,7 +22,7 @@
     <HelixSource>$(_HelixSource)</HelixSource>
     <HelixType>$(_HelixType)</HelixType>
 
-    <TimeoutInMinutes>$(_TimeoutInMinutes)</TimeoutInMinutes>
+    <TimeoutPerTestCollectionInMinutes>$(_TimeoutPerTestCollectionInMinutes)</TimeoutPerTestCollectionInMinutes>
     <TimeoutPerTestInMilliseconds Condition=" '$(_TimeoutPerTestInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(_TimeoutPerTestInMinutes)).TotalMilliseconds)</TimeoutPerTestInMilliseconds>
 
     <ReadyToRun>$(_ReadyToRun)</ReadyToRun>
@@ -85,6 +85,8 @@
       <_XUnitRunnerDll>%CORE_ROOT%\xunit.console.dll</_XUnitRunnerDll>
     </PropertyGroup>
 
+    <!-- WARNING: _HelixPreCommands collection is intentionally minimal and should be kept that way. -->
+
     <ItemGroup Condition=" '$(TargetsWindows)' == 'true' ">
       <_HelixPreCommands Include="set CORE_ROOT=%HELIX_CORRELATION_PAYLOAD%" />
       <_HelixPreCommands Include="set RunCrossGen=true" Condition=" '$(ReadyToRun)' == 'true' " />
@@ -115,7 +117,7 @@
         <PayloadDirectory>%(RootDir)%(Directory)</PayloadDirectory>
         <PreCommands>$(_HelixPreCommands)</PreCommands>
         <Command>$(_CoreRun) $(_XUnitRunnerDll) %(FileName)%(Extension) $(_XUnitRunnerArgs)</Command>
-        <Timeout Condition=" '$(TimeoutInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutInMinutes)))</Timeout>
+        <Timeout Condition=" '$(TimeoutPerTestCollectionInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</Timeout>
       </HelixWorkItem>
     </ItemGroup>
   </Target>