From e369d43e6630774ef73a18821488819fe5f1a596 Mon Sep 17 00:00:00 2001 From: Egor Chesakov Date: Wed, 30 Jan 2019 22:42:49 -0800 Subject: Add timeoutPerTestCollection and use this to control Helix workitem timeout (#22312) --- azure-pipelines.yml | 29 +++++++++++++++++++++-------- eng/send-to-helix-step.yml | 6 +++--- eng/test-job.yml | 7 ++++--- tests/helixpublishwitharcade.proj | 6 ++++-- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b010e5cb36..4e9967fc07 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -137,7 +137,14 @@ jobs: asArray: - normal - no_tiered_compilation - timeoutInMinutes: 240 + # "Total" includes building tests, waiting for a queue in Helix to become available, and running the tests. + # In case test-job.yml gets split into two separate jobs (e.g. build-test-job.yml and run-test.yml) + # this number should be adjusted accordingly. + timeoutTotalInMinutes: 240 + # "PerCollection" is time needed for the "biggest" xUnit test collection to complete. + # In case xUnit test wrappers get refactored this number should also be adjusted. + timeoutPerTestCollectionInMinutes: 30 + # "PerTest" corresponds to individual test running time (i.e. __TestTimeout). timeoutPerTestInMinutes: 10 # Pri1 (CI) @@ -153,7 +160,8 @@ jobs: asArray: - normal - no_tiered_compilation - timeoutInMinutes: 360 + timeoutTotalInMinutes: 360 + timeoutPerTestCollectionInMinutes: 60 timeoutPerTestInMinutes: 10 # Pri1 ReadyToRun (CI) @@ -170,8 +178,9 @@ jobs: asArray: - normal - no_tiered_compilation - timeoutInMinutes: 360 - timeoutPerTestInMinutes: 30 + timeoutTotalInMinutes: 360 + timeoutPerTestCollectionInMinutes: 60 + timeoutPerTestInMinutes: 10 # Pri1 (Schedule, Manual) - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule', 'Manual')) }}: @@ -199,7 +208,9 @@ jobs: - jitstressregs0x80 - jitstressregs0x1000 - jitminopts - timeoutInMinutes: 480 + timeoutTotalInMinutes: 480 + # TODO: Adjust this number as soon as we have more data on how long it takes to run these jobs in Helix. + timeoutPerTestCollectionInMinutes: 120 timeoutPerTestInMinutes: 30 @@ -220,7 +231,8 @@ jobs: asArray: - normal - no_tiered_compilation - timeoutInMinutes: 360 + timeoutTotalInMinutes: 360 + timeoutPerTestCollectionInMinutes: 60 timeoutPerTestInMinutes: 10 # Pri1 ReadyToRun (Official Build) @@ -237,8 +249,9 @@ jobs: asArray: - normal - no_tiered_compilation - timeoutInMinutes: 360 - timeoutPerTestInMinutes: 30 + timeoutTotalInMinutes: 360 + timeoutPerTestCollectionInMinutes: 60 + timeoutPerTestInMinutes: 10 # Publish build information to Build Assets Registry diff --git a/eng/send-to-helix-step.yml b/eng/send-to-helix-step.yml index 30526ca50f..73b989d696 100644 --- a/eng/send-to-helix-step.yml +++ b/eng/send-to-helix-step.yml @@ -12,7 +12,7 @@ parameters: helixQueues: '' helixType: '' scenarios: '' - timeoutInMinutes: '' + timeoutPerTestCollectionInMinutes: '' timeoutPerTestInMinutes: '' readyToRun: '' @@ -34,7 +34,7 @@ steps: _HelixTargetQueues: ${{ parameters.helixQueues }} _HelixType: ${{ parameters.helixType }} _Scenarios: ${{ parameters.scenarios }} - _TimeoutInMinutes: ${{ parameters.timeoutInMinutes }} + _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }} _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }} _ReadyToRun: ${{ parameters.readyToRun }} ${{ if eq(parameters.publishTestResults, 'true') }}: @@ -57,7 +57,7 @@ steps: _HelixTargetQueues: ${{ parameters.helixQueues }} _HelixType: ${{ parameters.helixType }} _Scenarios: ${{ parameters.scenarios }} - _TimeoutInMinutes: ${{ parameters.timeoutInMinutes }} + _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }} _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }} _ReadyToRun: ${{ parameters.readyToRun }} ${{ if eq(parameters.publishTestResults, 'true') }}: diff --git a/eng/test-job.yml b/eng/test-job.yml index a44acca5cd..c52a93d614 100644 --- a/eng/test-job.yml +++ b/eng/test-job.yml @@ -7,7 +7,8 @@ parameters: readyToRun: false scenarios: '' helixQueues: '' - timeoutInMinutes: '' + timeoutTotalInMinutes: '' + timeoutPerTestCollectionInMinutes: '' timeoutPerTestInMinutes: '' crossrootfsDir: '' @@ -77,7 +78,7 @@ jobs: # Note that the containers are resources defined in azure-pipelines.yml containerName: ${{ parameters.containerName }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + timeoutInMinutes: ${{ parameters.timeoutTotalInMinutes }} steps: @@ -129,7 +130,7 @@ jobs: condition: false publishTestResults: true - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + timeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }} timeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }} readyToRun: ${{ parameters.readyToRun }} diff --git a/tests/helixpublishwitharcade.proj b/tests/helixpublishwitharcade.proj index b15863d0f8..ac41ffcc5b 100644 --- a/tests/helixpublishwitharcade.proj +++ b/tests/helixpublishwitharcade.proj @@ -22,7 +22,7 @@ $(_HelixSource) $(_HelixType) - $(_TimeoutInMinutes) + $(_TimeoutPerTestCollectionInMinutes) $([System.TimeSpan]::FromMinutes($(_TimeoutPerTestInMinutes)).TotalMilliseconds) $(_ReadyToRun) @@ -85,6 +85,8 @@ <_XUnitRunnerDll>%CORE_ROOT%\xunit.console.dll + + <_HelixPreCommands Include="set CORE_ROOT=%HELIX_CORRELATION_PAYLOAD%" /> <_HelixPreCommands Include="set RunCrossGen=true" Condition=" '$(ReadyToRun)' == 'true' " /> @@ -115,7 +117,7 @@ %(RootDir)%(Directory) $(_HelixPreCommands) $(_CoreRun) $(_XUnitRunnerDll) %(FileName)%(Extension) $(_XUnitRunnerArgs) - $([System.TimeSpan]::FromMinutes($(TimeoutInMinutes))) + $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) -- cgit v1.2.3