diff options
| -rw-r--r-- | azure-pipelines.yml | 131 | ||||
| -rw-r--r-- | eng/send-to-helix-step.yml | 8 | ||||
| -rw-r--r-- | eng/test-job.yml | 85 |
3 files changed, 87 insertions, 137 deletions
diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 259b36210b..2029f7cba0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -92,7 +92,7 @@ jobs: # https://github.com/Microsoft/azure-pipelines-yaml/pull/46 for more information # -# Debug build (Pull Request) +# Debug build (Pull request) # - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-ci')) }}: - template: eng/platform-matrix.yml @@ -110,7 +110,7 @@ jobs: buildConfig: checked # -# Release build (Pull Request) +# Release build (Pull request) # - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-ci')) }}: - template: eng/platform-matrix.yml @@ -119,7 +119,7 @@ jobs: buildConfig: release # -# Release build (Official Build) +# Release build (Official build) # - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - template: eng/platform-matrix.yml @@ -136,151 +136,66 @@ jobs: # Checked test builds # -# Pri0 (Pull Request) -- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-ci')) }}: - - template: eng/platform-matrix.yml - parameters: - jobTemplate: test-job.yml - buildConfig: checked - jobParameters: - priority: 0 - scenarios: - asString: 'normal,no_tiered_compilation' - asArray: - - normal - - no_tiered_compilation - # "Total" includes building tests, waiting for a queue in Helix to become available, and running the tests. - # In case test-job.yml gets split into two separate jobs (e.g. build-test-job.yml and run-test.yml) - # this number should be adjusted accordingly. - timeoutTotalInMinutes: 240 - # "PerCollection" is time needed for the "biggest" xUnit test collection to complete. - # In case xUnit test wrappers get refactored this number should also be adjusted. - timeoutPerTestCollectionInMinutes: 30 - # "PerTest" corresponds to individual test running time (i.e. __TestTimeout). - timeoutPerTestInMinutes: 10 - -# Pri1 (Pull Request, Outerloop) -- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-outerloop')) }}: +# Pull request +- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked jobParameters: - priority: 1 - scenarios: - asString: 'normal,no_tiered_compilation' - asArray: - - normal - - no_tiered_compilation - timeoutTotalInMinutes: 360 - timeoutPerTestCollectionInMinutes: 60 - timeoutPerTestInMinutes: 10 - -# Pri1 (CI) + ${{ if eq(variables['Build.DefinitionName'], 'coreclr-ci') }}: + testGroup: innerloop + ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop') }}: + testGroup: outerloop + ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitminopts-jitstress1-jitstress2') }}: + testGroup: outerloop-jitminopts-jitstress1-jitstress2 + +# CI - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}: - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked jobParameters: - priority: 1 - scenarios: - asString: 'normal,no_tiered_compilation' - asArray: - - normal - - no_tiered_compilation - timeoutTotalInMinutes: 360 - timeoutPerTestCollectionInMinutes: 60 - timeoutPerTestInMinutes: 10 - -# Pri1 ReadyToRun (CI) -- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}: + testGroup: outerloop + - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked jobParameters: - priority: 1 readyToRun: true - scenarios: - asString: 'normal,no_tiered_compilation' - asArray: - - normal - - no_tiered_compilation - timeoutTotalInMinutes: 360 - timeoutPerTestCollectionInMinutes: 60 - timeoutPerTestInMinutes: 10 - -# Pri1 (Schedule, Manual) + testGroup: outerloop + +# Schedule or Manual - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule', 'Manual')) }}: - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked jobParameters: - priority: 1 - scenarios: - asString: 'normal,no_tiered_compilation,jitstress1,jitstress2,jitstress1_tiered,jitstress2_tiered,jitstressregs1,jitstressregs2,jitstressregs3,jitstressregs4,jitstressregs8,jitstressregs0x10,jitstressregs0x80,jitstressregs0x1000,jitminopts' - asArray: - - normal - - no_tiered_compilation - - jitstress1 - - jitstress2 - - jitstress1_tiered - - jitstress2_tiered - - jitstressregs1 - - jitstressregs2 - - jitstressregs3 - - jitstressregs4 - - jitstressregs8 - - jitstressregs0x10 - - jitstressregs0x80 - - jitstressregs0x1000 - - jitminopts - timeoutTotalInMinutes: 480 - # TODO: Adjust this number as soon as we have more data on how long it takes to run these jobs in Helix. - timeoutPerTestCollectionInMinutes: 120 - timeoutPerTestInMinutes: 30 - + testGroup: outerloop # # Release test builds # -# Pri1 (Official Build) +# Official build - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: release jobParameters: - priority: 1 - scenarios: - asString: 'normal,no_tiered_compilation' - asArray: - - normal - - no_tiered_compilation - timeoutTotalInMinutes: 360 - timeoutPerTestCollectionInMinutes: 60 - timeoutPerTestInMinutes: 10 - -# Pri1 ReadyToRun (Official Build) -- ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: + testGroup: outerloop + - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: release jobParameters: - priority: 1 + testGroup: outerloop readyToRun: true - scenarios: - asString: 'normal,no_tiered_compilation' - asArray: - - normal - - no_tiered_compilation - timeoutTotalInMinutes: 360 - timeoutPerTestCollectionInMinutes: 60 - timeoutPerTestInMinutes: 10 # Publish build information to Build Assets Registry diff --git a/eng/send-to-helix-step.yml b/eng/send-to-helix-step.yml index 802cc8c5be..cba652de09 100644 --- a/eng/send-to-helix-step.yml +++ b/eng/send-to-helix-step.yml @@ -31,10 +31,10 @@ steps: _HelixAccessToken: ${{ parameters.helixAccessToken }} _HelixBuild: ${{ parameters.helixBuild }} _HelixSource: ${{ parameters.helixSource }} - _HelixTargetQueues: ${{ parameters.helixQueues }} + _HelixTargetQueues: ${{ parameters.helixQueues.asString }} _HelixType: ${{ parameters.helixType }} _RunCrossGen: ${{ parameters.runCrossGen }} - _Scenarios: ${{ parameters.scenarios }} + _Scenarios: ${{ parameters.scenarios.asString }} _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }} _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }} ${{ if eq(parameters.publishTestResults, 'true') }}: @@ -54,10 +54,10 @@ steps: _HelixAccessToken: ${{ parameters.helixAccessToken }} _HelixBuild: ${{ parameters.helixBuild }} _HelixSource: ${{ parameters.helixSource }} - _HelixTargetQueues: ${{ parameters.helixQueues }} + _HelixTargetQueues: ${{ parameters.helixQueues.asString }} _HelixType: ${{ parameters.helixType }} _RunCrossGen: ${{ parameters.runCrossGen }} - _Scenarios: ${{ parameters.scenarios }} + _Scenarios: ${{ parameters.scenarios.asString }} _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }} _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }} ${{ if eq(parameters.publishTestResults, 'true') }}: diff --git a/eng/test-job.yml b/eng/test-job.yml index 483ad0809a..478a75e930 100644 --- a/eng/test-job.yml +++ b/eng/test-job.yml @@ -3,13 +3,9 @@ parameters: archType: '' osGroup: '' osIdentifier: '' - priority: 0 + testGroup: '' readyToRun: false - scenarios: '' helixQueues: '' - timeoutTotalInMinutes: '' - timeoutPerTestCollectionInMinutes: '' - timeoutPerTestInMinutes: '' crossrootfsDir: '' ### Test job @@ -26,35 +22,40 @@ jobs: osIdentifier: ${{ parameters.osIdentifier }} # Compute job name from template parameters - ${{ if eq(parameters.readyToRun, false) }}: - name: ${{ format('testbuild_pri{0}_{1}_{2}_{3}', parameters.priority, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Test Pri{0} {1} {2} {3}', parameters.priority, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - ${{ if eq(parameters.readyToRun, true) }}: - name: ${{ format('testbuild_pri{0}_r2r_{1}_{2}_{3}', parameters.priority, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Test Pri{0} R2R {1} {2} {3}', parameters.priority, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.readyToRun, false)) }}: + name: ${{ format('testbuild_pri0_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test Pri0 {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.readyToRun, true)) }}: + name: ${{ format('testbuild_pri0_r2r_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test Pri0 R2R {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + + ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.readyToRun, false)) }}: + name: ${{ format('testbuild_pri1_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test Pri1 {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.readyToRun, true)) }}: + name: ${{ format('testbuild_pri1_r2r_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test Pri1 R2R {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} crossrootfsDir: ${{ parameters.crossrootfsDir }} variables: - group: DotNet-HelixApi-Access # Map template parameters to command line arguments - - ${{ if eq(parameters.priority, '1') }}: + - name: priorityArg + value: '' + - ${{ if ne(parameters.testGroup, 'innerloop') }}: - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - name: priorityArg value: 'priority1' - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - name: priorityArg value: '-priority=1' - - ${{ if eq(parameters.priority, '0') }}: - - name: priorityArg - value: '' + - name: crossgenArg + value: '' - ${{ if eq(parameters.readyToRun, true) }}: - name: crossgenArg value: 'crossgen' - - ${{ if eq(parameters.readyToRun, false) }}: - - name: crossgenArg - value: '' - name: clangArg value: '' @@ -83,7 +84,15 @@ jobs: # Note that the containers are resources defined in azure-pipelines.yml containerName: ${{ parameters.containerName }} - timeoutInMinutes: ${{ parameters.timeoutTotalInMinutes }} + # "Total" means building tests, waiting for a queue in Helix to become available, and running the tests. + # In case test-job.yml gets split into two separate jobs (e.g. build-test-job.yml and run-test.yml) + # this number should be adjusted accordingly. + ${{ if eq(parameters.testGroup, 'innerloop') }}: + timeoutInMinutes: 240 + ${{ if eq(parameters.testGroup, 'outerloop') }}: + timeoutInMinutes: 360 + ${{ if eq(parameters.testGroup, 'outerloop-jitminopts-jitstress1-jitstress2') }}: + timeoutInMinutes: 480 steps: @@ -139,18 +148,30 @@ jobs: helixSource: ci/dotnet/coreclr/$(Build.SourceBranch) ${{ if eq(parameters.readyToRun, false) }}: - helixType: ${{ format('test/functional/cli/pri{0}', parameters.priority) }} + helixType: 'test/functional/cli/' ${{ if eq(parameters.readyToRun, true) }}: - helixType: ${{ format('test/functional/r2r/cli/pri{0}', parameters.priority) }} + helixType: 'test/functional/r2r/cli/' - helixQueues: ${{ parameters.helixQueues.asString }} + helixQueues: ${{ parameters.helixQueues }} ${{ if eq(parameters.helixQueues.asString, '') }}: condition: false publishTestResults: true - timeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }} - timeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }} + + ${{ if eq(parameters.testGroup, 'innerloop') }}: + # "PerCollection" is time needed for the "biggest" xUnit test collection to complete. + # In case xUnit test wrappers get refactored this number should also be adjusted. + timeoutPerTestCollectionInMinutes: 30 + # "PerTest" corresponds to individual test running time (i.e. __TestTimeout). + timeoutPerTestInMinutes: 10 + ${{ if eq(parameters.testGroup, 'outerloop') }}: + timeoutPerTestCollectionInMinutes: 60 + timeoutPerTestInMinutes: 10 + ${{ if eq(parameters.testGroup, 'outerloop-jitminopts-jitstress1-jitstress2') }}: + # TODO: Adjust this number as soon as we have more data on how long it takes to run these jobs in Helix. + timeoutPerTestCollectionInMinutes: 120 + timeoutPerTestInMinutes: 30 runCrossGen: ${{ parameters.readyToRun }} @@ -159,4 +180,18 @@ jobs: # DotNet-HelixApi-Access variable group helixAccessToken: $(HelixApiAccessToken) - scenarios: ${{ parameters.scenarios.asString }} + ${{ if in(parameters.testGroup, 'innerloop', 'outerloop') }}: + scenarios: + asString: 'normal,no_tiered_compilation' + asArray: + - normal + - no_tiered_compilation + ${{ if eq(parameters.testGroup, 'outerloop-jitminopts-jitstress1-jitstress2') }}: + scenarios: + asString: 'jitminopts,jitstress1,jitstress1_tiered,jitstress2,jitstress2_tiered' + asArray: + - jitminopts + - jitstress1 + - jitstress1_tiered + - jitstress2 + - jitstress2_tiered |