diff options
author | Bruce Forstall <brucefo@microsoft.com> | 2019-06-11 14:18:08 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-06-11 14:18:08 -0700 |
commit | 96c8a1259f5605f52a4d08d804c5f0cdb59308b2 (patch) | |
tree | e25828e971c91193d82e1d4b7cc0e855cf23a1a7 | |
parent | 94b6a89d491ef78792c491f83fd290948d2a838d (diff) | |
download | coreclr-96c8a1259f5605f52a4d08d804c5f0cdb59308b2.tar.gz coreclr-96c8a1259f5605f52a4d08d804c5f0cdb59308b2.tar.bz2 coreclr-96c8a1259f5605f52a4d08d804c5f0cdb59308b2.zip |
Improve corefx testing (#24962)
CoreFX tests are now used to test CoreCLR on Windows/x64 and Linux/x64.
PR testing includes normal and no-tiered modes. CI testing will include
all JIT stress modes that apply to CoreCLR testing. Many tests have been
disabled to allow the jobs to (mostly) run error-free.
Support for other platforms requires fixes to the testhost generation.
-rw-r--r-- | azure-pipelines.yml | 66 | ||||
-rw-r--r-- | eng/helixcorefxtests.proj | 275 | ||||
-rw-r--r-- | eng/test-job.yml | 99 | ||||
-rw-r--r-- | tests/CoreFX/CoreFX.issues.rsp | 44 | ||||
-rw-r--r-- | tests/helixpublishwitharcade.proj | 2 | ||||
-rw-r--r-- | tests/testenvironment.proj | 1 |
6 files changed, 377 insertions, 110 deletions
diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fb05c8f00c..0762ed408b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -151,6 +151,10 @@ jobs: - Linux_x64 - Windows_NT_x64 - Windows_NT_x86 + ${{ if in(variables['Build.DefinitionName'], 'coreclr-corefx', 'coreclr-corefx-jitstress', 'coreclr-corefx-jitstressregs', 'coreclr-corefx-jitstress2-jitstressregs') }}: + platforms: + - Linux_x64 + - Windows_NT_x64 # # Release builds @@ -194,7 +198,8 @@ jobs: # Checked test builds # # The test jobs that can be triggered by a PR, manually from ADO and that are scheduled -- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual', 'Schedule')) }}: +# TODO: This long 'in' condition needs to be make data driven, perhaps by adding another layer of template. +- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual', 'Schedule'), in(variables['Build.DefinitionName'], 'coreclr-ci', 'coreclr-outerloop', 'coreclr-outerloop-jitstress', 'coreclr-outerloop-jitstress-isas-arm', 'coreclr-outerloop-jitstress-isas-x86', 'coreclr-outerloop-jitstressregs-x86', 'coreclr-outerloop-jitstressregs', 'coreclr-outerloop-jitstress2-jitstressregs', 'coreclr-outerloop-gcstress0x3-gcstress0xc', 'coreclr-outerloop-gcstress-extra', 'coreclr-outerloop-r2r', 'coreclr-outerloop-r2r-extra')) }}: - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml @@ -238,31 +243,32 @@ jobs: ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop') }}: testGroup: outerloop ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstress') }}: - testGroup: outerloop-jitstress + testGroup: jitstress ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstress-isas-arm') }}: - testGroup: outerloop-jitstress-isas-arm + testGroup: jitstress-isas-arm ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstress-isas-x86') }}: - testGroup: outerloop-jitstress-isas-x86 + testGroup: jitstress-isas-x86 ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstressregs-x86') }}: - testGroup: outerloop-jitstressregs-x86 + testGroup: jitstressregs-x86 ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstressregs') }}: - testGroup: outerloop-jitstressregs + testGroup: jitstressregs ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-jitstress2-jitstressregs') }}: - testGroup: outerloop-jitstress2-jitstressregs + testGroup: jitstress2-jitstressregs ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-gcstress0x3-gcstress0xc') }}: - testGroup: outerloop-gcstress0x3-gcstress0xc + testGroup: gcstress0x3-gcstress0xc ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-gcstress-extra') }}: - testGroup: outerloop-gcstress-extra + testGroup: gcstress-extra ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-r2r') }}: testGroup: outerloop readyToRun: true displayNameArgs: R2R ${{ if eq(variables['Build.DefinitionName'], 'coreclr-outerloop-r2r-extra') }}: - testGroup: outerloop-r2r-extra + testGroup: r2r-extra readyToRun: true displayNameArgs: R2R -# The ReadyToRun test jobs that are triggered by default from a PR. +# ReadyToRun test jobs that are triggered by default from a PR. + - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-ci')) }}: - template: eng/platform-matrix.yml parameters: @@ -277,19 +283,43 @@ jobs: readyToRun: true testGroup: innerloop displayNameArgs: R2R - # The CoreFX runs against CoreCLR + +# +# CoreFX test runs against CoreCLR +# + +# PR innerloop checked build + +# TODO: This long 'in' condition needs to be make data driven, perhaps by adding another layer of template. +- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual', 'Schedule'), in(variables['Build.DefinitionName'], 'coreclr-ci', 'coreclr-corefx', 'coreclr-corefx-jitstress', 'coreclr-corefx-jitstressregs', 'coreclr-corefx-jitstress2-jitstressregs')) }}: - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml - buildConfig: release - platforms: - - Windows_NT_x64 + buildConfig: checked + ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'coreclr-ci')) }}: + platforms: + - Linux_x64 + - Windows_NT_x64 + ${{ if in(variables['Build.DefinitionName'], 'coreclr-corefx', 'coreclr-corefx-jitstress', 'coreclr-corefx-jitstressregs', 'coreclr-corefx-jitstress2-jitstressregs') }}: + platforms: + - Linux_x64 + - Windows_NT_x64 jobParameters: + ${{ if eq(variables['Build.DefinitionName'], 'coreclr-ci') }}: + testGroup: innerloop + ${{ if eq(variables['Build.DefinitionName'], 'coreclr-corefx') }}: + testGroup: outerloop + ${{ if eq(variables['Build.DefinitionName'], 'coreclr-corefx-jitstress') }}: + testGroup: jitstress + ${{ if eq(variables['Build.DefinitionName'], 'coreclr-corefx-jitstressregs') }}: + testGroup: jitstressregs + ${{ if eq(variables['Build.DefinitionName'], 'coreclr-corefx-jitstress2-jitstressregs') }}: + testGroup: jitstress2-jitstressregs corefxTests: true - testGroup: innerloop displayNameArgs: CoreFX -# CI +# CI (merge) jobs + - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}: - template: eng/platform-matrix.yml parameters: @@ -314,8 +344,8 @@ jobs: - Windows_NT_arm - Windows_NT_arm64 jobParameters: - readyToRun: true testGroup: outerloop + readyToRun: true displayNameArgs: R2R # diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj index 166eafe6b9..980c5c41ee 100644 --- a/eng/helixcorefxtests.proj +++ b/eng/helixcorefxtests.proj @@ -1,10 +1,77 @@ -<Project InitialTargets="BuildHelixWorkItems" Sdk="Microsoft.DotNet.Helix.Sdk"> +<Project InitialTargets="BuildHelixWorkItems" DefaultTargets="RunInParallelForEachScenario"> + + <!-- This project uses the helix SDK, documented at + https://github.com/dotnet/arcade/tree/master/src/Microsoft.DotNet.Helix/Sdk, + to send test jobs to helix. --> + + <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.props" Condition=" '$(UsesHelixSdk)' == 'true' " /> + + <!-- Import Microsoft.DotNet.Build.Tasks.Feed.targets for `ParseBuildManifest` --> + <Import Project="..\dir.props" Condition=" '$(NuGetPackageRoot)'=='' "/> <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" /> - <PropertyGroup> - <!-- Set the TargetFramework just to make the SDK happy --> - <TargetFramework>netstandard2.0</TargetFramework> - </PropertyGroup> + <!-- This project is copies much logic from helixpublishwitharcade.proj, used to send coreclr tests to Helix. --> + + <!-- + "RunInParallelForEachScenario" is the "root" target for this Project. It invokes other targets to set up the + information needed to submit a job to Helix. It does so by "recursively" invoking this project file with + different properties and targets to run, as well as invoking MSBuild on other project files. In particular, + it first creates the "correlation payload directory", which is the directory of files used by all Helix + submissions. Then, it recursively invokes this Project once per scenario (e.g., normal Pri-1 run, JitStress=2 + run, JitStressRegs=8 run, etc.), creating a new set of Helix work items for each scenario. For the CoreFX + tests, we create one Helix work item for each CoreFX test assembly; there is currently no concept of + grouping multiple assemblies into a single Helix work item to, for instance, consolidate multiple short-running + assemblies together. (Note that this grouping is done for the CoreCLR tests in helixpublishwitharcade.proj.) + --> + <Target Name="RunInParallelForEachScenario"> + <PropertyGroup> + <!-- This specifies what properties are needed to be passed down as global properties to a child project. --> + + <_PropertiesToPass> + __BuildArch=$(__BuildArch); + __BuildOS=$(__BuildOS); + __BuildType=$(__BuildType); + Creator=$(_Creator); + HelixAccessToken=$(_HelixAccessToken); + HelixBuild=$(_HelixBuild); + HelixSource=$(_HelixSource); + HelixTargetQueues=$(_HelixTargetQueues); + HelixType=$(_HelixType); + PublishTestResults=$(_PublishTestResults); + RunCrossGen=$(_RunCrossGen); + TimeoutPerTestCollectionInMinutes=$(_TimeoutPerTestCollectionInMinutes) + </_PropertiesToPass> + </PropertyGroup> + + <Error Condition="'$(_Scenarios)' == ''" Text="_Scenarios not set" /> + + <MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" Properties="Scenarios=$(_Scenarios)" /> + + <ItemGroup> + <_Scenarios Include="$(_Scenarios.Split(','))" /> + + <!-- MSBuild creates a new instance of the project for each %(_Scenarios.Identity) and can build them in parallel. --> + <_ProjectsToBuild Include="$(MSBuildProjectFile)"> + <AdditionalProperties>$(_PropertiesToPass);Scenario=%(_Scenarios.Identity)</AdditionalProperties> + </_ProjectsToBuild> + </ItemGroup> + + <PropertyGroup> + <_BuildInParallel>false</_BuildInParallel> + <_BuildInParallel Condition=" '@(_ProjectsToBuild->Count())' > '1' ">true</_BuildInParallel> + </PropertyGroup> + + <!-- Invoke MSBuild on this project file once for each Scenario (because of the "batching" defined in "_ProjectsToBuild"). + Set "UsesHelixSdk=true" to indicate we want to invoke Helix on this invocation: create the Helix work items and start + the jobs. This is done by invoking the "Test" Helix target. + --> + <MSBuild Projects="@(_ProjectsToBuild)" Targets="Test" BuildInParallel="$(_BuildInParallel)" StopOnFirstFailure="false" Properties="UsesHelixSdk=true" /> + </Target> + + <!-- Define a set of properties that are input to the Project, and that must be passed down to child processes. + (See "_PropertiesToPass" in target "RunInParallelForEachScenario".) If something can be computed, such as + properties needed by Helix, they should probably be set in target "BuildHelixWorkItems" instead. + --> <PropertyGroup> <Creator>$(_Creator)</Creator> @@ -13,8 +80,6 @@ <HelixSource>$(_HelixSource)</HelixSource> <HelixTargetQueues>$(_HelixTargetQueues)</HelixTargetQueues> <HelixType>$(_HelixType)</HelixType> - <HelixArchitecture>$(__BuildArch)</HelixArchitecture> - <HelixConfiguration>$(__BuildType)</HelixConfiguration> <!-- TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade. @@ -24,67 +89,104 @@ <TestWorkingDir Condition="'$(__TestWorkingDir)'==''">$(RootBinDir)tests\$(__BuildOS).$(__BuildArch).$(__BuildType)\</TestWorkingDir> <TargetsWindows Condition="'$(__BuildOS)' == 'Windows_NT'">true</TargetsWindows> + <!-- We have a single, universal exclusion file for all CoreFX test exclusions, for CoreFX tests run + in the CoreCLR repo. Note in particular that we don't have a unique exclusion file for each test + assembly, or for each processor architecture, or for each operating system, or for each stress + scenario type. This means that we generally exclude more tests than is strictly necessary for + any particular job. However, this mechanism is super simple; we don't need to manage fine- + grained exclusions; and there really shouldn't be any (or many) exclusions anyway. + --> + <CoreFXTestExclusionFile>$(ProjectDir)tests\CoreFX\CoreFX.issues.rsp</CoreFXTestExclusionFile> + <TestHostRootPath>$(TestWorkingDir)testhost\</TestHostRootPath> <TestArchiveRuntimeRoot>$(TestWorkingDir)helix\</TestArchiveRuntimeRoot> - <TestArchiveRuntimeFile>$(TestArchiveRuntimeRoot)testhost-runtime.zip</TestArchiveRuntimeFile> - <HelixCorrelationPayload></HelixCorrelationPayload> - <EnableAzurePipelinesReporter>$(_PublishTestResults)</EnableAzurePipelinesReporter> - <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter> - <EnableXUnitReporter>true</EnableXUnitReporter> - <FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure> - <FailOnWorkItemFailure>true</FailOnWorkItemFailure> - - <TimeoutInSeconds Condition="'$(TimeoutInSeconds)' == ''">600</TimeoutInSeconds> - <CommandTimeoutSpan>$([System.TimeSpan]::FromSeconds($(TimeoutInSeconds)))</CommandTimeoutSpan> - <MaxRetryCount Condition="'$(MaxRetryCount)' == ''">4</MaxRetryCount> - <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion> + <TestAssetBlobFeedUrl>https://dotnetfeed.blob.core.windows.net/dotnet-core</TestAssetBlobFeedUrl> </PropertyGroup> <PropertyGroup> <!-- - For windows we need to use call, since the command is going to be called from a bat script created by Helix - and we exit /b at the end of RunTests.cmd, Helix runs some other commands after ours within the bat script, - if we don't use call, then we cause the parent script to exit, and anything after will not be executed. - The arguments passed in to the run script in order are the runtime directory, the dotnet root directory (for - helix submissions same as the runtime directory) and the global tools directory. + For Windows, we need to use "call", since the command is going to be called from a batch script created by Helix. + We "exit /b" at the end of RunTests.cmd. Helix runs some other commands after ours within the batch script, + so if we don't use call, then we cause the parent script to exit, and anything after will not be executed. + The arguments passed in to the run script in are the runtime directory (-r) and the exclusion response file + passed to xunit (rsp-file). --> <HelixCommand Condition="'$(TargetsWindows)' == 'true'">call RunTests.cmd -r %HELIX_CORRELATION_PAYLOAD% --rsp-file %HELIX_CORRELATION_PAYLOAD%\CoreFX.issues.rsp </HelixCommand> <HelixCommand Condition="'$(TargetsWindows)' != 'true'">./RunTests.sh -r $HELIX_CORRELATION_PAYLOAD --rsp-file $HELIX_CORRELATION_PAYLOAD/CoreFX.issues.rsp </HelixCommand> </PropertyGroup> - <ItemGroup> - <TestArchiveRuntimeInputs Include="$(TestHostRootPath)**/*" /> - </ItemGroup> - + <!-- + Copy the CoreFX test exclusion response file (which is passed to the xunit console runner) to someplace we can + access it during the test run. We currently just copy it to the root of the generated .NET Core testhost + directory, which we use as the root of the correlation directory. + --> <Target Name="CopyRSPFile"> <Copy - SourceFiles="$(ProjectDir)\tests\CoreFX\CoreFX.issues.rsp" + SourceFiles="$(CoreFXTestExclusionFile)" DestinationFolder="$(TestHostRootPath)" /> + + <Message Importance="High" Text="Copied $(CoreFXTestExclusionFile) into $(TestHostRootPath)" /> </Target> - <Target Name="CompressRuntimeDirectory" - Inputs="@(TestArchiveRuntimeInputs)" - Outputs="$(TestArchiveRuntimeFile)" - DependsOnTargets="CopyRSPFile" > + <PropertyGroup> + <!-- Set the name of the scenario file. Note that this is only used in invocations where $(Scenario) is set. --> + <TestEnvFileName Condition=" '$(TargetsWindows)' == 'true' ">SetStressModes_$(Scenario).cmd</TestEnvFileName> + <TestEnvFileName Condition=" '$(TargetsWindows)' != 'true' ">SetStressModes_$(Scenario).sh</TestEnvFileName> + </PropertyGroup> - <MakeDir Directories="$(TestArchiveRuntimeRoot)" /> + <Target Name="CreateTestEnvFile"> + <!-- This target creates one __TestEnv file for the single $(Scenario). --> - <ZipDirectory - SourceDirectory="$(TestHostRootPath)" - DestinationFile="$(TestArchiveRuntimeFile)" - Overwrite="true" /> + <PropertyGroup> + <TestEnvFilePath>$(TestHostRootPath)$(TestEnvFileName)</TestEnvFilePath> + </PropertyGroup> - <Message Importance="High" Text="Zipped correlation payload into $(TestArchiveRuntimeFile)" /> + <ItemGroup> + <_ProjectsToBuild Include="..\tests\testenvironment.proj"> + <Properties>Scenario=$(Scenario);TestEnvFileName=$(TestEnvFilePath);TargetsWindows=$(TargetsWindows)</Properties> + </_ProjectsToBuild> + </ItemGroup> + + <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFile" StopOnFirstFailure="true" /> + + <Message Importance="High" Text="Created $(TestEnvFilePath) for scenario $(Scenario)" /> + <Error Condition="!Exists('$(TestEnvFilePath)')" Text="File $(TestEnvFilePath) not found!" /> </Target> - <PropertyGroup> - <TestAssetBlobFeedUrl>https://dotnetfeed.blob.core.windows.net/dotnet-core</TestAssetBlobFeedUrl> - </PropertyGroup> + <Target Name="CreateAllScenarioTestEnvFiles"> + <!-- This target creates one __TestEnv file for each of the scenarios in the $(Scenarios) comma-separated list. --> - <Target Name="GetTestAssetManifest" - Returns="@(TestAssetBlobInfos)"> + <ItemGroup> + <_Scenario Include="$(Scenarios.Split(','))" /> + <_ProjectsToBuild Include="$(MSBuildProjectFile)"> + <AdditionalProperties>Scenario=%(_Scenario.Identity)</AdditionalProperties> + </_ProjectsToBuild> + </ItemGroup> + + <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFile" StopOnFirstFailure="true" /> + </Target> + + <!-- + Collect all the tasks needed to be run once, to prepare the Helix correlation payload directory. + This just causes its dependent targets to be run. + --> + <Target Name="PrepareCorrelationPayloadDirectory" + DependsOnTargets="CopyRSPFile;CreateAllScenarioTestEnvFiles" > + </Target> + <!-- + Parse the test asset manifest at, e.g., + https://dotnetfeed.blob.core.windows.net/dotnet-core/corefx-tests/4.6.0-preview6.19264.9/Linux.arm64/netcoreapp/corefx-test-assets.xml + to figure out the tests to run. This manifest is created by the corefx official build, for use in the coreclr repo. + + This creates a "TestAssetBlobInfos" item group, which is used to create the Helix work items. + + NOTE: we are running this once for each Scenario, meaning we are reading and parsing it once per scenario. Ideally, we would + do this once and pass it down to the "BuildHelixWorkItems" task later. That's a little difficult given all the re-invocations + of this Project we do. It doesn't seem too expensive anyway. + --> + <Target Name="GetTestAssetManifest" Condition=" '$(UsesHelixSdk)' == 'true' " > <PropertyGroup> <_TargetGroup>netcoreapp</_TargetGroup> <_AssetManifestPath>$(TestAssetBlobFeedUrl)/corefx-tests/$(MicrosoftPrivateCoreFxNETCoreAppVersion)/$(__BuildOS).$(__BuildArch)/$(_TargetGroup)/corefx-test-assets.xml</_AssetManifestPath> @@ -95,22 +197,87 @@ </ParseBuildManifest> </Target> - <Target Name="BuildHelixWorkItems" - Inputs="$(TestArchiveRuntimeFile);@(TestAssetBlobInfos)" - Outputs="@(HelixCorrelationPayload);@(HelixWorkItem)" - DependsOnTargets="CompressRuntimeDirectory;GetTestAssetManifest"> + <!-- + The following item and property groups are setting things needed by Helix, to describe the jobs + we are creating. Ideally, we would put them inside the "BuildHelixWorkItems" task, which is what + we run to create the Helix correlation payload and work item groups. However, the Helix SDK + references some of these in "InitialTargets", and we don't have control over the order of + InitialTargets running, so we need to make these "global". All the "global" item and property + groups will be evaluated before the Helix InitialTargets are run. + --> - <ItemGroup> - <HelixCorrelationPayload - Include="$(TestArchiveRuntimeFile)" /> + <!-- HelixPreCommands is a set of commands run before the work item command. We use it here to inject + setting up the per-scenario environment. + --> + + <ItemGroup Condition=" '$(TargetsWindows)' == 'true' "> + <HelixPreCommand Include="set __TestEnv=%HELIX_CORRELATION_PAYLOAD%\$(TestEnvFileName)" /> + <HelixPreCommand Include="type %__TestEnv%" /> + <HelixPreCommand Include="call %__TestEnv%" /> + <!-- Display the interesting COMPlus variables that are set in the environment --> + <HelixPreCommand Include="set COMPlus" /> + </ItemGroup> + + <ItemGroup Condition=" '$(TargetsWindows)' != 'true' "> + <HelixPreCommand Include="export __TestEnv=$HELIX_CORRELATION_PAYLOAD/$(TestEnvFileName)" /> + <HelixPreCommand Include="cat $__TestEnv" /> + <HelixPreCommand Include=". $__TestEnv" /> <!-- Use "." not "source"; some clients appear to run scripts with "sh" not "bash" --> + <!-- Display the interesting COMPlus variables that are set in the environment --> + <HelixPreCommand Include="printenv | grep COMPlus" /> + </ItemGroup> + + <PropertyGroup> + <HelixPreCommands>@(HelixPreCommand)</HelixPreCommands> + </PropertyGroup> + + <PropertyGroup> + <HelixArchitecture>$(__BuildArch)</HelixArchitecture> + + <HelixConfiguration Condition=" '$(Scenario)' == 'normal' ">$(BuildType)</HelixConfiguration> + <HelixConfiguration Condition=" '$(Scenario)' != 'normal' ">$(BuildType)-$(Scenario)</HelixConfiguration> + + <TestRunNamePrefix>CoreFX </TestRunNamePrefix> + <TestRunNamePrefix Condition=" '$(Scenario)' == 'normal' ">$(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) @ </TestRunNamePrefix> + <TestRunNamePrefix Condition=" '$(Scenario)' != 'normal' ">$(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ </TestRunNamePrefix> + + <!-- REVIEW: it seems like this should be referencing "PublishTestResults" instead (without the underscore) --> + <EnableAzurePipelinesReporter>$(_PublishTestResults)</EnableAzurePipelinesReporter> + <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter> + + <EnableXUnitReporter>true</EnableXUnitReporter> + <FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure> + <FailOnWorkItemFailure>true</FailOnWorkItemFailure> + <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion> + + <!-- Specify the retry policy for Helix jobs --> + <!-- REVIEW: currently don't do any retries + <MaxRetryCount Condition="'$(MaxRetryCount)' == ''">4</MaxRetryCount> + --> + </PropertyGroup> + + <!-- + Create all the Helix data to start a set of jobs. This is invoked once for each Scenario (see + target "RunInParallelForEachScenario"). Create a set of work items, one for each CoreFX test assembly. + All will have the same Scenario specific command line. Note that this target is listed in the + InitialTargets for this Project. This causes it to be invoked (and the Helix data created, + such as the HelixWorkItem item group) before Helix "Test" target is invoked (as a normal target). + We conditionalize this (and its dependent targets) with "UsesHelixSdk=true" so they aren't invoked + during the other invocations of this Project, such as when "RunInParallelForEachScenario" is run. + --> + <Target Name="BuildHelixWorkItems" DependsOnTargets="GetTestAssetManifest" Condition=" '$(UsesHelixSdk)' == 'true' " > + <Error Condition=" '$(TimeoutPerTestCollectionInMinutes)' == '' " Text="TimeoutPerTestCollectionInMinutes not set" /> + + <ItemGroup Condition=" '$(UsesHelixSdk)' == 'true' "> + <HelixCorrelationPayload Include="$(TestHostRootPath)" /> <HelixWorkItem Include="@(TestAssetBlobInfos -> '%(FileName)')"> <PayloadUri>$(TestAssetBlobFeedUrl)/%(Identity)</PayloadUri> <Command>$(HelixCommand)</Command> - <Timeout>$(CommandTimeoutSpan)</Timeout> + <Timeout>$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</Timeout> </HelixWorkItem> </ItemGroup> - - <Message Importance="High" Text="@(HelixWorkItem -> '%(PayloadUri)')" /> </Target> + + <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.targets" Condition=" '$(UsesHelixSdk)' == 'true' " /> + </Project> diff --git a/eng/test-job.yml b/eng/test-job.yml index 010fb61359..522ef8bebd 100644 --- a/eng/test-job.yml +++ b/eng/test-job.yml @@ -26,19 +26,28 @@ jobs: helixType: 'build/tests/' # Compute job name from template parameters - ${{ if eq(parameters.testGroup, 'innerloop') }}: - name: ${{ format('testbuild_pri0_{0}_{1}_{2}_{3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Test Pri0 {0} {1} {2}_{3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}: + name: ${{ format('test_{0}_{1}_{2}_{3}', 'p0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test {0} {1} {2} {3}', 'Pri0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + + ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}: + name: ${{ format('test_{0}_{1}_{2}_{3}', 'p1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test {0} {1} {2} {3}', 'Pri1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.testGroup, 'innerloop') }}: - name: ${{ format('testbuild_pri1_{0}_{1}_{2}_{3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Test Pri1 {0} {1} {2} {3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + ${{ if and(eq(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}: + name: ${{ format('test_{0}_{1}_{2}_{3}_{4}', 'p0', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test {0} {1} {2} {3} {4}', 'Pri0', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + + ${{ if and(ne(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}: + name: ${{ format('test_{0}_{1}_{2}_{3}_{4}', 'p1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test {0} {1} {2} {3} {4}', 'Pri1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} crossrootfsDir: ${{ parameters.crossrootfsDir }} variables: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNet-HelixApi-Access + # Map template parameters to command line arguments - name: priorityArg value: '' @@ -92,11 +101,11 @@ jobs: # this number should be adjusted accordingly. ${{ if eq(parameters.testGroup, 'innerloop') }}: timeoutInMinutes: 240 - ${{ if eq(parameters.testGroup, 'outerloop') }}: + ${{ if in(parameters.testGroup, 'outerloop') }}: timeoutInMinutes: 360 - ${{ if in(parameters.testGroup, 'outerloop-jitstress', 'outerloop-jitstress-isas-arm', 'outerloop-jitstressregs-x86', 'outerloop-jitstressregs', 'outerloop-jitstress2-jitstressregs', 'outerloop-gcstress0x3-gcstress0xc') }}: + ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'gcstress0x3-gcstress0xc') }}: timeoutInMinutes: 480 - ${{ if in(parameters.testGroup, 'outerloop-jitstress-isas-x86', 'outerloop-gcstress-extra', 'outerloop-r2r-extra') }}: + ${{ if in(parameters.testGroup, 'jitstress-isas-x86', 'gcstress-extra', 'r2r-extra') }}: timeoutInMinutes: 600 steps: @@ -149,10 +158,16 @@ jobs: helixBuild: $(Build.BuildNumber) helixSource: $(_HelixSource) - ${{ if eq(parameters.readyToRun, false) }}: - helixType: 'test/functional/cli/' - ${{ if eq(parameters.readyToRun, true) }}: + # REVIEW: not sure why "cli" is part of the names here. Leave it for the ones that already had it, + # but don't add it to new ones. + ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: + helixType: 'test/functional/r2r_corefx/' + ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: + helixType: 'test/functional/corefx/' + ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: helixType: 'test/functional/r2r/cli/' + ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: + helixType: 'test/functional/cli/' helixQueues: ${{ parameters.helixQueues }} @@ -162,22 +177,30 @@ jobs: publishTestResults: true - ${{ if eq(parameters.testGroup, 'innerloop') }}: - # "PerCollection" is time needed for the "biggest" xUnit test collection to complete. - # In case xUnit test wrappers get refactored this number should also be adjusted. + # Set job timeouts + # + # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete. + # In case xUnit test wrappers get refactored this number should also be adjusted. + # + # "timeoutPerTestInMinutes" corresponds to individual test running time. This is implemented by setting + # the __TestTimeout variable, which is later read by the coreclr xunit test wrapper code (the code in the + # xunit test dlls that invokes the actual tests). Note this doesn't apply to CoreFX testing. + + ${{ if and(eq(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}: + timeoutPerTestCollectionInMinutes: 120 + ${{ if and(ne(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}: timeoutPerTestCollectionInMinutes: 30 - # "PerTest" corresponds to individual test running time (i.e. __TestTimeout). timeoutPerTestInMinutes: 10 - ${{ if eq(parameters.testGroup, 'outerloop') }}: + ${{ if in(parameters.testGroup, 'outerloop') }}: timeoutPerTestCollectionInMinutes: 120 timeoutPerTestInMinutes: 10 - ${{ if in(parameters.testGroup, 'outerloop-jitstress', 'outerloop-jitstress-isas-arm', 'outerloop-jitstress-isas-x86', 'outerloop-jitstressregs-x86', 'outerloop-jitstressregs', 'outerloop-jitstress2-jitstressregs') }}: + ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs' ) }}: timeoutPerTestCollectionInMinutes: 120 timeoutPerTestInMinutes: 30 - ${{ if in(parameters.testGroup, 'outerloop-gcstress0x3-gcstress0xc') }}: + ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}: timeoutPerTestCollectionInMinutes: 240 timeoutPerTestInMinutes: 60 - ${{ if in(parameters.testGroup, 'outerloop-gcstress-extra', 'outerloop-r2r-extra') }}: + ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra') }}: timeoutPerTestCollectionInMinutes: 300 timeoutPerTestInMinutes: 90 @@ -188,17 +211,17 @@ jobs: # DotNet-HelixApi-Access variable group helixAccessToken: $(HelixApiAccessToken) - # Sets up the template to run the corefx tests + # Choose which tests to send to Helix: CoreFX or CoreCLR. ${{ if eq(parameters.corefxTests, true) }}: - helixProjectArguments: 'eng/helixcorefxtests.proj /t:Test /p:ArcadeBuild=True' - ${{ if eq(parameters.corefxTests, false) }}: + helixProjectArguments: 'eng/helixcorefxtests.proj' + ${{ if ne(parameters.corefxTests, true) }}: helixProjectArguments: 'tests/helixpublishwitharcade.proj' ${{ if in(parameters.testGroup, 'innerloop', 'outerloop') }}: scenarios: - normal - no_tiered_compilation - ${{ if eq(parameters.testGroup, 'outerloop-jitstress') }}: + ${{ if in(parameters.testGroup, 'jitstress') }}: scenarios: - jitminopts - jitstress1 @@ -207,13 +230,13 @@ jobs: - jitstress2_tiered - zapdisable - tailcallstress - ${{ if eq(parameters.testGroup, 'outerloop-jitstress-isas-arm') }}: + ${{ if in(parameters.testGroup, 'jitstress-isas-arm') }}: scenarios: - jitstress_isas_incompletehwintrinsic - jitstress_isas_nohwintrinsic - jitstress_isas_nohwintrinsic_nosimd - jitstress_isas_nosimd - ${{ if eq(parameters.testGroup, 'outerloop-jitstress-isas-x86') }}: + ${{ if in(parameters.testGroup, 'jitstress-isas-x86') }}: scenarios: - jitstress_isas_incompletehwintrinsic - jitstress_isas_nohwintrinsic @@ -236,7 +259,7 @@ jobs: - jitstress_isas_x86_nosse41 - jitstress_isas_x86_nosse42 - jitstress_isas_x86_nossse3 - ${{ if eq(parameters.testGroup, 'outerloop-jitstressregs-x86') }}: + ${{ if in(parameters.testGroup, 'jitstressregs-x86') }}: scenarios: - jitstressregs1_x86_noavx - jitstressregs2_x86_noavx @@ -246,7 +269,7 @@ jobs: - jitstressregs0x10_x86_noavx - jitstressregs0x80_x86_noavx - jitstressregs0x1000_x86_noavx - ${{ if eq(parameters.testGroup, 'outerloop-jitstressregs') }}: + ${{ if in(parameters.testGroup, 'jitstressregs' ) }}: scenarios: - jitstressregs1 - jitstressregs2 @@ -256,7 +279,7 @@ jobs: - jitstressregs0x10 - jitstressregs0x80 - jitstressregs0x1000 - ${{ if eq(parameters.testGroup, 'outerloop-jitstress2-jitstressregs') }}: + ${{ if in(parameters.testGroup, 'jitstress2-jitstressregs') }}: scenarios: - jitstress2_jitstressregs1 - jitstress2_jitstressregs2 @@ -266,11 +289,11 @@ jobs: - jitstress2_jitstressregs0x10 - jitstress2_jitstressregs0x80 - jitstress2_jitstressregs0x1000 - ${{ if eq(parameters.testGroup, 'outerloop-gcstress0x3-gcstress0xc') }}: + ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}: scenarios: - gcstress0x3 - gcstress0xc - ${{ if eq(parameters.testGroup, 'outerloop-gcstress-extra') }}: + ${{ if in(parameters.testGroup, 'gcstress-extra') }}: scenarios: - heapverify1 - gcstress0xc_zapdisable @@ -279,7 +302,7 @@ jobs: - gcstress0xc_jitstress1 - gcstress0xc_jitstress2 - gcstress0xc_jitminopts_heapverify1 - ${{ if eq(parameters.testGroup, 'outerloop-r2r-extra') }}: + ${{ if in(parameters.testGroup, 'r2r-extra') }}: scenarios: - jitstress1 - jitstress2 @@ -303,9 +326,13 @@ jobs: displayName: Publish Logs inputs: pathtoPublish: $(Build.SourcesDirectory)/bin/Logs - ${{ if eq(parameters.readyToRun, false) }}: - artifactName: ${{ format('testbuild_{0}_{1}_{2}_{3}_Logs', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} - ${{ if eq(parameters.readyToRun, true) }}: - artifactName: ${{ format('testbuild_{0}_{1}_{2}_{3}_r2r_Logs', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: + artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '_r2r_corefx', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: + artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '_corefx', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: + artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '_r2r', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: + artifactName: ${{ format('test{0}_{1}_{2}_{3}_{4}_Logs', '', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} continueOnError: true condition: always() diff --git a/tests/CoreFX/CoreFX.issues.rsp b/tests/CoreFX/CoreFX.issues.rsp index 743f0d59c9..632658c129 100644 --- a/tests/CoreFX/CoreFX.issues.rsp +++ b/tests/CoreFX/CoreFX.issues.rsp @@ -70,4 +70,46 @@ -nomethod System.Tests.StringTests.CasingNegativeTest -nomethod System.Tests.StringTests.CompareNegativeTest -nomethod System.Tests.StringTests.ToLowerNullCulture --nomethod System.Tests.StringTests.ToUpperNullCulture
\ No newline at end of file +-nomethod System.Tests.StringTests.ToUpperNullCulture + +# +# Temporarily disable tests while bringing more CoreFX job configurations online. +# + +-nonamespace System.Net.Http.Functional.Tests +-nonamespace System.Net.Sockets.Tests + +-noclass System.Net.Tests.HttpListenerContextTests +-noclass System.Net.Tests.HttpListenerAuthenticationTests +-noclass System.Net.Tests.HttpListenerResponseTests +-noclass System.Net.Tests.HttpListenerResponseCookiesTests + +-nomethod System.Net.Tests.HttpListenerResponseTests.Redirect_Invoke_SetsRedirectionProperties +-nomethod System.Net.Mail.Tests.SmtpClientTest.TestMailDeliveryAsync + +-nomethod System.Net.Security.Tests.LoggingTest.EventSource_EventsRaisedAsExpected +-nomethod System.Net.Security.Tests.SslStreamSniTest.SslStream_NoSniFromClient_CallbackReturnsNull + +# System.Linq.Expressions.Tests: timeout on Linux/x64 +-nonamespace System.Linq.Expressions.Tests + +# assert/crashes +-nomethod System.Tests.RealFormatterTestsBase.TestFormatterSingle_F20 +-noclass System.Tests.RealFormatterTestsBase +-noclass System.Tests.RealFormatterTests +-noclass System.Buffers.Text.Tests.RealFormatterTests + +# System.Runtime.Serialization.Formatters.Tests: timeouts +-nonamespace System.Runtime.Serialization.Formatters.Tests + +-nomethod Generic.Dictionary.DictionaryConcurrentAccessDetectionTests.DictionaryConcurrentAccessDetection_ReferenceTypeKey +-nomethod Generic.Dictionary.DictionaryConcurrentAccessDetectionTests.DictionaryConcurrentAccessDetection_ValueTypeKey +-nomethod MonoTests.System.Runtime.Caching.MemoryCacheTest.Contains +-nomethod System.Numerics.Tests.ToStringTest.RunRegionSpecificStandardFormatToStringTests +-nomethod System.Text.RegularExpressions.Tests.RegexMatchTests.Match_ExcessPrefix + +# Assert with JITMinOpts=1: https://github.com/dotnet/coreclr/issues/25070 +-nonamespace Microsoft.VisualBasic.Tests + +# Assert: https://github.com/dotnet/coreclr/issues/25050 +-nonamespace System.Data.Common.Tests diff --git a/tests/helixpublishwitharcade.proj b/tests/helixpublishwitharcade.proj index 74d5eb78ac..c7994c3293 100644 --- a/tests/helixpublishwitharcade.proj +++ b/tests/helixpublishwitharcade.proj @@ -31,6 +31,8 @@ </_PropertiesToPass> </PropertyGroup> + <Error Condition="'$(_Scenarios)' == ''" Text="_Scenarios not set" /> + <MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" /> <MSBuild Projects="$(MSBuildProjectFile)" Targets="PreparePayloadsDirectories" Properties="Scenarios=$(_Scenarios)" StopOnFirstFailure="true" /> diff --git a/tests/testenvironment.proj b/tests/testenvironment.proj index ee6c039e65..7a0aa63272 100644 --- a/tests/testenvironment.proj +++ b/tests/testenvironment.proj @@ -154,7 +154,6 @@ <Target Name="CreateTestEnvFile" DependsOnTargets="OutputCOMPlusVariables"> <ItemGroup Condition="'$(TargetsWindows)' == 'true'"> - <_TestEnvFileLine Include="@echo off" /> <_TestEnvFileLine Include="@(_COMPlusVariable->'set %(Identity)=%(Value)')" /> </ItemGroup> |