Improve corefx testing (#24962)

CoreFX tests are now used to test CoreCLR on Windows/x64 and Linux/x64.
PR testing includes normal and no-tiered modes. CI testing will include
all JIT stress modes that apply to CoreCLR testing. Many tests have been
disabled to allow the jobs to (mostly) run error-free.

Support for other platforms requires fixes to the testhost generation.

1 files changed, 221 insertions, 54 deletions

diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj
index 166eafe6b9..980c5c41ee 100644
--- a/eng/helixcorefxtests.proj
+++ b/eng/helixcorefxtests.proj
@@ -1,10 +1,77 @@
-<Project InitialTargets="BuildHelixWorkItems" Sdk="Microsoft.DotNet.Helix.Sdk">
+<Project InitialTargets="BuildHelixWorkItems" DefaultTargets="RunInParallelForEachScenario">
+
+  <!-- This project uses the helix SDK, documented at
+       https://github.com/dotnet/arcade/tree/master/src/Microsoft.DotNet.Helix/Sdk,
+       to send test jobs to helix. -->
+
+  <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.props" Condition=" '$(UsesHelixSdk)' == 'true' " />
+
+  <!-- Import Microsoft.DotNet.Build.Tasks.Feed.targets for `ParseBuildManifest` -->
+  <Import Project="..\dir.props" Condition=" '$(NuGetPackageRoot)'=='' "/>
   <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />
 
-  <PropertyGroup>
-    <!-- Set the TargetFramework just to make the SDK happy -->
-    <TargetFramework>netstandard2.0</TargetFramework>
-  </PropertyGroup>
+  <!-- This project is copies much logic from helixpublishwitharcade.proj, used to send coreclr tests to Helix. -->
+
+  <!--
+    "RunInParallelForEachScenario" is the "root" target for this Project. It invokes other targets to set up the
+    information needed to submit a job to Helix. It does so by "recursively" invoking this project file with
+    different properties and targets to run, as well as invoking MSBuild on other project files. In particular,
+    it first creates the "correlation payload directory", which is the directory of files used by all Helix
+    submissions. Then, it recursively invokes this Project once per scenario (e.g., normal Pri-1 run, JitStress=2
+    run, JitStressRegs=8 run, etc.), creating a new set of Helix work items for each scenario. For the CoreFX
+    tests, we create one Helix work item for each CoreFX test assembly; there is currently no concept of
+    grouping multiple assemblies into a single Helix work item to, for instance, consolidate multiple short-running
+    assemblies together. (Note that this grouping is done for the CoreCLR tests in helixpublishwitharcade.proj.)
+  -->
+  <Target Name="RunInParallelForEachScenario">
+    <PropertyGroup>
+      <!-- This specifies what properties are needed to be passed down as global properties to a child project. -->
+
+      <_PropertiesToPass>
+        __BuildArch=$(__BuildArch);
+        __BuildOS=$(__BuildOS);
+        __BuildType=$(__BuildType);
+        Creator=$(_Creator);
+        HelixAccessToken=$(_HelixAccessToken);
+        HelixBuild=$(_HelixBuild);
+        HelixSource=$(_HelixSource);
+        HelixTargetQueues=$(_HelixTargetQueues);
+        HelixType=$(_HelixType);
+        PublishTestResults=$(_PublishTestResults);
+        RunCrossGen=$(_RunCrossGen);
+        TimeoutPerTestCollectionInMinutes=$(_TimeoutPerTestCollectionInMinutes)
+      </_PropertiesToPass>
+    </PropertyGroup>
+
+    <Error Condition="'$(_Scenarios)' == ''" Text="_Scenarios not set" />
+
+    <MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" Properties="Scenarios=$(_Scenarios)" />
+
+    <ItemGroup>
+      <_Scenarios Include="$(_Scenarios.Split(','))" />
+
+      <!-- MSBuild creates a new instance of the project for each %(_Scenarios.Identity) and can build them in parallel. -->
+      <_ProjectsToBuild Include="$(MSBuildProjectFile)">
+        <AdditionalProperties>$(_PropertiesToPass);Scenario=%(_Scenarios.Identity)</AdditionalProperties>
+      </_ProjectsToBuild>
+    </ItemGroup>
+
+    <PropertyGroup>
+      <_BuildInParallel>false</_BuildInParallel>
+      <_BuildInParallel Condition=" '@(_ProjectsToBuild->Count())' &gt; '1' ">true</_BuildInParallel>
+    </PropertyGroup>
+
+    <!-- Invoke MSBuild on this project file once for each Scenario (because of the "batching" defined in "_ProjectsToBuild").
+         Set "UsesHelixSdk=true" to indicate we want to invoke Helix on this invocation: create the Helix work items and start
+         the jobs. This is done by invoking the "Test" Helix target.
+    -->
+    <MSBuild Projects="@(_ProjectsToBuild)" Targets="Test" BuildInParallel="$(_BuildInParallel)" StopOnFirstFailure="false" Properties="UsesHelixSdk=true" />
+  </Target>
+
+  <!-- Define a set of properties that are input to the Project, and that must be passed down to child processes.
+      (See "_PropertiesToPass" in target "RunInParallelForEachScenario".) If something can be computed, such as
+      properties needed by Helix, they should probably be set in target "BuildHelixWorkItems" instead.
+  -->
 
   <PropertyGroup>
     <Creator>$(_Creator)</Creator>
@@ -13,8 +80,6 @@
     <HelixSource>$(_HelixSource)</HelixSource>
     <HelixTargetQueues>$(_HelixTargetQueues)</HelixTargetQueues>
     <HelixType>$(_HelixType)</HelixType>
-    <HelixArchitecture>$(__BuildArch)</HelixArchitecture>
-    <HelixConfiguration>$(__BuildType)</HelixConfiguration>
 
     <!--
       TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade.
@@ -24,67 +89,104 @@
     <TestWorkingDir Condition="'$(__TestWorkingDir)'==''">$(RootBinDir)tests\$(__BuildOS).$(__BuildArch).$(__BuildType)\</TestWorkingDir>
     <TargetsWindows Condition="'$(__BuildOS)' == 'Windows_NT'">true</TargetsWindows>
 
+    <!-- We have a single, universal exclusion file for all CoreFX test exclusions, for CoreFX tests run
+         in the CoreCLR repo. Note in particular that we don't have a unique exclusion file for each test
+         assembly, or for each processor architecture, or for each operating system, or for each stress
+         scenario type. This means that we generally exclude more tests than is strictly necessary for
+         any particular job. However, this mechanism is super simple; we don't need to manage fine-
+         grained exclusions; and there really shouldn't be any (or many) exclusions anyway.
+    -->
+    <CoreFXTestExclusionFile>$(ProjectDir)tests\CoreFX\CoreFX.issues.rsp</CoreFXTestExclusionFile>
+
     <TestHostRootPath>$(TestWorkingDir)testhost\</TestHostRootPath>
     <TestArchiveRuntimeRoot>$(TestWorkingDir)helix\</TestArchiveRuntimeRoot>
-    <TestArchiveRuntimeFile>$(TestArchiveRuntimeRoot)testhost-runtime.zip</TestArchiveRuntimeFile>
-    <HelixCorrelationPayload></HelixCorrelationPayload>
 
-    <EnableAzurePipelinesReporter>$(_PublishTestResults)</EnableAzurePipelinesReporter>
-    <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter>
-    <EnableXUnitReporter>true</EnableXUnitReporter>
-    <FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure>
-    <FailOnWorkItemFailure>true</FailOnWorkItemFailure>
-
-    <TimeoutInSeconds Condition="'$(TimeoutInSeconds)' == ''">600</TimeoutInSeconds>
-    <CommandTimeoutSpan>$([System.TimeSpan]::FromSeconds($(TimeoutInSeconds)))</CommandTimeoutSpan>
-    <MaxRetryCount Condition="'$(MaxRetryCount)' == ''">4</MaxRetryCount>
-    <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
+    <TestAssetBlobFeedUrl>https://dotnetfeed.blob.core.windows.net/dotnet-core</TestAssetBlobFeedUrl>
   </PropertyGroup>
 
   <PropertyGroup>
     <!--
-      For windows we need to use call, since the command is going to be called from a bat script created by Helix
-      and we exit /b at the end of RunTests.cmd, Helix runs some other commands after ours within the bat script,
-      if we don't use call, then we cause the parent script to exit, and anything after will not be executed.
-      The arguments passed in to the run script in order are the runtime directory, the dotnet root directory (for
-      helix submissions same as the runtime directory) and the global tools directory.
+      For Windows, we need to use "call", since the command is going to be called from a batch script created by Helix.
+      We "exit /b" at the end of RunTests.cmd. Helix runs some other commands after ours within the batch script,
+      so if we don't use call, then we cause the parent script to exit, and anything after will not be executed.
+      The arguments passed in to the run script in are the runtime directory (-r) and the exclusion response file
+      passed to xunit (rsp-file).
     -->
     <HelixCommand Condition="'$(TargetsWindows)' == 'true'">call RunTests.cmd -r %HELIX_CORRELATION_PAYLOAD% --rsp-file %HELIX_CORRELATION_PAYLOAD%\CoreFX.issues.rsp </HelixCommand>
     <HelixCommand Condition="'$(TargetsWindows)' != 'true'">./RunTests.sh -r $HELIX_CORRELATION_PAYLOAD --rsp-file $HELIX_CORRELATION_PAYLOAD/CoreFX.issues.rsp </HelixCommand>
   </PropertyGroup>
 
-  <ItemGroup>
-    <TestArchiveRuntimeInputs Include="$(TestHostRootPath)**/*" />
-  </ItemGroup>
-
+  <!--
+    Copy the CoreFX test exclusion response file (which is passed to the xunit console runner) to someplace we can
+    access it during the test run. We currently just copy it to the root of the generated .NET Core testhost
+    directory, which we use as the root of the correlation directory.
+  -->
   <Target Name="CopyRSPFile">
     <Copy
-      SourceFiles="$(ProjectDir)\tests\CoreFX\CoreFX.issues.rsp"
+      SourceFiles="$(CoreFXTestExclusionFile)"
       DestinationFolder="$(TestHostRootPath)" />
+
+    <Message Importance="High" Text="Copied $(CoreFXTestExclusionFile) into $(TestHostRootPath)" />
   </Target>
 
-  <Target Name="CompressRuntimeDirectory"
-          Inputs="@(TestArchiveRuntimeInputs)"
-          Outputs="$(TestArchiveRuntimeFile)"
-          DependsOnTargets="CopyRSPFile" >
+  <PropertyGroup>
+    <!-- Set the name of the scenario file. Note that this is only used in invocations where $(Scenario) is set. -->
+    <TestEnvFileName Condition=" '$(TargetsWindows)' == 'true' ">SetStressModes_$(Scenario).cmd</TestEnvFileName>
+    <TestEnvFileName Condition=" '$(TargetsWindows)' != 'true' ">SetStressModes_$(Scenario).sh</TestEnvFileName>
+  </PropertyGroup>
 
-    <MakeDir Directories="$(TestArchiveRuntimeRoot)" />
+  <Target Name="CreateTestEnvFile">
+    <!-- This target creates one __TestEnv file for the single $(Scenario). -->
 
-    <ZipDirectory
-        SourceDirectory="$(TestHostRootPath)"
-        DestinationFile="$(TestArchiveRuntimeFile)"
-        Overwrite="true" />
+    <PropertyGroup>
+      <TestEnvFilePath>$(TestHostRootPath)$(TestEnvFileName)</TestEnvFilePath>
+    </PropertyGroup>
 
-    <Message Importance="High" Text="Zipped correlation payload into $(TestArchiveRuntimeFile)" />
+    <ItemGroup>
+      <_ProjectsToBuild Include="..\tests\testenvironment.proj">
+        <Properties>Scenario=$(Scenario);TestEnvFileName=$(TestEnvFilePath);TargetsWindows=$(TargetsWindows)</Properties>
+      </_ProjectsToBuild>
+    </ItemGroup>
+
+    <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFile" StopOnFirstFailure="true" />
+
+    <Message Importance="High" Text="Created $(TestEnvFilePath) for scenario $(Scenario)" />
+    <Error Condition="!Exists('$(TestEnvFilePath)')" Text="File $(TestEnvFilePath) not found!" />
   </Target>
 
-  <PropertyGroup>
-    <TestAssetBlobFeedUrl>https://dotnetfeed.blob.core.windows.net/dotnet-core</TestAssetBlobFeedUrl>
-  </PropertyGroup>
+  <Target Name="CreateAllScenarioTestEnvFiles">
+    <!-- This target creates one __TestEnv file for each of the scenarios in the $(Scenarios) comma-separated list. -->
 
-  <Target Name="GetTestAssetManifest" 
-          Returns="@(TestAssetBlobInfos)">
+    <ItemGroup>
+      <_Scenario Include="$(Scenarios.Split(','))" />
+      <_ProjectsToBuild Include="$(MSBuildProjectFile)">
+        <AdditionalProperties>Scenario=%(_Scenario.Identity)</AdditionalProperties>
+      </_ProjectsToBuild>
+    </ItemGroup>
+
+    <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFile" StopOnFirstFailure="true" />
+  </Target>
+
+  <!--
+    Collect all the tasks needed to be run once, to prepare the Helix correlation payload directory.
+    This just causes its dependent targets to be run.
+  -->
+  <Target Name="PrepareCorrelationPayloadDirectory"
+          DependsOnTargets="CopyRSPFile;CreateAllScenarioTestEnvFiles" >
+  </Target>
 
+  <!--
+    Parse the test asset manifest at, e.g.,
+        https://dotnetfeed.blob.core.windows.net/dotnet-core/corefx-tests/4.6.0-preview6.19264.9/Linux.arm64/netcoreapp/corefx-test-assets.xml
+    to figure out the tests to run. This manifest is created by the corefx official build, for use in the coreclr repo.
+    
+    This creates a "TestAssetBlobInfos" item group, which is used to create the Helix work items.
+
+    NOTE: we are running this once for each Scenario, meaning we are reading and parsing it once per scenario. Ideally, we would
+    do this once and pass it down to the "BuildHelixWorkItems" task later. That's a little difficult given all the re-invocations
+    of this Project we do. It doesn't seem too expensive anyway.
+  -->
+  <Target Name="GetTestAssetManifest" Condition=" '$(UsesHelixSdk)' == 'true' " >
     <PropertyGroup>
       <_TargetGroup>netcoreapp</_TargetGroup>
       <_AssetManifestPath>$(TestAssetBlobFeedUrl)/corefx-tests/$(MicrosoftPrivateCoreFxNETCoreAppVersion)/$(__BuildOS).$(__BuildArch)/$(_TargetGroup)/corefx-test-assets.xml</_AssetManifestPath>
@@ -95,22 +197,87 @@
     </ParseBuildManifest>
   </Target>
 
-  <Target Name="BuildHelixWorkItems"
-          Inputs="$(TestArchiveRuntimeFile);@(TestAssetBlobInfos)"
-          Outputs="@(HelixCorrelationPayload);@(HelixWorkItem)"
-          DependsOnTargets="CompressRuntimeDirectory;GetTestAssetManifest">
+  <!--
+    The following item and property groups are setting things needed by Helix, to describe the jobs
+    we are creating. Ideally, we would put them inside the "BuildHelixWorkItems" task, which is what
+    we run to create the Helix correlation payload and work item groups. However, the Helix SDK
+    references some of these in "InitialTargets", and we don't have control over the order of
+    InitialTargets running, so we need to make these "global". All the "global" item and property
+    groups will be evaluated before the Helix InitialTargets are run.
+  -->
 
-    <ItemGroup>
-      <HelixCorrelationPayload 
-        Include="$(TestArchiveRuntimeFile)" />
+  <!-- HelixPreCommands is a set of commands run before the work item command. We use it here to inject
+       setting up the per-scenario environment.
+  -->
+
+  <ItemGroup Condition=" '$(TargetsWindows)' == 'true' ">
+    <HelixPreCommand Include="set __TestEnv=%HELIX_CORRELATION_PAYLOAD%\$(TestEnvFileName)" />
+    <HelixPreCommand Include="type %__TestEnv%" />
+    <HelixPreCommand Include="call %__TestEnv%" />
+    <!-- Display the interesting COMPlus variables that are set in the environment -->
+    <HelixPreCommand Include="set COMPlus" />
+  </ItemGroup>
+
+  <ItemGroup Condition=" '$(TargetsWindows)' != 'true' ">
+    <HelixPreCommand Include="export __TestEnv=$HELIX_CORRELATION_PAYLOAD/$(TestEnvFileName)" />
+    <HelixPreCommand Include="cat $__TestEnv" />
+    <HelixPreCommand Include=". $__TestEnv" /> <!-- Use "." not "source"; some clients appear to run scripts with "sh" not "bash" -->
+    <!-- Display the interesting COMPlus variables that are set in the environment -->
+    <HelixPreCommand Include="printenv | grep COMPlus" />
+  </ItemGroup>
+
+  <PropertyGroup>
+    <HelixPreCommands>@(HelixPreCommand)</HelixPreCommands>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <HelixArchitecture>$(__BuildArch)</HelixArchitecture>
+
+    <HelixConfiguration Condition=" '$(Scenario)' == 'normal' ">$(BuildType)</HelixConfiguration>
+    <HelixConfiguration Condition=" '$(Scenario)' != 'normal' ">$(BuildType)-$(Scenario)</HelixConfiguration>
+
+    <TestRunNamePrefix>CoreFX </TestRunNamePrefix>
+    <TestRunNamePrefix Condition=" '$(Scenario)' == 'normal' ">$(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) @ </TestRunNamePrefix>
+    <TestRunNamePrefix Condition=" '$(Scenario)' != 'normal' ">$(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ </TestRunNamePrefix>
+
+    <!-- REVIEW: it seems like this should be referencing "PublishTestResults" instead (without the underscore) -->
+    <EnableAzurePipelinesReporter>$(_PublishTestResults)</EnableAzurePipelinesReporter>
+    <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter>
+
+    <EnableXUnitReporter>true</EnableXUnitReporter>
+    <FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure>
+    <FailOnWorkItemFailure>true</FailOnWorkItemFailure>
+    <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
+
+    <!-- Specify the retry policy for Helix jobs -->
+    <!-- REVIEW: currently don't do any retries
+    <MaxRetryCount Condition="'$(MaxRetryCount)' == ''">4</MaxRetryCount>
+    -->
+  </PropertyGroup>
+
+  <!--
+    Create all the Helix data to start a set of jobs. This is invoked once for each Scenario (see
+    target "RunInParallelForEachScenario"). Create a set of work items, one for each CoreFX test assembly.
+    All will have the same Scenario specific command line.  Note that this target is listed in the
+    InitialTargets for this Project. This causes it to be invoked (and the Helix data created,
+    such as the HelixWorkItem item group) before Helix "Test" target is invoked (as a normal target).
+    We conditionalize this (and its dependent targets) with "UsesHelixSdk=true" so they aren't invoked
+    during the other invocations of this Project, such as when "RunInParallelForEachScenario" is run.
+  -->
+  <Target Name="BuildHelixWorkItems" DependsOnTargets="GetTestAssetManifest" Condition=" '$(UsesHelixSdk)' == 'true' " >
+    <Error Condition=" '$(TimeoutPerTestCollectionInMinutes)' == '' " Text="TimeoutPerTestCollectionInMinutes not set" />
+
+    <ItemGroup Condition=" '$(UsesHelixSdk)' == 'true' ">
+      <HelixCorrelationPayload Include="$(TestHostRootPath)" />
 
       <HelixWorkItem Include="@(TestAssetBlobInfos -> '%(FileName)')">
         <PayloadUri>$(TestAssetBlobFeedUrl)/%(Identity)</PayloadUri>
         <Command>$(HelixCommand)</Command>
-        <Timeout>$(CommandTimeoutSpan)</Timeout>
+        <Timeout>$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</Timeout>
       </HelixWorkItem>
     </ItemGroup>
-
-    <Message Importance="High" Text="@(HelixWorkItem -> '%(PayloadUri)')" />
   </Target>
+
+  <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.targets" Condition=" '$(UsesHelixSdk)' == 'true' " />
+
 </Project>