Use MSBuild to generate __TestEnv files (#21966)

3 files changed, 194 insertions, 135 deletions

diff --git a/tests/helixpublishwitharcade.proj b/tests/helixpublishwitharcade.proj
index 6a4622ee59..bd67b04546 100644
--- a/tests/helixpublishwitharcade.proj
+++ b/tests/helixpublishwitharcade.proj
@@ -8,7 +8,8 @@
 
   <PropertyGroup>
     <HelixArchitecture>$(BuildArch)</HelixArchitecture>
-    <HelixConfiguration>$(BuildType)</HelixConfiguration>
+    <HelixConfiguration Condition=" '$(Scenario)' == 'normal' ">$(BuildType)</HelixConfiguration>
+    <HelixConfiguration Condition=" '$(Scenario)' != 'normal' ">$(BuildType)-$(Scenario)</HelixConfiguration>
 
     <IsExternal>$(_IsExternal)</IsExternal>
     <IsExternal Condition=" '$(IsExternal)' == '' ">true</IsExternal>
@@ -29,57 +30,70 @@
 
     <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
     <CoreRootDirectory>$(TestWorkingDir)\Tests\Core_Root</CoreRootDirectory>
-  </PropertyGroup>
-
-  <PropertyGroup Condition=" '$(TargetsWindows)' == 'true' ">
-    <HelixPythonPath>%HELIX_PYTHONPATH%</HelixPythonPath>
-    <RunnerScript>%HELIX_CORRELATION_PAYLOAD%\runtest_helix.py</RunnerScript>
-  </PropertyGroup>
-
-  <PropertyGroup Condition=" '$(TargetsWindows)' != 'true' ">
-    <HelixPythonPath>$HELIX_PYTHONPATH</HelixPythonPath>
-    <RunnerScript>$HELIX_CORRELATION_PAYLOAD/runtest_helix.py</RunnerScript>
+    <TestEnvFileExtension Condition=" '$(TargetsWindows)' == 'true' ">.cmd</TestEnvFileExtension>
+    <TestEnvFileExtension Condition=" '$(TargetsWindows)' != 'true' ">.sh</TestEnvFileExtension>
   </PropertyGroup>
 
   <ItemGroup>
     <HelixCorrelationPayload Include="$(CoreRootDirectory)" />
     <XUnitWrapperDlls Include="$(TestWorkingDir)\**\*.XUnitWrapper.dll" />
+    <Scenarios Include="$(_Scenarios.Split(','))" />
   </ItemGroup>
 
-  <Target Name="CopyRunnerScriptToCoreRootDirectory" BeforeTargets="SubmitTestsToHelix">
-    <Copy SourceFiles="$(MSBuildThisFileDirectory)\runtest_helix.py" DestinationFolder="$(CoreRootDirectory)" />
+  <Target Name="CreateTestEnvFiles">
+    <!-- This target will create one __TestEnv file per XUnitWrapper/Scenario combination -->
+    <ItemGroup>
+      <_XUnitWrapperDirectoryWithScenario Include="@(XUnitWrapperDlls->'%(RootDir)%(Directory)')">
+        <Scenario>%(Scenarios.Identity)</Scenario>
+      </_XUnitWrapperDirectoryWithScenario>
+
+      <_TestEnvProjectsToBuild Include=".\testenvironment.proj">
+        <Properties>Scenario=%(_XUnitWrapperDirectoryWithScenario.Scenario);TestEnvFileName=%(Identity)\SetStressModes_%(Scenario)$(TestEnvFileExtension);TargetsWindows=$(TargetsWindows)</Properties>
+      </_TestEnvProjectsToBuild>
+    </ItemGroup>
+
+    <MSBuild Projects="@(_TestEnvProjectsToBuild)" Targets="CreateTestEnvFile" BuildInParallel="false" StopOnFirstFailure="true" UnloadProjectsOnCompletion="true" />
   </Target>
 
-  <Target Name="SubmitTestsToHelix">
+  <Target Name="SubmitTestsToHelix" DependsOnTargets="CreateTestEnvFiles">
     <ItemGroup>
-      <Scenarios Include="$(_Scenarios.Split(','))" />
-      <ProjectsToBuild Include="$(MSBuildProjectFile)">
-        <Properties>Scenario=%(_Scenarios.Identity)</Properties>
-      </ProjectsToBuild>
+      <_ProjectsToBuild Include="$(MSBuildProjectFile)">
+        <Properties>Scenario=%(Scenarios.Identity)</Properties>
+      </_ProjectsToBuild>
     </ItemGroup>
+
     <PropertyGroup>
-      <BuildInParallel>false</BuildInParallel>
-      <BuildInParallel Condition=" '@(ProjectsToBuild->Count())' > '1' ">true</BuildInParallel>
+      <_BuildInParallel>false</_BuildInParallel>
+      <_BuildInParallel Condition=" '@(_ProjectsToBuild->Count())' > '1' ">true</_BuildInParallel>
     </PropertyGroup>
-    <MSBuild Projects="@(ProjectsToBuild)" Targets="Test" BuildInParallel="$(BuildInParallel)" StopOnFirstFailure="false" UnloadProjectsOnCompletion="true" />
+
+    <MSBuild Projects="@(_ProjectsToBuild)" Targets="Test" BuildInParallel="$(_BuildInParallel)" StopOnFirstFailure="false" UnloadProjectsOnCompletion="true" />
   </Target>
 
   <Target Name="BuildHelixWorkItem" BeforeTargets="Test">
     <PropertyGroup>
-      <!-- The "normal" scenario is just a way to include the default
-           (empty) scenario when specifying multiple scenarios at
-           once. From here, on, treat it as the empty scenario so that
-           job names will not have a scenario prefix and the runtest
-           script doesn't have to define a "normal" scenario. -->
-      <Scenario Condition=" '$(Scenario)' == 'normal' "></Scenario>
-      <TestRunNamePrefix Condition=" '$(Scenario)' == '' ">$(BuildOS) $(BuildArch) $(BuildType) @ </TestRunNamePrefix>
-      <TestRunNamePrefix Condition=" '$(Scenario)' != '' ">$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ </TestRunNamePrefix>
+      <TestRunNamePrefix Condition=" '$(Scenario)' == 'normal' ">$(BuildOS) $(BuildArch) $(BuildType) @ </TestRunNamePrefix>
+      <TestRunNamePrefix Condition=" '$(Scenario)' != 'normal' ">$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ </TestRunNamePrefix>
+      <_XUnitRunnerArgs>-parallel collections -nocolor -noshadow -xml testResults.xml -notrait category=outerloop -notrait category=failing</_XUnitRunnerArgs>
+    </PropertyGroup>
+
+    <PropertyGroup Condition=" '$(TargetsWindows)' == 'true' ">
+      <_CoreRun>%CORE_ROOT%\CoreRun.exe</_CoreRun>
+      <_XUnitRunnerDll>%CORE_ROOT%\xunit.console.dll</_XUnitRunnerDll>
+      <_HelixPreCommands>set CORE_ROOT=%HELIX_CORRELATION_PAYLOAD%;set __TestEnv=%HELIX_WORKITEM_PAYLOAD%\SetStressModes_$(Scenario)$(TestEnvFileExtension);type %__TestEnv%</_HelixPreCommands>
     </PropertyGroup>
+
+    <PropertyGroup Condition=" '$(TargetsWindows)' != 'true' ">
+      <_CoreRun>$CORE_ROOT/corerun</_CoreRun>
+      <_XUnitRunnerDll>$CORE_ROOT/xunit.console.dll</_XUnitRunnerDll>
+      <_HelixPreCommands>export CORE_ROOT=$HELIX_CORRELATION_PAYLOAD;export __TestEnv=$HELIX_WORKITEM_PAYLOAD/SetStressModes_$(Scenario)$(TestEnvFileExtension);cat $__TestEnv</_HelixPreCommands>
+    </PropertyGroup>
+
     <ItemGroup>
       <HelixWorkItem Include="@(XUnitWrapperDlls->'%(FileName)'->Replace('.XUnitWrapper', ''))">
         <PayloadDirectory>%(RootDir)%(Directory)</PayloadDirectory>
-        <Command Condition=" '$(Scenario)' == '' ">$(HelixPythonPath) $(RunnerScript) -wrapper %(FileName)%(Extension)</Command>
-        <Command Condition=" '$(Scenario)' != '' ">$(HelixPythonPath) $(RunnerScript) -wrapper %(FileName)%(Extension) -scenario $(Scenario)</Command>
+        <PreCommands>$(_HelixPreCommands)</PreCommands>
+        <Command>$(_CoreRun) $(_XUnitRunnerDll) %(FileName)%(Extension) $(_XUnitRunnerArgs)</Command>
         <Timeout>$([System.TimeSpan]::FromMinutes($(TimeoutInMinutes)))</Timeout>
       </HelixWorkItem>
     </ItemGroup>
diff --git a/tests/runtest_helix.py b/tests/runtest_helix.py
deleted file mode 100755
index 10ef1a4853..0000000000
--- a/tests/runtest_helix.py
+++ /dev/null
@@ -1,104 +0,0 @@
-#!/usr/bin/env python
-
-# This script runs tests in helix. It defines a set of test scenarios
-# that enable various combinations of runtime configurations to be set
-# via the test process environment.
-
-# This script calls "corerun xunit.console.dll xunitwrapper.dll",
-# where the xunitwrapper.dll will run a .sh/.cmd script per test in a
-# separate process. This process will have the scenario environment
-# variables set, but the xunit process will not.
-
-# TODO: Factor out logic common with runtest.py
-
-import argparse
-import subprocess
-import os
-import sys
-import tempfile
-
-test_scenarios = {
-    "jitstress2": { "COMPlus_TieredCompilation": "0",
-                    "COMPlus_JitStress": "2" },
-}
-
-if sys.platform.startswith('linux') or sys.platform == "darwin":
-    platform_type = "unix"
-elif sys.platform == "win32":
-    platform_type = "windows"
-else:
-    print("unknown os: %s" % sys.platform)
-    sys.exit(1)
-
-def get_testenv_script(env_dict):
-    if platform_type == "unix":
-        return ''.join([ "export %s=%s%s" % (k, v, os.linesep) for k, v in env_dict.items() ])
-    elif platform_type == "windows":
-        return ''.join([ "set %s=%s%s" % (k, v, os.linesep) for k, v in env_dict.items() ])
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Parse arguments")
-    parser.add_argument("-scenario", dest="scenario", default=None)
-    parser.add_argument("-wrapper", dest="wrapper", default=None, required=True)
-    args = parser.parse_args()
-    scenario = args.scenario
-    wrapper = args.wrapper
-
-    if not "HELIX_CORRELATION_PAYLOAD" in os.environ:
-        print("HELIX_CORRELATION_PAYLOAD must be defined in environment")
-        sys.exit(1)
-
-    if not "HELIX_WORKITEM_PAYLOAD" in os.environ:
-        print("HELIX_WORKITEM_PAYLOAD must be defined in environment")
-        sys.exit(1)
-
-    core_root = os.environ["HELIX_CORRELATION_PAYLOAD"]
-
-    if platform_type == "unix":
-        corerun = os.path.join(core_root, "corerun")
-    else:
-        corerun = os.path.join(core_root, "corerun.exe")
-
-    # Unlike the old test wrapper, this runs xunit.console.dll from
-    # the correlation payload. This removes the need for redundant
-    # copies of the console runner in each test directory.
-    command = [corerun,
-               os.path.join(os.environ["HELIX_CORRELATION_PAYLOAD"], "xunit.console.dll"),
-               os.path.join(os.environ["HELIX_WORKITEM_PAYLOAD"], wrapper),
-               "-noshadow",
-               "-xml", "testResults.xml",
-               "-notrait", "category=outerloop",
-               "-notrait", "category=failing"]
-
-    if scenario is None:
-        print("CORE_ROOT=%s" % core_root)
-        os.environ["CORE_ROOT"] = core_root
-
-        print("BEGIN EXECUTION")
-        print(' '.join(command))
-        proc = subprocess.Popen(command)
-        proc.communicate()
-        print("Finished running tests. Exit code = %d" % proc.returncode)
-        sys.exit(proc.returncode)
-    else:
-        print("scenario: %s" % scenario)
-        with tempfile.NamedTemporaryFile(mode="w") as testenv:
-            testenv.write(get_testenv_script(test_scenarios[scenario]))
-            testenv.flush()
-
-            print("__TestEnv=%s" % testenv.name)
-            os.environ["__TestEnv"] = testenv.name
-
-            with open(testenv.name) as testenv_written:
-                contents = testenv_written.read()
-                print(contents)
-
-            print("CORE_ROOT=%s" % core_root)
-            os.environ["CORE_ROOT"] = core_root
-
-            print("BEGIN EXECUTION")
-            print(' '.join(command))
-            proc = subprocess.Popen(command)
-            proc.communicate()
-            print("Finished running tests. Exit code = %d" % proc.returncode)
-            sys.exit(proc.returncode)
diff --git a/tests/testenvironment.proj b/tests/testenvironment.proj
new file mode 100644
index 0000000000..f99209ba03
--- /dev/null
+++ b/tests/testenvironment.proj
@@ -0,0 +1,149 @@
+<Project DefaultTargets="CreateTestEnvFile">
+  <!-- This project creates a .cmd or .sh file that sets the environment variables corresponding to the specified Scenario property.
+       For example, to create SetStressModes.cmd file for scenario "jitstress1" you can use the following command:
+
+       dotnet msbuild testenvironment.proj /p:TestEnvFileName=SetStressModes.cmd /p:Scenario=jitstress1 /p:TargetsWindows=true
+  -->
+
+
+  <!-- COMPlus_* variables that can be specified for a test scenario -->
+  <PropertyGroup>
+    <COMPlusVariables>
+      COMPlus_TieredCompilation;
+      COMPlus_EnableAES;
+      COMPlus_EnableAVX;
+      COMPlus_EnableAVX2;
+      COMPlus_EnableBMI1;
+      COMPlus_EnableBMI2;
+      COMPlus_EnableFMA;
+      COMPlus_EnableIncompleteISAClass;
+      COMPlus_EnableLZCNT;
+      COMPlus_EnablePCLMULQDQ;
+      COMPlus_EnablePOPCNT;
+      COMPlus_EnableSSE;
+      COMPlus_EnableSSE2;
+      COMPlus_EnableSSE3;
+      COMPlus_EnableSSE3_4;
+      COMPlus_EnableSSSE3;
+      COMPlus_EnableSSE41;
+      COMPlus_EnableSSE42;
+      COMPlus_FeatureSIMD;
+      COMPlus_ForceRelocs;
+      COMPlus_GCStress;
+      COMPlus_HeapVerify;
+      COMPlus_JITMinOpts;
+      COMPlus_JitStress;
+      COMPlus_JitStressRegs;
+      COMPlus_TailcallStress;
+      COMPlus_ReadyToRun;
+      COMPlus_ZapDisable
+    </COMPlusVariables>
+  </PropertyGroup>
+  <ItemGroup>
+    <COMPlusVariable Include="$(COMPlusVariables)" />
+  </ItemGroup>
+
+  <!-- Default values for some COMPlus_* variables -->
+  <ItemDefinitionGroup>
+    <TestEnvironment>
+      <TieredCompilation>0</TieredCompilation>
+    </TestEnvironment>
+  </ItemDefinitionGroup>
+
+  <!-- TestEnvironment is a mapping between Scenario and COMPlus_* environment variable values that are written to __TestEnv file -->
+  <ItemGroup>
+    <!-- "normal" scenario doesn't define any COMPlus_* variables and uses the coreclr runtime default values
+         while other scenarios use the default values of COMPlus_* variables defined in ItemDefinitionGroup above -->
+    <TestEnvironment Include="normal" TieredCompilation="" />
+    <TestEnvironment Include="minopts" JITMinOpts="1" />
+    <TestEnvironment Include="no_tiered_compilation" TieredCompilation="0" />
+    <TestEnvironment Include="forcerelocs" ForceRelocs="1" />
+    <TestEnvironment Include="jitstress1" JitStress="1" />
+    <TestEnvironment Include="jitstress2" JitStress="2" />
+    <TestEnvironment Include="jitstress1_tiered" JitStress="1" TieredCompilation="1" />
+    <TestEnvironment Include="jitstress2_tiered" JitStress="2" TieredCompilation="1" />
+    <TestEnvironment Include="jitstressregs1" JitStressRegs="1" />
+    <TestEnvironment Include="jitstressregs2" JitStressRegs="2" />
+    <TestEnvironment Include="jitstressregs3" JitStressRegs="3" />
+    <TestEnvironment Include="jitstressregs4" JitStressRegs="4" />
+    <TestEnvironment Include="jitstressregs8" JitStressRegs="8" />
+    <TestEnvironment Include="jitstressregs0x10" JitStressRegs="0x10" />
+    <TestEnvironment Include="jitstressregs0x80" JitStressRegs="0x80" />
+    <TestEnvironment Include="jitstressregs0x1000" JitStressRegs="0x1000" />
+    <TestEnvironment Include="jitstress2_jitstressregs1" JitStress="2" JitStressRegs="1" />
+    <TestEnvironment Include="jitstress2_jitstressregs2" JitStress="2" JitStressRegs="2" />
+    <TestEnvironment Include="jitstress2_jitstressregs3" JitStress="2" JitStressRegs="3" />
+    <TestEnvironment Include="jitstress2_jitstressregs4" JitStress="2" JitStressRegs="4" />
+    <TestEnvironment Include="jitstress2_jitstressregs8" JitStress="2" JitStressRegs="8" />
+    <TestEnvironment Include="jitstress2_jitstressregs0x10" JitStress="2" JitStressRegs="0x10" />
+    <TestEnvironment Include="jitstress2_jitstressregs0x80" JitStress="2" JitStressRegs="0x80" />
+    <TestEnvironment Include="jitstress2_jitstressregs0x1000" JitStress="2" JitStressRegs="0x1000" />
+    <TestEnvironment Include="tailcallstress" TailcallStress="1" />
+    <TestEnvironment Include="jitsse2only" EnableAVX="0" EnableSSE3_4="0" />
+    <TestEnvironment Include="jitnosimd" FeatureSIMD="0" />
+    <TestEnvironment Include="jitincompletehwintrinsic" EnableIncompleteISAClass="1" />
+    <!-- testing the legacy SSE encoding -->
+    <TestEnvironment Include="jitx86hwintrinsicnoavx" EnableIncompleteISAClass="1" EnableAVX="0" />
+    <!-- testing SNB/IVB -->
+    <TestEnvironment Include="jitx86hwintrinsicnoavx2" EnableIncompleteISAClass="1" EnableAVX2="0" />
+    <!-- match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD -->
+    <TestEnvironment Include="jitx86hwintrinsicnosimd" EnableIncompleteISAClass="1" FeatureSIMD="0" />
+    <TestEnvironment Include="jitnox86hwintrinsic" EnableIncompleteISAClass="1" EnableSSE="0" EnableSSE2="0" EnableSSE3="0" EnableSSSE3="0" EnableSSE41="0" EnableSSE42="0" EnableAVX="0" EnableAVX2="0" EnableAES="0" EnableBMI1="0" EnableBMI2="0" EnableFMA="0" EnableLZCNT="0" EnablePCLMULQDQ="0" EnablePOPCNT="0" />
+    <TestEnvironment Include="gcstress0x3" GCStress="0x3" />
+    <TestEnvironment Include="gcstress0xc" GCStress="0xC" />
+    <TestEnvironment Include="zapdisable" ZapDisable="1" ReadyToRun="0" />
+    <TestEnvironment Include="heapverify1" HeapVerify="1" />
+    <TestEnvironment Include="gcstress0xc_zapdisable" GCStress="0xC" ZapDisable="1" ReadyToRun="0" />
+    <TestEnvironment Include="gcstress0xc_zapdisable_jitstress2" GCStress="0xC" ZapDisable="1" ReadyToRun="0" JitStress="2" />
+    <TestEnvironment Include="gcstress0xc_zapdisable_heapverify1" GCStress="0xC" ZapDisable="1" ReadyToRun="0" HeapVerify="1" />
+    <TestEnvironment Include="gcstress0xc_jitstress1" GCStress="0xC" JitStress="1" />
+    <TestEnvironment Include="gcstress0xc_jitstress2" GCStress="0xC" JitStress="2" />
+    <TestEnvironment Include="gcstress0xc_minopts_heapverify1" GCStress="0xC" JITMinOpts="1" HeapVerify="1" />
+  </ItemGroup>
+
+  <!-- We use target batching on the COMPlusVariable items to iterate over the all COMPlus_* environment variables
+       that can be specified by a test scenario. -->
+  <Target Name="OutputCOMPlusVariables" Inputs="@(COMPlusVariable)" Outputs="%(COMPlusVariable.Identity)">
+    <PropertyGroup>
+      <_COMPlusVariable>%(COMPlusVariable.Identity)</_COMPlusVariable>
+      <_COMPlusVariableMetadataName>$(_COMPlusVariable.Replace('COMPlus_', ''))</_COMPlusVariableMetadataName>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <_TestEnvironment Include="@(TestEnvironment)" Condition="'%(Identity)' == '$(Scenario)'" />
+      <_TestEnvironmentWithMetadata Include="@(_TestEnvironment->HasMetadata($(_COMPlusVariableMetadataName)))" />
+
+      <!-- _TestEnvironmentWithMetadata can be either empty or contain one item:
+
+       1) If _TestEnvironmentWithMetadata is empty, then this suggests that the test scenario with identity $(Scenario)
+          doesn't define COMPlus_* environment variable with name $(_COMPlusVariable);
+
+       2) Otherwise, the test scenario defines such COMPlus_* environment variable and the specified value
+          of this variable can be extracted by using Metadata() item function.
+      -->
+
+      <_COMPlusVariable Include="$(_COMPlusVariable)" Condition="@(_TestEnvironmentWithMetadata->Count()) == 1">
+        <Value>@(_TestEnvironmentWithMetadata->Metadata($(_COMPlusVariableMetadataName)))</Value>
+      </_COMPlusVariable>
+    </ItemGroup>
+
+    <!-- Conceptually, this target creates a collection of _COMPlusVariable items that corresponds to
+         COMPlus_* environment variables specified for the test scenario with identity $(Scenario) -->
+
+  </Target>
+
+  <Target Name="CreateTestEnvFile" DependsOnTargets="OutputCOMPlusVariables">
+    <ItemGroup Condition="'$(TargetsWindows)' == 'true'">
+      <_TestEnvFileLine Include="@echo off" />
+      <_TestEnvFileLine Include="@(_COMPlusVariable->'set %(Identity)=%(Value)')"  />
+    </ItemGroup>
+
+    <ItemGroup Condition="'$(TargetsWindows)' != 'true'">
+      <_TestEnvFileLine Include="#!/usr/bin/env bash" />
+      <_TestEnvFileLine Include="@(_COMPlusVariable->'export %(Identity)=%(Value)')"  />
+    </ItemGroup>
+
+    <WriteLinesToFile File="$(TestEnvFileName)" Lines="@(_TestEnvFileLine)" Overwrite="true" />
+  </Target>
+
+</Project>