Add azure-pipelines build and test definitions (#20840)

This adds an azure pipeline definition with a matrix of product and test builds, using helix to run tests. The intention is that this definition will eventually be used for both our official build and CI testing.

There is one build job for each OS/platform/arch, and one test job for each OS/platform/arch/priority/R2Rflag. The test job builds tests and then submits them to helix, passing along a number of test run modes. One helix test job will be created for each OS/platform/arch/priority/R2Rflag/helixtargetqueue/testscenario.

There is a lot of work left to be done to get this up to parity with our official builds and CI, which I've tried to call out in comments.

3 files changed, 226 insertions, 10 deletions

diff --git a/tests/helixprep.proj b/tests/helixprep.proj
index 1cbf0ed0f8..50d7d9bd76 100644
--- a/tests/helixprep.proj
+++ b/tests/helixprep.proj
@@ -1,23 +1,26 @@
 <Project ToolsVersion="12.0" DefaultTargets="ArchiveAll" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 
-  <UsingTask TaskName="ZipFileCreateFromDirectory" AssemblyFile="$(ToolsDir)\net46\Microsoft.DotNet.Build.Tasks.dll"/>
-  
+  <UsingTask TaskName="ZipFileCreateFromDirectory" AssemblyFile="$(ToolsDir)\net46\Microsoft.DotNet.Build.Tasks.dll"
+             Condition=" '$(MSBuildRuntimeType)' != 'Core' " />
+  <UsingTask TaskName="ZipFileCreateFromDirectory" AssemblyFile="$(ToolsDir)\Microsoft.DotNet.Build.Tasks.dll"
+             Condition=" '$(MSBuildRuntimeType)' == 'Core' " />
+
   <Import Project="dir.props" />
   <Import Project="..\dir.props" />
 
   <PropertyGroup>
     <DiscoveryDirectory>$(TestWorkingDir)</DiscoveryDirectory>
-    <CoreRootDir>$(CORE_ROOT)</CoreRootDir>
     <CoreRootName>Core_Root_$(RuntimeID)</CoreRootName>
+    <UsePython Condition=" '$(UsePython)' == '' ">false</UsePython>
   </PropertyGroup>
 
   <ItemGroup>
-    <TestCmds Include="$(DiscoveryDirectory)\**\*.cmd" ></TestCmds>
     <XunitDlls Include="$(DiscoveryDirectory)\**\*.XUnitWrapper.dll" ></XunitDlls>
     <RequiresSigningFilesToDelete Include="$(DiscoveryDirectory)\**\*.requires_signing" />
   </ItemGroup>
 
   <!-- Build the platform-specific wrapper to run an individual xunit wrapper -->
+  <!-- We should remove the generated wrappers once we have transitioned our official build -->
 
   <Target Name="GenerateWrapperExecutables"
     Inputs="@(XunitDlls)"
@@ -32,15 +35,26 @@
              Targets="GenerateWrapperSh" />
   </Target>
 
+  <!-- Copy the wrapper script to the xunit directory -->
+  <Target Name="CopyWrapperScript"
+    Inputs="@(XunitDlls)"
+    Outputs="$(TestWorkingDir)*\runtest_helix.py" >
+
+    <Copy SourceFiles="$(MSBuildThisFileDirectory)\runtest_helix.py"
+          DestinationFolder="%(XunitDlls.RootDir)%(XunitDlls.Directory)" />
+
+  </Target>
+
   <!-- Zip each top-level test folder to send to Helix -->
 
   <Target Name="ArchiveTests"
     Inputs="@(XunitDlls)"
     Outputs="$(TestWorkingDir)archive\**" >
-    
+
     <Copy SourceFiles="$(CORE_ROOT)\xunit.console.dll"
           DestinationFolder="%(XunitDlls.RootDir)%(XunitDlls.Directory)"
-    />
+          Condition=" '$(UsePython)' == 'false' " />
+
     <Message Text="Deleting '.requires_signing' files to avoid file name lengths exceeding MAX_PATH" Importance="Low" />
     <Delete Files="@(RequiresSigningFilesToDelete)" />
     <MSBuild Projects="helixprep.proj"
@@ -52,10 +66,10 @@
   <!-- Zip Core_Root & Packages payload to send to Helix -->
 
   <Target Name="ArchiveCoreRoot"
-    Inputs="$(CoreRootDir)"
+    Inputs="$(CORE_ROOT)"
     Outputs="$(TestWorkingDir)archive\Core_Root" >
     <MSBuild Projects="helixprep.proj"
-             Properties="BuildPath=$(CoreRootDir);ProjectName=$(CoreRootName);BuildArchiveDir=$(TestWorkingDir)archive\Core_Root\"
+             Properties="BuildPath=$(CORE_ROOT);ProjectName=$(CoreRootName);BuildArchiveDir=$(TestWorkingDir)archive\Core_Root\"
              Targets="ArchiveBuild" />
 
     <!-- Make dummy packages.zip to upload to Helix -->
@@ -180,8 +194,13 @@ EXIT /B %ERRORLEVEL%
   <!-- Default target to run - builds executables & archives everything needed for Helix run -->
 
   <Target Name="ArchiveAll" >
+    <PropertyGroup>
+      <_ArchiveTargets Condition=" '$(UsePython)' == 'false' ">GenerateWrapperExecutables</_ArchiveTargets>
+      <_ArchiveTargets Condition=" '$(UsePython)' == 'true' ">CopyWrapperScript</_ArchiveTargets>
+      <_ArchiveTargets>$(_ArchiveTargets);ArchiveTests;ArchiveCoreRoot</_ArchiveTargets>
+    </PropertyGroup>
     <MSBuild Projects="helixprep.proj"
-             Targets="GenerateWrapperExecutables;ArchiveTests;ArchiveCoreRoot" />
+             Targets="$(_ArchiveTargets)" />
   </Target>
 
-</Project>
\ No newline at end of file
+</Project>
diff --git a/tests/helixpublishwitharcade.proj b/tests/helixpublishwitharcade.proj
new file mode 100644
index 0000000000..38fb8d2589
--- /dev/null
+++ b/tests/helixpublishwitharcade.proj
@@ -0,0 +1,93 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk">
+
+  <!-- This project uses the helix SDK ,documented at
+       https://github.com/dotnet/arcade/tree/master/src/Microsoft.DotNet.Helix/Sdk,
+       to send test jobs to helix. -->
+
+  <Import Project="..\dir.props" />
+
+  <PropertyGroup>
+    <!-- TODO: pick appropriate helix source and type. -->
+    <HelixSource>pr/coreclr/master</HelixSource>
+    <HelixType>test/stuff</HelixType>
+    <HelixBuild>$(BUILD_BUILDNUMBER)</HelixBuild>
+
+    <!-- TODO: add target queues for rhel and linux-musl -->
+    <!-- TODO: why don't we currently run tests on windows x86? -->
+    <HelixTargetQueues Condition=" '$(BuildOS)' == 'Windows_NT' ">
+      Windows.10.Amd64;
+      Windows.10.Nano.Amd64;
+      Windows.10.Amd64.Core;
+      Windows.7.Amd64;
+      Windows.81.Amd64
+    </HelixTargetQueues>
+    <HelixTargetQueues Condition=" '$(BuildOS)' == 'Linux' ">
+      debian.82.amd64;
+      fedora.27.amd64;
+      fedora.28.amd64;
+      redhat.73.amd64;
+      ubuntu.1404.amd64;
+      ubuntu.1604.amd64;
+      ubuntu.1804.amd64;
+      opensuse.423.amd64;
+      sles.12.amd64
+    </HelixTargetQueues>
+    <HelixTargetQueues Condition=" '$(BuildOS)' == 'OSX' ">
+      osx.1012.amd64;
+      osx.1013.amd64
+    </HelixTargetQueues>
+
+    <EnableXUnitReporter>true</EnableXUnitReporter>
+    <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
+    <SourceDirectory>$(MSBuildProjectDirectory)/..</SourceDirectory>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <HelixCorrelationPayload Include="$(SourceDirectory)\bin\tests\*\archive\Core_Root\*.zip">
+      <PayloadArchive>%(Identity)</PayloadArchive>
+    </HelixCorrelationPayload>
+  </ItemGroup>
+
+  <Target Name="SubmitTestsToHelix">
+    <ItemGroup>
+      <Scenarios Include="$(Scenarios)" />
+    </ItemGroup>
+
+    <!-- If no scenario was specified, just run the normal test
+         scenario -->
+    <MSBuild Projects="$(MSBuildProjectFile)" Targets="Test"
+             Condition=" '@(Scenarios->Count())' == '0' " />
+
+    <!-- If scenarios were specified, submit jobs for each
+         scenario. -->
+    <MSBuild Projects="$(MSBuildProjectFile)" Targets="Test"
+             Properties="Scenario=%(Scenarios.Identity)"
+             BuildInParallel="true"
+             Condition=" '@(Scenarios->Count())' != '0' " />
+
+  </Target>
+
+  <Target Name="BuildHelixWorkItem"
+          BeforeTargets="Test">
+    <PropertyGroup>
+      <!-- The "normal" scenario is just a way to include the default
+           (empty) scenario when specifying multiple scenarios at
+           once. From here, on, treat it as the empty scenario so that
+           job names will not have a scenario prefix and the runtest
+           script doesn't have to define a "normal" scenario. -->
+      <Scenario Condition=" '$(Scenario)' == 'normal' "></Scenario>
+      <ScenarioPrefix Condition=" '$(Scenario)' == '' " />
+      <ScenarioPrefix Condition=" '$(Scenario)' != '' ">$(Scenario) </ScenarioPrefix>
+    </PropertyGroup>
+    <ItemGroup>
+      <TestZipFiles Include="$(SourceDirectory)\bin\tests\*\archive\tests\*.zip" />
+      <HelixWorkItem Include="@(TestZipFiles->'$(ScenarioPrefix)%(FileName)')" >
+        <PayloadArchive>%(Identity)</PayloadArchive>
+        <Command Condition=" '$(Scenario)' == '' ">python runtest_helix.py -wrapper %(FileName).dll</Command>
+        <Command Condition=" '$(Scenario)' != '' ">python runtest_helix.py -scenario $(Scenario) -wrapper %(FileName).dll</Command>
+      </HelixWorkItem>
+    </ItemGroup>
+
+  </Target>
+
+</Project>
diff --git a/tests/runtest_helix.py b/tests/runtest_helix.py
new file mode 100755
index 0000000000..910d9d77ce
--- /dev/null
+++ b/tests/runtest_helix.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+
+# This script runs tests in helix. It defines a set of test scenarios
+# that enable various combinations of runtime configurations to be set
+# via the test process environment.
+
+# This script calls "corerun xunit.console.dll xunitwrapper.dll",
+# where the xunitwrapper.dll will run a .sh/.cmd script per test in a
+# separate process. This process will have the scenario environment
+# variables set, but the xunit process will not.
+
+# TODO: Factor out logic common with runtest.py
+
+import argparse
+import subprocess
+import os
+import sys
+import tempfile
+
+test_scenarios = {
+    "jitstress2": { "COMPlus_TieredCompilation": "0",
+                    "COMPlus_JitStress": "2" },
+}
+
+if sys.platform == "linux" or sys.platform == "darwin":
+    platform_type = "unix"
+elif sys.platform == "win32":
+    platform_type = "windows"
+else:
+    print("unknown os: %s" % sys.platform)
+    sys.exit(1)
+
+def get_testenv_script(env_dict):
+    if platform_type == "unix":
+        return ''.join([ "export %s=%s%s" % (k, v, os.linesep) for k, v in env_dict.items() ])
+    elif platform_type == "windows":
+        return ''.join([ "set %s=%s%s" % (k, v, os.linesep) for k, v in env_dict.items() ])
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Parse arguments")
+    parser.add_argument("-scenario", dest="scenario", default=None)
+    parser.add_argument("-wrapper", dest="wrapper", default=None, required=True)
+    args = parser.parse_args()
+    scenario = args.scenario
+    wrapper = args.wrapper
+
+    if not "HELIX_CORRELATION_PAYLOAD" in os.environ:
+        print("HELIX_CORRELATION_PAYLOAD must be defined in environment")
+        sys.exit(1)
+
+    if not "HELIX_WORKITEM_PAYLOAD" in os.environ:
+        print("HELIX_WORKITEM_PAYLOAD must be defined in environment")
+        sys.exit(1)
+
+    core_root = os.environ["HELIX_CORRELATION_PAYLOAD"]
+
+    if platform_type == "unix":
+        corerun = os.path.join(core_root, "corerun")
+    else:
+        corerun = os.path.join(core_root, "corerun.exe")
+
+    # Unlike the old test wrapper, this runs xunit.console.dll from
+    # the correlation payload. This removes the need for redundant
+    # copies of the console runner in each test directory.
+    command = [corerun,
+               os.path.join(os.environ["HELIX_CORRELATION_PAYLOAD"], "xunit.console.dll"),
+               os.path.join(os.environ["HELIX_WORKITEM_PAYLOAD"], wrapper),
+               "-noshadow",
+               "-xml", "testResults.xml",
+               "-notrait", "category=outerloop",
+               "-notrait", "category=failing"]
+
+    if scenario is None:
+        print("CORE_ROOT=%s" % core_root)
+        os.environ["CORE_ROOT"] = core_root
+
+        print("BEGIN EXECUTION")
+        print(' '.join(command))
+        proc = subprocess.Popen(command)
+        proc.communicate()
+        print("Finished running tests. Exit code = %d" % proc.returncode)
+        sys.exit(proc.returncode)
+    else:
+        print("scenario: %s" % scenario)
+        with tempfile.NamedTemporaryFile(mode="w") as testenv:
+            testenv.write(get_testenv_script(test_scenarios[scenario]))
+            testenv.flush()
+
+            print("__TestEnv=%s" % testenv.name)
+            os.environ["__TestEnv"] = testenv.name
+
+            with open(testenv.name) as testenv_written:
+                contents = testenv_written.read()
+                print(contents)
+
+            print("CORE_ROOT=%s" % core_root)
+            os.environ["CORE_ROOT"] = core_root
+
+            print("BEGIN EXECUTION")
+            print(' '.join(command))
+            proc = subprocess.Popen(command)
+            proc.communicate()
+            print("Finished running tests. Exit code = %d" % proc.returncode)
+            sys.exit(proc.returncode)