Running and visualizing tests in AzureDevOps (Part 1) (#21502)

* Enable Azure Pipelines reporter so the test results can be seen at one place (i.e. "Tests" tab) - TestRunNamePrefix should encode all the needed information for a test run to be identifiable.

* Scenarios property should use comma (not semi-colon) to work correctly on non-Windows.

* Specify timeout to test jobs (the numbers are taken from netci.groovy) - we would adjust them as soon as we get this running in each PR.

* Specify timeout for Helix work items (the number is taken from buildpipeline) - same as above.
 
* Specify helixType and helixRepo properties to the job and use _HelixType and _HelixSource variables in MSBuild project instead of defining them manually.

*  Limit Pri0 Checked test jobs to running against pull requests. This also should be limited in the future to "default" build definition name - so when we have PR workload selection feature working we can distinguish between default PR validation and user triggered jobs.

* Remove "Prepare tests for helix" step - instead of archiving Core_Root and tests folders ourselves - directly specify them as correlation payload and work item payload directories and let the archiving be handled by Helix Sdk.

4 files changed, 50 insertions, 44 deletions

diff --git a/eng/build-job.yml b/eng/build-job.yml
index 3a5425faaa..98710b42aa 100644
--- a/eng/build-job.yml
+++ b/eng/build-job.yml
@@ -4,6 +4,7 @@ parameters:
   osGroup: ''
   osGroupName: ''
   containerName: ''
+  crossrootfsDir: ''
 
 ### Product build
 jobs:
@@ -21,6 +22,7 @@ jobs:
     # Run all steps in the container.
     # Note that the containers are resources defined in azure-pipelines.yml
     containerName: ${{ parameters.containerName }}
+    crossrootfsDir: ${{ parameters.crossrootfsDir }}
 
     steps:
 
@@ -36,14 +38,8 @@ jobs:
         displayName: Install native dependencies
 
     # Build
-    - ${{ if and(and(ne(parameters.archType, 'arm'), ne(parameters.archType, 'arm64')), ne(parameters.osGroup, 'Windows_NT')) }}:
-      - script: ./build.sh $(buildConfig) $(archType) -skiptests
-        displayName: Build product
-    - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm')) }}:
-      - script: ROOTFS_DIR=$(rootfsDir) CAC_ROOTFS_DIR=$(cacRootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skiptests
-        displayName: Build product
-    - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm64')) }}:
-      - script: ROOTFS_DIR=$(rootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skiptests
+    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
+      - script: ./build.sh $(buildConfig) $(archType) $(crossArg) -skiptests 
         displayName: Build product
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       # TODO: IBCOptimize? EnforcePGO? pass an OfficialBuildId? SignType? file logging parameters?
diff --git a/eng/platform-matrix.yml b/eng/platform-matrix.yml
index 6d84bbbf4a..647bd9ccef 100644
--- a/eng/platform-matrix.yml
+++ b/eng/platform-matrix.yml
@@ -17,6 +17,7 @@ jobs:
     osGroupName: Linux
     containerName: ubuntu_1404_arm_cross_build_image
     helixQueuesPublic: 'Ubuntu.1404.Arm32.Open'
+    crossrootfsDir: '/crossrootfs/arm'
     # Currently we don't have Linux/arm32 internal Helix queues
     ${{ insert }}: ${{ parameters.jobParameters }}
 
@@ -28,8 +29,11 @@ jobs:
     osGroup: Linux
     osGroupName: Linux
     containerName: ubuntu_1604_arm64_cross_build_image
-    helixQueuesPublic: 'Debian.9.Arm64.Open,Ubuntu.1604.Arm64.Open,Ubuntu.1804.Arm64.Open'
-    helixQueuesInternal: 'Debian.9.Arm64,Ubuntu.1604.Arm64,Ubuntu.1804.Arm64'
+    # TODO: enable Debian.9.Arm64.Open and Debian.9.Arm64 queues
+    # when https://github.com/dotnet/core-eng/issues/4805 is resolved
+    helixQueuesPublic: 'Ubuntu.1604.Arm64.Open,Ubuntu.1804.Arm64.Open'
+    helixQueuesInternal: 'Ubuntu.1604.Arm64,Ubuntu.1804.Arm64'
+    crossrootfsDir: '/crossrootfs/arm64'
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Linux musl
@@ -40,8 +44,7 @@ jobs:
     osGroup: Linux
     osGroupName: Linux_musl
     containerName: musl_x64_build_image
-    helixQueuesPublic: 'Alpine.3.Arm64.Open'
-    helixQueuesInternal: 'Alpine.3.Arm64'
+    # TODO: add Alpine.Amd64 queues
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 # RHEL 6
@@ -77,8 +80,8 @@ jobs:
     osGroup: Linux
     osGroupName: Linux
     containerName: ubuntu_1604_x64_build_image
-    helixQueuesPublic: 'Debian.8.Amd64.Open,Ubuntu.1604.Amd64.Open,Ubuntu.1804.Arm64.Open'
-    helixQueuesInternal: 'Debian.8.Amd64,Ubuntu.1604.Amd64,Ubuntu.1804.Arm64'
+    helixQueuesPublic: 'Debian.8.Amd64.Open,Ubuntu.1604.Amd64.Open,Ubuntu.1804.Amd64.Open'
+    helixQueuesInternal: 'Debian.8.Amd64,Ubuntu.1604.Amd64,Ubuntu.1804.Amd64'
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 # macOS x64
diff --git a/eng/test-job.yml b/eng/test-job.yml
index 72b37f6506..218f8abe34 100644
--- a/eng/test-job.yml
+++ b/eng/test-job.yml
@@ -8,6 +8,8 @@ parameters:
   scenarios: ''
   helixQueuesPublic: ''
   helixQueuesInternal: ''
+  timeoutInMinutes: ''
+  crossrootfsDir: ''
 
 ### Test job
 
@@ -25,11 +27,18 @@ jobs:
     # Compute job name from template parameters
     ${{ if eq(parameters.crossgen, 'false') }}:
         name: ${{ format('testbuild_pri{0}_{1}_{2}_{3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
-        displayName: ${{ format('Test pri{0} {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
+        displayName: ${{ format('Test Pri{0} {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
     ${{ if eq(parameters.crossgen, 'true') }}:
         name: ${{ format('testbuild_pri{0}_r2r_{1}_{2}_{3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
         displayName: ${{ format('Test Pri{0} R2R {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
 
+    ${{ if eq(parameters.crossgen, 'false') }}:
+        helixType: ${{ format('test/functional/cli/pri{0}', parameters.priority) }}
+    ${{ if eq(parameters.crossgen, 'true') }}:
+        helixType: ${{ format('test/functional/r2r/cli/pri{0}', parameters.priority) }}
+
+    crossrootfsDir: ${{ parameters.crossrootfsDir }}
+
     variables:
       # Map template parameters to command line arguments
       - ${{ if eq(parameters.priority, '1') }}:
@@ -61,6 +70,8 @@ jobs:
       - ${{ if ne(variables['System.TeamProject'], 'public') }}:
         - name: helixTargetQueuesArg
           value: ${{ format('/p:HelixTargetQueues=\"{0}\"', parameters.helixQueuesInternal) }}
+      - name: commonMSBuildArgs
+        value: ${{ format('/maxcpucount /p:__BuildOS={0} /p:__BuildArch={1} /p:__BuildType={2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
 
     # TODO: Enable crossgen in build-test.sh. It currently doesn't
     # accept a crossgen arg, so disable the macos/linux crossgen test
@@ -75,6 +86,8 @@ jobs:
     # Note that the containers are resources defined in azure-pipelines.yml
     containerName: ${{ parameters.containerName }}
 
+    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+
     steps:
 
     # Install test build dependencies
@@ -99,29 +112,18 @@ jobs:
 
       
     # Build tests
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
-        # TODO: enable crossgen in build-test.sh
-      - script: ./build-test.sh $(buildConfig) $(archType) $(priorityArg) $(crossgenArg)
+    # TODO: enable crossgen in build-test.sh
+    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
+      - script: ./build-test.sh $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(crossgenArg)
         displayName: Build tests
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       - script: build-test.cmd $(buildConfig) $(archType) $(priorityArg) $(crossgenArg)
         displayName: Build tests
 
 
-    # Prepare tests for helix
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
-      - script: ./Tools/dotnetcli/dotnet msbuild tests/helixprep.proj /p:CORE_ROOT=$(Build.SourcesDirectory)/bin/tests/$(osGroup).$(archType).$(buildConfigUpper)/Tests/Core_Root /p:__BuildType=$(buildConfig) /p:__BuildArch=$(archType) /p:UsePython=true
-        # TODO: remove UsePython argument once we've removed generated wrappers in helixprep.proj
-        displayName: Prepare test archives for Helix
-    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-        # TODO: remove UsePython argument once we've removed generated wrappers in helixprep.proj
-      - script: .\Tools\dotnetcli\dotnet.exe msbuild tests\helixprep.proj /p:CORE_ROOT=$(Build.SourcesDirectory)\bin\tests\Windows_NT.$(archType).$(buildConfigUpper)\tests\core_root /p:__BuildType=$(buildConfig) /p:__BuildArch=$(archType) /p:UsePython=true
-        displayName: Prepare test archives for Helix
-
-
     # Send tests to helix
     - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
-      - script: ./Tools/dotnetcli/dotnet msbuild tests/helixpublishwitharcade.proj /maxcpucount $(scenariosArg) $(helixTargetQueuesArg)
+      - script: ./Tools/dotnetcli/dotnet msbuild tests/helixpublishwitharcade.proj $(commonMSBuildArgs) $(scenariosArg) $(helixTargetQueuesArg)
         displayName: Send test jobs to Helix
         env:
           ${{ if eq(variables['System.TeamProject'], 'internal') }}:
@@ -130,8 +132,9 @@ jobs:
           ${{ if eq(variables['System.TeamProject'], 'public') }}:
             # Access token variable for public project
             HelixAccessToken: $(BotAccount-dotnet-github-anon-kaonashi-bot-helix-token)
+          SYSTEM_ACCESSTOKEN: $(System.AccessToken)
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-      - script: .\Tools\dotnetcli\dotnet msbuild tests\helixpublishwitharcade.proj /maxcpucount $(scenariosArg) $(helixTargetQueuesArg)
+      - script: .\Tools\dotnetcli\dotnet msbuild tests\helixpublishwitharcade.proj $(commonMSBuildArgs) $(scenariosArg) $(helixTargetQueuesArg)
         displayName: Send test jobs to Helix
         env:
           ${{ if eq(variables['System.TeamProject'], 'internal') }}:
@@ -140,3 +143,4 @@ jobs:
           ${{ if eq(variables['System.TeamProject'], 'public') }}:
             # Access token variable for public project
             HelixAccessToken: $(BotAccount-dotnet-github-anon-kaonashi-bot-helix-token)
+          SYSTEM_ACCESSTOKEN: $(System.AccessToken)
diff --git a/eng/xplat-job.yml b/eng/xplat-job.yml
index d9d45d1486..db8cf3ba50 100644
--- a/eng/xplat-job.yml
+++ b/eng/xplat-job.yml
@@ -8,6 +8,9 @@ parameters:
   condition: ''
   dependsOn: ''
   containerName: ''
+  timeoutInMinutes: ''
+  helixType: ''
+  crossrootfsDir: ''
   variables: {} ## any extra variables to add to the defaults defined below
 
 jobs:
@@ -18,8 +21,11 @@ jobs:
     displayName: ${{ parameters.displayName }}
 
     condition: ${{ parameters.condition }}
-
     dependsOn: ${{ parameters.dependsOn }}
+    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+
+    helixRepo: 'dotnet/coreclr'
+    helixType: ${{ parameters.helixType }}
 
     pool:
       ${{ if and(eq(parameters.osGroup, 'Linux'), eq(variables['System.TeamProject'], 'public')) }}:
@@ -34,7 +40,6 @@ jobs:
         name: dotnet-internal-temp
       ${{ if and(eq(parameters.osGroup, 'Windows_NT'), eq(variables['System.TeamProject'], 'public')) }}:
         name: dotnet-external-temp
-      timeoutInMinutes: 240
 
     workspace:
       clean: all
@@ -74,17 +79,15 @@ jobs:
       - name: osGroupName
         value: ${{ parameters.osGroupName }}
       
-      # Crossbuild specific variables.
-      - ${{ if eq(parameters.archType, 'arm') }}:
-          - name: rootfsDir
-            value: /crossrootfs/arm
-
-          - name: cacRootfsDir
-            value: /crossrootfs/x86
-
-      - ${{ if eq(parameters.archType, 'arm64') }}:
-          - name: rootfsDir
-            value: /crossrootfs/arm64
+      - ${{ if ne(parameters.crossrootfsDir, '') }}:
+          - name: crossArg
+            value: 'cross'
+          # This is only required for cross builds.
+          - name: ROOTFS_DIR
+            value: ${{ parameters.crossrootfsDir }}
+      - ${{ if eq(parameters.crossrootfsDir, '') }}:
+          - name: crossArg
+            value: ''
 
       - ${{ each variable in parameters.variables }}:
         - ${{insert}}: ${{ variable }}