Publish packages in Azure DevOps official build (#21536)

This enables coreclr participation in dependency flow.

11 files changed, 82 insertions, 43 deletions

diff --git a/eng/common/CIBuild.cmd b/eng/common/CIBuild.cmd
index 6544b0cd54..56c2f25ac2 100644
--- a/eng/common/CIBuild.cmd
+++ b/eng/common/CIBuild.cmd
@@ -1,3 +1,2 @@
 @echo off
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
-exit /b %ErrorLevel%
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
\ No newline at end of file
diff --git a/eng/common/PublishBuildAssets.cmd b/eng/common/PublishBuildAssets.cmd
index 399ca0bd32..ac629f00e1 100644
--- a/eng/common/PublishBuildAssets.cmd
+++ b/eng/common/PublishBuildAssets.cmd
@@ -1,3 +1,3 @@
 @echo off
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -msbuildEngine dotnet -restore -execute /p:PublishBuildAssets=true /p:SdkTaskProjects=PublishBuildAssets.proj %*"
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -msbuildEngine dotnet -restore -execute -binaryLog /p:PublishBuildAssets=true /p:SdkTaskProjects=PublishBuildAssets.proj %*"
 exit /b %ErrorLevel%
diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh
index 54b70f678b..bd799af2a4 100644
--- a/eng/common/init-tools-native.sh
+++ b/eng/common/init-tools-native.sh
@@ -10,7 +10,7 @@ force=false
 download_retries=5
 retry_wait_time_seconds=30
 global_json_file="${scriptroot}/../../global.json"
-declare -A native_assets
+declare -a native_assets
 
 . $scriptroot/native/common-library.sh
 
diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1
index 3cd667219a..5810a746b1 100644
--- a/eng/common/msbuild.ps1
+++ b/eng/common/msbuild.ps1
@@ -11,6 +11,10 @@ Param(
 . $PSScriptRoot\tools.ps1
 
 try {
+  if ($ci) {
+    $nodeReuse = $false
+  }
+
   MSBuild @extraArgs
 } 
 catch {
@@ -20,4 +24,4 @@ catch {
   ExitWithExitCode 1
 }
 
-ExitWithExitCode 0
\ No newline at end of file
+ExitWithExitCode 0
diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh
index 6fe07cb798..8160cd5a59 100755
--- a/eng/common/msbuild.sh
+++ b/eng/common/msbuild.sh
@@ -50,5 +50,9 @@ done
 
 . "$scriptroot/tools.sh"
 
+if [[ "$ci" == true ]]; then
+  node_reuse=false
+fi
+
 MSBuild $extra_args
 ExitWithExitCode 0
diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
index 9fb858e487..5e293db35d 100644
--- a/eng/common/templates/job/job.yml
+++ b/eng/common/templates/job/job.yml
@@ -95,11 +95,26 @@ jobs:
 
   variables:
   - ${{ each variable in parameters.variables }}:
+    # handle name-value variable syntax
+    # example:
+    # - name: [key]
+    #   value: [value]
     - ${{ if ne(variable.name, '') }}:
       - name: ${{ variable.name }}
         value: ${{ variable.value }}
+    
+    # handle variable groups
     - ${{ if ne(variable.group, '') }}:
       - group: ${{ variable.group }}
+
+    # handle key-value variable syntax.
+    # example:
+    # - [key]: [value]
+    - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}:
+      - ${{ each pair in variable }}:
+        - name: ${{ pair.key }}
+          value: ${{ pair.value }}
+
   # Add additional variables
   - ${{ if and(ne(parameters.helixRepo, ''), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}:
     - name: _HelixSource
diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml
index b40016f6fc..c094658fef 100644
--- a/eng/common/templates/job/publish-build-assets.yml
+++ b/eng/common/templates/job/publish-build-assets.yml
@@ -1,10 +1,4 @@
 parameters:
-  # Optional: dependencies of the job
-  dependsOn: ''
-
-  # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
-  pool: {}
-
   configuration: 'Debug'
 
   # Optional: condition for the job to run
@@ -13,6 +7,15 @@ parameters:
   # Optional: 'true' if future jobs should run even if this job fails
   continueOnError: false
 
+  # Optional: dependencies of the job
+  dependsOn: ''
+
+  # Optional: Include PublishBuildArtifacts task
+  enablePublishBuildArtifacts: false
+
+  # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+  pool: {}
+
   # Optional: should run as a public build even in the internal project
   #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
   runAsPublic: false
@@ -49,3 +52,12 @@ jobs:
       displayName: Publish Build Assets
       condition: ${{ parameters.condition }}
       continueOnError: ${{ parameters.continueOnError }}
+    - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
+      - task: PublishBuildArtifacts@1
+        displayName: Publish Logs to VSTS
+        inputs:
+          PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
+          PublishLocation: Container
+          ArtifactName: $(Agent.Os)_PublishBuildAssets
+        continueOnError: true
+        condition: always()      
diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml
index 6aa55e3afc..c7226b12ed 100644
--- a/eng/common/templates/jobs/jobs.yml
+++ b/eng/common/templates/jobs/jobs.yml
@@ -20,8 +20,8 @@ parameters:
   # Optional: enable sending telemetry
   #           if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
   #             _HelixBuildConfig - differentiate between Debug, Release, other
-  #             _HelixSource - Example: build/product/
-  #             _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch)
+  #             _HelixType - Example: build/product/
+  #             _HelixSource - Example: official/dotnet/arcade/$(Build.SourceBranch)
   enableTelemetry: false
 
   # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
@@ -73,3 +73,5 @@ jobs:
       pool:
         vmImage: vs2017-win2016
       runAsPublic: ${{ parameters.runAsPublic }}
+      enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
+
diff --git a/eng/common/templates/steps/helix-publish.yml b/eng/common/templates/steps/helix-publish.yml
index 6dada380ce..470ab65da0 100644
--- a/eng/common/templates/steps/helix-publish.yml
+++ b/eng/common/templates/steps/helix-publish.yml
@@ -40,7 +40,7 @@ steps:
       WorkItemCommand: ${{ parameters.WorkItemCommand }}
       CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
       XUnitProjects: ${{ parameters.XUnitProjects }}
-      XUnitTargetFramework: ${{ parameters.XUnitTargetFramework }}
+      XUnitRuntimeTargetFramework: ${{ parameters.XUnitTargetFramework }}
       XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
       IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
       DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml
index d0622f3caf..03f0e38662 100644
--- a/eng/common/templates/steps/send-to-helix.yml
+++ b/eng/common/templates/steps/send-to-helix.yml
@@ -1,29 +1,30 @@
 parameters:
-  HelixSource: 'pr/default'          # required
-  HelixType: 'tests/default/'        # required
-  HelixBuild: $(Build.BuildNumber)   # required
-  HelixTargetQueues: ''              # required
-  HelixAccessToken: ''               # required
-  HelixPreCommands: ''
-  HelixPostCommands: ''
-  WorkItemDirectory: ''
-  WorkItemCommand: ''
-  CorrelationPayloadDirectory: ''
-  XUnitProjects: ''
-  XUnitTargetFramework: ''
-  XUnitRunnerVersion: ''
-  IncludeDotNetCli: false
-  DotNetCliPackageType: ''
-  DotNetCliVersion: ''
-  EnableXUnitReporter: false
-  WaitForWorkItemCompletion: true
-  IsExternal: false
-  Creator: ''
-  condition: succeeded()
-  continueOnError: false
+  HelixSource: 'pr/default'          # required -- sources must start with pr/, official/, prodcon/, or agent/
+  HelixType: 'tests/default/'        # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
+  HelixBuild: $(Build.BuildNumber)   # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
+  HelixTargetQueues: ''              # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues
+  HelixAccessToken: ''               # required -- access token to make Helix API requests; should be provided by the appropriate variable group
+  HelixPreCommands: ''               # optional -- commands to run before Helix work item execution
+  HelixPostCommands: ''              # optional -- commands to run after Helix work item execution
+  WorkItemDirectory: ''              # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
+  WorkItemCommand: ''                # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
+  CorrelationPayloadDirectory: ''    # optional -- a directory to zip up and send to Helix as a correlation payload
+  XUnitProjects: ''                  # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
+  XUnitPublishTargetFramework: ''    # optional -- framework to use to publish your xUnit projects
+  XUnitRuntimeTargetFramework: ''    # optional -- framework to use for the xUnit console runner
+  XUnitRunnerVersion: ''             # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
+  IncludeDotNetCli: false            # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
+  DotNetCliPackageType: ''           # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  DotNetCliVersion: ''               # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  EnableXUnitReporter: false         # optional -- true enables XUnit result reporting to Mission Control
+  WaitForWorkItemCompletion: true    # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
+  IsExternal: false                  # optional -- true requires Creator and will make the Mission Control results visible to folks outside the Microsoft Org
+  Creator: ''                        # optional -- if the build is external, use this to specify who is sending the job
+  condition: succeeded()             # optional -- condition for step to execute; defaults to succeeded()
+  continueOnError: false             # optional -- determines whether to continue the build if the step errors; defaults to false
 
 steps:
-  - script: '%BUILD_SOURCESDIRECTORY%\eng\common\msbuild.ps1 %BUILD_SOURCESDIRECTORY%\eng\common\helixpublish.proj /bl:%BUILD_SOURCESDIRECTORY%\artifacts\log\%BuildConfig%\SendToHelix.binlog'
+  - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
     displayName: Send job to Helix (Windows)
     env:
       BuildConfig: $(_BuildConfig)
@@ -38,7 +39,8 @@ steps:
       WorkItemCommand: ${{ parameters.WorkItemCommand }}
       CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
       XUnitProjects: ${{ parameters.XUnitProjects }}
-      XUnitTargetFramework: ${{ parameters.XUnitTargetFramework }}
+      XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
+      XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
       XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
       IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
       DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
@@ -64,7 +66,8 @@ steps:
       WorkItemCommand: ${{ parameters.WorkItemCommand }}
       CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
       XUnitProjects: ${{ parameters.XUnitProjects }}
-      XUnitTargetFramework: ${{ parameters.XUnitTargetFramework }}
+      XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
+      XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
       XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
       IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
       DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index 0d0ebdc145..416eb30c8c 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -87,7 +87,7 @@ function Exec-Process([string]$command, [string]$commandArgs) {
 }
 
 function InitializeDotNetCli([bool]$install) {
-  if (Test-Path global:_DotNetInstallDir) {
+  if (Test-Path variable:global:_DotNetInstallDir) {
     return $global:_DotNetInstallDir
   }
 
@@ -170,7 +170,7 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version) {
 # Throws on failure.
 #
 function InitializeVisualStudioMSBuild([bool]$install) {
-  if (Test-Path global:_MSBuildExe) {
+  if (Test-Path variable:global:_MSBuildExe) {
     return $global:_MSBuildExe
   }
 
@@ -297,7 +297,7 @@ function LocateVisualStudio {
 }
 
 function InitializeBuildTool() {
-  if (Test-Path global:_BuildTool) {
+  if (Test-Path variable:global:_BuildTool) {
     return $global:_BuildTool
   }
 
@@ -364,7 +364,7 @@ function GetNuGetPackageCachePath() {
 }
 
 function InitializeToolset() {
-  if (Test-Path global:_ToolsetBuildProj) {
+  if (Test-Path variable:global:_ToolsetBuildProj) {
     return $global:_ToolsetBuildProj
   }