Update dependencies from https://github.com/dotnet/arcade build 420 (#21140)

This change updates the following dependencies
- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.18570.17
- Microsoft.DotNet.Helix.Sdk - 1.0.0-beta.18570.17

8 files changed, 336 insertions, 15 deletions

diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index bfc9d3e568..ef69040dc7 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -2,13 +2,13 @@
 <Dependencies>
   <ProductDependencies></ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.18569.4">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.18570.17">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>578524df5ce9f5785745727d406d4b7731a507cb</Sha>
+      <Sha>93b39c3209a5929662190c7e85b43b4f2a32bab1</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="1.0.0-beta.18569.4">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="1.0.0-beta.18570.17">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>578524df5ce9f5785745727d406d4b7731a507cb</Sha>
+      <Sha>93b39c3209a5929662190c7e85b43b4f2a32bab1</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>
diff --git a/eng/common/helixpublish.proj b/eng/common/helixpublish.proj
index 364af77668..72668a1337 100644
--- a/eng/common/helixpublish.proj
+++ b/eng/common/helixpublish.proj
@@ -1,4 +1,8 @@
-<Project Sdk="Microsoft.DotNet.Helix.Sdk">
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+
+  <PropertyGroup>
+    <Language>msbuild</Language>
+  </PropertyGroup>
 
   <ItemGroup>
     <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
@@ -12,4 +16,10 @@
       <Command>$(WorkItemCommand)</Command>
     </HelixWorkItem>
   </ItemGroup>
+
+  <ItemGroup>
+    <XUnitProject Include="$(XUnitProjects.Split(';'))">
+      <Arguments />
+    </XUnitProject>
+  </ItemGroup>
 </Project>
diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
new file mode 100644
index 0000000000..9f3e9389c8
--- /dev/null
+++ b/eng/common/templates/job/job.yml
@@ -0,0 +1,190 @@
+parameters:
+# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+  cancelTimeoutInMinutes: ''
+
+  condition: ''
+
+  continueOnError: false
+
+  container: ''
+
+  dependsOn: ''
+
+  displayName: ''
+
+  steps: []
+
+  pool: ''
+
+  strategy: ''
+
+  timeoutInMinutes: ''
+
+  variables: []
+
+  workspace: ''
+
+# Job base template specific parameters
+  # Optional: Enable installing Microbuild plugin
+  #           if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
+  #             _TeamName - the name of your team
+  #             _SignType - 'test' or 'real'
+  enableMicrobuild: false
+
+  # Optional: Include PublishBuildArtifacts task
+  enablePublishBuildArtifacts: false
+
+  # Optional: Enable publishing to the build asset registry
+  enablePublishBuildAssets: false
+
+  # Optional: Include PublishTestResults task
+  enablePublishTestResults: false
+
+  # Optional: enable sending telemetry
+  #           if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
+  #             _HelixBuildConfig - differentiate between Debug, Release, other
+  #             _HelixSource - Example: build/product
+  #             _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch)
+  enableTelemetry: false
+
+  # Optional: If specified, then automatically derive "_HelixSource" variable for telemetry
+  helixRepo: ''
+
+  # Optional: Define the type for helix telemetry (must end in '/')
+  helixType: build/product/
+
+  # Required: name of the job
+  name: ''
+
+  # Optional: should run as a public build even in the internal project
+  #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
+  runAsPublic: false
+
+# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
+# and some (Microbuild) should only be applied to non-PR cases for internal builds.
+
+jobs:
+- job: ${{ parameters.name }}
+
+  ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}:
+    cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }}
+
+  ${{ if ne(parameters.condition, '') }}:
+    condition: ${{ parameters.condition }}
+
+  ${{ if ne(parameters.container, '') }}:
+    container: ${{ parameters.container }}
+
+  ${{ if ne(parameters.continueOnError, '') }}:
+    continueOnError: ${{ parameters.continueOnError }}
+
+  ${{ if ne(parameters.dependsOn, '') }}:
+    dependsOn: ${{ parameters.dependsOn }}
+
+  ${{ if ne(parameters.displayName, '') }}:
+    displayName: ${{ parameters.displayName }}
+
+  ${{ if ne(parameters.pool, '') }}:
+    pool: ${{ parameters.pool }}
+
+  ${{ if ne(parameters.strategy, '') }}:
+    strategy: ${{ parameters.strategy }}
+
+  ${{ if ne(parameters.timeoutInMinutes, '') }}:
+    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+
+  variables:
+  - ${{ each variable in parameters.variables }}:
+    - ${{ if ne(variable.name, '') }}:
+      - name: ${{ variable.name }}
+        value: ${{ variable.value }}
+    - ${{ if ne(variable.group, '') }}:
+      - group: ${{ variable.group }}
+  # Add additional variables
+  - ${{ if and(ne(parameters.helixRepo, ''), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}:
+    - name: _HelixSource
+      value: official/${{ parameters.helixRepo }}/$(Build.SourceBranch)
+  - ${{ if and(ne(parameters.helixRepo, ''), or(ne(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'))) }}:
+    - name: _HelixSource
+      value: pr/${{ parameters.helixRepo }}/$(Build.SourceBranch)
+  - name: _HelixType
+    value: ${{ parameters.helixType }}
+  - name: _HelixBuildConfig
+    value: $(_BuildConfig)
+
+  ${{ if ne(parameters.workspace, '') }}:
+    workspace: ${{ parameters.workspace }}
+
+  steps:
+  - ${{ if eq(parameters.enableTelemetry, 'true') }}:
+    - template: /eng/common/templates/steps/telemetry-start.yml
+      parameters:
+        buildConfig: $(_HelixBuildConfig)
+        helixSource: $(_HelixSource)
+        helixType: $(_HelixType)
+        runAsPublic: ${{ parameters.runAsPublic }}
+
+  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      - task: MicroBuildSigningPlugin@2
+        displayName: Install MicroBuild plugin
+        inputs:
+          signType: $(_SignType)
+          zipSources: false
+          feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+        env:
+          TeamName: $(_TeamName)
+        continueOnError: ${{ parameters.continueOnError }}
+        condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+
+  - ${{ each step in parameters.steps }}:
+    - ${{ step }}
+
+  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      - task: MicroBuildCleanup@1
+        displayName: Execute Microbuild cleanup tasks  
+        condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+        continueOnError: ${{ parameters.continueOnError }}
+        env:
+          TeamName: $(_TeamName)
+
+  - ${{ if eq(parameters.enableTelemetry, 'true') }}:
+    - template: /eng/common/templates/steps/telemetry-end.yml
+
+  - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Logs to VSTS
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
+        PublishLocation: Container
+        ArtifactName: $(Agent.Os)_$(Agent.JobName)
+      continueOnError: true
+      condition: always()
+
+  - ${{ if eq(parameters.enablePublishTestResults, 'true') }}:
+    - task: PublishTestResults@2
+      displayName: Publish Test Results
+      inputs:
+        testResultsFormat: 'xUnit'
+        testResultsFiles: '*.xml' 
+        searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+      continueOnError: true
+      condition: always()
+    
+  - ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - task: CopyFiles@2
+      displayName: Gather Asset Manifests
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
+        TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
+      continueOnError: ${{ parameters.continueOnError }}
+      condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+    - task: PublishBuildArtifacts@1
+      displayName: Push Asset Manifests
+      inputs:
+        PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
+        PublishLocation: Container
+        ArtifactName: AssetManifests
+      continueOnError: ${{ parameters.continueOnError }}
+      condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
\ No newline at end of file
diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml
new file mode 100644
index 0000000000..b40016f6fc
--- /dev/null
+++ b/eng/common/templates/job/publish-build-assets.yml
@@ -0,0 +1,51 @@
+parameters:
+  # Optional: dependencies of the job
+  dependsOn: ''
+
+  # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+  pool: {}
+
+  configuration: 'Debug'
+
+  # Optional: condition for the job to run
+  condition: ''
+
+  # Optional: 'true' if future jobs should run even if this job fails
+  continueOnError: false
+
+  # Optional: should run as a public build even in the internal project
+  #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
+  runAsPublic: false
+
+jobs:
+- job: Asset_Registry_Publish
+
+  dependsOn: ${{ parameters.dependsOn }}
+
+  displayName: Publish to Build Asset Registry
+
+  pool: ${{ parameters.pool }}
+
+  variables:
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - name: _BuildConfig
+      value: ${{ parameters.configuration }}
+    - group: Publish-Build-Assets
+
+  steps:
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - task: DownloadBuildArtifacts@0
+      displayName: Download artifact
+      inputs:
+        artifactName: AssetManifests
+        downloadPath: '$(Build.StagingDirectory)/Download'
+      condition: ${{ parameters.condition }}
+      continueOnError: ${{ parameters.continueOnError }}
+    - script: eng\common\publishbuildassets.cmd
+        /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
+        /p:BuildAssetRegistryToken=$(MaestroAccessToken)
+        /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
+        /p:Configuration=$(_BuildConfig)
+      displayName: Publish Build Assets
+      condition: ${{ parameters.condition }}
+      continueOnError: ${{ parameters.continueOnError }}
diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml
new file mode 100644
index 0000000000..6aa55e3afc
--- /dev/null
+++ b/eng/common/templates/jobs/jobs.yml
@@ -0,0 +1,75 @@
+parameters:
+  # Optional: 'true' if failures in job.yml job should not fail the job
+  continueOnError: false
+
+  # Optional: Enable installing Microbuild plugin
+  #           if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
+  #             _TeamName - the name of your team
+  #             _SignType - 'test' or 'real'
+  enableMicrobuild: false
+
+  # Optional: Include PublishBuildArtifacts task
+  enablePublishBuildArtifacts: false
+
+  # Optional: Enable publishing to the build asset registry
+  enablePublishBuildAssets: false
+
+  # Optional: Include PublishTestResults task
+  enablePublishTestResults: false
+
+  # Optional: enable sending telemetry
+  #           if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
+  #             _HelixBuildConfig - differentiate between Debug, Release, other
+  #             _HelixSource - Example: build/product/
+  #             _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch)
+  enableTelemetry: false
+
+  # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+  jobs: []
+
+  # Optional: If specified, then automatically derive "_HelixSource" variable for telemetry
+  helixRepo: ''
+
+  # Optional: Define the type for helix telemetry (must end in '/')
+  helixType: build/product/
+
+  # Optional: Override automatically derived dependsOn value for "publish build assets" job
+  publishBuildAssetsDependsOn: ''
+
+  # Optional: should run as a public build even in the internal project
+  #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
+  runAsPublic: false
+
+# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
+# and some (Microbuild) should only be applied to non-PR cases for internal builds.
+
+jobs:
+- ${{ each job in parameters.jobs }}:
+  - template: ../job/job.yml
+    parameters: 
+      # pass along parameters
+      ${{ each parameter in parameters }}:
+        ${{ if ne(parameter.key, 'jobs') }}:
+          ${{ parameter.key }}: ${{ parameter.value }}
+
+      # pass along job properties
+      ${{ each property in job }}:
+        ${{ if ne(property.key, 'job') }}:
+          ${{ property.key }}: ${{ property.value }}
+
+      name: ${{ job.job }}
+
+- ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  - template: ../job/publish-build-assets.yml
+    parameters:
+      continueOnError: ${{ parameters.continueOnError }}
+      dependsOn:
+      - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
+        - ${{ each job in parameters.publishBuildAssetsDependsOn }}:
+          - ${{ job.job }}
+      - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
+        - ${{ each job in parameters.jobs }}:
+          - ${{ job.job }}
+      pool:
+        vmImage: vs2017-win2016
+      runAsPublic: ${{ parameters.runAsPublic }}
diff --git a/eng/common/templates/steps/helix-publish.yml b/eng/common/templates/steps/helix-publish.yml
index f261d0ad0e..6dada380ce 100644
--- a/eng/common/templates/steps/helix-publish.yml
+++ b/eng/common/templates/steps/helix-publish.yml
@@ -26,7 +26,7 @@ steps:
       command: custom
       projects: eng/common/helixpublish.proj
       custom: msbuild
-      arguments: '/t:test /p:Language=msbuild /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog'
+      arguments: '/bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog'
     displayName: Send job to Helix
     env:
       HelixSource: ${{ parameters.HelixSource }}
diff --git a/eng/common/templates/steps/telemetry-end.yml b/eng/common/templates/steps/telemetry-end.yml
index 9cfe3608b8..9b61481e7e 100644
--- a/eng/common/templates/steps/telemetry-end.yml
+++ b/eng/common/templates/steps/telemetry-end.yml
@@ -1,7 +1,3 @@
-parameters:
-  helixSource: 'undefined_defaulted_in_telemetry.yml'
-  helixType: 'undefined_defaulted_in_telemetry.yml'
-
 steps:
 - bash: |
     if [ "$AGENT_JOBSTATUS" = "Succeeded" ] || [ "$AGENT_JOBSTATUS" = "PartiallySucceeded" ]; then
diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml
index 07f1185d17..79c128c5de 100644
--- a/eng/common/templates/steps/telemetry-start.yml
+++ b/eng/common/templates/steps/telemetry-start.yml
@@ -31,17 +31,16 @@ steps:
     }
     JobListStuff
     
+    cat $jobInfo
+
     # create a temporary file for curl output
     res=`mktemp`
 
-    accessTokenParameter=''
-    if [[ ! "$HelixApiAccessToken" == "" ]]; then
-      accessTokenParameter="?access_token=$HelixApiAccessToken"
-    fi
+    accessTokenParameter="?access_token=$HelixApiAccessToken"
 
     curlResult=`
       cat $jobInfo |\
-      curl --verbose --output $res --write-out "%{http_code}" \
+      curl --trace - --verbose --output $res --write-out "%{http_code}" \
       -H 'Content-Type: application/json' \
       -X POST "https://helix.dot.net/api/2018-03-14/telemetry/job$accessTokenParameter" -d @-`
     curlStatus=$?