diff options
| author | Anipik <anirudhagnihotry098@gmail.com> | 2019-09-09 12:12:40 -0700 |
|---|---|---|
| committer | Anipik <anirudhagnihotry098@gmail.com> | 2019-09-09 12:12:40 -0700 |
| commit | 6b5c8ef1fdb23612cb09ac455000d3b92104b487 (patch) | |
| tree | b694ca6bf196bd4329f1cf0d06dc1cabab6d53c6 | |
| parent | 242b290169bf0af7f2b1f51273adbec17a180130 (diff) | |
| parent | 1533d6a5db52179c23a552878cd227fb2e7448bf (diff) | |
| download | coreclr-6b5c8ef1fdb23612cb09ac455000d3b92104b487.tar.gz coreclr-6b5c8ef1fdb23612cb09ac455000d3b92104b487.tar.bz2 coreclr-6b5c8ef1fdb23612cb09ac455000d3b92104b487.zip | |
Merge branch 'release/3.0' into release/3.1
32 files changed, 406 insertions, 523 deletions
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ea3d268fcc..7d1c55b6df 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -2,41 +2,41 @@ <ProductDependencies> </ProductDependencies> <ToolsetDependencies> - <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19455.3"> + <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19458.2"> <Uri>https://github.com/dotnet/arcade</Uri> - <Sha>fee4ebe849050a385a4999e4c84fa61c1dc234bb</Sha> + <Sha>29ee79a10c58dd6863a46157e374521cac610ad8</Sha> </Dependency> - <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19455.3"> + <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19458.2"> <Uri>https://github.com/dotnet/arcade</Uri> - <Sha>fee4ebe849050a385a4999e4c84fa61c1dc234bb</Sha> + <Sha>29ee79a10c58dd6863a46157e374521cac610ad8</Sha> </Dependency> - <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="2.2.0-beta.19455.3"> + <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="2.2.0-beta.19458.2"> <Uri>https://github.com/dotnet/arcade</Uri> - <Sha>fee4ebe849050a385a4999e4c84fa61c1dc234bb</Sha> + <Sha>29ee79a10c58dd6863a46157e374521cac610ad8</Sha> </Dependency> - <Dependency Name="Microsoft.DotNet.Build.Tasks.Packaging" Version="1.0.0-beta.19455.3"> + <Dependency Name="Microsoft.DotNet.Build.Tasks.Packaging" Version="1.0.0-beta.19458.2"> <Uri>https://github.com/dotnet/arcade</Uri> - <Sha>fee4ebe849050a385a4999e4c84fa61c1dc234bb</Sha> + <Sha>29ee79a10c58dd6863a46157e374521cac610ad8</Sha> </Dependency> - <Dependency Name="Microsoft.Private.CoreFx.NETCoreApp" Version="4.6.0-preview9.19419.5"> + <Dependency Name="Microsoft.Private.CoreFx.NETCoreApp" Version="4.6.0-rc2.19458.3"> <Uri>https://github.com/dotnet/corefx</Uri> - <Sha>961b953eb77922d3906506ee15a58068abfa3284</Sha> + <Sha>173d70b99a5dbb3ff9298a6e2e9c7f7e7b56dd7c</Sha> </Dependency> - <Dependency Name="Microsoft.NETCore.Platforms" Version="3.0.0-preview9.19419.5"> + <Dependency Name="Microsoft.NETCore.Platforms" Version="3.0.0-rc2.19458.3"> <Uri>https://github.com/dotnet/corefx</Uri> - <Sha>961b953eb77922d3906506ee15a58068abfa3284</Sha> + <Sha>173d70b99a5dbb3ff9298a6e2e9c7f7e7b56dd7c</Sha> </Dependency> <Dependency Name="Microsoft.NETCore.App" Version="3.1.0-preview1.19458.7"> <Uri>https://github.com/dotnet/core-setup</Uri> <Sha>604ef7a61aea16bc63a402dfcb2bbe5e6b7c5971</Sha> </Dependency> - <Dependency Name="optimization.IBC.CoreCLR" Version="99.99.99-master-20190807.1"> + <Dependency Name="optimization.IBC.CoreCLR" Version="99.99.99-master-20190909.1"> <Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-optimization</Uri> - <Sha>f721b85e367cd7dacaa74f5761cfd34d9e4fc423</Sha> + <Sha>48fd58c24cddbc6c0b0de1c00204aae1170f4864</Sha> </Dependency> - <Dependency Name="optimization.PGO.CoreCLR" Version="99.99.99-master-20190807.1"> + <Dependency Name="optimization.PGO.CoreCLR" Version="99.99.99-master-20190909.1"> <Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-optimization</Uri> - <Sha>f721b85e367cd7dacaa74f5761cfd34d9e4fc423</Sha> + <Sha>48fd58c24cddbc6c0b0de1c00204aae1170f4864</Sha> </Dependency> </ToolsetDependencies> </Dependencies> diff --git a/eng/Versions.props b/eng/Versions.props index 9e3fcdd7bf..924ed8899c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -15,22 +15,22 @@ <UsingToolXliff>false</UsingToolXliff> <!-- Package versions --> <!-- arcade --> - <MicrosoftDotNetBuildTasksFeedVersion>2.2.0-beta.19455.3</MicrosoftDotNetBuildTasksFeedVersion> - <MicrosoftDotNetBuildTasksPackagingVersion>1.0.0-beta.19455.3</MicrosoftDotNetBuildTasksPackagingVersion> + <MicrosoftDotNetBuildTasksFeedVersion>2.2.0-beta.19458.2</MicrosoftDotNetBuildTasksFeedVersion> + <MicrosoftDotNetBuildTasksPackagingVersion>1.0.0-beta.19458.2</MicrosoftDotNetBuildTasksPackagingVersion> <MicrosoftDotNetXUnitConsoleRunnerVersion>2.5.1-beta.19278.1</MicrosoftDotNetXUnitConsoleRunnerVersion> <!-- roslyn --> <MicrosoftNetCompilersToolsetVersion>3.3.0-beta2-19367-02</MicrosoftNetCompilersToolsetVersion> <!-- corefx --> - <MicrosoftPrivateCoreFxNETCoreAppVersion>4.6.0-preview9.19419.5</MicrosoftPrivateCoreFxNETCoreAppVersion> - <MicrosoftNETCorePlatformsVersion>3.0.0-preview9.19419.5</MicrosoftNETCorePlatformsVersion> + <MicrosoftPrivateCoreFxNETCoreAppVersion>4.6.0-rc2.19458.3</MicrosoftPrivateCoreFxNETCoreAppVersion> + <MicrosoftNETCorePlatformsVersion>3.0.0-rc2.19458.3</MicrosoftNETCorePlatformsVersion> <MicrosoftBclAsyncInterfacesVersion>1.0.0-preview7.19326.2</MicrosoftBclAsyncInterfacesVersion> <!-- core-setup --> <MicrosoftNETCoreAppVersion>3.1.0-preview1.19458.7</MicrosoftNETCoreAppVersion> <!-- dotnet-core-internal-tooling --> <IbcMergePackageVersion>5.0.6-beta.19203.1</IbcMergePackageVersion> <!-- dotnet-optimization --> - <optimizationIBCCoreCLRVersion>99.99.99-master-20190807.1</optimizationIBCCoreCLRVersion> - <optimizationPGOCoreCLRVersion>99.99.99-master-20190807.1</optimizationPGOCoreCLRVersion> + <optimizationIBCCoreCLRVersion>99.99.99-master-20190909.1</optimizationIBCCoreCLRVersion> + <optimizationPGOCoreCLRVersion>99.99.99-master-20190909.1</optimizationPGOCoreCLRVersion> </PropertyGroup> <!--Package names--> <PropertyGroup> diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index feb58d1419..e001ccb481 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -85,6 +85,10 @@ function Build { # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. # Explicitly set the type as string[] because otherwise PowerShell would make this char[] if $properties is empty. [string[]] $msbuildArgs = $properties + + # Resolve relative project paths into full paths + $projects = ($projects.Split(';').ForEach({Resolve-Path $_}) -join ';') + $msbuildArgs += "/p:Projects=$projects" $properties = $msbuildArgs } diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 268986246e..ec41965fc8 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -17,7 +17,7 @@ Param( [string] $Configurations="CompilationMode=$CompilationMode" ) -$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") +$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") $UseCoreRun = ($CoreRootDirectory -ne [string]::Empty) $UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty) diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 550b3ebf18..2f2092166e 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -113,7 +113,7 @@ while (($# > 0)); do esac done -if [[ "$repository" == "dotnet/performance" ]]; then +if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-performance" ]; then run_from_perf_repo=true fi diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 8db456bb7f..ffda80a197 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -24,7 +24,7 @@ parameters: workspace: '' -# Job base template specific parameters + # Job base template specific parameters # Optional: Enable installing Microbuild plugin # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix # _TeamName - the name of your team @@ -151,6 +151,9 @@ jobs: continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: NuGetAuthenticate@0 + - ${{ each step in parameters.steps }}: - ${{ step }} diff --git a/eng/common/templates/job/performance.yml b/eng/common/templates/job/performance.yml index ef809253d1..f877fd7a89 100644 --- a/eng/common/templates/job/performance.yml +++ b/eng/common/templates/job/performance.yml @@ -5,6 +5,7 @@ parameters: displayName: '' # optional -- display name for the job. Will use jobName if not passed pool: '' # required -- name of the Build pool container: '' # required -- name of the container + osGroup: '' # required -- operating system for the job extraSetupParameters: '' # optional -- extra arguments to pass to the setup script frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against continueOnError: 'false' # optional -- determines whether to continue the build if the step errors @@ -44,12 +45,13 @@ jobs: - HelixPreCommand: '' - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(variables['Agent.Os'], 'Windows_NT') }}: + - ${{ if eq( parameters.osGroup, 'Windows_NT') }}: - HelixPreCommand: 'set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"' - IsInternal: -Internal - - ${{ if ne(variables['Agent.Os'], 'Windows_NT') }}: + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - HelixPreCommand: 'export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' - IsInternal: --internal + - group: DotNet-HelixApi-Access - group: dotnet-benchview diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 9e77ef1b54..b722975f9c 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -47,6 +47,10 @@ jobs: downloadPath: '$(Build.StagingDirectory)/Download' condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: NuGetAuthenticate@0 + - task: PowerShell@2 displayName: Publish Build Assets inputs: @@ -59,6 +63,7 @@ jobs: /p:Configuration=$(_BuildConfig) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -67,12 +72,14 @@ jobs: Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + - task: PublishBuildArtifacts@1 displayName: Publish ReleaseConfigs Artifact inputs: PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' PublishLocation: Container ArtifactName: ReleaseConfigs + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - task: PublishBuildArtifacts@1 displayName: Publish Logs to VSTS diff --git a/eng/common/templates/post-build/channels/netcore-dev-30.yml b/eng/common/templates/post-build/channels/netcore-dev-30.yml index 3ce7ce4d6e..7984f06d1b 100644 --- a/eng/common/templates/post-build/channels/netcore-dev-30.yml +++ b/eng/common/templates/post-build/channels/netcore-dev-30.yml @@ -1,5 +1,4 @@ parameters: - enableSymbolValidation: true symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' publishInstallersAndChecksums: false @@ -47,7 +46,7 @@ stages: /p:Configuration=Release ${{ parameters.symbolPublishingAdditionalParameters }} - - job: + - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars variables: @@ -79,30 +78,41 @@ stages: buildType: current artifactName: AssetManifests + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 - displayName: Add Assets Location + displayName: Publish Assets env: - AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ChannelId=$(PublicDevRelease_30_Channel_Id) + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' - /p:Configuration=Release + /p:StaticInternalFeed=$(StaticInternalFeed) + /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl) + /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey) + /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) + /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) @@ -110,24 +120,6 @@ stages: /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) ${{ parameters.artifactsPublishingAdditionalParameters }} - - task: NuGetCommand@2 - displayName: Publish Packages to AzDO Feed - condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') - inputs: - command: push - vstsFeed: $(AzDoFeedName) - packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg - publishVstsFeed: $(AzDoFeedName) - - - task: PowerShell@2 - displayName: Publish Blobs to AzDO Feed - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) - -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) - enabled: false - - stage: NetCore_Dev30_Publish_Validation displayName: .NET Core 3.0 Dev Publish Validation @@ -136,30 +128,6 @@ stages: jobs: - template: ../setup-maestro-vars.yml - - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: - - job: - displayName: Symbol Availability - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_30_Channel_Id)) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Check Symbol Availability - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - - template: ../darc-gather-drop.yml - parameters: - ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} - - template: ../promote-build.yml parameters: ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-dev-31.yml b/eng/common/templates/post-build/channels/netcore-dev-31.yml index d40aaacc4f..bcedd0f075 100644 --- a/eng/common/templates/post-build/channels/netcore-dev-31.yml +++ b/eng/common/templates/post-build/channels/netcore-dev-31.yml @@ -1,5 +1,4 @@ parameters: - enableSymbolValidation: true symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' publishInstallersAndChecksums: false @@ -79,22 +78,33 @@ stages: buildType: current artifactName: AssetManifests + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 - displayName: Add Assets Location + displayName: Publish Assets env: - AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ChannelId=$(PublicDevRelease_31_Channel_Id) /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:StaticInternalFeed=$(StaticInternalFeed) + /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl) + /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey) + /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) + /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' /p:BARBuildId=$(BARBuildId) /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' @@ -110,24 +120,6 @@ stages: /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) ${{ parameters.artifactsPublishingAdditionalParameters }} - - task: NuGetCommand@2 - displayName: Publish Packages to AzDO Feed - condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') - inputs: - command: push - vstsFeed: $(AzDoFeedName) - packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg - publishVstsFeed: $(AzDoFeedName) - - - task: PowerShell@2 - displayName: Publish Blobs to AzDO Feed - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) - -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) - enabled: false - - stage: NetCore_Dev31_Publish_Validation displayName: .NET Core 3.1 Dev Publish Validation @@ -136,30 +128,6 @@ stages: jobs: - template: ../setup-maestro-vars.yml - - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: - - job: - displayName: Symbol Availability - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_31_Channel_Id)) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Check Symbol Availability - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - - template: ../darc-gather-drop.yml - parameters: - ChannelId: ${{ variables.PublicDevRelease_31_Channel_Id }} - - template: ../promote-build.yml parameters: ChannelId: ${{ variables.PublicDevRelease_31_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-dev-5.yml b/eng/common/templates/post-build/channels/netcore-dev-5.yml index 584185c2a7..18432cc60b 100644 --- a/eng/common/templates/post-build/channels/netcore-dev-5.yml +++ b/eng/common/templates/post-build/channels/netcore-dev-5.yml @@ -1,8 +1,8 @@ parameters: - enableSymbolValidation: true symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' publishInstallersAndChecksums: false + publishToAzureDevOpsFeeds: true stages: - stage: NetCore_Dev5_Publish @@ -47,7 +47,7 @@ stages: /p:Configuration=Release ${{ parameters.symbolPublishingAdditionalParameters }} - - job: + - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars variables: @@ -79,55 +79,53 @@ stages: buildType: current artifactName: AssetManifests + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 - displayName: Add Assets Location + displayName: Publish Assets env: - AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ChannelId=$(NetCore_5_Dev_Channel_Id) + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:StaticInternalFeed=$(StaticInternalFeed) + /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl) + /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey) + /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) + /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' /p:Configuration=Release + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) - /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:PublishToAzureDevOpsNuGetFeeds=${{ parameters.publishToAzureDevOpsFeeds }} + /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' ${{ parameters.artifactsPublishingAdditionalParameters }} - - task: NuGetCommand@2 - displayName: Publish Packages to AzDO Feed - condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') - inputs: - command: push - vstsFeed: $(AzDoFeedName) - packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg - publishVstsFeed: $(AzDoFeedName) - - - task: PowerShell@2 - displayName: Publish Blobs to AzDO Feed - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) - -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) - enabled: false - - stage: NetCore_Dev5_Publish_Validation displayName: .NET Core 5 Dev Publish Validation @@ -136,30 +134,6 @@ stages: jobs: - template: ../setup-maestro-vars.yml - - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: - - job: - displayName: Symbol Availability - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_5_Dev_Channel_Id)) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Check Symbol Availability - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - - template: ../darc-gather-drop.yml - parameters: - ChannelId: ${{ variables.NetCore_5_Dev_Channel_Id }} - - template: ../promote-build.yml parameters: ChannelId: ${{ variables.NetCore_5_Dev_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-internal-30.yml b/eng/common/templates/post-build/channels/netcore-internal-30.yml index 8bf88fd49e..36e1d1188b 100644 --- a/eng/common/templates/post-build/channels/netcore-internal-30.yml +++ b/eng/common/templates/post-build/channels/netcore-internal-30.yml @@ -1,5 +1,4 @@ parameters: - enableSymbolValidation: true symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' @@ -78,50 +77,42 @@ stages: buildType: current artifactName: AssetManifests + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 - displayName: Add Assets Location + displayName: Publish Assets env: - AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ChannelId=$(InternalServicing_30_Channel_Id) + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName) - /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1) - /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url) - /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url) - /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' - /p:Configuration=Release + /p:StaticInternalFeed=$(StaticInternalFeed) + /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl) + /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey) + /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) + /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release ${{ parameters.artifactsPublishingAdditionalParameters }} - - task: NuGetCommand@2 - displayName: Publish Packages to AzDO Feed - condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') - inputs: - command: push - vstsFeed: $(AzDoFeedName) - packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg - publishVstsFeed: $(AzDoFeedName) - - - task: PowerShell@2 - displayName: Publish Blobs to AzDO Feed - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) - -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) - enabled: false - - template: ../trigger-subscription.yml parameters: ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} @@ -133,26 +124,6 @@ stages: jobs: - template: ../setup-maestro-vars.yml - - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: - - job: - displayName: Symbol Availability - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.InternalServicing_30_Channel_Id)) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Check Symbol Availability - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - template: ../promote-build.yml parameters: ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-release-30.yml b/eng/common/templates/post-build/channels/netcore-release-30.yml index dc9541d9ec..abae985ab0 100644 --- a/eng/common/templates/post-build/channels/netcore-release-30.yml +++ b/eng/common/templates/post-build/channels/netcore-release-30.yml @@ -1,5 +1,4 @@ parameters: - enableSymbolValidation: true symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' @@ -78,51 +77,42 @@ stages: buildType: current artifactName: AssetManifests + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 - displayName: Publish + displayName: Publish Assets env: - AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ChannelId=$(PublicRelease_30_Channel_Id) + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName) - /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1) - /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url) - /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url) - /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' - /p:Configuration=Release + /p:StaticInternalFeed=$(StaticInternalFeed) + /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl) + /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey) + /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) + /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release ${{ parameters.artifactsPublishingAdditionalParameters }} - - task: NuGetCommand@2 - displayName: Publish Packages to AzDO Feed - condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') - inputs: - command: push - vstsFeed: $(AzDoFeedName) - packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg - publishVstsFeed: $(AzDoFeedName) - - - task: PowerShell@2 - displayName: Publish Blobs to AzDO Feed - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) - -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) - enabled: false - - template: ../trigger-subscription.yml parameters: ChannelId: ${{ variables.PublicRelease_30_Channel_Id }} @@ -134,27 +124,6 @@ stages: jobs: - template: ../setup-maestro-vars.yml - - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: - - job: - displayName: Symbol Availability - dependsOn: setupMaestroVars - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id)), - contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id))) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Check Symbol Availability - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - template: ../promote-build.yml parameters: ChannelId: ${{ variables.PublicRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-release-31.yml b/eng/common/templates/post-build/channels/netcore-release-31.yml index 4dcf4ba9c8..b2a6c7659d 100644 --- a/eng/common/templates/post-build/channels/netcore-release-31.yml +++ b/eng/common/templates/post-build/channels/netcore-release-31.yml @@ -1,5 +1,4 @@ parameters: - enableSymbolValidation: true symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' @@ -78,51 +77,48 @@ stages: buildType: current artifactName: AssetManifests + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 - displayName: Publish + displayName: Publish Assets env: - AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ChannelId=$(PublicRelease_31_Channel_Id) + /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) + /p:StaticInternalFeed=$(StaticInternalFeed) + /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl) + /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey) + /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) + /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName) - /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1) - /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url) - /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url) - /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe /p:BARBuildId=$(BARBuildId) /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' /p:Configuration=Release + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} + /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) + /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) + /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) + /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) ${{ parameters.artifactsPublishingAdditionalParameters }} - - task: NuGetCommand@2 - displayName: Publish Packages to AzDO Feed - condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') - inputs: - command: push - vstsFeed: $(AzDoFeedName) - packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg - publishVstsFeed: $(AzDoFeedName) - - - task: PowerShell@2 - displayName: Publish Blobs to AzDO Feed - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) - -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) - enabled: false - - template: ../trigger-subscription.yml parameters: ChannelId: ${{ variables.PublicRelease_31_Channel_Id }} @@ -134,27 +130,6 @@ stages: jobs: - template: ../setup-maestro-vars.yml - - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: - - job: - displayName: Symbol Availability - dependsOn: setupMaestroVars - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id)), - contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id))) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Check Symbol Availability - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - template: ../promote-build.yml parameters: ChannelId: ${{ variables.PublicRelease_31_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/netcore-tools-latest.yml b/eng/common/templates/post-build/channels/netcore-tools-latest.yml index 4d9f197d43..36f6dea628 100644 --- a/eng/common/templates/post-build/channels/netcore-tools-latest.yml +++ b/eng/common/templates/post-build/channels/netcore-tools-latest.yml @@ -1,10 +1,8 @@ parameters: - enableSymbolValidation: true symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' publishInstallersAndChecksums: false - publishToAzureDevOpsFeeds: false - azureDevOpsToolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + publishToAzureDevOpsFeeds: true stages: - stage: NetCore_Tools_Latest_Publish @@ -49,7 +47,7 @@ stages: /p:Configuration=Release ${{ parameters.symbolPublishingAdditionalParameters }} - - job: + - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars variables: @@ -81,60 +79,53 @@ stages: buildType: current artifactName: AssetManifests + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 - displayName: Add Assets Location + displayName: Publish Assets env: - AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ChannelId=$(NetCore_Tools_Latest_Channel_Id) + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:StaticInternalFeed=$(StaticInternalFeed) + /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl) + /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey) + /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) + /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' /p:Configuration=Release - /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} + /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) /p:PublishToAzureDevOpsNuGetFeeds=${{ parameters.publishToAzureDevOpsFeeds }} - /p:AzureDevOpsStaticShippingFeed=${{ parameters.azureDevOpsToolsFeed }} + /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' - /p:AzureDevOpsStaticTransportFeed=${{ parameters.azureDevOpsToolsFeed }} + /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw) ${{ parameters.artifactsPublishingAdditionalParameters }} - - - task: NuGetCommand@2 - displayName: Publish Packages to AzDO Feed - condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') - inputs: - command: push - vstsFeed: $(AzDoFeedName) - packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg - publishVstsFeed: $(AzDoFeedName) - - - task: PowerShell@2 - displayName: Publish Blobs to AzDO Feed - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) - -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) - enabled: false - + - stage: NetCore_Tools_Latest_PublishValidation displayName: .NET Tools - Latest Publish Validation @@ -143,30 +134,6 @@ stages: jobs: - template: ../setup-maestro-vars.yml - - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: - - job: - displayName: Symbol Availability - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_Tools_Latest_Channel_Id)) - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Check Symbol Availability - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - - template: ../darc-gather-drop.yml - parameters: - ChannelId: ${{ variables.NetCore_Tools_Latest_Channel_Id }} - - template: ../promote-build.yml parameters: ChannelId: ${{ variables.NetCore_Tools_Latest_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml index e306fa87bf..a5fcdc581a 100644 --- a/eng/common/templates/post-build/channels/public-validation-release.yml +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -1,8 +1,7 @@ parameters: artifactsPublishingAdditionalParameters: '' publishInstallersAndChecksums: false - publishToAzureDevOpsFeeds: false - azureDevOpsToolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + publishToAzureDevOpsFeeds: true stages: - stage: PVR_Publish @@ -13,7 +12,7 @@ stages: jobs: - template: ../setup-maestro-vars.yml - - job: + - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars variables: @@ -45,60 +44,48 @@ stages: buildType: current artifactName: AssetManifests + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 - displayName: Add Assets Location + displayName: Publish Assets env: - AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw) inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ChannelId=$(PublicValidationRelease_30_Channel_Id) + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ArtifactsCategory=$(_DotNetValidationArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' - /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' - /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' - /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' - /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' /p:Configuration=Release + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) - /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) /p:PublishToAzureDevOpsNuGetFeeds=${{ parameters.publishToAzureDevOpsFeeds }} - /p:AzureDevOpsStaticShippingFeed=${{ parameters.azureDevOpsToolsFeed }} + /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' - /p:AzureDevOpsStaticTransportFeed=${{ parameters.azureDevOpsToolsFeed }} + /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' ${{ parameters.artifactsPublishingAdditionalParameters }} - - task: NuGetCommand@2 - displayName: Publish Packages to AzDO Feed - condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') - inputs: - command: push - vstsFeed: $(AzDoFeedName) - packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg - publishVstsFeed: $(AzDoFeedName) - - - task: PowerShell@2 - displayName: Publish Blobs to AzDO Feed - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 - arguments: -FeedName $(AzDoFeedName) - -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) - enabled: false - - stage: PVR_PublishValidation displayName: .NET Tools - Validation Publish Validation @@ -107,10 +94,6 @@ stages: jobs: - template: ../setup-maestro-vars.yml - - template: ../darc-gather-drop.yml - parameters: - ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} - - template: ../promote-build.yml parameters: ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index f62c097b3d..c134764496 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,6 +1,7 @@ variables: - group: Publish-Build-Assets - group: DotNet-DotNetCli-Storage + - group: DotNet-MSRC-Storage # .NET Core 3 Dev - name: PublicDevRelease_30_Channel_Id @@ -38,10 +39,6 @@ variables: - name: IsInternalBuild value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - # Storage account name for proxy-backed feeds - - name: ProxyBackedFeedsAccountName - value: dotnetfeed - # Default Maestro++ API Endpoint and API Version - name: MaestroApiEndPoint value: "https://maestro-prod.westus2.cloudapp.azure.com" @@ -55,8 +52,27 @@ variables: - name: SymbolToolVersion value: 1.0.1 + # Feed Configurations + # These should include the suffix "/index.json" + + # Configuration for the feed where packages from internal non-stable builds will be published to + - name: StaticInternalFeed + value: 'https://dnceng.pkgs.visualstudio.com/_packaging/dotnet-core-internal/nuget/v3/index.json' + # Default locations for Installers and checksums + # Public Locations - name: ChecksumsBlobFeedUrl value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json - name: InstallersBlobFeedUrl value: https://dotnetcli.blob.core.windows.net/dotnet/index.json + + # Private Locations + - name: InternalChecksumsBlobFeedUrl + value: https://dotnetclichecksumsmsrc.blob.core.windows.net/dotnet/index.json + - name: InternalChecksumsBlobFeedKey + value: $(dotnetclichecksumsmsrc-storage-key) + + - name: InternalInstallersBlobFeedUrl + value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json + - name: InternalInstallersBlobFeedKey + value: $(dotnetclimsrc-access-key) diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index ae376b7747..e473cadbcc 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -98,28 +98,25 @@ stages: - template: \eng\common\templates\post-build\channels\netcore-dev-5.yml parameters: - enableSymbolValidation: ${{ parameters.enableSymbolValidation }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + enableAzDONuGetFeeds: ${{ parameters.enableAzDONuGetFeeds }} - template: \eng\common\templates\post-build\channels\netcore-dev-30.yml parameters: - enableSymbolValidation: ${{ parameters.enableSymbolValidation }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - template: \eng\common\templates\post-build\channels\netcore-dev-31.yml parameters: - enableSymbolValidation: ${{ parameters.enableSymbolValidation }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - template: \eng\common\templates\post-build\channels\netcore-tools-latest.yml parameters: - enableSymbolValidation: ${{ parameters.enableSymbolValidation }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} diff --git a/eng/pipelines/internal.yml b/eng/pipelines/internal.yml index 274d9be9d4..cacacffe29 100644 --- a/eng/pipelines/internal.yml +++ b/eng/pipelines/internal.yml @@ -44,32 +44,6 @@ stages: timeoutInMinutes: 120 # - # Release test builds - # - - template: /eng/platform-matrix.yml - parameters: - jobTemplate: test-job.yml - buildConfig: release - platformGroup: all - helixQueueGroup: all - jobParameters: - testGroup: outerloop - - # - # ReadyToRun test builds - # - - template: /eng/platform-matrix.yml - parameters: - jobTemplate: test-job.yml - buildConfig: release - platformGroup: all - helixQueueGroup: all - jobParameters: - testGroup: outerloop - readyToRun: true - displayNameArgs: R2R - - # # Publish build information to Build Assets Registry # # This job gathers build assets from the pipeline (from each official @@ -96,3 +70,34 @@ stages: parameters: # Symbol validation is not entirely reliable as of yet, so should be turned off until https://github.com/dotnet/arcade/issues/2871 is resolved. enableSymbolValidation: false + + - stage: test + displayName: Test + jobs: + # + # Release test builds + # + - template: /eng/platform-matrix.yml + parameters: + jobTemplate: test-job.yml + buildConfig: release + platformGroup: all + helixQueueGroup: all + ignoreDependencyOnBuildJobs: true + jobParameters: + testGroup: outerloop + + # + # ReadyToRun test builds + # + - template: /eng/platform-matrix.yml + parameters: + jobTemplate: test-job.yml + buildConfig: release + platformGroup: all + helixQueueGroup: all + ignoreDependencyOnBuildJobs: true + jobParameters: + testGroup: outerloop + readyToRun: true + displayNameArgs: R2R diff --git a/eng/platform-matrix.yml b/eng/platform-matrix.yml index 08d73183a6..29f11a6f19 100644 --- a/eng/platform-matrix.yml +++ b/eng/platform-matrix.yml @@ -13,6 +13,7 @@ parameters: # 'all' - the queues used for non-PR, non-CI test runs, e.g., Manual or Scheduled runs. Typically this is all available queues. # 'corefx' - the queues used for a corefx test run. helixQueueGroup: 'pr' + ignoreDependencyOnBuildJobs: false jobParameters: {} jobs: @@ -25,6 +26,7 @@ jobs: - ${{ if or(containsValue(parameters.platforms, 'Linux_arm'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - template: ${{ parameters.jobTemplate }} parameters: + ignoreDependencyOnBuildJobs: ${{ parameters.ignoreDependencyOnBuildJobs }} buildConfig: ${{ parameters.buildConfig }} archType: arm osGroup: Linux @@ -33,11 +35,8 @@ jobs: image: ubuntu-16.04-cross-14.04-23cacb0-20190528233931 registry: mcr helixQueues: - # Ubuntu.1404.Arm32.Open is used only by CI while Ubuntu.1604.Arm32.Open serves PRs and scheduled builds. - - ${{ if and(eq(variables['System.TeamProject'], 'public'), in(parameters.helixQueueGroup, 'ci')) }}: - - Ubuntu.1404.Arm32.Open - - ${{ if and(eq(variables['System.TeamProject'], 'public'), notIn(parameters.helixQueueGroup, 'ci')) }}: - - (Ubuntu.1804.Arm32.Open)Ubuntu.1604.Arm32.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-09a60ed-20190620155854 + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - (Ubuntu.1804.Arm32.Open)Ubuntu.1604.Arm32.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-30f6673-20190814153226 - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - (Debian.9.Arm32)Ubuntu.1604.Arm32@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-9-helix-arm32v7-74c9941-20190620155841 - (Ubuntu.1604.Arm32)Ubuntu.1604.Arm32@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-helix-arm32v7-a45aeeb-20190620160312 @@ -50,6 +49,7 @@ jobs: - ${{ if or(containsValue(parameters.platforms, 'Linux_arm64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - template: ${{ parameters.jobTemplate }} parameters: + ignoreDependencyOnBuildJobs: ${{ parameters.ignoreDependencyOnBuildJobs }} buildConfig: ${{ parameters.buildConfig }} archType: arm64 osGroup: Linux @@ -73,6 +73,7 @@ jobs: - ${{ if or(containsValue(parameters.platforms, 'Linux_musl_x64'), eq(parameters.platformGroup, 'all')) }}: - template: ${{ parameters.jobTemplate }} parameters: + ignoreDependencyOnBuildJobs: ${{ parameters.ignoreDependencyOnBuildJobs }} buildConfig: ${{ parameters.buildConfig }} archType: x64 osGroup: Linux @@ -93,6 +94,7 @@ jobs: - ${{ if or(containsValue(parameters.platforms, 'Linux_musl_arm64'), eq(parameters.platformGroup, 'all')) }}: - template: ${{ parameters.jobTemplate }} parameters: + ignoreDependencyOnBuildJobs: ${{ parameters.ignoreDependencyOnBuildJobs }} buildConfig: ${{ parameters.buildConfig }} archType: arm64 osGroup: Linux @@ -113,6 +115,7 @@ jobs: - ${{ if or(containsValue(parameters.platforms, 'Linux_rhel6_x64'), eq(parameters.platformGroup, 'all')) }}: - template: ${{ parameters.jobTemplate }} parameters: + ignoreDependencyOnBuildJobs: ${{ parameters.ignoreDependencyOnBuildJobs }} buildConfig: ${{ parameters.buildConfig }} archType: x64 osGroup: Linux @@ -131,6 +134,7 @@ jobs: - ${{ if or(containsValue(parameters.platforms, 'Linux_x64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - template: ${{ parameters.jobTemplate }} parameters: + ignoreDependencyOnBuildJobs: ${{ parameters.ignoreDependencyOnBuildJobs }} buildConfig: ${{ parameters.buildConfig }} archType: x64 osGroup: Linux @@ -178,6 +182,7 @@ jobs: - ${{ if or(containsValue(parameters.platforms, 'OSX_x64'), eq(parameters.platformGroup, 'all')) }}: - template: ${{ parameters.jobTemplate }} parameters: + ignoreDependencyOnBuildJobs: ${{ parameters.ignoreDependencyOnBuildJobs }} buildConfig: ${{ parameters.buildConfig }} archType: x64 osGroup: OSX @@ -200,6 +205,7 @@ jobs: - ${{ if or(containsValue(parameters.platforms, 'Windows_NT_x64'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - template: ${{ parameters.jobTemplate }} parameters: + ignoreDependencyOnBuildJobs: ${{ parameters.ignoreDependencyOnBuildJobs }} buildConfig: ${{ parameters.buildConfig }} archType: x64 osGroup: Windows_NT @@ -225,6 +231,7 @@ jobs: - ${{ if or(containsValue(parameters.platforms, 'Windows_NT_x86'), in(parameters.platformGroup, 'all', 'gcstress')) }}: - template: ${{ parameters.jobTemplate }} parameters: + ignoreDependencyOnBuildJobs: ${{ parameters.ignoreDependencyOnBuildJobs }} buildConfig: ${{ parameters.buildConfig }} archType: x86 osGroup: Windows_NT @@ -248,6 +255,7 @@ jobs: - ${{ if or(containsValue(parameters.platforms, 'Windows_NT_arm'), eq(parameters.platformGroup, 'all')) }}: - template: ${{ parameters.jobTemplate }} parameters: + ignoreDependencyOnBuildJobs: ${{ parameters.ignoreDependencyOnBuildJobs }} buildConfig: ${{ parameters.buildConfig }} archType: arm osGroup: Windows_NT @@ -267,6 +275,7 @@ jobs: - ${{ if or(containsValue(parameters.platforms, 'Windows_NT_arm64'), eq(parameters.platformGroup, 'all')) }}: - template: ${{ parameters.jobTemplate }} parameters: + ignoreDependencyOnBuildJobs: ${{ parameters.ignoreDependencyOnBuildJobs }} buildConfig: ${{ parameters.buildConfig }} archType: arm64 osGroup: Windows_NT diff --git a/eng/test-job.yml b/eng/test-job.yml index bc4e7aad1c..c8e56e58dd 100644 --- a/eng/test-job.yml +++ b/eng/test-job.yml @@ -12,6 +12,7 @@ parameters: corefxTests: false displayNameArgs: '' runInUnloadableContext: false + ignoreDependencyOnBuildJobs: false ### Test job @@ -96,7 +97,8 @@ jobs: condition: false # Test job depends on the corresponding build job - dependsOn: ${{ format('build_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + ${{ if ne(parameters.ignoreDependencyOnBuildJobs, true) }}: + dependsOn: ${{ format('build_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml diff --git a/global.json b/global.json index 9783bf9745..c374a7b017 100644 --- a/global.json +++ b/global.json @@ -7,8 +7,8 @@ "python": "2.7.15" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19455.3", - "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19455.3", + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19458.2", + "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19458.2", "Microsoft.Build.NoTargets": "1.0.53", "Microsoft.Build.Traversal": "2.0.2" } diff --git a/src/System.Private.CoreLib/shared/System/Buffers/ConfigurableArrayPool.cs b/src/System.Private.CoreLib/shared/System/Buffers/ConfigurableArrayPool.cs index 054a38fc60..6dd3063258 100644 --- a/src/System.Private.CoreLib/shared/System/Buffers/ConfigurableArrayPool.cs +++ b/src/System.Private.CoreLib/shared/System/Buffers/ConfigurableArrayPool.cs @@ -100,13 +100,13 @@ namespace System.Buffers // The pool was exhausted for this buffer size. Allocate a new buffer with a size corresponding // to the appropriate bucket. - buffer = GC.AllocateUninitializedArray<T>(_buckets[index]._bufferLength); + buffer = new T[_buckets[index]._bufferLength]; } else { // The request was for a size too large for the pool. Allocate an array of exactly the requested length. // When it's returned to the pool, we'll simply throw it away. - buffer = GC.AllocateUninitializedArray<T>(minimumLength); + buffer = new T[minimumLength]; } if (log.IsEnabled()) @@ -215,7 +215,7 @@ namespace System.Buffers // for that slot, in which case we should do so now. if (allocateBuffer) { - buffer = GC.AllocateUninitializedArray<T>(_bufferLength); + buffer = new T[_bufferLength]; var log = ArrayPoolEventSource.Log; if (log.IsEnabled()) diff --git a/src/System.Private.CoreLib/shared/System/Globalization/DateTimeFormatInfo.cs b/src/System.Private.CoreLib/shared/System/Globalization/DateTimeFormatInfo.cs index db64442035..a31d3bddaa 100644 --- a/src/System.Private.CoreLib/shared/System/Globalization/DateTimeFormatInfo.cs +++ b/src/System.Private.CoreLib/shared/System/Globalization/DateTimeFormatInfo.cs @@ -1349,7 +1349,7 @@ namespace System.Globalization /// <summary> /// Retrieve the array which contains the month names in genitive form. - /// If this culture does not use the gentive form, the normal month name is returned. + /// If this culture does not use the genitive form, the normal month name is returned. /// </summary> private string[] InternalGetGenitiveMonthNames(bool abbreviated) { @@ -2320,14 +2320,15 @@ namespace System.Globalization InsertHash(temp, GetAbbreviatedMonthName(i), TokenType.MonthToken, i); } - if ((FormatFlags & DateTimeFormatFlags.UseGenitiveMonth) != 0) { + string [] genitiveMonthNames = InternalGetGenitiveMonthNames(abbreviated: false); + string [] abbreviatedGenitiveMonthNames = InternalGetGenitiveMonthNames(abbreviated: true); + for (int i = 1; i <= 13; i++) { - string str; - str = InternalGetMonthName(i, MonthNameStyles.Genitive, false); - InsertHash(temp, str, TokenType.MonthToken, i); + InsertHash(temp, genitiveMonthNames[i - 1], TokenType.MonthToken, i); + InsertHash(temp, abbreviatedGenitiveMonthNames[i - 1], TokenType.MonthToken, i); } } diff --git a/src/System.Private.CoreLib/shared/System/Globalization/DateTimeFormatInfoScanner.cs b/src/System.Private.CoreLib/shared/System/Globalization/DateTimeFormatInfoScanner.cs index accc5b9b63..beef72a14e 100644 --- a/src/System.Private.CoreLib/shared/System/Globalization/DateTimeFormatInfoScanner.cs +++ b/src/System.Private.CoreLib/shared/System/Globalization/DateTimeFormatInfoScanner.cs @@ -354,10 +354,10 @@ namespace System.Globalization //////////////////////////////////////////////////////////////////////////// // - // Add the text that is a date separator but is treated like ignroable symbol. + // Add the text that is a date separator but is treated like ignorable symbol. // E.g. // hu-HU has: - // shrot date pattern: yyyy. MM. dd.;yyyy-MM-dd;yy-MM-dd + // short date pattern: yyyy. MM. dd.;yyyy-MM-dd;yy-MM-dd // long date pattern: yyyy. MMMM d. // Here, "." is the date separator (derived from short date pattern). However, // "." also appear at the end of long date pattern. In this case, we just @@ -614,7 +614,7 @@ namespace System.Globalization //----------------------------------------------------------------------------- // EqualStringArrays // compares two string arrays and return true if all elements of the first - // array equals to all elmentsof the second array. + // array equals to all elements of the second array. // otherwise it returns false. //----------------------------------------------------------------------------- diff --git a/src/System.Private.CoreLib/shared/System/Globalization/DateTimeParse.cs b/src/System.Private.CoreLib/shared/System/Globalization/DateTimeParse.cs index 511a440588..13e532f86e 100644 --- a/src/System.Private.CoreLib/shared/System/Globalization/DateTimeParse.cs +++ b/src/System.Private.CoreLib/shared/System/Globalization/DateTimeParse.cs @@ -3305,7 +3305,7 @@ new DS[] { DS.ERROR, DS.TX_NNN, DS.TX_NNN, DS.TX_NNN, DS.ERROR, DS.ERROR, /*=================================MatchAbbreviatedMonthName================================== **Action: Parse the abbreviated month name from string starting at str.Index. - **Returns: A value from 1 to 12 for the first month to the twelveth month. + **Returns: A value from 1 to 12 for the first month to the twelfth month. **Arguments: str: a __DTString. The parsing will start from the ** next character after str.Index. **Exceptions: FormatException if an abbreviated month name can not be found. @@ -3340,6 +3340,19 @@ new DS[] { DS.ERROR, DS.TX_NNN, DS.TX_NNN, DS.TX_NNN, DS.ERROR, DS.ERROR, } } + // Search genitive form. + if ((dtfi.FormatFlags & DateTimeFormatFlags.UseGenitiveMonth) != 0) + { + int tempResult = str.MatchLongestWords(dtfi.AbbreviatedMonthGenitiveNames, ref maxMatchStrLen); + + // We found a longer match in the genitive month name. Use this as the result. + // tempResult + 1 should be the month value. + if (tempResult >= 0) + { + result = tempResult + 1; + } + } + // Search leap year form. if ((dtfi.FormatFlags & DateTimeFormatFlags.UseLeapYearMonth) != 0) { @@ -3363,7 +3376,7 @@ new DS[] { DS.ERROR, DS.TX_NNN, DS.TX_NNN, DS.TX_NNN, DS.ERROR, DS.ERROR, /*=================================MatchMonthName================================== **Action: Parse the month name from string starting at str.Index. - **Returns: A value from 1 to 12 indicating the first month to the twelveth month. + **Returns: A value from 1 to 12 indicating the first month to the twelfth month. **Arguments: str: a __DTString. The parsing will start from the ** next character after str.Index. **Exceptions: FormatException if a month name can not be found. diff --git a/src/System.Private.CoreLib/shared/System/IO/BinaryReader.cs b/src/System.Private.CoreLib/shared/System/IO/BinaryReader.cs index 560c998a44..037088c49b 100644 --- a/src/System.Private.CoreLib/shared/System/IO/BinaryReader.cs +++ b/src/System.Private.CoreLib/shared/System/IO/BinaryReader.cs @@ -390,6 +390,28 @@ namespace System.IO { numBytes <<= 1; } + + // We do not want to read even a single byte more than necessary. + // + // Subtract pending bytes that the decoder may be holding onto. This assumes that each + // decoded char corresponds to one or more bytes. Note that custom encodings or encodings with + // a custom replacement sequence may violate this assumption. + if (numBytes > 1) + { + DecoderNLS? decoder = _decoder as DecoderNLS; + // For internal decoders, we can check whether the decoder has any pending state. + // For custom decoders, assume that the decoder has pending state. + if (decoder == null || decoder.HasState) + { + numBytes -= 1; + + // The worst case is charsRemaining = 2 and UTF32Decoder holding onto 3 pending bytes. We need to read just + // one byte in this case. + if (_2BytesPerChar && numBytes > 2) + numBytes -= 2; + } + } + if (numBytes > MaxCharBytesSize) { numBytes = MaxCharBytesSize; diff --git a/src/System.Private.CoreLib/shared/System/Text/DecoderNLS.cs b/src/System.Private.CoreLib/shared/System/Text/DecoderNLS.cs index 184d59d868..8cb3fffe1b 100644 --- a/src/System.Private.CoreLib/shared/System/Text/DecoderNLS.cs +++ b/src/System.Private.CoreLib/shared/System/Text/DecoderNLS.cs @@ -227,7 +227,7 @@ namespace System.Text { get { - return false; + return _leftoverByteCount != 0; } } diff --git a/src/inc/clrconfigvalues.h b/src/inc/clrconfigvalues.h index a97e53d4f8..c034f4f689 100644 --- a/src/inc/clrconfigvalues.h +++ b/src/inc/clrconfigvalues.h @@ -565,6 +565,7 @@ RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_ProfAPI_ValidateNGENInstrumentation, W("Pro RETAIL_CONFIG_DWORD_INFO_EX(EXTERNAL_PerfMapEnabled, W("PerfMapEnabled"), 0, "This flag is used on Linux to enable writing /tmp/perf-$pid.map. It is disabled by default", CLRConfig::REGUTIL_default) RETAIL_CONFIG_DWORD_INFO_EX(EXTERNAL_PerfMapIgnoreSignal, W("PerfMapIgnoreSignal"), 0, "When perf map is enabled, this option will configure the specified signal to be accepted and ignored as a marker in the perf logs. It is disabled by default", CLRConfig::REGUTIL_default) RETAIL_CONFIG_DWORD_INFO(EXTERNAL_PerfMapShowOptimizationTiers, W("PerfMapShowOptimizationTiers"), 1, "Shows optimization tiers in the perf map for methods, as part of the symbol name. Useful for seeing separate stack frames for different optimization tiers of each method.") +RETAIL_CONFIG_STRING_INFO(EXTERNAL_NativeImagePerfMapFormat, W("NativeImagePerfMapFormat"), "Specifies the format of native image perfmap files generated by crossgen. Valid options are RVA or OFFSET.") #endif RETAIL_CONFIG_STRING_INFO(EXTERNAL_StartupDelayMS, W("StartupDelayMS"), "") diff --git a/src/vm/perfinfo.cpp b/src/vm/perfinfo.cpp index 3c8841b010..7075137010 100644 --- a/src/vm/perfinfo.cpp +++ b/src/vm/perfinfo.cpp @@ -41,7 +41,9 @@ void PerfInfo::LogImage(PEFile* pFile, WCHAR* guid) SString value; const SString& path = pFile->GetPath(); - value.Printf("%S%c%S", path.GetUnicode(), sDelimiter, guid); + PEImageLayout *pLoadedLayout = pFile->GetLoaded(); + SIZE_T baseAddr = (SIZE_T)pLoadedLayout->GetBase(); + value.Printf("%S%c%S%c%p", path.GetUnicode(), sDelimiter, guid, sDelimiter, baseAddr); SString command; command.Printf("%s", "ImageLoad"); diff --git a/src/vm/perfmap.cpp b/src/vm/perfmap.cpp index ae2a5b07db..bc49bfef69 100644 --- a/src/vm/perfmap.cpp +++ b/src/vm/perfmap.cpp @@ -336,6 +336,14 @@ NativeImagePerfMap::NativeImagePerfMap(Assembly * pAssembly, BSTR pDestPath) // Open the perf map file. OpenFile(sDestPerfMapPath); + + // Determine whether to emit RVAs or file offsets based on the specified configuration. + m_EmitRVAs = true; + CLRConfigStringHolder wszFormat(CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_NativeImagePerfMapFormat)); + if(wszFormat != NULL && (wcsncmp(wszFormat, strOFFSET, wcslen(strOFFSET)) == 0)) + { + m_EmitRVAs = false; + } } // Log data to the perfmap for the specified module. @@ -371,7 +379,7 @@ void NativeImagePerfMap::LogDataForModule(Module * pModule) } // Log a pre-compiled method to the perfmap. -void NativeImagePerfMap::LogPreCompiledMethod(MethodDesc * pMethod, PCODE pCode, PEImageLayout * pLoadedLayout, const char *optimizationTier) +void NativeImagePerfMap::LogPreCompiledMethod(MethodDesc * pMethod, PCODE pCode, PEImageLayout *pLoadedLayout, const char *optimizationTier) { STANDARD_VM_CONTRACT; @@ -387,14 +395,25 @@ void NativeImagePerfMap::LogPreCompiledMethod(MethodDesc * pMethod, PCODE pCode, // NGEN can split code between hot and cold sections which are separate in memory. // Emit an entry for each section if it is used. + PCODE addr; if (methodRegionInfo.hotSize > 0) { - LogMethod(pMethod, pLoadedLayout->RvaToOffset((PCODE)methodRegionInfo.hotStartAddress - baseAddr), methodRegionInfo.hotSize, optimizationTier); + addr = (PCODE)methodRegionInfo.hotStartAddress - baseAddr; + if (!m_EmitRVAs) + { + addr = pLoadedLayout->RvaToOffset(addr); + } + LogMethod(pMethod, addr, methodRegionInfo.hotSize, optimizationTier); } if (methodRegionInfo.coldSize > 0) { - LogMethod(pMethod, pLoadedLayout->RvaToOffset((PCODE)methodRegionInfo.coldStartAddress - baseAddr), methodRegionInfo.coldSize, optimizationTier); + addr = (PCODE)methodRegionInfo.coldStartAddress - baseAddr; + if (!m_EmitRVAs) + { + addr = pLoadedLayout->RvaToOffset(addr); + } + LogMethod(pMethod, addr, methodRegionInfo.coldSize, optimizationTier); } } diff --git a/src/vm/perfmap.h b/src/vm/perfmap.h index 22d8452347..5788dcc60a 100644 --- a/src/vm/perfmap.h +++ b/src/vm/perfmap.h @@ -81,8 +81,13 @@ public: class NativeImagePerfMap : PerfMap { private: + const WCHAR *strOFFSET = W("OFFSET"); + + // Specify the address format since it's now possible for 'perf script' to output file offsets or RVAs. + bool m_EmitRVAs; + // Log a pre-compiled method to the map. - void LogPreCompiledMethod(MethodDesc * pMethod, PCODE pCode, PEImageLayout * pLoadedLayout, const char *optimizationTier); + void LogPreCompiledMethod(MethodDesc * pMethod, PCODE pCode, PEImageLayout *pLoadedLayout, const char *optimizationTier); public: // Construct a new map for a native image. |