diff options
Diffstat (limited to 'eng/common')
27 files changed, 412 insertions, 146 deletions
diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj index e17f72644e..9120b2d212 100644 --- a/eng/common/PublishToPackageFeed.proj +++ b/eng/common/PublishToPackageFeed.proj @@ -53,6 +53,7 @@ <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'TOOLSET'">https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json</TargetStaticFeed> <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'WINDOWSDESKTOP'">https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json</TargetStaticFeed> <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'NUGETCLIENT'">https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json</TargetStaticFeed> + <TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETENTITYFRAMEWORK6'">https://dotnetfeed.blob.core.windows.net/aspnet-entityframework6/index.json</TargetStaticFeed> </PropertyGroup> <Error diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index d0147db4bd..67046a43f8 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -1,6 +1,7 @@ [CmdletBinding(PositionalBinding=$false)] Param( [string][Alias('c')]$configuration = "Debug", + [string]$platform = $null, [string] $projects, [string][Alias('v')]$verbosity = "minimal", [string] $msbuildEngine = $null, @@ -29,6 +30,7 @@ Param( function Print-Usage() { Write-Host "Common settings:" Write-Host " -configuration <value> Build configuration: 'Debug' or 'Release' (short: -c)" + Write-Host " -platform <value> Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild" Write-Host " -verbosity <value> Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" Write-Host " -binaryLog Output binary log (short: -bl)" Write-Host " -help Print help and exit" @@ -77,6 +79,7 @@ function Build { InitializeCustomToolset $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } + $platformArg = if ($platform) { "/p:Platform=$platform" } else { "" } if ($projects) { # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. @@ -88,6 +91,7 @@ function Build { MSBuild $toolsetBuildProj ` $bl ` + $platformArg ` /p:Configuration=$configuration ` /p:RepoRoot=$RepoRoot ` /p:Restore=$restore ` @@ -122,6 +126,10 @@ try { . $configureToolsetScript } + if (($restore) -and ($null -eq $env:DisableNativeToolsetInstalls)) { + InitializeNativeTools + } + Build } catch { diff --git a/eng/common/build.sh b/eng/common/build.sh index 40b1e8ec73..09eb4bf68f 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -35,7 +35,7 @@ usage() echo " --nodeReuse <value> Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError <value> Sets warnaserror msbuild parameter ('true' or 'false')" echo "" - echo "Command line arguments starting with '/p:' are passed through to MSBuild." + echo "Command line arguments not listed above are passed thru to msbuild." echo "Arguments can also be passed in with a single hyphen." } @@ -66,6 +66,7 @@ ci=false warn_as_error=true node_reuse=true binary_log=false +pipelines_log=false projects='' configuration='Debug' @@ -92,6 +93,9 @@ while [[ $# > 0 ]]; do -binarylog|-bl) binary_log=true ;; + -pipelineslog|-pl) + pipelines_log=true + ;; -restore|-r) restore=true ;; @@ -137,22 +141,8 @@ while [[ $# > 0 ]]; do node_reuse=$2 shift ;; - -p:*|/p:*) - properties="$properties $1" - ;; - -m:*|/m:*) - properties="$properties $1" - ;; - -bl:*|/bl:*) - properties="$properties $1" - ;; - -dl:*|/dl:*) - properties="$properties $1" - ;; *) - echo "Invalid argument: $1" - usage - exit 1 + properties="$properties $1" ;; esac @@ -160,6 +150,7 @@ while [[ $# > 0 ]]; do done if [[ "$ci" == true ]]; then + pipelines_log=true binary_log=true node_reuse=false fi @@ -218,4 +209,9 @@ if [[ -n "${useInstalledDotNetCli:-}" ]]; then use_installed_dotnet_cli="$useInstalledDotNetCli" fi +# Workaround for https://github.com/dotnet/arcade/issues/2673 +# if [[ "$restore" == true && -z ${DisableNativeToolsetInstalls:-} ]]; then +# InitializeNativeTools +# fi + Build diff --git a/eng/common/cross/arm/sources.list.vivid b/eng/common/cross/arm/sources.list.vivid deleted file mode 100644 index 0b1215e475..0000000000 --- a/eng/common/cross/arm/sources.list.vivid +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ vivid main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ vivid-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ vivid-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ vivid-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-security main restricted universe multiverse
\ No newline at end of file diff --git a/eng/common/cross/arm/sources.list.wily b/eng/common/cross/arm/sources.list.wily deleted file mode 100644 index e23d1e02a0..0000000000 --- a/eng/common/cross/arm/sources.list.wily +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ wily main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted universe multiverse
\ No newline at end of file diff --git a/eng/common/cross/arm64/sources.list.buster b/eng/common/cross/arm64/sources.list.buster new file mode 100644 index 0000000000..7194ac64a9 --- /dev/null +++ b/eng/common/cross/arm64/sources.list.buster @@ -0,0 +1,11 @@ +deb http://deb.debian.org/debian buster main +deb-src http://deb.debian.org/debian buster main + +deb http://deb.debian.org/debian-security/ buster/updates main +deb-src http://deb.debian.org/debian-security/ buster/updates main + +deb http://deb.debian.org/debian buster-updates main +deb-src http://deb.debian.org/debian buster-updates main + +deb http://deb.debian.org/debian buster-backports main contrib non-free +deb-src http://deb.debian.org/debian buster-backports main contrib non-free diff --git a/eng/common/cross/arm64/sources.list.stretch b/eng/common/cross/arm64/sources.list.stretch new file mode 100644 index 0000000000..0e12157743 --- /dev/null +++ b/eng/common/cross/arm64/sources.list.stretch @@ -0,0 +1,12 @@ +deb http://deb.debian.org/debian stretch main +deb-src http://deb.debian.org/debian stretch main + +deb http://deb.debian.org/debian-security/ stretch/updates main +deb-src http://deb.debian.org/debian-security/ stretch/updates main + +deb http://deb.debian.org/debian stretch-updates main +deb-src http://deb.debian.org/debian stretch-updates main + +deb http://deb.debian.org/debian stretch-backports main contrib non-free +deb-src http://deb.debian.org/debian stretch-backports main contrib non-free + diff --git a/eng/common/cross/arm64/sources.list.vivid b/eng/common/cross/arm64/sources.list.vivid deleted file mode 100644 index 0b1215e475..0000000000 --- a/eng/common/cross/arm64/sources.list.vivid +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ vivid main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ vivid-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ vivid-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ vivid-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ vivid-security main restricted universe multiverse
\ No newline at end of file diff --git a/eng/common/cross/arm64/sources.list.wily b/eng/common/cross/arm64/sources.list.wily deleted file mode 100644 index e23d1e02a0..0000000000 --- a/eng/common/cross/arm64/sources.list.wily +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ wily main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted universe multiverse
\ No newline at end of file diff --git a/eng/common/cross/armel/tizen-build-rootfs.sh b/eng/common/cross/armel/tizen-build-rootfs.sh index 87c48e78fb..87c48e78fb 100644..100755 --- a/eng/common/cross/armel/tizen-build-rootfs.sh +++ b/eng/common/cross/armel/tizen-build-rootfs.sh diff --git a/eng/common/cross/armel/tizen-fetch.sh b/eng/common/cross/armel/tizen-fetch.sh index ba16e991c7..ba16e991c7 100644..100755 --- a/eng/common/cross/armel/tizen-fetch.sh +++ b/eng/common/cross/armel/tizen-fetch.sh diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index adceda877a..adceda877a 100644..100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 805948ca83..34d3d2ba1f 100644..100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -2,21 +2,21 @@ usage() { - echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount]" + echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]" echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "LinuxCodeName - optional, Code name for Linux, can be: trusty(default), vivid, wily, xenial, zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." - echo "lldbx.y - optional, LLDB version, can be: lldb3.6(default), lldb3.8, lldb3.9, lldb4.0, no-lldb. Ignored for alpine" + echo "LinuxCodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine" echo "--skipunmount - optional, will skip the unmount of rootfs folder." exit 1 } -__LinuxCodeName=trusty +__LinuxCodeName=xenial __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) __InitialDir=$PWD __BuildArch=arm __UbuntuArch=armhf __UbuntuRepo="http://ports.ubuntu.com/" -__LLDB_Package="lldb-3.6-dev" +__LLDB_Package="liblldb-3.9-dev" __SkipUnmount=0 # base development support @@ -53,8 +53,12 @@ __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" __UnprocessedBuildArgs= -for i in "$@" ; do - lowerI="$(echo $i | awk '{print tolower($0)}')" +while :; do + if [ $# -le 0 ]; then + break + fi + + lowerI="$(echo $1 | awk '{print tolower($0)}')" case $lowerI in -?|-h|--help) usage @@ -95,38 +99,49 @@ for i in "$@" ; do lldb4.0) __LLDB_Package="liblldb-4.0-dev" ;; + lldb5.0) + __LLDB_Package="liblldb-5.0-dev" + ;; + lldb6.0) + __LLDB_Package="liblldb-6.0-dev" + ;; no-lldb) unset __LLDB_Package ;; - vivid) + trusty) # Ubuntu 14.04 if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=vivid + __LinuxCodeName=trusty fi ;; - wily) - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=wily - fi - ;; - xenial) + xenial) # Ubuntu 16.04 if [ "$__LinuxCodeName" != "jessie" ]; then __LinuxCodeName=xenial fi ;; - zesty) + zesty) # Ubuntu 17.04 if [ "$__LinuxCodeName" != "jessie" ]; then __LinuxCodeName=zesty fi ;; - bionic) + bionic) # Ubuntu 18.04 if [ "$__LinuxCodeName" != "jessie" ]; then __LinuxCodeName=bionic fi ;; - jessie) + jessie) # Debian 8 __LinuxCodeName=jessie __UbuntuRepo="http://ftp.debian.org/debian/" ;; + stretch) # Debian 9 + __LinuxCodeName=stretch + __UbuntuRepo="http://ftp.debian.org/debian/" + __LLDB_Package="liblldb-6.0-dev" + ;; + buster) # Debian 10 + __LinuxCodeName=buster + __UbuntuRepo="http://ftp.debian.org/debian/" + __LLDB_Package="liblldb-6.0-dev" + ;; tizen) if [ "$__BuildArch" != "armel" ]; then echo "Tizen is available only for armel." @@ -144,10 +159,16 @@ for i in "$@" ; do --skipunmount) __SkipUnmount=1 ;; + --rootfsdir|-rootfsdir) + shift + __RootfsDir=$1 + ;; *) - __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" + __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" ;; esac + + shift done if [ "$__BuildArch" == "armel" ]; then @@ -155,12 +176,14 @@ if [ "$__BuildArch" == "armel" ]; then fi __UbuntuPackages+=" ${__LLDB_Package:-}" -__RootfsDir="$__CrossDir/rootfs/$__BuildArch" - -if [[ -n "$ROOTFS_DIR" ]]; then +if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then __RootfsDir=$ROOTFS_DIR fi +if [ -z "$__RootfsDir" ]; then + __RootfsDir="$__CrossDir/rootfs/$__BuildArch" +fi + if [ -d "$__RootfsDir" ]; then if [ $__SkipUnmount == 0 ]; then umount $__RootfsDir/* diff --git a/eng/common/cross/x86/sources.list.vivid b/eng/common/cross/x86/sources.list.vivid deleted file mode 100644 index 26d37b20fc..0000000000 --- a/eng/common/cross/x86/sources.list.vivid +++ /dev/null @@ -1,11 +0,0 @@ -deb http://archive.ubuntu.com/ubuntu/ vivid main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ vivid main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ vivid-updates main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ vivid-updates main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ vivid-backports main restricted -deb-src http://archive.ubuntu.com/ubuntu/ vivid-backports main restricted - -deb http://archive.ubuntu.com/ubuntu/ vivid-security main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu/ vivid-security main restricted universe multiverse diff --git a/eng/common/cross/x86/sources.list.wily b/eng/common/cross/x86/sources.list.wily deleted file mode 100644 index c4b0b442ab..0000000000 --- a/eng/common/cross/x86/sources.list.wily +++ /dev/null @@ -1,11 +0,0 @@ -deb http://archive.ubuntu.com/ubuntu/ wily main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ wily main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ wily-updates main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ wily-updates main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ wily-backports main restricted -deb-src http://archive.ubuntu.com/ubuntu/ wily-backports main restricted - -deb http://archive.ubuntu.com/ubuntu/ wily-security main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu/ wily-security main restricted universe multiverse diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 81ffd16779..dea7cdd903 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -1,5 +1,6 @@ param ( - $darcVersion = $null + $darcVersion = $null, + $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" ) $verbosity = "m" @@ -16,13 +17,13 @@ function InstallDarcCli ($darcVersion) { Invoke-Expression "& `"$dotnet`" tool uninstall $darcCliPackageName -g" } - # Until we can anonymously query the BAR API for the latest arcade-services - # build applied to the PROD channel, this is hardcoded. + # If the user didn't explicitly specify the darc version, + # query the Maestro API for the correct version of darc to install. if (-not $darcVersion) { - $darcVersion = '1.1.0-beta.19205.4' + $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content } - $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json' + $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json' Write-Host "Installing Darc CLI version $darcVersion..." Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index bd7eb46398..abdd0bc05a 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash source="${BASH_SOURCE[0]}" -darcVersion="1.1.0-beta.19205.4" +darcVersion='' +versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" while [[ $# > 0 ]]; do opt="$(echo "$1" | awk '{print tolower($0)}')" @@ -10,6 +11,10 @@ while [[ $# > 0 ]]; do darcVersion=$2 shift ;; + --versionendpoint) + versionEndpoint=$2 + shift + ;; *) echo "Invalid argument: $1" usage @@ -33,6 +38,10 @@ verbosity=m . "$scriptroot/tools.sh" +if [ -z "$darcVersion" ]; then + darcVersion=$(curl -X GET "$versionEndpoint" -H "accept: text/plain") +fi + function InstallDarcCli { local darc_cli_package_name="microsoft.dotnet.darc" @@ -45,9 +54,9 @@ function InstallDarcCli { echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g) fi - local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json" + local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" - echo "Installing Darc CLI version $toolset_version..." + echo "Installing Darc CLI version $darcVersion..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) } diff --git a/eng/common/dotnet-install.cmd b/eng/common/dotnet-install.cmd new file mode 100644 index 0000000000..b1c2642e76 --- /dev/null +++ b/eng/common/dotnet-install.cmd @@ -0,0 +1,2 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*"
\ No newline at end of file diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 new file mode 100644 index 0000000000..0b629b8301 --- /dev/null +++ b/eng/common/dotnet-install.ps1 @@ -0,0 +1,27 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $verbosity = "minimal", + [string] $architecture = "", + [string] $version = "Latest", + [string] $runtime = "dotnet" +) + +. $PSScriptRoot\tools.ps1 + +$dotnetRoot = Join-Path $RepoRoot ".dotnet" + +$installdir = $dotnetRoot +try { + if ($architecture -and $architecture.Trim() -eq "x86") { + $installdir = Join-Path $installdir "x86" + } + InstallDotNet $installdir $version $architecture $runtime $true +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} + +ExitWithExitCode 0 diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh new file mode 100644 index 0000000000..c3072c958a --- /dev/null +++ b/eng/common/dotnet-install.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +version='Latest' +architecture='' +runtime='dotnet' +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + -version|-v) + shift + version="$1" + ;; + -architecture|-a) + shift + architecture="$1" + ;; + -runtime|-r) + shift + runtime="$1" + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + shift +done + +. "$scriptroot/tools.sh" +dotnetRoot="$repo_root/.dotnet" +InstallDotNet $dotnetRoot $version "$architecture" $runtime true || { + local exit_code=$? + echo "dotnet-install.sh failed (exit code '$exit_code')." >&2 + ExitWithExitCode $exit_code +} + +ExitWithExitCode 0 diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index 495a563a75..a4306bd37e 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -41,9 +41,13 @@ Param ( [switch] $Force = $False, [int] $DownloadRetries = 5, [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile = "$PSScriptRoot\..\..\global.json" + [string] $GlobalJsonFile ) +if (!$GlobalJsonFile) { + $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName "global.json" +} + Set-StrictMode -version 2.0 $ErrorActionPreference="Stop" diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index 54b70f678b..fc72d13948 100644..100755 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -9,7 +9,7 @@ clean=false force=false download_retries=5 retry_wait_time_seconds=30 -global_json_file="${scriptroot}/../../global.json" +global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" declare -A native_assets . $scriptroot/native/common-library.sh @@ -71,6 +71,7 @@ function ReadGlobalJsonNativeTools { local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}') native_tools_list=${native_tools_list//[\" ]/} native_tools_list=${native_tools_list//,/$'\n'} + native_tools_list="$(echo -e "${native_tools_list}" | tr -d '[:space:]')" local old_IFS=$IFS while read -r line; do @@ -116,8 +117,6 @@ else installer_command+=" --clean" fi - echo "Installing $tool version $tool_version" - echo "Executing '$installer_command'" $installer_command if [[ $? != 0 ]]; then @@ -127,19 +126,16 @@ else done fi -if [[ ! -z $clean ]]; then +if [[ $clean = true ]]; then exit 0 fi if [[ -d $install_bin ]]; then echo "Native tools are available from $install_bin" - if [[ !-z BUILD_BUILDNUMBER ]]; then - echo "##vso[task.prependpath]$install_bin" - fi + echo "##vso[task.prependpath]$install_bin" else echo "Native tools install directory does not exist, installation failed" >&2 exit 1 fi exit 0 - diff --git a/eng/common/native/common-library.sh b/eng/common/native/common-library.sh index 271bddfac5..271bddfac5 100644..100755 --- a/eng/common/native/common-library.sh +++ b/eng/common/native/common-library.sh diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 7839b70bb7..1814e0ab61 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -90,6 +90,9 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} variables: + - ${{ if eq(parameters.enableTelemetry, 'true') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' - ${{ each variable in parameters.variables }}: # handle name-value variable syntax # example: diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index d1ce577db5..05df886f55 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -5,6 +5,7 @@ parameters: HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects @@ -35,6 +36,7 @@ steps: HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} HelixTargetQueues: ${{ parameters.HelixTargetQueues }} HelixAccessToken: ${{ parameters.HelixAccessToken }} HelixPreCommands: ${{ parameters.HelixPreCommands }} @@ -64,6 +66,7 @@ steps: HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} HelixTargetQueues: ${{ parameters.HelixTargetQueues }} HelixAccessToken: ${{ parameters.HelixAccessToken }} HelixPreCommands: ${{ parameters.HelixPreCommands }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 5c4a129c86..9cea610a27 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -11,6 +11,12 @@ # Binary log must be enabled on CI. [bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } +# Set to true to use the pipelines logger which will enable Azure logging output. +# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md +# This flag is meant as a temporary opt-opt for the feature while validate it across +# our consumers. It will be deleted in the future. +[bool]$pipelinesLog = if (Test-Path variable:pipelinesLog) { $pipelinesLog } else { $ci } + # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). [bool]$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false } @@ -108,7 +114,7 @@ function InitializeDotNetCli([bool]$install) { } # Find the first path on %PATH% that contains the dotnet.exe - if ($useInstalledDotNetCli -and ($env:DOTNET_INSTALL_DIR -eq $null)) { + if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { $dotnetCmd = Get-Command "dotnet.exe" -ErrorAction SilentlyContinue if ($dotnetCmd -ne $null) { $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent @@ -119,7 +125,7 @@ function InitializeDotNetCli([bool]$install) { # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if (($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { + if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { $dotnetRoot = Join-Path $RepoRoot ".dotnet" @@ -152,7 +158,7 @@ function InitializeDotNetCli([bool]$install) { } function GetDotNetInstallScript([string] $dotnetRoot) { - $installScript = "$dotnetRoot\dotnet-install.ps1" + $installScript = Join-Path $dotnetRoot "dotnet-install.ps1" if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot Invoke-WebRequest "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript @@ -162,9 +168,21 @@ function GetDotNetInstallScript([string] $dotnetRoot) { } function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "") { + InstallDotNet $dotnetRoot $version $architecture +} + +function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtime = "", [bool] $skipNonVersionedFiles = $false) { $installScript = GetDotNetInstallScript $dotnetRoot $installScript = GetDotNetInstallScript $dotnetRoot - $archArg = if ($architecture) { $architecture } else { "<auto>" } - & $installScript -Version $version -InstallDir $dotnetRoot -Architecture $archArg + $installParameters = @{ + Version = $version + InstallDir = $dotnetRoot + } + + if ($architecture) { $installParameters.Architecture = $architecture } + if ($runtime) { $installParameters.Runtime = $runtime } + if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } + + & $installScript @installParameters if ($lastExitCode -ne 0) { Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red ExitWithExitCode $lastExitCode @@ -195,7 +213,11 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = if ($env:VSINSTALLDIR -ne $null) { $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue if ($msbuildCmd -ne $null) { - if ($msbuildCmd.Version -ge $vsMinVersion) { + # Workaround for https://github.com/dotnet/roslyn/issues/35793 + # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+ + $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split(@('-', '+'))[0]) + + if ($msbuildVersion -ge $vsMinVersion) { return $global:_MSBuildExe = $msbuildCmd.Path } @@ -340,7 +362,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } - $buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild" } + $buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild"; Tool = "dotnet"; Framework = "netcoreapp2.1" } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore @@ -349,7 +371,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } - $buildTool = @{ Path = $msbuildPath; Command = "" } + $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } } else { Write-Host "Unexpected value of -msbuildEngine: '$msbuildEngine'." -ForegroundColor Red ExitWithExitCode 1 @@ -391,6 +413,16 @@ function GetSdkTaskProject([string]$taskName) { return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj" } +function InitializeNativeTools() { + if (Get-Member -InputObject $GlobalJson -Name "native-tools") { + $nativeArgs="" + if ($ci) { + $nativeArgs = "-InstallDirectory $ToolsDir" + } + Invoke-Expression "& `"$PSScriptRoot/init-tools-native.ps1`" $nativeArgs" + } +} + function InitializeToolset() { if (Test-Path variable:global:_ToolsetBuildProj) { return $global:_ToolsetBuildProj @@ -419,7 +451,8 @@ function InitializeToolset() { $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" } '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Set-Content $proj - MSBuild $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile + + MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile $path = Get-Content $toolsetLocationFile -TotalCount 1 if (!(Test-Path $path)) { @@ -449,6 +482,23 @@ function Stop-Processes() { # Terminates the script if the build fails. # function MSBuild() { + if ($pipelinesLog) { + $buildTool = InitializeBuildTool + $toolsetBuildProject = InitializeToolset + $path = Split-Path -parent $toolsetBuildProject + $path = Join-Path $path (Join-Path $buildTool.Framework "Microsoft.DotNet.Arcade.Sdk.dll") + $args += "/logger:$path" + } + + MSBuild-Core @args +} + +# +# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. +# The arguments are automatically quoted. +# Terminates the script if the build fails. +# +function MSBuild-Core() { if ($ci) { if (!$binaryLog) { throw "Binary log must be enabled in CI build." @@ -512,6 +562,8 @@ $ToolsDir = Join-Path $RepoRoot ".tools" $LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration $TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json +# true if global.json contains a "runtimes" section +$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false } Create-Directory $ToolsetDir Create-Directory $TempDir diff --git a/eng/common/tools.sh b/eng/common/tools.sh index ecdece1f85..fd26f6fbb2 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -1,8 +1,20 @@ +#!/usr/bin/env bash + # Initialize variables if they aren't already defined. # CI mode - set to true on CI server for PR validation build or official build. ci=${ci:-false} +# Set to true to use the pipelines logger which will enable Azure logging output. +# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md +# This flag is meant as a temporary opt-opt for the feature while validate it across +# our consumers. It will be deleted in the future. +if [[ "$ci" == true ]]; then + pipelines_log=${pipelines_log:-true} +else + pipelines_log=${pipelines_log:-false} +fi + # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. configuration=${configuration:-'Debug'} @@ -40,6 +52,78 @@ else use_global_nuget_cache=${use_global_nuget_cache:-true} fi +function EmitError { + if [[ "$ci" != true ]]; then + echo "$@" >&2 + return + fi + + message_type="error" + sourcepath='' + linenumber='' + columnnumber='' + error_code='' + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -type|-t) + message_type=$2 + shift + ;; + -sourcepath|-s) + sourcepath=$2 + shift + ;; + -linenumber|-l) + linenumber=$2 + shift + ;; + -columnnumber|-col) + columnnumber=$2 + shift + ;; + -code|-c) + error_code=$2 + shift + ;; + *) + break + ;; + esac + + shift + done + + message='##vso[task.logissue' + + message="$message type=$message_type" + + if [ -n "$sourcepath" ]; then + message="$message;sourcepath=$sourcepath" + else + message="$message;sourcepath=${BASH_SOURCE[1]}" + fi + + if [ -n "$linenumber" ]; then + message="$message;linenumber=$linenumber" + else + message="$message;linenumber=${BASH_LINENO[0]}" + fi + + if [ -n "$columnnumber" ]; then + message="$message;columnnumber=$columnnumber" + fi + + if [ -n "$error_code" ]; then + message="$message;code=$error_code" + fi + + message="$message]$*" + + echo "$message" +} + # Resolve any symlinks in the given path. function ResolvePath { local path=$1 @@ -65,7 +149,7 @@ function ReadGlobalVersion { local pattern="\"$key\" *: *\"(.*)\"" if [[ ! $line =~ $pattern ]]; then - echo "Error: Cannot find \"$key\" in $global_json_file" >&2 + EmitError "Error: Cannot find \"$key\" in $global_json_file" ExitWithExitCode 1 fi @@ -101,7 +185,7 @@ function InitializeDotNetCli { fi # Find the first path on $PATH that contains the dotnet.exe - if [[ "$use_installed_dotnet_cli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then + if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then local dotnet_path=`command -v dotnet` if [[ -n "$dotnet_path" ]]; then ResolvePath "$dotnet_path" @@ -115,17 +199,18 @@ function InitializeDotNetCli { # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if [[ -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then + if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else dotnet_root="$repo_root/.dotnet" + export DOTNET_INSTALL_DIR="$dotnet_root" if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then if [[ "$install" == true ]]; then InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version" else - echo "Unable to find dotnet with SDK version '$dotnet_sdk_version'" >&2 + EmitError "Unable to find dotnet with SDK version '$dotnet_sdk_version'" ExitWithExitCode 1 fi fi @@ -149,18 +234,36 @@ function InitializeDotNetCli { function InstallDotNetSdk { local root=$1 local version=$2 + local architecture="" + if [[ $# == 3 ]]; then + architecture=$3 + fi + InstallDotNet "$root" "$version" $architecture +} +function InstallDotNet { + local root=$1 + local version=$2 + GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript - local arch_arg="" - if [[ $# == 3 ]]; then - arch_arg="--architecture $3" + local archArg='' + if [[ -n "${3:-}" ]]; then + archArg="--architecture $3" + fi + local runtimeArg='' + if [[ -n "${4:-}" ]]; then + runtimeArg="--runtime $4" fi - bash "$install_script" --version $version --install-dir "$root" $arch_arg || { + local skipNonVersionedFilesArg="" + if [[ "$#" -ge "5" ]]; then + skipNonVersionedFilesArg="--skip-non-versioned-files" + fi + bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { local exit_code=$? - echo "Failed to install dotnet SDK (exit code '$exit_code')." >&2 + EmitError "Failed to install dotnet SDK (exit code '$exit_code')." ExitWithExitCode $exit_code } } @@ -197,6 +300,7 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" + _InitializeBuildToolFramework="netcoreapp2.1" } function GetNuGetPackageCachePath { @@ -212,6 +316,17 @@ function GetNuGetPackageCachePath { _GetNuGetPackageCachePath=$NUGET_PACKAGES } +function InitializeNativeTools() { + if grep -Fq "native-tools" $global_json_file + then + local nativeArgs="" + if [[ "$ci" == true ]]; then + nativeArgs="-InstallDirectory $tools_dir" + fi + "$_script_dir/init-tools-native.sh" $nativeArgs + fi +} + function InitializeToolset { if [[ -n "${_InitializeToolset:-}" ]]; then return @@ -234,7 +349,7 @@ function InitializeToolset { fi if [[ "$restore" != true ]]; then - echo "Toolset version $toolsetVersion has not been restored." >&2 + EmitError "Toolset version $toolsetVersion has not been restored." ExitWithExitCode 2 fi @@ -246,12 +361,12 @@ function InitializeToolset { fi echo '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' > "$proj" - MSBuild "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" + MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" local toolset_build_proj=`cat "$toolset_location_file"` if [[ ! -a "$toolset_build_proj" ]]; then - echo "Invalid toolset path: $toolset_build_proj" >&2 + EmitError "Invalid toolset path: $toolset_build_proj" ExitWithExitCode 3 fi @@ -274,14 +389,27 @@ function StopProcesses { } function MSBuild { + local args=$@ + if [[ "$pipelines_log" == true ]]; then + InitializeBuildTool + InitializeToolset + local toolset_dir="${_InitializeToolset%/*}" + local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" + args=( "${args[@]}" "-logger:$logger_path" ) + fi + + MSBuild-Core ${args[@]} +} + +function MSBuild-Core { if [[ "$ci" == true ]]; then if [[ "$binary_log" != true ]]; then - echo "Binary log must be enabled in CI build." >&2 + EmitError "Binary log must be enabled in CI build." ExitWithExitCode 1 fi if [[ "$node_reuse" == true ]]; then - echo "Node reuse must be disabled in CI build." >&2 + EmitError "Node reuse must be disabled in CI build." ExitWithExitCode 1 fi fi @@ -295,7 +423,7 @@ function MSBuild { "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { local exit_code=$? - echo "Build failed (exit code '$exit_code')." >&2 + EmitError "Build failed (exit code '$exit_code')." ExitWithExitCode $exit_code } } @@ -307,10 +435,17 @@ eng_root=`cd -P "$_script_dir/.." && pwd` repo_root=`cd -P "$_script_dir/../.." && pwd` artifacts_dir="$repo_root/artifacts" toolset_dir="$artifacts_dir/toolset" +tools_dir="$repo_root/.tools" log_dir="$artifacts_dir/log/$configuration" temp_dir="$artifacts_dir/tmp/$configuration" global_json_file="$repo_root/global.json" +# determine if global.json contains a "runtimes" entry +global_json_has_runtimes=false +dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true +if [[ -n "$dotnetlocal_key" ]]; then + global_json_has_runtimes=true +fi # HOME may not be defined in some scenarios, but it is required by NuGet if [[ -z $HOME ]]; then @@ -325,4 +460,4 @@ mkdir -p "$log_dir" if [[ $ci == true ]]; then export TEMP="$temp_dir" export TMP="$temp_dir" -fi +fi
\ No newline at end of file |