From 1b43bd8139d053e24ec51c193054555e3352371a Mon Sep 17 00:00:00 2001 From: Sven Boemer Date: Wed, 30 Jan 2019 16:24:33 -0800 Subject: Remove run.exe and config.json (#21608) This moves us one step away from buildtools and towards arcade. This replaces run.sh and run.cmd (and all invocations) with "dotnet.sh msbuild" and "msbuild.cmd". - I'm using these wrapper scripts for now instead of those in eng/common in order to retain the very helpful "Running " output in the build logs. - I'm using msbuild.cmd instead of dotnet.sh to match the current behavior that uses desktop msbuild on windows, instead of dotnet. All of the arguments that used to be implicitly generated by run.exe and config.json are now explicit, resulting in longer (but easier to copy+paste) commands. Some of these arguments are likely unnecessary, but in this change my goal is just to match the run.exe behavior. Later, I would like to go through and clean up parameters that don't need to be passed in every invocation. I might also consider moving more of the common arguments out into variables in a later change. Some of the wrapper scripts now have limited support for parsing "-Argument=Value" style parameters, to support our existing buildpipeline infrastructure, since I thought this was easier to test than changing our buildpipeline definitions. We can remove that parsing logic once we stop using buildpipeline (which has happened at this point). Some subtle parts of the change: * Add msbuild.cmd This simulates the behavior of Tools\msbuild.cmd, which calls desktop msbuild. * Fix BuildOS processing in package build and publish Previously, config.json had its own processing that would set OSName. Instead, we now pass it in explicitly where it's needed (building packages), or not at all (publishing them). * Handle "=" in publish-packages.cmd and other scripts This seems necessary to properly handle the azure access token * Set __BuildOS in PublishPackages Required for cases where the build OS isn't detected during the build such as freebsd. * Use dotnet msbuild in runtest.py This prevents us from having to deal with different quote escape behavior on windows and linux. Previously, arguments like fileloggerparameters and the logger were given quotes to escape semicolons in the argument. On unix, this prevented the argument from being split up by bash. On windows, it seems that the run.cmd/run.exe tools would prevent the extra quotes from being passed to msbuild.exe (desktop msbuild would choke on the quotes if they were passed along). Unlike desktop msbuild, dotnet msbuild is able to parse the quoted strings, so we simply psas the quoted arguments directly to it on windows. We may be able to do the same on unix. * Fix build-test.sh problem with BuildOS When copying native files during the unix test build, we rely on __BuildOS being set. Fixing the import order and always setting __BuildOS fixes this. We should eventually fix the inconsistent use of BuildOS vs __BuildOS. --- UpdatePublishedVersions.ps1 | 8 +- build-packages.cmd | 34 +- build-packages.sh | 41 ++- build-test.cmd | 77 +++-- build-test.sh | 60 ++-- build.cmd | 69 +++- build.sh | 79 ++++- clean.cmd | 7 +- clean.sh | 11 +- config.json | 810 -------------------------------------------- dir.common.props | 2 + dotnet.sh | 20 ++ msbuild.cmd | 30 ++ publish-packages.cmd | 36 +- publish-packages.sh | 87 ++++- run.cmd | 37 -- run.sh | 20 -- sync.cmd | 38 ++- sync.sh | 9 +- tests/dir.props | 4 +- tests/runtest.py | 88 ++--- tests/src/dir.props | 9 +- 22 files changed, 552 insertions(+), 1024 deletions(-) delete mode 100644 config.json create mode 100755 dotnet.sh create mode 100644 msbuild.cmd delete mode 100644 run.cmd delete mode 100755 run.sh diff --git a/UpdatePublishedVersions.ps1 b/UpdatePublishedVersions.ps1 index 4c9bd85966..d5724c930d 100644 --- a/UpdatePublishedVersions.ps1 +++ b/UpdatePublishedVersions.ps1 @@ -16,7 +16,13 @@ param( # A pattern matching all packages in the set that the versions repository should be set to. [Parameter(Mandatory=$true)][string]$nupkgPath) -& "$PSScriptRoot\run.cmd" build -- tests\build.proj /t:UpdatePublishedVersions ` +& "$PSScriptRoot\msbuild.cmd" /nologo /verbosity:minimal /clp:Summary /nodeReuse:false ` + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log ` + /p:RestoreDefaultOptimizationDataPackage=false ` + /p:PortableBuild=true ` + /p:UsePartialNGENOptimization=false ` + /maxcpucount ` + tests\build.proj /t:UpdatePublishedVersions ` /p:GitHubUser="$gitHubUser" ` /p:GitHubEmail="$gitHubEmail" ` /p:GitHubAuthToken="$gitHubAuthToken" ` diff --git a/build-packages.cmd b/build-packages.cmd index f80e39fd94..05af916a13 100644 --- a/build-packages.cmd +++ b/build-packages.cmd @@ -2,12 +2,42 @@ setlocal EnableDelayedExpansion set "__ProjectDir=%~dp0" -set allargs=%* +set "__args=%*" +set processedArgs= +set unprocessedArgs= +set __MSBuildArgs= + +:Arg_Loop +if "%1" == "" goto ArgsDone if /I [%1] == [/?] goto Usage if /I [%1] == [/help] goto Usage -call %__ProjectDir%/run.cmd build-packages -Project=%__ProjectDir%\src\.nuget\packages.builds -FilterToOSGroup=Windows_NT %allargs% +REM CMD eats "=" on the argument list. +REM TODO: remove all -Property=Value type arguments here once we get rid of them in buildpipeline. +if /i "%1" == "-BuildArch" (set processedArgs=!processedArgs! %1=%2&set __MSBuildArgs=!__MSBuildArgs! /p:__BuildArch=%2&shift&shift&goto Arg_Loop) +if /i "%1" == "-BuildType" (set processedArgs=!processedArgs! %1=%2&set __MSBuildArgs=!__MSBuildArgs! /p:__BuildType=%2&shift&shift&goto Arg_Loop) +if /i "%1" == "-OfficialBuildId" (set processedArgs=!processedArgs! %1=%2&set __MSBuildArgs=!__MSBuildArgs! /p:OfficialBuildId=%2&shift&shift&goto Arg_Loop) +if /i "%1" == "--" (set processedArgs=!processedArgs! %1&shift) + +REM handle any unprocessed arguments, assumed to go only after the processed arguments above +if [!processedArgs!]==[] ( + set unprocessedArgs=%__args% +) else ( + set unprocessedArgs=%__args% + for %%t in (!processedArgs!) do ( + REM strip out already-processed arguments from unprocessedArgs + set unprocessedArgs=!unprocessedArgs:*%%t=! + ) +) + +:ArgsDone + +call %__ProjectDir%/msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /p:__BuildOS=Windows_NT /flp:v=detailed;Append;LogFile=build-packages.log^ + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^ + /p:PortableBuild=true %__ProjectDir%\src\.nuget\packages.builds^ + /p:FilterToOSGroup=Windows_NT %__MSBuildArgs% %unprocessedArgs% if NOT [!ERRORLEVEL!]==[0] ( echo ERROR: An error occurred while building packages, see build-packages.log for more details. exit /b 1 diff --git a/build-packages.sh b/build-packages.sh index fce6fe343f..aba1fcc267 100755 --- a/build-packages.sh +++ b/build-packages.sh @@ -87,8 +87,11 @@ case $OSName in ;; esac +buildArgs= unprocessedBuildArgs= +# TODO: get rid of argument processing entirely once we remove the +# uses of -Arg=Value style in buildpipeline. while :; do if [ $# -le 0 ]; then break @@ -96,20 +99,33 @@ while :; do case "$1" in -\?|-h|--help) - usage - exit 1 - ;; + usage + exit 1 + ;; -BuildArch=*) - unprocessedBuildArgs="$unprocessedBuildArgs $1" - __Arch=$(echo $1| cut -d'=' -f 2) - ;; - + __Arch=$(echo $1| cut -d'=' -f 2) + buildArgs="$buildArgs /p:__BuildArch=$__Arch" + ;; + -BuildType=*) + __Type=$(echo $1| cut -d'=' -f 2) + buildArgs="$buildArgs /p:__BuildType=$__Type" + ;; + -OfficialBuildId=*) + __Id=$(echo $1| cut -d'=' -f 2) + buildArgs="$buildArgs /p:OfficialBuildId=$__Id" + ;; + -__DoCrossArchBuild=*) + __CrossBuild=$(echo $1| cut -d'=' -f 2) + buildArgs="$buildArgs /p:__DoCrossArchBuild=$__CrossBuild" + ;; -portablebuild=false) - unprocessedBuildArgs="$unprocessedBuildArgs $1" + buildArgs="$buildArgs /p:PortableBuild=false" __IsPortableBuild=0 ;; + --) + ;; *) - unprocessedBuildArgs="$unprocessedBuildArgs $1" + unprocessedBuildArgs="$unprocessedBuildArgs $1" esac shift done @@ -129,7 +145,12 @@ else export __DistroRid="$__HostDistroRid" fi -$__ProjectRoot/run.sh build-packages -Project=$__ProjectRoot/src/.nuget/packages.builds -DistroRid=$__DistroRid -UseSharedCompilation=false -BuildNugetPackage=false -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log" $unprocessedBuildArgs +$__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \ + /p:__BuildOS=$__BuildOS /flp:v=detailed\;Append\;LogFile=build-packages.log \ + /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \ + /p:PortableBuild=true src/.nuget/packages.builds \ + /p:__DistroRid=$__DistroRid /p:UseSharedCompilation=false /p:BuildNugetPackage=false \ + $buildArgs $unprocessedBuildArgs if [ $? -ne 0 ] then echo "ERROR: An error occurred while building packages; See build-packages.log for more details." diff --git a/build-test.cmd b/build-test.cmd index a12fa097c6..26b2b86a97 100644 --- a/build-test.cmd +++ b/build-test.cmd @@ -44,10 +44,9 @@ REM __UnprocessedBuildArgs are args that we pass to msbuild (e.g. /p:__BuildArch set "__args= %*" set processedArgs= set __UnprocessedBuildArgs= -set __RunArgs= +set __CommonMSBuildArgs= set __BuildAgainstPackagesArg= -set __BuildAgainstPackagesMsbuildArg= set __SkipRestorePackages= set __SkipManaged= set __SkipNative= @@ -62,7 +61,6 @@ set __DoCrossgen= @REM and allow the "-priority=1" syntax. set __Priority=0 set __PriorityArg= -set __PassThroughArg= :Arg_Loop if "%1" == "" goto ArgsDone @@ -87,7 +85,7 @@ if /i "%1" == "checked" (set __BuildType=Checked&set processedArgs if /i "%1" == "skipmanaged" (set __SkipManaged=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skipnative" (set __SkipNative=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "buildtesthostonly" (set __SkipNative=1&set __SkipManaged=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) -if /i "%1" == "buildagainstpackages" (set __ZipTests=1&set __BuildAgainstPackagesArg=-BuildTestsAgainstPackages&set __BuildAgainstPackagesMsbuildArg=/p:BuildTestsAgainstPackages=true&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "buildagainstpackages" (set __ZipTests=1&set __BuildAgainstPackagesArg=/p:BuildTestsAgainstPackages=true&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skiprestorepackages" (set __SkipRestorePackages=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "ziptests" (set __ZipTests=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "crossgen" (set __DoCrossgen=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) @@ -95,7 +93,7 @@ if /i "%1" == "runtimeid" (set __RuntimeId=%2&set processedArgs=!pro if /i "%1" == "targetsNonWindows" (set __TargetsWindows=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "Exclude" (set __Exclude=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%1" == "-priority" (set __Priority=%2&shift&set processedArgs=!processedArgs! %1=%2&shift&goto Arg_Loop) -if /i "%1" == "--" (set __PassThroughArg=%1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "--" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if [!processedArgs!]==[] ( set __UnprocessedBuildArgs=%__args% @@ -110,8 +108,7 @@ if [!processedArgs!]==[] ( @REM Special handling for -priority=N argument. if %__Priority% GTR 0 ( - set "__PriorityArg=-priority=%__Priority%" - set "__PriorityMsbuildArg=/p:CLRTestPriorityToBuild=%__Priority%" + set "__PriorityArg=/p:CLRTestPriorityToBuild=%__Priority%" ) if defined __BuildAgainstPackagesArg ( @@ -133,7 +130,7 @@ if "%__TargetsWindows%"=="1" ( @if defined _echo @echo on -set __RunArgs=-BuildOS=%__BuildOS% -BuildType=%__BuildType% -BuildArch=%__BuildArch% +set __CommonMSBuildArgs=/p:__BuildOS=%__BuildOS% /p:__BuildType=%__BuildType% /p:__BuildArch=%__BuildArch% REM As we move from buildtools to arcade, __RunArgs should be replaced with __msbuildArgs set __msbuildArgs=/p:__BuildOS=%__BuildOS% /p:__BuildType=%__BuildType% /p:__BuildArch=%__BuildArch% /nologo /verbosity:minimal /clp:Summary /maxcpucount @@ -235,9 +232,14 @@ set __BuildErr=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__B set __MsbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%" set __MsbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%" set __MsbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" -set __Logging=-MsBuildLog=!__MsbuildLog! -MsBuildWrn=!__MsbuildWrn! -MsBuildErr=!__MsbuildErr! +set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! -call "%__ProjectDir%\run.cmd" build -Project="%__NativeTestIntermediatesDir%\install.vcxproj" !__Logging! -configuration=%__BuildType% -platform=%__BuildArch% %__RunArgs% %__PriorityArg% %__PassThroughArg% %__UnprocessedBuildArgs% +call "%__ProjectDir%\msbuild.cmd" /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ + /p:UsePartialNGENOptimization=false /maxcpucount^ + "%__NativeTestIntermediatesDir%\install.vcxproj"^ + !__Logging! /p:Configuration=%__BuildType% /p:Platform=%__BuildArch% %__CommonMSBuildArgs% %__PriorityArg% %__UnprocessedBuildArgs% if errorlevel 1 ( echo %__MsgPrefix%Error: build failed. Refer to the build log files for details: echo %__BuildLog% @@ -268,9 +270,14 @@ set __BuildErr=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__B set __MsbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%" set __MsbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%" set __MsbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" -set __Logging=-MsBuildLog=!__MsbuildLog! -MsBuildWrn=!__MsbuildWrn! -MsBuildErr=!__MsbuildErr! +set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! -call "%__ProjectDir%\run.cmd" build -Project=%__ProjectDir%\tests\build.proj -BatchRestorePackages !__Logging! %__RunArgs% %__BuildAgainstPackagesArg% %__PriorityArg% %__PassThroughArg% %__UnprocessedBuildArgs% +call "%__ProjectDir%\msbuild.cmd" /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ + /p:UsePartialNGENOptimization=false /maxcpucount^ + %__ProjectDir%\tests\build.proj /t:BatchRestorePackages^ + !__Logging! %__CommonMSBuildArgs% %__BuildAgainstPackagesArg% %__PriorityArg% %__UnprocessedBuildArgs% if not defined __BuildAgainstPackagesArg goto SkipRestoreProduct @@ -283,9 +290,14 @@ set __BuildErr=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__B set __MsbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%" set __MsbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%" set __MsbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" -set __Logging=-MsBuildLog=!__MsbuildLog! -MsBuildWrn=!__MsbuildWrn! -MsBuildErr=!__MsbuildErr! +set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! -call "%__ProjectDir%\run.cmd" build -Project=%__ProjectDir%\tests\runtest.proj -BinPlaceRef -BinPlaceProduct -CopyCrossgenToProduct -RuntimeId="%__RuntimeId%" !__Logging! %__RunArgs% %__BuildAgainstPackagesArg% %__PriorityArg% %__PassThroughArg% %__UnprocessedBuildArgs% +call "%__ProjectDir%\msbuild.cmd" /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ + /p:UsePartialNGENOptimization=false /maxcpucount^ + %__Projectdir%\tests\runtest.proj /t:BinPlaceRef /t:BinPlaceProduct /t:CopyCrossgenToProduct /p:RuntimeId="%__RuntimeId%"^ + !__Logging! %__CommonMSBuildArgs% %__BuildAgainstPackagesArg% %__PriorityArg% %__UnprocessedBuildArgs% if errorlevel 1 ( echo %__MsgPrefix%Error: BinPlace of mscorlib.dll failed. Refer to the build log files for details: echo %__BuildLog% @@ -334,9 +346,9 @@ for /l %%G in (1, 1, %__BuildLoopCount%) do ( set __MsbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%";Append=!__AppendToLog! set TestBuildSlice=%%G - echo Running: msbuild %__ProjectDir%\tests\build.proj !__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! %TargetsWindowsMsbuildArg% %__msbuildArgs% %__BuildAgainstPackagesMsbuildArg% !__PriorityMsbuildArg! %__UnprocessedBuildArgs% + echo Running: msbuild %__ProjectDir%\tests\build.proj !__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! %TargetsWindowsMsbuildArg% %__msbuildArgs% %__BuildAgainstPackagesArg% !__PriorityArg! %__UnprocessedBuildArgs% - call msbuild %__ProjectDir%\tests\build.proj !__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! %TargetsWindowsMsbuildArg% %__msbuildArgs% %__BuildAgainstPackagesMsbuildArg% !__PriorityMsbuildArg! %__UnprocessedBuildArgs% + call msbuild %__ProjectDir%\tests\build.proj !__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! %TargetsWindowsMsbuildArg% %__msbuildArgs% %__BuildAgainstPackagesArg% !__PriorityArg! %__UnprocessedBuildArgs% if errorlevel 1 ( echo %__MsgPrefix%Error: build failed. Refer to the build log files for details: @@ -409,7 +421,7 @@ echo %__MsgPrefix%Creating test overlay set RuntimeIdArg= if defined __RuntimeId ( - set RuntimeIdArg=-RuntimeID="%__RuntimeId%" + set RuntimeIdArg=/p:RuntimeId="%__RuntimeId%" ) set __BuildLogRootName=Tests_Overlay_Managed @@ -419,9 +431,14 @@ set __BuildErr=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__B set __MsbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%" set __MsbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%" set __MsbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" -set __Logging=-MsBuildLog=!__MsbuildLog! -MsBuildWrn=!__MsbuildWrn! -MsBuildErr=!__MsbuildErr! +set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! -call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\runtest.proj -testOverlay !__Logging! %__RunArgs% %RuntimeIdArg% %__PriorityArg% %__PassThroughArg% %__UnprocessedBuildArgs% +call %__ProjectDir%\msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ + /p:UsePartialNGENOptimization=false /maxcpucount^ + %__ProjectDir%\tests\runtest.proj /t:CreateTestOverlay^ + !__Logging! %__CommonMSBuildArgs% %RuntimeIdArg% %__PriorityArg% %__UnprocessedBuildArgs% if errorlevel 1 ( echo %__MsgPrefix%Error: build failed. Refer to the build log files for details: echo %__BuildLog% @@ -448,9 +465,14 @@ set __BuildErr=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__B set __MsbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%" set __MsbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%" set __MsbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" -set __Logging=-MsBuildLog=!__MsbuildLog! -MsBuildWrn=!__MsbuildWrn! -MsBuildErr=!__MsbuildErr! +set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! -call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\runtest.proj -testHost !__Logging! %__RunArgs% %RuntimeIdArg% %__PriorityArg% %__PassThroughArg% %__UnprocessedBuildArgs% +call %__ProjectDir%\msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ + /p:UsePartialNGENOptimization=false /maxcpucount^ + %__ProjectDir%\tests\runtest.proj /t:CreateTestHost^ + !__Logging! %__CommonMSBuildArgs% %RuntimeIdArg% %__PriorityArg% %__UnprocessedBuildArgs% if errorlevel 1 ( echo %__MsgPrefix%Error: build failed. Refer to the build log files for details: echo %__BuildLog% @@ -479,7 +501,7 @@ set __MsbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! REM Build wrappers using the local SDK's msbuild. As we move to arcade, the other builds should be moved away from run.exe as well. -call %DotNetCli% msbuild %__ProjectDir%\tests\runtest.proj /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true !__Logging! %__msbuildArgs% %TargetsWindowsMsbuildArg% %__BuildAgainstPackagesMsbuildArg% %__UnprocessedBuildArgs% +call %DotNetCli% msbuild %__ProjectDir%\tests\runtest.proj /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true !__Logging! %__msbuildArgs% %TargetsWindowsMsbuildArg% %__BuildAgainstPackagesArg% %__UnprocessedBuildArgs% if errorlevel 1 ( echo %__MsgPrefix%Error: Xunit wrapper build failed. Refer to the build log files for details: echo %__BuildLog% @@ -500,7 +522,7 @@ REM ============================================================================ set __CrossgenArg = "" if defined __DoCrossgen ( - set __CrossgenArg="-Crossgen" + set __CrossgenArg="/p:Crossgen=true" if "%__TargetsWindows%" == "1" ( echo %__MsgPrefix%Running crossgen on framework assemblies call :PrecompileFX @@ -528,9 +550,14 @@ set __BuildErr=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__B set __MsbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%" set __MsbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%" set __MsbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%" -set __Logging=-MsBuildLog=!__MsbuildLog! -MsBuildWrn=!__MsbuildWrn! -MsBuildErr=!__MsbuildErr! +set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! -call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\helixprep.proj !__Logging! %__RunArgs% %RuntimeIdArg% %TargetsWindowsArg% %__CrossgenArg% %__PriorityArg% %__PassThroughArg% %__UnprocessedBuildArgs% +call %__ProjectDir%\msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ + /p:UsePartialNGENOptimization=false /maxcpucount^ + %__ProjectDir%\tests\helixprep.proj^ + !__Logging! %__CommonMSBuildArgs% %RuntimeIdArg% %TargetsWindowsMSBuildArg% %__CrossgenArg% %__PriorityArg% %__UnprocessedBuildArgs% if errorlevel 1 ( echo %__MsgPrefix%Error: build failed. Refer to the build log files for details: echo %__BuildLog% diff --git a/build-test.sh b/build-test.sh index 7542edb698..a0ba5da53c 100755 --- a/build-test.sh +++ b/build-test.sh @@ -189,10 +189,10 @@ generate_layout() # === # ========================================================================================= - build_MSBuild_projects "Restore_Packages" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "-BatchRestorePackages" + build_MSBuild_projects "Restore_Packages" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "/t:BatchRestorePackages" if [ -n "$__UpdateInvalidPackagesArg" ]; then - __up=-updateinvalidpackageversion + __up="/t:UpdateInvalidPackageVersions" fi echo "${__MsgPrefix}Creating test overlay..." @@ -209,7 +209,7 @@ generate_layout() mkdir -p $CORE_ROOT - build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "Creating test overlay" "-testOverlay" + build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "Creating test overlay" "/t:CreateTestOverlay" chmod +x $__BinDir/corerun chmod +x $__BinDir/crossgen @@ -230,7 +230,7 @@ generate_testhost() mkdir -p $TEST_HOST - build_MSBuild_projects "Tests_Generate_TestHost" "${__ProjectDir}/tests/runtest.proj" "Creating test host" "-testHost" + build_MSBuild_projects "Tests_Generate_TestHost" "${__ProjectDir}/tests/runtest.proj" "Creating test host" "/t:CreateTestHost" } @@ -293,7 +293,7 @@ build_Tests() # ========================================================================================= if [ ${__SkipRestorePackages} != 1 ]; then - build_MSBuild_projects "Restore_Product" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "-BatchRestorePackages" + build_MSBuild_projects "Restore_Product" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "/t:BatchRestorePackages" fi if [ $__SkipNative != 1 ]; then @@ -316,7 +316,7 @@ build_Tests() else echo "Checking the Managed Tests Build..." - build_MSBuild_projects "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "-ExtraParameters:/t:CheckTestBuild" + build_MSBuild_projects "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "/t:CheckTestBuild" if [ $? -ne 0 ]; then echo "${__MsgPrefix}Error: Check Test Build failed." @@ -330,7 +330,7 @@ build_Tests() build_test_wrappers if [ -n "$__UpdateInvalidPackagesArg" ]; then - __up=-updateinvalidpackageversion + __up="/t:UpdateInvalidPackageVersions" fi generate_layout @@ -358,10 +358,10 @@ build_MSBuild_projects() __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err" # Use binclashlogger by default if no other logger is specified - if [[ "${extraBuildParameters[*]}" == *"-MsBuildEventLogging"* ]]; then - msbuildEventLogging="" + if [[ "${extraBuildParameters[*]}" == *"/l:"* ]]; then + __msbuildEventLogging= else - msbuildEventLogging="-MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\"" + __msbuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log" fi if [[ "$subDirectoryName" == "Tests_Managed" ]]; then @@ -389,13 +389,17 @@ build_MSBuild_projects() export TestBuildSlice=$slice # Generate build command - buildArgs=("-Project=$projectName" "-MsBuildLog=${__msbuildLog}" "-MsBuildWrn=${__msbuildWrn}" "-MsBuildErr=${__msbuildErr}") - buildArgs+=("$msbuildEventLogging") + buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary") + buildArgs+=("/p:RestoreDefaultOptimizationDataPackage=false" "/p:PortableBuild=true") + buildArgs+=("/p:UsePartialNGENOptimization=false" "/maxcpucount") + + buildArgs+=("$projectName" "${__msbuildLog}" "${__msbuildWrn}" "${__msbuildErr}") + buildArgs+=("$__msbuildEventLogging") buildArgs+=("${extraBuildParameters[@]}") - buildArgs+=("${__RunArgs[@]}") + buildArgs+=("${__CommonMSBuildArgs[@]}") buildArgs+=("${__UnprocessedBuildArgs[@]}") - nextCommand="\"$__ProjectRoot/run.sh\" build ${buildArgs[@]}" + nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}" echo "Building step '$stepName' slice=$slice via $nextCommand" eval $nextCommand @@ -417,13 +421,17 @@ build_MSBuild_projects() __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\"" # Generate build command - buildArgs=("-Project=$projectName" "-MsBuildLog=${__msbuildLog}" "-MsBuildWrn=${__msbuildWrn}" "-MsBuildErr=${__msbuildErr}") - buildArgs+=("$msbuildEventLogging") + buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary") + buildArgs+=("/p:RestoreDefaultOptimizationDataPackage=false" "/p:PortableBuild=true") + buildArgs+=("/p:UsePartialNGENOptimization=false" "/maxcpucount") + + buildArgs+=("$projectName" "${__msbuildLog}" "${__msbuildWrn}" "${__msbuildErr}") + buildArgs+=("$__msbuildEventLogging") buildArgs+=("${extraBuildParameters[@]}") - buildArgs+=("${__RunArgs[@]}") + buildArgs+=("${__CommonMSBuildArgs[@]}") buildArgs+=("${__UnprocessedBuildArgs[@]}") - nextCommand="\"$__ProjectRoot/run.sh\" build ${buildArgs[@]}" + nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}" echo "Building step '$stepName' via $nextCommand" eval $nextCommand @@ -468,7 +476,13 @@ build_native_projects() __versionSourceFile="$intermediatesForBuild/version.cpp" if [ $__SkipGenerateVersion == 0 ]; then pwd - "$__ProjectRoot/run.sh" build -Project=$__ProjectDir/build.proj -generateHeaderUnix -NativeVersionSourceFile=$__versionSourceFile -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log" $__RunArgs $__UnprocessedBuildArgs + $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \ + /p:UsePartialNGENOptimization=false /maxcpucount \ + $__ProjectDir/build.proj /t:GenerateVersionHeader \ + /p:GenerateVersionHeader=true /p:NativeVersionSourceFile=$__versionSourceFile \ + /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \ + $__CommonMSBuildArgs $__UnprocessedBuildArgs else # Generate the dummy version.cpp, but only if it didn't exist to make sure we don't trigger unnecessary rebuild __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";" @@ -636,7 +650,7 @@ __RootBinDir="$__ProjectDir/bin" __BuildToolsDir="$__ProjectDir/Tools" __DotNetCli="${__BuildToolsDir}/dotnetcli/dotnet" __UnprocessedBuildArgs= -__RunArgs= +__CommonMSBuildArgs= __MSBCleanBuildArgs= __UseNinja=0 __VerboseBuild=0 @@ -839,7 +853,7 @@ while :; do priority1) __priority1=1 - __UnprocessedBuildArgs+=("-priority=1") + __UnprocessedBuildArgs+=("/p:CLRTestPriorityToBuild=1") ;; *) @@ -863,12 +877,12 @@ else __NumProc=$(nproc --all) fi -__RunArgs=("-BuildArch=$__BuildArch" "-BuildType=$__BuildType" "-BuildOS=$__BuildOS") +__CommonMSBuildArgs=("/p:__BuildArch=$__BuildArch" "/p:__BuildType=$__BuildType" "/p:__BuildOS=$__BuildOS") # Configure environment if we are doing a verbose build if [ $__VerboseBuild == 1 ]; then export VERBOSE=1 - __RunArgs+=("-verbose") + __CommonMSBuildArgs+=("/v:detailed") fi # Set default clang version diff --git a/build.cmd b/build.cmd index d710084fc3..9d0f0eb3bc 100644 --- a/build.cmd +++ b/build.cmd @@ -85,7 +85,7 @@ REM __UnprocessedBuildArgs are args that we pass to msbuild (e.g. /p:__BuildArch set "__args= %*" set processedArgs= set __UnprocessedBuildArgs= -set __RunArgs= +set __CommonMSBuildArgs= set __BuildCoreLib=1 set __BuildSOS=1 @@ -98,6 +98,8 @@ set __BuildManagedTools=1 set __RestoreOptData=1 set __GenerateLayout=0 set __CrossgenAltJit= +set __SkipRestoreArg= +set __OfficialBuildIdArg= set __CrossArch= set __SkipNugetPackage=0 set __PgoOptDataVersion= @@ -186,6 +188,9 @@ if /i "%1" == "-ibcoptimize" (set __IbcOptimize=1&set processedArgs=!pro if /i "%1" == "-ibconly" (set __IbcOptimize=1&set __IbcOnly=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-ibcinstrument" (set __IbcTuning=/Tuning&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-crossgenaltjit" (set __CrossgenAltJit=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) +REM TODO remove these once they are no longer used in buildpipeline +if /i "%1" == "-skiprestore" (set __SkipRestoreArg=/p:RestoreDuringBuild=false&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "-OfficialBuildId" (set __OfficialBuildIdArg=/p:OfficialBuildId=%2&set processedArgs=!processedArgs! %1=%2&shift&shift&goto Arg_Loop) REM TODO these are deprecated remove them eventually REM don't add more, use the - syntax instead @@ -210,6 +215,8 @@ if /i "%1" == "enforcepgo" (set __EnforcePgo=1&set processedArgs=!proce if /i "%1" == "ibcoptimize" (set __IbcOptimize=1&set __PartialNgen=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "ibcinstrument" (set __IbcTuning=/Tuning&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "crossgenaltjit" (set __CrossgenAltJit=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) +REM TODO remove this once it's no longer used in buildpipeline +if /i "%1" == "--" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if [!processedArgs!]==[] ( set __UnprocessedBuildArgs=%__args% @@ -229,7 +236,7 @@ if defined __Priority ( ) else ( set __PassThroughArgs=-priority=%__Priority% ) - set __UnprocessedBuildArgs=!__UnprocessedBuildArgs! -priority=%__Priority% + set __UnprocessedBuildArgs=!__UnprocessedBuildArgs! /p:CLRTestPriorityToBuild=%__Priority% ) if defined __BuildAll goto BuildAll @@ -261,7 +268,7 @@ if %__BuildTypeDebug%==1 set __BuildType=Debug if %__BuildTypeChecked%==1 set __BuildType=Checked if %__BuildTypeRelease%==1 set __BuildType=Release -set __RunArgs=-BuildOS=%__BuildOS% -BuildType=%__BuildType% -BuildArch=%__BuildArch% +set __CommonMSBuildArgs=/p:__BuildOS=%__BuildOS% /p:__BuildType=%__BuildType% /p:__BuildArch=%__BuildArch% !__SkipRestoreArg! !__OfficialBuildIdArg! if %__EnforcePgo%==1 ( if %__BuildArchArm%==1 ( @@ -350,7 +357,12 @@ REM ============================================================================ @if defined _echo @echo on -call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\build.proj -generateHeaderWindows -NativeVersionHeaderFile="%__RootBinDir%\obj\_version.h" %__RunArgs% %__UnprocessedBuildArgs% +call %__ProjectDir%\msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ + /p:UsePartialNGENOptimization=false /maxcpucount^ + %__ProjectDir%\build.proj /t:GenerateVersionHeader /p:GenerateVersionHeader=true /p:NativeVersionHeaderFile="%__RootBinDir%\obj\_version.h"^ + %__CommonMSBuildArgs% %__UnprocessedBuildArgs% REM ========================================================================================= REM === @@ -360,7 +372,11 @@ REM ============================================================================ if %__RestoreOptData% EQU 1 ( echo %__MsgPrefix%Restoring the OptimizationData Package - call %__ProjectDir%\run.cmd build -optdata %__RunArgs% %__UnprocessedBuildArgs% + call %__ProjectDir%\msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ + /p:UsePartialNGENOptimization=false /maxcpucount^ + ./build.proj /t:RestoreOptData^ + %__CommonMSBuildArgs% %__UnprocessedBuildArgs% if not !errorlevel! == 0 ( echo %__MsgPrefix%Error: Failed to restore the optimization data package. exit /b 1 @@ -500,9 +516,13 @@ if %__BuildNative% EQU 1 ( set __MsbuildLog=/flp:Verbosity=normal;LogFile=!__BuildLog! set __MsbuildWrn=/flp1:WarningsOnly;LogFile=!__BuildWrn! set __MsbuildErr=/flp2:ErrorsOnly;LogFile=!__BuildErr! - set __Logging=-MsBuildLog=!__MsbuildLog! -MsBuildWrn=!__MsbuildWrn! -MsBuildErr=!__MsbuildErr! + set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! - call %__ProjectDir%\run.cmd build -Project=%__IntermediatesDir%\install.vcxproj !__Logging! -configuration=%__BuildType% -platform=%__BuildArch% %__RunArgs% -MSBuildNodeCount="/m:2" %__UnprocessedBuildArgs% + call %__ProjectDir%\msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ + /p:UsePartialNGENOptimization=false /maxcpucount %__IntermediatesDir%\install.vcxproj^ + !__Logging! /p:Configuration=%__BuildType% /p:Platform=%__BuildArch% %__CommonMSBuildArgs% /m:2 %__UnprocessedBuildArgs% if not !errorlevel! == 0 ( echo %__MsgPrefix%Error: native component build failed. Refer to the build log files for details: @@ -563,9 +583,14 @@ if %__BuildCrossArchNative% EQU 1 ( set __MsbuildLog=/flp:Verbosity=normal;LogFile=!__BuildLog! set __MsbuildWrn=/flp1:WarningsOnly;LogFile=!__BuildWrn! set __MsbuildErr=/flp2:ErrorsOnly;LogFile=!__BuildErr! - set __Logging=-MsBuildLog=!__MsbuildLog! -MsBuildWrn=!__MsbuildWrn! -MsBuildErr=!__MsbuildErr! + set __Logging=!_MsbuildLog! !__MsbuildWrn! !__MsbuildErr! - call %__ProjectDir%\run.cmd build -Project=%__CrossCompIntermediatesDir%\install.vcxproj !__Logging! -configuration=%__BuildType% -platform=%__CrossArch% %__RunArgs% -MSBuildNodeCount="/m:2" %__UnprocessedBuildArgs% + call %__ProjectDir%\msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ + /p:UsePartialNGENOptimization=false /maxcpucount^ + %__CrossCompIntermediatesDir%\install.vcxproj^ + !__Logging! /p:Configuration=%__BuildType% /p:Platform=%__CrossArch% %__CommonMSBuildArgs% /m:2 %__UnprocessedBuildArgs% if not !errorlevel! == 0 ( echo %__MsgPrefix%Error: cross-arch components build failed. Refer to the build log files for details: @@ -598,17 +623,17 @@ if %__BuildCoreLib% EQU 1 ( set __ExtraBuildArgs= if "%__BuildSOS%" == "0" ( - set __ExtraBuildArgs=!__ExtraBuildArgs! -SkipSOS=true + set __ExtraBuildArgs=!__ExtraBuildArgs! /p:SkipSOS=true ) if "%__BuildManagedTools%" == "1" ( - set __ExtraBuildArgs=!__ExtraBuildArgs! -BuildManagedTools=true + set __ExtraBuildArgs=!__ExtraBuildArgs! /p:BuildManagedTools=true ) if "%__SkipNugetPackage%" == "1" ( - set __ExtraBuildArgs=!__ExtraBuildArgs! -buildNugetPackage=false + set __ExtraBuildArgs=!__ExtraBuildArgs! /p:BuildNugetPackage=false ) else ( - set __ExtraBuildArgs=!__ExtraBuildArgs! -buildNugetPackage=true + set __ExtraBuildArgs=!__ExtraBuildArgs! /p:BuildNugetPackage=true ) set __BuildLogRootName=System.Private.CoreLib @@ -618,9 +643,14 @@ if %__BuildCoreLib% EQU 1 ( set __MsbuildLog=/flp:Verbosity=normal;LogFile=!__BuildLog! set __MsbuildWrn=/flp1:WarningsOnly;LogFile=!__BuildWrn! set __MsbuildErr=/flp2:ErrorsOnly;LogFile=!__BuildErr! - set __Logging=-MsBuildLog=!__MsbuildLog! -MsBuildWrn=!__MsbuildWrn! -MsBuildErr=!__MsbuildErr! + set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! - call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\build.proj !__Logging! %__RunArgs% !__ExtraBuildArgs! %__UnprocessedBuildArgs% + call %__ProjectDir%\msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ + /p:UsePartialNGENOptimization=false /maxcpucount^ + %__ProjectDir%\build.proj^ + !__Logging! %__CommonMSBuildArgs% !__ExtraBuildArgs! %__UnprocessedBuildArgs% if not !errorlevel! == 0 ( echo %__MsgPrefix%Error: System.Private.CoreLib build failed. Refer to the build log files for details: @@ -811,10 +841,15 @@ if %__BuildPackages% EQU 1 ( set __MsbuildLog=/flp:Verbosity=normal;LogFile=!__BuildLog! set __MsbuildWrn=/flp1:WarningsOnly;LogFile=!__BuildWrn! set __MsbuildErr=/flp2:ErrorsOnly;LogFile=!__BuildErr! - set __Logging=-MsBuildLog=!__MsbuildLog! -MsBuildWrn=!__MsbuildWrn! -MsBuildErr=!__MsbuildErr! + set __Logging=!__MsbuildLog! !__MsbuildWrn! !__MsbuildErr! REM The conditions as to what to build are captured in the builds file. - call %__ProjectDir%\run.cmd build -Project=%__SourceDir%\.nuget\packages.builds !__Logging! -platform=%__BuildArch% %__RunArgs% %__UnprocessedBuildArgs% + call %__ProjectDir%\msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false^ + /l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log^ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true^ + /p:UsePartialNGENOptimization=false /maxcpucount^ + %__SourceDir%\.nuget\packages.builds^ + !__Logging! /p:Platform=%__BuildArch% %__CommonMSBuildArgs% %__UnprocessedBuildArgs% if not !errorlevel! == 0 ( echo %__MsgPrefix%Error: Nuget package generation failed. Refer to the build log files for details: diff --git a/build.sh b/build.sh index 14452ad5ac..0b3859c1ef 100755 --- a/build.sh +++ b/build.sh @@ -179,7 +179,11 @@ restore_optdata() if [[ ( $__SkipRestoreOptData == 0 ) && ( $__isMSBuildOnNETCoreSupported == 1 ) ]]; then echo "Restoring the OptimizationData package" - "$__ProjectRoot/run.sh" build -optdata $__RunArgs $__UnprocessedBuildArgs + "$__ProjectRoot/dotnet.sh" msbuild /nologo /verbosity:minimal /clp:Summary \ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \ + /p:UsePartialNGENOptimization=false /maxcpucount \ + /t:RestoreOptData ./build.proj \ + $__CommonMSBuildArgs $__UnprocessedBuildArgs if [ $? != 0 ]; then echo "Failed to restore the optimization data package." exit 1 @@ -292,7 +296,12 @@ build_native() __versionSourceFile="$intermediatesForBuild/version.cpp" if [ $__SkipGenerateVersion == 0 ]; then pwd - "$__ProjectRoot/run.sh" build -Project=$__ProjectDir/build.proj -generateHeaderUnix -NativeVersionSourceFile=$__versionSourceFile -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log" $__RunArgs $__UnprocessedBuildArgs + "$__ProjectRoot/dotnet.sh" msbuild /nologo /verbosity:minimal /clp:Summary \ + /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \ + /p:UsePartialNGENOptimization=false /maxcpucount \ + "$__ProjectDir/build.proj" /p:GenerateVersionSourceFile=true /t:GenerateVersionSourceFile /p:NativeVersionSourceFile=$__versionSourceFile \ + $__CommonMSBuildArgs $__UnprocessedBuildArgs else # Generate the dummy version.cpp, but only if it didn't exist to make sure we don't trigger unnecessary rebuild __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";" @@ -448,15 +457,22 @@ build_CoreLib() # Invoke MSBuild __ExtraBuildArgs="" if [[ "$__IbcTuning" == "" ]]; then - __ExtraBuildArgs="$__ExtraBuildArgs -OptimizationDataDir=\"$__PackagesDir/optimization.$__BuildOS-$__BuildArch.IBC.CoreCLR/$__IbcOptDataVersion/data/\"" - __ExtraBuildArgs="$__ExtraBuildArgs -EnableProfileGuidedOptimization=true" + __ExtraBuildArgs="$__ExtraBuildArgs /p:OptimizationDataDir=\"$__PackagesDir/optimization.$__BuildOS-$__BuildArch.IBC.CoreCLR/$__IbcOptDataVersion/data\"" + __ExtraBuildArgs="$__ExtraBuildArgs /p:EnableProfileGuidedOptimization=true" fi if [[ "$__BuildManagedTools" -eq "1" ]]; then - __ExtraBuildArgs="$__ExtraBuildArgs -BuildManagedTools=true" + __ExtraBuildArgs="$__ExtraBuildArgs /p:BuildManagedTools=true" fi - $__ProjectRoot/run.sh build -Project=$__ProjectDir/build.proj -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log" -MsBuildLog="/flp:Verbosity=normal;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log" -BuildTarget -__IntermediatesDir=$__IntermediatesDir -__RootBinDir=$__RootBinDir -BuildNugetPackage=false -UseSharedCompilation=false $__RunArgs $__ExtraBuildArgs $__UnprocessedBuildArgs + $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \ + /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \ + /p:UsePartialNGENOptimization=false /maxcpucount \ + $__ProjectDir/build.proj \ + /flp:Verbosity=normal\;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log \ + /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir /p:BuildNugetPackage=false /p:UseSharedCompilation=false \ + $__CommonMSBuildArgs $__ExtraBuildArgs $__UnprocessedBuildArgs if [ $? -ne 0 ]; then echo "Failed to build managed components." @@ -513,7 +529,14 @@ generate_NugetPackages() echo "DistroRid is "$__DistroRid echo "ROOTFS_DIR is "$ROOTFS_DIR # Build the packages - $__ProjectRoot/run.sh build -Project=$__SourceDir/.nuget/packages.builds -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log" -MsBuildLog="/flp:Verbosity=normal;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log" -BuildTarget -__IntermediatesDir=$__IntermediatesDir -__RootBinDir=$__RootBinDir -BuildNugetPackage=false -UseSharedCompilation=false -__DoCrossArchBuild=$__CrossBuild $__RunArgs $__UnprocessedBuildArgs + $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \ + /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \ + /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \ + /p:UsePartialNGENOptimization=false /maxcpucount \ + $__SourceDir/.nuget/packages.builds \ + /flp:Verbosity=normal\;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log \ + /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir /p:BuildNugetPackages=false /p:UseSharedCompilation=false /p:__DoCrossArchBuild=$__CrossBuild \ + $__CommonMSBuildArgs $__UnprocessedBuildArgs if [ $? -ne 0 ]; then echo "Failed to generate Nuget packages." @@ -624,7 +647,7 @@ __SourceDir="$__ProjectDir/src" __PackagesDir="${DotNetRestorePackagesPath:-${__ProjectDir}/packages}" __RootBinDir="$__ProjectDir/bin" __UnprocessedBuildArgs= -__RunArgs= +__CommonMSBuildArgs= __MSBCleanBuildArgs= __UseNinja=0 __VerboseBuild=0 @@ -656,6 +679,9 @@ __msbuildonunsupportedplatform=0 __PgoOptDataVersion="" __IbcOptDataVersion="" __BuildManagedTools=1 +__SkipRestoreArg="" +__SignTypeArg="" +__OfficialBuildIdArg="" # Get the number of processors available to the scheduler # Other techniques such as `nproc` only get the number of @@ -909,6 +935,23 @@ while :; do exit 1 ;; + -skiprestore) + __SkipRestoreArg="/p:RestoreDuringBuild=false" + ;; + + -disableoss) + __SignTypeArg="/p:SignType=real" + ;; + + -officialbuildid=*) + __Id=$(echo $1| cut -d'=' -f 2) + __OfficialBuildIdArg="/p:OfficialBuildId=$__Id" + ;; + + --) + # Skip -Option=Value style argument passing + ;; + *) __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" ;; @@ -917,23 +960,23 @@ while :; do shift done -__RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS" +__CommonMSBuildArgs="/p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__BuildOS=$__BuildOS $__OfficialBuildIdArg $__SignTypeArg $__SkipRestoreArg" # Configure environment if we are doing a verbose build if [ $__VerboseBuild == 1 ]; then export VERBOSE=1 - __RunArgs="$__RunArgs -verbose" + __CommonMSBuildArgs="$__CommonMSBuildArgs /v:detailed" fi # Set default clang version if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then - if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then - __ClangMajorVersion=5 - __ClangMinorVersion=0 - else - __ClangMajorVersion=3 - __ClangMinorVersion=9 - fi + if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then + __ClangMajorVersion=5 + __ClangMinorVersion=0 + else + __ClangMajorVersion=3 + __ClangMinorVersion=9 + fi fi if [[ "$__BuildArch" == "armel" ]]; then @@ -942,7 +985,7 @@ if [[ "$__BuildArch" == "armel" ]]; then fi if [ $__PortableBuild == 0 ]; then - __RunArgs="$__RunArgs -PortableBuild=false" + __CommonMSBuildArgs="$__CommonMSBuildArgs /p:PortableBuild=false" fi # Set dependent variables diff --git a/clean.cmd b/clean.cmd index 17f9e402c6..f4150d7811 100644 --- a/clean.cmd +++ b/clean.cmd @@ -26,8 +26,11 @@ if /I [%NO_DASHES_ARG:-=%] == [all] ( ) :no_args -if [%1]==[] set __args=-b -call %~dp0run.cmd clean %__args% %* +if [%1]==[] set __args=/t:CleanAllProjects +if [%1]==[-b] set __args=/t:CleanAllProjects +if [%1]==[-p] set __args=/t:CleanPackages +if [%1]==[-c] set __args=/t:CleanPackagesCache +call %~dp0msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false /flp:v=normal;LogFile=clean.log %__args% exit /b %ERRORLEVEL% :Usage diff --git a/clean.sh b/clean.sh index d4af84a02d..0a30b29b40 100755 --- a/clean.sh +++ b/clean.sh @@ -26,11 +26,14 @@ then echo "Removing all untracked files in the working tree" git clean -xdf $__working_tree_root exit $? +elif [ $# == 0 ] || [ "$*" == "-b" ]; then + __args="/t:CleanAllProjects" +elif [ "$*" == "-p" ]; then + __args="/t:CleanPackages" +elif [ "$*" == "-c" ]; then + __args="/t:CleanPackagesCache" fi -if [ $# == 0 ]; then - __args=-b -fi -$__working_tree_root/run.sh clean $__args $* +$__working_tree_root/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary /flp:v=normal\;LogFile=clean.log $__args exit $? diff --git a/config.json b/config.json deleted file mode 100644 index 32cd726387..0000000000 --- a/config.json +++ /dev/null @@ -1,810 +0,0 @@ -{ - "settings": { - "CloudDropAccessToken": { - "description": "Azure access token", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "CloudDropAccountName": { - "description": "Azure account name.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "__PublishPackages": { - "description": "Specifies whether to publish product packages.", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "__PublishSymbols": { - "description": "Specifies whether to publish symbol packages.", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "BuildNumberMajor": { - "description": "Product build major number.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "BuildNumberMinor": { - "description": "Product build minor number.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "CleanAllProjects": { - "description": "MsBuild target that deletes the binary output directory.", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "CleanPackages": { - "description": "MsBuild target that deletes the repo-local nuget package directory.", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "CleanPackagesCache": { - "description": "MsBuild target that deletes the user-local nuget package cache.", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "ContainerName": { - "description": "Container name for Azure upload.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "__BlobNamePrefix": { - "description": "Blob Prefix for Azure download.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "MsBuildFileLogging": { - "description": "MsBuild logging options.", - "valueType": "passThrough", - "values": [], - "defaultValue": "/flp:v=normal" - }, - "MsBuildEventLogging": { - "description": "MsBuild logging options.", - "valueType": "passThrough", - "values": [], - "defaultValue": "/l:BinClashLogger,Tools/net46/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log" - }, - "FilterToOSGroup": { - "description": "Specifies the OSGroup.", - "valueType": "property", - "values": [ "Windows_NT", "Linux", "Unix", "OSX", "Android" ], - "defaultValue": "${OSName}" - }, - "Sync": { - "description": "MsBuild target that restores the packages.", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "RestoreOptData": { - "description": "MsBuild target that restores optimization profile data.", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "RestoreDuringBuild": { - "description": "Enables/disables package restore.", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "Project": { - "description": "Project where the commands are going to be applied.", - "valueType": "passThrough", - "values": [], - "defaultValue": "" - }, - "__BuildType": { - "description": "Specifies the configuration to publish packages", - "valueType": "property", - "values": [ "release", "debug", "checked" ], - "defaultValue": "debug" - }, - "__BuildArch": { - "description": "Specifies the architecture to publish packages", - "valueType": "property", - "values": [ "x64", "x86", "arm", "arm64" ], - "defaultValue": "${CPUArch}" - }, - "__BuildOS": { - "description": "Specifies the OS to publish packages.", - "valueType": "property", - "values": [ "Linux", "OSX", "FreeBSD", "OpenBSD", "NetBSD", "SunOS", "Windows_NT", "Android" ], - "defaultValue": "${OSName}" - }, - "__DistroRid": { - "description": "Specifies the distro rid for Unix OS.", - "valueType": "property", - "values": [], - "defaultValue": "${OSRid}" - }, - "__VSVersion": { - "description": "Sets the Visual Studio version to use.", - "valueType": "property", - "values": [], - "defaultValue": "vs2015" - }, - "__ProjectDir": { - "description": "Directory of the dir.props file.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "__SourceDir": { - "description": "Source directory.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "__PackagesDir": { - "description": "Packages directory.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "__RootBinDir": { - "description": "Root bin directory.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "__BinDir": { - "description": "Bin directory of specific OS, Arch and Type being built.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "__IntermediatesDir": { - "description": "Intermediates bin directory.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "__PackagesBinDir": { - "description": "Packages bin directory.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "__TestWorkingDir": { - "description": "Test working directory.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "__DoCrossArchBuild": { - "description": "Specifies if we build cross-architecture components on Linux.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "GenerateVersionHeader": { - "description": "Generates _version.h", - "valueType": "passThrough", - "values": [], - "defaultValue": "/t:GenerateVersionHeader /p:GenerateVersionHeader=true" - }, - "GenerateVersionSourceFile": { - "description": "Generate version.cpp", - "valueType": "passThrough", - "values": [], - "defaultValue": "/t:GenerateVersionSourceFile /p:GenerateVersionSourceFile=true" - }, - "MSBuildNodeCount": { - "description": "Sets the value of the number of processors available for MSBuild.", - "valueType": "passThrough", - "values": [], - "defaultValue": "/maxcpucount" - }, - "NativeVersionSourceFile": { - "description": "Provides location for version.cpp", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "NativeVersionHeaderFile": { - "description": "Provides location for _version.h", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "OverwriteCoreClrPackageVersion": { - "description": "Sets the property specifying to overwrite CoreClr Package version using OfficialBuildId", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "OfficialBuildId": { - "description": "Official Build Id.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "Configuration": { - "description": "Specifies the build type for native build.", - "valueType": "property", - "values": [ "debug", "release", "checked" ], - "defaultValue": "debug" - }, - "EnableProfileGuidedOptimization": { - "description": "Enables IBC profile optimizations if profile data are available.", - "valueType": "property", - "values": [ true, false ], - "defaultValue": false - }, - "RestoreDefaultOptimizationDataPackage": { - "description": "Instructs BuildTools to restore the default OptimizationData package.", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "UsePartialNGENOptimization": { - "description": "Adds the `-partialNGEN -minify` arguments when invoking IBCMerge.", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "OptimizationDataDir": { - "description": "Sets the path where the build should look for IBC profile data.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "UseEnv": { - "description": "Set when building for arm64.", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "Platform": { - "description": "Sets the build arch for native build.", - "valueType": "property", - "values": [ "x86", "x64", "arm", "arm64" ], - "defaultValue": "${CPUArch}" - }, - "PortableBuild": { - "description": "Indicates if this is a portable build.", - "valueType": "property", - "values": [ "false", "true"], - "defaultValue": "true" - }, - "MsBuildLog": { - "description": "Specifies build logs for build alias.", - "valueType": "passThrough", - "values": [], - "defaultValue": "" - }, - "MsBuildWrn": { - "description": "Specifies build logs for build alias.", - "valueType": "passThrough", - "values": [], - "defaultValue": "" - }, - "MsBuildErr": { - "description": "Specifies build logs for build alias.", - "valueType": "passThrough", - "values": [], - "defaultValue": "" - }, - "BuildNugetPackage": { - "description": "Specifies to build nuget packages.", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "SkipSOS": { - "description": "", - "valueType": "property", - "values": [ true, false ], - "defaultValue": false - }, - "BuildManagedTools": { - "description": "", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "SignType": { - "description": "Sets the SignType.", - "valueType": "property", - "values": [ "real" ], - "defaultValue": "" - }, - "CLRTestPriorityToBuild": { - "description": "Sets priority to build test.", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "CreateTestOverlay": { - "description": "Runs CreateTestOverlay target.", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "CreateNonWindowsTestOverlay": { - "description": "Runs CreateNonWindowsTestOverlay target.", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "CreateTestHost": { - "description": "Place test dependencies in the /bin/testhost folder to be used to run CoreFX tests", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "Verbosity": { - "description": "Sets build verbosity.", - "valueType": "passThrough", - "values": [], - "defaultValue": "/v:normal" - }, - "Build": { - "description": "Runs the build target.", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "UseSharedCompilation": { - "description": "Sets the use shared compilation property.", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "UpdateInvalidPackageVersions": { - "description": "Runs the target to update package versions.", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "GenerateRuntimeLayout": { - "description": "Generates Core_Root folder", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "BuildTestsAgainstPackages": { - "description": "Sets the property specifying if we're building tests against packages", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "TargetsWindows": { - "description": "Sets the property specifying if the tests we're building will target Windows", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "PublishTestNativeBins": { - "description": "Publishes test native binaries to Azure on non-windows", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "BuildWrappers": { - "description": "Builds test Xunit wrappers", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "Crossgen": { - "description": "Determines if we're running Crossgen tests", - "valueType": "property", - "values": [ true, false ], - "defaultValue": true - }, - "RuntimeId": { - "description": "Specifies the OS to build Core_Root for", - "valueType": "property", - "values": [ "debian.8-x64", "fedora.24-x64", "fedora.25-x64", "opensuse.42.1-x64", "osx.10.12-x64", "rhel.7-x64", "ubuntu.14.04-x64", "ubuntu.16.04-x64", "ubuntu.16.10-x64", "android.21-arm64" ], - "defaultValue": "${__RuntimeId}" - }, - "BatchRestorePackages": { - "description": "MsBuild target that restores the packages.", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "BinPlaceRef": { - "description": "Place mscorlib.dll in bin/Product ref folder for building tests against", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "BinPlaceProduct": { - "description": "Place test dependencies in bin/Product folder for building tests against", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "CopyCrossgenToProduct": { - "description": "Place crossgen.exe in bin/Product folder for building tests against", - "valueType": "target", - "values": [], - "defaultValue": "" - }, - "OptionalToolSource": { - "description": "URL of the nuget feed used by 'optional-tools'", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "OptionalToolSourceUser": { - "description": "User name for authenticating to the optional tools feed", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "OptionalToolSourcePassword": { - "description": "VSTS token for authenticating to the optional tools feed; requires 'package(READ)' permission", - "valueType": "property", - "values": [], - "defaultValue": "" - }, - "ExtraParameters": { - "description": "Extra parameters will be passed to the selected command.", - "valueType": "passThrough", - "values": [], - "defaultValue": "" - } - }, - "commands": { - "build": { - "alias": { - "generateHeaderWindows": { - "description": "Generates _version.h", - "settings": { - "GenerateVersionHeader": "default", - "NativeVersionHeaderFile": "default" - } - }, - "generateHeaderUnix": { - "description": "Generates version.cpp", - "settings": { - "GenerateVersionSourceFile": "default", - "NativeVersionSourceFile": "default" - } - }, - "verbose": { - "description": "Sets the verbosity", - "settings": { - "Verbosity": "/v:detailed" - } - }, - "disableoss": { - "description": "Sets the sign type.", - "settings": { - "SignType": "real" - } - }, - "buildType": { - "description": "Sets buildtype.", - "settings": { - "__BuildType": "default" - } - }, - "buildArch": { - "description": "Sets buildarch.", - "settings": { - "__BuildArch": "default" - } - }, - "buildOS": { - "description": "Sets the OS to build.", - "settings": { - "__BuildOS": "default" - } - }, - "testOverlay": { - "description": "Runs test overlay target.", - "settings": { - "CreateTestOverlay": "default" - } - }, - "testHost": { - "description": "Runs test host target, which creates a dotnet executable from the built CoreCLR repo to run pre-built CoreFX test assemblies", - "settings": { - "CreateTestHost": "default" - } - }, - "priority": { - "description": "Sets CLRTestPriorityToBuild property.", - "settings": { - "CLRTestPriorityToBuild": "default" - } - }, - "skipRestore": { - "description": "Skips restoring packages during build.", - "settings": { - "RestoreDuringBuild": false - } - }, - "BuildTarget": { - "description": "Runs the build target.", - "settings": { - "Build": "default" - } - }, - "optdata": { - "description": "Restores optimization profile data for the repository.", - "settings": { - "Project": "./build.proj", - "RestoreOptData": "default", - "MsBuildEventLogging": " ", - } - } - }, - "defaultValues": { - "toolName": "msbuild", - "settings": { - "MsBuildLog": "default", - "MsBuildWrn": "default", - "MsBuildErr": "default", - "MsBuildEventLogging": "default", - "RestoreDefaultOptimizationDataPackage": "false", - "PortableBuild": "true", - "UsePartialNGENOptimization": "false", - "MSBuildNodeCount": "default" - } - } - }, - "clean": { - "alias": { - "b": { - "description": "Deletes the binary output directory.", - "settings": { - "CleanAllProjects": "default" - } - }, - "p": { - "description": "Deletes the repo-local nuget package directory.", - "settings": { - "CleanPackages": "default" - } - }, - "c": { - "description": "Deletes the user-local nuget package cache.", - "settings": { - "CleanPackagesCache": "default" - } - } - }, - "defaultValues": { - "toolName": "msbuild", - "settings": { - "MsBuildLog":"/flp:v=normal;LogFile=clean.log" - } - } - }, - "sync": { - "alias": { - "p": { - "description": "Restores all NuGet packages for repository.", - "settings": { - "Project": "./build.proj", - "RestoreDuringBuild": true, - "Sync": "default" - } - }, - "ab": { - "description": "Downloads the latests product packages from Azure. The values for '-AzureAccount' and '-AzureToken' are required", - "settings": { - "Project": "./src/syncAzure.proj" - } - }, - "n": { - "description": "Downloads test native binaries. The values for '-AzureAccount', '-AzureToken', and '-Container' are required", - "settings": { - "Project": "./src/syncAzure.proj", - "PublishTestNativeBins": "true" - } - }, - "azureToken": { - "description": "Account token to connect to Azure Blob storage.", - "settings": { - "CloudDropAccessToken": "default" - } - }, - "azureAccount": { - "description": "Account name to connect to Azure Blob storage.", - "settings": { - "CloudDropAccountName": "default" - } - }, - "container": { - "description": "Container name to download from in Azure Blob storage.", - "settings": { - "ContainerName": "default" - } - }, - "blobNamePrefix": { - "description": "Blob Prefix to download from in Azure container.", - "settings": { - "__BlobNamePrefix": "default" - } - }, - "runtimeId": { - "description": "Runtime ID of the binaries in the specified container.", - "settings": { - "RuntimeId": "default" - } - }, - "verbose": { - "description": "Passes /flp:v=diag to the msbuild command or the value passed by the user.", - "settings": { - "MsBuildFileLogging": "/flp:v=diag;LogFile=sync.log" - } - }, - "buildMajor": { - "description": "To download a specific group of product packages, specify build number. The value for -BuildMinor required.", - "settings": { - "BuildNumberMajor": "default" - } - }, - "buildMinor": { - "description": "To download a specific group of product packages, specify build number. The value for -BuildMajor required.", - "settings": { - "BuildNumberMinor": "default" - } - }, - "buildType": { - "description": "Sets buildtype.", - "settings": { - "__BuildType": "default" - } - } - }, - "defaultValues": { - "toolName": "msbuild", - "settings": { - "MsBuildFileLogging": "/flp:v=detailed;LogFile=sync.log" - } - } - }, - "publish-packages": { - "alias": { - "azureToken": { - "description": "Account token to connect to Azure Blob storage.", - "settings": { - "CloudDropAccessToken": "default" - } - }, - "azureAccount": { - "description": "Account name to connect to Azure Blob storage.", - "settings": { - "CloudDropAccountName": "default" - } - }, - "container": { - "description": "Container name to upload into in Azure Blob storage.", - "settings": { - "ContainerName": "default" - } - }, - "buildArch": { - "description": "Specifies architecture to publish, can be x64, x86, arm or arm64", - "settings": { - "__BuildArch": "default" - } - }, - "buildType": { - "description": "Specifies configuration to publish, can be Release, Debug or Checked.", - "settings": { - "__BuildType": "default" - } - }, - "buildOS": { - "description": "Specifies the OS to publish packages.", - "settings": { - "__BuildOS": "default" - } - }, - "distroRid": { - "description": "Specifies distro rid for Unix OS.", - "settings": { - "__DistroRid": "default" - } - }, - "PublishTestNativeBins": { - "description": "Publishes test native binaries.", - "settings": { - "PublishTestNativeBins": "default" - } - }, - "PublishPackages": { - "description": "Publishes product packages.", - "settings": { - "__PublishPackages": "default" - } - }, - "PublishSymbols": { - "description": "Publishes symbol packages.", - "settings": { - "__PublishSymbols": "default" - } - }, - }, - "defaultValues": { - "toolName": "msbuild", - "settings": { - "__BuildOS": "default", - "Project": "./src/publish.proj", - "MsBuildFileLogging": "/flp:v=detailed;LogFile=publish-packages.log /clp:v=detailed" - } - } - }, - "build-packages": { - "alias": { - "buildArch": { - "description": "Specifies architecture to publish, can be x64, x86, arm or arm64", - "settings": { - "__BuildArch": "default" - } - }, - "buildType": { - "description": "Specifies configuration to publish, can be Release, Debug or Checked.", - "settings": { - "__BuildType": "default" - } - }, - "distroRid": { - "description": "Specifies distro rid for Unix OS.", - "settings": { - "__DistroRid": "default" - } - } - }, - "defaultValues": { - "toolName": "msbuild", - "settings": { - "__BuildOS": "default", - "MsBuildFileLogging": "/flp:v=detailed;Append;LogFile=build-packages.log", - "MsBuildEventLogging": "default", - "PortableBuild": "true", - "Project": "src/.nuget/packages.builds" - } - } - } - }, - "tools": { - "msbuild": { - "osSpecific": { - "windows": { - "defaultParameters": "/nologo /verbosity:minimal /clp:Summary /nodeReuse:false", - "path": "Tools/msbuild.cmd" - }, - "unix": { - "defaultParameters": "/nologo /verbosity:minimal /clp:Summary", - "path": "Tools/msbuild.sh" - } - }, - "valueTypes": { - "property": "/p:{name}={value}", - "target": "/t:{name}", - "internal": "/{name}" - } - }, - "terminal": { - "run": { - "windows": "cmd", - "unix": "sh" - }, - "valueTypes": {} - } - } -} diff --git a/dir.common.props b/dir.common.props index 0c0096c9fb..dc9d8f1394 100644 --- a/dir.common.props +++ b/dir.common.props @@ -25,6 +25,8 @@ Windows_NT Linux OSX + + <__BuildOS>$(BuildOS) $(BuildType) $(BuildArch) diff --git a/dotnet.sh b/dotnet.sh new file mode 100755 index 0000000000..128f60b0bb --- /dev/null +++ b/dotnet.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +working_tree_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +echo "Running init-tools.sh" +source $working_tree_root/init-tools.sh + +toolRuntime=$working_tree_root/Tools +dotnet=$toolRuntime/dotnetcli/dotnet + +echo "Running: $dotnet $@" +$dotnet "$@" +if [ $? -ne 0 ] +then + echo "ERROR: An error occurred in $dotnet $@. Check logs under $working_tree_root." + exit 1 +fi + +echo "Command successfully completed." +exit 0 diff --git a/msbuild.cmd b/msbuild.cmd new file mode 100644 index 0000000000..eb99255a45 --- /dev/null +++ b/msbuild.cmd @@ -0,0 +1,30 @@ +@if not defined _echo @echo off +setlocal + +set "__ProjectDir=%~dp0" + +call "%__ProjectDir%"\setup_vs_tools.cmd + +REM setup_vs_tools.cmd will correctly echo error message. +if NOT '%ERRORLEVEL%' == '0' exit /b 1 + +:: Clear the 'Platform' env variable for this session, as it's a per-project setting within the build, and +:: misleading value (such as 'MCD' in HP PCs) may lead to build breakage (issue: #69). +set Platform= +set __ProjectDir= + +:: Restore the Tools directory +call %~dp0init-tools.cmd +if NOT [%ERRORLEVEL%]==[0] ( + exit /b 1 +) + +pushd %~dp0 +echo Running: msbuild.exe %* +call msbuild.exe %* +popd +if NOT [%ERRORLEVEL%]==[0] ( + exit /b 1 +) + +exit /b 0 diff --git a/publish-packages.cmd b/publish-packages.cmd index 3dd89ceecf..79adbe93da 100644 --- a/publish-packages.cmd +++ b/publish-packages.cmd @@ -1,11 +1,43 @@ @if not defined _echo @echo off setlocal EnableDelayedExpansion + +set "__args=%*" +set processedArgs= +set unprocessedArgs= +set __MSBuildArgs= + +:ArgLoop + +if "%1" == "" goto ArgsDone if /I [%1] == [-?] goto Usage if /I [%1] == [-help] goto Usage -echo %~dp0run.cmd publish-packages %* -call %~dp0run.cmd publish-packages %* +REM This for loop splits the remaining arguments, preserving "=". +REM x gets the next argument, and y gets all remaining arguments after the first. +FOR /f "tokens=1*" %%x IN ("%*") DO ( + set param=%%x + if /i "!param:~0,14!" == "-AzureAccount=" (set v=!param:~14!&set __MSBuildArgs=!__MSBuildArgs! /p:CloudDropAccountName=!v!) + if /i "!param:~0,12!" == "-AzureToken=" (set v=!param:~12!&set __MSBuildArgs=!__MSBuildArgs! /p:CloudDropAccessToken=!v!) + if /i "!param:~0,11!" == "-BuildArch=" (set v=!param:~11!&set __MSBuildArgs=!__MSBuildArgs! /p:__BuildArch=!v!) + if /i "!param:~0,11!" == "-BuildType=" (set v=!param:~11!&set __MSBuildArgs=!__MSBuildArgs! /p:__BuildType=!v!) + if /i "!param:~0,11!" == "-Container=" (set v=!param:~11!&set __MSBuildArgs=!__MSBuildArgs! /p:ContainerName=!v!) + if /i "!param!" == "-PublishPackages" (set __MSBuildArgs=!__MSBuildArgs! /p:__PublishPackages=true) + if /i "!param!" == "-PublishSymbols" (set __MSBuildArgs=!__MSBuildArgs! /p:__PublishSymbols=true) + REM all other arguments get passed through to msbuild unchanged. + if /i not "!param:~0,1!" == "-" (set __MSBuildArgs=!__MSBuildArgs! !param!) + + REM The innermost recursive invocation of :ArgLoop will execute + REM msbuild, and all other invocations simply exit. + call :ArgLoop %%y + exit /b +) + +:ArgsDone + +call %~dp0msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false /p:__BuildOS=Windows_NT^ + .\src\publish.proj^ + /flp:v=detailed;LogFile=publish-packages.log /clp:v=detailed %__MSBuildArgs% @exit /b %ERRORLEVEL% :Usage diff --git a/publish-packages.sh b/publish-packages.sh index 1d0f13b296..492fa462a2 100755 --- a/publish-packages.sh +++ b/publish-packages.sh @@ -22,7 +22,92 @@ usage() working_tree_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -$working_tree_root/run.sh publish-packages -BuildOS $* +# Use uname to determine what the OS is. +OSName=$(uname -s) +case $OSName in + Linux) + __BuildOS=Linux + ;; + Darwin) + __BuildOS=OSX + ;; + FreeBSD) + __BuildOS=FreeBSD + ;; + OpenBSD) + __BuildOS=OpenBSD + ;; + NetBSD) + __BuildOS=NetBSD + ;; + SunOS) + __BuildOS=SunOS + ;; + *) + echo "Unsupported OS $OSName detected, configuring as if for Linux" + __BuildOS=Linux + ;; +esac + +buildArgs= +unprocessedBuildArgs= + +# TODO: get rid of argument processing entirely once we remove the +# uses of -Arg=Value style in buildpipeline. +while :; do + if [ $# -le 0 ]; then + break + fi + case "$1" in + -\?|-h|--help) + usage + exit 1 + ;; + -AzureAccount=*) + __Account=$(echo $1| cut -d'=' -f 2) + buildArgs="$buildArgs /p:CloudDropAccountName=$__Account" + ;; + -AzureToken=*) + __Token=${1:12} + buildArgs="$buildArgs /p:CloudDropAccessToken=$__Token" + ;; + -BuildType=*) + __Type=$(echo $1| cut -d'=' -f 2) + buildArgs="$buildArgs /p:__BuildType=$__Type" + ;; + -BuildArch=*) + __Arch=$(echo $1| cut -d'=' -f 2) + buildArgs="$buildArgs /p:__BuildArch=$__Arch" + ;; + -Container=*) + __Container=$(echo $1| cut -d'=' -f 2) + buildArgs="$buildArgs /p:ContainerName=$__Container" + ;; + -distroRid=*) + __Rid=$(echo $1| cut -d'=' -f 2) + buildArgs="$buildArgs /p:__DistroRid=$__Rid" + ;; + -PublishPackages) + buildArgs="$buildArgs /p:__PublishPackages=true" + ;; + -PublishSymbols) + buildArgs="$buildArgs /p:__PublishSymbols=true" + ;; + -PublishTestNativeBins) + buildArgs="$buildArgs /p:PublishTestNativeBins=true" + ;; + --) + ;; + *) + unprocessedBuildArgs="$unprocessedBuildArgs $1" + esac + shift +done + +$working_tree_root/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \ + ./src/publish.proj \ + /flp:v=detailed\;LogFile=publish-packages.log /clp:v=detailed \ + /p:__BuildOS=$__BuildOS $buildArgs $unprocessedBuildArgs if [ $? -ne 0 ] then echo "ERROR: An error occurred while publishing packages; see $working_tree_root/publish-packages.log for more details. There may have been networking problems, so please try again in a few minutes." diff --git a/run.cmd b/run.cmd deleted file mode 100644 index bba65863a9..0000000000 --- a/run.cmd +++ /dev/null @@ -1,37 +0,0 @@ -@if not defined _echo @echo off -setlocal - -set "__ProjectDir=%~dp0" - -call "%__ProjectDir%"\setup_vs_tools.cmd - -REM setup_vs_tools.cmd will correctly echo error message. -if NOT '%ERRORLEVEL%' == '0' exit /b 1 - -:: Clear the 'Platform' env variable for this session, as it's a per-project setting within the build, and -:: misleading value (such as 'MCD' in HP PCs) may lead to build breakage (issue: #69). -set Platform= -set __ProjectDir= - -:: Restore the Tools directory -call %~dp0init-tools.cmd -if NOT [%ERRORLEVEL%]==[0] ( - exit /b 1 -) - -set _toolRuntime=%~dp0Tools -set _dotnet=%_toolRuntime%\dotnetcli\dotnet.exe -set _json=%~dp0config.json - -:: run.exe depends on running in the root directory, notably because the config.json specifies -:: a relative path to the binclash logger - -pushd %~dp0 -echo Running: %_dotnet% %_toolRuntime%\run.exe %~dp0config.json %* -call %_dotnet% %_toolRuntime%\run.exe "%_json%" %* -popd -if NOT [%ERRORLEVEL%]==[0] ( - exit /b 1 -) - -exit /b 0 diff --git a/run.sh b/run.sh deleted file mode 100755 index 4b8ffeea4c..0000000000 --- a/run.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -working_tree_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -echo "Running init-tools.sh" -source $working_tree_root/init-tools.sh - -toolRuntime=$working_tree_root/Tools -dotnet=$toolRuntime/dotnetcli/dotnet - -echo "Running: $dotnet $toolRuntime/run.exe $working_tree_root/config.json $*" -$dotnet $toolRuntime/run.exe $working_tree_root/config.json "$@" -if [ $? -ne 0 ] -then - echo "ERROR: An error occured in $dotnet $toolRuntime/run.exe $working_tree_root/config.json $*. Check logs under $working_tree_root." - exit 1 -fi - -echo "Command successfully completed." -exit 0 diff --git a/sync.cmd b/sync.cmd index 949f1605f9..89b9a9e6cb 100644 --- a/sync.cmd +++ b/sync.cmd @@ -1,12 +1,46 @@ @if not defined _echo @echo off setlocal EnableDelayedExpansion +set "__args=%*" +set processedArgs= +set unprocessedArgs= +set __MSBuildArgs= + +REM If no processed arguments are specified, -p is implied. +if "%1" == "" (set __MSBuildArgs=.\build.proj /p:RestoreDuringBuild=true /t:Sync&goto ArgsDone) +if "%1" == "--" (set __MSBuildArgs=.\build.proj /p:RestoreDuringBuild=true /t:Sync&goto ArgsDone) + +:ArgLoop + +if "%1" == "" goto ArgsDone if /I [%1] == [-?] goto Usage if /I [%1] == [-help] goto Usage -@if [%1]==[] set __args=-p +REM This for loop splits the remaining arguments, preserving "=". +REM x gets the next argument, and y gets all remaining arguments after the first. +FOR /f "tokens=1*" %%x IN ("%*") DO ( + set param=%%x + if /i "!param!" == "-p" (set __MSBuildArgs=!__MSBuildArgs! .\build.proj /p:RestoreDuringBuild=true /t:Sync) + if /i "!param!" == "-ab" (set __MSBuildArgs=!__MSBuildArgs! .\src\syncAzure.proj) + if /i "!param:~0,14!" == "-AzureAccount=" (set v=!param:~14!&set __MSBuildArgs=!__MSBuildArgs! /p:CloudDropAccountName=!v!) + if /i "!param:~0,12!" == "-AzureToken=" (set v=!param:~12!&set __MSBuildArgs=!__MSBuildArgs! /p:CloudDropAccessToken=!v!) + if /i "!param:~0,12!" == "-BuildMajor=" (set v=!param:~12!&set __MSBuildArgs=!__MSBuildArgs! /p:BuildNumberMajor=!v!) + if /i "!param:~0,12!" == "-BuildMinor=" (set v=!param:~12!&set __MSBuildArgs=!__MSBuildArgs! /p:BuildNumberMinor=!v!) + if /i "!param:~0,11!" == "-Container=" (set v=!param:~11!&set __MSBuildArgs=!__MSBuildArgs! /p:ContainerName=!v!) + if /i "!param:~0,16!" == "-BlobNamePrefix=" (set v=!param:~16!&set __MSBuildArgs=!__MSBuildArgs! /p:__BlobNamePrefix=!v!) + if /i "!param:~0,11!" == "-RuntimeId=" (set v=!param:~11!&set __MSBuildArgs=!__MSBuildArgs! /p:RuntimeId=!v!) + REM all other arguments get passed through to msbuild unchanged. + if /i not "!param:~0,1!" == "-" (set __MSBuildArgs=!__MSBuildArgs! !param!) + + REM The innermost recursive invocation of :ArgLoop will execute + REM msbuild, and all other invocations simply exit. + call :ArgLoop %%y + exit /b +) + +:ArgsDone - @call %~dp0run.cmd sync %__args% %* +@call %~dp0msbuild.cmd /nologo /verbosity:minimal /clp:Summary /nodeReuse:false /flp:v=detailed;LogFile=sync.log %__MSBuildArgs% @exit /b %ERRORLEVEL% :Usage diff --git a/sync.sh b/sync.sh index 8d8b16734f..3421c6203b 100755 --- a/sync.sh +++ b/sync.sh @@ -16,7 +16,10 @@ unprocessedBuildArgs= # Assume the default '-p' argument if the only arguments specified are specified after double dash. # Only position parameters can be specified after the double dash. if [ $# == 0 ] || [ $1 == '--' ]; then - buildArgs="-p" + buildArgs="./build.proj /p:RestoreDuringBuild=true /t:Sync" + if [ $1 == '--' ]; then + shift + fi fi while [[ $# -gt 0 ]] @@ -27,7 +30,7 @@ do usage ;; -p) - buildArgs="-p" + buildArgs="./build.proj /p:RestoreDuringBuild=true /t:Sync" ;; *) unprocessedBuildArgs="$unprocessedBuildArgs $1" @@ -35,7 +38,7 @@ do shift done -$working_tree_root/run.sh sync $buildArgs $unprocessedBuildArgs +$working_tree_root/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary /flp:v=detailed\;LogFile=sync.log $buildArgs $unprocessedBuildArgs if [ $? -ne 0 ] then echo "ERROR: An error occurred while syncing packages; See $working_tree_root/sync.log for more details. There may have been networking problems, so please try again in a few minutes." diff --git a/tests/dir.props b/tests/dir.props index 7a782a125e..2266e4eec2 100644 --- a/tests/dir.props +++ b/tests/dir.props @@ -78,7 +78,9 @@ - $(RootBinDir)obj\ + + $(RootBinDir)obj\ $(BaseIntermediateOutputPath)\$(BuildOS).$(BuildArch).$(BuildType) $(BaseIntermediateOutputPath)\$(BuildOS).$(BuildArch).$(BuildType) diff --git a/tests/runtest.py b/tests/runtest.py index b25e1c89b5..db58915792 100755 --- a/tests/runtest.py +++ b/tests/runtest.py @@ -1579,9 +1579,17 @@ def setup_core_root(host_os, if host_os != "Windows_NT": os.environ["__DistroRid"] = "%s-%s" % ("osx" if sys.platform == "darwin" else "linux", arch) - command = [os.path.join(coreclr_repo_location, "run.%s" % ("cmd" if host_os == "Windows_NT" else "sh")), - "build", - "-Project=%s" % os.path.join(coreclr_repo_location, "tests", "build.proj")] + command = [dotnetcli_location, "msbuild", "/nologo", "/verbosity:minimal", "/clp:Summary", + "\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\""] + + if host_os == "Windows_NT": + command += ["/nodeReuse:false"] + + command += ["/p:RestoreDefaultOptimizationDataPackage=false", + "/p:PortableBuild=true", + "/p:UsePartialNGENOptimization=false", + "/maxcpucount", + os.path.join(coreclr_repo_location, "tests", "build.proj")] logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs") if not os.path.isdir(logs_dir): @@ -1592,25 +1600,17 @@ def setup_core_root(host_os, wrn_log = log_path + ".wrn" err_log = log_path + ".err" - msbuild_log_params = "/fileloggerparameters:\"Verbosity=normal;LogFile=%s\"" % build_log - msbuild_wrn_params = "/fileloggerparameters1:\"WarningsOnly;LogFile=%s\"" % wrn_log - msbuild_err_params = "/fileloggerparameters2:\"ErrorsOnly;LogFile=%s\"" % err_log - - command += ["-MsBuildLog=%s" % msbuild_log_params, - "-MsBuildWrn=%s" % msbuild_wrn_params, - "-MsBuildErr=%s" % msbuild_err_params] - - if host_os != "Windows_NT": - command = ["bash"] + command - command += ["-MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\""] + command += ["/fileloggerparameters:\"Verbosity=normal;LogFile=%s\"" % build_log, + "/fileloggerparameters1:\"WarningsOnly;LogFile=%s\"" % wrn_log, + "/fileloggerparameters2:\"ErrorsOnly;LogFile=%s\"" % err_log] if g_verbose: - command += ["-verbose"] + command += ["/v:detailed"] - command += [ "-BatchRestorePackages", - "-BuildType=%s" % build_type, - "-BuildArch=%s" % arch, - "-BuildOS=%s" % host_os] + command += ["/t:BatchRestorePackages", + "/p:__BuildType=%s" % build_type, + "/p:__BuildArch=%s" % arch, + "/p:__BuildOS=%s" % host_os] print("Restoring packages...") print(" ".join(command)) @@ -1627,7 +1627,7 @@ def setup_core_root(host_os, proc.kill() sys.exit(1) - if proc.returncode == 1: + if proc.returncode != 0: print("Error: package restore failed.") return False @@ -1644,9 +1644,17 @@ def setup_core_root(host_os, os.environ["Core_Root"] = core_root os.environ["xUnitTestBinBase"] = os.path.dirname(os.path.dirname(core_root)) - command = [os.path.join(coreclr_repo_location, "run.%s" % ("cmd" if host_os == "Windows_NT" else "sh")), - "build", - "-Project=%s" % os.path.join(coreclr_repo_location, "tests", "runtest.proj")] + command = [dotnetcli_location, "msbuild", "/nologo", "/verbosity:minimal", "/clp:Summary", + "\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\""] + + if host_os == "Windows_NT": + command += ["/nodeReuse:false"] + + command += ["/p:RestoreDefaultOptimizationDataPackage=false", + "/p:PortableBuild=true", + "/p:UsePartialNGENOptimization=false", + "/maxcpucount", + os.path.join(coreclr_repo_location, "tests", "runtest.proj")] logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs") if not os.path.isdir(logs_dir): @@ -1657,25 +1665,17 @@ def setup_core_root(host_os, wrn_log = log_path + ".wrn" err_log = log_path + ".err" - msbuild_log_params = "/fileloggerparameters:\"Verbosity=normal;LogFile=%s\"" % build_log - msbuild_wrn_params = "/fileloggerparameters1:\"WarningsOnly;LogFile=%s\"" % wrn_log - msbuild_err_params = "/fileloggerparameters2:\"ErrorsOnly;LogFile=%s\"" % err_log - - command += ["-MsBuildLog=%s" % msbuild_log_params, - "-MsBuildWrn=%s" % msbuild_wrn_params, - "-MsBuildErr=%s" % msbuild_err_params] - - if host_os != "Windows_NT": - command = ["bash"] + command - command += ["-MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\""] + command += ["/fileloggerparameters:\"Verbosity=normal;LogFile=%s\"" % build_log, + "/fileloggerparameters1:\"WarningsOnly;LogFile=%s\"" % wrn_log, + "/fileloggerparameters2:\"ErrorsOnly;LogFile=%s\"" % err_log] if g_verbose: - command += ["-verbose"] + command += ["/v:detailed"] - command += [ "-testOverlay", - "-BuildType=%s" % build_type, - "-BuildArch=%s" % arch, - "-BuildOS=%s" % host_os] + command += ["/t:CreateTestOverlay", + "/p:__BuildType=%s" % build_type, + "/p:__BuildArch=%s" % arch, + "/p:__BuildOS=%s" % host_os] print("") print("Creating Core_Root...") @@ -1693,7 +1693,7 @@ def setup_core_root(host_os, proc.kill() sys.exit(1) - if proc.returncode == 1: + if proc.returncode != 0: print("Error: creating Core_Root failed.") return False @@ -1751,7 +1751,7 @@ def setup_core_root(host_os, proc = subprocess.Popen(msbuild_command) proc.communicate() - if not proc.returncode == 0: + if proc.returncode != 0: print("Error: generating test host failed.") return False @@ -1766,7 +1766,7 @@ def setup_core_root(host_os, proc = subprocess.Popen(msbuild_command) proc.communicate() - if proc.returncode == 1: + if proc.returncode != 0: print("Error: msbuild failed.") return False @@ -1789,7 +1789,7 @@ def setup_core_root(host_os, proc = subprocess.Popen(msbuild_command) proc.communicate() - if proc.returncode == 1: + if proc.returncode != 0: print("Error: msbuild failed.") return False @@ -1929,7 +1929,7 @@ def build_test_wrappers(host_os, proc.kill() sys.exit(1) - if proc.returncode == 1: + if proc.returncode != 0: print("Error: creating test wrappers failed.") return False diff --git a/tests/src/dir.props b/tests/src/dir.props index b56c686d1d..ce2d982efa 100644 --- a/tests/src/dir.props +++ b/tests/src/dir.props @@ -1,7 +1,12 @@ + + + - @@ -31,7 +36,7 @@ 32 - + $(ProjectDir)..\bin\tests $(__TestRootDir) -- cgit v1.2.3