From 8568bf1d9722657c602a1f1fe0148e28a93ea8ce Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Wed, 27 Apr 2016 14:35:45 -0700 Subject: Add dev workflow scripts for build pipeline --- BuildToolsVersion.txt | 2 +- build-packages.cmd | 47 ++++++++++++++++ build-packages.sh | 148 ++++++++++++++++++++++++++++++++++++++++++++++++++ build.cmd | 11 ++++ build.proj | 8 ++- build.sh | 17 +++++- clean.cmd | 24 ++++++++ clean.sh | 12 ++++ publish-packages.cmd | 35 ++++++++++++ publish-packages.sh | 77 ++++++++++++++++++++++++++ src/publish.proj | 20 +++++++ sync.cmd | 84 ++++++++++++++++++++++++++++ sync.sh | 76 ++++++++++++++++++++++++++ 13 files changed, 557 insertions(+), 4 deletions(-) create mode 100644 build-packages.cmd create mode 100755 build-packages.sh create mode 100644 clean.cmd create mode 100755 clean.sh create mode 100644 publish-packages.cmd create mode 100755 publish-packages.sh create mode 100644 src/publish.proj create mode 100644 sync.cmd create mode 100755 sync.sh diff --git a/BuildToolsVersion.txt b/BuildToolsVersion.txt index 407d4c968a..184804184a 100644 --- a/BuildToolsVersion.txt +++ b/BuildToolsVersion.txt @@ -1 +1 @@ -1.0.25-prerelease-00327-01 +1.0.25-prerelease-00416-04 \ No newline at end of file diff --git a/build-packages.cmd b/build-packages.cmd new file mode 100644 index 0000000000..0ae96fe911 --- /dev/null +++ b/build-packages.cmd @@ -0,0 +1,47 @@ +@if "%_echo%" neq "on" echo off +setlocal EnableDelayedExpansion + +set "__ProjectDir=%~dp0" +set packagesLog=build-packages.log +set binclashLoggerDll=%~dp0Tools\net45\Microsoft.DotNet.Build.Tasks.dll +set binclashlog=%~dp0binclash.log +echo Running build-packages.cmd %* > %packagesLog% + +set options=/nologo /maxcpucount /v:minimal /clp:Summary /nodeReuse:false /flp:v=detailed;Append;LogFile=%packagesLog% /l:BinClashLogger,%binclashLoggerDll%;LogFile=%binclashlog% /p:FilterToOSGroup=Windows_NT +set allargs=%* + +if /I [%1] == [/?] goto Usage +if /I [%1] == [/help] goto Usage + +REM ensure that msbuild is available +echo Running init-tools.cmd +call %~dp0init-tools.cmd + +set __msbuildArgs="%__ProjectDir%\src\.nuget\Microsoft.NETCore.Runtime.CoreClr\Microsoft.NETCore.Runtime.CoreCLR.builds" !allargs! +echo msbuild.exe %__msbuildArgs% !options! >> %packagesLog% +call msbuild.exe %__msbuildArgs% !options! +if NOT [!ERRORLEVEL!]==[0] ( + echo ERROR: An error occurred while building packages, see %packagesLog% for more details. + exit /b 1 +) + +set __msbuildArgs="%__ProjectDir%\src\.nuget\Microsoft.NETCore.Jit\Microsoft.NETCore.Jit.builds" !allargs! +echo msbuild.exe %__msbuildArgs% !options! >> %packagesLog% +call msbuild.exe %__msbuildArgs% !options! +if NOT [!ERRORLEVEL!]==[0] ( + echo ERROR: An error occurred while building packages, see %packagesLog% for more details. + exit /b 1 +) + + +echo Done Building Packages. +exit /b + +:Usage +echo. +echo Builds the NuGet packages from the binaries that were built in the Build product binaries step. +echo The following properties are required to define build architecture +echo /p:__BuildArch=[architecture] /p:__BuildType=[configuration] +echo Architecture can be x64, x86, arm, or arm64 +echo Configuration can be Release, Debug, or Checked +exit /b \ No newline at end of file diff --git a/build-packages.sh b/build-packages.sh new file mode 100755 index 0000000000..471b9ef819 --- /dev/null +++ b/build-packages.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash + +usage() +{ + echo "Builds the NuGet packages from the binaries that were built in the Build product binaries step." + echo "Usage: build-packages [arch] [configuration]" + echo "arch can be x64, x86, arm, arm64 (default is x64)" + echo "configuration can be release, checked, debug (default is debug)" + echo + exit 1 +} + +__ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +build_packages_log=$__ProjectRoot/build-packages.log +binclashlog=$__ProjectRoot/binclash.log +binclashloggerdll=$__ProjectRoot/Tools/Microsoft.DotNet.Build.Tasks.dll +RuntimeOS=ubuntu.$VERSION_ID + +__MSBuildPath=$__ProjectRoot/Tools/MSBuild.exe + +# Parse arguments +__BuildArch=x64 +__BuildType=Debug + +allargs="$@" + +echo -e "Running build-packages.sh $allargs" > $build_packages_log + +if [ "$allargs" == "-h" ] || [ "$allargs" == "--help" ]; then + usage +fi + +while :; do + if [ $# -le 0 ]; then + break + fi + + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + -\?|-h|--help) + usage + exit 1 + ;; + + x86) + __BuildArch=x86 + ;; + + x64) + __BuildArch=x64 + ;; + + arm) + __BuildArch=arm + ;; + + arm64) + __BuildArch=arm64 + ;; + debug) + __BuildType=Debug + ;; + release) + __BuildType=Release + ;; + checked) + __BuildType=Checked + esac + shift +done + +# Use uname to determine what the OS is. +OSName=$(uname -s) +case $OSName in + Linux) + __BuildOS=Linux + ;; + + Darwin) + __BuildOS=OSX + ;; + + FreeBSD) + __BuildOS=FreeBSD + ;; + + OpenBSD) + __BuildOS=OpenBSD + ;; + + NetBSD) + __BuildOS=NetBSD + ;; + + SunOS) + __BuildOS=SunOS + ;; + + *) + echo "Unsupported OS $OSName detected, configuring as if for Linux" + __BuildOS=Linux + ;; +esac + +if [ "$__BuildOS" == "Linux" ]; then + # Detect Distro + if [ "$(cat /etc/*-release | grep -cim1 ubuntu)" -eq 1 ]; then + export __DistroName=ubuntu + elif [ "$(cat /etc/*-release | grep -cim1 centos)" -eq 1 ]; then + export __DistroName=rhel + elif [ "$(cat /etc/*-release | grep -cim1 rhel)" -eq 1 ]; then + export __DistroName=rhel + elif [ "$(cat /etc/*-release | grep -cim1 debian)" -eq 1 ]; then + export __DistroName=debian + else + export __DistroName="" + fi +fi + +__IntermediatesDir="$__ProjectRoot/bin/obj/$__BuildOS.$__BuildArch.$__BuildType" + +# Ensure that MSBuild is available +echo "Running init-tools.sh" +$__ProjectRoot/init-tools.sh + + echo "Generating nuget packages for "$__BuildOS + + # Invoke MSBuild + $__ProjectRoot/Tools/corerun "$__MSBuildPath" /nologo "$__ProjectRoot/src/.nuget/Microsoft.NETCore.Runtime.CoreCLR/Microsoft.NETCore.Runtime.CoreCLR.builds" /verbosity:minimal "/fileloggerparameters:Verbosity=normal;LogFile=$binclashlog" /t:Build /p:__BuildOS=$__BuildOS /p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__IntermediatesDir=$__IntermediatesDir /p:BuildNugetPackage=false /p:UseSharedCompilation=false + +if [ $? -ne 0 ]; then + echo -e "\nAn error occurred. Aborting build-packages.sh ." >> $build_packages_log + echo "ERROR: An error occurred while building packages, see $build_packages_log for more details." + exit 1 +fi + + # Build the JIT packages + $__ProjectRoot/Tools/corerun "$__MSBuildPath" /nologo "$__ProjectRoot/src/.nuget/Microsoft.NETCore.Jit/Microsoft.NETCore.Jit.builds" /verbosity:minimal "/fileloggerparameters:Verbosity=normal;LogFile=$binclashlog" /t:Build /p:__BuildOS=$__BuildOS /p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__IntermediatesDir=$__IntermediatesDir /p:BuildNugetPackage=false /p:UseSharedCompilation=false + +if [ $? -ne 0 ]; then + echo -e "\nAn error occurred. Aborting build-packages.sh ." >> $build_packages_log + echo "ERROR: An error occurred while building packages, see $build_packages_log for more details." + exit 1 +fi + +echo "Done building packages." +echo -e "\nDone building packages." >> $build_packages_log +exit 0 \ No newline at end of file diff --git a/build.cmd b/build.cmd index d8c371a0ef..0e9cddca4d 100644 --- a/build.cmd +++ b/build.cmd @@ -52,6 +52,8 @@ set __SkipCoreLibBuild= set __SkipNativeBuild= set __SkipTestBuild= set __BuildSequential= +set __SkipRestore= +set __SkipNuget= set __msbuildCleanBuildArgs= set __msbuildExtraArgs= set __SignTypeReal= @@ -113,6 +115,8 @@ if /i "%1" == "skipconfigure" (set __SkipConfigure=1&shift&goto Arg_Loop) if /i "%1" == "skipmscorlib" (set __SkipCoreLibBuild=1&shift&goto Arg_Loop) if /i "%1" == "skipnative" (set __SkipNativeBuild=1&shift&goto Arg_Loop) if /i "%1" == "skiptests" (set __SkipTestBuild=1&shift&goto Arg_Loop) +if /i "%1" == "skiprestore" (set __SkipRestore=1&shift&goto Arg_Loop) +if /i "%1" == "skipnuget" (set __SkipNuget=1&shift&goto Arg_Loop) if /i "%1" == "sequential" (set __BuildSequential=1&shift&goto Arg_Loop) if /i "%1" == "disableoss" (set __SignTypeReal="/p:SignType=real"&shift&goto Arg_Loop) if /i "%1" == "priority" (set __TestPriority=%2&set __PassThroughArgs=%__PassThroughArgs% %2&shift&shift&goto Arg_Loop) @@ -274,6 +278,10 @@ set __msbuildCommonArgs=/nologo /nodeReuse:false %__msbuildCleanBuildArgs% %__ms if not defined __BuildSequential ( set __msbuildCommonArgs=%__msbuildCommonArgs% /maxcpucount ) +if defined __SkipRestore ( + set __msbuildCommonArgs=%__msbuildCommonArgs% /p:RestoreDuringBuild=false +) + REM ========================================================================================= REM === @@ -471,6 +479,7 @@ if NOT errorlevel 0 ( :GenerateNuget if /i "%__BuildArch%" =="arm64" goto :SkipNuget +if /i "%__SkipNuget%" == 1 goto :SkipNuget set "__BuildLog=%__LogsDir%\Nuget_%__BuildOS%__%__BuildArch%__%__BuildType%.log" set "__BuildWrn=%__LogsDir%\Nuget_%__BuildOS%__%__BuildArch%__%__BuildType%.wrn" @@ -679,6 +688,8 @@ echo skipconfigure: skip CMake ^(default: CMake is run^) echo skipmscorlib: skip building System.Private.CoreLib ^(default: System.Private.CoreLib is built^). echo skipnative: skip building native components ^(default: native components are built^). echo skiptests: skip building tests ^(default: tests are built^). +echo skiprestore: skip restoring packages ^(default: packages are restored during build^). +echo skipnuget: skip building nuget packages ^(default: packages are built^). echo disableoss: Disable Open Source Signing for System.Private.CoreLib. echo toolset_dir ^ : set the toolset directory -- Arm64 use only. Required for Arm64 builds. echo. diff --git a/build.proj b/build.proj index 42c5711b39..f3a9d40537 100644 --- a/build.proj +++ b/build.proj @@ -11,13 +11,19 @@ + + + + true + + - + diff --git a/build.sh b/build.sh index 3cc3ec803b..9dcc742e7c 100755 --- a/build.sh +++ b/build.sh @@ -19,6 +19,8 @@ usage() echo "skipnative - do not build native components." echo "skipmscorlib - do not build mscorlib.dll." echo "skiptests - skip the tests in the 'tests' subdirectory." + echo "skiprestore - skip restoring nuget packages." + echo "skipnuget - skip building nuget packages." echo "disableoss - Disable Open Source Signing for mscorlib." echo "skipgenerateversion - disable version generation even if MSBuild is supported." echo "cmakeargs - user-settable additional arguments passed to CMake." @@ -439,6 +441,8 @@ __MSBCleanBuildArgs= __UseNinja=0 __ConfigureOnly=0 __SkipConfigure=0 +__SkipRestore="" +__SkipNuget=0 __SkipCoreCLR=0 __SkipMSCorLib=0 __CleanBuild=0 @@ -574,6 +578,14 @@ while :; do __IncludeTests= ;; + skiprestore) + __SkipRestore="/p:RestoreDuringBuild=true" + ;; + + skipnuget) + __SkipNuget=1 + ;; + disableoss) __SignTypeReal="/p:SignType=real" ;; @@ -688,8 +700,9 @@ build_coreclr build_CoreLib # Generate nuget packages - -generate_NugetPackages +if [ $__SkipNuget != 1 ]; then + generate_NugetPackages +fi # Build complete diff --git a/clean.cmd b/clean.cmd new file mode 100644 index 0000000000..9dbeecf572 --- /dev/null +++ b/clean.cmd @@ -0,0 +1,24 @@ +@if not defined __echo @echo off +setlocal EnableDelayedExpansion + +echo Running clean.cmd + +if /I [%1] == [/?] goto Usage +if /I [%1] == [/help] goto Usage + +:: Set __ProjectDir to be the directory of this script +set "__ProjectDir=%~dp0" +:: remove trailing slash +if %__ProjectDir:~-1%==\ set "__ProjectDir=%__ProjectDir:~0,-1%" +set "__RootBinDir=%__ProjectDir%\bin" + +if exist "%__RootBinDir%" rd /s /q "%__RootBinDir%" +if exist "%__ProjectDir%\Tools" rd /s /q "%__ProjectDir%\Tools" + +exit /b 0 + +:Usage +echo. +echo Repository cleaning script. +echo No option parameters. +exit /b \ No newline at end of file diff --git a/clean.sh b/clean.sh new file mode 100755 index 0000000000..ade2f122f6 --- /dev/null +++ b/clean.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Obtain the location of the bash script to figure out where the root of the repo is. +__ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +echo Cleaning previous output for the selected configuration + +rm -rf "$__ProjectRoot/bin" + +rm -rf "$__ProjectRoot/Tools" + +exit 0 \ No newline at end of file diff --git a/publish-packages.cmd b/publish-packages.cmd new file mode 100644 index 0000000000..4179454327 --- /dev/null +++ b/publish-packages.cmd @@ -0,0 +1,35 @@ +@if "%_echo%" neq "on" echo off +setlocal EnableDelayedExpansion + +set packagesLog=publish-packages.log +echo Running publish-packages.cmd %* > %packagesLog% + +set options=/nologo /v:minimal /flp:v=detailed;Append;LogFile=%packagesLog% +set allargs=%* + +if /I [%1] == [/?] goto Usage +if /I [%1] == [/help] goto Usage + +REM ensure that msbuild is available +echo Running init-tools.cmd +call %~dp0init-tools.cmd + +echo msbuild.exe %~dp0src\publish.proj !options! !allargs! >> %packagesLog% +call msbuild.exe %~dp0src\publish.proj !options! !allargs! +if NOT [%ERRORLEVEL%]==[0] ( + echo ERROR: An error occurred while publishing packages, see %packagesLog% for more details. + exit /b 1 +) + +echo Done publishing packages. +exit /b + +:Usage +echo. +echo Publishes the NuGet packages to the specified location. +echo For publishing to Azure the following properties are required. +echo /p:CloudDropAccountName="account name" +echo /p:CloudDropAccessToken="access token" +echo /p:__BuildType="Configuration Group" +echo /p:__BuildArch="Architecture" +exit /b \ No newline at end of file diff --git a/publish-packages.sh b/publish-packages.sh new file mode 100755 index 0000000000..5c495e944c --- /dev/null +++ b/publish-packages.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +usage() +{ + echo "Publishes the NuGet packages to the specified location." + echo "For publishing to Azure the following properties are required." + echo " /p:CloudDropAccountName=\"account name\"" + echo " /p:CloudDropAccessToken=\"access token\"" + echo " /p:__BuildType=\"Configuration\"" + echo "Configuration can be Release, Checked, or Debug" + exit 1 +} + +working_tree_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +publish_log=$working_tree_root/publish.log + +while [[ $# > 0 ]] +do + unprocessedBuildArgs="$unprocessedBuildArgs $1" + shift +done + +# Use uname to determine what the OS is. +OSName=$(uname -s) +case $OSName in + Linux) + __BuildOS=Linux + ;; + + Darwin) + __BuildOS=OSX + ;; + + FreeBSD) + __BuildOS=FreeBSD + ;; + + OpenBSD) + __BuildOS=OpenBSD + ;; + + NetBSD) + __BuildOS=NetBSD + ;; + + SunOS) + __BuildOS=SunOS + ;; + + *) + echo "Unsupported OS $OSName detected, configuring as if for Linux" + __BuildOS=Linux + ;; +esac + +echo $unprocessedBuildArgs + +options="/nologo /v:minimal /flp:v=detailed;Append;LogFile=$publish_log" + +echo "Running publish-packages.sh $*" > $publish_log + +echo "Running init-tools.sh" +$working_tree_root/init-tools.sh + +echo "Restoring all packages..." +echo -e "\n$working_tree_root/Tools/corerun $working_tree_root/Tools/MSBuild.exe $working_tree_root/src/publish.proj $options $unprocessedBuildArgs" /p:__BuildOS=$__BuildOS >> $publish_log +$working_tree_root/Tools/corerun $working_tree_root/Tools/MSBuild.exe $working_tree_root/src/publish.proj $options $unprocessedBuildArgs /p:__BuildOS=$__BuildOS +if [ $? -ne 0 ] +then + echo -e "\nPackage publishing failed. Aborting." >> $publish_log + echo "ERROR: An error occurred while publishing packages; see $publish_log for more details. There may have been networking problems, so please try again in a few minutes." + exit 1 +fi + +echo "Publish completed successfully." +echo -e "\nPublish completed successfully." >> $publish_log +exit 0 \ No newline at end of file diff --git a/src/publish.proj b/src/publish.proj new file mode 100644 index 0000000000..5dad58c38b --- /dev/null +++ b/src/publish.proj @@ -0,0 +1,20 @@ + + + + + + + + $(PackageOutputPath)\*.nupkg;$(SymbolPackageOutputPath)\*.nupkg + + + + + coreclr-$(PreReleaseLabel)-$(BuildNumberMajor)-$(BuildNumberMinor) + + + + + \ No newline at end of file diff --git a/sync.cmd b/sync.cmd new file mode 100644 index 0000000000..70bde8ce87 --- /dev/null +++ b/sync.cmd @@ -0,0 +1,84 @@ +@if "%_echo%" neq "on" echo off +setlocal EnableDelayedExpansion + +set synclog=sync.log +echo Running Sync.cmd %* > %synclog% + +set options=/nologo /v:minimal /clp:Summary /flp:v=detailed;Append;LogFile=%synclog% +set unprocessedBuildArgs= +set allargs=%* +set thisArgs= + +set src=false +set packages=false + +if [%1]==[] ( + set src=true + set packages=true + goto Begin +) + +:Loop +if [%1]==[] goto Begin + +if /I [%1] == [/?] goto Usage +if /I [%1] == [/help] goto Usage + +if /I [%1] == [/p] ( + set packages=true + set thisArgs=!thisArgs!%1 + goto Next +) + +if /I [%1] == [/s] ( + set src=true + set thisArgs=!thisArgs!%1 + goto Next +) + +set unprocessedBuildArgs=!unprocessedBuildArgs! %1 + +:Next +shift /1 +goto Loop + +:Begin +echo Running init-tools.cmd +call %~dp0init-tools.cmd + +if [%src%] == [true] ( + echo Fetching git database from remote repos ... + call git fetch --all -p -v >> %synclog% 2>&1 + if NOT [!ERRORLEVEL!]==[0] ( + echo ERROR: An error occurred while fetching remote source code, see %synclog% for more details. + exit /b 1 + ) +) + +set targets=RestoreNETCorePlatforms + +if [%packages%] == [true] ( + set options=!options! /t:!targets! /p:RestoreDuringBuild=true + echo msbuild.exe %~dp0build.proj !options! !unprocessedBuildArgs! >> %synclog% + call msbuild.exe %~dp0build.proj !options! !unprocessedBuildArgs! + if NOT [!ERRORLEVEL!]==[0] ( + echo ERROR: An error occurred while syncing packages, see %synclog% for more details. There may have been networking problems so please try again in a few minutes. + exit /b 1 + ) +) + +echo Done Syncing. +exit /b 0 + +goto :EOF + +:Usage +echo. +echo Repository syncing script. +echo. +echo Options: +echo /s - Fetches source history from all configured remotes +echo (git fetch --all -p -v) +echo /p - Restores all nuget packages for repository +echo. +echo If no option is specified then sync.cmd /s /p is implied. \ No newline at end of file diff --git a/sync.sh b/sync.sh new file mode 100755 index 0000000000..e1b6401fc1 --- /dev/null +++ b/sync.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +usage() +{ + echo "Usage: sync [-p] [-s]" + echo "Repository syncing script." + echo " -s Fetch source history from all configured remotes" + echo " (git fetch --all -p -v)" + echo " -p Restore all NuGet packages for the repository" + echo + echo "If no option is specified, then \"sync.sh -p -s\" is implied." + exit 1 +} + +working_tree_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +sync_log=$working_tree_root/sync.log + +options="/nologo /v:minimal /clp:Summary /flp:v=detailed;Append;LogFile=$sync_log" +unprocessedBuildArgs= + +echo "Running sync.sh $*" > $sync_log + +# Parse arguments +if [ $# == 0 ]; then + sync_packages=true + sync_src=true +fi + +while [[ $# > 0 ]] +do + opt="$1" + case $opt in + -h|--help) + usage + ;; + -p) + sync_packages=true + ;; + -s) + sync_src=true + ;; + *) + unprocessedBuildArgs="$unprocessedBuildArgs $1" + esac + shift +done + +echo "Running init-tools.sh" +$working_tree_root/init-tools.sh + +if [ "$sync_src" == true ]; then + echo "Fetching git database from remote repos..." + git fetch --all -p -v >> $sync_log 2>&1 + if [ $? -ne 0 ]; then + echo -e "\ngit fetch failed. Aborting sync." >> $sync_log + echo "ERROR: An error occurred while fetching remote source code; see $sync_log for more details." + exit 1 + fi +fi + +if [ "$sync_packages" == true ]; then + options="$options /t:RestoreNETCorePlatforms /p:RestoreDuringBuild=true" + echo "Restoring all packages..." + echo -e "\n$working_tree_root/Tools/corerun $working_tree_root/Tools/MSBuild.exe $working_tree_root/build.proj $options $unprocessedBuildArgs" >> $sync_log + $working_tree_root/Tools/corerun $working_tree_root/Tools/MSBuild.exe $working_tree_root/build.proj $options $unprocessedBuildArgs + if [ $? -ne 0 ] + then + echo -e "\nPackage restored failed. Aborting sync." >> $sync_log + echo "ERROR: An error occurred while syncing packages; see $sync_log for more details. There may have been networking problems, so please try again in a few minutes." + exit 1 + fi +fi + +echo "Sync completed successfully." +echo -e "\nSync completed successfully." >> $sync_log +exit 0 \ No newline at end of file -- cgit v1.2.3