From ad0f22cd2f011e7112588415b13d615238b5acb4 Mon Sep 17 00:00:00 2001 From: Steve MacLean Date: Fri, 4 May 2018 15:53:44 -0400 Subject: Split unix test builds in slices (#17785) * Split unix test builds in slices Ports #17161 to linux * Address review feedback --- build-test.sh | 87 +++++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 70 insertions(+), 17 deletions(-) (limited to 'build-test.sh') diff --git a/build-test.sh b/build-test.sh index 77b98e76d7..5339dc1116 100755 --- a/build-test.sh +++ b/build-test.sh @@ -295,28 +295,75 @@ build_Tests_internal() __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log" __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn" __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err" - __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\"" - __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\"" - __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\"" - # Generate build command - buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs" + if [[ "$subDirectoryName" == "Tests_Managed" ]]; then + # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache + # See https://github.com/Microsoft/msbuild/issues/2993 - echo "Building step '$stepName' via $buildCommand" + # __SkipPackageRestore and __SkipTargetingPackBuild used to control build by tests/src/dirs.proj + export __SkipPackageRestore=false + export __SkipTargetingPackBuild=false + export __BuildLoopCount=2 + export __TestGroupToBuild=1 + __AppendToLog=false - # Invoke MSBuild - eval $buildCommand + if [ -n __priority1 ]; then + export __BuildLoopCount=16 + export __TestGroupToBuild=2 + fi - # Invoke MSBuild - # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs + for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 )) + do + __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\"" + __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\"" + __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\"" - # Make sure everything is OK - if [ $? -ne 0 ]; then - echo "${__MsgPrefix}Failed to build $stepName. See the build logs:" - echo " $__BuildLog" - echo " $__BuildWrn" - echo " $__BuildErr" - exit 1 + export TestBuildSlice=$slice + + # Generate build command + buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs" + + echo "Building step '$stepName' slice=$slice via $buildCommand" + + # Invoke MSBuild + eval $buildCommand + + # Make sure everything is OK + if [ $? -ne 0 ]; then + echo "${__MsgPrefix}Failed to build $stepName. See the build logs:" + echo " $__BuildLog" + echo " $__BuildWrn" + echo " $__BuildErr" + exit 1 + fi + export __SkipPackageRestore=true + export __SkipTargetingPackBuild=true + __AppendToLog=true + done + else + __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\"" + __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\"" + __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\"" + + # Generate build command + buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs" + + echo "Building step '$stepName' via $buildCommand" + + # Invoke MSBuild + eval $buildCommand + + # Invoke MSBuild + # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs + + # Make sure everything is OK + if [ $? -ne 0 ]; then + echo "${__MsgPrefix}Failed to build $stepName. See the build logs:" + echo " $__BuildLog" + echo " $__BuildWrn" + echo " $__BuildErr" + exit 1 + fi fi } @@ -342,6 +389,7 @@ usage() echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run" echo "bindir - output directory (defaults to $__ProjectRoot/bin)" echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported." + echo "priority1 - include priority=1 tests in the build" exit 1 } @@ -461,6 +509,7 @@ __RunTests=0 __RebuildTests=0 __BuildTestWrappers=0 __GenerateLayoutOnly= +__priority1= CORE_ROOT= while :; do @@ -615,6 +664,10 @@ while :; do msbuildonunsupportedplatform) __msbuildonunsupportedplatform=1 ;; + priority1) + __priority1=1 + __UnprocessedBuildArgs="$__UnprocessedBuildArgs -priority=1" + ;; *) __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" ;; -- cgit v1.2.3