Split unix test builds in slices (#17785)

* Split unix test builds in slices

Ports #17161 to linux

* Address review feedback

1 files changed, 70 insertions, 17 deletions

diff --git a/build-test.sh b/build-test.sh
index 77b98e76d7..5339dc1116 100755
--- a/build-test.sh
+++ b/build-test.sh
@@ -295,28 +295,75 @@ build_Tests_internal()
     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
-    __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
-    __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
-    __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 
-    # Generate build command
-    buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
+    if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
+        # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
+        # See https://github.com/Microsoft/msbuild/issues/2993
 
-    echo "Building step '$stepName' via $buildCommand"
+        # __SkipPackageRestore and __SkipTargetingPackBuild used  to control build by tests/src/dirs.proj
+        export __SkipPackageRestore=false
+        export __SkipTargetingPackBuild=false
+        export __BuildLoopCount=2
+        export __TestGroupToBuild=1
+        __AppendToLog=false
 
-    # Invoke MSBuild
-    eval $buildCommand
+        if [ -n __priority1 ]; then
+            export __BuildLoopCount=16
+            export __TestGroupToBuild=2
+        fi
 
-    # Invoke MSBuild
-    # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
+        for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 ))
+        do
+            __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
+            __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
+            __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
 
-    # Make sure everything is OK
-    if [ $? -ne 0 ]; then
-        echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
-        echo "    $__BuildLog"
-        echo "    $__BuildWrn"
-        echo "    $__BuildErr"
-        exit 1
+            export TestBuildSlice=$slice
+
+            # Generate build command
+            buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
+
+            echo "Building step '$stepName' slice=$slice via $buildCommand"
+
+            # Invoke MSBuild
+            eval $buildCommand
+
+            # Make sure everything is OK
+            if [ $? -ne 0 ]; then
+                echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
+                echo "    $__BuildLog"
+                echo "    $__BuildWrn"
+                echo "    $__BuildErr"
+                exit 1
+            fi
+            export __SkipPackageRestore=true
+            export __SkipTargetingPackBuild=true
+            __AppendToLog=true
+        done
+    else
+        __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
+        __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
+        __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
+
+        # Generate build command
+        buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
+
+        echo "Building step '$stepName' via $buildCommand"
+
+        # Invoke MSBuild
+        eval $buildCommand
+
+        # Invoke MSBuild
+        # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
+
+        # Make sure everything is OK
+        if [ $? -ne 0 ]; then
+            echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
+            echo "    $__BuildLog"
+            echo "    $__BuildWrn"
+            echo "    $__BuildErr"
+            exit 1
+        fi
     fi
 }
 
@@ -342,6 +389,7 @@ usage()
     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
+    echo "priority1 - include priority=1 tests in the build"
     exit 1
 }
 
@@ -461,6 +509,7 @@ __RunTests=0
 __RebuildTests=0
 __BuildTestWrappers=0
 __GenerateLayoutOnly=
+__priority1=
 CORE_ROOT=
 
 while :; do
@@ -615,6 +664,10 @@ while :; do
         msbuildonunsupportedplatform)
             __msbuildonunsupportedplatform=1
             ;;
+        priority1)
+            __priority1=1
+            __UnprocessedBuildArgs="$__UnprocessedBuildArgs -priority=1"
+            ;;
         *)
             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
             ;;