diff options
author | Steve MacLean <sdmaclea.qdt@qualcommdatacenter.com> | 2018-05-04 15:53:44 -0400 |
---|---|---|
committer | jashook <jashoo@microsoft.com> | 2018-05-11 10:10:52 -0700 |
commit | d0d7997f964b3e21ee4ed78d6f2dd2da0cf94ed6 (patch) | |
tree | d7e0513a8712388f4e87458e0b92d30b47f09b6d | |
parent | 6db42177ecabd5e81a5d3d4b0b51f7730e5fd782 (diff) | |
download | coreclr-d0d7997f964b3e21ee4ed78d6f2dd2da0cf94ed6.tar.gz coreclr-d0d7997f964b3e21ee4ed78d6f2dd2da0cf94ed6.tar.bz2 coreclr-d0d7997f964b3e21ee4ed78d6f2dd2da0cf94ed6.zip |
Split unix test builds in slices (#17785)
* Split unix test builds in slices
Ports #17161 to linux
* Address review feedback
-rwxr-xr-x | build-test.sh | 87 | ||||
-rw-r--r-- | tests/src/dirs.proj | 46 |
2 files changed, 88 insertions, 45 deletions
diff --git a/build-test.sh b/build-test.sh index 77b98e76d7..5339dc1116 100755 --- a/build-test.sh +++ b/build-test.sh @@ -295,28 +295,75 @@ build_Tests_internal() __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log" __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn" __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err" - __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\"" - __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\"" - __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\"" - # Generate build command - buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs" + if [[ "$subDirectoryName" == "Tests_Managed" ]]; then + # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache + # See https://github.com/Microsoft/msbuild/issues/2993 - echo "Building step '$stepName' via $buildCommand" + # __SkipPackageRestore and __SkipTargetingPackBuild used to control build by tests/src/dirs.proj + export __SkipPackageRestore=false + export __SkipTargetingPackBuild=false + export __BuildLoopCount=2 + export __TestGroupToBuild=1 + __AppendToLog=false - # Invoke MSBuild - eval $buildCommand + if [ -n __priority1 ]; then + export __BuildLoopCount=16 + export __TestGroupToBuild=2 + fi - # Invoke MSBuild - # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs + for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 )) + do + __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\"" + __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\"" + __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\"" - # Make sure everything is OK - if [ $? -ne 0 ]; then - echo "${__MsgPrefix}Failed to build $stepName. See the build logs:" - echo " $__BuildLog" - echo " $__BuildWrn" - echo " $__BuildErr" - exit 1 + export TestBuildSlice=$slice + + # Generate build command + buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs" + + echo "Building step '$stepName' slice=$slice via $buildCommand" + + # Invoke MSBuild + eval $buildCommand + + # Make sure everything is OK + if [ $? -ne 0 ]; then + echo "${__MsgPrefix}Failed to build $stepName. See the build logs:" + echo " $__BuildLog" + echo " $__BuildWrn" + echo " $__BuildErr" + exit 1 + fi + export __SkipPackageRestore=true + export __SkipTargetingPackBuild=true + __AppendToLog=true + done + else + __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\"" + __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\"" + __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\"" + + # Generate build command + buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs" + + echo "Building step '$stepName' via $buildCommand" + + # Invoke MSBuild + eval $buildCommand + + # Invoke MSBuild + # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs + + # Make sure everything is OK + if [ $? -ne 0 ]; then + echo "${__MsgPrefix}Failed to build $stepName. See the build logs:" + echo " $__BuildLog" + echo " $__BuildWrn" + echo " $__BuildErr" + exit 1 + fi fi } @@ -342,6 +389,7 @@ usage() echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run" echo "bindir - output directory (defaults to $__ProjectRoot/bin)" echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported." + echo "priority1 - include priority=1 tests in the build" exit 1 } @@ -461,6 +509,7 @@ __RunTests=0 __RebuildTests=0 __BuildTestWrappers=0 __GenerateLayoutOnly= +__priority1= CORE_ROOT= while :; do @@ -615,6 +664,10 @@ while :; do msbuildonunsupportedplatform) __msbuildonunsupportedplatform=1 ;; + priority1) + __priority1=1 + __UnprocessedBuildArgs="$__UnprocessedBuildArgs -priority=1" + ;; *) __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" ;; diff --git a/tests/src/dirs.proj b/tests/src/dirs.proj index f7d1aadea9..6009e2ed08 100644 --- a/tests/src/dirs.proj +++ b/tests/src/dirs.proj @@ -33,26 +33,16 @@ <DisabledProjects Include="Loader\classloader\generics\GenericMethods\VSW491668.csproj" /> <!-- issue 5501 --> </ItemGroup> - <!-- Unix builds do not support subgroups --> - <ItemGroup Condition="$(__BuildOS) != 'Windows_NT'" > - <Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)"> - <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> - </Project> - <Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)"> - <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> - </Project> - </ItemGroup> - <!-- Test build is divided in slices which can be created within Test Group Priority 0 tests are build using Test Group 1 with 2 subgroups or slices --> - <ItemGroup Condition="$(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'"> + <ItemGroup Condition="$(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'"> <Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> </ItemGroup> - <ItemGroup Condition="$(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '2'"> + <ItemGroup Condition="$(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '2'"> <Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> @@ -61,7 +51,7 @@ <!-- Test build is divided in slices which can be created within Test Group Priority 1 or higher tests are build using Test Group 2 with 16 subgroups or slices --> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '1')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '1')"> <Project Include="baseservices\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> @@ -76,7 +66,7 @@ </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '2')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '2')"> <Project Include="CoreMangLib\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> @@ -85,7 +75,7 @@ </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '3')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '3')"> <Project Include="E*\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> @@ -112,7 +102,7 @@ </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '4')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '4')"> <Project Include="JIT\B*\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> @@ -124,7 +114,7 @@ </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '5')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '5')"> <Project Include="JIT\B*\**\*.ilproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> @@ -136,7 +126,7 @@ </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '6')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '6')"> <Project Include="JIT\Generics\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> @@ -151,7 +141,7 @@ </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '7')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '7')"> <Project Include="JIT\IL_Conformance\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> @@ -160,7 +150,7 @@ </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '8')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '8')"> <Project Include="JIT\jit64\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> @@ -169,19 +159,19 @@ </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '9')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '9')"> <Project Include="JIT\Methodical\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '10')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '10')"> <Project Include="JIT\Methodical\**\*.ilproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '11')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '11')"> <Project Include="JIT\opt\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> @@ -202,31 +192,31 @@ </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '12')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '12')"> <Project Include="JIT\R*\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '13')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '13')"> <Project Include="JIT\R*\**\*.ilproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '14')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '14')"> <Project Include="Loader\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '15')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '15')"> <Project Include="Loader\**\*.ilproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> </ItemGroup> - <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '16')"> + <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '16')"> <Project Include="m*\**\*.csproj" Exclude="@(DisabledProjects)"> <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties> </Project> |