summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSteve MacLean <sdmaclea.qdt@qualcommdatacenter.com>2018-05-04 15:53:44 -0400
committerjashook <jashoo@microsoft.com>2018-05-11 10:10:52 -0700
commitd0d7997f964b3e21ee4ed78d6f2dd2da0cf94ed6 (patch)
treed7e0513a8712388f4e87458e0b92d30b47f09b6d
parent6db42177ecabd5e81a5d3d4b0b51f7730e5fd782 (diff)
downloadcoreclr-d0d7997f964b3e21ee4ed78d6f2dd2da0cf94ed6.tar.gz
coreclr-d0d7997f964b3e21ee4ed78d6f2dd2da0cf94ed6.tar.bz2
coreclr-d0d7997f964b3e21ee4ed78d6f2dd2da0cf94ed6.zip
Split unix test builds in slices (#17785)
* Split unix test builds in slices Ports #17161 to linux * Address review feedback
-rwxr-xr-xbuild-test.sh87
-rw-r--r--tests/src/dirs.proj46
2 files changed, 88 insertions, 45 deletions
diff --git a/build-test.sh b/build-test.sh
index 77b98e76d7..5339dc1116 100755
--- a/build-test.sh
+++ b/build-test.sh
@@ -295,28 +295,75 @@ build_Tests_internal()
__BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
__BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
__BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
- __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
- __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
- __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
- # Generate build command
- buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
+ if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
+ # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
+ # See https://github.com/Microsoft/msbuild/issues/2993
- echo "Building step '$stepName' via $buildCommand"
+ # __SkipPackageRestore and __SkipTargetingPackBuild used to control build by tests/src/dirs.proj
+ export __SkipPackageRestore=false
+ export __SkipTargetingPackBuild=false
+ export __BuildLoopCount=2
+ export __TestGroupToBuild=1
+ __AppendToLog=false
- # Invoke MSBuild
- eval $buildCommand
+ if [ -n __priority1 ]; then
+ export __BuildLoopCount=16
+ export __TestGroupToBuild=2
+ fi
- # Invoke MSBuild
- # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
+ for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 ))
+ do
+ __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
+ __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
+ __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
- # Make sure everything is OK
- if [ $? -ne 0 ]; then
- echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
- echo " $__BuildLog"
- echo " $__BuildWrn"
- echo " $__BuildErr"
- exit 1
+ export TestBuildSlice=$slice
+
+ # Generate build command
+ buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
+
+ echo "Building step '$stepName' slice=$slice via $buildCommand"
+
+ # Invoke MSBuild
+ eval $buildCommand
+
+ # Make sure everything is OK
+ if [ $? -ne 0 ]; then
+ echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
+ echo " $__BuildLog"
+ echo " $__BuildWrn"
+ echo " $__BuildErr"
+ exit 1
+ fi
+ export __SkipPackageRestore=true
+ export __SkipTargetingPackBuild=true
+ __AppendToLog=true
+ done
+ else
+ __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
+ __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
+ __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
+
+ # Generate build command
+ buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
+
+ echo "Building step '$stepName' via $buildCommand"
+
+ # Invoke MSBuild
+ eval $buildCommand
+
+ # Invoke MSBuild
+ # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
+
+ # Make sure everything is OK
+ if [ $? -ne 0 ]; then
+ echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
+ echo " $__BuildLog"
+ echo " $__BuildWrn"
+ echo " $__BuildErr"
+ exit 1
+ fi
fi
}
@@ -342,6 +389,7 @@ usage()
echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
+ echo "priority1 - include priority=1 tests in the build"
exit 1
}
@@ -461,6 +509,7 @@ __RunTests=0
__RebuildTests=0
__BuildTestWrappers=0
__GenerateLayoutOnly=
+__priority1=
CORE_ROOT=
while :; do
@@ -615,6 +664,10 @@ while :; do
msbuildonunsupportedplatform)
__msbuildonunsupportedplatform=1
;;
+ priority1)
+ __priority1=1
+ __UnprocessedBuildArgs="$__UnprocessedBuildArgs -priority=1"
+ ;;
*)
__UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
;;
diff --git a/tests/src/dirs.proj b/tests/src/dirs.proj
index f7d1aadea9..6009e2ed08 100644
--- a/tests/src/dirs.proj
+++ b/tests/src/dirs.proj
@@ -33,26 +33,16 @@
<DisabledProjects Include="Loader\classloader\generics\GenericMethods\VSW491668.csproj" /> <!-- issue 5501 -->
</ItemGroup>
- <!-- Unix builds do not support subgroups -->
- <ItemGroup Condition="$(__BuildOS) != 'Windows_NT'" >
- <Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)">
- <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
- </Project>
- <Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)">
- <AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
- </Project>
- </ItemGroup>
-
<!-- Test build is divided in slices which can be created within Test Group
Priority 0 tests are build using Test Group 1 with 2 subgroups or slices -->
- <ItemGroup Condition="$(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'">
+ <ItemGroup Condition="$(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'">
<Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
- <ItemGroup Condition="$(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '2'">
+ <ItemGroup Condition="$(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '2'">
<Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
@@ -61,7 +51,7 @@
<!-- Test build is divided in slices which can be created within Test Group
Priority 1 or higher tests are build using Test Group 2 with 16 subgroups or slices -->
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '1')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '1')">
<Project Include="baseservices\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
@@ -76,7 +66,7 @@
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '2')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '2')">
<Project Include="CoreMangLib\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
@@ -85,7 +75,7 @@
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '3')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '3')">
<Project Include="E*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
@@ -112,7 +102,7 @@
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '4')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '4')">
<Project Include="JIT\B*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
@@ -124,7 +114,7 @@
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '5')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '5')">
<Project Include="JIT\B*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
@@ -136,7 +126,7 @@
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '6')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '6')">
<Project Include="JIT\Generics\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
@@ -151,7 +141,7 @@
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '7')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '7')">
<Project Include="JIT\IL_Conformance\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
@@ -160,7 +150,7 @@
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '8')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '8')">
<Project Include="JIT\jit64\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
@@ -169,19 +159,19 @@
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '9')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '9')">
<Project Include="JIT\Methodical\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '10')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '10')">
<Project Include="JIT\Methodical\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '11')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '11')">
<Project Include="JIT\opt\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
@@ -202,31 +192,31 @@
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '12')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '12')">
<Project Include="JIT\R*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '13')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '13')">
<Project Include="JIT\R*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '14')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '14')">
<Project Include="Loader\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '15')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '15')">
<Project Include="Loader\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>
- <ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '16')">
+ <ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '16')">
<Project Include="m*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>