diff options
| -rw-r--r-- | build-test.cmd | 9 | ||||
| -rwxr-xr-x | build-test.sh | 14 | ||||
| -rw-r--r-- | tests/runtest.proj | 25 |
3 files changed, 47 insertions, 1 deletions
diff --git a/build-test.cmd b/build-test.cmd index 9df2033da5..242c39e70d 100644 --- a/build-test.cmd +++ b/build-test.cmd @@ -335,6 +335,15 @@ for /l %%G in (1, 1, %__BuildLoopCount%) do ( set __AppendToLog=true ) +REM Check that we've built about as many tests as we expect. This is primarily intended to prevent accidental changes that cause us to build +REM drastically fewer Pri-1 tests than expected. +echo %__MsgPrefix%Check the managed tests build +call %__DotnetHost% msbuild %__ProjectDir%\tests\runtest.proj /t:CheckTestBuild /p:CLRTestPriorityToBuild=%__Priority% %__msbuildArgs% %__unprocessedBuildArgs% +if errorlevel 1 ( + echo %__MsgPrefix%Error: build failed. + exit /b 1 +) + :SkipManagedBuild REM ========================================================================================= diff --git a/build-test.sh b/build-test.sh index fd2d8af3d9..b661ebaef9 100755 --- a/build-test.sh +++ b/build-test.sh @@ -252,6 +252,20 @@ build_Tests() echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)" exit 1 else + echo "Checking the Managed Tests Build..." + + if [ -n __priority1 ]; then + __Priority=1 + else + __Priority=0 + fi + build_Tests_internal "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "/t:CheckTestBuild /p:CLRTestPriorityToBuild=$__Priority" + + if [ $? -ne 0 ]; then + echo "${__MsgPrefix}Error: Check Test Build failed." + exit 1 + fi + echo "Managed tests build success!" fi diff --git a/tests/runtest.proj b/tests/runtest.proj index 3d12bd32fb..52eeaaf1e5 100644 --- a/tests/runtest.proj +++ b/tests/runtest.proj @@ -27,7 +27,7 @@ <Target Name="FindCmdDirectories" DependsOnTargets="GetListOfTestCmds"> <Error Condition="!Exists('$(XunitTestBinBase)')" - Text="$(XunitTestBinBase) does not exist. Please run buildtest.cmd from the (repo root)\tests at least once to get the tests built." /> + Text="$(XunitTestBinBase) does not exist. Please run build-test.cmd from the repo root at least once to get the tests built." /> <ItemGroup> @@ -54,6 +54,29 @@ </Target> + <!-- Target to check the test build, to see if it looks ok. We've had several cases where a change inadvertently and drastically changes + the set of tests that are built, and that change is unnoticed. The most common case is for a build of the Priority 1 tests + to only build the Priority 0 tests. This target is run after a test build to verify that the basic number of tests that were + built is basically what was expected. When this was written, there were about 2500 Priority 0 tests and about 12270 Priority 1 + tests (differing slightly based on platform). We currently check that the number of Priority 0 tests is greater than 2000 and + less than 3000, and the number of Priority 1 tests is greater than 12000. + --> + <Target Name="CheckTestBuild" DependsOnTargets="GetListOfTestCmds"> + <Error Condition="!Exists('$(XunitTestBinBase)')" + Text="$(XunitTestBinBase) does not exist. Please run build-test.cmd from the repo root at least once to get the tests built." /> + + <PropertyGroup> + <TestCount>@(AllRunnableTestPaths->Count())</TestCount> + </PropertyGroup> + + <Message Text="Found $(TestCount) built tests"/> + + <Error Condition="'$(CLRTestPriorityToBuild)' == '0' and '$(TestCount)' <= 2000" Text="Unexpected test count. Expected > 2000, found $(TestCount).'" /> + <Error Condition="'$(CLRTestPriorityToBuild)' == '0' and '$(TestCount)' >= 3000" Text="Unexpected test count. Expected < 3000, found $(TestCount).'" /> + <Error Condition="'$(CLRTestPriorityToBuild)' == '1' and '$(TestCount)' <= 12000" Text="Unexpected test count. Expected > 12000, found $(TestCount).'" /> + <Error Condition="'$(CLRTestPriorityToBuild)' != '0' and '$(CLRTestPriorityToBuild)' != '1'" Text="Unknown priority $(CLRTestPriorityToBuild)" /> + </Target> + <Import Project="$(__Exclude)" Condition="'$(__Exclude)' != '' AND '$(XunitTestBinBase)' != ''" /> <PropertyGroup> <HaveExcludes>False</HaveExcludes> |