Windows CoreFX CI (#18365)

* Add test list CL switch

* End-To-End Test Run on Windows

* Cleanup

* MAX_PATH Workaround

* Set Execution directory for CoreFX tests

* Add All CoreFX PR Tests

* Add test dependencies

* Add extra dependencies

* Add parallel test execution

* Disable OuterLoop tests and System.Data.SqlClient.*  tests

* Initialize maximum degree of parallelization to Environment.ProcessCount

* Remove unnecessary cli option

* Update Dependencies

* Add "enabled" property to tests

* Remove exclusions due to TestUtilities mismatch

* Add capability to run all tests for running Helix test lists directly

* Refactor build script to build testhost when skipping managed tests

* Disable failing System.Threading.Tests.EventWaitHandleTests.Ctor_InvalidMode

* Add switch to skip native test build

* Add testing documentation

* Don't run tests marked as "disabled" when running all available tests

* Add switch to build only testhost and remove Core_Root_Stage

* Clean up TopN.CoreFX.Windows.issues.json

* Refactor build-test.cmd

* PR feedback - build pipeline and documentation

* PR Feedback - Test Helper headers and comments

* Fix buildtesthost option for only building CoreFX test dependencies

* Disable intermittently failing test DrawBezier_PointFs

1 files changed, 61 insertions, 24 deletions

diff --git a/build-test.cmd b/build-test.cmd
index e65cbcacff..58c6d32f84 100644
--- a/build-test.cmd
+++ b/build-test.cmd
@@ -46,6 +46,9 @@ set processedArgs=
 set __unprocessedBuildArgs=
 set __RunArgs=
 set __BuildAgainstPackagesArg=
+set __SkipRestorePackages=
+set __SkipManaged=
+set __SkipNative=
 set __RuntimeId=
 set __ZipTests=
 set __TargetsWindows=1
@@ -74,8 +77,11 @@ if /i "%1" == "release"               (set __BuildType=Release&set processedArgs
 if /i "%1" == "checked"               (set __BuildType=Checked&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 
 if /i "%1" == "skipmanaged"           (set __SkipManaged=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "skipnative"            (set __SkipNative=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "buildtesthostonly"     (set __SkipNative=1&set __SkipManaged=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "toolset_dir"           (set __ToolsetDir=%2&set __PassThroughArgs=%__PassThroughArgs% %2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
 if /i "%1" == "buildagainstpackages"  (set __ZipTests=1&set __BuildAgainstPackagesArg=-BuildTestsAgainstPackages&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "skiprestorepackages"   (set __SkipRestorePackages=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "ziptests"              (set __ZipTests=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "crossgen"              (set __DoCrossgen=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "runtimeid"             (set __RuntimeId=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
@@ -160,6 +166,8 @@ REM ============================================================================
 call "%__TestDir%\setup-stress-dependencies.cmd" /arch %__BuildArch% /outputdir %__BinDir%
 @if defined _echo @echo on
 
+if defined __SkipNative goto skipnative
+
 REM =========================================================================================
 REM ===
 REM === Native test build section
@@ -224,8 +232,7 @@ if errorlevel 1 (
 :skipnative
 
 set "__TestWorkingDir=%__RootBinDir%\tests\%__BuildOS%.%__BuildArch%.%__BuildType%"
-
-if not defined __BuildAgainstPackagesArg goto SkipRestoreProduct
+if "%__SkipRestorePackages%" == 1 goto SkipRestoreProduct
 REM =========================================================================================
 REM ===
 REM === Restore product binaries from packages
@@ -245,6 +252,7 @@ set __msbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%"
 
 call "%__ProjectDir%\run.cmd" build -Project=%__ProjectDir%\tests\build.proj -BatchRestorePackages -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %__unprocessedBuildArgs%
 
+if not defined __BuildAgainstPackagesArg goto SkipRestoreProduct
 set __BuildLogRootName=Tests_GenerateRuntimeLayout
 set __BuildLog=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.log
 set __BuildWrn=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.wrn
@@ -263,7 +271,7 @@ echo %__MsgPrefix% Restored CoreCLR product from packages
 
 :SkipRestoreProduct
 
-if defined __SkipManaged exit /b 0
+if defined __SkipManaged goto SkipManagedBuild
 
 REM =========================================================================================
 REM ===
@@ -316,6 +324,7 @@ for /l %%G in (1, 1, %__BuildLoopCount%) do (
     set __AppendToLog=true
 )
 
+:SkipManagedBuild
 REM Prepare the Test Drop
 REM Cleans any NI from the last run
 powershell -NoProfile "Get-ChildItem -path %__TestWorkingDir% -Include '*.ni.*' -Recurse -Force | Remove-Item -force"
@@ -344,22 +353,14 @@ if defined __BuildAgainstPackagesArg (
   )
 )
 
-echo %__MsgPrefix%Creating test wrappers...
-
-set RuntimeIdArg=
-set TargetsWindowsArg=
-
-if defined __RuntimeId (
-    set RuntimeIdArg=-RuntimeID="%__RuntimeId%"
-)
-
-if "%__TargetsWindows%"=="1" (
-    set TargetsWindowsArg=-TargetsWindows=true
-) else if "%__TargetsWindows%"=="0" (
-    set TargetsWindowsArg=-TargetsWindows=false
-)
+REM =========================================================================================
+REM ===
+REM === Create the test overlay
+REM ===
+REM =========================================================================================
+echo %__MsgPrefix%Creating test overlay...
 
-set __BuildLogRootName=Tests_XunitWrapper
+set __BuildLogRootName=Tests_Overlay_Managed
 set __BuildLog=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.log
 set __BuildWrn=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.wrn
 set __BuildErr=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.err
@@ -367,15 +368,21 @@ set __msbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%"
 set __msbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%"
 set __msbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%"
 
-call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\runtest.proj -BuildWrappers -MsBuildEventLogging=" " -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %TargetsWindowsArg% %__unprocessedBuildArgs%
+call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\runtest.proj -testOverlay -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %RuntimeIdArg% %__unprocessedBuildArgs%
 if errorlevel 1 (
-    echo Xunit Wrapper build failed
+    echo %__MsgPrefix%Error: build failed. Refer to the build log files for details:
+    echo     %__BuildLog%
+    echo     %__BuildWrn%
+    echo     %__BuildErr%
     exit /b 1
 )
 
-echo %__MsgPrefix%Creating test overlay...
+xcopy /s /y "%CORE_ROOT_STAGE%" "%CORE_ROOT%"
 
-set __BuildLogRootName=Tests_Overlay_Managed
+REM Create the test host necessary for running CoreFX tests 
+REM The test host includes a dotnet executable, system libraries and CoreCLR assemblies found in CoreRoot
+
+set __BuildLogRootName=Tests_CoreFX_Testhost
 set __BuildLog=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.log
 set __BuildWrn=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.wrn
 set __BuildErr=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.err
@@ -383,7 +390,7 @@ set __msbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%"
 set __msbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%"
 set __msbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%"
 
-call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\runtest.proj -testOverlay -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %RuntimeIdArg% %__unprocessedBuildArgs%
+call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\runtest.proj -testHost -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %RuntimeIdArg% %__unprocessedBuildArgs%
 if errorlevel 1 (
     echo %__MsgPrefix%Error: build failed. Refer to the build log files for details:
     echo     %__BuildLog%
@@ -392,7 +399,37 @@ if errorlevel 1 (
     exit /b 1
 )
 
-xcopy /s /y "%CORE_ROOT_STAGE%" "%CORE_ROOT%"
+if defined __SkipManaged goto SkipBuildingWrappers
+
+echo %__MsgPrefix%Creating test wrappers...
+
+set RuntimeIdArg=
+set TargetsWindowsArg=
+
+if defined __RuntimeId (
+    set RuntimeIdArg=-RuntimeID="%__RuntimeId%"
+)
+
+if "%__TargetsWindows%"=="1" (
+    set TargetsWindowsArg=-TargetsWindows=true
+) else if "%__TargetsWindows%"=="0" (
+    set TargetsWindowsArg=-TargetsWindows=false
+)
+
+set __BuildLogRootName=Tests_XunitWrapper
+set __BuildLog=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.log
+set __BuildWrn=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.wrn
+set __BuildErr=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.err
+set __msbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%"
+set __msbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%"
+set __msbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%"
+
+call %__ProjectDir%\run.cmd build -Project=%__ProjectDir%\tests\runtest.proj -BuildWrappers -MsBuildEventLogging=" " -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %TargetsWindowsArg% %__unprocessedBuildArgs%
+if errorlevel 1 (
+    echo Xunit Wrapper build failed
+    exit /b 1
+)
+:SkipBuildingWrappers
 
 set __CrossgenArg = ""
 if defined __DoCrossgen (