CoreFX CI Unix (#18753)

* Full Unix test support

* Remove Test helper build from build-test

Add msg prefix

Change if condition

* Change Linux URL

* Syntax error

* Fix Merge Conflict

* Address PR Feedback

* Remove duplicate generate_testhost call

* Address PR comments

* Clean up exclusion list

* Disable Client_ReadWriteCancelledToken_Throws_OperationCanceledException

* Disable all flavors of CancelledToken_Throws_OperationCanceledException

* Disable X509StoreTests.Constructor_DefaultStoreName - flaky on OSX

* Update documentation with Linux instructions and workflow

* Disable EventWaitHandleTests.Ctor_InvalidMode - failing on OSX

* Disable Server_ReadWriteCancelledToken_Throws_OperationCanceledException

1 files changed, 118 insertions, 5 deletions

diff --git a/tests/runtest.sh b/tests/runtest.sh
index 58038f35f4..474b32cbc4 100755
--- a/tests/runtest.sh
+++ b/tests/runtest.sh
@@ -70,6 +70,9 @@ function print_usage {
     echo 'CoreFX Test Options '
     echo '  --corefxtests                    : Runs CoreFX tests'
     echo '  --corefxtestsall                 : Runs all available CoreFX tests'
+    echo '  --corefxtestlist=<path>          : Runs the CoreFX tests specified in the passed list'   
+    echo '  --testHostDir=<path>             : Directory containing a built test host including core binaries, test dependencies' 
+    echo '                                     and a dotnet executable'
     echo ''
     echo 'Runtime Code Coverage options:'
     echo '  --coreclr-coverage               : Optional argument to get coreclr code coverage reports'
@@ -411,9 +414,6 @@ function create_core_overlay {
     if [ ! -d "$coreClrBinDir" ]; then
         exit_with_error "$errorSource" "Directory specified by --coreClrBinDir does not exist: $coreClrBinDir"
     fi
-    if [ -z "$coreFxBinDir" ]; then
-        exit_with_error "$errorSource" "One of --coreOverlayDir or --coreFxBinDir must be specified." "$printUsage"
-    fi
 
     # Create the overlay
     coreOverlayDir=$testRootDir/Tests/coreoverlay
@@ -441,6 +441,81 @@ function create_core_overlay {
     copy_test_native_bin_to_test_root $coreOverlayDir
 }
 
+function create_testhost
+{
+    if [ ! -d "$testHostDir" ]; then
+        exit_with_error "$errorSource" "Did not find the test host directory: $testHostDir"
+    fi
+
+    # Initialize test variables
+    local buildToolsDir=$coreClrSrc/Tools
+    local dotnetExe=$buildToolsDir/dotnetcli/dotnet
+    local coreClrSrcTestDir=$coreClrSrc/tests
+    
+    if [ -z $coreClrBinDir ]; then
+        local coreClrBinDir=${coreClrSrc}/bin
+        export __CoreFXTestDir=${coreClrSrc}/bin/tests/CoreFX
+    else
+        export __CoreFXTestDir=${coreClrBinDir}/tests/CoreFX    
+    fi
+
+    local coreFXTestSetupUtilityName=CoreFX.TestUtils.TestFileSetup
+    local coreFXTestSetupUtility="${coreClrSrcTestDir}/src/Common/CoreFX/TestFileSetup/${coreFXTestSetupUtilityName}.csproj"
+    local coreFXTestSetupUtilityOutputPath=${__CoreFXTestDir}/TestUtilities
+    local coreFXTestBinariesOutputPath=${__CoreFXTestDir}/tests_downloaded
+    
+    if [ -z $CoreFXTestList]; then
+        local CoreFXTestList="${coreClrSrcTestDir}/CoreFX/TopN.CoreFX.x64.Unix.issues.json"
+    fi
+
+    case "${OSName}" in
+        # Check if we're running under OSX        
+        Darwin)
+            local coreFXTestRemoteURL=$(<${coreClrSrcTestDir}/CoreFX/CoreFXTestListURL_OSX.txt)
+            local coreFXTestExclusionDef=nonosxtests
+        ;;        
+        # Default to Linux        
+        *)
+            local coreFXTestRemoteURL=$(<${coreClrSrcTestDir}/CoreFX/CoreFXTestListURL_Linux.txt)
+            local coreFXTestExclusionDef=nonlinuxtests
+        ;;
+    esac
+
+    local coreFXTestExecutable=xunit.console.netcore.exe
+    local coreFXLogDir=${coreClrBinDir}/Logs/CoreFX/
+    local coreFXTestExecutableArgs="--notrait category=nonnetcoreapptests --notrait category=${coreFXTestExclusionDef} --notrait category=failing --notrait category=IgnoreForCI --notrait category=OuterLoop --notrait Benchmark=true"
+
+    chmod +x ${dotnetExe}
+    resetCommandArgs=("msbuild /t:Restore ${coreFXTestSetupUtility}")
+    echo "${dotnetExe} $resetCommandArgs"
+    "${dotnetExe}" $resetCommandArgs
+
+    buildCommandArgs=("msbuild ${coreFXTestSetupUtility} /p:OutputPath=${coreFXTestSetupUtilityOutputPath} /p:Platform=${_arch} /p:Configuration=Release")
+    echo "${dotnetExe} $buildCommandArgs"
+    "${dotnetExe}" $buildCommandArgs
+    
+    if [ "${RunCoreFXTestsAll}" == "1" ]; then
+        local coreFXRunCommand=--runAllTests
+    else
+        local coreFXRunCommand=--runSpecifiedTests
+    fi
+
+    local buildTestSetupUtilArgs=("${coreFXTestSetupUtilityOutputPath}/${coreFXTestSetupUtilityName}.dll --clean --outputDirectory ${coreFXTestBinariesOutputPath} --testListJsonPath ${CoreFXTestList} ${coreFXRunCommand} --dotnetPath ${testHostDir}/dotnet --testUrl ${coreFXTestRemoteURL} --executable ${coreFXTestExecutable} --log ${coreFXLogDir} ${coreFXTestExecutableArgs}")
+    echo "${dotnetExe} $buildTestSetupUtilArgs"
+    "${dotnetExe}" $buildTestSetupUtilArgs
+
+    local exitCode=$?
+    if [ $exitCode != 0 ]; then
+        echo Running CoreFX tests finished with failures
+    else
+        echo Running CoreFX tests finished successfully
+    fi    
+    
+    echo Check ${coreFXLogDir} for test run logs
+
+    exit ${exitCode}
+}
+
 declare -a skipCrossGenFiles
 
 function is_skip_crossgen_test {
@@ -1173,10 +1248,18 @@ do
             export RunCrossGen=1
             ;;
         --corefxtests)
-            exit 0
+            export RunCoreFXTests=1
             ;;
         --corefxtestsall)
-            exit 0
+            export RunCoreFXTests=1
+            export RunCoreFXTestsAll=1
+            ;;
+        --corefxtestlist)
+            export RunCoreFXTests=1
+            export CoreFXTestList=${i#*=} 
+            ;;
+        --testHostDir=*)
+            export testHostDir=${i#*=}
             ;;
         --sequential)
             ((maxProcesses = 1))
@@ -1248,6 +1331,36 @@ fi
 
 export COMPlus_gcServer="$serverGC"
 
+if [ "$RunCoreFXTests" == 1 ];
+then 
+    if [ -z "$coreClrSrc" ]
+    then
+        echo "Coreclr src files are required to run CoreFX tests"
+        echo "Coreclr src files root path can be passed using '--coreclr-src' argument"
+        print_usage
+        exit $EXIT_CODE_EXCEPTION
+    fi
+
+    if [ -z "$testHostDir" ]; then
+        echo "--testHostDir is required to run CoreFX tests"
+        print_usage
+        exit $EXIT_CODE_EXCEPTION
+    fi
+    
+    if [ ! -f "$testHostDir/dotnet" ]; then
+        echo "Executable dotnet not found in $testHostDir"
+        exit $EXIT_CODE_EXCEPTION
+    fi
+
+    if [ ! -d "$testHostDir" ]; then
+        echo "Directory specified by --testHostDir does not exist: $testRootDir"
+        exit $EXIT_CODE_EXCEPTION
+    fi
+
+    create_testhost
+    exit 0
+fi
+
 if [ -z "$testRootDir" ]; then
     echo "--testRootDir is required."
     print_usage