Enable fine-grained and universal exclusions for CoreFX tests

One problem we've had in coreclr Jenkins when running corefx
tests (in the coreclr outerloop runs -- the ones using the
run-corefx-tests.py harness) is the inability to exclude
failing tests for x86 or x64, and the inability to have
fine-grained exclusions. This has led to problems like
https://github.com/dotnet/coreclr/issues/22442 where
virtually all Windows x86 and x64 corefx runs fail due to
timeouts. And issues like https://github.com/dotnet/coreclr/issues/24159
where all corefx test legs fail, and we have no control
over exclusions to get them all passing again quickly.

Now that the corefx RunTests.cmd/sh wrapper scripts
parse named arguments, including a response file that
is passed on to xunit, and also that the corefx used
xunit has support for this response file as well as
fine-grained exclusions (per-method/per-class/per-namespace),
we can take advantage of it.

This change adds a single, global, corefx xunit exclusion
response file, that will be used for all platforms. Since this
run-corefx-tests.py mechanism is not expected to live much longer,
this seems sufficient.

1 files changed, 8 insertions, 4 deletions

diff --git a/netci.groovy b/netci.groovy
index 22ee7ffed5..fde15f9560 100755
--- a/netci.groovy
+++ b/netci.groovy
@@ -2169,8 +2169,9 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                                 def workspaceRelativeFxRoot = "_/fx"
                                 def absoluteFxRoot = "%WORKSPACE%\\_\\fx"
                                 def fxBranch = getFxBranch(branch)
+                                def exclusionRspPath = "%WORKSPACE%\\tests\\scripts\\run-corefx-tests-exclusions.txt"
 
-                                buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}"
+                                buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -exclusion_rsp_file ${exclusionRspPath}"
 
                                 // Archive and process (only) the test results
                                 Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
@@ -2403,8 +2404,9 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                             def workspaceRelativeFxRoot = "_/fx"
                             def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}"
                             def fxBranch = getFxBranch(branch)
+                            def exclusionRspPath = "\$WORKSPACE%/tests/scripts/run-corefx-tests-exclusions.txt"
 
-                            buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}"
+                            buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -exclusion_rsp_file ${exclusionRspPath}"
 
                             // Archive and process (only) the test results
                             Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
@@ -2989,7 +2991,8 @@ def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture
                 def corefx_runtime_path   = "%WORKSPACE%\\_\\fx\\artifacts\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}"
                 def corefx_tests_dir      = "%WORKSPACE%\\_\\fx\\artifacts\\bin\\tests"
                 def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt"
-                batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture}")
+                def exclusionRspPath      = "%WORKSPACE%\\tests\\scripts\\run-corefx-tests-exclusions.txt"
+                batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture} ${exclusionRspPath}")
 
             } else { // !isCoreFxScenario(scenario)
 
@@ -3374,8 +3377,9 @@ python -u \${WORKSPACE}/tests/scripts/run-pmi-diffs.py -arch ${architecture} -ci
                 shell("tar -czf dasm.${os}.${architecture}.${configuration}.tgz ./_/pmi/asm")
             }
             else if (doCoreFxTesting) {
+                def exclusionRspPath = "\${WORKSPACE}/tests/scripts/run-corefx-tests-exclusions.txt"
                 shell("""\
-\${WORKSPACE}/tests/scripts/run-corefx-tests.sh --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin --configurationGroup Release""")
+\${WORKSPACE}/tests/scripts/run-corefx-tests.sh --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin --configurationGroup Release --exclusion-rsp-file ${exclusionRspPath}""")
             }
             else {
                 def runScript = "${dockerCmd}./tests/runtest.sh"