Configure MSBuild debug path and save any logs

We have had many problems in the CI with MSBuild failing, and not
capturing the failure logs.

See https://github.com/dotnet/coreclr/issues/20236.

So, for all msbuild invocations, set MSBUILDDEBUGPATH to a new,
distinguished, subdirectory of the bin\Logs directory, named
"MsbuildDebugLogs". Change the CI to upload any log files found
there.

7 files changed, 51 insertions, 9 deletions

diff --git a/build-test.cmd b/build-test.cmd
index 26b2b86a97..0c1ded3ba8 100644
--- a/build-test.cmd
+++ b/build-test.cmd
@@ -36,6 +36,7 @@ set "__SourceDir=%__ProjectDir%\src"
 set "__PackagesDir=%__ProjectDir%\packages"
 set "__RootBinDir=%__ProjectDir%\bin"
 set "__LogsDir=%__RootBinDir%\Logs"
+set "__MsbuildDebugLogsDir=%__LogsDir%\MsbuildDebugLogs"
 
 :: Default __Exclude to issues.targets
 set __Exclude=%__TestDir%\issues.targets
@@ -158,6 +159,10 @@ if not exist "%__TestBinDir%"                   md "%__TestBinDir%"
 if not exist "%__NativeTestIntermediatesDir%"   md "%__NativeTestIntermediatesDir%"
 if not exist "%__ManagedTestIntermediatesDir%"  md "%__ManagedTestIntermediatesDir%"
 if not exist "%__LogsDir%"                      md "%__LogsDir%"
+if not exist "%__MsbuildDebugLogsDir%"          md "%__MsbuildDebugLogsDir%"
+
+REM Set up the directory for MSBuild debug logs.
+set MSBUILDDEBUGPATH=%__MsbuildDebugLogsDir%
 
 echo %__MsgPrefix%Checking prerequisites
 
diff --git a/build-test.sh b/build-test.sh
index 86472d1d73..75c8570f9a 100755
--- a/build-test.sh
+++ b/build-test.sh
@@ -180,6 +180,13 @@ generate_layout()
         echo "Creating LogsDir: ${__LogsDir}"
         mkdir -p $__LogsDir
     fi
+    if [ ! -f "$__MsbuildDebugLogsDir" ]; then
+        echo "Creating MsbuildDebugLogsDir: ${__MsbuildDebugLogsDir}"
+        mkdir -p $__MsbuildDebugLogsDir
+    fi
+
+    # Set up the directory for MSBuild debug logs.
+    export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
 
     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 
@@ -283,6 +290,13 @@ build_Tests()
         echo "Creating LogsDir: ${__LogsDir}"
         mkdir -p $__LogsDir
     fi
+    if [ ! -f "$__MsbuildDebugLogsDir" ]; then
+        echo "Creating MsbuildDebugLogsDir: ${__MsbuildDebugLogsDir}"
+        mkdir -p $__MsbuildDebugLogsDir
+    fi
+
+    # Set up the directory for MSBuild debug logs.
+    export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
 
     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 
@@ -898,6 +912,7 @@ fi
 
 # Set dependent variables
 __LogsDir="$__RootBinDir/Logs"
+__MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs"
 
 # init the host distro name
 initHostDistroRid
diff --git a/build.cmd b/build.cmd
index cc0345489d..dec253b5e3 100644
--- a/build.cmd
+++ b/build.cmd
@@ -58,6 +58,7 @@ set "__PackagesDir=%DotNetRestorePackagesPath%"
 if [%__PackagesDir%]==[] set "__PackagesDir=%__ProjectDir%\packages"
 set "__RootBinDir=%__ProjectDir%\bin"
 set "__LogsDir=%__RootBinDir%\Logs"
+set "__MsbuildDebugLogsDir=%__LogsDir%\MsbuildDebugLogs"
 
 set __BuildAll=
 
@@ -319,9 +320,13 @@ REM Generate path to be set for CMAKE_INSTALL_PREFIX to contain forward slash
 set "__CMakeBinDir=%__BinDir%"
 set "__CMakeBinDir=%__CMakeBinDir:\=/%"
 
-if not exist "%__BinDir%"           md "%__BinDir%"
-if not exist "%__IntermediatesDir%" md "%__IntermediatesDir%"
-if not exist "%__LogsDir%"          md "%__LogsDir%"
+if not exist "%__BinDir%"              md "%__BinDir%"
+if not exist "%__IntermediatesDir%"    md "%__IntermediatesDir%"
+if not exist "%__LogsDir%"             md "%__LogsDir%"
+if not exist "%__MsbuildDebugLogsDir%" md "%__MsbuildDebugLogsDir%"
+
+REM Set up the directory for MSBuild debug logs.
+set MSBUILDDEBUGPATH=%__MsbuildDebugLogsDir%
 
 REM It is convenient to have your Nuget search path include the location where the build
 REM will place packages.  However nuget used during the build will fail if that directory
diff --git a/build.sh b/build.sh
index 536dbe8f5b..ef50b3e8af 100755
--- a/build.sh
+++ b/build.sh
@@ -140,6 +140,7 @@ setup_dirs()
     mkdir -p "$__RootBinDir"
     mkdir -p "$__BinDir"
     mkdir -p "$__LogsDir"
+    mkdir -p "$__MsbuildDebugLogsDir"
     mkdir -p "$__IntermediatesDir"
 
     if [ $__CrossBuild == 1 ]; then
@@ -996,6 +997,7 @@ fi
 
 # Set dependent variables
 __LogsDir="$__RootBinDir/Logs"
+__MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs"
 
 # init the host distro name
 initHostDistroRid
@@ -1047,6 +1049,9 @@ initTargetDistroRid
 # Make the directories necessary for build if they don't exist
 setup_dirs
 
+# Set up the directory for MSBuild debug logs.
+export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
+
 # Check prereqs.
 check_prereqs
 
diff --git a/netci.groovy b/netci.groovy
index 00d42d14c3..ef2084b386 100755
--- a/netci.groovy
+++ b/netci.groovy
@@ -2246,7 +2246,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                     }
 
                     // Archive the logs, even if the build failed (which is when they are most interesting).
-                    Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
+                    Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err,bin/Logs/MsbuildDebugLogs/*", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
                     break
                 case 'arm':
                 case 'arm64':
@@ -2298,7 +2298,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                     }
 
                     // Archive the logs, even if the build failed (which is when they are most interesting).
-                    Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
+                    Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err,bin/Logs/MsbuildDebugLogs/*", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
                     break
                 default:
                     println("Unknown architecture: ${architecture}");
@@ -2422,7 +2422,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                     }
 
                     // Archive the logs, even if the build failed (which is when they are most interesting).
-                    Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
+                    Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err,bin/Logs/MsbuildDebugLogs/*", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
                     break
                 case 'armem':
                     // Emulator cross builds for ARM runs on Tizen currently
@@ -2547,7 +2547,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                     }
 
                     // Archive the logs, even if the build failed (which is when they are most interesting).
-                    Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
+                    Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err,bin/Logs/MsbuildDebugLogs/*", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false)
 
                     // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
                     // the cleanup task in Jenkins can't remove.
diff --git a/tests/runtest.cmd b/tests/runtest.cmd
index 37c534192a..e691e7fdb7 100644
--- a/tests/runtest.cmd
+++ b/tests/runtest.cmd
@@ -34,6 +34,7 @@ if %__ProjectDir:~-1%==\ set "__ProjectDir=%__ProjectDir:~0,-1%"
 set "__ProjectFilesDir=%__ProjectDir%"
 set "__RootBinDir=%__ProjectDir%\..\bin"
 set "__LogsDir=%__RootBinDir%\Logs"
+set "__MsbuildDebugLogsDir=%__LogsDir%\MsbuildDebugLogs"
 
 set __Sequential=
 set __msbuildExtraArgs=
@@ -283,7 +284,11 @@ if defined DoLink (
     set __msbuildCommonArgs=%__msbuildCommonArgs% /p:RunTestsViaIllink=true
 )
 
-if not exist %__LogsDir% md %__LogsDir%
+if not exist "%__LogsDir%"                      md "%__LogsDir%"
+if not exist "%__MsbuildDebugLogsDir%"          md "%__MsbuildDebugLogsDir%"
+
+REM Set up the directory for MSBuild debug logs.
+set MSBUILDDEBUGPATH=%__MsbuildDebugLogsDir%
 
 REM These log files are created automatically by the test run process. Q: what do they depend on being set?
 set __TestRunHtmlLog=%__LogsDir%\TestRun_%__BuildOS%__%__BuildArch%__%__BuildType%.html
diff --git a/tests/runtest.py b/tests/runtest.py
index db58915792..385406238f 100755
--- a/tests/runtest.py
+++ b/tests/runtest.py
@@ -640,7 +640,14 @@ def call_msbuild(coreclr_repo_location,
     logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs")
     if not os.path.isdir(logs_dir):
         os.makedirs(logs_dir)
-    
+
+    msbuild_debug_logs_dir = os.path.join(logs_dir, "MsbuildDebugLogs")
+    if not os.path.isdir(msbuild_debug_logs_dir):
+        os.makedirs(msbuild_debug_logs_dir)
+
+    # Set up the directory for MSBuild debug logs.
+    os.environ["MSBUILDDEBUGPATH"] = msbuild_debug_logs_dir
+
     command =   [dotnetcli_location,
                  "msbuild",
                  os.path.join(coreclr_repo_location, "tests", "runtest.proj"),