Clean up diagnosticserver socket on unexpected shutdown (#25976) (#26190)

* Clean up diagnosticserver socket/pipe on shutdown

* Refactor dbg transport pipe cleanup to be registered as signal handler from the vm side

* cleanup dead code

* Remove more dead code and fix windows builds

* Moving some ifdefs around

4 files changed, 45 insertions, 12 deletions

diff --git a/src/debug/ee/debugger.cpp b/src/debug/ee/debugger.cpp
index 0299cfef12..a5e081c02e 100644
--- a/src/debug/ee/debugger.cpp
+++ b/src/debug/ee/debugger.cpp
@@ -890,15 +890,6 @@ ShutdownTransport()
         g_pDbgTransport = NULL;
     }
 }
-
-void
-AbortTransport()
-{
-    if (g_pDbgTransport != NULL)
-    {
-        g_pDbgTransport->AbortConnection();
-    }
-}
 #endif // FEATURE_DBGIPC_TRANSPORT_VM
 
 
@@ -1895,6 +1886,16 @@ void NotifyDebuggerOfStartup()
 
 #endif // !FEATURE_PAL
 
+void Debugger::CleanupTransportSocket(void)
+{
+#if defined(FEATURE_PAL) && defined(FEATURE_DBGIPC_TRANSPORT_VM)
+    if (g_pDbgTransport != NULL)
+    {
+        g_pDbgTransport->AbortConnection();
+    }
+#endif // FEATURE_PAL && FEATURE_DBGIPC_TRANSPORT_VM
+}
+
 //---------------------------------------------------------------------------------------
 //
 // Initialize Left-Side debugger object
@@ -2047,9 +2048,6 @@ HRESULT Debugger::Startup(void)
             ShutdownTransport();
             ThrowHR(hr);
         }
-    #ifdef FEATURE_PAL
-        PAL_SetShutdownCallback(AbortTransport);
-    #endif // FEATURE_PAL
     #endif // FEATURE_DBGIPC_TRANSPORT_VM
 
         RaiseStartupNotification();
diff --git a/src/debug/ee/debugger.h b/src/debug/ee/debugger.h
index 67180c0a27..dc36784f5e 100644
--- a/src/debug/ee/debugger.h
+++ b/src/debug/ee/debugger.h
@@ -1852,6 +1852,8 @@ public:
 
     HRESULT StartupPhase2(Thread * pThread);
 
+    void CleanupTransportSocket();
+
     void InitializeLazyDataIfNecessary();
 
     void LazyInit(); // will throw
diff --git a/src/vm/ceemain.cpp b/src/vm/ceemain.cpp
index b91fe714e4..508693767f 100644
--- a/src/vm/ceemain.cpp
+++ b/src/vm/ceemain.cpp
@@ -596,6 +596,31 @@ do { \
 #define IfFailGoLog(EXPR) IfFailGotoLog(EXPR, ErrExit)
 #endif
 
+
+#ifndef CROSSGEN_COMPILE
+#ifdef FEATURE_PAL
+void EESocketCleanupHelper()
+{
+    CONTRACTL
+    {
+        GC_NOTRIGGER;
+        MODE_ANY;
+    } CONTRACTL_END;
+
+    // Close the debugger transport socket first
+    if (g_pDebugInterface != NULL)
+    {
+        g_pDebugInterface->CleanupTransportSocket();
+    }
+
+    // Close the diagnostic server socket.
+#ifdef FEATURE_PERFTRACING
+    DiagnosticServer::Shutdown();
+#endif // FEATURE_PERFTRACING
+}
+#endif // FEATURE_PAL
+#endif // CROSSGEN_COMPILE
+
 void EEStartupHelper(COINITIEE fFlags)
 {
     CONTRACTL
@@ -653,8 +678,13 @@ void EEStartupHelper(COINITIEE fFlags)
 #ifdef FEATURE_PERFTRACING
         // Initialize the event pipe.
         EventPipe::Initialize();
+
 #endif // FEATURE_PERFTRACING
 
+#ifdef FEATURE_PAL
+        PAL_SetShutdownCallback(EESocketCleanupHelper);
+#endif // FEATURE_PAL
+
 #ifdef FEATURE_GDBJIT
         // Initialize gdbjit
         NotifyGdb::Initialize();
diff --git a/src/vm/dbginterface.h b/src/vm/dbginterface.h
index ebf85e397a..96157bb6b9 100644
--- a/src/vm/dbginterface.h
+++ b/src/vm/dbginterface.h
@@ -399,6 +399,9 @@ public:
     virtual LONG FirstChanceSuspendHijackWorker(PCONTEXT pContext, PEXCEPTION_RECORD pExceptionRecord) = 0;
 #endif
 
+    // Helper method for cleaning up transport socket
+    virtual void CleanupTransportSocket(void) = 0;
+
 #endif // #ifndef DACCESS_COMPILE
 
 #ifdef DACCESS_COMPILE