Add support for the NetTrace EventPipe file format

Right now the new format is not on by default, but it can be enabled using COMPlus_EventPipeNetTraceFormat = 1 for testing purposes. The plan to have a follow up PR that will add shipping configuration mechanisms and change the default setting.

See the documentation in the PerfView repo for more details about the format. At a glance the goal is to create a format that is more efficient to produce, has a smaller on disk size, and offers enhanced functionality in a few areas:
a) 64 bit thread id support
b) Detection of dropped events via sequence numbers
c) Better support for extracting subsets of the file

Together with the change there was also some refactoring of the EventPipeBufferManager and EventPipeThread.

This change addresses (at least in part) the following issues:
#19688, #23414, #24188, #20751, #20555, #21827, #24852, #25046

5 files changed, 83 insertions, 12 deletions

diff --git a/src/pal/inc/pal.h b/src/pal/inc/pal.h
index 884b737d15..08a35c8f62 100644
--- a/src/pal/inc/pal.h
+++ b/src/pal/inc/pal.h
@@ -1585,6 +1585,17 @@ CreateThread(
          OUT LPDWORD lpThreadId);
 
 PALIMPORT
+HANDLE
+PALAPI
+PAL_CreateThread64(
+    IN LPSECURITY_ATTRIBUTES lpThreadAttributes,
+    IN DWORD dwStackSize,
+    IN LPTHREAD_START_ROUTINE lpStartAddress,
+    IN LPVOID lpParameter,
+    IN DWORD dwCreationFlags,
+    OUT SIZE_T* pThreadId);
+
+PALIMPORT
 PAL_NORETURN
 VOID
 PALAPI
diff --git a/src/pal/src/include/pal/thread.hpp b/src/pal/src/include/pal/thread.hpp
index 6227b721ce..2f067ecf3a 100644
--- a/src/pal/src/include/pal/thread.hpp
+++ b/src/pal/src/include/pal/thread.hpp
@@ -53,7 +53,7 @@ namespace CorUnix
         LPVOID lpParameter,
         DWORD dwCreationFlags,
         PalThreadType eThreadType,
-        LPDWORD lpThreadId,
+        SIZE_T* pThreadId,
         HANDLE *phThread
         );
 
@@ -215,7 +215,7 @@ namespace CorUnix
                 LPVOID,
                 DWORD,
                 PalThreadType,
-                LPDWORD,
+                SIZE_T*,
                 HANDLE*
                 );
 
diff --git a/src/pal/src/synchmgr/synchmanager.cpp b/src/pal/src/synchmgr/synchmanager.cpp
index 1bdfb9426b..c0631a6930 100644
--- a/src/pal/src/synchmgr/synchmanager.cpp
+++ b/src/pal/src/synchmgr/synchmanager.cpp
@@ -1486,6 +1486,7 @@ namespace CorUnix
         }
 
         HANDLE hWorkerThread = NULL;
+        SIZE_T osThreadId = 0;
         palErr = InternalCreateThread(pthrCurrent,
                                       NULL,
                                       0,
@@ -1493,11 +1494,12 @@ namespace CorUnix
                                       (PVOID)pSynchManager,
                                       0,
                                       PalWorkerThread,
-                                      &pSynchManager->m_dwWorkerThreadTid,
+                                      &osThreadId,
                                       &hWorkerThread);
 
         if (NO_ERROR == palErr)
         {
+            pSynchManager->m_dwWorkerThreadTid = (DWORD)osThreadId;
             palErr = InternalGetThreadDataFromHandle(pthrCurrent,
                                                      hWorkerThread,
                                                      0,
diff --git a/src/pal/src/thread/process.cpp b/src/pal/src/thread/process.cpp
index 724ddc8bd3..428cc2af9b 100644
--- a/src/pal/src/thread/process.cpp
+++ b/src/pal/src/thread/process.cpp
@@ -1674,6 +1674,7 @@ public:
         PAL_ERROR pe = NO_ERROR;
         BOOL ret;
         UnambiguousProcessDescriptor unambiguousProcessDescriptor;
+        SIZE_T osThreadId = 0;
 
 #ifdef __APPLE__
         if (lpApplicationGroupId != NULL)
@@ -1734,7 +1735,6 @@ public:
 
         // Add a reference for the thread handler
         AddRef();
-
         pe = InternalCreateThread(
             pThread,
             NULL,
@@ -1743,7 +1743,7 @@ public:
             this,
             0,
             UserCreatedThread,
-            &m_threadId,
+            &osThreadId,
             &m_threadHandle);
 
         if (NO_ERROR != pe)
@@ -1752,7 +1752,7 @@ public:
             Release();
             goto exit;
         }
-
+        m_threadId = (DWORD)osThreadId;
     exit:
         return pe;
     }
diff --git a/src/pal/src/thread/thread.cpp b/src/pal/src/thread/thread.cpp
index 34d780845f..42209a6028 100644
--- a/src/pal/src/thread/thread.cpp
+++ b/src/pal/src/thread/thread.cpp
@@ -510,7 +510,7 @@ CreateThread(
           dwCreationFlags, lpThreadId);
 
     pThread = InternalGetCurrentThread();
-
+    SIZE_T osThreadId = 0;
     palError = InternalCreateThread(
         pThread,
         lpThreadAttributes,
@@ -519,7 +519,7 @@ CreateThread(
         lpParameter,
         dwCreationFlags,
         UserCreatedThread,
-        lpThreadId,
+        &osThreadId,
         &hNewThread
         );
 
@@ -527,13 +527,71 @@ CreateThread(
     {
         pThread->SetLastError(palError);
     }
-
+    if(lpThreadId != nullptr)
+    {
+        *lpThreadId = (DWORD)osThreadId;
+    }
     LOGEXIT("CreateThread returns HANDLE %p\n", hNewThread);
     PERF_EXIT(CreateThread);
 
     return hNewThread;
 }
 
+/*++
+Function:
+  PAL_CreateThread64
+  Similar to CreateThread but passes out a 64 bit thread id on platforms which use one.
+
+Note:
+  lpThreadAttributes could be ignored.
+
+See MSDN doc.
+
+--*/
+HANDLE
+PALAPI
+PAL_CreateThread64(
+    IN LPSECURITY_ATTRIBUTES lpThreadAttributes,
+    IN DWORD dwStackSize,
+    IN LPTHREAD_START_ROUTINE lpStartAddress,
+    IN LPVOID lpParameter,
+    IN DWORD dwCreationFlags,
+    OUT SIZE_T* pThreadId)
+{
+    PAL_ERROR palError;
+    CPalThread *pThread;
+    HANDLE hNewThread = NULL;
+
+    PERF_ENTRY(PAL_CreateThread64);
+    ENTRY("PAL_CreateThread64(lpThreadAttr=%p, dwStackSize=%u, lpStartAddress=%p, "
+        "lpParameter=%p, dwFlags=%#x, pThreadId=%p)\n",
+        lpThreadAttributes, dwStackSize, lpStartAddress, lpParameter,
+        dwCreationFlags, pThreadId);
+
+    pThread = InternalGetCurrentThread();
+    palError = InternalCreateThread(
+        pThread,
+        lpThreadAttributes,
+        dwStackSize,
+        lpStartAddress,
+        lpParameter,
+        dwCreationFlags,
+        UserCreatedThread,
+        pThreadId,
+        &hNewThread
+    );
+
+    if (NO_ERROR != palError)
+    {
+        pThread->SetLastError(palError);
+    }
+
+    LOGEXIT("PAL_CreateThread64 returns HANDLE %p\n", hNewThread);
+    PERF_EXIT(PAL_CreateThread64);
+
+    return hNewThread;
+}
+
 PAL_ERROR
 CorUnix::InternalCreateThread(
     CPalThread *pThread,
@@ -543,7 +601,7 @@ CorUnix::InternalCreateThread(
     LPVOID lpParameter,
     DWORD dwCreationFlags,
     PalThreadType eThreadType,
-    LPDWORD lpThreadId,
+    SIZE_T* pThreadId,
     HANDLE *phThread
     )
 {
@@ -825,9 +883,9 @@ CorUnix::InternalCreateThread(
         //
         *phThread = hNewThread;
 
-        if (NULL != lpThreadId)
+        if (NULL != pThreadId)
         {
-            *lpThreadId = pNewThread->GetThreadId();
+            *pThreadId = pNewThread->GetThreadId();
         }
     }
     else