Pipe based communication between debugee and managed debugger on Linux

Goal of this change is to make managed debugging on Linux possible. (It is not fully achieved, but we're getting there)

So far our provision for debugging on Linux is somewhat different from debugging on Windows.
Instead of using WaitForDebugEvent and RaiseException as means of communication between debugger and debuggee, we're gonna use pipes.
Thankfully from old times of Silverlight Mac debugging we had debugging via network sockets under ifdefs FEATURE_DBGIPC_TRANSPORT_DI and FEATURE_DBGIPC_TRANSPORT_VM.
So this change is taking that old way of debugging, changing sockets for network pipes, removes lots of unused stuff and implements whatever is missing on Linux.

Testing:
Due to infrastructural issues I wasn't able to test debugging on Linux yet. So my testing consistent of
1. End to end net pipe debugging on Windows
2. Testing of twowaypipe implementation on Linux
3. Testing of search for loaded CoreCLR module on Linux.

1 files changed, 211 insertions, 0 deletions

diff --git a/src/debug/debug-pal/win/twowaypipe.cpp b/src/debug/debug-pal/win/twowaypipe.cpp
new file mode 100644
index 0000000000..49876871c0
--- /dev/null
+++ b/src/debug/debug-pal/win/twowaypipe.cpp
@@ -0,0 +1,211 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+#include <windows.h>
+#include <stdio.h> 
+#include <wchar.h>
+#include <assert.h>
+#include "twowaypipe.h"
+
+#define _ASSERTE assert
+
+// This file contains implementation of a simple IPC mechanism - bidirectional named pipe.
+// It is implemented on top of two one-directional names pipes (fifos on UNIX)
+
+
+// Creates a server side of the pipe. 
+// Id is used to create pipes names and uniquely identify the pipe on the machine. 
+// true - success, false - failure (use GetLastError() for more details)
+bool TwoWayPipe::CreateServer(DWORD id)
+{
+    _ASSERTE(m_state == NotInitialized);
+    if (m_state != NotInitialized)
+        return false;
+
+    m_inboundPipe = CreateOneWayPipe(id, true);
+    if (m_inboundPipe == INVALID_HANDLE_VALUE)
+    {
+        return false;
+    }
+
+    m_outboundPipe = CreateOneWayPipe(id, false);
+    if (m_outboundPipe == INVALID_HANDLE_VALUE)
+    {
+        CloseHandle(m_inboundPipe);
+        m_inboundPipe = INVALID_HANDLE_VALUE;
+        return false;
+    }
+
+    m_state = Created;
+    return true;
+}
+
+
+// Connects to a previously opened server side of the pipe.
+// Id is used to locate the pipe on the machine. 
+// true - success, false - failure (use GetLastError() for more details)
+bool TwoWayPipe::Connect(DWORD id)
+{
+    _ASSERTE(m_state == NotInitialized);
+    if (m_state != NotInitialized)
+        return false;
+
+    m_inboundPipe = OpenOneWayPipe(id, true);
+    if (m_inboundPipe == INVALID_HANDLE_VALUE)
+    {
+        return false;
+    }
+
+    m_outboundPipe = OpenOneWayPipe(id, false);
+    if (m_outboundPipe == INVALID_HANDLE_VALUE)
+    {
+        CloseHandle(m_inboundPipe);
+        m_inboundPipe = INVALID_HANDLE_VALUE;
+        return false;
+    }
+
+    m_state = ClientConnected;
+    return true;
+
+}
+
+// Waits for incoming client connections, assumes GetState() == Created
+// true - success, false - failure (use GetLastError() for more details)
+bool TwoWayPipe::WaitForConnection()
+{
+    _ASSERTE(m_state == Created);
+    if (m_state != Created)
+        return false;
+
+    if (!ConnectNamedPipe(m_inboundPipe, NULL))
+    {
+        auto error = GetLastError();
+        if (error != ERROR_PIPE_CONNECTED)
+            return false;
+    }
+
+    if (!ConnectNamedPipe(m_outboundPipe, NULL))
+    {
+        auto error = GetLastError();
+        if (error != ERROR_PIPE_CONNECTED)
+            return false;
+    }
+
+    m_state = ServerConnected;
+    return true;
+}
+
+// Reads data from pipe. Returns number of bytes read or a negative number in case of an error.
+// use GetLastError() for more details
+int TwoWayPipe::Read(void *buffer, DWORD bufferSize)
+{
+    _ASSERTE(m_state == ServerConnected || m_state == ClientConnected);
+    DWORD bytesRead;
+    BOOL ok = ReadFile(m_inboundPipe, buffer, bufferSize, &bytesRead, NULL);
+
+    if (ok)
+    {
+        return (int)bytesRead;
+    }
+    else
+    {
+        return -1;
+    }
+}
+
+// Writes data to pipe. Returns number of bytes written or a negative number in case of an error.
+// use GetLastError() for more details
+int TwoWayPipe::Write(const void *data, DWORD dataSize)
+{
+    _ASSERTE(m_state == ServerConnected || m_state == ClientConnected);
+    DWORD bytesWritten;
+    BOOL ok = WriteFile(m_outboundPipe, data, dataSize, &bytesWritten, NULL);
+
+    if (ok)
+    {
+        FlushFileBuffers(m_outboundPipe);
+        return (int)bytesWritten;
+    }
+    else
+    {
+        return -1;
+    }
+}
+
+// Disconnect server or client side of the pipe.
+// true - success, false - failure (use GetLastError() for more details)
+bool TwoWayPipe::Disconnect()
+{
+    if (m_state == ServerConnected)
+    {
+        DisconnectNamedPipe(m_outboundPipe);
+        DisconnectNamedPipe(m_inboundPipe);
+        CloseHandle(m_outboundPipe);
+        m_outboundPipe = INVALID_HANDLE_VALUE;
+        CloseHandle(m_inboundPipe);
+        m_inboundPipe = INVALID_HANDLE_VALUE;
+        m_state = NotInitialized;
+        return true;
+    }
+    else if (m_state == ClientConnected)
+    {
+        CloseHandle(m_outboundPipe);
+        m_outboundPipe = INVALID_HANDLE_VALUE;
+        CloseHandle(m_inboundPipe);
+        m_inboundPipe = INVALID_HANDLE_VALUE;
+        m_state = NotInitialized;
+        return true;
+    } 
+    else 
+    {
+        // nothign to do
+        return true;
+    }
+}
+
+#define PIPE_NAME_FORMAT_STR L"\\\\.\\pipe\\clr-debug-pipe-%d-%s"
+
+// Connects to a one sided pipe previously created by CreateOneWayPipe.
+// In order to successfully connect id and inbound flag should be the same.
+HANDLE TwoWayPipe::OpenOneWayPipe(DWORD id, bool inbound)
+{
+    WCHAR fullName[MAX_PATH];
+    // "in" and "out" are deliberately switched because we're opening a client side connection
+    int chars = swprintf_s(fullName, MAX_PATH, PIPE_NAME_FORMAT_STR, id, inbound ? L"out" : L"in");
+    _ASSERTE(chars > 0);
+
+    HANDLE handle = CreateFileW(
+        fullName,
+        inbound ? GENERIC_READ : GENERIC_WRITE,
+        0,              // no sharing 
+        NULL,           // default security attributes
+        OPEN_EXISTING,  // opens existing pipe 
+        0,              // default attributes 
+        NULL);          // no template file 
+
+    return handle;
+}
+
+
+// Creates a one way pipe, id and inboud flag are used for naming.
+// Created pipe is supposed to be connected to by OpenOneWayPipe.
+HANDLE TwoWayPipe::CreateOneWayPipe(DWORD id, bool inbound)
+{
+    WCHAR fullName[MAX_PATH];
+    int chars = swprintf_s(fullName, MAX_PATH, PIPE_NAME_FORMAT_STR, id, inbound ? L"in" : L"out");
+    _ASSERTE(chars > 0);
+
+    HANDLE handle = CreateNamedPipeW(fullName,
+        (inbound ? PIPE_ACCESS_INBOUND : PIPE_ACCESS_OUTBOUND) | FILE_FLAG_FIRST_PIPE_INSTANCE,
+        PIPE_TYPE_BYTE | PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS,
+        1,    // max number of instances
+        4000, //in buffer size
+        4000, //out buffer size
+        0,    // default timeout
+        NULL); // default security
+
+    return handle;
+}
+