Use dbi's path to load dac and add RPATH to make files.

Fixes VS's problems with debugging and running different versions of coreclr.

Set RPATH for OSX is @loader_path and removed now unnecessary load of mscordaccore
in the lldb sos plugin.

Simplify the GetDacModule code to use PAL_GetPALDirectoryW instead of depending on GetModuleInst()
and the module handle passed to DllMain.

Changed PAL_RegisterModule not to also call the DllMain of the module so it wouldn't be called twice.

2 files changed, 106 insertions, 90 deletions

diff --git a/src/pal/inc/pal.h b/src/pal/inc/pal.h
index 294502b73b..0bf1fb6de5 100644
--- a/src/pal/inc/pal.h
+++ b/src/pal/inc/pal.h
@@ -3642,13 +3642,13 @@ PALIMPORT
 HMODULE
 PALAPI
 LoadLibraryA(
-         IN LPCSTR lpLibFileName);
+        IN LPCSTR lpLibFileName);
 
 PALIMPORT
 HMODULE
 PALAPI
 LoadLibraryW(
-         IN LPCWSTR lpLibFileName);
+        IN LPCWSTR lpLibFileName);
 
 PALIMPORT
 HMODULE
@@ -3667,17 +3667,17 @@ LoadLibraryExW(
         IN DWORD dwFlags);
 
 PALIMPORT
-HMODULE
+void *
 PALAPI
 PAL_LoadLibraryDirect(
-         IN LPCWSTR lpLibFileName);
+        IN LPCWSTR lpLibFileName);
 
 PALIMPORT
 HMODULE
 PALAPI
 PAL_RegisterLibraryDirect(
-         IN HMODULE dl_handle,
-         IN LPCWSTR lpLibFileName);
+        IN void *dl_handle,
+        IN LPCWSTR lpLibFileName);
 
 /*++
 Function:
diff --git a/src/pal/src/loader/module.cpp b/src/pal/src/loader/module.cpp
index 4d0426d8b5..04e4b9cd59 100644
--- a/src/pal/src/loader/module.cpp
+++ b/src/pal/src/loader/module.cpp
@@ -107,11 +107,11 @@ bool LOADConvertLibraryPathWideStringToMultibyteString(
 
 static BOOL LOADValidateModule(MODSTRUCT *module);
 static LPWSTR LOADGetModuleFileName(MODSTRUCT *module);
-static HMODULE LOADLoadLibraryDirect(LPCSTR libraryNameOrPath, bool setLastError);
-static HMODULE LOADRegisterLibraryDirect(HMODULE dl_handle, LPCSTR libraryNameOrPath, BOOL fDynamic);
+static MODSTRUCT *LOADAddModule(void *dl_handle, LPCSTR libraryNameOrPath);
+static void *LOADLoadLibraryDirect(LPCSTR libraryNameOrPath);
+static HMODULE LOADRegisterLibraryDirect(void *dl_handle, LPCSTR libraryNameOrPath, BOOL fDynamic);
 static HMODULE LOADLoadLibrary(LPCSTR shortAsciiName, BOOL fDynamic);
 static void LOAD_SEH_CallDllMain(MODSTRUCT *module, DWORD dwReason, LPVOID lpReserved);
-static MODSTRUCT *LOADAllocModule(void *dl_handle, LPCSTR name);
 
 /* API function definitions ***************************************************/
 
@@ -267,7 +267,7 @@ Loads a library using a system call, without registering the library with the mo
 
 Returns the system handle to the loaded library, or nullptr upon failure (error is set via SetLastError()).
 */
-HMODULE
+void *
 PALAPI
 PAL_LoadLibraryDirect(
     IN LPCWSTR lpLibFileName)
@@ -275,7 +275,7 @@ PAL_LoadLibraryDirect(
     PathCharString pathstr;
     CHAR * lpstr = NULL;
     INT name_length;
-    HMODULE hModule = NULL;
+    void *dl_handle = NULL;
 
     PERF_ENTRY(LoadLibraryDirect);
     ENTRY("LoadLibraryDirect (lpLibFileName=%p (%S)) \n",
@@ -301,13 +301,12 @@ PAL_LoadLibraryDirect(
     FILEDosToUnixPathA(lpstr);
     pathstr.CloseBuffer(name_length);
 
-    /* let LOADLoadLibraryDirect call SetLastError in case of failure */
-    hModule = LOADLoadLibraryDirect(lpstr, true /* setLastError */);
+    dl_handle = LOADLoadLibraryDirect(lpstr);
 
 done:
-    LOGEXIT("LoadLibraryDirect returns HMODULE %p\n", hModule);
+    LOGEXIT("LoadLibraryDirect returns HMODULE %p\n", dl_handle);
     PERF_EXIT(LoadLibraryDirect);
-    return hModule;
+    return dl_handle;
 }
 
 /*
@@ -321,7 +320,7 @@ Returns a PAL handle to the loaded library, or nullptr upon failure (error is se
 HMODULE
 PALAPI
 PAL_RegisterLibraryDirect(
-    IN HMODULE dl_handle,
+    IN void *dl_handle,
     IN LPCWSTR lpLibFileName)
 {
     PathCharString pathstr;
@@ -355,7 +354,7 @@ PAL_RegisterLibraryDirect(
 
     /* let LOADRegisterLibraryDirect call SetLastError in case of failure */
     LockModuleList();
-    hModule = LOADRegisterLibraryDirect(dl_handle, lpstr, true /* fDynamic */);
+    hModule = LOADRegisterLibraryDirect((void *)dl_handle, lpstr, true /* fDynamic */);
     UnlockModuleList();
 
 done:
@@ -446,7 +445,7 @@ GetProcAddress(
     // inside the PAL, fall back to a normal search.
     if (ProcAddress == NULL)
     {
-            ProcAddress = (FARPROC) dlsym(module->dl_handle, lpProcName);
+        ProcAddress = (FARPROC) dlsym(module->dl_handle, lpProcName);
     }
 
     if (ProcAddress)
@@ -454,8 +453,7 @@ GetProcAddress(
         TRACE("Symbol %s found at address %p in module %p (named %S)\n",
               lpProcName, ProcAddress, module, MODNAME(module));
 
-        /* if we don't know the module's full name yet, this is our chance to
-           obtain it */
+        /* if we don't know the module's full name yet, this is our chance to obtain it */
         if (!module->lib_name && module->dl_handle)
         {
             const char* libName = PAL_dladdr((LPVOID)ProcAddress);
@@ -577,7 +575,7 @@ FreeLibrary(
         if (module->hinstance)
         {
             PUNREGISTER_MODULE unregisterModule = (PUNREGISTER_MODULE)dlsym(module->dl_handle, "PAL_UnregisterModule");
-            if (unregisterModule)
+            if (unregisterModule != nullptr)
             {
                  unregisterModule(module->hinstance);
             }
@@ -771,7 +769,8 @@ Function:
   PAL_RegisterModule
 
   Register the module with the target module and return a module handle in
-  the target module's context.
+  the target module's context. Doesn't call the DllMain because it is used
+  as part of calling DllMain in the calling module.
 
 --*/
 
@@ -788,7 +787,16 @@ PAL_RegisterModule(
         PERF_ENTRY(PAL_RegisterModule);
         ENTRY("PAL_RegisterModule(%s)\n", lpLibFileName ? lpLibFileName : "");
 
-        hinstance = (HINSTANCE)LOADLoadLibrary(lpLibFileName, FALSE);
+        LockModuleList();
+
+        void *dl_handle = LOADLoadLibraryDirect(lpLibFileName);
+        if (dl_handle)
+        {
+            // This only creates/adds the module handle and doesn't call DllMain
+            hinstance = LOADAddModule(dl_handle, lpLibFileName);
+        }
+
+        UnlockModuleList();
 
         LOGEXIT("PAL_RegisterModule returns HINSTANCE %p\n", hinstance);
         PERF_EXIT(PAL_RegisterModule);
@@ -811,6 +819,8 @@ PAL_UnregisterModule(
     PERF_ENTRY(PAL_UnregisterModule);
     ENTRY("PAL_UnregisterModule(hInstance=%p)\n", hInstance);
 
+    // DllMain won't be called because PAL_UnregisterModule didn't
+    // query its address and save it.
     FreeLibrary(hInstance);
 
     LOGEXIT("PAL_UnregisterModule returns\n");
@@ -1364,6 +1374,37 @@ static LPWSTR LOADGetModuleFileName(MODSTRUCT *module)
     return module->lib_name;
 }
 
+/*
+Function:
+    LOADLoadLibraryDirect [internal]
+
+    Loads a library using a system call, without registering the library with the module list.
+
+Parameters:
+    LPCSTR libraryNameOrPath:           The library to load.
+
+Return value:
+    System handle to the loaded library, or nullptr upon failure (error is set via SetLastError()).
+*/
+static void *LOADLoadLibraryDirect(LPCSTR libraryNameOrPath)
+{
+    _ASSERTE(libraryNameOrPath != nullptr);
+    _ASSERTE(libraryNameOrPath[0] != '\0');
+
+    void *dl_handle = dlopen(libraryNameOrPath, RTLD_LAZY);
+    if (dl_handle == nullptr)
+    {
+        WARN("dlopen() failed; dlerror says '%s'\n", dlerror());
+        SetLastError(ERROR_MOD_NOT_FOUND);
+    }
+    else
+    {
+        TRACE("dlopen() found module %s\n", libraryNameOrPath);
+    }
+
+    return dl_handle;
+}
+
 /*++
 Function :
     LOADAllocModule
@@ -1430,82 +1471,49 @@ static MODSTRUCT *LOADAllocModule(void *dl_handle, LPCSTR name)
 
 /*
 Function:
-    LOADLoadLibraryDirect [internal]
-
-    Loads a library using a system call, without registering the library with the module list.
-
-Parameters:
-    LPCSTR libraryNameOrPath:           The library to load.
-    bool setLastError:                  True to set the last error upon failure, false otherwise.
-
-Return value:
-    System handle to the loaded library, or nullptr upon failure (error is set via SetLastError() if 'setLastError' is true).
-*/
-static HMODULE LOADLoadLibraryDirect(LPCSTR libraryNameOrPath, bool setLastError)
-{
-    _ASSERTE(libraryNameOrPath != nullptr);
-    _ASSERTE(libraryNameOrPath[0] != '\0');
-
-    HMODULE dl_handle = dlopen(libraryNameOrPath, RTLD_LAZY);
-    if (dl_handle != nullptr)
-    {
-        TRACE("dlopen() found module %s\n", libraryNameOrPath);
-    }
-    else if (setLastError)
-    {
-        WARN("dlopen() failed; dlerror says '%s'\n", dlerror());
-        SetLastError(ERROR_MOD_NOT_FOUND);
-    }
-
-    return dl_handle;
-}
-
-/*
-Function:
-    LOADRegisterLibraryDirect [internal]
+    LOADAddModule [internal]
 
     Registers a system handle to a loaded library with the module list.
 
 Parameters:
-    HMODULE dl_handle:                  System handle to the loaded library.
+    void *dl_handle:                    System handle to the loaded library.
     LPCSTR libraryNameOrPath:           The library that was loaded.
-    BOOL fDynamic:                      TRUE if dynamic load through LoadLibrary, FALSE if static load through RegisterLibrary.
 
 Return value:
     PAL handle to the loaded library, or nullptr upon failure (error is set via SetLastError()).
 */
-static HMODULE LOADRegisterLibraryDirect(HMODULE dl_handle, LPCSTR libraryNameOrPath, BOOL fDynamic)
+static MODSTRUCT *LOADAddModule(void *dl_handle, LPCSTR libraryNameOrPath)
 {
     _ASSERTE(dl_handle != nullptr);
     _ASSERTE(libraryNameOrPath != nullptr);
     _ASSERTE(libraryNameOrPath[0] != '\0');
 
-    MODSTRUCT *module;
-    DWORD dwError;
-
 #if !RETURNS_NEW_HANDLES_ON_REPEAT_DLOPEN
     /* search module list for a match. */
-    module = &exe_module;
+    MODSTRUCT *module = &exe_module;
     do
     {
         if (dl_handle == module->dl_handle)
         {   
             /* found the handle. increment the refcount and return the 
                existing module structure */
-            TRACE("Found matching module %p for module name %s\n",
-                 module, libraryNameOrPath);
+            TRACE("Found matching module %p for module name %s\n", module, libraryNameOrPath);
+
             if (module->refcount != -1)
+            {
                 module->refcount++;
+            }
             dlclose(dl_handle);
             return module;
         }
         module = module->next;
+
     } while (module != &exe_module);
 #endif
 
     TRACE("Module doesn't exist : creating %s.\n", libraryNameOrPath);
-    module = LOADAllocModule(dl_handle, libraryNameOrPath);
 
+    module = LOADAllocModule(dl_handle, libraryNameOrPath);
     if (NULL == module)
     {
         ERROR("couldn't create new module\n");
@@ -1525,14 +1533,33 @@ static HMODULE LOADRegisterLibraryDirect(HMODULE dl_handle, LPCSTR libraryNameOr
     module->device = stat_buf.st_dev;
 #endif
 
-    /* We now get the address of DllMain if the module contains one. We save
-       the last error and restore it afterward, because our caller doesn't
-       care about GetProcAddress failures. */
-    dwError = GetLastError();
+    return module;
+}
+
+/*
+Function:
+    LOADRegisterLibraryDirect [internal]
 
-    module->pDllMain = (PDLLMAIN)GetProcAddress((HMODULE)module, "DllMain");
+    Registers a system handle to a loaded library with the module list.
 
-    SetLastError(dwError);
+Parameters:
+    void *dl_handle:                    System handle to the loaded library.
+    LPCSTR libraryNameOrPath:           The library that was loaded.
+    BOOL fDynamic:                      TRUE if dynamic load through LoadLibrary, FALSE if static load through RegisterLibrary.
+
+Return value:
+    PAL handle to the loaded library, or nullptr upon failure (error is set via SetLastError()).
+*/
+static HMODULE LOADRegisterLibraryDirect(void *dl_handle, LPCSTR libraryNameOrPath, BOOL fDynamic)
+{
+    MODSTRUCT *module = LOADAddModule(dl_handle, libraryNameOrPath);
+    if (module == nullptr)
+    {
+        return nullptr;
+    }
+
+    /* We now get the address of DllMain if the module contains one. */
+    module->pDllMain = (PDLLMAIN)dlsym(module->dl_handle, "DllMain");
 
     /* If it did contain a DllMain, call it. */
     if (module->pDllMain)
@@ -1544,7 +1571,7 @@ static HMODULE LOADRegisterLibraryDirect(HMODULE dl_handle, LPCSTR libraryNameOr
         if (NULL == module->hinstance)
         {
             PREGISTER_MODULE registerModule = (PREGISTER_MODULE)dlsym(module->dl_handle, "PAL_RegisterModule");
-            if (registerModule)
+            if (registerModule != nullptr)
             {
                 module->hinstance = registerModule(libraryNameOrPath);
             }
@@ -1615,7 +1642,7 @@ Return value :
 static HMODULE LOADLoadLibrary(LPCSTR shortAsciiName, BOOL fDynamic)
 {
     HMODULE module = NULL;
-    HMODULE dl_handle = NULL;
+    void *dl_handle = NULL;
 
     // Check whether we have been requested to load 'libc'. If that's the case then use the
     // full name of the library that is defined in <gnu/lib-names.h> by the LIBC_SO constant.
@@ -1635,25 +1662,14 @@ static HMODULE LOADLoadLibrary(LPCSTR shortAsciiName, BOOL fDynamic)
 
     LockModuleList();
 
-    // See if file can be dlopen()ed; this should work even if it's already loaded
-    {
-        // See GetProcAddress for an explanation why we leave the PAL.
-        PAL_LeaveHolder holder;
-
-        dl_handle = LOADLoadLibraryDirect(shortAsciiName, false /* setLastError */);
-    }
-
-    if (!dl_handle)
+    dl_handle = LOADLoadLibraryDirect(shortAsciiName);
+    if (dl_handle)
     {
-        WARN("dlopen() failed; dlerror says '%s'\n", dlerror()); 
-        SetLastError(ERROR_MOD_NOT_FOUND);
-        goto done;
+        module = LOADRegisterLibraryDirect(dl_handle, shortAsciiName, fDynamic);
     }
 
-    module = LOADRegisterLibraryDirect(dl_handle, shortAsciiName, fDynamic);
-
-done:
     UnlockModuleList();
+
     return module;
 }
 
@@ -1886,7 +1902,7 @@ MODSTRUCT *LOADGetPalLibrary()
 {
     if (pal_module == NULL)
     {
-        // Initialize the pal module (the module containing LOADPalLibrary). Assumes that 
+        // Initialize the pal module (the module containing LOADGetPalLibrary). Assumes that 
         // the PAL is linked into the coreclr module because we use the module name containing 
         // this function for the coreclr path.
         TRACE("Loading module for PAL library\n");