LoadLibrary refactoring (#20841)

* Refactor LoadLibrary Methods

This change refactors the code in DllImport in preparation
for implementing the new NativeLibrary API here:
dotnet/corefx#32015

The two main changes are:

1) A change in the semantics of the internal LoadLibrary helper functions.

When a native library is loaded, there are two categories of callers
expecting different return values:

External callers like AssemblyNative::InternalLoadUnmanagedDllFromPath()
and the upcoming System.Runtime.Interop.Marshall.LoadLibrary()
need the raw system handle
Internal callers like LoadLibraryModule() need the PAL registered handle
This change modifies the internal LoadLibraryModule* methods to work
in terms of native system handles, so that external callers can obrain
them directly. Methods requiring PAL-handles can register them explicitly.

There is no change in external signature of DllImport class, or the
native Dll cache in AppDomain class.

2) Differentiate HMODULE and NATIVE_LIBRARY_HANDLE

This change defines NATIVE_LIBRARY_HANDLE type to represent
raw system handles to native libraries that are not registered
with the PAL (On Unix systems).

The types on PAL and DlImport methods are adjusted to make
this semantic distinction explicit.

* 
Fix loading LibC via PAL_LoadLibraryDirect()

2 files changed, 55 insertions, 41 deletions

diff --git a/src/pal/inc/pal.h b/src/pal/inc/pal.h
index c4a1d64a29..45989998e8 100644
--- a/src/pal/inc/pal.h
+++ b/src/pal/inc/pal.h
@@ -68,6 +68,11 @@ extern "C" {
 #include <pal_error.h>
 #include <pal_mstypes.h>
 
+// Native system libray handle.
+// On Unix systems, NATIVE_LIBRARY_HANDLE type represents a library handle not registered with the PAL.
+// To get a HMODULE on Unix, call PAL_RegisterLibraryDirect() on a NATIVE_LIBRARY_HANDLE.
+typedef void * NATIVE_LIBRARY_HANDLE;
+
 /******************* Processor-specific glue  *****************************/
 
 #ifndef _MSC_VER
@@ -2590,7 +2595,7 @@ LoadLibraryExW(
         IN DWORD dwFlags);
 
 PALIMPORT
-void *
+NATIVE_LIBRARY_HANDLE
 PALAPI
 PAL_LoadLibraryDirect(
         IN LPCWSTR lpLibFileName);
diff --git a/src/pal/src/loader/module.cpp b/src/pal/src/loader/module.cpp
index 16bca0f929..04e41c38e5 100644
--- a/src/pal/src/loader/module.cpp
+++ b/src/pal/src/loader/module.cpp
@@ -103,10 +103,10 @@ static bool LOADConvertLibraryPathWideStringToMultibyteString(
     INT *multibyteLibraryPathLengthRef);
 static BOOL LOADValidateModule(MODSTRUCT *module);
 static LPWSTR LOADGetModuleFileName(MODSTRUCT *module);
-static MODSTRUCT *LOADAddModule(void *dl_handle, LPCSTR libraryNameOrPath);
-static void *LOADLoadLibraryDirect(LPCSTR libraryNameOrPath);
+static MODSTRUCT *LOADAddModule(NATIVE_LIBRARY_HANDLE dl_handle, LPCSTR libraryNameOrPath);
+static NATIVE_LIBRARY_HANDLE LOADLoadLibraryDirect(LPCSTR libraryNameOrPath);
 static BOOL LOADFreeLibrary(MODSTRUCT *module, BOOL fCallDllMain);
-static HMODULE LOADRegisterLibraryDirect(void *dl_handle, LPCSTR libraryNameOrPath, BOOL fDynamic);
+static HMODULE LOADRegisterLibraryDirect(NATIVE_LIBRARY_HANDLE dl_handle, LPCSTR libraryNameOrPath, BOOL fDynamic);
 static HMODULE LOADLoadLibrary(LPCSTR shortAsciiName, BOOL fDynamic);
 static BOOL LOADCallDllMainSafe(MODSTRUCT *module, DWORD dwReason, LPVOID lpReserved);
 
@@ -556,6 +556,33 @@ done:
     return retval;
 }
 
+LPCSTR FixLibCName(LPCSTR shortAsciiName)
+{
+    // Check whether we have been requested to load 'libc'. If that's the case, then:
+    // * For Linux, use the full name of the library that is defined in <gnu/lib-names.h> by the
+    //   LIBC_SO constant. The problem is that calling dlopen("libc.so") will fail for libc even
+    //   though it works for other libraries. The reason is that libc.so is just linker script
+    //   (i.e. a test file).
+    //   As a result, we have to use the full name (i.e. lib.so.6) that is defined by LIBC_SO.
+    // * For macOS, use constant value absolute path "/usr/lib/libc.dylib".
+    // * For FreeBSD, use constant value "libc.so.7".
+    // * For rest of Unices, use constant value "libc.so".
+    if (strcmp(shortAsciiName, LIBC_NAME_WITHOUT_EXTENSION) == 0)
+    {
+#if defined(__APPLE__)
+        return "/usr/lib/libc.dylib";
+#elif defined(__FreeBSD__)
+        return "libc.so.7";
+#elif defined(LIBC_SO)
+        return LIBC_SO;
+#else
+        return "libc.so";
+#endif
+    }
+
+    return shortAsciiName;
+}
+
 /*
 Function:
   PAL_LoadLibraryDirect
@@ -564,15 +591,16 @@ Function:
 
   Returns the system handle to the loaded library, or nullptr upon failure (error is set via SetLastError()).
 */
-void *
+NATIVE_LIBRARY_HANDLE
 PALAPI
 PAL_LoadLibraryDirect(
     IN LPCWSTR lpLibFileName)
 {
     PathCharString pathstr;
     CHAR * lpstr = nullptr;
+    LPCSTR lpcstr = nullptr;
     INT name_length;
-    void *dl_handle = nullptr;
+    NATIVE_LIBRARY_HANDLE dl_handle = nullptr;
 
     PERF_ENTRY(LoadLibraryDirect);
     ENTRY("LoadLibraryDirect (lpLibFileName=%p (%S)) \n",
@@ -597,8 +625,9 @@ PAL_LoadLibraryDirect(
     /* do the Dos/Unix conversion on our own copy of the name */
     FILEDosToUnixPathA(lpstr);
     pathstr.CloseBuffer(name_length);
+    lpcstr = FixLibCName(lpstr);
 
-    dl_handle = LOADLoadLibraryDirect(lpstr);
+    dl_handle = LOADLoadLibraryDirect(lpcstr);
 
 done:
     LOGEXIT("LoadLibraryDirect returns HMODULE %p\n", dl_handle);
@@ -617,7 +646,7 @@ Function:
 HMODULE
 PALAPI
 PAL_RegisterLibraryDirect(
-    IN void *dl_handle,
+    IN NATIVE_LIBRARY_HANDLE dl_handle,
     IN LPCWSTR lpLibFileName)
 {
     PathCharString pathstr;
@@ -651,7 +680,7 @@ PAL_RegisterLibraryDirect(
 
     /* let LOADRegisterLibraryDirect call SetLastError in case of failure */
     LockModuleList();
-    hModule = LOADRegisterLibraryDirect((void *)dl_handle, lpstr, true /* fDynamic */);
+    hModule = LOADRegisterLibraryDirect(dl_handle, lpstr, true /* fDynamic */);
     UnlockModuleList();
 
 done:
@@ -684,7 +713,7 @@ PAL_RegisterModule(
 
         LockModuleList();
 
-        void *dl_handle = LOADLoadLibraryDirect(lpLibFileName);
+        NATIVE_LIBRARY_HANDLE dl_handle = LOADLoadLibraryDirect(lpLibFileName);
         if (dl_handle)
         {
             // This only creates/adds the module handle and doesn't call DllMain
@@ -1395,12 +1424,12 @@ Parameters:
 Return value:
     System handle to the loaded library, or nullptr upon failure (error is set via SetLastError()).
 */
-static void *LOADLoadLibraryDirect(LPCSTR libraryNameOrPath)
+static NATIVE_LIBRARY_HANDLE LOADLoadLibraryDirect(LPCSTR libraryNameOrPath)
 {
     _ASSERTE(libraryNameOrPath != nullptr);
     _ASSERTE(libraryNameOrPath[0] != '\0');
 
-    void *dl_handle = dlopen(libraryNameOrPath, RTLD_LAZY);
+    NATIVE_LIBRARY_HANDLE dl_handle = dlopen(libraryNameOrPath, RTLD_LAZY);
     if (dl_handle == nullptr)
     {
         SetLastError(ERROR_MOD_NOT_FOUND);
@@ -1420,7 +1449,7 @@ Function :
     Allocate and initialize a new MODSTRUCT structure
 
 Parameters :
-    void *dl_handle :   handle returned by dl_open, goes in MODSTRUCT::dl_handle
+    NATIVE_LIBRARY_HANDLE dl_handle :   handle returned by dl_open, goes in MODSTRUCT::dl_handle
     
     char *name :        name of new module. after conversion to widechar, 
                         goes in MODSTRUCT::lib_name
@@ -1432,7 +1461,7 @@ Notes :
     'name' is used to initialize MODSTRUCT::lib_name. The other member is set to NULL
     In case of failure (in malloc or MBToWC), this function sets LastError.
 --*/
-static MODSTRUCT *LOADAllocModule(void *dl_handle, LPCSTR name)
+static MODSTRUCT *LOADAllocModule(NATIVE_LIBRARY_HANDLE dl_handle, LPCSTR name)
 {   
     MODSTRUCT *module;
     LPWSTR wide_name;
@@ -1485,13 +1514,13 @@ Function:
     Registers a system handle to a loaded library with the module list.
 
 Parameters:
-    void *dl_handle:                    System handle to the loaded library.
+    NATIVE_LIBRARY_HANDLE dl_handle:    System handle to the loaded library.
     LPCSTR libraryNameOrPath:           The library that was loaded.
 
 Return value:
     PAL handle to the loaded library, or nullptr upon failure (error is set via SetLastError()).
 */
-static MODSTRUCT *LOADAddModule(void *dl_handle, LPCSTR libraryNameOrPath)
+static MODSTRUCT *LOADAddModule(NATIVE_LIBRARY_HANDLE dl_handle, LPCSTR libraryNameOrPath)
 {
     _ASSERTE(dl_handle != nullptr);
     _ASSERTE(libraryNameOrPath != nullptr);
@@ -1555,14 +1584,14 @@ Function:
     Registers a system handle to a loaded library with the module list.
 
 Parameters:
-    void *dl_handle:                    System handle to the loaded library.
+    NATIVE_LIBRARY_HANDLE dl_handle:    System handle to the loaded library.
     LPCSTR libraryNameOrPath:           The library that was loaded.
     BOOL fDynamic:                      TRUE if dynamic load through LoadLibrary, FALSE if static load through RegisterLibrary.
 
 Return value:
     PAL handle to the loaded library, or nullptr upon failure (error is set via SetLastError()).
 */
-static HMODULE LOADRegisterLibraryDirect(void *dl_handle, LPCSTR libraryNameOrPath, BOOL fDynamic)
+static HMODULE LOADRegisterLibraryDirect(NATIVE_LIBRARY_HANDLE dl_handle, LPCSTR libraryNameOrPath, BOOL fDynamic)
 {
     MODSTRUCT *module = LOADAddModule(dl_handle, libraryNameOrPath);
     if (module == nullptr)
@@ -1631,30 +1660,10 @@ Return value :
 static HMODULE LOADLoadLibrary(LPCSTR shortAsciiName, BOOL fDynamic)
 {
     HMODULE module = nullptr;
-    void *dl_handle = nullptr;
-
-    // Check whether we have been requested to load 'libc'. If that's the case, then:
-    // * For Linux, use the full name of the library that is defined in <gnu/lib-names.h> by the
-    //   LIBC_SO constant. The problem is that calling dlopen("libc.so") will fail for libc even
-    //   though it works for other libraries. The reason is that libc.so is just linker script
-    //   (i.e. a test file).
-    //   As a result, we have to use the full name (i.e. lib.so.6) that is defined by LIBC_SO.
-    // * For macOS, use constant value absolute path "/usr/lib/libc.dylib".
-    // * For FreeBSD, use constant value "libc.so.7".
-    // * For rest of Unices, use constant value "libc.so".
-    if (strcmp(shortAsciiName, LIBC_NAME_WITHOUT_EXTENSION) == 0)
-    {
-#if defined(__APPLE__)
-        shortAsciiName = "/usr/lib/libc.dylib";
-#elif defined(__FreeBSD__)
-        shortAsciiName = "libc.so.7";
-#elif defined(LIBC_SO)
-        shortAsciiName = LIBC_SO;
-#else
-        shortAsciiName = "libc.so";
-#endif
-    }
+    NATIVE_LIBRARY_HANDLE dl_handle = nullptr;
 
+    shortAsciiName = FixLibCName(shortAsciiName);
+    
     LockModuleList();
 
     dl_handle = LOADLoadLibraryDirect(shortAsciiName);