improve DllImport library name variation trying (#15912)

* dllimport: refactor libname variation code

* Change order of libname variations depending on suffix presence

* make MAX_LIBNAME_VARIATIONS static

* PR feedback

* Fix non PAL build

1 files changed, 116 insertions, 87 deletions

diff --git a/src/vm/dllimport.cpp b/src/vm/dllimport.cpp
index b06de143db..345b3d6a70 100644
--- a/src/vm/dllimport.cpp
+++ b/src/vm/dllimport.cpp
@@ -6042,6 +6042,62 @@ HMODULE NDirect::LoadFromNativeDllSearchDirectories(AppDomain* pDomain, LPCWSTR
     return hmod;
 }
 
+#ifdef FEATURE_PAL
+static void DetermineLibNameVariations(const char* const** libNameVariations, int* numberOfVariations, const SString& libName, bool libNameIsRelativePath)
+{
+    if (libNameIsRelativePath)
+    {
+        // We check if the suffix is contained in the name, because on Linux it is common to append
+        // a version number to the library name (e.g. 'libicuuc.so.57').
+        bool containsSuffix;
+        SString::CIterator it = libName.Begin();
+        if (libName.FindASCII(it, PAL_SHLIB_SUFFIX))
+        {
+            it += strlen(PAL_SHLIB_SUFFIX);
+            containsSuffix = it == libName.End() || *it == (WCHAR)'.';
+        }
+        else
+        {
+            containsSuffix = false;
+        }
+
+        if (containsSuffix)
+        {
+            static const char* const SuffixLast[] =
+            {
+                "%.0s%s",   // name
+                "%s%s%.0s", // prefix+name
+                "%.0s%s%s", // name+suffix
+                "%s%s%s"    // prefix+name+suffix
+            };
+            *libNameVariations = SuffixLast;
+            *numberOfVariations = COUNTOF(SuffixLast);
+        }
+        else
+        {
+            static const char* const SuffixFirst[] =
+            {
+                "%.0s%s%s", // name+suffix
+                "%s%s%s",   // prefix+name+suffix
+                "%.0s%s",   // name
+                "%s%s%.0s"  // prefix+name
+            };
+            *libNameVariations = SuffixFirst;
+            *numberOfVariations = COUNTOF(SuffixFirst);
+        }
+    }
+    else
+    {
+        static const char* const NameOnly[] =
+        {
+            "%.0s%s"
+        };
+        *libNameVariations = NameOnly;
+        *numberOfVariations = COUNTOF(NameOnly);
+    }
+}
+#endif
+
 HINSTANCE NDirect::LoadLibraryModule(NDirectMethodDesc * pMD, LoadLibErrorTracker * pErrorTracker)
 {
     CONTRACTL
@@ -6105,68 +6161,84 @@ HINSTANCE NDirect::LoadLibraryModule(NDirectMethodDesc * pMD, LoadLibErrorTracke
     }
 #endif // FEATURE_CORESYSTEM && !FEATURE_PAL
 
-    if (hmod == NULL)
-    {
-        // NATIVE_DLL_SEARCH_DIRECTORIES set by host is considered well known path 
-        hmod = LoadFromNativeDllSearchDirectories(pDomain, wszLibName, loadWithAlteredPathFlags, pErrorTracker);
-    }
-
-    DWORD dllImportSearchPathFlag = 0;
-    BOOL searchAssemblyDirectory = TRUE;
     bool libNameIsRelativePath = Path::IsRelative(wszLibName);
-    if (hmod == NULL)
+    DWORD dllImportSearchPathFlag = 0;
+#ifdef FEATURE_PAL
+    // P/Invokes are often declared with variations on the actual library name.
+    // For example, it's common to leave off the extension/suffix of the library
+    // even if it has one, or to leave off a prefix like "lib" even if it has one
+    // (both of these are typically done to smooth over cross-platform differences). 
+    // We try to dlopen with such variations on the original.
+    const char* const* prefixSuffixCombinations = nullptr;
+    int numberOfVariations = 0;
+    DetermineLibNameVariations(&prefixSuffixCombinations, &numberOfVariations, wszLibName, libNameIsRelativePath);
+    for (int i = 0; hmod == NULL && i < numberOfVariations; i++)
+    {
+        SString currLibNameVariation;
+        currLibNameVariation.Printf(prefixSuffixCombinations[i], PAL_SHLIB_PREFIX, name, PAL_SHLIB_SUFFIX);
+#else
     {
-        // First checks if the method has DefaultDllImportSearchPathsAttribute. If method has the attribute
-        // then dllImportSearchPathFlag is set to its value.
-        // Otherwise checks if the assembly has the attribute. 
-        // If assembly has the attribute then flag ise set to its value.
-        BOOL attributeIsFound = FALSE;
-
-        if (pMD->HasDefaultDllImportSearchPathsAttribute())
+        LPCWSTR currLibNameVariation = wszLibName;
+#endif
+        if (hmod == NULL)
         {
-            dllImportSearchPathFlag = pMD->DefaultDllImportSearchPathsAttributeCachedValue();
-            searchAssemblyDirectory = pMD->DllImportSearchAssemblyDirectory();
-            attributeIsFound = TRUE;
+            // NATIVE_DLL_SEARCH_DIRECTORIES set by host is considered well known path 
+            hmod = LoadFromNativeDllSearchDirectories(pDomain, currLibNameVariation, loadWithAlteredPathFlags, pErrorTracker);
         }
-        else 
+
+        BOOL searchAssemblyDirectory = TRUE;
+        if (hmod == NULL)
         {
-            Module * pModule = pMD->GetModule();
+            // First checks if the method has DefaultDllImportSearchPathsAttribute. If method has the attribute
+            // then dllImportSearchPathFlag is set to its value.
+            // Otherwise checks if the assembly has the attribute. 
+            // If assembly has the attribute then flag ise set to its value.
+            BOOL attributeIsFound = FALSE;
 
-            if(pModule->HasDefaultDllImportSearchPathsAttribute())
+            if (pMD->HasDefaultDllImportSearchPathsAttribute())
             {
-                dllImportSearchPathFlag = pModule->DefaultDllImportSearchPathsAttributeCachedValue();
-                searchAssemblyDirectory = pModule->DllImportSearchAssemblyDirectory();
+                dllImportSearchPathFlag = pMD->DefaultDllImportSearchPathsAttributeCachedValue();
+                searchAssemblyDirectory = pMD->DllImportSearchAssemblyDirectory();
                 attributeIsFound = TRUE;
             }
-        }
+            else 
+            {
+                Module * pModule = pMD->GetModule();
 
+                if(pModule->HasDefaultDllImportSearchPathsAttribute())
+                {
+                    dllImportSearchPathFlag = pModule->DefaultDllImportSearchPathsAttributeCachedValue();
+                    searchAssemblyDirectory = pModule->DllImportSearchAssemblyDirectory();
+                    attributeIsFound = TRUE;
+                }
+            }
 
-        if (!libNameIsRelativePath)
-        {
-            DWORD flags = loadWithAlteredPathFlags;
-            if ((dllImportSearchPathFlag & LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR) != 0)
+            if (!libNameIsRelativePath)
             {
-                // LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR is the only flag affecting absolute path. Don't OR the flags
-                // unconditionally as all absolute path P/Invokes could then lose LOAD_WITH_ALTERED_SEARCH_PATH.
-                flags |= dllImportSearchPathFlag;
-            }
+                DWORD flags = loadWithAlteredPathFlags;
+                if ((dllImportSearchPathFlag & LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR) != 0)
+                {
+                    // LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR is the only flag affecting absolute path. Don't OR the flags
+                    // unconditionally as all absolute path P/Invokes could then lose LOAD_WITH_ALTERED_SEARCH_PATH.
+                    flags |= dllImportSearchPathFlag;
+                }
 
-            hmod = LocalLoadLibraryHelper(wszLibName, flags, pErrorTracker);
+                hmod = LocalLoadLibraryHelper(currLibNameVariation, flags, pErrorTracker);
+            }
+            else if (searchAssemblyDirectory)
+            {
+                Assembly* pAssembly = pMD->GetMethodTable()->GetAssembly();
+                hmod = LoadFromPInvokeAssemblyDirectory(pAssembly, currLibNameVariation, loadWithAlteredPathFlags | dllImportSearchPathFlag, pErrorTracker);
+            }
         }
-        else if (searchAssemblyDirectory)
-        {
-            Assembly* pAssembly = pMD->GetMethodTable()->GetAssembly();
-            hmod = LoadFromPInvokeAssemblyDirectory(pAssembly, wszLibName, loadWithAlteredPathFlags | dllImportSearchPathFlag, pErrorTracker);
 
+        // This call searches the application directory instead of the location for the library.
+        if (hmod == NULL)
+        {
+            hmod = LocalLoadLibraryHelper(currLibNameVariation, dllImportSearchPathFlag, pErrorTracker);
         }
     }
 
-    // This call searches the application directory instead of the location for the library.
-    if (hmod == NULL)
-    {
-        hmod = LocalLoadLibraryHelper(wszLibName, dllImportSearchPathFlag, pErrorTracker);
-    }
-
     // This may be an assembly name
     if (!hmod)
     {
@@ -6196,49 +6268,6 @@ HINSTANCE NDirect::LoadLibraryModule(NDirectMethodDesc * pMD, LoadLibErrorTracke
         }
     }
 
-#ifdef FEATURE_PAL
-    if (hmod == NULL)
-    {
-        // P/Invokes are often declared with variations on the actual library name.
-        // For example, it's common to leave off the extension/suffix of the library
-        // even if it has one, or to leave off a prefix like "lib" even if it has one
-        // (both of these are typically done to smooth over cross-platform differences). 
-        // We try to dlopen with such variations on the original.
-        const char* const prefixSuffixCombinations[] =
-        {
-            "%s%s%s",     // prefix+name+suffix
-            "%.0s%s%s",   // name+suffix
-            "%s%s%.0s",   // prefix+name
-        };
-
-        const int NUMBER_OF_LIB_NAME_VARIATIONS = COUNTOF(prefixSuffixCombinations);
-
-        // Try to load from places we tried above, but this time with variations on the
-        // name including the prefix, suffix, and both.
-        for (int i = 0; i < NUMBER_OF_LIB_NAME_VARIATIONS; i++)
-        {
-            SString currLibNameVariation;
-            currLibNameVariation.Printf(prefixSuffixCombinations[i], PAL_SHLIB_PREFIX, name, PAL_SHLIB_SUFFIX);
-
-            if (libNameIsRelativePath && searchAssemblyDirectory)
-            {
-                Assembly *pAssembly = pMD->GetMethodTable()->GetAssembly();
-                hmod = LoadFromPInvokeAssemblyDirectory(pAssembly, currLibNameVariation, loadWithAlteredPathFlags | dllImportSearchPathFlag, pErrorTracker);
-                if (hmod != NULL)
-                    break;
-            }
-
-            hmod = LoadFromNativeDllSearchDirectories(pDomain, currLibNameVariation, loadWithAlteredPathFlags, pErrorTracker);
-            if (hmod != NULL)
-                break;
-
-            hmod = LocalLoadLibraryHelper(currLibNameVariation, dllImportSearchPathFlag, pErrorTracker);
-            if (hmod != NULL)
-                break;
-        }
-    }
-#endif // FEATURE_PAL
-
     // After all this, if we have a handle add it to the cache.
     if (hmod)
     {