Enable support DllImport a native assembly whose name contains '.' (#17505)

11 files changed, 291 insertions, 27 deletions

diff --git a/src/pal/inc/pal.h b/src/pal/inc/pal.h
index 0e59006514..f89798ceab 100644
--- a/src/pal/inc/pal.h
+++ b/src/pal/inc/pal.h
@@ -5300,12 +5300,15 @@ public:
 #define MAKEDLLNAME(x) MAKEDLLNAME_A(x)
 #endif
 
-#define PAL_SHLIB_PREFIX "lib"
+#define PAL_SHLIB_PREFIX    "lib"
+#define PAL_SHLIB_PREFIX_W  u"lib"
 
 #if __APPLE__
-#define PAL_SHLIB_SUFFIX ".dylib"
+#define PAL_SHLIB_SUFFIX    ".dylib"
+#define PAL_SHLIB_SUFFIX_W  u".dylib"
 #else
-#define PAL_SHLIB_SUFFIX ".so"
+#define PAL_SHLIB_SUFFIX    ".so"
+#define PAL_SHLIB_SUFFIX_W  u".so"
 #endif
 
 #define DBG_EXCEPTION_HANDLED            ((DWORD   )0x00010001L)    
diff --git a/src/vm/dllimport.cpp b/src/vm/dllimport.cpp
index d0196317f0..5b51ee302a 100644
--- a/src/vm/dllimport.cpp
+++ b/src/vm/dllimport.cpp
@@ -5890,6 +5890,14 @@ bool         NDirect::s_fSecureLoadLibrarySupported = false;
 #define TOLOWER(a) (((a) >= W('A') && (a) <= W('Z')) ? (W('a') + (a - W('A'))) : (a))
 #define TOHEX(a)   ((a)>=10 ? W('a')+(a)-10 : W('0')+(a))
 
+#ifdef FEATURE_PAL
+#define PLATFORM_SHARED_LIB_SUFFIX_W PAL_SHLIB_SUFFIX_W
+#define PLATFORM_SHARED_LIB_PREFIX_W PAL_SHLIB_PREFIX_W
+#else // !FEATURE_PAL
+#define PLATFORM_SHARED_LIB_SUFFIX_W W(".dll")
+#define PLATFORM_SHARED_LIB_PREFIX_W W("")
+#endif // !FEATURE_PAL
+
 // static
 HMODULE NDirect::LoadLibraryFromPath(LPCWSTR libraryPath)
 {
@@ -6043,7 +6051,7 @@ HMODULE NDirect::LoadFromNativeDllSearchDirectories(AppDomain* pDomain, LPCWSTR
 }
 
 #ifdef FEATURE_PAL
-static void DetermineLibNameVariations(const char* const** libNameVariations, int* numberOfVariations, const SString& libName, bool libNameIsRelativePath)
+static void DetermineLibNameVariations(const WCHAR* const** libNameVariations, int* numberOfVariations, const SString& libName, bool libNameIsRelativePath)
 {
     if (libNameIsRelativePath)
     {
@@ -6051,9 +6059,9 @@ static void DetermineLibNameVariations(const char* const** libNameVariations, in
         // a version number to the library name (e.g. 'libicuuc.so.57').
         bool containsSuffix;
         SString::CIterator it = libName.Begin();
-        if (libName.FindASCII(it, PAL_SHLIB_SUFFIX))
+        if (libName.Find(it, PLATFORM_SHARED_LIB_SUFFIX_W))
         {
-            it += strlen(PAL_SHLIB_SUFFIX);
+            it += wcslen(PLATFORM_SHARED_LIB_SUFFIX_W);
             containsSuffix = it == libName.End() || *it == (WCHAR)'.';
         }
         else
@@ -6063,24 +6071,24 @@ static void DetermineLibNameVariations(const char* const** libNameVariations, in
 
         if (containsSuffix)
         {
-            static const char* const SuffixLast[] =
+            static const WCHAR* const SuffixLast[] =
             {
-                "%.0s%s",   // name
-                "%s%s%.0s", // prefix+name
-                "%.0s%s%s", // name+suffix
-                "%s%s%s"    // prefix+name+suffix
+                W("%.0s%s"),   // name
+                W("%s%s%.0s"), // prefix+name
+                W("%.0s%s%s"), // name+suffix
+                W("%s%s%s")    // prefix+name+suffix
             };
             *libNameVariations = SuffixLast;
             *numberOfVariations = COUNTOF(SuffixLast);
         }
         else
         {
-            static const char* const SuffixFirst[] =
+            static const WCHAR* const SuffixFirst[] =
             {
-                "%.0s%s%s", // name+suffix
-                "%s%s%s",   // prefix+name+suffix
-                "%.0s%s",   // name
-                "%s%s%.0s"  // prefix+name
+                W("%.0s%s%s"), // name+suffix
+                W("%s%s%s"),   // prefix+name+suffix
+                W("%.0s%s"),   // name
+                W("%s%s%.0s")  // prefix+name
             };
             *libNameVariations = SuffixFirst;
             *numberOfVariations = COUNTOF(SuffixFirst);
@@ -6088,15 +6096,50 @@ static void DetermineLibNameVariations(const char* const** libNameVariations, in
     }
     else
     {
-        static const char* const NameOnly[] =
+        static const WCHAR* const NameOnly[] =
         {
-            "%.0s%s"
+            W("%.0s%s")
         };
         *libNameVariations = NameOnly;
         *numberOfVariations = COUNTOF(NameOnly);
     }
 }
-#endif
+#else // FEATURE_PAL
+static void DetermineLibNameVariations(const WCHAR* const** libNameVariations, int* numberOfVariations, const SString& libName, bool libNameIsRelativePath)
+{
+    // The purpose of following code is to workaround LoadLibrary limitation: 
+    // LoadLibrary won't append extension if filename itself contains '.'. Thus it will break the following scenario:
+    // [DllImport("A.B")] // The full name for file is "A.B.dll". This is common code pattern for cross-platform PInvoke
+    // The workaround for above scenario is to call LoadLibrary with "A.B" first, if it fails, then call LoadLibrary with "A.B.dll"
+    if (!libNameIsRelativePath ||
+        !libName.Find(libName.Begin(), W('.')) || 
+        libName.EndsWith(W(".")) || 
+        libName.EndsWithCaseInsensitive(W(".dll")) || 
+        libName.EndsWithCaseInsensitive(W(".exe")))
+    {
+        // Follow LoadLibrary rules in MSDN doc: https://msdn.microsoft.com/en-us/library/windows/desktop/ms684175(v=vs.85).aspx
+        // If the string specifies a full path, the function searches only that path for the module.
+        // If the string specifies a module name without a path and the file name extension is omitted, the function appends the default library extension .dll to the module name.
+        // To prevent the function from appending .dll to the module name, include a trailing point character (.) in the module name string.
+        static const WCHAR* const NameOnly[] =
+        {
+            W("%.0s%s")
+        };
+        *libNameVariations = NameOnly;
+        *numberOfVariations = COUNTOF(NameOnly);
+    }
+    else
+    {
+        static const WCHAR* const SuffixLast[] =
+        {
+            W("%.0s%s"),   // name
+            W("%.0s%s%s"), // name+suffix
+        };
+        *libNameVariations = SuffixLast;
+        *numberOfVariations = COUNTOF(SuffixLast);
+    }
+}
+#endif // FEATURE_PAL
 
 HINSTANCE NDirect::LoadLibraryModule(NDirectMethodDesc * pMD, LoadLibErrorTracker * pErrorTracker)
 {
@@ -6163,23 +6206,18 @@ HINSTANCE NDirect::LoadLibraryModule(NDirectMethodDesc * pMD, LoadLibErrorTracke
 
     bool libNameIsRelativePath = Path::IsRelative(wszLibName);
     DWORD dllImportSearchPathFlag = 0;
-#ifdef FEATURE_PAL
     // P/Invokes are often declared with variations on the actual library name.
     // For example, it's common to leave off the extension/suffix of the library
     // even if it has one, or to leave off a prefix like "lib" even if it has one
     // (both of these are typically done to smooth over cross-platform differences). 
     // We try to dlopen with such variations on the original.
-    const char* const* prefixSuffixCombinations = nullptr;
+    const WCHAR* const* prefixSuffixCombinations = nullptr;
     int numberOfVariations = 0;
     DetermineLibNameVariations(&prefixSuffixCombinations, &numberOfVariations, wszLibName, libNameIsRelativePath);
     for (int i = 0; hmod == NULL && i < numberOfVariations; i++)
     {
         SString currLibNameVariation;
-        currLibNameVariation.Printf(prefixSuffixCombinations[i], PAL_SHLIB_PREFIX, name, PAL_SHLIB_SUFFIX);
-#else
-    {
-        LPCWSTR currLibNameVariation = wszLibName;
-#endif
+        currLibNameVariation.Printf(prefixSuffixCombinations[i], PLATFORM_SHARED_LIB_PREFIX_W, wszLibName, PLATFORM_SHARED_LIB_SUFFIX_W);
         if (hmod == NULL)
         {
             // NATIVE_DLL_SEARCH_DIRECTORIES set by host is considered well known path 
diff --git a/tests/src/Interop/CMakeLists.txt b/tests/src/Interop/CMakeLists.txt
index 9c217063f2..8780010154 100644
--- a/tests/src/Interop/CMakeLists.txt
+++ b/tests/src/Interop/CMakeLists.txt
@@ -26,4 +26,7 @@ add_subdirectory(StringMarshalling/UTF8)
 add_subdirectory(MarshalAPI/FunctionPointer)
 add_subdirectory(MarshalAPI/IUnknown)
 add_subdirectory(SizeConst)
-add_subdirectory(ClassicCOM)
\ No newline at end of file
+add_subdirectory(ClassicCOM)
+add_subdirectory(DllImportAttribute/ExeFile)
+add_subdirectory(DllImportAttribute/FileNameContainDot)
+add_subdirectory(DllImportAttribute/Simple)
diff --git a/tests/src/Interop/DllImportAttribute/ExeFile/CMakeLists.txt b/tests/src/Interop/DllImportAttribute/ExeFile/CMakeLists.txt
new file mode 100644
index 0000000000..388b7bc5ee
--- /dev/null
+++ b/tests/src/Interop/DllImportAttribute/ExeFile/CMakeLists.txt
@@ -0,0 +1,10 @@
+cmake_minimum_required (VERSION 2.6)
+project (ExeFile)
+set(SOURCES ExeFile.cpp )
+
+# add the executable
+add_executable (ExeFile ${SOURCES})
+target_link_libraries(ExeFile ${LINK_LIBRARIES_ADDITIONAL}) 
+
+# add the install targets
+install (TARGETS ExeFile DESTINATION bin)
\ No newline at end of file
diff --git a/tests/src/Interop/DllImportAttribute/ExeFile/ExeFile.cpp b/tests/src/Interop/DllImportAttribute/ExeFile/ExeFile.cpp
new file mode 100644
index 0000000000..97b79d995e
--- /dev/null
+++ b/tests/src/Interop/DllImportAttribute/ExeFile/ExeFile.cpp
@@ -0,0 +1,21 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <locale.h>
+#include <xplatform.h>
+
+#pragma warning( push )
+#pragma warning( disable : 4996)
+
+extern "C" DLL_EXPORT int _cdecl Sum(int a, int b)
+{
+    return a + b;
+}
+
+extern "C" int _cdecl main(int argc,  char **argv)
+{
+    return 0;
+}
diff --git a/tests/src/Interop/DllImportAttribute/FileExtensionProbe.cs b/tests/src/Interop/DllImportAttribute/FileExtensionProbe.cs
new file mode 100644
index 0000000000..df2090d6cb
--- /dev/null
+++ b/tests/src/Interop/DllImportAttribute/FileExtensionProbe.cs
@@ -0,0 +1,95 @@
+// Licensed to the .NET Foundation under one or more agreements.

+// The .NET Foundation licenses this file to you under the MIT license.

+// See the LICENSE file in the project root for more information.

+

+using System;

+using System.Text;

+using System.Security;

+using System.Runtime.InteropServices;

+using TestLibrary;

+

+public class FileExtensionProbe

+{

+    private static int s_failures = 0;

+

+    [DllImport("ExeFile.exe", CallingConvention = CallingConvention.Cdecl, EntryPoint = "Sum")]

+    public extern static int Exe_Sum(int a, int b);

+

+    [DllImport("DllFile.Probe", CallingConvention = CallingConvention.Cdecl, EntryPoint = "Sum")]

+    public extern static int FileNameContainDot_Sum(int a, int b);

+

+    [DllImport("DllFileProbe", CallingConvention = CallingConvention.Cdecl, EntryPoint = "Sum")]

+    public extern static int Simple_Sum(int a, int b);

+

+    private static void Simple()

+    {

+        try

+        {

+            if (5 != Simple_Sum(2, 3))

+            {

+                Console.WriteLine("Dll returns incorrectly result!");

+                s_failures++;

+            }

+        }

+        catch (Exception e)

+        {

+            Console.WriteLine("Dll throws unexpected exception: " + e.Message);

+            s_failures++;

+        }

+    }

+

+    private static void FileNameContainDot()

+    {

+        try

+        {

+            if (7 != FileNameContainDot_Sum(3, 4))

+            {

+                Console.WriteLine("FileNameContainDot returns incorrectly result!");

+                s_failures++;

+            }

+        }

+        catch (Exception e)

+        {

+            Console.WriteLine("FileNameContainDot throws unexpected exception: " + e.Message);

+            s_failures++;

+        }

+    }

+

+    private static void Exe()

+    {

+        try

+        {

+            if (9 != Exe_Sum(5, 4))

+            {

+                Console.WriteLine("Exe_Sum returns incorrectly result!");

+                s_failures++;

+            }

+        }

+        catch (Exception e)

+        {

+            Console.WriteLine("Exe_Sum throws unexpected exception: " + e.Message);

+            s_failures++;

+        }

+    }

+

+    public static int Main()

+    {

+        Simple();

+        FileNameContainDot();

+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))

+        {

+            Exe();

+        }

+

+        if (s_failures > 0)

+        {

+            Console.WriteLine("Failed!");

+            return 101;

+        }

+        else

+        {

+            Console.WriteLine("Succeed!");

+            return 100;

+        }

+    }

+}

diff --git a/tests/src/Interop/DllImportAttribute/FileExtensionProbe.csproj b/tests/src/Interop/DllImportAttribute/FileExtensionProbe.csproj
new file mode 100644
index 0000000000..3127984d5c
--- /dev/null
+++ b/tests/src/Interop/DllImportAttribute/FileExtensionProbe.csproj
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <AssemblyName>FileExtensionProbe</AssemblyName>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{1F247C2A-CFA5-477B-A588-EF8092BBC983}</ProjectGuid>
+    <OutputType>exe</OutputType>
+    <ProjectTypeGuids>{65B5AC5B-7A24-48CC-A4F1-195938998855};{A9E06A26-0976-4317-8ABD-1B9EE2E6A0DC}</ProjectTypeGuids>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <DefineConstants>$(DefineConstants);STATIC</DefineConstants>
+    <CLRTestPriority>1</CLRTestPriority>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"></PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'"></PropertyGroup>
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="*.cs" />
+    <Compile Include="..\common\Assertion.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\Common\CoreCLRTestLibrary\CoreCLRTestLibrary.csproj">
+      <Project>{c8c0dc74-fac4-45b1-81fe-70c4808366e0}</Project>
+      <Name>CoreCLRTestLibrary</Name>
+    </ProjectReference>
+    <ProjectReference Include="Simple\CMakeLists.txt"></ProjectReference>
+    <ProjectReference Include="FileNameContainDot\CMakeLists.txt"></ProjectReference>
+    <ProjectReference Include="ExeFile\CMakeLists.txt"></ProjectReference>
+  </ItemGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+</Project>
\ No newline at end of file
diff --git a/tests/src/Interop/DllImportAttribute/FileNameContainDot/CMakeLists.txt b/tests/src/Interop/DllImportAttribute/FileNameContainDot/CMakeLists.txt
new file mode 100644
index 0000000000..90de03eca8
--- /dev/null
+++ b/tests/src/Interop/DllImportAttribute/FileNameContainDot/CMakeLists.txt
@@ -0,0 +1,10 @@
+cmake_minimum_required (VERSION 2.6)
+project (DllFile.Probe)
+set(SOURCES DllFile.Probe.cpp )
+
+# add the executable
+add_library (DllFile.Probe SHARED ${SOURCES})
+target_link_libraries(DllFile.Probe ${LINK_LIBRARIES_ADDITIONAL}) 
+
+# add the install targets
+install (TARGETS DllFile.Probe DESTINATION bin)
\ No newline at end of file
diff --git a/tests/src/Interop/DllImportAttribute/FileNameContainDot/DllFile.Probe.cpp b/tests/src/Interop/DllImportAttribute/FileNameContainDot/DllFile.Probe.cpp
new file mode 100644
index 0000000000..d01fed04e0
--- /dev/null
+++ b/tests/src/Interop/DllImportAttribute/FileNameContainDot/DllFile.Probe.cpp
@@ -0,0 +1,16 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <locale.h>
+#include <xplatform.h>
+
+#pragma warning( push )
+#pragma warning( disable : 4996)
+
+extern "C" DLL_EXPORT int _cdecl Sum(int a, int b)
+{
+    return a + b;
+}
\ No newline at end of file
diff --git a/tests/src/Interop/DllImportAttribute/Simple/CMakeLists.txt b/tests/src/Interop/DllImportAttribute/Simple/CMakeLists.txt
new file mode 100644
index 0000000000..71c8d39d24
--- /dev/null
+++ b/tests/src/Interop/DllImportAttribute/Simple/CMakeLists.txt
@@ -0,0 +1,10 @@
+cmake_minimum_required (VERSION 2.6)
+project (DllFileProbe)
+set(SOURCES DllFileProbe.cpp )
+
+# add the executable
+add_library (DllFileProbe SHARED ${SOURCES})
+target_link_libraries(DllFileProbe ${LINK_LIBRARIES_ADDITIONAL}) 
+
+# add the install targets
+install (TARGETS DllFileProbe DESTINATION bin)
\ No newline at end of file
diff --git a/tests/src/Interop/DllImportAttribute/Simple/DllFileProbe.cpp b/tests/src/Interop/DllImportAttribute/Simple/DllFileProbe.cpp
new file mode 100644
index 0000000000..d01fed04e0
--- /dev/null
+++ b/tests/src/Interop/DllImportAttribute/Simple/DllFileProbe.cpp
@@ -0,0 +1,16 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <locale.h>
+#include <xplatform.h>
+
+#pragma warning( push )
+#pragma warning( disable : 4996)
+
+extern "C" DLL_EXPORT int _cdecl Sum(int a, int b)
+{
+    return a + b;
+}
\ No newline at end of file