Custom Marshalers in custom-ALC-loaded assemblies results in types loaded from crossing ALCs (#21606)

* Create repro for dotnet/coreclr#19654

* Update ICustomMarshaler.csproj

* Update ICustomMarshaler.csproj

* Clean up repro per feedback.

* Add test case for different assemblies with the same CustomMarshaler name.

* Move EEMarshalingData cache from AppDomain to LoaderAllocator. This fixes the custom-marshaler conflict when using unloadable assembly contexts.

* Internalize the LoaderHeap* parameter.

* Add the pointer to the requesting assembly to the hashtable key.

* Fix linux-musl build break.

* Move Crst out of FEATURE_COMINTEROP block.

* Make sure to copy over the assembly pointer to the key that's actually stored in the hash table.

* Add comment for m_invokingAssembly.

* Move all usages of EEMarshallingData to hang off the correct loader allocator instead of always the global one.

* Change to m_InteropDataCrst since this EEMarshallingData can be used in preemptive GC mode.

* Always init m_InteropDataCrst (since it's used by EEMarshallingData as well as COM).

* PR Feedback.

* Remove extraneous inlines.

12 files changed, 306 insertions, 4 deletions

diff --git a/tests/src/Interop/CMakeLists.txt b/tests/src/Interop/CMakeLists.txt
index e46e78a8ff..9b67844e90 100644
--- a/tests/src/Interop/CMakeLists.txt
+++ b/tests/src/Interop/CMakeLists.txt
@@ -62,6 +62,7 @@ add_subdirectory(DllImportAttribute/ExeFile)
 add_subdirectory(DllImportAttribute/FileNameContainDot)
 add_subdirectory(DllImportAttribute/Simple)
 add_subdirectory(ExecInDefAppDom)
+add_subdirectory(ICustomMarshaler/ConflictingNames)
 add_subdirectory(LayoutClass)
 
 if(WIN32)
diff --git a/tests/src/Interop/ICustomMarshaler/ConflictingNames/CMakeLists.txt b/tests/src/Interop/ICustomMarshaler/ConflictingNames/CMakeLists.txt
new file mode 100644
index 0000000000..b1743b761b
--- /dev/null
+++ b/tests/src/Interop/ICustomMarshaler/ConflictingNames/CMakeLists.txt
@@ -0,0 +1,11 @@
+cmake_minimum_required (VERSION 2.6)
+project (CustomMarshalersConflictingNames)
+include_directories(${INC_PLATFORM_DIR})
+set(SOURCES CustomMarshalerNative.cpp )
+
+# add the executable
+add_library (CustomMarshalerNative SHARED ${SOURCES})
+target_link_libraries(CustomMarshalerNative ${LINK_LIBRARIES_ADDITIONAL}) 
+
+# add the install targets
+install (TARGETS CustomMarshalerNative DESTINATION bin)
diff --git a/tests/src/Interop/ICustomMarshaler/ConflictingNames/CustomMarshaler.cs b/tests/src/Interop/ICustomMarshaler/ConflictingNames/CustomMarshaler.cs
new file mode 100644
index 0000000000..506f8b2181
--- /dev/null
+++ b/tests/src/Interop/ICustomMarshaler/ConflictingNames/CustomMarshaler.cs
@@ -0,0 +1,54 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.InteropServices;
+
+class WrappedString
+{
+    public WrappedString(string str)
+    {
+        _str = str;
+    }
+
+    internal string _str;
+}
+
+class WrappedStringCustomMarshaler : ICustomMarshaler
+{
+    public void CleanUpManagedData(object ManagedObj) { }
+    public void CleanUpNativeData(IntPtr pNativeData) { Marshal.ZeroFreeCoTaskMemAnsi(pNativeData); }
+
+    public int GetNativeDataSize() => IntPtr.Size;
+
+    public IntPtr MarshalManagedToNative(object ManagedObj) => Marshal.StringToCoTaskMemAnsi(((WrappedString)ManagedObj)._str);
+    public object MarshalNativeToManaged(IntPtr pNativeData) => new WrappedString(Marshal.PtrToStringAnsi(pNativeData));
+
+    public static ICustomMarshaler GetInstance(string cookie) => new WrappedStringCustomMarshaler();
+}
+
+// Use an ifdef here to give us two separate public API surfaces to call while allowing us to have the same implementation code
+// as well as allowing us to share the custom marshaler implementations above.
+// If we wanted to add more tests here, we would want to put the public API surface in the namespace and the private
+// details and marshalers in the global scope as done above.
+#if CUSTOMMARSHALERS2
+namespace CustomMarshalers2
+#else
+namespace CustomMarshalers
+#endif
+{
+    public class CustomMarshalerTest
+    {
+        [DllImport("CustomMarshalerNative", CharSet = CharSet.Ansi)]
+        private static extern int NativeParseInt([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(WrappedStringCustomMarshaler))] WrappedString str);
+
+        public int ParseInt(string str)
+        {
+            return NativeParseInt(new WrappedString(str));
+        }
+    }
+}
+
diff --git a/tests/src/Interop/ICustomMarshaler/ConflictingNames/CustomMarshaler.csproj b/tests/src/Interop/ICustomMarshaler/ConflictingNames/CustomMarshaler.csproj
new file mode 100644
index 0000000000..e04f27f049
--- /dev/null
+++ b/tests/src/Interop/ICustomMarshaler/ConflictingNames/CustomMarshaler.csproj
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), Interop.settings.targets))\Interop.settings.targets" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
+    <OutputType>library</OutputType>
+    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "></PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' " />
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="CustomMarshaler.cs" />
+  </ItemGroup>
+  <ItemGroup>
+  </ItemGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+</Project>
diff --git a/tests/src/Interop/ICustomMarshaler/ConflictingNames/CustomMarshaler2.csproj b/tests/src/Interop/ICustomMarshaler/ConflictingNames/CustomMarshaler2.csproj
new file mode 100644
index 0000000000..38c0cbacd2
--- /dev/null
+++ b/tests/src/Interop/ICustomMarshaler/ConflictingNames/CustomMarshaler2.csproj
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), Interop.settings.targets))\Interop.settings.targets" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
+    <OutputType>library</OutputType>
+    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+    <DefineConstants>$(DefineConstants);CUSTOMMARSHALERS2</DefineConstants>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "></PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' " />
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="CustomMarshaler.cs" />
+  </ItemGroup>
+  <ItemGroup>
+  </ItemGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+</Project>
diff --git a/tests/src/Interop/ICustomMarshaler/ConflictingNames/CustomMarshalerNative.cpp b/tests/src/Interop/ICustomMarshaler/ConflictingNames/CustomMarshalerNative.cpp
new file mode 100644
index 0000000000..11aee8008a
--- /dev/null
+++ b/tests/src/Interop/ICustomMarshaler/ConflictingNames/CustomMarshalerNative.cpp
@@ -0,0 +1,12 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#include <stdlib.h>
+#include <xplatform.h>
+
+
+extern "C" int DLL_EXPORT STDMETHODCALLTYPE NativeParseInt(LPCSTR str)
+{
+    return atoi(str);
+}
diff --git a/tests/src/Interop/ICustomMarshaler/ConflictingNames/MultipleALCs.csproj b/tests/src/Interop/ICustomMarshaler/ConflictingNames/MultipleALCs.csproj
new file mode 100644
index 0000000000..045a2b69dc
--- /dev/null
+++ b/tests/src/Interop/ICustomMarshaler/ConflictingNames/MultipleALCs.csproj
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <Import Project="../../Interop.settings.targets" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "></PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' " />
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="RunInALC.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="CMakeLists.txt" />
+    <ProjectReference Include="CustomMarshaler.csproj" />
+  </ItemGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+  <PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>
+</Project>
diff --git a/tests/src/Interop/ICustomMarshaler/ConflictingNames/RunInALC.cs b/tests/src/Interop/ICustomMarshaler/ConflictingNames/RunInALC.cs
new file mode 100644
index 0000000000..1d61f2deaf
--- /dev/null
+++ b/tests/src/Interop/ICustomMarshaler/ConflictingNames/RunInALC.cs
@@ -0,0 +1,72 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Runtime.Loader;
+using TestLibrary;
+
+public class RunInALC
+{
+    public static int Main(string[] args)
+    {
+        try
+        {
+            ConflictingCustomMarshalerNamesInCollectibleLoadContexts_Succeeds();
+            ConflictingCustomMarshalerNamesInNoncollectibleLoadContexts_Succeeds();
+        }
+        catch (System.Exception e)
+        {
+            Console.WriteLine(e.ToString());
+            return 101;
+        }
+        return 100;
+    }
+
+    static void ConflictingCustomMarshalerNamesInCollectibleLoadContexts_Succeeds()
+    {
+        Run(new UnloadableLoadContext());
+        Run(new UnloadableLoadContext());
+    }
+
+    static void ConflictingCustomMarshalerNamesInNoncollectibleLoadContexts_Succeeds()
+    {
+        Run(new CustomLoadContext());
+        Run(new CustomLoadContext());
+    }
+
+    static void Run(AssemblyLoadContext context)
+    {
+        string currentAssemblyDirectory = Path.GetDirectoryName(new Uri(Assembly.GetExecutingAssembly().CodeBase).AbsolutePath);
+        Assembly inContextAssembly = context.LoadFromAssemblyPath(Path.Combine(currentAssemblyDirectory, "CustomMarshaler.dll"));
+        Type inContextType = inContextAssembly.GetType("CustomMarshalers.CustomMarshalerTest");
+        object instance = Activator.CreateInstance(inContextType);
+        MethodInfo parseIntMethod = inContextType.GetMethod("ParseInt", BindingFlags.Instance | BindingFlags.Public);
+        Assert.AreEqual(1234, (int)parseIntMethod.Invoke(instance, new object[]{"1234"}));
+    }
+}
+
+class UnloadableLoadContext : AssemblyLoadContext
+{
+    public UnloadableLoadContext()
+        :base(true)
+    {
+        
+    }
+
+    protected override Assembly Load(AssemblyName assemblyName)
+    {
+        return null;
+    }
+}
+
+class CustomLoadContext : AssemblyLoadContext
+{
+    protected override Assembly Load(AssemblyName assemblyName)
+    {
+        return null;
+    }
+}
diff --git a/tests/src/Interop/ICustomMarshaler/ConflictingNames/SameNameDifferentAssembly.cs b/tests/src/Interop/ICustomMarshaler/ConflictingNames/SameNameDifferentAssembly.cs
new file mode 100644
index 0000000000..ce9113438c
--- /dev/null
+++ b/tests/src/Interop/ICustomMarshaler/ConflictingNames/SameNameDifferentAssembly.cs
@@ -0,0 +1,28 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Runtime.Loader;
+using TestLibrary;
+
+public class RunInALC
+{
+    public static int Main(string[] args)
+    {
+        try
+        {
+            Assert.AreEqual(123, new CustomMarshalers.CustomMarshalerTest().ParseInt("123"));
+            Assert.AreEqual(123, new CustomMarshalers2.CustomMarshalerTest().ParseInt("123"));
+            return 100;
+        }
+        catch (Exception e)
+        {
+            Console.WriteLine(e);
+            return 101;
+        }
+    }
+}
diff --git a/tests/src/Interop/ICustomMarshaler/ConflictingNames/SameNameDifferentAssembly.csproj b/tests/src/Interop/ICustomMarshaler/ConflictingNames/SameNameDifferentAssembly.csproj
new file mode 100644
index 0000000000..6b1f7df357
--- /dev/null
+++ b/tests/src/Interop/ICustomMarshaler/ConflictingNames/SameNameDifferentAssembly.csproj
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <Import Project="../../Interop.settings.targets" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "></PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' " />
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="SameNameDifferentAssembly.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="CMakeLists.txt" />
+    <ProjectReference Include="CustomMarshaler.csproj" />
+    <ProjectReference Include="CustomMarshaler2.csproj" />
+  </ItemGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+</Project>
diff --git a/tests/src/Interop/ICustomMarshaler/ICustomMarshaler.cs b/tests/src/Interop/ICustomMarshaler/Primitives/ICustomMarshaler.cs
index b07b8109cc..b07b8109cc 100644
--- a/tests/src/Interop/ICustomMarshaler/ICustomMarshaler.cs
+++ b/tests/src/Interop/ICustomMarshaler/Primitives/ICustomMarshaler.cs
diff --git a/tests/src/Interop/ICustomMarshaler/ICustomMarshaler.csproj b/tests/src/Interop/ICustomMarshaler/Primitives/ICustomMarshaler.csproj
index ae3ef0e4cf..0e68e23a9c 100644
--- a/tests/src/Interop/ICustomMarshaler/ICustomMarshaler.csproj
+++ b/tests/src/Interop/ICustomMarshaler/Primitives/ICustomMarshaler.csproj
@@ -21,11 +21,8 @@
     </CodeAnalysisDependentAssemblyPaths>
   </ItemGroup>
   <ItemGroup>
-    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
-  </ItemGroup>
-  <ItemGroup>
     <Compile Include="ICustomMarshaler.cs" />
-    <Compile Include="..\common\XunitBase.cs" />
+    <Compile Include="..\..\common\XunitBase.cs" />
   </ItemGroup>
   <ItemGroup>
   </ItemGroup>