Enable mixed mode assembly loading (#19542)

Bring back functionality for loading IJW assemblies and calling managed->native. Also add workaround to test case for the C++ compiler inserting calls to mscoree.

9 files changed, 187 insertions, 0 deletions

diff --git a/tests/src/Interop/CMakeLists.txt b/tests/src/Interop/CMakeLists.txt
index 0787ae0655..73189244cf 100644
--- a/tests/src/Interop/CMakeLists.txt
+++ b/tests/src/Interop/CMakeLists.txt
@@ -33,4 +33,6 @@ add_subdirectory(DllImportAttribute/Simple)
 
 if(WIN32)
     add_subdirectory(COM/NativeServer)
+    add_subdirectory(IJW/FakeMscoree)
+    add_subdirectory(IJW/ManagedCallingNative/IjwNativeDll)
 endif(WIN32)
diff --git a/tests/src/Interop/IJW/FakeMscoree/CMakeLists.txt b/tests/src/Interop/IJW/FakeMscoree/CMakeLists.txt
new file mode 100644
index 0000000000..cfb6ee2ab5
--- /dev/null
+++ b/tests/src/Interop/IJW/FakeMscoree/CMakeLists.txt
@@ -0,0 +1,11 @@
+cmake_minimum_required (VERSION 2.6)
+project (mscoree)
+include_directories( ${INC_PLATFORM_DIR} )
+set(SOURCES mscoree.cpp mscoree.def)
+
+# add the shared library
+add_library (mscoree SHARED ${SOURCES})
+target_link_libraries(mscoree ${LINK_LIBRARIES_ADDITIONAL})
+
+# add the install targets
+install (TARGETS mscoree DESTINATION bin)
diff --git a/tests/src/Interop/IJW/FakeMscoree/mscoree.cpp b/tests/src/Interop/IJW/FakeMscoree/mscoree.cpp
new file mode 100644
index 0000000000..415c8d5be0
--- /dev/null
+++ b/tests/src/Interop/IJW/FakeMscoree/mscoree.cpp
@@ -0,0 +1,7 @@
+#include <windows.h>
+
+// Entrypoint jumped to by IJW dlls when their dllmain is called
+extern "C" __declspec(dllexport) BOOL WINAPI _CorDllMain(HINSTANCE hInst, DWORD dwReason, LPVOID lpReserved)
+{
+    return TRUE;
+}
diff --git a/tests/src/Interop/IJW/FakeMscoree/mscoree.def b/tests/src/Interop/IJW/FakeMscoree/mscoree.def
new file mode 100644
index 0000000000..9279b0364b
--- /dev/null
+++ b/tests/src/Interop/IJW/FakeMscoree/mscoree.def
@@ -0,0 +1,3 @@
+LIBRARY MSCOREE
+EXPORTS
+    _CorDllMain
\ No newline at end of file
diff --git a/tests/src/Interop/IJW/ManagedCallingNative/IjwNativeDll/CMakeLists.txt b/tests/src/Interop/IJW/ManagedCallingNative/IjwNativeDll/CMakeLists.txt
new file mode 100644
index 0000000000..e0b91278f5
--- /dev/null
+++ b/tests/src/Interop/IJW/ManagedCallingNative/IjwNativeDll/CMakeLists.txt
@@ -0,0 +1,46 @@
+cmake_minimum_required (VERSION 2.6)
+project (IjwNativeDll)
+include_directories( ${INC_PLATFORM_DIR} )
+set(SOURCES IjwNativeDll.cpp)
+
+if (WIN32)
+  # 4365 - signed/unsigned mismatch
+  add_compile_options(-wd4365)
+
+  # IJW
+  add_compile_options(-clr)
+  
+  # IJW requires the CRT as a dll, not linked in
+  if(UPPERCASE_CMAKE_BUILD_TYPE STREQUAL DEBUG OR UPPERCASE_CMAKE_BUILD_TYPE STREQUAL CHECKED)
+    add_compile_options(-MDd)
+  else()
+    add_compile_options(-MD)
+  endif()
+
+  # CMake enables /RTC1 and /EHsc by default, but they're not compatible with /clr, so remove them
+  if(CMAKE_CXX_FLAGS_DEBUG MATCHES "/RTC1")
+    string(REPLACE "/RTC1" " " CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
+  endif()
+  
+  if(CMAKE_CXX_FLAGS MATCHES "/EHsc")
+    string(REPLACE "/EHsc" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+  endif()
+
+  # IJW isn't compatible with CFG
+   if(CMAKE_CXX_FLAGS MATCHES "/guard:cf")
+    string(REPLACE "/guard:cf" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+   endif()
+
+     # IJW isn't compatible with GR-
+   if(CMAKE_CXX_FLAGS MATCHES "/GR-")
+    string(REPLACE "/GR-" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+   endif()
+
+endif()
+
+# add the shared library
+add_library (IjwNativeDll SHARED ${SOURCES})
+target_link_libraries(IjwNativeDll ${LINK_LIBRARIES_ADDITIONAL})
+
+# add the install targets
+install (TARGETS IjwNativeDll DESTINATION bin)
diff --git a/tests/src/Interop/IJW/ManagedCallingNative/IjwNativeDll/IjwNativeDll.cpp b/tests/src/Interop/IJW/ManagedCallingNative/IjwNativeDll/IjwNativeDll.cpp
new file mode 100644
index 0000000000..6ac5601921
--- /dev/null
+++ b/tests/src/Interop/IJW/ManagedCallingNative/IjwNativeDll/IjwNativeDll.cpp
@@ -0,0 +1,15 @@
+#pragma unmanaged
+int NativeFunction()
+{
+    return 100;
+}
+
+#pragma managed
+public ref class TestClass
+{
+public:
+    int ManagedEntryPoint()
+    {
+        return NativeFunction();
+    }
+};
diff --git a/tests/src/Interop/IJW/ManagedCallingNative/ManagedCallingNative.cs b/tests/src/Interop/IJW/ManagedCallingNative/ManagedCallingNative.cs
new file mode 100644
index 0000000000..6f038e79fb
--- /dev/null
+++ b/tests/src/Interop/IJW/ManagedCallingNative/ManagedCallingNative.cs
@@ -0,0 +1,69 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.IO;
+using System.Reflection;
+using System.Runtime.InteropServices;
+using TestLibrary;
+
+namespace ManagedCallingNative
+{
+    class ManagedCallingNative
+    {
+        static int Main(string[] args)
+        {
+            bool success = true;
+            // Load a fake mscoree.dll to avoid starting desktop
+            LoadLibraryEx(Path.Combine(Environment.CurrentDirectory, "mscoree.dll"), IntPtr.Zero, 0);
+
+            TestFramework.BeginScenario("Calling from managed to native IJW code");
+
+            // Building with a reference to the IJW dll is difficult, so load via reflection instead
+            TestFramework.BeginTestCase("Load IJW dll via reflection");
+            Assembly ijwNativeDll = Assembly.Load("IjwNativeDll");
+            TestFramework.EndTestCase();
+
+            TestFramework.BeginTestCase("Call native method returning int");
+            Type testType = ijwNativeDll.GetType("TestClass");
+            object testInstance = Activator.CreateInstance(testType);
+            MethodInfo testMethod = testType.GetMethod("ManagedEntryPoint");
+            int result = (int)testMethod.Invoke(testInstance, null);
+            if (result != 100)
+            {
+                TestFramework.LogError("IJW", "Incorrect result returned: " + result);
+                success = false;
+            }
+            TestFramework.EndTestCase();
+
+            TestFramework.BeginTestCase("Negative: Load IJW dll as byte array");
+            byte[] ijwBytes = File.ReadAllBytes("IjwNativeDll.dll");
+            try
+            {
+                Assembly.Load(ijwBytes);
+                TestFramework.LogError("IJW", "Loading IJW dll as byte array should have thrown");
+                success = false;
+            }
+            catch { }
+            TestFramework.EndTestCase();
+
+            TestFramework.BeginTestCase("Ensure .NET Framework was not loaded");
+            IntPtr clrHandle = GetModuleHandle("mscoreei.dll");
+            if (clrHandle != IntPtr.Zero)
+            {
+                TestFramework.LogError("IJW", ".NET Framework loaded by IJw module load");
+                success = false;
+            }
+            TestFramework.EndTestCase();
+
+            return success ? 100 : 99;
+        }
+
+        [DllImport("kernel32.dll")]
+        static extern IntPtr LoadLibraryEx(string lpFileName, IntPtr hReservedNull, int dwFlags);
+
+        [DllImport("kernel32.dll")]
+        static extern IntPtr GetModuleHandle(string lpModuleName);
+    }
+}
diff --git a/tests/src/Interop/IJW/ManagedCallingNative/ManagedCallingNative.csproj b/tests/src/Interop/IJW/ManagedCallingNative/ManagedCallingNative.csproj
new file mode 100644
index 0000000000..d5decf8319
--- /dev/null
+++ b/tests/src/Interop/IJW/ManagedCallingNative/ManagedCallingNative.csproj
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <AssemblyName>ManagedCallingNative</AssemblyName>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{49D1D482-E783-4CA9-B6BA-A9714BF81036}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
+  </PropertyGroup>
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="ManagedCallingNative.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="IjwNativeDll/CMakeLists.txt" />
+    <ProjectReference Include="../FakeMscoree/CMakeLists.txt" />
+    <ProjectReference Include="../../../Common/CoreCLRTestLibrary/CoreCLRTestLibrary.csproj" />
+  </ItemGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+</Project>
diff --git a/tests/testsUnsupportedOutsideWindows.txt b/tests/testsUnsupportedOutsideWindows.txt
index aa56d76944..0c4e8f0fbb 100644
--- a/tests/testsUnsupportedOutsideWindows.txt
+++ b/tests/testsUnsupportedOutsideWindows.txt
@@ -126,6 +126,7 @@ GC/Coverage/smalloom/smalloom.sh
 Interop/COM/NETClients/Primitives/NETClientPrimitives/NETClientPrimitives.sh
 Interop/MarshalAPI/IUnknown/IUnknownTest/IUnknownTest.sh
 Interop/SizeConst/SizeConstTest/SizeConstTest.sh
+Interop/IJW/ManagedCallingNative/ManagedCallingNative/ManagedCallingNative.sh
 JIT/Directed/coverage/oldtests/callipinvoke/callipinvoke.sh
 JIT/Directed/coverage/oldtests/Desktop/callipinvoke_il_d/callipinvoke_il_d.sh
 JIT/Directed/coverage/oldtests/Desktop/callipinvoke_il_r/callipinvoke_il_r.sh