Fix the calling convention for P/Invokes and delegates to hostpolicy (#23249)

* Fix the calling convention for P/Invokes and delegates

* Update mock library to match official hostpolicy calling conventions

4 files changed, 20 insertions, 30 deletions

diff --git a/src/System.Private.CoreLib/src/System/Runtime/Loader/AssemblyDependencyResolver.cs b/src/System.Private.CoreLib/src/System/Runtime/Loader/AssemblyDependencyResolver.cs
index 584c60a146..68a9bb8f54 100644
--- a/src/System.Private.CoreLib/src/System/Runtime/Loader/AssemblyDependencyResolver.cs
+++ b/src/System.Private.CoreLib/src/System/Runtime/Loader/AssemblyDependencyResolver.cs
@@ -290,23 +290,23 @@ namespace System.Runtime.Loader
         }
 #endif
 
-        [UnmanagedFunctionPointer(CallingConvention.Winapi, CharSet = HostpolicyCharSet)]
+        [UnmanagedFunctionPointer(CallingConvention.Cdecl, CharSet = HostpolicyCharSet)]
         internal delegate void corehost_resolve_component_dependencies_result_fn(
             string assembly_paths,
             string native_search_paths,
             string resource_search_paths);
 
-        [UnmanagedFunctionPointer(CallingConvention.Winapi, CharSet = HostpolicyCharSet)]
+        [UnmanagedFunctionPointer(CallingConvention.Cdecl, CharSet = HostpolicyCharSet)]
         internal delegate void corehost_error_writer_fn(
             string message);
 
 #pragma warning disable BCL0015 // Disable Pinvoke analyzer errors.
-        [DllImport("hostpolicy", CharSet = HostpolicyCharSet)]
+        [DllImport("hostpolicy", CallingConvention = CallingConvention.Cdecl, CharSet = HostpolicyCharSet)]
         private static extern int corehost_resolve_component_dependencies(
             string component_main_assembly_path,
             corehost_resolve_component_dependencies_result_fn result);
 
-        [DllImport("hostpolicy", CharSet = HostpolicyCharSet)]
+        [DllImport("hostpolicy", CallingConvention = CallingConvention.Cdecl, CharSet = HostpolicyCharSet)]
         private static extern IntPtr corehost_set_error_writer(IntPtr error_writer);
 #pragma warning restore
     }
diff --git a/src/coreclr/hosts/coreshim/CoreShim.cpp b/src/coreclr/hosts/coreshim/CoreShim.cpp
index bb6ec8abab..2583215b20 100644
--- a/src/coreclr/hosts/coreshim/CoreShim.cpp
+++ b/src/coreclr/hosts/coreshim/CoreShim.cpp
@@ -181,7 +181,7 @@ HRESULT coreclr::GetCoreClrInstance(_Outptr_ coreclr **instance, _In_opt_z_ cons
                 return E_UNEXPECTED;
 
             // Initialize the hostpolicy mock to a default state
-            using Set_corehost_resolve_component_dependencies_Values_fn = void(STDMETHODCALLTYPE *)(
+            using Set_corehost_resolve_component_dependencies_Values_fn = void(__cdecl *)(
                 int returnValue,
                 const WCHAR *assemblyPaths,
                 const WCHAR *nativeSearchPaths,
diff --git a/tests/src/Common/CoreCLRTestLibrary/HostPolicyMock.cs b/tests/src/Common/CoreCLRTestLibrary/HostPolicyMock.cs
index afa1fbcd0d..f354d9ab56 100644
--- a/tests/src/Common/CoreCLRTestLibrary/HostPolicyMock.cs
+++ b/tests/src/Common/CoreCLRTestLibrary/HostPolicyMock.cs
@@ -16,31 +16,31 @@ namespace TestLibrary
         private const CharSet HostpolicyCharSet = CharSet.Ansi;
     #endif
 
-        [DllImport("hostpolicy", CharSet = HostpolicyCharSet)]
+        [DllImport("hostpolicy", CallingConvention = CallingConvention.Cdecl, CharSet = HostpolicyCharSet)]
         private static extern int Set_corehost_resolve_component_dependencies_Values(
             int returnValue,
             string assemblyPaths,
             string nativeSearchPaths,
             string resourceSearchPaths);
 
-        [DllImport("hostpolicy", CharSet = HostpolicyCharSet)]
+        [DllImport("hostpolicy", CallingConvention = CallingConvention.Cdecl, CharSet = HostpolicyCharSet)]
         private static extern void Set_corehost_set_error_writer_returnValue(IntPtr error_writer);
 
-        [DllImport("hostpolicy", CharSet = HostpolicyCharSet)]
+        [DllImport("hostpolicy", CallingConvention = CallingConvention.Cdecl, CharSet = HostpolicyCharSet)]
         private static extern IntPtr Get_corehost_set_error_writer_lastSet_error_writer();
 
-        [UnmanagedFunctionPointer(CallingConvention.Winapi, CharSet = HostpolicyCharSet)]
+        [UnmanagedFunctionPointer(CallingConvention.Cdecl, CharSet = HostpolicyCharSet)]
         internal delegate void Callback_corehost_resolve_component_dependencies(
             string component_main_assembly_path);
 
-        [DllImport("hostpolicy", CharSet = HostpolicyCharSet)]
+        [DllImport("hostpolicy", CallingConvention = CallingConvention.Cdecl, CharSet = HostpolicyCharSet)]
         private static extern void Set_corehost_resolve_component_dependencies_Callback(
             IntPtr callback);
 
         private static Type _assemblyDependencyResolverType;
         private static Type _corehost_error_writer_fnType;
 
-        [UnmanagedFunctionPointer(CallingConvention.Winapi, CharSet = HostpolicyCharSet)]
+        [UnmanagedFunctionPointer(CallingConvention.Cdecl, CharSet = HostpolicyCharSet)]
         public delegate void ErrorWriterDelegate(string message);
 
         public static string DeleteExistingHostpolicy(string coreRoot)
diff --git a/tests/src/Common/hostpolicymock/HostpolicyMock.cpp b/tests/src/Common/hostpolicymock/HostpolicyMock.cpp
index cd16afa1a0..73c92b8bcc 100644
--- a/tests/src/Common/hostpolicymock/HostpolicyMock.cpp
+++ b/tests/src/Common/hostpolicymock/HostpolicyMock.cpp
@@ -14,16 +14,6 @@
 typedef wchar_t char_t;
 typedef std::wstring string_t;
 
-// Only create undecorated exports on Windows x86
-#if defined _X86_
-
-// Define undecorated exports to ease test set up from native code
-#pragma comment(linker, "/export:Set_corehost_resolve_component_dependencies_Callback=_Set_corehost_resolve_component_dependencies_Callback@4")
-#pragma comment(linker, "/export:Set_corehost_resolve_component_dependencies_Values=_Set_corehost_resolve_component_dependencies_Values@16")
-#pragma comment(linker, "/export:Set_corehost_set_error_writer_returnValue=_Set_corehost_set_error_writer_returnValue@4")
-
-#endif
-
 #else //!_WIN32
 
 #if __GNUC__ >= 4
@@ -42,15 +32,15 @@ string_t g_corehost_resolve_component_dependencies_assemblyPaths;
 string_t g_corehost_resolve_component_dependencies_nativeSearchPaths;
 string_t g_corehost_resolve_component_dependencies_resourceSearchPaths;
 
-typedef void(*Callback_corehost_resolve_component_dependencies)(const char_t *component_main_assembly_path);
+typedef void(__cdecl *Callback_corehost_resolve_component_dependencies)(const char_t *component_main_assembly_path);
 Callback_corehost_resolve_component_dependencies g_corehost_resolve_component_dependencies_Callback;
 
-typedef void(*corehost_resolve_component_dependencies_result_fn)(
+typedef void(__cdecl *corehost_resolve_component_dependencies_result_fn)(
     const char_t* assembly_paths,
     const char_t* native_search_paths,
     const char_t* resource_search_paths);
 
-SHARED_API int corehost_resolve_component_dependencies(
+SHARED_API int __cdecl corehost_resolve_component_dependencies(
     const char_t *component_main_assembly_path,
     corehost_resolve_component_dependencies_result_fn result)
 {
@@ -70,7 +60,7 @@ SHARED_API int corehost_resolve_component_dependencies(
     return g_corehost_resolve_component_dependencies_returnValue;
 }
 
-SHARED_API void Set_corehost_resolve_component_dependencies_Values(
+SHARED_API void __cdecl Set_corehost_resolve_component_dependencies_Values(
     int returnValue,
     const char_t *assemblyPaths,
     const char_t *nativeSearchPaths,
@@ -82,29 +72,29 @@ SHARED_API void Set_corehost_resolve_component_dependencies_Values(
     g_corehost_resolve_component_dependencies_resourceSearchPaths.assign(resourceSearchPaths);
 }
 
-SHARED_API void Set_corehost_resolve_component_dependencies_Callback(
+SHARED_API void __cdecl Set_corehost_resolve_component_dependencies_Callback(
     Callback_corehost_resolve_component_dependencies callback)
 {
     g_corehost_resolve_component_dependencies_Callback = callback;
 }
 
 
-typedef void(*corehost_error_writer_fn)(const char_t* message);
+typedef void(__cdecl *corehost_error_writer_fn)(const char_t* message);
 corehost_error_writer_fn g_corehost_set_error_writer_lastSet_error_writer;
 corehost_error_writer_fn g_corehost_set_error_writer_returnValue;
 
-SHARED_API corehost_error_writer_fn corehost_set_error_writer(corehost_error_writer_fn error_writer)
+SHARED_API corehost_error_writer_fn __cdecl corehost_set_error_writer(corehost_error_writer_fn error_writer)
 {
     g_corehost_set_error_writer_lastSet_error_writer = error_writer;
     return g_corehost_set_error_writer_returnValue;
 }
 
-SHARED_API void Set_corehost_set_error_writer_returnValue(corehost_error_writer_fn error_writer)
+SHARED_API void __cdecl Set_corehost_set_error_writer_returnValue(corehost_error_writer_fn error_writer)
 {
     g_corehost_set_error_writer_returnValue = error_writer;
 }
 
-SHARED_API corehost_error_writer_fn Get_corehost_set_error_writer_lastSet_error_writer()
+SHARED_API corehost_error_writer_fn __cdecl Get_corehost_set_error_writer_lastSet_error_writer()
 {
     return g_corehost_set_error_writer_lastSet_error_writer;
 }