Add support for NativeCallableAttribute

Apply [NativeCallable] attribute to a managed method and then it can be
called from native code.Typical use would be passing a managed method as
callback to native, now it can be done by wrapping the method in a
delegate or directly using Marshal.GetFunctionPointerForDelegate.This's
fine as long as we make sure that delegate is not garbage
collected.[NativeCallable] introduce another way, where you can directly
load the function pointer of a native callable method and use it as
callback.This feature cannot be directly used from C#,but can be very
useful in dynamic code generation scenarios where you want a callback to
be passed to native.

Here's an example of how it can be used.

public static class NativeMethods {
 [DllImport("user32.dll")]
 public static extern int EnumWindows(IntPtr enumProc, IntPtr lParam);
}

//Method attributed with NativeCallable
[NativeCallable]
public static int CallbackMethod(IntPtr hWnd, IntPtr lParam){ return 1; }

Now you can generate the below IL to load native callable function pointer
( LDFTN) and then pass it a native method.
	.locals init ([0] native int ptr)
	nop
	ldftn      int32 CallbackMethod(native int,native int)
	stloc.0
	ldloc.0
	ldsfld     native int System.IntPtr::Zero
	call       bool NativeMethods::EnumWindows(native int,native int)
	pop
	ret

Encoding native callable methods as  ENCODE_METHOD_NATIVECALLABLE_HANDLE
so that we don't have to check for the custom attribute at runtime to
decode the method.Also fixing the remaining code review comments.

Adding runtime check to prevent Native Callable methods from being used as
calli target with an ldftn. Also adding some negative test cases , they
are disabled for now since the tests failfast and msbuild report it as
failure.

1 files changed, 77 insertions, 0 deletions

diff --git a/src/vm/comdelegate.cpp b/src/vm/comdelegate.cpp
index 66a19ac817..a6c7e063b1 100644
--- a/src/vm/comdelegate.cpp
+++ b/src/vm/comdelegate.cpp
@@ -29,6 +29,8 @@
 #include "security.h"
 #include "virtualcallstub.h"
 #include "callingconvention.h"
+#include "customattribute.h"
+#include "../md/compiler/custattr.h"
 #ifdef FEATURE_COMINTEROP
 #include "comcallablewrapper.h"
 #endif // FEATURE_COMINTEROP
@@ -1065,6 +1067,74 @@ BOOL COMDelegate::IsMethodAllowedToSinkReversePInvoke(MethodDesc *pMD)
 }
 #endif // FEATURE_CORECLR
 
+// Marshals a managed method to an unmanaged callback provided the 
+// managed method is static and it's parameters require no marshalling.
+PCODE COMDelegate::ConvertToCallback(MethodDesc* pMD)
+{
+    CONTRACTL
+    {
+        THROWS;
+    GC_TRIGGERS;
+    INJECT_FAULT(COMPlusThrowOM());
+    }
+    CONTRACTL_END;
+
+    PCODE pCode = NULL;
+
+    // only static methods are allowed
+    if (!pMD->IsStatic())
+        COMPlusThrow(kNotSupportedException, W("NotSupported_NonStaticMethod"));
+
+    // no generic methods
+    if (pMD->IsGenericMethodDefinition())
+        COMPlusThrow(kNotSupportedException, W("NotSupported_GenericMethod"));
+
+    // Arguments 
+    if (NDirect::MarshalingRequired(pMD, pMD->GetSig(), pMD->GetModule()))
+        COMPlusThrow(kNotSupportedException, W("NotSupported_NonBlittableTypes"));
+
+    // Get UMEntryThunk from appdomain thunkcache cache.
+    UMEntryThunk *pUMEntryThunk = GetAppDomain()->GetUMEntryThunkCache()->GetUMEntryThunk(pMD);
+
+#ifdef _TARGET_X86_
+
+    // System.Runtime.InteropServices.NativeCallableAttribute
+    BYTE* pData = NULL;
+    LONG cData = 0;
+    CorPinvokeMap callConv = (CorPinvokeMap)0;
+
+    HRESULT hr = pMD->GetMDImport()->GetCustomAttributeByName(pMD->GetMemberDef(), g_NativeCallableAttribute, (const VOID **)(&pData), (ULONG *)&cData);
+    IfFailThrow(hr);
+
+    if (cData > 0)
+    {
+        CustomAttributeParser ca(pData, cData);
+        // NativeCallable has two optional named arguments CallingConvention and EntryPoint.
+        CaNamedArg namedArgs[2];
+        CaTypeCtor caType(SERIALIZATION_TYPE_STRING);
+        // First, the void constructor.
+        IfFailThrow(ParseKnownCaArgs(ca, NULL, 0));
+
+        // Now the optional named properties
+        namedArgs[0].InitI4FieldEnum("CallingConvention", "System.Runtime.InteropServices.CallingConvention", (ULONG)callConv);
+        namedArgs[1].Init("EntryPoint", SERIALIZATION_TYPE_STRING, caType);
+        IfFailThrow(ParseKnownCaNamedArgs(ca, namedArgs, lengthof(namedArgs)));
+
+        callConv = (CorPinvokeMap)(namedArgs[0].val.u4 << 8);
+        // Let UMThunkMarshalInfo choose the default if calling convension not definied.
+        if (namedArgs[0].val.type.tag != SERIALIZATION_TYPE_UNDEFINED)
+        {
+            UMThunkMarshInfo* pUMThunkMarshalInfo = pUMEntryThunk->GetUMThunkMarshInfo();
+            pUMThunkMarshalInfo->SetCallingConvention(callConv);
+        }
+}
+#endif  //_TARGET_X86_
+
+    pCode = (PCODE)pUMEntryThunk->GetCode();
+    _ASSERTE(pCode != NULL);
+    return pCode;
+}
+
 // Marshals a delegate to a unmanaged callback.
 LPVOID COMDelegate::ConvertToCallback(OBJECTREF pDelegateObj)
 {
@@ -3188,6 +3258,13 @@ MethodDesc* COMDelegate::GetDelegateCtor(TypeHandle delegateType, MethodDesc *pT
     // that has the _methodBase field filled in with the LoaderAllocator of the collectible assembly
     // associated with the instantiation.
     BOOL fMaybeCollectibleAndStatic = FALSE;
+   
+    // Do not allow static methods with [NativeCallableAttribute] to be a delegate target.
+    // A native callable method is special and allowing it to be delegate target will destabilize the runtime.
+    if (pTargetMethod->HasNativeCallableAttribute())
+    {
+        COMPlusThrow(kNotSupportedException, W("NotSupported_NativeCallableTarget"));
+    }
 
     if (isStatic)
     {