Enable returning more complex structures via PInvoke returns. (#21470)

* Add test verifying behavior in dotnet/coreclr#19676.

* Clean up test code.

* Test what happens if we enable returning structures by value.

* Use braced initializer.

* Update Decimal tests to expect that returning a decimal by LPStruct or Currency works.

* Change handle-in-struct marshalling to expect a NotSupportedException thrown at marshal time instead of expecting a MarshalDirectiveException at signature time.

* Update Decimal Reverse-PInvoke tests.

* Disable some previously disabled return marshalling types and add a nice comment block explaining why they're disabled.

* Enable marshalling DateTime return values and add a test.

* Rename IsUnuspportedValueTypeReturn

* Add return test for ArrayWithOffset

* Remove extraneous P/Invoke.

* Fix spelling.

* Add test for successfully returning a struct that has one field of a type that is return-type blocked.

* Add explicit struct return test.

* Clean up tests.

* Fix grammer.

* Add test for struct whose managed layout doesn't require a stdcall return buffer but whose native layout does.

* Add test verifying HandleRef behavior.

* Clean up IsUnsupportedTypedefReturn per PR feedback.

1 files changed, 26 insertions, 19 deletions

diff --git a/src/vm/dllimport.cpp b/src/vm/dllimport.cpp
index 2ce100b4b9..8c86882006 100644
--- a/src/vm/dllimport.cpp
+++ b/src/vm/dllimport.cpp
@@ -3706,31 +3706,38 @@ static MarshalInfo::MarshalType DoMarshalReturnValue(MetaSig&           msig,
         else
 #endif // FEATURE_COMINTEROP
         {
-            if (marshalType > MarshalInfo::MARSHAL_TYPE_DOUBLE && IsUnsupportedValueTypeReturn(msig))
-            {
-                if (marshalType == MarshalInfo::MARSHAL_TYPE_BLITTABLEVALUECLASS
-                        || marshalType == MarshalInfo::MARSHAL_TYPE_GUID
-                        || marshalType == MarshalInfo::MARSHAL_TYPE_DECIMAL
+            if (marshalType == MarshalInfo::MARSHAL_TYPE_BLITTABLEVALUECLASS
+                    || marshalType == MarshalInfo::MARSHAL_TYPE_VALUECLASS
+                    || marshalType == MarshalInfo::MARSHAL_TYPE_GUID
+                    || marshalType == MarshalInfo::MARSHAL_TYPE_DECIMAL
 #ifdef FEATURE_COMINTEROP                    
-                        || marshalType == MarshalInfo::MARSHAL_TYPE_DATETIME
+                    || marshalType == MarshalInfo::MARSHAL_TYPE_DATETIME
 #endif // FEATURE_COMINTEROP
-                   )
-                {
-                    if (SF_IsHRESULTSwapping(dwStubFlags))
-                    {
-                        // V1 restriction: we could implement this but it's late in the game to do so.
-                        COMPlusThrow(kMarshalDirectiveException, IDS_EE_NDIRECT_UNSUPPORTED_SIG);
-                    }
-                }
-                else if (marshalType == MarshalInfo::MARSHAL_TYPE_HANDLEREF)
-                {
-                    COMPlusThrow(kMarshalDirectiveException, IDS_EE_BADMARSHAL_HANDLEREFRESTRICTION);
-                }
-                else
+                )
+            {
+                if (SF_IsHRESULTSwapping(dwStubFlags))
                 {
+                    // V1 restriction: we could implement this but it's late in the game to do so.
                     COMPlusThrow(kMarshalDirectiveException, IDS_EE_NDIRECT_UNSUPPORTED_SIG);
                 }
             }
+            else if (marshalType == MarshalInfo::MARSHAL_TYPE_CURRENCY
+                    || marshalType == MarshalInfo::MARSHAL_TYPE_ARRAYWITHOFFSET
+                    || marshalType == MarshalInfo::MARSHAL_TYPE_ARGITERATOR
+#ifdef FEATURE_COMINTEROP
+                    || marshalType == MarshalInfo::MARSHAL_TYPE_OLECOLOR
+#endif // FEATURE_COMINTEROP
+            )
+            {
+                // Each of these types are non-blittable and according to its managed size should be returned in a return buffer on x86 in stdcall.
+                // However, its native size is small enough to be returned by-value.
+                // We don't know the native type representation early enough to get this correct, so we throw an exception here.
+                COMPlusThrow(kMarshalDirectiveException, IDS_EE_NDIRECT_UNSUPPORTED_SIG);
+            }
+            else if (IsUnsupportedTypedrefReturn(msig))
+            {
+                COMPlusThrow(kMarshalDirectiveException, IDS_EE_NDIRECT_UNSUPPORTED_SIG);
+            }
 
 #ifdef FEATURE_COMINTEROP
             if (marshalType == MarshalInfo::MARSHAL_TYPE_OBJECT && !SF_IsHRESULTSwapping(dwStubFlags))