Fix a bug in implementation of ICastable when CoreCLR throws exception on IL isinst if IsInstanceOfInterface() sets the exception out param.

Correct behavior:   If false is returned when IsInstanceOfInterface() called as part of a castclass then the usual InvalidCastException
will be thrown unless an alternate exception is assigned to the castError output parameter.
This parameter is ignored on successful casts or during the evaluation of an isinst (which returns null rather than throwing on error).

1 files changed, 8 insertions, 12 deletions

diff --git a/src/vm/prestub.cpp b/src/vm/prestub.cpp
index 691be10594..c9a45bcc9a 100644
--- a/src/vm/prestub.cpp
+++ b/src/vm/prestub.cpp
@@ -2617,21 +2617,17 @@ extern "C" SIZE_T STDCALL DynamicHelperWorker(TransitionBlock * pTransitionBlock
         {
         case ENCODE_ISINSTANCEOF_HELPER:
         case ENCODE_CHKCAST_HELPER:
-            if (*(Object **)pArgument == NULL || ObjIsInstanceOf(*(Object **)pArgument, th))
             {
-                result = (SIZE_T)(*(Object **)pArgument);
-            }
-            else
-            {
-                if (kind == ENCODE_CHKCAST_HELPER)
+                BOOL throwInvalidCast = (kind == ENCODE_CHKCAST_HELPER);
+                if (*(Object **)pArgument == NULL || ObjIsInstanceOf(*(Object **)pArgument, th, throwInvalidCast))
                 {
-                    OBJECTREF obj = ObjectToOBJECTREF(*(Object **)pArgument);
-                    GCPROTECT_BEGIN(obj);
-                    COMPlusThrowInvalidCastException(&obj, th);
-                    GCPROTECT_END();
+                    result = (SIZE_T)(*(Object **)pArgument);
+                }
+                else
+                {
+                    _ASSERTE (!throwInvalidCast);
+                    result = NULL;
                 }
- 
-                result = NULL;
             }
             break;
         case ENCODE_STATIC_BASE_NONGC_HELPER: