[RyuJit/ARM32] Fix assertion of failure on "Instruction cannot be encoded" (#10789)

Fix assertion of failure on "Instruction cannot be encoded"

I figured out the bit set mask for '0xffffff00' which is the imm is not supportted on ARM.
So at first, It would be checked the validation of using instruction with immediate constants.
If not, it would be use the instruction with registers.

2 files changed, 38 insertions, 8 deletions

diff --git a/src/jit/codegenarmarch.cpp b/src/jit/codegenarmarch.cpp
index e887a8c0f1..c0ef471e36 100644
--- a/src/jit/codegenarmarch.cpp
+++ b/src/jit/codegenarmarch.cpp
@@ -1422,7 +1422,20 @@ void CodeGen::genIntToIntCast(GenTreePtr treeNode)
             // we only have to check for any bits set in 'typeMask'
 
             noway_assert(castInfo.typeMask != 0);
+#if defined(_TARGET_ARM_)
+            if (arm_Valid_Imm_For_Instr(INS_tst, castInfo.typeMask, INS_FLAGS_DONT_CARE))
+            {
+                emit->emitIns_R_I(INS_tst, cmpSize, sourceReg, castInfo.typeMask);
+            }
+            else
+            {
+                noway_assert(tmpReg != REG_NA);
+                instGen_Set_Reg_To_Imm(cmpSize, tmpReg, castInfo.typeMask);
+                emit->emitIns_R_R(INS_tst, cmpSize, sourceReg, tmpReg);
+            }
+#elif defined(_TARGET_ARM64_)
             emit->emitIns_R_I(INS_tst, cmpSize, sourceReg, castInfo.typeMask);
+#endif // _TARGET_ARM*
             emitJumpKind jmpNotEqual = genJumpKindForOper(GT_NE, CK_SIGNED);
             genJumpToThrowHlpBlk(jmpNotEqual, SCK_OVERFLOW);
         }
diff --git a/src/jit/lsraarm.cpp b/src/jit/lsraarm.cpp
index ecb7c46e92..1dd167c837 100644
--- a/src/jit/lsraarm.cpp
+++ b/src/jit/lsraarm.cpp
@@ -332,16 +332,33 @@ void Lowering::TreeNodeInfoInit(GenTree* tree)
                 var_types srcType = castOp->TypeGet();
                 emitAttr  cmpSize = EA_ATTR(genTypeSize(srcType));
 
-                // If we cannot store the comparisons in an immediate for either
-                // comparing against the max or min value, then we will need to
-                // reserve a temporary register.
+                // If we cannot store data in an immediate for instructions,
+                // then we will need to reserve a temporary register.
 
-                bool canStoreMaxValue = emitter::emitIns_valid_imm_for_cmp(castInfo.typeMax, INS_FLAGS_DONT_CARE);
-                bool canStoreMinValue = emitter::emitIns_valid_imm_for_cmp(castInfo.typeMin, INS_FLAGS_DONT_CARE);
-
-                if (!canStoreMaxValue || !canStoreMinValue)
+                if (!castInfo.signCheckOnly) // In case of only sign check, temp regs are not needeed.
                 {
-                    info->internalIntCount = 1;
+                    if (castInfo.unsignedSource || castInfo.unsignedDest)
+                    {
+                        // check typeMask
+                        bool canStoreTypeMask = emitter::emitIns_valid_imm_for_alu(castInfo.typeMask);
+                        if (!canStoreTypeMask)
+                        {
+                            info->internalIntCount = 1;
+                        }
+                    }
+                    else
+                    {
+                        // For comparing against the max or min value
+                        bool canStoreMaxValue =
+                            emitter::emitIns_valid_imm_for_cmp(castInfo.typeMax, INS_FLAGS_DONT_CARE);
+                        bool canStoreMinValue =
+                            emitter::emitIns_valid_imm_for_cmp(castInfo.typeMin, INS_FLAGS_DONT_CARE);
+
+                        if (!canStoreMaxValue || !canStoreMinValue)
+                        {
+                            info->internalIntCount = 1;
+                        }
+                    }
                 }
             }
         }