Handle multireg copies correctly (#19588)

* Handle multireg copies correctly

Fix #19029

4 files changed, 116 insertions, 6 deletions

diff --git a/src/jit/codegenarm.cpp b/src/jit/codegenarm.cpp
index f45bbfea69..19adebd08b 100644
--- a/src/jit/codegenarm.cpp
+++ b/src/jit/codegenarm.cpp
@@ -769,7 +769,7 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode)
         sourceIsLocal = true;
     }
 
-    bool dstOnStack = dstAddr->OperIsLocalAddr();
+    bool dstOnStack = dstAddr->gtSkipReloadOrCopy()->OperIsLocalAddr();
 
 #ifdef DEBUG
     assert(!dstAddr->isContained());
diff --git a/src/jit/codegenarm64.cpp b/src/jit/codegenarm64.cpp
index 30ab08f04d..d538896647 100644
--- a/src/jit/codegenarm64.cpp
+++ b/src/jit/codegenarm64.cpp
@@ -2469,7 +2469,7 @@ void CodeGen::genCodeForCpObj(GenTreeObj* cpObjNode)
         sourceIsLocal = true;
     }
 
-    bool dstOnStack = dstAddr->OperIsLocalAddr();
+    bool dstOnStack = dstAddr->gtSkipReloadOrCopy()->OperIsLocalAddr();
 
 #ifdef DEBUG
     assert(!dstAddr->isContained());
diff --git a/src/jit/codegenarmarch.cpp b/src/jit/codegenarmarch.cpp
index 9f0c75e00b..b66e309eaa 100644
--- a/src/jit/codegenarmarch.cpp
+++ b/src/jit/codegenarmarch.cpp
@@ -2069,7 +2069,8 @@ void CodeGen::genRegCopy(GenTree* treeNode)
     regNumber targetReg  = treeNode->gtRegNum;
     assert(targetReg != REG_NA);
 
-    GenTree* op1 = treeNode->gtOp.gtOp1;
+    GenTree*  op1       = treeNode->gtOp.gtOp1;
+    regNumber sourceReg = genConsumeReg(op1);
 
     // Check whether this node and the node from which we're copying the value have the same
     // register type.
@@ -2080,7 +2081,7 @@ void CodeGen::genRegCopy(GenTree* treeNode)
     if (varTypeIsFloating(treeNode) != varTypeIsFloating(op1))
     {
 #ifdef _TARGET_ARM64_
-        inst_RV_RV(INS_fmov, targetReg, genConsumeReg(op1), targetType);
+        inst_RV_RV(INS_fmov, targetReg, sourceReg, targetType);
 #else  // !_TARGET_ARM64_
         if (varTypeIsFloating(treeNode))
         {
@@ -2105,9 +2106,21 @@ void CodeGen::genRegCopy(GenTree* treeNode)
         }
 #endif // !_TARGET_ARM64_
     }
+    else if (targetType == TYP_STRUCT)
+    {
+        noway_assert(op1->IsMultiRegNode() && !op1->IsCopyOrReload());
+        unsigned regCount = op1->GetMultiRegCount();
+        for (unsigned i = 0; i < regCount; i++)
+        {
+            regNumber srcReg  = op1->GetRegByIndex(i);
+            regNumber tgtReg  = treeNode->AsCopyOrReload()->GetRegNumByIdx(i);
+            var_types regType = op1->GetRegTypeByIndex(i);
+            inst_RV_RV(ins_Copy(regType), tgtReg, srcReg, regType);
+        }
+    }
     else
     {
-        inst_RV_RV(ins_Copy(targetType), targetReg, genConsumeReg(op1), targetType);
+        inst_RV_RV(ins_Copy(targetType), targetReg, sourceReg, targetType);
     }
 
     if (op1->IsLocal())
diff --git a/src/jit/gentree.h b/src/jit/gentree.h
index 4195bf4650..cd402168fa 100644
--- a/src/jit/gentree.h
+++ b/src/jit/gentree.h
@@ -1735,9 +1735,15 @@ public:
     // Returns true if it is a node returning its value in more than one register
     inline bool IsMultiRegNode() const;
 
+    // Returns the number of registers defined by a multireg node.
+    unsigned GetMultiRegCount();
+
     // Returns the regIndex'th register defined by a possibly-multireg node.
     regNumber GetRegByIndex(int regIndex);
 
+    // Returns the type of the regIndex'th register defined by a multi-reg node.
+    var_types GetRegTypeByIndex(int regIndex);
+
     // Returns true if it is a GT_COPY or GT_RELOAD node
     inline bool IsCopyOrReload() const;
 
@@ -5431,7 +5437,7 @@ struct GenTreeCopyOrReload : public GenTreeUnOp
 {
 #if FEATURE_MULTIREG_RET
     // State required to support copy/reload of a multi-reg call node.
-    // First register is is always given by gtRegNum.
+    // The first register is always given by gtRegNum.
     //
     regNumberSmall gtOtherRegs[MAX_RET_REG_COUNT - 1];
 #endif
@@ -5536,6 +5542,26 @@ struct GenTreeCopyOrReload : public GenTreeUnOp
 #endif
     }
 
+    unsigned GetRegCount()
+    {
+        if (gtRegNum == REG_NA)
+        {
+            return 0;
+        }
+#if FEATURE_MULTIREG_RET
+        for (unsigned i = 0; i < MAX_RET_REG_COUNT - 1; ++i)
+        {
+            if (gtOtherRegs[i] == REG_NA)
+            {
+                return i + 1;
+            }
+        }
+        return MAX_RET_REG_COUNT;
+#else
+        return 1;
+#endif
+    }
+
     GenTreeCopyOrReload(genTreeOps oper, var_types type, GenTree* op1) : GenTreeUnOp(oper, type, op1)
     {
         gtRegNum = REG_NA;
@@ -6022,6 +6048,40 @@ inline bool GenTree::IsMultiRegNode() const
 
     return false;
 }
+//-----------------------------------------------------------------------------------
+// GetMultiRegCount: Return the register count for a multi-reg node.
+//
+// Arguments:
+//     None
+//
+// Return Value:
+//     Returns the number of registers defined by this node.
+inline unsigned GenTree::GetMultiRegCount()
+{
+    if (IsMultiRegCall())
+    {
+        return AsCall()->GetReturnTypeDesc()->GetReturnRegCount();
+    }
+
+#if FEATURE_ARG_SPLIT
+    if (OperIsPutArgSplit())
+    {
+        return AsPutArgSplit()->gtNumRegs;
+    }
+#endif
+#if defined(_TARGET_ARM_)
+    if (OperIsMultiRegOp())
+    {
+        return AsMultiRegOp()->GetRegCount();
+    }
+    if (OperIs(GT_COPY, GT_RELOAD))
+    {
+        return AsCopyOrReload()->GetRegCount();
+    }
+#endif
+    assert(!"GetMultiRegCount called with non-multireg node");
+    return 1;
+}
 
 //-----------------------------------------------------------------------------------
 // GetRegByIndex: Get a specific register, based on regIndex, that is produced
@@ -6064,6 +6124,43 @@ inline regNumber GenTree::GetRegByIndex(int regIndex)
     return REG_NA;
 }
 
+//-----------------------------------------------------------------------------------
+// GetRegTypeByIndex: Get a specific register's type, based on regIndex, that is produced
+//                    by this multi-reg node.
+//
+// Arguments:
+//     regIndex - which register type to return
+//
+// Return Value:
+//     The register type assigned to this index for this node.
+//
+// Notes:
+//     This must be a multireg node that is *not* a copy or reload (which must retrieve the
+//     type from its source), and 'regIndex' must be a valid index for this node.
+//
+inline var_types GenTree::GetRegTypeByIndex(int regIndex)
+{
+    if (IsMultiRegCall())
+    {
+        return AsCall()->AsCall()->GetReturnTypeDesc()->GetReturnRegType(regIndex);
+    }
+
+#if FEATURE_ARG_SPLIT
+    if (OperIsPutArgSplit())
+    {
+        return AsPutArgSplit()->GetRegType(regIndex);
+    }
+#endif
+#if defined(_TARGET_ARM_)
+    if (OperIsMultiRegOp())
+    {
+        return AsMultiRegOp()->GetRegType(regIndex);
+    }
+#endif
+    assert(!"Invalid node type for GetRegTypeByIndex");
+    return TYP_UNDEF;
+}
+
 //-------------------------------------------------------------------------
 // IsCopyOrReload: whether this is a GT_COPY or GT_RELOAD node.
 //