Fix legality checks in SetRegOptionalForBinOp.

It is not legal to make an operand register optional on xarch if the type
of the operand is narrower than the type of the operator and the contents
of the upper bits are not known to be zero. `SetRegOptionalForBinOp` was
not considering the case where `op1` was legal but `op2` was the preferred
register-optional operand and was illegal.

Fixes #11734.

1 files changed, 8 insertions, 6 deletions

diff --git a/src/jit/lower.h b/src/jit/lower.h
index bcc2bafdab..0d4067a4c9 100644
--- a/src/jit/lower.h
+++ b/src/jit/lower.h
@@ -178,15 +178,17 @@ private:
     {
         assert(GenTree::OperIsBinary(tree->OperGet()));
 
-        GenTree* op1 = tree->gtGetOp1();
-        GenTree* op2 = tree->gtGetOp2();
+        GenTree* const op1 = tree->gtGetOp1();
+        GenTree* const op2 = tree->gtGetOp2();
 
-        if (tree->OperIsCommutative() && tree->TypeGet() == op1->TypeGet())
+        const bool op1Legal = tree->OperIsCommutative() && (tree->TypeGet() == op1->TypeGet());
+        const bool op2Legal = tree->TypeGet() == op2->TypeGet();
+
+        if (op1Legal)
         {
-            GenTree* preferredOp = PreferredRegOptionalOperand(tree);
-            SetRegOptional(preferredOp);
+            SetRegOptional(op2Legal ? PreferredRegOptionalOperand(tree) : op1);
         }
-        else if (tree->TypeGet() == op2->TypeGet())
+        else if (op2Legal)
         {
             SetRegOptional(op2);
         }