diff options
-rw-r--r-- | src/jit/codegenarm64.cpp | 2 | ||||
-rw-r--r-- | src/jit/codegenlinear.h | 1 | ||||
-rw-r--r-- | src/jit/codegenxarch.cpp | 271 | ||||
-rw-r--r-- | src/jit/compiler.h | 2 | ||||
-rw-r--r-- | src/jit/flowgraph.cpp | 34 | ||||
-rw-r--r-- | src/jit/gentree.cpp | 10 | ||||
-rw-r--r-- | src/jit/gentree.h | 25 | ||||
-rw-r--r-- | src/jit/gtlist.h | 4 | ||||
-rw-r--r-- | src/jit/gtstructs.h | 1 | ||||
-rw-r--r-- | src/jit/lir.cpp | 2 | ||||
-rw-r--r-- | src/jit/lower.cpp | 257 | ||||
-rw-r--r-- | src/jit/lower.h | 4 | ||||
-rw-r--r-- | src/jit/lowerarm64.cpp | 4 | ||||
-rw-r--r-- | src/jit/lowerxarch.cpp | 11 | ||||
-rw-r--r-- | src/jit/lsra.cpp | 4 | ||||
-rw-r--r-- | src/jit/rationalize.cpp | 2 |
16 files changed, 403 insertions, 231 deletions
diff --git a/src/jit/codegenarm64.cpp b/src/jit/codegenarm64.cpp index ca0df53a34..14d20521a1 100644 --- a/src/jit/codegenarm64.cpp +++ b/src/jit/codegenarm64.cpp @@ -5147,7 +5147,7 @@ void CodeGen::genConsumeRegs(GenTree* tree) } else if (tree->OperGet() == GT_AND) { - // This is the special contained GT_AND that we created in Lowering::LowerCmp() + // This is the special contained GT_AND that we created in Lowering::TreeNodeInfoInitCmp() // Now we need to consume the operands of the GT_AND node. genConsumeOperands(tree->AsOp()); } diff --git a/src/jit/codegenlinear.h b/src/jit/codegenlinear.h index 7338b462da..6cc437bd8e 100644 --- a/src/jit/codegenlinear.h +++ b/src/jit/codegenlinear.h @@ -53,7 +53,6 @@ void genCompareInt(GenTreePtr treeNode); #if !defined(_TARGET_64BIT_) void genCompareLong(GenTreePtr treeNode); -void genJTrueLong(GenTreePtr treeNode); #endif #ifdef FEATURE_SIMD diff --git a/src/jit/codegenxarch.cpp b/src/jit/codegenxarch.cpp index e39fcf1555..402d80097d 100644 --- a/src/jit/codegenxarch.cpp +++ b/src/jit/codegenxarch.cpp @@ -2477,23 +2477,18 @@ void CodeGen::genCodeForTreeNode(GenTreePtr treeNode) // X86 Long comparison else if (varTypeIsLong(op1Type)) { - // When not materializing the result in a register, the compare logic is generated - // when we generate the GT_JTRUE. - if (treeNode->gtRegNum != REG_NA) - { - genCompareLong(treeNode); - } - else - { - if ((treeNode->gtNext != nullptr) && (treeNode->gtNext->OperGet() != GT_JTRUE)) - { - NYI("Long compare/reload/jtrue sequence"); - } +#ifdef DEBUG + // The result of an unlowered long compare on a 32-bit target must either be + // a) materialized into a register, or + // b) unused. + // + // A long compare that has a result that is used but not materialized into a register should + // have been handled by Lowering::LowerCompare. - // We generate the compare when we generate the GT_JTRUE, but we need to consume - // the operands now. - genConsumeOperands(treeNode->AsOp()); - } + LIR::Use use; + assert((treeNode->gtRegNum != REG_NA) || !LIR::AsRange(compiler->compCurBB).TryGetUse(treeNode, &use)); +#endif + genCompareLong(treeNode); } #endif // !defined(_TARGET_64BIT_) else @@ -2511,55 +2506,63 @@ void CodeGen::genCodeForTreeNode(GenTreePtr treeNode) assert(compiler->compCurBB->bbJumpKind == BBJ_COND); #if !defined(_TARGET_64BIT_) - // For long compares, we emit special logic - if (varTypeIsLong(cmp->gtGetOp1())) - { - genJTrueLong(cmp); - } - else + // Long-typed compares should have been handled by Lowering::LowerCompare. + assert(!varTypeIsLong(cmp->gtGetOp1())); #endif - { - // Get the "kind" and type of the comparison. Note that whether it is an unsigned cmp - // is governed by a flag NOT by the inherent type of the node - // TODO-XArch-CQ: Check if we can use the currently set flags. - emitJumpKind jumpKind[2]; - bool branchToTrueLabel[2]; - genJumpKindsForTree(cmp, jumpKind, branchToTrueLabel); - - BasicBlock* skipLabel = nullptr; - if (jumpKind[0] != EJ_NONE) - { - BasicBlock* jmpTarget; - if (branchToTrueLabel[0]) - { - jmpTarget = compiler->compCurBB->bbJumpDest; - } - else - { - // This case arises only for ordered GT_EQ right now - assert((cmp->gtOper == GT_EQ) && ((cmp->gtFlags & GTF_RELOP_NAN_UN) == 0)); - skipLabel = genCreateTempLabel(); - jmpTarget = skipLabel; - } - inst_JMP(jumpKind[0], jmpTarget); - } + // Get the "kind" and type of the comparison. Note that whether it is an unsigned cmp + // is governed by a flag NOT by the inherent type of the node + // TODO-XArch-CQ: Check if we can use the currently set flags. + emitJumpKind jumpKind[2]; + bool branchToTrueLabel[2]; + genJumpKindsForTree(cmp, jumpKind, branchToTrueLabel); - if (jumpKind[1] != EJ_NONE) + BasicBlock* skipLabel = nullptr; + if (jumpKind[0] != EJ_NONE) + { + BasicBlock* jmpTarget; + if (branchToTrueLabel[0]) { - // the second conditional branch always has to be to the true label - assert(branchToTrueLabel[1]); - inst_JMP(jumpKind[1], compiler->compCurBB->bbJumpDest); + jmpTarget = compiler->compCurBB->bbJumpDest; } - - if (skipLabel != nullptr) + else { - genDefineTempLabel(skipLabel); + // This case arises only for ordered GT_EQ right now + assert((cmp->gtOper == GT_EQ) && ((cmp->gtFlags & GTF_RELOP_NAN_UN) == 0)); + skipLabel = genCreateTempLabel(); + jmpTarget = skipLabel; } + + inst_JMP(jumpKind[0], jmpTarget); + } + + if (jumpKind[1] != EJ_NONE) + { + // the second conditional branch always has to be to the true label + assert(branchToTrueLabel[1]); + inst_JMP(jumpKind[1], compiler->compCurBB->bbJumpDest); + } + + if (skipLabel != nullptr) + { + genDefineTempLabel(skipLabel); } } break; + case GT_JCC: + { + GenTreeJumpCC* jcc = treeNode->AsJumpCC(); + + assert(compiler->compCurBB->bbJumpKind == BBJ_COND); + + CompareKind compareKind = ((jcc->gtFlags & GTF_UNSIGNED) != 0) ? CK_UNSIGNED : CK_SIGNED; + emitJumpKind jumpKind = genJumpKindForOper(jcc->gtCondition, compareKind); + + inst_JMP(jumpKind, compiler->compCurBB->bbJumpDest); + } + break; + case GT_RETURNTRAP: { // this is nothing but a conditional call to CORINFO_HELP_STOP_FOR_GC @@ -5246,7 +5249,7 @@ void CodeGen::genConsumeRegs(GenTree* tree) } else if (tree->OperGet() == GT_AND) { - // This is the special contained GT_AND that we created in Lowering::LowerCmp() + // This is the special contained GT_AND that we created in Lowering::TreeNodeInfoInitCmp() // Now we need to consume the operands of the GT_AND node. genConsumeOperands(tree->AsOp()); } @@ -7059,8 +7062,6 @@ void CodeGen::genCompareLong(GenTreePtr treeNode) genConsumeOperands(tree); - assert(targetReg != REG_NA); - GenTreePtr loOp1 = op1->gtGetOp1(); GenTreePtr hiOp1 = op1->gtGetOp2(); GenTreePtr loOp2 = op2->gtGetOp1(); @@ -7074,6 +7075,12 @@ void CodeGen::genCompareLong(GenTreePtr treeNode) // Emit the compare instruction getEmitter()->emitInsBinary(ins, cmpAttr, hiOp1, hiOp2); + // If the result is not being materialized in a register, we're done. + if (targetReg == REG_NA) + { + return; + } + // Generate the first jump for the high compare CompareKind compareKind = ((tree->gtFlags & GTF_UNSIGNED) != 0) ? CK_UNSIGNED : CK_SIGNED; @@ -7144,152 +7151,6 @@ void CodeGen::genCompareLong(GenTreePtr treeNode) genProduceReg(tree); } } - -//------------------------------------------------------------------------ -// genJTrueLong: Generate code for comparing two longs on x86 for the case where the result -// is not manifested in a register. -// -// Arguments: -// treeNode - the compare tree -// -// Return Value: -// None. -// Comments: -// For long compares, we need to compare the high parts of operands first, then the low parts. -// We only have to do the low compare if the high parts of the operands are equal. -// -// In the case where the result of a rel-op is not realized in a register, we generate: -// -// Opcode x86 equivalent Comment -// ------ -------------- ------- -// -// GT_LT; unsigned cmp hiOp1,hiOp2 -// jb trueLabel -// ja falseLabel -// cmp loOp1,loOp2 -// jb trueLabel -// falseLabel: -// -// GT_LE; unsigned cmp hiOp1,hiOp2 -// jb trueLabel -// ja falseLabel -// cmp loOp1,loOp2 -// jbe trueLabel -// falseLabel: -// -// GT_GT; unsigned cmp hiOp1,hiOp2 -// ja trueLabel -// jb falseLabel -// cmp loOp1,loOp2 -// ja trueLabel -// falseLabel: -// -// GT_GE; unsigned cmp hiOp1,hiOp2 -// ja trueLabel -// jb falseLabel -// cmp loOp1,loOp2 -// jae trueLabel -// falseLabel: -// -// GT_LT; signed cmp hiOp1,hiOp2 -// jl trueLabel -// jg falseLabel -// cmp loOp1,loOp2 -// jb trueLabel -// falseLabel: -// -// GT_LE; signed cmp hiOp1,hiOp2 -// jl trueLabel -// jg falseLabel -// cmp loOp1,loOp2 -// jbe trueLabel -// falseLabel: -// -// GT_GT; signed cmp hiOp1,hiOp2 -// jg trueLabel -// jl falseLabel -// cmp loOp1,loOp2 -// ja trueLabel -// falseLabel: -// -// GT_GE; signed cmp hiOp1,hiOp2 -// jg trueLabel -// jl falseLabel -// cmp loOp1,loOp2 -// jae trueLabel -// falseLabel: -// -// GT_EQ; cmp hiOp1,hiOp2 -// jne falseLabel -// cmp loOp1,loOp2 -// je trueLabel -// falseLabel: -// -// GT_NE; cmp hiOp1,hiOp2 -// jne labelTrue -// cmp loOp1,loOp2 -// jne trueLabel -// falseLabel: -// -// TODO-X86-CQ: Check if hi or lo parts of op2 are 0 and change the compare to a test. -void CodeGen::genJTrueLong(GenTreePtr treeNode) -{ - assert(treeNode->OperIsCompare()); - - GenTreeOp* tree = treeNode->AsOp(); - GenTreePtr op1 = tree->gtOp1; - GenTreePtr op2 = tree->gtOp2; - - assert(varTypeIsLong(op1->TypeGet())); - assert(varTypeIsLong(op2->TypeGet())); - - regNumber targetReg = treeNode->gtRegNum; - - assert(targetReg == REG_NA); - - GenTreePtr loOp1 = op1->gtGetOp1(); - GenTreePtr hiOp1 = op1->gtGetOp2(); - GenTreePtr loOp2 = op2->gtGetOp1(); - GenTreePtr hiOp2 = op2->gtGetOp2(); - - // Emit the compare instruction - getEmitter()->emitInsBinary(INS_cmp, EA_4BYTE, hiOp1, hiOp2); - - // Generate the first jump for the high compare - CompareKind compareKind = ((tree->gtFlags & GTF_UNSIGNED) != 0) ? CK_UNSIGNED : CK_SIGNED; - - // TODO-X86-CQ: If the next block is a BBJ_ALWAYS, we can set falseLabel = compiler->compCurBB->bbNext->bbJumpDest. - BasicBlock* falseLabel = genCreateTempLabel(); - - emitJumpKind jumpKindHi[2]; - - // Generate the jumps for the high compare - genJumpKindsForTreeLongHi(tree, jumpKindHi); - - BasicBlock* trueLabel = compiler->compCurBB->bbJumpDest; - - if (jumpKindHi[0] != EJ_NONE) - { - inst_JMP(jumpKindHi[0], trueLabel); - } - - if (jumpKindHi[1] != EJ_NONE) - { - inst_JMP(jumpKindHi[1], falseLabel); - } - - // The low jump must be unsigned - emitJumpKind jumpKindLo = genJumpKindForOper(tree->gtOper, CK_UNSIGNED); - - // Emit the comparison and the jump to the trueLabel - getEmitter()->emitInsBinary(INS_cmp, EA_4BYTE, loOp1, loOp2); - - inst_JMP(jumpKindLo, trueLabel); - - // Generate falseLabel, which is the false path. We will jump here if the high compare is false - // or fall through if the low compare is false. - genDefineTempLabel(falseLabel); -} #endif //! defined(_TARGET_64BIT_) //------------------------------------------------------------------------ @@ -7469,7 +7330,7 @@ void CodeGen::genCompareInt(GenTreePtr treeNode) { // Do we have a short compare against a constant in op2? // - // We checked for this case in LowerCmp() and if we can perform a small + // We checked for this case in TreeNodeInfoInitCmp() and if we can perform a small // compare immediate we labeled this compare with a GTF_RELOP_SMALL // and for unsigned small non-equality compares the GTF_UNSIGNED flag. // @@ -7524,7 +7385,7 @@ void CodeGen::genCompareInt(GenTreePtr treeNode) if (op1->isContained()) { // op1 can be a contained memory op - // or the special contained GT_AND that we created in Lowering::LowerCmp() + // or the special contained GT_AND that we created in Lowering::TreeNodeInfoInitCmp() // if ((op1->OperGet() == GT_AND)) { diff --git a/src/jit/compiler.h b/src/jit/compiler.h index 28dddb1375..50cc3f924d 100644 --- a/src/jit/compiler.h +++ b/src/jit/compiler.h @@ -4072,7 +4072,7 @@ public: void fgUnreachableBlock(BasicBlock* block); - void fgRemoveJTrue(BasicBlock* block); + void fgRemoveConditionalJump(BasicBlock* block); BasicBlock* fgLastBBInMainFunction(); diff --git a/src/jit/flowgraph.cpp b/src/jit/flowgraph.cpp index 9458967d25..97eb5cdaee 100644 --- a/src/jit/flowgraph.cpp +++ b/src/jit/flowgraph.cpp @@ -9339,6 +9339,7 @@ inline bool OperIsControlFlow(genTreeOps oper) switch (oper) { case GT_JTRUE: + case GT_JCC: case GT_SWITCH: case GT_LABEL: @@ -10019,10 +10020,10 @@ void Compiler::fgUnreachableBlock(BasicBlock* block) /***************************************************************************************************** * - * Function called to remove or morph a GT_JTRUE statement when we jump to the same + * Function called to remove or morph a jump when we jump to the same * block when both the condition is true or false. */ -void Compiler::fgRemoveJTrue(BasicBlock* block) +void Compiler::fgRemoveConditionalJump(BasicBlock* block) { noway_assert(block->bbJumpKind == BBJ_COND && block->bbJumpDest == block->bbNext); assert(compRationalIRForm == block->IsLIR()); @@ -10053,7 +10054,7 @@ void Compiler::fgRemoveJTrue(BasicBlock* block) LIR::Range& blockRange = LIR::AsRange(block); GenTree* test = blockRange.LastNode(); - assert(test->OperGet() == GT_JTRUE); + assert(test->OperIsConditionalJump()); bool isClosed; unsigned sideEffects; @@ -10109,7 +10110,7 @@ void Compiler::fgRemoveJTrue(BasicBlock* block) { test->gtStmtExpr = sideEffList; - fgMorphBlockStmt(block, test DEBUGARG("fgRemoveJTrue")); + fgMorphBlockStmt(block, test DEBUGARG("fgRemoveConditionalJump")); } } } @@ -10545,7 +10546,7 @@ void Compiler::fgRemoveBlock(BasicBlock* block, bool unreachable) // Make sure we are replacing "block" with "succBlock" in predBlock->bbJumpDest. noway_assert(predBlock->bbJumpDest == block); predBlock->bbJumpDest = succBlock; - fgRemoveJTrue(predBlock); + fgRemoveConditionalJump(predBlock); break; } @@ -10605,7 +10606,7 @@ void Compiler::fgRemoveBlock(BasicBlock* block, bool unreachable) /* Check for branch to next block */ if (bPrev->bbJumpDest == bPrev->bbNext) { - fgRemoveJTrue(bPrev); + fgRemoveConditionalJump(bPrev); } break; @@ -13793,7 +13794,7 @@ bool Compiler::fgOptimizeBranchToNext(BasicBlock* block, BasicBlock* bNext, Basi { LIR::Range& blockRange = LIR::AsRange(block); GenTree* jmp = blockRange.LastNode(); - assert(jmp->OperGet() == GT_JTRUE); + assert(jmp->OperIsConditionalJump()); bool isClosed; unsigned sideEffects; @@ -15879,11 +15880,18 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication) /* Reverse the jump condition */ GenTree* test = block->lastNode(); - noway_assert(test->gtOper == GT_JTRUE); + noway_assert(test->OperIsConditionalJump()); - GenTree* cond = gtReverseCond(test->gtOp.gtOp1); - assert(cond == test->gtOp.gtOp1); // Ensure `gtReverseCond` did not create a new node. - test->gtOp.gtOp1 = cond; + if (test->OperGet() == GT_JTRUE) + { + GenTree* cond = gtReverseCond(test->gtOp.gtOp1); + assert(cond == test->gtOp.gtOp1); // Ensure `gtReverseCond` did not create a new node. + test->gtOp.gtOp1 = cond; + } + else + { + gtReverseCond(test); + } // Optimize the Conditional JUMP to go to the new target block->bbJumpDest = bNext->bbJumpDest; @@ -19937,11 +19945,11 @@ void Compiler::fgDebugCheckBBlist(bool checkBBNum /* = false */, // If the block is a BBJ_COND, a BBJ_SWITCH or a // lowered GT_SWITCH_TABLE node then make sure it - // ends with a GT_JTRUE or a GT_SWITCH + // ends with a conditional jump or a GT_SWITCH if (block->bbJumpKind == BBJ_COND) { - noway_assert(block->lastNode()->gtNext == nullptr && block->lastNode()->gtOper == GT_JTRUE); + noway_assert(block->lastNode()->gtNext == nullptr && block->lastNode()->OperIsConditionalJump()); } else if (block->bbJumpKind == BBJ_SWITCH) { diff --git a/src/jit/gentree.cpp b/src/jit/gentree.cpp index 159469febe..8df171f287 100644 --- a/src/jit/gentree.cpp +++ b/src/jit/gentree.cpp @@ -3171,6 +3171,11 @@ GenTreePtr Compiler::gtReverseCond(GenTree* tree) tree->gtFlags ^= GTF_RELOP_NAN_UN; } } + else if (tree->OperGet() == GT_JCC) + { + GenTreeJumpCC* jcc = tree->AsJumpCC(); + jcc->gtCondition = GenTree::ReverseRelop(jcc->gtCondition); + } else { tree = gtNewOperNode(GT_NOT, TYP_INT, tree); @@ -10763,6 +10768,10 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) } break; + case GT_JCC: + printf(" cond=%s", GenTree::NodeName(tree->AsJumpCC()->gtCondition)); + break; + default: assert(!"don't know how to display tree leaf node"); } @@ -15381,6 +15390,7 @@ bool GenTree::isContained() const { case GT_STOREIND: case GT_JTRUE: + case GT_JCC: case GT_RETURN: case GT_RETFILT: case GT_STORE_LCL_FLD: diff --git a/src/jit/gentree.h b/src/jit/gentree.h index 18108b8000..53aca89fa8 100644 --- a/src/jit/gentree.h +++ b/src/jit/gentree.h @@ -1428,6 +1428,11 @@ public: return (gtOper == GT_LIST) && ((gtFlags & GTF_LIST_AGGREGATE) != 0); } + bool OperIsConditionalJump() const + { + return (gtOper == GT_JTRUE) || (gtOper == GT_JCC); + } + // Requires that "op" is an op= operator. Returns // the corresponding "op". static genTreeOps OpAsgToOper(genTreeOps op); @@ -4640,6 +4645,26 @@ struct GenTreeAllocObj final : public GenTreeUnOp #endif }; + +struct GenTreeJumpCC final : public GenTree +{ + genTreeOps gtCondition; // any relop + + GenTreeJumpCC(genTreeOps condition) + : GenTree(GT_JCC, TYP_VOID DEBUGARG(/*largeNode*/ FALSE)) + , gtCondition(condition) + { + assert(OperIsCompare(condition)); + } + +#if DEBUGGABLE_GENTREE + GenTreeJumpCC() + : GenTree() + { + } +#endif // DEBUGGABLE_GENTREE +}; + //------------------------------------------------------------------------ // Deferred inline functions of GenTree -- these need the subtypes above to // be defined already. diff --git a/src/jit/gtlist.h b/src/jit/gtlist.h index 4c69022ecf..da8246d20e 100644 --- a/src/jit/gtlist.h +++ b/src/jit/gtlist.h @@ -189,7 +189,11 @@ GTNODE(SIMD , "simd" ,0,GTK_BINOP|GTK_EXOP) // SIMD functions/oper // Other nodes that look like unary/binary operators: //----------------------------------------------------------------------------- +// The following are both conditional branches. GT_JTRUE has a single operand that computes a condition. GT_JCC +// implicitly reads the condition bits from a previous operation. The latter is allowed only in the LIR form +// used in the RyuJIT backend. GTNODE(JTRUE , "jmpTrue" ,0,GTK_UNOP|GTK_NOVALUE) +GTNODE(JCC , "jcc" ,0,GTK_LEAF|GTK_NOVALUE) GTNODE(LIST , "<list>" ,0,GTK_BINOP) diff --git a/src/jit/gtstructs.h b/src/jit/gtstructs.h index 895d3b6598..e0e1ee5fdc 100644 --- a/src/jit/gtstructs.h +++ b/src/jit/gtstructs.h @@ -100,6 +100,7 @@ GTSTRUCT_1(PhysReg , GT_PHYSREG) GTSTRUCT_1(SIMD , GT_SIMD) #endif // FEATURE_SIMD GTSTRUCT_1(AllocObj , GT_ALLOCOBJ) +GTSTRUCT_1(JumpCC , GT_JCC) /*****************************************************************************/ #undef GTSTRUCT_0 #undef GTSTRUCT_1 diff --git a/src/jit/lir.cpp b/src/jit/lir.cpp index 94206def1c..ed39dce21b 100644 --- a/src/jit/lir.cpp +++ b/src/jit/lir.cpp @@ -1617,7 +1617,7 @@ void LIR::InsertBeforeTerminator(BasicBlock* block, LIR::Range&& range) switch (block->bbJumpKind) { case BBJ_COND: - assert(insertionPoint->OperGet() == GT_JTRUE); + assert(insertionPoint->OperIsConditionalJump()); break; case BBJ_SWITCH: diff --git a/src/jit/lower.cpp b/src/jit/lower.cpp index 9cd41061e3..e059407ae2 100644 --- a/src/jit/lower.cpp +++ b/src/jit/lower.cpp @@ -149,6 +149,15 @@ GenTree* Lowering::LowerNode(GenTree* node) LowerCall(node); break; + case GT_LT: + case GT_LE: + case GT_GT: + case GT_GE: + case GT_EQ: + case GT_NE: + LowerCompare(node); + break; + case GT_JMP: LowerJmpMethod(node); break; @@ -1866,6 +1875,254 @@ GenTree* Lowering::LowerTailCallViaHelper(GenTreeCall* call, GenTree* callTarget return result; } +//------------------------------------------------------------------------ +// Lowering::LowerCompare: lowers a compare node. +// +// For 64-bit targets, this doesn't do much of anything: all comparisons +// that we support can be handled in code generation on such targets. +// +// For 32-bit targets, however, any comparison that feeds a `GT_JTRUE` +// node must be lowered such that the liveness of the operands to the +// comparison is properly visible to the rest of the backend. As such, +// a 64-bit comparison is lowered from something like this: +// +// ------------ BB02 [004..014) -> BB02 (cond), preds={BB02,BB01} succs={BB03,BB02} +// N001 ( 1, 1) [000006] ------------ t6 = lclVar int V02 loc0 u:5 $148 +// +// /--* t6 int +// N002 ( 2, 3) [000007] ---------U-- t7 = * cast long <- ulong <- uint $3c0 +// +// N003 ( 3, 10) [000009] ------------ t9 = lconst long 0x0000000000000003 $101 +// +// /--* t7 long +// +--* t9 long +// N004 ( 9, 17) [000010] N------N-U-- t10 = * < int $149 +// +// /--* t10 int +// N005 ( 11, 19) [000011] ------------ * jmpTrue void +// +// To something like this: +// +// ------------ BB02 [004..014) -> BB03 (cond), preds={BB06,BB07,BB01} succs={BB06,BB03} +// [000099] ------------ t99 = const int 0 +// +// [000101] ------------ t101 = const int 0 +// +// /--* t99 int +// +--* t101 int +// N004 ( 9, 17) [000010] N------N-U-- t10 = * > int $149 +// +// /--* t10 int +// N005 ( 11, 19) [000011] ------------ * jmpTrue void +// +// +// ------------ BB06 [???..???) -> BB02 (cond), preds={BB02} succs={BB07,BB02} +// [000105] -------N-U-- jcc void cond=< +// +// +// ------------ BB07 [???..???) -> BB02 (cond), preds={BB06} succs={BB03,BB02} +// N001 ( 1, 1) [000006] ------------ t6 = lclVar int V02 loc0 u:5 $148 +// +// N003 ( 3, 10) [000009] ------------ t9 = const int 3 +// +// /--* t6 int +// +--* t9 int +// [000106] N------N-U-- t106 = * < int +// +// /--* t106 int +// [000107] ------------ * jmpTrue void +// +// Which will eventually generate code similar to the following: +// +// 33DB xor ebx, ebx +// 85DB test ebx, ebx +// 7707 ja SHORT G_M50523_IG04 +// 72E7 jb SHORT G_M50523_IG03 +// 83F803 cmp eax, 3 +// 72E2 jb SHORT G_M50523_IG03 +// +void Lowering::LowerCompare(GenTree* cmp) +{ +#ifndef _TARGET_64BIT_ + if (cmp->gtGetOp1()->TypeGet() != TYP_LONG) + { + return; + } + + LIR::Use cmpUse; + + if (!BlockRange().TryGetUse(cmp, &cmpUse) || cmpUse.User()->OperGet() != GT_JTRUE) + { + return; + } + + GenTree* src1 = cmp->gtGetOp1(); + GenTree* src2 = cmp->gtGetOp2(); + unsigned weight = m_block->getBBWeight(comp); + + LIR::Use loSrc1(BlockRange(), &(src1->gtOp.gtOp1), src1); + LIR::Use loSrc2(BlockRange(), &(src2->gtOp.gtOp1), src2); + + if (loSrc1.Def()->OperGet() != GT_CNS_INT && loSrc1.Def()->OperGet() != GT_LCL_VAR) + { + loSrc1.ReplaceWithLclVar(comp, weight); + } + + if (loSrc2.Def()->OperGet() != GT_CNS_INT && loSrc2.Def()->OperGet() != GT_LCL_VAR) + { + loSrc2.ReplaceWithLclVar(comp, weight); + } + + BasicBlock* jumpDest = m_block->bbJumpDest; + BasicBlock* nextDest = m_block->bbNext; + BasicBlock* newBlock = comp->fgSplitBlockAtEnd(m_block); + + cmp->gtType = TYP_INT; + cmp->gtOp.gtOp1 = src1->gtOp.gtOp2; + cmp->gtOp.gtOp2 = src2->gtOp.gtOp2; + + if (cmp->OperGet() == GT_EQ || cmp->OperGet() == GT_NE) + { + // 64-bit equality comparisons (no matter the polarity) require two 32-bit comparisons: one for the upper 32 + // bits and one for the lower 32 bits. As such, we update the flow graph like so: + // + // Before: + // BB0: cond + // / \ + // false true + // | | + // BB1 BB2 + // + // After: + // BB0: cond(hi) + // / \ + // false true + // | | + // | BB3: cond(lo) + // | / \ + // | false true + // \ / | + // BB1 BB2 + // + + BlockRange().Remove(loSrc1.Def()); + BlockRange().Remove(loSrc2.Def()); + GenTree* loCmp = comp->gtNewOperNode(cmp->OperGet(), TYP_INT, loSrc1.Def(), loSrc2.Def()); + loCmp->gtFlags = cmp->gtFlags; + GenTree* loJtrue = comp->gtNewOperNode(GT_JTRUE, TYP_VOID, loCmp); + LIR::AsRange(newBlock).InsertAfter(nullptr, loSrc1.Def(), loSrc2.Def(), loCmp, loJtrue); + + m_block->bbJumpKind = BBJ_COND; + + if (cmp->OperGet() == GT_EQ) + { + cmp->gtOper = GT_NE; + m_block->bbJumpDest = nextDest; + nextDest->bbFlags |= BBF_JMP_TARGET; + comp->fgAddRefPred(nextDest, m_block); + } + else + { + m_block->bbJumpDest = jumpDest; + comp->fgAddRefPred(jumpDest, m_block); + } + + assert(newBlock->bbJumpKind == BBJ_COND); + assert(newBlock->bbJumpDest == jumpDest); + } + else + { + // 64-bit ordinal comparisons are more complicated: they require two comparisons for the upper 32 bits and one + // comparison for the lower 32 bits. We update the flowgraph as such: + // + // Before: + // BB0: cond + // / \ + // false true + // | | + // BB1 BB2 + // + // After: + // BB0: (!cond(hi) && !eq(hi)) + // / \ + // true false + // | | + // | BB3: (cond(hi) && !eq(hi)) + // | / \ + // | false true + // | | | + // | BB4: cond(lo) | + // | / \ | + // | false true | + // \ / \ / + // BB1 BB2 + // + // + // Note that the actual comparisons used to implement "(!cond(hi) && !eq(hi))" and "(cond(hi) && !eq(hi))" + // differ based on the original condition, and all consist of a single node. The switch statement below + // performs the necessary mapping. + // + + genTreeOps hiCmpOper; + genTreeOps loCmpOper; + + switch (cmp->OperGet()) + { + case GT_LT: + cmp->gtOper = GT_GT; + hiCmpOper = GT_LT; + loCmpOper = GT_LT; + break; + case GT_LE: + cmp->gtOper = GT_GT; + hiCmpOper = GT_LT; + loCmpOper = GT_LE; + break; + case GT_GT: + cmp->gtOper = GT_LT; + hiCmpOper = GT_GT; + loCmpOper = GT_GT; + break; + case GT_GE: + cmp->gtOper = GT_LT; + hiCmpOper = GT_GT; + loCmpOper = GT_GE; + break; + default: + unreached(); + } + + BasicBlock* newBlock2 = comp->fgSplitBlockAtEnd(newBlock); + + GenTree* hiJcc = new (comp, GT_JCC) GenTreeJumpCC(hiCmpOper); + hiJcc->gtFlags = cmp->gtFlags; + LIR::AsRange(newBlock).InsertAfter(nullptr, hiJcc); + + BlockRange().Remove(loSrc1.Def()); + BlockRange().Remove(loSrc2.Def()); + GenTree* loCmp = comp->gtNewOperNode(loCmpOper, TYP_INT, loSrc1.Def(), loSrc2.Def()); + loCmp->gtFlags = cmp->gtFlags | GTF_UNSIGNED; + GenTree* loJtrue = comp->gtNewOperNode(GT_JTRUE, TYP_VOID, loCmp); + LIR::AsRange(newBlock2).InsertAfter(nullptr, loSrc1.Def(), loSrc2.Def(), loCmp, loJtrue); + + m_block->bbJumpKind = BBJ_COND; + m_block->bbJumpDest = nextDest; + nextDest->bbFlags |= BBF_JMP_TARGET; + comp->fgAddRefPred(nextDest, m_block); + + newBlock->bbJumpKind = BBJ_COND; + newBlock->bbJumpDest = jumpDest; + comp->fgAddRefPred(jumpDest, newBlock); + + assert(newBlock2->bbJumpKind == BBJ_COND); + assert(newBlock2->bbJumpDest == jumpDest); + } + + BlockRange().Remove(src1); + BlockRange().Remove(src2); +#endif +} + // Lower "jmp <method>" tail call to insert PInvoke method epilog if required. void Lowering::LowerJmpMethod(GenTree* jmp) { diff --git a/src/jit/lower.h b/src/jit/lower.h index 1450f03d4a..5ce21e48f4 100644 --- a/src/jit/lower.h +++ b/src/jit/lower.h @@ -64,6 +64,7 @@ private: // Call Lowering // ------------------------------ void LowerCall(GenTree* call); + void LowerCompare(GenTree* tree); void LowerJmpMethod(GenTree* jmp); void LowerRet(GenTree* ret); GenTree* LowerDelegateInvoke(GenTreeCall* call); @@ -189,6 +190,7 @@ private: void TreeNodeInfoInitReturn(GenTree* tree); void TreeNodeInfoInitShiftRotate(GenTree* tree); void TreeNodeInfoInitCall(GenTreeCall* call); + void TreeNodeInfoInitCmp(GenTreePtr tree); void TreeNodeInfoInitStructArg(GenTreePtr structArg); void TreeNodeInfoInitBlockStore(GenTreeBlk* blkNode); void TreeNodeInfoInitLogicalOp(GenTree* tree); @@ -225,8 +227,6 @@ private: void SetMulOpCounts(GenTreePtr tree); #endif // defined(_TARGET_XARCH_) - void LowerCmp(GenTreePtr tree); - #if !CPU_LOAD_STORE_ARCH bool IsBinOpInRMWStoreInd(GenTreePtr tree); bool IsRMWMemOpRootedAtStoreInd(GenTreePtr storeIndTree, GenTreePtr* indirCandidate, GenTreePtr* indirOpSource); diff --git a/src/jit/lowerarm64.cpp b/src/jit/lowerarm64.cpp index 46a14211b3..d80691a9a4 100644 --- a/src/jit/lowerarm64.cpp +++ b/src/jit/lowerarm64.cpp @@ -484,7 +484,7 @@ void Lowering::TreeNodeInfoInit(GenTree* tree) case GT_LE: case GT_GE: case GT_GT: - LowerCmp(tree); + TreeNodeInfoInitCmp(tree); break; case GT_CKFINITE: @@ -1859,7 +1859,7 @@ void Lowering::SetIndirAddrOpCounts(GenTreePtr indirTree) } } -void Lowering::LowerCmp(GenTreePtr tree) +void Lowering::TreeNodeInfoInitCmp(GenTreePtr tree) { TreeNodeInfo* info = &(tree->gtLsraInfo); diff --git a/src/jit/lowerxarch.cpp b/src/jit/lowerxarch.cpp index 72c1ec82e3..c82603ba46 100644 --- a/src/jit/lowerxarch.cpp +++ b/src/jit/lowerxarch.cpp @@ -323,6 +323,11 @@ void Lowering::TreeNodeInfoInit(GenTree* tree) l->clearDstCount(tree->gtOp.gtOp1); break; + case GT_JCC: + info->srcCount = 0; + info->dstCount = 0; + break; + case GT_JMP: info->srcCount = 0; info->dstCount = 0; @@ -501,7 +506,7 @@ void Lowering::TreeNodeInfoInit(GenTree* tree) case GT_LE: case GT_GE: case GT_GT: - LowerCmp(tree); + TreeNodeInfoInitCmp(tree); break; case GT_CKFINITE: @@ -2894,7 +2899,7 @@ void Lowering::SetIndirAddrOpCounts(GenTreePtr indirTree) } } -void Lowering::LowerCmp(GenTreePtr tree) +void Lowering::TreeNodeInfoInitCmp(GenTreePtr tree) { TreeNodeInfo* info = &(tree->gtLsraInfo); @@ -3260,7 +3265,7 @@ void Lowering::LowerCmp(GenTreePtr tree) #ifdef DEBUG if (comp->verbose) { - printf("LowerCmp: Removing a GT_CAST to TYP_UBYTE and changing castOp1->gtType to " + printf("TreeNodeInfoInitCmp: Removing a GT_CAST to TYP_UBYTE and changing castOp1->gtType to " "TYP_UBYTE\n"); comp->gtDispTreeRange(BlockRange(), tree); } diff --git a/src/jit/lsra.cpp b/src/jit/lsra.cpp index 0ddf7ec340..6f017182c0 100644 --- a/src/jit/lsra.cpp +++ b/src/jit/lsra.cpp @@ -8526,7 +8526,7 @@ void LinearScan::insertMove( noway_assert(!blockRange.IsEmpty()); GenTree* branch = blockRange.LastNode(); - assert(branch->OperGet() == GT_JTRUE || branch->OperGet() == GT_SWITCH_TABLE || + assert(branch->OperIsConditionalJump() || branch->OperGet() == GT_SWITCH_TABLE || branch->OperGet() == GT_SWITCH); blockRange.InsertBefore(branch, std::move(treeRange)); @@ -8597,7 +8597,7 @@ void LinearScan::insertSwap( noway_assert(!blockRange.IsEmpty()); GenTree* branch = blockRange.LastNode(); - assert(branch->OperGet() == GT_JTRUE || branch->OperGet() == GT_SWITCH_TABLE || + assert(branch->OperIsConditionalJump() || branch->OperGet() == GT_SWITCH_TABLE || branch->OperGet() == GT_SWITCH); blockRange.InsertBefore(branch, std::move(swapRange)); diff --git a/src/jit/rationalize.cpp b/src/jit/rationalize.cpp index 03e0c9a27e..4a204c2ac7 100644 --- a/src/jit/rationalize.cpp +++ b/src/jit/rationalize.cpp @@ -931,6 +931,8 @@ Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<G #endif // FEATURE_SIMD default: + // JCC nodes should not be present in HIR. + assert(node->OperGet() != GT_JCC); break; } |