diff options
author | Pat Gavlin <pgavlin@gmail.com> | 2016-08-19 10:44:46 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-08-19 10:44:46 -0700 |
commit | 738f93e7baf5aef7639cdd4567e9cb1746aed619 (patch) | |
tree | 425681f48e72df83235e2a77a6c84cb93bed3927 /src | |
parent | 00c85b302a078d5d13cfe9ff8cb453cd8296d6aa (diff) | |
download | coreclr-738f93e7baf5aef7639cdd4567e9cb1746aed619.tar.gz coreclr-738f93e7baf5aef7639cdd4567e9cb1746aed619.tar.bz2 coreclr-738f93e7baf5aef7639cdd4567e9cb1746aed619.zip |
Implement the proposed design for RyuJIT's LIR. (#6689)
These changes implement the design for RyuJIT's LIR described in https://github.com/dotnet/coreclr/blob/master/Documentation/design-docs/removing-embedded-statements.md.
The following passes required changes:
Rationalize, which has been almost completely rewritten
Long decomposition
Target-independent lowering
Target-dependent lowering
LSRA
Liveness
Flowgraph optimization
Codegen
For the most part, these changes are confined to the backend. Common code that needed to be updated included liveness, flowgraph optimization, and a few miscellaneous utilities.
The utilities used to analyze and manipulate LIR live (almost) entirely in src/jit/lir.{cpp,h}. The core concepts that are unique to LIR are LIR::Use and LIR::Range. The latter is a tuple that captures an SDSU def (i.e. an LIR node) and its corresponding use->def edge and user. The former serves to abstract a self-contained sequence of LIR nodes that make up e.g. the contents of a basic block.
Testing indicates that neither JIT throughput nor code quality are significantly impacted by these changes.
Diffstat (limited to 'src')
37 files changed, 7328 insertions, 6993 deletions
diff --git a/src/jit/CMakeLists.txt b/src/jit/CMakeLists.txt index 4992074443..3835e99571 100644 --- a/src/jit/CMakeLists.txt +++ b/src/jit/CMakeLists.txt @@ -46,6 +46,7 @@ set( JIT_SOURCES jiteh.cpp jittelemetry.cpp lclvars.cpp + lir.cpp liveness.cpp loopcloning.cpp lower.cpp diff --git a/src/jit/block.cpp b/src/jit/block.cpp index 37c85dd39d..2d37754ec5 100644 --- a/src/jit/block.cpp +++ b/src/jit/block.cpp @@ -326,7 +326,9 @@ void BasicBlock::dspFlags() } #if FEATURE_EH_FUNCLETS && defined(_TARGET_ARM_) if (bbFlags & BBF_FINALLY_TARGET) + { printf("ftarget "); + } #endif // FEATURE_EH_FUNCLETS && defined(_TARGET_ARM_) if (bbFlags & BBF_BACKWARD_JUMP) { @@ -348,10 +350,17 @@ void BasicBlock::dspFlags() { printf("IBC "); } +#ifdef LEGACY_BACKEND if (bbFlags & BBF_FORWARD_SWITCH) { printf("fswitch "); } +#else // !LEGACY_BACKEND + if (bbFlags & BBF_IS_LIR) + { + printf("LIR "); + } +#endif // LEGACY_BACKEND if (bbFlags & BBF_KEEP_BBJ_ALWAYS) { printf("KEEP "); @@ -591,6 +600,33 @@ void BasicBlock::CloneBlockState(Compiler* compiler, BasicBlock* to, const Basic } } +// LIR helpers +void BasicBlock::MakeLIR(GenTree* firstNode, GenTree* lastNode) +{ +#ifdef LEGACY_BACKEND + unreached(); +#else // !LEGACY_BACKEND + assert(!IsLIR()); + assert((firstNode == nullptr) == (lastNode == nullptr)); + assert((firstNode == lastNode) || firstNode->Precedes(lastNode)); + + m_firstNode = firstNode; + m_lastNode = lastNode; + bbFlags |= BBF_IS_LIR; +#endif // LEGACY_BACKEND +} + +bool BasicBlock::IsLIR() +{ +#ifdef LEGACY_BACKEND + return false; +#else // !LEGACY_BACKEND + const bool isLIR = (bbFlags & BBF_IS_LIR) != 0; + assert((bbTreeList == nullptr) || ((isLIR) == !bbTreeList->IsStatement())); + return isLIR; +#endif // LEGACY_BACKEND +} + //------------------------------------------------------------------------ // firstStmt: Returns the first statement in the block // @@ -600,7 +636,6 @@ void BasicBlock::CloneBlockState(Compiler* compiler, BasicBlock* to, const Basic // Return Value: // The first statement in the block's bbTreeList. // - GenTreeStmt* BasicBlock::firstStmt() { if (bbTreeList == nullptr) @@ -620,9 +655,6 @@ GenTreeStmt* BasicBlock::firstStmt() // Return Value: // The last statement in the block's bbTreeList. // -// Notes: -// The last statement may be an embedded statement, when in linear order. - GenTreeStmt* BasicBlock::lastStmt() { if (bbTreeList == nullptr) @@ -635,37 +667,21 @@ GenTreeStmt* BasicBlock::lastStmt() return result->AsStmt(); } + //------------------------------------------------------------------------ -// lastTopLevelStmt: Returns the last top-level statement in the block -// -// Arguments: -// None. +// BasicBlock::firstNode: Returns the first node in the block. // -// Return Value: -// The last statement in the block's bbTreeList. -// -// Notes: -// The last statement may be an embedded statement, when in linear order, -// so this method is provided to obtain the last top-level statement, which -// will also contain the last tree nodes in execution order. - -GenTreeStmt* BasicBlock::lastTopLevelStmt() +GenTree* BasicBlock::firstNode() { - if (bbTreeList == nullptr) - { - return nullptr; - } - - GenTreePtr stmt = lastStmt(); - -#ifndef LEGACY_BACKEND - while ((stmt->gtFlags & GTF_STMT_TOP_LEVEL) == 0) - { - stmt = stmt->gtPrev; - } -#endif // !LEGACY_BACKEND + return IsLIR() ? bbTreeList : Compiler::fgGetFirstNode(firstStmt()->gtStmtExpr); +} - return stmt->AsStmt(); +//------------------------------------------------------------------------ +// BasicBlock::lastNode: Returns the last node in the block. +// +GenTree* BasicBlock::lastNode() +{ + return IsLIR() ? m_lastNode : lastStmt()->gtStmtExpr; } //------------------------------------------------------------------------ @@ -735,3 +751,21 @@ unsigned PtrKeyFuncs<BasicBlock>::GetHashCode(const BasicBlock* ptr) #endif return ptr->bbNum; } + +bool BasicBlock::isEmpty() +{ + if (!IsLIR()) + { + return (this->FirstNonPhiDef() == nullptr); + } + + for (GenTree* node : LIR::AsRange(this).NonPhiNodes()) + { + if (node->OperGet() != GT_IL_OFFSET) + { + return false; + } + } + + return true; +} diff --git a/src/jit/block.h b/src/jit/block.h index 969967f1a6..ecfbb620a1 100644 --- a/src/jit/block.h +++ b/src/jit/block.h @@ -268,13 +268,17 @@ public: } }; -/***************************************************************************** - * - * The following structure describes a basic block. - */ - -struct BasicBlock +//------------------------------------------------------------------------ +// BasicBlock: describes a basic block in the flowgraph. +// +// Note that this type derives from LIR::Range in order to make the LIR +// utilities that are polymorphic over basic block and scratch ranges +// faster and simpler. +// +struct BasicBlock : private LIR::Range { + friend class LIR; + BasicBlock* bbNext; // next BB in ascending PC offset order BasicBlock* bbPrev; @@ -287,7 +291,8 @@ struct BasicBlock } } - unsigned bbNum; // the block's number + unsigned bbNum; // the block's number + unsigned bbPostOrderNum; // the block's post order number in the graph. unsigned bbRefs; // number of blocks that can reach here, either by fall-through or a branch. If this falls to zero, // the block is unreachable. @@ -340,9 +345,13 @@ struct BasicBlock // BBJ_ALWAYS); see isBBCallAlwaysPair(). #define BBF_LOOP_PREHEADER 0x08000000 // BB is a loop preheader block -#define BBF_COLD 0x10000000 // BB is cold -#define BBF_PROF_WEIGHT 0x20000000 // BB weight is computed from profile data +#define BBF_COLD 0x10000000 // BB is cold +#define BBF_PROF_WEIGHT 0x20000000 // BB weight is computed from profile data +#ifdef LEGACY_BACKEND #define BBF_FORWARD_SWITCH 0x40000000 // Aux flag used in FP codegen to know if a jmptable entry has been forwarded +#else // !LEGACY_BACKEND +#define BBF_IS_LIR 0x40000000 // Set if the basic block contains LIR (as opposed to HIR) +#endif // LEGACY_BACKEND #define BBF_KEEP_BBJ_ALWAYS 0x80000000 // A special BBJ_ALWAYS block, used by EH code generation. Keep the jump kind // as BBJ_ALWAYS. Used for the paired BBJ_ALWAYS block following the // BBJ_CALLFINALLY block, as well as, on x86, the final step block out of a @@ -365,9 +374,14 @@ struct BasicBlock // Flags a block should not have had before it is split. +#ifdef LEGACY_BACKEND #define BBF_SPLIT_NONEXIST \ (BBF_CHANGED | BBF_LOOP_HEAD | BBF_LOOP_CALL0 | BBF_LOOP_CALL1 | BBF_RETLESS_CALL | BBF_LOOP_PREHEADER | \ BBF_COLD | BBF_FORWARD_SWITCH) +#else // !LEGACY_BACKEND +#define BBF_SPLIT_NONEXIST \ + (BBF_CHANGED | BBF_LOOP_HEAD | BBF_LOOP_CALL0 | BBF_LOOP_CALL1 | BBF_RETLESS_CALL | BBF_LOOP_PREHEADER | BBF_COLD) +#endif // LEGACY_BACKEND // Flags lost by the top block when a block is split. // Note, this is a conservative guess. @@ -532,10 +546,7 @@ struct BasicBlock // Returns "true" if the block is empty. Empty here means there are no statement // trees *except* PHI definitions. - bool isEmpty() - { - return (this->FirstNonPhiDef() == nullptr); - } + bool isEmpty(); // Returns "true" iff "this" is the first block of a BBJ_CALLFINALLY/BBJ_ALWAYS pair -- // a block corresponding to an exit from the try of a try/finally. In the flow graph, @@ -622,7 +633,18 @@ struct BasicBlock return bbRefs; } - GenTree* bbTreeList; // the body of the block + __declspec(property(get = getBBTreeList, put = setBBTreeList)) GenTree* bbTreeList; // the body of the block. + + GenTree* getBBTreeList() const + { + return m_firstNode; + } + + void setBBTreeList(GenTree* tree) + { + m_firstNode = tree; + } + EntryState* bbEntryState; // verifier tracked state of all entries in stack. #define NO_BASE_TMP UINT_MAX // base# to use when we have none @@ -965,6 +987,9 @@ struct BasicBlock GenTreeStmt* lastStmt(); GenTreeStmt* lastTopLevelStmt(); + GenTree* firstNode(); + GenTree* lastNode(); + bool containsStatement(GenTree* statement); bool endsWithJmpMethod(Compiler* comp); @@ -1072,6 +1097,9 @@ public: // Clone block state and statements from 'from' block to 'to' block. // Assumes that "to" is an empty block. static void CloneBlockState(Compiler* compiler, BasicBlock* to, const BasicBlock* from); + + void MakeLIR(GenTree* firstNode, GenTree* lastNode); + bool IsLIR(); }; template <> diff --git a/src/jit/codegen.h b/src/jit/codegen.h index 884a5ffa8c..a36973bcfe 100755 --- a/src/jit/codegen.h +++ b/src/jit/codegen.h @@ -172,6 +172,7 @@ private: void genGenerateStackProbe(); #endif +#ifdef LEGACY_BACKEND regMaskTP genNewLiveRegMask(GenTreePtr first, GenTreePtr second); // During codegen, determine the LiveSet after tree. @@ -179,6 +180,7 @@ private: // compCurLifeTree are being maintained, and tree must occur in the current // statement. VARSET_VALRET_TP genUpdateLiveSetForward(GenTreePtr tree); +#endif //------------------------------------------------------------------------- diff --git a/src/jit/codegenarm.cpp b/src/jit/codegenarm.cpp index e4df26d4e9..8ded006dc7 100644 --- a/src/jit/codegenarm.cpp +++ b/src/jit/codegenarm.cpp @@ -174,7 +174,6 @@ void CodeGen::genCodeForBBlist() #ifdef DEBUG genInterruptibleUsed = true; - unsigned stmtNum = 0; unsigned totalCostEx = 0; unsigned totalCostSz = 0; @@ -340,13 +339,12 @@ void CodeGen::genCodeForBBlist() if (handlerGetsXcptnObj(block->bbCatchTyp)) { - GenTreePtr firstStmt = block->FirstNonPhiDef(); - if (firstStmt != NULL) + for (GenTree* node : LIR::AsRange(block)) { - GenTreePtr firstTree = firstStmt->gtStmt.gtStmtExpr; - if (compiler->gtHasCatchArg(firstTree)) + if (node->OperGet() == GT_CATCH_ARG) { gcInfo.gcMarkRegSetGCref(RBM_EXCEPTION_OBJECT); + break; } } } @@ -474,93 +472,70 @@ void CodeGen::genCodeForBBlist() } #endif // FEATURE_EH_FUNCLETS - for (GenTreePtr stmt = block->FirstNonPhiDef(); stmt; stmt = stmt->gtNext) - { - noway_assert(stmt->gtOper == GT_STMT); - - if (stmt->AsStmt()->gtStmtIsEmbedded()) - continue; + // Clear compCurStmt and compCurLifeTree. + compiler->compCurStmt = nullptr; + compiler->compCurLifeTree = nullptr; - /* Get hold of the statement tree */ - GenTreePtr tree = stmt->gtStmt.gtStmtExpr; +#ifdef DEBUG + bool pastProfileUpdate = false; +#endif -#if defined(DEBUGGING_SUPPORT) +// Traverse the block in linear order, generating code for each node as we +// as we encounter it. +#ifdef DEBUGGING_SUPPORT + IL_OFFSETX currentILOffset = BAD_IL_OFFSET; +#endif + for (GenTree* node : LIR::AsRange(block)) + { +#ifdef DEBUGGING_SUPPORT + // Do we have a new IL offset? + if (node->OperGet() == GT_IL_OFFSET) + { + genEnsureCodeEmitted(currentILOffset); - /* Do we have a new IL-offset ? */ + currentILOffset = node->gtStmt.gtStmtILoffsx; - if (stmt->gtStmt.gtStmtILoffsx != BAD_IL_OFFSET) - { - /* Create and append a new IP-mapping entry */ - genIPmappingAdd(stmt->gtStmt.gtStmt.gtStmtILoffsx, firstMapping); + genIPmappingAdd(currentILOffset, firstMapping); firstMapping = false; } - #endif // DEBUGGING_SUPPORT #ifdef DEBUG - noway_assert(stmt->gtStmt.gtStmtLastILoffs <= compiler->info.compILCodeSize || - stmt->gtStmt.gtStmtLastILoffs == BAD_IL_OFFSET); - - if (compiler->opts.dspCode && compiler->opts.dspInstrs && stmt->gtStmt.gtStmtLastILoffs != BAD_IL_OFFSET) + if (node->OperGet() == GT_IL_OFFSET) { - while (genCurDispOffset <= stmt->gtStmt.gtStmtLastILoffs) + noway_assert(node->gtStmt.gtStmtLastILoffs <= compiler->info.compILCodeSize || + node->gtStmt.gtStmtLastILoffs == BAD_IL_OFFSET); + + if (compiler->opts.dspCode && compiler->opts.dspInstrs && + node->gtStmt.gtStmtLastILoffs != BAD_IL_OFFSET) { - genCurDispOffset += dumpSingleInstr(compiler->info.compCode, genCurDispOffset, "> "); + while (genCurDispOffset <= node->gtStmt.gtStmtLastILoffs) + { + genCurDispOffset += dumpSingleInstr(compiler->info.compCode, genCurDispOffset, "> "); + } } } - stmtNum++; - if (compiler->verbose) - { - printf("\nGenerating BB%02u, stmt %u\t\t", block->bbNum, stmtNum); - printf("Holding variables: "); - dspRegMask(regSet.rsMaskVars); - printf("\n\n"); - compiler->gtDispTree(compiler->opts.compDbgInfo ? stmt : tree); - printf("\n"); - } - totalCostEx += (stmt->gtCostEx * block->getBBWeight(compiler)); - totalCostSz += stmt->gtCostSz; -#endif // DEBUG - - // Traverse the tree in linear order, generating code for each node in the - // tree as we encounter it - - // If we have embedded statements, we need to keep track of the next top-level - // node in order, because if it produces a register, we need to consume it + // TODO-LIR: the cost accounting performed below is incorrect: each operator's cost includes the + // cost of its operands, so the total cost of the block is grossly overestimated. Fixing + // this requires the ability to calculate the cost of the operator itself. + // + // totalCostEx += (UINT64)node->gtCostEx * block->getBBWeight(compiler); + // totalCostSz += (UINT64)node->gtCostSz; - GenTreeStmt* curPossiblyEmbeddedStmt = stmt->gtStmt.gtStmtNextIfEmbedded(); - if (curPossiblyEmbeddedStmt == nullptr) - curPossiblyEmbeddedStmt = stmt->AsStmt(); +#endif // DEBUG - compiler->compCurLifeTree = NULL; - compiler->compCurStmt = stmt; - for (GenTreePtr treeNode = stmt->gtStmt.gtStmtList; treeNode != NULL; treeNode = treeNode->gtNext) + genCodeForTreeNode(node); + if (node->gtHasReg() && node->gtLsraInfo.isLocalDefUse) { - genCodeForTreeNode(treeNode); - - if (treeNode == curPossiblyEmbeddedStmt->gtStmtExpr) - { - // It is possible that the last top-level node may generate a result - // that is not used (but may still require a register, e.g. an indir - // that is evaluated only for the side-effect of a null check). - // In that case, we must "consume" the register now. - if (treeNode->gtHasReg()) - { - genConsumeReg(treeNode); - } - if (curPossiblyEmbeddedStmt != stmt) - { - curPossiblyEmbeddedStmt = curPossiblyEmbeddedStmt->gtStmt.gtStmtNextIfEmbedded(); - if (curPossiblyEmbeddedStmt == nullptr) - curPossiblyEmbeddedStmt = stmt->AsStmt(); - } - } + genConsumeReg(node); } +#ifdef DEBUG regSet.rsSpillChk(); -#ifdef DEBUG + assert((node->gtFlags & GTF_SPILL) == 0); + /* Make sure we didn't bungle pointer register tracking */ regMaskTP ptrRegs = (gcInfo.gcRegGCrefSetCur | gcInfo.gcRegByrefSetCur); @@ -571,26 +546,27 @@ void CodeGen::genCodeForBBlist() // even though we might return a ref. We can't use the compRetType // as the determiner because something we are tracking as a byref // might be used as a return value of a int function (which is legal) - if (tree->gtOper == GT_RETURN && (varTypeIsGC(compiler->info.compRetType) || - (tree->gtOp.gtOp1 != 0 && varTypeIsGC(tree->gtOp.gtOp1->TypeGet())))) + if (node->gtOper == GT_RETURN && (varTypeIsGC(compiler->info.compRetType) || + (node->gtOp.gtOp1 != 0 && varTypeIsGC(node->gtOp.gtOp1->TypeGet())))) { nonVarPtrRegs &= ~RBM_INTRET; } - // When profiling, the first statement in a catch block will be the - // harmless "inc" instruction (does not interfere with the exception - // object). - - if ((compiler->opts.eeFlags & CORJIT_FLG_BBINSTR) && (stmt == block->bbTreeList) && - handlerGetsXcptnObj(block->bbCatchTyp)) + // When profiling, the first few nodes in a catch block will be an update of + // the profile count (does not interfere with the exception object). + if (((compiler->opts.eeFlags & CORJIT_FLG_BBINSTR) != 0) && handlerGetsXcptnObj(block->bbCatchTyp)) { - nonVarPtrRegs &= ~RBM_EXCEPTION_OBJECT; + pastProfileUpdate = pastProfileUpdate || node->OperGet() == GT_CATCH_ARG; + if (!pastProfileUpdate) + { + nonVarPtrRegs &= ~RBM_EXCEPTION_OBJECT; + } } if (nonVarPtrRegs) { - printf("Regset after tree="); - Compiler::printTreeID(tree); + printf("Regset after node="); + Compiler::printTreeID(node); printf(" BB%02u gcr=", block->bbNum); printRegMaskInt(gcInfo.gcRegGCrefSetCur & ~regSet.rsMaskVars); compiler->getEmitter()->emitDispRegSet(gcInfo.gcRegGCrefSetCur & ~regSet.rsMaskVars); @@ -604,23 +580,21 @@ void CodeGen::genCodeForBBlist() } noway_assert(nonVarPtrRegs == 0); - - for (GenTree* node = stmt->gtStmt.gtStmtList; node; node = node->gtNext) - { - assert(!(node->gtFlags & GTF_SPILL)); - } - #endif // DEBUG - - noway_assert(stmt->gtOper == GT_STMT); - -#ifdef DEBUGGING_SUPPORT - genEnsureCodeEmitted(stmt->gtStmt.gtStmtILoffsx); -#endif - - } //-------- END-FOR each statement-tree of the current block --------- + } #ifdef DEBUGGING_SUPPORT + // It is possible to reach the end of the block without generating code for the current IL offset. + // For example, if the following IR ends the current block, no code will have been generated for + // offset 21: + // + // ( 0, 0) [000040] ------------ il_offset void IL offset: 21 + // + // N001 ( 0, 0) [000039] ------------ nop void + // + // This can lead to problems when debugging the generated code. To prevent these issues, make sure + // we've generated code for the last IL offset we saw in the block. + genEnsureCodeEmitted(currentILOffset); if (compiler->opts.compScopeInfo && (compiler->info.compVarScopesCount > 0)) { diff --git a/src/jit/codegenarm64.cpp b/src/jit/codegenarm64.cpp index 8ed2fcdbec..75d00b2bd6 100644 --- a/src/jit/codegenarm64.cpp +++ b/src/jit/codegenarm64.cpp @@ -1419,9 +1419,8 @@ void CodeGen::genCodeForBBlist() #ifdef DEBUG genInterruptibleUsed = true; - unsigned stmtNum = 0; - UINT64 totalCostEx = 0; - UINT64 totalCostSz = 0; + UINT64 totalCostEx = 0; + UINT64 totalCostSz = 0; // You have to be careful if you create basic blocks from now on compiler->fgSafeBasicBlockCreation = false; @@ -1616,13 +1615,12 @@ void CodeGen::genCodeForBBlist() if (handlerGetsXcptnObj(block->bbCatchTyp)) { - GenTreePtr firstStmt = block->FirstNonPhiDef(); - if (firstStmt != NULL) + for (GenTree* node : LIR::AsRange(block)) { - GenTreePtr firstTree = firstStmt->gtStmt.gtStmtExpr; - if (compiler->gtHasCatchArg(firstTree)) + if (node->OperGet() == GT_CATCH_ARG) { gcInfo.gcMarkRegSetGCref(RBM_EXCEPTION_OBJECT); + break; } } } @@ -1704,134 +1702,112 @@ void CodeGen::genCodeForBBlist() genReserveFuncletProlog(block); } - for (GenTreePtr stmt = block->FirstNonPhiDef(); stmt; stmt = stmt->gtNext) - { - noway_assert(stmt->gtOper == GT_STMT); - - if (stmt->AsStmt()->gtStmtIsEmbedded()) - continue; - - /* Get hold of the statement tree */ - GenTreePtr tree = stmt->gtStmt.gtStmtExpr; - -#if defined(DEBUGGING_SUPPORT) + // Clear compCurStmt and compCurLifeTree. + compiler->compCurStmt = nullptr; + compiler->compCurLifeTree = nullptr; - /* Do we have a new IL-offset ? */ + // Traverse the block in linear order, generating code for each node as we + // as we encounter it. + CLANG_FORMAT_COMMENT_ANCHOR; - if (stmt->gtStmt.gtStmtILoffsx != BAD_IL_OFFSET) +#ifdef DEBUGGING_SUPPORT + IL_OFFSETX currentILOffset = BAD_IL_OFFSET; +#endif + for (GenTree* node : LIR::AsRange(block).NonPhiNodes()) + { +#ifdef DEBUGGING_SUPPORT + // Do we have a new IL offset? + if (node->OperGet() == GT_IL_OFFSET) { - /* Create and append a new IP-mapping entry */ - genIPmappingAdd(stmt->gtStmt.gtStmt.gtStmtILoffsx, firstMapping); + genEnsureCodeEmitted(currentILOffset); + currentILOffset = node->gtStmt.gtStmtILoffsx; + genIPmappingAdd(currentILOffset, firstMapping); firstMapping = false; } - #endif // DEBUGGING_SUPPORT #ifdef DEBUG - noway_assert(stmt->gtStmt.gtStmtLastILoffs <= compiler->info.compILCodeSize || - stmt->gtStmt.gtStmtLastILoffs == BAD_IL_OFFSET); - - if (compiler->opts.dspCode && compiler->opts.dspInstrs && stmt->gtStmt.gtStmtLastILoffs != BAD_IL_OFFSET) + if (node->OperGet() == GT_IL_OFFSET) { - while (genCurDispOffset <= stmt->gtStmt.gtStmtLastILoffs) - { - genCurDispOffset += dumpSingleInstr(compiler->info.compCode, genCurDispOffset, "> "); - } - } + noway_assert(node->gtStmt.gtStmtLastILoffs <= compiler->info.compILCodeSize || + node->gtStmt.gtStmtLastILoffs == BAD_IL_OFFSET); - stmtNum++; - if (compiler->verbose) - { - printf("\nGenerating BB%02u, stmt %u\t\t", block->bbNum, stmtNum); - printf("Holding variables: "); - dspRegMask(regSet.rsMaskVars); - printf("\n\n"); - if (compiler->verboseTrees) + if (compiler->opts.dspCode && compiler->opts.dspInstrs && + node->gtStmt.gtStmtLastILoffs != BAD_IL_OFFSET) { - compiler->gtDispTree(compiler->opts.compDbgInfo ? stmt : tree); - printf("\n"); - } - } - totalCostEx += ((UINT64)stmt->gtCostEx * block->getBBWeight(compiler)); - totalCostSz += (UINT64)stmt->gtCostSz; -#endif // DEBUG - - // Traverse the tree in linear order, generating code for each node in the - // tree as we encounter it - - compiler->compCurLifeTree = NULL; - compiler->compCurStmt = stmt; - for (GenTreePtr treeNode = stmt->gtStmt.gtStmtList; treeNode != NULL; treeNode = treeNode->gtNext) - { - genCodeForTreeNode(treeNode); - if (treeNode->gtHasReg() && treeNode->gtLsraInfo.isLocalDefUse) - { - genConsumeReg(treeNode); + while (genCurDispOffset <= node->gtStmt.gtStmtLastILoffs) + { + genCurDispOffset += dumpSingleInstr(compiler->info.compCode, genCurDispOffset, "> "); + } } } - regSet.rsSpillChk(); - -#ifdef DEBUG - /* Make sure we didn't bungle pointer register tracking */ - - regMaskTP ptrRegs = (gcInfo.gcRegGCrefSetCur | gcInfo.gcRegByrefSetCur); - regMaskTP nonVarPtrRegs = ptrRegs & ~regSet.rsMaskVars; - - // If return is a GC-type, clear it. Note that if a common - // epilog is generated (genReturnBB) it has a void return - // even though we might return a ref. We can't use the compRetType - // as the determiner because something we are tracking as a byref - // might be used as a return value of a int function (which is legal) - if (tree->gtOper == GT_RETURN && (varTypeIsGC(compiler->info.compRetType) || - (tree->gtOp.gtOp1 != 0 && varTypeIsGC(tree->gtOp.gtOp1->TypeGet())))) - { - nonVarPtrRegs &= ~RBM_INTRET; - } - - // When profiling, the first statement in a catch block will be the - // harmless "inc" instruction (does not interfere with the exception - // object). - - if ((compiler->opts.eeFlags & CORJIT_FLG_BBINSTR) && (stmt == block->bbTreeList) && - handlerGetsXcptnObj(block->bbCatchTyp)) - { - nonVarPtrRegs &= ~RBM_EXCEPTION_OBJECT; - } - - if (nonVarPtrRegs) - { - printf("Regset after tree="); - compiler->printTreeID(tree); - printf(" BB%02u gcr=", block->bbNum); - printRegMaskInt(gcInfo.gcRegGCrefSetCur & ~regSet.rsMaskVars); - compiler->getEmitter()->emitDispRegSet(gcInfo.gcRegGCrefSetCur & ~regSet.rsMaskVars); - printf(", byr="); - printRegMaskInt(gcInfo.gcRegByrefSetCur & ~regSet.rsMaskVars); - compiler->getEmitter()->emitDispRegSet(gcInfo.gcRegByrefSetCur & ~regSet.rsMaskVars); - printf(", regVars="); - printRegMaskInt(regSet.rsMaskVars); - compiler->getEmitter()->emitDispRegSet(regSet.rsMaskVars); - printf("\n"); - } + // TODO-LIR: the cost accounting performed below is incorrect: each operator's cost includes the + // cost of its operands, so the total cost of the block is grossly overestimated. Fixing + // this requires the ability to calculate the cost of the operator itself. + // + // totalCostEx += (UINT64)node->gtCostEx * block->getBBWeight(compiler); + // totalCostSz += (UINT64)node->gtCostSz; - noway_assert(nonVarPtrRegs == 0); +#endif // DEBUG - for (GenTree* node = stmt->gtStmt.gtStmtList; node; node = node->gtNext) + genCodeForTreeNode(node); + if (node->gtHasReg() && node->gtLsraInfo.isLocalDefUse) { - assert(!(node->gtFlags & GTF_SPILL)); + genConsumeReg(node); } + } // end for each node in block +#ifdef DEBUG + // The following set of register spill checks and GC pointer tracking checks used to be + // performed at statement boundaries. Now, with LIR, there are no statements, so they are + // performed at the end of each block. + // TODO: could these checks be performed more frequently? E.g., at each location where + // the register allocator says there are no live non-variable registers. Perhaps this could + // be done by (a) keeping a running count of live non-variable registers by using + // gtLsraInfo.srcCount and gtLsraInfo.dstCount to decrement and increment the count, respectively, + // and running the checks when the count is zero. Or, (b) use the map maintained by LSRA + // (operandToLocationInfoMap) to mark a node somehow when, after the execution of that node, + // there will be no live non-variable registers. + + regSet.rsSpillChk(); + + /* Make sure we didn't bungle pointer register tracking */ + + regMaskTP ptrRegs = gcInfo.gcRegGCrefSetCur | gcInfo.gcRegByrefSetCur; + regMaskTP nonVarPtrRegs = ptrRegs & ~regSet.rsMaskVars; + + // If return is a GC-type, clear it. Note that if a common + // epilog is generated (genReturnBB) it has a void return + // even though we might return a ref. We can't use the compRetType + // as the determiner because something we are tracking as a byref + // might be used as a return value of a int function (which is legal) + GenTree* blockLastNode = block->lastNode(); + if ((blockLastNode != nullptr) && + (blockLastNode->gtOper == GT_RETURN) && + (varTypeIsGC(compiler->info.compRetType) || + (blockLastNode->gtOp.gtOp1 != nullptr && varTypeIsGC(blockLastNode->gtOp.gtOp1->TypeGet())))) + { + nonVarPtrRegs &= ~RBM_INTRET; + } + + if (nonVarPtrRegs) + { + printf("Regset after BB%02u gcr=", block->bbNum); + printRegMaskInt(gcInfo.gcRegGCrefSetCur & ~regSet.rsMaskVars); + compiler->getEmitter()->emitDispRegSet(gcInfo.gcRegGCrefSetCur & ~regSet.rsMaskVars); + printf(", byr="); + printRegMaskInt(gcInfo.gcRegByrefSetCur & ~regSet.rsMaskVars); + compiler->getEmitter()->emitDispRegSet(gcInfo.gcRegByrefSetCur & ~regSet.rsMaskVars); + printf(", regVars="); + printRegMaskInt(regSet.rsMaskVars); + compiler->getEmitter()->emitDispRegSet(regSet.rsMaskVars); + printf("\n"); + } + + noway_assert(nonVarPtrRegs == RBM_NONE); #endif // DEBUG - noway_assert(stmt->gtOper == GT_STMT); - -#ifdef DEBUGGING_SUPPORT - genEnsureCodeEmitted(stmt->gtStmt.gtStmtILoffsx); -#endif - - } //-------- END-FOR each statement-tree of the current block --------- - #if defined(DEBUG) && defined(_TARGET_ARM64_) if (block->bbNext == nullptr) { @@ -1844,6 +1820,17 @@ void CodeGen::genCodeForBBlist() #endif // defined(DEBUG) && defined(_TARGET_ARM64_) #ifdef DEBUGGING_SUPPORT + // It is possible to reach the end of the block without generating code for the current IL offset. + // For example, if the following IR ends the current block, no code will have been generated for + // offset 21: + // + // ( 0, 0) [000040] ------------ il_offset void IL offset: 21 + // + // N001 ( 0, 0) [000039] ------------ nop void + // + // This can lead to problems when debugging the generated code. To prevent these issues, make sure + // we've generated code for the last IL offset we saw in the block. + genEnsureCodeEmitted(currentILOffset); if (compiler->opts.compScopeInfo && (compiler->info.compVarScopesCount > 0)) { @@ -3608,6 +3595,10 @@ void CodeGen::genCodeForTreeNode(GenTreePtr treeNode) NYI("GT_CLS_VAR_ADDR"); break; + case GT_IL_OFFSET: + // Do nothing; these nodes are simply markers for debug info. + break; + default: { #ifdef DEBUG diff --git a/src/jit/codegencommon.cpp b/src/jit/codegencommon.cpp index 0bb8efc013..e5bd730900 100755 --- a/src/jit/codegencommon.cpp +++ b/src/jit/codegencommon.cpp @@ -462,6 +462,7 @@ void CodeGenInterface::genUpdateLife(VARSET_VALARG_TP newLife) compiler->compUpdateLife</*ForCodeGen*/ true>(newLife); } +#ifdef LEGACY_BACKEND // Returns the liveSet after tree has executed. // "tree" MUST occur in the current statement, AFTER the most recent // update of compiler->compCurLifeTree and compiler->compCurLife. @@ -498,6 +499,7 @@ regMaskTP CodeGen::genNewLiveRegMask(GenTreePtr first, GenTreePtr second) regMaskTP newLiveMask = genLiveMask(VarSetOps::Diff(compiler, secondLiveSet, firstLiveSet)); return newLiveMask; } +#endif // Return the register mask for the given register variable // inline @@ -1178,6 +1180,8 @@ void Compiler::compChangeLife(VARSET_VALARG_TP newLife DEBUGARG(GenTreePtr tree) // Need an explicit instantiation. template void Compiler::compChangeLife<true>(VARSET_VALARG_TP newLife DEBUGARG(GenTreePtr tree)); +#ifdef LEGACY_BACKEND + /***************************************************************************** * * Get the mask of integer registers that contain 'live' enregistered @@ -1356,6 +1360,8 @@ regMaskTP CodeGenInterface::genLiveMask(VARSET_VALARG_TP liveSet) return liveMask; } +#endif + /***************************************************************************** * * Generate a spill. @@ -9074,12 +9080,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) /* figure out what jump we have */ - GenTreePtr jmpNode = block->lastTopLevelStmt(); - - noway_assert(jmpNode && (jmpNode->gtNext == 0)); - noway_assert(jmpNode->gtOper == GT_STMT); - - jmpNode = jmpNode->gtStmt.gtStmtExpr; + GenTree* jmpNode = block->lastNode(); noway_assert(jmpNode->gtOper == GT_JMP); CORINFO_METHOD_HANDLE methHnd = (CORINFO_METHOD_HANDLE)jmpNode->gtVal.gtVal1; @@ -9199,18 +9200,14 @@ void CodeGen::genFnEpilog(BasicBlock* block) noway_assert(block->bbTreeList != nullptr); // figure out what jump we have - GenTreePtr jmpStmt = block->lastTopLevelStmt(); - noway_assert(jmpStmt && (jmpStmt->gtOper == GT_STMT)); + GenTree* jmpNode = block->lastNode(); #if !FEATURE_FASTTAILCALL - noway_assert(jmpStmt->gtNext == nullptr); - GenTreePtr jmpNode = jmpStmt->gtStmt.gtStmtExpr; noway_assert(jmpNode->gtOper == GT_JMP); #else // arm64 // If jmpNode is GT_JMP then gtNext must be null. // If jmpNode is a fast tail call, gtNext need not be null since it could have embedded stmts. - GenTreePtr jmpNode = jmpStmt->gtStmt.gtStmtExpr; - noway_assert((jmpNode->gtOper != GT_JMP) || (jmpStmt->gtNext == nullptr)); + noway_assert((jmpNode->gtOper != GT_JMP) || (jmpNode->gtNext == nullptr)); // Could either be a "jmp method" or "fast tail call" implemented as epilog+jmp noway_assert((jmpNode->gtOper == GT_JMP) || @@ -9475,20 +9472,15 @@ void CodeGen::genFnEpilog(BasicBlock* block) noway_assert(block->bbTreeList); // figure out what jump we have - GenTreePtr jmpStmt = block->lastTopLevelStmt(); - noway_assert(jmpStmt && (jmpStmt->gtOper == GT_STMT)); - + GenTree* jmpNode = block->lastNode(); #if !FEATURE_FASTTAILCALL // x86 - noway_assert(jmpStmt->gtNext == nullptr); - GenTreePtr jmpNode = jmpStmt->gtStmt.gtStmtExpr; noway_assert(jmpNode->gtOper == GT_JMP); #else // amd64 // If jmpNode is GT_JMP then gtNext must be null. // If jmpNode is a fast tail call, gtNext need not be null since it could have embedded stmts. - GenTreePtr jmpNode = jmpStmt->gtStmt.gtStmtExpr; - noway_assert((jmpNode->gtOper != GT_JMP) || (jmpStmt->gtNext == nullptr)); + noway_assert((jmpNode->gtOper != GT_JMP) || (jmpNode->gtNext == nullptr)); // Could either be a "jmp method" or "fast tail call" implemented as epilog+jmp noway_assert((jmpNode->gtOper == GT_JMP) || diff --git a/src/jit/codegeninterface.h b/src/jit/codegeninterface.h index f5eec89d33..e9abbe6b3c 100644 --- a/src/jit/codegeninterface.h +++ b/src/jit/codegeninterface.h @@ -144,8 +144,10 @@ protected: void genUpdateLife(GenTreePtr tree); void genUpdateLife(VARSET_VALARG_TP newLife); +#ifdef LEGACY_BACKEND regMaskTP genLiveMask(GenTreePtr tree); regMaskTP genLiveMask(VARSET_VALARG_TP liveSet); +#endif void genGetRegPairFromMask(regMaskTP regPairMask, regNumber* pLoReg, regNumber* pHiReg); diff --git a/src/jit/codegenxarch.cpp b/src/jit/codegenxarch.cpp index b5f85aa086..eb40069234 100755 --- a/src/jit/codegenxarch.cpp +++ b/src/jit/codegenxarch.cpp @@ -534,13 +534,12 @@ void CodeGen::genCodeForBBlist() if (handlerGetsXcptnObj(block->bbCatchTyp)) { - GenTreePtr firstStmt = block->FirstNonPhiDef(); - if (firstStmt != nullptr) + for (GenTree* node : LIR::AsRange(block)) { - GenTreePtr firstTree = firstStmt->gtStmt.gtStmtExpr; - if (compiler->gtHasCatchArg(firstTree)) + if (node->OperGet() == GT_CATCH_ARG) { gcInfo.gcMarkRegSetGCref(RBM_EXCEPTION_OBJECT); + break; } } } @@ -625,145 +624,112 @@ void CodeGen::genCodeForBBlist() } #endif // FEATURE_EH_FUNCLETS - for (GenTreePtr stmt = block->FirstNonPhiDef(); stmt; stmt = stmt->gtNext) - { - noway_assert(stmt->gtOper == GT_STMT); - - if (stmt->AsStmt()->gtStmtIsEmbedded()) - { - continue; - } - - /* Get hold of the statement tree */ - GenTreePtr tree = stmt->gtStmt.gtStmtExpr; - -#if defined(DEBUGGING_SUPPORT) + // Clear compCurStmt and compCurLifeTree. + compiler->compCurStmt = nullptr; + compiler->compCurLifeTree = nullptr; - /* Do we have a new IL-offset ? */ + // Traverse the block in linear order, generating code for each node as we + // as we encounter it. + CLANG_FORMAT_COMMENT_ANCHOR; - if (stmt->gtStmt.gtStmtILoffsx != BAD_IL_OFFSET) +#ifdef DEBUGGING_SUPPORT + IL_OFFSETX currentILOffset = BAD_IL_OFFSET; +#endif + for (GenTree* node : LIR::AsRange(block).NonPhiNodes()) + { +#ifdef DEBUGGING_SUPPORT + // Do we have a new IL offset? + if (node->OperGet() == GT_IL_OFFSET) { - /* Create and append a new IP-mapping entry */ - genIPmappingAdd(stmt->gtStmt.gtStmt.gtStmtILoffsx, firstMapping); + genEnsureCodeEmitted(currentILOffset); + currentILOffset = node->gtStmt.gtStmtILoffsx; + genIPmappingAdd(currentILOffset, firstMapping); firstMapping = false; } - #endif // DEBUGGING_SUPPORT #ifdef DEBUG - noway_assert(stmt->gtStmt.gtStmtLastILoffs <= compiler->info.compILCodeSize || - stmt->gtStmt.gtStmtLastILoffs == BAD_IL_OFFSET); - - if (compiler->opts.dspCode && compiler->opts.dspInstrs && stmt->gtStmt.gtStmtLastILoffs != BAD_IL_OFFSET) + if (node->OperGet() == GT_IL_OFFSET) { - while (genCurDispOffset <= stmt->gtStmt.gtStmtLastILoffs) - { - genCurDispOffset += dumpSingleInstr(compiler->info.compCode, genCurDispOffset, "> "); - } - } + noway_assert(node->gtStmt.gtStmtLastILoffs <= compiler->info.compILCodeSize || + node->gtStmt.gtStmtLastILoffs == BAD_IL_OFFSET); - stmtNum++; - if (compiler->verbose) - { - printf("\nGenerating BB%02u, stmt %u\t\t", block->bbNum, stmtNum); - printf("Holding variables: "); - dspRegMask(regSet.rsMaskVars); - printf("\n\n"); - if (compiler->verboseTrees) + if (compiler->opts.dspCode && compiler->opts.dspInstrs && + node->gtStmt.gtStmtLastILoffs != BAD_IL_OFFSET) { - compiler->gtDispTree(compiler->opts.compDbgInfo ? stmt : tree); - printf("\n"); - } - } - totalCostEx += ((UINT64)stmt->gtCostEx * block->getBBWeight(compiler)); - totalCostSz += (UINT64)stmt->gtCostSz; -#endif // DEBUG - - // Traverse the tree in linear order, generating code for each node in the - // tree as we encounter it - - compiler->compCurLifeTree = nullptr; - compiler->compCurStmt = stmt; - for (GenTreePtr treeNode = stmt->gtStmt.gtStmtList; treeNode != nullptr; treeNode = treeNode->gtNext) - { - genCodeForTreeNode(treeNode); - if (treeNode->gtHasReg() && treeNode->gtLsraInfo.isLocalDefUse) - { - genConsumeReg(treeNode); + while (genCurDispOffset <= node->gtStmt.gtStmtLastILoffs) + { + genCurDispOffset += dumpSingleInstr(compiler->info.compCode, genCurDispOffset, "> "); + } } } -#ifdef FEATURE_SIMD - // If the next statement expr is a SIMDIntrinsicUpperRestore, don't call rsSpillChk because we - // haven't yet restored spills from the most recent call. - GenTree* nextTopLevelStmt = stmt->AsStmt()->gtStmtNextTopLevelStmt(); - if ((nextTopLevelStmt == nullptr) || (nextTopLevelStmt->AsStmt()->gtStmtExpr->OperGet() != GT_SIMD) || - (nextTopLevelStmt->AsStmt()->gtStmtExpr->gtSIMD.gtSIMDIntrinsicID != SIMDIntrinsicUpperRestore)) -#endif // FEATURE_SIMD - { - regSet.rsSpillChk(); - } - -#ifdef DEBUG - /* Make sure we didn't bungle pointer register tracking */ - - regMaskTP ptrRegs = (gcInfo.gcRegGCrefSetCur | gcInfo.gcRegByrefSetCur); - regMaskTP nonVarPtrRegs = ptrRegs & ~regSet.rsMaskVars; - - // If return is a GC-type, clear it. Note that if a common - // epilog is generated (genReturnBB) it has a void return - // even though we might return a ref. We can't use the compRetType - // as the determiner because something we are tracking as a byref - // might be used as a return value of a int function (which is legal) - if (tree->gtOper == GT_RETURN && - (varTypeIsGC(compiler->info.compRetType) || - (tree->gtOp.gtOp1 != nullptr && varTypeIsGC(tree->gtOp.gtOp1->TypeGet())))) - { - nonVarPtrRegs &= ~RBM_INTRET; - } - // When profiling, the first statement in a catch block will be the - // harmless "inc" instruction (does not interfere with the exception - // object). + // TODO-LIR: the cost accounting performed below is incorrect: each operator's cost includes the + // cost of its operands, so the total cost of the block is grossly overestimated. Fixing + // this requires the ability to calculate the cost of the operator itself. + // + // totalCostEx += (UINT64)node->gtCostEx * block->getBBWeight(compiler); + // totalCostSz += (UINT64)node->gtCostSz; - if ((compiler->opts.eeFlags & CORJIT_FLG_BBINSTR) && (stmt == block->bbTreeList) && - handlerGetsXcptnObj(block->bbCatchTyp)) - { - nonVarPtrRegs &= ~RBM_EXCEPTION_OBJECT; - } +#endif // DEBUG - if (nonVarPtrRegs) + genCodeForTreeNode(node); + if (node->gtHasReg() && node->gtLsraInfo.isLocalDefUse) { - printf("Regset after tree="); - compiler->printTreeID(tree); - printf(" BB%02u gcr=", block->bbNum); - printRegMaskInt(gcInfo.gcRegGCrefSetCur & ~regSet.rsMaskVars); - compiler->getEmitter()->emitDispRegSet(gcInfo.gcRegGCrefSetCur & ~regSet.rsMaskVars); - printf(", byr="); - printRegMaskInt(gcInfo.gcRegByrefSetCur & ~regSet.rsMaskVars); - compiler->getEmitter()->emitDispRegSet(gcInfo.gcRegByrefSetCur & ~regSet.rsMaskVars); - printf(", regVars="); - printRegMaskInt(regSet.rsMaskVars); - compiler->getEmitter()->emitDispRegSet(regSet.rsMaskVars); - printf("\n"); + genConsumeReg(node); } + } // end for each node in block - noway_assert(nonVarPtrRegs == 0); - - for (GenTree* node = stmt->gtStmt.gtStmtList; node; node = node->gtNext) - { - assert(!(node->gtFlags & GTF_SPILL)); - } +#ifdef DEBUG + // The following set of register spill checks and GC pointer tracking checks used to be + // performed at statement boundaries. Now, with LIR, there are no statements, so they are + // performed at the end of each block. + // TODO: could these checks be performed more frequently? E.g., at each location where + // the register allocator says there are no live non-variable registers. Perhaps this could + // be done by (a) keeping a running count of live non-variable registers by using + // gtLsraInfo.srcCount and gtLsraInfo.dstCount to decrement and increment the count, respectively, + // and running the checks when the count is zero. Or, (b) use the map maintained by LSRA + // (operandToLocationInfoMap) to mark a node somehow when, after the execution of that node, + // there will be no live non-variable registers. + + regSet.rsSpillChk(); + + /* Make sure we didn't bungle pointer register tracking */ + + regMaskTP ptrRegs = gcInfo.gcRegGCrefSetCur | gcInfo.gcRegByrefSetCur; + regMaskTP nonVarPtrRegs = ptrRegs & ~regSet.rsMaskVars; + + // If return is a GC-type, clear it. Note that if a common + // epilog is generated (genReturnBB) it has a void return + // even though we might return a ref. We can't use the compRetType + // as the determiner because something we are tracking as a byref + // might be used as a return value of a int function (which is legal) + GenTree* blockLastNode = block->lastNode(); + if ((blockLastNode != nullptr) && + (blockLastNode->gtOper == GT_RETURN) && + (varTypeIsGC(compiler->info.compRetType) || + (blockLastNode->gtOp.gtOp1 != nullptr && varTypeIsGC(blockLastNode->gtOp.gtOp1->TypeGet())))) + { + nonVarPtrRegs &= ~RBM_INTRET; + } + + if (nonVarPtrRegs) + { + printf("Regset after BB%02u gcr=", block->bbNum); + printRegMaskInt(gcInfo.gcRegGCrefSetCur & ~regSet.rsMaskVars); + compiler->getEmitter()->emitDispRegSet(gcInfo.gcRegGCrefSetCur & ~regSet.rsMaskVars); + printf(", byr="); + printRegMaskInt(gcInfo.gcRegByrefSetCur & ~regSet.rsMaskVars); + compiler->getEmitter()->emitDispRegSet(gcInfo.gcRegByrefSetCur & ~regSet.rsMaskVars); + printf(", regVars="); + printRegMaskInt(regSet.rsMaskVars); + compiler->getEmitter()->emitDispRegSet(regSet.rsMaskVars); + printf("\n"); + } + noway_assert(nonVarPtrRegs == RBM_NONE); #endif // DEBUG - noway_assert(stmt->gtOper == GT_STMT); - -#ifdef DEBUGGING_SUPPORT - genEnsureCodeEmitted(stmt->gtStmt.gtStmtILoffsx); -#endif - - } //-------- END-FOR each statement-tree of the current block --------- - #if defined(DEBUG) && defined(LATE_DISASM) && defined(_TARGET_AMD64_) if (block->bbNext == nullptr) { @@ -776,6 +742,17 @@ void CodeGen::genCodeForBBlist() #endif // defined(DEBUG) && defined(LATE_DISASM) && defined(_TARGET_ARM64_) #ifdef DEBUGGING_SUPPORT + // It is possible to reach the end of the block without generating code for the current IL offset. + // For example, if the following IR ends the current block, no code will have been generated for + // offset 21: + // + // ( 0, 0) [000040] ------------ il_offset void IL offset: 21 + // + // N001 ( 0, 0) [000039] ------------ nop void + // + // This can lead to problems when debugging the generated code. To prevent these issues, make sure + // we've generated code for the last IL offset we saw in the block. + genEnsureCodeEmitted(currentILOffset); if (compiler->opts.compScopeInfo && (compiler->info.compVarScopesCount > 0)) { @@ -1071,14 +1048,12 @@ void CodeGen::genCodeForBBlist() case BBJ_EHFILTERRET: { // The last statement of the block must be a GT_RETFILT, which has already been generated. - GenTree* tmpNode = nullptr; - assert((block->bbTreeList != nullptr) && - ((tmpNode = block->bbTreeList->gtPrev->AsStmt()->gtStmtExpr) != nullptr) && - (tmpNode->gtOper == GT_RETFILT)); + assert(block->lastNode() != nullptr); + assert(block->lastNode()->OperGet() == GT_RETFILT); if (block->bbJumpKind == BBJ_EHFINALLYRET) { - assert(tmpNode->gtOp.gtOp1 == nullptr); // op1 == nullptr means endfinally + assert(block->lastNode()->gtOp.gtOp1 == nullptr); // op1 == nullptr means endfinally // Return using a pop-jmp sequence. As the "try" block calls // the finally with a jmp, this leaves the x86 call-ret stack @@ -2866,6 +2841,10 @@ void CodeGen::genCodeForTreeNode(GenTreePtr treeNode) break; #endif + case GT_IL_OFFSET: + // Do nothing; these nodes are simply markers for debug info. + break; + default: { #ifdef DEBUG diff --git a/src/jit/compiler.cpp b/src/jit/compiler.cpp index fd8e1d2846..6c38f87b48 100644 --- a/src/jit/compiler.cpp +++ b/src/jit/compiler.cpp @@ -4034,7 +4034,8 @@ void Compiler::compFunctionTraceStart() { printf(" "); } - printf("{ Start Jitting %s\n", info.compFullName); /* } editor brace matching workaround for this printf */ + printf("{ Start Jitting %s (MethodHash=%08x)\n", info.compFullName, + info.compMethodHash()); /* } editor brace matching workaround for this printf */ } #endif // DEBUG } @@ -8062,44 +8063,45 @@ void cBlockIR(Compiler* comp, BasicBlock* block) } printf("\n"); - for (GenTreeStmt* stmt = block->firstStmt(); stmt; stmt = stmt->gtNextStmt) - { - // Skip embedded stmts. They should have already been dumped prior to the stmt - // that they are embedded into. Even though they appear on the stmt list - // after the stmt they are embedded into. Don't understand the rationale for that - // but make the dataflow view look consistent. - if ((stmt->gtFlags & GTF_STMT_TOP_LEVEL) == 0) + if (!block->IsLIR()) + { + for (GenTreeStmt* stmt = block->firstStmt(); stmt; stmt = stmt->gtNextStmt) { - continue; - } + // Print current stmt. - // Print current stmt. + if (trees) + { + cTree(comp, stmt); + printf("\n"); + printf("=====================================================================\n"); + } - if (trees) - { - cTree(comp, stmt); - printf("\n"); - printf("=====================================================================\n"); - } + if (comp->compRationalIRForm) + { + GenTree* tree; - if (comp->compRationalIRForm) - { - GenTree* tree; + foreach_treenode_execution_order(tree, stmt) + { + cNodeIR(comp, tree); + } + } + else + { + cTreeIR(comp, stmt); + } - foreach_treenode_execution_order(tree, stmt) + if (!noStmts && !trees) { - cNodeIR(comp, tree); + printf("\n"); } } - else - { - cTreeIR(comp, stmt); - } - - if (!noStmts && !trees) + } + else + { + for (GenTree* node = block->bbTreeList; node != nullptr; node = node->gtNext) { - printf("\n"); + cNodeIR(comp, node); } } @@ -8817,14 +8819,6 @@ int cTreeFlagsIR(Compiler* comp, GenTree* tree) { chars += printf("[STMT_HAS_CSE]"); } - if (tree->gtFlags & GTF_STMT_TOP_LEVEL) - { - chars += printf("[STMT_TOP_LEVEL]"); - } - if (tree->gtFlags & GTF_STMT_SKIP_LOWER) - { - chars += printf("[STMT_SKIP_LOWER]"); - } break; default: @@ -9494,6 +9488,18 @@ int cLeafIR(Compiler* comp, GenTree* tree) } break; + case GT_IL_OFFSET: + + if (tree->gtStmt.gtStmtILoffsx == BAD_IL_OFFSET) + { + chars += printf("?"); + } + else + { + chars += printf("0x%x", jitGetILoffs(tree->gtStmt.gtStmtILoffsx)); + } + break; + case GT_CLS_VAR: case GT_CLS_VAR_ADDR: default: @@ -9861,6 +9867,8 @@ void cNodeIR(Compiler* comp, GenTree* tree) } } + bool nodeIsValue = tree->IsValue(); + // Dump tree id or dataflow destination. int chars = 0; @@ -9895,7 +9903,7 @@ void cNodeIR(Compiler* comp, GenTree* tree) chars += cValNumIR(comp, tree); } } - else + else if (nodeIsValue) { chars += printf("t%d", tree->gtTreeID); if (comp->dumpIRRegs) @@ -9920,7 +9928,7 @@ void cNodeIR(Compiler* comp, GenTree* tree) chars += dTabStopIR(chars, COLUMN_OPCODE); const char* opName = tree->OpName(op); - chars += printf(" = %s", opName); + chars += printf(" %c %s", nodeIsValue ? '=' : ' ', opName); if (dataflowView) { diff --git a/src/jit/compiler.h b/src/jit/compiler.h index fd3c800474..73d0d9533b 100644 --- a/src/jit/compiler.h +++ b/src/jit/compiler.h @@ -24,6 +24,9 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #include "jit.h" #include "opcode.h" +#include "varset.h" +#include "gentree.h" +#include "lir.h" #include "block.h" #include "inline.h" #include "jiteh.h" @@ -32,7 +35,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #include "sm.h" #include "simplerhash.h" #include "cycletimer.h" -#include "varset.h" #include "blockset.h" #include "jitstd.h" #include "arraystack.h" @@ -117,8 +119,6 @@ void* __cdecl operator new(size_t n, void* p, const jitstd::placement_t& syntax_ /* This is included here and not earlier as it needs the definition of "CSE" * which is defined in the section above */ -#include "gentree.h" - /*****************************************************************************/ unsigned genLog2(unsigned value); @@ -1400,6 +1400,7 @@ class Compiler friend class CodeGen; friend class LclVarDsc; friend class TempDsc; + friend class LIR; friend class ObjectAllocator; #ifndef _TARGET_64BIT_ @@ -2074,7 +2075,7 @@ public: // Functions to display the trees #ifdef DEBUG - void gtDispNode(GenTreePtr tree, IndentStack* indentStack, __in_z const char* msg); + void gtDispNode(GenTreePtr tree, IndentStack* indentStack, __in_z const char* msg, bool isLIR); void gtDispVN(GenTreePtr tree); void gtDispConst(GenTreePtr tree); @@ -2100,7 +2101,8 @@ public: void gtDispTree(GenTreePtr tree, IndentStack* indentStack = nullptr, __in_opt const char* msg = nullptr, - bool topOnly = false); + bool topOnly = false, + bool isLIR = false); void gtGetLclVarNameInfo(unsigned lclNum, const char** ilKindOut, const char** ilNameOut, unsigned* ilNumOut); int gtGetLclVarName(unsigned lclNum, char* buf, unsigned buf_remaining); char* gtGetLclVarName(unsigned lclNum); @@ -2111,12 +2113,11 @@ public: void gtDispArgList(GenTreePtr tree, IndentStack* indentStack); void gtDispFieldSeq(FieldSeqNode* pfsn); - GenTreePtr gtDispLinearTree(GenTreeStmt* curStmt, - GenTreePtr nextLinearNode, - GenTreePtr tree, - IndentStack* indentStack, - __in_opt const char* msg = nullptr); - GenTreePtr gtDispLinearStmt(GenTreeStmt* stmt, IndentStack* indentStack = nullptr); + void gtDispRange(LIR::ReadOnlyRange const& range); + + void gtDispTreeRange(LIR::Range& containingRange, GenTree* tree); + + void gtDispLIRNode(GenTree* node); #endif // For tree walks @@ -2476,6 +2477,7 @@ public: static fgWalkPreFn lvaDecRefCntsCB; void lvaDecRefCnts(GenTreePtr tree); + void lvaDecRefCnts(BasicBlock* basicBlock, GenTreePtr tree); void lvaRecursiveDecRefCounts(GenTreePtr tree); void lvaRecursiveIncRefCounts(GenTreePtr tree); @@ -3372,12 +3374,7 @@ public: // - In FGOrderTree, the dominant ordering is the tree order, and the nodes contained in // each tree and sub-tree are contiguous, and can be traversed (in gtNext/gtPrev order) // by traversing the tree according to the order of the operands. - // - In FGOrderLinear, the dominant ordering is the linear order. Each statement is either - // a top-level statement (GTF_STMT_TOP_LEVEL), or its nodes are ordered within the previous - // top-level statement. It is an invariant that such nodes are FULLY embedded in the top- - // level statement (i.e. both the first and last node, in execution order, for the top-level - // statement DO NOT belong to one of the embedded trees). It is possible that we will want - // to relax this requirement, but it makes it easier to validate the order. + // - In FGOrderLinear, the dominant ordering is the linear order. enum FlowGraphOrder { @@ -3469,6 +3466,7 @@ public: BasicBlock* fgSplitBlockAtBeginning(BasicBlock* curr); BasicBlock* fgSplitBlockAtEnd(BasicBlock* curr); BasicBlock* fgSplitBlockAfterStatement(BasicBlock* curr, GenTree* stmt); + BasicBlock* fgSplitBlockAfterNode(BasicBlock* curr, GenTree* node); // for LIR BasicBlock* fgSplitEdge(BasicBlock* curr, BasicBlock* succ); GenTreeStmt* fgNewStmtFromTree(GenTreePtr tree, BasicBlock* block, IL_OFFSETX offs); @@ -3476,8 +3474,6 @@ public: GenTreeStmt* fgNewStmtFromTree(GenTreePtr tree, BasicBlock* block); GenTreeStmt* fgNewStmtFromTree(GenTreePtr tree, IL_OFFSETX offs); - GenTreePtr fgGetLastTopLevelStmt(BasicBlock* block); - GenTreePtr fgGetTopLevelQmark(GenTreePtr expr, GenTreePtr* ppDst = nullptr); void fgExpandQmarkForCastInstOf(BasicBlock* block, GenTreePtr stmt); void fgExpandQmarkStmt(BasicBlock* block, GenTreePtr expr); @@ -3504,7 +3500,8 @@ public: #ifdef LEGACY_BACKEND GenTreePtr fgLegacyPerStatementLocalVarLiveness(GenTreePtr startNode, GenTreePtr relopNode, GenTreePtr asgdLclVar); #else - void fgPerStatementLocalVarLiveness(GenTreePtr startNode, GenTreePtr asgdLclVar); + void fgPerNodeLocalVarLiveness(GenTree* node, GenTree* asgdLclVar); + void fgPerStatementLocalVarLiveness(GenTree* node, GenTree* asgdLclVar); #endif void fgPerBlockLocalVarLiveness(); @@ -3535,12 +3532,16 @@ public: VARSET_VALARG_TP volatileVars, bool* pStmtInfoDirty DEBUGARG(bool* treeModf)); + VARSET_VALRET_TP fgComputeLifeLIR(VARSET_VALARG_TP life, BasicBlock* block, VARSET_VALARG_TP volatileVars); + bool fgRemoveDeadStore(GenTree** pTree, LclVarDsc* varDsc, VARSET_TP life, bool* doAgain, bool* pStmtInfoDirty DEBUGARG(bool* treeModf)); + bool fgTryRemoveDeadLIRStore(LIR::Range& blockRange, GenTree* node, GenTree** next); + // For updating liveset during traversal AFTER fgComputeLife has completed VARSET_VALRET_TP fgGetVarBits(GenTreePtr tree); VARSET_VALRET_TP fgUpdateLiveSet(VARSET_VALARG_TP liveSet, GenTreePtr tree); @@ -4058,8 +4059,6 @@ public: void fgRemoveEmptyBlocks(); - void fgRemoveLinearOrderDependencies(GenTreePtr stmt); - void fgRemoveStmt(BasicBlock* block, GenTreePtr stmt, bool updateRefCnt = true); bool fgCheckRemoveStmt(BasicBlock* block, GenTreePtr stmt); @@ -4189,6 +4188,7 @@ public: void fgDispBasicBlocks(BasicBlock* firstBlock, BasicBlock* lastBlock, bool dumpTrees); void fgDispBasicBlocks(bool dumpTrees = false); void fgDumpStmtTree(GenTreePtr stmt, unsigned blkNum); + void fgDumpBlock(BasicBlock* block); void fgDumpTrees(BasicBlock* firstBlock, BasicBlock* lastBlock); static fgWalkPreFn fgStress64RsltMulCB; @@ -4212,19 +4212,8 @@ public: regMaskTP* regsPtr); // OUT #endif // LEGACY_BACKEND - static GenTreeStmt* fgFindTopLevelStmtBackwards(GenTreeStmt* stmt); static GenTreePtr fgGetFirstNode(GenTreePtr tree); - static void fgSnipNode(GenTreeStmt* stmt, GenTreePtr node); - static void fgSnipInnerNode(GenTreePtr node); - static void fgDeleteTreeFromList(GenTreeStmt* stmt, GenTreePtr tree); static bool fgTreeIsInStmt(GenTree* tree, GenTreeStmt* stmt); - static void fgInsertTreeInListBefore(GenTree* tree, GenTree* insertionPoint, GenTreeStmt* stmt); - static void fgInsertTreeInListAfter(GenTree* tree, GenTree* insertionPoint, GenTreeStmt* stmt); - GenTreeStmt* fgInsertTreeBeforeAsEmbedded(GenTree* tree, GenTree* before, GenTreeStmt* stmt, BasicBlock* block); - GenTreeStmt* fgInsertTreeAfterAsEmbedded(GenTree* tree, GenTree* before, GenTreeStmt* stmt, BasicBlock* block); - bool fgNodeContainsEmbeddedStatement(GenTree* tree, GenTreeStmt* topLevel); - void fgRemoveContainedEmbeddedStatements(GenTreePtr tree, GenTreeStmt* topLevel, BasicBlock* block); - bool fgStmtContainsNode(GenTreeStmt* stmt, GenTree* tree); inline bool fgIsInlining() { @@ -4377,24 +4366,14 @@ private: public: // Used by linear scan register allocation GenTreePtr fgInsertStmtBefore(BasicBlock* block, GenTreePtr insertionPoint, GenTreePtr stmt); - void fgReplaceStmt(BasicBlock* block, GenTreeStmt* stmt, GenTreePtr newTree); private: GenTreePtr fgInsertStmtListAfter(BasicBlock* block, GenTreePtr stmtAfter, GenTreePtr stmtList); GenTreePtr fgMorphSplitTree(GenTree** splitPoint, GenTree* stmt, BasicBlock* blk); - // insert the given subtree as an embedded statement of parentStmt - GenTreeStmt* fgMakeEmbeddedStmt(BasicBlock* block, GenTreePtr tree, GenTreePtr parentStmt); - - // Insert the given single node before 'before'. - // Either the callee must ensure that 'before' is part of compCurStmt, - // or before->gtPrev must be non-null - void fgInsertLinearNodeBefore(GenTreePtr newNode, GenTreePtr before); - // Create a new temporary variable to hold the result of *ppTree, // and transform the graph accordingly. - GenTreeStmt* fgInsertEmbeddedFormTemp(GenTree** ppTree, unsigned lvaNum = BAD_VAR_NUM); GenTree* fgInsertCommaFormTemp(GenTree** ppTree, CORINFO_CLASS_HANDLE structType = nullptr); GenTree* fgMakeMultiUse(GenTree** ppTree); @@ -4402,8 +4381,10 @@ private: // if it happens to be an argument to a call. void fgFixupIfCallArg(ArrayStack<GenTree*>* parentStack, GenTree* oldChild, GenTree* newChild); +public: void fgFixupArgTabEntryPtr(GenTreePtr parentCall, GenTreePtr oldArg, GenTreePtr newArg); +private: // Recognize a bitwise rotation pattern and convert into a GT_ROL or a GT_ROR node. GenTreePtr fgRecognizeAndMorphBitwiseRotation(GenTreePtr tree); bool fgOperIsBitwiseRotationRoot(genTreeOps oper); @@ -4414,9 +4395,9 @@ private: GenTree* fgTreeSeqLst; GenTree* fgTreeSeqBeg; - GenTree* fgSetTreeSeq(GenTree* tree, GenTree* prev = nullptr); - void fgSetTreeSeqHelper(GenTree* tree); - void fgSetTreeSeqFinish(GenTreePtr tree); + GenTree* fgSetTreeSeq(GenTree* tree, GenTree* prev = nullptr, bool isLIR = false); + void fgSetTreeSeqHelper(GenTree* tree, bool isLIR); + void fgSetTreeSeqFinish(GenTreePtr tree, bool isLIR); void fgSetStmtSeq(GenTree* tree); void fgSetBlockOrder(BasicBlock* block); diff --git a/src/jit/compiler.hpp b/src/jit/compiler.hpp index a2af592733..29717864be 100644 --- a/src/jit/compiler.hpp +++ b/src/jit/compiler.hpp @@ -824,9 +824,10 @@ void* GenTree::operator new(size_t sz, Compiler* comp, genTreeOps oper) // GenTree constructor inline GenTree::GenTree(genTreeOps oper, var_types type DEBUGARG(bool largeNode)) { - gtOper = oper; - gtType = type; - gtFlags = 0; + gtOper = oper; + gtType = type; + gtFlags = 0; + gtLIRFlags = 0; #ifdef DEBUG gtDebugFlags = 0; #endif // DEBUG @@ -2768,7 +2769,23 @@ inline bool Compiler::fgIsThrowHlpBlk(BasicBlock* block) return false; } - GenTreePtr call = block->bbTreeList->gtStmt.gtStmtExpr; + GenTree* call = block->lastNode(); + +#ifdef DEBUG + if (block->IsLIR()) + { + LIR::Range& blockRange = LIR::AsRange(block); + for (LIR::Range::ReverseIterator node = blockRange.rbegin(), end = blockRange.rend(); node != end; ++node) + { + if (node->OperGet() == GT_CALL) + { + assert(*node == call); + assert(node == blockRange.rbegin()); + break; + } + } + } +#endif if (!call || (call->gtOper != GT_CALL)) { @@ -4569,9 +4586,9 @@ inline bool BasicBlock::endsWithJmpMethod(Compiler* comp) { if (comp->compJmpOpUsed && (bbJumpKind == BBJ_RETURN) && (bbFlags & BBF_HAS_JMP)) { - GenTreePtr last = comp->fgGetLastTopLevelStmt(this); - assert(last != nullptr); - return last->gtStmt.gtStmtExpr->gtOper == GT_JMP; + GenTree* lastNode = this->lastNode(); + assert(lastNode != nullptr); + return lastNode->OperGet() == GT_JMP; } return false; @@ -4635,12 +4652,10 @@ inline bool BasicBlock::endsWithTailCall(Compiler* comp, if (result) { - GenTreePtr last = comp->fgGetLastTopLevelStmt(this); - assert(last != nullptr); - last = last->gtStmt.gtStmtExpr; - if (last->OperGet() == GT_CALL) + GenTree* lastNode = this->lastNode(); + if (lastNode->OperGet() == GT_CALL) { - GenTreeCall* call = last->AsCall(); + GenTreeCall* call = lastNode->AsCall(); if (tailCallsConvertibleToLoopOnly) { result = call->IsTailCallConvertibleToLoop(); @@ -4687,19 +4702,6 @@ inline bool BasicBlock::endsWithTailCallConvertibleToLoop(Compiler* comp, GenTre return endsWithTailCall(comp, fastTailCallsOnly, tailCallsConvertibleToLoopOnly, tailCall); } -// Returns the last top level stmt of a given basic block. -// Returns nullptr if the block is empty. -inline GenTreePtr Compiler::fgGetLastTopLevelStmt(BasicBlock* block) -{ - // Return if the block is empty - if (block->bbTreeList == nullptr) - { - return nullptr; - } - - return fgFindTopLevelStmtBackwards(block->bbTreeList->gtPrev->AsStmt()); -} - inline GenTreeBlkOp* Compiler::gtCloneCpObjNode(GenTreeCpObj* source) { GenTreeCpObj* result = new (this, GT_COPYOBJ) GenTreeCpObj(source->gtGcPtrCount, source->gtSlots, source->gtGcPtrs); @@ -4717,6 +4719,31 @@ inline static bool StructHasCustomLayout(DWORD attribs) return ((attribs & CORINFO_FLG_CUSTOMLAYOUT) != 0); } +/***************************************************************************** + * This node should not be referenced by anyone now. Set its values to garbage + * to catch extra references + */ + +inline void DEBUG_DESTROY_NODE(GenTreePtr tree) +{ +#ifdef DEBUG + // printf("DEBUG_DESTROY_NODE for [0x%08x]\n", tree); + + // Save gtOper in case we want to find out what this node was + tree->gtOperSave = tree->gtOper; + + tree->gtType = TYP_UNDEF; + tree->gtFlags |= 0xFFFFFFFF & ~GTF_NODE_MASK; + if (tree->OperIsSimple()) + { + tree->gtOp.gtOp1 = tree->gtOp.gtOp2 = nullptr; + } + // Must do this last, because the "gtOp" check above will fail otherwise. + // Don't call SetOper, because GT_COUNT is not a valid value + tree->gtOper = GT_COUNT; +#endif +} + /*****************************************************************************/ #endif //_COMPILER_HPP_ /*****************************************************************************/ diff --git a/src/jit/decomposelongs.cpp b/src/jit/decomposelongs.cpp index 5e18c49212..8277a3c68a 100644 --- a/src/jit/decomposelongs.cpp +++ b/src/jit/decomposelongs.cpp @@ -66,69 +66,37 @@ void DecomposeLongs::PrepareForDecomposition() void DecomposeLongs::DecomposeBlock(BasicBlock* block) { assert(block == m_compiler->compCurBB); // compCurBB must already be set. + assert(block->isEmpty() || block->IsLIR()); - for (GenTree* stmt = block->bbTreeList; stmt != nullptr; stmt = stmt->gtNext) + m_block = block; + + GenTree* node = BlockRange().FirstNonPhiNode(); + while (node != nullptr) { -#ifdef DEBUG - if (m_compiler->verbose) + LIR::Use use; + if (!BlockRange().TryGetUse(node, &use)) { - printf("Decomposing BB%02u, stmt id %u\n", block->bbNum, stmt->gtTreeID); + use = LIR::Use::GetDummyUse(BlockRange(), node); } -#endif // DEBUG - DecomposeStmt(stmt->AsStmt()); + node = DecomposeNode(use); } -} - -//------------------------------------------------------------------------ -// DecomposeStmt: Do LONG decomposition to a statement tree. -// -// Arguments: -// stmt - the statement to process -// -// Return Value: -// None. -// -void DecomposeLongs::DecomposeStmt(GenTreeStmt* stmt) -{ - GenTree* savedStmt = m_compiler->compCurStmt; // We'll need to restore this later, in case this call was recursive. - m_compiler->compCurStmt = stmt; // Publish the current statement globally. One reason: - // fgInsertEmbeddedFormTemp requires it. - m_compiler->fgWalkTreePost(&stmt->gtStmt.gtStmtExpr, &DecomposeLongs::DecompNodeHelper, this, true); - m_compiler->compCurStmt = savedStmt; -} -//------------------------------------------------------------------------ -// DecompNodeHelper: fgWalkTreePost callback helper for LONG decomposition -// -// Arguments: -// ppTree - tree node we are working on. -// data - tree walk context, with data->pCallbackData as a DecomposeLongs* -// -// Return Value: -// Standard tree walk result. -// -// static -Compiler::fgWalkResult DecomposeLongs::DecompNodeHelper(GenTree** ppTree, Compiler::fgWalkData* data) -{ - DecomposeLongs* decomp = (DecomposeLongs*)data->pCallbackData; - decomp->DecomposeNode(ppTree, data); - return Compiler::WALK_CONTINUE; + assert(BlockRange().CheckLIR(m_compiler)); } //------------------------------------------------------------------------ // DecomposeNode: Decompose long-type trees into lower and upper halves. // // Arguments: -// *ppTree - A node that may or may not require decomposition. -// data - The tree-walk data that provides the context. +// use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: -// None. It the tree at *ppTree is of TYP_LONG, it will generally be replaced. +// The next node to process. // -void DecomposeLongs::DecomposeNode(GenTree** ppTree, Compiler::fgWalkData* data) +GenTree* DecomposeLongs::DecomposeNode(LIR::Use& use) { - GenTree* tree = *ppTree; + GenTree* tree = use.Def(); // Handle the case where we are implicitly using the lower half of a long lclVar. if ((tree->TypeGet() == TYP_INT) && tree->OperIsLocal()) @@ -141,58 +109,60 @@ void DecomposeLongs::DecomposeNode(GenTree** ppTree, Compiler::fgWalkData* data) { printf("Changing implicit reference to lo half of long lclVar to an explicit reference of its promoted " "half:\n"); - m_compiler->gtDispTree(tree); + m_compiler->gtDispTreeRange(BlockRange(), tree); } #endif // DEBUG m_compiler->lvaDecRefCnts(tree); unsigned loVarNum = varDsc->lvFieldLclStart; tree->AsLclVarCommon()->SetLclNum(loVarNum); m_compiler->lvaIncRefCnts(tree); - return; + return tree->gtNext; } } if (tree->TypeGet() != TYP_LONG) { - return; + return tree->gtNext; } #ifdef DEBUG if (m_compiler->verbose) { printf("Decomposing TYP_LONG tree. BEFORE:\n"); - m_compiler->gtDispTree(tree); + m_compiler->gtDispTreeRange(BlockRange(), tree); } #endif // DEBUG + GenTree* nextNode = nullptr; switch (tree->OperGet()) { case GT_PHI: case GT_PHI_ARG: + nextNode = tree->gtNext; break; case GT_LCL_VAR: - DecomposeLclVar(ppTree, data); + nextNode = DecomposeLclVar(use); break; case GT_LCL_FLD: - DecomposeLclFld(ppTree, data); + nextNode = DecomposeLclFld(use); break; case GT_STORE_LCL_VAR: - DecomposeStoreLclVar(ppTree, data); + nextNode = DecomposeStoreLclVar(use); break; case GT_CAST: - DecomposeCast(ppTree, data); + nextNode = DecomposeCast(use); break; case GT_CNS_LNG: - DecomposeCnsLng(ppTree, data); + nextNode = DecomposeCnsLng(use); break; case GT_CALL: - DecomposeCall(ppTree, data); + nextNode = DecomposeCall(use); break; case GT_RETURN: @@ -200,7 +170,7 @@ void DecomposeLongs::DecomposeNode(GenTree** ppTree, Compiler::fgWalkData* data) break; case GT_STOREIND: - DecomposeStoreInd(ppTree, data); + nextNode = DecomposeStoreInd(use); break; case GT_STORE_LCL_FLD: @@ -209,15 +179,15 @@ void DecomposeLongs::DecomposeNode(GenTree** ppTree, Compiler::fgWalkData* data) break; case GT_IND: - DecomposeInd(ppTree, data); + nextNode = DecomposeInd(use); break; case GT_NOT: - DecomposeNot(ppTree, data); + nextNode = DecomposeNot(use); break; case GT_NEG: - DecomposeNeg(ppTree, data); + nextNode = DecomposeNeg(use); break; // Binary operators. Those that require different computation for upper and lower half are @@ -227,7 +197,7 @@ void DecomposeLongs::DecomposeNode(GenTree** ppTree, Compiler::fgWalkData* data) case GT_OR: case GT_XOR: case GT_AND: - DecomposeArith(ppTree, data); + nextNode = DecomposeArith(use); break; case GT_MUL: @@ -283,10 +253,13 @@ void DecomposeLongs::DecomposeNode(GenTree** ppTree, Compiler::fgWalkData* data) #ifdef DEBUG if (m_compiler->verbose) { - printf(" AFTER:\n"); - m_compiler->gtDispTree(*ppTree); + // NOTE: st_lcl_var doesn't dump properly afterwards. + printf("Decomposing TYP_LONG tree. AFTER:\n"); + m_compiler->gtDispTreeRange(BlockRange(), use.Def()); } #endif + + return nextNode; } //------------------------------------------------------------------------ @@ -295,63 +268,56 @@ void DecomposeLongs::DecomposeNode(GenTree** ppTree, Compiler::fgWalkData* data) // with a new GT_LONG node that will replace the original node. // // Arguments: -// ppTree - the original tree node -// data - tree walk context +// use - the LIR::Use object for the def that needs to be decomposed. // loResult - the decomposed low part -// hiResult - the decomposed high part +// hiResult - the decomposed high part. This must follow loResult in the linear order, +// as the new GT_LONG node will be inserted immediately after it. // // Return Value: -// None. +// The next node to process. // -void DecomposeLongs::FinalizeDecomposition(GenTree** ppTree, - Compiler::fgWalkData* data, - GenTree* loResult, - GenTree* hiResult) +GenTree* DecomposeLongs::FinalizeDecomposition(LIR::Use& use, GenTree* loResult, GenTree* hiResult) { - assert(ppTree != nullptr); - assert(*ppTree != nullptr); - assert(data != nullptr); + assert(use.IsInitialized()); assert(loResult != nullptr); assert(hiResult != nullptr); - assert(m_compiler->compCurStmt != nullptr); + assert(BlockRange().Contains(loResult)); + assert(BlockRange().Contains(hiResult)); + assert(loResult->Precedes(hiResult)); - GenTree* tree = *ppTree; + GenTree* gtLong = new (m_compiler, GT_LONG) GenTreeOp(GT_LONG, TYP_LONG, loResult, hiResult); + BlockRange().InsertAfter(hiResult, gtLong); - m_compiler->fgInsertTreeInListAfter(hiResult, loResult, m_compiler->compCurStmt->AsStmt()); - hiResult->CopyCosts(tree); + use.ReplaceWith(m_compiler, gtLong); - GenTree* newTree = new (m_compiler, GT_LONG) GenTreeOp(GT_LONG, TYP_LONG, loResult, hiResult); - SimpleLinkNodeAfter(hiResult, newTree); - m_compiler->fgFixupIfCallArg(data->parentStack, tree, newTree); - newTree->CopyCosts(tree); - *ppTree = newTree; + return gtLong->gtNext; } //------------------------------------------------------------------------ // DecomposeLclVar: Decompose GT_LCL_VAR. // // Arguments: -// ppTree - the tree to decompose -// data - tree walk context +// use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: -// None. +// The next node to process. // -void DecomposeLongs::DecomposeLclVar(GenTree** ppTree, Compiler::fgWalkData* data) +GenTree* DecomposeLongs::DecomposeLclVar(LIR::Use& use) { - assert(ppTree != nullptr); - assert(*ppTree != nullptr); - assert(data != nullptr); - assert((*ppTree)->OperGet() == GT_LCL_VAR); + assert(use.IsInitialized()); + assert(use.Def()->OperGet() == GT_LCL_VAR); - GenTree* tree = *ppTree; + GenTree* tree = use.Def(); unsigned varNum = tree->AsLclVarCommon()->gtLclNum; LclVarDsc* varDsc = m_compiler->lvaTable + varNum; m_compiler->lvaDecRefCnts(tree); GenTree* loResult = tree; loResult->gtType = TYP_INT; + GenTree* hiResult = m_compiler->gtNewLclLNode(varNum, TYP_INT); + hiResult->CopyCosts(loResult); + BlockRange().InsertAfter(loResult, hiResult); if (varDsc->lvPromoted) { @@ -377,63 +343,55 @@ void DecomposeLongs::DecomposeLclVar(GenTree** ppTree, Compiler::fgWalkData* dat m_compiler->lvaIncRefCnts(loResult); m_compiler->lvaIncRefCnts(hiResult); - FinalizeDecomposition(ppTree, data, loResult, hiResult); + return FinalizeDecomposition(use, loResult, hiResult); } //------------------------------------------------------------------------ // DecomposeLclFld: Decompose GT_LCL_FLD. // // Arguments: -// ppTree - the tree to decompose -// data - tree walk context +// use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: -// None. +// The next node to process. // -void DecomposeLongs::DecomposeLclFld(GenTree** ppTree, Compiler::fgWalkData* data) +GenTree* DecomposeLongs::DecomposeLclFld(LIR::Use& use) { - assert(ppTree != nullptr); - assert(*ppTree != nullptr); - assert(data != nullptr); - assert((*ppTree)->OperGet() == GT_LCL_FLD); + assert(use.IsInitialized()); + assert(use.Def()->OperGet() == GT_LCL_FLD); - GenTree* tree = *ppTree; + GenTree* tree = use.Def(); GenTreeLclFld* loResult = tree->AsLclFld(); loResult->gtType = TYP_INT; GenTree* hiResult = m_compiler->gtNewLclFldNode(loResult->gtLclNum, TYP_INT, loResult->gtLclOffs + 4); + hiResult->CopyCosts(loResult); + BlockRange().InsertAfter(loResult, hiResult); - FinalizeDecomposition(ppTree, data, loResult, hiResult); + return FinalizeDecomposition(use, loResult, hiResult); } //------------------------------------------------------------------------ // DecomposeStoreLclVar: Decompose GT_STORE_LCL_VAR. // // Arguments: -// ppTree - the tree to decompose -// data - tree walk context +// use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: -// None. +// The next node to process. // -void DecomposeLongs::DecomposeStoreLclVar(GenTree** ppTree, Compiler::fgWalkData* data) +GenTree* DecomposeLongs::DecomposeStoreLclVar(LIR::Use& use) { - assert(ppTree != nullptr); - assert(*ppTree != nullptr); - assert(data != nullptr); - assert((*ppTree)->OperGet() == GT_STORE_LCL_VAR); - assert(m_compiler->compCurStmt != nullptr); + assert(use.IsInitialized()); + assert(use.Def()->OperGet() == GT_STORE_LCL_VAR); - GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt(); - - GenTree* tree = *ppTree; - GenTree* nextTree = tree->gtNext; - GenTree* rhs = tree->gtGetOp1(); + GenTree* tree = use.Def(); + GenTree* rhs = tree->gtGetOp1(); if ((rhs->OperGet() == GT_PHI) || (rhs->OperGet() == GT_CALL)) { // GT_CALLs are not decomposed, so will not be converted to GT_LONG // GT_STORE_LCL_VAR = GT_CALL are handled in genMultiRegCallStoreToLocal - return; + return tree->gtNext; } noway_assert(rhs->OperGet() == GT_LONG); @@ -468,62 +426,42 @@ void DecomposeLongs::DecomposeStoreLclVar(GenTree** ppTree, Compiler::fgWalkData hiStore->AsLclFld()->gtFieldSeq = FieldSeqStore::NotAField(); } + // 'tree' is going to steal the loRhs node for itself, so we need to remove the + // GT_LONG node from the threading. + BlockRange().Remove(rhs); + tree->gtOp.gtOp1 = loRhs; tree->gtType = TYP_INT; - loRhs->gtNext = tree; - tree->gtPrev = loRhs; - hiStore->gtOp.gtOp1 = hiRhs; - hiStore->CopyCosts(tree); hiStore->gtFlags |= GTF_VAR_DEF; m_compiler->lvaIncRefCnts(tree); m_compiler->lvaIncRefCnts(hiStore); - tree->gtNext = hiRhs; - hiRhs->gtPrev = tree; - hiRhs->gtNext = hiStore; - hiStore->gtPrev = hiRhs; - hiStore->gtNext = nextTree; - if (nextTree != nullptr) - { - nextTree->gtPrev = hiStore; - } - nextTree = hiRhs; - - bool isEmbeddedStmt = !curStmt->gtStmtIsTopLevel(); - if (!isEmbeddedStmt) - { - tree->gtNext = nullptr; - hiRhs->gtPrev = nullptr; - } + hiStore->CopyCosts(tree); + BlockRange().InsertAfter(tree, hiStore); - InsertNodeAsStmt(hiStore); + return hiStore->gtNext; } //------------------------------------------------------------------------ // DecomposeCast: Decompose GT_CAST. // // Arguments: -// ppTree - the tree to decompose -// data - tree walk context +// use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: -// None. +// The next node to process. // -void DecomposeLongs::DecomposeCast(GenTree** ppTree, Compiler::fgWalkData* data) +GenTree* DecomposeLongs::DecomposeCast(LIR::Use& use) { - assert(ppTree != nullptr); - assert(*ppTree != nullptr); - assert(data != nullptr); - assert((*ppTree)->OperGet() == GT_CAST); - assert(m_compiler->compCurStmt != nullptr); + assert(use.IsInitialized()); + assert(use.Def()->OperGet() == GT_CAST); - GenTree* tree = *ppTree; - GenTree* loResult = nullptr; - GenTree* hiResult = nullptr; - GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt(); + GenTree* tree = use.Def(); + GenTree* loResult = nullptr; + GenTree* hiResult = nullptr; assert(tree->gtPrev == tree->gtGetOp1()); NYI_IF(tree->gtOverflow(), "TYP_LONG cast with overflow"); @@ -533,8 +471,11 @@ void DecomposeLongs::DecomposeCast(GenTree** ppTree, Compiler::fgWalkData* data) if (tree->gtFlags & GTF_UNSIGNED) { loResult = tree->gtGetOp1(); + BlockRange().Remove(tree); + hiResult = new (m_compiler, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0); - m_compiler->fgSnipNode(curStmt, tree); + hiResult->CopyCosts(loResult); + BlockRange().InsertAfter(loResult, hiResult); } else { @@ -547,27 +488,24 @@ void DecomposeLongs::DecomposeCast(GenTree** ppTree, Compiler::fgWalkData* data) break; } - FinalizeDecomposition(ppTree, data, loResult, hiResult); + return FinalizeDecomposition(use, loResult, hiResult); } //------------------------------------------------------------------------ // DecomposeCnsLng: Decompose GT_CNS_LNG. // // Arguments: -// ppTree - the tree to decompose -// data - tree walk context +// use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: -// None. +// The next node to process. // -void DecomposeLongs::DecomposeCnsLng(GenTree** ppTree, Compiler::fgWalkData* data) +GenTree* DecomposeLongs::DecomposeCnsLng(LIR::Use& use) { - assert(ppTree != nullptr); - assert(*ppTree != nullptr); - assert(data != nullptr); - assert((*ppTree)->OperGet() == GT_CNS_LNG); + assert(use.IsInitialized()); + assert(use.Def()->OperGet() == GT_CNS_LNG); - GenTree* tree = *ppTree; + GenTree* tree = use.Def(); INT32 hiVal = tree->AsLngCon()->HiVal(); GenTree* loResult = tree; @@ -575,93 +513,77 @@ void DecomposeLongs::DecomposeCnsLng(GenTree** ppTree, Compiler::fgWalkData* dat loResult->gtType = TYP_INT; GenTree* hiResult = new (m_compiler, GT_CNS_INT) GenTreeIntCon(TYP_INT, hiVal); + hiResult->CopyCosts(loResult); + BlockRange().InsertAfter(loResult, hiResult); - FinalizeDecomposition(ppTree, data, loResult, hiResult); + return FinalizeDecomposition(use, loResult, hiResult); } //------------------------------------------------------------------------ // DecomposeCall: Decompose GT_CALL. // // Arguments: -// ppTree - the tree to decompose -// data - tree walk context +// use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: -// None. +// The next node to process. // -void DecomposeLongs::DecomposeCall(GenTree** ppTree, Compiler::fgWalkData* data) +GenTree* DecomposeLongs::DecomposeCall(LIR::Use& use) { - assert(ppTree != nullptr); - assert(*ppTree != nullptr); - assert(data != nullptr); - assert((*ppTree)->OperGet() == GT_CALL); + assert(use.IsInitialized()); + assert(use.Def()->OperGet() == GT_CALL); - GenTree* parent = data->parent; + // We only need to force var = call() if the call's result is used. + if (use.IsDummyUse()) + return use.Def()->gtNext; - // We only need to force var = call() if the call is not a top-level node. - if (parent == nullptr) - return; - - if (parent->gtOper == GT_STORE_LCL_VAR) + GenTree* user = use.User(); + if (user->OperGet() == GT_STORE_LCL_VAR) { // If parent is already a STORE_LCL_VAR, we can skip it if // it is already marked as lvIsMultiRegRet. - unsigned varNum = parent->AsLclVarCommon()->gtLclNum; + unsigned varNum = user->AsLclVarCommon()->gtLclNum; if (m_compiler->lvaTable[varNum].lvIsMultiRegRet) { - return; + return use.Def()->gtNext; } else if (!m_compiler->lvaTable[varNum].lvPromoted) { // If var wasn't promoted, we can just set lvIsMultiRegRet. m_compiler->lvaTable[varNum].lvIsMultiRegRet = true; - return; + return use.Def()->gtNext; } } - // Otherwise, we need to force var = call() - GenTree* tree = *ppTree; - GenTree** treePtr = nullptr; - parent = tree->gtGetParent(&treePtr); - - assert(treePtr != nullptr); - - GenTreeStmt* asgStmt = m_compiler->fgInsertEmbeddedFormTemp(treePtr); - GenTree* stLclVar = asgStmt->gtStmtExpr; - assert(stLclVar->OperIsLocalStore()); + GenTree* originalNode = use.Def(); - unsigned varNum = stLclVar->AsLclVarCommon()->gtLclNum; + // Otherwise, we need to force var = call() + unsigned varNum = use.ReplaceWithLclVar(m_compiler, m_block->getBBWeight(m_compiler)); m_compiler->lvaTable[varNum].lvIsMultiRegRet = true; - m_compiler->fgFixupIfCallArg(data->parentStack, tree, *treePtr); - // Decompose new node - DecomposeNode(treePtr, data); + // Decompose the new LclVar use + return DecomposeLclVar(use); } //------------------------------------------------------------------------ // DecomposeStoreInd: Decompose GT_STOREIND. // // Arguments: -// tree - the tree to decompose +// use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: -// None. +// The next node to process. // -void DecomposeLongs::DecomposeStoreInd(GenTree** ppTree, Compiler::fgWalkData* data) +// TODO-LIR: replace comments below that use embedded statements with ones that do not. +GenTree* DecomposeLongs::DecomposeStoreInd(LIR::Use& use) { - assert(ppTree != nullptr); - assert(*ppTree != nullptr); - assert(data != nullptr); - assert((*ppTree)->OperGet() == GT_STOREIND); - assert(m_compiler->compCurStmt != nullptr); + assert(use.IsInitialized()); + assert(use.Def()->OperGet() == GT_STOREIND); - GenTree* tree = *ppTree; + GenTree* tree = use.Def(); assert(tree->gtOp.gtOp2->OperGet() == GT_LONG); - GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt(); - bool isEmbeddedStmt = !curStmt->gtStmtIsTopLevel(); - // Example input trees (a nested embedded statement case) // // <linkBegin Node> @@ -689,27 +611,29 @@ void DecomposeLongs::DecomposeStoreInd(GenTree** ppTree, Compiler::fgWalkData* d // // (editor brace matching compensation: }}}}}}}}}}}}}}}}}}) - GenTree* linkBegin = m_compiler->fgGetFirstNode(tree)->gtPrev; - GenTree* linkEnd = tree->gtNext; - GenTree* gtLong = tree->gtOp.gtOp2; + GenTree* gtLong = tree->gtOp.gtOp2; + unsigned blockWeight = m_block->getBBWeight(m_compiler); // Save address to a temp. It is used in storeIndLow and storeIndHigh trees. - GenTreeStmt* addrStmt = CreateTemporary(&tree->gtOp.gtOp1); + LIR::Use address(BlockRange(), &tree->gtOp.gtOp1, tree); + address.ReplaceWithLclVar(m_compiler, blockWeight); JITDUMP("[DecomposeStoreInd]: Saving address tree to a temp var:\n"); - DISPTREE(addrStmt); + DISPTREERANGE(BlockRange(), address.Def()); if (!gtLong->gtOp.gtOp1->OperIsLeaf()) { - GenTreeStmt* dataLowStmt = CreateTemporary(>Long->gtOp.gtOp1); + LIR::Use op1(BlockRange(), >Long->gtOp.gtOp1, gtLong); + op1.ReplaceWithLclVar(m_compiler, blockWeight); JITDUMP("[DecomposeStoreInd]: Saving low data tree to a temp var:\n"); - DISPTREE(dataLowStmt); + DISPTREERANGE(BlockRange(), op1.Def()); } if (!gtLong->gtOp.gtOp2->OperIsLeaf()) { - GenTreeStmt* dataHighStmt = CreateTemporary(>Long->gtOp.gtOp2); + LIR::Use op2(BlockRange(), >Long->gtOp.gtOp2, gtLong); + op2.ReplaceWithLclVar(m_compiler, blockWeight); JITDUMP("[DecomposeStoreInd]: Saving high data tree to a temp var:\n"); - DISPTREE(dataHighStmt); + DISPTREERANGE(BlockRange(), op2.Def()); } // Example trees after embedded statements for address and data are added. @@ -765,8 +689,8 @@ void DecomposeLongs::DecomposeStoreInd(GenTree** ppTree, Compiler::fgWalkData* d // // (editor brace matching compensation: }}}}}}}}}) - m_compiler->fgSnipNode(curStmt, gtLong); - m_compiler->fgSnipNode(curStmt, dataHigh); + BlockRange().Remove(gtLong); + BlockRange().Remove(dataHigh); storeIndLow->gtOp.gtOp2 = dataLow; storeIndLow->gtType = TYP_INT; @@ -787,31 +711,12 @@ void DecomposeLongs::DecomposeStoreInd(GenTree** ppTree, Compiler::fgWalkData* d GenTree* storeIndHigh = new (m_compiler, GT_STOREIND) GenTreeStoreInd(TYP_INT, addrHigh, dataHigh); storeIndHigh->gtFlags = (storeIndLow->gtFlags & (GTF_ALL_EFFECT | GTF_LIVENESS_MASK)); storeIndHigh->gtFlags |= GTF_REVERSE_OPS; - storeIndHigh->CopyCosts(storeIndLow); - - // Internal links of storeIndHigh tree - dataHigh->gtPrev = nullptr; - dataHigh->gtNext = nullptr; - SimpleLinkNodeAfter(dataHigh, addrBaseHigh); - SimpleLinkNodeAfter(addrBaseHigh, addrHigh); - SimpleLinkNodeAfter(addrHigh, storeIndHigh); - - // External links of storeIndHigh tree - // dataHigh->gtPrev = nullptr; - if (isEmbeddedStmt) - { - // If storeIndTree is an embedded statement, connect storeIndLow - // and dataHigh - storeIndLow->gtNext = dataHigh; - dataHigh->gtPrev = storeIndLow; - } - storeIndHigh->gtNext = linkEnd; - if (linkEnd != nullptr) - { - linkEnd->gtPrev = storeIndHigh; - } - InsertNodeAsStmt(storeIndHigh); + m_compiler->gtPrepareCost(storeIndHigh); + + BlockRange().InsertAfter(storeIndLow, dataHigh, addrBaseHigh, addrHigh, storeIndHigh); + + return storeIndHigh; // Example final output // @@ -860,17 +765,19 @@ void DecomposeLongs::DecomposeStoreInd(GenTree** ppTree, Compiler::fgWalkData* d // DecomposeInd: Decompose GT_IND. // // Arguments: -// tree - the tree to decompose +// use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: -// None. +// The next node to process. // -void DecomposeLongs::DecomposeInd(GenTree** ppTree, Compiler::fgWalkData* data) +GenTree* DecomposeLongs::DecomposeInd(LIR::Use& use) { - GenTreePtr indLow = *ppTree; - GenTreeStmt* addrStmt = CreateTemporary(&indLow->gtOp.gtOp1); + GenTree* indLow = use.Def(); + + LIR::Use address(BlockRange(), &indLow->gtOp.gtOp1, indLow); + address.ReplaceWithLclVar(m_compiler, m_block->getBBWeight(m_compiler)); JITDUMP("[DecomposeInd]: Saving addr tree to a temp var:\n"); - DISPTREE(addrStmt); + DISPTREERANGE(BlockRange(), address.Def()); // Change the type of lower ind. indLow->gtType = TYP_INT; @@ -883,83 +790,78 @@ void DecomposeLongs::DecomposeInd(GenTree** ppTree, Compiler::fgWalkData* data) new (m_compiler, GT_LEA) GenTreeAddrMode(TYP_REF, addrBaseHigh, nullptr, 0, genTypeSize(TYP_INT)); GenTreePtr indHigh = new (m_compiler, GT_IND) GenTreeIndir(GT_IND, TYP_INT, addrHigh, nullptr); - // Connect linear links - SimpleLinkNodeAfter(addrBaseHigh, addrHigh); - SimpleLinkNodeAfter(addrHigh, indHigh); + m_compiler->gtPrepareCost(indHigh); + + BlockRange().InsertAfter(indLow, addrBaseHigh, addrHigh, indHigh); - FinalizeDecomposition(ppTree, data, indLow, indHigh); + return FinalizeDecomposition(use, indLow, indHigh); } //------------------------------------------------------------------------ // DecomposeNot: Decompose GT_NOT. // // Arguments: -// ppTree - the tree to decompose -// data - tree walk context +// use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: -// None. +// The next node to process. // -void DecomposeLongs::DecomposeNot(GenTree** ppTree, Compiler::fgWalkData* data) +GenTree* DecomposeLongs::DecomposeNot(LIR::Use& use) { - assert(ppTree != nullptr); - assert(*ppTree != nullptr); - assert(data != nullptr); - assert((*ppTree)->OperGet() == GT_NOT); - assert(m_compiler->compCurStmt != nullptr); + assert(use.IsInitialized()); + assert(use.Def()->OperGet() == GT_NOT); - GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt(); + GenTree* tree = use.Def(); + GenTree* gtLong = tree->gtGetOp1(); + noway_assert(gtLong->OperGet() == GT_LONG); + GenTree* loOp1 = gtLong->gtGetOp1(); + GenTree* hiOp1 = gtLong->gtGetOp2(); - GenTree* tree = *ppTree; - GenTree* op1 = tree->gtGetOp1(); - noway_assert(op1->OperGet() == GT_LONG); - GenTree* loOp1 = op1->gtGetOp1(); - GenTree* hiOp1 = op1->gtGetOp2(); - m_compiler->fgSnipNode(curStmt, op1); + BlockRange().Remove(gtLong); GenTree* loResult = tree; loResult->gtType = TYP_INT; loResult->gtOp.gtOp1 = loOp1; - loOp1->gtNext = loResult; - loResult->gtPrev = loOp1; GenTree* hiResult = new (m_compiler, GT_NOT) GenTreeOp(GT_NOT, TYP_INT, hiOp1, nullptr); - hiOp1->gtNext = hiResult; - hiResult->gtPrev = hiOp1; + hiResult->CopyCosts(loResult); + BlockRange().InsertAfter(loResult, hiResult); - FinalizeDecomposition(ppTree, data, loResult, hiResult); + return FinalizeDecomposition(use, loResult, hiResult); } //------------------------------------------------------------------------ // DecomposeNeg: Decompose GT_NEG. // // Arguments: -// ppTree - the tree to decompose -// data - tree walk context +// use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: -// None. +// The next node to process. // -void DecomposeLongs::DecomposeNeg(GenTree** ppTree, Compiler::fgWalkData* data) +GenTree* DecomposeLongs::DecomposeNeg(LIR::Use& use) { - assert(ppTree != nullptr); - assert(*ppTree != nullptr); - assert(data != nullptr); - assert((*ppTree)->OperGet() == GT_NEG); - assert(m_compiler->compCurStmt != nullptr); - - GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt(); - GenTree* tree = *ppTree; - GenTree* op1 = tree->gtGetOp1(); - noway_assert(op1->OperGet() == GT_LONG); - - CreateTemporary(&(op1->gtOp.gtOp1)); - CreateTemporary(&(op1->gtOp.gtOp2)); + assert(use.IsInitialized()); + assert(use.Def()->OperGet() == GT_NEG); + + GenTree* tree = use.Def(); + GenTree* gtLong = tree->gtGetOp1(); + noway_assert(gtLong->OperGet() == GT_LONG); + + unsigned blockWeight = m_block->getBBWeight(m_compiler); + + LIR::Use op1(BlockRange(), >Long->gtOp.gtOp1, gtLong); + op1.ReplaceWithLclVar(m_compiler, blockWeight); + + LIR::Use op2(BlockRange(), >Long->gtOp.gtOp2, gtLong); + op2.ReplaceWithLclVar(m_compiler, blockWeight); + // Neither GT_NEG nor the introduced temporaries have side effects. tree->gtFlags &= ~GTF_ALL_EFFECT; - GenTree* loOp1 = op1->gtGetOp1(); - GenTree* hiOp1 = op1->gtGetOp2(); - Compiler::fgSnipNode(curStmt, op1); + GenTree* loOp1 = gtLong->gtGetOp1(); + GenTree* hiOp1 = gtLong->gtGetOp2(); + + BlockRange().Remove(gtLong); GenTree* loResult = tree; loResult->gtType = TYP_INT; @@ -970,42 +872,32 @@ void DecomposeLongs::DecomposeNeg(GenTree** ppTree, Compiler::fgWalkData* data) GenTree* hiResult = m_compiler->gtNewOperNode(GT_NEG, TYP_INT, hiAdjust); hiResult->gtFlags = tree->gtFlags; - Compiler::fgSnipNode(curStmt, hiOp1); - // fgSnipNode doesn't clear gtNext/gtPrev... - hiOp1->gtNext = nullptr; - hiOp1->gtPrev = nullptr; - SimpleLinkNodeAfter(hiOp1, zero); - SimpleLinkNodeAfter(zero, hiAdjust); - SimpleLinkNodeAfter(hiAdjust, hiResult); + // Annotate new nodes with costs. This will re-cost the hiOp1 tree as well. + m_compiler->gtPrepareCost(hiResult); - FinalizeDecomposition(ppTree, data, loResult, hiResult); + BlockRange().InsertAfter(loResult, zero, hiAdjust, hiResult); + + return FinalizeDecomposition(use, loResult, hiResult); } //------------------------------------------------------------------------ // DecomposeArith: Decompose GT_ADD, GT_SUB, GT_OR, GT_XOR, GT_AND. // // Arguments: -// ppTree - the tree to decompose -// data - tree walk context +// use - the LIR::Use object for the def that needs to be decomposed. // // Return Value: -// None. +// The next node to process. // -void DecomposeLongs::DecomposeArith(GenTree** ppTree, Compiler::fgWalkData* data) +GenTree* DecomposeLongs::DecomposeArith(LIR::Use& use) { - assert(ppTree != nullptr); - assert(*ppTree != nullptr); - assert(data != nullptr); - assert(m_compiler->compCurStmt != nullptr); + assert(use.IsInitialized()); - GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt(); - GenTree* tree = *ppTree; - genTreeOps oper = tree->OperGet(); + GenTree* tree = use.Def(); + genTreeOps oper = tree->OperGet(); assert((oper == GT_ADD) || (oper == GT_SUB) || (oper == GT_OR) || (oper == GT_XOR) || (oper == GT_AND)); - NYI_IF((tree->gtFlags & GTF_REVERSE_OPS) != 0, "Binary operator with GTF_REVERSE_OPS"); - GenTree* op1 = tree->gtGetOp1(); GenTree* op2 = tree->gtGetOp2(); @@ -1036,8 +928,8 @@ void DecomposeLongs::DecomposeArith(GenTree** ppTree, Compiler::fgWalkData* data "Can't decompose expression tree TYP_LONG node"); // Now, remove op1 and op2 from the node list. - m_compiler->fgSnipNode(curStmt, op1); - m_compiler->fgSnipNode(curStmt, op2); + BlockRange().Remove(op1); + BlockRange().Remove(op2); // We will reuse "tree" for the loResult, which will now be of TYP_INT, and its operands // will be the lo halves of op1 from above. @@ -1047,35 +939,9 @@ void DecomposeLongs::DecomposeArith(GenTree** ppTree, Compiler::fgWalkData* data loResult->gtOp.gtOp1 = loOp1; loResult->gtOp.gtOp2 = loOp2; - // The various halves will be correctly threaded internally. We simply need to - // relink them into the proper order, i.e. loOp1 is followed by loOp2, and then - // the loResult node. - // (This rethreading, and that below, are where we need to address the reverse ops case). - // The current order is (after snipping op1 and op2): - // ... loOp1-> ... hiOp1->loOp2First ... loOp2->hiOp2First ... hiOp2 - // The order we want is: - // ... loOp1->loOp2First ... loOp2->loResult - // ... hiOp1->hiOp2First ... hiOp2->hiResult - // i.e. we swap hiOp1 and loOp2, and create (for now) separate loResult and hiResult trees - GenTree* loOp2First = hiOp1->gtNext; - GenTree* hiOp2First = loOp2->gtNext; - - // First, we will NYI if both hiOp1 and loOp2 have side effects. - NYI_IF(((loOp2->gtFlags & GTF_ALL_EFFECT) != 0) && ((hiOp1->gtFlags & GTF_ALL_EFFECT) != 0), - "Binary long operator with non-reorderable sub expressions"); - - // Now, we reorder the loOps and the loResult. - loOp1->gtNext = loOp2First; - loOp2First->gtPrev = loOp1; - loOp2->gtNext = loResult; - loResult->gtPrev = loOp2; - - // Next, reorder the hiOps and the hiResult. - GenTree* hiResult = new (m_compiler, oper) GenTreeOp(GetHiOper(oper), TYP_INT, hiOp1, hiOp2); - hiOp1->gtNext = hiOp2First; - hiOp2First->gtPrev = hiOp1; - hiOp2->gtNext = hiResult; - hiResult->gtPrev = hiOp2; + GenTree* hiResult = new (m_compiler, oper) GenTreeOp(GetHiOper(oper), TYP_INT, hiOp1, hiOp2); + hiResult->CopyCosts(loResult); + BlockRange().InsertAfter(loResult, hiResult); if ((oper == GT_ADD) || (oper == GT_SUB)) { @@ -1090,92 +956,7 @@ void DecomposeLongs::DecomposeArith(GenTree** ppTree, Compiler::fgWalkData* data } } - FinalizeDecomposition(ppTree, data, loResult, hiResult); -} - -//------------------------------------------------------------------------ -// CreateTemporary: call fgInsertEmbeddedFormTemp to replace *ppTree with -// a new temp that is assigned to the value previously at *ppTree by inserting -// an embedded statement. In addition, if the resulting statement actually ends -// up being top-level, it might pull along some embedded statements that have -// not yet been decomposed. So recursively decompose those before returning. -// -// Arguments: -// *ppTree - tree to replace with a temp. -// -// Return Value: -// The new statement that was created to create the temp. -// -GenTreeStmt* DecomposeLongs::CreateTemporary(GenTree** ppTree) -{ - GenTreeStmt* newStmt = m_compiler->fgInsertEmbeddedFormTemp(ppTree); - if (newStmt->gtStmtIsTopLevel()) - { - for (GenTreeStmt* nextEmbeddedStmt = newStmt->gtStmtNextIfEmbedded(); nextEmbeddedStmt != nullptr; - nextEmbeddedStmt = nextEmbeddedStmt->gtStmt.gtStmtNextIfEmbedded()) - { - DecomposeStmt(nextEmbeddedStmt); - } - } - - return newStmt; -} - -//------------------------------------------------------------------------ -// InsertNodeAsStmt: Insert a node as the root node of a new statement. -// If the current statement is embedded, the new statement will also be -// embedded. Otherwise, the new statement will be top level. -// -// Arguments: -// node - node to insert -// -// Return Value: -// None -// -// Notes: -// compCurStmt and compCurBB must be correctly set. -// -void DecomposeLongs::InsertNodeAsStmt(GenTree* node) -{ - assert(node != nullptr); - assert(m_compiler->compCurBB != nullptr); - assert(m_compiler->compCurStmt != nullptr); - - GenTreeStmt* curStmt = m_compiler->compCurStmt->AsStmt(); - GenTreeStmt* newStmt; - - if (curStmt->gtStmtIsTopLevel()) - { - newStmt = m_compiler->fgNewStmtFromTree(node); - - // Find an insert point. Skip all embedded statements. - GenTree* insertPt = curStmt; - while ((insertPt->gtNext != nullptr) && (!insertPt->gtNext->AsStmt()->gtStmtIsTopLevel())) - { - insertPt = insertPt->gtNext; - } - - m_compiler->fgInsertStmtAfter(m_compiler->compCurBB, insertPt, newStmt); - } - else - { - // The current statement is an embedded statement. Create a new embedded statement to - // contain the node. First, find the parent non-embedded statement containing the - // current statement. - GenTree* parentStmt = curStmt; - while ((parentStmt != nullptr) && (!parentStmt->AsStmt()->gtStmtIsTopLevel())) - { - parentStmt = parentStmt->gtPrev; - } - assert(parentStmt != nullptr); - - newStmt = m_compiler->fgMakeEmbeddedStmt(m_compiler->compCurBB, node, parentStmt); - } - - newStmt->gtStmtILoffsx = curStmt->gtStmtILoffsx; -#ifdef DEBUG - newStmt->gtStmtLastILoffs = curStmt->gtStmtLastILoffs; -#endif // DEBUG + return FinalizeDecomposition(use, loResult, hiResult); } //------------------------------------------------------------------------ @@ -1257,36 +1038,5 @@ genTreeOps DecomposeLongs::GetLoOper(genTreeOps oper) } } -//------------------------------------------------------------------------ -// SimpleLinkNodeAfter: insert a node after a given node in the execution order. -// NOTE: Does not support inserting after the last node of a statement, which -// would require updating the statement links. -// -// Arguments: -// insertionPoint - Insert after this tree node. -// node - The node to insert. -// -// Return Value: -// None -// -// Notes: -// Seems like this should be moved to someplace that houses all the flowgraph -// manipulation functions. -// -void DecomposeLongs::SimpleLinkNodeAfter(GenTree* insertionPoint, GenTree* node) -{ - assert(insertionPoint != nullptr); - assert(node != nullptr); - - GenTree* nextTree = insertionPoint->gtNext; - node->gtPrev = insertionPoint; - node->gtNext = nextTree; - insertionPoint->gtNext = node; - if (nextTree != nullptr) - { - nextTree->gtPrev = node; - } -} - #endif // !_TARGET_64BIT_ #endif // !LEGACY_BACKEND diff --git a/src/jit/decomposelongs.h b/src/jit/decomposelongs.h index fc02950f49..523a06a67a 100644 --- a/src/jit/decomposelongs.h +++ b/src/jit/decomposelongs.h @@ -27,34 +27,36 @@ public: void DecomposeBlock(BasicBlock* block); private: + inline LIR::Range& BlockRange() const + { + return LIR::AsRange(m_block); + } + // Driver functions - static Compiler::fgWalkResult DecompNodeHelper(GenTree** ppTree, Compiler::fgWalkData* data); - void DecomposeStmt(GenTreeStmt* stmt); - void DecomposeNode(GenTree** ppTree, Compiler::fgWalkData* data); + GenTree* DecomposeNode(LIR::Use& use); // Per-node type decompose cases - void DecomposeLclVar(GenTree** ppTree, Compiler::fgWalkData* data); - void DecomposeLclFld(GenTree** ppTree, Compiler::fgWalkData* data); - void DecomposeStoreLclVar(GenTree** ppTree, Compiler::fgWalkData* data); - void DecomposeCast(GenTree** ppTree, Compiler::fgWalkData* data); - void DecomposeCnsLng(GenTree** ppTree, Compiler::fgWalkData* data); - void DecomposeCall(GenTree** ppTree, Compiler::fgWalkData* data); - void DecomposeInd(GenTree** ppTree, Compiler::fgWalkData* data); - void DecomposeStoreInd(GenTree** ppTree, Compiler::fgWalkData* data); - void DecomposeNot(GenTree** ppTree, Compiler::fgWalkData* data); - void DecomposeNeg(GenTree** ppTree, Compiler::fgWalkData* data); - void DecomposeArith(GenTree** ppTree, Compiler::fgWalkData* data); + GenTree* DecomposeLclVar(LIR::Use& use); + GenTree* DecomposeLclFld(LIR::Use& use); + GenTree* DecomposeStoreLclVar(LIR::Use& use); + GenTree* DecomposeCast(LIR::Use& use); + GenTree* DecomposeCnsLng(LIR::Use& use); + GenTree* DecomposeCall(LIR::Use& use); + GenTree* DecomposeInd(LIR::Use& use); + GenTree* DecomposeStoreInd(LIR::Use& use); + GenTree* DecomposeNot(LIR::Use& use); + GenTree* DecomposeNeg(LIR::Use& use); + GenTree* DecomposeArith(LIR::Use& use); // Helper functions - void FinalizeDecomposition(GenTree** ppTree, Compiler::fgWalkData* data, GenTree* loResult, GenTree* hiResult); - void InsertNodeAsStmt(GenTree* node); - GenTreeStmt* CreateTemporary(GenTree** ppTree); + GenTree* FinalizeDecomposition(LIR::Use& use, GenTree* loResult, GenTree* hiResult); + static genTreeOps GetHiOper(genTreeOps oper); static genTreeOps GetLoOper(genTreeOps oper); - void SimpleLinkNodeAfter(GenTree* insertionPoint, GenTree* node); // Data - Compiler* m_compiler; + Compiler* m_compiler; + BasicBlock* m_block; }; #endif // _DECOMPOSELONGS_H_ diff --git a/src/jit/flowgraph.cpp b/src/jit/flowgraph.cpp index 4659f47dc7..9af603b714 100644 --- a/src/jit/flowgraph.cpp +++ b/src/jit/flowgraph.cpp @@ -626,7 +626,7 @@ GenTreeStmt* Compiler::fgInsertStmtNearEnd(BasicBlock* block, GenTreePtr node) { GenTreeStmt* stmt; - // This routine is not aware of embedded stmts and can only be used when in tree order. + // This routine can only be used when in tree order. assert(fgOrder == FGOrderTree); if ((block->bbJumpKind == BBJ_COND) || (block->bbJumpKind == BBJ_SWITCH) || (block->bbJumpKind == BBJ_RETURN)) @@ -844,133 +844,6 @@ void Compiler::fgRemoveReturnBlock(BasicBlock* block) } //------------------------------------------------------------------------ -// fgReplaceStmt: Replaces the top-level tree of 'stmt' with newTree -// -// Arguments: -// stmt - the statement whose tree we're replacing -// newTree - the new top-level tree for 'stmt' -// -// Return Value: -// None. -// -// Operation: -// This method has two main modes of operation: -// a) In case we're in Tree Order or we're replacing a top-level statement -// we first append the replacing statement ahead of the statement to replace -// and then remove the latter from the CFG. -// b) If we're replacing an embedded statement (and this naturally assumes we're -// in linear order), we proceed to do that in-place, i.e. replace the expression -// inside the statement to replace with the expression contained in the -// replacing node. -// -// Assumptions: -// This method will "fixup" any embedded statements from the old tree -// to the new. However, this will only work if the node which follows the -// embedded statement is preserved. This will be true if the newTree -// reuses the constituent nodes of the old tree (e.g. in the case where a -// node is replaced by a helper call with the original arguments to the node, -// but will not be true for arbitrary tree replacement.) -// -// Notes: -// This is currently only used in FGOrderLinear. -// TODO-Cleanup: This should probably simply replace the tree so that the information -// (such as IL offsets) is preserved, but currently it creates a new statement. - -void Compiler::fgReplaceStmt(BasicBlock* block, GenTreeStmt* stmt, GenTreePtr newTree) -{ - // fgNewStmtFromTree will sequence the nodes in newTree. Thus, if we are in FGOrderLinear, - // we will need to fixup any embedded statements after this call. - GenTreeStmt* newStmt = fgNewStmtFromTree(newTree, block); - - if (stmt->gtStmtIsTopLevel() || fgOrder == FGOrderTree) - { - assert(stmt->gtStmtIsTopLevel()); - fgInsertStmtAfter(block, stmt, newStmt); - - // Remove the old statement now we've inserted the new one. - fgRemoveStmt(block, stmt, false); - - if (fgOrder == FGOrderLinear) - { - // Because we are now in linear mode, we may have an embedded statement in the execution - // stream. It is too complex to try to sequence the new tree in an ad-hoc fashion, - // but we can't use the normal sequencing without bypassing the embedded statements. - // So, we fix them up now that we're done with the new tree. - // We preserve the order of the embedded statement relative to its gtNext. - // This is because the new tree may have a different order for its args than the - // block node did, and statements become embedded because they need to be ordered - // BEFORE something (not after). - // TODO-Cleanup: Consider finding an alternate approach to this - it seems risky - - for (GenTreeStmt* embeddedStmt = newStmt->gtNextStmt; - embeddedStmt != nullptr && embeddedStmt->gtStmtIsEmbedded(); embeddedStmt = embeddedStmt->gtNextStmt) - { - GenTreePtr firstEmbeddedNode = embeddedStmt->gtStmtList; - GenTreePtr lastEmbeddedNode = embeddedStmt->gtStmtExpr; - GenTreePtr nextNode = lastEmbeddedNode->gtNext; - GenTreePtr prevNode = nextNode->gtPrev; - assert(nextNode != nullptr); - if (prevNode == nullptr) - { - // We've reordered the nodes such that the embedded statement is now first. - // Extract it. - firstEmbeddedNode->gtPrev = nullptr; - lastEmbeddedNode->gtNext = nullptr; - fgRemoveStmt(block, embeddedStmt); - fgInsertStmtBefore(block, stmt, embeddedStmt); - embeddedStmt->gtFlags |= GTF_STMT_TOP_LEVEL; - } - else - { - prevNode->gtNext = firstEmbeddedNode; - firstEmbeddedNode->gtPrev = prevNode; - nextNode->gtPrev = lastEmbeddedNode; - lastEmbeddedNode->gtNext = nextNode; - } - } - } - } - else - { - assert(fgOrder == FGOrderLinear); - - GenTreePtr stmtExpr = stmt->gtStmtExpr; - GenTreePtr stmtList = stmt->gtStmtList; - - // First, proceed to wire the first node in - // execution order - if (stmtList->gtPrev != nullptr) - { - stmtList->gtPrev->gtNext = newStmt->gtStmtList; - } - newStmt->gtStmtList->gtPrev = stmtList->gtPrev; - - // Now, in order to wire the last execution order node - // in a statement, in case it's embedded, we have a special case - // since it *cannot* be null, its gtNext is connected to the - // 'resuming' next node in the containing statement. - // For this, we have to search for the last node in the - // newly created statement and wire it in accordingly to the - // rule just mentioned. - - assert(newStmt->gtStmtExpr->gtNext == nullptr); - - if (stmtExpr->gtNext != nullptr) - { - stmtExpr->gtNext->gtPrev = newStmt->gtStmtExpr; - } - newStmt->gtStmtExpr->gtNext = stmtExpr->gtNext; - - stmt->gtStmtExpr = newStmt->gtStmtExpr; - stmt->gtStmtList = newStmt->gtStmtList; - -#ifdef DEBUG - fgDebugCheckNodeLinks(compCurBB, stmt); -#endif // DEBUG - } -} - -//------------------------------------------------------------------------ // fgGetPredForBlock: Find and return the predecessor edge corresponding to a given predecessor block. // // Arguments: @@ -8707,6 +8580,10 @@ GenTreeStmt* Compiler::fgNewStmtFromTree(GenTreePtr tree, IL_OFFSETX offs) IL_OFFSET Compiler::fgFindBlockILOffset(BasicBlock* block) { + // This function searches for IL offsets in statement nodes, so it can't be used in LIR. We + // could have a similar function for LIR that searches for GT_IL_OFFSET nodes. + assert(!block->IsLIR()); + #if defined(DEBUGGING_SUPPORT) || defined(DEBUG) for (GenTree* stmt = block->bbTreeList; stmt != nullptr; stmt = stmt->gtNext) { @@ -8812,6 +8689,8 @@ BasicBlock* Compiler::fgSplitBlockAtEnd(BasicBlock* curr) //------------------------------------------------------------------------------ BasicBlock* Compiler::fgSplitBlockAfterStatement(BasicBlock* curr, GenTree* stmt) { + assert(!curr->IsLIR()); // No statements in LIR, so you can't use this function. + BasicBlock* newBlock = fgSplitBlockAtEnd(curr); if (stmt) @@ -8846,6 +8725,67 @@ BasicBlock* Compiler::fgSplitBlockAfterStatement(BasicBlock* curr, GenTree* stmt } //------------------------------------------------------------------------------ +// fgSplitBlockAfterNode - Split the given block, with all code after +// the given node going into the second block. +// This function is only used in LIR. +//------------------------------------------------------------------------------ +BasicBlock* Compiler::fgSplitBlockAfterNode(BasicBlock* curr, GenTree* node) +{ + assert(curr->IsLIR()); + + BasicBlock* newBlock = fgSplitBlockAtEnd(curr); + + if (node != nullptr) + { + LIR::Range& currBBRange = LIR::AsRange(curr); + + if (node != currBBRange.LastNode()) + { + LIR::Range nodesToMove = currBBRange.Remove(node->gtNext, currBBRange.LastNode()); + LIR::AsRange(newBlock).InsertAtBeginning(std::move(nodesToMove)); + } + + // Update the IL offsets of the blocks to match the split. + + assert(newBlock->bbCodeOffs == BAD_IL_OFFSET); + assert(newBlock->bbCodeOffsEnd == BAD_IL_OFFSET); + + // curr->bbCodeOffs remains the same + newBlock->bbCodeOffsEnd = curr->bbCodeOffsEnd; + + // Search backwards from the end of the current block looking for the IL offset to use + // for the end IL offset for the original block. + IL_OFFSET splitPointILOffset = BAD_IL_OFFSET; + LIR::Range::ReverseIterator riter; + LIR::Range::ReverseIterator riterEnd; + for (riter = currBBRange.rbegin(), riterEnd = currBBRange.rend(); riter != riterEnd; ++riter) + { + if ((*riter)->gtOper == GT_IL_OFFSET) + { + GenTreeStmt* stmt = (*riter)->AsStmt(); + if (stmt->gtStmtILoffsx != BAD_IL_OFFSET) + { + splitPointILOffset = jitGetILoffs(stmt->gtStmtILoffsx); + break; + } + } + } + + curr->bbCodeOffsEnd = splitPointILOffset; + + // Also use this as the beginning offset of the next block. Presumably we could/should + // look to see if the first node is a GT_IL_OFFSET node, and use that instead. + newBlock->bbCodeOffs = splitPointILOffset; + } + else + { + assert(curr->bbTreeList == nullptr); // if no node was given then it better be an empty block + } + + return newBlock; +} + +//------------------------------------------------------------------------------ // fgSplitBlockAtBeginning - Split the given block into two blocks. // Control falls through from original to new block, // and the new block is returned. @@ -8988,10 +8928,17 @@ void Compiler::fgSimpleLowering() // Walk the statement trees in this basic block, converting ArrLength nodes. compCurBB = block; // Used in fgRngChkTarget. +#ifdef LEGACY_BACKEND for (GenTreeStmt* stmt = block->FirstNonPhiDef(); stmt; stmt = stmt->gtNextStmt) { for (GenTreePtr tree = stmt->gtStmtList; tree; tree = tree->gtNext) { +#else + LIR::Range& range = LIR::AsRange(block); + for (GenTree* tree : range) + { + { +#endif if (tree->gtOper == GT_ARR_LENGTH) { GenTreeArrLen* arrLen = tree->AsArrLen(); @@ -9024,6 +8971,7 @@ void Compiler::fgSimpleLowering() add->gtRsvdRegs = arr->gtRsvdRegs; add->gtCopyFPlvl(arr); add->CopyCosts(arr); +#ifdef LEGACY_BACKEND arr->gtNext = con; con->gtPrev = arr; @@ -9032,6 +8980,9 @@ void Compiler::fgSimpleLowering() add->gtNext = tree; tree->gtPrev = add; +#else + range.InsertAfter(arr, con, add); +#endif } // Change to a GT_IND. @@ -9261,207 +9212,6 @@ void Compiler::fgRemoveEmptyBlocks() } /***************************************************************************** - * fgRemoveLinearOrderDependencies -- - * - * Remove stmt dependencies before removing the stmt itself. - * - * If called on a top level statement, - * - * All the first level (in a breadth-first order) embedded statements now become top - * level statements. In a comma world, it is analogous to retaining the exprs - * within the commas of a statement. - * - * If called on an embedded statement, - * - * Then the statement rooting the embedded statement's next links are - * correctly updated to point to any nested embedded statement nodes or - * the sibling embedded nodes and their prev links are updated to the - * rooting statement. Also the nodes of the embedded statement on which - * we are called are dropped from the list. - * - * Assumptions: - * "stmt" should be detached from the bbTreeList after the call. - * - */ -void Compiler::fgRemoveLinearOrderDependencies(GenTreePtr tree) -{ - assert(fgOrder == FGOrderLinear); - GenTreeStmt* stmt = tree->AsStmt(); - - // No embedded statements. - if (stmt->gtStmtIsTopLevel() && (stmt->gtNext == nullptr || stmt->gtNextStmt->gtStmtIsTopLevel())) - { - return; - } - // stmt is last embedded statement, assume we have a tree order: prevStmt->stmt->nextStmt. - // We are dropping "stmt". So fix the next link for "prevStmt" and prev link for "nextStmt". - if (stmt->gtStmtIsEmbedded() && (stmt->gtNext == nullptr || stmt->gtNextStmt->gtStmtIsTopLevel())) - { - if (stmt->gtStmtList->gtPrev) - { - stmt->gtStmtList->gtPrev->gtNext = stmt->gtStmtExpr->gtNext; - } - if (stmt->gtStmtExpr->gtNext) - { - stmt->gtStmtExpr->gtNext->gtPrev = stmt->gtStmtList->gtPrev; - } - return; - } - - // - // Walk the tree list, and define current statement as - // the immediate statement (embedded or top) in which - // the tree resides. - // - // Initially, next = stmt, cur is empty. - // - // While walking the tree list, we expect to see: - // 1. next stmt's list - // 2. or current stmt's expr - // - // If current stmt's expr is seen, pop to previous - // next and call it current. - // If next stmt's list is seen, then next becomes current. - // - - ArrayStack<GenTreePtr> stack(this); - - // Consider this example: - // - // In stmt order: - // (top (stmt (emb2, emb3) ) ) where "top" embeds "stmt" and "stmt" - // nests "emb2" and "emb3". Now we are removing "stmt." - // - // In the end we should obtain: - // (top (emb2, emb3) ). Callers should fix bbTreeList. We only fix tree order. - // - // So in tree order: - // BEFORE: top:t1 -> stmt:t1 -> emb2:t1 -> stmt:t2 -> emb3:t1 -> stmt:t3 -> top:t2 - // AFTER : top:t1 -> emb2:t1 -> emb3:t1 -> top:t2 - // - GenTreePtr lastNestEmbedNode = stmt->gtStmtList->gtPrev; // In the example, top:t1. - - GenTreePtr next = stmt; - GenTreePtr node = stmt->gtStmtList; - while (node != stmt->gtStmtExpr->gtNext) - { - // We are encountering a new stmt. Push it into the stack. It is now current. - if (next != nullptr && node == next->gtStmt.gtStmtList) - { - stack.Push(next); - next = next->gtNext; - - // Since stack height is 2, we are entering the next level embedded statement - // from stmt's level which is 1. Reminder: stmt is being removed. - // - // If stmt is top level, all level 2 stmts will become top level. - // So don't fix their prev next links. - if (stmt->gtStmtIsEmbedded() && stack.Height() == 2) - { - // clang-format off - // Two cases: - // Case 1 (Initial case -- we are discovering the first embedded stmt): - // Before: - // topList -> stmtList -> emb2List -> emb2Expr -> ... -> stmtExpr -> topExpr - // Currently: "node" is emb2List and "lastNestEmbedNode" is topList. We started the iteration from stmtList. - // After: - // topList -> emb2List -> emb2Expr -> ... -> stmtExpr -> topExpr. - // - // Case 2 (We already discovered an embedded stmt): - // Before: - // ... -> emb2List -> emb2Expr -> stmtNode -> stmtNode -> emb3List -> emb3Expr -> stmtNode -> ... -> stmtExpr - // Currently, "node" is emb3List and "lastNestEmbedNode" is emb2Expr. - // After: - // ... -> emb2List -> emb2Expr -> -> emb3List -> emb3Expr -> stmtNode -> ... -> stmtExpr - // clang-format on - - // Drop stmtNodes that occur between emb2Expr and emb3List. - if (lastNestEmbedNode) - { - lastNestEmbedNode->gtNext = node; - } - node->gtPrev = lastNestEmbedNode; - } - } - GenTreePtr cur = stack.Top(); - if (node == cur->gtStmt.gtStmtExpr) - { - // A stmt goes out of being current. - stack.Pop(); - - // Keep track of the last nested embedded stmt node. In the example, record emb2Expr or emb3Expr. - if (stack.Height() == 1) - { - lastNestEmbedNode = node; - } - - // Are we called on a top level statement? - if (stmt->gtStmtIsTopLevel() && stack.Height() == 1) - { - // We are just done visiting the last node of a first level embedded stmt. - - // Before: - // stmtList -> emb2List -> emb2Expr -> stmtNode -> stmtNode -> emb3List -> emb3Expr -> stmtNode -> ... - // -> stmtExpr - // "stmt" is top level. - // - // Currently, "node" is emb2Expr and "lastNestEmbedNode" is "don't care". - // - // After: - // node = stmtNode -> stmtNode -> emb3List -> emb3Expr -> stmtNode -> ... -> - // stmtExpr - // nullptr <- emb2List -> emb2Expr -> nullptr - // - // stmtList -> emb2List -> emb2Expr -> ... - // This is inconsistent for stmt, as there is no first level embedded statement now, but since callers - // are supposed to remove stmt, we don't care. - // - - // Advance node to next, so we don't set node->next to nullptr below. - node = node->gtNext; - - noway_assert(cur->gtStmt.gtStmtIsEmbedded()); - - // This embedded stmt is now top level since the original top level stmt - // is going to be removed. - cur->gtFlags |= GTF_STMT_TOP_LEVEL; - - cur->gtStmt.gtStmtList->gtPrev = nullptr; - cur->gtStmt.gtStmtExpr->gtNext = nullptr; - - // Don't bother updating stmt's pointers, as we are removing it. - continue; - } - } - node = node->gtNext; - } - - // Are we called on an embedded stmt? - if (stmt->gtStmtIsEmbedded()) - { - // - // Before: - // ... -> emb2List -> emb2Expr -> stmtNode -> stmtNode -> emb3List -> emb3Expr -> stmtNode -> ... -> stmtExpr -> - // topNode - // - // Currently, "node" is topNode (i.e., stmtExpr->gtNext) and "lastNestEmbedNode" is emb3Expr. - // - // After: - // ... -> emb2List -> emb2Expr -> -> emb3List -> emb3Expr -> -> - // topNode - // - if (node) - { - node->gtPrev = lastNestEmbedNode; - } - if (lastNestEmbedNode) - { - lastNestEmbedNode->gtNext = node; - } - } -} - -/***************************************************************************** * * Remove a useless statement from a basic block. * The default is to decrement ref counts of included vars @@ -9474,6 +9224,7 @@ void Compiler::fgRemoveStmt(BasicBlock* block, bool updateRefCount) { noway_assert(node); + assert(fgOrder == FGOrderTree); GenTreeStmt* tree = block->firstStmt(); GenTreeStmt* stmt = node->AsStmt(); @@ -9495,11 +9246,6 @@ void Compiler::fgRemoveStmt(BasicBlock* block, statement boundaries. Or should we leave a GT_NO_OP in its place? */ } - if (fgOrder == FGOrderLinear) - { - fgRemoveLinearOrderDependencies(stmt); - } - /* Is it the first statement in the list? */ GenTreeStmt* firstStmt = block->firstStmt(); @@ -9768,123 +9514,156 @@ void Compiler::fgCompactBlocks(BasicBlock* block, BasicBlock* bNext) // TODO-CQ: This may be the wrong thing to do. If we're compacting blocks, it's because a // control-flow choice was constant-folded away. So probably phi's need to go away, // as well, in favor of one of the incoming branches. Or at least be modified. - GenTreePtr blkNonPhi1 = block->FirstNonPhiDef(); - GenTreePtr bNextNonPhi1 = bNext->FirstNonPhiDef(); - GenTreePtr blkFirst = block->firstStmt(); - GenTreePtr bNextFirst = bNext->firstStmt(); - // Does the second have any phis? - if (bNextFirst != nullptr && bNextFirst != bNextNonPhi1) + assert(block->IsLIR() == bNext->IsLIR()); + if (block->IsLIR()) { - GenTreePtr bNextLast = bNextFirst->gtPrev; - assert(bNextLast->gtNext == nullptr); + LIR::Range& blockRange = LIR::AsRange(block); + LIR::Range& nextRange = LIR::AsRange(bNext); - // Does "blk" have phis? - if (blkNonPhi1 != blkFirst) + // Does the next block have any phis? + GenTree* nextFirstNonPhi = nullptr; + LIR::ReadOnlyRange nextPhis = nextRange.PhiNodes(); + if (!nextPhis.IsEmpty()) { - // Yes, has phis. - // Insert after the last phi of "block." - // First, bNextPhis after last phi of block. - GenTreePtr blkLastPhi; - if (blkNonPhi1 != nullptr) - { - blkLastPhi = blkNonPhi1->gtPrev; - } - else - { - blkLastPhi = blkFirst->gtPrev; - } + GenTree* blockLastPhi = blockRange.LastPhiNode(); + nextFirstNonPhi = nextPhis.LastNode()->gtNext; - blkLastPhi->gtNext = bNextFirst; - bNextFirst->gtPrev = blkLastPhi; - - // Now, rest of "block" after last phi of "bNext". - GenTreePtr bNextLastPhi = nullptr; - if (bNextNonPhi1 != nullptr) - { - bNextLastPhi = bNextNonPhi1->gtPrev; - } - else - { - bNextLastPhi = bNextFirst->gtPrev; - } - - bNextLastPhi->gtNext = blkNonPhi1; - if (blkNonPhi1 != nullptr) - { - blkNonPhi1->gtPrev = bNextLastPhi; - } - else - { - // block has no non phis, so make the last statement be the last added phi. - blkFirst->gtPrev = bNextLastPhi; - } - - // Now update the bbTreeList of "bNext". - bNext->bbTreeList = bNextNonPhi1; - if (bNextNonPhi1 != nullptr) - { - bNextNonPhi1->gtPrev = bNextLast; - } + LIR::Range phisToMove = nextRange.Remove(std::move(nextPhis)); + blockRange.InsertAfter(blockLastPhi, std::move(phisToMove)); } else { - if (blkFirst != nullptr) // If "block" has no statements, fusion will work fine... + nextFirstNonPhi = nextRange.FirstNode(); + } + + // Does the block have any other code? + if (nextFirstNonPhi != nullptr) + { + LIR::Range nextNodes = nextRange.Remove(nextFirstNonPhi, nextRange.LastNode()); + blockRange.InsertAtEnd(std::move(nextNodes)); + } + } + else + { + GenTreePtr blkNonPhi1 = block->FirstNonPhiDef(); + GenTreePtr bNextNonPhi1 = bNext->FirstNonPhiDef(); + GenTreePtr blkFirst = block->firstStmt(); + GenTreePtr bNextFirst = bNext->firstStmt(); + + // Does the second have any phis? + if (bNextFirst != nullptr && bNextFirst != bNextNonPhi1) + { + GenTreePtr bNextLast = bNextFirst->gtPrev; + assert(bNextLast->gtNext == nullptr); + + // Does "blk" have phis? + if (blkNonPhi1 != blkFirst) { - // First, bNextPhis at start of block. - GenTreePtr blkLast = blkFirst->gtPrev; - block->bbTreeList = bNextFirst; - // Now, rest of "block" (if it exists) after last phi of "bNext". + // Yes, has phis. + // Insert after the last phi of "block." + // First, bNextPhis after last phi of block. + GenTreePtr blkLastPhi; + if (blkNonPhi1 != nullptr) + { + blkLastPhi = blkNonPhi1->gtPrev; + } + else + { + blkLastPhi = blkFirst->gtPrev; + } + + blkLastPhi->gtNext = bNextFirst; + bNextFirst->gtPrev = blkLastPhi; + + // Now, rest of "block" after last phi of "bNext". GenTreePtr bNextLastPhi = nullptr; if (bNextNonPhi1 != nullptr) { - // There is a first non phi, so the last phi is before it. bNextLastPhi = bNextNonPhi1->gtPrev; } else { - // All the statements are phi defns, so the last one is the prev of the first. bNextLastPhi = bNextFirst->gtPrev; } - bNextFirst->gtPrev = blkLast; - bNextLastPhi->gtNext = blkFirst; - blkFirst->gtPrev = bNextLastPhi; - // Now update the bbTreeList of "bNext" + + bNextLastPhi->gtNext = blkNonPhi1; + if (blkNonPhi1 != nullptr) + { + blkNonPhi1->gtPrev = bNextLastPhi; + } + else + { + // block has no non phis, so make the last statement be the last added phi. + blkFirst->gtPrev = bNextLastPhi; + } + + // Now update the bbTreeList of "bNext". bNext->bbTreeList = bNextNonPhi1; if (bNextNonPhi1 != nullptr) { bNextNonPhi1->gtPrev = bNextLast; } } + else + { + if (blkFirst != nullptr) // If "block" has no statements, fusion will work fine... + { + // First, bNextPhis at start of block. + GenTreePtr blkLast = blkFirst->gtPrev; + block->bbTreeList = bNextFirst; + // Now, rest of "block" (if it exists) after last phi of "bNext". + GenTreePtr bNextLastPhi = nullptr; + if (bNextNonPhi1 != nullptr) + { + // There is a first non phi, so the last phi is before it. + bNextLastPhi = bNextNonPhi1->gtPrev; + } + else + { + // All the statements are phi defns, so the last one is the prev of the first. + bNextLastPhi = bNextFirst->gtPrev; + } + bNextFirst->gtPrev = blkLast; + bNextLastPhi->gtNext = blkFirst; + blkFirst->gtPrev = bNextLastPhi; + // Now update the bbTreeList of "bNext" + bNext->bbTreeList = bNextNonPhi1; + if (bNextNonPhi1 != nullptr) + { + bNextNonPhi1->gtPrev = bNextLast; + } + } + } } - } - // Now proceed with the updated bbTreeLists. - GenTreePtr stmtList1 = block->firstStmt(); - GenTreePtr stmtList2 = bNext->firstStmt(); + // Now proceed with the updated bbTreeLists. + GenTreePtr stmtList1 = block->firstStmt(); + GenTreePtr stmtList2 = bNext->firstStmt(); - /* the block may have an empty list */ + /* the block may have an empty list */ - if (stmtList1) - { - GenTreePtr stmtLast1 = block->lastStmt(); - - /* The second block may be a GOTO statement or something with an empty bbTreeList */ - if (stmtList2) + if (stmtList1) { - GenTreePtr stmtLast2 = bNext->lastStmt(); + GenTreePtr stmtLast1 = block->lastStmt(); + + /* The second block may be a GOTO statement or something with an empty bbTreeList */ + if (stmtList2) + { + GenTreePtr stmtLast2 = bNext->lastStmt(); - /* append list2 to list 1 */ + /* append list2 to list 1 */ - stmtLast1->gtNext = stmtList2; - stmtList2->gtPrev = stmtLast1; - stmtList1->gtPrev = stmtLast2; + stmtLast1->gtNext = stmtList2; + stmtList2->gtPrev = stmtLast1; + stmtList1->gtPrev = stmtLast2; + } + } + else + { + /* block was formerly empty and now has bNext's statements */ + block->bbTreeList = stmtList2; } - } - else - { - /* block was formerly empty and now has bNext's statements */ - block->bbTreeList = stmtList2; } // Note we could update the local variable weights here by @@ -10179,23 +9958,35 @@ void Compiler::fgUnreachableBlock(BasicBlock* block) /* Make the block publicly available */ compCurBB = block; - // TODO-Cleanup: I'm not sure why this happens -- if the block is unreachable, why does it have phis? - // Anyway, remove any phis. - GenTreePtr firstNonPhi = block->FirstNonPhiDef(); - if (block->bbTreeList != firstNonPhi) + if (block->IsLIR()) { - if (firstNonPhi != nullptr) + LIR::Range& blockRange = LIR::AsRange(block); + if (!blockRange.IsEmpty()) { - firstNonPhi->gtPrev = block->lastStmt(); + blockRange.Delete(this, block, blockRange.FirstNode(), blockRange.LastNode()); } - block->bbTreeList = firstNonPhi; } - - for (GenTreeStmt* stmt = block->firstStmt(); stmt; stmt = stmt->gtNextStmt) + else { - fgRemoveStmt(block, stmt); + // TODO-Cleanup: I'm not sure why this happens -- if the block is unreachable, why does it have phis? + // Anyway, remove any phis. + + GenTreePtr firstNonPhi = block->FirstNonPhiDef(); + if (block->bbTreeList != firstNonPhi) + { + if (firstNonPhi != nullptr) + { + firstNonPhi->gtPrev = block->lastStmt(); + } + block->bbTreeList = firstNonPhi; + } + + for (GenTreeStmt* stmt = block->firstStmt(); stmt; stmt = stmt->gtNextStmt) + { + fgRemoveStmt(block, stmt); + } + noway_assert(block->bbTreeList == nullptr); } - noway_assert(block->bbTreeList == nullptr); /* Next update the loop table and bbWeights */ optUpdateLoopsBeforeRemoveBlock(block); @@ -10215,6 +10006,7 @@ void Compiler::fgUnreachableBlock(BasicBlock* block) void Compiler::fgRemoveJTrue(BasicBlock* block) { noway_assert(block->bbJumpKind == BBJ_COND && block->bbJumpDest == block->bbNext); + assert(compRationalIRForm == block->IsLIR()); flowList* flow = fgGetPredForBlock(block->bbNext, block); noway_assert(flow->flDupCount == 2); @@ -10237,30 +10029,42 @@ void Compiler::fgRemoveJTrue(BasicBlock* block) /* Remove the block jump condition */ - GenTreeStmt* test = block->lastTopLevelStmt(); - - GenTree* tree = test->gtStmtExpr; + if (block->IsLIR()) + { + LIR::Range& blockRange = LIR::AsRange(block); - noway_assert(tree->gtOper == GT_JTRUE); + GenTree* test = blockRange.LastNode(); + assert(test->OperGet() == GT_JTRUE); - GenTree* sideEffList = nullptr; + bool isClosed; + unsigned sideEffects; + LIR::ReadOnlyRange testRange = blockRange.GetTreeRange(test, &isClosed, &sideEffects); - if (tree->gtFlags & GTF_SIDE_EFFECT) - { - if (compRationalIRForm) + // TODO-LIR: this should really be checking GTF_ALL_EFFECT, but that produces unacceptable + // diffs compared to the existing backend. + if (isClosed && ((sideEffects & GTF_SIDE_EFFECT) == 0)) { - // if we are in rational form don't try to extract the side effects - // because gtExtractSideEffList will create new comma nodes - // (which we would have to rationalize) and fgMorphBlockStmt can't - // handle embedded statements. - - // Instead just transform the JTRUE into a NEG which has the effect of - // evaluating the side-effecting tree and perform a benign operation on it. - tree->SetOper(GT_NEG); - tree->gtType = TYP_I_IMPL; + // If the jump and its operands form a contiguous, side-effect-free range, + // remove them. + blockRange.Delete(this, block, std::move(testRange)); } else { + // Otherwise, just remove the jump node itself. + blockRange.Remove(test); + } + } + else + { + GenTreeStmt* test = block->lastStmt(); + GenTree* tree = test->gtStmtExpr; + + noway_assert(tree->gtOper == GT_JTRUE); + + GenTree* sideEffList = nullptr; + + if (tree->gtFlags & GTF_SIDE_EFFECT) + { gtExtractSideEffList(tree, &sideEffList); if (sideEffList) @@ -10276,21 +10080,18 @@ void Compiler::fgRemoveJTrue(BasicBlock* block) #endif } } - } - // Delete the cond test or replace it with the side effect tree - if (sideEffList == nullptr) - { - if (!compRationalIRForm || (tree->gtFlags & GTF_SIDE_EFFECT) == 0) + // Delete the cond test or replace it with the side effect tree + if (sideEffList == nullptr) { fgRemoveStmt(block, test); } - } - else - { - test->gtStmtExpr = sideEffList; + else + { + test->gtStmtExpr = sideEffList; - fgMorphBlockStmt(block, test DEBUGARG("fgRemoveJTrue")); + fgMorphBlockStmt(block, test DEBUGARG("fgRemoveJTrue")); + } } } @@ -13381,9 +13182,18 @@ bool Compiler::fgOptimizeEmptyBlock(BasicBlock* block) { // Insert a NOP in the empty block to ensure we generate code // for the catchret target in the right EH region. - GenTreePtr nopStmt = fgInsertStmtAtEnd(block, new (this, GT_NO_OP) GenTree(GT_NO_OP, TYP_VOID)); - fgSetStmtSeq(nopStmt); - gtSetStmtInfo(nopStmt); + GenTree* nop = new (this, GT_NO_OP) GenTree(GT_NO_OP, TYP_VOID); + + if (block->IsLIR()) + { + LIR::AsRange(block).InsertAtEnd(nop); + } + else + { + GenTreePtr nopStmt = fgInsertStmtAtEnd(block, nop); + fgSetStmtSeq(nopStmt); + gtSetStmtInfo(nopStmt); + } #ifdef DEBUG if (verbose) @@ -13528,20 +13338,25 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) } } while (++jmpTab, --jmpCnt); - GenTreeStmt* switchStmt = block->lastTopLevelStmt(); - GenTreePtr switchTree = switchStmt->gtStmtExpr; + GenTreeStmt* switchStmt = nullptr; + LIR::Range* blockRange = nullptr; - // If this is a Lowered switch, it must have no embedded statements, because we pulled - // out any embedded statements when we cloned the switch value. - if (switchTree->gtOper == GT_SWITCH_TABLE) + GenTree* switchTree; + if (block->IsLIR()) { - noway_assert(fgOrder == FGOrderLinear); - assert(switchStmt->AsStmt()->gtStmtIsTopLevel() && (switchStmt->gtNext == nullptr)); + blockRange = &LIR::AsRange(block); + switchTree = blockRange->LastNode(); + + assert(switchTree->OperGet() == GT_SWITCH_TABLE); } else { - noway_assert(switchTree->gtOper == GT_SWITCH); + switchStmt = block->lastStmt(); + switchTree = switchStmt->gtStmtExpr; + + assert(switchTree->OperGet() == GT_SWITCH); } + noway_assert(switchTree->gtType == TYP_VOID); // At this point all of the case jump targets have been updated such @@ -13564,59 +13379,74 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) } #endif // DEBUG - /* check for SIDE_EFFECTS */ - - if (switchTree->gtFlags & GTF_SIDE_EFFECT) + if (block->IsLIR()) { - /* Extract the side effects from the conditional */ - GenTreePtr sideEffList = nullptr; + bool isClosed; + unsigned sideEffects; + LIR::ReadOnlyRange switchTreeRange = blockRange->GetTreeRange(switchTree, &isClosed, &sideEffects); - gtExtractSideEffList(switchTree, &sideEffList); + // The switch tree should form a contiguous, side-effect free range by construction. See + // Lowering::LowerSwitch for details. + assert(isClosed); + assert((sideEffects & GTF_ALL_EFFECT) == 0); - if (sideEffList == nullptr) + blockRange->Delete(this, block, std::move(switchTreeRange)); + } + else + { + /* check for SIDE_EFFECTS */ + if (switchTree->gtFlags & GTF_SIDE_EFFECT) { - goto NO_SWITCH_SIDE_EFFECT; - } + /* Extract the side effects from the conditional */ + GenTreePtr sideEffList = nullptr; - noway_assert(sideEffList->gtFlags & GTF_SIDE_EFFECT); + gtExtractSideEffList(switchTree, &sideEffList); + + if (sideEffList == nullptr) + { + goto NO_SWITCH_SIDE_EFFECT; + } + + noway_assert(sideEffList->gtFlags & GTF_SIDE_EFFECT); #ifdef DEBUG - if (verbose) - { - printf("\nSwitch expression has side effects! Extracting side effects...\n"); - gtDispTree(switchTree); - printf("\n"); - gtDispTree(sideEffList); - printf("\n"); - } + if (verbose) + { + printf("\nSwitch expression has side effects! Extracting side effects...\n"); + gtDispTree(switchTree); + printf("\n"); + gtDispTree(sideEffList); + printf("\n"); + } #endif // DEBUG - /* Replace the conditional statement with the list of side effects */ - noway_assert(sideEffList->gtOper != GT_STMT); - noway_assert(sideEffList->gtOper != GT_SWITCH); + /* Replace the conditional statement with the list of side effects */ + noway_assert(sideEffList->gtOper != GT_STMT); + noway_assert(sideEffList->gtOper != GT_SWITCH); - switchStmt->gtStmtExpr = sideEffList; + switchStmt->gtStmtExpr = sideEffList; - if (fgStmtListThreaded) - { - /* Update the lclvar ref counts */ - compCurBB = block; - fgUpdateRefCntForExtract(switchTree, sideEffList); + if (fgStmtListThreaded) + { + /* Update the lclvar ref counts */ + compCurBB = block; + fgUpdateRefCntForExtract(switchTree, sideEffList); - /* Update ordering, costs, FP levels, etc. */ - gtSetStmtInfo(switchStmt); + /* Update ordering, costs, FP levels, etc. */ + gtSetStmtInfo(switchStmt); - /* Re-link the nodes for this statement */ - fgSetStmtSeq(switchStmt); + /* Re-link the nodes for this statement */ + fgSetStmtSeq(switchStmt); + } } - } - else - { + else + { - NO_SWITCH_SIDE_EFFECT: + NO_SWITCH_SIDE_EFFECT: - /* conditional has NO side effect - remove it */ - fgRemoveStmt(block, switchStmt); + /* conditional has NO side effect - remove it */ + fgRemoveStmt(block, switchStmt); + } } // Change the switch jump into a BBJ_ALWAYS @@ -13640,6 +13470,14 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) GenTree* switchVal = switchTree->gtOp.gtOp1; noway_assert(genActualTypeIsIntOrI(switchVal->TypeGet())); + // If we are in LIR, remove the jump table from the block. + if (block->IsLIR()) + { + GenTree* jumpTable = switchTree->gtOp.gtOp2; + assert(jumpTable->OperGet() == GT_JMPTABLE); + blockRange->Remove(jumpTable); + } + // Change the GT_SWITCH(switchVal) into GT_JTRUE(GT_EQ(switchVal==0)). // Also mark the node as GTF_DONT_CSE as further down JIT is not capable of handling it. // For example CSE could determine that the expression rooted at GT_EQ is a candidate cse and @@ -13663,11 +13501,16 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) switchTree->gtOp.gtOp1 = condNode; switchTree->gtOp.gtOp1->gtFlags |= (GTF_RELOP_JMP_USED | GTF_DONT_CSE); - // Re-link the nodes for this statement. - // We know that this is safe for the Lowered form, because we will have eliminated any embedded trees - // when we cloned the switch condition (it is also asserted above). + if (block->IsLIR()) + { + blockRange->InsertAfter(switchVal, zeroConstNode, condNode); + } + else + { + // Re-link the nodes for this statement. + fgSetStmtSeq(switchStmt); + } - fgSetStmtSeq(switchStmt); block->bbJumpDest = block->bbJumpSwt->bbsDstTab[0]; block->bbJumpKind = BBJ_COND; @@ -13691,7 +13534,7 @@ bool Compiler::fgBlockEndFavorsTailDuplication(BasicBlock* block) return false; } - if (!block->lastTopLevelStmt()) + if (!block->lastStmt()) { return false; } @@ -13702,7 +13545,7 @@ bool Compiler::fgBlockEndFavorsTailDuplication(BasicBlock* block) // This is because these statements produce information about values // that would otherwise be lost at the upcoming merge point. - GenTreeStmt* lastStmt = block->lastTopLevelStmt(); + GenTreeStmt* lastStmt = block->lastStmt(); GenTree* tree = lastStmt->gtStmtExpr; if (tree->gtOper != GT_ASG) { @@ -13817,7 +13660,13 @@ bool Compiler::fgOptimizeUncondBranchToSimpleCond(BasicBlock* block, BasicBlock* return false; } + // NOTE: we do not currently hit this assert because this function is only called when + // `fgUpdateFlowGraph` has been called with `doTailDuplication` set to true, and the + // backend always calls `fgUpdateFlowGraph` with `doTailDuplication` set to false. + assert(!block->IsLIR()); + GenTreeStmt* stmt = target->FirstNonPhiDef(); + assert(stmt == target->lastStmt()); // Duplicate the target block at the end of this block @@ -13909,9 +13758,6 @@ bool Compiler::fgOptimizeBranchToNext(BasicBlock* block, BasicBlock* bNext, Basi noway_assert(block->bbJumpKind == BBJ_COND); noway_assert(block->bbTreeList); - GenTreeStmt* cond = block->lastTopLevelStmt(); - noway_assert(cond->gtStmtExpr->gtOper == GT_JTRUE); - #ifdef DEBUG if (verbose) { @@ -13919,22 +13765,38 @@ bool Compiler::fgOptimizeBranchToNext(BasicBlock* block, BasicBlock* bNext, Basi } #endif // DEBUG - /* check for SIDE_EFFECTS */ - - if (cond->gtStmtExpr->gtFlags & GTF_SIDE_EFFECT) + if (block->IsLIR()) { - if (compRationalIRForm) + LIR::Range& blockRange = LIR::AsRange(block); + GenTree* jmp = blockRange.LastNode(); + assert(jmp->OperGet() == GT_JTRUE); + + bool isClosed; + unsigned sideEffects; + LIR::ReadOnlyRange jmpRange = blockRange.GetTreeRange(jmp, &isClosed, &sideEffects); + + // TODO-LIR: this should really be checking GTF_ALL_EFFECT, but that produces unacceptable + // diffs compared to the existing backend. + if (isClosed && ((sideEffects & GTF_SIDE_EFFECT) == 0)) { - // Extracting side-effects won't work in rationalized form. - // Instead just transform the JTRUE into a NEG which has the effect of - // evaluating the side-effecting tree and perform a benign operation on it. - // TODO-CQ: [TFS:1121057] We should be able to simply remove the jump node, - // and change gtStmtExpr to its op1. - cond->gtStmtExpr->SetOper(GT_NEG); - cond->gtStmtExpr->gtType = TYP_I_IMPL; + // If the jump and its operands form a contiguous, side-effect-free range, + // remove them. + blockRange.Delete(this, block, std::move(jmpRange)); } else { + // Otherwise, just remove the jump node itself. + blockRange.Remove(jmp); + } + } + else + { + GenTreeStmt* cond = block->lastStmt(); + noway_assert(cond->gtStmtExpr->gtOper == GT_JTRUE); + + /* check for SIDE_EFFECTS */ + if (cond->gtStmtExpr->gtFlags & GTF_SIDE_EFFECT) + { /* Extract the side effects from the conditional */ GenTreePtr sideEffList = nullptr; @@ -13979,12 +13841,12 @@ bool Compiler::fgOptimizeBranchToNext(BasicBlock* block, BasicBlock* bNext, Basi } } } - } - else - { - compCurBB = block; - /* conditional has NO side effect - remove it */ - fgRemoveStmt(block, cond); + else + { + compCurBB = block; + /* conditional has NO side effect - remove it */ + fgRemoveStmt(block, cond); + } } /* Conditional is gone - simply fall into the next block */ @@ -14069,6 +13931,13 @@ bool Compiler::fgOptimizeBranch(BasicBlock* bJump) return false; } + // This function is only called by fgReorderBlocks, which we do not run in the backend. + // If we wanted to run block reordering in the backend, we would need to be able to + // calculate cost information for LIR on a per-node basis in order for this function + // to work. + assert(!bJump->IsLIR()); + assert(!bDest->IsLIR()); + GenTreeStmt* stmt; unsigned estDupCostSz = 0; for (stmt = bDest->firstStmt(); stmt; stmt = stmt->gtNextStmt) @@ -15981,12 +15850,12 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication) #endif // DEBUG /* Reverse the jump condition */ - GenTreePtr test = block->lastTopLevelStmt(); - - test = test->gtStmt.gtStmtExpr; + GenTree* test = block->lastNode(); noway_assert(test->gtOper == GT_JTRUE); - test->gtOp.gtOp1 = gtReverseCond(test->gtOp.gtOp1); + GenTree* cond = gtReverseCond(test->gtOp.gtOp1); + assert(cond == test->gtOp.gtOp1); // Ensure `gtReverseCond` did not create a new node. + test->gtOp.gtOp1 = cond; // Optimize the Conditional JUMP to go to the new target block->bbJumpDest = bNext->bbJumpDest; @@ -16201,6 +16070,15 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication) fgDispBasicBlocks(verboseTrees); fgDispHandlerTab(); } + + if (compRationalIRForm) + { + for (BasicBlock* block = fgFirstBB; block != nullptr; block = block->bbNext) + { + LIR::AsRange(block).CheckLIR(this); + } + } + fgVerifyHandlerTab(); // Make sure that the predecessor lists are accurate fgDebugCheckBBlist(); @@ -17697,8 +17575,15 @@ BasicBlock* Compiler::fgAddCodeRef(BasicBlock* srcBlk, unsigned refData, Special tree = fgMorphArgs(tree); // Store the tree in the new basic block. - - fgInsertStmtAtEnd(newBlk, fgNewStmtFromTree(tree)); + assert(!srcBlk->isEmpty()); + if (!srcBlk->IsLIR()) + { + fgInsertStmtAtEnd(newBlk, fgNewStmtFromTree(tree)); + } + else + { + LIR::AsRange(newBlk).InsertAtEnd(LIR::SeqTree(this, tree)); + } return add->acdDstBlk; } @@ -17745,7 +17630,10 @@ BasicBlock* Compiler::fgRngChkTarget(BasicBlock* block, unsigned stkDepth, Speci if (verbose) { printf("*** Computing fgRngChkTarget for block BB%02u to stkDepth %d\n", block->bbNum, stkDepth); - gtDispTree(compCurStmt); + if (!block->IsLIR()) + { + gtDispTree(compCurStmt); + } } #endif // DEBUG @@ -17757,7 +17645,7 @@ BasicBlock* Compiler::fgRngChkTarget(BasicBlock* block, unsigned stkDepth, Speci // Sequences the tree. // prevTree is what gtPrev of the first node in execution order gets set to. // Returns the first node (execution order) in the sequenced tree. -GenTree* Compiler::fgSetTreeSeq(GenTree* tree, GenTree* prevTree) +GenTree* Compiler::fgSetTreeSeq(GenTree* tree, GenTree* prevTree, bool isLIR) { GenTree list; @@ -17768,7 +17656,7 @@ GenTree* Compiler::fgSetTreeSeq(GenTree* tree, GenTree* prevTree) fgTreeSeqLst = prevTree; fgTreeSeqNum = 0; fgTreeSeqBeg = nullptr; - fgSetTreeSeqHelper(tree); + fgSetTreeSeqHelper(tree, isLIR); GenTree* result = prevTree->gtNext; if (prevTree == &list) @@ -17786,7 +17674,7 @@ GenTree* Compiler::fgSetTreeSeq(GenTree* tree, GenTree* prevTree) * Uses 'global' - fgTreeSeqLst */ -void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) +void Compiler::fgSetTreeSeqHelper(GenTreePtr tree, bool isLIR) { genTreeOps oper; unsigned kind; @@ -17804,7 +17692,7 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) if (kind & (GTK_CONST | GTK_LEAF)) { - fgSetTreeSeqFinish(tree); + fgSetTreeSeqFinish(tree, isLIR); return; } @@ -17826,9 +17714,9 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) // The use must appear before the def because of the case where a local is cpblk'ed to itself. // If it were otherwise, upstream stores to the local would appear to be dead. assert(tree->gtOp.gtOp1->gtOper != GT_LIST); - fgSetTreeSeqHelper(tree->gtOp.gtOp2); - fgSetTreeSeqHelper(tree->gtOp.gtOp1); - fgSetTreeSeqFinish(tree); + fgSetTreeSeqHelper(tree->gtOp.gtOp2, isLIR); + fgSetTreeSeqHelper(tree->gtOp.gtOp1, isLIR); + fgSetTreeSeqFinish(tree, isLIR); return; } @@ -17841,7 +17729,7 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) { list = nextList; GenTreePtr listItem = list->gtOp.gtOp1; - fgSetTreeSeqHelper(listItem); + fgSetTreeSeqHelper(listItem, isLIR); nextList = list->gtOp.gtOp2; if (nextList != nullptr) { @@ -17858,7 +17746,7 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) assert(list != nullptr); list = nextList; nextList = list->gtNext; - fgSetTreeSeqFinish(list); + fgSetTreeSeqFinish(list, isLIR); } while (list != tree); return; } @@ -17870,18 +17758,18 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) if (reverse) { assert(op1 != nullptr && op2 != nullptr); - fgSetTreeSeqHelper(op2); + fgSetTreeSeqHelper(op2, isLIR); } if (op1 != nullptr) { - fgSetTreeSeqHelper(op1); + fgSetTreeSeqHelper(op1, isLIR); } if (!reverse && op2 != nullptr) { - fgSetTreeSeqHelper(op2); + fgSetTreeSeqHelper(op2, isLIR); } - fgSetTreeSeqFinish(tree); + fgSetTreeSeqFinish(tree, isLIR); return; } @@ -17890,7 +17778,7 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) if (op1 == nullptr) { noway_assert(op2 == nullptr); - fgSetTreeSeqFinish(tree); + fgSetTreeSeqFinish(tree, isLIR); return; } @@ -17902,8 +17790,8 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) /* Visit the indirection first - op2 may point to the * jump Label for array-index-out-of-range */ - fgSetTreeSeqHelper(op1); - fgSetTreeSeqFinish(tree); + fgSetTreeSeqHelper(op1, isLIR); + fgSetTreeSeqFinish(tree, isLIR); return; } @@ -17913,8 +17801,8 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) { /* Visit the (only) operand and we're done */ - fgSetTreeSeqHelper(op1); - fgSetTreeSeqFinish(tree); + fgSetTreeSeqHelper(op1, isLIR); + fgSetTreeSeqFinish(tree, isLIR); return; } @@ -17933,22 +17821,22 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) { noway_assert((tree->gtFlags & GTF_REVERSE_OPS) == 0); - fgSetTreeSeqHelper(op1); + fgSetTreeSeqHelper(op1, isLIR); // Here, for the colon, the sequence does not actually represent "order of evaluation": // one or the other of the branches is executed, not both. Still, to make debugging checks // work, we want the sequence to match the order in which we'll generate code, which means // "else" clause then "then" clause. - fgSetTreeSeqHelper(op2->AsColon()->ElseNode()); - fgSetTreeSeqHelper(op2); - fgSetTreeSeqHelper(op2->AsColon()->ThenNode()); + fgSetTreeSeqHelper(op2->AsColon()->ElseNode(), isLIR); + fgSetTreeSeqHelper(op2, isLIR); + fgSetTreeSeqHelper(op2->AsColon()->ThenNode(), isLIR); - fgSetTreeSeqFinish(tree); + fgSetTreeSeqFinish(tree, isLIR); return; } if (oper == GT_COLON) { - fgSetTreeSeqFinish(tree); + fgSetTreeSeqFinish(tree, isLIR); return; } @@ -17956,16 +17844,16 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) if (tree->gtFlags & GTF_REVERSE_OPS) { - fgSetTreeSeqHelper(op2); - fgSetTreeSeqHelper(op1); + fgSetTreeSeqHelper(op2, isLIR); + fgSetTreeSeqHelper(op1, isLIR); } else { - fgSetTreeSeqHelper(op1); - fgSetTreeSeqHelper(op2); + fgSetTreeSeqHelper(op1, isLIR); + fgSetTreeSeqHelper(op2, isLIR); } - fgSetTreeSeqFinish(tree); + fgSetTreeSeqFinish(tree, isLIR); return; } @@ -17982,7 +17870,7 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) /* We'll evaluate the 'this' argument value first */ if (tree->gtCall.gtCallObjp) { - fgSetTreeSeqHelper(tree->gtCall.gtCallObjp); + fgSetTreeSeqHelper(tree->gtCall.gtCallObjp, isLIR); } /* We'll evaluate the arguments next, left to right @@ -17990,7 +17878,7 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) if (tree->gtCall.gtCallArgs) { - fgSetTreeSeqHelper(tree->gtCall.gtCallArgs); + fgSetTreeSeqHelper(tree->gtCall.gtCallArgs, isLIR); } /* Evaluate the temp register arguments list @@ -17999,49 +17887,49 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) if (tree->gtCall.gtCallLateArgs) { - fgSetTreeSeqHelper(tree->gtCall.gtCallLateArgs); + fgSetTreeSeqHelper(tree->gtCall.gtCallLateArgs, isLIR); } if ((tree->gtCall.gtCallType == CT_INDIRECT) && (tree->gtCall.gtCallCookie != nullptr)) { - fgSetTreeSeqHelper(tree->gtCall.gtCallCookie); + fgSetTreeSeqHelper(tree->gtCall.gtCallCookie, isLIR); } if (tree->gtCall.gtCallType == CT_INDIRECT) { - fgSetTreeSeqHelper(tree->gtCall.gtCallAddr); + fgSetTreeSeqHelper(tree->gtCall.gtCallAddr, isLIR); } if (tree->gtCall.gtControlExpr) { - fgSetTreeSeqHelper(tree->gtCall.gtControlExpr); + fgSetTreeSeqHelper(tree->gtCall.gtControlExpr, isLIR); } break; case GT_ARR_ELEM: - fgSetTreeSeqHelper(tree->gtArrElem.gtArrObj); + fgSetTreeSeqHelper(tree->gtArrElem.gtArrObj, isLIR); unsigned dim; for (dim = 0; dim < tree->gtArrElem.gtArrRank; dim++) { - fgSetTreeSeqHelper(tree->gtArrElem.gtArrInds[dim]); + fgSetTreeSeqHelper(tree->gtArrElem.gtArrInds[dim], isLIR); } break; case GT_ARR_OFFSET: - fgSetTreeSeqHelper(tree->gtArrOffs.gtOffset); - fgSetTreeSeqHelper(tree->gtArrOffs.gtIndex); - fgSetTreeSeqHelper(tree->gtArrOffs.gtArrObj); + fgSetTreeSeqHelper(tree->gtArrOffs.gtOffset, isLIR); + fgSetTreeSeqHelper(tree->gtArrOffs.gtIndex, isLIR); + fgSetTreeSeqHelper(tree->gtArrOffs.gtArrObj, isLIR); break; case GT_CMPXCHG: // Evaluate the trees left to right - fgSetTreeSeqHelper(tree->gtCmpXchg.gtOpLocation); - fgSetTreeSeqHelper(tree->gtCmpXchg.gtOpValue); - fgSetTreeSeqHelper(tree->gtCmpXchg.gtOpComparand); + fgSetTreeSeqHelper(tree->gtCmpXchg.gtOpLocation, isLIR); + fgSetTreeSeqHelper(tree->gtCmpXchg.gtOpValue, isLIR); + fgSetTreeSeqHelper(tree->gtCmpXchg.gtOpComparand, isLIR); break; case GT_ARR_BOUNDS_CHECK: @@ -18049,8 +17937,8 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) case GT_SIMD_CHK: #endif // FEATURE_SIMD // Evaluate the trees left to right - fgSetTreeSeqHelper(tree->gtBoundsChk.gtArrLen); - fgSetTreeSeqHelper(tree->gtBoundsChk.gtIndex); + fgSetTreeSeqHelper(tree->gtBoundsChk.gtArrLen, isLIR); + fgSetTreeSeqHelper(tree->gtBoundsChk.gtIndex, isLIR); break; default: @@ -18061,11 +17949,18 @@ void Compiler::fgSetTreeSeqHelper(GenTreePtr tree) break; } - fgSetTreeSeqFinish(tree); + fgSetTreeSeqFinish(tree, isLIR); } -void Compiler::fgSetTreeSeqFinish(GenTreePtr tree) +void Compiler::fgSetTreeSeqFinish(GenTreePtr tree, bool isLIR) { + // If we are sequencing a node that does not appear in LIR, + // do not add it to the list. + if (isLIR && ((tree->OperGet() == GT_LIST) || tree->OperGet() == GT_ARGPLACE)) + { + return; + } + /* Append to the node list */ ++fgTreeSeqNum; @@ -18245,7 +18140,6 @@ void Compiler::fgSetStmtSeq(GenTreePtr tree) // It's located in front of the first node in the list noway_assert(tree->gtOper == GT_STMT); - noway_assert(tree->gtNext == nullptr || tree->gtNext->gtFlags & GTF_STMT_TOP_LEVEL); /* Assign numbers and next/prev links for this tree */ @@ -18253,7 +18147,7 @@ void Compiler::fgSetStmtSeq(GenTreePtr tree) fgTreeSeqLst = &list; fgTreeSeqBeg = nullptr; - fgSetTreeSeqHelper(tree->gtStmt.gtStmtExpr); + fgSetTreeSeqHelper(tree->gtStmt.gtStmtExpr, false); /* Record the address of the first node */ @@ -18446,29 +18340,6 @@ void Compiler::fgOrderBlockOps(GenTreePtr tree, #endif // LEGACY_BACKEND //------------------------------------------------------------------------ -// fgFindTopLevelStmtBackwards: Find the nearest top-level statement to 'stmt', walking the gtPrev links. -// The nearest one might be 'stmt' itself. -// -// Arguments: -// stmt - The statement to start the search with. -// -// Return Value: -// The nearest top-level statement, walking backwards. -// -// Assumptions: -// We will find one! - -/* static */ -GenTreeStmt* Compiler::fgFindTopLevelStmtBackwards(GenTreeStmt* stmt) -{ - while (!stmt->gtStmtIsTopLevel()) - { - stmt = stmt->gtPrev->AsStmt(); - } - return stmt; -} - -//------------------------------------------------------------------------ // fgGetFirstNode: Get the first node in the tree, in execution order // // Arguments: @@ -18488,7 +18359,7 @@ GenTreePtr Compiler::fgGetFirstNode(GenTreePtr tree) GenTreePtr child = tree; while (child->NumChildren() > 0) { - if (child->OperIsBinary() && ((child->gtFlags & GTF_REVERSE_OPS) != 0)) + if (child->OperIsBinary() && child->IsReverseOp()) { child = child->GetChild(1); } @@ -18500,319 +18371,6 @@ GenTreePtr Compiler::fgGetFirstNode(GenTreePtr tree) return child; } -//------------------------------------------------------------------------ -// fgSnipNode: Remove a single tree node (and not its children, if any) from the execution order. -// -// Arguments: -// 'stmt' - The statement which currently contains 'node' -// 'node' - The tree node to be removed -// -// Return Value: -// None. -// -// Assumptions: -// 'stmt' must be non-null. -// -// Notes: -// The node may be any node in the statement, including the first or last node in the statement. -// This is similar to fgDeleteTreeFromList(), but it removes just a single node, not a whole tree. - -void Compiler::fgSnipNode(GenTreeStmt* stmt, GenTreePtr node) -{ - assert(stmt != nullptr); - assert(node != nullptr); - assert(stmt->gtOper == GT_STMT); - assert(node->gtOper != GT_STMT); - assert(fgTreeIsInStmt(node, stmt)); - - GenTreePtr prevNode = node->gtPrev; - GenTreePtr nextNode = node->gtNext; - - if (prevNode != nullptr) - { - prevNode->gtNext = nextNode; - } - else - { - // The node is the first in the statement in execution order. - assert(stmt->gtStmtList == node); - } - - // Note that the node may be first but also have a prevNode (if it is embedded) - if (stmt->gtStmtList == node) - { - stmt->gtStmtList = nextNode; - } - - if (nextNode != nullptr) - { - nextNode->gtPrev = prevNode; - } - else - { - // The node is the last in the statement in execution order. - assert(stmt->gtStmtExpr == node); - stmt->gtStmtExpr = prevNode; - } -} - -//------------------------------------------------------------------------ -// fgSnipInnerNode: Remove a single tree node (and not its children, if any) from the execution order. -// -// Arguments: -// 'node' - The tree node to be removed -// -// Return Value: -// None. -// -// Assumptions: -// The node may not be the first or last node in the statement. In those cases, fgSnipNode() must be used, -// which gets passed the parent GT_STMT node and can update gtStmtList and gtStmtExpr, respectively. - -/* static */ -void Compiler::fgSnipInnerNode(GenTreePtr node) -{ - assert(node != nullptr); - assert(node->gtOper != GT_STMT); - - GenTreePtr prevNode = node->gtPrev; - GenTreePtr nextNode = node->gtNext; - assert(prevNode != nullptr); - assert(nextNode != nullptr); - prevNode->gtNext = nextNode; - nextNode->gtPrev = prevNode; -} - -//------------------------------------------------------------------------ -// fgDeleteTreeFromList: Remove an entire tree from the execution order. -// -// Arguments: -// 'stmt' - The statement which currently contains 'tree' -// 'tree' - The tree to be removed -// -// Return Value: -// None. -// -// Assumptions: -// 'tree' is in the execution order list for 'stmt' -// -// Notes: -// This is similar to fgSnipNode(), but it removes a whole tree, not just a single node. - -void Compiler::fgDeleteTreeFromList(GenTreeStmt* stmt, GenTreePtr tree) -{ - assert(stmt != nullptr); - assert(tree != nullptr); - assert(stmt->gtOper == GT_STMT); - assert(tree->gtOper != GT_STMT); - assert(fgTreeIsInStmt(tree, stmt)); - - GenTreePtr firstNode = fgGetFirstNode(tree); - GenTreePtr prevNode = firstNode->gtPrev; - GenTreePtr nextNode = tree->gtNext; - - if (prevNode != nullptr) - { - prevNode->gtNext = nextNode; - } - else - { - // The first node in the tree is the first in the statement in execution order. - assert(stmt->gtStmtList == firstNode); - stmt->gtStmtList = nextNode; - } - - if (nextNode != nullptr) - { - nextNode->gtPrev = prevNode; - } - else - { - // The last node in the tree is the last in the statement in execution order. - assert(stmt->gtStmtExpr == tree); - stmt->gtStmtExpr = prevNode; - } -} - -//------------------------------------------------------------------------ -// fgTreeIsInStmt: return 'true' if 'tree' is in the execution order list of statement 'stmt'. -// This works for a single node or an entire tree, assuming a well-formed tree, where the entire -// tree's set of nodes are in the statement execution order list. -// -/* static */ -bool Compiler::fgTreeIsInStmt(GenTree* tree, GenTreeStmt* stmt) -{ - assert(tree != nullptr); - assert(stmt != nullptr); - assert(tree->gtOper != GT_STMT); - assert(stmt->gtOper == GT_STMT); - for (GenTree* curr = stmt->gtStmtList; curr != nullptr; curr = curr->gtNext) - { - if (tree == curr) - { - return true; - } - } - return false; -} - -//------------------------------------------------------------------------ -// fgInsertTreeInListAfter: Insert 'tree' in the execution order list before 'insertionPoint'. -// 'stmt' is required, so we can insert before the first node in the statement. -// Assumes that 'tree' and its children are disjoint from 'insertionPoint', and none of them are in 'stmt'. -// -/* static */ -void Compiler::fgInsertTreeInListBefore(GenTree* tree, GenTree* insertionPoint, GenTreeStmt* stmt) -{ - assert(tree != nullptr); - assert(insertionPoint != nullptr); - assert(stmt != nullptr); - assert(tree->gtOper != GT_STMT); - assert(insertionPoint->gtOper != GT_STMT); - assert(fgTreeIsInStmt(insertionPoint, stmt)); - assert(!fgTreeIsInStmt(tree, stmt)); - - GenTree* beforeTree = insertionPoint->gtPrev; - - insertionPoint->gtPrev = tree; - tree->gtNext = insertionPoint; - - GenTree* first = fgGetFirstNode(tree); - - first->gtPrev = beforeTree; - - if (beforeTree != nullptr) - { - beforeTree->gtNext = first; - - // If the insertionPoint is the gtStatementList, - // update the gtStatemenList to include the newly inserted tree. - if (stmt->gtStmtList == insertionPoint) - { - stmt->gtStmtList = first; - } - } - else - { - assert(stmt->gtStmtList == insertionPoint); - stmt->gtStmtList = first; - } -} - -//------------------------------------------------------------------------ -// fgInsertTreeInListAfter: Insert tree in execution order list after 'insertionPoint'. -// 'stmt' is required, so we can insert after the last node in the statement. -// Assumes that 'tree' and its children are disjoint from 'insertionPoint', and none of them are in 'stmt'. -// -/* static */ -void Compiler::fgInsertTreeInListAfter(GenTree* tree, GenTree* insertionPoint, GenTreeStmt* stmt) -{ - assert(tree != nullptr); - assert(insertionPoint != nullptr); - assert(stmt != nullptr); - assert(tree->gtOper != GT_STMT); - assert(insertionPoint->gtOper != GT_STMT); - assert(fgTreeIsInStmt(insertionPoint, stmt)); - assert(!fgTreeIsInStmt(tree, stmt)); - - GenTree* afterTree = insertionPoint->gtNext; - GenTree* first = fgGetFirstNode(tree); - - insertionPoint->gtNext = first; - first->gtPrev = insertionPoint; - - tree->gtNext = afterTree; - - if (afterTree != nullptr) - { - afterTree->gtPrev = tree; - } - else - { - assert(stmt->gtStmtExpr == insertionPoint); - stmt->gtStmtExpr = tree; - } -} - -//------------------------------------------------------------------------ -// fgInsertTreeBeforeAsEmbedded: Insert a tree before 'insertionPoint' as an embedded statement under 'stmt'. -// -GenTreeStmt* Compiler::fgInsertTreeBeforeAsEmbedded(GenTree* tree, - GenTree* insertionPoint, - GenTreeStmt* stmt, - BasicBlock* block) -{ - assert(tree->gtOper != GT_STMT); - assert(insertionPoint->gtOper != GT_STMT); - assert(stmt != nullptr); - assert(stmt->gtOper == GT_STMT); - assert(fgTreeIsInStmt(insertionPoint, stmt)); - assert(!fgTreeIsInStmt(tree, stmt)); - - gtSetEvalOrder(tree); - fgSetTreeSeq(tree); - fgInsertTreeInListBefore(tree, insertionPoint, stmt); - - // While inserting a statement as embedded, the parent specified has to be a top-level statement - // since we could be inserting it ahead of an already existing embedded statement - // in execution order. - GenTreeStmt* topStmt = fgFindTopLevelStmtBackwards(stmt); - GenTreeStmt* result = fgMakeEmbeddedStmt(block, tree, topStmt); - - DBEXEC(true, fgDebugCheckNodeLinks(block, result)); - return result; -} - -//------------------------------------------------------------------------ -// fgInsertTreeAfterAsEmbedded: Insert a tree after 'insertionPoint' as an embedded statement under 'stmt'. -// If it is inserted after all nodes in the given tree, just make it a new statement. -GenTreeStmt* Compiler::fgInsertTreeAfterAsEmbedded(GenTree* tree, - GenTree* insertionPoint, - GenTreeStmt* stmt, - BasicBlock* block) -{ - assert(tree->gtOper != GT_STMT); - assert(insertionPoint->gtOper != GT_STMT); - assert(stmt != nullptr); - assert(stmt->gtOper == GT_STMT); - assert(fgTreeIsInStmt(insertionPoint, stmt)); - assert(!fgTreeIsInStmt(tree, stmt)); - - GenTreeStmt* result; - - if (insertionPoint->gtNext == nullptr) - { - // We're just going to make it a new top-level statement, not an embedded statement, - // since we're inserting it at the end of the statement list's execution order. - - // we better have been given the right stmt - assert(insertionPoint == stmt->gtStmtExpr); - - // this sets the sequence - result = fgNewStmtFromTree(tree, block); - - // Skip all the embedded statements within 'stmt' (which immediately follow 'stmt' in the statement list). - // Insert after the last such embedded statement. - GenTreeStmt* stmtAfter = stmt; - while ((stmtAfter->gtNext != nullptr) && stmtAfter->gtNextStmt->gtStmtIsEmbedded()) - { - stmtAfter = stmtAfter->gtNextStmt; - } - - fgInsertStmtAfter(block, stmtAfter, result); - } - else - { - gtSetEvalOrder(tree); - fgSetTreeSeq(tree); - fgInsertTreeInListAfter(tree, insertionPoint, stmt); - result = fgMakeEmbeddedStmt(block, tree, stmt); - } - - DBEXEC(true, fgDebugCheckNodeLinks(block, result)); - return result; -} - // Examine the bbTreeList and return the estimated code size for this block unsigned Compiler::fgGetCodeEstimate(BasicBlock* block) { @@ -20088,8 +19646,7 @@ void Compiler::fgDumpStmtTree(GenTreePtr stmt, unsigned blkNum) { compCurStmtNum++; // Increment the current stmtNum - printf("\n***** BB%02u, stmt %d (%s)\n", blkNum, compCurStmtNum, - stmt->gtFlags & GTF_STMT_TOP_LEVEL ? "top level" : "embedded"); + printf("\n***** BB%02u, stmt %d\n", blkNum, compCurStmtNum); if (fgOrder == FGOrderLinear || opts.compDbgInfo) { @@ -20101,6 +19658,34 @@ void Compiler::fgDumpStmtTree(GenTreePtr stmt, unsigned blkNum) } } +//------------------------------------------------------------------------ +// Compiler::fgDumpBlock: dumps the contents of the given block to stdout. +// +// Arguments: +// block - The block to dump. +// +void Compiler::fgDumpBlock(BasicBlock* block) +{ + printf("\n------------ "); + block->dspBlockHeader(this); + + if (!block->IsLIR()) + { + for (GenTreeStmt* stmt = block->firstStmt(); stmt != nullptr; stmt = stmt->gtNextStmt) + { + fgDumpStmtTree(stmt, block->bbNum); + if (stmt == block->bbTreeList) + { + block->bbStmtNum = compCurStmtNum; // Set the block->bbStmtNum + } + } + } + else + { + gtDispRange(LIR::AsRange(block)); + } +} + /*****************************************************************************/ // Walk the BasicBlock list calling fgDumpTree once per Stmt // @@ -20116,17 +19701,8 @@ void Compiler::fgDumpTrees(BasicBlock* firstBlock, // for (BasicBlock* block = firstBlock; block; block = block->bbNext) { - printf("\n------------ "); - block->dspBlockHeader(this); + fgDumpBlock(block); - for (GenTreeStmt* stmt = block->firstStmt(); stmt; stmt = stmt->gtNextStmt) - { - fgDumpStmtTree(stmt, block->bbNum); - if (stmt == block->bbTreeList) - { - block->bbStmtNum = compCurStmtNum; // Set the block->bbStmtNum - } - } if (block == lastBlock) { break; } @@ -20262,18 +19838,17 @@ void Compiler::fgDebugCheckBBlist(bool checkBBNum /* = false */, if (block->bbJumpKind == BBJ_COND) { - noway_assert(block->lastStmt()->gtNext == nullptr && - block->lastTopLevelStmt()->gtStmtExpr->gtOper == GT_JTRUE); + noway_assert(block->lastNode()->gtNext == nullptr && block->lastNode()->gtOper == GT_JTRUE); } else if (block->bbJumpKind == BBJ_SWITCH) { #ifndef LEGACY_BACKEND - noway_assert(block->lastStmt()->gtNext == nullptr && - (block->lastTopLevelStmt()->gtStmtExpr->gtOper == GT_SWITCH || - block->lastTopLevelStmt()->gtStmtExpr->gtOper == GT_SWITCH_TABLE)); + noway_assert(block->lastNode()->gtNext == nullptr && + (block->lastNode()->gtOper == GT_SWITCH || + block->lastNode()->gtOper == GT_SWITCH_TABLE)); #else // LEGACY_BACKEND noway_assert(block->lastStmt()->gtNext == NULL && - block->lastTopLevelStmt()->gtStmtExpr->gtOper == GT_SWITCH); + block->lastStmt()->gtStmtExpr->gtOper == GT_SWITCH); #endif // LEGACY_BACKEND } else if (!( block->bbJumpKind == BBJ_ALWAYS @@ -20527,8 +20102,8 @@ PRED_OK:; if (genReturnBB) { noway_assert(genReturnBB->bbTreeList); - noway_assert(genReturnBB->bbTreeList->gtOper == GT_STMT); - noway_assert(genReturnBB->bbTreeList->gtType == TYP_VOID); + noway_assert(genReturnBB->IsLIR() || genReturnBB->bbTreeList->gtOper == GT_STMT); + noway_assert(genReturnBB->IsLIR() || genReturnBB->bbTreeList->gtType == TYP_VOID); } // The general encoder/decoder (currently) only reports "this" as a generics context as a stack location, @@ -20894,10 +20469,17 @@ void Compiler::fgDebugCheckFlags(GenTreePtr tree) // This calls an alternate method for FGOrderLinear. void Compiler::fgDebugCheckNodeLinks(BasicBlock* block, GenTree* node) { + // LIR blocks are checked using BasicBlock::CheckLIR(). + if (block->IsLIR()) + { + LIR::AsRange(block).CheckLIR(this); + // TODO: return? + } + GenTreeStmt* stmt = node->AsStmt(); assert(fgStmtListThreaded); - if (fgOrder == FGOrderLinear) + if (fgOrder == FGOrderLinear) // remove for LIR? { fgDebugCheckLinearNodeLinks(block, stmt); return; @@ -20999,6 +20581,7 @@ void Compiler::fgDebugCheckNodeLinks(BasicBlock* block, GenTree* node) } } +// TODO-LIR: remove? //------------------------------------------------------------------------ // fgDebugCheckLinearTree: Counts the nodes in a tree by doing a tree traversal, // and validates that GT_CATCH_ARG causes GTF_ORDER_SIDEEFF to be set on @@ -21059,6 +20642,7 @@ unsigned Compiler::fgDebugCheckLinearTree(BasicBlock* block, return nodeCount; } +// TODO-LIR: remove function? //------------------------------------------------------------------------ // fgDebugCheckLinearNodeLinks: DEBUG routine to check correctness of the internal // gtNext, gtPrev threading of a statement. @@ -21080,18 +20664,11 @@ void Compiler::fgDebugCheckLinearNodeLinks(BasicBlock* block, GenTreePtr topLevelStmt, bool printNodes) { + assert(!block->IsLIR()); assert(fgStmtListThreaded); assert(fgOrder == FGOrderLinear); assert(topLevelStmt->gtOper == GT_STMT); - // TODO-Cleanup: This is generally called for statements in order, so we'll skip the embedded ones. - // Consider whether we should do some alternate checking in that case (e.g. just validate - // the list is correct OR validate the corresponding top-level statement, which we probably - // just finished doing, OR fix all callees to check whether it's top-level before calling this). - if ((topLevelStmt->gtFlags & GTF_STMT_TOP_LEVEL) == 0) { - return; -} - // We're first going to traverse the statements in linear order, counting the nodes and ensuring that // the links are consistent. // We should be able to reach all the nodes by starting with topLevelStmt->gtStmt.gtStmtList. @@ -21152,12 +20729,7 @@ void Compiler::fgDebugCheckLinearNodeLinks(BasicBlock* block, JITDUMP("\nNow checking tree-ordering:\n"); } - do - { - treeNodeCount += fgDebugCheckLinearTree(block, stmt, stmt->gtStmtExpr, printNodes); - stmt = stmt->gtNextStmt; - } - while (stmt != nullptr && (stmt->gtFlags & GTF_STMT_TOP_LEVEL) == 0); + treeNodeCount += fgDebugCheckLinearTree(block, stmt, stmt->gtStmtExpr, printNodes); if (treeNodeCount != linearNodeCount) { @@ -21171,32 +20743,6 @@ void Compiler::fgDebugCheckLinearNodeLinks(BasicBlock* block, } -//------------------------------------------------------------------------ -// fgStmtContainsNode: -// Debugging method to check whether a tree is inside the given -// statement. -// -// Arguments: -// stmt - The statement whose tree is presumably contained inside -// tree - GenTree to be checked. -// -// Return Value: -// True in case 'tree' is contained inside statement 'stmt' -// -bool Compiler::fgStmtContainsNode(GenTreeStmt* stmt, GenTree* tree) -{ - GenTree* first = stmt->gtStmtList; - for (GenTree* actual = first; - actual != nullptr; - actual = actual->gtNext) - { - if (actual == tree) { - return true; -} - } - return false; -} - /***************************************************************************** * * A DEBUG routine to check the correctness of the links between GT_STMT nodes @@ -21223,60 +20769,66 @@ void Compiler::fgDebugCheckLinks(bool morphTrees) for (BasicBlock* block = fgFirstBB; block; block = block->bbNext) { PROCESS_BLOCK_AGAIN:; - - for (GenTreeStmt* stmt = block->firstStmt(); stmt; stmt = stmt->gtNextStmt) + if (block->IsLIR()) { - /* Verify that bbTreeList is threaded correctly */ - /* Note that for the GT_STMT list, the gtPrev list is circular. The gtNext list is not: gtNext of the last GT_STMT in a block is nullptr. */ + LIR::AsRange(block).CheckLIR(this); + } + else + { + for (GenTreeStmt* stmt = block->firstStmt(); stmt; stmt = stmt->gtNextStmt) + { + /* Verify that bbTreeList is threaded correctly */ + /* Note that for the GT_STMT list, the gtPrev list is circular. The gtNext list is not: gtNext of the last GT_STMT in a block is nullptr. */ - noway_assert(stmt->gtPrev); + noway_assert(stmt->gtPrev); - if (stmt == block->bbTreeList) - { - noway_assert(stmt->gtPrev->gtNext == nullptr); - } - else - { - noway_assert(stmt->gtPrev->gtNext == stmt); - } + if (stmt == block->bbTreeList) + { + noway_assert(stmt->gtPrev->gtNext == nullptr); + } + else + { + noway_assert(stmt->gtPrev->gtNext == stmt); + } - if (stmt->gtNext) - { - noway_assert(stmt->gtNext->gtPrev == stmt); - } - else - { - noway_assert(block->lastStmt() == stmt); - } + if (stmt->gtNext) + { + noway_assert(stmt->gtNext->gtPrev == stmt); + } + else + { + noway_assert(block->lastStmt() == stmt); + } - /* For each statement check that the exception flags are properly set */ + /* For each statement check that the exception flags are properly set */ - noway_assert(stmt->gtStmtExpr); + noway_assert(stmt->gtStmtExpr); - if (verbose && 0) - { - gtDispTree(stmt->gtStmtExpr); - } + if (verbose && 0) + { + gtDispTree(stmt->gtStmtExpr); + } - fgDebugCheckFlags(stmt->gtStmtExpr); + fgDebugCheckFlags(stmt->gtStmtExpr); - // Not only will this stress fgMorphBlockStmt(), but we also get all the checks - // done by fgMorphTree() + // Not only will this stress fgMorphBlockStmt(), but we also get all the checks + // done by fgMorphTree() - if (morphTrees) - { - // If 'stmt' is removed from the block, restart - if (fgMorphBlockStmt(block, stmt DEBUGARG("test morphing"))) + if (morphTrees) { - goto PROCESS_BLOCK_AGAIN; + // If 'stmt' is removed from the block, restart + if (fgMorphBlockStmt(block, stmt DEBUGARG("test morphing"))) + { + goto PROCESS_BLOCK_AGAIN; + } } - } - /* For each GT_STMT node check that the nodes are threaded correcly - gtStmtList */ + /* For each GT_STMT node check that the nodes are threaded correcly - gtStmtList */ - if (fgStmtListThreaded) - { - fgDebugCheckNodeLinks(block, stmt); + if (fgStmtListThreaded) + { + fgDebugCheckNodeLinks(block, stmt); + } } } } @@ -21344,85 +20896,6 @@ void Compiler::fgDebugCheckBlockLinks() /*****************************************************************************/ //------------------------------------------------------------------------ -// fgNodeContainsEmbeddedStatement: -// Predicate that verifies whether the given tree has an embedded statement -// in it. -// -// Arguments: -// tree - GenTree to be checked. -// topLevel - The top-level statement where 'tree' lives. -// -// Assumptions: -// The given 'tree' must be contained inside 'topLevel' (i.e. is a descendant -// of topLevel.gtStmtExpr -// -// Return Value: -// True in case 'tree' contains an embedded statement. -// -bool Compiler::fgNodeContainsEmbeddedStatement(GenTree* tree, GenTreeStmt* topLevel) -{ - assert(fgStmtContainsNode(topLevel, tree)); - - for (GenTree* actual = fgGetFirstNode(tree); - actual != tree; - actual = actual->gtNext) - { - for (GenTree* curStmt = topLevel->gtNext; - curStmt != nullptr && curStmt->gtStmt.gtStmtIsEmbedded(); - curStmt = curStmt->gtNext) - { - if (curStmt->gtStmt.gtStmtList == actual) { - return true; -} - } - } - return false; -} - -//------------------------------------------------------------------------ -// fgRemoveContainedEmbeddedStatements: -// If a tree contains a subtree, recursively remove all embedded -// statements "contained" in the subtree. -// -// Arguments: -// tree - GenTree to be checked. -// stmt - The statement where 'tree' lives. -// block - block where "topLevel" lives. -// -// Assumptions: -// The given 'tree' must be contained inside 'stmt' (i.e. is a descendant -// of stmt.gtStmtExpr and 'stmt' is in 'block' -// -// Return Value: -// None, but all embedded statements that the tree depends on are removed. -// -void Compiler::fgRemoveContainedEmbeddedStatements(GenTreePtr tree, GenTreeStmt* stmt, BasicBlock* block) -{ - assert(fgStmtContainsNode(stmt, tree)); - - GenTreePtr embCursor = stmt->gtNext; - // Get the first node that will be evaluated in the subtree, - // "tree" will be the last node to be evaluated. - for (GenTree* child = fgGetFirstNode(tree); child != tree; child = child->gtNext) - { - // Now check each following stmt to see if "tree" - // is actually the first node in its stmt list. - for (GenTreePtr cur = embCursor; - cur != nullptr && cur->gtStmt.gtStmtIsEmbedded(); - cur = cur->gtNext) - { - if (cur->gtStmt.gtStmtList == child) - { - fgRemoveContainedEmbeddedStatements(cur->gtStmt.gtStmtExpr, cur->AsStmt(), block); - fgRemoveStmt(block, cur); - embCursor = cur->gtNext; - break; - } - } - } -} - -//------------------------------------------------------------------------ // fgCheckForInlineDepthAndRecursion: compute depth of the candidate, and // check for recursion. // diff --git a/src/jit/gentree.cpp b/src/jit/gentree.cpp index 992e5d742f..00319033d6 100644 --- a/src/jit/gentree.cpp +++ b/src/jit/gentree.cpp @@ -767,49 +767,6 @@ Compiler::fgWalkResult Compiler::fgWalkTreePostRec(GenTreePtr* pTree, fgWalkData fgWalkData->parent = tree; - if (kind & GTK_SMPOP) - { - GenTree** op1Slot = &tree->gtOp.gtOp1; - - GenTree** op2Slot; - if (tree->OperIsBinary()) - { - if ((tree->gtFlags & GTF_REVERSE_OPS) == 0) - { - op2Slot = &tree->gtOp.gtOp2; - } - else - { - op2Slot = op1Slot; - op1Slot = &tree->gtOp.gtOp2; - } - } - else - { - op2Slot = nullptr; - } - - if (*op1Slot != nullptr) - { - result = fgWalkTreePostRec<computeStack>(op1Slot, fgWalkData); - if (result == WALK_ABORT) - { - return result; - } - } - - if (op2Slot != nullptr && *op2Slot != nullptr) - { - result = fgWalkTreePostRec<computeStack>(op2Slot, fgWalkData); - if (result == WALK_ABORT) - { - return result; - } - } - - goto DONE; - } - /* See what kind of a special operator we have here */ switch (oper) @@ -954,11 +911,97 @@ Compiler::fgWalkResult Compiler::fgWalkTreePostRec(GenTreePtr* pTree, fgWalkData } break; + case GT_PHI: + { + GenTreeUnOp* phi = tree->AsUnOp(); + if (phi->gtOp1 != nullptr) + { + for (GenTreeArgList* args = phi->gtOp1->AsArgList(); args != nullptr; args = args->Rest()) + { + result = fgWalkTreePostRec<computeStack>(&args->gtOp1, fgWalkData); + if (result == WALK_ABORT) + { + return result; + } + } + } + } + break; + + case GT_INITBLK: + case GT_COPYBLK: + case GT_COPYOBJ: + { + GenTreeBlkOp* blkOp = tree->AsBlkOp(); + result = fgWalkTreePostRec<computeStack>(&blkOp->gtOp1->AsArgList()->gtOp1, fgWalkData); + if (result == WALK_ABORT) + { + return result; + } + + result = fgWalkTreePostRec<computeStack>(&blkOp->gtOp1->AsArgList()->gtOp2, fgWalkData); + if (result == WALK_ABORT) + { + return result; + } + + result = fgWalkTreePostRec<computeStack>(&blkOp->gtOp2, fgWalkData); + if (result == WALK_ABORT) + { + return result; + } + } + break; + default: + if (kind & GTK_SMPOP) + { + GenTree** op1Slot = &tree->gtOp.gtOp1; + + GenTree** op2Slot; + if (tree->OperIsBinary()) + { + if ((tree->gtFlags & GTF_REVERSE_OPS) == 0) + { + op2Slot = &tree->gtOp.gtOp2; + } + else + { + op2Slot = op1Slot; + op1Slot = &tree->gtOp.gtOp2; + } + } + else + { + op2Slot = nullptr; + } + + if (*op1Slot != nullptr) + { + result = fgWalkTreePostRec<computeStack>(op1Slot, fgWalkData); + if (result == WALK_ABORT) + { + return result; + } + } + + if (op2Slot != nullptr && *op2Slot != nullptr) + { + result = fgWalkTreePostRec<computeStack>(op2Slot, fgWalkData); + if (result == WALK_ABORT) + { + return result; + } + } + } #ifdef DEBUG - fgWalkData->compiler->gtDispTree(tree); + else + { + fgWalkData->compiler->gtDispTree(tree); + assert(!"unexpected operator"); + } #endif - assert(!"unexpected operator"); + break; } DONE: @@ -5487,49 +5530,6 @@ bool GenTree::IsAddWithI32Const(GenTreePtr* addr, int* offset) } //------------------------------------------------------------------------ -// InsertAfterSelf: Insert 'node' after this node in execution order. -// If 'stmt' is not nullptr, then it is the parent statement of 'this', and we can insert at the -// end of the statement list. If 'stmt' is nullptr, we can't insert at the end of the statement list. -// -// Arguments: -// 'node' - The node to insert. We only insert a node, not a whole tree. -// 'stmt' - Optional. If set, the parent statement of 'this'. -// -// Return Value: -// None. -// -// Assumptions: -// 'node' is a single node to insert, not a tree to insert. -// -// Notes: -// Use Compiler::fgInsertTreeInListAfter() to insert a whole tree. - -void GenTree::InsertAfterSelf(GenTree* node, GenTreeStmt* stmt /* = nullptr */) -{ - // statements have crazy requirements - assert(this->gtOper != GT_STMT); - - node->gtNext = this->gtNext; - node->gtPrev = this; - - // Insertion at beginning and end of block are special cases - // and require more context. - if (this->gtNext == nullptr) - { - assert(stmt != nullptr); - assert(stmt->gtOper == GT_STMT); - assert(stmt->gtStmtExpr == this); - stmt->gtStmtExpr = node; - } - else - { - this->gtNext->gtPrev = node; - } - - this->gtNext = node; -} - -//------------------------------------------------------------------------ // gtGetChildPointer: If 'parent' is the parent of this node, return the pointer // to the child node so that it can be modified; otherwise, return nullptr. // @@ -5699,6 +5699,20 @@ GenTreePtr* GenTree::gtGetChildPointer(GenTreePtr parent) return nullptr; } +bool GenTree::TryGetUse(GenTree* def, GenTree*** use, bool expandMultiRegArgs) +{ + for (GenTree** useEdge : UseEdges(expandMultiRegArgs)) + { + if (*useEdge == def) + { + *use = useEdge; + return true; + } + } + + return false; +} + //------------------------------------------------------------------------ // gtGetParent: Get the parent of this node, and optionally capture the // pointer to the child so that it can be modified. @@ -7622,13 +7636,13 @@ DONE: // gtReplaceTree: Replace a tree with a new tree. // // Arguments: -// stmt - The top-level root stmt of the tree bing replaced. +// stmt - The top-level root stmt of the tree being replaced. // Must not be null. // tree - The tree being replaced. Must not be null. // replacementTree - The replacement tree. Must not be null. // // Return Value: -// Return the tree node actually replaces the old tree. +// The tree node that replaces the old tree. // // Assumptions: // The sequencing of the stmt has been done. @@ -8328,9 +8342,9 @@ GenTreePtr GenTree::GetChild(unsigned childNum) } } -GenTreeOperandIterator::GenTreeOperandIterator() +GenTreeUseEdgeIterator::GenTreeUseEdgeIterator() : m_node(nullptr) - , m_operand(nullptr) + , m_edge(nullptr) , m_argList(nullptr) , m_multiRegArg(nullptr) , m_expandMultiRegArgs(false) @@ -8338,9 +8352,9 @@ GenTreeOperandIterator::GenTreeOperandIterator() { } -GenTreeOperandIterator::GenTreeOperandIterator(GenTree* node, bool expandMultiRegArgs) +GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node, bool expandMultiRegArgs) : m_node(node) - , m_operand(nullptr) + , m_edge(nullptr) , m_argList(nullptr) , m_multiRegArg(nullptr) , m_expandMultiRegArgs(expandMultiRegArgs) @@ -8353,7 +8367,7 @@ GenTreeOperandIterator::GenTreeOperandIterator(GenTree* node, bool expandMultiRe } //------------------------------------------------------------------------ -// GenTreeOperandIterator::GetNextOperand: +// GenTreeUseEdgeIterator::GetNextUseEdge: // Gets the next operand of a node with a fixed number of operands. // This covers all nodes besides GT_CALL, GT_PHI, and GT_SIMD. For the // node types handled by this method, the `m_state` field indicates the @@ -8362,7 +8376,8 @@ GenTreeOperandIterator::GenTreeOperandIterator(GenTree* node, bool expandMultiRe // Returns: // The node's next operand or nullptr if all operands have been // produced. -GenTree* GenTreeOperandIterator::GetNextOperand() const +// +GenTree** GenTreeUseEdgeIterator::GetNextUseEdge() const { switch (m_node->OperGet()) { @@ -8370,11 +8385,11 @@ GenTree* GenTreeOperandIterator::GetNextOperand() const switch (m_state) { case 0: - return m_node->AsCmpXchg()->gtOpLocation; + return &m_node->AsCmpXchg()->gtOpLocation; case 1: - return m_node->AsCmpXchg()->gtOpValue; + return &m_node->AsCmpXchg()->gtOpValue; case 2: - return m_node->AsCmpXchg()->gtOpComparand; + return &m_node->AsCmpXchg()->gtOpComparand; default: return nullptr; } @@ -8385,9 +8400,9 @@ GenTree* GenTreeOperandIterator::GetNextOperand() const switch (m_state) { case 0: - return m_node->AsBoundsChk()->gtArrLen; + return &m_node->AsBoundsChk()->gtArrLen; case 1: - return m_node->AsBoundsChk()->gtIndex; + return &m_node->AsBoundsChk()->gtIndex; default: return nullptr; } @@ -8395,25 +8410,25 @@ GenTree* GenTreeOperandIterator::GetNextOperand() const case GT_FIELD: if (m_state == 0) { - return m_node->AsField()->gtFldObj; + return &m_node->AsField()->gtFldObj; } return nullptr; case GT_STMT: if (m_state == 0) { - return m_node->AsStmt()->gtStmtExpr; + return &m_node->AsStmt()->gtStmtExpr; } return nullptr; case GT_ARR_ELEM: if (m_state == 0) { - return m_node->AsArrElem()->gtArrObj; + return &m_node->AsArrElem()->gtArrObj; } else if (m_state <= m_node->AsArrElem()->gtArrRank) { - return m_node->AsArrElem()->gtArrInds[m_state - 1]; + return &m_node->AsArrElem()->gtArrInds[m_state - 1]; } return nullptr; @@ -8421,16 +8436,16 @@ GenTree* GenTreeOperandIterator::GetNextOperand() const switch (m_state) { case 0: - return m_node->AsArrOffs()->gtOffset; + return &m_node->AsArrOffs()->gtOffset; case 1: - return m_node->AsArrOffs()->gtIndex; + return &m_node->AsArrOffs()->gtIndex; case 2: - return m_node->AsArrOffs()->gtArrObj; + return &m_node->AsArrOffs()->gtArrObj; default: return nullptr; } - // Call, phi, and SIMD nodes are handled by MoveNext{Call,Phi,SIMD}Operand, repsectively. + // Call, phi, and SIMD nodes are handled by MoveNext{Call,Phi,SIMD}UseEdge, repsectively. case GT_CALL: case GT_PHI: #ifdef FEATURE_SIMD @@ -8452,11 +8467,11 @@ GenTree* GenTreeOperandIterator::GetNextOperand() const switch (m_state) { case 0: - return !srcDstReversed ? blkOp->gtOp1->AsArgList()->gtOp1 : blkOp->gtOp1->AsArgList()->gtOp2; + return !srcDstReversed ? &blkOp->gtOp1->AsArgList()->gtOp1 : &blkOp->gtOp1->AsArgList()->gtOp2; case 1: - return !srcDstReversed ? blkOp->gtOp1->AsArgList()->gtOp2 : blkOp->gtOp1->AsArgList()->gtOp1; + return !srcDstReversed ? &blkOp->gtOp1->AsArgList()->gtOp2 : &blkOp->gtOp1->AsArgList()->gtOp1; case 2: - return blkOp->gtOp2; + return &blkOp->gtOp2; default: return nullptr; } @@ -8466,11 +8481,11 @@ GenTree* GenTreeOperandIterator::GetNextOperand() const switch (m_state) { case 0: - return blkOp->gtOp2; + return &blkOp->gtOp2; case 1: - return !srcDstReversed ? blkOp->gtOp1->AsArgList()->gtOp1 : blkOp->gtOp1->AsArgList()->gtOp2; + return !srcDstReversed ? &blkOp->gtOp1->AsArgList()->gtOp1 : &blkOp->gtOp1->AsArgList()->gtOp2; case 2: - return !srcDstReversed ? blkOp->gtOp1->AsArgList()->gtOp2 : blkOp->gtOp1->AsArgList()->gtOp1; + return !srcDstReversed ? &blkOp->gtOp1->AsArgList()->gtOp2 : &blkOp->gtOp1->AsArgList()->gtOp1; default: return nullptr; } @@ -8485,16 +8500,16 @@ GenTree* GenTreeOperandIterator::GetNextOperand() const bool hasOp1 = lea->gtOp1 != nullptr; if (!hasOp1) { - return m_state == 0 ? lea->gtOp2 : nullptr; + return m_state == 0 ? &lea->gtOp2 : nullptr; } bool operandsReversed = (lea->gtFlags & GTF_REVERSE_OPS) != 0; switch (m_state) { case 0: - return !operandsReversed ? lea->gtOp1 : lea->gtOp2; + return !operandsReversed ? &lea->gtOp1 : &lea->gtOp2; case 1: - return !operandsReversed ? lea->gtOp2 : lea->gtOp1; + return !operandsReversed ? &lea->gtOp2 : &lea->gtOp1; default: return nullptr; } @@ -8508,7 +8523,7 @@ GenTree* GenTreeOperandIterator::GetNextOperand() const } else if (m_node->OperIsUnary()) { - return m_state == 0 ? m_node->AsUnOp()->gtOp1 : nullptr; + return m_state == 0 ? &m_node->AsUnOp()->gtOp1 : nullptr; } else if (m_node->OperIsBinary()) { @@ -8516,9 +8531,9 @@ GenTree* GenTreeOperandIterator::GetNextOperand() const switch (m_state) { case 0: - return !operandsReversed ? m_node->AsOp()->gtOp1 : m_node->AsOp()->gtOp2; + return !operandsReversed ? &m_node->AsOp()->gtOp1 : &m_node->AsOp()->gtOp2; case 1: - return !operandsReversed ? m_node->AsOp()->gtOp2 : m_node->AsOp()->gtOp1; + return !operandsReversed ? &m_node->AsOp()->gtOp2 : &m_node->AsOp()->gtOp1; default: return nullptr; } @@ -8529,13 +8544,13 @@ GenTree* GenTreeOperandIterator::GetNextOperand() const } //------------------------------------------------------------------------ -// GenTreeOperandIterator::MoveToNextCallOperand: +// GenTreeUseEdgeIterator::MoveToNextCallUseEdge: // Moves to the next operand of a call node. Unlike the simple nodes -// handled by `GetNextOperand`, call nodes have a variable number of +// handled by `GetNextUseEdge`, call nodes have a variable number of // operands stored in cons lists. This method expands the cons lists // into the operands stored within. // -void GenTreeOperandIterator::MoveToNextCallOperand() +void GenTreeUseEdgeIterator::MoveToNextCallUseEdge() { GenTreeCall* call = m_node->AsCall(); @@ -8549,7 +8564,7 @@ void GenTreeOperandIterator::MoveToNextCallOperand() if (call->gtCallObjp != nullptr) { - m_operand = call->gtCallObjp; + m_edge = &call->gtCallObjp; return; } break; @@ -8575,7 +8590,7 @@ void GenTreeOperandIterator::MoveToNextCallOperand() } else { - m_operand = argNode->gtOp1; + m_edge = &argNode->gtOp1; m_argList = argNode->Rest(); return; } @@ -8592,7 +8607,7 @@ void GenTreeOperandIterator::MoveToNextCallOperand() else { GenTreeArgList* regNode = m_multiRegArg->AsArgList(); - m_operand = regNode->gtOp1; + m_edge = ®Node->gtOp1; m_multiRegArg = regNode->Rest(); return; } @@ -8603,7 +8618,7 @@ void GenTreeOperandIterator::MoveToNextCallOperand() if (call->gtControlExpr != nullptr) { - m_operand = call->gtControlExpr; + m_edge = &call->gtControlExpr; return; } break; @@ -8615,7 +8630,7 @@ void GenTreeOperandIterator::MoveToNextCallOperand() if (call->gtCallCookie != nullptr) { - m_operand = call->gtCallCookie; + m_edge = &call->gtCallCookie; return; } break; @@ -8626,14 +8641,14 @@ void GenTreeOperandIterator::MoveToNextCallOperand() m_state = 8; if (call->gtCallAddr != nullptr) { - m_operand = call->gtCallAddr; + m_edge = &call->gtCallAddr; return; } break; default: m_node = nullptr; - m_operand = nullptr; + m_edge = nullptr; m_argList = nullptr; m_state = -1; return; @@ -8642,13 +8657,13 @@ void GenTreeOperandIterator::MoveToNextCallOperand() } //------------------------------------------------------------------------ -// GenTreeOperandIterator::MoveToNextPhiOperand: +// GenTreeUseEdgeIterator::MoveToNextPhiUseEdge: // Moves to the next operand of a phi node. Unlike the simple nodes -// handled by `GetNextOperand`, phi nodes have a variable number of +// handled by `GetNextUseEdge`, phi nodes have a variable number of // operands stored in a cons list. This method expands the cons list // into the operands stored within. // -void GenTreeOperandIterator::MoveToNextPhiOperand() +void GenTreeUseEdgeIterator::MoveToNextPhiUseEdge() { GenTreeUnOp* phi = m_node->AsUnOp(); @@ -8669,7 +8684,7 @@ void GenTreeOperandIterator::MoveToNextPhiOperand() else { GenTreeArgList* argNode = m_argList->AsArgList(); - m_operand = argNode->gtOp1; + m_edge = &argNode->gtOp1; m_argList = argNode->Rest(); return; } @@ -8677,7 +8692,7 @@ void GenTreeOperandIterator::MoveToNextPhiOperand() default: m_node = nullptr; - m_operand = nullptr; + m_edge = nullptr; m_argList = nullptr; m_state = -1; return; @@ -8687,14 +8702,14 @@ void GenTreeOperandIterator::MoveToNextPhiOperand() #ifdef FEATURE_SIMD //------------------------------------------------------------------------ -// GenTreeOperandIterator::MoveToNextSIMDOperand: +// GenTreeUseEdgeIterator::MoveToNextSIMDUseEdge: // Moves to the next operand of a SIMD node. Most SIMD nodes have a // fixed number of operands and are handled accordingly. // `SIMDIntrinsicInitN` nodes, however, have a variable number of // operands stored in a cons list. This method expands the cons list // into the operands stored within. // -void GenTreeOperandIterator::MoveToNextSIMDOperand() +void GenTreeUseEdgeIterator::MoveToNextSIMDUseEdge() { GenTreeSIMD* simd = m_node->AsSIMD(); @@ -8704,17 +8719,17 @@ void GenTreeOperandIterator::MoveToNextSIMDOperand() switch (m_state) { case 0: - m_operand = !operandsReversed ? simd->gtOp1 : simd->gtOp2; + m_edge = !operandsReversed ? &simd->gtOp1 : &simd->gtOp2; break; case 1: - m_operand = !operandsReversed ? simd->gtOp2 : simd->gtOp1; + m_edge = !operandsReversed ? &simd->gtOp2 : &simd->gtOp1; break; default: - m_operand = nullptr; + m_edge = nullptr; break; } - if (m_operand != nullptr) + if (m_edge != nullptr && *m_edge != nullptr) { m_state++; } @@ -8744,7 +8759,7 @@ void GenTreeOperandIterator::MoveToNextSIMDOperand() else { GenTreeArgList* argNode = m_argList->AsArgList(); - m_operand = argNode->gtOp1; + m_edge = &argNode->gtOp1; m_argList = argNode->Rest(); return; } @@ -8752,7 +8767,7 @@ void GenTreeOperandIterator::MoveToNextSIMDOperand() default: m_node = nullptr; - m_operand = nullptr; + m_edge = nullptr; m_argList = nullptr; m_state = -1; return; @@ -8762,16 +8777,16 @@ void GenTreeOperandIterator::MoveToNextSIMDOperand() #endif //------------------------------------------------------------------------ -// GenTreeOperandIterator::operator++: +// GenTreeUseEdgeIterator::operator++: // Advances the iterator to the next operand. // -GenTreeOperandIterator& GenTreeOperandIterator::operator++() +GenTreeUseEdgeIterator& GenTreeUseEdgeIterator::operator++() { if (m_state == -1) { // If we've reached the terminal state, do nothing. assert(m_node == nullptr); - assert(m_operand == nullptr); + assert(m_edge == nullptr); assert(m_argList == nullptr); } else @@ -8780,27 +8795,28 @@ GenTreeOperandIterator& GenTreeOperandIterator::operator++() genTreeOps op = m_node->OperGet(); if (op == GT_CALL) { - MoveToNextCallOperand(); + MoveToNextCallUseEdge(); } else if (op == GT_PHI) { - MoveToNextPhiOperand(); + MoveToNextPhiUseEdge(); } #ifdef FEATURE_SIMD else if (op == GT_SIMD) { - MoveToNextSIMDOperand(); + MoveToNextSIMDUseEdge(); } #endif else { - m_operand = GetNextOperand(); - if (m_operand != nullptr) + m_edge = GetNextUseEdge(); + if (m_edge != nullptr && *m_edge != nullptr) { m_state++; } else { + m_edge = nullptr; m_node = nullptr; m_state = -1; } @@ -8810,6 +8826,21 @@ GenTreeOperandIterator& GenTreeOperandIterator::operator++() return *this; } +GenTreeUseEdgeIterator GenTree::UseEdgesBegin(bool expandMultiRegArgs) +{ + return GenTreeUseEdgeIterator(this, expandMultiRegArgs); +} + +GenTreeUseEdgeIterator GenTree::UseEdgesEnd() +{ + return GenTreeUseEdgeIterator(); +} + +IteratorPair<GenTreeUseEdgeIterator> GenTree::UseEdges(bool expandMultiRegArgs) +{ + return MakeIteratorPair(UseEdgesBegin(expandMultiRegArgs), UseEdgesEnd()); +} + GenTreeOperandIterator GenTree::OperandsBegin(bool expandMultiRegArgs) { return GenTreeOperandIterator(this, expandMultiRegArgs); @@ -8825,6 +8856,21 @@ IteratorPair<GenTreeOperandIterator> GenTree::Operands(bool expandMultiRegArgs) return MakeIteratorPair(OperandsBegin(expandMultiRegArgs), OperandsEnd()); } +bool GenTree::Precedes(GenTree* other) +{ + assert(other != nullptr); + + for (GenTree* node = gtNext; node != nullptr; node = node->gtNext) + { + if (node == other) + { + return true; + } + } + + return false; +} + #ifdef DEBUG /* static */ int GenTree::gtDispFlags(unsigned flags, unsigned debugFlags) @@ -9060,7 +9106,7 @@ void Compiler::gtDispVN(GenTree* tree) // 'indentStack' may be null, in which case no indentation or arcs are printed // 'msg' may be null -void Compiler::gtDispNode(GenTreePtr tree, IndentStack* indentStack, __in __in_z __in_opt const char* msg) +void Compiler::gtDispNode(GenTreePtr tree, IndentStack* indentStack, __in __in_z __in_opt const char* msg, bool isLIR) { bool printPointer = true; // always true.. bool printFlags = true; // always true.. @@ -9411,6 +9457,28 @@ void Compiler::gtDispNode(GenTreePtr tree, IndentStack* indentStack, __in __in_z #endif // FEATURE_STACK_FP_X87 } + // If we're printing a node for LIR, we use the space normally associated with the message + // to display the node's temp name (if any) + const bool hasOperands = tree->OperandsBegin() != tree->OperandsEnd(); + if (isLIR) + { + assert(msg == nullptr); + + // If the tree does not have any operands, we do not display the indent stack. This gives us + // two additional characters for alignment. + if (!hasOperands) + { + msgLength += 1; + } + + if (tree->IsValue()) + { + const size_t bufLength = msgLength - 1; + msg = reinterpret_cast<char*>(alloca(bufLength * sizeof(char))); + sprintf_s(const_cast<char*>(msg), bufLength, "t%d = %s", tree->gtTreeID, hasOperands ? "" : " "); + } + } + /* print the msg associated with the node */ if (msg == nullptr) @@ -9422,10 +9490,13 @@ void Compiler::gtDispNode(GenTreePtr tree, IndentStack* indentStack, __in __in_z msgLength = 0; } - printf(" %-*s", msgLength, msg); + printf(isLIR ? " %+*s" : " %-*s", msgLength, msg); /* Indent the node accordingly */ - printIndent(indentStack); + if (!isLIR || hasOperands) + { + printIndent(indentStack); + } gtDispNodeName(tree); @@ -9459,15 +9530,6 @@ void Compiler::gtDispNode(GenTreePtr tree, IndentStack* indentStack, __in __in_z if (tree->gtOper == GT_STMT) { - if (tree->gtFlags & GTF_STMT_TOP_LEVEL) - { - printf("(top level) "); - } - else - { - printf("(embedded) "); - } - if (opts.compDbgInfo) { IL_OFFSET endIL = tree->gtStmt.gtStmtLastILoffs; @@ -10174,6 +10236,18 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) printf(" %s", getRegName(tree->gtPhysReg.gtSrcReg, varTypeIsFloating(tree))); break; + case GT_IL_OFFSET: + printf(" IL offset: "); + if (tree->gtStmt.gtStmtILoffsx == BAD_IL_OFFSET) + { + printf("???"); + } + else + { + printf("%d", jitGetILoffs(tree->gtStmt.gtStmtILoffsx)); + } + break; + default: assert(!"don't know how to display tree leaf node"); } @@ -10224,7 +10298,8 @@ extern const char* const simdIntrinsicNames[] = { void Compiler::gtDispTree(GenTreePtr tree, IndentStack* indentStack, /* = nullptr */ __in __in_z __in_opt const char* msg, /* = nullptr */ - bool topOnly) /* = false */ + bool topOnly, /* = false */ + bool isLIR) /* = false */ { if (tree == nullptr) { @@ -10233,19 +10308,6 @@ void Compiler::gtDispTree(GenTreePtr tree, return; } - if (fgOrder == FGOrderLinear && !topOnly) - { - if (tree->gtOper == GT_STMT) - { - (void)gtDispLinearStmt(tree->AsStmt()); - } - else - { - gtDispLinearTree(nullptr, fgGetFirstNode(tree), tree, new (this, CMK_DebugOnly) IndentStack(this)); - } - return; - } - if (indentStack == nullptr) { indentStack = new (this, CMK_DebugOnly) IndentStack(this); @@ -10260,7 +10322,7 @@ void Compiler::gtDispTree(GenTreePtr tree, if (tree->gtOper >= GT_COUNT) { - gtDispNode(tree, indentStack, msg); + gtDispNode(tree, indentStack, msg, isLIR); printf("Bogus operator!"); return; } @@ -10269,7 +10331,7 @@ void Compiler::gtDispTree(GenTreePtr tree, if (tree->OperIsLeaf() || tree->OperIsLocalStore()) // local stores used to be leaves { - gtDispNode(tree, indentStack, msg); + gtDispNode(tree, indentStack, msg, isLIR); gtDispLeaf(tree, indentStack); gtDispVN(tree); printf("\n"); @@ -10314,21 +10376,24 @@ void Compiler::gtDispTree(GenTreePtr tree, if (tree->OperGet() == GT_PHI) { - gtDispNode(tree, indentStack, msg); + gtDispNode(tree, indentStack, msg, isLIR); gtDispVN(tree); printf("\n"); - if (tree->gtOp.gtOp1 != nullptr) + if (!topOnly) { - IndentInfo arcType = IIArcTop; - for (GenTreeArgList* args = tree->gtOp.gtOp1->AsArgList(); args != nullptr; args = args->Rest()) + if (tree->gtOp.gtOp1 != nullptr) { - if (args->Rest() == nullptr) + IndentInfo arcType = IIArcTop; + for (GenTreeArgList* args = tree->gtOp.gtOp1->AsArgList(); args != nullptr; args = args->Rest()) { - arcType = IIArcBottom; + if (args->Rest() == nullptr) + { + arcType = IIArcBottom; + } + gtDispChild(args->Current(), indentStack, arcType); + arcType = IIArc; } - gtDispChild(args->Current(), indentStack, arcType); - arcType = IIArc; } } return; @@ -10361,7 +10426,8 @@ void Compiler::gtDispTree(GenTreePtr tree, indentStack->Pop(); indentStack->Push(myArc); } - gtDispNode(tree, indentStack, msg); + + gtDispNode(tree, indentStack, msg, isLIR); // Propagate lowerArc to the lower children. if (indentStack->Depth() > 0) @@ -10516,7 +10582,7 @@ void Compiler::gtDispTree(GenTreePtr tree, indentStack->Pop(); indentStack->Push(myArc); } - gtDispNode(tree, indentStack, msg); + gtDispNode(tree, indentStack, msg, isLIR); // Propagate lowerArc to the lower children. if (indentStack->Depth() > 0) @@ -10719,8 +10785,8 @@ void Compiler::gtDispTree(GenTreePtr tree, // call - The call for which 'arg' is an argument // arg - The argument for which a message should be constructed // argNum - The ordinal number of the arg in the argument list -// listCount - When printing in Linear form this is the count for a multireg GT_LIST -// or -1 if we are not printing in Linear form +// listCount - When printing in LIR form this is the count for a multireg GT_LIST +// or -1 if we are not printing in LIR form // bufp - A pointer to the buffer into which the message is written // bufLength - The length of the buffer pointed to by bufp // @@ -10775,8 +10841,8 @@ void Compiler::gtGetArgMsg( // call - The call for which 'arg' is an argument // argx - The argument for which a message should be constructed // lateArgIndex - The ordinal number of the arg in the lastArg list -// listCount - When printing in Linear form this is the count for a multireg GT_LIST -// or -1 if we are not printing in Linear form +// listCount - When printing in LIR form this is the count for a multireg GT_LIST +// or -1 if we are not printing in LIR form // bufp - A pointer to the buffer into which the message is written // bufLength - The length of the buffer pointed to by bufp // @@ -10920,360 +10986,164 @@ void Compiler::gtDispTreeList(GenTreePtr tree, IndentStack* indentStack /* = nul } //------------------------------------------------------------------------ -// nextPrintable: Retrieves the next gtNode that can be dumped in linear order +// Compiler::gtDispRange: dumps a range of LIR. // // Arguments: -// next - The call for which 'arg' is an argument -// tree - the specification for the current level of indentation & arcs -// -// Return Value: -// The next node to be printed in linear order. +// range - the range of LIR to display. // -GenTree* nextPrintable(GenTree* next, GenTree* tree) +void Compiler::gtDispRange(LIR::ReadOnlyRange const& range) { - assert(next != nullptr); - assert(tree != nullptr); - - // Skip any nodes that are in the linear order, but that we don't actually visit - while (next != tree && (next->IsList() || next->IsArgPlaceHolderNode())) + for (GenTree* node : range) { - next = next->gtNext; + gtDispLIRNode(node); } - return next; } //------------------------------------------------------------------------ -// gtDispLinearTree: Dump a tree in linear order +// Compiler::gtDispTreeRange: dumps the LIR range that contains all of the +// nodes in the dataflow tree rooted at a given +// node. // // Arguments: -// curStmt - The current statement being dumped -// nextLinearNode - The next node to be printed -// tree - The current tree being traversed -// indentStack - the specification for the current level of indentation & arcs -// msg - a contextual method (i.e. from the parent) to print +// containingRange - the LIR range that contains the root node. +// tree - the root of the dataflow tree. // -// Return Value: -// None. +void Compiler::gtDispTreeRange(LIR::Range& containingRange, GenTree* tree) +{ + bool unused; + gtDispRange(containingRange.GetTreeRange(tree, &unused)); +} + +//------------------------------------------------------------------------ +// Compiler::gtDispLIRNode: dumps a single LIR node. // -// Assumptions: -// 'tree' must be a GT_LIST node +// Arguments: +// node - the LIR node to dump. // -// Notes: -// 'nextLinearNode' tracks the node we should be printing next. -// In general, we should encounter it as we traverse the tree. If not, we -// have an embedded statement, so that statement is then printed within -// the dump for this statement. - -GenTreePtr Compiler::gtDispLinearTree(GenTreeStmt* curStmt, - GenTreePtr nextLinearNode, - GenTreePtr tree, - IndentStack* indentStack, - __in __in_z __in_opt const char* msg /* = nullptr */) +void Compiler::gtDispLIRNode(GenTree* node) { - const int BufLength = 256; - char buf[BufLength]; - char* bufp = &buf[0]; + auto displayOperand = [](GenTree* operand, const char* message, IndentInfo operandArc, IndentStack& indentStack) + { + assert(operand != nullptr); + assert(message != nullptr); - // Determine what kind of arc to propagate - IndentInfo myArc = IINone; - if (indentStack->Depth() > 0) + // 49 spaces for alignment + printf("%-49s", ""); + + indentStack.Push(operandArc); + indentStack.print(); + indentStack.Pop(); + operandArc = IIArc; + + printf(" t%-5d %-6s %s\n", operand->gtTreeID, varTypeName(operand->TypeGet()), message); + + }; + + IndentStack indentStack(this); + + const int bufLength = 256; + char buf[bufLength]; + + const bool nodeIsCall = node->IsCall(); + + int numCallEarlyArgs = 0; + if (nodeIsCall) { - myArc = indentStack->Pop(); - if (myArc == IIArcBottom || myArc == IIArc) + GenTreeCall* call = node->AsCall(); + for (GenTreeArgList* args = call->gtCallArgs; args != nullptr; args = args->Rest()) { - indentStack->Push(IIArc); - } - else - { - assert(myArc == IIArcTop); - indentStack->Push(IINone); + if (!args->Current()->IsArgPlaceHolderNode() && args->Current()->IsValue()) + { + numCallEarlyArgs++; + } } } - // Visit children - unsigned childCount = tree->NumChildren(); - GenTreePtr deferChild = nullptr; - for (unsigned i = 0; i < childCount; i++) + // Visit operands + IndentInfo operandArc = IIArcTop; + int callArgNumber = 0; + const bool expandMultiRegArgs = false; + for (GenTree* operand : node->Operands(expandMultiRegArgs)) { - unsigned childIndex = i; - if (tree->OperIsBinary() && tree->IsReverseOp()) + if (operand->IsArgPlaceHolderNode() || !operand->IsValue()) { - childIndex = (i == 0) ? 1 : 0; - } - - GenTreePtr child = tree->GetChild(childIndex); - IndentInfo indentInfo = (i == 0) ? IIArcTop : IIArc; - - if (tree->OperGet() == GT_COLON && i == 1) - { - deferChild = child; + // Either of these situations may happen with calls. continue; } - unsigned listElemNum = 0; - const char* childMsg = nullptr; - if (tree->IsCall()) + if (nodeIsCall) { - if (child == tree->gtCall.gtCallObjp) + GenTreeCall* call = node->AsCall(); + if (operand == call->gtCallObjp) { - if (child->gtOper == GT_ASG) - { - sprintf_s(bufp, sizeof(buf), "this SETUP%c", 0); - } - else - { - sprintf_s(bufp, sizeof(buf), "this in %s%c", compRegVarName(REG_ARG_0), 0); - } - childMsg = bufp; - } - else if (child == tree->gtCall.gtCallAddr) - { - childMsg = "calli tgt"; - } - else if (child == tree->gtCall.gtControlExpr) - { - childMsg = "control expr"; - } - else if (child == tree->gtCall.gtCallCookie) - { - childMsg = "cookie"; + sprintf_s(buf, sizeof(buf), "this in %s", compRegVarName(REG_ARG_0)); + displayOperand(operand, buf, operandArc, indentStack); } - else if (child == tree->gtCall.gtCallArgs) + else if (operand == call->gtCallAddr) { - // List is handled below, but adjust listElemNum to account for "this" if necessary - if (tree->gtCall.gtCallObjp != nullptr) - { - listElemNum = 1; - } + displayOperand(operand, "calli tgt", operandArc, indentStack); } - else + else if (operand == call->gtControlExpr) { - // Late args list is handled below - assert(child == tree->gtCall.gtCallLateArgs); + displayOperand(operand, "control expr", operandArc, indentStack); } - } - - if (child->OperGet() == GT_LIST) - { - // For each list element - GenTreePtr nextList = nullptr; - if (child->gtOp.gtOp2 != nullptr && child->gtOp.gtOp2->gtOper != GT_LIST) + else if (operand == call->gtCallCookie) { - // special case for child of initblk and cpblk - // op1 is dst, op2 is src, and op2 must show up first - assert(tree->OperIsBlkOp()); - sprintf_s(bufp, sizeof(buf), "Source"); - indentStack->Push(indentInfo); - nextLinearNode = gtDispLinearTree(curStmt, nextLinearNode, child->gtOp.gtOp2, indentStack, bufp); - indentStack->Pop(); - - indentInfo = IIArc; - sprintf_s(bufp, sizeof(buf), "Destination"); - indentStack->Push(indentInfo); - nextLinearNode = gtDispLinearTree(curStmt, nextLinearNode, child->gtOp.gtOp1, indentStack, bufp); - indentStack->Pop(); + displayOperand(operand, "cookie", operandArc, indentStack); } else { - // normal null-terminated list case - - for (GenTreePtr list = child; list != nullptr; list = nextList) + int callLateArgNumber = callArgNumber - numCallEarlyArgs; + if (operand->OperGet() == GT_LIST) { - GenTreePtr listElem; - if (list->gtOper == GT_LIST) - { - nextList = list->gtGetOp2(); - listElem = list->gtGetOp1(); - } - else - { - // GT_LIST nodes (under initBlk, others?) can have a non-null op2 that's not a GT_LIST - nextList = nullptr; - listElem = list; - } - - // get child msg - if (tree->IsCall()) + int listIndex = 0; + for (GenTreeArgList* element = operand->AsArgList(); element != nullptr; element = element->Rest()) { - // If this is a call and the arg (listElem) is a GT_LIST (Unix LCL_FLD for passing a var in - // multiple registers) print the nodes of the nested list and continue to the next argument. - if (listElem->gtOper == GT_LIST) - { - int listCount = 0; - GenTreePtr nextListNested = nullptr; - for (GenTreePtr listNested = listElem; listNested != nullptr; listNested = nextListNested) - { - GenTreePtr listElemNested; - if (listNested->gtOper == GT_LIST) - { - nextListNested = listNested->MoveNext(); - listElemNested = listNested->Current(); - } - else - { - // GT_LIST nodes (under initBlk, others?) can have a non-null op2 that's not a - // GT_LIST - nextListNested = nullptr; - listElemNested = listNested; - } - - indentStack->Push(indentInfo); - if (child == tree->gtCall.gtCallArgs) - { - gtGetArgMsg(tree, listNested, listElemNum, listCount, bufp, BufLength); - } - else - { - assert(child == tree->gtCall.gtCallLateArgs); - gtGetLateArgMsg(tree, listNested, listElemNum, listCount, bufp, BufLength); - } - listCount++; - nextLinearNode = - gtDispLinearTree(curStmt, nextLinearNode, listElemNested, indentStack, bufp); - indentStack->Pop(); - } - - // Skip the GT_LIST nodes, as we do not print them, and the next node to print will occur - // after the list. - while (nextLinearNode->OperGet() == GT_LIST) - { - nextLinearNode = nextLinearNode->gtNext; - } - - listElemNum++; - continue; - } - - if (child == tree->gtCall.gtCallArgs) + operand = element->Current(); + if (callLateArgNumber < 0) { - gtGetArgMsg(tree, listElem, listElemNum, -1, bufp, BufLength); + gtGetArgMsg(call, operand, callArgNumber, listIndex, buf, sizeof(buf)); } else { - assert(child == tree->gtCall.gtCallLateArgs); - gtGetLateArgMsg(tree, listElem, listElemNum, -1, bufp, BufLength); + gtGetLateArgMsg(call, operand, callLateArgNumber, listIndex, buf, sizeof(buf)); } + + displayOperand(operand, buf, operandArc, indentStack); + operandArc = IIArc; + } + } + else + { + if (callLateArgNumber < 0) + { + gtGetArgMsg(call, operand, callArgNumber, -1, buf, sizeof(buf)); } else { - sprintf_s(bufp, sizeof(buf), "List Item %d", listElemNum); + gtGetLateArgMsg(call, operand, callLateArgNumber, -1, buf, sizeof(buf)); } - indentStack->Push(indentInfo); - nextLinearNode = gtDispLinearTree(curStmt, nextLinearNode, listElem, indentStack, bufp); - indentStack->Pop(); - indentInfo = IIArc; - listElemNum++; + displayOperand(operand, buf, operandArc, indentStack); } - } - // Skip the GT_LIST nodes, as we do not print them, and the next node to print will occur - // after the list. - while (nextLinearNode->OperGet() == GT_LIST) - { - nextLinearNode = nextLinearNode->gtNext; + callArgNumber++; } } else { - indentStack->Push(indentInfo); - nextLinearNode = gtDispLinearTree(curStmt, nextLinearNode, child, indentStack, childMsg); - indentStack->Pop(); + displayOperand(operand, "", operandArc, indentStack); } - } - // This sometimes gets called before nodes have been properly sequenced. - // TODO-Cleanup: Determine whether this needs to be hardened in some way. - if (nextLinearNode == nullptr) - { - printf("BROKEN LINEAR ORDER\n"); - nextLinearNode = tree; - } - - // If we don't have a 'curStmt', we're only printing the local tree, so skip - // any embedded statements - if (curStmt != nullptr) - { - while (nextLinearNode != tree) - { - // Get the next statement, which had better be embedded - GenTreePtr nextStmt = curStmt->gtNext; - while (nextStmt != nullptr && nextStmt->gtStmt.gtStmtIsEmbedded() && - nextStmt->gtStmt.gtStmtList != nextLinearNode) - { - nextStmt = nextStmt->gtNext; - } - if (nextStmt != nullptr && nextStmt->gtStmt.gtStmtList == nextLinearNode) - { - indentStack->Push(IIEmbedded); - nextLinearNode = gtDispLinearStmt(nextStmt->AsStmt(), indentStack); - indentStack->Pop(); - } - else if (nextLinearNode != nullptr) - { - // If we have an inconsistency, attempt to print the rest of the broken tree, but don't assert, - // since we don't really want to have different asserts when dumping. - // The method should fail later with an assert in fgDebugCheckNodeLinks() the next time it's called. - // Print the next node in linear order, and eventually we will reach the end of the statement, - // or sync up to 'tree' - IndentInfo saveInfo = indentStack->Pop(); - indentStack->Push(IIError); - gtDispTree(nextLinearNode, indentStack, msg, true /*topOnly*/); - nextLinearNode = nextLinearNode->gtNext; - indentStack->Pop(); - indentStack->Push(saveInfo); - } - else - { - break; - } - } + operandArc = IIArc; } - // Now, get the right type of arc for this node - if (myArc != IINone) - { - indentStack->Pop(); - indentStack->Push(myArc); - } - gtDispTree(tree, indentStack, msg, true /*topOnly*/); - nextLinearNode = tree->gtNext; - - if (deferChild != nullptr) - { - indentStack->Push(IIArcBottom); - nextLinearNode = gtDispLinearTree(curStmt, nextLinearNode, deferChild, indentStack); - indentStack->Pop(); - } + // Visit the operator + const bool topOnly = true; + const bool isLIR = true; + gtDispTree(node, &indentStack, nullptr, topOnly, isLIR); - return nextLinearNode; -} - -//------------------------------------------------------------------------ -// gtDispLinearStmt: Dump a statement in linear order -// -// Arguments: -// stmt - The current statement being dumped -// indentStack - the specification for the current level of indentation & arcs -// -// Return Value: -// A pointer to the tree that is next in the linear traversal. -// This will generally be null, except when this statement is embedded. -// -// Assumptions: -// 'stmt' must be a GT_STMT node - -GenTreePtr Compiler::gtDispLinearStmt(GenTreeStmt* stmt, IndentStack* indentStack /* = nullptr */) -{ - if (indentStack == nullptr) - { - indentStack = new (this, CMK_DebugOnly) IndentStack(this); - } - gtDispTree(stmt, indentStack, nullptr, true /*topOnly*/); - indentStack->Push(IIArcBottom); - GenTreePtr nextLinearNode = gtDispLinearTree(stmt, stmt->gtStmtList, stmt->gtStmtExpr, indentStack); - indentStack->Pop(); - return nextLinearNode; + printf("\n"); } /*****************************************************************************/ @@ -13303,9 +13173,9 @@ GenTreePtr Compiler::gtNewTempAssign(unsigned tmp, GenTreePtr val) } #ifndef LEGACY_BACKEND - if (fgOrder == FGOrderLinear) + if (compRationalIRForm) { - Rationalizer::MorphAsgIntoStoreLcl(nullptr, asg); + Rationalizer::RewriteAssignmentIntoStoreLcl(asg->AsOp()); } #endif // !LEGACY_BACKEND @@ -14210,6 +14080,13 @@ BasicBlock* Compiler::bbNewBasicBlock(BBjumpKinds jumpKind) block->bbNum = ++fgBBNumMax; } +#ifndef LEGACY_BACKEND + if (compRationalIRForm) + { + block->bbFlags |= BBF_IS_LIR; + } +#endif // !LEGACY_BACKEND + block->bbRefs = 1; block->bbWeight = BB_UNITY_WEIGHT; @@ -14529,10 +14406,16 @@ bool GenTree::IsRegOptional() const #endif } +bool GenTree::IsPhiNode() +{ + return (OperGet() == GT_PHI_ARG) || (OperGet() == GT_PHI) || IsPhiDefn(); +} + bool GenTree::IsPhiDefn() { - bool res = OperGet() == GT_ASG && gtOp.gtOp2 != nullptr && gtOp.gtOp2->OperGet() == GT_PHI; - assert(!res || gtOp.gtOp1->OperGet() == GT_LCL_VAR); + bool res = ((OperGet() == GT_ASG) && (gtOp.gtOp2 != nullptr) && (gtOp.gtOp2->OperGet() == GT_PHI)) || + ((OperGet() == GT_STORE_LCL_VAR) && (gtOp.gtOp1 != nullptr) && (gtOp.gtOp1->OperGet() == GT_PHI)); + assert(!res || OperGet() == GT_STORE_LCL_VAR || gtOp.gtOp1->OperGet() == GT_LCL_VAR); return res; } @@ -14950,6 +14833,7 @@ bool GenTree::isContained() const case GT_LCLHEAP: case GT_CKFINITE: case GT_JMP: + case GT_IL_OFFSET: #ifdef FEATURE_SIMD case GT_SIMD_CHK: #endif // FEATURE_SIMD diff --git a/src/jit/gentree.h b/src/jit/gentree.h index 4892d1c95d..cb77fa887d 100644 --- a/src/jit/gentree.h +++ b/src/jit/gentree.h @@ -117,6 +117,9 @@ enum genTreeKinds GTK_LOCAL = 0x0200, // is a local access (load, store, phi) + GTK_NOVALUE = 0x0400, // node does not produce a value + GTK_NOTLIR = 0x0800, // node is not allowed in LIR + /* Define composite value(s) */ GTK_SMPOP = (GTK_UNOP | GTK_BINOP | GTK_RELOP | GTK_LOGOP) @@ -235,6 +238,7 @@ public: } }; +class GenTreeUseEdgeIterator; class GenTreeOperandIterator; /*****************************************************************************/ @@ -387,6 +391,8 @@ struct GenTree #endif // FEATURE_ANYCSE + unsigned char gtLIRFlags; // Used for nodes that are in LIR. See LIR::Flags in lir.h for the various flags. + #if ASSERTION_PROP unsigned short gtAssertionNum; // 0 or Assertion table index // valid only for non-GT_STMT nodes @@ -936,12 +942,8 @@ public: //---------------------------------------------------------------- -#define GTF_STMT_CMPADD 0x80000000 // GT_STMT -- added by compiler -#define GTF_STMT_HAS_CSE 0x40000000 // GT_STMT -- CSE def or use was subsituted -#define GTF_STMT_TOP_LEVEL 0x20000000 // GT_STMT -- Top-level statement - - // true iff gtStmtList->gtPrev == nullptr - // True for all stmts when in FGOrderTree -#define GTF_STMT_SKIP_LOWER 0x10000000 // GT_STMT -- Skip lowering if we already lowered an embedded stmt. +#define GTF_STMT_CMPADD 0x80000000 // GT_STMT -- added by compiler +#define GTF_STMT_HAS_CSE 0x40000000 // GT_STMT -- CSE def or use was subsituted //---------------------------------------------------------------- @@ -992,6 +994,55 @@ public: return opKind & ~GTK_EXOP; } + bool IsValue() const + { + if ((OperKind(gtOper) & GTK_NOVALUE) != 0) + { + return false; + } + + if (gtOper == GT_NOP || gtOper == GT_CALL) + { + return gtType != TYP_VOID; + } + + return true; + } + + bool IsLIR() const + { + if ((OperKind(gtOper) & GTK_NOTLIR) != 0) + { + return false; + } + + switch (gtOper) + { + case GT_NOP: + // NOPs may only be present in LIR if they do not produce a value. + return IsNothingNode(); + + case GT_ARGPLACE: + case GT_LIST: + // ARGPLACE and LIST nodes may not be present in a block's LIR sequence, but they may + // be present as children of an LIR node. + return (gtNext == nullptr) && (gtPrev == nullptr); + + case GT_ADDR: + { + // ADDR ndoes may only be present in LIR if the location they refer to is not a + // local, class variable, or IND node. + GenTree* location = const_cast<GenTree*>(this)->gtGetOp1(); + genTreeOps locationOp = location->OperGet(); + return !location->IsLocal() && (locationOp != GT_CLS_VAR) && (locationOp != GT_IND); + } + + default: + // All other nodes are assumed to be correct. + return true; + } + } + static bool OperIsConst(genTreeOps gtOper) { return (OperKind(gtOper) & GTK_CONST) != 0; @@ -1096,6 +1147,16 @@ public: return gtOper == GT_PUTARG_STK; } + bool OperIsPutArgReg() const + { + return gtOper == GT_PUTARG_REG; + } + + bool OperIsPutArg() const + { + return OperIsPutArgStk() || OperIsPutArgReg(); + } + bool OperIsAddrMode() const { return OperIsAddrMode(OperGet()); @@ -1421,6 +1482,10 @@ public: // can be modified; otherwise, return null. GenTreePtr* gtGetChildPointer(GenTreePtr parent); + // Given a tree node, if this node uses that node, return the use as an out parameter and return true. + // Otherwise, return false. + bool TryGetUse(GenTree* def, GenTree*** use, bool expandMultiRegArgs = true); + // Get the parent of this node, and optionally capture the pointer to the child so that it can be modified. GenTreePtr gtGetParent(GenTreePtr** parentChildPtrPtr); @@ -1449,9 +1514,6 @@ public: // "*addr" to the other argument. bool IsAddWithI32Const(GenTreePtr* addr, int* offset); - // Insert 'node' after this node in execution order. - void InsertAfterSelf(GenTree* node, GenTreeStmt* stmt = nullptr); - public: #if SMALL_TREE_NODES static unsigned char s_gtNodeSizes[]; @@ -1710,6 +1772,9 @@ public: // register. bool IsRegOptional() const; + // Returns "true" iff "this" is a phi-related node (i.e. a GT_PHI_ARG, GT_PHI, or a PhiDefn). + bool IsPhiNode(); + // Returns "true" iff "*this" is an assignment (GT_ASG) tree that defines an SSA name (lcl = phi(...)); bool IsPhiDefn(); @@ -1740,15 +1805,26 @@ public: // Requires "childNum < NumChildren()". Returns the "n"th child of "this." GenTreePtr GetChild(unsigned childNum); + // Returns an iterator that will produce the use edge to each operand of this node. Differs + // from the sequence of nodes produced by a loop over `GetChild` in its handling of call, phi, + // and block op nodes. If `expandMultiRegArgs` is true, an multi-reg args passed to a call + // will appear be expanded from their GT_LIST node into that node's contents. + GenTreeUseEdgeIterator GenTree::UseEdgesBegin(bool expandMultiRegArgs = true); + GenTreeUseEdgeIterator GenTree::UseEdgesEnd(); + + IteratorPair<GenTreeUseEdgeIterator> GenTree::UseEdges(bool expandMultiRegArgs = true); + // Returns an iterator that will produce each operand of this node. Differs from the sequence // of nodes produced by a loop over `GetChild` in its handling of call, phi, and block op // nodes. If `expandMultiRegArgs` is true, an multi-reg args passed to a call will appear // be expanded from their GT_LIST node into that node's contents. - GenTreeOperandIterator OperandsBegin(bool expandMultiRegArgs = false); + GenTreeOperandIterator OperandsBegin(bool expandMultiRegArgs = true); GenTreeOperandIterator OperandsEnd(); // Returns a range that will produce the operands of this node in use order. - IteratorPair<GenTreeOperandIterator> Operands(bool expandMultiRegArgs = false); + IteratorPair<GenTreeOperandIterator> Operands(bool expandMultiRegArgs = true); + + bool Precedes(GenTree* other); // The maximum possible # of children of any node. static const int MAX_CHILDREN = 6; @@ -1774,6 +1850,7 @@ public: } #ifdef DEBUG + private: GenTree& operator=(const GenTree& gt) { @@ -1804,70 +1881,124 @@ public: }; //------------------------------------------------------------------------ -// GenTreeOperandIterator: an iterator that will produce each operand of a +// GenTreeUseEdgeIterator: an iterator that will produce each use edge of a // GenTree node in the order in which they are -// used. Note that the operands of a node may not +// used. Note that the use edges of a node may not // correspond exactly to the nodes on the other // ends of its use edges: in particular, GT_LIST // nodes are expanded into their component parts // (with the optional exception of multi-reg // arguments). This differs from the behavior of -// GenTree::GetChild(), which does not expand +// GenTree::GetChildPointer(), which does not expand // lists. // // Note: valid values of this type may be obtained by calling -// `GenTree::OperandsBegin` and `GenTree::OperandsEnd`. -class GenTreeOperandIterator +// `GenTree::UseEdgesBegin` and `GenTree::UseEdgesEnd`. +// +class GenTreeUseEdgeIterator final { - friend GenTreeOperandIterator GenTree::OperandsBegin(bool expandMultiRegArgs); - friend GenTreeOperandIterator GenTree::OperandsEnd(); - - GenTree* m_node; - GenTree* m_operand; - GenTree* m_argList; - GenTree* m_multiRegArg; - bool m_expandMultiRegArgs; - int m_state; - - GenTreeOperandIterator(GenTree* node, bool expandMultiRegArgs); - - GenTree* GetNextOperand() const; - void MoveToNextCallOperand(); - void MoveToNextPhiOperand(); + friend class GenTreeOperandIterator; + friend GenTreeUseEdgeIterator GenTree::UseEdgesBegin(bool expandMultiRegArgs); + friend GenTreeUseEdgeIterator GenTree::UseEdgesEnd(); + + GenTree* m_node; + GenTree** m_edge; + GenTree* m_argList; + GenTree* m_multiRegArg; + bool m_expandMultiRegArgs; + int m_state; + + GenTreeUseEdgeIterator(GenTree* node, bool expandMultiRegArgs); + + GenTree** GetNextUseEdge() const; + void MoveToNextCallUseEdge(); + void MoveToNextPhiUseEdge(); #ifdef FEATURE_SIMD - void MoveToNextSIMDOperand(); + void MoveToNextSIMDUseEdge(); #endif public: - GenTreeOperandIterator(); + GenTreeUseEdgeIterator(); - inline GenTree*& operator*() + inline GenTree** operator*() { - return m_operand; + return m_edge; } inline GenTree** operator->() { - return &m_operand; + return m_edge; } - inline bool operator==(const GenTreeOperandIterator& other) const + inline bool operator==(const GenTreeUseEdgeIterator& other) const { if (m_state == -1 || other.m_state == -1) { return m_state == other.m_state; } - return (m_node == other.m_node) && (m_operand == other.m_operand) && (m_argList == other.m_argList) && + return (m_node == other.m_node) && (m_edge == other.m_edge) && (m_argList == other.m_argList) && (m_state == other.m_state); } + inline bool operator!=(const GenTreeUseEdgeIterator& other) const + { + return !(operator==(other)); + } + + GenTreeUseEdgeIterator& operator++(); +}; + +//------------------------------------------------------------------------ +// GenTreeOperandIterator: an iterator that will produce each operand of a +// GenTree node in the order in which they are +// used. This uses `GenTreeUseEdgeIterator` under +// the covers and comes with the same caveats +// w.r.t. `GetChild`. +// +// Note: valid values of this type may be obtained by calling +// `GenTree::OperandsBegin` and `GenTree::OperandsEnd`. +class GenTreeOperandIterator final +{ + friend GenTreeOperandIterator GenTree::OperandsBegin(bool expandMultiRegArgs); + friend GenTreeOperandIterator GenTree::OperandsEnd(); + + GenTreeUseEdgeIterator m_useEdges; + + GenTreeOperandIterator(GenTree* node, bool expandMultiRegArgs) : m_useEdges(node, expandMultiRegArgs) + { + } + +public: + GenTreeOperandIterator() : m_useEdges() + { + } + + inline GenTree* operator*() + { + return *(*m_useEdges); + } + + inline GenTree* operator->() + { + return *(*m_useEdges); + } + + inline bool operator==(const GenTreeOperandIterator& other) const + { + return m_useEdges == other.m_useEdges; + } + inline bool operator!=(const GenTreeOperandIterator& other) const { return !(operator==(other)); } - GenTreeOperandIterator& operator++(); + inline GenTreeOperandIterator& operator++() + { + ++m_useEdges; + return *this; + } }; /*****************************************************************************/ @@ -4032,40 +4163,6 @@ struct GenTreeStmt : public GenTree IL_OFFSET gtStmtLastILoffs; // instr offset at end of stmt #endif - bool gtStmtIsTopLevel() - { - return (gtFlags & GTF_STMT_TOP_LEVEL) != 0; - } - - bool gtStmtIsEmbedded() - { - return !gtStmtIsTopLevel(); - } - - // Return the next statement, if it is embedded, otherwise nullptr - GenTreeStmt* gtStmtNextIfEmbedded() - { - GenTree* nextStmt = gtNext; - if (nextStmt != nullptr && nextStmt->gtStmt.gtStmtIsEmbedded()) - { - return nextStmt->AsStmt(); - } - else - { - return nullptr; - } - } - - GenTree* gtStmtNextTopLevelStmt() - { - GenTree* nextStmt = gtNext; - while (nextStmt != nullptr && nextStmt->gtStmt.gtStmtIsEmbedded()) - { - nextStmt = nextStmt->gtNext; - } - return nextStmt; - } - __declspec(property(get = getNextStmt)) GenTreeStmt* gtNextStmt; __declspec(property(get = getPrevStmt)) GenTreeStmt* gtPrevStmt; @@ -4109,8 +4206,6 @@ struct GenTreeStmt : public GenTree // Statements can't have statements as part of their expression tree. assert(expr->gtOper != GT_STMT); - gtFlags |= GTF_STMT_TOP_LEVEL; - // Set the statement to have the same costs as the top node of the tree. // This is used long before costs have been assigned, so we need to copy // the raw costs. diff --git a/src/jit/gtlist.h b/src/jit/gtlist.h index 97db5b3c0e..59e59b972e 100644 --- a/src/jit/gtlist.h +++ b/src/jit/gtlist.h @@ -20,16 +20,16 @@ GTNODE(NONE , "<none>" ,0,GTK_SPECIAL) // Leaf nodes (i.e. these nodes have no sub-operands): //----------------------------------------------------------------------------- -GTNODE(LCL_VAR , "lclVar" ,0,GTK_LEAF|GTK_LOCAL) // local variable -GTNODE(LCL_FLD , "lclFld" ,0,GTK_LEAF|GTK_LOCAL) // field in a non-primitive variable -GTNODE(LCL_VAR_ADDR , "&lclVar" ,0,GTK_LEAF) // address of local variable -GTNODE(LCL_FLD_ADDR , "&lclFld" ,0,GTK_LEAF) // address of field in a non-primitive variable -GTNODE(STORE_LCL_VAR , "st.lclVar" ,0,GTK_UNOP|GTK_LOCAL) // store to local variable -GTNODE(STORE_LCL_FLD , "st.lclFld" ,0,GTK_UNOP|GTK_LOCAL) // store to field in a non-primitive variable -GTNODE(CATCH_ARG , "catchArg" ,0,GTK_LEAF) // Exception object in a catch block -GTNODE(LABEL , "codeLabel" ,0,GTK_LEAF) // Jump-target -GTNODE(FTN_ADDR , "ftnAddr" ,0,GTK_LEAF) // Address of a function -GTNODE(RET_EXPR , "retExpr" ,0,GTK_LEAF) // Place holder for the return expression from an inline candidate +GTNODE(LCL_VAR , "lclVar" ,0,GTK_LEAF|GTK_LOCAL) // local variable +GTNODE(LCL_FLD , "lclFld" ,0,GTK_LEAF|GTK_LOCAL) // field in a non-primitive variable +GTNODE(LCL_VAR_ADDR , "&lclVar" ,0,GTK_LEAF) // address of local variable +GTNODE(LCL_FLD_ADDR , "&lclFld" ,0,GTK_LEAF) // address of field in a non-primitive variable +GTNODE(STORE_LCL_VAR , "st.lclVar" ,0,GTK_UNOP|GTK_LOCAL|GTK_NOVALUE) // store to local variable +GTNODE(STORE_LCL_FLD , "st.lclFld" ,0,GTK_UNOP|GTK_LOCAL|GTK_NOVALUE) // store to field in a non-primitive variable +GTNODE(CATCH_ARG , "catchArg" ,0,GTK_LEAF) // Exception object in a catch block +GTNODE(LABEL , "codeLabel" ,0,GTK_LEAF) // Jump-target +GTNODE(FTN_ADDR , "ftnAddr" ,0,GTK_LEAF) // Address of a function +GTNODE(RET_EXPR , "retExpr" ,0,GTK_LEAF) // Place holder for the return expression from an inline candidate //----------------------------------------------------------------------------- // Constant nodes: @@ -50,39 +50,40 @@ GTNODE(NEG , "unary -" ,0,GTK_UNOP) GTNODE(COPY , "copy" ,0,GTK_UNOP) // Copies a variable from its current location to a register that satisfies // code generation constraints. The child is the actual lclVar node. GTNODE(RELOAD , "reload" ,0,GTK_UNOP) -GTNODE(CHS , "flipsign" ,0,GTK_BINOP|GTK_ASGOP) // GT_CHS is actually unary -- op2 is ignored. - // Changing to unary presently causes problems, though -- take a little work to fix. +GTNODE(CHS , "flipsign" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) // GT_CHS is actually unary -- op2 is ignored. + // Changing to unary presently causes problems, though -- take a little work to fix. GTNODE(ARR_LENGTH , "arrLen" ,0,GTK_UNOP|GTK_EXOP) // array-length GTNODE(INTRINSIC , "intrinsic" ,0,GTK_BINOP|GTK_EXOP) // intrinsics -GTNODE(LOCKADD , "lockAdd" ,0,GTK_BINOP) +GTNODE(LOCKADD , "lockAdd" ,0,GTK_BINOP|GTK_NOVALUE) GTNODE(XADD , "XAdd" ,0,GTK_BINOP) GTNODE(XCHG , "Xchg" ,0,GTK_BINOP) GTNODE(CMPXCHG , "cmpxchg" ,0,GTK_SPECIAL) -GTNODE(MEMORYBARRIER , "memoryBarrier" ,0,GTK_LEAF) +GTNODE(MEMORYBARRIER , "memoryBarrier" ,0,GTK_LEAF|GTK_NOVALUE) -GTNODE(CAST , "cast" ,0,GTK_UNOP|GTK_EXOP) // conversion to another type -GTNODE(CKFINITE , "ckfinite" ,0,GTK_UNOP) // Check for NaN -GTNODE(LCLHEAP , "lclHeap" ,0,GTK_UNOP) // alloca() -GTNODE(JMP , "jump" ,0,GTK_LEAF) // Jump to another function +GTNODE(CAST , "cast" ,0,GTK_UNOP|GTK_EXOP) // conversion to another type +GTNODE(CKFINITE , "ckfinite" ,0,GTK_UNOP) // Check for NaN +GTNODE(LCLHEAP , "lclHeap" ,0,GTK_UNOP) // alloca() +GTNODE(JMP , "jump" ,0,GTK_LEAF|GTK_NOVALUE) // Jump to another function -GTNODE(ADDR , "addr" ,0,GTK_UNOP) // address of -GTNODE(IND , "indir" ,0,GTK_UNOP) // load indirection -GTNODE(STOREIND , "storeIndir" ,0,GTK_BINOP) // store indirection - // TODO-Cleanup: GT_ARR_BOUNDS_CHECK should be made a GTK_BINOP now that it has only two child nodes -GTNODE(ARR_BOUNDS_CHECK , "arrBndsChk" ,0,GTK_SPECIAL) // array bounds check +GTNODE(ADDR , "addr" ,0,GTK_UNOP) // address of +GTNODE(IND , "indir" ,0,GTK_UNOP) // load indirection +GTNODE(STOREIND , "storeIndir" ,0,GTK_BINOP|GTK_NOVALUE) // store indirection + + // TODO-Cleanup: GT_ARR_BOUNDS_CHECK should be made a GTK_BINOP now that it has only two child nodes +GTNODE(ARR_BOUNDS_CHECK , "arrBndsChk" ,0,GTK_SPECIAL|GTK_NOVALUE) // array bounds check GTNODE(OBJ , "obj" ,0,GTK_UNOP|GTK_EXOP) -GTNODE(BOX , "box" ,0,GTK_UNOP|GTK_EXOP) +GTNODE(BOX , "box" ,0,GTK_UNOP|GTK_EXOP|GTK_NOTLIR) #ifdef FEATURE_SIMD -GTNODE(SIMD_CHK , "simdChk" ,0,GTK_SPECIAL) // Compare whether an index is less than the given SIMD vector length, and call CORINFO_HELP_RNGCHKFAIL if not. - // TODO-CQ: In future may want to add a field that specifies different exceptions but we'll - // need VM assistance for that. - // TODO-CQ: It would actually be very nice to make this an unconditional throw, and expose the control flow that - // does the compare, so that it can be more easily optimized. But that involves generating qmarks at import time... +GTNODE(SIMD_CHK , "simdChk" ,0,GTK_SPECIAL|GTK_NOVALUE) // Compare whether an index is less than the given SIMD vector length, and call CORINFO_HELP_RNGCHKFAIL if not. + // TODO-CQ: In future may want to add a field that specifies different exceptions but we'll + // need VM assistance for that. + // TODO-CQ: It would actually be very nice to make this an unconditional throw, and expose the control flow that + // does the compare, so that it can be more easily optimized. But that involves generating qmarks at import time... #endif // FEATURE_SIMD GTNODE(ALLOCOBJ , "allocObj" ,0,GTK_UNOP|GTK_EXOP) // object allocator @@ -111,22 +112,22 @@ GTNODE(ROL , "rol" ,0,GTK_BINOP) GTNODE(ROR , "ror" ,0,GTK_BINOP) GTNODE(MULHI , "mulhi" ,1,GTK_BINOP) // returns high bits (top N bits of the 2N bit result of an NxN multiply) -GTNODE(ASG , "=" ,0,GTK_BINOP|GTK_ASGOP) -GTNODE(ASG_ADD , "+=" ,0,GTK_BINOP|GTK_ASGOP) -GTNODE(ASG_SUB , "-=" ,0,GTK_BINOP|GTK_ASGOP) -GTNODE(ASG_MUL , "*=" ,0,GTK_BINOP|GTK_ASGOP) -GTNODE(ASG_DIV , "/=" ,0,GTK_BINOP|GTK_ASGOP) -GTNODE(ASG_MOD , "%=" ,0,GTK_BINOP|GTK_ASGOP) +GTNODE(ASG , "=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) +GTNODE(ASG_ADD , "+=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) +GTNODE(ASG_SUB , "-=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) +GTNODE(ASG_MUL , "*=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) +GTNODE(ASG_DIV , "/=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) +GTNODE(ASG_MOD , "%=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) -GTNODE(ASG_UDIV , "/=" ,0,GTK_BINOP|GTK_ASGOP) -GTNODE(ASG_UMOD , "%=" ,0,GTK_BINOP|GTK_ASGOP) +GTNODE(ASG_UDIV , "/=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) +GTNODE(ASG_UMOD , "%=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) -GTNODE(ASG_OR , "|=" ,0,GTK_BINOP|GTK_ASGOP) -GTNODE(ASG_XOR , "^=" ,0,GTK_BINOP|GTK_ASGOP) -GTNODE(ASG_AND , "&=" ,0,GTK_BINOP|GTK_ASGOP) -GTNODE(ASG_LSH , "<<=" ,0,GTK_BINOP|GTK_ASGOP) -GTNODE(ASG_RSH , ">>=" ,0,GTK_BINOP|GTK_ASGOP) -GTNODE(ASG_RSZ , ">>>=" ,0,GTK_BINOP|GTK_ASGOP) +GTNODE(ASG_OR , "|=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) +GTNODE(ASG_XOR , "^=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) +GTNODE(ASG_AND , "&=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) +GTNODE(ASG_LSH , "<<=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) +GTNODE(ASG_RSH , ">>=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) +GTNODE(ASG_RSZ , ">>>=" ,0,GTK_BINOP|GTK_ASGOP|GTK_NOTLIR) GTNODE(EQ , "==" ,0,GTK_BINOP|GTK_RELOP) GTNODE(NE , "!=" ,0,GTK_BINOP|GTK_RELOP) @@ -135,12 +136,12 @@ GTNODE(LE , "<=" ,0,GTK_BINOP|GTK_RELOP) GTNODE(GE , ">=" ,0,GTK_BINOP|GTK_RELOP) GTNODE(GT , ">" ,0,GTK_BINOP|GTK_RELOP) -GTNODE(COMMA , "comma" ,0,GTK_BINOP) +GTNODE(COMMA , "comma" ,0,GTK_BINOP|GTK_NOTLIR) -GTNODE(QMARK , "qmark" ,0,GTK_BINOP|GTK_EXOP) -GTNODE(COLON , "colon" ,0,GTK_BINOP) +GTNODE(QMARK , "qmark" ,0,GTK_BINOP|GTK_EXOP|GTK_NOTLIR) +GTNODE(COLON , "colon" ,0,GTK_BINOP|GTK_NOTLIR) -GTNODE(INDEX , "[]" ,0,GTK_BINOP|GTK_EXOP) // SZ-array-element +GTNODE(INDEX , "[]" ,0,GTK_BINOP|GTK_EXOP|GTK_NOTLIR) // SZ-array-element GTNODE(MKREFANY , "mkrefany" ,0,GTK_BINOP) @@ -173,9 +174,9 @@ GTNODE(SIMD , "simd" ,0,GTK_BINOP|GTK_EXOP) // SIMD functions/oper // Other nodes that look like unary/binary operators: //----------------------------------------------------------------------------- -GTNODE(JTRUE , "jmpTrue" ,0,GTK_UNOP) +GTNODE(JTRUE , "jmpTrue" ,0,GTK_UNOP|GTK_NOVALUE) -GTNODE(LIST , "<list>" ,0,GTK_BINOP) +GTNODE(LIST , "<list>" ,0,GTK_BINOP|GTK_NOVALUE) //----------------------------------------------------------------------------- // Other nodes that have special structure: @@ -191,60 +192,61 @@ GTNODE(CALL , "call()" ,0,GTK_SPECIAL) // Statement operator nodes: //----------------------------------------------------------------------------- -GTNODE(BEG_STMTS , "begStmts" ,0,GTK_SPECIAL) // used only temporarily in importer by impBegin/EndTreeList() -GTNODE(STMT , "stmtExpr" ,0,GTK_SPECIAL) // top-level list nodes in bbTreeList +GTNODE(BEG_STMTS , "begStmts" ,0,GTK_SPECIAL|GTK_NOVALUE) // used only temporarily in importer by impBegin/EndTreeList() +GTNODE(STMT , "stmtExpr" ,0,GTK_SPECIAL|GTK_NOVALUE) // top-level list nodes in bbTreeList -GTNODE(RETURN , "return" ,0,GTK_UNOP) // return from current function -GTNODE(SWITCH , "switch" ,0,GTK_UNOP) // switch +GTNODE(RETURN , "return" ,0,GTK_UNOP|GTK_NOVALUE) // return from current function +GTNODE(SWITCH , "switch" ,0,GTK_UNOP|GTK_NOVALUE) // switch -GTNODE(NO_OP , "no_op" ,0,GTK_LEAF) // nop! +GTNODE(NO_OP , "no_op" ,0,GTK_LEAF|GTK_NOVALUE) // nop! -GTNODE(START_NONGC, "start_nongc",0,GTK_LEAF) // starts a new instruction group that will be non-gc interruptible +GTNODE(START_NONGC, "start_nongc",0,GTK_LEAF|GTK_NOVALUE) // starts a new instruction group that will be non-gc interruptible -GTNODE(PROF_HOOK , "prof_hook" ,0,GTK_LEAF) // profiler Enter/Leave/TailCall hook +GTNODE(PROF_HOOK , "prof_hook" ,0,GTK_LEAF|GTK_NOVALUE) // profiler Enter/Leave/TailCall hook -GTNODE(RETFILT , "retfilt", 0,GTK_UNOP) // end filter with TYP_I_IMPL return value +GTNODE(RETFILT , "retfilt", 0,GTK_UNOP|GTK_NOVALUE) // end filter with TYP_I_IMPL return value #if !FEATURE_EH_FUNCLETS -GTNODE(END_LFIN , "endLFin" ,0,GTK_LEAF) // end locally-invoked finally +GTNODE(END_LFIN , "endLFin" ,0,GTK_LEAF|GTK_NOVALUE) // end locally-invoked finally #endif // !FEATURE_EH_FUNCLETS -GTNODE(INITBLK , "initBlk" ,0,GTK_BINOP) -GTNODE(COPYBLK , "copyBlk" ,0,GTK_BINOP) -GTNODE(COPYOBJ , "copyObj" ,0,GTK_BINOP) +GTNODE(INITBLK , "initBlk" ,0,GTK_BINOP|GTK_NOVALUE) +GTNODE(COPYBLK , "copyBlk" ,0,GTK_BINOP|GTK_NOVALUE) +GTNODE(COPYOBJ , "copyObj" ,0,GTK_BINOP|GTK_NOVALUE) //----------------------------------------------------------------------------- // Nodes used for optimizations. //----------------------------------------------------------------------------- -GTNODE(PHI , "phi" ,0,GTK_UNOP) // phi node for ssa. -GTNODE(PHI_ARG , "phiArg" ,0,GTK_LEAF|GTK_LOCAL) // phi(phiarg, phiarg, phiarg) +GTNODE(PHI , "phi" ,0,GTK_UNOP) // phi node for ssa. +GTNODE(PHI_ARG , "phiArg" ,0,GTK_LEAF|GTK_LOCAL) // phi(phiarg, phiarg, phiarg) //----------------------------------------------------------------------------- // Nodes used by Lower to generate a closer CPU representation of other nodes //----------------------------------------------------------------------------- -GTNODE(JMPTABLE , "jumpTable" , 0, GTK_LEAF) // Generates the jump table for switches -GTNODE(SWITCH_TABLE, "tableSwitch", 0, GTK_BINOP) // Jump Table based switch construct +GTNODE(JMPTABLE , "jumpTable" , 0, GTK_LEAF) // Generates the jump table for switches +GTNODE(SWITCH_TABLE, "tableSwitch", 0, GTK_BINOP|GTK_NOVALUE) // Jump Table based switch construct //----------------------------------------------------------------------------- // Nodes used only within the code generator: //----------------------------------------------------------------------------- -GTNODE(REG_VAR , "regVar" ,0,GTK_LEAF|GTK_LOCAL) // register variable -GTNODE(CLS_VAR , "clsVar" ,0,GTK_LEAF) // static data member -GTNODE(CLS_VAR_ADDR , "&clsVar" ,0,GTK_LEAF) // static data member address -GTNODE(STORE_CLS_VAR, "st.clsVar" ,0,GTK_LEAF) // store to static data member -GTNODE(ARGPLACE , "argPlace" ,0,GTK_LEAF) // placeholder for a register arg -GTNODE(NULLCHECK , "nullcheck" ,0,GTK_UNOP) // null checks the source -GTNODE(PHYSREG , "physregSrc" ,0,GTK_LEAF) // read from a physical register -GTNODE(PHYSREGDST , "physregDst" ,0,GTK_UNOP) // write to a physical register -GTNODE(EMITNOP , "emitnop" ,0,GTK_LEAF) // emitter-placed nop -GTNODE(PINVOKE_PROLOG,"pinvoke_prolog",0,GTK_LEAF) // pinvoke prolog seq -GTNODE(PINVOKE_EPILOG,"pinvoke_epilog",0,GTK_LEAF) // pinvoke epilog seq -GTNODE(PUTARG_REG , "putarg_reg" ,0,GTK_UNOP) // operator that places outgoing arg in register -GTNODE(PUTARG_STK , "putarg_stk" ,0,GTK_UNOP) // operator that places outgoing arg in stack -GTNODE(RETURNTRAP , "returnTrap" ,0,GTK_UNOP) // a conditional call to wait on gc -GTNODE(SWAP , "swap" ,0,GTK_BINOP) // op1 and op2 swap (registers) +GTNODE(REG_VAR , "regVar" ,0,GTK_LEAF|GTK_LOCAL) // register variable +GTNODE(CLS_VAR , "clsVar" ,0,GTK_LEAF) // static data member +GTNODE(CLS_VAR_ADDR , "&clsVar" ,0,GTK_LEAF) // static data member address +GTNODE(STORE_CLS_VAR, "st.clsVar" ,0,GTK_LEAF|GTK_NOVALUE) // store to static data member +GTNODE(ARGPLACE , "argPlace" ,0,GTK_LEAF) // placeholder for a register arg +GTNODE(NULLCHECK , "nullcheck" ,0,GTK_UNOP|GTK_NOVALUE) // null checks the source +GTNODE(PHYSREG , "physregSrc" ,0,GTK_LEAF) // read from a physical register +GTNODE(PHYSREGDST , "physregDst" ,0,GTK_UNOP|GTK_NOVALUE) // write to a physical register +GTNODE(EMITNOP , "emitnop" ,0,GTK_LEAF|GTK_NOVALUE) // emitter-placed nop +GTNODE(PINVOKE_PROLOG,"pinvoke_prolog",0,GTK_LEAF|GTK_NOVALUE) // pinvoke prolog seq +GTNODE(PINVOKE_EPILOG,"pinvoke_epilog",0,GTK_LEAF|GTK_NOVALUE) // pinvoke epilog seq +GTNODE(PUTARG_REG , "putarg_reg" ,0,GTK_UNOP) // operator that places outgoing arg in register +GTNODE(PUTARG_STK , "putarg_stk" ,0,GTK_UNOP) // operator that places outgoing arg in stack +GTNODE(RETURNTRAP , "returnTrap" ,0,GTK_UNOP|GTK_NOVALUE) // a conditional call to wait on gc +GTNODE(SWAP , "swap" ,0,GTK_BINOP|GTK_NOVALUE) // op1 and op2 swap (registers) +GTNODE(IL_OFFSET , "il_offset" ,0,GTK_LEAF|GTK_NOVALUE) // marks an IL offset for debugging purposes /*****************************************************************************/ #undef GTNODE diff --git a/src/jit/gtstructs.h b/src/jit/gtstructs.h index ae7311ace5..43483804b6 100644 --- a/src/jit/gtstructs.h +++ b/src/jit/gtstructs.h @@ -80,7 +80,7 @@ GTSTRUCT_1(ArrElem , GT_ARR_ELEM) GTSTRUCT_1(ArrOffs , GT_ARR_OFFSET) GTSTRUCT_1(ArrIndex , GT_ARR_INDEX) GTSTRUCT_1(RetExpr , GT_RET_EXPR) -GTSTRUCT_1(Stmt , GT_STMT) +GTSTRUCT_2(Stmt , GT_STMT, GT_IL_OFFSET) GTSTRUCT_1(Obj , GT_OBJ) GTSTRUCT_2(CopyOrReload, GT_COPY, GT_RELOAD) GTSTRUCT_2(ClsVar , GT_CLS_VAR, GT_CLS_VAR_ADDR) diff --git a/src/jit/jit.h b/src/jit/jit.h index 205ccf03a6..7bf5cd4051 100644 --- a/src/jit/jit.h +++ b/src/jit/jit.h @@ -523,9 +523,15 @@ void JitDump(const char* pcFormat, ...); #define DISPNODE(t) \ if (JitTls::GetCompiler()->verbose) \ JitTls::GetCompiler()->gtDispTree(t, nullptr, nullptr, true); -#define DISPTREE(x) \ +#define DISPTREE(t) \ if (JitTls::GetCompiler()->verbose) \ - JitTls::GetCompiler()->gtDispTree(x) + JitTls::GetCompiler()->gtDispTree(t); +#define DISPRANGE(range) \ + if (JitTls::GetCompiler()->verbose) \ + JitTls::GetCompiler()->gtDispRange(range); +#define DISPTREERANGE(range, t) \ + if (JitTls::GetCompiler()->verbose) \ + JitTls::GetCompiler()->gtDispTreeRange(range, t); #define VERBOSE JitTls::GetCompiler()->verbose #else // !DEBUG #define JITDUMP(...) @@ -533,7 +539,9 @@ void JitDump(const char* pcFormat, ...); #define JITLOG_THIS(t, x) #define DBEXEC(flg, expr) #define DISPNODE(t) -#define DISPTREE(x) +#define DISPTREE(t) +#define DISPRANGE(range) +#define DISPTREERANGE(range, t) #define VERBOSE 0 #endif // !DEBUG diff --git a/src/jit/jit.settings.targets b/src/jit/jit.settings.targets index b8e109e761..27c7680ee5 100644 --- a/src/jit/jit.settings.targets +++ b/src/jit/jit.settings.targets @@ -55,6 +55,7 @@ <CppCompile Include="..\Instr.cpp" /> <CppCompile Include="..\JitTelemetry.cpp" /> <CppCompile Include="..\LclVars.cpp" /> + <CppCompile Include="..\LIR.cpp" /> <CppCompile Include="..\Liveness.cpp" /> <CppCompile Include="..\Morph.cpp" /> <CppCompile Include="..\Optimizer.cpp" /> diff --git a/src/jit/lclvars.cpp b/src/jit/lclvars.cpp index 71df266e83..369c96322d 100644 --- a/src/jit/lclvars.cpp +++ b/src/jit/lclvars.cpp @@ -2249,6 +2249,15 @@ void Compiler::lvaRecursiveIncRefCounts(GenTreePtr tree) */ void Compiler::lvaDecRefCnts(GenTreePtr tree) { + assert(compCurBB != nullptr); + lvaDecRefCnts(compCurBB, tree); +} + +void Compiler::lvaDecRefCnts(BasicBlock* block, GenTreePtr tree) +{ + assert(block != nullptr); + assert(tree != nullptr); + unsigned lclNum; LclVarDsc* varDsc; @@ -2268,8 +2277,8 @@ void Compiler::lvaDecRefCnts(GenTreePtr tree) /* Decrement the reference counts twice */ - varDsc->decRefCnts(compCurBB->getBBWeight(this), this); - varDsc->decRefCnts(compCurBB->getBBWeight(this), this); + varDsc->decRefCnts(block->getBBWeight(this), this); + varDsc->decRefCnts(block->getBBWeight(this), this); } } else @@ -2287,7 +2296,7 @@ void Compiler::lvaDecRefCnts(GenTreePtr tree) /* Decrement its lvRefCnt and lvRefCntWtd */ - varDsc->decRefCnts(compCurBB->getBBWeight(this), this); + varDsc->decRefCnts(block->getBBWeight(this), this); } } diff --git a/src/jit/lir.cpp b/src/jit/lir.cpp new file mode 100644 index 0000000000..1aab172a41 --- /dev/null +++ b/src/jit/lir.cpp @@ -0,0 +1,1594 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#include "jitpch.h" +#include "smallhash.h" + +#ifdef _MSC_VER +#pragma hdrstop +#endif + +LIR::Use::Use() : m_range(nullptr), m_edge(nullptr), m_user(nullptr) +{ +} + +LIR::Use::Use(const Use& other) +{ + *this = other; +} + +//------------------------------------------------------------------------ +// LIR::Use::Use: Constructs a use <-> def edge given the range that +// contains the use and the def, the use -> def edge, and +// the user. +// +// Arguments: +// range - The range that contains the use and the def. +// edge - The use -> def edge. +// user - The node that uses the def. +// +// Return Value: +// +LIR::Use::Use(Range& range, GenTree** edge, GenTree* user) : m_range(&range), m_edge(edge), m_user(user) +{ + AssertIsValid(); +} + +LIR::Use& LIR::Use::operator=(const Use& other) +{ + m_range = other.m_range; + m_user = other.m_user; + m_edge = other.IsDummyUse() ? &m_user : other.m_edge; + + assert(IsDummyUse() == other.IsDummyUse()); + return *this; +} + +LIR::Use& LIR::Use::operator=(Use&& other) +{ + *this = other; + return *this; +} + +//------------------------------------------------------------------------ +// LIR::Use::GetDummyUse: Returns a dummy use for a node. +// +// This method is provided as a convenience to allow transforms to work +// uniformly over Use values. It allows the creation of a Use given a node +// that is not used. +// +// Arguments: +// range - The range that contains the node. +// node - The node for which to create a dummy use. +// +// Return Value: +// +LIR::Use LIR::Use::GetDummyUse(Range& range, GenTree* node) +{ + assert(node != nullptr); + + Use dummyUse; + dummyUse.m_range = ⦥ + dummyUse.m_user = node; + dummyUse.m_edge = &dummyUse.m_user; + + assert(dummyUse.IsInitialized()); + return dummyUse; +} + +//------------------------------------------------------------------------ +// LIR::Use::IsDummyUse: Indicates whether or not a use is a dummy use. +// +// This method must be called before attempting to call the User() method +// below: for dummy uses, the user is the same node as the def. +// +// Return Value: true if this use is a dummy use; false otherwise. +// +bool LIR::Use::IsDummyUse() const +{ + return m_edge == &m_user; +} + +//------------------------------------------------------------------------ +// LIR::Use::Def: Returns the node that produces the def for this use. +// +GenTree* LIR::Use::Def() const +{ + assert(IsInitialized()); + + return *m_edge; +} + +//------------------------------------------------------------------------ +// LIR::Use::User: Returns the node that uses the def for this use. +/// +GenTree* LIR::Use::User() const +{ + assert(IsInitialized()); + assert(!IsDummyUse()); + + return m_user; +} + +//------------------------------------------------------------------------ +// LIR::Use::IsInitialized: Returns true if the use is minimally valid; false otherwise. +// +bool LIR::Use::IsInitialized() const +{ + return (m_range != nullptr) && (m_user != nullptr) && (m_edge != nullptr); +} + +//------------------------------------------------------------------------ +// LIR::Use::AssertIsValid: DEBUG function to assert on many validity conditions. +// +void LIR::Use::AssertIsValid() const +{ + assert(IsInitialized()); + assert(m_range->Contains(m_user)); + assert(Def() != nullptr); + + GenTree** useEdge = nullptr; + assert(m_user->TryGetUse(Def(), &useEdge)); + assert(useEdge == m_edge); +} + +//------------------------------------------------------------------------ +// LIR::Use::ReplaceWith: Changes the use to point to a new value. +// +// For example, given the following LIR: +// +// t15 = lclVar int arg1 +// t16 = lclVar int arg1 +// +// /--* t15 int +// +--* t16 int +// t17 = * == int +// +// /--* t17 int +// * jmpTrue void +// +// If we wanted to replace the use of t17 with a use of the constant "1", we +// might do the following (where `opEq` is a `Use` value that represents the +// use of t17): +// +// GenTree* constantOne = compiler->gtNewIconNode(1); +// range.InsertAfter(opEq.Def(), constantOne); +// opEq.ReplaceWith(compiler, constantOne); +// +// Which would produce something like the following LIR: +// +// t15 = lclVar int arg1 +// t16 = lclVar int arg1 +// +// /--* t15 int +// +--* t16 int +// t17 = * == int +// +// t18 = const int 1 +// +// /--* t18 int +// * jmpTrue void +// +// Elminating the now-dead compare and its operands using `LIR::Range::Remove` +// would then give us: +// +// t18 = const int 1 +// +// /--* t18 int +// * jmpTrue void +// +// Arguments: +// compiler - The Compiler context. +// replacement - The replacement node. +// +void LIR::Use::ReplaceWith(Compiler* compiler, GenTree* replacement) +{ + assert(IsInitialized()); + assert(compiler != nullptr); + assert(replacement != nullptr); + assert(IsDummyUse() || m_range->Contains(m_user)); + assert(m_range->Contains(replacement)); + + GenTree* replacedNode = *m_edge; + + *m_edge = replacement; + if (!IsDummyUse() && m_user->IsCall()) + { + compiler->fgFixupArgTabEntryPtr(m_user, replacedNode, replacement); + } +} + +//------------------------------------------------------------------------ +// LIR::Use::ReplaceWithLclVar: Assigns the def for this use to a local +// var and points the use to a use of that +// local var. If no local number is provided, +// creates a new local var. +// +// For example, given the following IR: +// +// t15 = lclVar int arg1 +// t16 = lclVar int arg1 +// +// /--* t15 int +// +--* t16 int +// t17 = * == int +// +// /--* t17 int +// * jmpTrue void +// +// If we wanted to replace the use of t17 with a use of a new local var +// that holds the value represented by t17, we might do the following +// (where `opEq` is a `Use` value that represents the use of t17): +// +// opEq.ReplaceUseWithLclVar(compiler, block->getBBWeight(compiler)); +// +// This would produce the following LIR: +// +// t15 = lclVar int arg1 +// t16 = lclVar int arg1 +// +// /--* t15 int +// +--* t16 int +// t17 = * == int +// +// /--* t17 int +// * st.lclVar int tmp0 +// +// t18 = lclVar int tmp0 +// +// /--* t18 int +// * jmpTrue void +// +// Arguments: +// compiler - The Compiler context. +// blockWeight - The weight of the basic block that contains the use. +// lclNum - The local to use for temporary storage. If BAD_VAR_NUM (the +// default) is provided, this method will create and use a new +// local var. +// +// Return Value: The number of the local var used for temporary storage. +// +unsigned LIR::Use::ReplaceWithLclVar(Compiler* compiler, unsigned blockWeight, unsigned lclNum) +{ + assert(IsInitialized()); + assert(compiler != nullptr); + assert(m_range->Contains(m_user)); + assert(m_range->Contains(*m_edge)); + + GenTree* node = *m_edge; + + if (lclNum == BAD_VAR_NUM) + { + lclNum = compiler->lvaGrabTemp(true DEBUGARG("ReplaceWithLclVar is creating a new local variable")); + } + + // Increment its lvRefCnt and lvRefCntWtd twice, one for the def and one for the use + compiler->lvaTable[lclNum].incRefCnts(blockWeight, compiler); + compiler->lvaTable[lclNum].incRefCnts(blockWeight, compiler); + + GenTreeLclVar* store = compiler->gtNewTempAssign(lclNum, node)->AsLclVar(); + store->CopyCosts(node); + + GenTree* load = + new (compiler, GT_LCL_VAR) GenTreeLclVar(store->TypeGet(), store->AsLclVarCommon()->GetLclNum(), BAD_IL_OFFSET); + compiler->gtPrepareCost(load); + + m_range->InsertAfter(node, store, load); + + ReplaceWith(compiler, load); + + JITDUMP("ReplaceWithLclVar created store :\n"); + DISPNODE(store); + + return lclNum; +} + +LIR::ReadOnlyRange::ReadOnlyRange() : m_firstNode(nullptr), m_lastNode(nullptr) +{ +} + +LIR::ReadOnlyRange::ReadOnlyRange(ReadOnlyRange&& other) : m_firstNode(other.m_firstNode), m_lastNode(other.m_lastNode) +{ +#ifdef DEBUG + other.m_firstNode = nullptr; + other.m_lastNode = nullptr; +#endif +} + +//------------------------------------------------------------------------ +// LIR::ReadOnlyRange::ReadOnlyRange: +// Creates a `ReadOnlyRange` value given the first and last node in +// the range. +// +// Arguments: +// firstNode - The first node in the range. +// lastNode - The last node in the range. +// +LIR::ReadOnlyRange::ReadOnlyRange(GenTree* firstNode, GenTree* lastNode) : m_firstNode(firstNode), m_lastNode(lastNode) +{ + assert((m_firstNode == nullptr) == (m_lastNode == nullptr)); + assert((m_firstNode == m_lastNode) || (Contains(m_lastNode))); +} + +//------------------------------------------------------------------------ +// LIR::ReadOnlyRange::FirstNode: Returns the first node in the range. +// +GenTree* LIR::ReadOnlyRange::FirstNode() const +{ + return m_firstNode; +} + +//------------------------------------------------------------------------ +// LIR::ReadOnlyRange::LastNode: Returns the last node in the range. +// +GenTree* LIR::ReadOnlyRange::LastNode() const +{ + return m_lastNode; +} + +//------------------------------------------------------------------------ +// LIR::ReadOnlyRange::IsEmpty: Returns true if the range is empty; false +// otherwise. +// +bool LIR::ReadOnlyRange::IsEmpty() const +{ + assert((m_firstNode == nullptr) == (m_lastNode == nullptr)); + return m_firstNode == nullptr; +} + +//------------------------------------------------------------------------ +// LIR::ReadOnlyRange::begin: Returns an iterator positioned at the first +// node in the range. +// +LIR::ReadOnlyRange::Iterator LIR::ReadOnlyRange::begin() const +{ + return Iterator(m_firstNode); +} + +//------------------------------------------------------------------------ +// LIR::ReadOnlyRange::end: Returns an iterator positioned after the last +// node in the range. +// +LIR::ReadOnlyRange::Iterator LIR::ReadOnlyRange::end() const +{ + return Iterator(m_lastNode == nullptr ? nullptr : m_lastNode->gtNext); +} + +//------------------------------------------------------------------------ +// LIR::ReadOnlyRange::rbegin: Returns an iterator positioned at the last +// node in the range. +// +LIR::ReadOnlyRange::ReverseIterator LIR::ReadOnlyRange::rbegin() const +{ + return ReverseIterator(m_lastNode); +} + +//------------------------------------------------------------------------ +// LIR::ReadOnlyRange::rend: Returns an iterator positioned before the first +// node in the range. +// +LIR::ReadOnlyRange::ReverseIterator LIR::ReadOnlyRange::rend() const +{ + return ReverseIterator(m_firstNode == nullptr ? nullptr : m_firstNode->gtPrev); +} + +#ifdef DEBUG + +//------------------------------------------------------------------------ +// LIR::ReadOnlyRange::Contains: Indicates whether or not this range +// contains a given node. +// +// Arguments: +// node - The node to find. +// +// Return Value: True if this range contains the given node; false +// otherwise. +// +bool LIR::ReadOnlyRange::Contains(GenTree* node) const +{ + assert(node != nullptr); + + // TODO-LIR: derive this from the # of nodes in the function as well as + // the debug level. Checking small functions is pretty cheap; checking + // large functions is not. + if (JitConfig.JitExpensiveDebugCheckLevel() < 2) + { + return true; + } + + for (GenTree* n : *this) + { + if (n == node) + { + return true; + } + } + + return false; +} + +#endif + +LIR::Range::Range() : ReadOnlyRange() +{ +} + +LIR::Range::Range(Range&& other) : ReadOnlyRange(std::move(other)) +{ +} + +//------------------------------------------------------------------------ +// LIR::Range::Range: Creates a `Range` value given the first and last +// node in the range. +// +// Arguments: +// firstNode - The first node in the range. +// lastNode - The last node in the range. +// +LIR::Range::Range(GenTree* firstNode, GenTree* lastNode) : ReadOnlyRange(firstNode, lastNode) +{ +} + +//------------------------------------------------------------------------ +// LIR::Range::LastPhiNode: Returns the last phi node in the range or +// `nullptr` if no phis exist. +// +GenTree* LIR::Range::LastPhiNode() const +{ + GenTree* lastPhiNode = nullptr; + for (GenTree* node : *this) + { + if (!node->IsPhiNode()) + { + break; + } + + lastPhiNode = node; + } + + return lastPhiNode; +} + +//------------------------------------------------------------------------ +// LIR::Range::FirstNonPhiNode: Returns the first non-phi node in the +// range or `nullptr` if no non-phi nodes +// exist. +// +GenTree* LIR::Range::FirstNonPhiNode() const +{ + for (GenTree* node : *this) + { + if (!node->IsPhiNode()) + { + return node; + } + } + + return nullptr; +} + +//------------------------------------------------------------------------ +// LIR::Range::FirstNonPhiOrCatchArgNode: Returns the first node after all +// phi or catch arg nodes in this +// range. +// +GenTree* LIR::Range::FirstNonPhiOrCatchArgNode() const +{ + for (GenTree* node : NonPhiNodes()) + { + if (node->OperGet() == GT_CATCH_ARG) + { + continue; + } + else if ((node->OperGet() == GT_STORE_LCL_VAR) && (node->gtGetOp1()->OperGet() == GT_CATCH_ARG)) + { + continue; + } + + return node; + } + + return nullptr; +} + +//------------------------------------------------------------------------ +// LIR::Range::PhiNodes: Returns the range of phi nodes inside this range. +// +LIR::ReadOnlyRange LIR::Range::PhiNodes() const +{ + GenTree* lastPhiNode = LastPhiNode(); + if (lastPhiNode == nullptr) + { + return ReadOnlyRange(); + } + + return ReadOnlyRange(m_firstNode, lastPhiNode); +} + +//------------------------------------------------------------------------ +// LIR::Range::PhiNodes: Returns the range of non-phi nodes inside this +// range. +// +LIR::ReadOnlyRange LIR::Range::NonPhiNodes() const +{ + GenTree* firstNonPhiNode = FirstNonPhiNode(); + if (firstNonPhiNode == nullptr) + { + return ReadOnlyRange(); + } + + return ReadOnlyRange(firstNonPhiNode, m_lastNode); +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertBefore: Inserts a node before another node in this range. +// +// Arguments: +// insertionPoint - The node before which `node` will be inserted. If non-null, must be part +// of this range. If null, insert at the end of the range. +// node - The node to insert. Must not be part of any range. +// +void LIR::Range::InsertBefore(GenTree* insertionPoint, GenTree* node) +{ + assert(node != nullptr); + assert(node->gtPrev == nullptr); + assert(node->gtNext == nullptr); + + FinishInsertBefore(insertionPoint, node, node); +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertBefore: Inserts 2 nodes before another node in this range. +// +// Arguments: +// insertionPoint - The node before which the nodes will be inserted. If non-null, must be part +// of this range. If null, insert at the end of the range. +// node1 - The first node to insert. Must not be part of any range. +// node2 - The second node to insert. Must not be part of any range. +// +// Notes: +// Resulting order: +// previous insertionPoint->gtPrev <-> node1 <-> node2 <-> insertionPoint +// +void LIR::Range::InsertBefore(GenTree* insertionPoint, GenTree* node1, GenTree* node2) +{ + assert(node1 != nullptr); + assert(node2 != nullptr); + + assert(node1->gtNext == nullptr); + assert(node1->gtPrev == nullptr); + assert(node2->gtNext == nullptr); + assert(node2->gtPrev == nullptr); + + node1->gtNext = node2; + node2->gtPrev = node1; + + FinishInsertBefore(insertionPoint, node1, node2); +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertBefore: Inserts 3 nodes before another node in this range. +// +// Arguments: +// insertionPoint - The node before which the nodes will be inserted. If non-null, must be part +// of this range. If null, insert at the end of the range. +// node1 - The first node to insert. Must not be part of any range. +// node2 - The second node to insert. Must not be part of any range. +// node3 - The third node to insert. Must not be part of any range. +// +// Notes: +// Resulting order: +// previous insertionPoint->gtPrev <-> node1 <-> node2 <-> node3 <-> insertionPoint +// +void LIR::Range::InsertBefore(GenTree* insertionPoint, GenTree* node1, GenTree* node2, GenTree* node3) +{ + assert(node1 != nullptr); + assert(node2 != nullptr); + assert(node3 != nullptr); + + assert(node1->gtNext == nullptr); + assert(node1->gtPrev == nullptr); + assert(node2->gtNext == nullptr); + assert(node2->gtPrev == nullptr); + assert(node3->gtNext == nullptr); + assert(node3->gtPrev == nullptr); + + node1->gtNext = node2; + + node2->gtPrev = node1; + node2->gtNext = node3; + + node3->gtPrev = node2; + + FinishInsertBefore(insertionPoint, node1, node3); +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertBefore: Inserts 4 nodes before another node in this range. +// +// Arguments: +// insertionPoint - The node before which the nodes will be inserted. If non-null, must be part +// of this range. If null, insert at the end of the range. +// node1 - The first node to insert. Must not be part of any range. +// node2 - The second node to insert. Must not be part of any range. +// node3 - The third node to insert. Must not be part of any range. +// node4 - The fourth node to insert. Must not be part of any range. +// +// Notes: +// Resulting order: +// previous insertionPoint->gtPrev <-> node1 <-> node2 <-> node3 <-> node4 <-> insertionPoint +// +void LIR::Range::InsertBefore(GenTree* insertionPoint, GenTree* node1, GenTree* node2, GenTree* node3, GenTree* node4) +{ + assert(node1 != nullptr); + assert(node2 != nullptr); + assert(node3 != nullptr); + assert(node4 != nullptr); + + assert(node1->gtNext == nullptr); + assert(node1->gtPrev == nullptr); + assert(node2->gtNext == nullptr); + assert(node2->gtPrev == nullptr); + assert(node3->gtNext == nullptr); + assert(node3->gtPrev == nullptr); + assert(node4->gtNext == nullptr); + assert(node4->gtPrev == nullptr); + + node1->gtNext = node2; + + node2->gtPrev = node1; + node2->gtNext = node3; + + node3->gtPrev = node2; + node3->gtNext = node4; + + node4->gtPrev = node3; + + FinishInsertBefore(insertionPoint, node1, node4); +} + +//------------------------------------------------------------------------ +// LIR::Range::FinishInsertBefore: Helper function to finalize InsertBefore processing: link the +// range to insertionPoint. gtNext/gtPrev links between first and last are already set. +// +// Arguments: +// insertionPoint - The node before which the nodes will be inserted. If non-null, must be part +// of this range. If null, indicates to insert at the end of the range. +// first - The first node of the range to insert. +// last - The last node of the range to insert. +// +// Notes: +// Resulting order: +// previous insertionPoint->gtPrev <-> first <-> ... <-> last <-> insertionPoint +// +void LIR::Range::FinishInsertBefore(GenTree* insertionPoint, GenTree* first, GenTree* last) +{ + assert(first != nullptr); + assert(last != nullptr); + assert(first->gtPrev == nullptr); + assert(last->gtNext == nullptr); + + if (insertionPoint == nullptr) + { + if (m_firstNode == nullptr) + { + m_firstNode = first; + } + else + { + assert(m_lastNode != nullptr); + assert(m_lastNode->gtNext == nullptr); + m_lastNode->gtNext = first; + first->gtPrev = m_lastNode; + } + m_lastNode = last; + } + else + { + assert(Contains(insertionPoint)); + + first->gtPrev = insertionPoint->gtPrev; + if (first->gtPrev == nullptr) + { + assert(insertionPoint == m_firstNode); + m_firstNode = first; + } + else + { + first->gtPrev->gtNext = first; + } + + last->gtNext = insertionPoint; + insertionPoint->gtPrev = last; + } +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertAfter: Inserts a node after another node in this range. +// +// Arguments: +// insertionPoint - The node after which `node` will be inserted. If non-null, must be part +// of this range. If null, insert at the beginning of the range. +// node - The node to insert. Must not be part of any range. +// +// Notes: +// Resulting order: +// insertionPoint <-> node <-> previous insertionPoint->gtNext +// +void LIR::Range::InsertAfter(GenTree* insertionPoint, GenTree* node) +{ + assert(node != nullptr); + + assert(node->gtNext == nullptr); + assert(node->gtPrev == nullptr); + + FinishInsertAfter(insertionPoint, node, node); +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertAfter: Inserts 2 nodes after another node in this range. +// +// Arguments: +// insertionPoint - The node after which the nodes will be inserted. If non-null, must be part +// of this range. If null, insert at the beginning of the range. +// node1 - The first node to insert. Must not be part of any range. +// node2 - The second node to insert. Must not be part of any range. Inserted after node1. +// +// Notes: +// Resulting order: +// insertionPoint <-> node1 <-> node2 <-> previous insertionPoint->gtNext +// +void LIR::Range::InsertAfter(GenTree* insertionPoint, GenTree* node1, GenTree* node2) +{ + assert(node1 != nullptr); + assert(node2 != nullptr); + + assert(node1->gtNext == nullptr); + assert(node1->gtPrev == nullptr); + assert(node2->gtNext == nullptr); + assert(node2->gtPrev == nullptr); + + node1->gtNext = node2; + node2->gtPrev = node1; + + FinishInsertAfter(insertionPoint, node1, node2); +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertAfter: Inserts 3 nodes after another node in this range. +// +// Arguments: +// insertionPoint - The node after which the nodes will be inserted. If non-null, must be part +// of this range. If null, insert at the beginning of the range. +// node1 - The first node to insert. Must not be part of any range. +// node2 - The second node to insert. Must not be part of any range. Inserted after node1. +// node3 - The third node to insert. Must not be part of any range. Inserted after node2. +// +// Notes: +// Resulting order: +// insertionPoint <-> node1 <-> node2 <-> node3 <-> previous insertionPoint->gtNext +// +void LIR::Range::InsertAfter(GenTree* insertionPoint, GenTree* node1, GenTree* node2, GenTree* node3) +{ + assert(node1 != nullptr); + assert(node2 != nullptr); + assert(node3 != nullptr); + + assert(node1->gtNext == nullptr); + assert(node1->gtPrev == nullptr); + assert(node2->gtNext == nullptr); + assert(node2->gtPrev == nullptr); + assert(node3->gtNext == nullptr); + assert(node3->gtPrev == nullptr); + + node1->gtNext = node2; + + node2->gtPrev = node1; + node2->gtNext = node3; + + node3->gtPrev = node2; + + FinishInsertAfter(insertionPoint, node1, node3); +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertAfter: Inserts 4 nodes after another node in this range. +// +// Arguments: +// insertionPoint - The node after which the nodes will be inserted. If non-null, must be part +// of this range. If null, insert at the beginning of the range. +// node1 - The first node to insert. Must not be part of any range. +// node2 - The second node to insert. Must not be part of any range. Inserted after node1. +// node3 - The third node to insert. Must not be part of any range. Inserted after node2. +// node4 - The fourth node to insert. Must not be part of any range. Inserted after node3. +// +// Notes: +// Resulting order: +// insertionPoint <-> node1 <-> node2 <-> node3 <-> node4 <-> previous insertionPoint->gtNext +// +void LIR::Range::InsertAfter(GenTree* insertionPoint, GenTree* node1, GenTree* node2, GenTree* node3, GenTree* node4) +{ + assert(node1 != nullptr); + assert(node2 != nullptr); + assert(node3 != nullptr); + assert(node4 != nullptr); + + assert(node1->gtNext == nullptr); + assert(node1->gtPrev == nullptr); + assert(node2->gtNext == nullptr); + assert(node2->gtPrev == nullptr); + assert(node3->gtNext == nullptr); + assert(node3->gtPrev == nullptr); + assert(node4->gtNext == nullptr); + assert(node4->gtPrev == nullptr); + + node1->gtNext = node2; + + node2->gtPrev = node1; + node2->gtNext = node3; + + node3->gtPrev = node2; + node3->gtNext = node4; + + node4->gtPrev = node3; + + FinishInsertAfter(insertionPoint, node1, node4); +} + +//------------------------------------------------------------------------ +// LIR::Range::FinishInsertAfter: Helper function to finalize InsertAfter processing: link the +// range to insertionPoint. gtNext/gtPrev links between first and last are already set. +// +// Arguments: +// insertionPoint - The node after which the nodes will be inserted. If non-null, must be part +// of this range. If null, insert at the beginning of the range. +// first - The first node of the range to insert. +// last - The last node of the range to insert. +// +// Notes: +// Resulting order: +// insertionPoint <-> first <-> ... <-> last <-> previous insertionPoint->gtNext +// +void LIR::Range::FinishInsertAfter(GenTree* insertionPoint, GenTree* first, GenTree* last) +{ + assert(first != nullptr); + assert(last != nullptr); + assert(first->gtPrev == nullptr); + assert(last->gtNext == nullptr); + + if (insertionPoint == nullptr) + { + if (m_lastNode == nullptr) + { + m_lastNode = last; + } + else + { + assert(m_firstNode != nullptr); + assert(m_firstNode->gtPrev == nullptr); + m_firstNode->gtPrev = last; + last->gtNext = m_firstNode; + } + m_firstNode = first; + } + else + { + assert(Contains(insertionPoint)); + + last->gtNext = insertionPoint->gtNext; + if (last->gtNext == nullptr) + { + assert(insertionPoint == m_lastNode); + m_lastNode = last; + } + else + { + last->gtNext->gtPrev = last; + } + + first->gtPrev = insertionPoint; + insertionPoint->gtNext = first; + } +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertBefore: Inserts a range before another node in `this` range. +// +// Arguments: +// insertionPoint - The node before which the nodes will be inserted. If non-null, must be part +// of this range. If null, insert at the end of the range. +// range - The range to splice in. +// +void LIR::Range::InsertBefore(GenTree* insertionPoint, Range&& range) +{ + assert(!range.IsEmpty()); + FinishInsertBefore(insertionPoint, range.m_firstNode, range.m_lastNode); +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertAfter: Inserts a range after another node in `this` range. +// +// Arguments: +// insertionPoint - The node after which the nodes will be inserted. If non-null, must be part +// of this range. If null, insert at the beginning of the range. +// range - The range to splice in. +// +void LIR::Range::InsertAfter(GenTree* insertionPoint, Range&& range) +{ + assert(!range.IsEmpty()); + FinishInsertAfter(insertionPoint, range.m_firstNode, range.m_lastNode); +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertAtBeginning: Inserts a node at the beginning of this range. +// +// Arguments: +// node - The node to insert. Must not be part of any range. +// +void LIR::Range::InsertAtBeginning(GenTree* node) +{ + InsertBefore(m_firstNode, node); +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertAtEnd: Inserts a node at the end of this range. +// +// Arguments: +// node - The node to insert. Must not be part of any range. +// +void LIR::Range::InsertAtEnd(GenTree* node) +{ + InsertAfter(m_lastNode, node); +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertAtBeginning: Inserts a range at the beginning of `this` range. +// +// Arguments: +// range - The range to splice in. +// +void LIR::Range::InsertAtBeginning(Range&& range) +{ + InsertBefore(m_firstNode, std::move(range)); +} + +//------------------------------------------------------------------------ +// LIR::Range::InsertAtEnd: Inserts a range at the end of `this` range. +// +// Arguments: +// range - The range to splice in. +// +void LIR::Range::InsertAtEnd(Range&& range) +{ + InsertAfter(m_lastNode, std::move(range)); +} + +//------------------------------------------------------------------------ +// LIR::Range::Remove: Removes a node from this range. +// +// Arguments: +// node - The node to remove. Must be part of this range. +// +void LIR::Range::Remove(GenTree* node) +{ + assert(node != nullptr); + assert(Contains(node)); + + GenTree* prev = node->gtPrev; + GenTree* next = node->gtNext; + + if (prev != nullptr) + { + prev->gtNext = next; + } + else + { + assert(node == m_firstNode); + m_firstNode = next; + } + + if (next != nullptr) + { + next->gtPrev = prev; + } + else + { + assert(node == m_lastNode); + m_lastNode = prev; + } + + node->gtPrev = nullptr; + node->gtNext = nullptr; +} + +//------------------------------------------------------------------------ +// LIR::Range::Remove: Removes a subrange from this range. +// +// Both the start and the end of the subrange must be part of this range. +// +// Arguments: +// firstNode - The first node in the subrange. +// lastNode - The last node in the subrange. +// +// Returns: +// A mutable range containing the removed nodes. +// +LIR::Range LIR::Range::Remove(GenTree* firstNode, GenTree* lastNode) +{ + assert(firstNode != nullptr); + assert(lastNode != nullptr); + assert(Contains(firstNode)); + assert((firstNode == lastNode) || firstNode->Precedes(lastNode)); + + GenTree* prev = firstNode->gtPrev; + GenTree* next = lastNode->gtNext; + + if (prev != nullptr) + { + prev->gtNext = next; + } + else + { + assert(firstNode == m_firstNode); + m_firstNode = next; + } + + if (next != nullptr) + { + next->gtPrev = prev; + } + else + { + assert(lastNode == m_lastNode); + m_lastNode = prev; + } + + firstNode->gtPrev = nullptr; + lastNode->gtNext = nullptr; + + return Range(firstNode, lastNode); +} + +//------------------------------------------------------------------------ +// LIR::Range::Remove: Removes a subrange from this range. +// +// Arguments: +// range - The subrange to remove. Must be part of this range. +// +// Returns: +// A mutable range containing the removed nodes. +// +LIR::Range LIR::Range::Remove(ReadOnlyRange&& range) +{ + return Remove(range.m_firstNode, range.m_lastNode); +} + +//------------------------------------------------------------------------ +// LIR::Range::Delete: Deletes a node from this range. +// +// Note that the deleted node must not be used after this function has +// been called. If the deleted node is part of a block, this function also +// calls `Compiler::lvaDecRefCnts` as necessary. +// +// Arguments: +// node - The node to delete. Must be part of this range. +// block - The block that contains the node, if any. May be null. +// compiler - The compiler context. May be null if block is null. +// +void LIR::Range::Delete(Compiler* compiler, BasicBlock* block, GenTree* node) +{ + assert(node != nullptr); + assert((block == nullptr) == (compiler == nullptr)); + + Remove(node); + + if (block != nullptr) + { + if (((node->OperGet() == GT_CALL) && ((node->gtFlags & GTF_CALL_UNMANAGED) != 0)) || + (node->OperIsLocal() && !node->IsPhiNode())) + { + compiler->lvaDecRefCnts(block, node); + } + } + + DEBUG_DESTROY_NODE(node); +} + +//------------------------------------------------------------------------ +// LIR::Range::Delete: Deletes a subrange from this range. +// +// Both the start and the end of the subrange must be part of this range. +// Note that the deleted nodes must not be used after this function has +// been called. If the deleted nodes are part of a block, this function +// also calls `Compiler::lvaDecRefCnts` as necessary. +// +// Arguments: +// firstNode - The first node in the subrange. +// lastNode - The last node in the subrange. +// block - The block that contains the subrange, if any. May be null. +// compiler - The compiler context. May be null if block is null. +// +void LIR::Range::Delete(Compiler* compiler, BasicBlock* block, GenTree* firstNode, GenTree* lastNode) +{ + assert(firstNode != nullptr); + assert(lastNode != nullptr); + assert((block == nullptr) == (compiler == nullptr)); + + Remove(firstNode, lastNode); + + assert(lastNode->gtNext == nullptr); + + if (block != nullptr) + { + for (GenTree* node = firstNode; node != nullptr; node = node->gtNext) + { + if (((node->OperGet() == GT_CALL) && ((node->gtFlags & GTF_CALL_UNMANAGED) != 0)) || + (node->OperIsLocal() && !node->IsPhiNode())) + { + compiler->lvaDecRefCnts(block, node); + } + } + } + +#ifdef DEBUG + // We can't do this in the loop above because it causes `IsPhiNode` to return a false negative + // for `GT_STORE_LCL_VAR` nodes that participate in phi definitions. + for (GenTree* node = firstNode; node != nullptr; node = node->gtNext) + { + DEBUG_DESTROY_NODE(node); + } +#endif +} + +//------------------------------------------------------------------------ +// LIR::Range::Delete: Deletes a subrange from this range. +// +// Both the start and the end of the subrange must be part of this range. +// Note that the deleted nodes must not be used after this function has +// been called. If the deleted nodes are part of a block, this function +// also calls `Compiler::lvaDecRefCnts` as necessary. +// +// Arguments: +// range - The subrange to delete. +// block - The block that contains the subrange, if any. May be null. +// compiler - The compiler context. May be null if block is null. +// +void LIR::Range::Delete(Compiler* compiler, BasicBlock* block, ReadOnlyRange&& range) +{ + Delete(compiler, block, range.m_firstNode, range.m_lastNode); +} + + +//------------------------------------------------------------------------ +// LIR::Range::TryGetUse: Try to find the use for a given node. +// +// Arguments: +// node - The node for which to find the corresponding use. +// use (out) - The use of the corresponding node, if any. Invalid if +// this method returns false. +// +// Return Value: Returns true if a use was found; false otherwise. +// +bool LIR::Range::TryGetUse(GenTree* node, Use* use) +{ + assert(node != nullptr); + assert(use != nullptr); + assert(Contains(node)); + + // Don't bother looking for uses of nodes that are not values. + // If the node is the last node, we won't find a use (and we would + // end up creating an illegal range if we tried). + if (node->IsValue() && (node != LastNode())) + { + for (GenTree* n : ReadOnlyRange(node->gtNext, m_lastNode)) + { + GenTree** edge; + if (n->TryGetUse(node, &edge)) + { + *use = Use(*this, edge, n); + return true; + } + } + } + + *use = Use(); + return false; +} + +//------------------------------------------------------------------------ +// LIR::Range::GetTreeRange: Computes the subrange that includes all nodes +// in the dataflow trees rooted at a particular +// set of nodes. +// +// This method logically uses the following algorithm to compute the +// range: +// +// worklist = { set } +// firstNode = start +// isClosed = true +// +// while not worklist.isEmpty: +// if not worklist.contains(firstNode): +// isClosed = false +// else: +// for operand in firstNode: +// worklist.add(operand) +// +// worklist.remove(firstNode) +// +// firstNode = firstNode.previousNode +// +// return firstNode +// +// Instead of using a set for the worklist, the implementation uses the +// `LIR::Mark` bit of the `GenTree::LIRFlags` field to track whether or +// not a node is in the worklist. +// +// Note also that this algorithm depends LIR nodes being SDSU, SDSU defs +// and uses occurring in the same block, and correct dataflow (i.e. defs +// occurring before uses). +// +// Arguments: +// root - The root of the dataflow tree. +// isClosed - An output parameter that is set to true if the returned +// range contains only nodes in the dataflow tree and false +// otherwise. +// +// Returns: +// The computed subrange. +// +LIR::ReadOnlyRange LIR::Range::GetMarkedRange(unsigned markCount, + GenTree* start, + bool* isClosed, + unsigned* sideEffects) const +{ + assert(markCount != 0); + assert(start != nullptr); + assert(isClosed != nullptr); + assert(sideEffects != nullptr); + + bool sawUnmarkedNode = false; + unsigned sideEffectsInRange = 0; + + GenTree* firstNode = start; + GenTree* lastNode = nullptr; + for (;;) + { + if ((firstNode->gtLIRFlags & LIR::Flags::Mark) != 0) + { + if (lastNode == nullptr) + { + lastNode = firstNode; + } + + // Mark the node's operands + for (GenTree* operand : firstNode->Operands()) + { + // Do not mark nodes that do not appear in the execution order + assert(operand->OperGet() != GT_LIST); + if (operand->OperGet() == GT_ARGPLACE) + { + continue; + } + + operand->gtLIRFlags |= LIR::Flags::Mark; + markCount++; + } + + // Unmark the the node and update `firstNode` + firstNode->gtLIRFlags &= ~LIR::Flags::Mark; + markCount--; + } + else if (lastNode != nullptr) + { + sawUnmarkedNode = true; + } + + if (lastNode != nullptr) + { + sideEffectsInRange |= (firstNode->gtFlags & GTF_ALL_EFFECT); + } + + if (markCount == 0) + { + break; + } + + firstNode = firstNode->gtPrev; + + // This assert will fail if the dataflow that feeds the root node + // is incorrect in that it crosses a block boundary or if it involves + // a use that occurs before its corresponding def. + assert(firstNode != nullptr); + } + + assert(lastNode != nullptr); + + *isClosed = !sawUnmarkedNode; + *sideEffects = sideEffectsInRange; + return ReadOnlyRange(firstNode, lastNode); +} + +//------------------------------------------------------------------------ +// LIR::Range::GetTreeRange: Computes the subrange that includes all nodes +// in the dataflow tree rooted at a particular +// node. +// +// Arguments: +// root - The root of the dataflow tree. +// isClosed - An output parameter that is set to true if the returned +// range contains only nodes in the dataflow tree and false +// otherwise. +// +// Returns: +// The computed subrange. +LIR::ReadOnlyRange LIR::Range::GetTreeRange(GenTree* root, bool* isClosed) const +{ + unsigned unused; + return GetTreeRange(root, isClosed, &unused); +} + +//------------------------------------------------------------------------ +// LIR::Range::GetTreeRange: Computes the subrange that includes all nodes +// in the dataflow tree rooted at a particular +// node. +// +// Arguments: +// root - The root of the dataflow tree. +// isClosed - An output parameter that is set to true if the returned +// range contains only nodes in the dataflow tree and false +// otherwise. +// sideEffects - An output parameter that summarizes the side effects +// contained in the returned range. +// +// Returns: +// The computed subrange. +LIR::ReadOnlyRange LIR::Range::GetTreeRange(GenTree* root, bool* isClosed, unsigned* sideEffects) const +{ + assert(root != nullptr); + + // Mark the root of the tree + const unsigned markCount = 1; + root->gtLIRFlags |= LIR::Flags::Mark; + + return GetMarkedRange(markCount, root, isClosed, sideEffects); +} + +//------------------------------------------------------------------------ +// LIR::Range::GetTreeRange: Computes the subrange that includes all nodes +// in the dataflow trees rooted by the operands +// to a particular node. +// +// Arguments: +// root - The root of the dataflow tree. +// isClosed - An output parameter that is set to true if the returned +// range contains only nodes in the dataflow tree and false +// otherwise. +// sideEffects - An output parameter that summarizes the side effects +// contained in the returned range. +// +// Returns: +// The computed subrange. +// +LIR::ReadOnlyRange LIR::Range::GetRangeOfOperandTrees(GenTree* root, bool* isClosed, unsigned* sideEffects) const +{ + assert(root != nullptr); + assert(isClosed != nullptr); + assert(sideEffects != nullptr); + + // Mark the root node's operands + unsigned markCount = 0; + for (GenTree* operand : root->Operands()) + { + operand->gtLIRFlags |= LIR::Flags::Mark; + markCount++; + } + + if (markCount == 0) + { + *isClosed = true; + *sideEffects = 0; + return ReadOnlyRange(); + } + + return GetMarkedRange(markCount, root, isClosed, sideEffects); +} + +#ifdef DEBUG + +//------------------------------------------------------------------------ +// LIR::Range::CheckLIR: Performs a set of correctness checks on the LIR +// contained in this range. +// +// This method checks the following properties: +// - Defs are singly-used +// - Uses follow defs +// - Uses are correctly linked into the block +// - Nodes that do not produce values are not used +// - Only LIR nodes are present in the block +// - If any phi nodes are present in the range, they precede all other +// nodes +// +// The first four properties are verified by walking the range's LIR in execution order, +// inserting defs into a set as they are visited, and removing them as they are used. The +// different cases are distinguished only when an error is detected. +// +// Arguments: +// compiler - A compiler context. +// +// Return Value: +// 'true' if the LIR for the specified range is legal. +// +bool LIR::Range::CheckLIR(Compiler* compiler, bool checkUnusedValues) const +{ + if (IsEmpty()) + { + // Nothing more to check. + return true; + } + + // Check the gtNext/gtPrev links: (1) ensure there are no circularities, (2) ensure the gtPrev list is + // precisely the inverse of the gtNext list. + // + // To detect circularity, use the "tortoise and hare" 2-pointer algorithm. + + GenTree* slowNode = FirstNode(); + assert(slowNode != nullptr); // because it's a non-empty range + GenTree* fastNode1 = nullptr; + GenTree* fastNode2 = slowNode; + GenTree* prevSlowNode = nullptr; + while (((fastNode1 = fastNode2->gtNext) != nullptr) && ((fastNode2 = fastNode1->gtNext) != nullptr)) + { + if ((slowNode == fastNode1) || (slowNode == fastNode2)) + { + assert(!"gtNext nodes have a circularity!"); + } + assert(slowNode->gtPrev == prevSlowNode); + prevSlowNode = slowNode; + slowNode = slowNode->gtNext; + assert(slowNode != nullptr); // the fastNodes would have gone null first. + } + // If we get here, the list had no circularities, so either fastNode1 or fastNode2 must be nullptr. + assert((fastNode1 == nullptr) || (fastNode2 == nullptr)); + + // Need to check the rest of the gtPrev links. + while (slowNode != nullptr) + { + assert(slowNode->gtPrev == prevSlowNode); + prevSlowNode = slowNode; + slowNode = slowNode->gtNext; + } + + SmallHashTable<GenTree*, bool, 32> unusedDefs(compiler); + + bool pastPhis = false; + GenTree* prev = nullptr; + for (Iterator node = begin(), end = this->end(); node != end; prev = *node, ++node) + { + // Verify that the node is allowed in LIR. + assert(node->IsLIR()); + + // TODO: validate catch arg stores + + // Check that all phi nodes (if any) occur at the start of the range. + if ((node->OperGet() == GT_PHI_ARG) || (node->OperGet() == GT_PHI) || node->IsPhiDefn()) + { + assert(!pastPhis); + } + else + { + pastPhis = true; + } + + for (GenTree** useEdge : node->UseEdges()) + { + GenTree* def = *useEdge; + + assert((!checkUnusedValues || ((def->gtLIRFlags & LIR::Flags::IsUnusedValue) == 0)) && + "operands should never be marked as unused values"); + + if (def->OperGet() == GT_ARGPLACE) + { + // ARGPLACE nodes are not represented in the LIR sequence. Ignore them. + continue; + } + else if (!def->IsValue()) + { + // Calls may contain "uses" of nodes that do not produce a value. This is an artifact of + // the HIR and should probably be fixed, but doing so is an unknown amount of work. + assert(node->OperGet() == GT_CALL); + continue; + } + + bool v; + bool foundDef = unusedDefs.TryRemove(def, &v); + if (!foundDef) + { + // First, scan backwards and look for a preceding use. + for (GenTree* prev = *node; prev != nullptr; prev = prev->gtPrev) + { + // TODO: dump the users and the def + GenTree** earlierUseEdge; + bool foundEarlierUse = prev->TryGetUse(def, &earlierUseEdge) && earlierUseEdge != useEdge; + assert(!foundEarlierUse && "found multiply-used LIR node"); + } + + // The def did not precede the use. Check to see if it exists in the block at all. + for (GenTree* next = node->gtNext; next != nullptr; next = next->gtNext) + { + // TODO: dump the user and the def + assert(next != def && "found def after use"); + } + + // The def might not be a node that produces a value. + assert(def->IsValue() && "found use of a node that does not produce a value"); + + // By this point, the only possibility is that the def is not threaded into the LIR sequence. + assert(false && "found use of a node that is not in the LIR sequence"); + } + } + + if (node->IsValue()) + { + bool added = unusedDefs.AddOrUpdate(*node, true); + assert(added); + } + } + + assert(prev == m_lastNode); + + // At this point the unusedDefs map should contain only unused values. + if (checkUnusedValues) + { + for (auto kvp : unusedDefs) + { + GenTree* node = kvp.Key(); + assert(((node->gtLIRFlags & LIR::Flags::IsUnusedValue) != 0) && "found an unmarked unused value"); + } + } + + return true; +} + +#endif // DEBUG + +//------------------------------------------------------------------------ +// LIR::AsRange: Returns an LIR view of the given basic block. +// +LIR::Range& LIR::AsRange(BasicBlock* block) +{ + return *static_cast<Range*>(block); +} + +//------------------------------------------------------------------------ +// LIR::EmptyRange: Constructs and returns an empty range. +// +// static +LIR::Range LIR::EmptyRange() +{ + return Range(nullptr, nullptr); +} + +//------------------------------------------------------------------------ +// LIR::SeqTree: Given a newly created, unsequenced HIR tree, set the evaluation +// order (call gtSetEvalOrder) and sequence the tree (set gtNext/gtPrev pointers +// by calling fgSetTreeSeq), and return a Range representing the list of nodes. +// It is expected this will later be spliced into the LIR graph. +// +// Arguments: +// compiler - The Compiler context. +// tree - The tree to sequence. +// +// Return Value: The newly constructed range. +// +// static +LIR::Range LIR::SeqTree(Compiler* compiler, GenTree* tree) +{ + // TODO-LIR: it would be great to assert that the tree has not already been + // threaded into an order, but I'm not sure that will be practical at this + // point. + + compiler->gtSetEvalOrder(tree); + return Range(compiler->fgSetTreeSeq(tree, nullptr, true), tree); +} diff --git a/src/jit/lir.h b/src/jit/lir.h new file mode 100644 index 0000000000..f7307409c8 --- /dev/null +++ b/src/jit/lir.h @@ -0,0 +1,308 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#ifndef _LIR_H_ +#define _LIR_H_ + +class Compiler; +struct GenTree; +struct BasicBlock; + +class LIR final +{ +public: + class Range; + + //------------------------------------------------------------------------ + // LIR::Flags: Defines the set of flags that may appear in the + // GenTree::gtLIRFlags field. + class Flags final + { + // Disallow the creation of values of this type. + Flags() = delete; + + public: + enum : unsigned char + { + None = 0x00, + + Mark = 0x01, // An aribtrary "mark" bit that can be used in place of + // a more expensive data structure when processing a set + // of LIR nodes. See for example `LIR::GetTreeRange`. + + IsUnusedValue = 0x02, // Set on a node if it produces a value that is not + // subsequently used. Should never be set on nodes + // that return `false` for `GenTree::IsValue`. Note + // that this bit should not be assumed to be valid + // at all points during compilation: it is currently + // only computed during target-dependent lowering. + }; + }; + + //------------------------------------------------------------------------ + // LIR::Use: Represents a use <-> def edge between two nodes in a range + // of LIR. Provides utilities to point the use to a different + // def. Note that because this type deals in edges between + // nodes, it represents the single use of the def. + // + class Use final + { + private: + Range* m_range; + GenTree** m_edge; + GenTree* m_user; + + public: + Use(); + Use(const Use& other); + Use(Range& range, GenTree** edge, GenTree* user); + + Use& operator=(const Use& other); + Use& operator=(Use&& other); + + static Use GetDummyUse(Range& range, GenTree* node); + + GenTree* Def() const; + GenTree* User() const; + + bool IsInitialized() const; + void AssertIsValid() const; + bool IsDummyUse() const; + + void ReplaceWith(Compiler* compiler, GenTree* replacement); + unsigned ReplaceWithLclVar(Compiler* compiler, unsigned blockWeight, unsigned lclNum = BAD_VAR_NUM); + }; + + //------------------------------------------------------------------------ + // LIR::ReadOnlyRange: + // + // Represents a contiguous range of LIR nodes that may be a subrange of + // a containing range. Provides a small set of utilities for iteration. + // Instances of this type are primarily created by and provided to + // analysis and utility methods on LIR::Range. + // + // Although some pains have been taken to help guard against the existence + // of invalid subranges, it remains possible to create them. For example, + // consider the following: + // + // // View the block as a range + // LIR::Range& blockRange = LIR::AsRange(block); + // + // // Create a range from the first non-phi node in the block to the + // // last node in the block + // LIR::ReadOnlyRange nonPhis = blockRange.NonPhiNodes(); + // + // // Remove the last node from the block + // blockRange.Remove(blockRange.LastNode()); + // + // After the removal of the last node in the block, the last node of + // nonPhis is no longer linked to any of the other nodes in nonPhis. Due + // to issues such as the above, some care must be taken in order to + // ensure that ranges are not used once they have been invalidated. + // + class ReadOnlyRange + { + friend class LIR; + friend class Range; + friend struct BasicBlock; + + private: + GenTree* m_firstNode; + GenTree* m_lastNode; + + ReadOnlyRange(GenTree* firstNode, GenTree* lastNode); + + ReadOnlyRange(const ReadOnlyRange& other) = delete; + ReadOnlyRange& operator=(const ReadOnlyRange& other) = delete; + + public: + class Iterator + { + friend class ReadOnlyRange; + + GenTree* m_node; + + Iterator(GenTree* begin) : m_node(begin) + { + } + + public: + Iterator() : m_node(nullptr) + { + } + + inline GenTree* operator*() + { + return m_node; + } + + inline GenTree* operator->() + { + return m_node; + } + + inline bool operator==(const Iterator& other) const + { + return m_node == other.m_node; + } + + inline bool operator!=(const Iterator& other) const + { + return m_node != other.m_node; + } + + inline Iterator& operator++() + { + m_node = (m_node == nullptr) ? nullptr : m_node->gtNext; + return *this; + } + }; + + class ReverseIterator + { + friend class ReadOnlyRange; + + GenTree* m_node; + + ReverseIterator(GenTree* begin) : m_node(begin) + { + } + + public: + ReverseIterator() : m_node(nullptr) + { + } + + inline GenTree* operator*() + { + return m_node; + } + + inline GenTree* operator->() + { + return m_node; + } + + inline bool operator==(const ReverseIterator& other) const + { + return m_node == other.m_node; + } + + inline bool operator!=(const ReverseIterator& other) const + { + return m_node != other.m_node; + } + + inline ReverseIterator& operator++() + { + m_node = (m_node == nullptr) ? nullptr : m_node->gtPrev; + return *this; + } + }; + + ReadOnlyRange(); + ReadOnlyRange(ReadOnlyRange&& other); + + GenTree* FirstNode() const; + GenTree* LastNode() const; + + bool IsEmpty() const; + + Iterator begin() const; + Iterator end() const; + + ReverseIterator rbegin() const; + ReverseIterator rend() const; + +#ifdef DEBUG + bool Contains(GenTree* node) const; +#endif + }; + + //------------------------------------------------------------------------ + // LIR::Range: + // + // Represents a contiguous range of LIR nodes. Provides a variety of + // variety of utilites that modify the LIR contained in the range. Unlike + // `ReadOnlyRange`, values of this type may be edited. + // + // Because it is not a final class, it is possible to slice values of this + // type; this is especially dangerous when the Range value is actually of + // type `BasicBlock`. As a result, this type is not copyable and it is + // not possible to view a `BasicBlock` as anything other than a `Range&`. + // + class Range : public ReadOnlyRange + { + friend class LIR; + friend struct BasicBlock; + + private: + Range(GenTree* firstNode, GenTree* lastNode); + + Range(const Range& other) = delete; + Range& operator=(const Range& other) = delete; + + ReadOnlyRange GetMarkedRange(unsigned markCount, GenTree* start, bool* isClosed, unsigned* sideEffects) const; + + void FinishInsertBefore(GenTree* insertionPoint, GenTree* first, GenTree* last); + void FinishInsertAfter(GenTree* insertionPoint, GenTree* first, GenTree* last); + + public: + Range(); + Range(Range&& other); + + GenTree* LastPhiNode() const; + GenTree* FirstNonPhiNode() const; + GenTree* FirstNonPhiOrCatchArgNode() const; + + ReadOnlyRange PhiNodes() const; + ReadOnlyRange NonPhiNodes() const; + + void InsertBefore(GenTree* insertionPoint, GenTree* node); + void InsertAfter(GenTree* insertionPoint, GenTree* node); + + void InsertBefore(GenTree* insertionPoint, GenTree* node1, GenTree* node2); + void InsertBefore(GenTree* insertionPoint, GenTree* node1, GenTree* node2, GenTree* node3); + void InsertBefore(GenTree* insertionPoint, GenTree* node1, GenTree* node2, GenTree* node3, GenTree* node4); + + void InsertAfter(GenTree* insertionPoint, GenTree* node1, GenTree* node2); + void InsertAfter(GenTree* insertionPoint, GenTree* node1, GenTree* node2, GenTree* node3); + void InsertAfter(GenTree* insertionPoint, GenTree* node1, GenTree* node2, GenTree* node3, GenTree* node4); + + void InsertBefore(GenTree* insertionPoint, Range&& range); + void InsertAfter(GenTree* insertionPoint, Range&& range); + + void InsertAtBeginning(GenTree* node); + void InsertAtEnd(GenTree* node); + + void InsertAtBeginning(Range&& range); + void InsertAtEnd(Range&& range); + + void Remove(GenTree* node); + Range Remove(GenTree* firstNode, GenTree* lastNode); + Range Remove(ReadOnlyRange&& range); + + void Delete(Compiler* compiler, BasicBlock* block, GenTree* node); + void Delete(Compiler* compiler, BasicBlock* block, GenTree* firstNode, GenTree* lastNode); + void Delete(Compiler* compiler, BasicBlock* block, ReadOnlyRange&& range); + + bool TryGetUse(GenTree* node, Use* use); + + ReadOnlyRange GetTreeRange(GenTree* root, bool* isClosed) const; + ReadOnlyRange GetTreeRange(GenTree* root, bool* isClosed, unsigned* sideEffects) const; + ReadOnlyRange GetRangeOfOperandTrees(GenTree* root, bool* isClosed, unsigned* sideEffects) const; + +#ifdef DEBUG + bool CheckLIR(Compiler* compiler, bool checkUnusedValues = false) const; +#endif + }; + +public: + static Range& AsRange(BasicBlock* block); + + static Range EmptyRange(); + static Range SeqTree(Compiler* compiler, GenTree* tree); +}; + +#endif // _LIR_H_ diff --git a/src/jit/liveness.cpp b/src/jit/liveness.cpp index 9b07f8cd94..c12301c97a 100644 --- a/src/jit/liveness.cpp +++ b/src/jit/liveness.cpp @@ -52,8 +52,15 @@ void Compiler::fgMarkUseDef(GenTreeLclVarCommon* tree, GenTree* asgdLclVar) varDsc->lvRefCnt = 1; } - if (asgdLclVar) + // NOTE: the analysis done below is neither necessary nor correct for LIR: it depends on + // the nodes that precede `asgdLclVar` in execution order to factor into the dataflow for the + // value being assigned to the local var, which is not necessarily the case without tree + // order. Furthermore, LIR is always traversed in an order that reflects the dataflow for the + // block. + if (asgdLclVar != nullptr) { + assert(!compCurBB->IsLIR()); + /* we have an assignment to a local var : asgdLclVar = ... tree ... * check for x = f(x) case */ @@ -274,195 +281,203 @@ void Compiler::fgLocalVarLivenessInit() // #ifndef LEGACY_BACKEND //------------------------------------------------------------------------ -// fgPerStatementLocalVarLiveness: +// fgPerNodeLocalVarLiveness: // Set fgCurHeapUse and fgCurHeapDef when the global heap is read or updated // Call fgMarkUseDef for any Local variables encountered // // Arguments: -// startNode must be the first node in the statement -// asgdLclVar is either nullptr or the assignement's left-hand-side GT_LCL_VAR -// it is used as an argument to fgMarkUseDef() +// tree - The current node. +// asgdLclVar - Either nullptr or the assignement's left-hand-side GT_LCL_VAR. +// Used as an argument to fgMarkUseDef(); only valid for HIR blocks. // -void Compiler::fgPerStatementLocalVarLiveness(GenTreePtr startNode, GenTreePtr asgdLclVar) +void Compiler::fgPerNodeLocalVarLiveness(GenTree* tree, GenTree* asgdLclVar) { - // The startNode must be the 1st node of the statement. - assert(startNode == compCurStmt->gtStmt.gtStmtList); + assert(tree != nullptr); + assert(asgdLclVar == nullptr || !compCurBB->IsLIR()); - // The asgdLclVar node must be either nullptr or a GT_LCL_VAR or GT_STORE_LCL_VAR - assert((asgdLclVar == nullptr) || (asgdLclVar->gtOper == GT_LCL_VAR || asgdLclVar->gtOper == GT_STORE_LCL_VAR)); - - // We always walk every node in statement list - for (GenTreePtr tree = startNode; (tree != nullptr); tree = tree->gtNext) + switch (tree->gtOper) { - switch (tree->gtOper) - { - case GT_QMARK: - case GT_COLON: - // We never should encounter a GT_QMARK or GT_COLON node - noway_assert(!"unexpected GT_QMARK/GT_COLON"); - break; + case GT_QMARK: + case GT_COLON: + // We never should encounter a GT_QMARK or GT_COLON node + noway_assert(!"unexpected GT_QMARK/GT_COLON"); + break; - case GT_LCL_VAR: - case GT_LCL_FLD: - case GT_LCL_VAR_ADDR: - case GT_LCL_FLD_ADDR: - case GT_STORE_LCL_VAR: - case GT_STORE_LCL_FLD: - fgMarkUseDef(tree->AsLclVarCommon(), asgdLclVar); - break; + case GT_LCL_VAR: + case GT_LCL_FLD: + case GT_LCL_VAR_ADDR: + case GT_LCL_FLD_ADDR: + case GT_STORE_LCL_VAR: + case GT_STORE_LCL_FLD: + fgMarkUseDef(tree->AsLclVarCommon(), asgdLclVar); + break; - case GT_CLS_VAR: - // For Volatile indirection, first mutate the global heap - // see comments in ValueNum.cpp (under case GT_CLS_VAR) - // This models Volatile reads as def-then-use of the heap. - // and allows for a CSE of a subsequent non-volatile read - if ((tree->gtFlags & GTF_FLD_VOLATILE) != 0) - { - // For any Volatile indirection, we must handle it as a - // definition of the global heap - fgCurHeapDef = true; - } - // If the GT_CLS_VAR is the lhs of an assignment, we'll handle it as a heap def, when we get to - // assignment. - // Otherwise, we treat it as a use here. - if (!fgCurHeapDef && (tree->gtFlags & GTF_CLS_VAR_ASG_LHS) == 0) - { - fgCurHeapUse = true; - } - break; + case GT_CLS_VAR: + // For Volatile indirection, first mutate the global heap + // see comments in ValueNum.cpp (under case GT_CLS_VAR) + // This models Volatile reads as def-then-use of the heap. + // and allows for a CSE of a subsequent non-volatile read + if ((tree->gtFlags & GTF_FLD_VOLATILE) != 0) + { + // For any Volatile indirection, we must handle it as a + // definition of the global heap + fgCurHeapDef = true; + } + // If the GT_CLS_VAR is the lhs of an assignment, we'll handle it as a heap def, when we get to assignment. + // Otherwise, we treat it as a use here. + if (!fgCurHeapDef && (tree->gtFlags & GTF_CLS_VAR_ASG_LHS) == 0) + { + fgCurHeapUse = true; + } + break; - case GT_IND: - // For Volatile indirection, first mutate the global heap - // see comments in ValueNum.cpp (under case GT_CLS_VAR) - // This models Volatile reads as def-then-use of the heap. - // and allows for a CSE of a subsequent non-volatile read - if ((tree->gtFlags & GTF_IND_VOLATILE) != 0) - { - // For any Volatile indirection, we must handle it as a - // definition of the global heap - fgCurHeapDef = true; - } + case GT_IND: + // For Volatile indirection, first mutate the global heap + // see comments in ValueNum.cpp (under case GT_CLS_VAR) + // This models Volatile reads as def-then-use of the heap. + // and allows for a CSE of a subsequent non-volatile read + if ((tree->gtFlags & GTF_IND_VOLATILE) != 0) + { + // For any Volatile indirection, we must handle it as a + // definition of the global heap + fgCurHeapDef = true; + } - // If the GT_IND is the lhs of an assignment, we'll handle it - // as a heap def, when we get to assignment. - // Otherwise, we treat it as a use here. - if ((tree->gtFlags & GTF_IND_ASG_LHS) == 0) + // If the GT_IND is the lhs of an assignment, we'll handle it + // as a heap def, when we get to assignment. + // Otherwise, we treat it as a use here. + if ((tree->gtFlags & GTF_IND_ASG_LHS) == 0) + { + GenTreeLclVarCommon* dummyLclVarTree = nullptr; + bool dummyIsEntire = false; + GenTreePtr addrArg = tree->gtOp.gtOp1->gtEffectiveVal(/*commaOnly*/ true); + if (!addrArg->DefinesLocalAddr(this, /*width doesn't matter*/ 0, &dummyLclVarTree, &dummyIsEntire)) { - GenTreeLclVarCommon* dummyLclVarTree = nullptr; - bool dummyIsEntire = false; - GenTreePtr addrArg = tree->gtOp.gtOp1->gtEffectiveVal(/*commaOnly*/ true); - if (!addrArg->DefinesLocalAddr(this, /*width doesn't matter*/ 0, &dummyLclVarTree, &dummyIsEntire)) - { - if (!fgCurHeapDef) - { - fgCurHeapUse = true; - } - } - else + if (!fgCurHeapDef) { - // Defines a local addr - assert(dummyLclVarTree != nullptr); - fgMarkUseDef(dummyLclVarTree->AsLclVarCommon(), asgdLclVar); + fgCurHeapUse = true; } } - break; - - // These should have been morphed away to become GT_INDs: - case GT_FIELD: - case GT_INDEX: - unreached(); - break; - - // We'll assume these are use-then-defs of the heap. - case GT_LOCKADD: - case GT_XADD: - case GT_XCHG: - case GT_CMPXCHG: - if (!fgCurHeapDef) + else { - fgCurHeapUse = true; + // Defines a local addr + assert(dummyLclVarTree != nullptr); + fgMarkUseDef(dummyLclVarTree->AsLclVarCommon(), asgdLclVar); } - fgCurHeapDef = true; - fgCurHeapHavoc = true; - break; + } + break; - case GT_MEMORYBARRIER: - // Simliar to any Volatile indirection, we must handle this as a definition of the global heap - fgCurHeapDef = true; - break; + // These should have been morphed away to become GT_INDs: + case GT_FIELD: + case GT_INDEX: + unreached(); + break; - // For now, all calls read/write the heap, the latter in its entirety. Might tighten this case later. - case GT_CALL: + // We'll assume these are use-then-defs of the heap. + case GT_LOCKADD: + case GT_XADD: + case GT_XCHG: + case GT_CMPXCHG: + if (!fgCurHeapDef) { - GenTreeCall* call = tree->AsCall(); - bool modHeap = true; - if (call->gtCallType == CT_HELPER) - { - CorInfoHelpFunc helpFunc = eeGetHelperNum(call->gtCallMethHnd); + fgCurHeapUse = true; + } + fgCurHeapDef = true; + fgCurHeapHavoc = true; + break; - if (!s_helperCallProperties.MutatesHeap(helpFunc) && !s_helperCallProperties.MayRunCctor(helpFunc)) - { - modHeap = false; - } + case GT_MEMORYBARRIER: + // Simliar to any Volatile indirection, we must handle this as a definition of the global heap + fgCurHeapDef = true; + break; + + // For now, all calls read/write the heap, the latter in its entirety. Might tighten this case later. + case GT_CALL: + { + GenTreeCall* call = tree->AsCall(); + bool modHeap = true; + if (call->gtCallType == CT_HELPER) + { + CorInfoHelpFunc helpFunc = eeGetHelperNum(call->gtCallMethHnd); + + if (!s_helperCallProperties.MutatesHeap(helpFunc) && !s_helperCallProperties.MayRunCctor(helpFunc)) + { + modHeap = false; } - if (modHeap) + } + if (modHeap) + { + if (!fgCurHeapDef) { - if (!fgCurHeapDef) - { - fgCurHeapUse = true; - } - fgCurHeapDef = true; - fgCurHeapHavoc = true; + fgCurHeapUse = true; } + fgCurHeapDef = true; + fgCurHeapHavoc = true; } + } - // If this is a p/invoke unmanaged call or if this is a tail-call - // and we have an unmanaged p/invoke call in the method, - // then we're going to run the p/invoke epilog. - // So we mark the FrameRoot as used by this instruction. - // This ensures that the block->bbVarUse will contain - // the FrameRoot local var if is it a tracked variable. + // If this is a p/invoke unmanaged call or if this is a tail-call + // and we have an unmanaged p/invoke call in the method, + // then we're going to run the p/invoke epilog. + // So we mark the FrameRoot as used by this instruction. + // This ensures that the block->bbVarUse will contain + // the FrameRoot local var if is it a tracked variable. - if ((tree->gtCall.IsUnmanaged() || (tree->gtCall.IsTailCall() && info.compCallUnmanaged))) + if ((tree->gtCall.IsUnmanaged() || (tree->gtCall.IsTailCall() && info.compCallUnmanaged))) + { + assert((!opts.ShouldUsePInvokeHelpers()) || (info.compLvFrameListRoot == BAD_VAR_NUM)); + if (!opts.ShouldUsePInvokeHelpers()) { - assert((!opts.ShouldUsePInvokeHelpers()) || (info.compLvFrameListRoot == BAD_VAR_NUM)); - if (!opts.ShouldUsePInvokeHelpers()) - { - /* Get the TCB local and mark it as used */ + /* Get the TCB local and mark it as used */ - noway_assert(info.compLvFrameListRoot < lvaCount); + noway_assert(info.compLvFrameListRoot < lvaCount); - LclVarDsc* varDsc = &lvaTable[info.compLvFrameListRoot]; + LclVarDsc* varDsc = &lvaTable[info.compLvFrameListRoot]; - if (varDsc->lvTracked) + if (varDsc->lvTracked) + { + if (!VarSetOps::IsMember(this, fgCurDefSet, varDsc->lvVarIndex)) { - if (!VarSetOps::IsMember(this, fgCurDefSet, varDsc->lvVarIndex)) - { - VarSetOps::AddElemD(this, fgCurUseSet, varDsc->lvVarIndex); - } + VarSetOps::AddElemD(this, fgCurUseSet, varDsc->lvVarIndex); } } } + } - break; + break; - default: + default: - // Determine whether it defines a heap location. - if (tree->OperIsAssignment() || tree->OperIsBlkOp()) + // Determine whether it defines a heap location. + if (tree->OperIsAssignment() || tree->OperIsBlkOp()) + { + GenTreeLclVarCommon* dummyLclVarTree = nullptr; + if (!tree->DefinesLocal(this, &dummyLclVarTree)) { - GenTreeLclVarCommon* dummyLclVarTree = nullptr; - if (!tree->DefinesLocal(this, &dummyLclVarTree)) - { - // If it doesn't define a local, then it might update the heap. - fgCurHeapDef = true; - } + // If it doesn't define a local, then it might update the heap. + fgCurHeapDef = true; } - break; - } + } + break; } } -#endif // LEGACY_BACKEND + +void Compiler::fgPerStatementLocalVarLiveness(GenTree* startNode, GenTree* asgdLclVar) +{ + // The startNode must be the 1st node of the statement. + assert(startNode == compCurStmt->gtStmt.gtStmtList); + + // The asgdLclVar node must be either nullptr or a GT_LCL_VAR or GT_STORE_LCL_VAR + assert((asgdLclVar == nullptr) || (asgdLclVar->gtOper == GT_LCL_VAR || asgdLclVar->gtOper == GT_STORE_LCL_VAR)); + + // We always walk every node in statement list + for (GenTreePtr node = startNode; node != nullptr; node = node->gtNext) + { + fgPerNodeLocalVarLiveness(node, asgdLclVar); + } +} + +#endif // !LEGACY_BACKEND /*****************************************************************************/ void Compiler::fgPerBlockLocalVarLiveness() @@ -546,65 +561,79 @@ void Compiler::fgPerBlockLocalVarLiveness() compCurBB = block; - for (stmt = block->FirstNonPhiDef(); stmt; stmt = stmt->gtNext) + if (!block->IsLIR()) { - noway_assert(stmt->gtOper == GT_STMT); - - if (!stmt->gtStmt.gtStmtIsTopLevel()) + for (stmt = block->FirstNonPhiDef(); stmt; stmt = stmt->gtNext) { - continue; - } + noway_assert(stmt->gtOper == GT_STMT); - compCurStmt = stmt; + compCurStmt = stmt; - asgdLclVar = nullptr; - tree = stmt->gtStmt.gtStmtExpr; - noway_assert(tree); + asgdLclVar = nullptr; + tree = stmt->gtStmt.gtStmtExpr; + noway_assert(tree); - // The following code checks if we have an assignment expression - // which may become a GTF_VAR_USEDEF - x=f(x). - // consider if LHS is local var - ignore if RHS contains SIDE_EFFECTS + // The following code checks if we have an assignment expression + // which may become a GTF_VAR_USEDEF - x=f(x). + // consider if LHS is local var - ignore if RHS contains SIDE_EFFECTS - if ((tree->gtOper == GT_ASG && tree->gtOp.gtOp1->gtOper == GT_LCL_VAR) || tree->gtOper == GT_STORE_LCL_VAR) - { - noway_assert(tree->gtOp.gtOp1); - GenTreePtr rhsNode; - if (tree->gtOper == GT_ASG) - { - noway_assert(tree->gtOp.gtOp2); - asgdLclVar = tree->gtOp.gtOp1; - rhsNode = tree->gtOp.gtOp2; - } - else + if ((tree->gtOper == GT_ASG && tree->gtOp.gtOp1->gtOper == GT_LCL_VAR) || + tree->gtOper == GT_STORE_LCL_VAR) { - asgdLclVar = tree; - rhsNode = tree->gtOp.gtOp1; - } + noway_assert(tree->gtOp.gtOp1); + GenTreePtr rhsNode; + if (tree->gtOper == GT_ASG) + { + noway_assert(tree->gtOp.gtOp2); + asgdLclVar = tree->gtOp.gtOp1; + rhsNode = tree->gtOp.gtOp2; + } + else + { + asgdLclVar = tree; + rhsNode = tree->gtOp.gtOp1; + } - // If this is an assignment to local var with no SIDE EFFECTS, - // set asgdLclVar so that genMarkUseDef will flag potential - // x=f(x) expressions as GTF_VAR_USEDEF. - // Reset the flag before recomputing it - it may have been set before, - // but subsequent optimizations could have removed the rhs reference. - asgdLclVar->gtFlags &= ~GTF_VAR_USEDEF; - if ((rhsNode->gtFlags & GTF_SIDE_EFFECT) == 0) - { - noway_assert(asgdLclVar->gtFlags & GTF_VAR_DEF); - } - else - { - asgdLclVar = nullptr; + // If this is an assignment to local var with no SIDE EFFECTS, + // set asgdLclVar so that genMarkUseDef will flag potential + // x=f(x) expressions as GTF_VAR_USEDEF. + // Reset the flag before recomputing it - it may have been set before, + // but subsequent optimizations could have removed the rhs reference. + asgdLclVar->gtFlags &= ~GTF_VAR_USEDEF; + if ((rhsNode->gtFlags & GTF_SIDE_EFFECT) == 0) + { + noway_assert(asgdLclVar->gtFlags & GTF_VAR_DEF); + } + else + { + asgdLclVar = nullptr; + } } - } #ifdef LEGACY_BACKEND - tree = fgLegacyPerStatementLocalVarLiveness(stmt->gtStmt.gtStmtList, NULL, asgdLclVar); + tree = fgLegacyPerStatementLocalVarLiveness(stmt->gtStmt.gtStmtList, NULL, asgdLclVar); - // We must have walked to the end of this statement. - noway_assert(!tree); -#else - fgPerStatementLocalVarLiveness(stmt->gtStmt.gtStmtList, asgdLclVar); -#endif // LEGACY_BACKEND + // We must have walked to the end of this statement. + noway_assert(!tree); +#else // !LEGACY_BACKEND + fgPerStatementLocalVarLiveness(stmt->gtStmt.gtStmtList, asgdLclVar); +#endif // !LEGACY_BACKEND + } + } + else + { +#ifdef LEGACY_BACKEND + unreached(); +#else // !LEGACY_BACKEND + // NOTE: the `asgdLclVar` analysis done above is not correct for LIR: it depends + // on all of the nodes that precede `asgdLclVar` in execution order to factor into the + // dataflow for the value being assigned to the local var, which is not necessarily the + // case without tree order. As a result, we simply pass `nullptr` for `asgdLclVar`. + for (GenTree* node : LIR::AsRange(block).NonPhiNodes()) + { + fgPerNodeLocalVarLiveness(node, nullptr); + } +#endif // !LEGACY_BACKEND } /* Get the TCB local and mark it as used */ @@ -1069,9 +1098,15 @@ void Compiler::fgExtendDbgLifetimes() continue; } + // TODO-LIR: the code below does not work for blocks that contain LIR. As a result, + // we must run liveness at least once before any LIR is created in order + // to ensure that this code doesn't attempt to insert HIR into LIR blocks. + // If we haven't already done this ... if (!fgLocalVarLivenessDone) { + assert(!block->IsLIR()); + // Create a "zero" node GenTreePtr zero = gtNewZeroConNode(genActualType(type)); @@ -1726,7 +1761,13 @@ bool Compiler::fgComputeLifeLocal(VARSET_TP& life, VARSET_TP& keepAliveVars, Gen GTF_VAR_USEASG and we are in an interior statement that will be used (e.g. while (i++) or a GT_COMMA) */ - return true; + // Do not consider this store dead if the target local variable represents + // a promoted struct field of an address exposed local or if the address + // of the variable has been exposed. Improved alias analysis could allow + // stores to these sorts of variables to be removed at the cost of compile + // time. + return !varDsc->lvAddrExposed && + !(varDsc->lvIsStructField && lvaTable[varDsc->lvParentLcl].lvAddrExposed); } } @@ -1894,6 +1935,46 @@ VARSET_VALRET_TP Compiler::fgComputeLife(VARSET_VALARG_TP lifeArg, return life; } +VARSET_VALRET_TP Compiler::fgComputeLifeLIR(VARSET_VALARG_TP lifeArg, BasicBlock* block, VARSET_VALARG_TP volatileVars) +{ + VARSET_TP VARSET_INIT(this, life, lifeArg); // lifeArg is const ref; copy to allow modification. + + VARSET_TP VARSET_INIT(this, keepAliveVars, volatileVars); +#ifdef DEBUGGING_SUPPORT + VarSetOps::UnionD(this, keepAliveVars, block->bbScope); // Don't kill vars in scope +#endif + + noway_assert(VarSetOps::Equal(this, VarSetOps::Intersection(this, keepAliveVars, life), keepAliveVars)); + + LIR::Range& blockRange = LIR::AsRange(block); + GenTree* firstNonPhiNode = blockRange.FirstNonPhiNode(); + if (firstNonPhiNode == nullptr) + { + return life; + } + + for (GenTree *node = blockRange.LastNode(), *next = nullptr, *end = firstNonPhiNode->gtPrev; node != end; + node = next) + { + next = node->gtPrev; + + if (node->OperGet() == GT_CALL) + { + fgComputeLifeCall(life, node->AsCall()); + } + else if (node->OperIsNonPhiLocal() || node->OperIsLocalAddr()) + { + bool isDeadStore = fgComputeLifeLocal(life, keepAliveVars, node, node); + if (isDeadStore) + { + fgTryRemoveDeadLIRStore(blockRange, node, &next); + } + } + } + + return life; +} + #else // LEGACY_BACKEND #ifdef _PREFAST_ @@ -2245,6 +2326,84 @@ VARSET_VALRET_TP Compiler::fgComputeLife(VARSET_VALARG_TP lifeArg, #endif // !LEGACY_BACKEND +bool Compiler::fgTryRemoveDeadLIRStore(LIR::Range& blockRange, GenTree* node, GenTree** next) +{ + assert(node != nullptr); + assert(next != nullptr); + + assert(node->OperIsLocalStore() || node->OperIsLocalAddr()); + + GenTree* store = nullptr; + GenTree* value = nullptr; + if (node->OperIsLocalStore()) + { + store = node; + value = store->gtGetOp1(); + } + else if (node->OperIsLocalAddr()) + { + LIR::Use addrUse; + if (!blockRange.TryGetUse(node, &addrUse) || (addrUse.User()->OperGet() != GT_STOREIND)) + { + *next = node->gtPrev; + return false; + } + + store = addrUse.User(); + value = store->gtGetOp2(); + } + + bool isClosed = false; + unsigned sideEffects = 0; + LIR::ReadOnlyRange operandsRange = blockRange.GetRangeOfOperandTrees(store, &isClosed, &sideEffects); + if (!isClosed || ((sideEffects & GTF_SIDE_EFFECT) != 0) || + (((sideEffects & GTF_ORDER_SIDEEFF) != 0) && (value->OperGet() == GT_CATCH_ARG))) + { + // If the range of the operands contains unrelated code or if it contains any side effects, + // do not remove it. Instead, just remove the store. + + *next = node->gtPrev; + } + else + { + // Okay, the operands to the store form a contiguous range that has no side effects. Remove the + // range containing the operands and decrement the local var ref counts appropriately. + + // Compute the next node to process. Note that we must be careful not to set the next node to + // process to a node that we are about to remove. + if (node->OperIsLocalStore()) + { + assert(node == store); + *next = (operandsRange.LastNode()->gtNext == store) ? operandsRange.FirstNode()->gtPrev : node->gtPrev; + } + else + { + assert(operandsRange.Contains(node)); + *next = operandsRange.FirstNode()->gtPrev; + } + + blockRange.Delete(this, compCurBB, std::move(operandsRange)); + } + + // If the store is marked as a late argument, it is referenced by a call. Instead of removing it, + // bash it to a NOP. + if ((store->gtFlags & GTF_LATE_ARG) != 0) + { + if (store->IsLocal()) + { + lvaDecRefCnts(compCurBB, store); + } + + store->gtBashToNOP(); + } + else + { + blockRange.Delete(this, compCurBB, store); + } + + return true; +} + // fgRemoveDeadStore - remove a store to a local which has no exposed uses. // // pTree - GenTree** to local, including store-form local or local addr (post-rationalize) @@ -2258,6 +2417,12 @@ VARSET_VALRET_TP Compiler::fgComputeLife(VARSET_VALARG_TP lifeArg, bool Compiler::fgRemoveDeadStore( GenTree** pTree, LclVarDsc* varDsc, VARSET_TP life, bool* doAgain, bool* pStmtInfoDirty DEBUGARG(bool* treeModf)) { + assert(!compRationalIRForm); + + // Vars should have already been checked for address exposure by this point. + assert(!varDsc->lvIsStructField || !lvaTable[varDsc->lvParentLcl].lvAddrExposed); + assert(!varDsc->lvAddrExposed); + GenTree* asgNode = nullptr; GenTree* rhsNode = nullptr; GenTree* addrNode = nullptr; @@ -2442,11 +2607,6 @@ bool Compiler::fgRemoveDeadStore( if (rhsNode->gtFlags & GTF_SIDE_EFFECT) { - if (compRationalIRForm) - { - return false; - } - EXTRACT_SIDE_EFFECTS: /* Extract the side effects */ @@ -2517,14 +2677,6 @@ bool Compiler::fgRemoveDeadStore( } } - // If there is an embedded statement this could be tricky because we need to - // walk them next, and we have already skipped over them because they were - // not top level (but will be if we delete the top level statement) - if (compCurStmt->gtStmt.gtNextStmt && !compCurStmt->gtStmt.gtNextStmt->gtStmtIsTopLevel()) - { - return false; - } - /* No side effects - remove the whole statement from the block->bbTreeList */ fgRemoveStmt(compCurBB, compCurStmt); @@ -2540,18 +2692,8 @@ bool Compiler::fgRemoveDeadStore( { /* This is an INTERIOR STATEMENT with a dead assignment - remove it */ - // don't want to deal with this - if (compRationalIRForm) - { - return false; - } - noway_assert(!VarSetOps::IsMember(this, life, varDsc->lvVarIndex)); - JITDUMP("interior assign\n"); - DISPTREE(asgNode); - JITDUMP("\n"); - if (rhsNode->gtFlags & GTF_SIDE_EFFECT) { /* :-( we have side effects */ @@ -2633,17 +2775,6 @@ bool Compiler::fgRemoveDeadStore( /* Change the assignment to a GT_NOP node */ - if (compRationalIRForm) - { - JITDUMP("deleting tree:\n"); - DISPTREE(rhsNode); - fgDeleteTreeFromList(compCurStmt->AsStmt(), rhsNode); - if (tree->gtOper == GT_STOREIND) - { - fgDeleteTreeFromList(compCurStmt->AsStmt(), asgNode->gtOp.gtOp1); - } - } - asgNode->gtBashToNOP(); #ifdef DEBUG @@ -2653,17 +2784,14 @@ bool Compiler::fgRemoveDeadStore( /* Re-link the nodes for this statement - Do not update ordering! */ - if (!compRationalIRForm) - { - // Do not update costs by calling gtSetStmtInfo. fgSetStmtSeq modifies - // the tree threading based on the new costs. Removing nodes could - // cause a subtree to get evaluated first (earlier second) during the - // liveness walk. Instead just set a flag that costs are dirty and - // caller has to call gtSetStmtInfo. - *pStmtInfoDirty = true; + // Do not update costs by calling gtSetStmtInfo. fgSetStmtSeq modifies + // the tree threading based on the new costs. Removing nodes could + // cause a subtree to get evaluated first (earlier second) during the + // liveness walk. Instead just set a flag that costs are dirty and + // caller has to call gtSetStmtInfo. + *pStmtInfoDirty = true; - fgSetStmtSeq(compCurStmt); - } + fgSetStmtSeq(compCurStmt); /* Continue analysis from this node */ @@ -2857,56 +2985,62 @@ void Compiler::fgInterBlockLocalVarLiveness() fgMarkIntf(life); - /* Get the first statement in the block */ + if (!block->IsLIR()) + { + /* Get the first statement in the block */ - GenTreePtr firstStmt = block->FirstNonPhiDef(); + GenTreePtr firstStmt = block->FirstNonPhiDef(); - if (!firstStmt) - { - continue; - } + if (!firstStmt) + { + continue; + } - /* Walk all the statements of the block backwards - Get the LAST stmt */ + /* Walk all the statements of the block backwards - Get the LAST stmt */ - GenTreePtr nextStmt = block->bbTreeList->gtPrev; + GenTreePtr nextStmt = block->bbTreeList->gtPrev; - do - { + do + { #ifdef DEBUG - bool treeModf = false; + bool treeModf = false; #endif // DEBUG - noway_assert(nextStmt); - noway_assert(nextStmt->gtOper == GT_STMT); + noway_assert(nextStmt); + noway_assert(nextStmt->gtOper == GT_STMT); - compCurStmt = nextStmt; - nextStmt = nextStmt->gtPrev; + compCurStmt = nextStmt; + nextStmt = nextStmt->gtPrev; - if (!compCurStmt->gtStmt.gtStmtIsTopLevel()) - { - continue; - } - - /* Compute the liveness for each tree node in the statement */ - bool stmtInfoDirty = false; + /* Compute the liveness for each tree node in the statement */ + bool stmtInfoDirty = false; - VarSetOps::AssignNoCopy(this, life, fgComputeLife(life, compCurStmt->gtStmt.gtStmtExpr, nullptr, - volatileVars, &stmtInfoDirty DEBUGARG(&treeModf))); + VarSetOps::AssignNoCopy(this, life, fgComputeLife(life, compCurStmt->gtStmt.gtStmtExpr, nullptr, + volatileVars, &stmtInfoDirty DEBUGARG(&treeModf))); - if (stmtInfoDirty) - { - gtSetStmtInfo(compCurStmt); - fgSetStmtSeq(compCurStmt); - } + if (stmtInfoDirty) + { + gtSetStmtInfo(compCurStmt); + fgSetStmtSeq(compCurStmt); + } #ifdef DEBUG - if (verbose && treeModf) - { - printf("\nfgComputeLife modified tree:\n"); - gtDispTree(compCurStmt->gtStmt.gtStmtExpr); - printf("\n"); - } + if (verbose && treeModf) + { + printf("\nfgComputeLife modified tree:\n"); + gtDispTree(compCurStmt->gtStmt.gtStmtExpr); + printf("\n"); + } #endif // DEBUG - } while (compCurStmt != firstStmt); + } while (compCurStmt != firstStmt); + } + else + { +#ifdef LEGACY_BACKEND + unreached(); +#else // !LEGACY_BACKEND + VarSetOps::AssignNoCopy(this, life, fgComputeLifeLIR(life, block, volatileVars)); +#endif // !LEGACY_BACKEND + } /* Done with the current block - if we removed any statements, some * variables may have become dead at the beginning of the block diff --git a/src/jit/lower.cpp b/src/jit/lower.cpp index 5c57a88f70..51a56b3ae9 100644 --- a/src/jit/lower.cpp +++ b/src/jit/lower.cpp @@ -30,11 +30,11 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #endif // !defined(_TARGET_64BIT_) //------------------------------------------------------------------------ -// MakeSrcContained: Make "tree" a contained node +// MakeSrcContained: Make "childNode" a contained node // // Arguments: -// 'parentNode' is a non-leaf node that can contain its 'childNode' -// 'childNode' is an op that will now be contained by its parent. +// parentNode - is a non-leaf node that can contain its 'childNode' +// childNode - is an op that will now be contained by its parent. // // Notes: // If 'childNode' it has any existing sources, they will now be sources for the parent. @@ -50,16 +50,15 @@ void Lowering::MakeSrcContained(GenTreePtr parentNode, GenTreePtr childNode) } //------------------------------------------------------------------------ -// CheckImmedAndMakeContained: Check and make 'childNode' contained +// CheckImmedAndMakeContained: Checks if the 'childNode' is a containable immediate +// and, if so, makes it contained. +// // Arguments: -// 'parentNode' is any non-leaf node -// 'childNode' is an child op of 'parentNode' -// Return value: -// returns true if we are able to make childNode contained immediate +// parentNode - is any non-leaf node +// childNode - is an child op of 'parentNode' // -// Notes: -// Checks if the 'childNode' is a containable immediate -// and then makes it contained +// Return value: +// true if we are able to make childNode a contained immediate // bool Lowering::CheckImmedAndMakeContained(GenTree* parentNode, GenTree* childNode) { @@ -75,29 +74,28 @@ bool Lowering::CheckImmedAndMakeContained(GenTree* parentNode, GenTree* childNod } //------------------------------------------------------------------------ -// IsSafeToContainMem: Checks for conflicts between childNode and parentNode. +// IsSafeToContainMem: Checks for conflicts between childNode and parentNode, +// and returns 'true' iff memory operand childNode can be contained in parentNode. // // Arguments: // parentNode - a non-leaf binary node // childNode - a memory op that is a child op of 'parentNode' // // Return value: -// returns true if it is safe to make childNode a contained memory op +// true if it is safe to make childNode a contained memory operand. // -// Notes: -// Checks for memory conflicts in the instructions between childNode and parentNode, -// and returns true iff childNode can be contained. - bool Lowering::IsSafeToContainMem(GenTree* parentNode, GenTree* childNode) { assert(parentNode->OperIsBinary()); assert(childNode->isMemoryOp()); - // Check conflicts against nodes between 'childNode' and 'parentNode' - GenTree* node; unsigned int childFlags = (childNode->gtFlags & GTF_ALL_EFFECT); - for (node = childNode->gtNext; (node != parentNode) && (node != nullptr); node = node->gtNext) + + GenTree* node; + for (node = childNode; node != parentNode; node = node->gtNext) { + assert(node != nullptr); + if ((childFlags != 0) && node->IsCall()) { bool isPureHelper = (node->gtCall.gtCallType == CT_HELPER) && @@ -112,208 +110,78 @@ bool Lowering::IsSafeToContainMem(GenTree* parentNode, GenTree* childNode) return false; } } - if (node != parentNode) - { - assert(!"Ran off end of stmt\n"); - return false; - } + return true; } //------------------------------------------------------------------------ -// static -Compiler::fgWalkResult Lowering::LowerNodeHelper(GenTreePtr* pTree, Compiler::fgWalkData* data) -{ - Lowering* lower = (Lowering*)data->pCallbackData; - lower->LowerNode(pTree, data); - return Compiler::WALK_CONTINUE; -} - -/** Creates an assignment of an existing tree to a new temporary local variable - * and the specified reference count for the new variable. - */ -GenTreePtr Lowering::CreateLocalTempAsg(GenTreePtr rhs, unsigned refCount, GenTreePtr* ppLclVar) // out legacy arg -{ - unsigned lclNum = comp->lvaGrabTemp(true DEBUGARG("Lowering is creating a new local variable")); - comp->lvaSortAgain = true; - comp->lvaTable[lclNum].lvType = rhs->TypeGet(); - - // Make sure we don't lose precision when downgrading to short - noway_assert(FitsIn<short>(refCount)); - comp->lvaTable[lclNum].lvRefCnt = (short)(refCount); - JITDUMP("Lowering has requested a new temporary local variable: V%02u with refCount %u \n", lclNum, refCount); - - GenTreeLclVar* store = - new (comp, GT_STORE_LCL_VAR) GenTreeLclVar(GT_STORE_LCL_VAR, rhs->TypeGet(), lclNum, BAD_IL_OFFSET); - store->gtOp1 = rhs; - store->gtFlags = (rhs->gtFlags & GTF_COMMON_MASK); - store->gtFlags |= GTF_VAR_DEF; - return store; -} - -//----------------------------------------------------------------------------------------------- -// CreateTemporary: Store the result of the given tree in a newly created temporary local -// variable and replace the original use of the tree with the temporary. -// -// Arguments: -// ppTree - a pointer to the tree use to replace. -// -// Return Value: -// The newly created store statement. -// -// Assumptions: -// This may only be called during tree lowering. The callee must ensure that the tree has already -// been lowered and is part of compCurStmt and that compCurStmt is in compCurBB. -// -// Notes: -// The newly created statement is usually an embedded statement but it can also be a top-level -// statement if the tree to be replaced extends to the begining of the current statement. If -// a top-level statement is created any embedded statements contained in the tree move to the -// the new top-level statement, before the current statement. Such embedded statements need to -// be lowered here because the normal lowering code path won't reach them anymore. -// -// TODO-Cleanup: -// Some uses of fgInsertEmbeddedFormTemp in lowering could be replaced with this to avoid -// duplication, see LowerArrElem for example. - -GenTreeStmt* Lowering::CreateTemporary(GenTree** ppTree) -{ - GenTreeStmt* newStmt = comp->fgInsertEmbeddedFormTemp(ppTree); - - // The tree is assumed to be already lowered so the newly created statement - // should not be lowered again. - newStmt->gtFlags |= GTF_STMT_SKIP_LOWER; - - assert(newStmt->gtStmtExpr->OperIsLocalStore()); - - // If the newly created statement is top-level then we need to manually lower its embedded - // statements, the tree is lowered but some of its embedded statements are yet to be lowered. - if (newStmt->gtStmtIsTopLevel()) - { - GenTree* curStmt = comp->compCurStmt; - - for (GenTree* nextEmbeddedStmt = newStmt->gtStmtNextIfEmbedded(); nextEmbeddedStmt != nullptr; - nextEmbeddedStmt = nextEmbeddedStmt->gtStmt.gtStmtNextIfEmbedded()) - { - // A previous call to CreateTemporary could have created embedded statements - // from the tree and those are already lowered. - if ((nextEmbeddedStmt->gtFlags & GTF_STMT_SKIP_LOWER) != 0) - { - continue; - } - -#ifdef DEBUG - if (comp->verbose) - { - printf("Lowering BB%02u, stmt id %u\n", currBlock->bbNum, nextEmbeddedStmt->gtTreeID); - } -#endif - comp->compCurStmt = nextEmbeddedStmt; - comp->fgWalkTreePost(&nextEmbeddedStmt->gtStmt.gtStmtExpr, &Lowering::LowerNodeHelper, this, true); - nextEmbeddedStmt->gtFlags |= GTF_STMT_SKIP_LOWER; - - // Lowering can remove the statement and set compCurStmt to another suitable statement. - // Currently only switch lowering does this and since embedded statements can't contain - // a GT_SWITCH this case should never be hit here. - assert(comp->compCurStmt == nextEmbeddedStmt); - } - - comp->compCurStmt = curStmt; - } - - return newStmt; -} - // This is the main entry point for Lowering. - -// In addition to that, LowerNode is also responsible for initializing the -// treeNodeMap data structure consumed by LSRA. This map is a 1:1 mapping between -// expression trees and TreeNodeInfo structs. Currently, Lowering initializes -// treeNodeMap with new instances of TreeNodeInfo for each tree and also annotates them -// with the register requirements needed for each tree. -// We receive a double pointer to a tree in order to be able, if needed, to entirely -// replace the tree by creating a new one and updating the underying pointer so this -// enables in-place tree manipulation. -// The current design is made in such a way we perform a helper call for each different -// type of tree. Currently, the only supported node is GT_IND and for that we call the -// LowerInd private method. The build system picks up the appropiate Lower.cpp (either -// LowerArm/LowerX86/LowerAMD64) that has the machine dependent logic to lower each node. -// TODO-Throughput: Modify post-order traversal to propagate parent info OR -// implement child iterator directly on GenTree, so that we can -// lower in-place. -void Lowering::LowerNode(GenTreePtr* ppTree, Compiler::fgWalkData* data) +GenTree* Lowering::LowerNode(GenTree* node) { - // First, lower any child nodes (done via post-order walk) - assert(ppTree); - assert(*ppTree); - switch ((*ppTree)->gtOper) + assert(node != nullptr); + switch (node->gtOper) { case GT_IND: + TryCreateAddrMode(LIR::Use(BlockRange(), &node->gtOp.gtOp1, node), true); + break; + case GT_STOREIND: - LowerInd(ppTree); + LowerStoreInd(node); break; case GT_ADD: - LowerAdd(ppTree, data); - break; + return LowerAdd(node); case GT_UDIV: case GT_UMOD: - LowerUnsignedDivOrMod(*ppTree); + LowerUnsignedDivOrMod(node); break; case GT_DIV: case GT_MOD: - LowerSignedDivOrMod(ppTree, data); - break; + return LowerSignedDivOrMod(node); case GT_SWITCH: - LowerSwitch(ppTree); - break; + return LowerSwitch(node); case GT_CALL: - LowerCall(*ppTree); + LowerCall(node); break; case GT_JMP: - LowerJmpMethod(*ppTree); + LowerJmpMethod(node); break; case GT_RETURN: - LowerRet(*ppTree); + LowerRet(node); break; case GT_CAST: - LowerCast(ppTree); + LowerCast(node); break; case GT_ARR_ELEM: - { - GenTree* oldTree = *ppTree; - LowerArrElem(ppTree, data); - comp->fgFixupIfCallArg(data->parentStack, oldTree, *ppTree); - } - break; + return LowerArrElem(node); case GT_ROL: case GT_ROR: - LowerRotate(*ppTree); + LowerRotate(node); break; #ifdef FEATURE_SIMD case GT_SIMD: - if ((*ppTree)->TypeGet() == TYP_SIMD12) + if (node->TypeGet() == TYP_SIMD12) { // GT_SIMD node requiring to produce TYP_SIMD12 in fact // produces a TYP_SIMD16 result - (*ppTree)->gtType = TYP_SIMD16; + node->gtType = TYP_SIMD16; } break; case GT_LCL_VAR: case GT_STORE_LCL_VAR: - if ((*ppTree)->TypeGet() == TYP_SIMD12) + if (node->TypeGet() == TYP_SIMD12) { #ifdef _TARGET_64BIT_ // Assumption 1: @@ -338,7 +206,7 @@ void Lowering::LowerNode(GenTreePtr* ppTree, Compiler::fgWalkData* data) // Vector3 return values are returned two return registers and Caller assembles them into a // single xmm reg. Hence RyuJIT explicitly generates code to clears upper 4-bytes of Vector3 // type args in prolog and Vector3 type return value of a call - (*ppTree)->gtType = TYP_SIMD16; + node->gtType = TYP_SIMD16; #else NYI("Lowering of TYP_SIMD12 locals"); #endif // _TARGET_64BIT_ @@ -346,8 +214,10 @@ void Lowering::LowerNode(GenTreePtr* ppTree, Compiler::fgWalkData* data) #endif // FEATURE_SIMD default: - return; + break; } + + return node->gtNext; } /** -- Switch Lowering -- @@ -379,11 +249,12 @@ void Lowering::LowerNode(GenTreePtr* ppTree, Compiler::fgWalkData* data) * Now, the way we morph a GT_SWITCH node into this lowered switch table node form is the following: * * Input: GT_SWITCH (inside a basic block whose Branch Type is BBJ_SWITCH) - * |_____ expr (an arbitrary complex GT_NODE that represents the switch index) + * |_____ expr (an arbitrarily complex GT_NODE that represents the switch index) * * This gets transformed into the following statements inside a BBJ_COND basic block (the target would be * the default case of the switch in case the conditional is evaluated to true). * + * ----- original block, transformed * GT_ASG * |_____ tempLocal (a new temporary local variable used to store the switch index) * |_____ expr (the index expression) @@ -395,6 +266,7 @@ void Lowering::LowerNode(GenTreePtr* ppTree, Compiler::fgWalkData* data) * that happens to be the highest index in the jump table). * |___ tempLocal (The local variable were we stored the index expression). * + * ----- new basic block * GT_SWITCH_TABLE * |_____ tempLocal * |_____ jumpTable (a new jump table node that now LSRA can allocate registers for explicitly @@ -414,21 +286,21 @@ void Lowering::LowerNode(GenTreePtr* ppTree, Compiler::fgWalkData* data) * InstrGroups downstream. */ -void Lowering::LowerSwitch(GenTreePtr* pTree) +GenTree* Lowering::LowerSwitch(GenTree* node) { unsigned jumpCnt; unsigned targetCnt; BasicBlock** jumpTab; - GenTreePtr tree = *pTree; - assert(tree->gtOper == GT_SWITCH); + assert(node->gtOper == GT_SWITCH); // The first step is to build the default case conditional construct that is // shared between both kinds of expansion of the switch node. - // To avoid confusion, we'll alias compCurBB to originalSwitchBB + // To avoid confusion, we'll alias m_block to originalSwitchBB // that represents the node we're morphing. - BasicBlock* originalSwitchBB = comp->compCurBB; + BasicBlock* originalSwitchBB = m_block; + LIR::Range& switchBBRange = LIR::AsRange(originalSwitchBB); // jumpCnt is the number of elements in the jump table array. // jumpTab is the actual pointer to the jump table array. @@ -437,6 +309,14 @@ void Lowering::LowerSwitch(GenTreePtr* pTree) jumpTab = originalSwitchBB->bbJumpSwt->bbsDstTab; targetCnt = originalSwitchBB->NumSucc(comp); +// GT_SWITCH must be a top-level node with no use. +#ifdef DEBUG + { + LIR::Use use; + assert(!switchBBRange.TryGetUse(node, &use)); + } +#endif + JITDUMP("Lowering switch BB%02u, %d cases\n", originalSwitchBB->bbNum, jumpCnt); // Handle a degenerate case: if the switch has only a default case, just convert it @@ -461,40 +341,42 @@ void Lowering::LowerSwitch(GenTreePtr* pTree) { (void)comp->fgRemoveRefPred(jumpTab[i], originalSwitchBB); } + // We have to get rid of the GT_SWITCH node but a child might have side effects so just assign // the result of the child subtree to a temp. - GenTree* store = CreateLocalTempAsg(tree->gtOp.gtOp1, 1); - tree->InsertAfterSelf(store, comp->compCurStmt->AsStmt()); - Compiler::fgSnipNode(comp->compCurStmt->AsStmt(), tree); - *pTree = store; + GenTree* rhs = node->gtOp.gtOp1; - return; + unsigned lclNum = comp->lvaGrabTemp(true DEBUGARG("Lowering is creating a new local variable")); + comp->lvaSortAgain = true; + comp->lvaTable[lclNum].lvType = rhs->TypeGet(); + comp->lvaTable[lclNum].lvRefCnt = 1; + + GenTreeLclVar* store = + new (comp, GT_STORE_LCL_VAR) GenTreeLclVar(GT_STORE_LCL_VAR, rhs->TypeGet(), lclNum, BAD_IL_OFFSET); + store->gtOp1 = rhs; + store->gtFlags = (rhs->gtFlags & GTF_COMMON_MASK); + store->gtFlags |= GTF_VAR_DEF; + + switchBBRange.InsertAfter(node, store); + switchBBRange.Remove(node); + + return store; } noway_assert(jumpCnt >= 2); - // Split the switch node to insert an assignment to a temporary variable. - // Note that 'tree' is the GT_SWITCH, and its op1 may be overwritten by SplitTree - // - GenTreeStmt* asgStmt = comp->fgInsertEmbeddedFormTemp(&(tree->gtOp.gtOp1)); + // Spill the argument to the switch node into a local so that it can be used later. + unsigned blockWeight = originalSwitchBB->getBBWeight(comp); + + LIR::Use use(switchBBRange, &(node->gtOp.gtOp1), node); + use.ReplaceWithLclVar(comp, blockWeight); // GT_SWITCH(indexExpression) is now two statements: // 1. a statement containing 'asg' (for temp = indexExpression) // 2. and a statement with GT_SWITCH(temp) - // The return value of fgInsertEmbeddedFormTemp is stmt 1 - // The 'asg' can either be a GT_ASG or a GT_STORE_LCL_VAR - // 'tree' is still a GT_SWITCH but tree->gtOp.gtOp1 is modified to be 'temp' - - // The asgStmt needs to pickup the IL offsets from the current statement - // - asgStmt->gtStmtILoffsx = comp->compCurStmt->gtStmt.gtStmtILoffsx; -#ifdef DEBUG - asgStmt->gtStmtLastILoffs = comp->compCurStmt->gtStmt.gtStmtLastILoffs; -#endif // DEBUG - - assert(tree->gtOper == GT_SWITCH); - GenTreePtr temp = tree->gtOp.gtOp1; + assert(node->gtOper == GT_SWITCH); + GenTreePtr temp = node->gtOp.gtOp1; assert(temp->gtOper == GT_LCL_VAR); unsigned tempLclNum = temp->gtLclVarCommon.gtLclNum; LclVarDsc* tempVarDsc = comp->lvaTable + tempLclNum; @@ -522,40 +404,39 @@ void Lowering::LowerSwitch(GenTreePtr* pTree) // table is huge and hideous due to the relocation... :( minSwitchTabJumpCnt += 2; #endif // _TARGET_ARM_ + // Once we have the temporary variable, we construct the conditional branch for // the default case. As stated above, this conditional is being shared between // both GT_SWITCH lowering code paths. // This condition is of the form: if (temp > jumpTableLength - 2){ goto jumpTable[jumpTableLength - 1]; } GenTreePtr gtDefaultCaseCond = comp->gtNewOperNode(GT_GT, TYP_INT, comp->gtNewLclvNode(tempLclNum, tempLclType), comp->gtNewIconNode(jumpCnt - 2, TYP_INT)); - // + // Make sure we perform an unsigned comparison, just in case the switch index in 'temp' // is now less than zero 0 (that would also hit the default case). gtDefaultCaseCond->gtFlags |= GTF_UNSIGNED; /* Increment the lvRefCnt and lvRefCntWtd for temp */ - tempVarDsc->incRefCnts(originalSwitchBB->getBBWeight(comp), comp); + tempVarDsc->incRefCnts(blockWeight, comp); GenTreePtr gtDefaultCaseJump = comp->gtNewOperNode(GT_JTRUE, TYP_VOID, gtDefaultCaseCond); - gtDefaultCaseJump->gtFlags = tree->gtFlags; - - GenTreePtr condStmt = - comp->fgNewStmtFromTree(gtDefaultCaseJump, originalSwitchBB, comp->compCurStmt->gtStmt.gtStmtILoffsx); - -#ifdef DEBUG - condStmt->gtStmt.gtStmtLastILoffs = comp->compCurStmt->gtStmt.gtStmtLastILoffs; -#endif // DEBUG + gtDefaultCaseJump->gtFlags = node->gtFlags; - comp->fgInsertStmtAfter(originalSwitchBB, comp->compCurStmt, condStmt); + LIR::Range condRange = LIR::SeqTree(comp, gtDefaultCaseJump); + switchBBRange.InsertAtEnd(std::move(condRange)); - BasicBlock* afterDefCondBlock = comp->fgSplitBlockAfterStatement(originalSwitchBB, condStmt); + BasicBlock* afterDefaultCondBlock = comp->fgSplitBlockAfterNode(originalSwitchBB, condRange.LastNode()); - // afterDefCondBlock is now the switch, and all the switch targets have it as a predecessor. - // originalSwitchBB is now a BBJ_NONE, and there is a predecessor edge in afterDefCondBlock + // afterDefaultCondBlock is now the switch, and all the switch targets have it as a predecessor. + // originalSwitchBB is now a BBJ_NONE, and there is a predecessor edge in afterDefaultCondBlock // representing the fall-through flow from originalSwitchBB. assert(originalSwitchBB->bbJumpKind == BBJ_NONE); - assert(afterDefCondBlock->bbJumpKind == BBJ_SWITCH); - assert(afterDefCondBlock->bbJumpSwt->bbsHasDefault); + assert(originalSwitchBB->bbNext == afterDefaultCondBlock); + assert(afterDefaultCondBlock->bbJumpKind == BBJ_SWITCH); + assert(afterDefaultCondBlock->bbJumpSwt->bbsHasDefault); + assert(afterDefaultCondBlock->isEmpty()); // Nothing here yet. + + // The GT_SWITCH code is still in originalSwitchBB (it will be removed later). // Turn originalSwitchBB into a BBJ_COND. originalSwitchBB->bbJumpKind = BBJ_COND; @@ -563,8 +444,8 @@ void Lowering::LowerSwitch(GenTreePtr* pTree) // Fix the pred for the default case: the default block target still has originalSwitchBB // as a predecessor, but the fgSplitBlockAfterStatement() moved all predecessors to point - // to afterDefCondBlock. - flowList* oldEdge = comp->fgRemoveRefPred(jumpTab[jumpCnt - 1], afterDefCondBlock); + // to afterDefaultCondBlock. + flowList* oldEdge = comp->fgRemoveRefPred(jumpTab[jumpCnt - 1], afterDefaultCondBlock); comp->fgAddRefPred(jumpTab[jumpCnt - 1], originalSwitchBB, oldEdge); // If we originally had 2 unique successors, check to see whether there is a unique @@ -596,17 +477,17 @@ void Lowering::LowerSwitch(GenTreePtr* pTree) for (unsigned i = 1; i < jumpCnt - 1; ++i) { assert(jumpTab[i] == uniqueSucc); - (void)comp->fgRemoveRefPred(uniqueSucc, afterDefCondBlock); + (void)comp->fgRemoveRefPred(uniqueSucc, afterDefaultCondBlock); } - if (afterDefCondBlock->bbNext == uniqueSucc) + if (afterDefaultCondBlock->bbNext == uniqueSucc) { - afterDefCondBlock->bbJumpKind = BBJ_NONE; - afterDefCondBlock->bbJumpDest = nullptr; + afterDefaultCondBlock->bbJumpKind = BBJ_NONE; + afterDefaultCondBlock->bbJumpDest = nullptr; } else { - afterDefCondBlock->bbJumpKind = BBJ_ALWAYS; - afterDefCondBlock->bbJumpDest = uniqueSucc; + afterDefaultCondBlock->bbJumpKind = BBJ_ALWAYS; + afterDefaultCondBlock->bbJumpDest = uniqueSucc; } } // If the number of possible destinations is small enough, we proceed to expand the switch @@ -616,7 +497,7 @@ void Lowering::LowerSwitch(GenTreePtr* pTree) { // Lower the switch into a series of compare and branch IR trees. // - // In this case we will morph the tree in the following way: + // In this case we will morph the node in the following way: // 1. Generate a JTRUE statement to evaluate the default case. (This happens above.) // 2. Start splitting the switch basic block into subsequent basic blocks, each of which will contain // a statement that is responsible for performing a comparison of the table index and conditional @@ -624,11 +505,12 @@ void Lowering::LowerSwitch(GenTreePtr* pTree) JITDUMP("Lowering switch BB%02u: using compare/branch expansion\n", originalSwitchBB->bbNum); - // We'll use 'afterDefCondBlock' for the first conditional. After that, we'll add new + // We'll use 'afterDefaultCondBlock' for the first conditional. After that, we'll add new // blocks. If we end up not needing it at all (say, if all the non-default cases just fall through), // we'll delete it. - bool fUsedAfterDefCondBlock = false; - BasicBlock* currentBlock = afterDefCondBlock; + bool fUsedAfterDefaultCondBlock = false; + BasicBlock* currentBlock = afterDefaultCondBlock; + LIR::Range* currentBBRange = &LIR::AsRange(currentBlock); // Walk to entries 0 to jumpCnt - 1. If a case target follows, ignore it and let it fall through. // If no case target follows, the last one doesn't need to be a compare/branch: it can be an @@ -640,7 +522,7 @@ void Lowering::LowerSwitch(GenTreePtr* pTree) // Remove the switch from the predecessor list of this case target's block. // We'll add the proper new predecessor edge later. - flowList* oldEdge = comp->fgRemoveRefPred(jumpTab[i], afterDefCondBlock); + flowList* oldEdge = comp->fgRemoveRefPred(jumpTab[i], afterDefaultCondBlock); if (jumpTab[i] == followingBB) { @@ -650,17 +532,18 @@ void Lowering::LowerSwitch(GenTreePtr* pTree) } // We need a block to put in the new compare and/or branch. - // If we haven't used the afterDefCondBlock yet, then use that. - if (fUsedAfterDefCondBlock) + // If we haven't used the afterDefaultCondBlock yet, then use that. + if (fUsedAfterDefaultCondBlock) { BasicBlock* newBlock = comp->fgNewBBafter(BBJ_NONE, currentBlock, true); comp->fgAddRefPred(newBlock, currentBlock); // The fall-through predecessor. - currentBlock = newBlock; + currentBlock = newBlock; + currentBBRange = &LIR::AsRange(currentBlock); } else { - assert(currentBlock == afterDefCondBlock); - fUsedAfterDefCondBlock = true; + assert(currentBlock == afterDefaultCondBlock); + fUsedAfterDefaultCondBlock = true; } // We're going to have a branch, either a conditional or unconditional, @@ -696,11 +579,11 @@ void Lowering::LowerSwitch(GenTreePtr* pTree) comp->gtNewOperNode(GT_EQ, TYP_INT, comp->gtNewLclvNode(tempLclNum, tempLclType), comp->gtNewIconNode(i, TYP_INT)); /* Increment the lvRefCnt and lvRefCntWtd for temp */ - tempVarDsc->incRefCnts(originalSwitchBB->getBBWeight(comp), comp); + tempVarDsc->incRefCnts(blockWeight, comp); GenTreePtr gtCaseBranch = comp->gtNewOperNode(GT_JTRUE, TYP_VOID, gtCaseCond); - GenTreePtr gtCaseStmt = comp->fgNewStmtFromTree(gtCaseBranch, currentBlock); - comp->fgInsertStmtAtEnd(currentBlock, gtCaseStmt); + LIR::Range caseRange = LIR::SeqTree(comp, gtCaseBranch); + currentBBRange->InsertAtEnd(std::move(condRange)); } } @@ -712,13 +595,13 @@ void Lowering::LowerSwitch(GenTreePtr* pTree) comp->fgAddRefPred(currentBlock->bbNext, currentBlock); } - if (!fUsedAfterDefCondBlock) + if (!fUsedAfterDefaultCondBlock) { // All the cases were fall-through! We don't need this block. // Convert it from BBJ_SWITCH to BBJ_NONE and unset the BBF_DONT_REMOVE flag // so fgRemoveBlock() doesn't complain. JITDUMP("Lowering switch BB%02u: all switch cases were fall-through\n", originalSwitchBB->bbNum); - assert(currentBlock == afterDefCondBlock); + assert(currentBlock == afterDefaultCondBlock); assert(currentBlock->bbJumpKind == BBJ_SWITCH); currentBlock->bbJumpKind = BBJ_NONE; currentBlock->bbFlags &= ~BBF_DONT_REMOVE; @@ -744,50 +627,43 @@ void Lowering::LowerSwitch(GenTreePtr* pTree) comp->gtNewOperNode(GT_SWITCH_TABLE, TYP_VOID, comp->gtNewLclvNode(tempLclNum, tempLclType), comp->gtNewJmpTableNode()); /* Increment the lvRefCnt and lvRefCntWtd for temp */ - tempVarDsc->incRefCnts(originalSwitchBB->getBBWeight(comp), comp); + tempVarDsc->incRefCnts(blockWeight, comp); // this block no longer branches to the default block - afterDefCondBlock->bbJumpSwt->removeDefault(); - comp->fgInvalidateSwitchDescMapEntry(afterDefCondBlock); + afterDefaultCondBlock->bbJumpSwt->removeDefault(); + comp->fgInvalidateSwitchDescMapEntry(afterDefaultCondBlock); - GenTreeStmt* stmt = comp->fgNewStmtFromTree(gtTableSwitch); - comp->fgInsertStmtAtEnd(afterDefCondBlock, stmt); + LIR::Range& afterDefaultCondBBRange = LIR::AsRange(afterDefaultCondBlock); + afterDefaultCondBBRange.InsertAtEnd(LIR::SeqTree(comp, gtTableSwitch)); } - // Get rid of the original GT_SWITCH. - comp->fgRemoveStmt(originalSwitchBB, comp->compCurStmt, false); - // Set compCurStmt. If asgStmt is top-level, we need to set it to that, so that any of - // its embedded statements are traversed. Otherwise, set it to condStmt, which will - // contain the embedded asgStmt. - if (asgStmt->gtStmtIsTopLevel()) - { - comp->compCurStmt = asgStmt; - } - else - { -#ifdef DEBUG - GenTree* nextStmt = condStmt->gtNext; - while (nextStmt != nullptr && nextStmt != asgStmt) - { - nextStmt = nextStmt->gtNext; - } - assert(nextStmt == asgStmt); -#endif // DEBUG - comp->compCurStmt = condStmt; - } + GenTree* next = node->gtNext; + + // Get rid of the GT_SWITCH(temp). + switchBBRange.Remove(node->gtOp.gtOp1); + switchBBRange.Remove(node); + + return next; } -// splice in a unary op, between the child and parent -// resulting in parent->newNode->child -void Lowering::SpliceInUnary(GenTreePtr parent, GenTreePtr* ppChild, GenTreePtr newNode) +// NOTE: this method deliberately does not update the call arg table. It must only +// be used by NewPutArg and LowerArg; these functions are responsible for updating +// the call arg table as necessary. +void Lowering::ReplaceArgWithPutArgOrCopy(GenTree** argSlot, GenTree* putArgOrCopy) { - GenTreePtr oldChild = *ppChild; + assert(argSlot != nullptr); + assert(*argSlot != nullptr); + assert(putArgOrCopy->OperGet() == GT_PUTARG_REG || putArgOrCopy->OperGet() == GT_PUTARG_STK || + putArgOrCopy->OperGet() == GT_COPY); - // Replace tree in the parent node - *ppChild = newNode; - newNode->gtOp.gtOp1 = oldChild; + GenTree* arg = *argSlot; - oldChild->InsertAfterSelf(newNode); + // Replace the argument with the putarg/copy + *argSlot = putArgOrCopy; + putArgOrCopy->gtOp.gtOp1 = arg; + + // Insert the putarg/copy into the block + BlockRange().InsertAfter(arg, putArgOrCopy); } //------------------------------------------------------------------------ @@ -958,7 +834,7 @@ GenTreePtr Lowering::NewPutArg(GenTreeCall* call, GenTreePtr arg, fgArgTabEntryP newOper->CopyCosts(argListPtr->gtOp.gtOp1); // Splice in the new GT_PUTARG_REG node in the GT_LIST - SpliceInUnary(argListPtr, &argListPtr->gtOp.gtOp1, newOper); + ReplaceArgWithPutArgOrCopy(&argListPtr->gtOp.gtOp1, newOper); } // Just return arg. The GT_LIST is not replaced. @@ -991,7 +867,7 @@ GenTreePtr Lowering::NewPutArg(GenTreeCall* call, GenTreePtr arg, fgArgTabEntryP newOper->CopyCosts(argListPtr->gtOp.gtOp1); // Splice in the new GT_PUTARG_REG node in the GT_LIST - SpliceInUnary(argListPtr, &argListPtr->gtOp.gtOp1, newOper); + ReplaceArgWithPutArgOrCopy(&argListPtr->gtOp.gtOp1, newOper); } // Just return arg. The GT_LIST is not replaced. @@ -1087,10 +963,20 @@ GenTreePtr Lowering::NewPutArg(GenTreeCall* call, GenTreePtr arg, fgArgTabEntryP return putArg; } -// lower one arg of a call -// this currently entails splicing in a "putarg" node in between the arg evaluation and call -// These are the point at which the source is consumed and the values transition from control -// of the register allocator to the calling convention. +//------------------------------------------------------------------------ +// LowerArg: Lower one argument of a call. This entails splicing a "putarg" node between +// the argument evaluation and the call. This is the point at which the source is +// consumed and the value transitions from control of the register allocator to the calling +// convention. +// +// Arguments: +// call - The call node +// ppArg - Pointer to the call argument pointer. We might replace the call argument by +// changing *ppArg. +// +// Return Value: +// None. +// void Lowering::LowerArg(GenTreeCall* call, GenTreePtr* ppArg) { GenTreePtr arg = *ppArg; @@ -1100,118 +986,91 @@ void Lowering::LowerArg(GenTreeCall* call, GenTreePtr* ppArg) // No assignments should remain by Lowering. assert(!arg->OperIsAssignment()); + assert(!arg->OperIsPutArgStk()); - // assignments/stores at this level are not really placing an arg - // they are setting up temporary locals that will later be placed into - // outgoing regs or stack - if (!arg->OperIsAssignment() && !arg->OperIsStore() && !arg->IsArgPlaceHolderNode() && !arg->IsNothingNode() && -#ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING - !arg->OperIsPutArgStk() && -#endif // FEATURE_UNIX_AMD64_STRUCT_PASSING - !arg->OperIsCopyBlkOp()) // these are de facto placeholders (apparently) + // Assignments/stores at this level are not really placing an argument. + // They are setting up temporary locals that will later be placed into + // outgoing regs or stack. + if (arg->OperIsStore() || arg->IsArgPlaceHolderNode() || arg->IsNothingNode() || arg->OperIsCopyBlkOp()) { - fgArgTabEntryPtr info = comp->gtArgEntryByNode(call, arg); - assert(info->node == arg); - bool isReg = (info->regNum != REG_STK); - var_types type = arg->TypeGet(); + return; + } - if (varTypeIsSmall(type)) - { - // Normalize 'type', it represents the item that we will be storing in the Outgoing Args - type = TYP_INT; - } + fgArgTabEntryPtr info = comp->gtArgEntryByNode(call, arg); + assert(info->node == arg); + bool isReg = (info->regNum != REG_STK); + var_types type = arg->TypeGet(); - GenTreePtr putArg; + if (varTypeIsSmall(type)) + { + // Normalize 'type', it represents the item that we will be storing in the Outgoing Args + type = TYP_INT; + } + + GenTreePtr putArg; - // if we hit this we are probably double-lowering - assert(arg->gtOper != GT_PUTARG_REG && arg->gtOper != GT_PUTARG_STK); + // If we hit this we are probably double-lowering. + assert(!arg->OperIsPutArg()); #if !defined(_TARGET_64BIT_) - if (varTypeIsLong(type)) + if (varTypeIsLong(type)) + { + if (isReg) { - if (isReg) - { - NYI("Lowering of long register argument"); - } - // For longs, we will create two PUTARG_STKs below the GT_LONG. - // This is because the lo/hi values will be marked localDefUse, and we need to ensure that - // they are pushed onto the stack as soon as they are created. - // We also need to reverse the order, since the hi argument needs to be pushed first. - noway_assert(arg->OperGet() == GT_LONG); - GenTreePtr argLo = arg->gtGetOp1(); - GenTreePtr argHi = arg->gtGetOp2(); + NYI("Lowering of long register argument"); + } - NYI_IF((argHi->OperGet() == GT_ADD_HI) || (argHi->OperGet() == GT_SUB_HI) || (argHi->OperGet() == GT_NEG), - "Hi and Lo cannot be reordered"); + // For longs, we will create two PUTARG_STKs below the GT_LONG. The hi argument needs to + // be pushed first, so the hi PUTARG_STK will precede the lo PUTARG_STK in execution order. + noway_assert(arg->OperGet() == GT_LONG); + GenTreePtr argLo = arg->gtGetOp1(); + GenTreePtr argHi = arg->gtGetOp2(); - GenTreePtr putArgLo = NewPutArg(call, argLo, info, type); - GenTreePtr putArgHi = NewPutArg(call, argHi, info, type); + GenTreePtr putArgLo = NewPutArg(call, argLo, info, type); + GenTreePtr putArgHi = NewPutArg(call, argHi, info, type); - arg->gtOp.gtOp1 = putArgLo; - arg->gtOp.gtOp2 = putArgHi; + arg->gtOp.gtOp1 = putArgLo; + arg->gtOp.gtOp2 = putArgHi; - // Now, reorder the arguments and insert the putArg in the right place. + BlockRange().InsertBefore(arg, putArgHi, putArgLo); - GenTreePtr argLoFirst = comp->fgGetFirstNode(argLo); - GenTreePtr argHiFirst = comp->fgGetFirstNode(argHi); - GenTreePtr argLoPrev = argLoFirst->gtPrev; - noway_assert(argHiFirst->gtPrev == argLo); - noway_assert(arg->gtPrev == argHi); + // The execution order now looks like this: + // argLoPrev <-> argLoFirst ... argLo <-> argHiFirst ... argHi <-> putArgHi <-> putArgLo <-> arg(GT_LONG) - argHiFirst->gtPrev = argLoPrev; - if (argLoPrev != nullptr) - { - argLoPrev->gtNext = argHiFirst; - } - else - { - assert(comp->compCurStmt->gtStmt.gtStmtList == argLoFirst); - comp->compCurStmt->gtStmt.gtStmtList = argHiFirst; - } - argHi->gtNext = putArgHi; - putArgHi->gtPrev = argHi; - putArgHi->gtNext = argLoFirst; - argLoFirst->gtPrev = putArgHi; - argLo->gtNext = putArgLo; - putArgLo->gtPrev = argLo; - putArgLo->gtNext = arg; - arg->gtPrev = putArgLo; - - assert((arg->gtFlags & GTF_REVERSE_OPS) == 0); - arg->gtFlags |= GTF_REVERSE_OPS; - } - else + assert((arg->gtFlags & GTF_REVERSE_OPS) == 0); + arg->gtFlags |= GTF_REVERSE_OPS; // We consume the high arg (op2) first. + } + else #endif // !defined(_TARGET_64BIT_) - { + { #ifdef _TARGET_ARM64_ - // For vararg call, reg args should be all integer. - // Insert a copy to move float value to integer register. - if (call->IsVarargs() && varTypeIsFloating(type)) - { - var_types intType = (type == TYP_DOUBLE) ? TYP_LONG : TYP_INT; - GenTreePtr intArg = comp->gtNewOperNode(GT_COPY, intType, arg); + // For vararg call, reg args should be all integer. + // Insert a copy to move float value to integer register. + if (call->IsVarargs() && varTypeIsFloating(type)) + { + var_types intType = (type == TYP_DOUBLE) ? TYP_LONG : TYP_INT; + GenTreePtr intArg = comp->gtNewOperNode(GT_COPY, intType, arg); - intArg->CopyCosts(arg); - info->node = intArg; - SpliceInUnary(call, ppArg, intArg); + intArg->CopyCosts(arg); + info->node = intArg; + ReplaceArgWithPutArgOrCopy(ppArg, intArg); - // Update arg/type with new ones. - arg = intArg; - type = intType; - } + // Update arg/type with new ones. + arg = intArg; + type = intType; + } #endif - putArg = NewPutArg(call, arg, info, type); + putArg = NewPutArg(call, arg, info, type); - // In the case of register passable struct (in one or two registers) - // the NewPutArg returns a new node (GT_PUTARG_REG or a GT_LIST with two GT_PUTARG_REGs.) - // If an extra node is returned, splice it in the right place in the tree. - if (arg != putArg) - { - // putArg and arg are equals if arg is GT_LIST (a list of multiple LCL_FLDs to be passed in registers.) - SpliceInUnary(call, ppArg, putArg); - } + // In the case of register passable struct (in one or two registers) + // the NewPutArg returns a new node (GT_PUTARG_REG or a GT_LIST with two GT_PUTARG_REGs.) + // If an extra node is returned, splice it in the right place in the tree. + if (arg != putArg) + { + // putArg and arg are equals if arg is GT_LIST (a list of multiple LCL_FLDs to be passed in registers.) + ReplaceArgWithPutArgOrCopy(ppArg, putArg); } } } @@ -1258,14 +1117,6 @@ GenTree* Lowering::AddrGen(void* addr, regNumber reg) return AddrGen((ssize_t)addr, reg); } -// do some common operations on trees before they are inserted as top level statements -GenTreeStmt* Lowering::LowerMorphAndSeqTree(GenTree* tree) -{ - tree = comp->fgMorphTree(tree); - GenTreeStmt* stmt = comp->fgNewStmtFromTree(tree); - return stmt; -} - // do lowering steps for a call // this includes: // - adding the placement nodes (either stack or register variety) for arguments @@ -1275,12 +1126,10 @@ GenTreeStmt* Lowering::LowerMorphAndSeqTree(GenTree* tree) // void Lowering::LowerCall(GenTree* node) { - GenTreeCall* call = node->AsCall(); - GenTreeStmt* callStmt = comp->compCurStmt->AsStmt(); - assert(comp->fgTreeIsInStmt(call, callStmt)); + GenTreeCall* call = node->AsCall(); JITDUMP("lowering call (before):\n"); - DISPTREE(call); + DISPTREERANGE(BlockRange(), call); JITDUMP("\n"); LowerArgsForCall(call); @@ -1335,80 +1184,72 @@ void Lowering::LowerCall(GenTree* node) } } -#ifdef DEBUG - comp->fgDebugCheckNodeLinks(comp->compCurBB, comp->compCurStmt); -#endif - - if (result) - { - // The controlExpr is newly constructed, so we can use tree sequencing - comp->gtSetEvalOrder(result); - comp->fgSetTreeSeq(result, nullptr); - - JITDUMP("results of lowering call:\n"); - DISPTREE(result); - } - if (call->IsTailCallViaHelper()) { // Either controlExpr or gtCallAddr must contain real call target. if (result == nullptr) { + assert(call->gtCallType == CT_INDIRECT); assert(call->gtCallAddr != nullptr); result = call->gtCallAddr; } result = LowerTailCallViaHelper(call, result); - - if (result != nullptr) - { - // We got a new call target constructed, so resequence it. - comp->gtSetEvalOrder(result); - comp->fgSetTreeSeq(result, nullptr); - JITDUMP("results of lowering tail call via helper:\n"); - DISPTREE(result); - } } else if (call->IsFastTailCall()) { LowerFastTailCall(call); } - if (result) + if (result != nullptr) { + LIR::Range resultRange = LIR::SeqTree(comp, result); + + JITDUMP("results of lowering call:\n"); + DISPRANGE(resultRange); + GenTree* insertionPoint = call; if (!call->IsTailCallViaHelper()) { // The controlExpr should go before the gtCallCookie and the gtCallAddr, if they exist + // + // TODO-LIR: find out what's really required here, as this is currently a tree order + // dependency. if (call->gtCallType == CT_INDIRECT) { + bool isClosed = false; if (call->gtCallCookie != nullptr) { - insertionPoint = comp->fgGetFirstNode(call->gtCallCookie); +#ifdef DEBUG + GenTree* firstCallAddrNode = BlockRange().GetTreeRange(call->gtCallAddr, &isClosed).FirstNode(); + assert(isClosed); + assert(call->gtCallCookie->Precedes(firstCallAddrNode)); +#endif // DEBUG + + insertionPoint = BlockRange().GetTreeRange(call->gtCallCookie, &isClosed).FirstNode(); + assert(isClosed); } else if (call->gtCallAddr != nullptr) { - insertionPoint = comp->fgGetFirstNode(call->gtCallAddr); + insertionPoint = BlockRange().GetTreeRange(call->gtCallAddr, &isClosed).FirstNode(); + assert(isClosed); } } } - comp->fgInsertTreeInListBefore(result, insertionPoint, callStmt); + BlockRange().InsertBefore(insertionPoint, std::move(resultRange)); + call->gtControlExpr = result; } #endif //!_TARGET_ARM_ -#ifdef DEBUG - comp->fgDebugCheckNodeLinks(comp->compCurBB, callStmt); -#endif - if (comp->opts.IsJit64Compat()) { CheckVSQuirkStackPaddingNeeded(call); } JITDUMP("lowering call (after):\n"); - DISPTREE(call); + DISPTREERANGE(BlockRange(), call); JITDUMP("\n"); } @@ -1545,7 +1386,7 @@ void Lowering::CheckVSQuirkStackPaddingNeeded(GenTreeCall* call) // control expr | +--* const(h) long 0x7ffe8e910e98 ftn REG NA // \--* call void System.Runtime.Remoting.Identity.RemoveAppNameOrAppGuidIfNecessary $VN.Void // -// In this case, the GT_PUTARG_REG src is a nested call. We need to put the embedded statement after that call +// In this case, the GT_PUTARG_REG src is a nested call. We need to put the instructions after that call // (as shown). We assume that of all the GT_PUTARG_*, only the first one can have a nested call. // // Params: @@ -1595,9 +1436,8 @@ void Lowering::InsertProfTailCallHook(GenTreeCall* call, GenTree* insertionPoint } assert(insertionPoint != nullptr); - GenTreeStmt* callStmt = comp->compCurStmt->AsStmt(); - GenTreePtr profHookNode = new (comp, GT_PROF_HOOK) GenTree(GT_PROF_HOOK, TYP_VOID); - comp->fgInsertTreeBeforeAsEmbedded(profHookNode, insertionPoint, callStmt, comp->compCurBB); + GenTreePtr profHookNode = new (comp, GT_PROF_HOOK) GenTree(GT_PROF_HOOK, TYP_VOID); + BlockRange().InsertBefore(insertionPoint, profHookNode); } // Lower fast tail call implemented as epilog+jmp. @@ -1700,7 +1540,6 @@ void Lowering::LowerFastTailCall(GenTreeCall* call) // // The below logic is meant to detect cases like this and introduce // temps to set up args correctly for Callee. - GenTreeStmt* callStmt = comp->compCurStmt->AsStmt(); for (int i = 0; i < putargs.Height(); i++) { @@ -1779,7 +1618,7 @@ void Lowering::LowerFastTailCall(GenTreeCall* call) GenTreeLclVar* local = new (comp, GT_LCL_VAR) GenTreeLclVar(GT_LCL_VAR, tmpType, callerArgLclNum, BAD_IL_OFFSET); GenTree* assignExpr = comp->gtNewTempAssign(tmpLclNum, local); - comp->fgInsertTreeBeforeAsEmbedded(assignExpr, firstPutArgStk, callStmt, comp->compCurBB); + BlockRange().InsertBefore(firstPutArgStk, LIR::SeqTree(comp, assignExpr)); } } @@ -1790,7 +1629,7 @@ void Lowering::LowerFastTailCall(GenTreeCall* call) if (firstPutArgStk != nullptr) { startNonGCNode = new (comp, GT_START_NONGC) GenTree(GT_START_NONGC, TYP_VOID); - comp->fgInsertTreeBeforeAsEmbedded(startNonGCNode, firstPutArgStk, callStmt, comp->compCurBB); + BlockRange().InsertBefore(firstPutArgStk, startNonGCNode); // Gc-interruptability in the following case: // foo(a, b, c, d, e) { bar(a, b, c, d, e); } @@ -1806,7 +1645,7 @@ void Lowering::LowerFastTailCall(GenTreeCall* call) { assert(comp->fgFirstBB == comp->compCurBB); GenTreePtr noOp = new (comp, GT_NO_OP) GenTree(GT_NO_OP, TYP_VOID); - comp->fgInsertTreeBeforeAsEmbedded(noOp, startNonGCNode, callStmt, comp->compCurBB); + BlockRange().InsertBefore(startNonGCNode, noOp); } } @@ -1882,14 +1721,21 @@ GenTree* Lowering::LowerTailCallViaHelper(GenTreeCall* call, GenTree* callTarget InsertPInvokeMethodEpilog(comp->compCurBB DEBUGARG(call)); } - // Remove gtCallAddr from execution order if one present. - GenTreeStmt* callStmt = comp->compCurStmt->AsStmt(); + // Remove gtCallAddr from execution order if present. if (call->gtCallType == CT_INDIRECT) { assert(call->gtCallAddr != nullptr); - comp->fgDeleteTreeFromList(callStmt, call->gtCallAddr); + + bool isClosed; + LIR::ReadOnlyRange callAddrRange = BlockRange().GetTreeRange(call->gtCallAddr, &isClosed); + assert(isClosed); + + BlockRange().Remove(std::move(callAddrRange)); } + // The callTarget tree needs to be sequenced. + LIR::Range callTargetRange = LIR::SeqTree(comp, callTarget); + fgArgTabEntry* argEntry; #if defined(_TARGET_AMD64_) @@ -1910,8 +1756,14 @@ GenTree* Lowering::LowerTailCallViaHelper(GenTreeCall* call, GenTree* callTarget assert(argEntry->node->gtOper == GT_PUTARG_REG); GenTree* secondArg = argEntry->node->gtOp.gtOp1; - comp->fgInsertTreeInListAfter(callTarget, secondArg, callStmt); - comp->fgDeleteTreeFromList(callStmt, secondArg); + BlockRange().InsertAfter(secondArg, std::move(callTargetRange)); + + bool isClosed; + LIR::ReadOnlyRange secondArgRange = BlockRange().GetTreeRange(secondArg, &isClosed); + assert(isClosed); + + BlockRange().Remove(std::move(secondArgRange)); + argEntry->node->gtOp.gtOp1 = callTarget; #elif defined(_TARGET_X86_) @@ -1932,8 +1784,12 @@ GenTree* Lowering::LowerTailCallViaHelper(GenTreeCall* call, GenTree* callTarget assert(argEntry->node->gtOper == GT_PUTARG_STK); GenTree* arg0 = argEntry->node->gtOp.gtOp1; - comp->fgInsertTreeInListAfter(callTarget, arg0, callStmt); - comp->fgDeleteTreeFromList(callStmt, arg0); + BlockRange().InsertAfter(arg0, std::move(callTargetRange)); + + bool isClosed; + LIR::ReadOnlyRange secondArgRange = BlockRange().GetTreeRange(arg0, &isClosed); + assert(isClosed); + argEntry->node->gtOp.gtOp1 = callTarget; // arg 1 == flags @@ -1997,7 +1853,7 @@ void Lowering::LowerJmpMethod(GenTree* jmp) assert(jmp->OperGet() == GT_JMP); JITDUMP("lowering GT_JMP\n"); - DISPTREE(jmp); + DISPNODE(jmp); JITDUMP("============"); // If PInvokes are in-lined, we have to remember to execute PInvoke method epilog anywhere that @@ -2014,7 +1870,7 @@ void Lowering::LowerRet(GenTree* ret) assert(ret->OperGet() == GT_RETURN); JITDUMP("lowering GT_RETURN\n"); - DISPTREE(ret); + DISPNODE(ret); JITDUMP("============"); // Method doing PInvokes has exactly one return block unless it has tail calls. @@ -2196,13 +2052,13 @@ GenTree* Lowering::LowerDelegateInvoke(GenTreeCall* call) else #endif // _TARGET_X86_ { - unsigned delegateInvokeTmp = comp->lvaGrabTemp(true DEBUGARG("delegate invoke call")); - GenTreeStmt* newStmt = comp->fgInsertEmbeddedFormTemp(&thisArgNode->gtOp.gtOp1, delegateInvokeTmp); - originalThisExpr = thisArgNode->gtOp.gtOp1; // it's changed; reload it. - newStmt->gtFlags |= GTF_STMT_SKIP_LOWER; // we're in postorder so we have already processed this subtree - GenTree* stLclVar = newStmt->gtStmtExpr; - assert(stLclVar->OperIsLocalStore()); - lclNum = stLclVar->AsLclVarCommon()->GetLclNum(); + unsigned delegateInvokeTmp = comp->lvaGrabTemp(true DEBUGARG("delegate invoke call")); + + LIR::Use thisExprUse(BlockRange(), &thisArgNode->gtOp.gtOp1, thisArgNode); + thisExprUse.ReplaceWithLclVar(comp, m_block->getBBWeight(comp), delegateInvokeTmp); + + originalThisExpr = thisExprUse.Def(); // it's changed; reload it. + lclNum = delegateInvokeTmp; } // replace original expression feeding into thisPtr with @@ -2210,11 +2066,12 @@ GenTree* Lowering::LowerDelegateInvoke(GenTreeCall* call) GenTree* newThisAddr = new (comp, GT_LEA) GenTreeAddrMode(TYP_REF, originalThisExpr, nullptr, 0, comp->eeGetEEInfo()->offsetOfDelegateInstance); - originalThisExpr->InsertAfterSelf(newThisAddr); GenTree* newThis = comp->gtNewOperNode(GT_IND, TYP_REF, newThisAddr); newThis->SetCosts(IND_COST_EX, 2); - newThisAddr->InsertAfterSelf(newThis); + + BlockRange().InsertAfter(originalThisExpr, newThisAddr, newThis); + thisArgNode->gtOp.gtOp1 = newThis; // the control target is @@ -2409,6 +2266,8 @@ void Lowering::InsertPInvokeMethodProlog() JITDUMP("======= Inserting PInvoke method prolog\n"); + LIR::Range& firstBlockRange = LIR::AsRange(comp->fgFirstBB); + const CORINFO_EE_INFO* pInfo = comp->eeGetEEInfo(); const CORINFO_EE_INFO::InlinedCallFrameInfo& callFrameInfo = pInfo->inlinedCallFrameInfo; @@ -2441,10 +2300,11 @@ void Lowering::InsertPInvokeMethodProlog() store->gtOp.gtOp1 = call; store->gtFlags |= GTF_VAR_DEF; - GenTreeStmt* stmt = LowerMorphAndSeqTree(store); - comp->fgInsertStmtAtBeg(comp->fgFirstBB, stmt); - GenTree* lastStmt = stmt; - DISPTREE(lastStmt); + GenTree* insertionPoint = firstBlockRange.FirstNonPhiOrCatchArgNode(); + + comp->fgMorphTree(store); + firstBlockRange.InsertBefore(insertionPoint, LIR::SeqTree(comp, store)); + DISPTREERANGE(firstBlockRange, store); #ifndef _TARGET_X86_ // For x86, this step is done at the call site (due to stack pointer not being static in the // function). @@ -2456,10 +2316,8 @@ void Lowering::InsertPInvokeMethodProlog() GenTreeLclFld(GT_STORE_LCL_FLD, TYP_I_IMPL, comp->lvaInlinedPInvokeFrameVar, callFrameInfo.offsetOfCallSiteSP); storeSP->gtOp1 = PhysReg(REG_SPBASE); - GenTreeStmt* storeSPStmt = LowerMorphAndSeqTree(storeSP); - comp->fgInsertStmtAfter(comp->fgFirstBB, lastStmt, storeSPStmt); - lastStmt = storeSPStmt; - DISPTREE(lastStmt); + firstBlockRange.InsertBefore(insertionPoint, LIR::SeqTree(comp, storeSP)); + DISPTREERANGE(firstBlockRange, storeSP); #endif // !_TARGET_X86_ @@ -2471,10 +2329,8 @@ void Lowering::InsertPInvokeMethodProlog() callFrameInfo.offsetOfCalleeSavedFP); storeFP->gtOp1 = PhysReg(REG_FPBASE); - GenTreeStmt* storeFPStmt = LowerMorphAndSeqTree(storeFP); - comp->fgInsertStmtAfter(comp->fgFirstBB, lastStmt, storeFPStmt); - lastStmt = storeFPStmt; - DISPTREE(lastStmt); + firstBlockRange.InsertBefore(insertionPoint, LIR::SeqTree(comp, storeFP)); + DISPTREERANGE(firstBlockRange, storeFP); // -------------------------------------------------------- @@ -2483,10 +2339,8 @@ void Lowering::InsertPInvokeMethodProlog() // Push a frame - if we are NOT in an IL stub, this is done right before the call // The init routine sets InlinedCallFrame's m_pNext, so we just set the thead's top-of-stack GenTree* frameUpd = CreateFrameLinkUpdate(PushFrame); - - GenTreeStmt* frameUpdStmt = LowerMorphAndSeqTree(frameUpd); - comp->fgInsertStmtAfter(comp->fgFirstBB, lastStmt, frameUpdStmt); - DISPTREE(frameUpdStmt); + firstBlockRange.InsertBefore(insertionPoint, LIR::SeqTree(comp, frameUpd)); + DISPTREERANGE(firstBlockRange, frameUpd); } } @@ -2518,11 +2372,10 @@ void Lowering::InsertPInvokeMethodEpilog(BasicBlock* returnBB DEBUGARG(GenTreePt assert(((returnBB == comp->genReturnBB) && (returnBB->bbJumpKind == BBJ_RETURN)) || returnBB->endsWithTailCallOrJmp(comp)); - GenTreeStmt* lastTopLevelStmt = comp->fgGetLastTopLevelStmt(returnBB)->AsStmt(); - GenTreePtr lastTopLevelStmtExpr = lastTopLevelStmt->gtStmtExpr; + LIR::Range& returnBlockRange = LIR::AsRange(returnBB); - // Gentree of the last top level stmnt should match. - assert(lastTopLevelStmtExpr == lastExpr); + GenTree* insertionPoint = returnBlockRange.LastNode(); + assert(insertionPoint == lastExpr); // Note: PInvoke Method Epilog (PME) needs to be inserted just before GT_RETURN, GT_JMP or GT_CALL node in execution // order so that it is guaranteed that there will be no further PInvokes after that point in the method. @@ -2549,14 +2402,13 @@ void Lowering::InsertPInvokeMethodEpilog(BasicBlock* returnBB DEBUGARG(GenTreePt // Thread.offsetOfGcState = 0/1 // That is [tcb + offsetOfGcState] = 1 GenTree* storeGCState = SetGCState(1); - comp->fgInsertTreeBeforeAsEmbedded(storeGCState, lastTopLevelStmtExpr, lastTopLevelStmt, returnBB); + returnBlockRange.InsertBefore(insertionPoint, LIR::SeqTree(comp, storeGCState)); if (comp->opts.eeFlags & CORJIT_FLG_IL_STUB) { // Pop the frame, in non-stubs we do this around each PInvoke call GenTree* frameUpd = CreateFrameLinkUpdate(PopFrame); - - comp->fgInsertTreeBeforeAsEmbedded(frameUpd, lastTopLevelStmtExpr, lastTopLevelStmt, returnBB); + returnBlockRange.InsertBefore(insertionPoint, LIR::SeqTree(comp, frameUpd)); } } @@ -2577,7 +2429,9 @@ void Lowering::InsertPInvokeCallProlog(GenTreeCall* call) GenTree* insertBefore = call; if (call->gtCallType == CT_INDIRECT) { - insertBefore = comp->fgGetFirstNode(call->gtCallAddr); + bool isClosed; + insertBefore = BlockRange().GetTreeRange(call->gtCallAddr, &isClosed).FirstNode(); + assert(isClosed); } const CORINFO_EE_INFO::InlinedCallFrameInfo& callFrameInfo = comp->eeGetEEInfo()->inlinedCallFrameInfo; @@ -2598,7 +2452,7 @@ void Lowering::InsertPInvokeCallProlog(GenTreeCall* call) comp->gtNewHelperCallNode(CORINFO_HELP_JIT_PINVOKE_BEGIN, TYP_VOID, 0, comp->gtNewArgList(frameAddr)); comp->fgMorphTree(helperCall); - comp->fgInsertTreeBeforeAsEmbedded(helperCall, insertBefore, comp->compCurStmt->AsStmt(), currBlock); + BlockRange().InsertBefore(insertBefore, LIR::SeqTree(comp, helperCall)); return; } #endif @@ -2654,8 +2508,8 @@ void Lowering::InsertPInvokeCallProlog(GenTreeCall* call) new (comp, GT_STORE_LCL_FLD) GenTreeLclFld(GT_STORE_LCL_FLD, TYP_I_IMPL, comp->lvaInlinedPInvokeFrameVar, callFrameInfo.offsetOfCallTarget); store->gtOp1 = src; - comp->fgInsertTreeBeforeAsEmbedded(store, insertBefore, comp->compCurStmt->AsStmt(), currBlock); - DISPTREE(comp->compCurStmt); + + BlockRange().InsertBefore(insertBefore, LIR::SeqTree(comp, store)); } #ifdef _TARGET_X86_ @@ -2668,8 +2522,7 @@ void Lowering::InsertPInvokeCallProlog(GenTreeCall* call) storeCallSiteSP->gtOp1 = PhysReg(REG_SPBASE); - comp->fgInsertTreeBeforeAsEmbedded(storeCallSiteSP, insertBefore, comp->compCurStmt->AsStmt(), currBlock); - DISPTREE(comp->compCurStmt); + BlockRange().InsertBefore(insertBefore, LIR::SeqTree(comp, storeCallSiteSP)); #endif @@ -2686,8 +2539,7 @@ void Lowering::InsertPInvokeCallProlog(GenTreeCall* call) labelRef->gtType = TYP_I_IMPL; storeLab->gtOp1 = labelRef; - comp->fgInsertTreeBeforeAsEmbedded(storeLab, insertBefore, comp->compCurStmt->AsStmt(), currBlock); - DISPTREE(comp->compCurStmt); + BlockRange().InsertBefore(insertBefore, LIR::SeqTree(comp, storeLab)); if (!(comp->opts.eeFlags & CORJIT_FLG_IL_STUB)) { @@ -2697,8 +2549,7 @@ void Lowering::InsertPInvokeCallProlog(GenTreeCall* call) // // Stubs do this once per stub, not once per call. GenTree* frameUpd = CreateFrameLinkUpdate(PushFrame); - comp->fgInsertTreeBeforeAsEmbedded(frameUpd, insertBefore, comp->compCurStmt->AsStmt(), currBlock); - DISPTREE(comp->compCurStmt); + BlockRange().InsertBefore(insertBefore, LIR::SeqTree(comp, frameUpd)); } // IMPORTANT **** This instruction must come last!!! **** @@ -2707,8 +2558,7 @@ void Lowering::InsertPInvokeCallProlog(GenTreeCall* call) // [tcb + offsetOfGcState] = 0 GenTree* storeGCState = SetGCState(0); - comp->fgInsertTreeBeforeAsEmbedded(storeGCState, insertBefore, comp->compCurStmt->AsStmt(), currBlock); - DISPTREE(comp->compCurStmt); + BlockRange().InsertBefore(insertBefore, LIR::SeqTree(comp, storeGCState)); } //------------------------------------------------------------------------ @@ -2739,47 +2589,25 @@ void Lowering::InsertPInvokeCallEpilog(GenTreeCall* call) comp->gtNewHelperCallNode(CORINFO_HELP_JIT_PINVOKE_END, TYP_VOID, 0, comp->gtNewArgList(frameAddr)); comp->fgMorphTree(helperCall); - comp->fgInsertTreeAfterAsEmbedded(helperCall, call, comp->compCurStmt->AsStmt(), currBlock); - DISPTREE(comp->compCurStmt); + BlockRange().InsertAfter(call, LIR::SeqTree(comp, helperCall)); return; } #endif - GenTreeStmt* newStmt; - GenTreeStmt* topStmt = comp->compCurStmt->AsStmt(); - // gcstate = 1 - GenTree* latest = call; - GenTree* tree = SetGCState(1); - newStmt = comp->fgInsertTreeAfterAsEmbedded(tree, latest, topStmt, currBlock); - DISPTREE(newStmt); - latest = tree; - if (newStmt->gtStmtIsTopLevel()) - { - topStmt = newStmt; - } + GenTree* insertionPoint = call->gtNext; - tree = CreateReturnTrapSeq(); - newStmt = comp->fgInsertTreeAfterAsEmbedded(tree, latest, topStmt, currBlock); - DISPTREE(newStmt); - latest = tree; - if (newStmt->gtStmtIsTopLevel()) - { - topStmt = newStmt; - } + GenTree* tree = SetGCState(1); + BlockRange().InsertBefore(insertionPoint, LIR::SeqTree(comp, tree)); + + tree = CreateReturnTrapSeq(); + BlockRange().InsertBefore(insertionPoint, LIR::SeqTree(comp, tree)); - // Pop the frame + // Pop the frame if necessasry if (!(comp->opts.eeFlags & CORJIT_FLG_IL_STUB)) { - GenTree* frameUpd = CreateFrameLinkUpdate(PopFrame); - - newStmt = comp->fgInsertTreeAfterAsEmbedded(frameUpd, latest, topStmt, currBlock); - DISPTREE(newStmt); - latest = frameUpd; - if (newStmt->gtStmtIsTopLevel()) - { - topStmt = newStmt; - } + tree = CreateFrameLinkUpdate(PopFrame); + BlockRange().InsertBefore(insertionPoint, LIR::SeqTree(comp, tree)); } } @@ -2855,7 +2683,7 @@ GenTree* Lowering::LowerNonvirtPinvokeCall(GenTreeCall* call) // The PINVOKE_PROLOG op signals this to the code generator/emitter. GenTree* prolog = new (comp, GT_NOP) GenTree(GT_PINVOKE_PROLOG, TYP_VOID); - comp->fgInsertTreeBeforeAsEmbedded(prolog, call, comp->compCurStmt->AsStmt(), currBlock); + BlockRange().InsertBefore(call, prolog); InsertPInvokeCallProlog(call); @@ -2956,11 +2784,11 @@ GenTree* Lowering::LowerVirtualVtableCall(GenTreeCall* call) { vtableCallTemp = comp->lvaGrabTemp(true DEBUGARG("virtual vtable call")); } - GenTreeStmt* newStmt = comp->fgInsertEmbeddedFormTemp(&(argEntry->node->gtOp.gtOp1), vtableCallTemp); - newStmt->gtFlags |= GTF_STMT_SKIP_LOWER; // we're in postorder so we have already processed this subtree - GenTree* stLclVar = newStmt->gtStmtExpr; - assert(stLclVar->OperIsLocalStore()); - lclNum = stLclVar->gtLclVar.gtLclNum; + + LIR::Use thisPtrUse(BlockRange(), &(argEntry->node->gtOp.gtOp1), argEntry->node); + thisPtrUse.ReplaceWithLclVar(comp, m_block->getBBWeight(comp), vtableCallTemp); + + lclNum = vtableCallTemp; } // We'll introduce another use of this local so increase its ref count. @@ -3057,7 +2885,7 @@ GenTree* Lowering::LowerVirtualStubCall(GenTreeCall* call) // All we have to do here is add an indirection to generate the actual call target. GenTree* ind = Ind(call->gtCallAddr); - call->gtCallAddr->InsertAfterSelf(ind); + BlockRange().InsertAfter(call->gtCallAddr, ind); call->gtCallAddr = ind; } else @@ -3106,96 +2934,96 @@ GenTree* Lowering::LowerVirtualStubCall(GenTreeCall* call) } //------------------------------------------------------------------------ -// LowerIndCleanupHelper: Remove the nodes that are no longer used after an +// AddrModeCleanupHelper: Remove the nodes that are no longer used after an // addressing mode is constructed // // Arguments: // addrMode - A pointer to a new GenTreeAddrMode -// tree - The tree currently being considered to removal +// node - The node currently being considered for removal // // Return Value: // None. // // Assumptions: -// 'addrMode' and 'tree' must be contained in comp->compCurStmt - -void Lowering::LowerIndCleanupHelper(GenTreeAddrMode* addrMode, GenTreePtr tree) +// 'addrMode' and 'node' must be contained in the current block +// +void Lowering::AddrModeCleanupHelper(GenTreeAddrMode* addrMode, GenTree* node) { - if (tree == addrMode->Base() || tree == addrMode->Index()) + if (node == addrMode->Base() || node == addrMode->Index()) { return; } - unsigned childCount = tree->NumChildren(); - for (unsigned i = 0; i < childCount; i++) + + // TODO-LIR: change this to use the LIR mark bit and iterate instead of recursing + for (GenTree* operand : node->Operands()) { - LowerIndCleanupHelper(addrMode, tree->GetChild(i)); + AddrModeCleanupHelper(addrMode, operand); } - Compiler::fgSnipNode(comp->compCurStmt->AsStmt(), tree); + + BlockRange().Remove(node); } -// given two nodes which will be used in an addressing mode (src1, src2) +// given two nodes which will be used in an addressing mode (base, index) // walk backwards from the use to those nodes to determine if they are // potentially modified in that range // // returns: true if the sources given may be modified before they are used -bool Lowering::AreSourcesPossiblyModified(GenTree* use, GenTree* src1, GenTree* src2) +bool Lowering::AreSourcesPossiblyModified(GenTree* addr, GenTree* base, GenTree* index) { - GenTree* cursor = use; - GenTree* firstTree = comp->compCurStmt->AsStmt()->gtStmtList; + assert(addr != nullptr); - while (cursor && cursor != firstTree) + for (GenTree* cursor = addr; cursor != nullptr; cursor = cursor->gtPrev) { - cursor = cursor->gtPrev; - - if (cursor == src1) + if (cursor == base) { - src1 = nullptr; + base = nullptr; } - if (cursor == src2) + + if (cursor == index) { - src2 = nullptr; + index = nullptr; } - if (src2 == nullptr && src1 == nullptr) + + if (base == nullptr && index == nullptr) { return false; } - if (src1 && comp->fgNodesMayInterfere(src1, cursor)) + if (base != nullptr && comp->fgNodesMayInterfere(base, cursor)) { return true; } - if (src2 && comp->fgNodesMayInterfere(src2, cursor)) + if (index != nullptr && comp->fgNodesMayInterfere(index, cursor)) { return true; } } - assert(!"ran off beginning of stmt\n"); - return true; + + unreached(); } //------------------------------------------------------------------------ -// LowerAddrMode: recognize trees which can be implemented using an addressing -// mode and transform them to a GT_LEA +// TryCreateAddrMode: recognize trees which can be implemented using an +// addressing mode and transform them to a GT_LEA // // Arguments: -// pTree: pointer to the parent node's link to the node we care about -// before: node to insert the new GT_LEA before -// data: fgWalkData which is used to get info about parents and fixup call args +// use: the use of the address we want to transform // isIndir: true if this addressing mode is the child of an indir // -void Lowering::LowerAddrMode(GenTreePtr* pTree, GenTree* before, Compiler::fgWalkData* data, bool isIndir) +// Returns: +// The created LEA node or the original address node if an LEA could +// not be formed. +// +GenTree* Lowering::TryCreateAddrMode(LIR::Use&& use, bool isIndir) { - GenTree* addr = *pTree; + GenTree* addr = use.Def(); GenTreePtr base = nullptr; GenTreePtr index = nullptr; unsigned scale = 0; unsigned offset = 0; bool rev = false; - // If it's not an indir, we need the fgWalkData to get info about the parent. - assert(isIndir || data); - // Find out if an addressing mode can be constructed bool doAddrMode = comp->codeGen->genCreateAddrMode(addr, -1, true, 0, &rev, &base, &index, &scale, &offset, true /*nogen*/); @@ -3210,134 +3038,129 @@ void Lowering::LowerAddrMode(GenTreePtr* pTree, GenTree* before, Compiler::fgWal // this is just a reg-const add if (index == nullptr) { - return; + return addr; } // this is just a reg-reg add if (scale == 1 && offset == 0) { - return; + return addr; } } // make sure there are not any side effects between def of leaves and use - if (doAddrMode && !AreSourcesPossiblyModified(addr, base, index)) + if (!doAddrMode || AreSourcesPossiblyModified(addr, base, index)) { - GenTreePtr arrLength = nullptr; - - JITDUMP("Addressing mode:\n"); - JITDUMP(" Base\n"); - DISPNODE(base); - if (index != nullptr) - { - JITDUMP(" + Index * %u + %u\n", scale, offset); - DISPNODE(index); - } - else - { - JITDUMP(" + %u\n", offset); - } - - var_types addrModeType = addr->TypeGet(); - if (addrModeType == TYP_REF) - { - addrModeType = TYP_BYREF; - } - - GenTreeAddrMode* addrMode = new (comp, GT_LEA) GenTreeAddrMode(addrModeType, base, index, scale, offset); + JITDUMP(" No addressing mode\n"); + return addr; + } - addrMode->CopyCosts(addr); - addrMode->gtRsvdRegs = addr->gtRsvdRegs; - addrMode->gtFlags |= (addr->gtFlags & (GTF_ALL_EFFECT | GTF_IND_FLAGS)); + GenTreePtr arrLength = nullptr; - JITDUMP("New addressing mode node:\n"); - DISPNODE(addrMode); - JITDUMP("\n"); + JITDUMP("Addressing mode:\n"); + JITDUMP(" Base\n"); + DISPNODE(base); + if (index != nullptr) + { + JITDUMP(" + Index * %u + %u\n", scale, offset); + DISPNODE(index); + } + else + { + JITDUMP(" + %u\n", offset); + } - // Required to prevent assert failure: - // Assertion failed 'op1 && op2' in flowgraph.cpp, Line: 34431 - // when iterating the operands of a GT_LEA - // Test Case: self_host_tests_amd64\jit\jit64\opt\cse\VolatileTest_op_mul.exe - // Method: TestCSE:.cctor - // The method genCreateAddrMode() above probably should be fixed - // to not return rev=true, when index is returned as NULL - // - if (rev && index == nullptr) - { - rev = false; - } + var_types addrModeType = addr->TypeGet(); + if (addrModeType == TYP_REF) + { + addrModeType = TYP_BYREF; + } - if (rev) - { - addrMode->gtFlags |= GTF_REVERSE_OPS; - } - else - { - addrMode->gtFlags &= ~(GTF_REVERSE_OPS); - } + GenTreeAddrMode* addrMode = new (comp, GT_LEA) GenTreeAddrMode(addrModeType, base, index, scale, offset); - comp->fgInsertLinearNodeBefore(addrMode, before); + addrMode->CopyCosts(addr); + addrMode->gtRsvdRegs = addr->gtRsvdRegs; + addrMode->gtFlags |= (addr->gtFlags & (GTF_ALL_EFFECT | GTF_IND_FLAGS)); - // Now we need to snip from the linear order all the nodes subsumed by the addrMode - LowerIndCleanupHelper(addrMode, addr); + JITDUMP("New addressing mode node:\n"); + DISPNODE(addrMode); + JITDUMP("\n"); - GenTree* old = *pTree; - *pTree = addrMode; + // Required to prevent assert failure: + // Assertion failed 'op1 && op2' in flowgraph.cpp, Line: 34431 + // when iterating the operands of a GT_LEA + // Test Case: self_host_tests_amd64\jit\jit64\opt\cse\VolatileTest_op_mul.exe + // Method: TestCSE:.cctor + // The method genCreateAddrMode() above probably should be fixed + // to not return rev=true, when index is returned as NULL + // + if (rev && index == nullptr) + { + rev = false; + } - if (!isIndir) - { - // this could be an arg to a call - comp->fgFixupIfCallArg(data->parentStack, old, addrMode); - } + if (rev) + { + addrMode->gtFlags |= GTF_REVERSE_OPS; } else { - JITDUMP(" No addressing mode\n"); + addrMode->gtFlags &= ~(GTF_REVERSE_OPS); } + + BlockRange().InsertAfter(addr, addrMode); + + // Now we need to remove all the nodes subsumed by the addrMode + AddrModeCleanupHelper(addrMode, addr); + + // Replace the original address node with the addrMode. + use.ReplaceWith(comp, addrMode); + + return addrMode; } //------------------------------------------------------------------------ // LowerAdd: turn this add into a GT_LEA if that would be profitable // // Arguments: -// pTree: pointer to the parent node's link to the node we care about -// data: fgWalkData which is used to get info about parents and fixup call args - -void Lowering::LowerAdd(GenTreePtr* pTree, Compiler::fgWalkData* data) +// node - the node we care about +// +// Returns: +// The next node to lower. +// +GenTree* Lowering::LowerAdd(GenTree* node) { - GenTreePtr newNode = nullptr; - - GenTreePtr addr = *pTree; + GenTree* next = node->gtNext; #ifdef _TARGET_ARMARCH_ // For ARM architectures we don't have the LEA instruction // therefore we won't get much benefit from doing this. - return; + return next; #else // _TARGET_ARMARCH_ - if (data->parentStack->Height() < 2) + if (!varTypeIsIntegralOrI(node)) { - return; + return next; } - // If this is a child of an indir, and it is not a block op, let the parent handle it. - GenTree* parent = data->parentStack->Index(1); - if (parent->OperIsIndir() && !varTypeIsStruct(parent)) + LIR::Use use; + if (!BlockRange().TryGetUse(node, &use)) { - return; + return next; } - // if there is a chain of adds, only look at the topmost one - if (parent->gtOper == GT_ADD) + // if this is a child of an indir, let the parent handle it + if (use.User()->OperIsIndir()) { - return; + return next; } - if (!varTypeIsIntegralOrI(addr)) + // if there is a chain of adds, only look at the topmost one + if (use.User()->gtOper == GT_ADD) { - return; + return next; } - LowerAddrMode(pTree, addr, data, false); + return TryCreateAddrMode(std::move(use), false)->gtNext; #endif // !_TARGET_ARMARCH_ } @@ -3346,13 +3169,13 @@ void Lowering::LowerAdd(GenTreePtr* pTree, Compiler::fgWalkData* data) // divisor into GT_RSZ/GT_AND nodes. // // Arguments: -// tree: pointer to the GT_UDIV/GT_UMOD node to be lowered - -void Lowering::LowerUnsignedDivOrMod(GenTree* tree) +// node - pointer to the GT_UDIV/GT_UMOD node to be lowered +// +void Lowering::LowerUnsignedDivOrMod(GenTree* node) { - assert(tree->OperGet() == GT_UDIV || tree->OperGet() == GT_UMOD); + assert((node->OperGet() == GT_UDIV) || (node->OperGet() == GT_UMOD)); - GenTree* divisor = tree->gtGetOp2(); + GenTree* divisor = node->gtGetOp2(); if (divisor->IsCnsIntOrI()) { @@ -3362,7 +3185,7 @@ void Lowering::LowerUnsignedDivOrMod(GenTree* tree) { genTreeOps newOper; - if (tree->OperGet() == GT_UDIV) + if (node->OperGet() == GT_UDIV) { newOper = GT_RSZ; divisorValue = genLog2(divisorValue); @@ -3373,7 +3196,7 @@ void Lowering::LowerUnsignedDivOrMod(GenTree* tree) divisorValue -= 1; } - tree->SetOper(newOper); + node->SetOper(newOper); divisor->gtIntCon.SetIconValue(divisorValue); } } @@ -3384,181 +3207,181 @@ void Lowering::LowerUnsignedDivOrMod(GenTree* tree) // const divisor into equivalent but faster sequences. // // Arguments: -// pTree: pointer to the parent node's link to the node we care about -// data: fgWalkData which is used to get info about parents and fixup call args - -void Lowering::LowerSignedDivOrMod(GenTreePtr* ppTree, Compiler::fgWalkData* data) +// node - pointer to node we care about +// +// Returns: +// The next node to lower. +// +GenTree* Lowering::LowerSignedDivOrMod(GenTreePtr node) { - GenTree* divMod = *ppTree; - assert(divMod->OperGet() == GT_DIV || divMod->OperGet() == GT_MOD); + assert((node->OperGet() == GT_DIV) || (node->OperGet() == GT_MOD)); + + GenTree* next = node->gtNext; + GenTree* divMod = node; GenTree* divisor = divMod->gtGetOp2(); - if (divisor->IsCnsIntOrI()) + if (!divisor->IsCnsIntOrI()) { - const var_types type = divMod->TypeGet(); - assert(type == TYP_INT || type == TYP_LONG); + return next; // no transformations to make + } - GenTree* dividend = divMod->gtGetOp1(); + const var_types type = divMod->TypeGet(); + assert((type == TYP_INT) || (type == TYP_LONG)); - if (dividend->IsCnsIntOrI()) - { - // We shouldn't see a divmod with constant operands here but if we do then it's likely - // because optimizations are disabled or it's a case that's supposed to throw an exception. - // Don't optimize this. - return; - } + GenTree* dividend = divMod->gtGetOp1(); + + if (dividend->IsCnsIntOrI()) + { + // We shouldn't see a divmod with constant operands here but if we do then it's likely + // because optimizations are disabled or it's a case that's supposed to throw an exception. + // Don't optimize this. + return next; + } - ssize_t divisorValue = divisor->gtIntCon.IconValue(); + ssize_t divisorValue = divisor->gtIntCon.IconValue(); - if (divisorValue == -1) - { - // x / -1 can't be optimized because INT_MIN / -1 is required to throw an exception. + if (divisorValue == -1) + { + // x / -1 can't be optimized because INT_MIN / -1 is required to throw an exception. - // x % -1 is always 0 and the IL spec says that the rem instruction "can" throw an exception if x is - // the minimum representable integer. However, the C# spec says that an exception "is" thrown in this - // case so optimizing this case would break C# code. + // x % -1 is always 0 and the IL spec says that the rem instruction "can" throw an exception if x is + // the minimum representable integer. However, the C# spec says that an exception "is" thrown in this + // case so optimizing this case would break C# code. - // A runtime check could be used to handle this case but it's probably too rare to matter. - return; - } + // A runtime check could be used to handle this case but it's probably too rare to matter. + return next; + } - bool isDiv = divMod->OperGet() == GT_DIV; + bool isDiv = divMod->OperGet() == GT_DIV; - if (isDiv) + if (isDiv) + { + if ((type == TYP_INT && divisorValue == INT_MIN) || (type == TYP_LONG && divisorValue == INT64_MIN)) { - if ((type == TYP_INT && divisorValue == INT_MIN) || (type == TYP_LONG && divisorValue == INT64_MIN)) - { - // If the divisor is the minimum representable integer value then we can use a compare, - // the result is 1 iff the dividend equals divisor. - divMod->SetOper(GT_EQ); - return; - } + // If the divisor is the minimum representable integer value then we can use a compare, + // the result is 1 iff the dividend equals divisor. + divMod->SetOper(GT_EQ); + return next; } + } - size_t absDivisorValue = - (divisorValue == SSIZE_T_MIN) ? static_cast<size_t>(divisorValue) : static_cast<size_t>(abs(divisorValue)); + size_t absDivisorValue = + (divisorValue == SSIZE_T_MIN) ? static_cast<size_t>(divisorValue) : static_cast<size_t>(abs(divisorValue)); - if (isPow2(absDivisorValue)) - { - // We need to use the dividend node multiple times so its value needs to be - // computed once and stored in a temp variable. - CreateTemporary(&(divMod->gtOp.gtOp1)); - dividend = divMod->gtGetOp1(); + if (!isPow2(absDivisorValue)) + { + return next; + } - GenTreeStmt* curStmt = comp->compCurStmt->AsStmt(); - unsigned curBBWeight = currBlock->getBBWeight(comp); - unsigned dividendLclNum = dividend->gtLclVar.gtLclNum; + // We're committed to the conversion now. Go find the use. + LIR::Use use; + if (!BlockRange().TryGetUse(node, &use)) + { + assert(!"signed DIV/MOD node is unused"); + return next; + } - GenTree* adjustment = - comp->gtNewOperNode(GT_RSH, type, dividend, comp->gtNewIconNode(type == TYP_INT ? 31 : 63)); + // We need to use the dividend node multiple times so its value needs to be + // computed once and stored in a temp variable. - if (absDivisorValue == 2) - { - // If the divisor is +/-2 then we'd end up with a bitwise and between 0/-1 and 1. - // We can get the same result by using GT_RSZ instead of GT_RSH. - adjustment->SetOper(GT_RSZ); - } - else - { - adjustment = - comp->gtNewOperNode(GT_AND, type, adjustment, comp->gtNewIconNode(absDivisorValue - 1, type)); - } + unsigned curBBWeight = comp->compCurBB->getBBWeight(comp); - GenTree* adjustedDividend = - comp->gtNewOperNode(GT_ADD, type, adjustment, comp->gtNewLclvNode(dividendLclNum, type)); + LIR::Use opDividend(BlockRange(), &divMod->gtOp.gtOp1, divMod); + opDividend.ReplaceWithLclVar(comp, curBBWeight); - comp->lvaTable[dividendLclNum].incRefCnts(curBBWeight, comp); + dividend = divMod->gtGetOp1(); + assert(dividend->OperGet() == GT_LCL_VAR); - GenTree* newDivMod; + unsigned dividendLclNum = dividend->gtLclVar.gtLclNum; - if (isDiv) - { - // perform the division by right shifting the adjusted dividend - divisor->gtIntCon.SetIconValue(genLog2(absDivisorValue)); + GenTree* adjustment = comp->gtNewOperNode(GT_RSH, type, dividend, comp->gtNewIconNode(type == TYP_INT ? 31 : 63)); - newDivMod = comp->gtNewOperNode(GT_RSH, type, adjustedDividend, divisor); + if (absDivisorValue == 2) + { + // If the divisor is +/-2 then we'd end up with a bitwise and between 0/-1 and 1. + // We can get the same result by using GT_RSZ instead of GT_RSH. + adjustment->SetOper(GT_RSZ); + } + else + { + adjustment = comp->gtNewOperNode(GT_AND, type, adjustment, comp->gtNewIconNode(absDivisorValue - 1, type)); + } - if (divisorValue < 0) - { - // negate the result if the divisor is negative - newDivMod = comp->gtNewOperNode(GT_NEG, type, newDivMod); - } - } - else - { - // divisor % dividend = dividend - divisor x (dividend / divisor) - // divisor x (dividend / divisor) translates to (dividend >> log2(divisor)) << log2(divisor) - // which simply discards the low log2(divisor) bits, that's just dividend & ~(divisor - 1) - divisor->gtIntCon.SetIconValue(~(absDivisorValue - 1)); + GenTree* adjustedDividend = + comp->gtNewOperNode(GT_ADD, type, adjustment, comp->gtNewLclvNode(dividendLclNum, type)); - newDivMod = comp->gtNewOperNode(GT_SUB, type, comp->gtNewLclvNode(dividendLclNum, type), - comp->gtNewOperNode(GT_AND, type, adjustedDividend, divisor)); + comp->lvaTable[dividendLclNum].incRefCnts(curBBWeight, comp); - comp->lvaTable[dividendLclNum].incRefCnts(curBBWeight, comp); - } + GenTree* newDivMod; - // Remove the divisor and dividend nodes from the linear order, - // since we have reused them and will resequence the tree - comp->fgSnipNode(curStmt, divisor); - comp->fgSnipNode(curStmt, dividend); + if (isDiv) + { + // perform the division by right shifting the adjusted dividend + divisor->gtIntCon.SetIconValue(genLog2(absDivisorValue)); - // linearize and insert the new tree before the original divMod node - comp->gtSetEvalOrder(newDivMod); - comp->fgSetTreeSeq(newDivMod); - comp->fgInsertTreeInListBefore(newDivMod, divMod, curStmt); - comp->fgSnipNode(curStmt, divMod); + newDivMod = comp->gtNewOperNode(GT_RSH, type, adjustedDividend, divisor); - // the divMod that we've replaced could have been a call arg - comp->fgFixupIfCallArg(data->parentStack, divMod, newDivMod); + if (divisorValue < 0) + { + // negate the result if the divisor is negative + newDivMod = comp->gtNewOperNode(GT_NEG, type, newDivMod); + } + } + else + { + // divisor % dividend = dividend - divisor x (dividend / divisor) + // divisor x (dividend / divisor) translates to (dividend >> log2(divisor)) << log2(divisor) + // which simply discards the low log2(divisor) bits, that's just dividend & ~(divisor - 1) + divisor->gtIntCon.SetIconValue(~(absDivisorValue - 1)); - // replace the original divmod node with the new divmod tree - *ppTree = newDivMod; + newDivMod = comp->gtNewOperNode(GT_SUB, type, comp->gtNewLclvNode(dividendLclNum, type), + comp->gtNewOperNode(GT_AND, type, adjustedDividend, divisor)); - return; - } + comp->lvaTable[dividendLclNum].incRefCnts(curBBWeight, comp); } + + // Remove the divisor and dividend nodes from the linear order, + // since we have reused them and will resequence the tree + BlockRange().Remove(divisor); + BlockRange().Remove(dividend); + + // linearize and insert the new tree before the original divMod node + BlockRange().InsertBefore(divMod, LIR::SeqTree(comp, newDivMod)); + BlockRange().Remove(divMod); + + // replace the original divmod node with the new divmod tree + use.ReplaceWith(comp, newDivMod); + + return newDivMod->gtNext; } //------------------------------------------------------------------------ -// LowerInd: attempt to transform indirected expression into an addressing mode +// LowerStoreInd: attempt to transform an indirect store to use an +// addressing mode // // Arguments: -// pTree: pointer to the parent node's link to the node we care about - -void Lowering::LowerInd(GenTreePtr* pTree) +// node - the node we care about +// +void Lowering::LowerStoreInd(GenTree* node) { - GenTreePtr newNode = nullptr; - GenTreePtr cTree = *pTree; + assert(node != nullptr); + assert(node->OperGet() == GT_STOREIND); - JITDUMP("\n"); - DISPNODE(cTree); - - GenTreePtr addr = cTree->gtOp.gtOp1; - - GenTreePtr before = cTree; - if (cTree->OperGet() == GT_STOREIND && !cTree->IsReverseOp()) - { - before = comp->fgGetFirstNode(cTree->gtGetOp2()); - } - - LowerAddrMode(&cTree->gtOp.gtOp1, before, nullptr, true); + TryCreateAddrMode(LIR::Use(BlockRange(), &node->gtOp.gtOp1, node), true); // Mark all GT_STOREIND nodes to indicate that it is not known // whether it represents a RMW memory op. - if (cTree->OperGet() == GT_STOREIND) - { - cTree->AsStoreInd()->SetRMWStatusDefault(); - } + node->AsStoreInd()->SetRMWStatusDefault(); } //------------------------------------------------------------------------ // LowerArrElem: Lower a GT_ARR_ELEM node // // Arguments: -// pTree - pointer to the field in the parent node that holds the pointer to the GT_ARR_ELEM node. +// node - the GT_ARR_ELEM node to lower. // // Return Value: -// None. +// The next node to lower. // // Assumptions: // pTree points to a pointer to a GT_ARR_ELEM node. @@ -3593,19 +3416,17 @@ void Lowering::LowerInd(GenTreePtr* pTree) // Note that the arrMDOffs is the INDEX of the lea, but is evaluated before the BASE (which is the second // reference to NewTemp), because that provides more accurate lifetimes. // There may be 1, 2 or 3 dimensions, with 1, 2 or 3 arrMDIdx nodes, respectively. - -void Lowering::LowerArrElem(GenTree** ppTree, Compiler::fgWalkData* data) +// +GenTree* Lowering::LowerArrElem(GenTree* node) { - GenTreePtr tree = *ppTree; // This will assert if we don't have an ArrElem node - GenTreeArrElem* arrElem = tree->AsArrElem(); - Compiler* comp = data->compiler; - GenTreePtr curStmt = comp->compCurStmt; - unsigned char rank = arrElem->gtArrElem.gtArrRank; + GenTreeArrElem* arrElem = node->AsArrElem(); + const unsigned char rank = arrElem->gtArrElem.gtArrRank; + const unsigned blockWeight = m_block->getBBWeight(comp); JITDUMP("Lowering ArrElem\n"); JITDUMP("============\n"); - DISPTREE(arrElem); + DISPTREERANGE(BlockRange(), arrElem); JITDUMP("\n"); assert(arrElem->gtArrObj->TypeGet() == TYP_REF); @@ -3613,60 +3434,22 @@ void Lowering::LowerArrElem(GenTree** ppTree, Compiler::fgWalkData* data) // We need to have the array object in a lclVar. if (!arrElem->gtArrObj->IsLocal()) { - // Split off the array object and store to a temporary variable. - GenTreeStmt* newStmt = comp->fgInsertEmbeddedFormTemp(&(arrElem->gtArrObj)); - newStmt->gtFlags |= GTF_STMT_SKIP_LOWER; - GenTreePtr stLclVar = newStmt->gtStmtExpr; - assert(stLclVar->OperIsLocalStore()); - - // If we have made a new top-level statement, and it has inherited any - // embedded statements from curStmt, they have not yet been lowered. - if (newStmt->gtStmtIsTopLevel()) - { - for (GenTreePtr nextEmbeddedStmt = newStmt->gtStmtNextIfEmbedded(); nextEmbeddedStmt != nullptr; - nextEmbeddedStmt = nextEmbeddedStmt->gtStmt.gtStmtNextIfEmbedded()) - { - comp->compCurStmt = nextEmbeddedStmt; - comp->fgWalkTreePost(&nextEmbeddedStmt->gtStmt.gtStmtExpr, &Lowering::LowerNodeHelper, this, true); - nextEmbeddedStmt->gtFlags |= GTF_STMT_SKIP_LOWER; - } - } - // Restore curStmt. - comp->compCurStmt = curStmt; + LIR::Use arrObjUse(BlockRange(), &arrElem->gtArrObj, arrElem); + arrObjUse.ReplaceWithLclVar(comp, blockWeight); } - GenTreePtr arrObjNode = arrElem->gtArrObj; - assert(arrObjNode->IsLocal()); - GenTreePtr nextNode = arrElem; + GenTree* arrObjNode = arrElem->gtArrObj; + assert(arrObjNode->IsLocal()); - // We need to evaluate the index expressions up-front if they have side effects. - for (unsigned char dim = 0; dim < rank; dim++) - { - GenTree* currIndexNode = arrElem->gtArrElem.gtArrInds[dim]; - assert(varTypeIsIntegral(currIndexNode->TypeGet())); - if ((currIndexNode->gtFlags & GTF_SIDE_EFFECT) != 0) - { - // Split off this index computation and store to a temporary variable. - GenTreeStmt* newStmt = comp->fgInsertEmbeddedFormTemp(&(arrElem->gtArrElem.gtArrInds[dim])); - GenTreePtr stLclVar = newStmt->gtStmtExpr; - assert(stLclVar->OperIsLocalStore()); - // We can't have made a new top-level statement, because we know we've got an ArrObj - // prior to the index nodes. - assert(newStmt->gtStmtIsEmbedded()); - newStmt->gtFlags |= GTF_STMT_SKIP_LOWER; - // Restore curStmt (we've already lowered the tree we just split off). - comp->compCurStmt = curStmt; - } - } + GenTree* insertionPoint = arrElem; // The first ArrOffs node will have 0 for the offset of the previous dimension. GenTree* prevArrOffs = new (comp, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, 0); - comp->fgInsertLinearNodeBefore(prevArrOffs, arrObjNode); + BlockRange().InsertBefore(insertionPoint, prevArrOffs); for (unsigned char dim = 0; dim < rank; dim++) { - GenTree* currIndexTree = arrElem->gtArrElem.gtArrInds[dim]; - GenTree* insertBeforeNode = nextNode; + GenTree* indexNode = arrElem->gtArrElem.gtArrInds[dim]; // Use the original arrObjNode on the 0th ArrIndex node, and clone it for subsequent ones. GenTreePtr idxArrObjNode; @@ -3677,53 +3460,32 @@ void Lowering::LowerArrElem(GenTree** ppTree, Compiler::fgWalkData* data) else { idxArrObjNode = comp->gtClone(arrObjNode); - comp->fgInsertLinearNodeBefore(idxArrObjNode, nextNode); - } - - // Move the index (temp created above, or non-side-effect computation) if needed. - // (All side-effecting computations we've split above need to come before the GT_ARR_INDEX nodes.) - if (currIndexTree->gtNext != insertBeforeNode) - { - GenTree* firstIndexNode = comp->fgGetFirstNode(currIndexTree); - GenTree* oldPrevNode = firstIndexNode->gtPrev; - GenTree* oldNextNode = currIndexTree->gtNext; - GenTree* newPrevNode = insertBeforeNode->gtPrev; - // All these are inner nodes, so they cannot be null. - assert(oldPrevNode != nullptr && oldNextNode != nullptr && newPrevNode != nullptr); - - oldPrevNode->gtNext = oldNextNode; - oldNextNode->gtPrev = oldPrevNode; - - firstIndexNode->gtPrev = newPrevNode; - newPrevNode->gtNext = firstIndexNode; - - currIndexTree->gtNext = insertBeforeNode; - insertBeforeNode->gtPrev = currIndexTree; + BlockRange().InsertBefore(insertionPoint, idxArrObjNode); } // Next comes the GT_ARR_INDEX node. GenTreeArrIndex* arrMDIdx = new (comp, GT_ARR_INDEX) - GenTreeArrIndex(TYP_INT, idxArrObjNode, currIndexTree, dim, rank, arrElem->gtArrElem.gtArrElemType); - arrMDIdx->gtFlags |= ((idxArrObjNode->gtFlags | currIndexTree->gtFlags) & GTF_ALL_EFFECT); - comp->fgInsertLinearNodeBefore(arrMDIdx, insertBeforeNode); + GenTreeArrIndex(TYP_INT, idxArrObjNode, indexNode, dim, rank, arrElem->gtArrElem.gtArrElemType); + arrMDIdx->gtFlags |= ((idxArrObjNode->gtFlags | indexNode->gtFlags) & GTF_ALL_EFFECT); + BlockRange().InsertBefore(insertionPoint, arrMDIdx); GenTree* offsArrObjNode = comp->gtClone(arrObjNode); - comp->fgInsertLinearNodeBefore(offsArrObjNode, insertBeforeNode); + BlockRange().InsertBefore(insertionPoint, offsArrObjNode); GenTreeArrOffs* arrOffs = new (comp, GT_ARR_OFFSET) GenTreeArrOffs(TYP_I_IMPL, prevArrOffs, arrMDIdx, offsArrObjNode, dim, rank, arrElem->gtArrElem.gtArrElemType); - comp->fgInsertLinearNodeBefore(arrOffs, insertBeforeNode); arrOffs->gtFlags |= ((prevArrOffs->gtFlags | arrMDIdx->gtFlags | offsArrObjNode->gtFlags) & GTF_ALL_EFFECT); + BlockRange().InsertBefore(insertionPoint, arrOffs); prevArrOffs = arrOffs; } // Generate the LEA and make it reverse evaluation, because we want to evaluate the index expression before the // base. - GenTreePtr leaBase = comp->gtClone(arrObjNode); - unsigned scale = arrElem->gtArrElem.gtArrElemSize; - unsigned offset = comp->eeGetMDArrayDataOffset(arrElem->gtArrElem.gtArrElemType, arrElem->gtArrElem.gtArrRank); + unsigned scale = arrElem->gtArrElem.gtArrElemSize; + unsigned offset = comp->eeGetMDArrayDataOffset(arrElem->gtArrElem.gtArrElemType, arrElem->gtArrElem.gtArrRank); + GenTreePtr leaIndexNode = prevArrOffs; if (!jitIsScaleIndexMul(scale)) { @@ -3731,39 +3493,36 @@ void Lowering::LowerArrElem(GenTree** ppTree, Compiler::fgWalkData* data) // TYP_INT GenTreePtr scaleNode = new (comp, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, scale); GenTreePtr mulNode = new (comp, GT_MUL) GenTreeOp(GT_MUL, TYP_I_IMPL, leaIndexNode, scaleNode); - comp->fgInsertLinearNodeBefore(scaleNode, nextNode); - comp->fgInsertLinearNodeBefore(mulNode, nextNode); + BlockRange().InsertBefore(insertionPoint, scaleNode, mulNode); leaIndexNode = mulNode; scale = 1; } - comp->fgInsertLinearNodeBefore(leaBase, nextNode); + + GenTreePtr leaBase = comp->gtClone(arrObjNode); + BlockRange().InsertBefore(insertionPoint, leaBase); + GenTreePtr leaNode = new (comp, GT_LEA) GenTreeAddrMode(arrElem->TypeGet(), leaBase, leaIndexNode, scale, offset); leaNode->gtFlags |= GTF_REVERSE_OPS; - comp->fgInsertLinearNodeBefore(leaNode, nextNode); - *ppTree = leaNode; + // Set the costs for all of the new nodes. Depends on the new nodes all participating in the + // dataflow tree rooted at `leaNode`. + comp->gtPrepareCost(leaNode); - if (arrElem->gtNext != nullptr) - { - comp->fgSnipInnerNode(arrElem); - } - else + BlockRange().InsertBefore(insertionPoint, leaNode); + + LIR::Use arrElemUse; + if (BlockRange().TryGetUse(arrElem, &arrElemUse)) { - // We can have a top-level GT_ARR_ELEM. For example, a function call - // with a parameter of GT_ARR_ELEM can end up being simplified by the - // inliner to single GT_ARR_ELEM node if the function has an empty body. - arrElem->gtPrev->gtNext = nullptr; - curStmt->gtStmt.gtStmtExpr = *ppTree; + arrElemUse.ReplaceWith(comp, leaNode); } - // Update the costs. - comp->gtSetStmtInfo(curStmt); + BlockRange().Remove(arrElem); JITDUMP("Results of lowering ArrElem:\n"); - DISPTREE(leaNode); - JITDUMP("\nResulting statement:\n"); - DISPTREE(curStmt); + DISPTREERANGE(BlockRange(), leaNode); JITDUMP("\n\n"); + + return leaNode; } void Lowering::DoPhase() @@ -3803,34 +3562,14 @@ void Lowering::DoPhase() for (BasicBlock* block = comp->fgFirstBB; block; block = block->bbNext) { - GenTreePtr stmt; - /* Make the block publicly available */ - currBlock = block; comp->compCurBB = block; #if !defined(_TARGET_64BIT_) decomp.DecomposeBlock(block); #endif //!_TARGET_64BIT_ - // Walk the statement trees in this basic block - for (stmt = block->bbTreeList; stmt; stmt = stmt->gtNext) - { - if (stmt->gtFlags & GTF_STMT_SKIP_LOWER) - { - continue; - } -#ifdef DEBUG - if (comp->verbose) - { - printf("Lowering BB%02u, stmt id %u\n", block->bbNum, stmt->gtTreeID); - } -#endif - comp->compCurStmt = stmt; - comp->fgWalkTreePost(&stmt->gtStmt.gtStmtExpr, &Lowering::LowerNodeHelper, this, true); - // We may have removed "stmt" in LowerNode(). - stmt = comp->compCurStmt; - } + LowerBlock(block); } // If we have any PInvoke calls, insert the one-time prolog code. We've already inserted the epilog code in the @@ -3910,60 +3649,49 @@ void Lowering::DoPhase() // are in increasing location order. currentLoc += 2; - for (stmt = block->FirstNonPhiDef(); stmt; stmt = stmt->gtNext) + m_block = block; + for (GenTree* node : BlockRange().NonPhiNodes()) { - if (stmt->gtStmt.gtStmtIsEmbedded()) - { - continue; - } - - /* We increment the number position of each tree node by 2 to - * simplify the logic when there's the case of a tree that implicitly - * does a dual-definition of temps (the long case). In this case - * is easier to already have an idle spot to handle a dual-def instead - * of making some messy adjustments if we only increment the - * number position by one. - */ - GenTreePtr node; - foreach_treenode_execution_order(node, stmt) - { +/* We increment the number position of each tree node by 2 to +* simplify the logic when there's the case of a tree that implicitly +* does a dual-definition of temps (the long case). In this case +* is easier to already have an idle spot to handle a dual-def instead +* of making some messy adjustments if we only increment the +* number position by one. +*/ #ifdef DEBUG - node->gtSeqNum = currentLoc; + node->gtSeqNum = currentLoc; #endif - node->gtLsraInfo.Initialize(m_lsra, node, currentLoc); - node->gtClearReg(comp); - currentLoc += 2; + node->gtLsraInfo.Initialize(m_lsra, node, currentLoc); + node->gtClearReg(comp); + + // Mark the node's operands as used + for (GenTree* operand : node->Operands()) + { + operand->gtLIRFlags &= ~LIR::Flags::IsUnusedValue; } - } - for (stmt = block->FirstNonPhiDef(); stmt; stmt = stmt->gtNext) - { - if (stmt->gtStmt.gtStmtIsEmbedded()) + // If the node produces a value, mark it as unused. + if (node->IsValue()) { - continue; + node->gtLIRFlags |= LIR::Flags::IsUnusedValue; } - comp->compCurStmt = stmt; + currentLoc += 2; + } - TreeNodeInfoInit(stmt); + for (GenTree* node : BlockRange().NonPhiNodes()) + { + TreeNodeInfoInit(node); - // In the special case where a comma node is at the top level, make it consume - // its (op2) source - GenTreePtr tree = stmt->gtStmt.gtStmtExpr; - if (tree->gtOper == GT_COMMA && tree->TypeGet() != TYP_VOID) - { - tree->gtLsraInfo.srcCount = 1; - } - // In the special case where a lclVar node is at the top level, set it as - // localDefUse - // TODO-Cleanup: This used to be isCandidateLocalRef, but we haven't initialized the - // lvLRACandidate field yet. Fix this. - else if (comp->optIsTrackedLocal(tree)) + // If the node produces an unused value, mark it as a local def-use + if ((node->gtLIRFlags & LIR::Flags::IsUnusedValue) != 0) { - tree->gtLsraInfo.isLocalDefUse = true; - tree->gtLsraInfo.dstCount = 0; + node->gtLsraInfo.isLocalDefUse = true; + node->gtLsraInfo.dstCount = 0; } + #if 0 // TODO-CQ: Enable this code after fixing the isContained() logic to not abort for these // top-level nodes that throw away their result. @@ -3978,10 +3706,153 @@ void Lowering::DoPhase() } #endif } + + assert(BlockRange().CheckLIR(comp, true)); } DBEXEC(VERBOSE, DumpNodeInfoMap()); } +#ifdef DEBUG + +//------------------------------------------------------------------------ +// Lowering::CheckCallArg: check that a call argument is in an expected +// form after lowering. +// +// Arguments: +// arg - the argument to check. +// +void Lowering::CheckCallArg(GenTree* arg) +{ + if (arg->OperIsStore() || arg->IsArgPlaceHolderNode() || arg->IsNothingNode() || arg->OperIsCopyBlkOp()) + { + return; + } + + switch (arg->OperGet()) + { +#if !defined(_TARGET_64BIT_) + case GT_LONG: + assert(arg->gtGetOp1()->OperIsPutArg()); + assert(arg->gtGetOp2()->OperIsPutArg()); + break; +#endif + + case GT_LIST: + for (GenTreeArgList* list = arg->AsArgList(); list != nullptr; list = list->Rest()) + { + assert(list->Current()->OperIsPutArg()); + } + break; + + default: + assert(arg->OperIsPutArg()); + break; + } +} + +//------------------------------------------------------------------------ +// Lowering::CheckCall: check that a call is in an expected form after +// lowering. Currently this amounts to checking its +// arguments, but could be expanded to verify more +// properties in the future. +// +// Arguments: +// call - the call to check. +// +void Lowering::CheckCall(GenTreeCall* call) +{ + if (call->gtCallObjp != nullptr) + { + CheckCallArg(call->gtCallObjp); + } + + for (GenTreeArgList* args = call->gtCallArgs; args != nullptr; args = args->Rest()) + { + CheckCallArg(args->Current()); + } + + for (GenTreeArgList* args = call->gtCallLateArgs; args != nullptr; args = args->Rest()) + { + CheckCallArg(args->Current()); + } +} + +//------------------------------------------------------------------------ +// Lowering::CheckNode: check that an LIR node is in an expected form +// after lowering. +// +// Arguments: +// node - the node to check. +// +void Lowering::CheckNode(GenTree* node) +{ + switch (node->OperGet()) + { + case GT_CALL: + CheckCall(node->AsCall()); + break; + +#ifdef FEATURE_SIMD + case GT_SIMD: +#ifdef _TARGET_64BIT_ + case GT_LCL_VAR: + case GT_STORE_LCL_VAR: +#endif // _TARGET_64BIT_ + assert(node->TypeGet() != TYP_SIMD12); + break; +#endif + + default: + break; + } +} + +//------------------------------------------------------------------------ +// Lowering::CheckBlock: check that the contents of an LIR block are in an +// expected form after lowering. +// +// Arguments: +// compiler - the compiler context. +// block - the block to check. +// +bool Lowering::CheckBlock(Compiler* compiler, BasicBlock* block) +{ + assert(block->isEmpty() || block->IsLIR()); + + LIR::Range& blockRange = LIR::AsRange(block); + for (GenTree* node : blockRange) + { + CheckNode(node); + } + + assert(blockRange.CheckLIR(compiler)); + return true; +} +#endif + +void Lowering::LowerBlock(BasicBlock* block) +{ + assert(block == comp->compCurBB); // compCurBB must already be set. + assert(block->isEmpty() || block->IsLIR()); + + m_block = block; + + // NOTE: some of the lowering methods insert calls before the node being + // lowered (See e.g. InsertPInvoke{Method,Call}{Prolog,Epilog}). In + // general, any code that is inserted before the current node should be + // "pre-lowered" as they won't be subject to further processing. + // Lowering::CheckBlock() runs some extra checks on call arguments in + // order to help catch unlowered nodes. + + GenTree* node = BlockRange().FirstNode(); + while (node != nullptr) + { + node = LowerNode(node); + } + + assert(CheckBlock(comp, block)); +} + /** Verifies if both of these trees represent the same indirection. * Used by Lower to annotate if CodeGen generate an instruction of the * form *addrMode BinOp= expr @@ -4103,83 +3974,6 @@ bool Lowering::NodesAreEquivalentLeaves(GenTreePtr tree1, GenTreePtr tree2) } } -/** - * Takes care of replacing a GenTree node's child with a new tree. - * - * Assumptions: - * a) replacementNode has been unlinked (orphaned) and the expression it represents - * is a valid tree, and correctly sequenced internally in case it's not a leaf node. - * b) The location specified in ppTreeLocation must be a descendant of 'stmt'. - * - */ -void Lowering::ReplaceNode(GenTree** ppTreeLocation, GenTree* replacementNode, GenTree* stmt, BasicBlock* block) -{ - assert(ppTreeLocation != nullptr); - GenTreePtr& treeLocation = *ppTreeLocation; - - assert(treeLocation != nullptr); - assert(replacementNode != nullptr); - JITDUMP("The node to replace is:\n"); - DISPNODE(treeLocation); - JITDUMP("The node that replaces it is:\n"); - DISPTREE(replacementNode); - - assert(comp->fgStmtContainsNode((GenTreeStmt*)stmt, treeLocation)); - - GenTreePtr first = comp->fgGetFirstNode(treeLocation); - comp->fgRemoveContainedEmbeddedStatements(treeLocation, stmt->AsStmt(), block); - - assert(first != nullptr); - - GenTreePtr gtPrev = first->gtPrev; - GenTreePtr gtNext = treeLocation->gtNext; - - assert(!treeLocation->OperIsLeaf() || gtPrev == treeLocation->gtPrev); - - if (gtPrev == nullptr) - { - stmt->gtStmt.gtStmtList = replacementNode; - } - else - { - gtPrev->gtNext = replacementNode; - } - - // If we have an embedded statement, and the node we want to - // replace it's the first one in execution order, it won't fit - // the special case of having gtPrev == nullptr, so we have to - // ask directly whether is the first or not. - if (stmt->gtStmt.gtStmtIsEmbedded() && stmt->gtStmt.gtStmtList == first) - { - stmt->gtStmt.gtStmtList = replacementNode; - } - - replacementNode->gtPrev = gtPrev; - - if (gtNext != nullptr) - { - gtNext->gtPrev = replacementNode; - } - - replacementNode->gtNext = gtNext; - treeLocation = replacementNode; -#ifdef DEBUG - comp->fgDebugCheckLinks(); -#endif -} - -/** - * Unlinks a node hanging from the specified location and replaces it with a GT_NOP - * - * Assumptions: - * The location specified in ppParentLink must be a descendant of stmt. - * - */ -void Lowering::UnlinkNode(GenTree** ppParentLink, GenTree* stmt, BasicBlock* block) -{ - ReplaceNode(ppParentLink, comp->gtNewNothingNode(), stmt, block); -} - #ifdef _TARGET_64BIT_ /** * Get common information required to handle a cast instruction @@ -4298,28 +4092,17 @@ void Lowering::getCastDescription(GenTreePtr treeNode, CastInfo* castInfo) #ifdef DEBUG void Lowering::DumpNodeInfoMap() { - // dump tree node info printf("-----------------------------\n"); printf("TREE NODE INFO DUMP\n"); printf("-----------------------------\n"); - for (BasicBlock* block = comp->fgFirstBB; block; block = block->bbNext) + for (BasicBlock* block = comp->fgFirstBB; block != nullptr; block = block->bbNext) { - GenTreePtr stmt; - GenTreePtr tree; - for (stmt = block->FirstNonPhiDef(); stmt; stmt = stmt->gtNext) + for (GenTree* node : LIR::AsRange(block).NonPhiNodes()) { - GenTreePtr node; - foreach_treenode_execution_order(node, stmt) - { - if (stmt->gtStmt.gtStmtIsEmbedded()) - { - continue; - } - comp->gtDispTree(node, nullptr, nullptr, true); - printf(" +"); - node->gtLsraInfo.dump(m_lsra); - } + comp->gtDispTree(node, nullptr, nullptr, true); + printf(" +"); + node->gtLsraInfo.dump(m_lsra); } } } diff --git a/src/jit/lower.h b/src/jit/lower.h index 98e43fff7e..b3e7c7872b 100644 --- a/src/jit/lower.h +++ b/src/jit/lower.h @@ -49,13 +49,15 @@ public: #endif // _TARGET_64BIT_ private: - // Friends - static Compiler::fgWalkResult LowerNodeHelper(GenTreePtr* ppTree, Compiler::fgWalkData* data); - static Compiler::fgWalkResult TreeInfoInitHelper(GenTreePtr* ppTree, Compiler::fgWalkData* data); - - // Member Functions - void LowerNode(GenTreePtr* tree, Compiler::fgWalkData* data); - GenTreeStmt* LowerMorphAndSeqTree(GenTree* tree); +#ifdef DEBUG + static void CheckCallArg(GenTree* arg); + static void CheckCall(GenTreeCall* call); + static void CheckNode(GenTree* node); + static bool CheckBlock(Compiler* compiler, BasicBlock* block); +#endif // DEBUG + + void LowerBlock(BasicBlock* block); + GenTree* LowerNode(GenTree* node); void CheckVSQuirkStackPaddingNeeded(GenTreeCall* call); // ------------------------------ @@ -74,6 +76,7 @@ private: GenTree* LowerVirtualVtableCall(GenTreeCall* call); GenTree* LowerVirtualStubCall(GenTreeCall* call); void LowerArgsForCall(GenTreeCall* call); + void ReplaceArgWithPutArgOrCopy(GenTreePtr* ppChild, GenTreePtr newNode); GenTree* NewPutArg(GenTreeCall* call, GenTreePtr arg, fgArgTabEntryPtr info, var_types type); void LowerArg(GenTreeCall* call, GenTreePtr* ppTree); void InsertPInvokeCallProlog(GenTreeCall* call); @@ -91,25 +94,6 @@ private: GenTree* AddrGen(ssize_t addr, regNumber reg = REG_NA); GenTree* AddrGen(void* addr, regNumber reg = REG_NA); - // return concatenation of two trees, which currently uses a comma and really should not - // because we're not supposed to have commas in codegen - GenTree* Concat(GenTree* first, GenTree* second) - { - // if any is null, it must be the first - if (first == nullptr) - { - return second; - } - else if (second == nullptr) - { - return first; - } - else - { - return comp->gtNewOperNode(GT_COMMA, TYP_I_IMPL, first, second); - } - } - GenTree* Ind(GenTree* tree) { return comp->gtNewOperNode(GT_IND, TYP_I_IMPL, tree); @@ -143,7 +127,7 @@ private: bool IsCallTargetInRange(void* addr); void TreeNodeInfoInit(GenTree* stmt); - void TreeNodeInfoInit(GenTreePtr* tree, GenTree* parent); + #if defined(_TARGET_XARCH_) void TreeNodeInfoInitSimple(GenTree* tree); @@ -221,23 +205,24 @@ private: #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING void TreeNodeInfoInitLclHeap(GenTree* tree); - void SpliceInUnary(GenTreePtr parent, GenTreePtr* ppChild, GenTreePtr newNode); void DumpNodeInfoMap(); // Per tree node member functions - void LowerInd(GenTreePtr* ppTree); - void LowerAddrMode(GenTreePtr* ppTree, GenTree* before, Compiler::fgWalkData* data, bool isIndir); - void LowerAdd(GenTreePtr* ppTree, Compiler::fgWalkData* data); - void LowerUnsignedDivOrMod(GenTree* tree); - void LowerSignedDivOrMod(GenTreePtr* ppTree, Compiler::fgWalkData* data); - - // Remove the nodes that are no longer used after an addressing mode is constructed under a GT_IND - void LowerIndCleanupHelper(GenTreeAddrMode* addrMode, GenTreePtr tree); - void LowerSwitch(GenTreePtr* ppTree); - void LowerCast(GenTreePtr* ppTree); - void LowerCntBlockOp(GenTreePtr* ppTree); + void LowerStoreInd(GenTree* node); + GenTree* LowerAdd(GenTree* node); + void LowerUnsignedDivOrMod(GenTree* node); + GenTree* LowerSignedDivOrMod(GenTree* node); + + GenTree* TryCreateAddrMode(LIR::Use&& use, bool isIndir); + void AddrModeCleanupHelper(GenTreeAddrMode* addrMode, GenTree* node); + + GenTree* LowerSwitch(GenTree* node); + void LowerCast(GenTree* node); +#if defined(_TARGET_XARCH_) void SetMulOpCounts(GenTreePtr tree); +#endif // defined(_TARGET_XARCH_) + void LowerCmp(GenTreePtr tree); #if !CPU_LOAD_STORE_ARCH @@ -248,7 +233,7 @@ private: void LowerStoreLoc(GenTreeLclVarCommon* tree); void SetIndirAddrOpCounts(GenTree* indirTree); void LowerGCWriteBarrier(GenTree* tree); - void LowerArrElem(GenTree** ppTree, Compiler::fgWalkData* data); + GenTree* LowerArrElem(GenTree* node); void LowerRotate(GenTree* tree); // Utility functions @@ -260,12 +245,7 @@ public: private: static bool NodesAreEquivalentLeaves(GenTreePtr candidate, GenTreePtr storeInd); - GenTreePtr CreateLocalTempAsg(GenTreePtr rhs, unsigned refCount, GenTreePtr* ppLclVar = nullptr); - GenTreeStmt* CreateTemporary(GenTree** ppTree); - bool AreSourcesPossiblyModified(GenTree* use, GenTree* src1, GenTree* src2); - void ReplaceNode(GenTree** ppTreeLocation, GenTree* replacementNode, GenTree* stmt, BasicBlock* block); - - void UnlinkNode(GenTree** ppParentLink, GenTree* stmt, BasicBlock* block); + bool AreSourcesPossiblyModified(GenTree* addr, GenTree* base, GenTree* index); // return true if 'childNode' is an immediate that can be contained // by the 'parentNode' (i.e. folded into an instruction) @@ -282,9 +262,14 @@ private: // can be contained. bool IsSafeToContainMem(GenTree* parentNode, GenTree* childNode); + inline LIR::Range& BlockRange() const + { + return LIR::AsRange(m_block); + } + LinearScan* m_lsra; - BasicBlock* currBlock; unsigned vtableCallTemp; // local variable we use as a temp for vtable calls + BasicBlock* m_block; }; #endif // _LOWER_H_ diff --git a/src/jit/lowerarm.cpp b/src/jit/lowerarm.cpp index 2acb7498a2..4c9648598c 100644 --- a/src/jit/lowerarm.cpp +++ b/src/jit/lowerarm.cpp @@ -32,13 +32,9 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #include "lsra.h" /* Lowering of GT_CAST nodes */ -void Lowering::LowerCast(GenTreePtr* ppTree) +void Lowering::LowerCast(GenTree* tree) { -} - -void Lowering::LowerCntBlockOp(GenTreePtr* ppTree) -{ - NYI_ARM("ARM Lowering for BlockOp"); + NYI_ARM("ARM Lowering for cast"); } void Lowering::LowerRotate(GenTreePtr tree) @@ -46,20 +42,8 @@ void Lowering::LowerRotate(GenTreePtr tree) NYI_ARM("ARM Lowering for ROL and ROR"); } -Compiler::fgWalkResult Lowering::TreeInfoInitHelper(GenTreePtr* pTree, Compiler::fgWalkData* data) -{ - Lowering* lower = (Lowering*)data->pCallbackData; - lower->TreeNodeInfoInit(pTree, data->parent); - return Compiler::WALK_CONTINUE; -} - void Lowering::TreeNodeInfoInit(GenTree* stmt) { - comp->fgWalkTreePost(&stmt->gtStmt.gtStmtExpr, &Lowering::TreeInfoInitHelper, this); -} - -void Lowering::TreeNodeInfoInit(GenTreePtr* pTree, GenTree* parent) -{ NYI("ARM TreeNodInfoInit"); } diff --git a/src/jit/lowerarm64.cpp b/src/jit/lowerarm64.cpp index a9c5709209..97b2456b0b 100644 --- a/src/jit/lowerarm64.cpp +++ b/src/jit/lowerarm64.cpp @@ -116,684 +116,675 @@ void Lowering::LowerStoreLoc(GenTreeLclVarCommon* storeLoc) * destination and internal [temp] register counts). * This code is refactored originally from LSRA. */ -void Lowering::TreeNodeInfoInit(GenTree* stmt) +void Lowering::TreeNodeInfoInit(GenTree* tree) { LinearScan* l = m_lsra; Compiler* compiler = comp; - assert(stmt->gtStmt.gtStmtIsTopLevel()); - GenTree* tree = stmt->gtStmt.gtStmtList; + unsigned kind = tree->OperKind(); + TreeNodeInfo* info = &(tree->gtLsraInfo); + RegisterType registerType = TypeGet(tree); - while (tree) + switch (tree->OperGet()) { - unsigned kind = tree->OperKind(); - TreeNodeInfo* info = &(tree->gtLsraInfo); - RegisterType registerType = TypeGet(tree); - GenTree* next = tree->gtNext; + GenTree* op1; + GenTree* op2; - switch (tree->OperGet()) - { - GenTree* op1; - GenTree* op2; - - default: - info->dstCount = (tree->TypeGet() == TYP_VOID) ? 0 : 1; - if (kind & (GTK_CONST | GTK_LEAF)) - { - info->srcCount = 0; - } - else if (kind & (GTK_SMPOP)) - { - if (tree->gtGetOp2() != nullptr) - { - info->srcCount = 2; - } - else - { - info->srcCount = 1; - } - } - else - { - unreached(); - } - break; - - case GT_STORE_LCL_FLD: - case GT_STORE_LCL_VAR: - info->srcCount = 1; - info->dstCount = 0; - LowerStoreLoc(tree->AsLclVarCommon()); - break; - - case GT_BOX: - noway_assert(!"box should not exist here"); - // The result of 'op1' is also the final result + default: + info->dstCount = (tree->TypeGet() == TYP_VOID) ? 0 : 1; + if (kind & (GTK_CONST | GTK_LEAF)) + { info->srcCount = 0; - info->dstCount = 0; - break; - - case GT_PHYSREGDST: - info->srcCount = 1; - info->dstCount = 0; - break; - - case GT_COMMA: + } + else if (kind & (GTK_SMPOP)) { - GenTreePtr firstOperand; - GenTreePtr secondOperand; - if (tree->gtFlags & GTF_REVERSE_OPS) + if (tree->gtGetOp2() != nullptr) { - firstOperand = tree->gtOp.gtOp2; - secondOperand = tree->gtOp.gtOp1; + info->srcCount = 2; } else { - firstOperand = tree->gtOp.gtOp1; - secondOperand = tree->gtOp.gtOp2; - } - if (firstOperand->TypeGet() != TYP_VOID) - { - firstOperand->gtLsraInfo.isLocalDefUse = true; - firstOperand->gtLsraInfo.dstCount = 0; - } - if (tree->TypeGet() == TYP_VOID && secondOperand->TypeGet() != TYP_VOID) - { - secondOperand->gtLsraInfo.isLocalDefUse = true; - secondOperand->gtLsraInfo.dstCount = 0; + info->srcCount = 1; } } + else + { + unreached(); + } + break; - __fallthrough; - - case GT_LIST: - case GT_ARGPLACE: - case GT_NO_OP: - case GT_START_NONGC: - case GT_PROF_HOOK: - info->srcCount = 0; - info->dstCount = 0; - break; - - case GT_CNS_DBL: - info->srcCount = 0; - info->dstCount = 1; - { - GenTreeDblCon* dblConst = tree->AsDblCon(); - double constValue = dblConst->gtDblCon.gtDconVal; + case GT_STORE_LCL_FLD: + case GT_STORE_LCL_VAR: + info->srcCount = 1; + info->dstCount = 0; + LowerStoreLoc(tree->AsLclVarCommon()); + break; - if (emitter::emitIns_valid_imm_for_fmov(constValue)) - { - // Directly encode constant to instructions. - } - else - { - // Reserve int to load constant from memory (IF_LARGELDC) - info->internalIntCount = 1; - } - } - break; + case GT_BOX: + noway_assert(!"box should not exist here"); + // The result of 'op1' is also the final result + info->srcCount = 0; + info->dstCount = 0; + break; - case GT_QMARK: - case GT_COLON: - info->srcCount = 0; - info->dstCount = 0; - unreached(); - break; + case GT_PHYSREGDST: + info->srcCount = 1; + info->dstCount = 0; + break; - case GT_RETURN: - TreeNodeInfoInitReturn(tree); - break; + case GT_COMMA: + { + GenTreePtr firstOperand; + GenTreePtr secondOperand; + if (tree->gtFlags & GTF_REVERSE_OPS) + { + firstOperand = tree->gtOp.gtOp2; + secondOperand = tree->gtOp.gtOp1; + } + else + { + firstOperand = tree->gtOp.gtOp1; + secondOperand = tree->gtOp.gtOp2; + } + if (firstOperand->TypeGet() != TYP_VOID) + { + firstOperand->gtLsraInfo.isLocalDefUse = true; + firstOperand->gtLsraInfo.dstCount = 0; + } + if (tree->TypeGet() == TYP_VOID && secondOperand->TypeGet() != TYP_VOID) + { + secondOperand->gtLsraInfo.isLocalDefUse = true; + secondOperand->gtLsraInfo.dstCount = 0; + } + } - case GT_RETFILT: - if (tree->TypeGet() == TYP_VOID) - { - info->srcCount = 0; - info->dstCount = 0; - } - else - { - assert(tree->TypeGet() == TYP_INT); + __fallthrough; - info->srcCount = 1; - info->dstCount = 1; + case GT_LIST: + case GT_ARGPLACE: + case GT_NO_OP: + case GT_START_NONGC: + case GT_PROF_HOOK: + info->srcCount = 0; + info->dstCount = 0; + break; - info->setSrcCandidates(l, RBM_INTRET); - tree->gtOp.gtOp1->gtLsraInfo.setSrcCandidates(l, RBM_INTRET); - } - break; + case GT_CNS_DBL: + info->srcCount = 0; + info->dstCount = 1; + { + GenTreeDblCon* dblConst = tree->AsDblCon(); + double constValue = dblConst->gtDblCon.gtDconVal; - case GT_NOP: - // A GT_NOP is either a passthrough (if it is void, or if it has - // a child), but must be considered to produce a dummy value if it - // has a type but no child - info->srcCount = 0; - if (tree->TypeGet() != TYP_VOID && tree->gtOp.gtOp1 == nullptr) + if (emitter::emitIns_valid_imm_for_fmov(constValue)) { - info->dstCount = 1; + // Directly encode constant to instructions. } else { - info->dstCount = 0; + // Reserve int to load constant from memory (IF_LARGELDC) + info->internalIntCount = 1; } - break; + } + break; - case GT_JTRUE: - info->srcCount = 0; - info->dstCount = 0; - l->clearDstCount(tree->gtOp.gtOp1); - break; + case GT_QMARK: + case GT_COLON: + info->srcCount = 0; + info->dstCount = 0; + unreached(); + break; - case GT_JMP: - info->srcCount = 0; - info->dstCount = 0; - break; + case GT_RETURN: + TreeNodeInfoInitReturn(tree); + break; - case GT_SWITCH: - // This should never occur since switch nodes must not be visible at this - // point in the JIT. + case GT_RETFILT: + if (tree->TypeGet() == TYP_VOID) + { info->srcCount = 0; - info->dstCount = 0; // To avoid getting uninit errors. - noway_assert(!"Switch must be lowered at this point"); - break; + info->dstCount = 0; + } + else + { + assert(tree->TypeGet() == TYP_INT); - case GT_JMPTABLE: - info->srcCount = 0; + info->srcCount = 1; info->dstCount = 1; - break; - case GT_SWITCH_TABLE: - info->srcCount = 2; - info->internalIntCount = 1; - info->dstCount = 0; - break; + info->setSrcCandidates(l, RBM_INTRET); + tree->gtOp.gtOp1->gtLsraInfo.setSrcCandidates(l, RBM_INTRET); + } + break; - case GT_ASG: - case GT_ASG_ADD: - case GT_ASG_SUB: - noway_assert(!"We should never hit any assignment operator in lowering"); - info->srcCount = 0; + case GT_NOP: + // A GT_NOP is either a passthrough (if it is void, or if it has + // a child), but must be considered to produce a dummy value if it + // has a type but no child + info->srcCount = 0; + if (tree->TypeGet() != TYP_VOID && tree->gtOp.gtOp1 == nullptr) + { + info->dstCount = 1; + } + else + { info->dstCount = 0; - break; + } + break; - case GT_ADD: - case GT_SUB: - if (varTypeIsFloating(tree->TypeGet())) - { - // overflow operations aren't supported on float/double types. - assert(!tree->gtOverflow()); + case GT_JTRUE: + info->srcCount = 0; + info->dstCount = 0; + l->clearDstCount(tree->gtOp.gtOp1); + break; - // No implicit conversions at this stage as the expectation is that - // everything is made explicit by adding casts. - assert(tree->gtOp.gtOp1->TypeGet() == tree->gtOp.gtOp2->TypeGet()); + case GT_JMP: + info->srcCount = 0; + info->dstCount = 0; + break; - info->srcCount = 2; - info->dstCount = 1; + case GT_SWITCH: + // This should never occur since switch nodes must not be visible at this + // point in the JIT. + info->srcCount = 0; + info->dstCount = 0; // To avoid getting uninit errors. + noway_assert(!"Switch must be lowered at this point"); + break; - break; - } + case GT_JMPTABLE: + info->srcCount = 0; + info->dstCount = 1; + break; - __fallthrough; + case GT_SWITCH_TABLE: + info->srcCount = 2; + info->internalIntCount = 1; + info->dstCount = 0; + break; + + case GT_ASG: + case GT_ASG_ADD: + case GT_ASG_SUB: + noway_assert(!"We should never hit any assignment operator in lowering"); + info->srcCount = 0; + info->dstCount = 0; + break; + + case GT_ADD: + case GT_SUB: + if (varTypeIsFloating(tree->TypeGet())) + { + // overflow operations aren't supported on float/double types. + assert(!tree->gtOverflow()); + + // No implicit conversions at this stage as the expectation is that + // everything is made explicit by adding casts. + assert(tree->gtOp.gtOp1->TypeGet() == tree->gtOp.gtOp2->TypeGet()); - case GT_AND: - case GT_OR: - case GT_XOR: info->srcCount = 2; info->dstCount = 1; - // Check and make op2 contained (if it is a containable immediate) - CheckImmedAndMakeContained(tree, tree->gtOp.gtOp2); - break; - case GT_RETURNTRAP: - // this just turns into a compare of its child with an int - // + a conditional call - info->srcCount = 1; - info->dstCount = 1; break; + } - case GT_MOD: - case GT_UMOD: - NYI_IF(varTypeIsFloating(tree->TypeGet()), "FP Remainder in ARM64"); - assert(!"Shouldn't see an integer typed GT_MOD node in ARM64"); - break; + __fallthrough; - case GT_MUL: - if (tree->gtOverflow()) - { - // Need a register different from target reg to check for overflow. - info->internalIntCount = 2; - } - __fallthrough; + case GT_AND: + case GT_OR: + case GT_XOR: + info->srcCount = 2; + info->dstCount = 1; + // Check and make op2 contained (if it is a containable immediate) + CheckImmedAndMakeContained(tree, tree->gtOp.gtOp2); + break; - case GT_DIV: - case GT_MULHI: - case GT_UDIV: - { - info->srcCount = 2; - info->dstCount = 1; - } + case GT_RETURNTRAP: + // this just turns into a compare of its child with an int + // + a conditional call + info->srcCount = 1; + info->dstCount = 1; break; - case GT_INTRINSIC: + case GT_MOD: + case GT_UMOD: + NYI_IF(varTypeIsFloating(tree->TypeGet()), "FP Remainder in ARM64"); + assert(!"Shouldn't see an integer typed GT_MOD node in ARM64"); + break; + + case GT_MUL: + if (tree->gtOverflow()) { - // TODO-ARM64-NYI - // Right now only Abs/Round/Sqrt are treated as math intrinsics - noway_assert((tree->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Abs) || - (tree->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Round) || - (tree->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Sqrt)); + // Need a register different from target reg to check for overflow. + info->internalIntCount = 2; + } + __fallthrough; - // Both operand and its result must be of the same floating point type. - op1 = tree->gtOp.gtOp1; - assert(varTypeIsFloating(op1)); - assert(op1->TypeGet() == tree->TypeGet()); + case GT_DIV: + case GT_MULHI: + case GT_UDIV: + { + info->srcCount = 2; + info->dstCount = 1; + } + break; - info->srcCount = 1; - info->dstCount = 1; - } - break; + case GT_INTRINSIC: + { + // TODO-ARM64-NYI + // Right now only Abs/Round/Sqrt are treated as math intrinsics + noway_assert((tree->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Abs) || + (tree->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Round) || + (tree->gtIntrinsic.gtIntrinsicId == CORINFO_INTRINSIC_Sqrt)); + + // Both operand and its result must be of the same floating point type. + op1 = tree->gtOp.gtOp1; + assert(varTypeIsFloating(op1)); + assert(op1->TypeGet() == tree->TypeGet()); + + info->srcCount = 1; + info->dstCount = 1; + } + break; #ifdef FEATURE_SIMD - case GT_SIMD: - TreeNodeInfoInitSIMD(tree); - break; + case GT_SIMD: + TreeNodeInfoInitSIMD(tree); + break; #endif // FEATURE_SIMD - case GT_CAST: - { - // TODO-ARM64-CQ: Int-To-Int conversions - castOp cannot be a memory op and must have an assigned - // register. - // see CodeGen::genIntToIntCast() + case GT_CAST: + { + // TODO-ARM64-CQ: Int-To-Int conversions - castOp cannot be a memory op and must have an assigned + // register. + // see CodeGen::genIntToIntCast() - info->srcCount = 1; - info->dstCount = 1; + info->srcCount = 1; + info->dstCount = 1; - // Non-overflow casts to/from float/double are done using SSE2 instructions - // and that allow the source operand to be either a reg or memop. Given the - // fact that casts from small int to float/double are done as two-level casts, - // the source operand is always guaranteed to be of size 4 or 8 bytes. - var_types castToType = tree->CastToType(); - GenTreePtr castOp = tree->gtCast.CastOp(); - var_types castOpType = castOp->TypeGet(); - if (tree->gtFlags & GTF_UNSIGNED) - { - castOpType = genUnsignedType(castOpType); - } + // Non-overflow casts to/from float/double are done using SSE2 instructions + // and that allow the source operand to be either a reg or memop. Given the + // fact that casts from small int to float/double are done as two-level casts, + // the source operand is always guaranteed to be of size 4 or 8 bytes. + var_types castToType = tree->CastToType(); + GenTreePtr castOp = tree->gtCast.CastOp(); + var_types castOpType = castOp->TypeGet(); + if (tree->gtFlags & GTF_UNSIGNED) + { + castOpType = genUnsignedType(castOpType); + } #ifdef DEBUG - if (!tree->gtOverflow() && (varTypeIsFloating(castToType) || varTypeIsFloating(castOpType))) + if (!tree->gtOverflow() && (varTypeIsFloating(castToType) || varTypeIsFloating(castOpType))) + { + // If converting to float/double, the operand must be 4 or 8 byte in size. + if (varTypeIsFloating(castToType)) { - // If converting to float/double, the operand must be 4 or 8 byte in size. - if (varTypeIsFloating(castToType)) - { - unsigned opSize = genTypeSize(castOpType); - assert(opSize == 4 || opSize == 8); - } + unsigned opSize = genTypeSize(castOpType); + assert(opSize == 4 || opSize == 8); } + } #endif // DEBUG - // Some overflow checks need a temp reg + // Some overflow checks need a temp reg - CastInfo castInfo; + CastInfo castInfo; - // Get information about the cast. - getCastDescription(tree, &castInfo); + // Get information about the cast. + getCastDescription(tree, &castInfo); - if (castInfo.requiresOverflowCheck) - { - var_types srcType = castOp->TypeGet(); - emitAttr cmpSize = EA_ATTR(genTypeSize(srcType)); + if (castInfo.requiresOverflowCheck) + { + var_types srcType = castOp->TypeGet(); + emitAttr cmpSize = EA_ATTR(genTypeSize(srcType)); - // If we cannot store the comparisons in an immediate for either - // comparing against the max or min value, then we will need to - // reserve a temporary register. + // If we cannot store the comparisons in an immediate for either + // comparing against the max or min value, then we will need to + // reserve a temporary register. - bool canStoreMaxValue = emitter::emitIns_valid_imm_for_cmp(castInfo.typeMax, cmpSize); - bool canStoreMinValue = emitter::emitIns_valid_imm_for_cmp(castInfo.typeMin, cmpSize); + bool canStoreMaxValue = emitter::emitIns_valid_imm_for_cmp(castInfo.typeMax, cmpSize); + bool canStoreMinValue = emitter::emitIns_valid_imm_for_cmp(castInfo.typeMin, cmpSize); - if (!canStoreMaxValue || !canStoreMinValue) - { - info->internalIntCount = 1; - } + if (!canStoreMaxValue || !canStoreMinValue) + { + info->internalIntCount = 1; } } + } + break; + + case GT_NEG: + info->srcCount = 1; + info->dstCount = 1; break; - case GT_NEG: - info->srcCount = 1; - info->dstCount = 1; - break; + case GT_NOT: + info->srcCount = 1; + info->dstCount = 1; + break; - case GT_NOT: - info->srcCount = 1; - info->dstCount = 1; - break; + case GT_LSH: + case GT_RSH: + case GT_RSZ: + case GT_ROR: + { + info->srcCount = 2; + info->dstCount = 1; - case GT_LSH: - case GT_RSH: - case GT_RSZ: - case GT_ROR: + GenTreePtr shiftBy = tree->gtOp.gtOp2; + GenTreePtr source = tree->gtOp.gtOp1; + if (shiftBy->IsCnsIntOrI()) { - info->srcCount = 2; - info->dstCount = 1; - - GenTreePtr shiftBy = tree->gtOp.gtOp2; - GenTreePtr source = tree->gtOp.gtOp1; - if (shiftBy->IsCnsIntOrI()) - { - l->clearDstCount(shiftBy); - info->srcCount--; - } + l->clearDstCount(shiftBy); + info->srcCount--; } + } + break; + + case GT_EQ: + case GT_NE: + case GT_LT: + case GT_LE: + case GT_GE: + case GT_GT: + LowerCmp(tree); break; - case GT_EQ: - case GT_NE: - case GT_LT: - case GT_LE: - case GT_GE: - case GT_GT: - LowerCmp(tree); - break; + case GT_CKFINITE: + info->srcCount = 1; + info->dstCount = 1; + info->internalIntCount = 1; + break; - case GT_CKFINITE: - info->srcCount = 1; - info->dstCount = 1; - info->internalIntCount = 1; - break; + case GT_CMPXCHG: + info->srcCount = 3; + info->dstCount = 1; - case GT_CMPXCHG: - info->srcCount = 3; - info->dstCount = 1; + // TODO-ARM64-NYI + NYI("CMPXCHG"); + break; - // TODO-ARM64-NYI - NYI("CMPXCHG"); - break; + case GT_LOCKADD: + info->srcCount = 2; + info->dstCount = 0; + CheckImmedAndMakeContained(tree, tree->gtOp.gtOp2); + break; - case GT_LOCKADD: - info->srcCount = 2; - info->dstCount = 0; - CheckImmedAndMakeContained(tree, tree->gtOp.gtOp2); - break; + case GT_CALL: + TreeNodeInfoInitCall(tree->AsCall()); + break; - case GT_CALL: - TreeNodeInfoInitCall(tree->AsCall()); - break; + case GT_ADDR: + { + // For a GT_ADDR, the child node should not be evaluated into a register + GenTreePtr child = tree->gtOp.gtOp1; + assert(!l->isCandidateLocalRef(child)); + l->clearDstCount(child); + info->srcCount = 0; + info->dstCount = 1; + } + break; - case GT_ADDR: - { - // For a GT_ADDR, the child node should not be evaluated into a register - GenTreePtr child = tree->gtOp.gtOp1; - assert(!l->isCandidateLocalRef(child)); - l->clearDstCount(child); - info->srcCount = 0; - info->dstCount = 1; - } + case GT_INITBLK: + case GT_COPYBLK: + case GT_COPYOBJ: + TreeNodeInfoInitBlockStore(tree->AsBlkOp()); break; - case GT_INITBLK: - case GT_COPYBLK: - case GT_COPYOBJ: - TreeNodeInfoInitBlockStore(tree->AsBlkOp()); - break; - - case GT_LCLHEAP: - { - info->srcCount = 1; - info->dstCount = 1; + case GT_LCLHEAP: + { + info->srcCount = 1; + info->dstCount = 1; - // Need a variable number of temp regs (see genLclHeap() in codegenamd64.cpp): - // Here '-' means don't care. - // - // Size? Init Memory? # temp regs - // 0 - 0 - // const and <=6 ptr words - 0 - // const and <PageSize No 0 - // >6 ptr words Yes hasPspSym ? 1 : 0 - // Non-const Yes hasPspSym ? 1 : 0 - // Non-const No 2 - // - // PSPSym - If the method has PSPSym increment internalIntCount by 1. - // - bool hasPspSym; + // Need a variable number of temp regs (see genLclHeap() in codegenamd64.cpp): + // Here '-' means don't care. + // + // Size? Init Memory? # temp regs + // 0 - 0 + // const and <=6 ptr words - 0 + // const and <PageSize No 0 + // >6 ptr words Yes hasPspSym ? 1 : 0 + // Non-const Yes hasPspSym ? 1 : 0 + // Non-const No 2 + // + // PSPSym - If the method has PSPSym increment internalIntCount by 1. + // + bool hasPspSym; #if FEATURE_EH_FUNCLETS - hasPspSym = (compiler->lvaPSPSym != BAD_VAR_NUM); + hasPspSym = (compiler->lvaPSPSym != BAD_VAR_NUM); #else - hasPspSym = false; + hasPspSym = false; #endif - GenTreePtr size = tree->gtOp.gtOp1; - if (size->IsCnsIntOrI()) - { - MakeSrcContained(tree, size); + GenTreePtr size = tree->gtOp.gtOp1; + if (size->IsCnsIntOrI()) + { + MakeSrcContained(tree, size); - size_t sizeVal = size->gtIntCon.gtIconVal; + size_t sizeVal = size->gtIntCon.gtIconVal; - if (sizeVal == 0) + if (sizeVal == 0) + { + info->internalIntCount = 0; + } + else + { + // Compute the amount of memory to properly STACK_ALIGN. + // Note: The Gentree node is not updated here as it is cheap to recompute stack aligned size. + // This should also help in debugging as we can examine the original size specified with + // localloc. + sizeVal = AlignUp(sizeVal, STACK_ALIGN); + size_t cntStackAlignedWidthItems = (sizeVal >> STACK_ALIGN_SHIFT); + + // For small allocations upto 4 'stp' instructions (i.e. 64 bytes of localloc) + // + if (cntStackAlignedWidthItems <= 4) { info->internalIntCount = 0; } - else + else if (!compiler->info.compInitMem) { - // Compute the amount of memory to properly STACK_ALIGN. - // Note: The Gentree node is not updated here as it is cheap to recompute stack aligned size. - // This should also help in debugging as we can examine the original size specified with - // localloc. - sizeVal = AlignUp(sizeVal, STACK_ALIGN); - size_t cntStackAlignedWidthItems = (sizeVal >> STACK_ALIGN_SHIFT); - - // For small allocations upto 4 'stp' instructions (i.e. 64 bytes of localloc) - // - if (cntStackAlignedWidthItems <= 4) + // No need to initialize allocated stack space. + if (sizeVal < compiler->eeGetPageSize()) { info->internalIntCount = 0; } - else if (!compiler->info.compInitMem) - { - // No need to initialize allocated stack space. - if (sizeVal < compiler->eeGetPageSize()) - { - info->internalIntCount = 0; - } - else - { - // We need two registers: regCnt and RegTmp - info->internalIntCount = 2; - } - } else { - // greater than 4 and need to zero initialize allocated stack space. - // If the method has PSPSym, we need an internal register to hold regCnt - // since targetReg allocated to GT_LCLHEAP node could be the same as one of - // the the internal registers. - info->internalIntCount = hasPspSym ? 1 : 0; + // We need two registers: regCnt and RegTmp + info->internalIntCount = 2; } } - } - else - { - if (!compiler->info.compInitMem) - { - info->internalIntCount = 2; - } else { + // greater than 4 and need to zero initialize allocated stack space. // If the method has PSPSym, we need an internal register to hold regCnt // since targetReg allocated to GT_LCLHEAP node could be the same as one of // the the internal registers. info->internalIntCount = hasPspSym ? 1 : 0; } } - - // If the method has PSPSym, we would need an addtional register to relocate it on stack. - if (hasPspSym) - { - // Exclude const size 0 - if (!size->IsCnsIntOrI() || (size->gtIntCon.gtIconVal > 0)) - info->internalIntCount++; - } } - break; - - case GT_ARR_BOUNDS_CHECK: -#ifdef FEATURE_SIMD - case GT_SIMD_CHK: -#endif // FEATURE_SIMD + else { - GenTreeBoundsChk* node = tree->AsBoundsChk(); - // Consumes arrLen & index - has no result - info->srcCount = 2; - info->dstCount = 0; - - GenTree* intCns = nullptr; - GenTree* other = nullptr; - if (CheckImmedAndMakeContained(tree, node->gtIndex)) - { - intCns = node->gtIndex; - other = node->gtArrLen; - } - else if (CheckImmedAndMakeContained(tree, node->gtArrLen)) + if (!compiler->info.compInitMem) { - intCns = node->gtArrLen; - other = node->gtIndex; + info->internalIntCount = 2; } else { - other = node->gtIndex; + // If the method has PSPSym, we need an internal register to hold regCnt + // since targetReg allocated to GT_LCLHEAP node could be the same as one of + // the the internal registers. + info->internalIntCount = hasPspSym ? 1 : 0; } } - break; - case GT_ARR_ELEM: - // These must have been lowered to GT_ARR_INDEX - noway_assert(!"We should never see a GT_ARR_ELEM in lowering"); - info->srcCount = 0; - info->dstCount = 0; - break; + // If the method has PSPSym, we would need an addtional register to relocate it on stack. + if (hasPspSym) + { + // Exclude const size 0 + if (!size->IsCnsIntOrI() || (size->gtIntCon.gtIconVal > 0)) + info->internalIntCount++; + } + } + break; - case GT_ARR_INDEX: - info->srcCount = 2; - info->dstCount = 1; + case GT_ARR_BOUNDS_CHECK: +#ifdef FEATURE_SIMD + case GT_SIMD_CHK: +#endif // FEATURE_SIMD + { + GenTreeBoundsChk* node = tree->AsBoundsChk(); + // Consumes arrLen & index - has no result + info->srcCount = 2; + info->dstCount = 0; - // We need one internal register when generating code for GT_ARR_INDEX, however the - // register allocator always may just give us the same one as it gives us for the 'dst' - // as a workaround we will just ask for two internal registers. - // - info->internalIntCount = 2; + GenTree* intCns = nullptr; + GenTree* other = nullptr; + if (CheckImmedAndMakeContained(tree, node->gtIndex)) + { + intCns = node->gtIndex; + other = node->gtArrLen; + } + else if (CheckImmedAndMakeContained(tree, node->gtArrLen)) + { + intCns = node->gtArrLen; + other = node->gtIndex; + } + else + { + other = node->gtIndex; + } + } + break; - // For GT_ARR_INDEX, the lifetime of the arrObj must be extended because it is actually used multiple - // times while the result is being computed. - tree->AsArrIndex()->ArrObj()->gtLsraInfo.isDelayFree = true; - info->hasDelayFreeSrc = true; - break; + case GT_ARR_ELEM: + // These must have been lowered to GT_ARR_INDEX + noway_assert(!"We should never see a GT_ARR_ELEM in lowering"); + info->srcCount = 0; + info->dstCount = 0; + break; - case GT_ARR_OFFSET: - // This consumes the offset, if any, the arrObj and the effective index, - // and produces the flattened offset for this dimension. - info->srcCount = 3; - info->dstCount = 1; - info->internalIntCount = 1; + case GT_ARR_INDEX: + info->srcCount = 2; + info->dstCount = 1; - // we don't want to generate code for this - if (tree->gtArrOffs.gtOffset->IsIntegralConst(0)) - { - MakeSrcContained(tree, tree->gtArrOffs.gtOffset); - } - break; + // We need one internal register when generating code for GT_ARR_INDEX, however the + // register allocator always may just give us the same one as it gives us for the 'dst' + // as a workaround we will just ask for two internal registers. + // + info->internalIntCount = 2; - case GT_LEA: + // For GT_ARR_INDEX, the lifetime of the arrObj must be extended because it is actually used multiple + // times while the result is being computed. + tree->AsArrIndex()->ArrObj()->gtLsraInfo.isDelayFree = true; + info->hasDelayFreeSrc = true; + break; + + case GT_ARR_OFFSET: + // This consumes the offset, if any, the arrObj and the effective index, + // and produces the flattened offset for this dimension. + info->srcCount = 3; + info->dstCount = 1; + info->internalIntCount = 1; + + // we don't want to generate code for this + if (tree->gtArrOffs.gtOffset->IsIntegralConst(0)) { - GenTreeAddrMode* lea = tree->AsAddrMode(); + MakeSrcContained(tree, tree->gtArrOffs.gtOffset); + } + break; - GenTree* base = lea->Base(); - GenTree* index = lea->Index(); - unsigned cns = lea->gtOffset; + case GT_LEA: + { + GenTreeAddrMode* lea = tree->AsAddrMode(); - // This LEA is instantiating an address, - // so we set up the srcCount and dstCount here. - info->srcCount = 0; - if (base != nullptr) - { - info->srcCount++; - } - if (index != nullptr) - { - info->srcCount++; - } - info->dstCount = 1; + GenTree* base = lea->Base(); + GenTree* index = lea->Index(); + unsigned cns = lea->gtOffset; - // On ARM64 we may need a single internal register - // (when both conditions are true then we still only need a single internal register) - if ((index != nullptr) && (cns != 0)) - { - // ARM64 does not support both Index and offset so we need an internal register - info->internalIntCount = 1; - } - else if (!emitter::emitIns_valid_imm_for_add(cns, EA_8BYTE)) - { - // This offset can't be contained in the add instruction, so we need an internal register - info->internalIntCount = 1; - } + // This LEA is instantiating an address, + // so we set up the srcCount and dstCount here. + info->srcCount = 0; + if (base != nullptr) + { + info->srcCount++; } - break; + if (index != nullptr) + { + info->srcCount++; + } + info->dstCount = 1; - case GT_STOREIND: + // On ARM64 we may need a single internal register + // (when both conditions are true then we still only need a single internal register) + if ((index != nullptr) && (cns != 0)) { - info->srcCount = 2; - info->dstCount = 0; - GenTree* src = tree->gtOp.gtOp2; + // ARM64 does not support both Index and offset so we need an internal register + info->internalIntCount = 1; + } + else if (!emitter::emitIns_valid_imm_for_add(cns, EA_8BYTE)) + { + // This offset can't be contained in the add instruction, so we need an internal register + info->internalIntCount = 1; + } + } + break; - if (compiler->codeGen->gcInfo.gcIsWriteBarrierAsgNode(tree)) - { - LowerGCWriteBarrier(tree); - break; - } - if (!varTypeIsFloating(src->TypeGet()) && src->IsIntegralConst(0)) - { - // an integer zero for 'src' can be contained. - MakeSrcContained(tree, src); - } + case GT_STOREIND: + { + info->srcCount = 2; + info->dstCount = 0; + GenTree* src = tree->gtOp.gtOp2; - SetIndirAddrOpCounts(tree); + if (compiler->codeGen->gcInfo.gcIsWriteBarrierAsgNode(tree)) + { + LowerGCWriteBarrier(tree); + break; + } + if (!varTypeIsFloating(src->TypeGet()) && src->IsIntegralConst(0)) + { + // an integer zero for 'src' can be contained. + MakeSrcContained(tree, src); } - break; - case GT_NULLCHECK: - info->dstCount = 0; - info->srcCount = 1; - info->isLocalDefUse = true; - // null check is an indirection on an addr - SetIndirAddrOpCounts(tree); - break; + SetIndirAddrOpCounts(tree); + } + break; - case GT_IND: - info->dstCount = 1; - info->srcCount = 1; - SetIndirAddrOpCounts(tree); - break; + case GT_NULLCHECK: + info->dstCount = 0; + info->srcCount = 1; + info->isLocalDefUse = true; + // null check is an indirection on an addr + SetIndirAddrOpCounts(tree); + break; - case GT_CATCH_ARG: - info->srcCount = 0; - info->dstCount = 1; - info->setDstCandidates(l, RBM_EXCEPTION_OBJECT); - break; + case GT_IND: + info->dstCount = 1; + info->srcCount = 1; + SetIndirAddrOpCounts(tree); + break; - case GT_CLS_VAR: - info->srcCount = 0; - // GT_CLS_VAR, by the time we reach the backend, must always - // be a pure use. - // It will produce a result of the type of the - // node, and use an internal register for the address. + case GT_CATCH_ARG: + info->srcCount = 0; + info->dstCount = 1; + info->setDstCandidates(l, RBM_EXCEPTION_OBJECT); + break; - info->dstCount = 1; - assert((tree->gtFlags & (GTF_VAR_DEF | GTF_VAR_USEASG | GTF_VAR_USEDEF)) == 0); - info->internalIntCount = 1; - break; - } // end switch (tree->OperGet()) + case GT_CLS_VAR: + info->srcCount = 0; + // GT_CLS_VAR, by the time we reach the backend, must always + // be a pure use. + // It will produce a result of the type of the + // node, and use an internal register for the address. - // We need to be sure that we've set info->srcCount and info->dstCount appropriately - assert((info->dstCount < 2) || tree->IsMultiRegCall()); + info->dstCount = 1; + assert((tree->gtFlags & (GTF_VAR_DEF | GTF_VAR_USEASG | GTF_VAR_USEDEF)) == 0); + info->internalIntCount = 1; + break; + } // end switch (tree->OperGet()) - tree = next; - } + // We need to be sure that we've set info->srcCount and info->dstCount appropriately + assert((info->dstCount < 2) || tree->IsMultiRegCall()); } //------------------------------------------------------------------------ // TreeNodeInfoInitReturn: Set the NodeInfo for a GT_RETURN. @@ -1048,7 +1039,7 @@ void Lowering::TreeNodeInfoInitCall(GenTreeCall* call) else { #ifdef DEBUG - compiler->gtDispTree(argNode); + compiler->gtDispTreeRange(BlockRange(), argNode); #endif noway_assert(!"Unsupported TYP_STRUCT arg kind"); } @@ -1829,9 +1820,8 @@ void Lowering::LowerCmp(GenTreePtr tree) * i) GT_CAST(float/double, int type with overflow detection) * */ -void Lowering::LowerCast(GenTreePtr* ppTree) +void Lowering::LowerCast(GenTree* tree) { - GenTreePtr tree = *ppTree; assert(tree->OperGet() == GT_CAST); GenTreePtr op1 = tree->gtOp.gtOp1; @@ -1869,7 +1859,7 @@ void Lowering::LowerCast(GenTreePtr* ppTree) tree->gtFlags &= ~GTF_UNSIGNED; tree->gtOp.gtOp1 = tmp; - op1->InsertAfterSelf(tmp); + BlockRange().InsertAfter(op1, tmp); } } @@ -1892,7 +1882,7 @@ void Lowering::LowerRotate(GenTreePtr tree) { GenTreePtr tmp = comp->gtNewOperNode(GT_NEG, genActualType(rotateLeftIndexNode->gtType), rotateLeftIndexNode); - rotateLeftIndexNode->InsertAfterSelf(tmp); + BlockRange().InsertAfter(rotateLeftIndexNode, tmp); tree->gtOp.gtOp2 = tmp; } tree->ChangeOper(GT_ROR); diff --git a/src/jit/lowerxarch.cpp b/src/jit/lowerxarch.cpp index 318ef2c5e1..e8080ba945 100644 --- a/src/jit/lowerxarch.cpp +++ b/src/jit/lowerxarch.cpp @@ -141,718 +141,709 @@ void Lowering::LowerStoreLoc(GenTreeLclVarCommon* storeLoc) * destination and internal [temp] register counts). * This code is refactored originally from LSRA. */ -void Lowering::TreeNodeInfoInit(GenTree* stmt) +void Lowering::TreeNodeInfoInit(GenTree* tree) { LinearScan* l = m_lsra; Compiler* compiler = comp; - assert(stmt->gtStmt.gtStmtIsTopLevel()); - GenTree* tree = stmt->gtStmt.gtStmtList; + TreeNodeInfo* info = &(tree->gtLsraInfo); - while (tree) + switch (tree->OperGet()) { - TreeNodeInfo* info = &(tree->gtLsraInfo); - GenTree* next = tree->gtNext; - - switch (tree->OperGet()) - { - GenTree* op1; - GenTree* op2; + GenTree* op1; + GenTree* op2; - default: - TreeNodeInfoInitSimple(tree); - break; + default: + TreeNodeInfoInitSimple(tree); + break; - case GT_LCL_FLD: - info->srcCount = 0; - info->dstCount = 1; + case GT_LCL_FLD: + info->srcCount = 0; + info->dstCount = 1; #ifdef FEATURE_SIMD - // Need an additional register to read upper 4 bytes of Vector3. - if (tree->TypeGet() == TYP_SIMD12) - { - // We need an internal register different from targetReg in which 'tree' produces its result - // because both targetReg and internal reg will be in use at the same time. This is achieved - // by asking for two internal registers. - info->internalFloatCount = 2; - info->setInternalCandidates(m_lsra, m_lsra->allSIMDRegs()); - } + // Need an additional register to read upper 4 bytes of Vector3. + if (tree->TypeGet() == TYP_SIMD12) + { + // We need an internal register different from targetReg in which 'tree' produces its result + // because both targetReg and internal reg will be in use at the same time. This is achieved + // by asking for two internal registers. + info->internalFloatCount = 2; + info->setInternalCandidates(m_lsra, m_lsra->allSIMDRegs()); + } #endif - break; + break; - case GT_STORE_LCL_FLD: - case GT_STORE_LCL_VAR: - info->srcCount = 1; - info->dstCount = 0; - LowerStoreLoc(tree->AsLclVarCommon()); - break; + case GT_STORE_LCL_FLD: + case GT_STORE_LCL_VAR: + info->srcCount = 1; + info->dstCount = 0; + LowerStoreLoc(tree->AsLclVarCommon()); + break; - case GT_BOX: - noway_assert(!"box should not exist here"); - // The result of 'op1' is also the final result - info->srcCount = 0; - info->dstCount = 0; - break; + case GT_BOX: + noway_assert(!"box should not exist here"); + // The result of 'op1' is also the final result + info->srcCount = 0; + info->dstCount = 0; + break; - case GT_PHYSREGDST: - info->srcCount = 1; - info->dstCount = 0; - break; + case GT_PHYSREGDST: + info->srcCount = 1; + info->dstCount = 0; + break; - case GT_COMMA: + case GT_COMMA: + { + GenTreePtr firstOperand; + GenTreePtr secondOperand; + if (tree->gtFlags & GTF_REVERSE_OPS) { - GenTreePtr firstOperand; - GenTreePtr secondOperand; - if (tree->gtFlags & GTF_REVERSE_OPS) - { - firstOperand = tree->gtOp.gtOp2; - secondOperand = tree->gtOp.gtOp1; - } - else - { - firstOperand = tree->gtOp.gtOp1; - secondOperand = tree->gtOp.gtOp2; - } - if (firstOperand->TypeGet() != TYP_VOID) - { - firstOperand->gtLsraInfo.isLocalDefUse = true; - firstOperand->gtLsraInfo.dstCount = 0; - } - if (tree->TypeGet() == TYP_VOID && secondOperand->TypeGet() != TYP_VOID) - { - secondOperand->gtLsraInfo.isLocalDefUse = true; - secondOperand->gtLsraInfo.dstCount = 0; - } + firstOperand = tree->gtOp.gtOp2; + secondOperand = tree->gtOp.gtOp1; } - info->srcCount = 0; - info->dstCount = 0; - break; + else + { + firstOperand = tree->gtOp.gtOp1; + secondOperand = tree->gtOp.gtOp2; + } + if (firstOperand->TypeGet() != TYP_VOID) + { + firstOperand->gtLsraInfo.isLocalDefUse = true; + firstOperand->gtLsraInfo.dstCount = 0; + } + if (tree->TypeGet() == TYP_VOID && secondOperand->TypeGet() != TYP_VOID) + { + secondOperand->gtLsraInfo.isLocalDefUse = true; + secondOperand->gtLsraInfo.dstCount = 0; + } + } + info->srcCount = 0; + info->dstCount = 0; + break; - case GT_LIST: - case GT_ARGPLACE: - case GT_NO_OP: - case GT_START_NONGC: - case GT_PROF_HOOK: - info->srcCount = 0; - info->dstCount = 0; - break; + case GT_LIST: + case GT_ARGPLACE: + case GT_NO_OP: + case GT_START_NONGC: + case GT_PROF_HOOK: + info->srcCount = 0; + info->dstCount = 0; + break; - case GT_CNS_DBL: - info->srcCount = 0; - info->dstCount = 1; - break; + case GT_CNS_DBL: + info->srcCount = 0; + info->dstCount = 1; + break; #if !defined(_TARGET_64BIT_) - case GT_LONG: - if (tree->gtNext == nullptr) - { - // An uncontained GT_LONG node needs to consume its source operands - info->srcCount = 2; - } - else - { - // Passthrough - info->srcCount = 0; - } - info->dstCount = 0; - break; + case GT_LONG: + if (tree->gtNext == nullptr) + { + // An uncontained GT_LONG node needs to consume its source operands + info->srcCount = 2; + } + else + { + // Passthrough + info->srcCount = 0; + } + info->dstCount = 0; + break; #endif // !defined(_TARGET_64BIT_) - case GT_QMARK: - case GT_COLON: - info->srcCount = 0; - info->dstCount = 0; - unreached(); - break; - - case GT_RETURN: - TreeNodeInfoInitReturn(tree); - break; + case GT_QMARK: + case GT_COLON: + info->srcCount = 0; + info->dstCount = 0; + unreached(); + break; - case GT_RETFILT: - if (tree->TypeGet() == TYP_VOID) - { - info->srcCount = 0; - info->dstCount = 0; - } - else - { - assert(tree->TypeGet() == TYP_INT); + case GT_RETURN: + TreeNodeInfoInitReturn(tree); + break; - info->srcCount = 1; - info->dstCount = 1; + case GT_RETFILT: + if (tree->TypeGet() == TYP_VOID) + { + info->srcCount = 0; + info->dstCount = 0; + } + else + { + assert(tree->TypeGet() == TYP_INT); - info->setSrcCandidates(l, RBM_INTRET); - tree->gtOp.gtOp1->gtLsraInfo.setSrcCandidates(l, RBM_INTRET); - } - break; + info->srcCount = 1; + info->dstCount = 1; - // A GT_NOP is either a passthrough (if it is void, or if it has - // a child), but must be considered to produce a dummy value if it - // has a type but no child - case GT_NOP: - info->srcCount = 0; - if (tree->TypeGet() != TYP_VOID && tree->gtOp.gtOp1 == nullptr) - { - info->dstCount = 1; - } - else - { - info->dstCount = 0; - } - break; + info->setSrcCandidates(l, RBM_INTRET); + tree->gtOp.gtOp1->gtLsraInfo.setSrcCandidates(l, RBM_INTRET); + } + break; - case GT_JTRUE: - info->srcCount = 0; + // A GT_NOP is either a passthrough (if it is void, or if it has + // a child), but must be considered to produce a dummy value if it + // has a type but no child + case GT_NOP: + info->srcCount = 0; + if (tree->TypeGet() != TYP_VOID && tree->gtOp.gtOp1 == nullptr) + { + info->dstCount = 1; + } + else + { info->dstCount = 0; - l->clearDstCount(tree->gtOp.gtOp1); - break; + } + break; - case GT_JMP: - info->srcCount = 0; - info->dstCount = 0; - break; + case GT_JTRUE: + info->srcCount = 0; + info->dstCount = 0; + l->clearDstCount(tree->gtOp.gtOp1); + break; - case GT_SWITCH: - // This should never occur since switch nodes must not be visible at this - // point in the JIT. - info->srcCount = 0; - info->dstCount = 0; // To avoid getting uninit errors. - noway_assert(!"Switch must be lowered at this point"); - break; + case GT_JMP: + info->srcCount = 0; + info->dstCount = 0; + break; - case GT_JMPTABLE: - info->srcCount = 0; - info->dstCount = 1; - break; + case GT_SWITCH: + // This should never occur since switch nodes must not be visible at this + // point in the JIT. + info->srcCount = 0; + info->dstCount = 0; // To avoid getting uninit errors. + noway_assert(!"Switch must be lowered at this point"); + break; - case GT_SWITCH_TABLE: - info->srcCount = 2; - info->internalIntCount = 1; - info->dstCount = 0; - break; + case GT_JMPTABLE: + info->srcCount = 0; + info->dstCount = 1; + break; - case GT_ASG: - case GT_ASG_ADD: - case GT_ASG_SUB: - noway_assert(!"We should never hit any assignment operator in lowering"); - info->srcCount = 0; - info->dstCount = 0; - break; + case GT_SWITCH_TABLE: + info->srcCount = 2; + info->internalIntCount = 1; + info->dstCount = 0; + break; + + case GT_ASG: + case GT_ASG_ADD: + case GT_ASG_SUB: + noway_assert(!"We should never hit any assignment operator in lowering"); + info->srcCount = 0; + info->dstCount = 0; + break; #if !defined(_TARGET_64BIT_) - case GT_ADD_LO: - case GT_ADD_HI: - case GT_SUB_LO: - case GT_SUB_HI: + case GT_ADD_LO: + case GT_ADD_HI: + case GT_SUB_LO: + case GT_SUB_HI: #endif - case GT_ADD: - case GT_SUB: - // SSE2 arithmetic instructions doesn't support the form "op mem, xmm". - // Rather they only support "op xmm, mem/xmm" form. - if (varTypeIsFloating(tree->TypeGet())) - { - // overflow operations aren't supported on float/double types. - assert(!tree->gtOverflow()); + case GT_ADD: + case GT_SUB: + // SSE2 arithmetic instructions doesn't support the form "op mem, xmm". + // Rather they only support "op xmm, mem/xmm" form. + if (varTypeIsFloating(tree->TypeGet())) + { + // overflow operations aren't supported on float/double types. + assert(!tree->gtOverflow()); - op1 = tree->gtGetOp1(); - op2 = tree->gtGetOp2(); + op1 = tree->gtGetOp1(); + op2 = tree->gtGetOp2(); - // No implicit conversions at this stage as the expectation is that - // everything is made explicit by adding casts. - assert(op1->TypeGet() == op2->TypeGet()); + // No implicit conversions at this stage as the expectation is that + // everything is made explicit by adding casts. + assert(op1->TypeGet() == op2->TypeGet()); - info->srcCount = 2; - info->dstCount = 1; + info->srcCount = 2; + info->dstCount = 1; - if (op2->isMemoryOp() || op2->IsCnsNonZeroFltOrDbl()) - { - MakeSrcContained(tree, op2); - } - else if (tree->OperIsCommutative() && - (op1->IsCnsNonZeroFltOrDbl() || (op1->isMemoryOp() && IsSafeToContainMem(tree, op1)))) - { - // Though we have GT_ADD(op1=memOp, op2=non-memOp, we try to reorder the operands - // as long as it is safe so that the following efficient code sequence is generated: - // addss/sd targetReg, memOp (if op1Reg == targetReg) OR - // movaps targetReg, op2Reg; addss/sd targetReg, [memOp] - // - // Instead of - // movss op1Reg, [memOp]; addss/sd targetReg, Op2Reg (if op1Reg == targetReg) OR - // movss op1Reg, [memOp]; movaps targetReg, op1Reg, addss/sd targetReg, Op2Reg - MakeSrcContained(tree, op1); - } - else - { - // If there are no containable operands, we can make an operand reg optional. - SetRegOptionalForBinOp(tree); - } - break; + if (op2->isMemoryOp() || op2->IsCnsNonZeroFltOrDbl()) + { + MakeSrcContained(tree, op2); } - - __fallthrough; - - case GT_AND: - case GT_OR: - case GT_XOR: - TreeNodeInfoInitLogicalOp(tree); - break; - - case GT_RETURNTRAP: - // this just turns into a compare of its child with an int - // + a conditional call - info->srcCount = 1; - info->dstCount = 0; - if (tree->gtOp.gtOp1->isIndir()) + else if (tree->OperIsCommutative() && + (op1->IsCnsNonZeroFltOrDbl() || (op1->isMemoryOp() && IsSafeToContainMem(tree, op1)))) { - MakeSrcContained(tree, tree->gtOp.gtOp1); + // Though we have GT_ADD(op1=memOp, op2=non-memOp, we try to reorder the operands + // as long as it is safe so that the following efficient code sequence is generated: + // addss/sd targetReg, memOp (if op1Reg == targetReg) OR + // movaps targetReg, op2Reg; addss/sd targetReg, [memOp] + // + // Instead of + // movss op1Reg, [memOp]; addss/sd targetReg, Op2Reg (if op1Reg == targetReg) OR + // movss op1Reg, [memOp]; movaps targetReg, op1Reg, addss/sd targetReg, Op2Reg + MakeSrcContained(tree, op1); + } + else + { + // If there are no containable operands, we can make an operand reg optional. + SetRegOptionalForBinOp(tree); } - info->internalIntCount = 1; - info->setInternalCandidates(l, l->allRegs(TYP_INT)); break; + } - case GT_MOD: - case GT_DIV: - case GT_UMOD: - case GT_UDIV: - TreeNodeInfoInitModDiv(tree); - break; + __fallthrough; - case GT_MUL: - case GT_MULHI: - SetMulOpCounts(tree); - break; + case GT_AND: + case GT_OR: + case GT_XOR: + TreeNodeInfoInitLogicalOp(tree); + break; - case GT_INTRINSIC: - TreeNodeInfoInitIntrinsic(tree); - break; + case GT_RETURNTRAP: + // this just turns into a compare of its child with an int + // + a conditional call + info->srcCount = 1; + info->dstCount = 0; + if (tree->gtOp.gtOp1->isIndir()) + { + MakeSrcContained(tree, tree->gtOp.gtOp1); + } + info->internalIntCount = 1; + info->setInternalCandidates(l, l->allRegs(TYP_INT)); + break; + + case GT_MOD: + case GT_DIV: + case GT_UMOD: + case GT_UDIV: + TreeNodeInfoInitModDiv(tree); + break; + + case GT_MUL: + case GT_MULHI: + SetMulOpCounts(tree); + break; + + case GT_INTRINSIC: + TreeNodeInfoInitIntrinsic(tree); + break; #ifdef FEATURE_SIMD - case GT_SIMD: - TreeNodeInfoInitSIMD(tree); - break; + case GT_SIMD: + TreeNodeInfoInitSIMD(tree); + break; #endif // FEATURE_SIMD - case GT_CAST: - TreeNodeInfoInitCast(tree); - break; - - case GT_NEG: - info->srcCount = 1; - info->dstCount = 1; + case GT_CAST: + TreeNodeInfoInitCast(tree); + break; - // TODO-XArch-CQ: - // SSE instruction set doesn't have an instruction to negate a number. - // The recommended way is to xor the float/double number with a bitmask. - // The only way to xor is using xorps or xorpd both of which operate on - // 128-bit operands. To hold the bit-mask we would need another xmm - // register or a 16-byte aligned 128-bit data constant. Right now emitter - // lacks the support for emitting such constants or instruction with mem - // addressing mode referring to a 128-bit operand. For now we use an - // internal xmm register to load 32/64-bit bitmask from data section. - // Note that by trading additional data section memory (128-bit) we can - // save on the need for an internal register and also a memory-to-reg - // move. - // - // Note: another option to avoid internal register requirement is by - // lowering as GT_SUB(0, src). This will generate code different from - // Jit64 and could possibly result in compat issues (?). - if (varTypeIsFloating(tree)) - { - info->internalFloatCount = 1; - info->setInternalCandidates(l, l->internalFloatRegCandidates()); - } - break; + case GT_NEG: + info->srcCount = 1; + info->dstCount = 1; - case GT_NOT: - info->srcCount = 1; - info->dstCount = 1; - break; + // TODO-XArch-CQ: + // SSE instruction set doesn't have an instruction to negate a number. + // The recommended way is to xor the float/double number with a bitmask. + // The only way to xor is using xorps or xorpd both of which operate on + // 128-bit operands. To hold the bit-mask we would need another xmm + // register or a 16-byte aligned 128-bit data constant. Right now emitter + // lacks the support for emitting such constants or instruction with mem + // addressing mode referring to a 128-bit operand. For now we use an + // internal xmm register to load 32/64-bit bitmask from data section. + // Note that by trading additional data section memory (128-bit) we can + // save on the need for an internal register and also a memory-to-reg + // move. + // + // Note: another option to avoid internal register requirement is by + // lowering as GT_SUB(0, src). This will generate code different from + // Jit64 and could possibly result in compat issues (?). + if (varTypeIsFloating(tree)) + { + info->internalFloatCount = 1; + info->setInternalCandidates(l, l->internalFloatRegCandidates()); + } + break; - case GT_LSH: - case GT_RSH: - case GT_RSZ: - case GT_ROL: - case GT_ROR: - TreeNodeInfoInitShiftRotate(tree); - break; + case GT_NOT: + info->srcCount = 1; + info->dstCount = 1; + break; - case GT_EQ: - case GT_NE: - case GT_LT: - case GT_LE: - case GT_GE: - case GT_GT: - LowerCmp(tree); - break; + case GT_LSH: + case GT_RSH: + case GT_RSZ: + case GT_ROL: + case GT_ROR: + TreeNodeInfoInitShiftRotate(tree); + break; - case GT_CKFINITE: - info->srcCount = 1; - info->dstCount = 1; - info->internalIntCount = 1; - break; + case GT_EQ: + case GT_NE: + case GT_LT: + case GT_LE: + case GT_GE: + case GT_GT: + LowerCmp(tree); + break; - case GT_CMPXCHG: - info->srcCount = 3; - info->dstCount = 1; + case GT_CKFINITE: + info->srcCount = 1; + info->dstCount = 1; + info->internalIntCount = 1; + break; - // comparand is preferenced to RAX. - // Remaining two operands can be in any reg other than RAX. - tree->gtCmpXchg.gtOpComparand->gtLsraInfo.setSrcCandidates(l, RBM_RAX); - tree->gtCmpXchg.gtOpLocation->gtLsraInfo.setSrcCandidates(l, l->allRegs(TYP_INT) & ~RBM_RAX); - tree->gtCmpXchg.gtOpValue->gtLsraInfo.setSrcCandidates(l, l->allRegs(TYP_INT) & ~RBM_RAX); - tree->gtLsraInfo.setDstCandidates(l, RBM_RAX); - break; + case GT_CMPXCHG: + info->srcCount = 3; + info->dstCount = 1; - case GT_LOCKADD: - info->srcCount = 2; - info->dstCount = 0; + // comparand is preferenced to RAX. + // Remaining two operands can be in any reg other than RAX. + tree->gtCmpXchg.gtOpComparand->gtLsraInfo.setSrcCandidates(l, RBM_RAX); + tree->gtCmpXchg.gtOpLocation->gtLsraInfo.setSrcCandidates(l, l->allRegs(TYP_INT) & ~RBM_RAX); + tree->gtCmpXchg.gtOpValue->gtLsraInfo.setSrcCandidates(l, l->allRegs(TYP_INT) & ~RBM_RAX); + tree->gtLsraInfo.setDstCandidates(l, RBM_RAX); + break; - CheckImmedAndMakeContained(tree, tree->gtOp.gtOp2); - break; + case GT_LOCKADD: + info->srcCount = 2; + info->dstCount = 0; - case GT_CALL: - TreeNodeInfoInitCall(tree->AsCall()); - break; + CheckImmedAndMakeContained(tree, tree->gtOp.gtOp2); + break; - case GT_ADDR: - { - // For a GT_ADDR, the child node should not be evaluated into a register - GenTreePtr child = tree->gtOp.gtOp1; - assert(!l->isCandidateLocalRef(child)); - l->clearDstCount(child); - info->srcCount = 0; - info->dstCount = 1; - } + case GT_CALL: + TreeNodeInfoInitCall(tree->AsCall()); break; + case GT_ADDR: + { + // For a GT_ADDR, the child node should not be evaluated into a register + GenTreePtr child = tree->gtOp.gtOp1; + assert(!l->isCandidateLocalRef(child)); + l->clearDstCount(child); + info->srcCount = 0; + info->dstCount = 1; + } + break; + #ifdef _TARGET_X86_ - case GT_OBJ: - NYI_X86("GT_OBJ"); + case GT_OBJ: + NYI_X86("GT_OBJ"); #endif //_TARGET_X86_ - case GT_INITBLK: - case GT_COPYBLK: - case GT_COPYOBJ: - TreeNodeInfoInitBlockStore(tree->AsBlkOp()); - break; + case GT_INITBLK: + case GT_COPYBLK: + case GT_COPYOBJ: + TreeNodeInfoInitBlockStore(tree->AsBlkOp()); + break; #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING - case GT_PUTARG_STK: - TreeNodeInfoInitPutArgStk(tree); - break; + case GT_PUTARG_STK: + TreeNodeInfoInitPutArgStk(tree); + break; #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING - case GT_LCLHEAP: - TreeNodeInfoInitLclHeap(tree); - break; + case GT_LCLHEAP: + TreeNodeInfoInitLclHeap(tree); + break; - case GT_ARR_BOUNDS_CHECK: + case GT_ARR_BOUNDS_CHECK: #ifdef FEATURE_SIMD - case GT_SIMD_CHK: + case GT_SIMD_CHK: #endif // FEATURE_SIMD + { + GenTreeBoundsChk* node = tree->AsBoundsChk(); + // Consumes arrLen & index - has no result + info->srcCount = 2; + info->dstCount = 0; + + GenTreePtr other; + if (CheckImmedAndMakeContained(tree, node->gtIndex)) { - GenTreeBoundsChk* node = tree->AsBoundsChk(); - // Consumes arrLen & index - has no result - info->srcCount = 2; - info->dstCount = 0; + other = node->gtArrLen; + } + else if (CheckImmedAndMakeContained(tree, node->gtArrLen)) + { + other = node->gtIndex; + } + else if (node->gtIndex->isMemoryOp()) + { + other = node->gtIndex; + } + else + { + other = node->gtArrLen; + } - GenTreePtr other; - if (CheckImmedAndMakeContained(tree, node->gtIndex)) - { - other = node->gtArrLen; - } - else if (CheckImmedAndMakeContained(tree, node->gtArrLen)) - { - other = node->gtIndex; - } - else if (node->gtIndex->isMemoryOp()) + if (node->gtIndex->TypeGet() == node->gtArrLen->TypeGet()) + { + if (other->isMemoryOp()) { - other = node->gtIndex; + MakeSrcContained(tree, other); } else { - other = node->gtArrLen; - } - - if (node->gtIndex->TypeGet() == node->gtArrLen->TypeGet()) - { - if (other->isMemoryOp()) - { - MakeSrcContained(tree, other); - } - else - { - // We can mark 'other' as reg optional, since it is not contained. - SetRegOptional(other); - } + // We can mark 'other' as reg optional, since it is not contained. + SetRegOptional(other); } } + } + break; + + case GT_ARR_ELEM: + // These must have been lowered to GT_ARR_INDEX + noway_assert(!"We should never see a GT_ARR_ELEM in lowering"); + info->srcCount = 0; + info->dstCount = 0; break; - case GT_ARR_ELEM: - // These must have been lowered to GT_ARR_INDEX - noway_assert(!"We should never see a GT_ARR_ELEM in lowering"); - info->srcCount = 0; - info->dstCount = 0; - break; + case GT_ARR_INDEX: + info->srcCount = 2; + info->dstCount = 1; + // For GT_ARR_INDEX, the lifetime of the arrObj must be extended because it is actually used multiple + // times while the result is being computed. + tree->AsArrIndex()->ArrObj()->gtLsraInfo.isDelayFree = true; + info->hasDelayFreeSrc = true; + break; - case GT_ARR_INDEX: - info->srcCount = 2; - info->dstCount = 1; - // For GT_ARR_INDEX, the lifetime of the arrObj must be extended because it is actually used multiple - // times while the result is being computed. - tree->AsArrIndex()->ArrObj()->gtLsraInfo.isDelayFree = true; - info->hasDelayFreeSrc = true; - break; + case GT_ARR_OFFSET: + // This consumes the offset, if any, the arrObj and the effective index, + // and produces the flattened offset for this dimension. + info->srcCount = 3; + info->dstCount = 1; + info->internalIntCount = 1; + // we don't want to generate code for this + if (tree->gtArrOffs.gtOffset->IsIntegralConst(0)) + { + MakeSrcContained(tree, tree->gtArrOffs.gtOffset); + } + break; - case GT_ARR_OFFSET: - // This consumes the offset, if any, the arrObj and the effective index, - // and produces the flattened offset for this dimension. - info->srcCount = 3; - info->dstCount = 1; - info->internalIntCount = 1; - // we don't want to generate code for this - if (tree->gtArrOffs.gtOffset->IsIntegralConst(0)) - { - MakeSrcContained(tree, tree->gtArrOffs.gtOffset); - } - break; + case GT_LEA: + // The LEA usually passes its operands through to the GT_IND, in which case we'll + // clear the info->srcCount and info->dstCount later, but we may be instantiating an address, + // so we set them here. + info->srcCount = 0; + if (tree->AsAddrMode()->HasBase()) + { + info->srcCount++; + } + if (tree->AsAddrMode()->HasIndex()) + { + info->srcCount++; + } + info->dstCount = 1; + break; - case GT_LEA: - // The LEA usually passes its operands through to the GT_IND, in which case we'll - // clear the info->srcCount and info->dstCount later, but we may be instantiating an address, - // so we set them here. - info->srcCount = 0; - if (tree->AsAddrMode()->HasBase()) - { - info->srcCount++; - } - if (tree->AsAddrMode()->HasIndex()) - { - info->srcCount++; - } - info->dstCount = 1; - break; + case GT_STOREIND: + { + info->srcCount = 2; + info->dstCount = 0; + GenTree* src = tree->gtOp.gtOp2; - case GT_STOREIND: + if (compiler->codeGen->gcInfo.gcIsWriteBarrierAsgNode(tree)) { - info->srcCount = 2; - info->dstCount = 0; - GenTree* src = tree->gtOp.gtOp2; + LowerGCWriteBarrier(tree); + break; + } - if (compiler->codeGen->gcInfo.gcIsWriteBarrierAsgNode(tree)) + // If the source is a containable immediate, make it contained, unless it is + // an int-size or larger store of zero to memory, because we can generate smaller code + // by zeroing a register and then storing it. + if (IsContainableImmed(tree, src) && + (!src->IsIntegralConst(0) || varTypeIsSmall(tree) || tree->gtGetOp1()->OperGet() == GT_CLS_VAR_ADDR)) + { + MakeSrcContained(tree, src); + } + else if (!varTypeIsFloating(tree)) + { + // Perform recognition of trees with the following structure: + // StoreInd(addr, BinOp(expr, GT_IND(addr))) + // to be able to fold this into an instruction of the form + // BINOP [addr], register + // where register is the actual place where 'expr' is computed. + // + // SSE2 doesn't support RMW form of instructions. + if (SetStoreIndOpCountsIfRMWMemOp(tree)) { - LowerGCWriteBarrier(tree); break; } + } - // If the source is a containable immediate, make it contained, unless it is - // an int-size or larger store of zero to memory, because we can generate smaller code - // by zeroing a register and then storing it. - if (IsContainableImmed(tree, src) && (!src->IsIntegralConst(0) || varTypeIsSmall(tree) || - tree->gtGetOp1()->OperGet() == GT_CLS_VAR_ADDR)) - { - MakeSrcContained(tree, src); - } - else if (!varTypeIsFloating(tree)) - { - // Perform recognition of trees with the following structure: - // StoreInd(addr, BinOp(expr, GT_IND(addr))) - // to be able to fold this into an instruction of the form - // BINOP [addr], register - // where register is the actual place where 'expr' is computed. - // - // SSE2 doesn't support RMW form of instructions. - if (SetStoreIndOpCountsIfRMWMemOp(tree)) - { - break; - } - } + SetIndirAddrOpCounts(tree); + } + break; - SetIndirAddrOpCounts(tree); - } + case GT_NULLCHECK: + info->dstCount = 0; + info->srcCount = 1; + info->isLocalDefUse = true; break; - case GT_NULLCHECK: - info->dstCount = 0; - info->srcCount = 1; - info->isLocalDefUse = true; - break; - - case GT_IND: - info->dstCount = 1; - info->srcCount = 1; - SetIndirAddrOpCounts(tree); - break; + case GT_IND: + info->dstCount = 1; + info->srcCount = 1; + SetIndirAddrOpCounts(tree); + break; - case GT_CATCH_ARG: - info->srcCount = 0; - info->dstCount = 1; - info->setDstCandidates(l, RBM_EXCEPTION_OBJECT); - break; + case GT_CATCH_ARG: + info->srcCount = 0; + info->dstCount = 1; + info->setDstCandidates(l, RBM_EXCEPTION_OBJECT); + break; #if !FEATURE_EH_FUNCLETS - case GT_END_LFIN: - info->srcCount = 0; - info->dstCount = 0; - break; + case GT_END_LFIN: + info->srcCount = 0; + info->dstCount = 0; + break; #endif - case GT_CLS_VAR: - info->srcCount = 0; - // GT_CLS_VAR, by the time we reach the backend, must always - // be a pure use. - // It will produce a result of the type of the - // node, and use an internal register for the address. + case GT_CLS_VAR: + info->srcCount = 0; + // GT_CLS_VAR, by the time we reach the backend, must always + // be a pure use. + // It will produce a result of the type of the + // node, and use an internal register for the address. - info->dstCount = 1; - assert((tree->gtFlags & (GTF_VAR_DEF | GTF_VAR_USEASG | GTF_VAR_USEDEF)) == 0); - info->internalIntCount = 1; - break; - } // end switch (tree->OperGet()) + info->dstCount = 1; + assert((tree->gtFlags & (GTF_VAR_DEF | GTF_VAR_USEASG | GTF_VAR_USEDEF)) == 0); + info->internalIntCount = 1; + break; + } // end switch (tree->OperGet()) - // If op2 of a binary-op gets marked as contained, then binary-op srcCount will be 1. - // Even then we would like to set isTgtPref on Op1. - if (tree->OperIsBinary() && info->srcCount >= 1) + // If op2 of a binary-op gets marked as contained, then binary-op srcCount will be 1. + // Even then we would like to set isTgtPref on Op1. + if (tree->OperIsBinary() && info->srcCount >= 1) + { + if (isRMWRegOper(tree)) { - if (isRMWRegOper(tree)) - { - GenTree* op1 = tree->gtOp.gtOp1; - GenTree* op2 = tree->gtOp.gtOp2; + GenTree* op1 = tree->gtOp.gtOp1; + GenTree* op2 = tree->gtOp.gtOp2; - // Commutative opers like add/mul/and/or/xor could reverse the order of - // operands if it is safe to do so. In such a case we would like op2 to be - // target preferenced instead of op1. - if (tree->OperIsCommutative() && op1->gtLsraInfo.dstCount == 0 && op2 != nullptr) - { - op1 = op2; - op2 = tree->gtOp.gtOp1; - } + // Commutative opers like add/mul/and/or/xor could reverse the order of + // operands if it is safe to do so. In such a case we would like op2 to be + // target preferenced instead of op1. + if (tree->OperIsCommutative() && op1->gtLsraInfo.dstCount == 0 && op2 != nullptr) + { + op1 = op2; + op2 = tree->gtOp.gtOp1; + } - // If we have a read-modify-write operation, we want to preference op1 to the target. - // If op1 is contained, we don't want to preference it, but it won't - // show up as a source in that case, so it will be ignored. - op1->gtLsraInfo.isTgtPref = true; - - // Is this a non-commutative operator, or is op2 a contained memory op? - // (Note that we can't call IsContained() at this point because it uses exactly the - // same information we're currently computing.) - // In either case, we need to make op2 remain live until the op is complete, by marking - // the source(s) associated with op2 as "delayFree". - // Note that if op2 of a binary RMW operator is a memory op, even if the operator - // is commutative, codegen cannot reverse them. - // TODO-XArch-CQ: This is not actually the case for all RMW binary operators, but there's - // more work to be done to correctly reverse the operands if they involve memory - // operands. Also, we may need to handle more cases than GT_IND, especially once - // we've modified the register allocator to not require all nodes to be assigned - // a register (e.g. a spilled lclVar can often be referenced directly from memory). - // Note that we may have a null op2, even with 2 sources, if op1 is a base/index memory op. - - GenTree* delayUseSrc = nullptr; - // TODO-XArch-Cleanup: We should make the indirection explicit on these nodes so that we don't have - // to special case them. - if (tree->OperGet() == GT_XADD || tree->OperGet() == GT_XCHG || tree->OperGet() == GT_LOCKADD) - { - delayUseSrc = op1; - } - else if ((op2 != nullptr) && - (!tree->OperIsCommutative() || (op2->isMemoryOp() && (op2->gtLsraInfo.srcCount == 0)))) - { - delayUseSrc = op2; - } - if (delayUseSrc != nullptr) - { - // If delayUseSrc is an indirection and it doesn't produce a result, then we need to set "delayFree' - // on the base & index, if any. - // Otherwise, we set it on delayUseSrc itself. - if (delayUseSrc->isIndir() && (delayUseSrc->gtLsraInfo.dstCount == 0)) + // If we have a read-modify-write operation, we want to preference op1 to the target. + // If op1 is contained, we don't want to preference it, but it won't + // show up as a source in that case, so it will be ignored. + op1->gtLsraInfo.isTgtPref = true; + + // Is this a non-commutative operator, or is op2 a contained memory op? + // (Note that we can't call IsContained() at this point because it uses exactly the + // same information we're currently computing.) + // In either case, we need to make op2 remain live until the op is complete, by marking + // the source(s) associated with op2 as "delayFree". + // Note that if op2 of a binary RMW operator is a memory op, even if the operator + // is commutative, codegen cannot reverse them. + // TODO-XArch-CQ: This is not actually the case for all RMW binary operators, but there's + // more work to be done to correctly reverse the operands if they involve memory + // operands. Also, we may need to handle more cases than GT_IND, especially once + // we've modified the register allocator to not require all nodes to be assigned + // a register (e.g. a spilled lclVar can often be referenced directly from memory). + // Note that we may have a null op2, even with 2 sources, if op1 is a base/index memory op. + + GenTree* delayUseSrc = nullptr; + // TODO-XArch-Cleanup: We should make the indirection explicit on these nodes so that we don't have + // to special case them. + if (tree->OperGet() == GT_XADD || tree->OperGet() == GT_XCHG || tree->OperGet() == GT_LOCKADD) + { + delayUseSrc = op1; + } + else if ((op2 != nullptr) && + (!tree->OperIsCommutative() || (op2->isMemoryOp() && (op2->gtLsraInfo.srcCount == 0)))) + { + delayUseSrc = op2; + } + if (delayUseSrc != nullptr) + { + // If delayUseSrc is an indirection and it doesn't produce a result, then we need to set "delayFree' + // on the base & index, if any. + // Otherwise, we set it on delayUseSrc itself. + if (delayUseSrc->isIndir() && (delayUseSrc->gtLsraInfo.dstCount == 0)) + { + GenTree* base = delayUseSrc->AsIndir()->Base(); + GenTree* index = delayUseSrc->AsIndir()->Index(); + if (base != nullptr) { - GenTree* base = delayUseSrc->AsIndir()->Base(); - GenTree* index = delayUseSrc->AsIndir()->Index(); - if (base != nullptr) - { - base->gtLsraInfo.isDelayFree = true; - } - if (index != nullptr) - { - index->gtLsraInfo.isDelayFree = true; - } + base->gtLsraInfo.isDelayFree = true; } - else + if (index != nullptr) { - delayUseSrc->gtLsraInfo.isDelayFree = true; + index->gtLsraInfo.isDelayFree = true; } - info->hasDelayFreeSrc = true; } + else + { + delayUseSrc->gtLsraInfo.isDelayFree = true; + } + info->hasDelayFreeSrc = true; } } + } #ifdef _TARGET_X86_ - // Exclude RBM_NON_BYTE_REGS from dst candidates of tree node and src candidates of operands - // if the tree node is a byte type. - // - // Example1: GT_STOREIND(byte, addr, op2) - storeind of byte sized value from op2 into mem 'addr' - // Storeind itself will not produce any value and hence dstCount=0. But op2 could be TYP_INT - // value. In this case we need to exclude esi/edi from the src candidates of op2. - // - // Example2: GT_CAST(int <- bool <- int) - here type of GT_CAST node is int and castToType is bool. - // - // Example3: GT_EQ(int, op1 of type ubyte, op2 of type ubyte) - in this case codegen uses - // ubyte as the result of comparison and if the result needs to be materialized into a reg - // simply zero extend it to TYP_INT size. Here is an example of generated code: - // cmp dl, byte ptr[addr mode] - // movzx edx, dl - // - // Though this looks conservative in theory, in practice we could not think of a case where - // the below logic leads to conservative register specification. In future when or if we find - // one such case, this logic needs to be fine tuned for that case(s). - if (varTypeIsByte(tree) || ((tree->OperGet() == GT_CAST) && varTypeIsByte(tree->CastToType())) || - (tree->OperIsCompare() && varTypeIsByte(tree->gtGetOp1()) && varTypeIsByte(tree->gtGetOp2()))) + // Exclude RBM_NON_BYTE_REGS from dst candidates of tree node and src candidates of operands + // if the tree node is a byte type. + // + // Example1: GT_STOREIND(byte, addr, op2) - storeind of byte sized value from op2 into mem 'addr' + // Storeind itself will not produce any value and hence dstCount=0. But op2 could be TYP_INT + // value. In this case we need to exclude esi/edi from the src candidates of op2. + // + // Example2: GT_CAST(int <- bool <- int) - here type of GT_CAST node is int and castToType is bool. + // + // Example3: GT_EQ(int, op1 of type ubyte, op2 of type ubyte) - in this case codegen uses + // ubyte as the result of comparison and if the result needs to be materialized into a reg + // simply zero extend it to TYP_INT size. Here is an example of generated code: + // cmp dl, byte ptr[addr mode] + // movzx edx, dl + // + // Though this looks conservative in theory, in practice we could not think of a case where + // the below logic leads to conservative register specification. In future when or if we find + // one such case, this logic needs to be fine tuned for that case(s). + if (varTypeIsByte(tree) || ((tree->OperGet() == GT_CAST) && varTypeIsByte(tree->CastToType())) || + (tree->OperIsCompare() && varTypeIsByte(tree->gtGetOp1()) && varTypeIsByte(tree->gtGetOp2()))) + { + regMaskTP regMask; + if (info->dstCount > 0) { - regMaskTP regMask; - if (info->dstCount > 0) + regMask = info->getDstCandidates(l); + assert(regMask != RBM_NONE); + info->setDstCandidates(l, regMask & ~RBM_NON_BYTE_REGS); + } + + if (tree->OperIsSimple() && (info->srcCount > 0)) + { + // No need to set src candidates on a contained child operand. + GenTree* op = tree->gtOp.gtOp1; + assert(op != nullptr); + bool containedNode = (op->gtLsraInfo.srcCount == 0) && (op->gtLsraInfo.dstCount == 0); + if (!containedNode) { - regMask = info->getDstCandidates(l); + regMask = op->gtLsraInfo.getSrcCandidates(l); assert(regMask != RBM_NONE); - info->setDstCandidates(l, regMask & ~RBM_NON_BYTE_REGS); + op->gtLsraInfo.setSrcCandidates(l, regMask & ~RBM_NON_BYTE_REGS); } - if (tree->OperIsSimple() && (info->srcCount > 0)) + if (tree->OperIsBinary() && (tree->gtOp.gtOp2 != nullptr)) { - // No need to set src candidates on a contained child operand. - GenTree* op = tree->gtOp.gtOp1; - assert(op != nullptr); - bool containedNode = (op->gtLsraInfo.srcCount == 0) && (op->gtLsraInfo.dstCount == 0); + op = tree->gtOp.gtOp2; + containedNode = (op->gtLsraInfo.srcCount == 0) && (op->gtLsraInfo.dstCount == 0); if (!containedNode) { regMask = op->gtLsraInfo.getSrcCandidates(l); assert(regMask != RBM_NONE); op->gtLsraInfo.setSrcCandidates(l, regMask & ~RBM_NON_BYTE_REGS); } - - if (tree->OperIsBinary() && (tree->gtOp.gtOp2 != nullptr)) - { - op = tree->gtOp.gtOp2; - containedNode = (op->gtLsraInfo.srcCount == 0) && (op->gtLsraInfo.dstCount == 0); - if (!containedNode) - { - regMask = op->gtLsraInfo.getSrcCandidates(l); - assert(regMask != RBM_NONE); - op->gtLsraInfo.setSrcCandidates(l, regMask & ~RBM_NON_BYTE_REGS); - } - } } } + } #endif //_TARGET_X86_ - // We need to be sure that we've set info->srcCount and info->dstCount appropriately - assert((info->dstCount < 2) || (tree->IsMultiRegCall() && info->dstCount == MAX_RET_REG_COUNT)); - - tree = next; - } + // We need to be sure that we've set info->srcCount and info->dstCount appropriately + assert((info->dstCount < 2) || (tree->IsMultiRegCall() && info->dstCount == MAX_RET_REG_COUNT)); } //------------------------------------------------------------------------ @@ -3148,13 +3139,14 @@ void Lowering::LowerCmp(GenTreePtr tree) } } } - comp->fgSnipNode(comp->compCurStmt->AsStmt(), removeTreeNode); + + BlockRange().Remove(removeTreeNode); #ifdef DEBUG if (comp->verbose) { printf("LowerCmp: Removing a GT_CAST to TYP_UBYTE and changing castOp1->gtType to " "TYP_UBYTE\n"); - comp->gtDispTree(tree); + comp->gtDispTreeRange(BlockRange(), tree); } #endif } @@ -3233,9 +3225,8 @@ void Lowering::LowerCmp(GenTreePtr tree) * system.windows.forms, scimark, fractals, bio mums). If we ever find evidence that * doing this optimization is a win, should consider generating in-lined code. */ -void Lowering::LowerCast(GenTreePtr* ppTree) +void Lowering::LowerCast(GenTree* tree) { - GenTreePtr tree = *ppTree; assert(tree->OperGet() == GT_CAST); GenTreePtr op1 = tree->gtOp.gtOp1; @@ -3294,7 +3285,7 @@ void Lowering::LowerCast(GenTreePtr* ppTree) tree->gtFlags &= ~GTF_UNSIGNED; tree->gtOp.gtOp1 = tmp; - op1->InsertAfterSelf(tmp); + BlockRange().InsertAfter(op1, tmp); } } @@ -3324,8 +3315,8 @@ bool Lowering::IsBinOpInRMWStoreInd(GenTreePtr tree) return false; } - GenTreePtr parent = tree->gtGetParent(nullptr); - if (parent == nullptr || parent->OperGet() != GT_STOREIND || parent->gtGetOp2() != tree) + LIR::Use use; + if (!BlockRange().TryGetUse(tree, &use) || use.User()->OperGet() != GT_STOREIND || use.User()->gtGetOp2() != tree) { return false; } @@ -3335,7 +3326,7 @@ bool Lowering::IsBinOpInRMWStoreInd(GenTreePtr tree) // we can use the result. GenTreePtr indirCandidate = nullptr; GenTreePtr indirOpSource = nullptr; - return IsRMWMemOpRootedAtStoreInd(parent, &indirCandidate, &indirOpSource); + return IsRMWMemOpRootedAtStoreInd(use.User(), &indirCandidate, &indirOpSource); } //---------------------------------------------------------------------------------------------- @@ -3562,7 +3553,7 @@ bool Lowering::SetStoreIndOpCountsIfRMWMemOp(GenTreePtr storeInd) { JITDUMP("Lower of StoreInd didn't mark the node as self contained for reason: %d\n", storeInd->AsStoreInd()->GetRMWStatus()); - DISPTREE(storeInd); + DISPTREERANGE(BlockRange(), storeInd); return false; } @@ -3604,7 +3595,7 @@ bool Lowering::SetStoreIndOpCountsIfRMWMemOp(GenTreePtr storeInd) JITDUMP("Lower succesfully detected an assignment of the form: *addrMode = UnaryOp(*addrMode)\n"); info->srcCount = 0; } - DISPTREE(storeInd); + DISPTREERANGE(BlockRange(), storeInd); m_lsra->clearOperandCounts(indirSrc); m_lsra->clearOperandCounts(indirCandidate); diff --git a/src/jit/lsra.cpp b/src/jit/lsra.cpp index e2c1930e2a..6d87fa9fb1 100644 --- a/src/jit/lsra.cpp +++ b/src/jit/lsra.cpp @@ -2488,6 +2488,7 @@ void LinearScan::addRefsForPhysRegMask(regMaskTP mask, LsraLocation currentLoc, // getKillSetForNode: Return the registers killed by the given tree node. // // Arguments: +// compiler - the compiler context to use // tree - the tree for which the kill set is needed. // // Return Value: a register mask of the registers killed @@ -3189,8 +3190,12 @@ static int ComputeOperandDstCount(GenTree* operand) } else if (operandInfo.srcCount != 0) { - // If an operand has no destination registers but does have source registers, it must be a store. - assert(operand->OperIsStore() || operand->OperIsBlkOp() || operand->OperIsPutArgStk()); + // If an operand has no destination registers but does have source registers, it must be a store + // or a compare. + assert(operand->OperIsStore() || + operand->OperIsBlkOp() || + operand->OperIsPutArgStk() || + operand->OperIsCompare()); return 0; } else if (operand->OperIsStore() || operand->TypeGet() == TYP_VOID) @@ -3860,12 +3865,11 @@ void LinearScan::insertZeroInitRefPositions() if (!varDsc->lvIsParam && isCandidateVar(varDsc) && (compiler->info.compInitMem || varTypeIsGC(varDsc->TypeGet()))) { - GenTree* firstStmt = getNonEmptyBlock(compiler->fgFirstBB)->bbTreeList; + GenTree* firstNode = getNonEmptyBlock(compiler->fgFirstBB)->firstNode(); JITDUMP("V%02u was live in\n", varNum); - DISPTREE(firstStmt); Interval* interval = getIntervalForLocalVar(varNum); RefPosition* pos = - newRefPosition(interval, MinLocation, RefTypeZeroInit, firstStmt, allRegs(interval->registerType)); + newRefPosition(interval, MinLocation, RefTypeZeroInit, firstNode, allRegs(interval->registerType)); varDsc->lvMustInit = true; } } @@ -4314,76 +4318,23 @@ void LinearScan::buildIntervals() VarSetOps::Assign(compiler, currentLiveVars, block->bbLiveIn); - for (GenTree* statement = block->FirstNonPhiDef(); statement; statement = statement->gtNext) + LIR::Range& blockRange = LIR::AsRange(block); + for (GenTree* node : blockRange.NonPhiNodes()) { - if (statement->gtStmt.gtStmtIsEmbedded()) - { - continue; - } - - GenTree* treeNode; - int dstCount = 0; - - GenTree* stmtExpr = statement->gtStmt.gtStmtExpr; - - // If we have a dead lclVar use, we have to generate a RefPosition for it, - // otherwise the dataflow won't match the allocations. - // TODO-Cleanup: Delete these dead lclVar uses before LSRA, and remove from the - // live sets as appropriate. - // - // Do this for the top level statement and all of its embedded statements. - // The embedded statements need to be traversed because otherwise we cannot - // identify a dead use. We skip them in the outer loop because we need - // this data for the whole statement before we build refpositions. - GenTree* nextStmt = statement; - do - { - GenTree* nextStmtExpr = nextStmt->gtStmt.gtStmtExpr; - if (nextStmtExpr->gtLsraInfo.dstCount > 0) - { - nextStmtExpr->gtLsraInfo.isLocalDefUse = true; - nextStmtExpr->gtLsraInfo.dstCount = 0; - } + assert(node->gtLsraInfo.loc >= currentLoc); + assert(((node->gtLIRFlags & LIR::Flags::IsUnusedValue) == 0) || node->gtLsraInfo.isLocalDefUse); - nextStmt = nextStmt->gtNext; - } while (nextStmt && nextStmt->gtStmt.gtStmtIsEmbedded()); - - // Go through the statement nodes in execution order, and build RefPositions - foreach_treenode_execution_order(treeNode, statement) - { - assert(treeNode->gtLsraInfo.loc >= currentLoc); - currentLoc = treeNode->gtLsraInfo.loc; - dstCount = treeNode->gtLsraInfo.dstCount; - buildRefPositionsForNode(treeNode, block, listNodePool, operandToLocationInfoMap, currentLoc); -#ifdef DEBUG - if (currentLoc > maxNodeLocation) - { - maxNodeLocation = currentLoc; - } -#endif // DEBUG - } + currentLoc = node->gtLsraInfo.loc; + buildRefPositionsForNode(node, block, listNodePool, operandToLocationInfoMap, currentLoc); #ifdef DEBUG - // At this point the map should be empty, unless: we have a node that - // produces a result that's ignored, in which case the map should contain - // one element that maps to dstCount locations. - JITDUMP("map size after tree processed was %d\n", operandToLocationInfoMap.Count()); - - int locCount = 0; - for (auto kvp : operandToLocationInfoMap) + if (currentLoc > maxNodeLocation) { - LocationInfoList defList = kvp.Value(); - for (LocationInfoListNode *def = defList.Begin(), *end = defList.End(); def != end; def = def->Next()) - { - locCount++; - } + maxNodeLocation = currentLoc; } - - assert(locCount == dstCount); -#endif - - operandToLocationInfoMap.Clear(); +#endif // DEBUG } + // Increment the LsraLocation at this point, so that the dummy RefPositions // will not have the same LsraLocation as any "real" RefPosition. currentLoc += 2; @@ -7297,8 +7248,10 @@ void LinearScan::allocateRegisters() // NICE: Consider tracking whether an Interval is always in the same location (register/stack) // in which case it will require no resolution. // -void LinearScan::resolveLocalRef(GenTreePtr treeNode, RefPosition* currentRefPosition) +void LinearScan::resolveLocalRef(BasicBlock* block, GenTreePtr treeNode, RefPosition* currentRefPosition) { + assert((block == nullptr) == (treeNode == nullptr)); + // Is this a tracked local? Or just a register allocated for loading // a non-tracked one? Interval* interval = currentRefPosition->getInterval(); @@ -7448,6 +7401,7 @@ void LinearScan::resolveLocalRef(GenTreePtr treeNode, RefPosition* currentRefPos // currently lives. For moveReg, the homeReg is the new register (as assigned above). // But for copyReg, the homeReg remains unchanged. + assert(treeNode != nullptr); treeNode->gtRegNum = interval->physReg; if (currentRefPosition->copyReg) @@ -7462,7 +7416,7 @@ void LinearScan::resolveLocalRef(GenTreePtr treeNode, RefPosition* currentRefPos if (!currentRefPosition->isFixedRegRef || currentRefPosition->moveReg) { // This is the second case, where we need to generate a copy - insertCopyOrReload(treeNode, currentRefPosition->getMultiRegIdx(), currentRefPosition); + insertCopyOrReload(block, treeNode, currentRefPosition->getMultiRegIdx(), currentRefPosition); } } else @@ -7586,11 +7540,15 @@ void LinearScan::writeRegisters(RefPosition* currentRefPosition, GenTree* tree) // and the unspilling code automatically reuses the same register, and does the reload when it notices that flag // when considering a node's operands. // -void LinearScan::insertCopyOrReload(GenTreePtr tree, unsigned multiRegIdx, RefPosition* refPosition) +void LinearScan::insertCopyOrReload(BasicBlock* block, GenTreePtr tree, unsigned multiRegIdx, RefPosition* refPosition) { - GenTreePtr* parentChildPointer = nullptr; - GenTreePtr parent = tree->gtGetParent(&parentChildPointer); - noway_assert(parent != nullptr && parentChildPointer != nullptr); + LIR::Range& blockRange = LIR::AsRange(block); + + LIR::Use treeUse; + bool foundUse = blockRange.TryGetUse(tree, &treeUse); + assert(foundUse); + + GenTree* parent = treeUse.User(); genTreeOps oper; if (refPosition->reload) @@ -7654,12 +7612,10 @@ void LinearScan::insertCopyOrReload(GenTreePtr tree, unsigned multiRegIdx, RefPo newNode->gtFlags |= GTF_VAR_DEATH; } - // Replace tree in the parent node. - *parentChildPointer = newNode; - - // we insert this directly after the spilled node. it does not reload at that point but - // just updates registers - tree->InsertAfterSelf(newNode); + // Insert the copy/reload after the spilled node and replace the use of the original node with a use + // of the copy/reload. + blockRange.InsertAfter(tree, newNode); + treeUse.ReplaceWith(compiler, newNode); } } @@ -7691,21 +7647,7 @@ void LinearScan::insertUpperVectorSaveAndReload(GenTreePtr tree, RefPosition* re regNumber spillReg = refPosition->assignedReg(); bool spillToMem = refPosition->spillAfter; - // We will insert the save before the statement containing 'tree', and the restore after it. - // They will each be inserted as embedded statements. - // In order to do this, we need to find the current statement. - // TODO-Througput: There's got to be a better way to do this. - GenTree* lastNodeInCurrentStatement = tree; - while (lastNodeInCurrentStatement->gtNext != nullptr) - { - lastNodeInCurrentStatement = lastNodeInCurrentStatement->gtNext; - } - GenTree* stmt = block->bbTreeList; - while (stmt != nullptr && stmt->gtStmt.gtStmtExpr != lastNodeInCurrentStatement) - { - stmt = stmt->gtNext; - } - noway_assert(stmt != nullptr); + LIR::Range& blockRange = LIR::AsRange(block); // First, insert the save as an embedded statement before the call. @@ -7724,7 +7666,8 @@ void LinearScan::insertUpperVectorSaveAndReload(GenTreePtr tree, RefPosition* re { simdNode->gtFlags |= GTF_SPILL; } - compiler->fgInsertTreeBeforeAsEmbedded(simdNode, tree, stmt->AsStmt(), block); + + blockRange.InsertBefore(tree, LIR::SeqTree(compiler, simdNode)); // Now insert the restore after the call. @@ -7743,7 +7686,7 @@ void LinearScan::insertUpperVectorSaveAndReload(GenTreePtr tree, RefPosition* re simdNode->gtFlags |= GTF_SPILLED; } - compiler->fgInsertTreeAfterAsEmbedded(simdNode, tree, stmt->AsStmt(), block); + blockRange.InsertAfter(tree, LIR::SeqTree(compiler, simdNode)); } #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE @@ -7975,7 +7918,7 @@ void LinearScan::resolveRegisters() { Interval* interval = currentRefPosition->getInterval(); assert(interval != nullptr && interval->isLocalVar); - resolveLocalRef(nullptr, currentRefPosition); + resolveLocalRef(nullptr, nullptr, currentRefPosition); regNumber reg = REG_STK; int varIndex = interval->getVarIndex(compiler); @@ -7997,7 +7940,7 @@ void LinearScan::resolveRegisters() JITDUMP("------------------------\n"); BasicBlock* insertionBlock = compiler->fgFirstBB; - GenTreePtr insertionPoint = insertionBlock->FirstNonPhiDef(); + GenTreePtr insertionPoint = LIR::AsRange(insertionBlock).FirstNonPhiNode(); // write back assignments for (block = startBlockSequence(); block != nullptr; block = moveToNextBlock()) @@ -8028,7 +7971,7 @@ void LinearScan::resolveRegisters() assert(currentRefPosition->isIntervalRef()); // Don't mark dummy defs as reload currentRefPosition->reload = false; - resolveLocalRef(nullptr, currentRefPosition); + resolveLocalRef(nullptr, nullptr, currentRefPosition); regNumber reg; if (currentRefPosition->registerAssignment != RBM_NONE) { @@ -8198,7 +8141,7 @@ void LinearScan::resolveRegisters() if (treeNode->IsLocal() && currentRefPosition->getInterval()->isLocalVar) { - resolveLocalRef(treeNode, currentRefPosition); + resolveLocalRef(block, treeNode, currentRefPosition); } // Mark spill locations on temps @@ -8241,7 +8184,8 @@ void LinearScan::resolveRegisters() { if (nextRefPosition->assignedReg() != REG_NA) { - insertCopyOrReload(treeNode, currentRefPosition->getMultiRegIdx(), nextRefPosition); + insertCopyOrReload(block, treeNode, currentRefPosition->getMultiRegIdx(), + nextRefPosition); } else { @@ -8450,6 +8394,7 @@ void LinearScan::resolveRegisters() printf("Trees after linear scan register allocator (LSRA)\n"); compiler->fgDispBasicBlocks(true); } + verifyFinalAllocation(); #endif // DEBUG @@ -8537,18 +8482,13 @@ void LinearScan::insertMove( top->gtLsraInfo.isLsraAdded = true; } top->gtLsraInfo.isLocalDefUse = true; - GenTreePtr stmt = compiler->gtNewStmt(top); - compiler->gtSetStmtInfo(stmt); - // The top-level node has no gtNext, and src has no gtPrev - they are set that way - // when created. - assert(top->gtNext == nullptr); - assert(src->gtPrev == nullptr); - stmt->gtStmt.gtStmtList = src; + LIR::Range treeRange = LIR::SeqTree(compiler, top); + LIR::Range& blockRange = LIR::AsRange(block); if (insertionPoint != nullptr) { - compiler->fgInsertStmtBefore(block, insertionPoint, stmt); + blockRange.InsertBefore(insertionPoint, std::move(treeRange)); } else { @@ -8556,29 +8496,18 @@ void LinearScan::insertMove( // If there's a branch, make an embedded statement that executes just prior to the branch if (block->bbJumpKind == BBJ_COND || block->bbJumpKind == BBJ_SWITCH) { - stmt->gtFlags &= ~GTF_STMT_TOP_LEVEL; - noway_assert(block->bbTreeList != nullptr); - GenTreePtr lastStmt = block->lastStmt(); - GenTreePtr branchStmt = block->lastTopLevelStmt(); - GenTreePtr branch = branchStmt->gtStmt.gtStmtExpr; + noway_assert(!blockRange.IsEmpty()); + + GenTree* branch = blockRange.LastNode(); assert(branch->OperGet() == GT_JTRUE || branch->OperGet() == GT_SWITCH_TABLE || branch->OperGet() == GT_SWITCH); - GenTreePtr prev = branch->gtPrev; - prev->gtNext = src; - src->gtPrev = prev; - branch->gtPrev = top; - top->gtNext = branch; - - stmt->gtNext = nullptr; - stmt->gtPrev = lastStmt; - lastStmt->gtNext = stmt; - block->bbTreeList->gtPrev = stmt; + blockRange.InsertBefore(branch, std::move(treeRange)); } else { assert(block->bbJumpKind == BBJ_NONE || block->bbJumpKind == BBJ_ALWAYS); - compiler->fgInsertStmtAtEnd(block, stmt); + blockRange.InsertAtEnd(std::move(treeRange)); } } } @@ -8625,18 +8554,12 @@ void LinearScan::insertSwap( lcl2->gtNext = swap; swap->gtPrev = lcl2; - GenTreePtr stmt = compiler->gtNewStmt(swap); - compiler->gtSetStmtInfo(stmt); - - // The top-level node has no gtNext, and lcl1 has no gtPrev - they are set that way - // when created. - assert(swap->gtNext == nullptr); - assert(lcl1->gtPrev == nullptr); - stmt->gtStmt.gtStmtList = lcl1; + LIR::Range swapRange = LIR::SeqTree(compiler, swap); + LIR::Range& blockRange = LIR::AsRange(block); if (insertionPoint != nullptr) { - compiler->fgInsertStmtBefore(block, insertionPoint, stmt); + blockRange.InsertBefore(insertionPoint, std::move(swapRange)); } else { @@ -8644,28 +8567,18 @@ void LinearScan::insertSwap( // If there's a branch, make an embedded statement that executes just prior to the branch if (block->bbJumpKind == BBJ_COND || block->bbJumpKind == BBJ_SWITCH) { - stmt->gtFlags &= ~GTF_STMT_TOP_LEVEL; - noway_assert(block->bbTreeList != nullptr); - GenTreePtr lastStmt = block->lastStmt(); - GenTreePtr branchStmt = block->lastTopLevelStmt(); - GenTreePtr branch = branchStmt->gtStmt.gtStmtExpr; - assert(branch->OperGet() == GT_JTRUE || branch->OperGet() == GT_SWITCH); + noway_assert(!blockRange.IsEmpty()); - GenTreePtr prev = branch->gtPrev; - prev->gtNext = lcl1; - lcl1->gtPrev = prev; - branch->gtPrev = swap; - swap->gtNext = branch; + GenTree* branch = blockRange.LastNode(); + assert(branch->OperGet() == GT_JTRUE || branch->OperGet() == GT_SWITCH_TABLE || + branch->OperGet() == GT_SWITCH); - stmt->gtNext = nullptr; - stmt->gtPrev = lastStmt; - lastStmt->gtNext = stmt; - block->bbTreeList->gtPrev = stmt; + blockRange.InsertBefore(branch, std::move(swapRange)); } else { assert(block->bbJumpKind == BBJ_NONE || block->bbJumpKind == BBJ_ALWAYS); - compiler->fgInsertStmtAtEnd(block, stmt); + blockRange.InsertAtEnd(std::move(swapRange)); } } } @@ -8828,12 +8741,12 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) { // At this point, Lowering has transformed any non-switch-table blocks into // cascading ifs. - GenTree* lastStmt = block->lastStmt(); - assert(lastStmt != nullptr && lastStmt->gtStmt.gtStmtExpr->gtOper == GT_SWITCH_TABLE); - GenTree* switchTable = lastStmt->gtStmt.gtStmtExpr; - switchRegs = switchTable->gtRsvdRegs; - GenTree* op1 = switchTable->gtGetOp1(); - GenTree* op2 = switchTable->gtGetOp2(); + GenTree* switchTable = LIR::AsRange(block).LastNode(); + assert(switchTable != nullptr && switchTable->OperGet() == GT_SWITCH_TABLE); + + switchRegs = switchTable->gtRsvdRegs; + GenTree* op1 = switchTable->gtGetOp1(); + GenTree* op2 = switchTable->gtGetOp2(); noway_assert(op1 != nullptr && op2 != nullptr); assert(op1->gtRegNum != REG_NA && op2->gtRegNum != REG_NA); switchRegs |= genRegMask(op1->gtRegNum); @@ -9305,7 +9218,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, GenTreePtr insertionPoint = nullptr; if (resolveType == ResolveSplit || resolveType == ResolveCritical) { - insertionPoint = block->FirstNonPhiDef(); + insertionPoint = LIR::AsRange(block).FirstNonPhiNode(); } // First: @@ -10028,28 +9941,69 @@ void LinearScan::lsraDispNode(GenTreePtr tree, LsraTupleDumpMode mode, bool hasD } } -GenTreePtr popAndPrintLclVarUses(ArrayStack<GenTreePtr>* stack, int* remainingUses) +//------------------------------------------------------------------------ +// ComputeOperandDstCount: computes the number of registers defined by a +// node. +// +// For most nodes, this is simple: +// - Nodes that do not produce values (e.g. stores and other void-typed +// nodes) and nodes that immediately use the registers they define +// produce no registers +// - Nodes that are marked as defining N registers define N registers. +// +// For contained nodes, however, things are more complicated: for purposes +// of bookkeeping, a contained node is treated as producing the transitive +// closure of the registers produced by its sources. +// +// Arguments: +// operand - The operand for which to compute a register count. +// +// Returns: +// The number of registers defined by `operand`. +// +void LinearScan::DumpOperandDefs(GenTree* operand, + bool& first, + LsraTupleDumpMode mode, + char* operandString, + const unsigned operandStringLength) { - while (*remainingUses != 0) + assert(operand != nullptr); + assert(operandString != nullptr); + + if (ComputeOperandDstCount(operand) == 0) { - GenTreePtr nextUseNode = stack->Pop(); - (*remainingUses)--; - if (nextUseNode->IsLocal()) + return; + } + + if (operand->gtLsraInfo.dstCount != 0) + { + // This operand directly produces registers; print it. + for (int i = 0; i < operand->gtLsraInfo.dstCount; i++) { - printf(" V%02u"); + if (!first) + { + printf(","); + } + + lsraGetOperandString(operand, mode, operandString, operandStringLength); + printf("%s", operandString); + + first = false; } - else + } + else + { + // This is a contained node. Dump the defs produced by its operands. + for (GenTree* op : operand->Operands()) { - return nextUseNode; + DumpOperandDefs(op, first, mode, operandString, operandStringLength); } } - return nullptr; } void LinearScan::TupleStyleDump(LsraTupleDumpMode mode) { BasicBlock* block; - ArrayStack<GenTreePtr> stack(compiler, CMK_LSRA); LsraLocation currentLoc = 1; // 0 is the entry const unsigned operandStringLength = 16; char operandString[operandStringLength]; @@ -10173,167 +10127,113 @@ void LinearScan::TupleStyleDump(LsraTupleDumpMode mode) splitEdgeInfo.toBBNum); } - for (GenTree* statement = block->FirstNonPhiDef(); statement; statement = statement->gtNext) + for (GenTree* node : LIR::AsRange(block).NonPhiNodes()) { - if ((statement->gtFlags & GTF_STMT_TOP_LEVEL) == 0) - { - continue; - } + GenTree* tree = node; - for (GenTree *tree = statement->gtStmt.gtStmtList; tree; tree = tree->gtNext, currentLoc += 2) + genTreeOps oper = tree->OperGet(); + TreeNodeInfo& info = tree->gtLsraInfo; + if (tree->gtLsraInfo.isLsraAdded) { - genTreeOps oper = tree->OperGet(); - if (oper == GT_ARGPLACE) + // This must be one of the nodes that we add during LSRA + + if (oper == GT_LCL_VAR) { - continue; + info.srcCount = 0; + info.dstCount = 1; } - TreeNodeInfo& info = tree->gtLsraInfo; - if (tree->gtLsraInfo.isLsraAdded) + else if (oper == GT_RELOAD || oper == GT_COPY) { - // This must be one of the nodes that we add during LSRA - - if (oper == GT_LCL_VAR) - { - info.srcCount = 0; - info.dstCount = 1; - } - else if (oper == GT_RELOAD || oper == GT_COPY) + info.srcCount = 1; + info.dstCount = 1; + } +#ifdef FEATURE_SIMD + else if (oper == GT_SIMD) + { + if (tree->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicUpperSave) { info.srcCount = 1; info.dstCount = 1; } -#ifdef FEATURE_SIMD - else if (oper == GT_SIMD) - { - if (tree->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicUpperSave) - { - info.srcCount = 1; - info.dstCount = 1; - } - else - { - assert(tree->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicUpperRestore); - info.srcCount = 2; - info.dstCount = 0; - } - } -#endif // FEATURE_SIMD else { - assert(oper == GT_SWAP); + assert(tree->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicUpperRestore); info.srcCount = 2; info.dstCount = 0; } - info.internalIntCount = 0; - info.internalFloatCount = 0; } - - int consume = info.srcCount; - int produce = info.dstCount; - regMaskTP killMask = RBM_NONE; - regMaskTP fixedMask = RBM_NONE; - - if (tree->OperGet() == GT_LIST) +#endif // FEATURE_SIMD + else { - continue; + assert(oper == GT_SWAP); + info.srcCount = 2; + info.dstCount = 0; } + info.internalIntCount = 0; + info.internalFloatCount = 0; + } + + int consume = info.srcCount; + int produce = info.dstCount; + regMaskTP killMask = RBM_NONE; + regMaskTP fixedMask = RBM_NONE; - lsraDispNode(tree, mode, produce != 0 && mode != LSRA_DUMP_REFPOS); + lsraDispNode(tree, mode, produce != 0 && mode != LSRA_DUMP_REFPOS); - if (mode != LSRA_DUMP_REFPOS) + if (mode != LSRA_DUMP_REFPOS) + { + if (consume > 0) { - if (consume > stack.Height()) - { - printf("about to consume %d, height is only %d\n", consume, stack.Height()); - } + printf("; "); - if (!(tree->gtFlags & GTF_REVERSE_OPS)) - { - stack.ReverseTop(consume); - } - if (consume > 0) - { - printf("; "); - } - while (consume--) + bool first = true; + for (GenTree* operand : tree->Operands()) { - lsraGetOperandString(stack.Pop(), mode, operandString, operandStringLength); - printf("%s", operandString); - if (consume) - { - printf(","); - } + DumpOperandDefs(operand, first, mode, operandString, operandStringLength); } - while (produce--) + } + } + else + { + // Print each RefPosition on a new line, but + // printing all the kills for each node on a single line + // and combining the fixed regs with their associated def or use + bool killPrinted = false; + RefPosition* lastFixedRegRefPos = nullptr; + for (; currentRefPosition != refPositions.end() && + (currentRefPosition->refType == RefTypeUse || currentRefPosition->refType == RefTypeFixedReg || + currentRefPosition->refType == RefTypeKill || currentRefPosition->refType == RefTypeDef) && + (currentRefPosition->nodeLocation == tree->gtSeqNum || + currentRefPosition->nodeLocation == tree->gtSeqNum + 1); + ++currentRefPosition) + { + Interval* interval = nullptr; + if (currentRefPosition->isIntervalRef()) { - stack.Push(tree); + interval = currentRefPosition->getInterval(); } - } - else - { - // Print each RefPosition on a new line, but - // printing all the kills for each node on a single line - // and combining the fixed regs with their associated def or use - bool killPrinted = false; - RefPosition* lastFixedRegRefPos = nullptr; - for (; - currentRefPosition != refPositions.end() && - (currentRefPosition->refType == RefTypeUse || currentRefPosition->refType == RefTypeFixedReg || - currentRefPosition->refType == RefTypeKill || currentRefPosition->refType == RefTypeDef) && - (currentRefPosition->nodeLocation == tree->gtSeqNum || - currentRefPosition->nodeLocation == tree->gtSeqNum + 1); - ++currentRefPosition) + switch (currentRefPosition->refType) { - Interval* interval = nullptr; - if (currentRefPosition->isIntervalRef()) - { - interval = currentRefPosition->getInterval(); - } - switch (currentRefPosition->refType) - { - case RefTypeUse: - if (currentRefPosition->isPhysRegRef) - { - printf("\n Use:R%d(#%d)", - currentRefPosition->getReg()->regNum, currentRefPosition->rpNum); - } - else - { - assert(interval != nullptr); - printf("\n Use:"); - interval->microDump(); - printf("(#%d)", currentRefPosition->rpNum); - if (currentRefPosition->isFixedRegRef) - { - assert(genMaxOneBit(currentRefPosition->registerAssignment)); - assert(lastFixedRegRefPos != nullptr); - printf(" Fixed:%s(#%d)", getRegName(currentRefPosition->assignedReg(), - isFloatRegType(interval->registerType)), - lastFixedRegRefPos->rpNum); - lastFixedRegRefPos = nullptr; - } - if (currentRefPosition->isLocalDefUse) - { - printf(" LocalDefUse"); - } - if (currentRefPosition->lastUse) - { - printf(" *"); - } - } - break; - case RefTypeDef: + case RefTypeUse: + if (currentRefPosition->isPhysRegRef) + { + printf("\n Use:R%d(#%d)", + currentRefPosition->getReg()->regNum, currentRefPosition->rpNum); + } + else { - // Print each def on a new line assert(interval != nullptr); - printf("\n Def:"); + printf("\n Use:"); interval->microDump(); printf("(#%d)", currentRefPosition->rpNum); if (currentRefPosition->isFixedRegRef) { assert(genMaxOneBit(currentRefPosition->registerAssignment)); - printf(" %s", getRegName(currentRefPosition->assignedReg(), - isFloatRegType(interval->registerType))); + assert(lastFixedRegRefPos != nullptr); + printf(" Fixed:%s(#%d)", getRegName(currentRefPosition->assignedReg(), + isFloatRegType(interval->registerType)), + lastFixedRegRefPos->rpNum); + lastFixedRegRefPos = nullptr; } if (currentRefPosition->isLocalDefUse) { @@ -10343,61 +10243,82 @@ void LinearScan::TupleStyleDump(LsraTupleDumpMode mode) { printf(" *"); } - if (interval->relatedInterval != nullptr) - { - printf(" Pref:"); - interval->relatedInterval->microDump(); - } } break; - case RefTypeKill: - if (!killPrinted) - { - printf("\n Kill: "); - killPrinted = true; - } - printf(getRegName(currentRefPosition->assignedReg(), - isFloatRegType(currentRefPosition->getReg()->registerType))); - printf(" "); - break; - case RefTypeFixedReg: - lastFixedRegRefPos = currentRefPosition; - break; - default: - printf("Unexpected RefPosition type at #%d\n", currentRefPosition->rpNum); - break; + case RefTypeDef: + { + // Print each def on a new line + assert(interval != nullptr); + printf("\n Def:"); + interval->microDump(); + printf("(#%d)", currentRefPosition->rpNum); + if (currentRefPosition->isFixedRegRef) + { + assert(genMaxOneBit(currentRefPosition->registerAssignment)); + printf(" %s", getRegName(currentRefPosition->assignedReg(), + isFloatRegType(interval->registerType))); + } + if (currentRefPosition->isLocalDefUse) + { + printf(" LocalDefUse"); + } + if (currentRefPosition->lastUse) + { + printf(" *"); + } + if (interval->relatedInterval != nullptr) + { + printf(" Pref:"); + interval->relatedInterval->microDump(); + } } + break; + case RefTypeKill: + if (!killPrinted) + { + printf("\n Kill: "); + killPrinted = true; + } + printf(getRegName(currentRefPosition->assignedReg(), + isFloatRegType(currentRefPosition->getReg()->registerType))); + printf(" "); + break; + case RefTypeFixedReg: + lastFixedRegRefPos = currentRefPosition; + break; + default: + printf("Unexpected RefPosition type at #%d\n", currentRefPosition->rpNum); + break; } } + } + printf("\n"); + if (info.internalIntCount != 0 && mode != LSRA_DUMP_REFPOS) + { + printf("\tinternal (%d):\t", info.internalIntCount); + if (mode == LSRA_DUMP_POST) + { + dumpRegMask(tree->gtRsvdRegs); + } + else if ((info.getInternalCandidates(this) & allRegs(TYP_INT)) != allRegs(TYP_INT)) + { + dumpRegMask(info.getInternalCandidates(this) & allRegs(TYP_INT)); + } printf("\n"); - if (info.internalIntCount != 0 && mode != LSRA_DUMP_REFPOS) + } + if (info.internalFloatCount != 0 && mode != LSRA_DUMP_REFPOS) + { + printf("\tinternal (%d):\t", info.internalFloatCount); + if (mode == LSRA_DUMP_POST) { - printf("\tinternal (%d):\t", info.internalIntCount); - if (mode == LSRA_DUMP_POST) - { - dumpRegMask(tree->gtRsvdRegs); - } - else if ((info.getInternalCandidates(this) & allRegs(TYP_INT)) != allRegs(TYP_INT)) - { - dumpRegMask(info.getInternalCandidates(this) & allRegs(TYP_INT)); - } - printf("\n"); + dumpRegMask(tree->gtRsvdRegs); } - if (info.internalFloatCount != 0 && mode != LSRA_DUMP_REFPOS) + else if ((info.getInternalCandidates(this) & allRegs(TYP_INT)) != allRegs(TYP_INT)) { - printf("\tinternal (%d):\t", info.internalFloatCount); - if (mode == LSRA_DUMP_POST) - { - dumpRegMask(tree->gtRsvdRegs); - } - else if ((info.getInternalCandidates(this) & allRegs(TYP_INT)) != allRegs(TYP_INT)) - { - dumpRegMask(info.getInternalCandidates(this) & allRegs(TYP_INT)); - } - printf("\n"); + dumpRegMask(info.getInternalCandidates(this) & allRegs(TYP_INT)); } + printf("\n"); } - printf("\n"); } if (mode == LSRA_DUMP_POST) { @@ -11072,6 +10993,66 @@ void LinearScan::dumpRefPositionShort(RefPosition* refPosition, BasicBlock* curr } //------------------------------------------------------------------------ +// LinearScan::IsResolutionMove: +// Returns true if the given node is a move inserted by LSRA +// resolution. +// +// Arguments: +// node - the node to check. +// +bool LinearScan::IsResolutionMove(GenTree* node) +{ + if (!node->gtLsraInfo.isLsraAdded) + { + return false; + } + + switch (node->OperGet()) + { + case GT_LCL_VAR: + case GT_COPY: + return node->gtLsraInfo.isLocalDefUse; + + case GT_SWAP: + return true; + + default: + return false; + } +} + +//------------------------------------------------------------------------ +// LinearScan::IsResolutionNode: +// Returns true if the given node is either a move inserted by LSRA +// resolution or an operand to such a move. +// +// Arguments: +// containingRange - the range that contains the node to check. +// node - the node to check. +// +bool LinearScan::IsResolutionNode(LIR::Range& containingRange, GenTree* node) +{ + for (;;) + { + if (IsResolutionMove(node)) + { + return true; + } + + if (!node->gtLsraInfo.isLsraAdded || (node->OperGet() != GT_LCL_VAR)) + { + return false; + } + + LIR::Use use; + bool foundUse = containingRange.TryGetUse(node, &use); + assert(foundUse); + + node = use.User(); + } +} + +//------------------------------------------------------------------------ // verifyFinalAllocation: Traverse the RefPositions and verify various invariants. // // Arguments: @@ -11105,7 +11086,7 @@ void LinearScan::verifyFinalAllocation() DBEXEC(VERBOSE, dumpRegRecordTitle()); BasicBlock* currentBlock = nullptr; - GenTreeStmt* firstBlockEndResolutionStmt = nullptr; + GenTree* firstBlockEndResolutionNode = nullptr; regMaskTP regsToFree = RBM_NONE; regMaskTP delayRegsToFree = RBM_NONE; LsraLocation currentLocation = MinLocation; @@ -11184,9 +11165,14 @@ void LinearScan::verifyFinalAllocation() else { // Verify the resolution moves at the end of the previous block. - for (GenTreeStmt* stmt = firstBlockEndResolutionStmt; stmt != nullptr; stmt = stmt->getNextStmt()) + for (GenTree* node = firstBlockEndResolutionNode; node != nullptr; node = node->gtNext) { - verifyResolutionMove(stmt, currentLocation); + // Only verify nodes that are actually moves; don't bother with the nodes that are + // operands to moves. + if (IsResolutionMove(node)) + { + verifyResolutionMove(node, currentLocation); + } } // Validate the locations at the end of the previous block. @@ -11236,33 +11222,30 @@ void LinearScan::verifyFinalAllocation() } // Finally, handle the resolution moves, if any, at the beginning of the next block. - firstBlockEndResolutionStmt = nullptr; - bool foundNonResolutionStmt = false; - if (currentBlock != nullptr) + firstBlockEndResolutionNode = nullptr; + bool foundNonResolutionNode = false; + + LIR::Range& currentBlockRange = LIR::AsRange(currentBlock); + for (GenTree* node : currentBlockRange.NonPhiNodes()) { - for (GenTreeStmt* stmt = currentBlock->FirstNonPhiDef(); - stmt != nullptr && firstBlockEndResolutionStmt == nullptr; stmt = stmt->getNextStmt()) + if (IsResolutionNode(currentBlockRange, node)) { - if (stmt->gtStmtExpr->gtLsraInfo.isLsraAdded -#ifdef FEATURE_SIMD - && stmt->gtStmtExpr->OperGet() != GT_SIMD -#endif // FEATURE_SIMD - ) + if (foundNonResolutionNode) { - if (foundNonResolutionStmt) - { - firstBlockEndResolutionStmt = stmt; - } - else - { - verifyResolutionMove(stmt, currentLocation); - } + firstBlockEndResolutionNode = node; + break; } - else + else if (IsResolutionMove(node)) { - foundNonResolutionStmt = true; + // Only verify nodes that are actually moves; don't bother with the nodes that are + // operands to moves. + verifyResolutionMove(node, currentLocation); } } + else + { + foundNonResolutionNode = true; + } } } } @@ -11467,10 +11450,16 @@ void LinearScan::verifyFinalAllocation() } // Verify the moves in this block - for (GenTreeStmt* stmt = currentBlock->FirstNonPhiDef(); stmt != nullptr; stmt = stmt->getNextStmt()) + LIR::Range& currentBlockRange = LIR::AsRange(currentBlock); + for (GenTree* node : currentBlockRange.NonPhiNodes()) { - assert(stmt->gtStmtExpr->gtLsraInfo.isLsraAdded); - verifyResolutionMove(stmt, currentLocation); + assert(IsResolutionNode(currentBlockRange, node)); + if (IsResolutionMove(node)) + { + // Only verify nodes that are actually moves; don't bother with the nodes that are + // operands to moves. + verifyResolutionMove(node, currentLocation); + } } // Verify the outgoing register assignments @@ -11498,7 +11487,7 @@ void LinearScan::verifyFinalAllocation() // verifyResolutionMove: Verify a resolution statement. Called by verifyFinalAllocation() // // Arguments: -// resolutionStmt - A GenTreeStmt* that must be a resolution move. +// resolutionMove - A GenTree* that must be a resolution move. // currentLocation - The LsraLocation of the most recent RefPosition that has been verified. // // Return Value: @@ -11506,9 +11495,11 @@ void LinearScan::verifyFinalAllocation() // // Notes: // If verbose is set, this will also dump the moves into the table of final allocations. -void LinearScan::verifyResolutionMove(GenTreeStmt* resolutionStmt, LsraLocation currentLocation) +void LinearScan::verifyResolutionMove(GenTree* resolutionMove, LsraLocation currentLocation) { - GenTree* dst = resolutionStmt->gtStmtExpr; + GenTree* dst = resolutionMove; + assert(IsResolutionMove(dst)); + if (dst->OperGet() == GT_SWAP) { GenTreeLclVarCommon* left = dst->gtGetOp1()->AsLclVarCommon(); diff --git a/src/jit/lsra.h b/src/jit/lsra.h index 05d6ecf16d..a3c41fe1e3 100644 --- a/src/jit/lsra.h +++ b/src/jit/lsra.h @@ -398,7 +398,7 @@ public: // Insert a copy in the case where a tree node value must be moved to a different // register at the point of use, or it is reloaded to a different register // than the one it was spilled from - void insertCopyOrReload(GenTreePtr tree, unsigned multiRegIdx, RefPosition* refPosition); + void insertCopyOrReload(BasicBlock* block, GenTreePtr tree, unsigned multiRegIdx, RefPosition* refPosition); #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE // Insert code to save and restore the upper half of a vector that lives @@ -613,8 +613,12 @@ private: void lsraDumpIntervals(const char* msg); void dumpRefPositions(const char* msg); void dumpVarRefPositions(const char* msg); + + static bool IsResolutionMove(GenTree* node); + static bool IsResolutionNode(LIR::Range& containingRange, GenTree* node); + void verifyFinalAllocation(); - void verifyResolutionMove(GenTreeStmt* resolutionStmt, LsraLocation currentLocation); + void verifyResolutionMove(GenTree* resolutionNode, LsraLocation currentLocation); #else // !DEBUG bool doSelectNearest() { @@ -743,6 +747,7 @@ private: // Return the registers killed by the given tree node. regMaskTP getKillSetForNode(GenTree* tree); + // Given some tree node add refpositions for all the registers this node kills bool buildKillPositionsForNode(GenTree* tree, LsraLocation currentLoc); @@ -770,7 +775,7 @@ private: void buildInternalRegisterUsesForNode(GenTree* tree, LsraLocation currentLoc, RefPosition* defs[], int total); - void resolveLocalRef(GenTreePtr treeNode, RefPosition* currentRefPosition); + void resolveLocalRef(BasicBlock* block, GenTreePtr treeNode, RefPosition* currentRefPosition); void insertMove(BasicBlock* block, GenTreePtr insertionPoint, unsigned lclNum, regNumber inReg, regNumber outReg); @@ -931,6 +936,11 @@ private: char* operandString, unsigned operandStringLength); void lsraDispNode(GenTreePtr tree, LsraTupleDumpMode mode, bool hasDest); + void DumpOperandDefs(GenTree* operand, + bool& first, + LsraTupleDumpMode mode, + char* operandString, + const unsigned operandStringLength); void TupleStyleDump(LsraTupleDumpMode mode); bool dumpTerse; diff --git a/src/jit/morph.cpp b/src/jit/morph.cpp index 7f55b6ad72..e055bd3a94 100755 --- a/src/jit/morph.cpp +++ b/src/jit/morph.cpp @@ -113,31 +113,6 @@ GenTreePtr Compiler::fgMorphIntoHelperCall(GenTreePtr tree, int helper, GenTreeA } /***************************************************************************** - * This node should not be referenced by anyone now. Set its values to garbage - * to catch extra references - */ - -inline void DEBUG_DESTROY_NODE(GenTreePtr tree) -{ -#ifdef DEBUG - // printf("DEBUG_DESTROY_NODE for [0x%08x]\n", tree); - - // Save gtOper in case we want to find out what this node was - tree->gtOperSave = tree->gtOper; - - tree->gtType = TYP_UNDEF; - tree->gtFlags |= 0xFFFFFFFF & ~GTF_NODE_MASK; - if (tree->OperIsSimple()) - { - tree->gtOp.gtOp1 = tree->gtOp.gtOp2 = nullptr; - } - // Must do this last, because the "gtOp" check above will fail otherwise. - // Don't call SetOper, because GT_COUNT is not a valid value - tree->gtOper = GT_COUNT; -#endif -} - -/***************************************************************************** * * Determine if a relop must be morphed to a qmark to manifest a boolean value. * This is done when code generation can't create straight-line code to do it. @@ -7109,7 +7084,7 @@ void Compiler::fgMorphTailCall(GenTreeCall* call) void Compiler::fgMorphRecursiveFastTailCallIntoLoop(BasicBlock* block, GenTreeCall* recursiveTailCall) { assert(recursiveTailCall->IsTailCallConvertibleToLoop()); - GenTreePtr last = fgGetLastTopLevelStmt(block); + GenTreePtr last = block->lastStmt(); assert(recursiveTailCall == last->gtStmt.gtStmtExpr); // Transform recursive tail call into a loop. @@ -17364,9 +17339,7 @@ void Compiler::fgMarkAddressExposedLocals() bool Compiler::fgNodesMayInterfere(GenTree* write, GenTree* read) { - LclVarDsc* srcVar = nullptr; - bool srcAliased = false; - bool dstAliased = false; + LclVarDsc* srcVar = nullptr; bool readIsIndir = read->OperIsIndir() || read->OperIsImplicitIndir(); bool writeIsIndir = write->OperIsIndir() || write->OperIsImplicitIndir(); diff --git a/src/jit/nodeinfo.h b/src/jit/nodeinfo.h index 8373dcf29b..a73033a91f 100644 --- a/src/jit/nodeinfo.h +++ b/src/jit/nodeinfo.h @@ -5,6 +5,8 @@ #ifndef _NODEINFO_H_ #define _NODEINFO_H_ +struct GenTree; + class LinearScan; typedef unsigned int LsraLocation; diff --git a/src/jit/rationalize.cpp b/src/jit/rationalize.cpp index 6856ff9ae7..1127c8f542 100644 --- a/src/jit/rationalize.cpp +++ b/src/jit/rationalize.cpp @@ -7,66 +7,13 @@ #pragma hdrstop #endif -#include "hashbv.h" - -#ifdef DEBUG - -void dumpMethod() -{ - if (VERBOSE) - { - JitTls::GetCompiler()->fgDispBasicBlocks(true); - } -} - -void dumpTreeStack(Compiler* comp, ArrayStack<GenTree*>* stack) -{ - printf("=TOS=================\n"); - for (int i = 0; i < stack->Height(); i++) - { - comp->gtDispNode(stack->Index(i), nullptr, ""); - printf("\n"); - } - printf("=====================\n"); -} - -void dumpArgTable(Compiler* comp, GenTree* call) -{ - noway_assert(call->IsCall()); - fgArgInfoPtr argInfo = call->gtCall.fgArgInfo; - noway_assert(argInfo != nullptr); - - unsigned argCount = argInfo->ArgCount(); - fgArgTabEntryPtr* argTable = argInfo->ArgTable(); - fgArgTabEntryPtr curArgTabEntry = nullptr; - - JITDUMP("ARG TABLE for call "); - Compiler::printTreeID(call); - JITDUMP(":\n"); - for (unsigned i = 0; i < argCount; i++) - { - curArgTabEntry = argTable[i]; - JITDUMP("entry %d\n", i); - DISPTREE(curArgTabEntry->node); - } - JITDUMP("--------------ARG TABLE END --------------\n"); -} - -#endif // DEBUG - // state carried over the tree walk, to be used in making // a splitting decision. struct SplitData { - // callbacks to determine if we should split here, in pre and post order traversals - Compiler::fgSplitPredicate* predicatePre; - Compiler::fgSplitPredicate* predicatePost; - GenTree* root; // root stmt of tree being processed BasicBlock* block; Rationalizer* thisPhase; - - bool continueSubtrees; // whether to continue after splitting off a tree (in pre-order) }; //------------------------------------------------------------------------------ @@ -107,224 +54,6 @@ GenTree* isNodeCallArg(ArrayStack<GenTree*>* parentStack) return nullptr; } -//------------------------------------------------------------------------------ -// fgMakeEmbeddedStmt: insert the given subtree as an embedded statement -// -// Arguments: -// block - The block containing the parentStmt, into which the new embedded -// statement will go -// tree - The tree that will be the gtStmtExpr of the new embedded statement -// parentStmt - A statement (top-level or embedded) that 'tree' is fully contained in -// -// Return Value: -// A pointer to the new statement. -// -// Assumptions: -// 'tree' is fully contained in the linear order of parentStmt -// -// Notes: -// If 'tree' is at the beginning of the linear order of 'parentStmt', it -// is made into a top-level statement. - -GenTreeStmt* Compiler::fgMakeEmbeddedStmt(BasicBlock* block, GenTree* tree, GenTree* parentStmt) -{ - assert(tree->gtOper != GT_STMT); - assert(parentStmt->gtOper == GT_STMT); - assert(fgBlockContainsStatementBounded(block, parentStmt)); - - GenTreePtr newStmtFirstNode = fgGetFirstNode(tree); - GenTreePtr parentStmtFirstNode = parentStmt->gtStmt.gtStmtList; - GenTreePtr prevStmt = parentStmt; - bool newTopLevelStmt = false; - bool splitParentStmt = false; - if (newStmtFirstNode == parentStmtFirstNode) - { - // If this is the first node of the new statement, split them. - parentStmt->gtStmt.gtStmtList = tree->gtNext; - prevStmt = parentStmt->gtPrev; - splitParentStmt = true; - } - GenTreeStmt* newStmt = gtNewStmt(tree, parentStmt->gtStmt.gtStmtILoffsx); // Use same IL offset as parent statement - newStmt->CopyCosts(tree); - newStmt->gtStmtList = newStmtFirstNode; - if (splitParentStmt && parentStmt->gtStmt.gtStmtIsTopLevel()) - { - newTopLevelStmt = true; - tree->gtNext->gtPrev = nullptr; - tree->gtNext = nullptr; - } - else - { - newStmt->gtFlags &= ~(GTF_STMT_TOP_LEVEL); - } - - // Does parentStmt already have embedded statements? - // If so, determine where this fits in the linear order. - // Note that if we have the splitParentStmt case, some of parentStmt's embedded statements - // may need to move with the new statement - GenTreePtr nextStmt = parentStmt->gtNext; - GenTreePtr nextLinearNode; - GenTreePtr searchNode; - if (splitParentStmt) - { - nextLinearNode = newStmtFirstNode; - // In this case, we're going to search for the LAST linear node in the new statement - // in order to determine which embedded statements will move with this one. - searchNode = tree; - } - else - { - nextLinearNode = parentStmt->gtStmt.gtStmtList; - // In this case, we're going to search for the FIRST linear node in the new statement - // so that we can insert this after any embedded statements that START before it. - searchNode = newStmtFirstNode; - } - // Remember if we find any embedded statements before encountering 'searchNode'. - bool foundEmbeddedStmts = false; - while (nextStmt != nullptr && nextStmt->gtStmt.gtStmtIsEmbedded()) - { - GenTreePtr nextEmbeddedNode = nextStmt->gtStmt.gtStmtList; - while (nextLinearNode != searchNode && nextLinearNode != nextEmbeddedNode) - { - nextLinearNode = nextLinearNode->gtNext; - assert(nextLinearNode != nullptr); - } - if (nextLinearNode == searchNode) - { - break; - } - prevStmt = nextStmt; - nextStmt = nextStmt->gtNext; - foundEmbeddedStmts = true; - } - - if (newTopLevelStmt) - { - // For this case, we are actually going to insert it BEFORE parentStmt. - // However if we have a new prevStmt (i.e. there are some embedded statements - // to be included in newStmt) then those need to be moved as well. - // Note, however, that all the tree links have already been fixed up. - fgInsertStmtBefore(block, parentStmt, newStmt); - if (foundEmbeddedStmts) - { - GenTreePtr firstEmbeddedStmt = parentStmt->gtNext; - assert(firstEmbeddedStmt->gtStmt.gtStmtIsEmbedded()); - assert(prevStmt->gtStmt.gtStmtIsEmbedded()); - parentStmt->gtNext = prevStmt->gtNext; - if (parentStmt->gtNext != nullptr) - { - parentStmt->gtNext->gtPrev = parentStmt; - } - else - { - block->bbTreeList->gtPrev = parentStmt; - } - - parentStmt->gtPrev = prevStmt; - prevStmt->gtNext = parentStmt; - - newStmt->gtNext = firstEmbeddedStmt; - firstEmbeddedStmt->gtPrev = newStmt; - } - } - else - { - fgInsertStmtAfter(block, prevStmt, newStmt); - } - - return newStmt; -} - -//------------------------------------------------------------------------------ -// fgInsertLinearNodeBefore: insert the given single node before 'before'. -// -// Arguments: -// newNode - The node to be inserted -// before - The node to insert it before -// -// Return Value: -// None. -// -// Assumptions: -// Either the callee must ensure that 'before' is part of compCurStmt, -// or before->gtPrev must be non-null - -void Compiler::fgInsertLinearNodeBefore(GenTreePtr newNode, GenTreePtr before) -{ - GenTreePtr prevNode = before->gtPrev; - newNode->gtPrev = prevNode; - if (prevNode == nullptr) - { - assert(compCurStmt->gtStmt.gtStmtList == before && compCurStmt->gtStmt.gtStmtIsTopLevel()); - } - else - { - prevNode->gtNext = newNode; - } - // Note that 'before' may be the first node in gtStmtList even if its gtPrev is non-null, - // since compCurStmt may be embedded. - if (compCurStmt->gtStmt.gtStmtList == before) - { - compCurStmt->gtStmt.gtStmtList = newNode; - } - newNode->gtNext = before; - before->gtPrev = newNode; -} - -//----------------------------------------------------------------------------------------------- -// fgInsertEmbeddedFormTemp: Assign a variable to hold the result of *ppTree, possibly creating a new variable -// and creating a new (possibly embedded) statement for it. The original -// subtree will be replaced with a use of the temp. -// -// Arguments: -// ppTree - a pointer to the child node we will be replacing with a reference to the new temp. -// lclNum - local var to use, or BAD_VAR_NUM to create one -// -// Return Value: -// The new statement. -// -// Assumptions: -// The caller must ensure that '*ppTree' is part of compCurStmt, and that -// compCurStmt is in compCurBB; - -GenTreeStmt* Compiler::fgInsertEmbeddedFormTemp(GenTree** ppTree, unsigned lclNum) -{ - GenTree* subTree = *ppTree; - - if (lclNum == BAD_VAR_NUM) - { - lclNum = lvaGrabTemp(true DEBUGARG("fgInsertEmbeddedFormTemp is creating a new local variable")); - } - - // Increment its lvRefCnt and lvRefCntWtd twice, one for the def and one for the use - lvaTable[lclNum].incRefCnts(compCurBB->getBBWeight(this), this); - lvaTable[lclNum].incRefCnts(compCurBB->getBBWeight(this), this); - - GenTreeLclVar* store = gtNewTempAssign(lclNum, subTree)->AsLclVar(); - gtSetEvalOrder(store); - - subTree->InsertAfterSelf(store); - - GenTree* load = - new (this, GT_LCL_VAR) GenTreeLclVar(store->TypeGet(), store->AsLclVarCommon()->GetLclNum(), BAD_IL_OFFSET); - gtSetEvalOrder(load); - - store->InsertAfterSelf(load); - - *ppTree = load; - - JITDUMP("fgInsertEmbeddedFormTemp created store :\n"); - DISPTREE(store); - - GenTreeStmt* stmt = fgMakeEmbeddedStmt(compCurBB, store, compCurStmt); - stmt->gtStmtILoffsx = compCurStmt->gtStmt.gtStmtILoffsx; -#ifdef DEBUG - stmt->gtStmtLastILoffs = compCurStmt->gtStmt.gtStmtLastILoffs; -#endif // DEBUG - - return stmt; -} - // return op that is the store equivalent of the given load opcode genTreeOps storeForm(genTreeOps loadForm) { @@ -358,162 +87,26 @@ genTreeOps addrForm(genTreeOps loadForm) } } -// copy the flags determined by mask from src to dst -void copyFlags(GenTree* dst, GenTree* src, unsigned mask) +// return op that is the load equivalent of the given addr opcode +genTreeOps loadForm(genTreeOps addrForm) { - dst->gtFlags &= ~mask; - dst->gtFlags |= (src->gtFlags & mask); -} - -//-------------------------------------------------------------------------------------- -// RewriteTopLevelComma - remove a top-level comma by creating a new preceding statement -// from its LHS and replacing the comma with its RHS (unless the -// comma's RHS is a NOP, in which case the comma is replaced with -// its LHS and no new statement is created) -// -// Returns the location of the statement that contains the LHS of the removed comma. -//-------------------------------------------------------------------------------------- - -Location Rationalizer::RewriteTopLevelComma(Location loc) -{ - GenTreeStmt* commaStmt = loc.tree->AsStmt(); - - GenTree* commaOp = commaStmt->gtStmtExpr; - assert(commaOp->OperGet() == GT_COMMA); - - GenTree* commaOp1 = commaOp->gtGetOp1(); - GenTree* commaOp2 = commaOp->gtGetOp2(); - - if (commaOp2->IsNothingNode()) + switch (addrForm) { -#ifdef DEBUG - if (comp->verbose) - { - printf("Replacing GT_COMMA(X, GT_NOP) by X\n"); - comp->gtDispTree(commaOp); - printf("\n"); - } -#endif // DEBUG - - comp->fgSnipNode(commaStmt, commaOp); - comp->fgDeleteTreeFromList(commaStmt, commaOp2); - commaStmt->gtStmtExpr = commaOp1; - - return loc; - } - - JITDUMP("splitting top level comma!\n"); - - // Replace the comma node in the original statement with the RHS of the comma node. - comp->fgDeleteTreeFromList(commaStmt, commaOp1); - comp->fgSnipNode(commaStmt, commaOp); - commaStmt->gtStmtExpr = commaOp2; - - // Create and insert a new preceding statement from the LHS of the comma node. - GenTreeStmt* newStatement = comp->gtNewStmt(commaOp1, commaStmt->gtStmtILoffsx); - newStatement->CopyCosts(commaOp1); - newStatement->gtStmtList = Compiler::fgGetFirstNode(commaOp1); - newStatement->gtStmtList->gtPrev = nullptr; - commaOp1->gtNext = nullptr; - - comp->fgInsertStmtBefore(loc.block, commaStmt, newStatement); - - return Location(newStatement, loc.block); -} - -//------------------------------------------------------------------------------ -// MorphAsgIntoStoreLcl - -// Receives an assignment of type GT_ASG(Lhs, Rhs) where: -// -- Lhs can be GT_LCL_VAR or GT_LCL_FLD -// -- Rhs is an arbitrary tree and converts that into its corresponding -// store local form. -// -// Returns the tree converted into GT_STORE_LCL_VAR or GT_STORE_LCL_FLD form. -// -// If stmt is null, this is a newly created tree that is not yet contained in -// a stmt. -//------------------------------------------------------------------------------ -void Rationalizer::MorphAsgIntoStoreLcl(GenTreeStmt* stmt, GenTreePtr pTree) -{ - assert(pTree->OperGet() == GT_ASG); - - GenTreePtr lhs = pTree->gtGetOp1(); - GenTreePtr rhs = pTree->gtGetOp2(); - - genTreeOps lhsOper = lhs->OperGet(); - genTreeOps storeOper; - - assert(lhsOper == GT_LCL_VAR || lhsOper == GT_LCL_FLD); - - storeOper = storeForm(lhsOper); -#ifdef DEBUG - JITDUMP("rewriting asg(%s, X) to %s(X)\n", GenTree::NodeName(lhsOper), GenTree::NodeName(storeOper)); -#endif // DEBUG - - GenTreeLclVarCommon* var = lhs->AsLclVarCommon(); - pTree->SetOper(storeOper); - GenTreeLclVarCommon* dst = pTree->AsLclVarCommon(); - dst->SetLclNum(var->gtLclNum); - dst->SetSsaNum(var->gtSsaNum); - dst->gtType = lhs->gtType; - - if (lhs->OperGet() == GT_LCL_FLD) - { - dst->gtLclFld.gtLclOffs = lhs->gtLclFld.gtLclOffs; - dst->gtLclFld.gtFieldSeq = lhs->gtLclFld.gtFieldSeq; - } - - copyFlags(dst, var, GTF_LIVENESS_MASK); - dst->gtOp.gtOp1 = rhs; - - if (stmt != nullptr) - { - assert(stmt->OperGet() == GT_STMT); - Compiler::fgDeleteTreeFromList(stmt, lhs); + case GT_LCL_VAR_ADDR: + return GT_LCL_VAR; + case GT_LCL_FLD_ADDR: + return GT_LCL_FLD; + default: + noway_assert(!"not a local address opcode\n"); + unreached(); } - - DISPNODE(pTree); - JITDUMP("\n"); -} - -//------------------------------------------------------------------------------ -// CreateTempAssignment - -// Constructs an assignment where its left hand side is a GenTree node -// representing the given local variable number and the right hand side is -// the given tree. -// -// This calls gtNewTempAssig(), which produces a GT_STORE_LCL_VAR instead of a -// GT_ASG when we are in linear order, which we are in the Rationalizer. -// -//------------------------------------------------------------------------------ -GenTreePtr Rationalizer::CreateTempAssignment(Compiler* comp, unsigned lclNum, GenTreePtr rhs) -{ - GenTreePtr gtAsg = comp->gtNewTempAssign(lclNum, rhs); - return gtAsg; } -// turn "comma(lcl x, lcl x)" into "lcl x" -// this is produced by earlier transformations - -void Rationalizer::DuplicateCommaProcessOneTree(Compiler* comp, - Rationalizer* irt, - BasicBlock* block, - GenTree* statement) +// copy the flags determined by mask from src to dst +void copyFlags(GenTree* dst, GenTree* src, unsigned mask) { - SplitData tmpState = {nullptr}; - tmpState.root = statement; - tmpState.continueSubtrees = true; - tmpState.thisPhase = irt; - tmpState.block = block; - - assert(statement->IsStatement()); - - comp->fgWalkTree(&(statement->gtStmt.gtStmtExpr), nullptr, CommaHelper, &tmpState); - -#if 0 - JITDUMP("resulting block\n"); - DBEXEC(VERBOSE, comp->fgDispBasicBlocks(block, block, true)); -#endif + dst->gtFlags &= ~mask; + dst->gtFlags |= (src->gtFlags & mask); } // call args have other pointers to them which must be fixed up if @@ -523,7 +116,6 @@ void Compiler::fgFixupIfCallArg(ArrayStack<GenTree*>* parentStack, GenTree* oldC GenTree* parentCall = isNodeCallArg(parentStack); if (!parentCall) { - DBEXEC(VERBOSE, dumpTreeStack(JitTls::GetCompiler(), parentStack)); return; } @@ -548,10 +140,10 @@ void Compiler::fgFixupArgTabEntryPtr(GenTreePtr parentCall, GenTreePtr oldArg, G assert(newArg != nullptr); JITDUMP("parent call was :\n"); - DISPTREE(parentCall); + DISPNODE(parentCall); JITDUMP("old child was :\n"); - DISPTREE(oldArg); + DISPNODE(oldArg); if (oldArg->gtFlags & GTF_LATE_ARG) { @@ -563,336 +155,6 @@ void Compiler::fgFixupArgTabEntryPtr(GenTreePtr parentCall, GenTreePtr oldArg, G assert(fp->node == oldArg); fp->node = newArg; } - - JITDUMP("parent call:\n"); - DISPTREE(parentCall); -} - -//------------------------------------------------------------------------ -// CommaUselessChild: removes commas with useless first child: -// - Turns "comma(lcl x, Y)" into "Y" -// - Turns "comma(NOP, Y)" into "Y" -// -// Arguments: -// ppTree - a pointer to the parent pointer for a comma node -// data - the traversal data -// -// Return Value: -// Returns "true" if it found a comma with a useless child, and transformed it. -// -// Notes: -// These comma forms are produced by earlier transformations. - -bool Rationalizer::CommaUselessChild(GenTree** ppTree, Compiler::fgWalkData* data) -{ - GenTree* tree = *ppTree; - GenTree * subChild1, *subChild2; - SplitData* tmpState = (SplitData*)data->pCallbackData; - - assert(tree->OperGet() == GT_COMMA); - - subChild1 = tree->gtGetOp1(); - subChild2 = tree->gtGetOp2(); - - if (subChild1->OperGet() == GT_COMMA) - { - data->parentStack->Push(tree->gtOp.gtOp1); - CommaUselessChild(&(tree->gtOp.gtOp1), data); - subChild1 = tree->gtGetOp1(); - data->parentStack->Pop(); - } - - if (subChild2->OperGet() == GT_COMMA) - { - data->parentStack->Push(tree->gtOp.gtOp2); - CommaUselessChild(&(tree->gtOp.gtOp2), data); - subChild2 = tree->gtGetOp2(); - data->parentStack->Pop(); - } - - if (subChild1 != nullptr && subChild2 != nullptr && - (subChild1->OperIsLocalRead() || (subChild1->OperGet() == GT_NOP && subChild1->gtGetOp1() == nullptr))) - { - JITDUMP("found comma subtree with useless child:\n"); - DISPTREE(tree); - JITDUMP("\n"); - -#ifdef DEBUG - if (isNodeCallArg(data->parentStack)) - { - JITDUMP("PARENT TREE:\n"); - DISPTREE(isNodeCallArg(data->parentStack)); - JITDUMP("\n"); - } -#endif // DEBUG - - Compiler::fgSnipNode(tmpState->root->AsStmt(), tree); - Compiler::fgSnipNode(tmpState->root->AsStmt(), subChild1); - *ppTree = subChild2; - - if (tree->gtFlags & GTF_LATE_ARG) - { - subChild2->gtFlags |= GTF_LATE_ARG; - // If we just have a bare local as a late ("SETUP") arg then that is effectively a NOP - // however if that local node is a last use, codegen will not count it as such, and blow up - // so get rid of those here - if (subChild2->IsLocal()) - { - subChild2->gtBashToNOP(); - } - } - - tmpState->thisPhase->comp->fgFixupIfCallArg(data->parentStack, tree, subChild2); - return true; - } - return false; -} - -// Call CommaUselessChild() to turn "comma(lcl x, lcl x)" into "lcl x" - -Compiler::fgWalkResult Rationalizer::CommaHelper(GenTree** ppTree, Compiler::fgWalkData* data) -{ - GenTree* tree = *ppTree; - Compiler* comp = data->compiler; - - SplitData* tmpState = (SplitData*)data->pCallbackData; - - if (tree->OperGet() == GT_COMMA && CommaUselessChild(ppTree, data)) - { - return Compiler::WALK_SKIP_SUBTREES; - } - - return Compiler::WALK_CONTINUE; -} - -// rewrite ASG nodes as either local store or indir store forms -// also remove ADDR nodes -Location Rationalizer::TreeTransformRationalization(Location loc) -{ - GenTree* savedCurStmt = comp->compCurStmt; - GenTreeStmt* statement = (loc.tree)->AsStmt(); - GenTree* tree = statement->gtStmt.gtStmtExpr; - - JITDUMP("TreeTransformRationalization, with statement:\n"); - DISPTREE(statement); - JITDUMP("\n"); - - DBEXEC(TRUE, loc.Validate()); - DBEXEC(TRUE, ValidateStatement(loc)); - - if (statement->gtStmtIsTopLevel()) - { - comp->compCurBB = loc.block; - comp->compCurStmt = statement; - - while (tree->OperGet() == GT_COMMA) - { - // RewriteTopLevelComma may create a new preceding statement for the LHS of a - // top-level comma. If it does, we need to process that statement now. - Location newLoc = RewriteTopLevelComma(loc); - if (newLoc.tree != statement) - { - (void)TreeTransformRationalization(newLoc); - } - - // RewriteTopLevelComma also replaces the tree for this statement with the RHS - // of the comma (or the LHS, if the RHS is a NOP), so we must reload it for - // correctness. - tree = statement->gtStmt.gtStmtExpr; - } - - if (tree->OperKind() & GTK_CONST) - { - // Don't bother generating a top level statement that is just a constant. - // We can get these if we decide to hoist a large constant value out of a loop. - tree->gtBashToNOP(); - } - } - - SplitData tmpState = {nullptr}; - tmpState.root = statement; - tmpState.continueSubtrees = true; - tmpState.thisPhase = this; - tmpState.block = loc.block; - - comp->fgWalkTree(&(statement->gtStmt.gtStmtExpr), SimpleTransformHelper, nullptr, &tmpState); - - tree = statement->gtStmt.gtStmtExpr; - if (tree->OperIsLocalRead()) - { - comp->lvaTable[tree->AsLclVarCommon()->gtLclNum].decRefCnts(comp->compCurBB->getBBWeight(comp), comp); - tree->gtBashToNOP(); - } - - DuplicateCommaProcessOneTree(comp, this, loc.block, loc.tree); - - JITDUMP("After simple transforms:\n"); - DISPTREE(statement); - JITDUMP("\n"); - - DBEXEC(TRUE, ValidateStatement(loc)); - - comp->compCurStmt = savedCurStmt; - return loc; -} - -// RecursiveRewriteComma -// -// This routine deals with subtrees composed entirely of commas, and the expressions that hang off of them. -// The degenerate case is a single comma but (?????) -// -// ppTree : pointer to a link to a comma node -// discard: true if any value produced by the node will ultimately be discarded. -// In a tree of commas with some non-comma expressions hanging off the terminal commas, -// ultimately all results of those expressions will be discarded except for -// the expression reached by following the second link of of all commas on a path from the base -// ex: in "comma(comma(exp1, exp2), comma(exp3, comma(exp4, exp5)))" -// the only expression whose value makes it to the root of the comma tree is exp5 -// nested: true if there is another comma as the parent -// -void Rationalizer::RecursiveRewriteComma(GenTree** ppTree, Compiler::fgWalkData* data, bool discard, bool nested) -{ - GenTree* comma = *ppTree; - assert(comma->gtOper == GT_COMMA); - GenTreePtr op2 = comma->gtOp.gtOp2; - GenTreePtr op1 = comma->gtOp.gtOp1; - SplitData* tmpState = (SplitData*)data->pCallbackData; - GenTreePtr stmt = tmpState->root; - Compiler* comp = data->compiler; - - JITDUMP("recursive rewrite comma :\n"); - DISPTREE(comma); - JITDUMP("\n"); - - if (op1->gtOper == GT_COMMA) - { - // embed all of the expressions reachable from op1. - // Since they feed into op1, their results are discarded (not used up the tree) - RecursiveRewriteComma(&(comma->gtOp.gtOp1), data, true, true); - } - - // Although most top-level commas have already been handled, we may create new ones - // (for example by splitting a comma above another comma). - Compiler::fgSnipNode(stmt->AsStmt(), comma); - *ppTree = op2; - JITDUMP("pptree now : "); - DISPNODE(op2); - if (data->parentStack->Top() == comma) - { - data->parentStack->Pop(); - data->parentStack->Push(op2); - } - - GenTree* commaNext = comma->gtNext; - - op1 = comma->gtOp.gtOp1; - - // op1 of the comma will now be a new statement, either top-level or embedded - // depending on the execution order. - // The comma is simply eliminated. - GenTreePtr newStmt = comp->fgMakeEmbeddedStmt(tmpState->block, op1, tmpState->root); - - if (!nested) - { - comp->fgFixupIfCallArg(data->parentStack, comma, *ppTree); - } - - JITDUMP("Split comma into %s statements. New statement:\n", - (newStmt->gtFlags & GTF_STMT_TOP_LEVEL) ? "top-level" : "embedded"); - DISPTREE(newStmt); - JITDUMP("\nOld statement:\n"); - DISPTREE(stmt); - JITDUMP("\n"); - - (void)((Rationalizer*)tmpState->thisPhase)->TreeTransformRationalization(Location(newStmt, tmpState->block)); - - // In a sense, assignment nodes have two destinations: 1) whatever they are writing to - // and 2) they also produce the value that was written so their parent can consume it. - // In the case where the parent is going to consume the value, - // insert the assign as an embedded statement and clone the destination to replace itself in the tree. - - if (op2->OperGet() == GT_ASG && !discard) - { - JITDUMP("op2 of comma was an assignment, doing additional work\n"); - assert(op2->gtNext); - GenTree* dst = op2->gtOp.gtOp1; - GenTree* newSrc = nullptr; - GenTreeStmt* newStmt; - - newStmt = comp->fgMakeEmbeddedStmt(tmpState->block, op2, tmpState->root); - - // can this happen ? - assert(dst->OperIsLocal()); - - newSrc = comp->gtClone(dst); - newSrc->gtFlags &= ~GTF_VAR_DEF; - - *ppTree = newSrc; - comp->fgInsertTreeInListBefore(newSrc, commaNext, stmt->AsStmt()); - - JITDUMP("Split comma into %s statements. New statement:\n", - (newStmt->gtFlags & GTF_STMT_TOP_LEVEL) ? "top-level" : "embedded"); - DISPTREE(newStmt); - JITDUMP("\nOld statement:\n"); - DISPTREE(stmt); - JITDUMP("\n"); - - (void)((Rationalizer*)tmpState->thisPhase)->TreeTransformRationalization(Location(newStmt, tmpState->block)); - - if (!nested) - { - comp->fgFixupIfCallArg(data->parentStack, comma, newSrc); - } - - (void)((Rationalizer*)tmpState->thisPhase)->TreeTransformRationalization(Location(newStmt, tmpState->block)); - - return; - } - JITDUMP("\nreturning from RecursiveRewriteComma\n"); -} - -//------------------------------------------------------------------------ -// RewriteOneComma: Rewrites the trees to remove a comma -// -// Arguments: -// ppTree - a pointer to the parent pointer for a comma node -// data - the traversal data -// -// Return Value: -// None. -// -// Assumptions: -// This method is always called during a traversal (hence the fgWalkData). -// 'ppTree' must point to a GT_COMMA GenTreePtr -// -// Notes: -// If op1 of the comma is a (unused) lclVar, it is deleted by CommmaUselessChild() - -void Rationalizer::RewriteOneComma(GenTree** ppTree, Compiler::fgWalkData* data) -{ - GenTreePtr comma = *ppTree; - Compiler* comp = data->compiler; - SplitData* tmpState = (SplitData*)data->pCallbackData; - GenTreePtr stmt = tmpState->root; - - assert(comma->gtOper == GT_COMMA); - GenTreePtr op2 = comma->gtOp.gtOp2; - GenTreePtr op1 = comma->gtOp.gtOp1; - - // Remove the comma from the tree; we know it has non-null gtPrev, otherwise - // we would have handled it as a top-level comma. - assert(comma->gtPrev != nullptr); - JITDUMP("Rationalizing comma:"); - DISPNODE(comma); - - if (!CommaUselessChild(ppTree, data)) - { - // Set 'discard' to true when the comma tree does not return a value - // If the comma's type is TYP_VOID then 'discard' is set to true - // otherwise 'discard' is set to false - bool discard = (comma->TypeGet() == TYP_VOID); - RecursiveRewriteComma(ppTree, data, discard, false); - } } // Rewrite InitBlk involving SIMD vector into stlcl.var of a SIMD type. @@ -907,11 +169,9 @@ void Rationalizer::RewriteOneComma(GenTree** ppTree, Compiler::fgWalkData* data) // TODO-Cleanup: Once SIMD types are plumbed through the frontend, this will no longer // be required. // -void Rationalizer::RewriteInitBlk(GenTreePtr* ppTree, Compiler::fgWalkData* data) +void Rationalizer::RewriteInitBlk(LIR::Use& use) { #ifdef FEATURE_SIMD - Compiler* comp = data->compiler; - // No lowering is needed for non-SIMD nodes, so early out if featureSIMD is not enabled. if (!comp->featureSIMD) { @@ -919,65 +179,57 @@ void Rationalizer::RewriteInitBlk(GenTreePtr* ppTree, Compiler::fgWalkData* data } // See if this is a SIMD initBlk that needs to be changed to a simple st.lclVar. - GenTreeInitBlk* tree = (*ppTree)->AsInitBlk(); + GenTreeInitBlk* initBlk = use.Def()->AsInitBlk(); // Is the dstAddr is addr of a SIMD type lclVar? - GenTreePtr dstAddr = tree->Dest(); - if (dstAddr->OperGet() != GT_ADDR) + GenTree* dstAddr = initBlk->Dest(); + if (!comp->isAddrOfSIMDType(dstAddr) || !dstAddr->OperIsLocalAddr()) { return; } - GenTree* dst = dstAddr->gtGetOp1(); - var_types baseType = comp->getBaseTypeOfSIMDLocal(dst); - if (baseType == TYP_UNKNOWN) + unsigned lclNum = dstAddr->AsLclVarCommon()->gtLclNum; + if (!comp->lvaTable[lclNum].lvSIMDType) { return; } - CORINFO_CLASS_HANDLE typeHnd = comp->lvaTable[dst->AsLclVarCommon()->gtLclNum].lvVerTypeInfo.GetClassHandle(); + + var_types baseType = comp->lvaTable[lclNum].lvBaseType; + CORINFO_CLASS_HANDLE typeHnd = comp->lvaTable[lclNum].lvVerTypeInfo.GetClassHandle(); unsigned simdLocalSize = comp->getSIMDTypeSizeInBytes(typeHnd); JITDUMP("Rewriting SIMD InitBlk\n"); - DISPTREE(tree); - - // Get rid of the parent node in GT_ADDR(GT_LCL_VAR) - comp->fgSnipInnerNode(dstAddr); + DISPTREERANGE(BlockRange(), initBlk); - assert((dst->gtFlags & GTF_VAR_USEASG) == 0); + assert((dstAddr->gtFlags & GTF_VAR_USEASG) == 0); - // Remove 'size' from execution order // There are currently only three sizes supported: 8 bytes, 16 bytes or the vector register length. - GenTreeIntConCommon* sizeNode = tree->Size()->AsIntConCommon(); + GenTreeIntConCommon* sizeNode = initBlk->Size()->AsIntConCommon(); unsigned int size = (unsigned int)roundUp(sizeNode->IconValue(), TARGET_POINTER_SIZE); var_types simdType = comp->getSIMDTypeForSize(size); assert(roundUp(simdLocalSize, TARGET_POINTER_SIZE) == size); - comp->fgSnipInnerNode(sizeNode); - GenTree* initVal = tree->InitVal(); - GenTreeSIMD* simdTree = new (comp, GT_SIMD) + GenTree* initVal = initBlk->InitVal(); + GenTreeSIMD* simdNode = new (comp, GT_SIMD) GenTreeSIMD(simdType, initVal, SIMDIntrinsicInit, baseType, (unsigned)sizeNode->IconValue()); - dst->SetOper(GT_STORE_LCL_VAR); - dst->gtType = simdType; - dst->gtOp.gtOp1 = simdTree; - dst->gtFlags |= (simdTree->gtFlags & GTF_ALL_EFFECT); - initVal->gtNext = simdTree; - simdTree->gtPrev = initVal; + dstAddr->SetOper(GT_STORE_LCL_VAR); + GenTreeLclVar* store = dstAddr->AsLclVar(); + store->gtType = simdType; + store->gtOp.gtOp1 = simdNode; + store->gtFlags |= ((simdNode->gtFlags & GTF_ALL_EFFECT) | GTF_ASG); + BlockRange().Remove(store); - simdTree->gtNext = dst; - dst->gtPrev = simdTree; + // Insert the new nodes into the block + BlockRange().InsertAfter(initVal, simdNode, store); + use.ReplaceWith(comp, store); - GenTree* nextNode = tree->gtNext; - dst->gtNext = nextNode; - if (nextNode != nullptr) - { - nextNode->gtPrev = dst; - } - - *ppTree = dst; + // Remove the old size and GT_INITBLK nodes. + BlockRange().Remove(sizeNode); + BlockRange().Remove(initBlk); JITDUMP("After rewriting SIMD InitBlk:\n"); - DISPTREE(*ppTree); + DISPTREERANGE(BlockRange(), use.Def()); JITDUMP("\n"); #endif // FEATURE_SIMD } @@ -1005,11 +257,9 @@ void Rationalizer::RewriteInitBlk(GenTreePtr* ppTree, Compiler::fgWalkData* data // TODO-Cleanup: Once SIMD types are plumbed through the frontend, this will no longer // be required. // -void Rationalizer::RewriteCopyBlk(GenTreePtr* ppTree, Compiler::fgWalkData* data) +void Rationalizer::RewriteCopyBlk(LIR::Use& use) { #ifdef FEATURE_SIMD - Compiler* comp = data->compiler; - // No need to transofrm non-SIMD nodes, if featureSIMD is not enabled. if (!comp->featureSIMD) { @@ -1017,15 +267,17 @@ void Rationalizer::RewriteCopyBlk(GenTreePtr* ppTree, Compiler::fgWalkData* data } // See if this is a SIMD copyBlk - GenTreeCpBlk* tree = (*ppTree)->AsCpBlk(); - genTreeOps oper = GT_NONE; - GenTreePtr dstAddr = tree->Dest(); - GenTree* srcAddr = tree->Source(); + GenTreeCpBlk* cpBlk = use.Def()->AsCpBlk(); + GenTreePtr dstAddr = cpBlk->Dest(); + GenTree* srcAddr = cpBlk->Source(); + + const bool srcIsSIMDAddr = comp->isAddrOfSIMDType(srcAddr); + const bool dstIsSIMDAddr = comp->isAddrOfSIMDType(dstAddr); // Do not transform if neither src or dst is known to be a SIMD type. // If src tree type is something we cannot reason but if dst is known to be of a SIMD type // we will treat src tree as a SIMD type and vice versa. - if (!(comp->isAddrOfSIMDType(srcAddr) || comp->isAddrOfSIMDType(dstAddr))) + if (!srcIsSIMDAddr && !dstIsSIMDAddr) { return; } @@ -1034,22 +286,22 @@ void Rationalizer::RewriteCopyBlk(GenTreePtr* ppTree, Compiler::fgWalkData* data // start transforming the original tree. Prior to this point do not perform // any modifications to the original tree. JITDUMP("\nRewriting SIMD CopyBlk\n"); - DISPTREE(tree); + DISPTREERANGE(BlockRange(), cpBlk); - // Remove 'size' from execution order // There are currently only three sizes supported: 8 bytes, 12 bytes, 16 bytes or the vector register length. - GenTreeIntConCommon* sizeNode = tree->Size()->AsIntConCommon(); + GenTreeIntConCommon* sizeNode = cpBlk->Size()->AsIntConCommon(); var_types simdType = comp->getSIMDTypeForSize((unsigned int)sizeNode->IconValue()); - comp->fgSnipInnerNode(sizeNode); + + // Remove 'size' from execution order + BlockRange().Remove(sizeNode); // Is destination a lclVar which is not an arg? // If yes then we can turn it to a stlcl.var, otherwise turn into stind. - GenTree* simdDst = nullptr; - if (dstAddr->OperGet() == GT_ADDR && comp->isSIMDTypeLocal(dstAddr->gtGetOp1())) + GenTree* simdDst = nullptr; + genTreeOps oper = GT_NONE; + if (dstIsSIMDAddr && dstAddr->OperIsLocalAddr()) { - // Get rid of parent node in GT_ADDR(GT_LCL_VAR) - comp->fgSnipInnerNode(dstAddr); - simdDst = dstAddr->gtGetOp1(); + simdDst = dstAddr; simdDst->gtType = simdType; oper = GT_STORE_LCL_VAR; @@ -1064,13 +316,20 @@ void Rationalizer::RewriteCopyBlk(GenTreePtr* ppTree, Compiler::fgWalkData* data oper = GT_STOREIND; } - // Src: Get rid of parent node of GT_ADDR(..) if its child happens to be of a SIMD type. GenTree* simdSrc = nullptr; - if (srcAddr->OperGet() == GT_ADDR && varTypeIsSIMD(srcAddr->gtGetOp1())) + if ((srcAddr->OperGet() == GT_ADDR) && varTypeIsSIMD(srcAddr->gtGetOp1())) { - comp->fgSnipInnerNode(srcAddr); + // Get rid of parent node of GT_ADDR(..) if its child happens to be of a SIMD type. + BlockRange().Remove(srcAddr); simdSrc = srcAddr->gtGetOp1(); } + else if (srcIsSIMDAddr && srcAddr->OperIsLocalAddr()) + { + // If the source has been rewritten into a local addr node, rewrite it back into a + // local var node. + simdSrc = srcAddr; + simdSrc->SetOper(loadForm(srcAddr->OperGet())); + } else { // Since destination is known to be a SIMD type, src must be a SIMD type too @@ -1082,10 +341,10 @@ void Rationalizer::RewriteCopyBlk(GenTreePtr* ppTree, Compiler::fgWalkData* data // but setting them to a reasonable value based on the logic in gtSetEvalOrder(). GenTree* indir = comp->gtNewOperNode(GT_IND, simdType, srcAddr); indir->SetCosts(IND_COST_EX, 2); - srcAddr->InsertAfterSelf(indir); + BlockRange().InsertAfter(srcAddr, indir); - tree->gtGetOp1()->gtOp.gtOp2 = indir; - simdSrc = indir; + cpBlk->gtGetOp1()->gtOp.gtOp2 = indir; + simdSrc = indir; } simdSrc->gtType = simdType; @@ -1097,45 +356,41 @@ void Rationalizer::RewriteCopyBlk(GenTreePtr* ppTree, Compiler::fgWalkData* data assert(simdDst != nullptr); assert(simdSrc != nullptr); - GenTree* newTree = nullptr; - GenTree* list = tree->gtGetOp1(); + GenTree* newNode = nullptr; if (oper == GT_STORE_LCL_VAR) { - // get rid of the list node - comp->fgSnipInnerNode(list); - - newTree = simdDst; - newTree->SetOper(oper); - newTree->gtOp.gtOp1 = simdSrc; - newTree->gtType = simdType; - newTree->gtFlags |= (simdSrc->gtFlags & GTF_ALL_EFFECT); - simdSrc->gtNext = newTree; - newTree->gtPrev = simdSrc; + newNode = simdDst; + newNode->SetOper(oper); + + GenTreeLclVar* store = newNode->AsLclVar(); + store->gtOp1 = simdSrc; + store->gtType = simdType; + store->gtFlags |= ((simdSrc->gtFlags & GTF_ALL_EFFECT) | GTF_ASG); + + BlockRange().Remove(simdDst); + BlockRange().InsertAfter(simdSrc, store); } else { assert(oper == GT_STOREIND); - newTree = list; - newTree->SetOper(oper); - newTree->gtType = simdType; - newTree->gtFlags |= (simdSrc->gtFlags & GTF_ALL_EFFECT); - newTree->gtOp.gtOp1 = simdDst; - newTree->gtOp.gtOp2 = simdSrc; - } + newNode = cpBlk->gtGetOp1(); + newNode->SetOper(oper); - assert(newTree != nullptr); - GenTree* nextNode = tree->gtNext; - newTree->gtNext = nextNode; - if (nextNode != nullptr) - { - nextNode->gtPrev = newTree; + GenTreeStoreInd* storeInd = newNode->AsStoreInd(); + storeInd->gtType = simdType; + storeInd->gtFlags |= ((simdSrc->gtFlags & GTF_ALL_EFFECT) | GTF_ASG); + storeInd->gtOp1 = simdDst; + storeInd->gtOp2 = simdSrc; + + BlockRange().InsertBefore(cpBlk, storeInd); } - *ppTree = newTree; + use.ReplaceWith(comp, newNode); + BlockRange().Remove(cpBlk); JITDUMP("After rewriting SIMD CopyBlk:\n"); - DISPTREE(*ppTree); + DISPTREERANGE(BlockRange(), use.Def()); JITDUMP("\n"); #endif // FEATURE_SIMD } @@ -1152,58 +407,37 @@ void Rationalizer::RewriteCopyBlk(GenTreePtr* ppTree, Compiler::fgWalkData* data // TODO-Cleanup: Once SIMD types are plumbed through the frontend, this will no longer // be required. // -void Rationalizer::RewriteObj(GenTreePtr* ppTree, Compiler::fgWalkData* data) +void Rationalizer::RewriteObj(LIR::Use& use) { #ifdef FEATURE_SIMD - Compiler* comp = data->compiler; - GenTreeObj* obj = (*ppTree)->AsObj(); + GenTreeObj* obj = use.Def()->AsObj(); +// For UNIX struct passing, we can have Obj nodes for arguments. +// For other cases, we should never see a non-SIMD type here. #ifdef FEATURE_UNIX_AMD64_STRUCT_PASSING - // For UNIX struct passing, we can have Obj nodes for arguments. - // For other cases, we should never see a non-SIMD type here. - if (!varTypeIsSIMD(obj)) { return; } #endif // FEATURE_UNIX_AMD64_STRUCT_PASSING + // Should come here only if featureSIMD is enabled noway_assert(comp->featureSIMD); - // On we should only call this with a SIMD type. + + // We should only call this with a SIMD type. noway_assert(varTypeIsSIMD(obj)); var_types simdType = obj->TypeGet(); // If the operand of obj is a GT_ADDR(GT_LCL_VAR) and LclVar is known to be a SIMD type, // replace obj by GT_LCL_VAR. GenTree* srcAddr = obj->gtGetOp1(); - if (srcAddr->OperGet() == GT_ADDR && comp->isSIMDTypeLocal(srcAddr->gtGetOp1())) + if (srcAddr->OperIsLocalAddr() && comp->isAddrOfSIMDType(srcAddr)) { - GenTree* src = srcAddr->gtGetOp1(); - comp->fgSnipInnerNode(srcAddr); - // It is possible for the obj to be the last node in the tree, if its result is - // not actually stored anywhere and is not eliminated. - // This can happen with an unused SIMD expression involving a localVar or temporary value, - // where the SIMD expression is returning a non-SIMD value, and the expression is sufficiently - // complex (e.g. a call to vector * scalar which is inlined but not an intrinsic). - // The obj of the localVar is not eliminated, because it involves an indirection, - // and therefore appears potentially unsafe to eliminate. However, when we transform the obj into - // a plain localVar during the Rationalizer, we need to correctly handle the case where it has - // no parent. - // This happens, for example, with this source code: - // Vector4.Dot(default(Vector4) * 2f, Vector4.One); - if (obj->gtNext == nullptr) - { - SplitData* tmpState = (SplitData*)data->pCallbackData; - comp->fgSnipNode(tmpState->root->AsStmt(), obj); - } - else - { - comp->fgSnipInnerNode(obj); - } - comp->fgFixupIfCallArg(data->parentStack, obj, src); - src->gtType = simdType; + BlockRange().Remove(obj); - *ppTree = src; + srcAddr->SetOper(loadForm(srcAddr->OperGet())); + srcAddr->gtType = simdType; + use.ReplaceWith(comp, srcAddr); } else { @@ -1230,7 +464,7 @@ void Rationalizer::RewriteObj(GenTreePtr* ppTree, Compiler::fgWalkData* data) // None. // -void Rationalizer::RewriteNodeAsCall(GenTreePtr* ppTree, +void Rationalizer::RewriteNodeAsCall(GenTree** use, Compiler::fgWalkData* data, CORINFO_METHOD_HANDLE callHnd, #ifdef FEATURE_READYTORUN_COMPILER @@ -1238,7 +472,7 @@ void Rationalizer::RewriteNodeAsCall(GenTreePtr* ppTree, #endif GenTreeArgList* args) { - GenTreePtr tree = *ppTree; + GenTreePtr tree = *use; Compiler* comp = data->compiler; SplitData* tmpState = (SplitData*)data->pCallbackData; GenTreePtr root = tmpState->root; @@ -1256,7 +490,7 @@ void Rationalizer::RewriteNodeAsCall(GenTreePtr* ppTree, #endif // Replace "tree" with "call" - *ppTree = call; + *use = call; // Rebuild the evaluation order. comp->gtSetStmtInfo(root); @@ -1302,8 +536,6 @@ void Rationalizer::RewriteNodeAsCall(GenTreePtr* ppTree, assert(data->parentStack->Top() == tree); (void)data->parentStack->Pop(); data->parentStack->Push(call); - - DBEXEC(TRUE, ValidateStatement(root, tmpState->block)); } // RewriteIntrinsicAsUserCall : Rewrite an intrinsic operator as a GT_CALL to the original method. @@ -1320,356 +552,28 @@ void Rationalizer::RewriteNodeAsCall(GenTreePtr* ppTree, // Conceptually, the lower is the right place to do the rewrite. Keeping it in rationalization is // mainly for throughput issue. -void Rationalizer::RewriteIntrinsicAsUserCall(GenTreePtr* ppTree, Compiler::fgWalkData* data) +void Rationalizer::RewriteIntrinsicAsUserCall(GenTree** use, Compiler::fgWalkData* data) { - GenTreePtr tree = *ppTree; - Compiler* comp = data->compiler; - GenTreeArgList* args; - - assert(tree->OperGet() == GT_INTRINSIC); + GenTreeIntrinsic* intrinsic = (*use)->AsIntrinsic(); + Compiler* comp = data->compiler; - if (tree->gtOp.gtOp2 == nullptr) + GenTreeArgList* args; + if (intrinsic->gtOp.gtOp2 == nullptr) { - args = comp->gtNewArgList(tree->gtOp.gtOp1); + args = comp->gtNewArgList(intrinsic->gtGetOp1()); } else { - args = comp->gtNewArgList(tree->gtOp.gtOp1, tree->gtOp.gtOp2); + args = comp->gtNewArgList(intrinsic->gtGetOp1(), intrinsic->gtGetOp2()); } - RewriteNodeAsCall(ppTree, data, tree->gtIntrinsic.gtMethodHandle, + RewriteNodeAsCall(use, data, intrinsic->gtMethodHandle, #ifdef FEATURE_READYTORUN_COMPILER - tree->gtIntrinsic.gtEntryPoint, + intrinsic->gtEntryPoint, #endif args); } -// tree walker callback function that rewrites ASG and ADDR nodes -Compiler::fgWalkResult Rationalizer::SimpleTransformHelper(GenTree** ppTree, Compiler::fgWalkData* data) -{ - GenTree* tree = *ppTree; - Compiler* comp = data->compiler; - SplitData* tmpState = (SplitData*)data->pCallbackData; - - while (tree->OperGet() == GT_COMMA) - { - RewriteOneComma(ppTree, data); - tree = *ppTree; - } - - if (tree->OperIsAssignment()) - { - GenTree* lhs = tree->gtGetOp1(); - GenTree* dataSrc = tree->gtGetOp2(); - - // the other assign ops should have already been rewritten to ASG - assert(tree->OperGet() == GT_ASG); - - while (lhs->OperGet() == GT_COMMA) - { - RewriteOneComma(&(tree->gtOp.gtOp1), data); - lhs = tree->gtGetOp1(); - } - switch (lhs->OperGet()) - { - case GT_LCL_VAR: - case GT_LCL_FLD: - case GT_REG_VAR: - case GT_PHI_ARG: - MorphAsgIntoStoreLcl(tmpState->root->AsStmt(), tree); - tree->gtFlags &= ~GTF_REVERSE_OPS; - break; - - case GT_IND: - { - GenTreeStoreInd* store = - new (comp, GT_STOREIND) GenTreeStoreInd(lhs->TypeGet(), lhs->gtGetOp1(), dataSrc); - if (tree->IsReverseOp()) - { - store->gtFlags |= GTF_REVERSE_OPS; - } - store->gtFlags |= (lhs->gtFlags & GTF_IND_FLAGS); - store->CopyCosts(tree); - - JITDUMP("Rewriting GT_ASG(GT_IND, X) to GT_STOREIND(X):\n"); - DISPTREE(store); - JITDUMP("\n"); - - // Snip out the old GT_IND node - GenTreePtr indPrev = lhs->gtPrev; - indPrev->gtNext = lhs->gtNext; - indPrev->gtNext->gtPrev = indPrev; - - // Replace "tree" with "store" - *ppTree = store; - store->gtNext = tree->gtNext; - store->gtPrev = tree->gtPrev; - if (store->gtNext != nullptr) - { - store->gtNext->gtPrev = store; - } - assert(store->gtPrev != nullptr); - store->gtPrev->gtNext = store; - - // Since "tree" is replaced with "store", pop "tree" node (i.e the current node) - // and replace it with "store" on parent stack. - assert(data->parentStack->Top() == tree); - (void)data->parentStack->Pop(); - data->parentStack->Push(store); - - JITDUMP("root:\n"); - DISPTREE(tmpState->root); - JITDUMP("\n"); - } - break; - - case GT_CLS_VAR: - { - lhs->gtOper = GT_CLS_VAR_ADDR; - lhs->gtType = TYP_BYREF; - tree->gtOper = GT_STOREIND; - - JITDUMP("Rewriting GT_ASG(GT_CLS_VAR, X) to GT_STOREIND(GT_CLS_VAR_ADDR, X):\n"); - DISPTREE(tree); - JITDUMP("\n"); - } - break; - - default: - assert(!"unhandled op\n"); - break; - } - } - else if (tree->OperGet() == GT_BOX) - { - // GT_BOX at this level just passes through so get rid of it - Compiler::fgSnipNode(tmpState->root->AsStmt(), tree); - *ppTree = tree->gtOp.gtOp1; - comp->fgFixupIfCallArg(data->parentStack, tree, *ppTree); - JITDUMP("Rewriting GT_BOX(X) to X:\n"); - DISPTREE(*ppTree); - JITDUMP("\n"); - return SimpleTransformHelper(ppTree, data); - } - else if (tree->gtOper == GT_ADDR) - { - GenTree* child = tree->gtOp.gtOp1; - if (child->IsLocal()) - { - // We are changing the child from GT_LCL_VAR TO GT_LCL_VAR_ADDR. - // Therefore gtType of the child needs to be changed to a TYP_BYREF - CLANG_FORMAT_COMMENT_ANCHOR; -#ifdef DEBUG - if (child->gtOper == GT_LCL_VAR) - { - JITDUMP("Rewriting GT_ADDR(GT_LCL_VAR) to GT_LCL_VAR_ADDR:\n"); - } - else - { - assert(child->gtOper == GT_LCL_FLD); - JITDUMP("Rewriting GT_ADDR(GT_LCL_FLD) to GT_LCL_FLD_ADDR:\n"); - } -#endif // DEBUG - - Compiler::fgSnipNode(tmpState->root->AsStmt(), tree); - child->gtOper = addrForm(child->gtOper); - child->gtType = TYP_BYREF; - copyFlags(child, tree, GTF_ALL_EFFECT); - *ppTree = child; - } - else if (child->gtOper == GT_CLS_VAR) - { - Compiler::fgSnipNode(tmpState->root->AsStmt(), tree); - child->gtOper = GT_CLS_VAR_ADDR; - child->gtType = TYP_BYREF; - copyFlags(child, tree, GTF_ALL_EFFECT); - *ppTree = child; - - JITDUMP("Rewriting GT_ADDR(GT_CLS_VAR) to GT_CLS_VAR_ADDR:\n"); - } - else if (child->gtOper == GT_IND) - { - Compiler::fgSnipNode(tmpState->root->AsStmt(), tree); - Compiler::fgSnipNode(tmpState->root->AsStmt(), child); - *ppTree = child->gtOp.gtOp1; - JITDUMP("Rewriting GT_ADDR(GT_IND(X)) to X:\n"); - } - comp->fgFixupIfCallArg(data->parentStack, tree, *ppTree); - DISPTREE(*ppTree); - JITDUMP("\n"); - } - else if (tree->gtOper == GT_NOP && tree->gtOp.gtOp1) - { - // fgmorph sometimes inserts NOP nodes between def and use - // supposedly 'to prevent constant folding' - Compiler::fgSnipNode(tmpState->root->AsStmt(), tree); - *ppTree = tree->gtOp.gtOp1; - comp->fgFixupIfCallArg(data->parentStack, tree, *ppTree); - - // Since GT_NOP(op1) is replaced with op1, pop GT_NOP node (i.e the current node) - // and replace it with op1 on parent stack. - (void)data->parentStack->Pop(); - data->parentStack->Push(tree->gtOp.gtOp1); - - JITDUMP("Rewriting GT_NOP(X) to X:\n"); - DISPTREE(*ppTree); - JITDUMP("\n"); - return SimpleTransformHelper(ppTree, data); - } -#ifdef _TARGET_XARCH_ - else if (tree->gtOper == GT_CLS_VAR) - { - // rewrite "clsvar" as [&clsvar] so indirs are explicit - tree->gtOper = GT_CLS_VAR_ADDR; - GenTree* ind = comp->gtNewOperNode(GT_IND, tree->TypeGet(), tree); - tree->gtType = TYP_BYREF; - ind->CopyCosts(tree); - tree->InsertAfterSelf(ind, tmpState->root->AsStmt()); - *ppTree = ind; - comp->fgFixupIfCallArg(data->parentStack, tree, ind); - - JITDUMP("Rewriting GT_CLS_VAR to GT_IND(GT_CLS_VAR_ADDR(GT_CLS_VAR)):\n"); - DISPTREE(tmpState->root); - JITDUMP("\n"); - } -#endif // _TARGET_XARCH_ - else if ((tree->gtOper == GT_INTRINSIC) && - Compiler::IsIntrinsicImplementedByUserCall(tree->gtIntrinsic.gtIntrinsicId)) - { - RewriteIntrinsicAsUserCall(ppTree, data); - } -#ifdef FEATURE_SIMD - else - { - assert(tree->gtOper != GT_INTRINSIC || Compiler::IsTargetIntrinsic(tree->gtIntrinsic.gtIntrinsicId)); - - // Transform the treeNode types for SIMD nodes. - // If we have a SIMD type, set its size in simdSize, and later we will - // set the actual type according to its size (which may be less than a full - // vector register). - unsigned simdSize = 0; - switch (tree->gtOper) - { - default: - // Nothing to do for most nodes. - break; - - case GT_INITBLK: - RewriteInitBlk(ppTree, data); - break; - - case GT_COPYBLK: - RewriteCopyBlk(ppTree, data); - break; - - case GT_OBJ: - RewriteObj(ppTree, data); - break; - - case GT_LCL_FLD: - case GT_STORE_LCL_FLD: - // TODO-1stClassStructs: Eliminate this. - FixupIfSIMDLocal(comp, tree->AsLclVarCommon()); - break; - - case GT_STOREIND: - case GT_IND: - if (tree->gtType == TYP_STRUCT) - { - GenTree* addr = tree->AsIndir()->Addr(); - assert(addr->OperIsLocal() && addr->TypeGet() == TYP_BYREF); - LclVarDsc* varDsc = &(comp->lvaTable[addr->AsLclVarCommon()->gtLclNum]); - assert(varDsc->lvSIMDType); - simdSize = (unsigned int)roundUp(varDsc->lvExactSize, TARGET_POINTER_SIZE); - tree->gtType = comp->getSIMDTypeForSize(simdSize); - } - break; - - case GT_SIMD: - { - noway_assert(comp->featureSIMD); - GenTreeSIMD* simdTree = (*ppTree)->AsSIMD(); - simdSize = simdTree->gtSIMDSize; - var_types simdType = comp->getSIMDTypeForSize(simdSize); - // TODO-1stClassStructs: This should be handled more generally for enregistered or promoted - // structs that are passed or returned in a different register type than their enregistered - // type(s). - if (simdTree->gtType == TYP_I_IMPL && simdTree->gtSIMDSize == TARGET_POINTER_SIZE) - { - // This happens when it is consumed by a GT_RET_EXPR. - // It can only be a Vector2f or Vector2i. - assert(genTypeSize(simdTree->gtSIMDBaseType) == 4); - simdTree->gtType = TYP_SIMD8; - } - else if (simdTree->gtType == TYP_STRUCT || varTypeIsSIMD(simdTree)) - { - tree->gtType = simdType; - } - // Certain SIMD trees require rationalizing. - if (simdTree->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray) - { - // Rewrite this as an explicit load. - JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n"); - unsigned int baseTypeSize = genTypeSize(simdTree->gtSIMDBaseType); - GenTree* address = - new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdTree->gtOp1, simdTree->gtOp2, baseTypeSize, - offsetof(CORINFO_Array, u1Elems)); - GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address); - address->CopyCosts(simdTree); - ind->CopyCosts(simdTree); - - // Fix up the links. - GenTreePtr addressPrev = simdTree->gtPrev; - assert(addressPrev != nullptr); - GenTree* indNext = simdTree->gtNext; - // We don't have any top-level GT_SIMD nodes. - assert(addressPrev != nullptr); - - address->gtPrev = addressPrev; - addressPrev->gtNext = address; - - ind->gtPrev = address; - address->gtNext = ind; - - indNext->gtPrev = ind; - ind->gtNext = indNext; - - // Replace "simdTree" with "ind" - *ppTree = ind; - - DISPTREE(tmpState->root); - JITDUMP("\n"); - } - else - { - // This code depends on the fact that NONE of the SIMD intrinsics take vector operands - // of a different width. If that assumption changes, we will EITHER have to make these type - // transformations during importation, and plumb the types all the way through the JIT, - // OR add a lot of special handling here. - GenTree* op1 = simdTree->gtGetOp1(); - if (op1 != nullptr && op1->gtType == TYP_STRUCT) - { - op1->gtType = simdType; - } - GenTree* op2 = simdTree->gtGetOp2(); - if (op2 != nullptr && op2->gtType == TYP_STRUCT) - { - op2->gtType = simdType; - } - } - } - break; - } - if ((*ppTree) != tree) - { - return SimpleTransformHelper(ppTree, data); - } - } -#endif // FEATURE_SIMD - - return Compiler::WALK_CONTINUE; -} - // FixupIfSIMDLocal: Fixup the type of a lclVar tree, as needed, if it is a SIMD type vector. // // Arguments: @@ -1683,7 +587,7 @@ Compiler::fgWalkResult Rationalizer::SimpleTransformHelper(GenTree** ppTree, Com // desirable to change the lclFld nodes back to TYP_SIMD (it will cause them to be loaded // into a vector register, and then moved to an int register). -void Rationalizer::FixupIfSIMDLocal(Compiler* comp, GenTreeLclVarCommon* tree) +void Rationalizer::FixupIfSIMDLocal(GenTreeLclVarCommon* node) { #ifdef FEATURE_SIMD if (!comp->featureSIMD) @@ -1691,7 +595,7 @@ void Rationalizer::FixupIfSIMDLocal(Compiler* comp, GenTreeLclVarCommon* tree) return; } - LclVarDsc* varDsc = &(comp->lvaTable[tree->gtLclNum]); + LclVarDsc* varDsc = &(comp->lvaTable[node->gtLclNum]); // Don't mark byref of SIMD vector as a SIMD type. // Note that struct args though marked as lvIsSIMD=true, @@ -1701,7 +605,7 @@ void Rationalizer::FixupIfSIMDLocal(Compiler* comp, GenTreeLclVarCommon* tree) { return; } - switch (tree->OperGet()) + switch (node->OperGet()) { default: // Nothing to do for most tree nodes. @@ -1712,11 +616,11 @@ void Rationalizer::FixupIfSIMDLocal(Compiler* comp, GenTreeLclVarCommon* tree) // case we can change it to GT_LCL_VAR. // However, we may also see a lclFld with FieldSeqStore::NotAField() for structs that can't // be analyzed, e.g. those with overlapping fields such as the IL implementation of Vector<T>. - if ((tree->AsLclFld()->gtFieldSeq == FieldSeqStore::NotAField()) && (tree->AsLclFld()->gtLclOffs == 0) && - (tree->gtType == TYP_I_IMPL) && (varDsc->lvExactSize == TARGET_POINTER_SIZE)) + if ((node->AsLclFld()->gtFieldSeq == FieldSeqStore::NotAField()) && (node->AsLclFld()->gtLclOffs == 0) && + (node->gtType == TYP_I_IMPL) && (varDsc->lvExactSize == TARGET_POINTER_SIZE)) { - tree->SetOper(GT_LCL_VAR); - tree->gtFlags &= ~(GTF_VAR_USEASG); + node->SetOper(GT_LCL_VAR); + node->gtFlags &= ~(GTF_VAR_USEASG); } else { @@ -1727,23 +631,18 @@ void Rationalizer::FixupIfSIMDLocal(Compiler* comp, GenTreeLclVarCommon* tree) } break; case GT_STORE_LCL_FLD: - assert(tree->gtType == TYP_I_IMPL); - tree->SetOper(GT_STORE_LCL_VAR); - tree->gtFlags &= ~(GTF_VAR_USEASG); + assert(node->gtType == TYP_I_IMPL); + node->SetOper(GT_STORE_LCL_VAR); + node->gtFlags &= ~(GTF_VAR_USEASG); break; } unsigned simdSize = (unsigned int)roundUp(varDsc->lvExactSize, TARGET_POINTER_SIZE); - tree->gtType = comp->getSIMDTypeForSize(simdSize); + node->gtType = comp->getSIMDTypeForSize(simdSize); #endif // FEATURE_SIMD } #ifdef DEBUG -void Rationalizer::ValidateStatement(Location loc) -{ - ValidateStatement(loc.tree, loc.block); -} - void Rationalizer::ValidateStatement(GenTree* tree, BasicBlock* block) { assert(tree->gtOper == GT_STMT); @@ -1753,6 +652,7 @@ void Rationalizer::ValidateStatement(GenTree* tree, BasicBlock* block) // sanity checks that apply to all kinds of IR void Rationalizer::SanityCheck() { + // TODO: assert(!IsLIR()); BasicBlock* block; foreach_block(comp, block) { @@ -1790,6 +690,448 @@ void Rationalizer::SanityCheckRational() #endif // DEBUG +static void RewriteAssignmentIntoStoreLclCore(GenTreeOp* assignment, + GenTree* location, + GenTree* value, + genTreeOps locationOp) +{ + assert(assignment != nullptr); + assert(assignment->OperGet() == GT_ASG); + assert(location != nullptr); + assert(value != nullptr); + + genTreeOps storeOp = storeForm(locationOp); + +#ifdef DEBUG + JITDUMP("rewriting asg(%s, X) to %s(X)\n", GenTree::NodeName(locationOp), GenTree::NodeName(storeOp)); +#endif // DEBUG + + assignment->SetOper(storeOp); + GenTreeLclVarCommon* store = assignment->AsLclVarCommon(); + + GenTreeLclVarCommon* var = location->AsLclVarCommon(); + store->SetLclNum(var->gtLclNum); + store->SetSsaNum(var->gtSsaNum); + + if (locationOp == GT_LCL_FLD) + { + store->gtLclFld.gtLclOffs = var->gtLclFld.gtLclOffs; + store->gtLclFld.gtFieldSeq = var->gtLclFld.gtFieldSeq; + } + + copyFlags(store, var, GTF_LIVENESS_MASK); + store->gtFlags &= ~GTF_REVERSE_OPS; + + store->gtType = var->TypeGet(); + store->gtOp1 = value; + + DISPNODE(store); + JITDUMP("\n"); +} + +void Rationalizer::RewriteAssignmentIntoStoreLcl(GenTreeOp* assignment) +{ + assert(assignment != nullptr); + assert(assignment->OperGet() == GT_ASG); + + GenTree* location = assignment->gtGetOp1(); + GenTree* value = assignment->gtGetOp2(); + + RewriteAssignmentIntoStoreLclCore(assignment, location, value, location->OperGet()); +} + +void Rationalizer::RewriteAssignment(LIR::Use& use) +{ + assert(use.IsInitialized()); + + GenTreeOp* assignment = use.Def()->AsOp(); + assert(assignment->OperGet() == GT_ASG); + + GenTree* location = assignment->gtGetOp1(); + GenTree* value = assignment->gtGetOp2(); + + genTreeOps locationOp = location->OperGet(); + switch (locationOp) + { + case GT_LCL_VAR: + case GT_LCL_FLD: + case GT_REG_VAR: + case GT_PHI_ARG: + RewriteAssignmentIntoStoreLclCore(assignment, location, value, locationOp); + BlockRange().Remove(location); + break; + + case GT_IND: + { + GenTreeStoreInd* store = + new (comp, GT_STOREIND) GenTreeStoreInd(location->TypeGet(), location->gtGetOp1(), value); + + copyFlags(store, assignment, GTF_ALL_EFFECT); + copyFlags(store, location, GTF_IND_FLAGS); + + if (assignment->IsReverseOp()) + { + store->gtFlags |= GTF_REVERSE_OPS; + } + + store->CopyCosts(assignment); + + // TODO: JIT dump + + // Remove the GT_IND node and replace the assignment node with the store + BlockRange().Remove(location); + BlockRange().InsertBefore(assignment, store); + use.ReplaceWith(comp, store); + BlockRange().Remove(assignment); + } + break; + + case GT_CLS_VAR: + { + location->SetOper(GT_CLS_VAR_ADDR); + location->gtType = TYP_BYREF; + + assignment->SetOper(GT_STOREIND); + + // TODO: JIT dump + } + break; + + default: + unreached(); + break; + } +} + +void Rationalizer::RewriteAddress(LIR::Use& use) +{ + assert(use.IsInitialized()); + + GenTreeUnOp* address = use.Def()->AsUnOp(); + assert(address->OperGet() == GT_ADDR); + + GenTree* location = address->gtGetOp1(); + genTreeOps locationOp = location->OperGet(); + + if (location->IsLocal()) + { +// We are changing the child from GT_LCL_VAR TO GT_LCL_VAR_ADDR. +// Therefore gtType of the child needs to be changed to a TYP_BYREF +#ifdef DEBUG + if (locationOp == GT_LCL_VAR) + { + JITDUMP("Rewriting GT_ADDR(GT_LCL_VAR) to GT_LCL_VAR_ADDR:\n"); + } + else + { + assert(locationOp == GT_LCL_FLD); + JITDUMP("Rewriting GT_ADDR(GT_LCL_FLD) to GT_LCL_FLD_ADDR:\n"); + } +#endif // DEBUG + + location->SetOper(addrForm(locationOp)); + location->gtType = TYP_BYREF; + copyFlags(location, address, GTF_ALL_EFFECT); + + use.ReplaceWith(comp, location); + BlockRange().Remove(address); + } + else if (locationOp == GT_CLS_VAR) + { + location->SetOper(GT_CLS_VAR_ADDR); + location->gtType = TYP_BYREF; + copyFlags(location, address, GTF_ALL_EFFECT); + + use.ReplaceWith(comp, location); + BlockRange().Remove(address); + + JITDUMP("Rewriting GT_ADDR(GT_CLS_VAR) to GT_CLS_VAR_ADDR:\n"); + } + else if (locationOp == GT_IND) + { + use.ReplaceWith(comp, location->gtGetOp1()); + BlockRange().Remove(location); + BlockRange().Remove(address); + + JITDUMP("Rewriting GT_ADDR(GT_IND(X)) to X:\n"); + } + + DISPTREERANGE(BlockRange(), use.Def()); + JITDUMP("\n"); +} + +Compiler::fgWalkResult Rationalizer::RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parentStack) +{ + assert(useEdge != nullptr); + + GenTree* node = *useEdge; + assert(node != nullptr); + +#ifdef DEBUG + const bool isLateArg = (node->gtFlags & GTF_LATE_ARG) != 0; +#endif + + // First, remove any preceeding GT_LIST nodes, which are not otherwise visited by the tree walk. + // + // NOTE: GT_LIST nodes that are used by block ops and phi nodes will in fact be visited. + for (GenTree* prev = node->gtPrev; prev != nullptr && prev->OperGet() == GT_LIST; prev = node->gtPrev) + { + BlockRange().Remove(prev); + } + + // In addition, remove the current node if it is a GT_LIST node. + if ((*useEdge)->OperGet() == GT_LIST) + { + BlockRange().Remove(*useEdge); + return Compiler::WALK_CONTINUE; + } + + LIR::Use use; + if (parentStack.Height() < 2) + { + use = LIR::Use::GetDummyUse(BlockRange(), *useEdge); + } + else + { + use = LIR::Use(BlockRange(), useEdge, parentStack.Index(1)); + } + + assert(node == use.Def()); + switch (node->OperGet()) + { + case GT_ASG: + RewriteAssignment(use); + break; + + case GT_BOX: + // GT_BOX at this level just passes through so get rid of it + use.ReplaceWith(comp, node->gtGetOp1()); + BlockRange().Remove(node); + break; + + case GT_ADDR: + RewriteAddress(use); + break; + + case GT_NOP: + // fgMorph sometimes inserts NOP nodes between defs and uses + // supposedly 'to prevent constant folding'. In this case, remove the + // NOP. + if (node->gtGetOp1() != nullptr) + { + use.ReplaceWith(comp, node->gtGetOp1()); + BlockRange().Remove(node); + } + break; + + case GT_COMMA: + { + GenTree* op1 = node->gtGetOp1(); + if ((op1->gtFlags & GTF_ALL_EFFECT) == 0) + { + // The LHS has no side effects. Remove it. + bool isClosed = false; + unsigned sideEffects = 0; + LIR::ReadOnlyRange lhsRange = BlockRange().GetTreeRange(op1, &isClosed, &sideEffects); + + // None of the transforms performed herein violate tree order, so these + // should always be true. + assert(isClosed); + assert((sideEffects & GTF_ALL_EFFECT) == 0); + + BlockRange().Delete(comp, m_block, std::move(lhsRange)); + } + + GenTree* replacement = node->gtGetOp2(); + if (!use.IsDummyUse()) + { + use.ReplaceWith(comp, replacement); + } + else + { + // This is a top-level comma. If the RHS has no side effects we can remove + // it as well. + if ((replacement->gtFlags & GTF_ALL_EFFECT) == 0) + { + bool isClosed = false; + unsigned sideEffects = 0; + LIR::ReadOnlyRange rhsRange = BlockRange().GetTreeRange(replacement, &isClosed, &sideEffects); + + // None of the transforms performed herein violate tree order, so these + // should always be true. + assert(isClosed); + assert((sideEffects & GTF_ALL_EFFECT) == 0); + + BlockRange().Delete(comp, m_block, std::move(rhsRange)); + } + } + + BlockRange().Remove(node); + } + break; + + case GT_ARGPLACE: + // Remove argplace and list nodes from the execution order. + // + // TODO: remove phi args and phi nodes as well? + BlockRange().Remove(node); + break; + +#ifdef _TARGET_XARCH_ + case GT_CLS_VAR: + { + // Class vars that are the target of an assignment will get rewritten into + // GT_STOREIND(GT_CLS_VAR_ADDR, val) by RewriteAssignment. This check is + // not strictly necessary--the GT_IND(GT_CLS_VAR_ADDR) pattern that would + // otherwise be generated would also be picked up by RewriteAssignment--but + // skipping the rewrite here saves an allocation and a bit of extra work. + const bool isLHSOfAssignment = (use.User()->OperGet() == GT_ASG) && (use.User()->gtGetOp1() == node); + if (!isLHSOfAssignment) + { + GenTree* ind = comp->gtNewOperNode(GT_IND, node->TypeGet(), node); + ind->CopyCosts(node); + + node->SetOper(GT_CLS_VAR_ADDR); + node->gtType = TYP_BYREF; + + BlockRange().InsertAfter(node, ind); + use.ReplaceWith(comp, ind); + + // TODO: JIT dump + } + } + break; +#endif // _TARGET_XARCH_ + + case GT_INTRINSIC: + // Non-target intrinsics should have already been rewritten back into user calls. + assert(Compiler::IsTargetIntrinsic(node->gtIntrinsic.gtIntrinsicId)); + break; + +#ifdef FEATURE_SIMD + case GT_INITBLK: + RewriteInitBlk(use); + break; + + case GT_COPYBLK: + RewriteCopyBlk(use); + break; + + case GT_OBJ: + RewriteObj(use); + break; + + case GT_LCL_FLD: + case GT_STORE_LCL_FLD: + // TODO-1stClassStructs: Eliminate this. + FixupIfSIMDLocal(node->AsLclVarCommon()); + break; + + case GT_STOREIND: + case GT_IND: + if (node->gtType == TYP_STRUCT) + { + GenTree* addr = node->AsIndir()->Addr(); + assert(addr->TypeGet() == TYP_BYREF); + + if (addr->OperIsLocal()) + { + LclVarDsc* varDsc = &(comp->lvaTable[addr->AsLclVarCommon()->gtLclNum]); + assert(varDsc->lvSIMDType); + unsigned simdSize = (unsigned int)roundUp(varDsc->lvExactSize, TARGET_POINTER_SIZE); + node->gtType = comp->getSIMDTypeForSize(simdSize); + } +#if DEBUG + else + { + // If the address is not a local var, assert that the user of this IND is an ADDR node. + assert((use.User()->OperGet() == GT_ADDR) || use.User()->OperIsLocalAddr()); + } +#endif + } + break; + + case GT_SIMD: + { + noway_assert(comp->featureSIMD); + GenTreeSIMD* simdNode = node->AsSIMD(); + unsigned simdSize = simdNode->gtSIMDSize; + var_types simdType = comp->getSIMDTypeForSize(simdSize); + + // TODO-1stClassStructs: This should be handled more generally for enregistered or promoted + // structs that are passed or returned in a different register type than their enregistered + // type(s). + if (simdNode->gtType == TYP_I_IMPL && simdNode->gtSIMDSize == TARGET_POINTER_SIZE) + { + // This happens when it is consumed by a GT_RET_EXPR. + // It can only be a Vector2f or Vector2i. + assert(genTypeSize(simdNode->gtSIMDBaseType) == 4); + simdNode->gtType = TYP_SIMD8; + } + else if (simdNode->gtType == TYP_STRUCT || varTypeIsSIMD(simdNode)) + { + node->gtType = simdType; + } + + // Certain SIMD trees require rationalizing. + if (simdNode->gtSIMD.gtSIMDIntrinsicID == SIMDIntrinsicInitArray) + { + // Rewrite this as an explicit load. + JITDUMP("Rewriting GT_SIMD array init as an explicit load:\n"); + unsigned int baseTypeSize = genTypeSize(simdNode->gtSIMDBaseType); + GenTree* address = new (comp, GT_LEA) GenTreeAddrMode(TYP_BYREF, simdNode->gtOp1, simdNode->gtOp2, + baseTypeSize, offsetof(CORINFO_Array, u1Elems)); + GenTree* ind = comp->gtNewOperNode(GT_IND, simdType, address); + address->CopyCosts(simdNode); + ind->CopyCosts(simdNode); + + BlockRange().InsertBefore(simdNode, address, ind); + use.ReplaceWith(comp, ind); + BlockRange().Remove(simdNode); + + DISPTREERANGE(BlockRange(), use.Def()); + JITDUMP("\n"); + } + else + { + // This code depends on the fact that NONE of the SIMD intrinsics take vector operands + // of a different width. If that assumption changes, we will EITHER have to make these type + // transformations during importation, and plumb the types all the way through the JIT, + // OR add a lot of special handling here. + GenTree* op1 = simdNode->gtGetOp1(); + if (op1 != nullptr && op1->gtType == TYP_STRUCT) + { + op1->gtType = simdType; + } + + GenTree* op2 = simdNode->gtGetOp2(); + if (op2 != nullptr && op2->gtType == TYP_STRUCT) + { + op2->gtType = simdType; + } + } + } + break; +#endif // FEATURE_SIMD + + default: + break; + } + + // Do some extra processing on top-level nodes to remove unused local reads. + if (use.IsDummyUse() && node->OperIsLocalRead()) + { + assert((node->gtFlags & GTF_ALL_EFFECT) == 0); + + comp->lvaDecRefCnts(node); + BlockRange().Remove(node); + } + + assert(isLateArg == ((node->gtFlags & GTF_LATE_ARG) != 0)); + + return Compiler::WALK_CONTINUE; +} + void Rationalizer::DoPhase() { DBEXEC(TRUE, SanityCheck()); @@ -1797,15 +1139,102 @@ void Rationalizer::DoPhase() comp->compCurBB = nullptr; comp->fgOrder = Compiler::FGOrderLinear; - // break up the trees at side effects, etc - Location loc(comp->fgFirstBB); - while (loc.block) + BasicBlock* firstBlock = comp->fgFirstBB; + + for (BasicBlock* block = comp->fgFirstBB; block != nullptr; block = block->bbNext) { - loc = TreeTransformRationalization(loc); - loc = loc.Next(); - } + comp->compCurBB = block; + m_block = block; - DBEXEC(TRUE, SanityCheckRational()); + // Establish the first and last nodes for the block. This is necessary in order for the LIR + // utilities that hang off the BasicBlock type to work correctly. + GenTreeStmt* firstStatement = block->firstStmt(); + if (firstStatement == nullptr) + { + // No statements in this block; skip it. + block->MakeLIR(nullptr, nullptr); + continue; + } + + GenTreeStmt* lastStatement = block->lastStmt(); + + // Rewrite intrinsics that are not supported by the target back into user calls. + // This needs to be done before the transition to LIR because it relies on the use + // of fgMorphArgs, which is designed to operate on HIR. Once this is done for a + // particular statement, link that statement's nodes into the current basic block. + // + // This walk also clears the GTF_VAR_USEDEF bit on locals, which is not necessary + // in the backend. + GenTree* lastNodeInPreviousStatement = nullptr; + for (GenTreeStmt* statement = firstStatement; statement != nullptr; statement = statement->getNextStmt()) + { + assert(statement->gtStmtList != nullptr); + assert(statement->gtStmtList->gtPrev == nullptr); + assert(statement->gtStmtExpr != nullptr); + assert(statement->gtStmtExpr->gtNext == nullptr); + + SplitData splitData; + splitData.root = statement; + splitData.block = block; + splitData.thisPhase = this; + + comp->fgWalkTreePost(&statement->gtStmtExpr, + [](GenTree** use, Compiler::fgWalkData* walkData) -> Compiler::fgWalkResult { + GenTree* node = *use; + if (node->OperGet() == GT_INTRINSIC && + Compiler::IsIntrinsicImplementedByUserCall(node->gtIntrinsic.gtIntrinsicId)) + { + RewriteIntrinsicAsUserCall(use, walkData); + } + else if (node->OperIsLocal()) + { + node->gtFlags &= ~GTF_VAR_USEDEF; + } + + return Compiler::WALK_CONTINUE; + }, + &splitData, true); + + GenTree* firstNodeInStatement = statement->gtStmtList; + if (lastNodeInPreviousStatement != nullptr) + { + lastNodeInPreviousStatement->gtNext = firstNodeInStatement; + } + + firstNodeInStatement->gtPrev = lastNodeInPreviousStatement; + lastNodeInPreviousStatement = statement->gtStmtExpr; + } + + block->MakeLIR(firstStatement->gtStmtList, lastStatement->gtStmtExpr); + + // Rewrite HIR nodes into LIR nodes. + for (GenTreeStmt *statement = firstStatement, *nextStatement; statement != nullptr; statement = nextStatement) + { + nextStatement = statement->getNextStmt(); + + // If this statement has correct offset information, change it into an IL offset + // node and insert it into the LIR. + if (statement->gtStmtILoffsx != BAD_IL_OFFSET) + { + assert(!statement->IsPhiDefnStmt()); + statement->SetOper(GT_IL_OFFSET); + statement->gtNext = nullptr; + statement->gtPrev = nullptr; + + BlockRange().InsertBefore(statement->gtStmtList, statement); + } + + m_statement = statement; + comp->fgWalkTreePost(&statement->gtStmtExpr, + [](GenTree** use, Compiler::fgWalkData* walkData) -> Compiler::fgWalkResult { + return reinterpret_cast<Rationalizer*>(walkData->pCallbackData) + ->RewriteNode(use, *walkData->parentStack); + }, + this, true); + } + + assert(BlockRange().CheckLIR(comp)); + } comp->compRationalIRForm = true; } diff --git a/src/jit/rationalize.h b/src/jit/rationalize.h index fe8118b429..03e6aabeb6 100644 --- a/src/jit/rationalize.h +++ b/src/jit/rationalize.h @@ -5,100 +5,16 @@ //=============================================================================== #include "phase.h" -//------------------------------------------------------------------------------ -// Location - (tree, block) tuple is minimum context required to manipulate trees in the JIT -//------------------------------------------------------------------------------ -class Location +class Rationalizer : public Phase { -public: - GenTree* tree; - BasicBlock* block; - - Location() : tree(nullptr), block(nullptr) - { - } - - Location(GenTree* t, BasicBlock* b) : tree(t), block(b) - { - DBEXEC(TRUE, Validate()); - } - - // construct a location consisting of the first tree after the start of the given block - // (and the corresponding block, which may not be the same as the one passed in) - Location(BasicBlock* b) : tree(nullptr), block(b) - { - Initialize(); - } - -#ifdef DEBUG - // Validate - basic validation that this (tree, block) tuple forms a real location - void Validate() - { - if (tree != nullptr) - { - assert(Compiler::fgBlockContainsStatementBounded(block, tree)); - assert(tree->gtOper == GT_STMT); - } - } -#endif // DEBUG - - // Next - skip to next location, - // which means next tree in block, or next block's first tree, or a null location - Location Next() - { - tree = tree->gtNext; - while (tree == nullptr) - { - block = block->bbNext; - if (block == nullptr) - { - return Location(); - } - tree = block->bbTreeList; - } - assert(tree != nullptr); - assert(tree->gtOper == GT_STMT); - return *this; - } - - void Reset(Compiler* comp) - { - block = comp->fgFirstBB; - tree = nullptr; - Initialize(); - } - private: - void Initialize() - { - assert(tree == nullptr); - tree = block->bbTreeList; - while (tree == nullptr) - { - block = block->bbNext; - if (block == nullptr) - { - block = nullptr; - tree = nullptr; - break; - } - tree = block->bbTreeList; - } - DBEXEC(TRUE, Validate()); - } -}; + BasicBlock* m_block; + GenTreeStmt* m_statement; -class Rationalizer : public Phase -{ - //=============================================================================== - // Methods public: Rationalizer(Compiler* comp); - Location TreeTransformRationalization(Location loc); #ifdef DEBUG - - static void ValidateStatement(Location loc); static void ValidateStatement(GenTree* tree, BasicBlock* block); // general purpose sanity checking of de facto standard GenTree @@ -109,35 +25,23 @@ public: #endif // DEBUG - virtual void DoPhase(); - typedef ArrayStack<GenTree*> GenTreeStack; - static void MorphAsgIntoStoreLcl(GenTreeStmt* stmt, GenTreePtr pTree); - -private: - static Compiler::fgWalkResult CommaHelper(GenTree** ppTree, Compiler::fgWalkData* data); - static void RewriteOneComma(GenTree** ppTree, Compiler::fgWalkData* data); - static bool CommaUselessChild(GenTree** ppTree, Compiler::fgWalkData* data); - static void RecursiveRewriteComma(GenTree** ppTree, Compiler::fgWalkData* data, bool discard, bool nested); - static bool RewriteArrElem(GenTree** ppTree, Compiler::fgWalkData* data); - - static Compiler::fgWalkResult SimpleTransformHelper(GenTree** ppTree, Compiler::fgWalkData* data); - - static void DuplicateCommaProcessOneTree(Compiler* comp, Rationalizer* irt, BasicBlock* block, GenTree* tree); - - static void FixupIfCallArg(GenTreeStack* parentStack, GenTree* oldChild, GenTree* newChild); + virtual void DoPhase() override; - static void FixupIfSIMDLocal(Compiler* comp, GenTreeLclVarCommon* tree); + static void RewriteAssignmentIntoStoreLcl(GenTreeOp* assignment); - static GenTreePtr CreateTempAssignment(Compiler* comp, unsigned lclNum, GenTreePtr rhs); - - Location RewriteTopLevelComma(Location loc); +private: + inline LIR::Range& BlockRange() const + { + return LIR::AsRange(m_block); + } // SIMD related transformations - static void RewriteObj(GenTreePtr* ppTree, Compiler::fgWalkData* data); - static void RewriteCopyBlk(GenTreePtr* ppTree, Compiler::fgWalkData* data); - static void RewriteInitBlk(GenTreePtr* ppTree, Compiler::fgWalkData* data); + void RewriteInitBlk(LIR::Use& use); + void RewriteCopyBlk(LIR::Use& use); + void RewriteObj(LIR::Use& use); + void FixupIfSIMDLocal(GenTreeLclVarCommon* node); - // Intrinsic related + // Intrinsic related transformations static void RewriteNodeAsCall(GenTreePtr* ppTree, Compiler::fgWalkData* data, CORINFO_METHOD_HANDLE callHnd, @@ -145,7 +49,15 @@ private: CORINFO_CONST_LOOKUP entryPoint, #endif GenTreeArgList* args); + static void RewriteIntrinsicAsUserCall(GenTreePtr* ppTree, Compiler::fgWalkData* data); + + // Other transformations + void RewriteAssignment(LIR::Use& use); + void RewriteAddress(LIR::Use& use); + + // Root visitor + Compiler::fgWalkResult RewriteNode(GenTree** useEdge, ArrayStack<GenTree*>& parents); }; inline Rationalizer::Rationalizer(Compiler* _comp) : Phase(_comp, "IR Rationalize", PHASE_RATIONALIZE) diff --git a/src/jit/smallhash.h b/src/jit/smallhash.h index 7a3b6d32e9..71ea4a6269 100644 --- a/src/jit/smallhash.h +++ b/src/jit/smallhash.h @@ -135,13 +135,15 @@ private: { // The home bucket is empty; use it. // - // Note that the next offset does not need to be updated: whether or not it is non-zero, - // it is already correct, since we're inserting at the head of the list. - home->m_isFull = true; - home->m_firstOffset = 0; - home->m_hash = hash; - home->m_key = key; - home->m_value = value; + // Note that `m_firstOffset` does not need to be updated: whether or not it is non-zero, + // it is already correct, since we're inserting at the head of the list. `m_nextOffset` + // must be 0, however, since this node should not be part of a list. + assert(home->m_nextOffset == 0); + + home->m_isFull = true; + home->m_hash = hash; + home->m_key = key; + home->m_value = value; return true; } @@ -172,6 +174,8 @@ private: } unsigned offset = (bucketIndex - precedingIndexInChain) & mask; + assert(offset != 0); + if (precedingIndexInChain == homeIndex) { buckets[precedingIndexInChain].m_firstOffset = offset; @@ -473,8 +477,7 @@ public: return false; } - Bucket* bucket = &m_buckets[bucketIndex]; - bucket->m_isFull = false; + Bucket* bucket = &m_buckets[bucketIndex]; if (precedingIndexInChain != bucketIndex) { @@ -502,6 +505,9 @@ public: } } + bucket->m_isFull = false; + bucket->m_nextOffset = 0; + m_numFullBuckets--; *value = bucket->m_value; |