summaryrefslogtreecommitdiff
path: root/src/jit/inlinepolicy.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/jit/inlinepolicy.cpp')
-rw-r--r--src/jit/inlinepolicy.cpp2857
1 files changed, 2857 insertions, 0 deletions
diff --git a/src/jit/inlinepolicy.cpp b/src/jit/inlinepolicy.cpp
new file mode 100644
index 0000000000..f80f3a5ec0
--- /dev/null
+++ b/src/jit/inlinepolicy.cpp
@@ -0,0 +1,2857 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#include "jitpch.h"
+#ifdef _MSC_VER
+#pragma hdrstop
+#endif
+
+#include "inlinepolicy.h"
+#include "sm.h"
+
+//------------------------------------------------------------------------
+// getPolicy: Factory method for getting an InlinePolicy
+//
+// Arguments:
+// compiler - the compiler instance that will evaluate inlines
+// isPrejitRoot - true if this policy is evaluating a prejit root
+//
+// Return Value:
+// InlinePolicy to use in evaluating an inline.
+//
+// Notes:
+// Determines which of the various policies should apply,
+// and creates (or reuses) a policy instance to use.
+
+InlinePolicy* InlinePolicy::GetPolicy(Compiler* compiler, bool isPrejitRoot)
+{
+
+#ifdef DEBUG
+
+ // Optionally install the RandomPolicy.
+ bool useRandomPolicy = compiler->compRandomInlineStress();
+
+ if (useRandomPolicy)
+ {
+ unsigned seed = getJitStressLevel();
+ assert(seed != 0);
+ return new (compiler, CMK_Inlining) RandomPolicy(compiler, isPrejitRoot, seed);
+ }
+
+#endif // DEBUG
+
+#if defined(DEBUG) || defined(INLINE_DATA)
+
+ // Optionally install the ReplayPolicy.
+ bool useReplayPolicy = JitConfig.JitInlinePolicyReplay() != 0;
+
+ if (useReplayPolicy)
+ {
+ return new (compiler, CMK_Inlining) ReplayPolicy(compiler, isPrejitRoot);
+ }
+
+ // Optionally install the SizePolicy.
+ bool useSizePolicy = JitConfig.JitInlinePolicySize() != 0;
+
+ if (useSizePolicy)
+ {
+ return new (compiler, CMK_Inlining) SizePolicy(compiler, isPrejitRoot);
+ }
+
+ // Optionally install the FullPolicy.
+ bool useFullPolicy = JitConfig.JitInlinePolicyFull() != 0;
+
+ if (useFullPolicy)
+ {
+ return new (compiler, CMK_Inlining) FullPolicy(compiler, isPrejitRoot);
+ }
+
+ // Optionally install the DiscretionaryPolicy.
+ bool useDiscretionaryPolicy = JitConfig.JitInlinePolicyDiscretionary() != 0;
+
+ if (useDiscretionaryPolicy)
+ {
+ return new (compiler, CMK_Inlining) DiscretionaryPolicy(compiler, isPrejitRoot);
+ }
+
+#endif // defined(DEBUG) || defined(INLINE_DATA)
+
+ // Optionally install the ModelPolicy.
+ bool useModelPolicy = JitConfig.JitInlinePolicyModel() != 0;
+
+ if (useModelPolicy)
+ {
+ return new (compiler, CMK_Inlining) ModelPolicy(compiler, isPrejitRoot);
+ }
+
+ // Optionally fallback to the original legacy policy
+ bool useLegacyPolicy = JitConfig.JitInlinePolicyLegacy() != 0;
+
+ if (useLegacyPolicy)
+ {
+ return new (compiler, CMK_Inlining) LegacyPolicy(compiler, isPrejitRoot);
+ }
+
+ // Use the enhanced legacy policy by default
+ return new (compiler, CMK_Inlining) EnhancedLegacyPolicy(compiler, isPrejitRoot);
+}
+
+//------------------------------------------------------------------------
+// NoteFatal: handle an observation with fatal impact
+//
+// Arguments:
+// obs - the current obsevation
+
+void LegalPolicy::NoteFatal(InlineObservation obs)
+{
+ // As a safeguard, all fatal impact must be
+ // reported via noteFatal.
+ assert(InlGetImpact(obs) == InlineImpact::FATAL);
+ NoteInternal(obs);
+ assert(InlDecisionIsFailure(m_Decision));
+}
+
+//------------------------------------------------------------------------
+// NoteInternal: helper for handling an observation
+//
+// Arguments:
+// obs - the current obsevation
+
+void LegalPolicy::NoteInternal(InlineObservation obs)
+{
+ // Note any INFORMATION that reaches here will now cause failure.
+ // Non-fatal INFORMATION observations must be handled higher up.
+ InlineTarget target = InlGetTarget(obs);
+
+ if (target == InlineTarget::CALLEE)
+ {
+ this->SetNever(obs);
+ }
+ else
+ {
+ this->SetFailure(obs);
+ }
+}
+
+//------------------------------------------------------------------------
+// SetFailure: helper for setting a failing decision
+//
+// Arguments:
+// obs - the current obsevation
+
+void LegalPolicy::SetFailure(InlineObservation obs)
+{
+ // Expect a valid observation
+ assert(InlIsValidObservation(obs));
+
+ switch (m_Decision)
+ {
+ case InlineDecision::FAILURE:
+ // Repeated failure only ok if evaluating a prejit root
+ // (since we can't fail fast because we're not inlining)
+ // or if inlining and the observation is CALLSITE_TOO_MANY_LOCALS
+ // (since we can't fail fast from lvaGrabTemp).
+ assert(m_IsPrejitRoot || (obs == InlineObservation::CALLSITE_TOO_MANY_LOCALS));
+ break;
+ case InlineDecision::UNDECIDED:
+ case InlineDecision::CANDIDATE:
+ m_Decision = InlineDecision::FAILURE;
+ m_Observation = obs;
+ break;
+ default:
+ // SUCCESS, NEVER, or ??
+ assert(!"Unexpected m_Decision");
+ unreached();
+ }
+}
+
+//------------------------------------------------------------------------
+// SetNever: helper for setting a never decision
+//
+// Arguments:
+// obs - the current obsevation
+
+void LegalPolicy::SetNever(InlineObservation obs)
+{
+ // Expect a valid observation
+ assert(InlIsValidObservation(obs));
+
+ switch (m_Decision)
+ {
+ case InlineDecision::NEVER:
+ // Repeated never only ok if evaluating a prejit root
+ assert(m_IsPrejitRoot);
+ break;
+ case InlineDecision::UNDECIDED:
+ case InlineDecision::CANDIDATE:
+ m_Decision = InlineDecision::NEVER;
+ m_Observation = obs;
+ break;
+ default:
+ // SUCCESS, FAILURE or ??
+ assert(!"Unexpected m_Decision");
+ unreached();
+ }
+}
+
+//------------------------------------------------------------------------
+// SetCandidate: helper updating candidacy
+//
+// Arguments:
+// obs - the current obsevation
+//
+// Note:
+// Candidate observations are handled here. If the inline has already
+// failed, they're ignored. If there's already a candidate reason,
+// this new reason trumps it.
+
+void LegalPolicy::SetCandidate(InlineObservation obs)
+{
+ // Ignore if this inline is going to fail.
+ if (InlDecisionIsFailure(m_Decision))
+ {
+ return;
+ }
+
+ // We should not have declared success yet.
+ assert(!InlDecisionIsSuccess(m_Decision));
+
+ // Update, overriding any previous candidacy.
+ m_Decision = InlineDecision::CANDIDATE;
+ m_Observation = obs;
+}
+
+//------------------------------------------------------------------------
+// NoteSuccess: handle finishing all the inlining checks successfully
+
+void LegacyPolicy::NoteSuccess()
+{
+ assert(InlDecisionIsCandidate(m_Decision));
+ m_Decision = InlineDecision::SUCCESS;
+}
+
+//------------------------------------------------------------------------
+// NoteBool: handle a boolean observation with non-fatal impact
+//
+// Arguments:
+// obs - the current obsevation
+// value - the value of the observation
+void LegacyPolicy::NoteBool(InlineObservation obs, bool value)
+{
+ // Check the impact
+ InlineImpact impact = InlGetImpact(obs);
+
+ // As a safeguard, all fatal impact must be
+ // reported via noteFatal.
+ assert(impact != InlineImpact::FATAL);
+
+ // Handle most information here
+ bool isInformation = (impact == InlineImpact::INFORMATION);
+ bool propagate = !isInformation;
+
+ if (isInformation)
+ {
+ switch (obs)
+ {
+ case InlineObservation::CALLEE_IS_FORCE_INLINE:
+ // We may make the force-inline observation more than
+ // once. All observations should agree.
+ assert(!m_IsForceInlineKnown || (m_IsForceInline == value));
+ m_IsForceInline = value;
+ m_IsForceInlineKnown = true;
+ break;
+
+ case InlineObservation::CALLEE_IS_INSTANCE_CTOR:
+ m_IsInstanceCtor = value;
+ break;
+
+ case InlineObservation::CALLEE_CLASS_PROMOTABLE:
+ m_IsFromPromotableValueClass = value;
+ break;
+
+ case InlineObservation::CALLEE_HAS_SIMD:
+ m_HasSimd = value;
+ break;
+
+ case InlineObservation::CALLEE_LOOKS_LIKE_WRAPPER:
+ // LegacyPolicy ignores this for prejit roots.
+ if (!m_IsPrejitRoot)
+ {
+ m_LooksLikeWrapperMethod = value;
+ }
+ break;
+
+ case InlineObservation::CALLEE_ARG_FEEDS_CONSTANT_TEST:
+ // LegacyPolicy ignores this for prejit roots.
+ if (!m_IsPrejitRoot)
+ {
+ m_ArgFeedsConstantTest++;
+ }
+ break;
+
+ case InlineObservation::CALLEE_ARG_FEEDS_RANGE_CHECK:
+ // LegacyPolicy ignores this for prejit roots.
+ if (!m_IsPrejitRoot)
+ {
+ m_ArgFeedsRangeCheck++;
+ }
+ break;
+
+ case InlineObservation::CALLEE_HAS_SWITCH:
+ case InlineObservation::CALLEE_UNSUPPORTED_OPCODE:
+ // LegacyPolicy ignores these for prejit roots.
+ if (!m_IsPrejitRoot)
+ {
+ // Pass these on, they should cause inlining to fail.
+ propagate = true;
+ }
+ break;
+
+ case InlineObservation::CALLSITE_CONSTANT_ARG_FEEDS_TEST:
+ // We shouldn't see this for a prejit root since
+ // we don't know anything about callers.
+ assert(!m_IsPrejitRoot);
+ m_ConstantArgFeedsConstantTest++;
+ break;
+
+ case InlineObservation::CALLEE_BEGIN_OPCODE_SCAN:
+ {
+ // Set up the state machine, if this inline is
+ // discretionary and is still a candidate.
+ if (InlDecisionIsCandidate(m_Decision) &&
+ (m_Observation == InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE))
+ {
+ // Better not have a state machine already.
+ assert(m_StateMachine == nullptr);
+ m_StateMachine = new (m_RootCompiler, CMK_Inlining) CodeSeqSM;
+ m_StateMachine->Start(m_RootCompiler);
+ }
+ break;
+ }
+
+ case InlineObservation::CALLEE_END_OPCODE_SCAN:
+ {
+ if (m_StateMachine != nullptr)
+ {
+ m_StateMachine->End();
+ }
+
+ // If this function is mostly loads and stores, we
+ // should try harder to inline it. You can't just use
+ // the percentage test because if the method has 8
+ // instructions and 6 are loads, it's only 75% loads.
+ // This allows for CALL, RET, and one more non-ld/st
+ // instruction.
+ if (((m_InstructionCount - m_LoadStoreCount) < 4) ||
+ (((double)m_LoadStoreCount / (double)m_InstructionCount) > .90))
+ {
+ m_MethodIsMostlyLoadStore = true;
+ }
+
+ // Budget check.
+ //
+ // Conceptually this should happen when we
+ // observe the candidate's IL size.
+ //
+ // However, we do this here to avoid potential
+ // inconsistency between the state of the budget
+ // during candidate scan and the state when the IL is
+ // being scanned.
+ //
+ // Consider the case where we're just below the budget
+ // during candidate scan, and we have three possible
+ // inlines, any two of which put us over budget. We
+ // allow them all to become candidates. We then move
+ // on to inlining and the first two get inlined and
+ // put us over budget. Now the third can't be inlined
+ // anymore, but we have a policy that when we replay
+ // the candidate IL size during the inlining pass it
+ // "reestablishes" candidacy rather than alters
+ // candidacy ... so instead we bail out here.
+
+ if (!m_IsPrejitRoot)
+ {
+ InlineStrategy* strategy = m_RootCompiler->m_inlineStrategy;
+ bool overBudget = strategy->BudgetCheck(m_CodeSize);
+ if (overBudget)
+ {
+ SetFailure(InlineObservation::CALLSITE_OVER_BUDGET);
+ }
+ }
+
+ break;
+ }
+
+ default:
+ // Ignore the remainder for now
+ break;
+ }
+ }
+
+ if (propagate)
+ {
+ NoteInternal(obs);
+ }
+}
+
+//------------------------------------------------------------------------
+// NoteInt: handle an observed integer value
+//
+// Arguments:
+// obs - the current obsevation
+// value - the value being observed
+
+void LegacyPolicy::NoteInt(InlineObservation obs, int value)
+{
+ switch (obs)
+ {
+ case InlineObservation::CALLEE_MAXSTACK:
+ {
+ assert(m_IsForceInlineKnown);
+
+ unsigned calleeMaxStack = static_cast<unsigned>(value);
+
+ if (!m_IsForceInline && (calleeMaxStack > SMALL_STACK_SIZE))
+ {
+ SetNever(InlineObservation::CALLEE_MAXSTACK_TOO_BIG);
+ }
+
+ break;
+ }
+
+ case InlineObservation::CALLEE_NUMBER_OF_BASIC_BLOCKS:
+ {
+ assert(m_IsForceInlineKnown);
+ assert(value != 0);
+
+ unsigned basicBlockCount = static_cast<unsigned>(value);
+
+ if (!m_IsForceInline && (basicBlockCount > MAX_BASIC_BLOCKS))
+ {
+ SetNever(InlineObservation::CALLEE_TOO_MANY_BASIC_BLOCKS);
+ }
+
+ break;
+ }
+
+ case InlineObservation::CALLEE_IL_CODE_SIZE:
+ {
+ assert(m_IsForceInlineKnown);
+ assert(value != 0);
+ m_CodeSize = static_cast<unsigned>(value);
+
+ // Now that we know size and forceinline state,
+ // update candidacy.
+ if (m_CodeSize <= InlineStrategy::ALWAYS_INLINE_SIZE)
+ {
+ // Candidate based on small size
+ SetCandidate(InlineObservation::CALLEE_BELOW_ALWAYS_INLINE_SIZE);
+ }
+ else if (m_IsForceInline)
+ {
+ // Candidate based on force inline
+ SetCandidate(InlineObservation::CALLEE_IS_FORCE_INLINE);
+ }
+ else if (m_CodeSize <= m_RootCompiler->m_inlineStrategy->GetMaxInlineILSize())
+ {
+ // Candidate, pending profitability evaluation
+ SetCandidate(InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE);
+ }
+ else
+ {
+ // Callee too big, not a candidate
+ SetNever(InlineObservation::CALLEE_TOO_MUCH_IL);
+ }
+
+ break;
+ }
+
+ case InlineObservation::CALLSITE_DEPTH:
+ {
+ unsigned depth = static_cast<unsigned>(value);
+
+ if (depth > m_RootCompiler->m_inlineStrategy->GetMaxInlineDepth())
+ {
+ SetFailure(InlineObservation::CALLSITE_IS_TOO_DEEP);
+ }
+
+ break;
+ }
+
+ case InlineObservation::CALLEE_OPCODE_NORMED:
+ case InlineObservation::CALLEE_OPCODE:
+ {
+ m_InstructionCount++;
+ OPCODE opcode = static_cast<OPCODE>(value);
+
+ if (m_StateMachine != nullptr)
+ {
+ SM_OPCODE smOpcode = CodeSeqSM::MapToSMOpcode(opcode);
+ noway_assert(smOpcode < SM_COUNT);
+ noway_assert(smOpcode != SM_PREFIX_N);
+ if (obs == InlineObservation::CALLEE_OPCODE_NORMED)
+ {
+ if (smOpcode == SM_LDARGA_S)
+ {
+ smOpcode = SM_LDARGA_S_NORMED;
+ }
+ else if (smOpcode == SM_LDLOCA_S)
+ {
+ smOpcode = SM_LDLOCA_S_NORMED;
+ }
+ }
+
+ m_StateMachine->Run(smOpcode DEBUGARG(0));
+ }
+
+ // Look for opcodes that imply loads and stores.
+ // Logic here is as it is to match legacy behavior.
+ if ((opcode >= CEE_LDARG_0 && opcode <= CEE_STLOC_S) || (opcode >= CEE_LDARG && opcode <= CEE_STLOC) ||
+ (opcode >= CEE_LDNULL && opcode <= CEE_LDC_R8) || (opcode >= CEE_LDIND_I1 && opcode <= CEE_STIND_R8) ||
+ (opcode >= CEE_LDFLD && opcode <= CEE_STOBJ) || (opcode >= CEE_LDELEMA && opcode <= CEE_STELEM) ||
+ (opcode == CEE_POP))
+ {
+ m_LoadStoreCount++;
+ }
+
+ break;
+ }
+
+ case InlineObservation::CALLSITE_FREQUENCY:
+ assert(m_CallsiteFrequency == InlineCallsiteFrequency::UNUSED);
+ m_CallsiteFrequency = static_cast<InlineCallsiteFrequency>(value);
+ assert(m_CallsiteFrequency != InlineCallsiteFrequency::UNUSED);
+ break;
+
+ default:
+ // Ignore all other information
+ break;
+ }
+}
+
+//------------------------------------------------------------------------
+// DetermineMultiplier: determine benefit multiplier for this inline
+//
+// Notes: uses the accumulated set of observations to compute a
+// profitability boost for the inline candidate.
+
+double LegacyPolicy::DetermineMultiplier()
+{
+ double multiplier = 0;
+
+ // Bump up the multiplier for instance constructors
+
+ if (m_IsInstanceCtor)
+ {
+ multiplier += 1.5;
+ JITDUMP("\nmultiplier in instance constructors increased to %g.", multiplier);
+ }
+
+ // Bump up the multiplier for methods in promotable struct
+
+ if (m_IsFromPromotableValueClass)
+ {
+ multiplier += 3;
+ JITDUMP("\nmultiplier in methods of promotable struct increased to %g.", multiplier);
+ }
+
+#ifdef FEATURE_SIMD
+
+ if (m_HasSimd)
+ {
+ multiplier += JitConfig.JitInlineSIMDMultiplier();
+ JITDUMP("\nInline candidate has SIMD type args, locals or return value. Multiplier increased to %g.",
+ multiplier);
+ }
+
+#endif // FEATURE_SIMD
+
+ if (m_LooksLikeWrapperMethod)
+ {
+ multiplier += 1.0;
+ JITDUMP("\nInline candidate looks like a wrapper method. Multiplier increased to %g.", multiplier);
+ }
+
+ if (m_ArgFeedsConstantTest > 0)
+ {
+ multiplier += 1.0;
+ JITDUMP("\nInline candidate has an arg that feeds a constant test. Multiplier increased to %g.", multiplier);
+ }
+
+ if (m_MethodIsMostlyLoadStore)
+ {
+ multiplier += 3.0;
+ JITDUMP("\nInline candidate is mostly loads and stores. Multiplier increased to %g.", multiplier);
+ }
+
+ if (m_ArgFeedsRangeCheck > 0)
+ {
+ multiplier += 0.5;
+ JITDUMP("\nInline candidate has arg that feeds range check. Multiplier increased to %g.", multiplier);
+ }
+
+ if (m_ConstantArgFeedsConstantTest > 0)
+ {
+ multiplier += 3.0;
+ JITDUMP("\nInline candidate has const arg that feeds a conditional. Multiplier increased to %g.", multiplier);
+ }
+
+ switch (m_CallsiteFrequency)
+ {
+ case InlineCallsiteFrequency::RARE:
+ // Note this one is not additive, it uses '=' instead of '+='
+ multiplier = 1.3;
+ JITDUMP("\nInline candidate callsite is rare. Multiplier limited to %g.", multiplier);
+ break;
+ case InlineCallsiteFrequency::BORING:
+ multiplier += 1.3;
+ JITDUMP("\nInline candidate callsite is boring. Multiplier increased to %g.", multiplier);
+ break;
+ case InlineCallsiteFrequency::WARM:
+ multiplier += 2.0;
+ JITDUMP("\nInline candidate callsite is warm. Multiplier increased to %g.", multiplier);
+ break;
+ case InlineCallsiteFrequency::LOOP:
+ multiplier += 3.0;
+ JITDUMP("\nInline candidate callsite is in a loop. Multiplier increased to %g.", multiplier);
+ break;
+ case InlineCallsiteFrequency::HOT:
+ multiplier += 3.0;
+ JITDUMP("\nInline candidate callsite is hot. Multiplier increased to %g.", multiplier);
+ break;
+ default:
+ assert(!"Unexpected callsite frequency");
+ break;
+ }
+
+#ifdef DEBUG
+
+ int additionalMultiplier = JitConfig.JitInlineAdditionalMultiplier();
+
+ if (additionalMultiplier != 0)
+ {
+ multiplier += additionalMultiplier;
+ JITDUMP("\nmultiplier increased via JitInlineAdditonalMultiplier=%d to %g.", additionalMultiplier, multiplier);
+ }
+
+ if (m_RootCompiler->compInlineStress())
+ {
+ multiplier += 10;
+ JITDUMP("\nmultiplier increased via inline stress to %g.", multiplier);
+ }
+
+#endif // DEBUG
+
+ return multiplier;
+}
+
+//------------------------------------------------------------------------
+// DetermineNativeSizeEstimate: return estimated native code size for
+// this inline candidate.
+//
+// Notes:
+// This is an estimate for the size of the inlined callee.
+// It does not include size impact on the caller side.
+//
+// Uses the results of a state machine model for discretionary
+// candidates. Should not be needed for forced or always
+// candidates.
+
+int LegacyPolicy::DetermineNativeSizeEstimate()
+{
+ // Should be a discretionary candidate.
+ assert(m_StateMachine != nullptr);
+
+ return m_StateMachine->NativeSize;
+}
+
+//------------------------------------------------------------------------
+// DetermineCallsiteNativeSizeEstimate: estimate native size for the
+// callsite.
+//
+// Arguments:
+// methInfo -- method info for the callee
+//
+// Notes:
+// Estimates the native size (in bytes, scaled up by 10x) for the
+// call site. While the quality of the estimate here is questionable
+// (especially for x64) it is being left as is for legacy compatibility.
+
+int LegacyPolicy::DetermineCallsiteNativeSizeEstimate(CORINFO_METHOD_INFO* methInfo)
+{
+ int callsiteSize = 55; // Direct call take 5 native bytes; indirect call takes 6 native bytes.
+
+ bool hasThis = methInfo->args.hasThis();
+
+ if (hasThis)
+ {
+ callsiteSize += 30; // "mov" or "lea"
+ }
+
+ CORINFO_ARG_LIST_HANDLE argLst = methInfo->args.args;
+ COMP_HANDLE comp = m_RootCompiler->info.compCompHnd;
+
+ for (unsigned i = (hasThis ? 1 : 0); i < methInfo->args.totalILArgs(); i++, argLst = comp->getArgNext(argLst))
+ {
+ var_types sigType = (var_types)m_RootCompiler->eeGetArgType(argLst, &methInfo->args);
+
+ if (sigType == TYP_STRUCT)
+ {
+ typeInfo verType = m_RootCompiler->verParseArgSigToTypeInfo(&methInfo->args, argLst);
+
+ /*
+
+ IN0028: 00009B lea EAX, bword ptr [EBP-14H]
+ IN0029: 00009E push dword ptr [EAX+4]
+ IN002a: 0000A1 push gword ptr [EAX]
+ IN002b: 0000A3 call [MyStruct.staticGetX2(struct):int]
+
+ */
+
+ callsiteSize += 10; // "lea EAX, bword ptr [EBP-14H]"
+
+ // NB sizeof (void*) fails to convey intent when cross-jitting.
+
+ unsigned opsz = (unsigned)(roundUp(comp->getClassSize(verType.GetClassHandle()), sizeof(void*)));
+ unsigned slots = opsz / sizeof(void*);
+
+ callsiteSize += slots * 20; // "push gword ptr [EAX+offs] "
+ }
+ else
+ {
+ callsiteSize += 30; // push by average takes 3 bytes.
+ }
+ }
+
+ return callsiteSize;
+}
+
+//------------------------------------------------------------------------
+// DetermineProfitability: determine if this inline is profitable
+//
+// Arguments:
+// methodInfo -- method info for the callee
+//
+// Notes:
+// A profitable inline is one that is projected to have a beneficial
+// size/speed tradeoff.
+//
+// It is expected that this method is only invoked for discretionary
+// candidates, since it does not make sense to do this assessment for
+// failed, always, or forced inlines.
+
+void LegacyPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
+{
+
+#if defined(DEBUG)
+
+ // Punt if we're inlining and we've reached the acceptance limit.
+ int limit = JitConfig.JitInlineLimit();
+ unsigned current = m_RootCompiler->m_inlineStrategy->GetInlineCount();
+
+ if (!m_IsPrejitRoot && (limit >= 0) && (current >= static_cast<unsigned>(limit)))
+ {
+ SetFailure(InlineObservation::CALLSITE_OVER_INLINE_LIMIT);
+ return;
+ }
+
+#endif // defined(DEBUG)
+
+ assert(InlDecisionIsCandidate(m_Decision));
+ assert(m_Observation == InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE);
+
+ m_CalleeNativeSizeEstimate = DetermineNativeSizeEstimate();
+ m_CallsiteNativeSizeEstimate = DetermineCallsiteNativeSizeEstimate(methodInfo);
+ m_Multiplier = DetermineMultiplier();
+ const int threshold = (int)(m_CallsiteNativeSizeEstimate * m_Multiplier);
+
+ // Note the LegacyPolicy estimates are scaled up by SIZE_SCALE
+ JITDUMP("\ncalleeNativeSizeEstimate=%d\n", m_CalleeNativeSizeEstimate)
+ JITDUMP("callsiteNativeSizeEstimate=%d\n", m_CallsiteNativeSizeEstimate);
+ JITDUMP("benefit multiplier=%g\n", m_Multiplier);
+ JITDUMP("threshold=%d\n", threshold);
+
+ // Reject if callee size is over the threshold
+ if (m_CalleeNativeSizeEstimate > threshold)
+ {
+ // Inline appears to be unprofitable
+ JITLOG_THIS(m_RootCompiler,
+ (LL_INFO100000, "Native estimate for function size exceeds threshold"
+ " for inlining %g > %g (multiplier = %g)\n",
+ (double)m_CalleeNativeSizeEstimate / SIZE_SCALE, (double)threshold / SIZE_SCALE, m_Multiplier));
+
+ // Fail the inline
+ if (m_IsPrejitRoot)
+ {
+ SetNever(InlineObservation::CALLEE_NOT_PROFITABLE_INLINE);
+ }
+ else
+ {
+ SetFailure(InlineObservation::CALLSITE_NOT_PROFITABLE_INLINE);
+ }
+ }
+ else
+ {
+ // Inline appears to be profitable
+ JITLOG_THIS(m_RootCompiler,
+ (LL_INFO100000, "Native estimate for function size is within threshold"
+ " for inlining %g <= %g (multiplier = %g)\n",
+ (double)m_CalleeNativeSizeEstimate / SIZE_SCALE, (double)threshold / SIZE_SCALE, m_Multiplier));
+
+ // Update candidacy
+ if (m_IsPrejitRoot)
+ {
+ SetCandidate(InlineObservation::CALLEE_IS_PROFITABLE_INLINE);
+ }
+ else
+ {
+ SetCandidate(InlineObservation::CALLSITE_IS_PROFITABLE_INLINE);
+ }
+ }
+}
+
+//------------------------------------------------------------------------
+// CodeSizeEstimate: estimated code size impact of the inline
+//
+// Return Value:
+// Estimated code size impact, in bytes * 10
+//
+// Notes:
+// Only meaningful for discretionary inlines (whether successful or
+// not). For always or force inlines the legacy policy doesn't
+// estimate size impact.
+
+int LegacyPolicy::CodeSizeEstimate()
+{
+ if (m_StateMachine != nullptr)
+ {
+ // This is not something the LegacyPolicy explicitly computed,
+ // since it uses a blended evaluation model (mixing size and time
+ // together for overall profitability). But it's effecitvely an
+ // estimate of the size impact.
+ return (m_CalleeNativeSizeEstimate - m_CallsiteNativeSizeEstimate);
+ }
+ else
+ {
+ return 0;
+ }
+}
+
+//------------------------------------------------------------------------
+// NoteBool: handle a boolean observation with non-fatal impact
+//
+// Arguments:
+// obs - the current obsevation
+// value - the value of the observation
+
+void EnhancedLegacyPolicy::NoteBool(InlineObservation obs, bool value)
+{
+ switch (obs)
+ {
+ case InlineObservation::CALLEE_DOES_NOT_RETURN:
+ m_IsNoReturn = value;
+ m_IsNoReturnKnown = true;
+ break;
+
+ default:
+ // Pass all other information to the legacy policy
+ LegacyPolicy::NoteBool(obs, value);
+ break;
+ }
+}
+
+//------------------------------------------------------------------------
+// NoteInt: handle an observed integer value
+//
+// Arguments:
+// obs - the current obsevation
+// value - the value being observed
+
+void EnhancedLegacyPolicy::NoteInt(InlineObservation obs, int value)
+{
+ switch (obs)
+ {
+ case InlineObservation::CALLEE_NUMBER_OF_BASIC_BLOCKS:
+ {
+ assert(value != 0);
+ assert(m_IsNoReturnKnown);
+
+ //
+ // Let's be conservative for now and reject inlining of "no return" methods only
+ // if the callee contains a single basic block. This covers most of the use cases
+ // (typical throw helpers simply do "throw new X();" and so they have a single block)
+ // without affecting more exotic cases (loops that do actual work for example) where
+ // failure to inline could negatively impact code quality.
+ //
+
+ unsigned basicBlockCount = static_cast<unsigned>(value);
+
+ if (m_IsNoReturn && (basicBlockCount == 1))
+ {
+ SetNever(InlineObservation::CALLEE_DOES_NOT_RETURN);
+ }
+ else
+ {
+ LegacyPolicy::NoteInt(obs, value);
+ }
+
+ break;
+ }
+
+ default:
+ // Pass all other information to the legacy policy
+ LegacyPolicy::NoteInt(obs, value);
+ break;
+ }
+}
+
+//------------------------------------------------------------------------
+// PropagateNeverToRuntime: determine if a never result should cause the
+// method to be marked as un-inlinable.
+
+bool EnhancedLegacyPolicy::PropagateNeverToRuntime() const
+{
+ //
+ // Do not propagate the "no return" observation. If we do this then future inlining
+ // attempts will fail immediately without marking the call node as "no return".
+ // This can have an adverse impact on caller's code quality as it may have to preserve
+ // registers across the call.
+ // TODO-Throughput: We should persist the "no return" information in the runtime
+ // so we don't need to re-analyze the inlinee all the time.
+ //
+
+ bool propagate = (m_Observation != InlineObservation::CALLEE_DOES_NOT_RETURN);
+
+ propagate &= LegacyPolicy::PropagateNeverToRuntime();
+
+ return propagate;
+}
+
+#ifdef DEBUG
+
+//------------------------------------------------------------------------
+// RandomPolicy: construct a new RandomPolicy
+//
+// Arguments:
+// compiler -- compiler instance doing the inlining (root compiler)
+// isPrejitRoot -- true if this compiler is prejitting the root method
+// seed -- seed value for the random number generator
+
+RandomPolicy::RandomPolicy(Compiler* compiler, bool isPrejitRoot, unsigned seed)
+ : LegalPolicy(isPrejitRoot)
+ , m_RootCompiler(compiler)
+ , m_Random(nullptr)
+ , m_CodeSize(0)
+ , m_IsForceInline(false)
+ , m_IsForceInlineKnown(false)
+{
+ // If necessary, setup and seed the random state.
+ if (compiler->inlRNG == nullptr)
+ {
+ compiler->inlRNG = new (compiler, CMK_Inlining) CLRRandom();
+
+ unsigned hash = m_RootCompiler->info.compMethodHash();
+ assert(hash != 0);
+ assert(seed != 0);
+ int hashSeed = static_cast<int>(hash ^ seed);
+ compiler->inlRNG->Init(hashSeed);
+ }
+
+ m_Random = compiler->inlRNG;
+}
+
+//------------------------------------------------------------------------
+// NoteSuccess: handle finishing all the inlining checks successfully
+
+void RandomPolicy::NoteSuccess()
+{
+ assert(InlDecisionIsCandidate(m_Decision));
+ m_Decision = InlineDecision::SUCCESS;
+}
+
+//------------------------------------------------------------------------
+// NoteBool: handle a boolean observation with non-fatal impact
+//
+// Arguments:
+// obs - the current obsevation
+// value - the value of the observation
+void RandomPolicy::NoteBool(InlineObservation obs, bool value)
+{
+ // Check the impact
+ InlineImpact impact = InlGetImpact(obs);
+
+ // As a safeguard, all fatal impact must be
+ // reported via noteFatal.
+ assert(impact != InlineImpact::FATAL);
+
+ // Handle most information here
+ bool isInformation = (impact == InlineImpact::INFORMATION);
+ bool propagate = !isInformation;
+
+ if (isInformation)
+ {
+ switch (obs)
+ {
+ case InlineObservation::CALLEE_IS_FORCE_INLINE:
+ // The RandomPolicy still honors force inlines.
+ //
+ // We may make the force-inline observation more than
+ // once. All observations should agree.
+ assert(!m_IsForceInlineKnown || (m_IsForceInline == value));
+ m_IsForceInline = value;
+ m_IsForceInlineKnown = true;
+ break;
+
+ case InlineObservation::CALLEE_HAS_SWITCH:
+ case InlineObservation::CALLEE_UNSUPPORTED_OPCODE:
+ // Pass these on, they should cause inlining to fail.
+ propagate = true;
+ break;
+
+ default:
+ // Ignore the remainder for now
+ break;
+ }
+ }
+
+ if (propagate)
+ {
+ NoteInternal(obs);
+ }
+}
+
+//------------------------------------------------------------------------
+// NoteInt: handle an observed integer value
+//
+// Arguments:
+// obs - the current obsevation
+// value - the value being observed
+
+void RandomPolicy::NoteInt(InlineObservation obs, int value)
+{
+ switch (obs)
+ {
+
+ case InlineObservation::CALLEE_IL_CODE_SIZE:
+ {
+ assert(m_IsForceInlineKnown);
+ assert(value != 0);
+ m_CodeSize = static_cast<unsigned>(value);
+
+ if (m_IsForceInline)
+ {
+ // Candidate based on force inline
+ SetCandidate(InlineObservation::CALLEE_IS_FORCE_INLINE);
+ }
+ else
+ {
+ // Candidate, pending profitability evaluation
+ SetCandidate(InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE);
+ }
+
+ break;
+ }
+
+ default:
+ // Ignore all other information
+ break;
+ }
+}
+
+//------------------------------------------------------------------------
+// DetermineProfitability: determine if this inline is profitable
+//
+// Arguments:
+// methodInfo -- method info for the callee
+//
+// Notes:
+// The random policy makes random decisions about profitablity.
+// Generally we aspire to inline differently, not necessarily to
+// inline more.
+
+void RandomPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
+{
+ assert(InlDecisionIsCandidate(m_Decision));
+ assert(m_Observation == InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE);
+
+ // Budget check.
+ if (!m_IsPrejitRoot)
+ {
+ InlineStrategy* strategy = m_RootCompiler->m_inlineStrategy;
+ bool overBudget = strategy->BudgetCheck(m_CodeSize);
+ if (overBudget)
+ {
+ SetFailure(InlineObservation::CALLSITE_OVER_BUDGET);
+ return;
+ }
+ }
+
+ // Use a probability curve that roughly matches the observed
+ // behavior of the LegacyPolicy. That way we're inlining
+ // differently but not creating enormous methods.
+ //
+ // We vary a bit at the extremes. The RandomPolicy won't always
+ // inline the small methods (<= 16 IL bytes) and won't always
+ // reject the large methods (> 100 IL bytes).
+
+ unsigned threshold = 0;
+
+ if (m_CodeSize <= 16)
+ {
+ threshold = 75;
+ }
+ else if (m_CodeSize <= 30)
+ {
+ threshold = 50;
+ }
+ else if (m_CodeSize <= 40)
+ {
+ threshold = 40;
+ }
+ else if (m_CodeSize <= 50)
+ {
+ threshold = 30;
+ }
+ else if (m_CodeSize <= 75)
+ {
+ threshold = 20;
+ }
+ else if (m_CodeSize <= 100)
+ {
+ threshold = 10;
+ }
+ else if (m_CodeSize <= 200)
+ {
+ threshold = 5;
+ }
+ else
+ {
+ threshold = 1;
+ }
+
+ unsigned randomValue = m_Random->Next(1, 100);
+
+ // Reject if callee size is over the threshold
+ if (randomValue > threshold)
+ {
+ // Inline appears to be unprofitable
+ JITLOG_THIS(m_RootCompiler, (LL_INFO100000, "Random rejection (r=%d > t=%d)\n", randomValue, threshold));
+
+ // Fail the inline
+ if (m_IsPrejitRoot)
+ {
+ SetNever(InlineObservation::CALLEE_RANDOM_REJECT);
+ }
+ else
+ {
+ SetFailure(InlineObservation::CALLSITE_RANDOM_REJECT);
+ }
+ }
+ else
+ {
+ // Inline appears to be profitable
+ JITLOG_THIS(m_RootCompiler, (LL_INFO100000, "Random acceptance (r=%d <= t=%d)\n", randomValue, threshold));
+
+ // Update candidacy
+ if (m_IsPrejitRoot)
+ {
+ SetCandidate(InlineObservation::CALLEE_RANDOM_ACCEPT);
+ }
+ else
+ {
+ SetCandidate(InlineObservation::CALLSITE_RANDOM_ACCEPT);
+ }
+ }
+}
+
+#endif // DEBUG
+
+#ifdef _MSC_VER
+// Disable warning about new array member initialization behavior
+#pragma warning(disable : 4351)
+#endif
+
+//------------------------------------------------------------------------
+// DiscretionaryPolicy: construct a new DiscretionaryPolicy
+//
+// Arguments:
+// compiler -- compiler instance doing the inlining (root compiler)
+// isPrejitRoot -- true if this compiler is prejitting the root method
+
+// clang-format off
+DiscretionaryPolicy::DiscretionaryPolicy(Compiler* compiler, bool isPrejitRoot)
+ : LegacyPolicy(compiler, isPrejitRoot)
+ , m_Depth(0)
+ , m_BlockCount(0)
+ , m_Maxstack(0)
+ , m_ArgCount(0)
+ , m_ArgType()
+ , m_ArgSize()
+ , m_LocalCount(0)
+ , m_ReturnType(CORINFO_TYPE_UNDEF)
+ , m_ReturnSize(0)
+ , m_ArgAccessCount(0)
+ , m_LocalAccessCount(0)
+ , m_IntConstantCount(0)
+ , m_FloatConstantCount(0)
+ , m_IntLoadCount(0)
+ , m_FloatLoadCount(0)
+ , m_IntStoreCount(0)
+ , m_FloatStoreCount(0)
+ , m_SimpleMathCount(0)
+ , m_ComplexMathCount(0)
+ , m_OverflowMathCount(0)
+ , m_IntArrayLoadCount(0)
+ , m_FloatArrayLoadCount(0)
+ , m_RefArrayLoadCount(0)
+ , m_StructArrayLoadCount(0)
+ , m_IntArrayStoreCount(0)
+ , m_FloatArrayStoreCount(0)
+ , m_RefArrayStoreCount(0)
+ , m_StructArrayStoreCount(0)
+ , m_StructOperationCount(0)
+ , m_ObjectModelCount(0)
+ , m_FieldLoadCount(0)
+ , m_FieldStoreCount(0)
+ , m_StaticFieldLoadCount(0)
+ , m_StaticFieldStoreCount(0)
+ , m_LoadAddressCount(0)
+ , m_ThrowCount(0)
+ , m_ReturnCount(0)
+ , m_CallCount(0)
+ , m_CallSiteWeight(0)
+ , m_ModelCodeSizeEstimate(0)
+ , m_PerCallInstructionEstimate(0)
+ , m_IsClassCtor(false)
+ , m_IsSameThis(false)
+ , m_CallerHasNewArray(false)
+ , m_CallerHasNewObj(false)
+{
+ // Empty
+}
+// clang-format on
+
+//------------------------------------------------------------------------
+// NoteBool: handle an observed boolean value
+//
+// Arguments:
+// obs - the current obsevation
+// value - the value being observed
+
+void DiscretionaryPolicy::NoteBool(InlineObservation obs, bool value)
+{
+ switch (obs)
+ {
+ case InlineObservation::CALLEE_LOOKS_LIKE_WRAPPER:
+ m_LooksLikeWrapperMethod = value;
+ break;
+
+ case InlineObservation::CALLEE_ARG_FEEDS_CONSTANT_TEST:
+ assert(value);
+ m_ArgFeedsConstantTest++;
+ break;
+
+ case InlineObservation::CALLEE_ARG_FEEDS_RANGE_CHECK:
+ assert(value);
+ m_ArgFeedsRangeCheck++;
+ break;
+
+ case InlineObservation::CALLSITE_CONSTANT_ARG_FEEDS_TEST:
+ assert(value);
+ m_ConstantArgFeedsConstantTest++;
+ break;
+
+ case InlineObservation::CALLEE_IS_CLASS_CTOR:
+ m_IsClassCtor = value;
+ break;
+
+ case InlineObservation::CALLSITE_IS_SAME_THIS:
+ m_IsSameThis = value;
+ break;
+
+ case InlineObservation::CALLER_HAS_NEWARRAY:
+ m_CallerHasNewArray = value;
+ break;
+
+ case InlineObservation::CALLER_HAS_NEWOBJ:
+ m_CallerHasNewObj = value;
+ break;
+
+ default:
+ LegacyPolicy::NoteBool(obs, value);
+ break;
+ }
+}
+
+//------------------------------------------------------------------------
+// NoteInt: handle an observed integer value
+//
+// Arguments:
+// obs - the current obsevation
+// value - the value being observed
+
+void DiscretionaryPolicy::NoteInt(InlineObservation obs, int value)
+{
+ switch (obs)
+ {
+
+ case InlineObservation::CALLEE_IL_CODE_SIZE:
+ // Override how code size is handled
+ {
+ assert(m_IsForceInlineKnown);
+ assert(value != 0);
+ m_CodeSize = static_cast<unsigned>(value);
+
+ if (m_IsForceInline)
+ {
+ // Candidate based on force inline
+ SetCandidate(InlineObservation::CALLEE_IS_FORCE_INLINE);
+ }
+ else
+ {
+ // Candidate, pending profitability evaluation
+ SetCandidate(InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE);
+ }
+
+ break;
+ }
+
+ case InlineObservation::CALLEE_OPCODE:
+ {
+ // This tries to do a rough binning of opcodes based
+ // on similarity of impact on codegen.
+ OPCODE opcode = static_cast<OPCODE>(value);
+ ComputeOpcodeBin(opcode);
+ LegacyPolicy::NoteInt(obs, value);
+ break;
+ }
+
+ case InlineObservation::CALLEE_MAXSTACK:
+ m_Maxstack = value;
+ break;
+
+ case InlineObservation::CALLEE_NUMBER_OF_BASIC_BLOCKS:
+ m_BlockCount = value;
+ break;
+
+ case InlineObservation::CALLSITE_DEPTH:
+ m_Depth = value;
+ break;
+
+ case InlineObservation::CALLSITE_WEIGHT:
+ m_CallSiteWeight = static_cast<unsigned>(value);
+ break;
+
+ default:
+ // Delegate remainder to the LegacyPolicy.
+ LegacyPolicy::NoteInt(obs, value);
+ break;
+ }
+}
+
+//------------------------------------------------------------------------
+// ComputeOpcodeBin: simple histogramming of opcodes based on presumably
+// similar codegen impact.
+//
+// Arguments:
+// opcode - an MSIL opcode from the callee
+
+void DiscretionaryPolicy::ComputeOpcodeBin(OPCODE opcode)
+{
+ switch (opcode)
+ {
+ case CEE_LDARG_0:
+ case CEE_LDARG_1:
+ case CEE_LDARG_2:
+ case CEE_LDARG_3:
+ case CEE_LDARG_S:
+ case CEE_LDARG:
+ case CEE_STARG_S:
+ case CEE_STARG:
+ m_ArgAccessCount++;
+ break;
+
+ case CEE_LDLOC_0:
+ case CEE_LDLOC_1:
+ case CEE_LDLOC_2:
+ case CEE_LDLOC_3:
+ case CEE_LDLOC_S:
+ case CEE_STLOC_0:
+ case CEE_STLOC_1:
+ case CEE_STLOC_2:
+ case CEE_STLOC_3:
+ case CEE_STLOC_S:
+ case CEE_LDLOC:
+ case CEE_STLOC:
+ m_LocalAccessCount++;
+ break;
+
+ case CEE_LDNULL:
+ case CEE_LDC_I4_M1:
+ case CEE_LDC_I4_0:
+ case CEE_LDC_I4_1:
+ case CEE_LDC_I4_2:
+ case CEE_LDC_I4_3:
+ case CEE_LDC_I4_4:
+ case CEE_LDC_I4_5:
+ case CEE_LDC_I4_6:
+ case CEE_LDC_I4_7:
+ case CEE_LDC_I4_8:
+ case CEE_LDC_I4_S:
+ m_IntConstantCount++;
+ break;
+
+ case CEE_LDC_R4:
+ case CEE_LDC_R8:
+ m_FloatConstantCount++;
+ break;
+
+ case CEE_LDIND_I1:
+ case CEE_LDIND_U1:
+ case CEE_LDIND_I2:
+ case CEE_LDIND_U2:
+ case CEE_LDIND_I4:
+ case CEE_LDIND_U4:
+ case CEE_LDIND_I8:
+ case CEE_LDIND_I:
+ m_IntLoadCount++;
+ break;
+
+ case CEE_LDIND_R4:
+ case CEE_LDIND_R8:
+ m_FloatLoadCount++;
+ break;
+
+ case CEE_STIND_I1:
+ case CEE_STIND_I2:
+ case CEE_STIND_I4:
+ case CEE_STIND_I8:
+ case CEE_STIND_I:
+ m_IntStoreCount++;
+ break;
+
+ case CEE_STIND_R4:
+ case CEE_STIND_R8:
+ m_FloatStoreCount++;
+ break;
+
+ case CEE_SUB:
+ case CEE_AND:
+ case CEE_OR:
+ case CEE_XOR:
+ case CEE_SHL:
+ case CEE_SHR:
+ case CEE_SHR_UN:
+ case CEE_NEG:
+ case CEE_NOT:
+ case CEE_CONV_I1:
+ case CEE_CONV_I2:
+ case CEE_CONV_I4:
+ case CEE_CONV_I8:
+ case CEE_CONV_U4:
+ case CEE_CONV_U8:
+ case CEE_CONV_U2:
+ case CEE_CONV_U1:
+ case CEE_CONV_I:
+ case CEE_CONV_U:
+ m_SimpleMathCount++;
+ break;
+
+ case CEE_MUL:
+ case CEE_DIV:
+ case CEE_DIV_UN:
+ case CEE_REM:
+ case CEE_REM_UN:
+ case CEE_CONV_R4:
+ case CEE_CONV_R8:
+ case CEE_CONV_R_UN:
+ m_ComplexMathCount++;
+ break;
+
+ case CEE_CONV_OVF_I1_UN:
+ case CEE_CONV_OVF_I2_UN:
+ case CEE_CONV_OVF_I4_UN:
+ case CEE_CONV_OVF_I8_UN:
+ case CEE_CONV_OVF_U1_UN:
+ case CEE_CONV_OVF_U2_UN:
+ case CEE_CONV_OVF_U4_UN:
+ case CEE_CONV_OVF_U8_UN:
+ case CEE_CONV_OVF_I_UN:
+ case CEE_CONV_OVF_U_UN:
+ case CEE_CONV_OVF_I1:
+ case CEE_CONV_OVF_U1:
+ case CEE_CONV_OVF_I2:
+ case CEE_CONV_OVF_U2:
+ case CEE_CONV_OVF_I4:
+ case CEE_CONV_OVF_U4:
+ case CEE_CONV_OVF_I8:
+ case CEE_CONV_OVF_U8:
+ case CEE_ADD_OVF:
+ case CEE_ADD_OVF_UN:
+ case CEE_MUL_OVF:
+ case CEE_MUL_OVF_UN:
+ case CEE_SUB_OVF:
+ case CEE_SUB_OVF_UN:
+ case CEE_CKFINITE:
+ m_OverflowMathCount++;
+ break;
+
+ case CEE_LDELEM_I1:
+ case CEE_LDELEM_U1:
+ case CEE_LDELEM_I2:
+ case CEE_LDELEM_U2:
+ case CEE_LDELEM_I4:
+ case CEE_LDELEM_U4:
+ case CEE_LDELEM_I8:
+ case CEE_LDELEM_I:
+ m_IntArrayLoadCount++;
+ break;
+
+ case CEE_LDELEM_R4:
+ case CEE_LDELEM_R8:
+ m_FloatArrayLoadCount++;
+ break;
+
+ case CEE_LDELEM_REF:
+ m_RefArrayLoadCount++;
+ break;
+
+ case CEE_LDELEM:
+ m_StructArrayLoadCount++;
+ break;
+
+ case CEE_STELEM_I:
+ case CEE_STELEM_I1:
+ case CEE_STELEM_I2:
+ case CEE_STELEM_I4:
+ case CEE_STELEM_I8:
+ m_IntArrayStoreCount++;
+ break;
+
+ case CEE_STELEM_R4:
+ case CEE_STELEM_R8:
+ m_FloatArrayStoreCount++;
+ break;
+
+ case CEE_STELEM_REF:
+ m_RefArrayStoreCount++;
+ break;
+
+ case CEE_STELEM:
+ m_StructArrayStoreCount++;
+ break;
+
+ case CEE_CPOBJ:
+ case CEE_LDOBJ:
+ case CEE_CPBLK:
+ case CEE_INITBLK:
+ case CEE_STOBJ:
+ m_StructOperationCount++;
+ break;
+
+ case CEE_CASTCLASS:
+ case CEE_ISINST:
+ case CEE_UNBOX:
+ case CEE_BOX:
+ case CEE_UNBOX_ANY:
+ case CEE_LDFTN:
+ case CEE_LDVIRTFTN:
+ case CEE_SIZEOF:
+ m_ObjectModelCount++;
+ break;
+
+ case CEE_LDFLD:
+ case CEE_LDLEN:
+ case CEE_REFANYTYPE:
+ case CEE_REFANYVAL:
+ m_FieldLoadCount++;
+ break;
+
+ case CEE_STFLD:
+ m_FieldStoreCount++;
+ break;
+
+ case CEE_LDSFLD:
+ m_StaticFieldLoadCount++;
+ break;
+
+ case CEE_STSFLD:
+ m_StaticFieldStoreCount++;
+ break;
+
+ case CEE_LDELEMA:
+ case CEE_LDSFLDA:
+ case CEE_LDFLDA:
+ case CEE_LDSTR:
+ case CEE_LDARGA:
+ case CEE_LDLOCA:
+ m_LoadAddressCount++;
+ break;
+
+ case CEE_CALL:
+ case CEE_CALLI:
+ case CEE_CALLVIRT:
+ case CEE_NEWOBJ:
+ case CEE_NEWARR:
+ case CEE_JMP:
+ m_CallCount++;
+ break;
+
+ case CEE_THROW:
+ case CEE_RETHROW:
+ m_ThrowCount++;
+ break;
+
+ case CEE_RET:
+ m_ReturnCount++;
+
+ default:
+ break;
+ }
+}
+
+//------------------------------------------------------------------------
+// PropagateNeverToRuntime: determine if a never result should cause the
+// method to be marked as un-inlinable.
+
+bool DiscretionaryPolicy::PropagateNeverToRuntime() const
+{
+ // Propagate most failures, but don't propagate when the inline
+ // was viable but unprofitable.
+ bool propagate = (m_Observation != InlineObservation::CALLEE_NOT_PROFITABLE_INLINE);
+
+ return propagate;
+}
+
+//------------------------------------------------------------------------
+// DetermineProfitability: determine if this inline is profitable
+//
+// Arguments:
+// methodInfo -- method info for the callee
+
+void DiscretionaryPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
+{
+
+#if defined(DEBUG)
+
+ // Punt if we're inlining and we've reached the acceptance limit.
+ int limit = JitConfig.JitInlineLimit();
+ unsigned current = m_RootCompiler->m_inlineStrategy->GetInlineCount();
+
+ if (!m_IsPrejitRoot && (limit >= 0) && (current >= static_cast<unsigned>(limit)))
+ {
+ SetFailure(InlineObservation::CALLSITE_OVER_INLINE_LIMIT);
+ return;
+ }
+
+#endif // defined(DEBUG)
+
+ // Make additional observations based on the method info
+ MethodInfoObservations(methodInfo);
+
+ // Estimate the code size impact. This is just for model
+ // evaluation purposes -- we'll still use the legacy policy's
+ // model for actual inlining.
+ EstimateCodeSize();
+
+ // Estimate peformance impact. This is just for model
+ // evaluation purposes -- we'll still use the legacy policy's
+ // model for actual inlining.
+ EstimatePerformanceImpact();
+
+ // Delegate to LegacyPolicy for the rest
+ LegacyPolicy::DetermineProfitability(methodInfo);
+}
+
+//------------------------------------------------------------------------
+// MethodInfoObservations: make observations based on information from
+// the method info for the callee.
+//
+// Arguments:
+// methodInfo -- method info for the callee
+
+void DiscretionaryPolicy::MethodInfoObservations(CORINFO_METHOD_INFO* methodInfo)
+{
+ CORINFO_SIG_INFO& locals = methodInfo->locals;
+ m_LocalCount = locals.numArgs;
+
+ CORINFO_SIG_INFO& args = methodInfo->args;
+ const unsigned argCount = args.numArgs;
+ m_ArgCount = argCount;
+
+ const unsigned pointerSize = sizeof(void*);
+ unsigned i = 0;
+
+ // Implicit arguments
+
+ const bool hasThis = args.hasThis();
+
+ if (hasThis)
+ {
+ m_ArgType[i] = CORINFO_TYPE_CLASS;
+ m_ArgSize[i] = pointerSize;
+ i++;
+ m_ArgCount++;
+ }
+
+ const bool hasTypeArg = args.hasTypeArg();
+
+ if (hasTypeArg)
+ {
+ m_ArgType[i] = CORINFO_TYPE_NATIVEINT;
+ m_ArgSize[i] = pointerSize;
+ i++;
+ m_ArgCount++;
+ }
+
+ // Explicit arguments
+
+ unsigned j = 0;
+ CORINFO_ARG_LIST_HANDLE argListHandle = args.args;
+ COMP_HANDLE comp = m_RootCompiler->info.compCompHnd;
+
+ while ((i < MAX_ARGS) && (j < argCount))
+ {
+ CORINFO_CLASS_HANDLE classHandle;
+ CorInfoType type = strip(comp->getArgType(&args, argListHandle, &classHandle));
+
+ m_ArgType[i] = type;
+
+ if (type == CORINFO_TYPE_VALUECLASS)
+ {
+ assert(classHandle != nullptr);
+ m_ArgSize[i] = roundUp(comp->getClassSize(classHandle), pointerSize);
+ }
+ else
+ {
+ m_ArgSize[i] = pointerSize;
+ }
+
+ argListHandle = comp->getArgNext(argListHandle);
+ i++;
+ j++;
+ }
+
+ while (i < MAX_ARGS)
+ {
+ m_ArgType[i] = CORINFO_TYPE_UNDEF;
+ m_ArgSize[i] = 0;
+ i++;
+ }
+
+ // Return Type
+
+ m_ReturnType = args.retType;
+
+ if (m_ReturnType == CORINFO_TYPE_VALUECLASS)
+ {
+ assert(args.retTypeClass != nullptr);
+ m_ReturnSize = roundUp(comp->getClassSize(args.retTypeClass), pointerSize);
+ }
+ else if (m_ReturnType == CORINFO_TYPE_VOID)
+ {
+ m_ReturnSize = 0;
+ }
+ else
+ {
+ m_ReturnSize = pointerSize;
+ }
+}
+
+//------------------------------------------------------------------------
+// EstimateCodeSize: produce (various) code size estimates based on
+// observations.
+//
+// The "Baseline" code size model used by the legacy policy is
+// effectively
+//
+// 0.100 * m_CalleeNativeSizeEstimate +
+// -0.100 * m_CallsiteNativeSizeEstimate
+//
+// On the inlines in CoreCLR's mscorlib, release windows x64, this
+// yields scores of R=0.42, MSE=228, and MAE=7.25.
+//
+// This estimate can be improved slighly by refitting, resulting in
+//
+// -1.451 +
+// 0.095 * m_CalleeNativeSizeEstimate +
+// -0.104 * m_CallsiteNativeSizeEstimate
+//
+// With R=0.44, MSE=220, and MAE=6.93.
+
+void DiscretionaryPolicy::EstimateCodeSize()
+{
+ // Ensure we have this available.
+ m_CalleeNativeSizeEstimate = DetermineNativeSizeEstimate();
+
+ // Size estimate based on GLMNET model.
+ // R=0.55, MSE=177, MAE=6.59
+ //
+ // Suspect it doesn't handle factors properly...
+ // clang-format off
+ double sizeEstimate =
+ -13.532 +
+ 0.359 * (int) m_CallsiteFrequency +
+ -0.015 * m_ArgCount +
+ -1.553 * m_ArgSize[5] +
+ 2.326 * m_LocalCount +
+ 0.287 * m_ReturnSize +
+ 0.561 * m_IntConstantCount +
+ 1.932 * m_FloatConstantCount +
+ -0.822 * m_SimpleMathCount +
+ -7.591 * m_IntArrayLoadCount +
+ 4.784 * m_RefArrayLoadCount +
+ 12.778 * m_StructArrayLoadCount +
+ 1.452 * m_FieldLoadCount +
+ 8.811 * m_StaticFieldLoadCount +
+ 2.752 * m_StaticFieldStoreCount +
+ -6.566 * m_ThrowCount +
+ 6.021 * m_CallCount +
+ -0.238 * m_IsInstanceCtor +
+ -5.357 * m_IsFromPromotableValueClass +
+ -7.901 * (m_ConstantArgFeedsConstantTest > 0 ? 1 : 0) +
+ 0.065 * m_CalleeNativeSizeEstimate;
+ // clang-format on
+
+ // Scaled up and reported as an integer value.
+ m_ModelCodeSizeEstimate = (int)(SIZE_SCALE * sizeEstimate);
+}
+
+//------------------------------------------------------------------------
+// EstimatePeformanceImpact: produce performance estimates based on
+// observations.
+//
+// Notes:
+// Attempts to predict the per-call savings in instructions executed.
+//
+// A negative value indicates the doing the inline will save instructions
+// and likely time.
+
+void DiscretionaryPolicy::EstimatePerformanceImpact()
+{
+ // Performance estimate based on GLMNET model.
+ // R=0.24, RMSE=16.1, MAE=8.9.
+ // clang-format off
+ double perCallSavingsEstimate =
+ -7.35
+ + (m_CallsiteFrequency == InlineCallsiteFrequency::BORING ? 0.76 : 0)
+ + (m_CallsiteFrequency == InlineCallsiteFrequency::LOOP ? -2.02 : 0)
+ + (m_ArgType[0] == CORINFO_TYPE_CLASS ? 3.51 : 0)
+ + (m_ArgType[3] == CORINFO_TYPE_BOOL ? 20.7 : 0)
+ + (m_ArgType[4] == CORINFO_TYPE_CLASS ? 0.38 : 0)
+ + (m_ReturnType == CORINFO_TYPE_CLASS ? 2.32 : 0);
+ // clang-format on
+
+ // Scaled up and reported as an integer value.
+ m_PerCallInstructionEstimate = (int)(SIZE_SCALE * perCallSavingsEstimate);
+}
+
+//------------------------------------------------------------------------
+// CodeSizeEstimate: estimated code size impact of the inline
+//
+// Return Value:
+// Estimated code size impact, in bytes * 10
+
+int DiscretionaryPolicy::CodeSizeEstimate()
+{
+ return m_ModelCodeSizeEstimate;
+}
+
+#if defined(DEBUG) || defined(INLINE_DATA)
+
+//------------------------------------------------------------------------
+// DumpSchema: dump names for all the supporting data for the
+// inline decision in CSV format.
+//
+// Arguments:
+// file -- file to write to
+
+void DiscretionaryPolicy::DumpSchema(FILE* file) const
+{
+ fprintf(file, ",ILSize");
+ fprintf(file, ",CallsiteFrequency");
+ fprintf(file, ",InstructionCount");
+ fprintf(file, ",LoadStoreCount");
+ fprintf(file, ",Depth");
+ fprintf(file, ",BlockCount");
+ fprintf(file, ",Maxstack");
+ fprintf(file, ",ArgCount");
+
+ for (unsigned i = 0; i < MAX_ARGS; i++)
+ {
+ fprintf(file, ",Arg%uType", i);
+ }
+
+ for (unsigned i = 0; i < MAX_ARGS; i++)
+ {
+ fprintf(file, ",Arg%uSize", i);
+ }
+
+ fprintf(file, ",LocalCount");
+ fprintf(file, ",ReturnType");
+ fprintf(file, ",ReturnSize");
+ fprintf(file, ",ArgAccessCount");
+ fprintf(file, ",LocalAccessCount");
+ fprintf(file, ",IntConstantCount");
+ fprintf(file, ",FloatConstantCount");
+ fprintf(file, ",IntLoadCount");
+ fprintf(file, ",FloatLoadCount");
+ fprintf(file, ",IntStoreCount");
+ fprintf(file, ",FloatStoreCount");
+ fprintf(file, ",SimpleMathCount");
+ fprintf(file, ",ComplexMathCount");
+ fprintf(file, ",OverflowMathCount");
+ fprintf(file, ",IntArrayLoadCount");
+ fprintf(file, ",FloatArrayLoadCount");
+ fprintf(file, ",RefArrayLoadCount");
+ fprintf(file, ",StructArrayLoadCount");
+ fprintf(file, ",IntArrayStoreCount");
+ fprintf(file, ",FloatArrayStoreCount");
+ fprintf(file, ",RefArrayStoreCount");
+ fprintf(file, ",StructArrayStoreCount");
+ fprintf(file, ",StructOperationCount");
+ fprintf(file, ",ObjectModelCount");
+ fprintf(file, ",FieldLoadCount");
+ fprintf(file, ",FieldStoreCount");
+ fprintf(file, ",StaticFieldLoadCount");
+ fprintf(file, ",StaticFieldStoreCount");
+ fprintf(file, ",LoadAddressCount");
+ fprintf(file, ",ThrowCount");
+ fprintf(file, ",ReturnCount");
+ fprintf(file, ",CallCount");
+ fprintf(file, ",CallSiteWeight");
+ fprintf(file, ",IsForceInline");
+ fprintf(file, ",IsInstanceCtor");
+ fprintf(file, ",IsFromPromotableValueClass");
+ fprintf(file, ",HasSimd");
+ fprintf(file, ",LooksLikeWrapperMethod");
+ fprintf(file, ",ArgFeedsConstantTest");
+ fprintf(file, ",IsMostlyLoadStore");
+ fprintf(file, ",ArgFeedsRangeCheck");
+ fprintf(file, ",ConstantArgFeedsConstantTest");
+ fprintf(file, ",CalleeNativeSizeEstimate");
+ fprintf(file, ",CallsiteNativeSizeEstimate");
+ fprintf(file, ",ModelCodeSizeEstimate");
+ fprintf(file, ",ModelPerCallInstructionEstimate");
+ fprintf(file, ",IsClassCtor");
+ fprintf(file, ",IsSameThis");
+ fprintf(file, ",CallerHasNewArray");
+ fprintf(file, ",CallerHasNewObj");
+}
+
+//------------------------------------------------------------------------
+// DumpData: dump all the supporting data for the inline decision
+// in CSV format.
+//
+// Arguments:
+// file -- file to write to
+
+void DiscretionaryPolicy::DumpData(FILE* file) const
+{
+ fprintf(file, ",%u", m_CodeSize);
+ fprintf(file, ",%u", m_CallsiteFrequency);
+ fprintf(file, ",%u", m_InstructionCount);
+ fprintf(file, ",%u", m_LoadStoreCount);
+ fprintf(file, ",%u", m_Depth);
+ fprintf(file, ",%u", m_BlockCount);
+ fprintf(file, ",%u", m_Maxstack);
+ fprintf(file, ",%u", m_ArgCount);
+
+ for (unsigned i = 0; i < MAX_ARGS; i++)
+ {
+ fprintf(file, ",%u", m_ArgType[i]);
+ }
+
+ for (unsigned i = 0; i < MAX_ARGS; i++)
+ {
+ fprintf(file, ",%u", (unsigned)m_ArgSize[i]);
+ }
+
+ fprintf(file, ",%u", m_LocalCount);
+ fprintf(file, ",%u", m_ReturnType);
+ fprintf(file, ",%u", (unsigned)m_ReturnSize);
+ fprintf(file, ",%u", m_ArgAccessCount);
+ fprintf(file, ",%u", m_LocalAccessCount);
+ fprintf(file, ",%u", m_IntConstantCount);
+ fprintf(file, ",%u", m_FloatConstantCount);
+ fprintf(file, ",%u", m_IntLoadCount);
+ fprintf(file, ",%u", m_FloatLoadCount);
+ fprintf(file, ",%u", m_IntStoreCount);
+ fprintf(file, ",%u", m_FloatStoreCount);
+ fprintf(file, ",%u", m_SimpleMathCount);
+ fprintf(file, ",%u", m_ComplexMathCount);
+ fprintf(file, ",%u", m_OverflowMathCount);
+ fprintf(file, ",%u", m_IntArrayLoadCount);
+ fprintf(file, ",%u", m_FloatArrayLoadCount);
+ fprintf(file, ",%u", m_RefArrayLoadCount);
+ fprintf(file, ",%u", m_StructArrayLoadCount);
+ fprintf(file, ",%u", m_IntArrayStoreCount);
+ fprintf(file, ",%u", m_FloatArrayStoreCount);
+ fprintf(file, ",%u", m_RefArrayStoreCount);
+ fprintf(file, ",%u", m_StructArrayStoreCount);
+ fprintf(file, ",%u", m_StructOperationCount);
+ fprintf(file, ",%u", m_ObjectModelCount);
+ fprintf(file, ",%u", m_FieldLoadCount);
+ fprintf(file, ",%u", m_FieldStoreCount);
+ fprintf(file, ",%u", m_StaticFieldLoadCount);
+ fprintf(file, ",%u", m_StaticFieldStoreCount);
+ fprintf(file, ",%u", m_LoadAddressCount);
+ fprintf(file, ",%u", m_ReturnCount);
+ fprintf(file, ",%u", m_ThrowCount);
+ fprintf(file, ",%u", m_CallCount);
+ fprintf(file, ",%u", m_CallSiteWeight);
+ fprintf(file, ",%u", m_IsForceInline ? 1 : 0);
+ fprintf(file, ",%u", m_IsInstanceCtor ? 1 : 0);
+ fprintf(file, ",%u", m_IsFromPromotableValueClass ? 1 : 0);
+ fprintf(file, ",%u", m_HasSimd ? 1 : 0);
+ fprintf(file, ",%u", m_LooksLikeWrapperMethod ? 1 : 0);
+ fprintf(file, ",%u", m_ArgFeedsConstantTest);
+ fprintf(file, ",%u", m_MethodIsMostlyLoadStore ? 1 : 0);
+ fprintf(file, ",%u", m_ArgFeedsRangeCheck);
+ fprintf(file, ",%u", m_ConstantArgFeedsConstantTest);
+ fprintf(file, ",%d", m_CalleeNativeSizeEstimate);
+ fprintf(file, ",%d", m_CallsiteNativeSizeEstimate);
+ fprintf(file, ",%d", m_ModelCodeSizeEstimate);
+ fprintf(file, ",%d", m_PerCallInstructionEstimate);
+ fprintf(file, ",%u", m_IsClassCtor ? 1 : 0);
+ fprintf(file, ",%u", m_IsSameThis ? 1 : 0);
+ fprintf(file, ",%u", m_CallerHasNewArray ? 1 : 0);
+ fprintf(file, ",%u", m_CallerHasNewObj ? 1 : 0);
+}
+
+#endif // defined(DEBUG) || defined(INLINE_DATA)
+
+//------------------------------------------------------------------------/
+// ModelPolicy: construct a new ModelPolicy
+//
+// Arguments:
+// compiler -- compiler instance doing the inlining (root compiler)
+// isPrejitRoot -- true if this compiler is prejitting the root method
+
+ModelPolicy::ModelPolicy(Compiler* compiler, bool isPrejitRoot) : DiscretionaryPolicy(compiler, isPrejitRoot)
+{
+ // Empty
+}
+
+//------------------------------------------------------------------------
+// NoteInt: handle an observed integer value
+//
+// Arguments:
+// obs - the current obsevation
+// value - the value being observed
+//
+// Notes:
+// The ILSize threshold used here should be large enough that
+// it does not generally influence inlining decisions -- it only
+// helps to make them faster.
+//
+// The value is determined as follows. We figure out the maximum
+// possible code size estimate that will lead to an inline. This is
+// found by determining the maximum possible inline benefit and
+// working backwards.
+//
+// In the current ModelPolicy, the maximum benefit is -28.1, which
+// comes from a CallSiteWeight of 3 and a per call benefit of
+// -9.37. This implies that any candidate with code size larger
+// than (28.1/0.2) will not pass the threshold. So maximum code
+// size estimate (in bytes) for any inlinee is 140.55, and hence
+// maximum estimate is 1405.
+//
+// Since we are trying to short circuit early in the evaluation
+// process we don't have the code size estimate in hand. We need to
+// estimate the possible code size estimate based on something we
+// know cheaply and early -- the ILSize. So we use quantile
+// regression to project how ILSize predicts the model code size
+// estimate. Note that ILSize does not currently directly enter
+// into the model.
+//
+// The median value for the model code size estimate based on
+// ILSize is given by -107 + 12.6 * ILSize for the V9 data. This
+// means an ILSize of 120 is likely to lead to a size estimate of
+// at least 1405 at least 50% of the time. So we choose this as the
+// early rejection threshold.
+
+void ModelPolicy::NoteInt(InlineObservation obs, int value)
+{
+ // Let underlying policy do its thing.
+ DiscretionaryPolicy::NoteInt(obs, value);
+
+ // Fail fast for inlinees that are too large to ever inline.
+ // The value of 120 is model-dependent; see notes above.
+ if (!m_IsForceInline && (obs == InlineObservation::CALLEE_IL_CODE_SIZE) && (value >= 120))
+ {
+ // Callee too big, not a candidate
+ SetNever(InlineObservation::CALLEE_TOO_MUCH_IL);
+ return;
+ }
+
+ // Safeguard against overly deep inlines
+ if (obs == InlineObservation::CALLSITE_DEPTH)
+ {
+ unsigned depthLimit = m_RootCompiler->m_inlineStrategy->GetMaxInlineDepth();
+
+ if (m_Depth > depthLimit)
+ {
+ SetFailure(InlineObservation::CALLSITE_IS_TOO_DEEP);
+ return;
+ }
+ }
+}
+
+//------------------------------------------------------------------------
+// DetermineProfitability: determine if this inline is profitable
+//
+// Arguments:
+// methodInfo -- method info for the callee
+//
+// Notes:
+// There are currently two parameters that are ad-hoc: the
+// per-call-site weight and the size/speed threshold. Ideally this
+// policy would have just one tunable parameter, the threshold,
+// which describes how willing we are to trade size for speed.
+
+void ModelPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
+{
+ // Do some homework
+ MethodInfoObservations(methodInfo);
+ EstimateCodeSize();
+ EstimatePerformanceImpact();
+
+ // Preliminary inline model.
+ //
+ // If code size is estimated to increase, look at
+ // the profitability model for guidance.
+ //
+ // If code size will decrease, just inline.
+
+ if (m_ModelCodeSizeEstimate <= 0)
+ {
+ // Inline will likely decrease code size
+ JITLOG_THIS(m_RootCompiler, (LL_INFO100000, "Inline profitable, will decrease code size by %g bytes\n",
+ (double)-m_ModelCodeSizeEstimate / SIZE_SCALE));
+
+ if (m_IsPrejitRoot)
+ {
+ SetCandidate(InlineObservation::CALLEE_IS_SIZE_DECREASING_INLINE);
+ }
+ else
+ {
+ SetCandidate(InlineObservation::CALLSITE_IS_SIZE_DECREASING_INLINE);
+ }
+ }
+ else
+ {
+ // We estimate that this inline will increase code size. Only
+ // inline if the performance win is sufficiently large to
+ // justify bigger code.
+
+ // First compute the number of instruction executions saved
+ // via inlining per call to the callee per byte of code size
+ // impact.
+ //
+ // The per call instruction estimate is negative if the inline
+ // will reduce instruction count. Flip the sign here to make
+ // positive be better and negative worse.
+ double perCallBenefit = -((double)m_PerCallInstructionEstimate / (double)m_ModelCodeSizeEstimate);
+
+ // Now estimate the local call frequency.
+ //
+ // Todo: use IBC data, or a better local profile estimate, or
+ // try and incorporate this into the model. For instance if we
+ // tried to predict the benefit per call to the root method
+ // then the model would have to incorporate the local call
+ // frequency, somehow.
+ double callSiteWeight = 1.0;
+
+ switch (m_CallsiteFrequency)
+ {
+ case InlineCallsiteFrequency::RARE:
+ callSiteWeight = 0.1;
+ break;
+ case InlineCallsiteFrequency::BORING:
+ callSiteWeight = 1.0;
+ break;
+ case InlineCallsiteFrequency::WARM:
+ callSiteWeight = 1.5;
+ break;
+ case InlineCallsiteFrequency::LOOP:
+ case InlineCallsiteFrequency::HOT:
+ callSiteWeight = 3.0;
+ break;
+ default:
+ assert(false);
+ break;
+ }
+
+ // Determine the estimated number of instructions saved per
+ // call to the root method per byte of code size impact. This
+ // is our benefit figure of merit.
+ double benefit = callSiteWeight * perCallBenefit;
+
+ // Compare this to the threshold, and inline if greater.
+ //
+ // The threshold is interpretable as a size/speed tradeoff:
+ // the value of 0.2 below indicates we'll allow inlines that
+ // grow code by as many as 5 bytes to save 1 instruction
+ // execution (per call to the root method).
+ double threshold = 0.20;
+ bool shouldInline = (benefit > threshold);
+
+ JITLOG_THIS(m_RootCompiler,
+ (LL_INFO100000, "Inline %s profitable: benefit=%g (weight=%g, percall=%g, size=%g)\n",
+ shouldInline ? "is" : "is not", benefit, callSiteWeight,
+ (double)m_PerCallInstructionEstimate / SIZE_SCALE, (double)m_ModelCodeSizeEstimate / SIZE_SCALE));
+
+ if (!shouldInline)
+ {
+ // Fail the inline
+ if (m_IsPrejitRoot)
+ {
+ SetNever(InlineObservation::CALLEE_NOT_PROFITABLE_INLINE);
+ }
+ else
+ {
+ SetFailure(InlineObservation::CALLSITE_NOT_PROFITABLE_INLINE);
+ }
+ }
+ else
+ {
+ // Update candidacy
+ if (m_IsPrejitRoot)
+ {
+ SetCandidate(InlineObservation::CALLEE_IS_PROFITABLE_INLINE);
+ }
+ else
+ {
+ SetCandidate(InlineObservation::CALLSITE_IS_PROFITABLE_INLINE);
+ }
+ }
+ }
+}
+
+#if defined(DEBUG) || defined(INLINE_DATA)
+
+//------------------------------------------------------------------------/
+// FullPolicy: construct a new FullPolicy
+//
+// Arguments:
+// compiler -- compiler instance doing the inlining (root compiler)
+// isPrejitRoot -- true if this compiler is prejitting the root method
+
+FullPolicy::FullPolicy(Compiler* compiler, bool isPrejitRoot) : DiscretionaryPolicy(compiler, isPrejitRoot)
+{
+ // Empty
+}
+
+//------------------------------------------------------------------------
+// DetermineProfitability: determine if this inline is profitable
+//
+// Arguments:
+// methodInfo -- method info for the callee
+
+void FullPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
+{
+ // Check depth
+
+ unsigned depthLimit = m_RootCompiler->m_inlineStrategy->GetMaxInlineDepth();
+
+ if (m_Depth > depthLimit)
+ {
+ SetFailure(InlineObservation::CALLSITE_IS_TOO_DEEP);
+ return;
+ }
+
+ // Check size
+
+ unsigned sizeLimit = m_RootCompiler->m_inlineStrategy->GetMaxInlineILSize();
+
+ if (m_CodeSize > sizeLimit)
+ {
+ SetFailure(InlineObservation::CALLEE_TOO_MUCH_IL);
+ return;
+ }
+
+ // Otherwise, we're good to go
+
+ if (m_IsPrejitRoot)
+ {
+ SetCandidate(InlineObservation::CALLEE_IS_PROFITABLE_INLINE);
+ }
+ else
+ {
+ SetCandidate(InlineObservation::CALLSITE_IS_PROFITABLE_INLINE);
+ }
+
+ return;
+}
+
+//------------------------------------------------------------------------/
+// SizePolicy: construct a new SizePolicy
+//
+// Arguments:
+// compiler -- compiler instance doing the inlining (root compiler)
+// isPrejitRoot -- true if this compiler is prejitting the root method
+
+SizePolicy::SizePolicy(Compiler* compiler, bool isPrejitRoot) : DiscretionaryPolicy(compiler, isPrejitRoot)
+{
+ // Empty
+}
+
+//------------------------------------------------------------------------
+// DetermineProfitability: determine if this inline is profitable
+//
+// Arguments:
+// methodInfo -- method info for the callee
+
+void SizePolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
+{
+ // Do some homework
+ MethodInfoObservations(methodInfo);
+ EstimateCodeSize();
+
+ // Does this inline increase the estimated size beyond
+ // the original size estimate?
+ const InlineStrategy* strategy = m_RootCompiler->m_inlineStrategy;
+ const int initialSize = strategy->GetInitialSizeEstimate();
+ const int currentSize = strategy->GetCurrentSizeEstimate();
+ const int newSize = currentSize + m_ModelCodeSizeEstimate;
+
+ if (newSize <= initialSize)
+ {
+ // Estimated size impact is acceptable, so inline here.
+ JITLOG_THIS(m_RootCompiler,
+ (LL_INFO100000, "Inline profitable, root size estimate %d is less than initial size %d\n",
+ newSize / SIZE_SCALE, initialSize / SIZE_SCALE));
+
+ if (m_IsPrejitRoot)
+ {
+ SetCandidate(InlineObservation::CALLEE_IS_SIZE_DECREASING_INLINE);
+ }
+ else
+ {
+ SetCandidate(InlineObservation::CALLSITE_IS_SIZE_DECREASING_INLINE);
+ }
+ }
+ else
+ {
+ // Estimated size increase is too large, so no inline here.
+ //
+ // Note that we ought to reconsider this inline if we make
+ // room in the budget by inlining a bunch of size decreasing
+ // inlines after this one. But for now, we won't do this.
+ if (m_IsPrejitRoot)
+ {
+ SetNever(InlineObservation::CALLEE_NOT_PROFITABLE_INLINE);
+ }
+ else
+ {
+ SetFailure(InlineObservation::CALLSITE_NOT_PROFITABLE_INLINE);
+ }
+ }
+
+ return;
+}
+
+// Statics to track emission of the replay banner
+// and provide file access to the inline xml
+
+bool ReplayPolicy::s_WroteReplayBanner = false;
+FILE* ReplayPolicy::s_ReplayFile = nullptr;
+CritSecObject ReplayPolicy::s_XmlReaderLock;
+
+//------------------------------------------------------------------------/
+// ReplayPolicy: construct a new ReplayPolicy
+//
+// Arguments:
+// compiler -- compiler instance doing the inlining (root compiler)
+// isPrejitRoot -- true if this compiler is prejitting the root method
+
+ReplayPolicy::ReplayPolicy(Compiler* compiler, bool isPrejitRoot)
+ : DiscretionaryPolicy(compiler, isPrejitRoot)
+ , m_InlineContext(nullptr)
+ , m_Offset(BAD_IL_OFFSET)
+ , m_WasForceInline(false)
+{
+ // Is there a log file open already? If so, we can use it.
+ if (s_ReplayFile == nullptr)
+ {
+ // Did we already try and open and fail?
+ if (!s_WroteReplayBanner)
+ {
+ // Nope, open it up.
+ const wchar_t* replayFileName = JitConfig.JitInlineReplayFile();
+ s_ReplayFile = _wfopen(replayFileName, W("r"));
+
+ // Display banner to stderr, unless we're dumping inline Xml,
+ // in which case the policy name is captured in the Xml.
+ if (JitConfig.JitInlineDumpXml() == 0)
+ {
+ fprintf(stderr, "*** %s inlines from %ws\n", s_ReplayFile == nullptr ? "Unable to replay" : "Replaying",
+ replayFileName);
+ }
+
+ s_WroteReplayBanner = true;
+ }
+ }
+}
+
+//------------------------------------------------------------------------
+// ReplayPolicy: Finalize reading of inline Xml
+//
+// Notes:
+// Called during jitShutdown()
+
+void ReplayPolicy::FinalizeXml()
+{
+ if (s_ReplayFile != nullptr)
+ {
+ fclose(s_ReplayFile);
+ s_ReplayFile = nullptr;
+ }
+}
+
+//------------------------------------------------------------------------
+// FindMethod: find the root method in the inline Xml
+//
+// ReturnValue:
+// true if found. File position left pointing just after the
+// <Token> entry for the method.
+
+bool ReplayPolicy::FindMethod()
+{
+ if (s_ReplayFile == nullptr)
+ {
+ return false;
+ }
+
+ // See if we've already found this method.
+ InlineStrategy* inlineStrategy = m_RootCompiler->m_inlineStrategy;
+ long filePosition = inlineStrategy->GetMethodXmlFilePosition();
+
+ if (filePosition == -1)
+ {
+ // Past lookup failed
+ return false;
+ }
+ else if (filePosition > 0)
+ {
+ // Past lookup succeeded, jump there
+ fseek(s_ReplayFile, filePosition, SEEK_SET);
+ return true;
+ }
+
+ // Else, scan the file. Might be nice to build an index
+ // or something, someday.
+ const mdMethodDef methodToken =
+ m_RootCompiler->info.compCompHnd->getMethodDefFromMethod(m_RootCompiler->info.compMethodHnd);
+ const unsigned methodHash = m_RootCompiler->info.compMethodHash();
+
+ bool foundMethod = false;
+ char buffer[256];
+ fseek(s_ReplayFile, 0, SEEK_SET);
+
+ while (!foundMethod)
+ {
+ // Get next line
+ if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
+ {
+ break;
+ }
+
+ // Look for next method entry
+ if (strstr(buffer, "<Method>") == nullptr)
+ {
+ continue;
+ }
+
+ // Get next line
+ if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
+ {
+ break;
+ }
+
+ // See if token matches
+ unsigned token = 0;
+ int count = sscanf(buffer, " <Token>%u</Token> ", &token);
+ if ((count != 1) || (token != methodToken))
+ {
+ continue;
+ }
+
+ // Get next line
+ if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
+ {
+ break;
+ }
+
+ // See if hash matches
+ unsigned hash = 0;
+ count = sscanf(buffer, " <Hash>%u</Hash> ", &hash);
+ if ((count != 1) || (hash != methodHash))
+ {
+ continue;
+ }
+
+ // Found a match...
+ foundMethod = true;
+ break;
+ }
+
+ // Update file position cache for this method
+ long foundPosition = -1;
+
+ if (foundMethod)
+ {
+ foundPosition = ftell(s_ReplayFile);
+ }
+
+ inlineStrategy->SetMethodXmlFilePosition(foundPosition);
+
+ return foundMethod;
+}
+
+//------------------------------------------------------------------------
+// FindContext: find an inline context in the inline Xml
+//
+// Notes:
+// Assumes file position within the relevant method has just been
+// set by a successful call to FindMethod().
+//
+// Arguments:
+// context -- context of interest
+//
+// ReturnValue:
+// true if found. File position left pointing just after the
+// <Token> entry for the context.
+
+bool ReplayPolicy::FindContext(InlineContext* context)
+{
+ // Make sure we've found the parent context.
+ if (context->IsRoot())
+ {
+ // We've already found the method context so we're good.
+ return true;
+ }
+
+ bool foundParent = FindContext(context->GetParent());
+
+ if (!foundParent)
+ {
+ return false;
+ }
+
+ // File pointer should be pointing at the parent context level.
+ // See if we see an inline entry for this context.
+ //
+ // Token and Hash we're looking for.
+ mdMethodDef contextToken = m_RootCompiler->info.compCompHnd->getMethodDefFromMethod(context->GetCallee());
+ unsigned contextHash = m_RootCompiler->info.compCompHnd->getMethodHash(context->GetCallee());
+ unsigned contextOffset = (unsigned)context->GetOffset();
+
+ return FindInline(contextToken, contextHash, contextOffset);
+}
+
+//------------------------------------------------------------------------
+// FindInline: find entry for the current inline in inline Xml.
+//
+// Arguments:
+// token -- token describing the inline
+// hash -- hash describing the inline
+// offset -- IL offset of the call site in the parent method
+//
+// ReturnValue:
+// true if the inline entry was found
+//
+// Notes:
+// Assumes file position has just been set by a successful call to
+// FindMethod or FindContext.
+//
+// Token and Hash will not be sufficiently unique to identify a
+// particular inline, if there are multiple calls to the same
+// method.
+
+bool ReplayPolicy::FindInline(unsigned token, unsigned hash, unsigned offset)
+{
+ char buffer[256];
+ bool foundInline = false;
+ int depth = 0;
+
+ while (!foundInline)
+ {
+ // Get next line
+ if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
+ {
+ break;
+ }
+
+ // If we hit </Method> we've gone too far,
+ // and the XML is messed up.
+ if (strstr(buffer, "</Method>") != nullptr)
+ {
+ break;
+ }
+
+ // Look for <Inlines />....
+ if (strstr(buffer, "<Inlines />") != nullptr)
+ {
+ if (depth == 0)
+ {
+ // Exited depth 1, failed to find the context
+ break;
+ }
+ else
+ {
+ // Exited nested, keep looking
+ continue;
+ }
+ }
+
+ // Look for <Inlines>....
+ if (strstr(buffer, "<Inlines>") != nullptr)
+ {
+ depth++;
+ continue;
+ }
+
+ // If we hit </Inlines> we've exited a nested entry
+ // or the current entry.
+ if (strstr(buffer, "</Inlines>") != nullptr)
+ {
+ depth--;
+
+ if (depth == 0)
+ {
+ // Exited depth 1, failed to find the context
+ break;
+ }
+ else
+ {
+ // Exited nested, keep looking
+ continue;
+ }
+ }
+
+ // Look for start of inline section at the right depth
+ if ((depth != 1) || (strstr(buffer, "<Inline>") == nullptr))
+ {
+ continue;
+ }
+
+ // Get next line
+ if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
+ {
+ break;
+ }
+
+ // Match token
+ unsigned inlineToken = 0;
+ int count = sscanf(buffer, " <Token>%u</Token> ", &inlineToken);
+
+ if ((count != 1) || (inlineToken != token))
+ {
+ continue;
+ }
+
+ // Get next line
+ if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
+ {
+ break;
+ }
+
+ // Match hash
+ unsigned inlineHash = 0;
+ count = sscanf(buffer, " <Hash>%u</Hash> ", &inlineHash);
+
+ if ((count != 1) || (inlineHash != hash))
+ {
+ continue;
+ }
+
+ // Get next line
+ if (fgets(buffer, sizeof(buffer), s_ReplayFile) == nullptr)
+ {
+ break;
+ }
+
+ // Match offset
+ unsigned inlineOffset = 0;
+ count = sscanf(buffer, " <Offset>%u</Offset> ", &inlineOffset);
+ if ((count != 1) || (inlineOffset != offset))
+ {
+ continue;
+ }
+
+ // Token,Hash,Offset may still not be unique enough, but it's
+ // all we have right now.
+
+ // We're good!
+ foundInline = true;
+
+ // Check for a data collection marker. This does not affect
+ // matching...
+
+ // Get next line
+ if (fgets(buffer, sizeof(buffer), s_ReplayFile) != nullptr)
+ {
+ unsigned collectData = 0;
+ count = sscanf(buffer, " <CollectData>%u</CollectData> ", &collectData);
+
+ if (count == 1)
+ {
+ m_IsDataCollectionTarget = (collectData == 1);
+ }
+ }
+
+ break;
+ }
+
+ return foundInline;
+}
+
+//------------------------------------------------------------------------
+// FindInline: find entry for a particular callee in inline Xml.
+//
+// Arguments:
+// callee -- handle for the callee method
+//
+// ReturnValue:
+// true if the inline should be performed.
+//
+// Notes:
+// Assumes file position has just been set by a successful call to
+// FindContext(...);
+//
+// callee handle will not be sufficiently unique to identify a
+// particular inline, if there are multiple calls to the same
+// method.
+
+bool ReplayPolicy::FindInline(CORINFO_METHOD_HANDLE callee)
+{
+ // Token and Hash we're looking for
+ mdMethodDef calleeToken = m_RootCompiler->info.compCompHnd->getMethodDefFromMethod(callee);
+ unsigned calleeHash = m_RootCompiler->info.compCompHnd->getMethodHash(callee);
+
+ // Abstract this or just pass through raw bits
+ // See matching code in xml writer
+ int offset = -1;
+ if (m_Offset != BAD_IL_OFFSET)
+ {
+ offset = (int)jitGetILoffs(m_Offset);
+ }
+
+ unsigned calleeOffset = (unsigned)offset;
+
+ bool foundInline = FindInline(calleeToken, calleeHash, calleeOffset);
+
+ return foundInline;
+}
+
+//------------------------------------------------------------------------
+// NoteBool: handle an observed boolean value
+//
+// Arguments:
+// obs - the current obsevation
+// value - the value being observed
+//
+// Notes:
+// Overrides parent so Replay can control force inlines.
+
+void ReplayPolicy::NoteBool(InlineObservation obs, bool value)
+{
+ // When inlining, let log override force inline.
+ // Make a note of the actual value for later reporting during observations.
+ if (!m_IsPrejitRoot && (obs == InlineObservation::CALLEE_IS_FORCE_INLINE))
+ {
+ m_WasForceInline = value;
+ value = false;
+ }
+
+ DiscretionaryPolicy::NoteBool(obs, value);
+}
+
+//------------------------------------------------------------------------
+// DetermineProfitability: determine if this inline is profitable
+//
+// Arguments:
+// methodInfo -- method info for the callee
+
+void ReplayPolicy::DetermineProfitability(CORINFO_METHOD_INFO* methodInfo)
+{
+ // TODO: handle prejit root case....need to record this in the
+ // root method XML.
+ if (m_IsPrejitRoot)
+ {
+ // Fall back to discretionary policy for now.
+ return DiscretionaryPolicy::DetermineProfitability(methodInfo);
+ }
+
+ // If we're also dumping inline data, make additional observations
+ // based on the method info, and estimate code size and perf
+ // impact, so that the reports have the necessary data.
+ if (JitConfig.JitInlineDumpData() != 0)
+ {
+ MethodInfoObservations(methodInfo);
+ EstimateCodeSize();
+ EstimatePerformanceImpact();
+ m_IsForceInline = m_WasForceInline;
+ }
+
+ // Try and find this candiate in the Xml.
+ // If we fail to find it, then don't inline.
+ bool accept = false;
+
+ // Grab the reader lock, since we'll be manipulating
+ // the file pointer as we look for the relevant inline xml.
+ {
+ CritSecHolder readerLock(s_XmlReaderLock);
+
+ // First, locate the entries for the root method.
+ bool foundMethod = FindMethod();
+
+ if (foundMethod && (m_InlineContext != nullptr))
+ {
+ // Next, navigate the context tree to find the entries
+ // for the context that contains this candidate.
+ bool foundContext = FindContext(m_InlineContext);
+
+ if (foundContext)
+ {
+ // Finally, find this candidate within its context
+ CORINFO_METHOD_HANDLE calleeHandle = methodInfo->ftn;
+ accept = FindInline(calleeHandle);
+ }
+ }
+ }
+
+ if (accept)
+ {
+ JITLOG_THIS(m_RootCompiler, (LL_INFO100000, "Inline accepted via log replay"));
+
+ if (m_IsPrejitRoot)
+ {
+ SetCandidate(InlineObservation::CALLEE_LOG_REPLAY_ACCEPT);
+ }
+ else
+ {
+ SetCandidate(InlineObservation::CALLSITE_LOG_REPLAY_ACCEPT);
+ }
+ }
+ else
+ {
+ JITLOG_THIS(m_RootCompiler, (LL_INFO100000, "Inline rejected via log replay"));
+
+ if (m_IsPrejitRoot)
+ {
+ SetNever(InlineObservation::CALLEE_LOG_REPLAY_REJECT);
+ }
+ else
+ {
+ SetFailure(InlineObservation::CALLSITE_LOG_REPLAY_REJECT);
+ }
+ }
+
+ return;
+}
+
+#endif // defined(DEBUG) || defined(INLINE_DATA)