diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index fa5bd6087220cf..6afd014edcf21a 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4581,10 +4581,11 @@ class Compiler GenTreeCall::Use* args = nullptr, CORINFO_LOOKUP_KIND* pGenericLookupKind = nullptr); - GenTree* impCastClassOrIsInstToTree(GenTree* op1, - GenTree* op2, - CORINFO_RESOLVED_TOKEN* pResolvedToken, - bool isCastClass); + bool impIsCastHelperEligibleForClassProbe(GenTree* tree); + bool impIsCastHelperMayHaveProfileData(GenTree* tree); + + GenTree* impCastClassOrIsInstToTree( + GenTree* op1, GenTree* op2, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass, IL_OFFSET ilOffset); GenTree* impOptimizeCastClassOrIsInst(GenTree* op1, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass); diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 34ab390a9b899e..aa5d1034eeda3b 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -1410,7 +1410,8 @@ void EfficientEdgeCountInstrumentor::Instrument(BasicBlock* block, Schema& schem } //------------------------------------------------------------------------ -// ClassProbeVisitor: invoke functor on each virtual call in a tree +// ClassProbeVisitor: invoke functor on each virtual call or cast-related +// helper calls in a tree // template class ClassProbeVisitor final : public GenTreeVisitor> @@ -1436,6 +1437,12 @@ class ClassProbeVisitor final : public GenTreeVisitorAsCall(); if (call->IsVirtual() && (call->gtCallType != CT_INDIRECT)) { + // virtual call + m_functor(m_compiler, call); + } + else if (m_compiler->impIsCastHelperEligibleForClassProbe(call)) + { + // isinst/cast helper m_functor(m_compiler, call); } } @@ -1469,7 +1476,7 @@ class BuildClassProbeSchemaGen } else { - assert(call->IsVirtualVtable()); + assert(call->IsVirtualVtable() || compiler->impIsCastHelperEligibleForClassProbe(call)); } schemaElem.InstrumentationKind = JitConfig.JitCollect64BitCounts() @@ -1524,8 +1531,6 @@ class ClassProbeInserter // ... args ...) // - assert(call->gtCallThisArg->GetNode()->TypeGet() == TYP_REF); - // Sanity check that we're looking at the right schema entry // assert(m_schema[*m_currentSchemaIndex].ILOffset == (int32_t)call->gtClassProfileCandidateInfo->ilOffset); @@ -1540,6 +1545,20 @@ class ClassProbeInserter uint8_t* classProfile = m_schema[*m_currentSchemaIndex].Offset + m_profileMemory; *m_currentSchemaIndex += 2; // There are 2 schema entries per class probe + GenTreeCall::Use* objUse = nullptr; + if (compiler->impIsCastHelperEligibleForClassProbe(call)) + { + // Grab the second arg of cast/isinst helper call + objUse = call->gtCallArgs->GetNext(); + } + else + { + // Grab 'this' arg + objUse = call->gtCallThisArg; + } + + assert(objUse->GetNode()->TypeIs(TYP_REF)); + // Grab a temp to hold the 'this' object as it will be used three times // unsigned const tmpNum = compiler->lvaGrabTemp(true DEBUGARG("class profile tmp")); @@ -1556,12 +1575,12 @@ class ClassProbeInserter GenTree* const tmpNode2 = compiler->gtNewLclvNode(tmpNum, TYP_REF); GenTree* const callCommaNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, helperCallNode, tmpNode2); GenTree* const tmpNode3 = compiler->gtNewLclvNode(tmpNum, TYP_REF); - GenTree* const asgNode = compiler->gtNewOperNode(GT_ASG, TYP_REF, tmpNode3, call->gtCallThisArg->GetNode()); - GenTree* const asgCommaNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, asgNode, callCommaNode); + GenTree* const asgNode = compiler->gtNewOperNode(GT_ASG, TYP_REF, tmpNode3, objUse->GetNode()); + GenTree* const asgCommaNode = compiler->gtNewOperNode(GT_COMMA, TYP_REF, asgNode, callCommaNode); // Update the call // - call->gtCallThisArg->SetNode(asgCommaNode); + objUse->SetNode(asgCommaNode); JITDUMP("Modified call is now\n"); DISPTREE(call); diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 9e00910d5d85ef..01942a28f3fe6f 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -376,12 +376,6 @@ inline void Compiler::impEndTreeList(BasicBlock* block, Statement* firstStmt, St block->bbFlags |= BBF_IMPORTED; } -//------------------------------------------------------------------------ -// impEndTreeList: Store the current tree list in the given basic block. -// -// Arguments: -// block - the basic block to store into. -// inline void Compiler::impEndTreeList(BasicBlock* block) { if (impStmtList == nullptr) @@ -2108,6 +2102,64 @@ GenTree* Compiler::impReadyToRunLookupToTree(CORINFO_CONST_LOOKUP* pLookup, return addr; } +//------------------------------------------------------------------------ +// impIsCastHelperEligibleForClassProbe: Checks whether a tree is a cast helper eligible to +// to be profiled and then optimized with PGO data +// +// Arguments: +// tree - the tree object to check +// +// Returns: +// true if the tree is a cast helper eligible to be profiled +// +bool Compiler::impIsCastHelperEligibleForClassProbe(GenTree* tree) +{ + if (!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) || (JitConfig.JitCastProfiling() != 1)) + { + return false; + } + + if (tree->IsCall() && tree->AsCall()->gtCallType == CT_HELPER) + { + const CorInfoHelpFunc helper = eeGetHelperNum(tree->AsCall()->gtCallMethHnd); + if ((helper == CORINFO_HELP_ISINSTANCEOFINTERFACE) || (helper == CORINFO_HELP_ISINSTANCEOFCLASS) || + (helper == CORINFO_HELP_CHKCASTCLASS) || (helper == CORINFO_HELP_CHKCASTINTERFACE)) + { + return true; + } + } + return false; +} + +//------------------------------------------------------------------------ +// impIsCastHelperMayHaveProfileData: Checks whether a tree is a cast helper that might +// have profile data +// +// Arguments: +// tree - the tree object to check +// +// Returns: +// true if the tree is a cast helper with potential profile data +// +bool Compiler::impIsCastHelperMayHaveProfileData(GenTree* tree) +{ + if (!opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBOPT) || (JitConfig.JitCastProfiling() != 1)) + { + return false; + } + + if (tree->IsCall() && tree->AsCall()->gtCallType == CT_HELPER) + { + const CorInfoHelpFunc helper = eeGetHelperNum(tree->AsCall()->gtCallMethHnd); + if ((helper == CORINFO_HELP_ISINSTANCEOFINTERFACE) || (helper == CORINFO_HELP_ISINSTANCEOFCLASS) || + (helper == CORINFO_HELP_CHKCASTCLASS) || (helper == CORINFO_HELP_CHKCASTINTERFACE)) + { + return true; + } + } + return false; +} + GenTreeCall* Compiler::impReadyToRunHelperToTree( CORINFO_RESOLVED_TOKEN* pResolvedToken, CorInfoHelpFunc helper, @@ -11458,10 +11510,8 @@ GenTree* Compiler::impOptimizeCastClassOrIsInst(GenTree* op1, CORINFO_RESOLVED_T // Notes: // May expand into a series of runtime checks or a helper call. -GenTree* Compiler::impCastClassOrIsInstToTree(GenTree* op1, - GenTree* op2, - CORINFO_RESOLVED_TOKEN* pResolvedToken, - bool isCastClass) +GenTree* Compiler::impCastClassOrIsInstToTree( + GenTree* op1, GenTree* op2, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass, IL_OFFSET ilOffset) { assert(op1->TypeGet() == TYP_REF); @@ -11483,6 +11533,16 @@ GenTree* Compiler::impCastClassOrIsInstToTree(GenTree* op1, // not worth creating an untracked local variable shouldExpandInline = false; } + else if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) && (JitConfig.JitCastProfiling() == 1)) + { + // Optimizations are enabled but we're still instrumenting (including casts) + if (isCastClass && !impIsClassExact(pResolvedToken->hClass)) + { + // Usually, we make a speculative assumption that it makes sense to expand castclass + // even for non-sealed classes, but let's rely on PGO in this specific case + shouldExpandInline = false; + } + } // Pessimistically assume the jit cannot expand this as an inline test bool canExpandInline = false; @@ -11521,7 +11581,16 @@ GenTree* Compiler::impCastClassOrIsInstToTree(GenTree* op1, // op2->gtFlags |= GTF_DONT_CSE; - return gtNewHelperCallNode(helper, TYP_REF, gtNewCallArgs(op2, op1)); + GenTreeCall* call = gtNewHelperCallNode(helper, TYP_REF, gtNewCallArgs(op2, op1)); + if (impIsCastHelperEligibleForClassProbe(call) && !impIsClassExact(pResolvedToken->hClass)) + { + ClassProfileCandidateInfo* pInfo = new (this, CMK_Inlining) ClassProfileCandidateInfo; + pInfo->ilOffset = ilOffset; + pInfo->probeIndex = info.compClassProbeCount++; + call->gtClassProfileCandidateInfo = pInfo; + compCurBB->bbFlags |= BBF_HAS_CLASS_PROFILE; + } + return call; } JITDUMP("\nExpanding %s inline\n", isCastClass ? "castclass" : "isinst"); @@ -15765,7 +15834,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) if (!usingReadyToRunHelper) #endif { - op1 = impCastClassOrIsInstToTree(op1, op2, &resolvedToken, false); + op1 = impCastClassOrIsInstToTree(op1, op2, &resolvedToken, false, opcodeOffs); } if (compDonotInline()) { @@ -16290,7 +16359,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) if (!usingReadyToRunHelper) #endif { - op1 = impCastClassOrIsInstToTree(op1, op2, &resolvedToken, true); + op1 = impCastClassOrIsInstToTree(op1, op2, &resolvedToken, true, opcodeOffs); } if (compDonotInline()) { diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 355bbe1f9f3ee3..051fcc4affd335 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -486,8 +486,9 @@ CONFIG_STRING(JitEnablePatchpointRange, W("JitEnablePatchpointRange")) // Profile instrumentation options CONFIG_INTEGER(JitMinimalJitProfiling, W("JitMinimalJitProfiling"), 1) CONFIG_INTEGER(JitMinimalPrejitProfiling, W("JitMinimalPrejitProfiling"), 0) -CONFIG_INTEGER(JitClassProfiling, W("JitClassProfiling"), 1) -CONFIG_INTEGER(JitEdgeProfiling, W("JitEdgeProfiling"), 1) +CONFIG_INTEGER(JitCastProfiling, W("JitCastProfiling"), 0) // Profile castclass and isinst +CONFIG_INTEGER(JitClassProfiling, W("JitClassProfiling"), 1) // Profile virtual and interface calls +CONFIG_INTEGER(JitEdgeProfiling, W("JitEdgeProfiling"), 1) // Profile edges instead of blocks CONFIG_INTEGER(JitCollect64BitCounts, W("JitCollect64BitCounts"), 0) // Collect counts as 64-bit values. // Profile consumption options