Skip to content

Expose the x86 WaitPkg intrinsics #117437

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 8 commits into from
Closed

Expose the x86 WaitPkg intrinsics #117437

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
9695a3a
Expose the x86 WaitPkg intrinsics
tannergooding Jul 8, 2025
995a1ea
Merge branch 'main' into fix-66873
tannergooding Jul 12, 2025
5005115
Check for the right return type
tannergooding Jul 13, 2025
361dfcc
Add throughput/latency info
tannergooding Jul 13, 2025
40eb88c
Fix the comment header for an intrinsic group
tannergooding Jul 13, 2025
aa59d38
Merge remote-tracking branch 'dotnet/main' into fix-66873
tannergooding Jul 13, 2025
b3fda0a
Merge branch 'main' into fix-66873
tannergooding Jul 23, 2025
8a016cf
Merge remote-tracking branch 'dotnet/main' into fix-66873
tannergooding Aug 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions src/coreclr/jit/codegen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -941,12 +941,11 @@ class CodeGen final : public CodeGenInterface
GenTreeHWIntrinsic* node, instruction ins, emitAttr attr, int8_t ival, insOpts instOptions);

void genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions);
void genX86BaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions);
void genHWIntrinsicSpecial(GenTreeHWIntrinsic* node, insOpts instOptions);
void genAvxFamilyIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions);
void genFmaIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions);
void genPermuteVar2x(GenTreeHWIntrinsic* node, insOpts instOptions);
void genXCNTIntrinsic(GenTreeHWIntrinsic* node, instruction ins);
void genX86SerializeIntrinsic(GenTreeHWIntrinsic* node);

template <typename HWIntrinsicSwitchCaseBody>
void genHWIntrinsicJumpTableFallback(NamedIntrinsic intrinsic,
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/jit/emitxarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8040,7 +8040,7 @@ void emitter::emitIns_R_R_I(
void emitter::emitIns_AR(instruction ins, emitAttr attr, regNumber base, int offs, insOpts instOptions)
{
assert(ins == INS_prefetcht0 || ins == INS_prefetcht1 || ins == INS_prefetcht2 || ins == INS_prefetchnta ||
ins == INS_inc || ins == INS_dec);
ins == INS_inc || ins == INS_dec || ins == INS_umonitor);

instrDesc* id = emitNewInstrAmd(attr, offs);

Expand Down
3 changes: 3 additions & 0 deletions src/coreclr/jit/fgdiagnostic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3405,6 +3405,9 @@ void Compiler::fgDebugCheckFlags(GenTree* tree, BasicBlock* block)
case NI_X86Base_Prefetch1:
case NI_X86Base_Prefetch2:
case NI_X86Base_PrefetchNonTemporal:
case NI_WAITPKG_SetUpUserLevelMonitor:
case NI_WAITPKG_TimedPause:
case NI_WAITPKG_WaitForUserLevelMonitor:
{
assert(tree->OperRequiresCallFlag(this));
expectedFlags |= GTF_GLOB_REF;
Expand Down
6 changes: 6 additions & 0 deletions src/coreclr/jit/gentree.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28579,6 +28579,9 @@ bool GenTreeHWIntrinsic::OperRequiresCallFlag() const
case NI_X86Base_Prefetch1:
case NI_X86Base_Prefetch2:
case NI_X86Base_PrefetchNonTemporal:
case NI_WAITPKG_SetUpUserLevelMonitor:
case NI_WAITPKG_TimedPause:
case NI_WAITPKG_WaitForUserLevelMonitor:
{
return true;
}
Expand Down Expand Up @@ -28784,6 +28787,9 @@ void GenTreeHWIntrinsic::Initialize(NamedIntrinsic intrinsicId)
case NI_X86Base_Prefetch1:
case NI_X86Base_Prefetch2:
case NI_X86Base_PrefetchNonTemporal:
case NI_WAITPKG_SetUpUserLevelMonitor:
case NI_WAITPKG_TimedPause:
case NI_WAITPKG_WaitForUserLevelMonitor:
{
// Mark as a call and global reference, much as is done for GT_KEEPALIVE
gtFlags |= (GTF_CALL | GTF_GLOB_REF);
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/jit/hwintrinsic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -960,7 +960,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = {
{ FIRST_NI_GFNI_V256, LAST_NI_GFNI_V256 }, // GFNI_V256
{ FIRST_NI_GFNI_V512, LAST_NI_GFNI_V512 }, // GFNI_V512
{ NI_Illegal, NI_Illegal }, // SHA
{ NI_Illegal, NI_Illegal }, // WAITPKG
{ FIRST_NI_WAITPKG, LAST_NI_WAITPKG }, // WAITPKG
{ FIRST_NI_X86Serialize, LAST_NI_X86Serialize }, // X86Serialize
{ FIRST_NI_Vector128, LAST_NI_Vector128 }, // Vector128
{ FIRST_NI_Vector256, LAST_NI_Vector256 }, // Vector256
Expand Down
90 changes: 50 additions & 40 deletions src/coreclr/jit/hwintrinsiccodegenxarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -998,10 +998,12 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
break;
}

case InstructionSet_WAITPKG:
case InstructionSet_X86Base:
case InstructionSet_X86Base_X64:
case InstructionSet_X86Serialize:
{
genX86BaseIntrinsic(node, instOptions);
genHWIntrinsicSpecial(node, instOptions);
break;
}

Expand All @@ -1018,14 +1020,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
break;
}

case InstructionSet_X86Serialize:
case InstructionSet_X86Serialize_X64:
{
assert(instOptions == INS_OPTS_NONE);
genX86SerializeIntrinsic(node);
break;
}

default:
unreached();
break;
Expand Down Expand Up @@ -2378,12 +2372,12 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions)
}

//------------------------------------------------------------------------
// genX86BaseIntrinsic: Generates the code for an X86 base hardware intrinsic node
// genHWIntrinsicSpecial: Generates the code for an special hardware intrinsic node
//
// Arguments:
// node - The hardware intrinsic node
//
void CodeGen::genX86BaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions)
void CodeGen::genHWIntrinsicSpecial(GenTreeHWIntrinsic* node, insOpts instOptions)
{
NamedIntrinsic intrinsicId = node->GetHWIntrinsicId();
regNumber targetReg = node->GetRegNum();
Expand Down Expand Up @@ -2466,6 +2460,7 @@ void CodeGen::genX86BaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions)
case NI_X86Base_Prefetch1:
case NI_X86Base_Prefetch2:
case NI_X86Base_PrefetchNonTemporal:
case NI_WAITPKG_SetUpUserLevelMonitor:
{
assert(baseType == TYP_UBYTE);
assert(instOptions == INS_OPTS_NONE);
Expand Down Expand Up @@ -2534,6 +2529,14 @@ void CodeGen::genX86BaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions)
break;
}

case NI_X86Serialize_Serialize:
{
assert(instOptions == INS_OPTS_NONE);
assert(node->GetSimdBaseType() == TYP_UNKNOWN);
GetEmitter()->emitIns(INS_serialize);
break;
}

case NI_X86Base_ConvertToVector128Int16:
case NI_X86Base_ConvertToVector128Int32:
case NI_X86Base_ConvertToVector128Int64:
Expand Down Expand Up @@ -2652,6 +2655,42 @@ void CodeGen::genX86BaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions)
break;
}

case NI_WAITPKG_TimedPause:
case NI_WAITPKG_WaitForUserLevelMonitor:
{
assert(node->GetSimdBaseType() == TYP_INT);

assert(node->GetOperandCount() == 3);
assert(instOptions == INS_OPTS_NONE);

GenTree* op1 = node->Op(1);
GenTree* op2 = node->Op(2);
GenTree* op3 = node->Op(3);

instruction ins = HWIntrinsicInfo::lookupIns(intrinsicId, TYP_INT, compiler);

regNumber op1Reg = op1->GetRegNum();
regNumber op2Reg = op2->GetRegNum();
regNumber op3Reg = op3->GetRegNum();

emitAttr attr = emitTypeSize(TYP_INT);

// op1: free, op2: EDX, op3: EAX

assert(op1Reg != REG_EDX);
assert(op1Reg != REG_EAX);

assert(op2Reg != REG_EAX);
assert(op3Reg != REG_EDX);

emit->emitIns_Mov(INS_mov, attr, REG_EDX, op2Reg, /* canSkip */ true);
emit->emitIns_Mov(INS_mov, attr, REG_EAX, op3Reg, /* canSkip */ true);

// emit the TPAUSE/UMWAIT instruction
emit->emitIns_R(ins, attr, op1Reg);
break;
}

default:
unreached();
break;
Expand Down Expand Up @@ -3902,33 +3941,4 @@ void CodeGen::genXCNTIntrinsic(GenTreeHWIntrinsic* node, instruction ins)
genHWIntrinsic_R_RM(node, ins, emitTypeSize(node->TypeGet()), targetReg, op1, INS_OPTS_NONE);
}

//------------------------------------------------------------------------
// genX86SerializeIntrinsic: Generates the code for an X86 serialize hardware intrinsic node
//
// Arguments:
// node - The hardware intrinsic node
//
void CodeGen::genX86SerializeIntrinsic(GenTreeHWIntrinsic* node)
{
NamedIntrinsic intrinsicId = node->GetHWIntrinsicId();

genConsumeMultiOpOperands(node);

switch (intrinsicId)
{
case NI_X86Serialize_Serialize:
{
assert(node->GetSimdBaseType() == TYP_UNKNOWN);
GetEmitter()->emitIns(INS_serialize);
break;
}

default:
unreached();
break;
}

genProduceReg(node);
}

#endif // FEATURE_HW_INTRINSICS
13 changes: 12 additions & 1 deletion src/coreclr/jit/hwintrinsiclistxarch.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1158,7 +1158,7 @@ HARDWARE_INTRINSIC(AES_V512, CarrylessMultiply,
// ISA Function name SIMD size NumArg Instructions Category Flags
// {TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE}
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
// AVX512 Intrinsics for X86Serialize
// Intrinsics for X86Serialize
#define FIRST_NI_X86Serialize NI_X86Serialize_Serialize
HARDWARE_INTRINSIC(X86Serialize, Serialize, 0, 0, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Barrier)
#define LAST_NI_X86Serialize NI_X86Serialize_Serialize
Expand Down Expand Up @@ -1196,6 +1196,17 @@ HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldAffineTransformInverse,
HARDWARE_INTRINSIC(GFNI_V512, GaloisFieldMultiply, 64, 2, {INS_invalid, INS_gf2p8mulb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_NoFlag)
#define LAST_NI_GFNI_V512 NI_GFNI_V512_GaloisFieldMultiply

// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
// ISA Function name SIMD size NumArg Instructions Category Flags
// {TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE}
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
// Intrinsics for WaitPkg
#define FIRST_NI_WAITPKG NI_WAITPKG_SetUpUserLevelMonitor
HARDWARE_INTRINSIC(WAITPKG, SetUpUserLevelMonitor, 0, 1, {INS_invalid, INS_umonitor, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(WAITPKG, TimedPause, 0, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_tpause, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(WAITPKG, WaitForUserLevelMonitor, 0, -1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_umwait, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Special, HW_Flag_NoContainment|HW_Flag_NoRMWSemantics|HW_Flag_SpecialSideEffect_Other)
#define LAST_NI_WAITPKG NI_WAITPKG_WaitForUserLevelMonitor

// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
// ISA Function name SIMD size NumArg Instructions Category Flags
// {TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE}
Expand Down
24 changes: 24 additions & 0 deletions src/coreclr/jit/hwintrinsicxarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4295,6 +4295,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
case NI_X86Base_Prefetch1:
case NI_X86Base_Prefetch2:
case NI_X86Base_PrefetchNonTemporal:
case NI_WAITPKG_SetUpUserLevelMonitor:
{
assert(sig->numArgs == 1);
assert(JITtype2varType(sig->retType) == TYP_VOID);
Expand Down Expand Up @@ -4335,6 +4336,29 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
break;
}

case NI_WAITPKG_TimedPause:
case NI_WAITPKG_WaitForUserLevelMonitor:
{
assert(sig->numArgs == 2);
assert(JITtype2varType(sig->retType) == TYP_UBYTE);
assert(simdSize == 0);

op2 = impPopStack().val;
op1 = impPopStack().val;

GenTree* op2Hi = nullptr;
GenTree* op2Lo = nullptr;

GenTree* op2Dup = fgMakeMultiUse(&op2);

op2Hi = gtFoldExpr(gtNewOperNode(GT_RSZ, TYP_LONG, op2, gtNewIconNode(32)));
op2Hi = gtFoldExpr(gtNewCastNode(TYP_INT, op2Hi, false, TYP_INT));
op2Lo = gtFoldExpr(gtNewCastNode(TYP_INT, op2Dup, false, TYP_INT));

retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2Hi, op2Lo, intrinsic, CORINFO_TYPE_INT, 0);
break;
}

case NI_AVX2_PermuteVar8x32:
case NI_AVX512_PermuteVar4x64:
case NI_AVX512_PermuteVar8x16:
Expand Down
6 changes: 3 additions & 3 deletions src/coreclr/jit/instrsxarch.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1197,9 +1197,9 @@ INST3(popcnt_apx, "popcnt", IUM_WR, BAD_CODE, BAD_CODE,
#endif

// WAITPKG
INST3(tpause, "tpause", IUM_RD, BAD_CODE, BAD_CODE, PCKDBL(0xAE), ILLEGAL, ILLEGAL, INS_TT_NONE, Resets_OF | Resets_SF | Resets_ZF | Resets_AF | Resets_PF | Writes_CF) // Timed PAUSE
INST3(umonitor, "umonitor", IUM_RD, BAD_CODE, BAD_CODE, SSEFLT(0xAE), ILLEGAL, ILLEGAL, INS_TT_NONE, INS_FLAGS_None) // User Level Set Up Monitor Address
INST3(umwait, "umwait", IUM_RD, BAD_CODE, BAD_CODE, SSEDBL(0xAE), ILLEGAL, ILLEGAL, INS_TT_NONE, Resets_OF | Resets_SF | Resets_ZF | Resets_AF | Resets_PF | Writes_CF) // User Level Monitor Wait
INST3(tpause, "tpause", IUM_RD, PCKDBL(0xAE), BAD_CODE, BAD_CODE, 140C, 1C, INS_TT_NONE, Resets_OF | Resets_SF | Resets_ZF | Resets_AF | Resets_PF | Writes_CF) // Timed PAUSE
INST3(umonitor, "umonitor", IUM_RD, SSEFLT(0xAE), BAD_CODE, BAD_CODE, 140C, 1C, INS_TT_NONE, INS_FLAGS_None) // User Level Set Up Monitor Address
INST3(umwait, "umwait", IUM_RD, SSEDBL(0xAE), BAD_CODE, BAD_CODE, 140C, 1C, INS_TT_NONE, Resets_OF | Resets_SF | Resets_ZF | Resets_AF | Resets_PF | Writes_CF) // User Level Monitor Wait

INST3(neg, "neg", IUM_RW, 0x0018F6, BAD_CODE, 0x0018F6, ILLEGAL, ILLEGAL, INS_TT_NONE, Writes_OF | Writes_SF | Writes_ZF | Writes_AF | Writes_PF | Writes_CF | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD | INS_Flags_Has_NF)
INST3(not, "not", IUM_RW, 0x0010F6, BAD_CODE, 0x0010F6, ILLEGAL, ILLEGAL, INS_TT_NONE, INS_FLAGS_None | INS_FLAGS_Has_Wbit | Encoding_REX2 | INS_Flags_Has_NDD)
Expand Down
10 changes: 10 additions & 0 deletions src/coreclr/jit/lowerxarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2615,6 +2615,16 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node)
break;
}

case NI_WAITPKG_TimedPause:
case NI_WAITPKG_WaitForUserLevelMonitor:
{
GenTree* cc = LowerNodeCC(node, GenCondition::C);

node->gtType = TYP_VOID;
node->ClearUnusedValue();
break;
}

default:
break;
}
Expand Down
Loading
Loading