Skip to content

Commit ceccafa

Browse files
a74nha74nh
committed
UnzipEven() and Count16BitElements() intrinsics for SVE
Many of the files touched here are autogenerated via InstructionSetDesc.txt
1 parent 72a5613 commit ceccafa

28 files changed

+475
-24
lines changed

‎src/coreclr/inc/clrconfigvalues.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -798,6 +798,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sha1, W("EnableArm64Sh
798798
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sha256, W("EnableArm64Sha256"), 1, "Allows Arm64 Sha256+ hardware intrinsics to be disabled")
799799
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Rcpc, W("EnableArm64Rcpc"), 1, "Allows Arm64 Rcpc+ hardware intrinsics to be disabled")
800800
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Rcpc2, W("EnableArm64Rcpc2"), 1, "Allows Arm64 Rcpc2+ hardware intrinsics to be disabled")
801+
RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sve, W("EnableArm64Sve"), 1, "Allows Arm64 SVE hardware intrinsics to be disabled")
801802
#endif
802803

803804
///

‎src/coreclr/inc/corinfoinstructionset.h‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,16 @@ enum CORINFO_InstructionSet
3131
InstructionSet_Rcpc=13,
3232
InstructionSet_VectorT128=14,
3333
InstructionSet_Rcpc2=15,
34-
InstructionSet_ArmBase_Arm64=16,
35-
InstructionSet_AdvSimd_Arm64=17,
36-
InstructionSet_Aes_Arm64=18,
37-
InstructionSet_Crc32_Arm64=19,
38-
InstructionSet_Dp_Arm64=20,
39-
InstructionSet_Rdm_Arm64=21,
40-
InstructionSet_Sha1_Arm64=22,
41-
InstructionSet_Sha256_Arm64=23,
34+
InstructionSet_Sve=16,
35+
InstructionSet_ArmBase_Arm64=17,
36+
InstructionSet_AdvSimd_Arm64=18,
37+
InstructionSet_Aes_Arm64=19,
38+
InstructionSet_Crc32_Arm64=20,
39+
InstructionSet_Dp_Arm64=21,
40+
InstructionSet_Rdm_Arm64=22,
41+
InstructionSet_Sha1_Arm64=23,
42+
InstructionSet_Sha256_Arm64=24,
43+
InstructionSet_Sve_Arm64=25,
4244
#endif // TARGET_ARM64
4345
#ifdef TARGET_AMD64
4446
InstructionSet_X86Base=1,
@@ -282,6 +284,8 @@ struct CORINFO_InstructionSetFlags
282284
AddInstructionSet(InstructionSet_Sha1_Arm64);
283285
if (HasInstructionSet(InstructionSet_Sha256))
284286
AddInstructionSet(InstructionSet_Sha256_Arm64);
287+
if (HasInstructionSet(InstructionSet_Sve))
288+
AddInstructionSet(InstructionSet_Sve_Arm64);
285289
#endif // TARGET_ARM64
286290
#ifdef TARGET_AMD64
287291
if (HasInstructionSet(InstructionSet_X86Base))
@@ -394,6 +398,10 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins
394398
resultflags.RemoveInstructionSet(InstructionSet_Sha256);
395399
if (resultflags.HasInstructionSet(InstructionSet_Sha256_Arm64) && !resultflags.HasInstructionSet(InstructionSet_Sha256))
396400
resultflags.RemoveInstructionSet(InstructionSet_Sha256_Arm64);
401+
if (resultflags.HasInstructionSet(InstructionSet_Sve) && !resultflags.HasInstructionSet(InstructionSet_Sve_Arm64))
402+
resultflags.RemoveInstructionSet(InstructionSet_Sve);
403+
if (resultflags.HasInstructionSet(InstructionSet_Sve_Arm64) && !resultflags.HasInstructionSet(InstructionSet_Sve))
404+
resultflags.RemoveInstructionSet(InstructionSet_Sve_Arm64);
397405
if (resultflags.HasInstructionSet(InstructionSet_AdvSimd) && !resultflags.HasInstructionSet(InstructionSet_ArmBase))
398406
resultflags.RemoveInstructionSet(InstructionSet_AdvSimd);
399407
if (resultflags.HasInstructionSet(InstructionSet_Aes) && !resultflags.HasInstructionSet(InstructionSet_ArmBase))
@@ -414,6 +422,8 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins
414422
resultflags.RemoveInstructionSet(InstructionSet_Vector128);
415423
if (resultflags.HasInstructionSet(InstructionSet_VectorT128) && !resultflags.HasInstructionSet(InstructionSet_AdvSimd))
416424
resultflags.RemoveInstructionSet(InstructionSet_VectorT128);
425+
if (resultflags.HasInstructionSet(InstructionSet_Sve) && !resultflags.HasInstructionSet(InstructionSet_AdvSimd))
426+
resultflags.RemoveInstructionSet(InstructionSet_Sve);
417427
#endif // TARGET_ARM64
418428
#ifdef TARGET_AMD64
419429
if (resultflags.HasInstructionSet(InstructionSet_X86Base) && !resultflags.HasInstructionSet(InstructionSet_X86Base_X64))
@@ -764,6 +774,10 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet)
764774
return "VectorT128";
765775
case InstructionSet_Rcpc2 :
766776
return "Rcpc2";
777+
case InstructionSet_Sve :
778+
return "Sve";
779+
case InstructionSet_Sve_Arm64 :
780+
return "Sve_Arm64";
767781
#endif // TARGET_ARM64
768782
#ifdef TARGET_AMD64
769783
case InstructionSet_X86Base :
@@ -998,6 +1012,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst
9981012
case READYTORUN_INSTRUCTION_Rcpc: return InstructionSet_Rcpc;
9991013
case READYTORUN_INSTRUCTION_VectorT128: return InstructionSet_VectorT128;
10001014
case READYTORUN_INSTRUCTION_Rcpc2: return InstructionSet_Rcpc2;
1015+
case READYTORUN_INSTRUCTION_Sve: return InstructionSet_Sve;
10011016
#endif // TARGET_ARM64
10021017
#ifdef TARGET_AMD64
10031018
case READYTORUN_INSTRUCTION_X86Base: return InstructionSet_X86Base;

‎src/coreclr/inc/jiteeversionguid.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,13 @@ typedef const GUID *LPCGUID;
4343
#define GUID_DEFINED
4444
#endif // !GUID_DEFINED
4545

46-
constexpr GUID JITEEVersionIdentifier = { /* a2974440-e8ee-4d95-9e6e-799a330be1a0 */
47-
0xa2974440,
48-
0xe8ee,
49-
0x4d95,
50-
{0x9e, 0x6e, 0x79, 0x9a, 0x33, 0x0b, 0xe1, 0xa0}
46+
constexpr GUID JITEEVersionIdentifier = { /* e15e62ce-d9c6-418a-a5a7-26ad17fcf4bf */
47+
0xe15e62ce,
48+
0xd9c6,
49+
0x418a,
50+
{0xa5, 0xa7, 0x26, 0xad, 0x17, 0xfc, 0xf4, 0xbf}
5151
};
52-
52+
5353
//////////////////////////////////////////////////////////////////////////////////////////////////////////
5454
//
5555
// END JITEEVersionIdentifier

‎src/coreclr/inc/readytoruninstructionset.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ enum ReadyToRunInstructionSet
5151
READYTORUN_INSTRUCTION_VectorT256=40,
5252
READYTORUN_INSTRUCTION_VectorT512=41,
5353
READYTORUN_INSTRUCTION_Rcpc2=42,
54+
READYTORUN_INSTRUCTION_Sve=43,
5455

5556
};
5657

‎src/coreclr/jit/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,7 @@ set( JIT_ARM64_HEADERS
400400
emitfmtsarm64.h
401401
emitfmtsarm64sve.h
402402
hwintrinsiclistarm64.h
403+
hwintrinsiclistarm64sve.h
403404
instrsarm64.h
404405
instrsarm64sve.h
405406
registerarm64.h

‎src/coreclr/jit/compiler.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6013,6 +6013,11 @@ int Compiler::compCompile(CORINFO_MODULE_HANDLE classPtr,
60136013
{
60146014
instructionSetFlags.AddInstructionSet(InstructionSet_Dczva);
60156015
}
6016+
6017+
if (JitConfig.EnableArm64Sve() != 0)
6018+
{
6019+
instructionSetFlags.AddInstructionSet(InstructionSet_Sve);
6020+
}
60166021
#elif defined(TARGET_XARCH)
60176022
if (JitConfig.EnableHWIntrinsic() != 0)
60186023
{

‎src/coreclr/jit/emitarm64.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8035,6 +8035,14 @@ void emitter::emitIns_R_R_R(
80358035
fmt = IF_DV_3A;
80368036
break;
80378037

8038+
case INS_sve_uzp1:
8039+
// TODO: This could also be SVE_BR_3B or SVE_CI_3A
8040+
assert(isVectorRegister(reg1));
8041+
assert(isVectorRegister(reg2));
8042+
assert(isVectorRegister(reg3));
8043+
fmt = IF_SVE_BR_3A;
8044+
break;
8045+
80388046
default:
80398047
unreached();
80408048
break;

‎src/coreclr/jit/hwintrinsic.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1391,8 +1391,8 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
13911391
{
13921392
case 0:
13931393
{
1394-
assert(!isScalar);
1395-
retNode = gtNewSimdHWIntrinsicNode(retType, intrinsic, simdBaseJitType, simdSize);
1394+
retNode = isScalar ? gtNewScalarHWIntrinsicNode(retType, intrinsic)
1395+
: gtNewSimdHWIntrinsicNode(retType, intrinsic, simdBaseJitType, simdSize);
13961396
break;
13971397
}
13981398

‎src/coreclr/jit/hwintrinsicarm64.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ static CORINFO_InstructionSet Arm64VersionOfIsa(CORINFO_InstructionSet isa)
3434
return InstructionSet_Sha256_Arm64;
3535
case InstructionSet_Rdm:
3636
return InstructionSet_Rdm_Arm64;
37+
case InstructionSet_Sve:
38+
return InstructionSet_Sve_Arm64;
3739
default:
3840
return InstructionSet_NONE;
3941
}
@@ -97,6 +99,10 @@ static CORINFO_InstructionSet lookupInstructionSet(const char* className)
9799
{
98100
return InstructionSet_Sha256;
99101
}
102+
if (strcmp(className, "Sve") == 0)
103+
{
104+
return InstructionSet_Sve;
105+
}
100106
}
101107
else if (className[0] == 'V')
102108
{
@@ -166,6 +172,8 @@ bool HWIntrinsicInfo::isFullyImplementedIsa(CORINFO_InstructionSet isa)
166172
case InstructionSet_Sha1_Arm64:
167173
case InstructionSet_Sha256:
168174
case InstructionSet_Sha256_Arm64:
175+
case InstructionSet_Sve:
176+
case InstructionSet_Sve_Arm64:
169177
case InstructionSet_Vector64:
170178
case InstructionSet_Vector128:
171179
return true;

‎src/coreclr/jit/hwintrinsiccodegenarm64.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,10 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
419419
GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op2Reg, op3Reg, opt);
420420
break;
421421

422+
case 0:
423+
GetEmitter()->emitIns_R(ins, emitSize, targetReg);
424+
break;
425+
422426
default:
423427
unreached();
424428
}

0 commit comments

Comments
 (0)