Count16BitElements() intrinsic for SVE

src/coreclr/inc/clrconfigvalues.h

@@ -798,6 +798,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sha1,              W("EnableArm64Sh
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sha256,            W("EnableArm64Sha256"),         1, "Allows Arm64 Sha256+ hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Rcpc,              W("EnableArm64Rcpc"),           1, "Allows Arm64 Rcpc+ hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Rcpc2,             W("EnableArm64Rcpc2"),          1, "Allows Arm64 Rcpc2+ hardware intrinsics to be disabled")
+RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sve,               W("EnableArm64Sve"),            1, "Allows Arm64 SVE hardware intrinsics to be disabled")
 #endif
 
 ///

src/coreclr/inc/corinfoinstructionset.h

@@ -31,14 +31,16 @@ enum CORINFO_InstructionSet
     InstructionSet_Rcpc=13,
     InstructionSet_VectorT128=14,
     InstructionSet_Rcpc2=15,
-    InstructionSet_ArmBase_Arm64=16,
-    InstructionSet_AdvSimd_Arm64=17,
-    InstructionSet_Aes_Arm64=18,
-    InstructionSet_Crc32_Arm64=19,
-    InstructionSet_Dp_Arm64=20,
-    InstructionSet_Rdm_Arm64=21,
-    InstructionSet_Sha1_Arm64=22,
-    InstructionSet_Sha256_Arm64=23,
+    InstructionSet_Sve=16,
+    InstructionSet_ArmBase_Arm64=17,
+    InstructionSet_AdvSimd_Arm64=18,
+    InstructionSet_Aes_Arm64=19,
+    InstructionSet_Crc32_Arm64=20,
+    InstructionSet_Dp_Arm64=21,
+    InstructionSet_Rdm_Arm64=22,
+    InstructionSet_Sha1_Arm64=23,
+    InstructionSet_Sha256_Arm64=24,
+    InstructionSet_Sve_Arm64=25,
 #endif // TARGET_ARM64
 #ifdef TARGET_AMD64
     InstructionSet_X86Base=1,
@@ -282,6 +284,8 @@ struct CORINFO_InstructionSetFlags
             AddInstructionSet(InstructionSet_Sha1_Arm64);
         if (HasInstructionSet(InstructionSet_Sha256))
             AddInstructionSet(InstructionSet_Sha256_Arm64);
+        if (HasInstructionSet(InstructionSet_Sve))
+            AddInstructionSet(InstructionSet_Sve_Arm64);
 #endif // TARGET_ARM64
 #ifdef TARGET_AMD64
         if (HasInstructionSet(InstructionSet_X86Base))
@@ -394,6 +398,10 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins
             resultflags.RemoveInstructionSet(InstructionSet_Sha256);
         if (resultflags.HasInstructionSet(InstructionSet_Sha256_Arm64) && !resultflags.HasInstructionSet(InstructionSet_Sha256))
             resultflags.RemoveInstructionSet(InstructionSet_Sha256_Arm64);
+        if (resultflags.HasInstructionSet(InstructionSet_Sve) && !resultflags.HasInstructionSet(InstructionSet_Sve_Arm64))
+            resultflags.RemoveInstructionSet(InstructionSet_Sve);
+        if (resultflags.HasInstructionSet(InstructionSet_Sve_Arm64) && !resultflags.HasInstructionSet(InstructionSet_Sve))
+            resultflags.RemoveInstructionSet(InstructionSet_Sve_Arm64);
         if (resultflags.HasInstructionSet(InstructionSet_AdvSimd) && !resultflags.HasInstructionSet(InstructionSet_ArmBase))
             resultflags.RemoveInstructionSet(InstructionSet_AdvSimd);
         if (resultflags.HasInstructionSet(InstructionSet_Aes) && !resultflags.HasInstructionSet(InstructionSet_ArmBase))
@@ -414,6 +422,8 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins
             resultflags.RemoveInstructionSet(InstructionSet_Vector128);
         if (resultflags.HasInstructionSet(InstructionSet_VectorT128) && !resultflags.HasInstructionSet(InstructionSet_AdvSimd))
             resultflags.RemoveInstructionSet(InstructionSet_VectorT128);
+        if (resultflags.HasInstructionSet(InstructionSet_Sve) && !resultflags.HasInstructionSet(InstructionSet_AdvSimd))
+            resultflags.RemoveInstructionSet(InstructionSet_Sve);
 #endif // TARGET_ARM64
 #ifdef TARGET_AMD64
         if (resultflags.HasInstructionSet(InstructionSet_X86Base) && !resultflags.HasInstructionSet(InstructionSet_X86Base_X64))
@@ -764,6 +774,10 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet)
             return "VectorT128";
         case InstructionSet_Rcpc2 :
             return "Rcpc2";
+        case InstructionSet_Sve :
+            return "Sve";
+        case InstructionSet_Sve_Arm64 :
+            return "Sve_Arm64";
 #endif // TARGET_ARM64
 #ifdef TARGET_AMD64
         case InstructionSet_X86Base :
@@ -998,6 +1012,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst
         case READYTORUN_INSTRUCTION_Rcpc: return InstructionSet_Rcpc;
         case READYTORUN_INSTRUCTION_VectorT128: return InstructionSet_VectorT128;
         case READYTORUN_INSTRUCTION_Rcpc2: return InstructionSet_Rcpc2;
+        case READYTORUN_INSTRUCTION_Sve: return InstructionSet_Sve;
 #endif // TARGET_ARM64
 #ifdef TARGET_AMD64
         case READYTORUN_INSTRUCTION_X86Base: return InstructionSet_X86Base;

src/coreclr/inc/jiteeversionguid.h

@@ -43,13 +43,13 @@ typedef const GUID *LPCGUID;
 #define GUID_DEFINED
 #endif // !GUID_DEFINED
 
-constexpr GUID JITEEVersionIdentifier = { /* a2974440-e8ee-4d95-9e6e-799a330be1a0 */
-    0xa2974440,
-    0xe8ee,
-    0x4d95,
-    {0x9e, 0x6e, 0x79, 0x9a, 0x33, 0x0b, 0xe1, 0xa0}
+constexpr GUID JITEEVersionIdentifier = { /* e15e62ce-d9c6-418a-a5a7-26ad17fcf4bf */
+    0xe15e62ce,
+    0xd9c6,
+    0x418a,
+    {0xa5, 0xa7, 0x26, 0xad, 0x17, 0xfc, 0xf4, 0xbf}
   };
-  
+
 //////////////////////////////////////////////////////////////////////////////////////////////////////////
 //
 // END JITEEVersionIdentifier

src/coreclr/inc/readytoruninstructionset.h

@@ -51,6 +51,7 @@ enum ReadyToRunInstructionSet
     READYTORUN_INSTRUCTION_VectorT256=40,
     READYTORUN_INSTRUCTION_VectorT512=41,
     READYTORUN_INSTRUCTION_Rcpc2=42,
+    READYTORUN_INSTRUCTION_Sve=43,
 
 };
 

src/coreclr/jit/CMakeLists.txt

@@ -398,8 +398,11 @@ set( JIT_I386_HEADERS ${JIT_AMD64_HEADERS} )
 set( JIT_ARM64_HEADERS
     emitarm64.h
     emitfmtsarm64.h
+    emitfmtsarm64sve.h
     hwintrinsiclistarm64.h
+    hwintrinsiclistarm64sve.h
     instrsarm64.h
+    instrsarm64sve.h
     registerarm64.h
     simdashwintrinsiclistarm64.h
 )

src/coreclr/jit/codegenarm64.cpp

@@ -6702,6 +6702,14 @@ void CodeGen::genArm64EmitterUnitTests()
     theEmitter->emitIns_R_R_R(INS_asrv, EA_4BYTE, REG_R8, REG_R9, REG_R10);
     theEmitter->emitIns_R_R_R(INS_rorv, EA_4BYTE, REG_R8, REG_R9, REG_R10);
 
+    // TODO-SVE: Fix once we add predicate registers
+    theEmitter->emitIns_R_R_R(INS_sve_uzp1, EA_8BYTE, REG_R0, REG_R1, REG_R2,
+                              INS_OPTS_8B); // UZP1    <Zd>.<T>, <Zn>.<T>, <Zm>.<T>
+    theEmitter->emitIns_R_R_R(INS_sve_uzp1, EA_8BYTE, REG_R0, REG_R1, REG_R2,
+                              INS_OPTS_8B); // UZP1    <Zd>.Q, <Zn>.Q, <Zm>.Q
+    theEmitter->emitIns_R_R_R(INS_sve_uzp1, EA_8BYTE, REG_R0, REG_R1, REG_R2,
+                              INS_OPTS_8B); // UZP1    <Pd>.<T>, <Pn>.<T>, <Pm>.<T>
+
 #endif // ALL_ARM64_EMITTER_UNIT_TESTS
 
 #ifdef ALL_ARM64_EMITTER_UNIT_TESTS

src/coreclr/jit/compiler.cpp

@@ -6013,6 +6013,11 @@ int Compiler::compCompile(CORINFO_MODULE_HANDLE classPtr,
         {
             instructionSetFlags.AddInstructionSet(InstructionSet_Dczva);
         }
+
+        if (JitConfig.EnableArm64Sve() != 0)
+        {
+            instructionSetFlags.AddInstructionSet(InstructionSet_Sve);
+        }
 #elif defined(TARGET_XARCH)
         if (JitConfig.EnableHWIntrinsic() != 0)
         {

src/coreclr/jit/emit.cpp

@@ -130,6 +130,10 @@ const char* emitter::emitIfName(unsigned f)
     static const char* const ifNames[] = {
 #define IF_DEF(en, op1, op2) "IF_" #en,
 #include "emitfmts.h"
+#if defined(TARGET_ARM64)
+#define IF_DEF(en, op1, op2) "IF_" #en,
+#include "emitfmtsarm64sve.h"
+#endif
     };
 
     static char errBuff[32];
@@ -3767,6 +3771,10 @@ emitter::instrDesc* emitter::emitNewInstrCallDir(int              argCnt,
 const BYTE emitter::emitFmtToOps[] = {
 #define IF_DEF(en, op1, op2) ID_OP_##op2,
 #include "emitfmts.h"
+#if defined(TARGET_ARM64)
+#define IF_DEF(en, op1, op2) ID_OP_##op2,
+#include "emitfmtsarm64sve.h"
+#endif
 };
 
 #ifdef DEBUG

src/coreclr/jit/emit.h

@@ -565,7 +565,10 @@ class emitter
     {
 #define IF_DEF(en, op1, op2) IF_##en,
 #include "emitfmts.h"
-
+#if defined(TARGET_ARM64)
+#define IF_DEF(en, op1, op2) IF_##en,
+#include "emitfmtsarm64sve.h"
+#endif
         IF_COUNT
     };
 
@@ -628,8 +631,8 @@ class emitter
 #define MAX_ENCODED_SIZE 15
 #elif defined(TARGET_ARM64)
 #define INSTR_ENCODED_SIZE 4
-        static_assert_no_msg(INS_count <= 1024);
-        instruction _idIns : 10;
+        static_assert_no_msg(INS_count <= 2048);
+        instruction _idIns : 11;
 #elif defined(TARGET_LOONGARCH64)
         // TODO-LoongArch64: not include SIMD-vector.
         static_assert_no_msg(INS_count <= 512);
@@ -647,6 +650,9 @@ class emitter
         unsigned    _idCodeSize : 5; // the instruction(s) size of this instrDesc described.
 #elif defined(TARGET_RISCV64)
         unsigned    _idCodeSize : 6; // the instruction(s) size of this instrDesc described.
+#elif defined(TARGET_ARM64)
+        static_assert_no_msg(IF_COUNT <= 1024);
+        insFormat _idInsFmt : 10;
 #else
         static_assert_no_msg(IF_COUNT <= 256);
         insFormat _idInsFmt : 8;
@@ -712,7 +718,7 @@ class emitter
         // x86:         17 bits
         // amd64:       17 bits
         // arm:         16 bits
-        // arm64:       18 bits
+        // arm64:       21 bits
         // loongarch64: 14 bits
         // risc-v:      14 bits
 
@@ -722,10 +728,13 @@ class emitter
         opSize   _idOpSize : 3;   // operand size: 0=1 , 1=2 , 2=4 , 3=8, 4=16, 5=32
                                   // At this point we have fully consumed first DWORD so that next field
                                   // doesn't cross a byte boundary.
-#elif defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
+#elif defined(TARGET_ARM64)
+        opSize  _idOpSize : 3; // operand size: 0=1 , 1=2 , 2=4 , 3=8, 4=16
+        insOpts _idInsOpt : 6; // options for instructions
+#elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
 /* _idOpSize defined below. */
 #else
-        opSize _idOpSize : 2; // operand size: 0=1 , 1=2 , 2=4 , 3=8
+        opSize    _idOpSize : 2; // operand size: 0=1 , 1=2 , 2=4 , 3=8
 #endif // TARGET_ARM64 || TARGET_LOONGARCH64 || TARGET_RISCV64
 
         // On Amd64, this is where the second DWORD begins
@@ -754,7 +763,7 @@ class emitter
         // x86:         38 bits
         // amd64:       38 bits
         // arm:         32 bits
-        // arm64:       32 bits
+        // arm64:       44 bits
         // loongarch64: 28 bits
         // risc-v:      28 bits
 
@@ -774,8 +783,7 @@ class emitter
 #endif                                //  TARGET_XARCH
 
 #ifdef TARGET_ARM64
-        opSize   _idOpSize : 3;    // operand size: 0=1 , 1=2 , 2=4 , 3=8, 4=16
-        insOpts  _idInsOpt : 6;    // options for instructions
+
         unsigned _idLclVar : 1;    // access a local on stack
         unsigned _idLclVarPair : 1 // carries information for 2 GC lcl vars.
 #endif
@@ -808,7 +816,7 @@ class emitter
         // x86:         47 bits
         // amd64:       47 bits
         // arm:         48 bits
-        // arm64:       50 bits
+        // arm64:       53 bits
         // loongarch64: 46 bits
         // risc-v:      46 bits
 
@@ -821,7 +829,7 @@ class emitter
 #if defined(TARGET_ARM)
 #define ID_EXTRA_BITFIELD_BITS (16)
 #elif defined(TARGET_ARM64)
-#define ID_EXTRA_BITFIELD_BITS (18)
+#define ID_EXTRA_BITFIELD_BITS (21)
 #elif defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
 #define ID_EXTRA_BITFIELD_BITS (14)
 #elif defined(TARGET_XARCH)
@@ -863,7 +871,7 @@ class emitter
         // x86:         53/49 bits
         // amd64:       54/49 bits
         // arm:         54/50 bits
-        // arm64:       57/52 bits
+        // arm64:       60/55 bits
         // loongarch64: 53/48 bits
         // risc-v:      53/48 bits
         CLANG_FORMAT_COMMENT_ANCHOR;
@@ -880,7 +888,7 @@ class emitter
         // x86:         11/15 bits
         // amd64:       10/15 bits
         // arm:         10/14 bits
-        // arm64:        7/12 bits
+        // arm64:        4/9 bits
         // loongarch64: 11/16 bits
         // risc-v:      11/16 bits
         CLANG_FORMAT_COMMENT_ANCHOR;
@@ -1132,7 +1140,7 @@ class emitter
             _idCodeSize = sz;
         }
 #elif defined(TARGET_RISCV64)
-        unsigned idCodeSize() const
+        unsigned  idCodeSize() const
         {
             return _idCodeSize;
         }