SVE2 API for AddCarryWideningLower

Change-Id: I24b0cc3474dcfaee40ad735d3ec274ed48e97035

Author: jacob-crawley

src/coreclr/jit/hwintrinsiclistarm64sve.h

@@ -318,6 +318,7 @@ HARDWARE_INTRINSIC(Sve2,          AbsoluteDifferenceAddWideningLower,
 HARDWARE_INTRINSIC(Sve2,          AbsoluteDifferenceAddWideningUpper,                               -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sabalt,     INS_sve_uabalt,     INS_sve_sabalt,     INS_sve_uabalt,     INS_sve_sabalt,     INS_sve_uabalt,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics)
 HARDWARE_INTRINSIC(Sve2,          AbsoluteDifferenceWideningLower,                                  -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_sabdlb,     INS_sve_uabdlb,     INS_sve_sabdlb,     INS_sve_uabdlb,     INS_sve_sabdlb,     INS_sve_uabdlb,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
 HARDWARE_INTRINSIC(Sve2,          AbsoluteDifferenceWideningUpper,                                  -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_sabdlt,     INS_sve_uabdlt,     INS_sve_sabdlt,     INS_sve_uabdlt,     INS_sve_sabdlt,     INS_sve_uabdlt,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
+HARDWARE_INTRINSIC(Sve2,          AddCarryWideningLower,                                            -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_adclb,      INS_invalid,        INS_sve_adclb,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
 HARDWARE_INTRINSIC(Sve2,          BitwiseClearXor,                                                  -1,      3,     {INS_sve_bcax,       INS_sve_bcax,       INS_sve_bcax,       INS_sve_bcax,       INS_sve_bcax,       INS_sve_bcax,       INS_sve_bcax,       INS_sve_bcax,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
 HARDWARE_INTRINSIC(Sve2,          BitwiseSelect,                                                    -1,      3,     {INS_sve_bsl,        INS_sve_bsl,        INS_sve_bsl,        INS_sve_bsl,        INS_sve_bsl,        INS_sve_bsl,        INS_sve_bsl,        INS_sve_bsl,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
 HARDWARE_INTRINSIC(Sve2,          BitwiseSelectLeftInverted,                                        -1,      3,     {INS_sve_bsl1n,      INS_sve_bsl1n,      INS_sve_bsl1n,      INS_sve_bsl1n,      INS_sve_bsl1n,      INS_sve_bsl1n,      INS_sve_bsl1n,      INS_sve_bsl1n,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)

src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.PlatformNotSupported.cs

@@ -230,6 +230,20 @@ internal Arm64() { }
         /// </summary>
         public static Vector<ulong> AbsoluteDifferenceWideningUpper(Vector<uint> left, Vector<uint> right) { throw new PlatformNotSupportedException(); }
 
+        // Add with carry long (bottom)
+
+        /// <summary>
+        /// svuint32_t svadclb[_u32](svuint32_t op1, svuint32_t op2, svuint32_t op3)
+        ///   ADCLB Ztied1.S, Zop2.S, Zop3.S
+        /// </summary>
+        public static unsafe Vector<uint> AddCarryWideningLower(Vector<uint> op1, Vector<uint> op2, Vector<uint> op3) { throw new PlatformNotSupportedException(); }
+
+        /// <summary>
+        /// svuint64_t svadclb[_u64](svuint64_t op1, svuint64_t op2, svuint64_t op3)
+        ///   ADCLB Ztied1.D, Zop2.D, Zop3.D
+        /// </summary>
+        public static unsafe Vector<ulong> AddCarryWideningLower(Vector<ulong> op1, Vector<ulong> op2, Vector<ulong> op3) { throw new PlatformNotSupportedException(); }
+
         // Bitwise clear and exclusive OR
 
         /// <summary>

src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.cs

@@ -230,6 +230,20 @@ internal Arm64() { }
         /// </summary>
         public static Vector<ulong> AbsoluteDifferenceWideningUpper(Vector<uint> left, Vector<uint> right) => AbsoluteDifferenceWideningUpper(left, right);
 
+        // Add with carry long (bottom)
+
+        /// <summary>
+        /// svuint32_t svadclb[_u32](svuint32_t op1, svuint32_t op2, svuint32_t op3)
+        ///   ADCLB Ztied1.S, Zop2.S, Zop3.S
+        /// </summary>
+        public static unsafe Vector<uint> AddCarryWideningLower(Vector<uint> op1, Vector<uint> op2, Vector<uint> op3) => AddCarryWideningLower(op1, op2, op3);
+
+        /// <summary>
+        /// svuint64_t svadclb[_u64](svuint64_t op1, svuint64_t op2, svuint64_t op3)
+        ///   ADCLB Ztied1.D, Zop2.D, Zop3.D
+        /// </summary>
+        public static unsafe Vector<ulong> AddCarryWideningLower(Vector<ulong> op1, Vector<ulong> op2, Vector<ulong> op3) => AddCarryWideningLower(op1, op2, op3);
+
         // Bitwise clear and exclusive OR
 
         /// <summary>

src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs

@@ -6116,6 +6116,9 @@ internal Arm64() { }
         public static System.Numerics.Vector<uint> AbsoluteDifferenceWideningUpper(System.Numerics.Vector<ushort> left, System.Numerics.Vector<ushort> right) { throw null; }
         public static System.Numerics.Vector<ulong> AbsoluteDifferenceWideningUpper(System.Numerics.Vector<uint> left, System.Numerics.Vector<uint> right) { throw null; }
 
+        public static System.Numerics.Vector<uint> AddCarryWideningLower(System.Numerics.Vector<uint> op1, System.Numerics.Vector<uint> op2, System.Numerics.Vector<uint> op3) { throw null; }
+        public static System.Numerics.Vector<ulong> AddCarryWideningLower(System.Numerics.Vector<ulong> op1, System.Numerics.Vector<ulong> op2, System.Numerics.Vector<ulong> op3) { throw null; }
+
         public static System.Numerics.Vector<byte> BitwiseClearXor(System.Numerics.Vector<byte> xor, System.Numerics.Vector<byte> value, System.Numerics.Vector<byte> mask) { throw null; }
         public static System.Numerics.Vector<short> BitwiseClearXor(System.Numerics.Vector<short> xor, System.Numerics.Vector<short> value, System.Numerics.Vector<short> mask) { throw null; }
         public static System.Numerics.Vector<int> BitwiseClearXor(System.Numerics.Vector<int> xor, System.Numerics.Vector<int> value, System.Numerics.Vector<int> mask) { throw null; }

src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs

@@ -4817,6 +4817,9 @@
     ("SveVecBinOpDifferentRetType.template",      new Dictionary<string, string> {["TestName"] = "Sve2_AbsoluteDifferenceWideningUpper_uint_ushort",                                             ["Isa"] = "Sve2",          ["LoadIsa"] = "Sve2",    ["Method"] = "AbsoluteDifferenceWideningUpper",                                      ["RetVectorType"] = "Vector",    ["RetBaseType"] = "UInt32",  ["Op1VectorType"] = "Vector",    ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector",    ["Op2BaseType"] = "UInt16",                                                              ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()",        ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()",                                                                                       ["ConvertFunc"] = "",                                   ["ValidateIterResult"] = "Helpers.AbsoluteDifferenceWideningOdd(left, right, i) != result[i]", ["GetIterResult"] = "(UInt32) Helpers.AbsoluteDifferenceWideningOdd(left, right, i)"}),
     ("SveVecBinOpDifferentRetType.template",      new Dictionary<string, string> {["TestName"] = "Sve2_AbsoluteDifferenceWideningUpper_ulong_uint",                                              ["Isa"] = "Sve2",          ["LoadIsa"] = "Sve2",    ["Method"] = "AbsoluteDifferenceWideningUpper",                                      ["RetVectorType"] = "Vector",    ["RetBaseType"] = "UInt64",  ["Op1VectorType"] = "Vector",    ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector",    ["Op2BaseType"] = "UInt32",                                                              ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()",        ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()",                                                                                       ["ConvertFunc"] = "",                                   ["ValidateIterResult"] = "Helpers.AbsoluteDifferenceWideningOdd(left, right, i) != result[i]", ["GetIterResult"] = "(UInt64) Helpers.AbsoluteDifferenceWideningOdd(left, right, i)"}),
 
+    ("SveVecTernOpTest.template",            new Dictionary<string, string> { ["TestName"] = "Sve2_AddCarryWideningLower_uint",                                                                            ["Isa"] = "Sve2",          ["LoadIsa"] = "Sve2",    ["Method"] = "AddCarryWideningLower",                                                ["RetVectorType"] = "Vector",    ["RetBaseType"] = "UInt32",  ["Op1VectorType"] = "Vector",    ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector",    ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector",    ["Op3BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()",        ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()",        ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()",                        ["ConvertFunc"] = "",                                   ["ValidateIterResult"] = "Helpers.AddCarryWideningEven(firstOp, secondOp, thirdOp, i) != result[i]", ["GetIterResult"] = "(UInt32) Helpers.AddCarryWideningEven(first, second, third, i)"}),
+    ("SveVecTernOpTest.template",            new Dictionary<string, string> { ["TestName"] = "Sve2_AddCarryWideningLower_ulong",                                                                           ["Isa"] = "Sve2",          ["LoadIsa"] = "Sve2",    ["Method"] = "AddCarryWideningLower",                                                ["RetVectorType"] = "Vector",    ["RetBaseType"] = "UInt64",  ["Op1VectorType"] = "Vector",    ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector",    ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector",    ["Op3BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()",        ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()",        ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()",                        ["ConvertFunc"] = "",                                   ["ValidateIterResult"] = "Helpers.AddCarryWideningEven(firstOp, secondOp, thirdOp, i) != result[i]", ["GetIterResult"] = "(UInt64) Helpers.AddCarryWideningEven(first, second, third, i)"}),
+
     ("SveVecTernOpTest.template",      new Dictionary<string, string> { ["TestName"] = "Sve2_BitwiseClearXor_sbyte",                                                                            ["Isa"] = "Sve2",          ["LoadIsa"] = "Sve2",    ["Method"] = "BitwiseClearXor",                                                      ["RetVectorType"] = "Vector",    ["RetBaseType"] = "SByte",   ["Op1VectorType"] = "Vector",    ["Op1BaseType"] = "SByte",  ["Op2VectorType"] = "Vector",    ["Op2BaseType"] = "SByte",  ["Op3VectorType"] = "Vector",    ["Op3BaseType"] = "SByte",  ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()",         ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()",         ["NextValueOp3"] = "TestLibrary.Generator.GetSByte()",                         ["ConvertFunc"] = "",                                   ["ValidateIterResult"] = "result[i] != (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))",              ["GetIterResult"] = "(SByte) (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))"}),
     ("SveVecTernOpTest.template",      new Dictionary<string, string> { ["TestName"] = "Sve2_BitwiseClearXor_short",                                                                            ["Isa"] = "Sve2",          ["LoadIsa"] = "Sve2",    ["Method"] = "BitwiseClearXor",                                                      ["RetVectorType"] = "Vector",    ["RetBaseType"] = "Int16",   ["Op1VectorType"] = "Vector",    ["Op1BaseType"] = "Int16",  ["Op2VectorType"] = "Vector",    ["Op2BaseType"] = "Int16",  ["Op3VectorType"] = "Vector",    ["Op3BaseType"] = "Int16",  ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()",         ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()",         ["NextValueOp3"] = "TestLibrary.Generator.GetInt16()",                         ["ConvertFunc"] = "",                                   ["ValidateIterResult"] = "result[i] != (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))",              ["GetIterResult"] = "(Int16) (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))"}),
     ("SveVecTernOpTest.template",      new Dictionary<string, string> { ["TestName"] = "Sve2_BitwiseClearXor_int",                                                                              ["Isa"] = "Sve2",          ["LoadIsa"] = "Sve2",    ["Method"] = "BitwiseClearXor",                                                      ["RetVectorType"] = "Vector",    ["RetBaseType"] = "Int32",   ["Op1VectorType"] = "Vector",    ["Op1BaseType"] = "Int32",  ["Op2VectorType"] = "Vector",    ["Op2BaseType"] = "Int32",  ["Op3VectorType"] = "Vector",    ["Op3BaseType"] = "Int32",  ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()",         ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()",         ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()",                         ["ConvertFunc"] = "",                                   ["ValidateIterResult"] = "result[i] != (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))",              ["GetIterResult"] = "(Int32) (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))"}),

src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs

@@ -2293,6 +2293,27 @@ private static long Reduce(Func<long, sbyte, long> reduceOp, sbyte[] op1)
 
         public static int AddPairwiseWideningAndAdd(int[] op1, short[] op2, int i) => (int)(op1[i] + AddWidening(op2[2 * i], op2[2 * i + 1]));
 
+        public static uint AddCarryWideningEven(uint[] op1, uint[] op2, uint[] op3, int i)
+        {
+            uint lsb;
+            ulong res;
+
+            if (i % 2 == 0)
+            {
+                lsb = op3[i + 1] & 1u;
+                res = (ulong)op1[i] + op2[i] + lsb;
+                return (uint)res;
+            }
+            else
+            {
+                lsb = op3[i] & 1u;
+                res = (ulong)op1[i - 1] + op2[i - 1] + lsb;
+
+                // Shift result to get the carry bit
+                return (uint)(res >> 32);
+            }
+        }
+
         private static short HighNarrowing(int op1, bool round)
         {
             uint roundConst = 0;
@@ -2415,6 +2436,32 @@ private static long Reduce(Func<long, short, long> reduceOp, short[] op1)
 
         public static long AddPairwiseWideningAndAdd(long[] op1, int[] op2, int i) => (long)(op1[i] + AddWidening(op2[2 * i], op2[2 * i + 1]));
 
+        public static ulong AddCarryWideningEven(ulong[] op1, ulong[] op2, ulong[] op3, int i)
+        {
+            ulong lsb;
+            ulong res;
+
+            if (i % 2 == 0)
+            {
+                lsb = op3[i + 1] & 1UL;
+                res = op1[i] + op2[i] + lsb;
+                return res;
+            }
+            else
+            {
+                lsb = op3[i] & 1UL;
+
+                // Look for an overflow in the addition to get the carry bit
+                ulong sum1 = op1[i - 1] + op2[i - 1];
+                bool overflow1 = sum1 < op1[i - 1];
+
+                ulong sum2 = sum1 + lsb;
+                bool overflow2 = sum2 < sum1;
+
+                return (overflow1 || overflow2) ? 1UL : 0UL;
+            }
+        }
+
         private static int HighNarrowing(long op1, bool round)
         {
             ulong roundConst = 0;