Skip to content

Commit 8fe2d2b

Browse files
fanyang-monofanyang-mono
authored
[Mono] Enable generating SIMD intrinsics for System.Numerics.Vector on Arm64 (#65486)
* Enable SIMD intrinsics for System.Numerics.Vector on Arm64 * Minor adjustment * Use the correct op code for BitwiseAnd, BitwiseOr and Xor * Check if vector element type is a primitive type or not * Remove dead code and fix constant formet * Add type checks for each method and refactor * Remove extra condition check * Loose the type check for vector creation methods * Remove type checks for Create* functions * Remove some of the type checks
1 parent 3959a4a commit 8fe2d2b

File tree

3 files changed

+74
-46
lines changed

3 files changed

+74
-46
lines changed

‎src/mono/mono/mini/llvm-intrinsics-types.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@ typedef enum {
1818
} IntrinsicId;
1919

2020
enum {
21-
XBINOP_FORCEINT_and,
22-
XBINOP_FORCEINT_or,
23-
XBINOP_FORCEINT_ornot,
24-
XBINOP_FORCEINT_xor,
21+
XBINOP_FORCEINT_AND,
22+
XBINOP_FORCEINT_OR,
23+
XBINOP_FORCEINT_ORNOT,
24+
XBINOP_FORCEINT_XOR,
2525
};
2626

2727
#endif /* __MONO_MINI_LLVM_INTRINSICS_TYPES_H__ */

‎src/mono/mono/mini/mini-llvm.c‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7895,17 +7895,17 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb)
78957895
LLVMValueRef rhs_int = convert (ctx, rhs, intermediate_t);
78967896
LLVMValueRef result = NULL;
78977897
switch (ins->inst_c0) {
7898-
case XBINOP_FORCEINT_and:
7898+
case XBINOP_FORCEINT_AND:
78997899
result = LLVMBuildAnd (builder, lhs_int, rhs_int, "");
79007900
break;
7901-
case XBINOP_FORCEINT_or:
7901+
case XBINOP_FORCEINT_OR:
79027902
result = LLVMBuildOr (builder, lhs_int, rhs_int, "");
79037903
break;
7904-
case XBINOP_FORCEINT_ornot:
7904+
case XBINOP_FORCEINT_ORNOT:
79057905
result = LLVMBuildNot (builder, rhs_int, "");
79067906
result = LLVMBuildOr (builder, result, lhs_int, "");
79077907
break;
7908-
case XBINOP_FORCEINT_xor:
7908+
case XBINOP_FORCEINT_XOR:
79097909
result = LLVMBuildXor (builder, lhs_int, rhs_int, "");
79107910
break;
79117911
}

‎src/mono/mono/mini/simd-intrinsics.c‎

Lines changed: 66 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -600,13 +600,13 @@ static guint16 sri_vector_methods [] = {
600600
SN_AsUInt16,
601601
SN_AsUInt32,
602602
SN_AsUInt64,
603-
SN_BitwiseAnd,
604-
SN_BitwiseOr,
605603
SN_AsVector128,
606604
SN_AsVector2,
607605
SN_AsVector256,
608606
SN_AsVector3,
609607
SN_AsVector4,
608+
SN_BitwiseAnd,
609+
SN_BitwiseOr,
610610
SN_Ceiling,
611611
SN_ConditionalSelect,
612612
SN_ConvertToDouble,
@@ -669,24 +669,33 @@ is_create_from_half_vectors_overload (MonoMethodSignature *fsig)
669669
return mono_metadata_type_equal (fsig->params [0], fsig->params [1]);
670670
}
671671

672+
static gboolean
673+
is_element_type_primitive (MonoType *vector_type)
674+
{
675+
MonoType *element_type = get_vector_t_elem_type (vector_type);
676+
return MONO_TYPE_IS_INTRINSICS_VECTOR_PRIMITIVE (element_type);
677+
}
678+
672679
static MonoInst*
673680
emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args)
674681
{
675682
if (!COMPILE_LLVM (cfg))
676683
return NULL;
677684

678-
MonoClass *klass = cmethod->klass;
679685
int id = lookup_intrins (sri_vector_methods, sizeof (sri_vector_methods), cmethod);
680686
if (id == -1)
681687
return NULL;
682688

683689
if (!strcmp (m_class_get_name (cfg->method->klass), "Vector256"))
684690
return NULL; // TODO: Fix Vector256.WithUpper/WithLower
685-
691+
692+
MonoClass *klass = cmethod->klass;
686693
MonoTypeEnum arg0_type = fsig->param_count > 0 ? get_underlying_type (fsig->params [0]) : MONO_TYPE_VOID;
687694

688695
switch (id) {
689696
case SN_Abs: {
697+
if (!is_element_type_primitive (fsig->params [0]))
698+
return NULL;
690699
#ifdef TARGET_ARM64
691700
switch (arg0_type) {
692701
case MONO_TYPE_U1:
@@ -704,16 +713,22 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
704713
#endif
705714
}
706715
case SN_Add:
716+
case SN_Divide:
707717
case SN_Max:
708718
case SN_Min:
709719
case SN_Multiply:
710720
case SN_Subtract: {
721+
if (!is_element_type_primitive (fsig->params [0]))
722+
return NULL;
711723
int instc0 = -1;
712724
if (arg0_type == MONO_TYPE_R4 || arg0_type == MONO_TYPE_R8) {
713725
switch (id) {
714726
case SN_Add:
715727
instc0 = OP_FADD;
716728
break;
729+
case SN_Divide:
730+
instc0 = OP_FDIV;
731+
break;
717732
case SN_Max:
718733
instc0 = OP_FMAX;
719734
break;
@@ -734,6 +749,8 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
734749
case SN_Add:
735750
instc0 = OP_IADD;
736751
break;
752+
case SN_Divide:
753+
return NULL;
737754
case SN_Max:
738755
instc0 = OP_IMAX;
739756
break;
@@ -752,25 +769,34 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
752769
}
753770
return emit_simd_ins_for_sig (cfg, klass, OP_XBINOP, instc0, arg0_type, fsig, args);
754771
}
755-
case SN_Divide: {
756-
if ((arg0_type != MONO_TYPE_R4) && (arg0_type != MONO_TYPE_R8))
757-
return NULL;
758-
return emit_simd_ins_for_sig (cfg, klass, OP_XBINOP, OP_FDIV, arg0_type, fsig, args);
759-
}
760772
case SN_AndNot:
773+
if (!is_element_type_primitive (fsig->params [0]))
774+
return NULL;
761775
#ifdef TARGET_ARM64
762776
return emit_simd_ins_for_sig (cfg, klass, OP_ARM64_BIC, -1, arg0_type, fsig, args);
763777
#else
764778
return NULL;
765779
#endif
766780
case SN_BitwiseAnd:
767-
return emit_simd_ins_for_sig (cfg, klass, OP_XBINOP, OP_IAND, arg0_type, fsig, args);
768781
case SN_BitwiseOr:
769-
return emit_simd_ins_for_sig (cfg, klass, OP_XBINOP, OP_IOR, arg0_type, fsig, args);
770782
case SN_Xor: {
771-
if ((arg0_type == MONO_TYPE_R4) || (arg0_type == MONO_TYPE_R8))
783+
if (!is_element_type_primitive (fsig->params [0]))
772784
return NULL;
773-
return emit_simd_ins_for_sig (cfg, klass, OP_XBINOP, OP_IXOR, arg0_type, fsig, args);
785+
int instc0 = -1;
786+
switch (id) {
787+
case SN_BitwiseAnd:
788+
instc0 = XBINOP_FORCEINT_AND;
789+
break;
790+
case SN_BitwiseOr:
791+
instc0 = XBINOP_FORCEINT_OR;
792+
break;
793+
case SN_Xor:
794+
instc0 = XBINOP_FORCEINT_XOR;
795+
break;
796+
default:
797+
g_assert_not_reached ();
798+
}
799+
return emit_simd_ins_for_sig (cfg, klass, OP_XBINOP_FORCEINT, instc0, arg0_type, fsig, args);
774800
}
775801
case SN_As:
776802
case SN_AsByte:
@@ -783,9 +809,7 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
783809
case SN_AsUInt16:
784810
case SN_AsUInt32:
785811
case SN_AsUInt64: {
786-
MonoType *ret_type = get_vector_t_elem_type (fsig->ret);
787-
MonoType *arg_type = get_vector_t_elem_type (fsig->params [0]);
788-
if (!MONO_TYPE_IS_INTRINSICS_VECTOR_PRIMITIVE (ret_type) || !MONO_TYPE_IS_INTRINSICS_VECTOR_PRIMITIVE (arg_type))
812+
if (!is_element_type_primitive (fsig->ret) || !is_element_type_primitive (fsig->params [0]))
789813
return NULL;
790814
return emit_simd_ins (cfg, klass, OP_XCAST, args [0]->dreg, -1);
791815
}
@@ -801,6 +825,8 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
801825
#endif
802826
}
803827
case SN_ConditionalSelect: {
828+
if (!is_element_type_primitive (fsig->params [0]))
829+
return NULL;
804830
#ifdef TARGET_ARM64
805831
return emit_simd_ins_for_sig (cfg, klass, OP_ARM64_BSL, -1, arg0_type, fsig, args);
806832
#else
@@ -851,10 +877,8 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
851877
case SN_Equals:
852878
case SN_EqualsAll:
853879
case SN_EqualsAny: {
854-
MonoType *arg_type = get_vector_t_elem_type (fsig->params [0]);
855-
if (!MONO_TYPE_IS_INTRINSICS_VECTOR_PRIMITIVE (arg_type))
880+
if (!is_element_type_primitive (fsig->params [0]))
856881
return NULL;
857-
858882
switch (id) {
859883
case SN_Equals:
860884
return emit_xcompare (cfg, klass, arg0_type, args [0], args [1]);
@@ -870,10 +894,10 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
870894
}
871895
}
872896
case SN_GetElement: {
897+
if (!is_element_type_primitive (fsig->params [0]))
898+
return NULL;
873899
MonoClass *arg_class = mono_class_from_mono_type_internal (fsig->params [0]);
874900
MonoType *etype = mono_class_get_context (arg_class)->class_inst->type_argv [0];
875-
if (!MONO_TYPE_IS_INTRINSICS_VECTOR_PRIMITIVE (etype))
876-
return NULL;
877901
int size = mono_class_value_size (arg_class, NULL);
878902
int esize = mono_class_value_size (mono_class_from_mono_type_internal (etype), NULL);
879903
int elems = size / esize;
@@ -884,8 +908,7 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
884908
}
885909
case SN_GetLower:
886910
case SN_GetUpper: {
887-
MonoType *arg_type = get_vector_t_elem_type (fsig->params [0]);
888-
if (!MONO_TYPE_IS_INTRINSICS_VECTOR_PRIMITIVE (arg_type))
911+
if (!is_element_type_primitive (fsig->params [0]))
889912
return NULL;
890913
int op = id == SN_GetLower ? OP_XLOWER : OP_XUPPER;
891914
return emit_simd_ins_for_sig (cfg, klass, op, 0, arg0_type, fsig, args);
@@ -894,10 +917,8 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
894917
case SN_GreaterThanOrEqual:
895918
case SN_LessThan:
896919
case SN_LessThanOrEqual: {
897-
MonoType *arg_type = get_vector_t_elem_type (fsig->params [0]);
898-
if (!MONO_TYPE_IS_INTRINSICS_VECTOR_PRIMITIVE (arg_type))
920+
if (!is_element_type_primitive (fsig->params [0]))
899921
return NULL;
900-
901922
gboolean is_unsigned = type_is_unsigned (fsig->params [0]);
902923
MonoInst *ins = emit_xcompare (cfg, klass, arg0_type, args [0], args [1]);
903924
switch (id) {
@@ -920,6 +941,8 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
920941
}
921942
case SN_Negate:
922943
case SN_OnesComplement: {
944+
if (!is_element_type_primitive (fsig->params [0]))
945+
return NULL;
923946
#ifdef TARGET_ARM64
924947
int op = id == SN_Negate ? OP_ARM64_XNEG : OP_ARM64_MVN;
925948
return emit_simd_ins_for_sig (cfg, klass, op, -1, arg0_type, fsig, args);
@@ -928,6 +951,8 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
928951
#endif
929952
}
930953
case SN_Sqrt: {
954+
if (!is_element_type_primitive (fsig->params [0]))
955+
return NULL;
931956
#ifdef TARGET_ARM64
932957
if ((arg0_type != MONO_TYPE_R4) && (arg0_type != MONO_TYPE_R8))
933958
return NULL;
@@ -937,25 +962,23 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
937962
#endif
938963
}
939964
case SN_ToScalar: {
940-
MonoType *arg_type = get_vector_t_elem_type (fsig->params [0]);
941-
if (!MONO_TYPE_IS_INTRINSICS_VECTOR_PRIMITIVE (arg_type))
965+
if (!is_element_type_primitive (fsig->params [0]))
942966
return NULL;
943967
int extract_op = type_to_extract_op (arg0_type);
944968
return emit_simd_ins_for_sig (cfg, klass, extract_op, 0, arg0_type, fsig, args);
945969
}
946970
case SN_ToVector128:
947971
case SN_ToVector128Unsafe: {
948-
MonoType *arg_type = get_vector_t_elem_type (fsig->params [0]);
949-
if (!MONO_TYPE_IS_INTRINSICS_VECTOR_PRIMITIVE (arg_type))
972+
if (!is_element_type_primitive (fsig->params [0]))
950973
return NULL;
951974
int op = id == SN_ToVector128 ? OP_XWIDEN : OP_XWIDEN_UNSAFE;
952975
return emit_simd_ins_for_sig (cfg, klass, op, 0, arg0_type, fsig, args);
953976
}
954977
case SN_WithElement: {
978+
if (!is_element_type_primitive (fsig->params [0]))
979+
return NULL;
955980
MonoClass *arg_class = mono_class_from_mono_type_internal (fsig->params [0]);
956981
MonoType *etype = mono_class_get_context (arg_class)->class_inst->type_argv [0];
957-
if (!MONO_TYPE_IS_INTRINSICS_VECTOR_PRIMITIVE (etype))
958-
return NULL;
959982
int size = mono_class_value_size (arg_class, NULL);
960983
int esize = mono_class_value_size (mono_class_from_mono_type_internal (etype), NULL);
961984
int elems = size / esize;
@@ -969,8 +992,7 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi
969992
}
970993
case SN_WithLower:
971994
case SN_WithUpper: {
972-
MonoType *arg_type = get_vector_t_elem_type (fsig->params [0]);
973-
if (!MONO_TYPE_IS_INTRINSICS_VECTOR_PRIMITIVE (arg_type))
995+
if (!is_element_type_primitive (fsig->params [0]))
974996
return NULL;
975997
int op = id == SN_GetLower ? OP_XINSERT_LOWER : OP_XINSERT_UPPER;
976998
return emit_simd_ins_for_sig (cfg, klass, op, 0, arg0_type, fsig, args);
@@ -1559,7 +1581,7 @@ static SimdIntrinsic advsimd_methods [] = {
15591581
{SN_AddScalar, OP_XBINOP_SCALAR, OP_IADD, None, None, OP_XBINOP_SCALAR, OP_FADD},
15601582
{SN_AddWideningLower, OP_ARM64_SADD, None, OP_ARM64_UADD},
15611583
{SN_AddWideningUpper, OP_ARM64_SADD2, None, OP_ARM64_UADD2},
1562-
{SN_And, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_and},
1584+
{SN_And, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_AND},
15631585
{SN_BitwiseClear, OP_ARM64_BIC},
15641586
{SN_BitwiseSelect, OP_ARM64_BSL},
15651587
{SN_Ceiling, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP},
@@ -1762,8 +1784,8 @@ static SimdIntrinsic advsimd_methods [] = {
17621784
{SN_NegateSaturateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG},
17631785
{SN_NegateScalar, OP_ARM64_XNEG_SCALAR},
17641786
{SN_Not, OP_ARM64_MVN},
1765-
{SN_Or, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_or},
1766-
{SN_OrNot, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_ornot},
1787+
{SN_Or, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_OR},
1788+
{SN_OrNot, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_ORNOT},
17671789
{SN_PolynomialMultiply, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_PMUL},
17681790
{SN_PolynomialMultiplyWideningLower, OP_ARM64_PMULL},
17691791
{SN_PolynomialMultiplyWideningUpper, OP_ARM64_PMULL2},
@@ -1883,7 +1905,7 @@ static SimdIntrinsic advsimd_methods [] = {
18831905
{SN_UnzipOdd, OP_ARM64_UZP2},
18841906
{SN_VectorTableLookup, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_TBL1},
18851907
{SN_VectorTableLookupExtension, OP_XOP_OVR_X_X_X_X, INTRINS_AARCH64_ADV_SIMD_TBX1},
1886-
{SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_xor},
1908+
{SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_XOR},
18871909
{SN_ZeroExtendWideningLower, OP_ARM64_UXTL},
18881910
{SN_ZeroExtendWideningUpper, OP_ARM64_UXTL2},
18891911
{SN_ZipHigh, OP_ARM64_ZIP2},
@@ -3351,6 +3373,12 @@ mono_emit_simd_intrinsics (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign
33513373
}
33523374
#endif // defined(TARGET_ARM64) || defined(TARGET_AMD64)
33533375

3376+
#if defined(TARGET_ARM64)
3377+
if (!strcmp (class_ns, "System.Numerics") && !strcmp (class_name, "Vector")){
3378+
return emit_sri_vector (cfg, cmethod, fsig, args);
3379+
}
3380+
#endif // defined(TARGET_ARM64)
3381+
33543382
return emit_simd_intrinsics (class_ns, class_name, cfg, cmethod, fsig, args);
33553383
}
33563384

0 commit comments

Comments
 (0)