Skip to content

Commit 8462e20

Browse files
davemgreendavemgreen
authored
[AArch64] Push mul into extend operands (#94960)
In a similar way to how we push vector adds into extends, this pushed 'mul(zext,zext)' into 'zext(mul(zext,zext))' if the extend can be done in two or more steps. https://alive2.llvm.org/ce/z/WjU7Kr
1 parent 0c97ac0 commit 8462e20

File tree

6 files changed

+754
-910
lines changed

6 files changed

+754
-910
lines changed

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp‎

Lines changed: 44 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -17720,6 +17720,47 @@ static SDValue performMulVectorCmpZeroCombine(SDNode *N, SelectionDAG &DAG) {
1772017720
return DAG.getNode(AArch64ISD::NVCAST, DL, VT, CM);
1772117721
}
1772217722

17723+
// Transform vector add(zext i8 to i32, zext i8 to i32)
17724+
// into sext(add(zext(i8 to i16), zext(i8 to i16)) to i32)
17725+
// This allows extra uses of saddl/uaddl at the lower vector widths, and less
17726+
// extends.
17727+
static SDValue performVectorExtCombine(SDNode *N, SelectionDAG &DAG) {
17728+
EVT VT = N->getValueType(0);
17729+
if (!VT.isFixedLengthVector() || VT.getSizeInBits() <= 128 ||
17730+
(N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND &&
17731+
N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND) ||
17732+
(N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND &&
17733+
N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND) ||
17734+
N->getOperand(0).getOperand(0).getValueType() !=
17735+
N->getOperand(1).getOperand(0).getValueType())
17736+
return SDValue();
17737+
17738+
if (N->getOpcode() == ISD::MUL &&
17739+
N->getOperand(0).getOpcode() != N->getOperand(1).getOpcode())
17740+
return SDValue();
17741+
17742+
SDValue N0 = N->getOperand(0).getOperand(0);
17743+
SDValue N1 = N->getOperand(1).getOperand(0);
17744+
EVT InVT = N0.getValueType();
17745+
17746+
EVT S1 = InVT.getScalarType();
17747+
EVT S2 = VT.getScalarType();
17748+
if ((S2 == MVT::i32 && S1 == MVT::i8) ||
17749+
(S2 == MVT::i64 && (S1 == MVT::i8 || S1 == MVT::i16))) {
17750+
SDLoc DL(N);
17751+
EVT HalfVT = EVT::getVectorVT(*DAG.getContext(),
17752+
S2.getHalfSizedIntegerVT(*DAG.getContext()),
17753+
VT.getVectorElementCount());
17754+
SDValue NewN0 = DAG.getNode(N->getOperand(0).getOpcode(), DL, HalfVT, N0);
17755+
SDValue NewN1 = DAG.getNode(N->getOperand(1).getOpcode(), DL, HalfVT, N1);
17756+
SDValue NewOp = DAG.getNode(N->getOpcode(), DL, HalfVT, NewN0, NewN1);
17757+
return DAG.getNode(N->getOpcode() == ISD::MUL ? N->getOperand(0).getOpcode()
17758+
: (unsigned)ISD::SIGN_EXTEND,
17759+
DL, VT, NewOp);
17760+
}
17761+
return SDValue();
17762+
}
17763+
1772317764
static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
1772417765
TargetLowering::DAGCombinerInfo &DCI,
1772517766
const AArch64Subtarget *Subtarget) {
@@ -17728,6 +17769,8 @@ static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
1772817769
return Ext;
1772917770
if (SDValue Ext = performMulVectorCmpZeroCombine(N, DAG))
1773017771
return Ext;
17772+
if (SDValue Ext = performVectorExtCombine(N, DAG))
17773+
return Ext;
1773117774

1773217775
if (DCI.isBeforeLegalizeOps())
1773317776
return SDValue();
@@ -19604,41 +19647,6 @@ static SDValue foldADCToCINC(SDNode *N, SelectionDAG &DAG) {
1960419647
return DAG.getNode(AArch64ISD::CSINC, DL, VT, LHS, LHS, CC, Cond);
1960519648
}
1960619649

19607-
// Transform vector add(zext i8 to i32, zext i8 to i32)
19608-
// into sext(add(zext(i8 to i16), zext(i8 to i16)) to i32)
19609-
// This allows extra uses of saddl/uaddl at the lower vector widths, and less
19610-
// extends.
19611-
static SDValue performVectorAddSubExtCombine(SDNode *N, SelectionDAG &DAG) {
19612-
EVT VT = N->getValueType(0);
19613-
if (!VT.isFixedLengthVector() || VT.getSizeInBits() <= 128 ||
19614-
(N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND &&
19615-
N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND) ||
19616-
(N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND &&
19617-
N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND) ||
19618-
N->getOperand(0).getOperand(0).getValueType() !=
19619-
N->getOperand(1).getOperand(0).getValueType())
19620-
return SDValue();
19621-
19622-
SDValue N0 = N->getOperand(0).getOperand(0);
19623-
SDValue N1 = N->getOperand(1).getOperand(0);
19624-
EVT InVT = N0.getValueType();
19625-
19626-
EVT S1 = InVT.getScalarType();
19627-
EVT S2 = VT.getScalarType();
19628-
if ((S2 == MVT::i32 && S1 == MVT::i8) ||
19629-
(S2 == MVT::i64 && (S1 == MVT::i8 || S1 == MVT::i16))) {
19630-
SDLoc DL(N);
19631-
EVT HalfVT = EVT::getVectorVT(*DAG.getContext(),
19632-
S2.getHalfSizedIntegerVT(*DAG.getContext()),
19633-
VT.getVectorElementCount());
19634-
SDValue NewN0 = DAG.getNode(N->getOperand(0).getOpcode(), DL, HalfVT, N0);
19635-
SDValue NewN1 = DAG.getNode(N->getOperand(1).getOpcode(), DL, HalfVT, N1);
19636-
SDValue NewOp = DAG.getNode(N->getOpcode(), DL, HalfVT, NewN0, NewN1);
19637-
return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewOp);
19638-
}
19639-
return SDValue();
19640-
}
19641-
1964219650
static SDValue performBuildVectorCombine(SDNode *N,
1964319651
TargetLowering::DAGCombinerInfo &DCI,
1964419652
SelectionDAG &DAG) {
@@ -20260,7 +20268,7 @@ static SDValue performAddSubCombine(SDNode *N,
2026020268
return Val;
2026120269
if (SDValue Val = performNegCSelCombine(N, DCI.DAG))
2026220270
return Val;
20263-
if (SDValue Val = performVectorAddSubExtCombine(N, DCI.DAG))
20271+
if (SDValue Val = performVectorExtCombine(N, DCI.DAG))
2026420272
return Val;
2026520273
if (SDValue Val = performAddCombineForShiftedOperands(N, DCI.DAG))
2026620274
return Val;

‎llvm/test/CodeGen/AArch64/aarch64-wide-mul.ll‎

Lines changed: 41 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,12 @@ entry:
2828
define <16 x i32> @mul_i32(<16 x i8> %a, <16 x i8> %b) {
2929
; CHECK-SD-LABEL: mul_i32:
3030
; CHECK-SD: // %bb.0: // %entry
31-
; CHECK-SD-NEXT: ushll v2.8h, v0.8b, #0
32-
; CHECK-SD-NEXT: ushll v4.8h, v1.8b, #0
33-
; CHECK-SD-NEXT: ushll2 v5.8h, v0.16b, #0
34-
; CHECK-SD-NEXT: ushll2 v6.8h, v1.16b, #0
35-
; CHECK-SD-NEXT: umull v0.4s, v2.4h, v4.4h
36-
; CHECK-SD-NEXT: umull2 v1.4s, v2.8h, v4.8h
37-
; CHECK-SD-NEXT: umull2 v3.4s, v5.8h, v6.8h
38-
; CHECK-SD-NEXT: umull v2.4s, v5.4h, v6.4h
31+
; CHECK-SD-NEXT: umull v2.8h, v0.8b, v1.8b
32+
; CHECK-SD-NEXT: umull2 v4.8h, v0.16b, v1.16b
33+
; CHECK-SD-NEXT: ushll v0.4s, v2.4h, #0
34+
; CHECK-SD-NEXT: ushll2 v3.4s, v4.8h, #0
35+
; CHECK-SD-NEXT: ushll2 v1.4s, v2.8h, #0
36+
; CHECK-SD-NEXT: ushll v2.4s, v4.4h, #0
3937
; CHECK-SD-NEXT: ret
4038
;
4139
; CHECK-GI-LABEL: mul_i32:
@@ -59,26 +57,20 @@ entry:
5957
define <16 x i64> @mul_i64(<16 x i8> %a, <16 x i8> %b) {
6058
; CHECK-SD-LABEL: mul_i64:
6159
; CHECK-SD: // %bb.0: // %entry
62-
; CHECK-SD-NEXT: ushll v2.8h, v0.8b, #0
63-
; CHECK-SD-NEXT: ushll2 v0.8h, v0.16b, #0
64-
; CHECK-SD-NEXT: ushll v3.8h, v1.8b, #0
65-
; CHECK-SD-NEXT: ushll2 v1.8h, v1.16b, #0
66-
; CHECK-SD-NEXT: ushll v4.4s, v2.4h, #0
67-
; CHECK-SD-NEXT: ushll v5.4s, v0.4h, #0
68-
; CHECK-SD-NEXT: ushll v6.4s, v3.4h, #0
60+
; CHECK-SD-NEXT: umull v2.8h, v0.8b, v1.8b
61+
; CHECK-SD-NEXT: umull2 v0.8h, v0.16b, v1.16b
62+
; CHECK-SD-NEXT: ushll v3.4s, v2.4h, #0
6963
; CHECK-SD-NEXT: ushll2 v2.4s, v2.8h, #0
70-
; CHECK-SD-NEXT: ushll v16.4s, v1.4h, #0
71-
; CHECK-SD-NEXT: ushll2 v7.4s, v3.8h, #0
72-
; CHECK-SD-NEXT: ushll2 v17.4s, v0.8h, #0
73-
; CHECK-SD-NEXT: ushll2 v18.4s, v1.8h, #0
74-
; CHECK-SD-NEXT: umull2 v1.2d, v4.4s, v6.4s
75-
; CHECK-SD-NEXT: umull v0.2d, v4.2s, v6.2s
76-
; CHECK-SD-NEXT: umull2 v3.2d, v2.4s, v7.4s
77-
; CHECK-SD-NEXT: umull v2.2d, v2.2s, v7.2s
78-
; CHECK-SD-NEXT: umull v4.2d, v5.2s, v16.2s
79-
; CHECK-SD-NEXT: umull2 v7.2d, v17.4s, v18.4s
80-
; CHECK-SD-NEXT: umull2 v5.2d, v5.4s, v16.4s
81-
; CHECK-SD-NEXT: umull v6.2d, v17.2s, v18.2s
64+
; CHECK-SD-NEXT: ushll v5.4s, v0.4h, #0
65+
; CHECK-SD-NEXT: ushll2 v6.4s, v0.8h, #0
66+
; CHECK-SD-NEXT: ushll2 v1.2d, v3.4s, #0
67+
; CHECK-SD-NEXT: ushll v0.2d, v3.2s, #0
68+
; CHECK-SD-NEXT: ushll2 v3.2d, v2.4s, #0
69+
; CHECK-SD-NEXT: ushll v2.2d, v2.2s, #0
70+
; CHECK-SD-NEXT: ushll v4.2d, v5.2s, #0
71+
; CHECK-SD-NEXT: ushll2 v7.2d, v6.4s, #0
72+
; CHECK-SD-NEXT: ushll2 v5.2d, v5.4s, #0
73+
; CHECK-SD-NEXT: ushll v6.2d, v6.2s, #0
8274
; CHECK-SD-NEXT: ret
8375
;
8476
; CHECK-GI-LABEL: mul_i64:
@@ -139,17 +131,12 @@ entry:
139131
define <16 x i32> @mla_i32(<16 x i8> %a, <16 x i8> %b, <16 x i32> %c) {
140132
; CHECK-SD-LABEL: mla_i32:
141133
; CHECK-SD: // %bb.0: // %entry
142-
; CHECK-SD-NEXT: ushll v6.8h, v0.8b, #0
143-
; CHECK-SD-NEXT: ushll v7.8h, v1.8b, #0
144-
; CHECK-SD-NEXT: ushll2 v0.8h, v0.16b, #0
145-
; CHECK-SD-NEXT: ushll2 v1.8h, v1.16b, #0
146-
; CHECK-SD-NEXT: umlal v2.4s, v6.4h, v7.4h
147-
; CHECK-SD-NEXT: umlal2 v3.4s, v6.8h, v7.8h
148-
; CHECK-SD-NEXT: umlal2 v5.4s, v0.8h, v1.8h
149-
; CHECK-SD-NEXT: umlal v4.4s, v0.4h, v1.4h
150-
; CHECK-SD-NEXT: mov v0.16b, v2.16b
151-
; CHECK-SD-NEXT: mov v1.16b, v3.16b
152-
; CHECK-SD-NEXT: mov v2.16b, v4.16b
134+
; CHECK-SD-NEXT: umull2 v7.8h, v0.16b, v1.16b
135+
; CHECK-SD-NEXT: umull v6.8h, v0.8b, v1.8b
136+
; CHECK-SD-NEXT: uaddw2 v5.4s, v5.4s, v7.8h
137+
; CHECK-SD-NEXT: uaddw v0.4s, v2.4s, v6.4h
138+
; CHECK-SD-NEXT: uaddw2 v1.4s, v3.4s, v6.8h
139+
; CHECK-SD-NEXT: uaddw v2.4s, v4.4s, v7.4h
153140
; CHECK-SD-NEXT: mov v3.16b, v5.16b
154141
; CHECK-SD-NEXT: ret
155142
;
@@ -179,35 +166,22 @@ entry:
179166
define <16 x i64> @mla_i64(<16 x i8> %a, <16 x i8> %b, <16 x i64> %c) {
180167
; CHECK-SD-LABEL: mla_i64:
181168
; CHECK-SD: // %bb.0: // %entry
182-
; CHECK-SD-NEXT: mov v17.16b, v7.16b
183-
; CHECK-SD-NEXT: mov v16.16b, v6.16b
184-
; CHECK-SD-NEXT: ushll v6.8h, v0.8b, #0
185-
; CHECK-SD-NEXT: ushll2 v0.8h, v0.16b, #0
186-
; CHECK-SD-NEXT: ushll v7.8h, v1.8b, #0
187-
; CHECK-SD-NEXT: ushll2 v1.8h, v1.16b, #0
188-
; CHECK-SD-NEXT: ushll v18.4s, v6.4h, #0
189-
; CHECK-SD-NEXT: ushll2 v21.4s, v6.8h, #0
190-
; CHECK-SD-NEXT: ushll v19.4s, v0.4h, #0
191-
; CHECK-SD-NEXT: ushll v20.4s, v7.4h, #0
192-
; CHECK-SD-NEXT: ushll v22.4s, v1.4h, #0
193-
; CHECK-SD-NEXT: ushll2 v23.4s, v7.8h, #0
194-
; CHECK-SD-NEXT: ldp q6, q7, [sp]
195-
; CHECK-SD-NEXT: ushll2 v0.4s, v0.8h, #0
196-
; CHECK-SD-NEXT: ushll2 v1.4s, v1.8h, #0
197-
; CHECK-SD-NEXT: umlal2 v3.2d, v18.4s, v20.4s
198-
; CHECK-SD-NEXT: umlal v2.2d, v18.2s, v20.2s
199-
; CHECK-SD-NEXT: umlal v16.2d, v19.2s, v22.2s
200-
; CHECK-SD-NEXT: umlal2 v5.2d, v21.4s, v23.4s
201-
; CHECK-SD-NEXT: umlal v4.2d, v21.2s, v23.2s
202-
; CHECK-SD-NEXT: umlal2 v17.2d, v19.4s, v22.4s
203-
; CHECK-SD-NEXT: umlal2 v7.2d, v0.4s, v1.4s
204-
; CHECK-SD-NEXT: umlal v6.2d, v0.2s, v1.2s
205-
; CHECK-SD-NEXT: mov v0.16b, v2.16b
206-
; CHECK-SD-NEXT: mov v1.16b, v3.16b
207-
; CHECK-SD-NEXT: mov v2.16b, v4.16b
208-
; CHECK-SD-NEXT: mov v3.16b, v5.16b
209-
; CHECK-SD-NEXT: mov v4.16b, v16.16b
210-
; CHECK-SD-NEXT: mov v5.16b, v17.16b
169+
; CHECK-SD-NEXT: umull v16.8h, v0.8b, v1.8b
170+
; CHECK-SD-NEXT: umull2 v0.8h, v0.16b, v1.16b
171+
; CHECK-SD-NEXT: ldp q20, q21, [sp]
172+
; CHECK-SD-NEXT: ushll v17.4s, v16.4h, #0
173+
; CHECK-SD-NEXT: ushll2 v16.4s, v16.8h, #0
174+
; CHECK-SD-NEXT: ushll2 v19.4s, v0.8h, #0
175+
; CHECK-SD-NEXT: ushll v18.4s, v0.4h, #0
176+
; CHECK-SD-NEXT: uaddw2 v1.2d, v3.2d, v17.4s
177+
; CHECK-SD-NEXT: uaddw v0.2d, v2.2d, v17.2s
178+
; CHECK-SD-NEXT: uaddw2 v3.2d, v5.2d, v16.4s
179+
; CHECK-SD-NEXT: uaddw v2.2d, v4.2d, v16.2s
180+
; CHECK-SD-NEXT: uaddw2 v16.2d, v21.2d, v19.4s
181+
; CHECK-SD-NEXT: uaddw v4.2d, v6.2d, v18.2s
182+
; CHECK-SD-NEXT: uaddw2 v5.2d, v7.2d, v18.4s
183+
; CHECK-SD-NEXT: uaddw v6.2d, v20.2d, v19.2s
184+
; CHECK-SD-NEXT: mov v7.16b, v16.16b
211185
; CHECK-SD-NEXT: ret
212186
;
213187
; CHECK-GI-LABEL: mla_i64:

‎llvm/test/CodeGen/AArch64/addp-shuffle.ll‎

Lines changed: 14 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -136,15 +136,13 @@ define <4 x double> @deinterleave_shuffle_v8f64(<8 x double> %a) {
136136
define <4 x i32> @udot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) {
137137
; CHECK-LABEL: udot:
138138
; CHECK: // %bb.0:
139-
; CHECK-NEXT: ushll v3.8h, v1.8b, #0
140-
; CHECK-NEXT: ushll v4.8h, v2.8b, #0
141-
; CHECK-NEXT: ushll2 v1.8h, v1.16b, #0
142-
; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0
143-
; CHECK-NEXT: umull2 v5.4s, v3.8h, v4.8h
144-
; CHECK-NEXT: umull v3.4s, v3.4h, v4.4h
145-
; CHECK-NEXT: umull2 v4.4s, v1.8h, v2.8h
146-
; CHECK-NEXT: umull v1.4s, v1.4h, v2.4h
147-
; CHECK-NEXT: addp v2.4s, v3.4s, v5.4s
139+
; CHECK-NEXT: umull v3.8h, v1.8b, v2.8b
140+
; CHECK-NEXT: umull2 v1.8h, v1.16b, v2.16b
141+
; CHECK-NEXT: ushll2 v2.4s, v3.8h, #0
142+
; CHECK-NEXT: ushll v3.4s, v3.4h, #0
143+
; CHECK-NEXT: ushll2 v4.4s, v1.8h, #0
144+
; CHECK-NEXT: ushll v1.4s, v1.4h, #0
145+
; CHECK-NEXT: addp v2.4s, v3.4s, v2.4s
148146
; CHECK-NEXT: addp v1.4s, v1.4s, v4.4s
149147
; CHECK-NEXT: addp v1.4s, v2.4s, v1.4s
150148
; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
@@ -165,15 +163,13 @@ define <4 x i32> @udot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) {
165163
define <4 x i32> @sdot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) {
166164
; CHECK-LABEL: sdot:
167165
; CHECK: // %bb.0:
168-
; CHECK-NEXT: sshll v3.8h, v1.8b, #0
169-
; CHECK-NEXT: sshll v4.8h, v2.8b, #0
170-
; CHECK-NEXT: sshll2 v1.8h, v1.16b, #0
171-
; CHECK-NEXT: sshll2 v2.8h, v2.16b, #0
172-
; CHECK-NEXT: smull2 v5.4s, v3.8h, v4.8h
173-
; CHECK-NEXT: smull v3.4s, v3.4h, v4.4h
174-
; CHECK-NEXT: smull2 v4.4s, v1.8h, v2.8h
175-
; CHECK-NEXT: smull v1.4s, v1.4h, v2.4h
176-
; CHECK-NEXT: addp v2.4s, v3.4s, v5.4s
166+
; CHECK-NEXT: smull v3.8h, v1.8b, v2.8b
167+
; CHECK-NEXT: smull2 v1.8h, v1.16b, v2.16b
168+
; CHECK-NEXT: sshll2 v2.4s, v3.8h, #0
169+
; CHECK-NEXT: sshll v3.4s, v3.4h, #0
170+
; CHECK-NEXT: sshll2 v4.4s, v1.8h, #0
171+
; CHECK-NEXT: sshll v1.4s, v1.4h, #0
172+
; CHECK-NEXT: addp v2.4s, v3.4s, v2.4s
177173
; CHECK-NEXT: addp v1.4s, v1.4s, v4.4s
178174
; CHECK-NEXT: addp v1.4s, v2.4s, v1.4s
179175
; CHECK-NEXT: add v0.4s, v0.4s, v1.4s

0 commit comments

Comments
 (0)