Skip to content

Commit ff3e2ba

Browse files
mstorsjomstorsjo
committed
Revert "[VPlan] Add transformation to narrow interleave groups. (#106441)"
This reverts commit dfa665f. This commit caused miscompilations in ffmpeg, see #106441 for details.
1 parent b2b3cb5 commit ff3e2ba

File tree

4 files changed

+92
-126
lines changed

4 files changed

+92
-126
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 12 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -2236,36 +2236,6 @@ void VPlanTransforms::materializeBroadcasts(VPlan &Plan) {
22362236
}
22372237
}
22382238

2239-
/// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
2240-
/// converted to a narrower recipe. \p V is used by a wide recipe \p WideMember
2241-
/// that feeds a store interleave group at index \p Idx, \p WideMember0 is the
2242-
/// recipe feeding the same interleave group at index 0. A VPWidenLoadRecipe can
2243-
/// be narrowed to an index-independent load if it feeds all wide ops at all
2244-
/// indices (checked by via the operands of the wide recipe at lane0, \p
2245-
/// WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if \p V
2246-
/// is defined at \p Idx of a load interleave group.
2247-
static bool canNarrowLoad(VPWidenRecipe *WideMember0, VPWidenRecipe *WideMember,
2248-
VPValue *V, unsigned Idx) {
2249-
auto *DefR = V->getDefiningRecipe();
2250-
if (!DefR)
2251-
return false;
2252-
if (auto *W = dyn_cast<VPWidenLoadRecipe>(DefR))
2253-
return !W->getMask() &&
2254-
all_of(zip(WideMember0->operands(), WideMember->operands()),
2255-
[V](const auto P) {
2256-
// V must be as at the same places in both WideMember0 and
2257-
// WideMember.
2258-
const auto &[WideMember0Op, WideMemberOp] = P;
2259-
return (WideMember0Op == V) == (WideMemberOp == V);
2260-
});
2261-
2262-
if (auto *IR = dyn_cast<VPInterleaveRecipe>(DefR))
2263-
return IR->getInterleaveGroup()->getFactor() ==
2264-
IR->getInterleaveGroup()->getNumMembers() &&
2265-
IR->getVPValue(Idx) == V;
2266-
return false;
2267-
}
2268-
22692239
/// Returns true if \p IR is a full interleave group with factor and number of
22702240
/// members both equal to \p VF. The interleave group must also access the full
22712241
/// vector width \p VectorRegWidth.
@@ -2328,8 +2298,6 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
23282298
if (R.mayWriteToMemory() && !InterleaveR)
23292299
return;
23302300

2331-
// All other ops are allowed, but we reject uses that cannot be converted
2332-
// when checking all allowed consumers (store interleave groups) below.
23332301
if (!InterleaveR)
23342302
continue;
23352303

@@ -2344,7 +2312,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
23442312

23452313
// For now, we only support full interleave groups storing load interleave
23462314
// groups.
2347-
if (all_of(enumerate(InterleaveR->getStoredValues()), [](auto Op) {
2315+
if (!all_of(enumerate(InterleaveR->getStoredValues()), [](auto Op) {
23482316
VPRecipeBase *DefR = Op.value()->getDefiningRecipe();
23492317
if (!DefR)
23502318
return false;
@@ -2354,67 +2322,31 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
23542322
IR->getInterleaveGroup()->getNumMembers() &&
23552323
IR->getVPValue(Op.index()) == Op.value();
23562324
})) {
2357-
StoreGroups.push_back(InterleaveR);
2358-
continue;
2359-
}
2360-
2361-
// Check if all values feeding InterleaveR are matching wide recipes, which
2362-
// operands that can be narrowed.
2363-
auto *WideMember0 = dyn_cast_or_null<VPWidenRecipe>(
2364-
InterleaveR->getStoredValues()[0]->getDefiningRecipe());
2365-
if (!WideMember0)
23662325
return;
2367-
for (const auto &[I, V] : enumerate(InterleaveR->getStoredValues())) {
2368-
auto *R = dyn_cast<VPWidenRecipe>(V->getDefiningRecipe());
2369-
if (!R || R->getOpcode() != WideMember0->getOpcode() ||
2370-
R->getNumOperands() > 2)
2371-
return;
2372-
if (any_of(R->operands(), [WideMember0, Idx = I, R](VPValue *V) {
2373-
return !canNarrowLoad(WideMember0, R, V, Idx);
2374-
}))
2375-
return;
23762326
}
23772327
StoreGroups.push_back(InterleaveR);
23782328
}
23792329

23802330
if (StoreGroups.empty())
23812331
return;
23822332

2383-
// Convert InterleaveGroup \p R to a single VPWidenLoadRecipe.
2333+
// Convert InterleaveGroup R to a single VPWidenLoadRecipe.
23842334
auto NarrowOp = [](VPRecipeBase *R) -> VPValue * {
2385-
if (auto *LoadGroup = dyn_cast<VPInterleaveRecipe>(R)) {
2386-
// Narrow interleave group to wide load, as transformed VPlan will only
2387-
// process one original iteration.
2388-
auto *L = new VPWidenLoadRecipe(
2389-
*cast<LoadInst>(LoadGroup->getInterleaveGroup()->getInsertPos()),
2390-
LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true,
2391-
/*Reverse=*/false, LoadGroup->getDebugLoc());
2392-
L->insertBefore(LoadGroup);
2393-
return L;
2394-
}
2395-
2396-
auto *WideLoad = cast<VPWidenLoadRecipe>(R);
2397-
2398-
// Narrow wide load to uniform scalar load, as transformed VPlan will only
2335+
auto *LoadGroup = cast<VPInterleaveRecipe>(R);
2336+
// Narrow interleave group to wide load, as transformed VPlan will only
23992337
// process one original iteration.
2400-
auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(),
2401-
WideLoad->operands(), /*IsUniform*/ true);
2402-
N->insertBefore(WideLoad);
2403-
return N;
2338+
auto *L = new VPWidenLoadRecipe(
2339+
*cast<LoadInst>(LoadGroup->getInterleaveGroup()->getInsertPos()),
2340+
LoadGroup->getAddr(), LoadGroup->getMask(), /*Consecutive=*/true,
2341+
/*Reverse=*/false, LoadGroup->getDebugLoc());
2342+
L->insertBefore(LoadGroup);
2343+
return L;
24042344
};
24052345

24062346
// Narrow operation tree rooted at store groups.
24072347
for (auto *StoreGroup : StoreGroups) {
2408-
VPValue *Res = nullptr;
2409-
if (auto *WideMember0 = dyn_cast<VPWidenRecipe>(
2410-
StoreGroup->getStoredValues()[0]->getDefiningRecipe())) {
2411-
for (unsigned Idx = 0, E = WideMember0->getNumOperands(); Idx != E; ++Idx)
2412-
WideMember0->setOperand(
2413-
Idx, NarrowOp(WideMember0->getOperand(Idx)->getDefiningRecipe()));
2414-
Res = WideMember0;
2415-
} else {
2416-
Res = NarrowOp(StoreGroup->getStoredValues()[0]->getDefiningRecipe());
2417-
}
2348+
VPValue *Res =
2349+
NarrowOp(StoreGroup->getStoredValues()[0]->getDefiningRecipe());
24182350

24192351
auto *S = new VPWidenStoreRecipe(
24202352
*cast<StoreInst>(StoreGroup->getInterleaveGroup()->getInsertPos()),

‎llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -99,17 +99,31 @@ define void @test_complex_add_double(ptr %res, ptr noalias %A, ptr noalias %B, i
9999
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[A]], i64 [[TMP1]]
100100
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i64 [[TMP0]]
101101
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[B]], i64 [[TMP1]]
102-
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = load <2 x double>, ptr [[TMP2]], align 4
103-
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP3]], align 4
104-
; CHECK-NEXT: [[STRIDED_VEC10:%.*]] = load <2 x double>, ptr [[TMP4]], align 4
105-
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <2 x double>, ptr [[TMP5]], align 4
102+
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP2]], align 4
103+
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
104+
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
105+
; CHECK-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP3]], align 4
106+
; CHECK-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
107+
; CHECK-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
108+
; CHECK-NEXT: [[WIDE_VEC5:%.*]] = load <4 x double>, ptr [[TMP4]], align 4
109+
; CHECK-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <4 x double> [[WIDE_VEC5]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
110+
; CHECK-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <4 x double> [[WIDE_VEC5]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
111+
; CHECK-NEXT: [[WIDE_VEC8:%.*]] = load <4 x double>, ptr [[TMP5]], align 4
112+
; CHECK-NEXT: [[STRIDED_VEC9:%.*]] = shufflevector <4 x double> [[WIDE_VEC8]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
113+
; CHECK-NEXT: [[STRIDED_VEC10:%.*]] = shufflevector <4 x double> [[WIDE_VEC8]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
114+
; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[STRIDED_VEC]], [[STRIDED_VEC6]]
115+
; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> [[STRIDED_VEC3]], [[STRIDED_VEC9]]
116+
; CHECK-NEXT: [[TMP8:%.*]] = fadd <2 x double> [[STRIDED_VEC1]], [[STRIDED_VEC7]]
106117
; CHECK-NEXT: [[TMP9:%.*]] = fadd <2 x double> [[STRIDED_VEC4]], [[STRIDED_VEC10]]
107-
; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
108118
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP0]]
109119
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[RES]], i64 [[TMP1]]
110-
; CHECK-NEXT: store <2 x double> [[TMP9]], ptr [[TMP10]], align 4
111-
; CHECK-NEXT: store <2 x double> [[TMP7]], ptr [[TMP11]], align 4
112-
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
120+
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
121+
; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
122+
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP10]], align 4
123+
; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
124+
; CHECK-NEXT: [[INTERLEAVED_VEC11:%.*]] = shufflevector <4 x double> [[TMP13]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
125+
; CHECK-NEXT: store <4 x double> [[INTERLEAVED_VEC11]], ptr [[TMP11]], align 4
126+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
113127
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
114128
; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
115129
; CHECK: [[MIDDLE_BLOCK]]:

‎llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops.ll‎

Lines changed: 41 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,24 @@ define void @test_2xi64_unary_op_load_interleave_group(ptr noalias %data, ptr no
1919
; VF2-NEXT: [[TMP1:%.*]] = shl nsw i64 [[TMP0]], 1
2020
; VF2-NEXT: [[TMP12:%.*]] = shl nsw i64 [[TMP10]], 1
2121
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP1]]
22-
; VF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP12]]
23-
; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 8
24-
; VF2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x double>, ptr [[TMP5]], align 8
25-
; VF2-NEXT: [[TMP9:%.*]] = fneg <2 x double> [[WIDE_LOAD]]
26-
; VF2-NEXT: [[TMP11:%.*]] = fneg <2 x double> [[WIDE_LOAD1]]
27-
; VF2-NEXT: store <2 x double> [[TMP9]], ptr [[TMP2]], align 8
28-
; VF2-NEXT: store <2 x double> [[TMP11]], ptr [[TMP5]], align 8
29-
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
22+
; VF2-NEXT: [[TMP13:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP12]]
23+
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP2]], align 8
24+
; VF2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
25+
; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
26+
; VF2-NEXT: [[WIDE_VEC2:%.*]] = load <4 x double>, ptr [[TMP13]], align 8
27+
; VF2-NEXT: [[STRIDED_VEC3:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
28+
; VF2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x double> [[WIDE_VEC2]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
29+
; VF2-NEXT: [[TMP3:%.*]] = fneg <2 x double> [[STRIDED_VEC]]
30+
; VF2-NEXT: [[TMP14:%.*]] = fneg <2 x double> [[STRIDED_VEC3]]
31+
; VF2-NEXT: [[TMP4:%.*]] = fneg <2 x double> [[STRIDED_VEC1]]
32+
; VF2-NEXT: [[TMP9:%.*]] = fneg <2 x double> [[STRIDED_VEC4]]
33+
; VF2-NEXT: [[TMP5:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
34+
; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
35+
; VF2-NEXT: store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
36+
; VF2-NEXT: [[TMP11:%.*]] = shufflevector <2 x double> [[TMP14]], <2 x double> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
37+
; VF2-NEXT: [[INTERLEAVED_VEC5:%.*]] = shufflevector <4 x double> [[TMP11]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
38+
; VF2-NEXT: store <4 x double> [[INTERLEAVED_VEC5]], ptr [[TMP13]], align 8
39+
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
3040
; VF2-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
3141
; VF2-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3242
; VF2: [[MIDDLE_BLOCK]]:
@@ -190,15 +200,18 @@ define void @test_2xi64(ptr noalias %data, ptr noalias %factor) {
190200
; VF2-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
191201
; VF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[TMP0]]
192202
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 0
193-
; VF2-NEXT: [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 8
194-
; VF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i64 0
195-
; VF2-NEXT: [[WIDE_LOAD:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
203+
; VF2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
196204
; VF2-NEXT: [[TMP6:%.*]] = shl nsw i64 [[TMP0]], 1
197205
; VF2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP6]]
198-
; VF2-NEXT: [[STRIDED_VEC1:%.*]] = load <2 x i64>, ptr [[TMP7]], align 8
199-
; VF2-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
200-
; VF2-NEXT: store <2 x i64> [[TMP8]], ptr [[TMP7]], align 8
201-
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
206+
; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP7]], align 8
207+
; VF2-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
208+
; VF2-NEXT: [[TMP23:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
209+
; VF2-NEXT: [[TMP12:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[TMP11]]
210+
; VF2-NEXT: [[TMP24:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[TMP23]]
211+
; VF2-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP24]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
212+
; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
213+
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
214+
; VF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
202215
; VF2-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
203216
; VF2-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
204217
; VF2: [[MIDDLE_BLOCK]]:
@@ -1001,30 +1014,28 @@ define void @test_2xi64_sub_of_wide_loads(ptr noalias %data, ptr noalias %A, ptr
10011014
; VF2-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
10021015
; VF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
10031016
; VF2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 2
1004-
; VF2-NEXT: [[TMP5:%.*]] = load i64, ptr [[TMP3]], align 8
1005-
; VF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP5]], i64 0
1006-
; VF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
1007-
; VF2-NEXT: [[TMP6:%.*]] = load i64, ptr [[TMP4]], align 8
1008-
; VF2-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i64 0
1009-
; VF2-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT3]], <2 x i64> poison, <2 x i32> zeroinitializer
1017+
; VF2-NEXT: [[BROADCAST_SPLAT:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8
1018+
; VF2-NEXT: [[BROADCAST_SPLAT4:%.*]] = load <2 x i64>, ptr [[TMP4]], align 8
10101019
; VF2-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]]
10111020
; VF2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0
10121021
; VF2-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 2
1013-
; VF2-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP8]], align 8
1014-
; VF2-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i64 0
1015-
; VF2-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
1016-
; VF2-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP9]], align 8
1017-
; VF2-NEXT: [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <2 x i64> poison, i64 [[TMP11]], i64 0
1018-
; VF2-NEXT: [[BROADCAST_SPLAT6:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT5]], <2 x i64> poison, <2 x i32> zeroinitializer
1022+
; VF2-NEXT: [[BROADCAST_SPLAT2:%.*]] = load <2 x i64>, ptr [[TMP8]], align 8
1023+
; VF2-NEXT: [[BROADCAST_SPLAT6:%.*]] = load <2 x i64>, ptr [[TMP9]], align 8
10191024
; VF2-NEXT: [[TMP12:%.*]] = sub <2 x i64> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT2]]
10201025
; VF2-NEXT: [[TMP13:%.*]] = sub <2 x i64> [[BROADCAST_SPLAT4]], [[BROADCAST_SPLAT6]]
10211026
; VF2-NEXT: [[TMP19:%.*]] = shl nsw i64 [[TMP0]], 1
10221027
; VF2-NEXT: [[TMP20:%.*]] = shl nsw i64 [[TMP1]], 1
10231028
; VF2-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP19]]
10241029
; VF2-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP20]]
1025-
; VF2-NEXT: store <2 x i64> [[TMP12]], ptr [[DATA_0]], align 8
1026-
; VF2-NEXT: store <2 x i64> [[TMP13]], ptr [[DATA_1]], align 8
1027-
; VF2-NEXT: [[IV_NEXT]] = add nuw i64 [[INDEX]], 2
1030+
; VF2-NEXT: [[TMP14:%.*]] = sub <2 x i64> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT2]]
1031+
; VF2-NEXT: [[TMP15:%.*]] = sub <2 x i64> [[BROADCAST_SPLAT4]], [[BROADCAST_SPLAT6]]
1032+
; VF2-NEXT: [[TMP16:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1033+
; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP16]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
1034+
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC]], ptr [[DATA_0]], align 8
1035+
; VF2-NEXT: [[TMP17:%.*]] = shufflevector <2 x i64> [[TMP13]], <2 x i64> [[TMP15]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
1036+
; VF2-NEXT: [[INTERLEAVED_VEC4:%.*]] = shufflevector <4 x i64> [[TMP17]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
1037+
; VF2-NEXT: store <4 x i64> [[INTERLEAVED_VEC4]], ptr [[DATA_1]], align 8
1038+
; VF2-NEXT: [[IV_NEXT]] = add nuw i64 [[INDEX]], 4
10281039
; VF2-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
10291040
; VF2-NEXT: br i1 [[EC]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
10301041
; VF2: [[MIDDLE_BLOCK]]:

0 commit comments

Comments
 (0)