@@ -92,13 +92,14 @@ class AArch64ExpandPseudo : public MachineFunctionPass {
9292 bool expandCALL_BTI (MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
9393 bool expandStoreSwiftAsyncContext (MachineBasicBlock &MBB,
9494 MachineBasicBlock::iterator MBBI);
95-
96- MachineBasicBlock *
97- expandConditionalPseudo (MachineBasicBlock &MBB,
98- MachineBasicBlock::iterator MBBI, DebugLoc DL,
99- MachineInstrBuilder &Branch,
100- function_ref<void (MachineBasicBlock &)> InsertBody);
101-
95+ struct ConditionalBlocks {
96+ MachineBasicBlock &CondBB;
97+ MachineBasicBlock &EndBB;
98+ };
99+ ConditionalBlocks expandConditionalPseudo (MachineBasicBlock &MBB,
100+ MachineBasicBlock::iterator MBBI,
101+ DebugLoc DL,
102+ MachineInstrBuilder &Branch);
102103 MachineBasicBlock *expandRestoreZASave (MachineBasicBlock &MBB,
103104 MachineBasicBlock::iterator MBBI);
104105 MachineBasicBlock *expandCommitZASave (MachineBasicBlock &MBB,
@@ -999,10 +1000,11 @@ bool AArch64ExpandPseudo::expandStoreSwiftAsyncContext(
9991000 return true ;
10001001}
10011002
1002- MachineBasicBlock *AArch64ExpandPseudo::expandConditionalPseudo (
1003- MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL,
1004- MachineInstrBuilder &Branch,
1005- function_ref<void (MachineBasicBlock &)> InsertBody) {
1003+ AArch64ExpandPseudo::ConditionalBlocks
1004+ AArch64ExpandPseudo::expandConditionalPseudo (MachineBasicBlock &MBB,
1005+ MachineBasicBlock::iterator MBBI,
1006+ DebugLoc DL,
1007+ MachineInstrBuilder &Branch) {
10061008 MachineInstr &MI = *MBBI;
10071009 assert ((std::next (MBBI) != MBB.end () ||
10081010 MI.getParent ()->successors ().begin () !=
@@ -1011,26 +1013,24 @@ MachineBasicBlock *AArch64ExpandPseudo::expandConditionalPseudo(
10111013
10121014 // Split MBB and create two new blocks:
10131015 // - MBB now contains all instructions before the conditional pseudo.
1014- // - SMBB contains the conditional pseudo instruction only.
1016+ // - CondBB contains the conditional pseudo instruction only.
10151017 // - EndBB contains all instructions after the conditional pseudo.
10161018 MachineInstr &PrevMI = *std::prev (MBBI);
1017- MachineBasicBlock *SMBB = MBB.splitAt (PrevMI, /* UpdateLiveIns*/ true );
1018- MachineBasicBlock *EndBB = std::next (MI.getIterator ()) == SMBB ->end ()
1019- ? *SMBB ->successors ().begin ()
1020- : SMBB ->splitAt (MI, /* UpdateLiveIns*/ true );
1019+ MachineBasicBlock *CondBB = MBB.splitAt (PrevMI, /* UpdateLiveIns*/ true );
1020+ MachineBasicBlock *EndBB = std::next (MI.getIterator ()) == CondBB ->end ()
1021+ ? *CondBB ->successors ().begin ()
1022+ : CondBB ->splitAt (MI, /* UpdateLiveIns*/ true );
10211023
10221024 // Add the SMBB label to the branch instruction & create a branch to EndBB.
1023- Branch.addMBB (SMBB );
1025+ Branch.addMBB (CondBB );
10241026 BuildMI (&MBB, DL, TII->get (AArch64::B))
10251027 .addMBB (EndBB);
10261028 MBB.addSuccessor (EndBB);
10271029
1028- // Insert the conditional pseudo expansion.
1029- InsertBody (*SMBB);
1030-
1031- BuildMI (SMBB, DL, TII->get (AArch64::B)).addMBB (EndBB);
1032- MI.eraseFromParent ();
1033- return EndBB;
1030+ // Create branch from CondBB to EndBB. Users of this helper should insert new
1031+ // instructions at CondBB.back() -- i.e. before the branch.
1032+ BuildMI (CondBB, DL, TII->get (AArch64::B)).addMBB (EndBB);
1033+ return {*CondBB, *EndBB};
10341034}
10351035
10361036MachineBasicBlock *
@@ -1043,17 +1043,18 @@ AArch64ExpandPseudo::expandRestoreZASave(MachineBasicBlock &MBB,
10431043 MachineInstrBuilder Branch =
10441044 BuildMI (MBB, MBBI, DL, TII->get (AArch64::CBZX)).add (MI.getOperand (0 ));
10451045
1046- return expandConditionalPseudo (
1047- MBB, MBBI, DL, Branch, [&](MachineBasicBlock &SMBB) {
1048- // Replace the pseudo with a call (BL).
1049- MachineInstrBuilder MIB =
1050- BuildMI (SMBB, SMBB.end (), DL, TII->get (AArch64::BL));
1051- // Copy operands (mainly the regmask) from the pseudo.
1052- for (unsigned I = 2 ; I < MI.getNumOperands (); ++I)
1053- MIB.add (MI.getOperand (I));
1054- // Mark the TPIDR2 block pointer (X0) as an implicit use.
1055- MIB.addReg (MI.getOperand (1 ).getReg (), RegState::Implicit);
1056- });
1046+ auto [CondBB, EndBB] = expandConditionalPseudo (MBB, MBBI, DL, Branch);
1047+ // Replace the pseudo with a call (BL).
1048+ MachineInstrBuilder MIB =
1049+ BuildMI (CondBB, CondBB.back (), DL, TII->get (AArch64::BL));
1050+ // Copy operands (mainly the regmask) from the pseudo.
1051+ for (unsigned I = 2 ; I < MI.getNumOperands (); ++I)
1052+ MIB.add (MI.getOperand (I));
1053+ // Mark the TPIDR2 block pointer (X0) as an implicit use.
1054+ MIB.addReg (MI.getOperand (1 ).getReg (), RegState::Implicit);
1055+
1056+ MI.eraseFromParent ();
1057+ return &EndBB;
10571058}
10581059
10591060static constexpr unsigned ZERO_ALL_ZA_MASK = 0b11111111 ;
@@ -1070,26 +1071,27 @@ AArch64ExpandPseudo::expandCommitZASave(MachineBasicBlock &MBB,
10701071 MachineInstrBuilder Branch =
10711072 BuildMI (MBB, MBBI, DL, TII->get (AArch64::CBNZX)).add (MI.getOperand (0 ));
10721073
1073- return expandConditionalPseudo (
1074- MBB, MBBI, DL, Branch, [&](MachineBasicBlock &SMBB) {
1075- // Replace the pseudo with a call (BL).
1076- MachineInstrBuilder MIB =
1077- BuildMI (SMBB, SMBB.end (), DL, TII->get (AArch64::BL));
1078- // Copy operands (mainly the regmask) from the pseudo.
1079- for (unsigned I = 2 ; I < MI.getNumOperands (); ++I)
1080- MIB.add (MI.getOperand (I));
1081- // Clear TPIDR2_EL0.
1082- BuildMI (SMBB, SMBB.end (), DL, TII->get (AArch64::MSR))
1083- .addImm (AArch64SysReg::TPIDR2_EL0)
1084- .addReg (AArch64::XZR);
1085- bool ZeroZA = MI.getOperand (1 ).getImm () != 0 ;
1086- if (ZeroZA) {
1087- assert (MI.definesRegister (AArch64::ZAB0, TRI) && " should define ZA!" );
1088- BuildMI (SMBB, SMBB.end (), DL, TII->get (AArch64::ZERO_M))
1089- .addImm (ZERO_ALL_ZA_MASK)
1090- .addDef (AArch64::ZAB0, RegState::ImplicitDefine);
1091- }
1092- });
1074+ auto [CondBB, EndBB] = expandConditionalPseudo (MBB, MBBI, DL, Branch);
1075+ // Replace the pseudo with a call (BL).
1076+ MachineInstrBuilder MIB =
1077+ BuildMI (CondBB, CondBB.back (), DL, TII->get (AArch64::BL));
1078+ // Copy operands (mainly the regmask) from the pseudo.
1079+ for (unsigned I = 2 ; I < MI.getNumOperands (); ++I)
1080+ MIB.add (MI.getOperand (I));
1081+ // Clear TPIDR2_EL0.
1082+ BuildMI (CondBB, CondBB.back (), DL, TII->get (AArch64::MSR))
1083+ .addImm (AArch64SysReg::TPIDR2_EL0)
1084+ .addReg (AArch64::XZR);
1085+ bool ZeroZA = MI.getOperand (1 ).getImm () != 0 ;
1086+ if (ZeroZA) {
1087+ assert (MI.definesRegister (AArch64::ZAB0, TRI) && " should define ZA!" );
1088+ BuildMI (CondBB, CondBB.back (), DL, TII->get (AArch64::ZERO_M))
1089+ .addImm (ZERO_ALL_ZA_MASK)
1090+ .addDef (AArch64::ZAB0, RegState::ImplicitDefine);
1091+ }
1092+
1093+ MI.eraseFromParent ();
1094+ return &EndBB;
10931095}
10941096
10951097MachineBasicBlock *
@@ -1163,19 +1165,20 @@ AArch64ExpandPseudo::expandCondSMToggle(MachineBasicBlock &MBB,
11631165 MachineInstrBuilder Tbx =
11641166 BuildMI (MBB, MBBI, DL, TII->get (Opc)).addReg (SMReg32).addImm (0 );
11651167
1166- return expandConditionalPseudo (
1167- MBB, MBBI, DL, Tbx, [&](MachineBasicBlock &SMBB) {
1168- // Create the SMSTART/SMSTOP (MSRpstatesvcrImm1) instruction in SMBB.
1169- MachineInstrBuilder MIB = BuildMI (SMBB, SMBB.begin (), MI.getDebugLoc (),
1170- TII->get (AArch64::MSRpstatesvcrImm1));
1171- // Copy all but the second and third operands of MSRcond_pstatesvcrImm1
1172- // (as these contain the CopyFromReg for the first argument and the flag
1173- // to indicate whether the callee is streaming or normal).
1174- MIB.add (MI.getOperand (0 ));
1175- MIB.add (MI.getOperand (1 ));
1176- for (unsigned i = 4 ; i < MI.getNumOperands (); ++i)
1177- MIB.add (MI.getOperand (i));
1178- });
1168+ auto [CondBB, EndBB] = expandConditionalPseudo (MBB, MBBI, DL, Tbx);
1169+ // Create the SMSTART/SMSTOP (MSRpstatesvcrImm1) instruction in SMBB.
1170+ MachineInstrBuilder MIB = BuildMI (CondBB, CondBB.back (), MI.getDebugLoc (),
1171+ TII->get (AArch64::MSRpstatesvcrImm1));
1172+ // Copy all but the second and third operands of MSRcond_pstatesvcrImm1
1173+ // (as these contain the CopyFromReg for the first argument and the flag
1174+ // to indicate whether the callee is streaming or normal).
1175+ MIB.add (MI.getOperand (0 ));
1176+ MIB.add (MI.getOperand (1 ));
1177+ for (unsigned i = 4 ; i < MI.getNumOperands (); ++i)
1178+ MIB.add (MI.getOperand (i));
1179+
1180+ MI.eraseFromParent ();
1181+ return &EndBB;
11791182}
11801183
11811184bool AArch64ExpandPseudo::expandMultiVecPseudo (
0 commit comments