//===- HexagonEarlyIfConv.cpp ---------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This implements a Hexagon-specific if-conversion pass that runs on the
// SSA form.
// In SSA it is not straightforward to represent instructions that condi-
// tionally define registers, since a conditionally-defined register may
// only be used under the same condition on which the definition was based.
// To avoid complications of this nature, this patch will only generate
// predicated stores, and speculate other instructions from the "if-conver-
// ted" block.
// The code will recognize CFG patterns where a block with a conditional
// branch "splits" into a "true block" and a "false block". Either of these
// could be omitted (in case of a triangle, for example).
// If after conversion of the side block(s) the CFG allows it, the resul-
// ting blocks may be merged. If the "join" block contained PHI nodes, they
// will be replaced with MUX (or MUX-like) instructions to maintain the
// semantics of the PHI.
//
// Example:
//
// %40 = L2_loadrub_io killed %39, 1
// %41 = S2_tstbit_i killed %40, 0
// J2_jumpt killed %41, <%bb.5>, implicit dead %pc
// J2_jump <%bb.4>, implicit dead %pc
// Successors according to CFG: %bb.4(62) %bb.5(62)
//
// %bb.4: derived from LLVM BB %if.then
// Predecessors according to CFG: %bb.3
// %11 = A2_addp %6, %10
// S2_storerd_io %32, 16, %11
// Successors according to CFG: %bb.5
//
// %bb.5: derived from LLVM BB %if.end
// Predecessors according to CFG: %bb.3 %bb.4
// %12 = PHI %6, <%bb.3>, %11, <%bb.4>
// %13 = A2_addp %7, %12
// %42 = C2_cmpeqi %9, 10
// J2_jumpf killed %42, <%bb.3>, implicit dead %pc
// J2_jump <%bb.6>, implicit dead %pc
// Successors according to CFG: %bb.6(4) %bb.3(124)
//
// would become:
//
// %40 = L2_loadrub_io killed %39, 1
// %41 = S2_tstbit_i killed %40, 0
// spec-> %11 = A2_addp %6, %10
// pred-> S2_pstorerdf_io %41, %32, 16, %11
// %46 = PS_pselect %41, %6, %11
// %13 = A2_addp %7, %46
// %42 = C2_cmpeqi %9, 10
// J2_jumpf killed %42, <%bb.3>, implicit dead %pc
// J2_jump <%bb.6>, implicit dead %pc
// Successors according to CFG: %bb.6 %bb.3
#include "Hexagon.h"
#include "HexagonInstrInfo.h"
#include "HexagonSubtarget.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include <cassert>
#include <iterator>
#define DEBUG_TYPE "hexagon-eif"
using namespace llvm;
namespace llvm {
FunctionPass *createHexagonEarlyIfConversion();
void initializeHexagonEarlyIfConversionPass(PassRegistry& Registry);
} // end namespace llvm
static cl::opt<bool> EnableHexagonBP("enable-hexagon-br-prob", cl::Hidden,
cl::init(true), cl::desc("Enable branch probability info"));
static cl::opt<unsigned> SizeLimit("eif-limit", cl::init(6), cl::Hidden,
cl::desc("Size limit in Hexagon early if-conversion"));
static cl::opt<bool> SkipExitBranches("eif-no-loop-exit", cl::init(false),
cl::Hidden, cl::desc("Do not convert branches that may exit the loop"));
namespace {
struct PrintMB {
PrintMB(const MachineBasicBlock *B) : MB(B) {}
const MachineBasicBlock *MB;
};
raw_ostream &operator<< (raw_ostream &OS, const PrintMB &P) {
if (!P.MB)
return OS << "<none>";
return OS << '#' << P.MB->getNumber();
}
struct FlowPattern {
FlowPattern() = default;
FlowPattern(MachineBasicBlock *B, unsigned PR, MachineBasicBlock *TB,
MachineBasicBlock *FB, MachineBasicBlock *JB)
: SplitB(B), TrueB(TB), FalseB(FB), JoinB(JB), PredR(PR) {}
MachineBasicBlock *SplitB = nullptr;
MachineBasicBlock *TrueB = nullptr;
MachineBasicBlock *FalseB = nullptr;
MachineBasicBlock *JoinB = nullptr;
unsigned PredR = 0;
};
struct PrintFP {
PrintFP(const FlowPattern &P, const TargetRegisterInfo &T)
: FP(P), TRI(T) {}
const FlowPattern &FP;
const TargetRegisterInfo &TRI;
friend raw_ostream &operator<< (raw_ostream &OS, const PrintFP &P);
};
raw_ostream &operator<<(raw_ostream &OS,
const PrintFP &P) LLVM_ATTRIBUTE_UNUSED;
raw_ostream &operator<<(raw_ostream &OS, const PrintFP &P) {
OS << "{ SplitB:" << PrintMB(P.FP.SplitB)
<< ", PredR:" << printReg(P.FP.PredR, &P.TRI)
<< ", TrueB:" << PrintMB(P.FP.TrueB)
<< ", FalseB:" << PrintMB(P.FP.FalseB)
<< ", JoinB:" << PrintMB(P.FP.JoinB) << " }";
return OS;
}
class HexagonEarlyIfConversion : public MachineFunctionPass {
public:
static char ID;
HexagonEarlyIfConversion() : MachineFunctionPass(ID) {}
StringRef getPassName() const override {
return "Hexagon early if conversion";
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<MachineBranchProbabilityInfo>();
AU.addRequired<MachineDominatorTree>();
AU.addPreserved<MachineDominatorTree>();
AU.addRequired<MachineLoopInfo>();
MachineFunctionPass::getAnalysisUsage(AU);
}
bool runOnMachineFunction(MachineFunction &MF) override;
private:
using BlockSetType = DenseSet<MachineBasicBlock *>;
bool isPreheader(const MachineBasicBlock *B) const;
bool matchFlowPattern(MachineBasicBlock *B, MachineLoop *L,
FlowPattern &FP);
bool visitBlock(MachineBasicBlock *B, MachineLoop *L);
bool visitLoop(MachineLoop *L);
bool hasEHLabel(const MachineBasicBlock *B) const;
bool hasUncondBranch(const MachineBasicBlock *B) const;
bool isValidCandidate(const MachineBasicBlock *B) const;
bool usesUndefVReg(const MachineInstr *MI) const;
bool isValid(const FlowPattern &FP) const;
unsigned countPredicateDefs(const MachineBasicBlock *B) const;
unsigned computePhiCost(const MachineBasicBlock *B,
const FlowPattern &FP) const;
bool isProfitable(const FlowPattern &FP) const;
bool isPredicableStore(const MachineInstr *MI) const;
bool isSafeToSpeculate(const MachineInstr *MI) const;
bool isPredicate(unsigned R) const;
unsigned getCondStoreOpcode(unsigned Opc, bool IfTrue) const;
void predicateInstr(MachineBasicBlock *ToB, MachineBasicBlock::iterator At,
MachineInstr *MI, unsigned PredR, bool IfTrue);
void predicateBlockNB(MachineBasicBlock *ToB,
MachineBasicBlock::iterator At, MachineBasicBlock *FromB,
unsigned PredR, bool IfTrue);
unsigned buildMux(MachineBasicBlock *B, MachineBasicBlock::iterator At,
const TargetRegisterClass *DRC, unsigned PredR, unsigned TR,
unsigned TSR, unsigned FR, unsigned FSR);
void updatePhiNodes(MachineBasicBlock *WhereB, const FlowPattern &FP);
void convert(const FlowPattern &FP);
void removeBlock(MachineBasicBlock *B);
void eliminatePhis(MachineBasicBlock *B);
void mergeBlocks(MachineBasicBlock *PredB, MachineBasicBlock *SuccB);
void simplifyFlowGraph(const FlowPattern &FP);
const HexagonInstrInfo *HII = nullptr;
const TargetRegisterInfo *TRI = nullptr;
MachineFunction *MFN = nullptr;
MachineRegisterInfo *MRI = nullptr;
MachineDominatorTree *MDT = nullptr;
MachineLoopInfo *MLI = nullptr;
BlockSetType Deleted;
const MachineBranchProbabilityInfo *MBPI = nullptr;
};
} // end anonymous namespace
char HexagonEarlyIfConversion::ID = 0;
INITIALIZE_PASS(HexagonEarlyIfConversion, "hexagon-early-if",
"Hexagon early if conversion", false, false)
bool HexagonEarlyIfConversion::isPreheader(const MachineBasicBlock *B) const {
if (B->succ_size() != 1)
return false;
MachineBasicBlock *SB = *B->succ_begin();
MachineLoop *L = MLI->getLoopFor(SB);
return L && SB == L->getHeader() && MDT->dominates(B, SB);
}
bool HexagonEarlyIfConversion::matchFlowPattern(MachineBasicBlock *B,
MachineLoop *L, FlowPattern &FP) {
LLVM_DEBUG(dbgs() << "Checking flow pattern at " << printMBBReference(*B)
<< "\n");
// Interested only in conditional branches, no .new, no new-value, etc.
// Check the terminators directly, it's easier than handling all responses
// from analyzeBranch.
MachineBasicBlock *TB = nullptr, *FB = nullptr;
MachineBasicBlock::const_iterator T1I = B->getFirstTerminator();
if (T1I == B->end())
return false;
unsigned Opc = T1I->getOpcode();
if (Opc != Hexagon::J2_jumpt && Opc != Hexagon::J2_jumpf)
return false;
Register PredR = T1I->getOperand(0).getReg();
// Get the layout successor, or 0 if B does not have one.
MachineFunction::iterator NextBI = std::next(MachineFunction::iterator(B));
MachineBasicBlock *NextB = (NextBI != MFN->end()) ? &*NextBI : nullptr;
MachineBasicBlock *T1B = T1I->getOperand(1).getMBB();
MachineBasicBlock::const_iterator T2I = std::next(T1I);
// The second terminator should be an unconditional branch.
assert(T2I == B->end() || T2I->getOpcode() == Hexagon::J2_jump);
MachineBasicBlock *T2B = (T2I == B->end()) ? NextB
: T2I->getOperand(0).getMBB();
if (T1B == T2B) {
// XXX merge if T1B == NextB, or convert branch to unconditional.
// mark as diamond with both sides equal?
return false;
}
// Record the true/false blocks in such a way that "true" means "if (PredR)",
// and "false" means "if (!PredR)".
if (Opc == Hexagon::J2_jumpt)
TB = T1B, FB = T2B;
else
TB = T2B, FB = T1B;
if (!MDT->properlyDominates(B, TB) || !MDT->properlyDominates(B, FB))
return false;
// Detect triangle first. In case of a triangle, one of the blocks TB/FB
// can fall through into the other, in other words, it will be executed
// in both cases. We only want to predicate the block that is executed
// conditionally.
assert(TB && FB && "Failed to find triangle control flow blocks");
unsigned TNP = TB->pred_size(), FNP = FB->pred_size();
unsigned TNS = TB->succ_size(), FNS = FB->succ_size();
// A block is predicable if it has one predecessor (it must be B), and
// it has a single successor. In fact, the block has to end either with
// an unconditional branch (which can be predicated), or with a fall-
// through.
// Also, skip blocks that do not belong to the same loop.
bool TOk = (TNP == 1 && TNS == 1 && MLI->getLoopFor(TB) == L);
bool FOk = (FNP == 1 && FNS == 1 && MLI->getLoopFor(FB) == L);
// If requested (via an option), do not consider branches where the
// true and false targets do not belong to the same loop.
if (SkipExitBranches && MLI->getLoopFor(TB) != MLI->getLoopFor(FB))
return false;
// If neither is predicable, there is nothing interesting.
if (!TOk && !FOk)
return false;
MachineBasicBlock *TSB = (TNS > 0) ? *TB->succ_begin() : nullptr;
MachineBasicBlock *FSB = (FNS > 0) ? *FB->succ_begin() : nullptr;
MachineBasicBlock *JB = nullptr;
if (TOk) {
if (FOk) {
if (TSB == FSB)
JB = TSB;
// Diamond: "if (P) then TB; else FB;".
} else {
// TOk && !FOk
if (TSB == FB)
JB = FB;
FB = nullptr;
}
} else {
// !TOk && FOk (at least one must be true by now).
if (FSB == TB)
JB = TB;
TB = nullptr;
}
// Don't try to predicate loop preheaders.
if ((TB && isPreheader(TB)) || (FB && isPreheader(FB))) {
LLVM_DEBUG(dbgs() << "One of blocks " << PrintMB(TB) << ", " << PrintMB(FB)
<< " is a loop preheader. Skipping.\n");
return false;
}
FP = FlowPattern(B, PredR, TB, FB, JB);
LLVM_DEBUG(dbgs() << "Detected " << PrintFP(FP, *TRI) << "\n");
return true;
}
// KLUDGE: HexagonInstrInfo::analyzeBranch won't work on a block that
// contains EH_LABEL.
bool HexagonEarlyIfConversion::hasEHLabel(const MachineBasicBlock *B) const {
for (auto &I : *B)
if (I.isEHLabel())
return true;
return false;
}
// KLUDGE: HexagonInstrInfo::analyzeBranch may be unable to recognize
// that a block can never fall-through.
bool HexagonEarlyIfConversion::hasUncondBranch(const MachineBasicBlock *B)
const {
MachineBasicBlock::const_iterator I = B->getFirstTerminator(), E = B->end();
while (I != E) {
if (I->isBarrier())
return true;
++I;
}
return false;
}
bool HexagonEarlyIfConversion::isValidCandidate(const MachineBasicBlock *B)
const {
if (!B)
return true;
if (B->isEHPad() || B->hasAddressTaken())
return false;
if (B->succ_size() == 0)
return false;
for (auto &MI : *B) {
if (MI.isDebugInstr())
continue;
if (MI.isConditionalBranch())
return false;
unsigned Opc = MI.getOpcode();
bool IsJMP = (Opc == Hexagon::J2_jump);
if (!isPredicableStore(&MI) && !IsJMP && !isSafeToSpeculate(&MI))
return false;
// Look for predicate registers defined by this instruction. It's ok
// to speculate such an instruction, but the predicate register cannot
// be used outside of this block (or else it won't be possible to
// update the use of it after predication). PHI uses will be updated
// to use a result of a MUX, and a MUX cannot be created for predicate
// registers.
for (const MachineOperand &MO : MI.operands()) {
if (!MO.isReg() || !MO.isDef())
continue;
Register R = MO.getReg();
if (!Register::isVirtualRegister(R))
continue;
if (!isPredicate(R))
continue;
for (auto U = MRI->use_begin(R); U != MRI->use_end(); ++U)
if (U->getParent()->isPHI())
return false;
}
}
return true;
}
bool HexagonEarlyIfConversion::usesUndefVReg(const MachineInstr *MI) const {
for (const MachineOperand &MO : MI->operands()) {
if (!MO.isReg() || !MO.isUse())
continue;
Register R = MO.getReg();
if (!Register::isVirtualRegister(R))
continue;
const MachineInstr *DefI = MRI->getVRegDef(R);
// "Undefined" virtual registers are actually defined via IMPLICIT_DEF.
assert(DefI && "Expecting a reaching def in MRI");
if (DefI->isImplicitDef())
return true;
}
return false;
}
bool HexagonEarlyIfConversion::isValid(const FlowPattern &FP) const {
if (hasEHLabel(FP.SplitB)) // KLUDGE: see function definition
return false;
if (FP.TrueB && !isValidCandidate(FP.TrueB))
return false;
if (FP.FalseB && !isValidCandidate(FP.FalseB))
return false;
// Check the PHIs in the join block. If any of them use a register
// that is defined as IMPLICIT_DEF, do not convert this. This can
// legitimately happen if one side of the split never executes, but
// the compiler is unable to prove it. That side may then seem to
// provide an "undef" value to the join block, however it will never
// execute at run-time. If we convert this case, the "undef" will
// be used in a MUX instruction, and that may seem like actually
// using an undefined value to other optimizations. This could lead
// to trouble further down the optimization stream, cause assertions
// to fail, etc.
if (FP.JoinB) {
const MachineBasicBlock &B = *FP.JoinB;
for (auto &MI : B) {
if (!MI.isPHI())
break;
if (usesUndefVReg(&MI))
return false;
Register DefR = MI.getOperand(0).getReg();
if (isPredicate(DefR))
return false;
}
}
return true;
}
unsigned HexagonEarlyIfConversion::computePhiCost(const MachineBasicBlock *B,
const FlowPattern &FP) const {
if (B->pred_size() < 2)
return 0;
unsigned Cost = 0;
for (const MachineInstr &MI : *B) {
if (!MI.isPHI())
break;
// If both incoming blocks are one of the TrueB/FalseB/SplitB, then
// a MUX may be needed. Otherwise the PHI will need to be updated at
// no extra cost.
// Find the interesting PHI operands for further checks.
SmallVector<unsigned,2> Inc;
for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
const MachineBasicBlock *BB = MI.getOperand(i+1).getMBB();
if (BB == FP.SplitB || BB == FP.TrueB || BB == FP.FalseB)
Inc.push_back(i);
}
assert(Inc.size() <= 2);
if (Inc.size() < 2)
continue;
const MachineOperand &RA = MI.getOperand(1);
const MachineOperand &RB = MI.getOperand(3);
assert(RA.isReg() && RB.isReg());
// Must have a MUX if the phi uses a subregister.
if (RA.getSubReg() != 0 || RB.getSubReg() != 0) {
Cost++;
continue;
}
const MachineInstr *Def1 = MRI->getVRegDef(RA.getReg());
const MachineInstr *Def3 = MRI->getVRegDef(RB.getReg());
if (!HII->isPredicable(*Def1) || !HII->isPredicable(*Def3))
Cost++;
}
return Cost;
}
unsigned HexagonEarlyIfConversion::countPredicateDefs(
const MachineBasicBlock *B) const {
unsigned PredDefs = 0;
for (auto &MI : *B) {
for (const MachineOperand &MO : MI.operands()) {
if (!MO.isReg() || !MO.isDef())
continue;
Register R = MO.getReg();
if (!Register::isVirtualRegister(R))
continue;
if (isPredicate(R))
PredDefs++;
}
}
return PredDefs;
}
bool HexagonEarlyIfConversion::isProfitable(const FlowPattern &FP) const {
BranchProbability JumpProb(1, 10);
BranchProbability Prob(9, 10);
if (MBPI && FP.TrueB && !FP.FalseB &&
(MBPI->getEdgeProbability(FP.SplitB, FP.TrueB) < JumpProb ||
MBPI->getEdgeProbability(FP.SplitB, FP.TrueB) > Prob))
return false;
if (MBPI && !FP.TrueB && FP.FalseB &&
(MBPI->getEdgeProbability(FP.SplitB, FP.FalseB) < JumpProb ||
MBPI->getEdgeProbability(FP.SplitB, FP.FalseB) > Prob))
return false;
if (FP.TrueB && FP.FalseB) {
// Do not IfCovert if the branch is one sided.
if (MBPI) {
if (MBPI->getEdgeProbability(FP.SplitB, FP.TrueB) > Prob)
return false;
if (MBPI->getEdgeProbability(FP.SplitB, FP.FalseB) > Prob)
return false;
}
// If both sides are predicable, convert them if they join, and the
// join block has no other predecessors.
MachineBasicBlock *TSB = *FP.TrueB->succ_begin();
MachineBasicBlock *FSB = *FP.FalseB->succ_begin();
if (TSB != FSB)
return false;
if (TSB->pred_size() != 2)
return false;
}
// Calculate the total size of the predicated blocks.
// Assume instruction counts without branches to be the approximation of
// the code size. If the predicated blocks are smaller than a packet size,
// approximate the spare room in the packet that could be filled with the
// predicated/speculated instructions.
auto TotalCount = [] (const MachineBasicBlock *B, unsigned &Spare) {
if (!B)
return 0u;
unsigned T = std::count_if(B->begin(), B->getFirstTerminator(),
[](const MachineInstr &MI) {
return !MI.isMetaInstruction();
});
if (T < HEXAGON_PACKET_SIZE)
Spare += HEXAGON_PACKET_SIZE-T;
return T;
};
unsigned Spare = 0;
unsigned TotalIn = TotalCount(FP.TrueB, Spare) + TotalCount(FP.FalseB, Spare);
LLVM_DEBUG(
dbgs() << "Total number of instructions to be predicated/speculated: "
<< TotalIn << ", spare room: " << Spare << "\n");
if (TotalIn >= SizeLimit+Spare)
return false;
// Count the number of PHI nodes that will need to be updated (converted
// to MUX). Those can be later converted to predicated instructions, so
// they aren't always adding extra cost.
// KLUDGE: Also, count the number of predicate register definitions in
// each block. The scheduler may increase the pressure of these and cause
// expensive spills (e.g. bitmnp01).
unsigned TotalPh = 0;
unsigned PredDefs = countPredicateDefs(FP.SplitB);
if (FP.JoinB) {
TotalPh = computePhiCost(FP.JoinB, FP);
PredDefs += countPredicateDefs(FP.JoinB);
} else {
if (FP.TrueB && FP.TrueB->succ_size() > 0) {
MachineBasicBlock *SB = *FP.TrueB->succ_begin();
TotalPh += computePhiCost(SB, FP);
PredDefs += countPredicateDefs(SB);
}
if (FP.FalseB && FP.FalseB->succ_size() > 0) {
MachineBasicBlock *SB = *FP.FalseB->succ_begin();
TotalPh += computePhiCost(SB, FP);
PredDefs += countPredicateDefs(SB);
}
}
LLVM_DEBUG(dbgs() << "Total number of extra muxes from converted phis: "
<< TotalPh << "\n");
if (TotalIn+TotalPh >= SizeLimit+Spare)
return false;
LLVM_DEBUG(dbgs() << "Total number of predicate registers: " << PredDefs
<< "\n");
if (PredDefs > 4)
return false;
return true;
}
bool HexagonEarlyIfConversion::visitBlock(MachineBasicBlock *B,
MachineLoop *L) {
bool Changed = false;
// Visit all dominated blocks from the same loop first, then process B.
MachineDomTreeNode *N = MDT->getNode(B);
using GTN = GraphTraits<MachineDomTreeNode *>;
// We will change CFG/DT during this traversal, so take precautions to
// avoid problems related to invalidated iterators. In fact, processing
// a child C of B cannot cause another child to be removed, but it can
// cause a new child to be added (which was a child of C before C itself
// was removed. This new child C, however, would have been processed
// prior to processing B, so there is no need to process it again.
// Simply keep a list of children of B, and traverse that list.
using DTNodeVectType = SmallVector<MachineDomTreeNode *, 4>;
DTNodeVectType Cn(GTN::child_begin(N), GTN::child_end(N));
for (DTNodeVectType::iterator I = Cn.begin(), E = Cn.end(); I != E; ++I) {
MachineBasicBlock *SB = (*I)->getBlock();
if (!Deleted.count(SB))
Changed |= visitBlock(SB, L);
}
// When walking down the dominator tree, we want to traverse through
// blocks from nested (other) loops, because they can dominate blocks
// that are in L. Skip the non-L blocks only after the tree traversal.
if (MLI->getLoopFor(B) != L)
return Changed;
FlowPattern FP;
if (!matchFlowPattern(B, L, FP))
return Changed;
if (!isValid(FP)) {
LLVM_DEBUG(dbgs() << "Conversion is not valid\n");
return Changed;
}
if (!isProfitable(FP)) {
LLVM_DEBUG(dbgs() << "Conversion is not profitable\n");
return Changed;
}
convert(FP);
simplifyFlowGraph(FP);
return true;
}
bool HexagonEarlyIfConversion::visitLoop(MachineLoop *L) {
MachineBasicBlock *HB = L ? L->getHeader() : nullptr;
LLVM_DEBUG((L ? dbgs() << "Visiting loop H:" << PrintMB(HB)
: dbgs() << "Visiting function")
<< "\n");
bool Changed = false;
if (L) {
for (MachineLoop::iterator I = L->begin(), E = L->end(); I != E; ++I)
Changed |= visitLoop(*I);
}
MachineBasicBlock *EntryB = GraphTraits<MachineFunction*>::getEntryNode(MFN);
Changed |= visitBlock(L ? HB : EntryB, L);
return Changed;
}
bool HexagonEarlyIfConversion::isPredicableStore(const MachineInstr *MI)
const {
// HexagonInstrInfo::isPredicable will consider these stores are non-
// -predicable if the offset would become constant-extended after
// predication.
unsigned Opc = MI->getOpcode();
switch (Opc) {
case Hexagon::S2_storerb_io:
case Hexagon::S2_storerbnew_io:
case Hexagon::S2_storerh_io:
case Hexagon::S2_storerhnew_io:
case Hexagon::S2_storeri_io:
case Hexagon::S2_storerinew_io:
case Hexagon::S2_storerd_io:
case Hexagon::S4_storeirb_io:
case Hexagon::S4_storeirh_io:
case Hexagon::S4_storeiri_io:
return true;
}
// TargetInstrInfo::isPredicable takes a non-const pointer.
return MI->mayStore() && HII->isPredicable(const_cast<MachineInstr&>(*MI));
}
bool HexagonEarlyIfConversion::isSafeToSpeculate(const MachineInstr *MI)
const {
if (MI->mayLoadOrStore())
return false;
if (MI->isCall() || MI->isBarrier() || MI->isBranch())
return false;
if (MI->hasUnmodeledSideEffects())
return false;
if (MI->getOpcode() == TargetOpcode::LIFETIME_END)
return false;
return true;
}
bool HexagonEarlyIfConversion::isPredicate(unsigned R) const {
const TargetRegisterClass *RC = MRI->getRegClass(R);
return RC == &Hexagon::PredRegsRegClass ||
RC == &Hexagon::HvxQRRegClass;
}
unsigned HexagonEarlyIfConversion::getCondStoreOpcode(unsigned Opc,
bool IfTrue) const {
return HII->getCondOpcode(Opc, !IfTrue);
}
void HexagonEarlyIfConversion::predicateInstr(MachineBasicBlock *ToB,
MachineBasicBlock::iterator At, MachineInstr *MI,
unsigned PredR, bool IfTrue) {
DebugLoc DL;
if (At != ToB->end())
DL = At->getDebugLoc();
else if (!ToB->empty())
DL = ToB->back().getDebugLoc();
unsigned Opc = MI->getOpcode();
if (isPredicableStore(MI)) {
unsigned COpc = getCondStoreOpcode(Opc, IfTrue);
assert(COpc);
MachineInstrBuilder MIB = BuildMI(*ToB, At, DL, HII->get(COpc));
MachineInstr::mop_iterator MOI = MI->operands_begin();
if (HII->isPostIncrement(*MI)) {
MIB.add(*MOI);
++MOI;
}
MIB.addReg(PredR);
for (const MachineOperand &MO : make_range(MOI, MI->operands_end()))
MIB.add(MO);
// Set memory references.
MIB.cloneMemRefs(*MI);
MI->eraseFromParent();
return;
}
if (Opc == Hexagon::J2_jump) {
MachineBasicBlock *TB = MI->getOperand(0).getMBB();
const MCInstrDesc &D = HII->get(IfTrue ? Hexagon::J2_jumpt
: Hexagon::J2_jumpf);
BuildMI(*ToB, At, DL, D)
.addReg(PredR)
.addMBB(TB);
MI->eraseFromParent();
return;
}
// Print the offending instruction unconditionally as we are about to
// abort.
dbgs() << *MI;
llvm_unreachable("Unexpected instruction");
}
// Predicate/speculate non-branch instructions from FromB into block ToB.
// Leave the branches alone, they will be handled later. Btw, at this point
// FromB should have at most one branch, and it should be unconditional.
void HexagonEarlyIfConversion::predicateBlockNB(MachineBasicBlock *ToB,
MachineBasicBlock::iterator At, MachineBasicBlock *FromB,
unsigned PredR, bool IfTrue) {
LLVM_DEBUG(dbgs() << "Predicating block " << PrintMB(FromB) << "\n");
MachineBasicBlock::iterator End = FromB->getFirstTerminator();
MachineBasicBlock::iterator I, NextI;
for (I = FromB->begin(); I != End; I = NextI) {
assert(!I->isPHI());
NextI = std::next(I);
if (isSafeToSpeculate(&*I))
ToB->splice(At, FromB, I);
else
predicateInstr(ToB, At, &*I, PredR, IfTrue);
}
}
unsigned HexagonEarlyIfConversion::buildMux(MachineBasicBlock *B,
MachineBasicBlock::iterator At, const TargetRegisterClass *DRC,
unsigned PredR, unsigned TR, unsigned TSR, unsigned FR, unsigned FSR) {
unsigned Opc = 0;
switch (DRC->getID()) {
case Hexagon::IntRegsRegClassID:
case Hexagon::IntRegsLow8RegClassID:
Opc = Hexagon::C2_mux;
break;
case Hexagon::DoubleRegsRegClassID:
case Hexagon::GeneralDoubleLow8RegsRegClassID:
Opc = Hexagon::PS_pselect;
break;
case Hexagon::HvxVRRegClassID:
Opc = Hexagon::PS_vselect;
break;
case Hexagon::HvxWRRegClassID:
Opc = Hexagon::PS_wselect;
break;
default:
llvm_unreachable("unexpected register type");
}
const MCInstrDesc &D = HII->get(Opc);
DebugLoc DL = B->findBranchDebugLoc();
Register MuxR = MRI->createVirtualRegister(DRC);
BuildMI(*B, At, DL, D, MuxR)
.addReg(PredR)
.addReg(TR, 0, TSR)
.addReg(FR, 0, FSR);
return MuxR;
}
void HexagonEarlyIfConversion::updatePhiNodes(MachineBasicBlock *WhereB,
const FlowPattern &FP) {
// Visit all PHI nodes in the WhereB block and generate MUX instructions
// in the split block. Update the PHI nodes with the values of the MUX.
auto NonPHI = WhereB->getFirstNonPHI();
for (auto I = WhereB->begin(); I != NonPHI; ++I) {
MachineInstr *PN = &*I;
// Registers and subregisters corresponding to TrueB, FalseB and SplitB.
unsigned TR = 0, TSR = 0, FR = 0, FSR = 0, SR = 0, SSR = 0;
for (int i = PN->getNumOperands()-2; i > 0; i -= 2) {
const MachineOperand &RO = PN->getOperand(i), &BO = PN->getOperand(i+1);
if (BO.getMBB() == FP.SplitB)
SR = RO.getReg(), SSR = RO.getSubReg();
else if (BO.getMBB() == FP.TrueB)
TR = RO.getReg(), TSR = RO.getSubReg();
else if (BO.getMBB() == FP.FalseB)
FR = RO.getReg(), FSR = RO.getSubReg();
else
continue;
PN->RemoveOperand(i+1);
PN->RemoveOperand(i);
}
if (TR == 0)
TR = SR, TSR = SSR;
else if (FR == 0)
FR = SR, FSR = SSR;
assert(TR || FR);
unsigned MuxR = 0, MuxSR = 0;
if (TR && FR) {
Register DR = PN->getOperand(0).getReg();
const TargetRegisterClass *RC = MRI->getRegClass(DR);
MuxR = buildMux(FP.SplitB, FP.SplitB->getFirstTerminator(), RC,
FP.PredR, TR, TSR, FR, FSR);
} else if (TR) {
MuxR = TR;
MuxSR = TSR;
} else {
MuxR = FR;
MuxSR = FSR;
}
PN->addOperand(MachineOperand::CreateReg(MuxR, false, false, false, false,
false, false, MuxSR));
PN->addOperand(MachineOperand::CreateMBB(FP.SplitB));
}
}
void HexagonEarlyIfConversion::convert(const FlowPattern &FP) {
MachineBasicBlock *TSB = nullptr, *FSB = nullptr;
MachineBasicBlock::iterator OldTI = FP.SplitB->getFirstTerminator();
assert(OldTI != FP.SplitB->end());
DebugLoc DL = OldTI->getDebugLoc();
if (FP.TrueB) {
TSB = *FP.TrueB->succ_begin();
predicateBlockNB(FP.SplitB, OldTI, FP.TrueB, FP.PredR, true);
}
if (FP.FalseB) {
FSB = *FP.FalseB->succ_begin();
MachineBasicBlock::iterator At = FP.SplitB->getFirstTerminator();
predicateBlockNB(FP.SplitB, At, FP.FalseB, FP.PredR, false);
}
// Regenerate new terminators in the split block and update the successors.
// First, remember any information that may be needed later and remove the
// existing terminators/successors from the split block.
MachineBasicBlock *SSB = nullptr;
FP.SplitB->erase(OldTI, FP.SplitB->end());
while (FP.SplitB->succ_size() > 0) {
MachineBasicBlock *T = *FP.SplitB->succ_begin();
// It's possible that the split block had a successor that is not a pre-
// dicated block. This could only happen if there was only one block to
// be predicated. Example:
// split_b:
// if (p) jump true_b
// jump unrelated2_b
// unrelated1_b:
// ...
// unrelated2_b: ; can have other predecessors, so it's not "false_b"
// jump other_b
// true_b: ; only reachable from split_b, can be predicated
// ...
//
// Find this successor (SSB) if it exists.
if (T != FP.TrueB && T != FP.FalseB) {
assert(!SSB);
SSB = T;
}
FP.SplitB->removeSuccessor(FP.SplitB->succ_begin());
}
// Insert new branches and update the successors of the split block. This
// may create unconditional branches to the layout successor, etc., but
// that will be cleaned up later. For now, make sure that correct code is
// generated.
if (FP.JoinB) {
assert(!SSB || SSB == FP.JoinB);
BuildMI(*FP.SplitB, FP.SplitB->end(), DL, HII->get(Hexagon::J2_jump))
.addMBB(FP.JoinB);
FP.SplitB->addSuccessor(FP.JoinB);
} else {
bool HasBranch = false;
if (TSB) {
BuildMI(*FP.SplitB, FP.SplitB->end(), DL, HII->get(Hexagon::J2_jumpt))
.addReg(FP.PredR)
.addMBB(TSB);
FP.SplitB->addSuccessor(TSB);
HasBranch = true;
}
if (FSB) {
const MCInstrDesc &D = HasBranch ? HII->get(Hexagon::J2_jump)
: HII->get(Hexagon::J2_jumpf);
MachineInstrBuilder MIB = BuildMI(*FP.SplitB, FP.SplitB->end(), DL, D);
if (!HasBranch)
MIB.addReg(FP.PredR);
MIB.addMBB(FSB);
FP.SplitB->addSuccessor(FSB);
}
if (SSB) {
// This cannot happen if both TSB and FSB are set. [TF]SB are the
// successor blocks of the TrueB and FalseB (or null of the TrueB
// or FalseB block is null). SSB is the potential successor block
// of the SplitB that is neither TrueB nor FalseB.
BuildMI(*FP.SplitB, FP.SplitB->end(), DL, HII->get(Hexagon::J2_jump))
.addMBB(SSB);
FP.SplitB->addSuccessor(SSB);
}
}
// What is left to do is to update the PHI nodes that could have entries
// referring to predicated blocks.
if (FP.JoinB) {
updatePhiNodes(FP.JoinB, FP);
} else {
if (TSB)
updatePhiNodes(TSB, FP);
if (FSB)
updatePhiNodes(FSB, FP);
// Nothing to update in SSB, since SSB's predecessors haven't changed.
}
}
void HexagonEarlyIfConversion::removeBlock(MachineBasicBlock *B) {
LLVM_DEBUG(dbgs() << "Removing block " << PrintMB(B) << "\n");
// Transfer the immediate dominator information from B to its descendants.
MachineDomTreeNode *N = MDT->getNode(B);
MachineDomTreeNode *IDN = N->getIDom();
if (IDN) {
MachineBasicBlock *IDB = IDN->getBlock();
using GTN = GraphTraits<MachineDomTreeNode *>;
using DTNodeVectType = SmallVector<MachineDomTreeNode *, 4>;
DTNodeVectType Cn(GTN::child_begin(N), GTN::child_end(N));
for (DTNodeVectType::iterator I = Cn.begin(), E = Cn.end(); I != E; ++I) {
MachineBasicBlock *SB = (*I)->getBlock();
MDT->changeImmediateDominator(SB, IDB);
}
}
while (B->succ_size() > 0)
B->removeSuccessor(B->succ_begin());
for (auto I = B->pred_begin(), E = B->pred_end(); I != E; ++I)
(*I)->removeSuccessor(B, true);
Deleted.insert(B);
MDT->eraseNode(B);
MFN->erase(B->getIterator());
}
void HexagonEarlyIfConversion::eliminatePhis(MachineBasicBlock *B) {
LLVM_DEBUG(dbgs() << "Removing phi nodes from block " << PrintMB(B) << "\n");
MachineBasicBlock::iterator I, NextI, NonPHI = B->getFirstNonPHI();
for (I = B->begin(); I != NonPHI; I = NextI) {
NextI = std::next(I);
MachineInstr *PN = &*I;
assert(PN->getNumOperands() == 3 && "Invalid phi node");
MachineOperand &UO = PN->getOperand(1);
Register UseR = UO.getReg(), UseSR = UO.getSubReg();
Register DefR = PN->getOperand(0).getReg();
unsigned NewR = UseR;
if (UseSR) {
// MRI.replaceVregUsesWith does not allow to update the subregister,
// so instead of doing the use-iteration here, create a copy into a
// "non-subregistered" register.
const DebugLoc &DL = PN->getDebugLoc();
const TargetRegisterClass *RC = MRI->getRegClass(DefR);
NewR = MRI->createVirtualRegister(RC);
NonPHI = BuildMI(*B, NonPHI, DL, HII->get(TargetOpcode::COPY), NewR)
.addReg(UseR, 0, UseSR);
}
MRI->replaceRegWith(DefR, NewR);
B->erase(I);
}
}
void HexagonEarlyIfConversion::mergeBlocks(MachineBasicBlock *PredB,
MachineBasicBlock *SuccB) {
LLVM_DEBUG(dbgs() << "Merging blocks " << PrintMB(PredB) << " and "
<< PrintMB(SuccB) << "\n");
bool TermOk = hasUncondBranch(SuccB);
eliminatePhis(SuccB);
HII->removeBranch(*PredB);
PredB->removeSuccessor(SuccB);
PredB->splice(PredB->end(), SuccB, SuccB->begin(), SuccB->end());
PredB->transferSuccessorsAndUpdatePHIs(SuccB);
MachineBasicBlock *OldLayoutSuccessor = SuccB->getNextNode();
removeBlock(SuccB);
if (!TermOk)
PredB->updateTerminator(OldLayoutSuccessor);
}
void HexagonEarlyIfConversion::simplifyFlowGraph(const FlowPattern &FP) {
MachineBasicBlock *OldLayoutSuccessor = FP.SplitB->getNextNode();
if (FP.TrueB)
removeBlock(FP.TrueB);
if (FP.FalseB)
removeBlock(FP.FalseB);
FP.SplitB->updateTerminator(OldLayoutSuccessor);
if (FP.SplitB->succ_size() != 1)
return;
MachineBasicBlock *SB = *FP.SplitB->succ_begin();
if (SB->pred_size() != 1)
return;
// By now, the split block has only one successor (SB), and SB has only
// one predecessor. We can try to merge them. We will need to update ter-
// minators in FP.Split+SB, and that requires working analyzeBranch, which
// fails on Hexagon for blocks that have EH_LABELs. However, if SB ends
// with an unconditional branch, we won't need to touch the terminators.
if (!hasEHLabel(SB) || hasUncondBranch(SB))
mergeBlocks(FP.SplitB, SB);
}
bool HexagonEarlyIfConversion::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
auto &ST = MF.getSubtarget<HexagonSubtarget>();
HII = ST.getInstrInfo();
TRI = ST.getRegisterInfo();
MFN = &MF;
MRI = &MF.getRegInfo();
MDT = &getAnalysis<MachineDominatorTree>();
MLI = &getAnalysis<MachineLoopInfo>();
MBPI = EnableHexagonBP ? &getAnalysis<MachineBranchProbabilityInfo>() :
nullptr;
Deleted.clear();
bool Changed = false;
for (MachineLoopInfo::iterator I = MLI->begin(), E = MLI->end(); I != E; ++I)
Changed |= visitLoop(*I);
Changed |= visitLoop(nullptr);
return Changed;
}
//===----------------------------------------------------------------------===//
// Public Constructor Functions
//===----------------------------------------------------------------------===//
FunctionPass *llvm::createHexagonEarlyIfConversion() {
return new HexagonEarlyIfConversion();
}