#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
#include "llvm/CodeGen/MachinePostDominators.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/CodeGen/TargetFrameLowering.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Function.h"
#include "llvm/InitializePasses.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include <cassert>
#include <cstdint>
#include <memory>
using namespace llvm;
#define DEBUG_TYPE "shrink-wrap"
STATISTIC(NumFunc, "Number of functions");
STATISTIC(NumCandidates, "Number of shrink-wrapping candidates");
STATISTIC(NumCandidatesDropped,
"Number of shrink-wrapping candidates dropped because of frequency");
static cl::opt<cl::boolOrDefault>
EnableShrinkWrapOpt("enable-shrink-wrap", cl::Hidden,
cl::desc("enable the shrink-wrapping pass"));
namespace {
class ShrinkWrap : public MachineFunctionPass {
RegisterClassInfo RCI;
MachineDominatorTree *MDT;
MachinePostDominatorTree *MPDT;
MachineBasicBlock *Save;
MachineBasicBlock *Restore;
MachineBlockFrequencyInfo *MBFI;
MachineLoopInfo *MLI;
MachineOptimizationRemarkEmitter *ORE = nullptr;
uint64_t EntryFreq;
unsigned FrameSetupOpcode;
unsigned FrameDestroyOpcode;
Register SP;
const MachineBasicBlock *Entry;
using SetOfRegs = SmallSetVector<unsigned, 16>;
mutable SetOfRegs CurrentCSRs;
MachineFunction *MachineFunc;
bool useOrDefCSROrFI(const MachineInstr &MI, RegScavenger *RS) const;
const SetOfRegs &getCurrentCSRs(RegScavenger *RS) const {
if (CurrentCSRs.empty()) {
BitVector SavedRegs;
const TargetFrameLowering *TFI =
MachineFunc->getSubtarget().getFrameLowering();
TFI->determineCalleeSaves(*MachineFunc, SavedRegs, RS);
for (int Reg = SavedRegs.find_first(); Reg != -1;
Reg = SavedRegs.find_next(Reg))
CurrentCSRs.insert((unsigned)Reg);
}
return CurrentCSRs;
}
void updateSaveRestorePoints(MachineBasicBlock &MBB, RegScavenger *RS);
void init(MachineFunction &MF) {
RCI.runOnMachineFunction(MF);
MDT = &getAnalysis<MachineDominatorTree>();
MPDT = &getAnalysis<MachinePostDominatorTree>();
Save = nullptr;
Restore = nullptr;
MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
MLI = &getAnalysis<MachineLoopInfo>();
ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
EntryFreq = MBFI->getEntryFreq();
const TargetSubtargetInfo &Subtarget = MF.getSubtarget();
const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
FrameSetupOpcode = TII.getCallFrameSetupOpcode();
FrameDestroyOpcode = TII.getCallFrameDestroyOpcode();
SP = Subtarget.getTargetLowering()->getStackPointerRegisterToSaveRestore();
Entry = &MF.front();
CurrentCSRs.clear();
MachineFunc = &MF;
++NumFunc;
}
bool ArePointsInteresting() const { return Save != Entry && Save && Restore; }
static bool isShrinkWrapEnabled(const MachineFunction &MF);
public:
static char ID;
ShrinkWrap() : MachineFunctionPass(ID) {
initializeShrinkWrapPass(*PassRegistry::getPassRegistry());
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesAll();
AU.addRequired<MachineBlockFrequencyInfo>();
AU.addRequired<MachineDominatorTree>();
AU.addRequired<MachinePostDominatorTree>();
AU.addRequired<MachineLoopInfo>();
AU.addRequired<MachineOptimizationRemarkEmitterPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
MachineFunctionProperties getRequiredProperties() const override {
return MachineFunctionProperties().set(
MachineFunctionProperties::Property::NoVRegs);
}
StringRef getPassName() const override { return "Shrink Wrapping analysis"; }
bool runOnMachineFunction(MachineFunction &MF) override;
};
}
char ShrinkWrap::ID = 0;
char &llvm::ShrinkWrapID = ShrinkWrap::ID;
INITIALIZE_PASS_BEGIN(ShrinkWrap, DEBUG_TYPE, "Shrink Wrap Pass", false, false)
INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo)
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTree)
INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass)
INITIALIZE_PASS_END(ShrinkWrap, DEBUG_TYPE, "Shrink Wrap Pass", false, false)
bool ShrinkWrap::useOrDefCSROrFI(const MachineInstr &MI,
RegScavenger *RS) const {
if (MI.mayLoadOrStore())
return true;
if (MI.getOpcode() == FrameSetupOpcode ||
MI.getOpcode() == FrameDestroyOpcode) {
LLVM_DEBUG(dbgs() << "Frame instruction: " << MI << '\n');
return true;
}
const MachineFunction *MF = MI.getParent()->getParent();
const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
for (const MachineOperand &MO : MI.operands()) {
bool UseOrDefCSR = false;
if (MO.isReg()) {
if (!MO.isDef() && !MO.readsReg())
continue;
Register PhysReg = MO.getReg();
if (!PhysReg)
continue;
assert(Register::isPhysicalRegister(PhysReg) && "Unallocated register?!");
UseOrDefCSR =
(!MI.isCall() && PhysReg == SP) ||
RCI.getLastCalleeSavedAlias(PhysReg) ||
(!MI.isReturn() && TRI->isNonallocatableRegisterCalleeSave(PhysReg));
} else if (MO.isRegMask()) {
for (unsigned Reg : getCurrentCSRs(RS)) {
if (MO.clobbersPhysReg(Reg)) {
UseOrDefCSR = true;
break;
}
}
}
if (UseOrDefCSR || (MO.isFI() && !MI.isDebugValue())) {
LLVM_DEBUG(dbgs() << "Use or define CSR(" << UseOrDefCSR << ") or FI("
<< MO.isFI() << "): " << MI << '\n');
return true;
}
}
return false;
}
template <typename ListOfBBs, typename DominanceAnalysis>
static MachineBasicBlock *FindIDom(MachineBasicBlock &Block, ListOfBBs BBs,
DominanceAnalysis &Dom) {
MachineBasicBlock *IDom = &Block;
for (MachineBasicBlock *BB : BBs) {
IDom = Dom.findNearestCommonDominator(IDom, BB);
if (!IDom)
break;
}
if (IDom == &Block)
return nullptr;
return IDom;
}
void ShrinkWrap::updateSaveRestorePoints(MachineBasicBlock &MBB,
RegScavenger *RS) {
if (!Save)
Save = &MBB;
else
Save = MDT->findNearestCommonDominator(Save, &MBB);
assert(Save);
if (!Restore)
Restore = &MBB;
else if (MPDT->getNode(&MBB)) Restore = MPDT->findNearestCommonDominator(Restore, &MBB);
else
Restore = nullptr;
if (Restore == &MBB) {
for (const MachineInstr &Terminator : MBB.terminators()) {
if (!useOrDefCSROrFI(Terminator, RS))
continue;
if (MBB.succ_empty()) {
Restore = nullptr; break;
}
Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT);
break;
}
}
if (!Restore) {
LLVM_DEBUG(
dbgs() << "Restore point needs to be spanned on several blocks\n");
return;
}
bool SaveDominatesRestore = false;
bool RestorePostDominatesSave = false;
while (Restore &&
(!(SaveDominatesRestore = MDT->dominates(Save, Restore)) ||
!(RestorePostDominatesSave = MPDT->dominates(Restore, Save)) ||
MLI->getLoopFor(Save) || MLI->getLoopFor(Restore))) {
if (!SaveDominatesRestore) {
Save = MDT->findNearestCommonDominator(Save, Restore);
continue;
}
if (!RestorePostDominatesSave)
Restore = MPDT->findNearestCommonDominator(Restore, Save);
if (Restore && (MLI->getLoopFor(Save) || MLI->getLoopFor(Restore))) {
if (MLI->getLoopDepth(Save) > MLI->getLoopDepth(Restore)) {
Save = FindIDom<>(*Save, Save->predecessors(), *MDT);
if (!Save)
break;
} else {
SmallVector<MachineBasicBlock*, 4> ExitBlocks;
MLI->getLoopFor(Restore)->getExitingBlocks(ExitBlocks);
MachineBasicBlock *IPdom = Restore;
for (MachineBasicBlock *LoopExitBB: ExitBlocks) {
IPdom = FindIDom<>(*IPdom, LoopExitBB->successors(), *MPDT);
if (!IPdom)
break;
}
if (IPdom && MLI->getLoopDepth(IPdom) < MLI->getLoopDepth(Restore))
Restore = IPdom;
else {
Restore = nullptr;
break;
}
}
}
}
}
static bool giveUpWithRemarks(MachineOptimizationRemarkEmitter *ORE,
StringRef RemarkName, StringRef RemarkMessage,
const DiagnosticLocation &Loc,
const MachineBasicBlock *MBB) {
ORE->emit([&]() {
return MachineOptimizationRemarkMissed(DEBUG_TYPE, RemarkName, Loc, MBB)
<< RemarkMessage;
});
LLVM_DEBUG(dbgs() << RemarkMessage << '\n');
return false;
}
bool ShrinkWrap::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()) || MF.empty() || !isShrinkWrapEnabled(MF))
return false;
LLVM_DEBUG(dbgs() << "**** Analysing " << MF.getName() << '\n');
init(MF);
ReversePostOrderTraversal<MachineBasicBlock *> RPOT(&*MF.begin());
if (containsIrreducibleCFG<MachineBasicBlock *>(RPOT, *MLI)) {
return giveUpWithRemarks(ORE, "UnsupportedIrreducibleCFG",
"Irreducible CFGs are not supported yet.",
MF.getFunction().getSubprogram(), &MF.front());
}
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
std::unique_ptr<RegScavenger> RS(
TRI->requiresRegisterScavenging(MF) ? new RegScavenger() : nullptr);
for (MachineBasicBlock &MBB : MF) {
LLVM_DEBUG(dbgs() << "Look into: " << MBB.getNumber() << ' '
<< MBB.getName() << '\n');
if (MBB.isEHFuncletEntry())
return giveUpWithRemarks(ORE, "UnsupportedEHFunclets",
"EH Funclets are not supported yet.",
MBB.front().getDebugLoc(), &MBB);
if (MBB.isEHPad() || MBB.isInlineAsmBrIndirectTarget()) {
updateSaveRestorePoints(MBB, RS.get());
if (!ArePointsInteresting()) {
LLVM_DEBUG(dbgs() << "EHPad/inlineasm_br prevents shrink-wrapping\n");
return false;
}
continue;
}
for (const MachineInstr &MI : MBB) {
if (!useOrDefCSROrFI(MI, RS.get()))
continue;
updateSaveRestorePoints(MBB, RS.get());
if (!ArePointsInteresting()) {
LLVM_DEBUG(dbgs() << "No Shrink wrap candidate found\n");
return false;
}
break;
}
}
if (!ArePointsInteresting()) {
assert(!Save && !Restore && "We miss a shrink-wrap opportunity?!");
LLVM_DEBUG(dbgs() << "Nothing to shrink-wrap\n");
return false;
}
LLVM_DEBUG(dbgs() << "\n ** Results **\nFrequency of the Entry: " << EntryFreq
<< '\n');
const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
do {
LLVM_DEBUG(dbgs() << "Shrink wrap candidates (#, Name, Freq):\nSave: "
<< Save->getNumber() << ' ' << Save->getName() << ' '
<< MBFI->getBlockFreq(Save).getFrequency()
<< "\nRestore: " << Restore->getNumber() << ' '
<< Restore->getName() << ' '
<< MBFI->getBlockFreq(Restore).getFrequency() << '\n');
bool IsSaveCheap, TargetCanUseSaveAsPrologue = false;
if (((IsSaveCheap = EntryFreq >= MBFI->getBlockFreq(Save).getFrequency()) &&
EntryFreq >= MBFI->getBlockFreq(Restore).getFrequency()) &&
((TargetCanUseSaveAsPrologue = TFI->canUseAsPrologue(*Save)) &&
TFI->canUseAsEpilogue(*Restore)))
break;
LLVM_DEBUG(
dbgs() << "New points are too expensive or invalid for the target\n");
MachineBasicBlock *NewBB;
if (!IsSaveCheap || !TargetCanUseSaveAsPrologue) {
Save = FindIDom<>(*Save, Save->predecessors(), *MDT);
if (!Save)
break;
NewBB = Save;
} else {
Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT);
if (!Restore)
break;
NewBB = Restore;
}
updateSaveRestorePoints(*NewBB, RS.get());
} while (Save && Restore);
if (!ArePointsInteresting()) {
++NumCandidatesDropped;
return false;
}
LLVM_DEBUG(dbgs() << "Final shrink wrap candidates:\nSave: "
<< Save->getNumber() << ' ' << Save->getName()
<< "\nRestore: " << Restore->getNumber() << ' '
<< Restore->getName() << '\n');
MachineFrameInfo &MFI = MF.getFrameInfo();
MFI.setSavePoint(Save);
MFI.setRestorePoint(Restore);
++NumCandidates;
return false;
}
bool ShrinkWrap::isShrinkWrapEnabled(const MachineFunction &MF) {
const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
switch (EnableShrinkWrapOpt) {
case cl::BOU_UNSET:
return TFI->enableShrinkWrapping(MF) &&
!MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
!(MF.getFunction().hasFnAttribute(Attribute::SanitizeAddress) ||
MF.getFunction().hasFnAttribute(Attribute::SanitizeThread) ||
MF.getFunction().hasFnAttribute(Attribute::SanitizeMemory) ||
MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress));
case cl::BOU_TRUE:
return true;
case cl::BOU_FALSE:
return false;
}
llvm_unreachable("Invalid shrink-wrapping state");
}