#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/LiveInterval.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/SlotIndexes.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cassert>
#include <limits>
#include <memory>
#include <utility>
using namespace llvm;
#define DEBUG_TYPE "stack-coloring"
static cl::opt<bool>
DisableColoring("no-stack-coloring",
cl::init(false), cl::Hidden,
cl::desc("Disable stack coloring"));
static cl::opt<bool>
ProtectFromEscapedAllocas("protect-from-escaped-allocas",
cl::init(false), cl::Hidden,
cl::desc("Do not optimize lifetime zones that "
"are broken"));
static cl::opt<bool>
LifetimeStartOnFirstUse("stackcoloring-lifetime-start-on-first-use",
cl::init(true), cl::Hidden,
cl::desc("Treat stack lifetimes as starting on first use, not on START marker."));
STATISTIC(NumMarkerSeen, "Number of lifetime markers found.");
STATISTIC(StackSpaceSaved, "Number of bytes saved due to merging slots.");
STATISTIC(StackSlotMerged, "Number of stack slot merged.");
STATISTIC(EscapedAllocas, "Number of allocas that escaped the lifetime region");
namespace {
class StackColoring : public MachineFunctionPass {
MachineFrameInfo *MFI;
MachineFunction *MF;
struct BlockLifetimeInfo {
BitVector Begin;
BitVector End;
BitVector LiveIn;
BitVector LiveOut;
};
using LivenessMap = DenseMap<const MachineBasicBlock *, BlockLifetimeInfo>;
LivenessMap BlockLiveness;
DenseMap<const MachineBasicBlock *, int> BasicBlocks;
SmallVector<const MachineBasicBlock *, 8> BasicBlockNumbering;
SmallVector<std::unique_ptr<LiveInterval>, 16> Intervals;
SmallVector<SmallVector<SlotIndex, 4>, 16> LiveStarts;
VNInfo::Allocator VNInfoAllocator;
SlotIndexes *Indexes;
SmallVector<MachineInstr*, 8> Markers;
BitVector InterestingSlots;
BitVector ConservativeSlots;
BitVector StoreSlots;
unsigned NumIterations;
public:
static char ID;
StackColoring() : MachineFunctionPass(ID) {
initializeStackColoringPass(*PassRegistry::getPassRegistry());
}
void getAnalysisUsage(AnalysisUsage &AU) const override;
bool runOnMachineFunction(MachineFunction &Func) override;
private:
using BlockBitVecMap = DenseMap<const MachineBasicBlock *, BitVector>;
void dump() const;
void dumpIntervals() const;
void dumpBB(MachineBasicBlock *MBB) const;
void dumpBV(const char *tag, const BitVector &BV) const;
bool removeAllMarkers();
unsigned collectMarkers(unsigned NumSlot);
void calculateLocalLiveness();
bool applyFirstUse(int Slot) {
if (!LifetimeStartOnFirstUse || ProtectFromEscapedAllocas)
return false;
if (ConservativeSlots.test(Slot))
return false;
return true;
}
bool isLifetimeStartOrEnd(const MachineInstr &MI,
SmallVector<int, 4> &slots,
bool &isStart);
void calculateLiveIntervals(unsigned NumSlots);
void remapInstructions(DenseMap<int, int> &SlotRemap);
void removeInvalidSlotRanges();
void expungeSlotMap(DenseMap<int, int> &SlotRemap, unsigned NumSlots);
};
}
char StackColoring::ID = 0;
char &llvm::StackColoringID = StackColoring::ID;
INITIALIZE_PASS_BEGIN(StackColoring, DEBUG_TYPE,
"Merge disjoint stack slots", false, false)
INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
INITIALIZE_PASS_END(StackColoring, DEBUG_TYPE,
"Merge disjoint stack slots", false, false)
void StackColoring::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<SlotIndexes>();
MachineFunctionPass::getAnalysisUsage(AU);
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void StackColoring::dumpBV(const char *tag,
const BitVector &BV) const {
dbgs() << tag << " : { ";
for (unsigned I = 0, E = BV.size(); I != E; ++I)
dbgs() << BV.test(I) << " ";
dbgs() << "}\n";
}
LLVM_DUMP_METHOD void StackColoring::dumpBB(MachineBasicBlock *MBB) const {
LivenessMap::const_iterator BI = BlockLiveness.find(MBB);
assert(BI != BlockLiveness.end() && "Block not found");
const BlockLifetimeInfo &BlockInfo = BI->second;
dumpBV("BEGIN", BlockInfo.Begin);
dumpBV("END", BlockInfo.End);
dumpBV("LIVE_IN", BlockInfo.LiveIn);
dumpBV("LIVE_OUT", BlockInfo.LiveOut);
}
LLVM_DUMP_METHOD void StackColoring::dump() const {
for (MachineBasicBlock *MBB : depth_first(MF)) {
dbgs() << "Inspecting block #" << MBB->getNumber() << " ["
<< MBB->getName() << "]\n";
dumpBB(MBB);
}
}
LLVM_DUMP_METHOD void StackColoring::dumpIntervals() const {
for (unsigned I = 0, E = Intervals.size(); I != E; ++I) {
dbgs() << "Interval[" << I << "]:\n";
Intervals[I]->dump();
}
}
#endif
static inline int getStartOrEndSlot(const MachineInstr &MI)
{
assert((MI.getOpcode() == TargetOpcode::LIFETIME_START ||
MI.getOpcode() == TargetOpcode::LIFETIME_END) &&
"Expected LIFETIME_START or LIFETIME_END op");
const MachineOperand &MO = MI.getOperand(0);
int Slot = MO.getIndex();
if (Slot >= 0)
return Slot;
return -1;
}
bool StackColoring::isLifetimeStartOrEnd(const MachineInstr &MI,
SmallVector<int, 4> &slots,
bool &isStart) {
if (MI.getOpcode() == TargetOpcode::LIFETIME_START ||
MI.getOpcode() == TargetOpcode::LIFETIME_END) {
int Slot = getStartOrEndSlot(MI);
if (Slot < 0)
return false;
if (!InterestingSlots.test(Slot))
return false;
slots.push_back(Slot);
if (MI.getOpcode() == TargetOpcode::LIFETIME_END) {
isStart = false;
return true;
}
if (!applyFirstUse(Slot)) {
isStart = true;
return true;
}
} else if (LifetimeStartOnFirstUse && !ProtectFromEscapedAllocas) {
if (!MI.isDebugInstr()) {
bool found = false;
for (const MachineOperand &MO : MI.operands()) {
if (!MO.isFI())
continue;
int Slot = MO.getIndex();
if (Slot<0)
continue;
if (InterestingSlots.test(Slot) && applyFirstUse(Slot)) {
slots.push_back(Slot);
found = true;
}
}
if (found) {
isStart = true;
return true;
}
}
}
return false;
}
unsigned StackColoring::collectMarkers(unsigned NumSlot) {
unsigned MarkersFound = 0;
BlockBitVecMap SeenStartMap;
InterestingSlots.clear();
InterestingSlots.resize(NumSlot);
ConservativeSlots.clear();
ConservativeSlots.resize(NumSlot);
StoreSlots.clear();
StoreSlots.resize(NumSlot);
SmallVector<int, 8> NumStartLifetimes(NumSlot, 0);
SmallVector<int, 8> NumEndLifetimes(NumSlot, 0);
SmallVector<int, 8> NumLoadInCatchPad(NumSlot, 0);
for (MachineBasicBlock *MBB : depth_first(MF)) {
BitVector BetweenStartEnd;
BetweenStartEnd.resize(NumSlot);
for (const MachineBasicBlock *Pred : MBB->predecessors()) {
BlockBitVecMap::const_iterator I = SeenStartMap.find(Pred);
if (I != SeenStartMap.end()) {
BetweenStartEnd |= I->second;
}
}
for (MachineInstr &MI : *MBB) {
if (MI.isDebugInstr())
continue;
if (MI.getOpcode() == TargetOpcode::LIFETIME_START ||
MI.getOpcode() == TargetOpcode::LIFETIME_END) {
int Slot = getStartOrEndSlot(MI);
if (Slot < 0)
continue;
InterestingSlots.set(Slot);
if (MI.getOpcode() == TargetOpcode::LIFETIME_START) {
BetweenStartEnd.set(Slot);
NumStartLifetimes[Slot] += 1;
} else {
BetweenStartEnd.reset(Slot);
NumEndLifetimes[Slot] += 1;
}
const AllocaInst *Allocation = MFI->getObjectAllocation(Slot);
if (Allocation) {
LLVM_DEBUG(dbgs() << "Found a lifetime ");
LLVM_DEBUG(dbgs() << (MI.getOpcode() == TargetOpcode::LIFETIME_START
? "start"
: "end"));
LLVM_DEBUG(dbgs() << " marker for slot #" << Slot);
LLVM_DEBUG(dbgs()
<< " with allocation: " << Allocation->getName() << "\n");
}
Markers.push_back(&MI);
MarkersFound += 1;
} else {
for (const MachineOperand &MO : MI.operands()) {
if (!MO.isFI())
continue;
int Slot = MO.getIndex();
if (Slot < 0)
continue;
if (! BetweenStartEnd.test(Slot)) {
ConservativeSlots.set(Slot);
}
if (MI.mayStore())
StoreSlots.set(Slot);
if (MF->getWinEHFuncInfo() && MBB->isEHPad() && MI.mayLoad())
NumLoadInCatchPad[Slot] += 1;
}
}
}
BitVector &SeenStart = SeenStartMap[MBB];
SeenStart |= BetweenStartEnd;
}
if (!MarkersFound) {
return 0;
}
for (unsigned slot = 0; slot < NumSlot; ++slot) {
if (NumStartLifetimes[slot] > 1 || NumEndLifetimes[slot] > 1 ||
(NumLoadInCatchPad[slot] > 1 && !StoreSlots.test(slot)))
ConservativeSlots.set(slot);
}
LLVM_DEBUG(dumpBV("Conservative slots", ConservativeSlots));
for (MachineBasicBlock *MBB : depth_first(MF)) {
BasicBlocks[MBB] = BasicBlockNumbering.size();
BasicBlockNumbering.push_back(MBB);
BlockLifetimeInfo &BlockInfo = BlockLiveness[MBB];
BlockInfo.Begin.resize(NumSlot);
BlockInfo.End.resize(NumSlot);
SmallVector<int, 4> slots;
for (MachineInstr &MI : *MBB) {
bool isStart = false;
slots.clear();
if (isLifetimeStartOrEnd(MI, slots, isStart)) {
if (!isStart) {
assert(slots.size() == 1 && "unexpected: MI ends multiple slots");
int Slot = slots[0];
if (BlockInfo.Begin.test(Slot)) {
BlockInfo.Begin.reset(Slot);
}
BlockInfo.End.set(Slot);
} else {
for (auto Slot : slots) {
LLVM_DEBUG(dbgs() << "Found a use of slot #" << Slot);
LLVM_DEBUG(dbgs()
<< " at " << printMBBReference(*MBB) << " index ");
LLVM_DEBUG(Indexes->getInstructionIndex(MI).print(dbgs()));
const AllocaInst *Allocation = MFI->getObjectAllocation(Slot);
if (Allocation) {
LLVM_DEBUG(dbgs()
<< " with allocation: " << Allocation->getName());
}
LLVM_DEBUG(dbgs() << "\n");
if (BlockInfo.End.test(Slot)) {
BlockInfo.End.reset(Slot);
}
BlockInfo.Begin.set(Slot);
}
}
}
}
}
NumMarkerSeen += MarkersFound;
return MarkersFound;
}
void StackColoring::calculateLocalLiveness() {
unsigned NumIters = 0;
bool changed = true;
while (changed) {
changed = false;
++NumIters;
for (const MachineBasicBlock *BB : BasicBlockNumbering) {
LivenessMap::iterator BI = BlockLiveness.find(BB);
assert(BI != BlockLiveness.end() && "Block not found");
BlockLifetimeInfo &BlockInfo = BI->second;
BitVector LocalLiveIn;
for (MachineBasicBlock *Pred : BB->predecessors()) {
LivenessMap::const_iterator I = BlockLiveness.find(Pred);
if (I != BlockLiveness.end())
LocalLiveIn |= I->second.LiveOut;
}
BitVector LocalLiveOut = LocalLiveIn;
LocalLiveOut.reset(BlockInfo.End);
LocalLiveOut |= BlockInfo.Begin;
if (LocalLiveIn.test(BlockInfo.LiveIn)) {
changed = true;
BlockInfo.LiveIn |= LocalLiveIn;
}
if (LocalLiveOut.test(BlockInfo.LiveOut)) {
changed = true;
BlockInfo.LiveOut |= LocalLiveOut;
}
}
}
NumIterations = NumIters;
}
void StackColoring::calculateLiveIntervals(unsigned NumSlots) {
SmallVector<SlotIndex, 16> Starts;
SmallVector<bool, 16> DefinitelyInUse;
for (const MachineBasicBlock &MBB : *MF) {
Starts.clear();
Starts.resize(NumSlots);
DefinitelyInUse.clear();
DefinitelyInUse.resize(NumSlots);
BlockLifetimeInfo &MBBLiveness = BlockLiveness[&MBB];
for (int pos = MBBLiveness.LiveIn.find_first(); pos != -1;
pos = MBBLiveness.LiveIn.find_next(pos)) {
Starts[pos] = Indexes->getMBBStartIdx(&MBB);
}
for (const MachineInstr &MI : MBB) {
SmallVector<int, 4> slots;
bool IsStart = false;
if (!isLifetimeStartOrEnd(MI, slots, IsStart))
continue;
SlotIndex ThisIndex = Indexes->getInstructionIndex(MI);
for (auto Slot : slots) {
if (IsStart) {
if (!DefinitelyInUse[Slot]) {
LiveStarts[Slot].push_back(ThisIndex);
DefinitelyInUse[Slot] = true;
}
if (!Starts[Slot].isValid())
Starts[Slot] = ThisIndex;
} else {
if (Starts[Slot].isValid()) {
VNInfo *VNI = Intervals[Slot]->getValNumInfo(0);
Intervals[Slot]->addSegment(
LiveInterval::Segment(Starts[Slot], ThisIndex, VNI));
Starts[Slot] = SlotIndex(); DefinitelyInUse[Slot] = false;
}
}
}
}
for (unsigned i = 0; i < NumSlots; ++i) {
if (!Starts[i].isValid())
continue;
SlotIndex EndIdx = Indexes->getMBBEndIdx(&MBB);
VNInfo *VNI = Intervals[i]->getValNumInfo(0);
Intervals[i]->addSegment(LiveInterval::Segment(Starts[i], EndIdx, VNI));
}
}
}
bool StackColoring::removeAllMarkers() {
unsigned Count = 0;
for (MachineInstr *MI : Markers) {
MI->eraseFromParent();
Count++;
}
Markers.clear();
LLVM_DEBUG(dbgs() << "Removed " << Count << " markers.\n");
return Count;
}
void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
unsigned FixedInstr = 0;
unsigned FixedMemOp = 0;
unsigned FixedDbg = 0;
for (auto &VI : MF->getVariableDbgInfo()) {
if (!VI.Var)
continue;
if (SlotRemap.count(VI.Slot)) {
LLVM_DEBUG(dbgs() << "Remapping debug info for ["
<< cast<DILocalVariable>(VI.Var)->getName() << "].\n");
VI.Slot = SlotRemap[VI.Slot];
FixedDbg++;
}
}
DenseMap<const AllocaInst*, const AllocaInst*> Allocas;
SmallPtrSet<const AllocaInst*, 32> MergedAllocas;
for (const std::pair<int, int> &SI : SlotRemap) {
const AllocaInst *From = MFI->getObjectAllocation(SI.first);
const AllocaInst *To = MFI->getObjectAllocation(SI.second);
assert(To && From && "Invalid allocation object");
Allocas[From] = To;
if (From->comesBefore(To))
const_cast<AllocaInst*>(To)->moveBefore(const_cast<AllocaInst*>(From));
Instruction *Inst = const_cast<AllocaInst *>(To);
if (From->getType() != To->getType()) {
BitCastInst *Cast = new BitCastInst(Inst, From->getType());
Cast->insertAfter(Inst);
Inst = Cast;
}
MergedAllocas.insert(From);
MergedAllocas.insert(To);
MachineFrameInfo::SSPLayoutKind FromKind
= MFI->getObjectSSPLayout(SI.first);
MachineFrameInfo::SSPLayoutKind ToKind = MFI->getObjectSSPLayout(SI.second);
if (FromKind != MachineFrameInfo::SSPLK_None &&
(ToKind == MachineFrameInfo::SSPLK_None ||
(ToKind != MachineFrameInfo::SSPLK_LargeArray &&
FromKind != MachineFrameInfo::SSPLK_AddrOf)))
MFI->setObjectSSPLayout(SI.second, FromKind);
AllocaInst *FromAI = const_cast<AllocaInst *>(From);
if (FromAI->isUsedByMetadata())
ValueAsMetadata::handleRAUW(FromAI, UndefValue::get(FromAI->getType()));
for (auto &Use : FromAI->uses()) {
if (BitCastInst *BCI = dyn_cast<BitCastInst>(Use.get()))
if (BCI->isUsedByMetadata())
ValueAsMetadata::handleRAUW(BCI, UndefValue::get(BCI->getType()));
}
FromAI->replaceAllUsesWith(Inst);
}
std::vector<std::vector<MachineMemOperand *>> SSRefs(
MFI->getObjectIndexEnd());
for (MachineBasicBlock &BB : *MF)
for (MachineInstr &I : BB) {
if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
I.getOpcode() == TargetOpcode::LIFETIME_END)
continue;
for (MachineMemOperand *MMO : I.memoperands()) {
const AllocaInst *AI = dyn_cast_or_null<AllocaInst>(MMO->getValue());
if (!AI)
continue;
if (!Allocas.count(AI))
continue;
MMO->setValue(Allocas[AI]);
FixedMemOp++;
}
for (MachineOperand &MO : I.operands()) {
if (!MO.isFI())
continue;
int FromSlot = MO.getIndex();
if (FromSlot<0)
continue;
if (!SlotRemap.count(FromSlot))
continue;
#ifndef NDEBUG
bool TouchesMemory = I.mayLoadOrStore();
if (!I.isDebugInstr() && TouchesMemory && ProtectFromEscapedAllocas) {
SlotIndex Index = Indexes->getInstructionIndex(I);
const LiveInterval *Interval = &*Intervals[FromSlot];
assert(Interval->find(Index) != Interval->end() &&
"Found instruction usage outside of live range.");
}
#endif
int ToSlot = SlotRemap[FromSlot];
MO.setIndex(ToSlot);
FixedInstr++;
}
SmallVector<MachineMemOperand *, 2> NewMMOs;
bool ReplaceMemOps = false;
for (MachineMemOperand *MMO : I.memoperands()) {
if (const auto *FSV = dyn_cast_or_null<FixedStackPseudoSourceValue>(
MMO->getPseudoValue())) {
int FI = FSV->getFrameIndex();
auto To = SlotRemap.find(FI);
if (To != SlotRemap.end())
SSRefs[FI].push_back(MMO);
}
bool MayHaveConflictingAAMD = false;
if (MMO->getAAInfo()) {
if (const Value *MMOV = MMO->getValue()) {
SmallVector<Value *, 4> Objs;
getUnderlyingObjectsForCodeGen(MMOV, Objs);
if (Objs.empty())
MayHaveConflictingAAMD = true;
else
for (Value *V : Objs) {
const AllocaInst *AI = dyn_cast_or_null<AllocaInst>(V);
if (AI && MergedAllocas.count(AI)) {
MayHaveConflictingAAMD = true;
break;
}
}
}
}
if (MayHaveConflictingAAMD) {
NewMMOs.push_back(MF->getMachineMemOperand(MMO, AAMDNodes()));
ReplaceMemOps = true;
} else {
NewMMOs.push_back(MMO);
}
}
if (ReplaceMemOps)
I.setMemRefs(*MF, NewMMOs);
}
for (auto E : enumerate(SSRefs))
if (!E.value().empty()) {
const PseudoSourceValue *NewSV =
MF->getPSVManager().getFixedStack(SlotRemap.find(E.index())->second);
for (MachineMemOperand *Ref : E.value())
Ref->setValue(NewSV);
}
if (WinEHFuncInfo *EHInfo = MF->getWinEHFuncInfo())
for (WinEHTryBlockMapEntry &TBME : EHInfo->TryBlockMap)
for (WinEHHandlerType &H : TBME.HandlerArray)
if (H.CatchObj.FrameIndex != std::numeric_limits<int>::max() &&
SlotRemap.count(H.CatchObj.FrameIndex))
H.CatchObj.FrameIndex = SlotRemap[H.CatchObj.FrameIndex];
LLVM_DEBUG(dbgs() << "Fixed " << FixedMemOp << " machine memory operands.\n");
LLVM_DEBUG(dbgs() << "Fixed " << FixedDbg << " debug locations.\n");
LLVM_DEBUG(dbgs() << "Fixed " << FixedInstr << " machine instructions.\n");
(void) FixedMemOp;
(void) FixedDbg;
(void) FixedInstr;
}
void StackColoring::removeInvalidSlotRanges() {
for (MachineBasicBlock &BB : *MF)
for (MachineInstr &I : BB) {
if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
I.getOpcode() == TargetOpcode::LIFETIME_END || I.isDebugInstr())
continue;
if (!I.mayLoad() && !I.mayStore())
continue;
for (const MachineOperand &MO : I.operands()) {
if (!MO.isFI())
continue;
int Slot = MO.getIndex();
if (Slot<0)
continue;
if (Intervals[Slot]->empty())
continue;
LiveInterval *Interval = &*Intervals[Slot];
SlotIndex Index = Indexes->getInstructionIndex(I);
if (Interval->find(Index) == Interval->end()) {
Interval->clear();
LLVM_DEBUG(dbgs() << "Invalidating range #" << Slot << "\n");
EscapedAllocas++;
}
}
}
}
void StackColoring::expungeSlotMap(DenseMap<int, int> &SlotRemap,
unsigned NumSlots) {
for (unsigned i=0; i < NumSlots; ++i) {
if (SlotRemap.count(i)) {
int Target = SlotRemap[i];
while (SlotRemap.count(Target)) {
Target = SlotRemap[Target];
SlotRemap[i] = Target;
}
}
}
}
bool StackColoring::runOnMachineFunction(MachineFunction &Func) {
LLVM_DEBUG(dbgs() << "********** Stack Coloring **********\n"
<< "********** Function: " << Func.getName() << '\n');
MF = &Func;
MFI = &MF->getFrameInfo();
Indexes = &getAnalysis<SlotIndexes>();
BlockLiveness.clear();
BasicBlocks.clear();
BasicBlockNumbering.clear();
Markers.clear();
Intervals.clear();
LiveStarts.clear();
VNInfoAllocator.Reset();
unsigned NumSlots = MFI->getObjectIndexEnd();
if (!NumSlots)
return false;
SmallVector<int, 8> SortedSlots;
SortedSlots.reserve(NumSlots);
Intervals.reserve(NumSlots);
LiveStarts.resize(NumSlots);
unsigned NumMarkers = collectMarkers(NumSlots);
unsigned TotalSize = 0;
LLVM_DEBUG(dbgs() << "Found " << NumMarkers << " markers and " << NumSlots
<< " slots\n");
LLVM_DEBUG(dbgs() << "Slot structure:\n");
for (int i=0; i < MFI->getObjectIndexEnd(); ++i) {
LLVM_DEBUG(dbgs() << "Slot #" << i << " - " << MFI->getObjectSize(i)
<< " bytes.\n");
TotalSize += MFI->getObjectSize(i);
}
LLVM_DEBUG(dbgs() << "Total Stack size: " << TotalSize << " bytes\n\n");
if (NumMarkers < 2 || TotalSize < 16 || DisableColoring ||
skipFunction(Func.getFunction())) {
LLVM_DEBUG(dbgs() << "Will not try to merge slots.\n");
return removeAllMarkers();
}
for (unsigned i=0; i < NumSlots; ++i) {
std::unique_ptr<LiveInterval> LI(new LiveInterval(i, 0));
LI->getNextValue(Indexes->getZeroIndex(), VNInfoAllocator);
Intervals.push_back(std::move(LI));
SortedSlots.push_back(i);
}
calculateLocalLiveness();
LLVM_DEBUG(dbgs() << "Dataflow iterations: " << NumIterations << "\n");
LLVM_DEBUG(dump());
calculateLiveIntervals(NumSlots);
LLVM_DEBUG(dumpIntervals());
if (ProtectFromEscapedAllocas)
removeInvalidSlotRanges();
DenseMap<int, int> SlotRemap;
unsigned RemovedSlots = 0;
unsigned ReducedSize = 0;
for (unsigned I = 0; I < NumSlots; ++I) {
if (Intervals[SortedSlots[I]]->empty())
SortedSlots[I] = -1;
}
llvm::stable_sort(SortedSlots, [this](int LHS, int RHS) {
if (LHS == -1)
return false;
if (RHS == -1)
return true;
return MFI->getObjectSize(LHS) > MFI->getObjectSize(RHS);
});
for (auto &s : LiveStarts)
llvm::sort(s);
bool Changed = true;
while (Changed) {
Changed = false;
for (unsigned I = 0; I < NumSlots; ++I) {
if (SortedSlots[I] == -1)
continue;
for (unsigned J=I+1; J < NumSlots; ++J) {
if (SortedSlots[J] == -1)
continue;
int FirstSlot = SortedSlots[I];
int SecondSlot = SortedSlots[J];
if (MFI->getStackID(FirstSlot) != MFI->getStackID(SecondSlot))
continue;
LiveInterval *First = &*Intervals[FirstSlot];
LiveInterval *Second = &*Intervals[SecondSlot];
auto &FirstS = LiveStarts[FirstSlot];
auto &SecondS = LiveStarts[SecondSlot];
assert(!First->empty() && !Second->empty() && "Found an empty range");
if (!First->isLiveAtIndexes(SecondS) &&
!Second->isLiveAtIndexes(FirstS)) {
Changed = true;
First->MergeSegmentsInAsValue(*Second, First->getValNumInfo(0));
int OldSize = FirstS.size();
FirstS.append(SecondS.begin(), SecondS.end());
auto Mid = FirstS.begin() + OldSize;
std::inplace_merge(FirstS.begin(), Mid, FirstS.end());
SlotRemap[SecondSlot] = FirstSlot;
SortedSlots[J] = -1;
LLVM_DEBUG(dbgs() << "Merging #" << FirstSlot << " and slots #"
<< SecondSlot << " together.\n");
Align MaxAlignment = std::max(MFI->getObjectAlign(FirstSlot),
MFI->getObjectAlign(SecondSlot));
assert(MFI->getObjectSize(FirstSlot) >=
MFI->getObjectSize(SecondSlot) &&
"Merging a small object into a larger one");
RemovedSlots+=1;
ReducedSize += MFI->getObjectSize(SecondSlot);
MFI->setObjectAlignment(FirstSlot, MaxAlignment);
MFI->RemoveStackObject(SecondSlot);
}
}
}
}
StackSpaceSaved += ReducedSize;
StackSlotMerged += RemovedSlots;
LLVM_DEBUG(dbgs() << "Merge " << RemovedSlots << " slots. Saved "
<< ReducedSize << " bytes\n");
expungeSlotMap(SlotRemap, NumSlots);
remapInstructions(SlotRemap);
return removeAllMarkers();
}