#ifndef LLVM_IR_MODULESUMMARYINDEX_H
#define LLVM_IR_MODULESUMMARYINDEX_H
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/IR/ConstantRange.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/ScaledNumber.h"
#include "llvm/Support/StringSaver.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <array>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>
namespace llvm {
template <class GraphType> struct GraphTraits;
namespace yaml {
template <typename T> struct MappingTraits;
}
struct CalleeInfo {
enum class HotnessType : uint8_t {
Unknown = 0,
Cold = 1,
None = 2,
Hot = 3,
Critical = 4
};
uint32_t Hotness : 3;
uint32_t RelBlockFreq : 29;
static constexpr int32_t ScaleShift = 8;
static constexpr uint64_t MaxRelBlockFreq = (1 << 29) - 1;
CalleeInfo()
: Hotness(static_cast<uint32_t>(HotnessType::Unknown)), RelBlockFreq(0) {}
explicit CalleeInfo(HotnessType Hotness, uint64_t RelBF)
: Hotness(static_cast<uint32_t>(Hotness)), RelBlockFreq(RelBF) {}
void updateHotness(const HotnessType OtherHotness) {
Hotness = std::max(Hotness, static_cast<uint32_t>(OtherHotness));
}
HotnessType getHotness() const { return HotnessType(Hotness); }
void updateRelBlockFreq(uint64_t BlockFreq, uint64_t EntryFreq) {
if (EntryFreq == 0)
return;
using Scaled64 = ScaledNumber<uint64_t>;
Scaled64 Temp(BlockFreq, ScaleShift);
Temp /= Scaled64::get(EntryFreq);
uint64_t Sum =
SaturatingAdd<uint64_t>(Temp.toInt<uint64_t>(), RelBlockFreq);
Sum = std::min(Sum, uint64_t(MaxRelBlockFreq));
RelBlockFreq = static_cast<uint32_t>(Sum);
}
};
inline const char *getHotnessName(CalleeInfo::HotnessType HT) {
switch (HT) {
case CalleeInfo::HotnessType::Unknown:
return "unknown";
case CalleeInfo::HotnessType::Cold:
return "cold";
case CalleeInfo::HotnessType::None:
return "none";
case CalleeInfo::HotnessType::Hot:
return "hot";
case CalleeInfo::HotnessType::Critical:
return "critical";
}
llvm_unreachable("invalid hotness");
}
class GlobalValueSummary;
using GlobalValueSummaryList = std::vector<std::unique_ptr<GlobalValueSummary>>;
struct alignas(8) GlobalValueSummaryInfo {
union NameOrGV {
NameOrGV(bool HaveGVs) {
if (HaveGVs)
GV = nullptr;
else
Name = "";
}
const GlobalValue *GV;
StringRef Name;
} U;
GlobalValueSummaryInfo(bool HaveGVs) : U(HaveGVs) {}
GlobalValueSummaryList SummaryList;
};
using GlobalValueSummaryMapTy =
std::map<GlobalValue::GUID, GlobalValueSummaryInfo>;
struct ValueInfo {
enum Flags { HaveGV = 1, ReadOnly = 2, WriteOnly = 4 };
PointerIntPair<const GlobalValueSummaryMapTy::value_type *, 3, int>
RefAndFlags;
ValueInfo() = default;
ValueInfo(bool HaveGVs, const GlobalValueSummaryMapTy::value_type *R) {
RefAndFlags.setPointer(R);
RefAndFlags.setInt(HaveGVs);
}
explicit operator bool() const { return getRef(); }
GlobalValue::GUID getGUID() const { return getRef()->first; }
const GlobalValue *getValue() const {
assert(haveGVs());
return getRef()->second.U.GV;
}
ArrayRef<std::unique_ptr<GlobalValueSummary>> getSummaryList() const {
return getRef()->second.SummaryList;
}
StringRef name() const {
return haveGVs() ? getRef()->second.U.GV->getName()
: getRef()->second.U.Name;
}
bool haveGVs() const { return RefAndFlags.getInt() & HaveGV; }
bool isReadOnly() const {
assert(isValidAccessSpecifier());
return RefAndFlags.getInt() & ReadOnly;
}
bool isWriteOnly() const {
assert(isValidAccessSpecifier());
return RefAndFlags.getInt() & WriteOnly;
}
unsigned getAccessSpecifier() const {
assert(isValidAccessSpecifier());
return RefAndFlags.getInt() & (ReadOnly | WriteOnly);
}
bool isValidAccessSpecifier() const {
unsigned BadAccessMask = ReadOnly | WriteOnly;
return (RefAndFlags.getInt() & BadAccessMask) != BadAccessMask;
}
void setReadOnly() {
assert(getAccessSpecifier() == 0);
RefAndFlags.setInt(RefAndFlags.getInt() | ReadOnly);
}
void setWriteOnly() {
assert(getAccessSpecifier() == 0);
RefAndFlags.setInt(RefAndFlags.getInt() | WriteOnly);
}
const GlobalValueSummaryMapTy::value_type *getRef() const {
return RefAndFlags.getPointer();
}
GlobalValue::VisibilityTypes getELFVisibility() const;
bool isDSOLocal(bool WithDSOLocalPropagation = false) const;
bool canAutoHide() const;
};
inline raw_ostream &operator<<(raw_ostream &OS, const ValueInfo &VI) {
OS << VI.getGUID();
if (!VI.name().empty())
OS << " (" << VI.name() << ")";
return OS;
}
inline bool operator==(const ValueInfo &A, const ValueInfo &B) {
assert(A.getRef() && B.getRef() &&
"Need ValueInfo with non-null Ref for comparison");
return A.getRef() == B.getRef();
}
inline bool operator!=(const ValueInfo &A, const ValueInfo &B) {
assert(A.getRef() && B.getRef() &&
"Need ValueInfo with non-null Ref for comparison");
return A.getRef() != B.getRef();
}
inline bool operator<(const ValueInfo &A, const ValueInfo &B) {
assert(A.getRef() && B.getRef() &&
"Need ValueInfo with non-null Ref to compare GUIDs");
return A.getGUID() < B.getGUID();
}
template <> struct DenseMapInfo<ValueInfo> {
static inline ValueInfo getEmptyKey() {
return ValueInfo(false, (GlobalValueSummaryMapTy::value_type *)-8);
}
static inline ValueInfo getTombstoneKey() {
return ValueInfo(false, (GlobalValueSummaryMapTy::value_type *)-16);
}
static inline bool isSpecialKey(ValueInfo V) {
return V == getTombstoneKey() || V == getEmptyKey();
}
static bool isEqual(ValueInfo L, ValueInfo R) {
assert(isSpecialKey(L) || isSpecialKey(R) || (L.haveGVs() == R.haveGVs()));
return L.getRef() == R.getRef();
}
static unsigned getHashValue(ValueInfo I) { return (uintptr_t)I.getRef(); }
};
class GlobalValueSummary {
public:
enum SummaryKind : unsigned { AliasKind, FunctionKind, GlobalVarKind };
struct GVFlags {
unsigned Linkage : 4;
unsigned Visibility : 2;
unsigned NotEligibleToImport : 1;
unsigned Live : 1;
unsigned DSOLocal : 1;
unsigned CanAutoHide : 1;
explicit GVFlags(GlobalValue::LinkageTypes Linkage,
GlobalValue::VisibilityTypes Visibility,
bool NotEligibleToImport, bool Live, bool IsLocal,
bool CanAutoHide)
: Linkage(Linkage), Visibility(Visibility),
NotEligibleToImport(NotEligibleToImport), Live(Live),
DSOLocal(IsLocal), CanAutoHide(CanAutoHide) {}
};
private:
SummaryKind Kind;
GVFlags Flags;
GlobalValue::GUID OriginalName = 0;
StringRef ModulePath;
std::vector<ValueInfo> RefEdgeList;
protected:
GlobalValueSummary(SummaryKind K, GVFlags Flags, std::vector<ValueInfo> Refs)
: Kind(K), Flags(Flags), RefEdgeList(std::move(Refs)) {
assert((K != AliasKind || Refs.empty()) &&
"Expect no references for AliasSummary");
}
public:
virtual ~GlobalValueSummary() = default;
GlobalValue::GUID getOriginalName() const { return OriginalName; }
void setOriginalName(GlobalValue::GUID Name) { OriginalName = Name; }
SummaryKind getSummaryKind() const { return Kind; }
void setModulePath(StringRef ModPath) { ModulePath = ModPath; }
StringRef modulePath() const { return ModulePath; }
GVFlags flags() const { return Flags; }
GlobalValue::LinkageTypes linkage() const {
return static_cast<GlobalValue::LinkageTypes>(Flags.Linkage);
}
void setLinkage(GlobalValue::LinkageTypes Linkage) {
Flags.Linkage = Linkage;
}
bool notEligibleToImport() const { return Flags.NotEligibleToImport; }
bool isLive() const { return Flags.Live; }
void setLive(bool Live) { Flags.Live = Live; }
void setDSOLocal(bool Local) { Flags.DSOLocal = Local; }
bool isDSOLocal() const { return Flags.DSOLocal; }
void setCanAutoHide(bool CanAutoHide) { Flags.CanAutoHide = CanAutoHide; }
bool canAutoHide() const { return Flags.CanAutoHide; }
GlobalValue::VisibilityTypes getVisibility() const {
return (GlobalValue::VisibilityTypes)Flags.Visibility;
}
void setVisibility(GlobalValue::VisibilityTypes Vis) {
Flags.Visibility = (unsigned)Vis;
}
void setNotEligibleToImport() { Flags.NotEligibleToImport = true; }
ArrayRef<ValueInfo> refs() const { return RefEdgeList; }
GlobalValueSummary *getBaseObject();
const GlobalValueSummary *getBaseObject() const;
friend class ModuleSummaryIndex;
};
class AliasSummary : public GlobalValueSummary {
ValueInfo AliaseeValueInfo;
GlobalValueSummary *AliaseeSummary;
public:
AliasSummary(GVFlags Flags)
: GlobalValueSummary(AliasKind, Flags, ArrayRef<ValueInfo>{}),
AliaseeSummary(nullptr) {}
static bool classof(const GlobalValueSummary *GVS) {
return GVS->getSummaryKind() == AliasKind;
}
void setAliasee(ValueInfo &AliaseeVI, GlobalValueSummary *Aliasee) {
AliaseeValueInfo = AliaseeVI;
AliaseeSummary = Aliasee;
}
bool hasAliasee() const {
assert(!!AliaseeSummary == (AliaseeValueInfo &&
!AliaseeValueInfo.getSummaryList().empty()) &&
"Expect to have both aliasee summary and summary list or neither");
return !!AliaseeSummary;
}
const GlobalValueSummary &getAliasee() const {
assert(AliaseeSummary && "Unexpected missing aliasee summary");
return *AliaseeSummary;
}
GlobalValueSummary &getAliasee() {
return const_cast<GlobalValueSummary &>(
static_cast<const AliasSummary *>(this)->getAliasee());
}
ValueInfo getAliaseeVI() const {
assert(AliaseeValueInfo && "Unexpected missing aliasee");
return AliaseeValueInfo;
}
GlobalValue::GUID getAliaseeGUID() const {
assert(AliaseeValueInfo && "Unexpected missing aliasee");
return AliaseeValueInfo.getGUID();
}
};
const inline GlobalValueSummary *GlobalValueSummary::getBaseObject() const {
if (auto *AS = dyn_cast<AliasSummary>(this))
return &AS->getAliasee();
return this;
}
inline GlobalValueSummary *GlobalValueSummary::getBaseObject() {
if (auto *AS = dyn_cast<AliasSummary>(this))
return &AS->getAliasee();
return this;
}
class FunctionSummary : public GlobalValueSummary {
public:
using EdgeTy = std::pair<ValueInfo, CalleeInfo>;
enum ForceSummaryHotnessType : unsigned {
FSHT_None,
FSHT_AllNonCritical,
FSHT_All
};
struct VFuncId {
GlobalValue::GUID GUID;
uint64_t Offset;
};
struct ConstVCall {
VFuncId VFunc;
std::vector<uint64_t> Args;
};
struct TypeIdInfo {
std::vector<GlobalValue::GUID> TypeTests;
std::vector<VFuncId> TypeTestAssumeVCalls, TypeCheckedLoadVCalls;
std::vector<ConstVCall> TypeTestAssumeConstVCalls,
TypeCheckedLoadConstVCalls;
};
struct FFlags {
unsigned ReadNone : 1;
unsigned ReadOnly : 1;
unsigned NoRecurse : 1;
unsigned ReturnDoesNotAlias : 1;
unsigned NoInline : 1;
unsigned AlwaysInline : 1;
unsigned NoUnwind : 1;
unsigned MayThrow : 1;
unsigned HasUnknownCall : 1;
unsigned MustBeUnreachable : 1;
FFlags &operator&=(const FFlags &RHS) {
this->ReadNone &= RHS.ReadNone;
this->ReadOnly &= RHS.ReadOnly;
this->NoRecurse &= RHS.NoRecurse;
this->ReturnDoesNotAlias &= RHS.ReturnDoesNotAlias;
this->NoInline &= RHS.NoInline;
this->AlwaysInline &= RHS.AlwaysInline;
this->NoUnwind &= RHS.NoUnwind;
this->MayThrow &= RHS.MayThrow;
this->HasUnknownCall &= RHS.HasUnknownCall;
this->MustBeUnreachable &= RHS.MustBeUnreachable;
return *this;
}
bool anyFlagSet() {
return this->ReadNone | this->ReadOnly | this->NoRecurse |
this->ReturnDoesNotAlias | this->NoInline | this->AlwaysInline |
this->NoUnwind | this->MayThrow | this->HasUnknownCall |
this->MustBeUnreachable;
}
operator std::string() {
std::string Output;
raw_string_ostream OS(Output);
OS << "funcFlags: (";
OS << "readNone: " << this->ReadNone;
OS << ", readOnly: " << this->ReadOnly;
OS << ", noRecurse: " << this->NoRecurse;
OS << ", returnDoesNotAlias: " << this->ReturnDoesNotAlias;
OS << ", noInline: " << this->NoInline;
OS << ", alwaysInline: " << this->AlwaysInline;
OS << ", noUnwind: " << this->NoUnwind;
OS << ", mayThrow: " << this->MayThrow;
OS << ", hasUnknownCall: " << this->HasUnknownCall;
OS << ", mustBeUnreachable: " << this->MustBeUnreachable;
OS << ")";
return OS.str();
}
};
struct ParamAccess {
static constexpr uint32_t RangeWidth = 64;
struct Call {
uint64_t ParamNo = 0;
ValueInfo Callee;
ConstantRange Offsets{RangeWidth, true};
Call() = default;
Call(uint64_t ParamNo, ValueInfo Callee, const ConstantRange &Offsets)
: ParamNo(ParamNo), Callee(Callee), Offsets(Offsets) {}
};
uint64_t ParamNo = 0;
ConstantRange Use{RangeWidth, true};
std::vector<Call> Calls;
ParamAccess() = default;
ParamAccess(uint64_t ParamNo, const ConstantRange &Use)
: ParamNo(ParamNo), Use(Use) {}
};
static FunctionSummary
makeDummyFunctionSummary(std::vector<FunctionSummary::EdgeTy> Edges) {
return FunctionSummary(
FunctionSummary::GVFlags(
GlobalValue::LinkageTypes::AvailableExternallyLinkage,
GlobalValue::DefaultVisibility,
true, true, false,
false),
0, FunctionSummary::FFlags{}, 0,
std::vector<ValueInfo>(), std::move(Edges),
std::vector<GlobalValue::GUID>(),
std::vector<FunctionSummary::VFuncId>(),
std::vector<FunctionSummary::VFuncId>(),
std::vector<FunctionSummary::ConstVCall>(),
std::vector<FunctionSummary::ConstVCall>(),
std::vector<FunctionSummary::ParamAccess>());
}
static FunctionSummary ExternalNode;
private:
unsigned InstCount;
FFlags FunFlags;
uint64_t EntryCount = 0;
std::vector<EdgeTy> CallGraphEdgeList;
std::unique_ptr<TypeIdInfo> TIdInfo;
using ParamAccessesTy = std::vector<ParamAccess>;
std::unique_ptr<ParamAccessesTy> ParamAccesses;
public:
FunctionSummary(GVFlags Flags, unsigned NumInsts, FFlags FunFlags,
uint64_t EntryCount, std::vector<ValueInfo> Refs,
std::vector<EdgeTy> CGEdges,
std::vector<GlobalValue::GUID> TypeTests,
std::vector<VFuncId> TypeTestAssumeVCalls,
std::vector<VFuncId> TypeCheckedLoadVCalls,
std::vector<ConstVCall> TypeTestAssumeConstVCalls,
std::vector<ConstVCall> TypeCheckedLoadConstVCalls,
std::vector<ParamAccess> Params)
: GlobalValueSummary(FunctionKind, Flags, std::move(Refs)),
InstCount(NumInsts), FunFlags(FunFlags), EntryCount(EntryCount),
CallGraphEdgeList(std::move(CGEdges)) {
if (!TypeTests.empty() || !TypeTestAssumeVCalls.empty() ||
!TypeCheckedLoadVCalls.empty() || !TypeTestAssumeConstVCalls.empty() ||
!TypeCheckedLoadConstVCalls.empty())
TIdInfo = std::make_unique<TypeIdInfo>(
TypeIdInfo{std::move(TypeTests), std::move(TypeTestAssumeVCalls),
std::move(TypeCheckedLoadVCalls),
std::move(TypeTestAssumeConstVCalls),
std::move(TypeCheckedLoadConstVCalls)});
if (!Params.empty())
ParamAccesses = std::make_unique<ParamAccessesTy>(std::move(Params));
}
std::pair<unsigned, unsigned> specialRefCounts() const;
static bool classof(const GlobalValueSummary *GVS) {
return GVS->getSummaryKind() == FunctionKind;
}
FFlags fflags() const { return FunFlags; }
void setNoRecurse() { FunFlags.NoRecurse = true; }
void setNoUnwind() { FunFlags.NoUnwind = true; }
unsigned instCount() const { return InstCount; }
uint64_t entryCount() const { return EntryCount; }
void setEntryCount(uint64_t EC) { EntryCount = EC; }
ArrayRef<EdgeTy> calls() const { return CallGraphEdgeList; }
std::vector<EdgeTy> &mutableCalls() { return CallGraphEdgeList; }
void addCall(EdgeTy E) { CallGraphEdgeList.push_back(E); }
ArrayRef<GlobalValue::GUID> type_tests() const {
if (TIdInfo)
return TIdInfo->TypeTests;
return {};
}
ArrayRef<VFuncId> type_test_assume_vcalls() const {
if (TIdInfo)
return TIdInfo->TypeTestAssumeVCalls;
return {};
}
ArrayRef<VFuncId> type_checked_load_vcalls() const {
if (TIdInfo)
return TIdInfo->TypeCheckedLoadVCalls;
return {};
}
ArrayRef<ConstVCall> type_test_assume_const_vcalls() const {
if (TIdInfo)
return TIdInfo->TypeTestAssumeConstVCalls;
return {};
}
ArrayRef<ConstVCall> type_checked_load_const_vcalls() const {
if (TIdInfo)
return TIdInfo->TypeCheckedLoadConstVCalls;
return {};
}
ArrayRef<ParamAccess> paramAccesses() const {
if (ParamAccesses)
return *ParamAccesses;
return {};
}
void setParamAccesses(std::vector<ParamAccess> NewParams) {
if (NewParams.empty())
ParamAccesses.reset();
else if (ParamAccesses)
*ParamAccesses = std::move(NewParams);
else
ParamAccesses = std::make_unique<ParamAccessesTy>(std::move(NewParams));
}
void addTypeTest(GlobalValue::GUID Guid) {
if (!TIdInfo)
TIdInfo = std::make_unique<TypeIdInfo>();
TIdInfo->TypeTests.push_back(Guid);
}
const TypeIdInfo *getTypeIdInfo() const { return TIdInfo.get(); };
friend struct GraphTraits<ValueInfo>;
};
template <> struct DenseMapInfo<FunctionSummary::VFuncId> {
static FunctionSummary::VFuncId getEmptyKey() { return {0, uint64_t(-1)}; }
static FunctionSummary::VFuncId getTombstoneKey() {
return {0, uint64_t(-2)};
}
static bool isEqual(FunctionSummary::VFuncId L, FunctionSummary::VFuncId R) {
return L.GUID == R.GUID && L.Offset == R.Offset;
}
static unsigned getHashValue(FunctionSummary::VFuncId I) { return I.GUID; }
};
template <> struct DenseMapInfo<FunctionSummary::ConstVCall> {
static FunctionSummary::ConstVCall getEmptyKey() {
return {{0, uint64_t(-1)}, {}};
}
static FunctionSummary::ConstVCall getTombstoneKey() {
return {{0, uint64_t(-2)}, {}};
}
static bool isEqual(FunctionSummary::ConstVCall L,
FunctionSummary::ConstVCall R) {
return DenseMapInfo<FunctionSummary::VFuncId>::isEqual(L.VFunc, R.VFunc) &&
L.Args == R.Args;
}
static unsigned getHashValue(FunctionSummary::ConstVCall I) {
return I.VFunc.GUID;
}
};
struct VirtFuncOffset {
VirtFuncOffset(ValueInfo VI, uint64_t Offset)
: FuncVI(VI), VTableOffset(Offset) {}
ValueInfo FuncVI;
uint64_t VTableOffset;
};
using VTableFuncList = std::vector<VirtFuncOffset>;
class GlobalVarSummary : public GlobalValueSummary {
private:
std::unique_ptr<VTableFuncList> VTableFuncs;
public:
struct GVarFlags {
GVarFlags(bool ReadOnly, bool WriteOnly, bool Constant,
GlobalObject::VCallVisibility Vis)
: MaybeReadOnly(ReadOnly), MaybeWriteOnly(WriteOnly),
Constant(Constant), VCallVisibility(Vis) {}
unsigned MaybeReadOnly : 1;
unsigned MaybeWriteOnly : 1;
unsigned Constant : 1;
unsigned VCallVisibility : 2;
} VarFlags;
GlobalVarSummary(GVFlags Flags, GVarFlags VarFlags,
std::vector<ValueInfo> Refs)
: GlobalValueSummary(GlobalVarKind, Flags, std::move(Refs)),
VarFlags(VarFlags) {}
static bool classof(const GlobalValueSummary *GVS) {
return GVS->getSummaryKind() == GlobalVarKind;
}
GVarFlags varflags() const { return VarFlags; }
void setReadOnly(bool RO) { VarFlags.MaybeReadOnly = RO; }
void setWriteOnly(bool WO) { VarFlags.MaybeWriteOnly = WO; }
bool maybeReadOnly() const { return VarFlags.MaybeReadOnly; }
bool maybeWriteOnly() const { return VarFlags.MaybeWriteOnly; }
bool isConstant() const { return VarFlags.Constant; }
void setVCallVisibility(GlobalObject::VCallVisibility Vis) {
VarFlags.VCallVisibility = Vis;
}
GlobalObject::VCallVisibility getVCallVisibility() const {
return (GlobalObject::VCallVisibility)VarFlags.VCallVisibility;
}
void setVTableFuncs(VTableFuncList Funcs) {
assert(!VTableFuncs);
VTableFuncs = std::make_unique<VTableFuncList>(std::move(Funcs));
}
ArrayRef<VirtFuncOffset> vTableFuncs() const {
if (VTableFuncs)
return *VTableFuncs;
return {};
}
};
struct TypeTestResolution {
enum Kind {
Unsat, ByteArray, Inline, Single, AllOnes, Unknown, } TheKind = Unknown;
unsigned SizeM1BitWidth = 0;
uint64_t AlignLog2 = 0;
uint64_t SizeM1 = 0;
uint8_t BitMask = 0;
uint64_t InlineBits = 0;
};
struct WholeProgramDevirtResolution {
enum Kind {
Indir, SingleImpl, BranchFunnel, } TheKind = Indir;
std::string SingleImplName;
struct ByArg {
enum Kind {
Indir, UniformRetVal, UniqueRetVal, VirtualConstProp, } TheKind = Indir;
uint64_t Info = 0;
uint32_t Byte = 0;
uint32_t Bit = 0;
};
std::map<std::vector<uint64_t>, ByArg> ResByArg;
};
struct TypeIdSummary {
TypeTestResolution TTRes;
std::map<uint64_t, WholeProgramDevirtResolution> WPDRes;
};
using ModuleHash = std::array<uint32_t, 5>;
using const_gvsummary_iterator = GlobalValueSummaryMapTy::const_iterator;
using gvsummary_iterator = GlobalValueSummaryMapTy::iterator;
using ModulePathStringTableTy = StringMap<std::pair<uint64_t, ModuleHash>>;
using GVSummaryMapTy = DenseMap<GlobalValue::GUID, GlobalValueSummary *>;
using TypeIdSummaryMapTy =
std::multimap<GlobalValue::GUID, std::pair<std::string, TypeIdSummary>>;
struct TypeIdOffsetVtableInfo {
TypeIdOffsetVtableInfo(uint64_t Offset, ValueInfo VI)
: AddressPointOffset(Offset), VTableVI(VI) {}
uint64_t AddressPointOffset;
ValueInfo VTableVI;
};
using TypeIdCompatibleVtableInfo = std::vector<TypeIdOffsetVtableInfo>;
class ModuleSummaryIndex {
private:
GlobalValueSummaryMapTy GlobalValueMap;
ModulePathStringTableTy ModulePathStringTable;
TypeIdSummaryMapTy TypeIdMap;
std::map<std::string, TypeIdCompatibleVtableInfo, std::less<>>
TypeIdCompatibleVtableMap;
std::map<GlobalValue::GUID, GlobalValue::GUID> OidGuidMap;
bool WithGlobalValueDeadStripping = false;
bool WithAttributePropagation = false;
bool WithDSOLocalPropagation = false;
bool WithWholeProgramVisibility = false;
bool HasSyntheticEntryCounts = false;
bool SkipModuleByDistributedBackend = false;
bool HaveGVs;
bool EnableSplitLTOUnit;
bool PartiallySplitLTOUnits = false;
bool HasParamAccess = false;
std::set<std::string> CfiFunctionDefs;
std::set<std::string> CfiFunctionDecls;
BumpPtrAllocator Alloc;
StringSaver Saver;
uint64_t BlockCount;
friend yaml::MappingTraits<ModuleSummaryIndex>;
GlobalValueSummaryMapTy::value_type *
getOrInsertValuePtr(GlobalValue::GUID GUID) {
return &*GlobalValueMap.emplace(GUID, GlobalValueSummaryInfo(HaveGVs))
.first;
}
public:
ModuleSummaryIndex(bool HaveGVs, bool EnableSplitLTOUnit = false)
: HaveGVs(HaveGVs), EnableSplitLTOUnit(EnableSplitLTOUnit), Saver(Alloc),
BlockCount(0) {}
static constexpr uint64_t BitcodeSummaryVersion = 9;
static constexpr const char *getRegularLTOModuleName() {
return "[Regular LTO]";
}
bool haveGVs() const { return HaveGVs; }
uint64_t getFlags() const;
void setFlags(uint64_t Flags);
uint64_t getBlockCount() const { return BlockCount; }
void addBlockCount(uint64_t C) { BlockCount += C; }
void setBlockCount(uint64_t C) { BlockCount = C; }
gvsummary_iterator begin() { return GlobalValueMap.begin(); }
const_gvsummary_iterator begin() const { return GlobalValueMap.begin(); }
gvsummary_iterator end() { return GlobalValueMap.end(); }
const_gvsummary_iterator end() const { return GlobalValueMap.end(); }
size_t size() const { return GlobalValueMap.size(); }
static void discoverNodes(ValueInfo V,
std::map<ValueInfo, bool> &FunctionHasParent) {
if (!V.getSummaryList().size())
return;
auto S = FunctionHasParent.emplace(V, false);
if (!S.second)
return;
FunctionSummary *F =
dyn_cast<FunctionSummary>(V.getSummaryList().front().get());
assert(F != nullptr && "Expected FunctionSummary node");
for (auto &C : F->calls()) {
auto S = FunctionHasParent.emplace(C.first, true);
if (!S.second && S.first->second)
continue;
if (S.second)
discoverNodes(C.first, FunctionHasParent);
else
S.first->second = true;
}
}
FunctionSummary calculateCallGraphRoot() {
std::map<ValueInfo, bool> FunctionHasParent;
for (auto &S : *this) {
if (!S.second.SummaryList.size() ||
!isa<FunctionSummary>(S.second.SummaryList.front().get()))
continue;
discoverNodes(ValueInfo(HaveGVs, &S), FunctionHasParent);
}
std::vector<FunctionSummary::EdgeTy> Edges;
for (auto &P : FunctionHasParent) {
if (P.second)
continue; Edges.push_back(std::make_pair(P.first, CalleeInfo{}));
}
if (Edges.empty()) {
return FunctionSummary::makeDummyFunctionSummary({});
}
auto CallGraphRoot = FunctionSummary::makeDummyFunctionSummary(Edges);
return CallGraphRoot;
}
bool withGlobalValueDeadStripping() const {
return WithGlobalValueDeadStripping;
}
void setWithGlobalValueDeadStripping() {
WithGlobalValueDeadStripping = true;
}
bool withAttributePropagation() const { return WithAttributePropagation; }
void setWithAttributePropagation() {
WithAttributePropagation = true;
}
bool withDSOLocalPropagation() const { return WithDSOLocalPropagation; }
void setWithDSOLocalPropagation() { WithDSOLocalPropagation = true; }
bool withWholeProgramVisibility() const { return WithWholeProgramVisibility; }
void setWithWholeProgramVisibility() { WithWholeProgramVisibility = true; }
bool isReadOnly(const GlobalVarSummary *GVS) const {
return WithAttributePropagation && GVS->maybeReadOnly();
}
bool isWriteOnly(const GlobalVarSummary *GVS) const {
return WithAttributePropagation && GVS->maybeWriteOnly();
}
bool hasSyntheticEntryCounts() const { return HasSyntheticEntryCounts; }
void setHasSyntheticEntryCounts() { HasSyntheticEntryCounts = true; }
bool skipModuleByDistributedBackend() const {
return SkipModuleByDistributedBackend;
}
void setSkipModuleByDistributedBackend() {
SkipModuleByDistributedBackend = true;
}
bool enableSplitLTOUnit() const { return EnableSplitLTOUnit; }
void setEnableSplitLTOUnit() { EnableSplitLTOUnit = true; }
bool partiallySplitLTOUnits() const { return PartiallySplitLTOUnits; }
void setPartiallySplitLTOUnits() { PartiallySplitLTOUnits = true; }
bool hasParamAccess() const { return HasParamAccess; }
bool isGlobalValueLive(const GlobalValueSummary *GVS) const {
return !WithGlobalValueDeadStripping || GVS->isLive();
}
bool isGUIDLive(GlobalValue::GUID GUID) const;
ValueInfo getValueInfo(const GlobalValueSummaryMapTy::value_type &R) const {
return ValueInfo(HaveGVs, &R);
}
ValueInfo getValueInfo(GlobalValue::GUID GUID) const {
auto I = GlobalValueMap.find(GUID);
return ValueInfo(HaveGVs, I == GlobalValueMap.end() ? nullptr : &*I);
}
ValueInfo getOrInsertValueInfo(GlobalValue::GUID GUID) {
return ValueInfo(HaveGVs, getOrInsertValuePtr(GUID));
}
StringRef saveString(StringRef String) { return Saver.save(String); }
ValueInfo getOrInsertValueInfo(GlobalValue::GUID GUID, StringRef Name) {
assert(!HaveGVs);
auto VP = getOrInsertValuePtr(GUID);
VP->second.U.Name = Name;
return ValueInfo(HaveGVs, VP);
}
ValueInfo getOrInsertValueInfo(const GlobalValue *GV) {
assert(HaveGVs);
auto VP = getOrInsertValuePtr(GV->getGUID());
VP->second.U.GV = GV;
return ValueInfo(HaveGVs, VP);
}
GlobalValue::GUID getGUIDFromOriginalID(GlobalValue::GUID OriginalID) const {
const auto I = OidGuidMap.find(OriginalID);
return I == OidGuidMap.end() ? 0 : I->second;
}
std::set<std::string> &cfiFunctionDefs() { return CfiFunctionDefs; }
const std::set<std::string> &cfiFunctionDefs() const { return CfiFunctionDefs; }
std::set<std::string> &cfiFunctionDecls() { return CfiFunctionDecls; }
const std::set<std::string> &cfiFunctionDecls() const { return CfiFunctionDecls; }
void addGlobalValueSummary(const GlobalValue &GV,
std::unique_ptr<GlobalValueSummary> Summary) {
addGlobalValueSummary(getOrInsertValueInfo(&GV), std::move(Summary));
}
void addGlobalValueSummary(StringRef ValueName,
std::unique_ptr<GlobalValueSummary> Summary) {
addGlobalValueSummary(getOrInsertValueInfo(GlobalValue::getGUID(ValueName)),
std::move(Summary));
}
void addGlobalValueSummary(ValueInfo VI,
std::unique_ptr<GlobalValueSummary> Summary) {
if (const FunctionSummary *FS = dyn_cast<FunctionSummary>(Summary.get()))
HasParamAccess |= !FS->paramAccesses().empty();
addOriginalName(VI.getGUID(), Summary->getOriginalName());
const_cast<GlobalValueSummaryMapTy::value_type *>(VI.getRef())
->second.SummaryList.push_back(std::move(Summary));
}
void addOriginalName(GlobalValue::GUID ValueGUID,
GlobalValue::GUID OrigGUID) {
if (OrigGUID == 0 || ValueGUID == OrigGUID)
return;
if (OidGuidMap.count(OrigGUID) && OidGuidMap[OrigGUID] != ValueGUID)
OidGuidMap[OrigGUID] = 0;
else
OidGuidMap[OrigGUID] = ValueGUID;
}
GlobalValueSummary *findSummaryInModule(ValueInfo VI, StringRef ModuleId) const {
auto SummaryList = VI.getSummaryList();
auto Summary =
llvm::find_if(SummaryList,
[&](const std::unique_ptr<GlobalValueSummary> &Summary) {
return Summary->modulePath() == ModuleId;
});
if (Summary == SummaryList.end())
return nullptr;
return Summary->get();
}
GlobalValueSummary *findSummaryInModule(GlobalValue::GUID ValueGUID,
StringRef ModuleId) const {
auto CalleeInfo = getValueInfo(ValueGUID);
if (!CalleeInfo)
return nullptr; return findSummaryInModule(CalleeInfo, ModuleId);
}
GlobalValueSummary *getGlobalValueSummary(const GlobalValue &GV,
bool PerModuleIndex = true) const {
assert(GV.hasName() && "Can't get GlobalValueSummary for GV with no name");
return getGlobalValueSummary(GV.getGUID(), PerModuleIndex);
}
GlobalValueSummary *getGlobalValueSummary(GlobalValue::GUID ValueGUID,
bool PerModuleIndex = true) const;
const StringMap<std::pair<uint64_t, ModuleHash>> &modulePaths() const {
return ModulePathStringTable;
}
StringMap<std::pair<uint64_t, ModuleHash>> &modulePaths() {
return ModulePathStringTable;
}
uint64_t getModuleId(const StringRef ModPath) const {
return ModulePathStringTable.lookup(ModPath).first;
}
const ModuleHash &getModuleHash(const StringRef ModPath) const {
auto It = ModulePathStringTable.find(ModPath);
assert(It != ModulePathStringTable.end() && "Module not registered");
return It->second.second;
}
static std::string getGlobalNameForLocal(StringRef Name, ModuleHash ModHash) {
SmallString<256> NewName(Name);
NewName += ".llvm.";
NewName += utostr((uint64_t(ModHash[0]) << 32) |
ModHash[1]); return std::string(NewName.str());
}
static StringRef getOriginalNameBeforePromote(StringRef Name) {
std::pair<StringRef, StringRef> Pair = Name.rsplit(".llvm.");
return Pair.first;
}
typedef ModulePathStringTableTy::value_type ModuleInfo;
ModuleInfo *addModule(StringRef ModPath, uint64_t ModId,
ModuleHash Hash = ModuleHash{{0}}) {
return &*ModulePathStringTable.insert({ModPath, {ModId, Hash}}).first;
}
ModuleInfo *getModule(StringRef ModPath) {
auto It = ModulePathStringTable.find(ModPath);
assert(It != ModulePathStringTable.end() && "Module not registered");
return &*It;
}
bool hasExportedFunctions(const Module &M) const {
return ModulePathStringTable.count(M.getModuleIdentifier());
}
const TypeIdSummaryMapTy &typeIds() const { return TypeIdMap; }
TypeIdSummary &getOrInsertTypeIdSummary(StringRef TypeId) {
auto TidIter = TypeIdMap.equal_range(GlobalValue::getGUID(TypeId));
for (auto It = TidIter.first; It != TidIter.second; ++It)
if (It->second.first == TypeId)
return It->second.second;
auto It = TypeIdMap.insert(
{GlobalValue::getGUID(TypeId), {std::string(TypeId), TypeIdSummary()}});
return It->second.second;
}
const TypeIdSummary *getTypeIdSummary(StringRef TypeId) const {
auto TidIter = TypeIdMap.equal_range(GlobalValue::getGUID(TypeId));
for (auto It = TidIter.first; It != TidIter.second; ++It)
if (It->second.first == TypeId)
return &It->second.second;
return nullptr;
}
TypeIdSummary *getTypeIdSummary(StringRef TypeId) {
return const_cast<TypeIdSummary *>(
static_cast<const ModuleSummaryIndex *>(this)->getTypeIdSummary(
TypeId));
}
const auto &typeIdCompatibleVtableMap() const {
return TypeIdCompatibleVtableMap;
}
TypeIdCompatibleVtableInfo &
getOrInsertTypeIdCompatibleVtableSummary(StringRef TypeId) {
return TypeIdCompatibleVtableMap[std::string(TypeId)];
}
Optional<TypeIdCompatibleVtableInfo>
getTypeIdCompatibleVtableSummary(StringRef TypeId) const {
auto I = TypeIdCompatibleVtableMap.find(TypeId);
if (I == TypeIdCompatibleVtableMap.end())
return None;
return I->second;
}
void collectDefinedFunctionsForModule(StringRef ModulePath,
GVSummaryMapTy &GVSummaryMap) const;
template <class Map>
void
collectDefinedGVSummariesPerModule(Map &ModuleToDefinedGVSummaries) const {
for (auto &GlobalList : *this) {
auto GUID = GlobalList.first;
for (auto &Summary : GlobalList.second.SummaryList) {
ModuleToDefinedGVSummaries[Summary->modulePath()][GUID] = Summary.get();
}
}
}
void print(raw_ostream &OS, bool IsForDebug = false) const;
void dump() const;
void
exportToDot(raw_ostream &OS,
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) const;
void dumpSCCs(raw_ostream &OS);
void propagateAttributes(const DenseSet<GlobalValue::GUID> &PreservedSymbols);
bool canImportGlobalVar(GlobalValueSummary *S, bool AnalyzeRefs) const;
};
template <> struct GraphTraits<ValueInfo> {
typedef ValueInfo NodeRef;
using EdgeRef = FunctionSummary::EdgeTy &;
static NodeRef valueInfoFromEdge(FunctionSummary::EdgeTy &P) {
return P.first;
}
using ChildIteratorType =
mapped_iterator<std::vector<FunctionSummary::EdgeTy>::iterator,
decltype(&valueInfoFromEdge)>;
using ChildEdgeIteratorType = std::vector<FunctionSummary::EdgeTy>::iterator;
static NodeRef getEntryNode(ValueInfo V) { return V; }
static ChildIteratorType child_begin(NodeRef N) {
if (!N.getSummaryList().size()) return ChildIteratorType(
FunctionSummary::ExternalNode.CallGraphEdgeList.begin(),
&valueInfoFromEdge);
FunctionSummary *F =
cast<FunctionSummary>(N.getSummaryList().front()->getBaseObject());
return ChildIteratorType(F->CallGraphEdgeList.begin(), &valueInfoFromEdge);
}
static ChildIteratorType child_end(NodeRef N) {
if (!N.getSummaryList().size()) return ChildIteratorType(
FunctionSummary::ExternalNode.CallGraphEdgeList.end(),
&valueInfoFromEdge);
FunctionSummary *F =
cast<FunctionSummary>(N.getSummaryList().front()->getBaseObject());
return ChildIteratorType(F->CallGraphEdgeList.end(), &valueInfoFromEdge);
}
static ChildEdgeIteratorType child_edge_begin(NodeRef N) {
if (!N.getSummaryList().size()) return FunctionSummary::ExternalNode.CallGraphEdgeList.begin();
FunctionSummary *F =
cast<FunctionSummary>(N.getSummaryList().front()->getBaseObject());
return F->CallGraphEdgeList.begin();
}
static ChildEdgeIteratorType child_edge_end(NodeRef N) {
if (!N.getSummaryList().size()) return FunctionSummary::ExternalNode.CallGraphEdgeList.end();
FunctionSummary *F =
cast<FunctionSummary>(N.getSummaryList().front()->getBaseObject());
return F->CallGraphEdgeList.end();
}
static NodeRef edge_dest(EdgeRef E) { return E.first; }
};
template <>
struct GraphTraits<ModuleSummaryIndex *> : public GraphTraits<ValueInfo> {
static NodeRef getEntryNode(ModuleSummaryIndex *I) {
std::unique_ptr<GlobalValueSummary> Root =
std::make_unique<FunctionSummary>(I->calculateCallGraphRoot());
GlobalValueSummaryInfo G(I->haveGVs());
G.SummaryList.push_back(std::move(Root));
static auto P =
GlobalValueSummaryMapTy::value_type(GlobalValue::GUID(0), std::move(G));
return ValueInfo(I->haveGVs(), &P);
}
};
}
#endif