#ifndef LLVM_TOOLS_LLVM_PROFGEN_PERFREADER_H
#define LLVM_TOOLS_LLVM_PROFGEN_PERFREADER_H
#include "ErrorHandling.h"
#include "ProfiledBinary.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Regex.h"
#include <cstdint>
#include <fstream>
#include <list>
#include <map>
#include <vector>
using namespace llvm;
using namespace sampleprof;
namespace llvm {
namespace sampleprof {
class TraceStream {
std::string CurrentLine;
std::ifstream Fin;
bool IsAtEoF = false;
uint64_t LineNumber = 0;
public:
TraceStream(StringRef Filename) : Fin(Filename.str()) {
if (!Fin.good())
exitWithError("Error read input perf script file", Filename);
advance();
}
StringRef getCurrentLine() {
assert(!IsAtEoF && "Line iterator reaches the End-of-File!");
return CurrentLine;
}
uint64_t getLineNumber() { return LineNumber; }
bool isAtEoF() { return IsAtEoF; }
void advance() {
if (!std::getline(Fin, CurrentLine)) {
IsAtEoF = true;
return;
}
LineNumber++;
}
};
enum PerfFormat {
UnknownFormat = 0,
PerfData = 1, PerfScript = 2, UnsymbolizedProfile = 3,
};
enum PerfContent {
UnknownContent = 0,
LBR = 1, LBRStack = 2, };
struct PerfInputFile {
std::string InputFile;
PerfFormat Format = PerfFormat::UnknownFormat;
PerfContent Content = PerfContent::UnknownContent;
};
struct LBREntry {
uint64_t Source = 0;
uint64_t Target = 0;
LBREntry(uint64_t S, uint64_t T) : Source(S), Target(T) {}
#ifndef NDEBUG
void print() const {
dbgs() << "from " << format("%#010x", Source) << " to "
<< format("%#010x", Target);
}
#endif
};
#ifndef NDEBUG
static inline void printLBRStack(const SmallVectorImpl<LBREntry> &LBRStack) {
for (size_t I = 0; I < LBRStack.size(); I++) {
dbgs() << "[" << I << "] ";
LBRStack[I].print();
dbgs() << "\n";
}
}
static inline void printCallStack(const SmallVectorImpl<uint64_t> &CallStack) {
for (size_t I = 0; I < CallStack.size(); I++) {
dbgs() << "[" << I << "] " << format("%#010x", CallStack[I]) << "\n";
}
}
#endif
template <class T> class Hashable {
public:
std::shared_ptr<T> Data;
Hashable(const std::shared_ptr<T> &D) : Data(D) {}
struct Hash {
uint64_t operator()(const Hashable<T> &Key) const {
uint64_t Hash = Key.Data->getHashCode();
assert(Hash && "Should generate HashCode for it!");
return Hash;
}
};
struct Equal {
bool operator()(const Hashable<T> &LHS, const Hashable<T> &RHS) const {
return LHS.Data->isEqual(RHS.Data.get());
}
};
T *getPtr() const { return Data.get(); }
};
struct PerfSample {
SmallVector<LBREntry, 16> LBRStack;
SmallVector<uint64_t, 16> CallStack;
virtual ~PerfSample() = default;
uint64_t getHashCode() const {
auto HashCombine = [](uint64_t H, uint64_t V) {
return ((H << 5) + H) + V;
};
uint64_t Hash = 5381;
for (const auto &Value : CallStack) {
Hash = HashCombine(Hash, Value);
}
for (const auto &Entry : LBRStack) {
Hash = HashCombine(Hash, Entry.Source);
Hash = HashCombine(Hash, Entry.Target);
}
return Hash;
}
bool isEqual(const PerfSample *Other) const {
const SmallVector<uint64_t, 16> &OtherCallStack = Other->CallStack;
const SmallVector<LBREntry, 16> &OtherLBRStack = Other->LBRStack;
if (CallStack.size() != OtherCallStack.size() ||
LBRStack.size() != OtherLBRStack.size())
return false;
if (!std::equal(CallStack.begin(), CallStack.end(), OtherCallStack.begin()))
return false;
for (size_t I = 0; I < OtherLBRStack.size(); I++) {
if (LBRStack[I].Source != OtherLBRStack[I].Source ||
LBRStack[I].Target != OtherLBRStack[I].Target)
return false;
}
return true;
}
#ifndef NDEBUG
uint64_t Linenum = 0;
void print() const {
dbgs() << "Line " << Linenum << "\n";
dbgs() << "LBR stack\n";
printLBRStack(LBRStack);
dbgs() << "Call stack\n";
printCallStack(CallStack);
}
#endif
};
using AggregatedCounter =
std::unordered_map<Hashable<PerfSample>, uint64_t,
Hashable<PerfSample>::Hash, Hashable<PerfSample>::Equal>;
using SampleVector = SmallVector<std::tuple<uint64_t, uint64_t, uint64_t>, 16>;
inline bool isValidFallThroughRange(uint64_t Start, uint64_t End,
ProfiledBinary *Binary) {
return Start <= End && !Binary->rangeCrossUncondBranch(Start, End);
}
struct UnwindState {
const ProfiledBinary *Binary;
struct ProfiledFrame {
const uint64_t Address = DummyRoot;
ProfiledFrame *Parent;
SampleVector RangeSamples;
SampleVector BranchSamples;
std::unordered_map<uint64_t, std::unique_ptr<ProfiledFrame>> Children;
ProfiledFrame(uint64_t Addr = 0, ProfiledFrame *P = nullptr)
: Address(Addr), Parent(P) {}
ProfiledFrame *getOrCreateChildFrame(uint64_t Address) {
assert(Address && "Address can't be zero!");
auto Ret = Children.emplace(
Address, std::make_unique<ProfiledFrame>(Address, this));
return Ret.first->second.get();
}
void recordRangeCount(uint64_t Start, uint64_t End, uint64_t Count) {
RangeSamples.emplace_back(std::make_tuple(Start, End, Count));
}
void recordBranchCount(uint64_t Source, uint64_t Target, uint64_t Count) {
BranchSamples.emplace_back(std::make_tuple(Source, Target, Count));
}
bool isDummyRoot() { return Address == DummyRoot; }
bool isExternalFrame() { return Address == ExternalAddr; }
bool isLeafFrame() { return Children.empty(); }
};
ProfiledFrame DummyTrieRoot;
ProfiledFrame *CurrentLeafFrame;
uint32_t LBRIndex = 0;
const SmallVector<LBREntry, 16> &LBRStack;
InstructionPointer InstPtr;
bool Invalid = false;
UnwindState(const PerfSample *Sample, const ProfiledBinary *Binary)
: Binary(Binary), LBRStack(Sample->LBRStack),
InstPtr(Binary, Sample->CallStack.front()) {
initFrameTrie(Sample->CallStack);
}
bool validateInitialState() {
uint64_t LBRLeaf = LBRStack[LBRIndex].Target;
uint64_t LeafAddr = CurrentLeafFrame->Address;
assert((LBRLeaf != ExternalAddr || LBRLeaf == LeafAddr) &&
"External leading LBR should match the leaf frame.");
if (LeafAddr < LBRLeaf || LeafAddr - LBRLeaf >= 0x100) {
WithColor::warning() << "Bogus trace: stack tip = "
<< format("%#010x", LeafAddr)
<< ", LBR tip = " << format("%#010x\n", LBRLeaf);
return false;
}
return true;
}
void checkStateConsistency() {
assert(InstPtr.Address == CurrentLeafFrame->Address &&
"IP should align with context leaf");
}
void setInvalid() { Invalid = true; }
bool hasNextLBR() const { return LBRIndex < LBRStack.size(); }
uint64_t getCurrentLBRSource() const { return LBRStack[LBRIndex].Source; }
uint64_t getCurrentLBRTarget() const { return LBRStack[LBRIndex].Target; }
const LBREntry &getCurrentLBR() const { return LBRStack[LBRIndex]; }
bool IsLastLBR() const { return LBRIndex == 0; }
bool getLBRStackSize() const { return LBRStack.size(); }
void advanceLBR() { LBRIndex++; }
ProfiledFrame *getParentFrame() { return CurrentLeafFrame->Parent; }
void pushFrame(uint64_t Address) {
CurrentLeafFrame = CurrentLeafFrame->getOrCreateChildFrame(Address);
}
void switchToFrame(uint64_t Address) {
if (CurrentLeafFrame->Address == Address)
return;
CurrentLeafFrame = CurrentLeafFrame->Parent->getOrCreateChildFrame(Address);
}
void popFrame() { CurrentLeafFrame = CurrentLeafFrame->Parent; }
void clearCallStack() { CurrentLeafFrame = &DummyTrieRoot; }
void initFrameTrie(const SmallVectorImpl<uint64_t> &CallStack) {
ProfiledFrame *Cur = &DummyTrieRoot;
for (auto Address : reverse(CallStack)) {
Cur = Cur->getOrCreateChildFrame(Address);
}
CurrentLeafFrame = Cur;
}
ProfiledFrame *getDummyRootPtr() { return &DummyTrieRoot; }
};
struct ContextKey {
uint64_t HashCode = 0;
virtual ~ContextKey() = default;
uint64_t getHashCode() {
if (HashCode == 0)
genHashCode();
return HashCode;
}
virtual void genHashCode() = 0;
virtual bool isEqual(const ContextKey *K) const {
return HashCode == K->HashCode;
};
enum ContextKind { CK_StringBased, CK_AddrBased };
const ContextKind Kind;
ContextKind getKind() const { return Kind; }
ContextKey(ContextKind K) : Kind(K){};
};
struct StringBasedCtxKey : public ContextKey {
SampleContextFrameVector Context;
bool WasLeafInlined;
StringBasedCtxKey() : ContextKey(CK_StringBased), WasLeafInlined(false){};
static bool classof(const ContextKey *K) {
return K->getKind() == CK_StringBased;
}
bool isEqual(const ContextKey *K) const override {
const StringBasedCtxKey *Other = dyn_cast<StringBasedCtxKey>(K);
return Context == Other->Context;
}
void genHashCode() override {
HashCode = hash_value(SampleContextFrames(Context));
}
};
struct AddrBasedCtxKey : public ContextKey {
SmallVector<uint64_t, 16> Context;
bool WasLeafInlined;
AddrBasedCtxKey() : ContextKey(CK_AddrBased), WasLeafInlined(false){};
static bool classof(const ContextKey *K) {
return K->getKind() == CK_AddrBased;
}
bool isEqual(const ContextKey *K) const override {
const AddrBasedCtxKey *Other = dyn_cast<AddrBasedCtxKey>(K);
return Context == Other->Context;
}
void genHashCode() override {
HashCode = hash_combine_range(Context.begin(), Context.end());
}
};
using BranchSample = std::map<std::pair<uint64_t, uint64_t>, uint64_t>;
using RangeSample = std::map<std::pair<uint64_t, uint64_t>, uint64_t>;
struct SampleCounter {
RangeSample RangeCounter;
BranchSample BranchCounter;
void recordRangeCount(uint64_t Start, uint64_t End, uint64_t Repeat) {
assert(Start <= End && "Invalid instruction range");
RangeCounter[{Start, End}] += Repeat;
}
void recordBranchCount(uint64_t Source, uint64_t Target, uint64_t Repeat) {
BranchCounter[{Source, Target}] += Repeat;
}
};
using ContextSampleCounterMap =
std::unordered_map<Hashable<ContextKey>, SampleCounter,
Hashable<ContextKey>::Hash, Hashable<ContextKey>::Equal>;
struct FrameStack {
SmallVector<uint64_t, 16> Stack;
ProfiledBinary *Binary;
FrameStack(ProfiledBinary *B) : Binary(B) {}
bool pushFrame(UnwindState::ProfiledFrame *Cur) {
assert(!Cur->isExternalFrame() &&
"External frame's not expected for context stack.");
Stack.push_back(Cur->Address);
return true;
}
void popFrame() {
if (!Stack.empty())
Stack.pop_back();
}
std::shared_ptr<StringBasedCtxKey> getContextKey();
};
struct AddressStack {
SmallVector<uint64_t, 16> Stack;
ProfiledBinary *Binary;
AddressStack(ProfiledBinary *B) : Binary(B) {}
bool pushFrame(UnwindState::ProfiledFrame *Cur) {
assert(!Cur->isExternalFrame() &&
"External frame's not expected for context stack.");
Stack.push_back(Cur->Address);
return true;
}
void popFrame() {
if (!Stack.empty())
Stack.pop_back();
}
std::shared_ptr<AddrBasedCtxKey> getContextKey();
};
class VirtualUnwinder {
public:
VirtualUnwinder(ContextSampleCounterMap *Counter, ProfiledBinary *B)
: CtxCounterMap(Counter), Binary(B) {}
bool unwind(const PerfSample *Sample, uint64_t Repeat);
std::set<uint64_t> &getUntrackedCallsites() { return UntrackedCallsites; }
uint64_t NumTotalBranches = 0;
uint64_t NumExtCallBranch = 0;
uint64_t NumMissingExternalFrame = 0;
uint64_t NumMismatchedProEpiBranch = 0;
uint64_t NumMismatchedExtCallBranch = 0;
uint64_t NumUnpairedExtAddr = 0;
uint64_t NumPairedExtAddr = 0;
private:
bool isSourceExternal(UnwindState &State) const {
return State.getCurrentLBRSource() == ExternalAddr;
}
bool isTargetExternal(UnwindState &State) const {
return State.getCurrentLBRTarget() == ExternalAddr;
}
bool isReturnFromExternal(UnwindState &State) const {
return isSourceExternal(State) &&
(Binary->getCallAddrFromFrameAddr(State.getCurrentLBRTarget()) != 0);
}
bool isCallFromExternal(UnwindState &State) const {
return isSourceExternal(State) &&
Binary->getCallAddrFromFrameAddr(State.getCurrentLBRTarget()) == 0;
}
bool isCallState(UnwindState &State) const {
if (!isValidState(State))
return false;
if (Binary->addressIsCall(State.getCurrentLBRSource()))
return true;
return isCallFromExternal(State);
}
bool isReturnState(UnwindState &State) const {
if (!isValidState(State))
return false;
if (Binary->addressIsReturn(State.getCurrentLBRSource()))
return true;
return isReturnFromExternal(State);
}
bool isValidState(UnwindState &State) const { return !State.Invalid; }
void unwindCall(UnwindState &State);
void unwindLinear(UnwindState &State, uint64_t Repeat);
void unwindReturn(UnwindState &State);
void unwindBranch(UnwindState &State);
template <typename T>
void collectSamplesFromFrame(UnwindState::ProfiledFrame *Cur, T &Stack);
template <typename T>
void collectSamplesFromFrameTrie(UnwindState::ProfiledFrame *Cur, T &Stack);
void collectSamplesFromFrameTrie(UnwindState::ProfiledFrame *Cur);
void recordRangeCount(uint64_t Start, uint64_t End, UnwindState &State,
uint64_t Repeat);
void recordBranchCount(const LBREntry &Branch, UnwindState &State,
uint64_t Repeat);
ContextSampleCounterMap *CtxCounterMap;
ProfiledBinary *Binary;
std::set<uint64_t> UntrackedCallsites;
};
class PerfReaderBase {
public:
PerfReaderBase(ProfiledBinary *B, StringRef PerfTrace)
: Binary(B), PerfTraceFile(PerfTrace) {
Binary->setBaseAddress(Binary->getPreferredBaseAddress());
};
virtual ~PerfReaderBase() = default;
static std::unique_ptr<PerfReaderBase> create(ProfiledBinary *Binary,
PerfInputFile &PerfInput,
Optional<uint32_t> PIDFilter);
virtual void parsePerfTraces() = 0;
const ContextSampleCounterMap &getSampleCounters() const {
return SampleCounters;
}
bool profileIsCS() { return ProfileIsCS; }
protected:
ProfiledBinary *Binary = nullptr;
StringRef PerfTraceFile;
ContextSampleCounterMap SampleCounters;
bool ProfileIsCS = false;
uint64_t NumTotalSample = 0;
uint64_t NumLeafExternalFrame = 0;
uint64_t NumLeadingOutgoingLBR = 0;
};
class PerfScriptReader : public PerfReaderBase {
public:
PerfScriptReader(ProfiledBinary *B, StringRef PerfTrace,
Optional<uint32_t> PID)
: PerfReaderBase(B, PerfTrace), PIDFilter(PID){};
void parsePerfTraces() override;
static PerfInputFile convertPerfDataToTrace(ProfiledBinary *Binary,
PerfInputFile &File,
Optional<uint32_t> PIDFilter);
static PerfContent checkPerfScriptType(StringRef FileName);
protected:
struct MMapEvent {
uint64_t PID = 0;
uint64_t Address = 0;
uint64_t Size = 0;
uint64_t Offset = 0;
StringRef BinaryPath;
};
static bool isLBRSample(StringRef Line);
static bool isMMap2Event(StringRef Line);
static bool extractMMap2EventForBinary(ProfiledBinary *Binary, StringRef Line,
MMapEvent &MMap);
void updateBinaryAddress(const MMapEvent &Event);
void parseMMap2Event(TraceStream &TraceIt);
void parseAndAggregateTrace();
void parseEventOrSample(TraceStream &TraceIt);
void warnIfMissingMMap();
void warnTruncatedStack();
void warnInvalidRange();
bool extractCallstack(TraceStream &TraceIt,
SmallVectorImpl<uint64_t> &CallStack);
bool extractLBRStack(TraceStream &TraceIt,
SmallVectorImpl<LBREntry> &LBRStack);
uint64_t parseAggregatedCount(TraceStream &TraceIt);
void parseSample(TraceStream &TraceIt);
virtual void parseSample(TraceStream &TraceIt, uint64_t Count){};
void computeCounterFromLBR(const PerfSample *Sample, uint64_t Repeat);
virtual void generateUnsymbolizedProfile();
void writeUnsymbolizedProfile(StringRef Filename);
void writeUnsymbolizedProfile(raw_fd_ostream &OS);
AggregatedCounter AggregatedSamples;
std::set<uint64_t> InvalidReturnAddresses;
Optional<uint32_t> PIDFilter;
};
class LBRPerfReader : public PerfScriptReader {
public:
LBRPerfReader(ProfiledBinary *Binary, StringRef PerfTrace,
Optional<uint32_t> PID)
: PerfScriptReader(Binary, PerfTrace, PID){};
void parseSample(TraceStream &TraceIt, uint64_t Count) override;
};
class HybridPerfReader : public PerfScriptReader {
public:
HybridPerfReader(ProfiledBinary *Binary, StringRef PerfTrace,
Optional<uint32_t> PID)
: PerfScriptReader(Binary, PerfTrace, PID){};
void parseSample(TraceStream &TraceIt, uint64_t Count) override;
void generateUnsymbolizedProfile() override;
private:
void unwindSamples();
};
class UnsymbolizedProfileReader : public PerfReaderBase {
public:
UnsymbolizedProfileReader(ProfiledBinary *Binary, StringRef PerfTrace)
: PerfReaderBase(Binary, PerfTrace){};
void parsePerfTraces() override;
private:
void readSampleCounters(TraceStream &TraceIt, SampleCounter &SCounters);
void readUnsymbolizedProfile(StringRef Filename);
std::unordered_set<std::string> ContextStrSet;
};
} }
#endif