/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
|*
|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|* See https://llvm.org/LICENSE.txt for license information.
|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|*
\*===----------------------------------------------------------------------===*/
/*
* This is the main file that defines all the data structure, signature,
* constant literals that are shared across profiling runtime library,
* compiler (instrumentation), and host tools (reader/writer). The entities
* defined in this file affect the profile runtime ABI, the raw profile format,
* or both.
*
* The file has two identical copies. The primary copy lives in LLVM and
* the other one sits in compiler-rt/lib/profile directory. To make changes
* in this file, first modify the primary copy and copy it over to compiler-rt.
* Testing of any change in this file can start only after the two copies are
* synced up.
*
* The first part of the file includes macros that defines types, names, and
* initializers for the member fields of the core data structures. The field
* declarations for one structure is enabled by defining the field activation
* macro associated with that structure. Only one field activation record
* can be defined at one time and the rest definitions will be filtered out by
* the preprocessor.
*
* Examples of how the template is used to instantiate structure definition:
* 1. To declare a structure:
*
* struct ProfData {
* #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
* Type Name;
* #include "llvm/ProfileData/InstrProfData.inc"
* };
*
* 2. To construct LLVM type arrays for the struct type:
*
* Type *DataTypes[] = {
* #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
* LLVMType,
* #include "llvm/ProfileData/InstrProfData.inc"
* };
*
* 4. To construct constant array for the initializers:
* #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
* Initializer,
* Constant *ConstantVals[] = {
* #include "llvm/ProfileData/InstrProfData.inc"
* };
*
*
* The second part of the file includes definitions all other entities that
* are related to runtime ABI and format. When no field activation macro is
* defined, this file can be included to introduce the definitions.
*
\*===----------------------------------------------------------------------===*/
/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
* the compiler runtime. */
#ifndef INSTR_PROF_VISIBILITY
#define INSTR_PROF_VISIBILITY
#endif
/* INSTR_PROF_DATA start. */
/* Definition of member fields of the per-function control structure. */
#ifndef INSTR_PROF_DATA
#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
Inc->getHash()->getZExtValue()))
INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr)
/* This is used to map function pointers for the indirect call targets to
* function name hashes during the conversion from raw to merged profile
* data.
*/
INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
FunctionAddr)
INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
ValuesPtrExpr)
INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
#undef INSTR_PROF_DATA
/* INSTR_PROF_DATA end. */
/* This is an internal data structure used by value profiler. It
* is defined here to allow serialization code sharing by LLVM
* to be used in unit test.
*
* typedef struct ValueProfNode {
* // InstrProfValueData VData;
* uint64_t Value;
* uint64_t Count;
* struct ValueProfNode *Next;
* } ValueProfNode;
*/
/* INSTR_PROF_VALUE_NODE start. */
#ifndef INSTR_PROF_VALUE_NODE
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \
ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \
ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0))
#undef INSTR_PROF_VALUE_NODE
/* INSTR_PROF_VALUE_NODE end. */
/* INSTR_PROF_RAW_HEADER start */
/* Definition of member fields of the raw profile header data structure. */
#ifndef INSTR_PROF_RAW_HEADER
#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
/* FIXME: A more accurate name is NumData */
INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
/* FIXME: A more accurate name is NumCounters */
INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
(uintptr_t)CountersBegin - (uintptr_t)DataBegin)
INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
#undef INSTR_PROF_RAW_HEADER
/* INSTR_PROF_RAW_HEADER end */
/* VALUE_PROF_FUNC_PARAM start */
/* Definition of parameter types of the runtime API used to do value profiling
* for a given value site.
*/
#ifndef VALUE_PROF_FUNC_PARAM
#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
#define INSTR_PROF_COMMA
#else
#define INSTR_PROF_DATA_DEFINED
#define INSTR_PROF_COMMA ,
#endif
VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
INSTR_PROF_COMMA
VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
#undef VALUE_PROF_FUNC_PARAM
#undef INSTR_PROF_COMMA
/* VALUE_PROF_FUNC_PARAM end */
/* VALUE_PROF_KIND start */
#ifndef VALUE_PROF_KIND
#define VALUE_PROF_KIND(Enumerator, Value, Descr)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
/* For indirect function call value profiling, the addresses of the target
* functions are profiled by the instrumented code. The target addresses are
* written in the raw profile data and converted to target function name's MD5
* hash by the profile reader during deserialization. Typically, this happens
* when the raw profile data is read during profile merging.
*
* For this remapping the ProfData is used. ProfData contains both the function
* name hash and the function address.
*/
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
/* For memory intrinsic functions size profiling. */
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
/* These two kinds must be the last to be
* declared. This is to make sure the string
* array created with the template can be
* indexed with the kind value.
*/
VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
#undef VALUE_PROF_KIND
/* VALUE_PROF_KIND end */
#undef COVMAP_V2_OR_V3
#ifdef COVMAP_V2
#define COVMAP_V2_OR_V3
#endif
#ifdef COVMAP_V3
#define COVMAP_V2_OR_V3
#endif
/* COVMAP_FUNC_RECORD start */
/* Definition of member fields of the function record structure in coverage
* map.
*/
#ifndef COVMAP_FUNC_RECORD
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
#ifdef COVMAP_V1
COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
llvm::Type::getInt8PtrTy(Ctx)))
COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
NameValue.size()))
#endif
#ifdef COVMAP_V2_OR_V3
COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
llvm::ConstantInt::get( \
llvm::Type::getInt64Ty(Ctx), NameHash))
#endif
COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
llvm::ConstantInt::get( \
llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
llvm::ConstantInt::get( \
llvm::Type::getInt64Ty(Ctx), FuncHash))
#ifdef COVMAP_V3
COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
llvm::ConstantInt::get( \
llvm::Type::getInt64Ty(Ctx), FilenamesRef))
COVMAP_FUNC_RECORD(const char, \
llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \
CoverageMapping.size()), \
CoverageMapping,
llvm::ConstantDataArray::getRaw( \
CoverageMapping, CoverageMapping.size(), \
llvm::Type::getInt8Ty(Ctx)))
#endif
#undef COVMAP_FUNC_RECORD
/* COVMAP_FUNC_RECORD end. */
/* COVMAP_HEADER start */
/* Definition of member fields of coverage map header.
*/
#ifndef COVMAP_HEADER
#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
#else
#define INSTR_PROF_DATA_DEFINED
#endif
COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
llvm::ConstantInt::get(Int32Ty, NRecords))
COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
llvm::ConstantInt::get(Int32Ty, FilenamesSize))
COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
COVMAP_HEADER(uint32_t, Int32Ty, Version, \
llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
#undef COVMAP_HEADER
/* COVMAP_HEADER end. */
#ifdef INSTR_PROF_SECT_ENTRY
#define INSTR_PROF_DATA_DEFINED
INSTR_PROF_SECT_ENTRY(IPSK_data, \
INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
INSTR_PROF_DATA_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
INSTR_PROF_CNTS_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_name, \
INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
INSTR_PROF_NAME_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vals, \
INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
INSTR_PROF_VALS_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
INSTR_PROF_VNODES_COFF, "__DATA,")
INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
#undef INSTR_PROF_SECT_ENTRY
#endif
#ifdef INSTR_PROF_VALUE_PROF_DATA
#define INSTR_PROF_DATA_DEFINED
#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
/*!
* This is the header of the data structure that defines the on-disk
* layout of the value profile data of a particular kind for one function.
*/
typedef struct ValueProfRecord {
/* The kind of the value profile record. */
uint32_t Kind;
/*
* The number of value profile sites. It is guaranteed to be non-zero;
* otherwise the record for this kind won't be emitted.
*/
uint32_t NumValueSites;
/*
* The first element of the array that stores the number of profiled
* values for each value site. The size of the array is NumValueSites.
* Since NumValueSites is greater than zero, there is at least one
* element in the array.
*/
uint8_t SiteCountArray[1];
/*
* The fake declaration is for documentation purpose only.
* Align the start of next field to be on 8 byte boundaries.
uint8_t Padding[X];
*/
/* The array of value profile data. The size of the array is the sum
* of all elements in SiteCountArray[].
InstrProfValueData ValueData[];
*/
#ifdef __cplusplus
/*!
* Return the number of value sites.
*/
uint32_t getNumValueSites() const { return NumValueSites; }
/*!
* Read data from this record and save it to Record.
*/
void deserializeTo(InstrProfRecord &Record,
InstrProfSymtab *SymTab);
/*
* In-place byte swap:
* Do byte swap for this instance. \c Old is the original order before
* the swap, and \c New is the New byte order.
*/
void swapBytes(support::endianness Old, support::endianness New);
#endif
} ValueProfRecord;
/*!
* Per-function header/control data structure for value profiling
* data in indexed format.
*/
typedef struct ValueProfData {
/*
* Total size in bytes including this field. It must be a multiple
* of sizeof(uint64_t).
*/
uint32_t TotalSize;
/*
*The number of value profile kinds that has value profile data.
* In this implementation, a value profile kind is considered to
* have profile data if the number of value profile sites for the
* kind is not zero. More aggressively, the implementation can
* choose to check the actual data value: if none of the value sites
* has any profiled values, the kind can be skipped.
*/
uint32_t NumValueKinds;
/*
* Following are a sequence of variable length records. The prefix/header
* of each record is defined by ValueProfRecord type. The number of
* records is NumValueKinds.
* ValueProfRecord Record_1;
* ValueProfRecord Record_N;
*/
#if __cplusplus
/*!
* Return the total size in bytes of the on-disk value profile data
* given the data stored in Record.
*/
static uint32_t getSize(const InstrProfRecord &Record);
/*!
* Return a pointer to \c ValueProfData instance ready to be streamed.
*/
static std::unique_ptr<ValueProfData>
serializeFrom(const InstrProfRecord &Record);
/*!
* Check the integrity of the record.
*/
Error checkIntegrity();
/*!
* Return a pointer to \c ValueProfileData instance ready to be read.
* All data in the instance are properly byte swapped. The input
* data is assumed to be in little endian order.
*/
static Expected<std::unique_ptr<ValueProfData>>
getValueProfData(const unsigned char *SrcBuffer,
const unsigned char *const SrcBufferEnd,
support::endianness SrcDataEndianness);
/*!
* Swap byte order from \c Endianness order to host byte order.
*/
void swapBytesToHost(support::endianness Endianness);
/*!
* Swap byte order from host byte order to \c Endianness order.
*/
void swapBytesFromHost(support::endianness Endianness);
/*!
* Return the total size of \c ValueProfileData.
*/
uint32_t getSize() const { return TotalSize; }
/*!
* Read data from this data and save it to \c Record.
*/
void deserializeTo(InstrProfRecord &Record,
InstrProfSymtab *SymTab);
void operator delete(void *ptr) { ::operator delete(ptr); }
#endif
} ValueProfData;
/*
* The closure is designed to abstact away two types of value profile data:
* - InstrProfRecord which is the primary data structure used to
* represent profile data in host tools (reader, writer, and profile-use)
* - value profile runtime data structure suitable to be used by C
* runtime library.
*
* Both sources of data need to serialize to disk/memory-buffer in common
* format: ValueProfData. The abstraction allows compiler-rt's raw profiler
* writer to share the same format and code with indexed profile writer.
*
* For documentation of the member methods below, refer to corresponding methods
* in class InstrProfRecord.
*/
typedef struct ValueProfRecordClosure {
const void *Record;
uint32_t (*GetNumValueKinds)(const void *Record);
uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
/*
* After extracting the value profile data from the value profile record,
* this method is used to map the in-memory value to on-disk value. If
* the method is null, value will be written out untranslated.
*/
uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
uint32_t S);
ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
} ValueProfRecordClosure;
INSTR_PROF_VISIBILITY ValueProfRecord *
getFirstValueProfRecord(ValueProfData *VPD);
INSTR_PROF_VISIBILITY ValueProfRecord *
getValueProfRecordNext(ValueProfRecord *VPR);
INSTR_PROF_VISIBILITY InstrProfValueData *
getValueProfRecordValueData(ValueProfRecord *VPR);
INSTR_PROF_VISIBILITY uint32_t
getValueProfRecordHeaderSize(uint32_t NumValueSites);
#undef INSTR_PROF_VALUE_PROF_DATA
#endif /* INSTR_PROF_VALUE_PROF_DATA */
#ifdef INSTR_PROF_COMMON_API_IMPL
#define INSTR_PROF_DATA_DEFINED
#ifdef __cplusplus
#define INSTR_PROF_INLINE inline
#define INSTR_PROF_NULLPTR nullptr
#else
#define INSTR_PROF_INLINE
#define INSTR_PROF_NULLPTR NULL
#endif
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
/*!
* Return the \c ValueProfRecord header size including the
* padding bytes.
*/
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
sizeof(uint8_t) * NumValueSites;
/* Round the size to multiple of 8 bytes. */
Size = (Size + 7) & ~7;
return Size;
}
/*!
* Return the total size of the value profile record including the
* header and the value data.
*/
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
uint32_t getValueProfRecordSize(uint32_t NumValueSites,
uint32_t NumValueData) {
return getValueProfRecordHeaderSize(NumValueSites) +
sizeof(InstrProfValueData) * NumValueData;
}
/*!
* Return the pointer to the start of value data array.
*/
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
This->NumValueSites));
}
/*!
* Return the total number of value data for \c This record.
*/
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
uint32_t NumValueData = 0;
uint32_t I;
for (I = 0; I < This->NumValueSites; I++)
NumValueData += This->SiteCountArray[I];
return NumValueData;
}
/*!
* Use this method to advance to the next \c This \c ValueProfRecord.
*/
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
uint32_t NumValueData = getValueProfRecordNumValueData(This);
return (ValueProfRecord *)((char *)This +
getValueProfRecordSize(This->NumValueSites,
NumValueData));
}
/*!
* Return the first \c ValueProfRecord instance.
*/
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
}
/* Closure based interfaces. */
/*!
* Return the total size in bytes of the on-disk value profile data
* given the data stored in Record.
*/
INSTR_PROF_VISIBILITY uint32_t
getValueProfDataSize(ValueProfRecordClosure *Closure) {
uint32_t Kind;
uint32_t TotalSize = sizeof(ValueProfData);
const void *Record = Closure->Record;
for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
if (!NumValueSites)
continue;
TotalSize += getValueProfRecordSize(NumValueSites,
Closure->GetNumValueData(Record, Kind));
}
return TotalSize;
}
/*!
* Extract value profile data of a function for the profile kind \c ValueKind
* from the \c Closure and serialize the data into \c This record instance.
*/
INSTR_PROF_VISIBILITY void
serializeValueProfRecordFrom(ValueProfRecord *This,
ValueProfRecordClosure *Closure,
uint32_t ValueKind, uint32_t NumValueSites) {
uint32_t S;
const void *Record = Closure->Record;
This->Kind = ValueKind;
This->NumValueSites = NumValueSites;
InstrProfValueData *DstVD = getValueProfRecordValueData(This);
for (S = 0; S < NumValueSites; S++) {
uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
This->SiteCountArray[S] = ND;
Closure->GetValueForSite(Record, DstVD, ValueKind, S);
DstVD += ND;
}
}
/*!
* Extract value profile data of a function from the \c Closure
* and serialize the data into \c DstData if it is not NULL or heap
* memory allocated by the \c Closure's allocator method. If \c
* DstData is not null, the caller is expected to set the TotalSize
* in DstData.
*/
INSTR_PROF_VISIBILITY ValueProfData *
serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
ValueProfData *DstData) {
uint32_t Kind;
uint32_t TotalSize =
DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
ValueProfData *VPD =
DstData ? DstData : Closure->AllocValueProfData(TotalSize);
VPD->TotalSize = TotalSize;
VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
ValueProfRecord *VR = getFirstValueProfRecord(VPD);
for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
if (!NumValueSites)
continue;
serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
VR = getValueProfRecordNext(VR);
}
return VPD;
}
#undef INSTR_PROF_COMMON_API_IMPL
#endif /* INSTR_PROF_COMMON_API_IMPL */
/*============================================================================*/
#ifndef INSTR_PROF_DATA_DEFINED
#ifndef INSTR_PROF_DATA_INC
#define INSTR_PROF_DATA_INC
/* Helper macros. */
#define INSTR_PROF_SIMPLE_QUOTE(x) #x
#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
/* Magic number to detect file format and endianness.
* Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
* so that utilities, like strings, don't grab it as a string. 129 is also
* invalid UTF-8, and high enough to be interesting.
* Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
* for 32-bit platforms.
*/
#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
(uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
(uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
(uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
(uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
/* FIXME: Please remedy the fixme in the header before bumping the version. */
/* Raw profile format version (start from 1). */
#define INSTR_PROF_RAW_VERSION 8
/* Indexed profile format version (start from 1). */
#define INSTR_PROF_INDEX_VERSION 8
/* Coverage mapping format version (start from 0). */
#define INSTR_PROF_COVMAP_VERSION 5
/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
* version for other variants of profile. We set the lowest bit of the upper 8
* bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
* generated profile, and 0 if this is a Clang FE generated profile.
* 1 in bit 57 indicates there are context-sensitive records in the profile.
* The 59th bit indicates whether to use debug info to correlate profiles.
* The 60th bit indicates single byte coverage instrumentation.
* The 61st bit indicates function entry instrumentation only.
* The 62nd bit indicates whether memory profile information is present.
*/
#define VARIANT_MASKS_ALL 0xff00000000000000ULL
#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)
#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)
#define VARIANT_MASK_MEMPROF (0x1ULL << 62)
#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
/* The variable that holds the name of the profile data
* specified via command line. */
#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
/* section name strings common to all targets other
than WIN32 */
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
#define INSTR_PROF_COVFUN_COMMON __llvm_covfun
#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
/* Windows section names. Because these section names contain dollar characters,
* they must be quoted.
*/
#define INSTR_PROF_DATA_COFF ".lprfd$M"
#define INSTR_PROF_NAME_COFF ".lprfn$M"
#define INSTR_PROF_CNTS_COFF ".lprfc$M"
#define INSTR_PROF_VALS_COFF ".lprfv$M"
#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
#define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
#ifdef _WIN32
/* Runtime section names and name strings. */
#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
/* Array of pointers. Each pointer points to a list
* of value nodes associated with one value site.
*/
#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
/* Value profile nodes section. */
#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
#else
/* Runtime section names and name strings. */
#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
/* Array of pointers. Each pointer points to a list
* of value nodes associated with one value site.
*/
#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
/* Value profile nodes section. */
#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
/* Order file instrumentation. */
#define INSTR_PROF_ORDERFILE_SECT_NAME \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
#endif
#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
/* Macros to define start/stop section symbol for a given
* section on Linux. For instance
* INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
* expand to __start___llvm_prof_data
*/
#define INSTR_PROF_SECT_START(Sect) \
INSTR_PROF_CONCAT(__start_,Sect)
#define INSTR_PROF_SECT_STOP(Sect) \
INSTR_PROF_CONCAT(__stop_,Sect)
/* Value Profiling API linkage name. */
#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
#define INSTR_PROF_VALUE_PROF_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
/* InstrProfile per-function control data alignment. */
#define INSTR_PROF_DATA_ALIGNMENT 8
/* The data structure that represents a tracked value by the
* value profiler.
*/
typedef struct InstrProfValueData {
/* Profiled value. */
uint64_t Value;
/* Number of times the value appears in the training run. */
uint64_t Count;
} InstrProfValueData;
#endif /* INSTR_PROF_DATA_INC */
#ifndef INSTR_ORDER_FILE_INC
/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
#define INSTR_ORDER_FILE_BUFFER_BITS 17
#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
#endif /* INSTR_ORDER_FILE_INC */
#else
#undef INSTR_PROF_DATA_DEFINED
#endif
#undef COVMAP_V2_OR_V3
#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
#ifdef __cplusplus
#define INSTR_PROF_INLINE inline
#else
#define INSTR_PROF_INLINE
#endif
/* The value range buckets (22 buckets) for the memop size value profiling looks
* like:
*
* [0, 0]
* [1, 1]
* [2, 2]
* [3, 3]
* [4, 4]
* [5, 5]
* [6, 6]
* [7, 7]
* [8, 8]
* [9, 15]
* [16, 16]
* [17, 31]
* [32, 32]
* [33, 63]
* [64, 64]
* [65, 127]
* [128, 128]
* [129, 255]
* [256, 256]
* [257, 511]
* [512, 512]
* [513, UINT64_MAX]
*
* Each range has a 'representative value' which is the lower end value of the
* range and used to store in the runtime profile data records and the VP
* metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
*/
#define INSTR_PROF_NUM_BUCKETS 22
/*
* Clz and Popcount. This code was copied from
* compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
* llvm/include/llvm/Support/MathExtras.h.
*/
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
int InstProfClzll(unsigned long long X) {
unsigned long LeadZeroIdx = 0;
#if !defined(_M_ARM64) && !defined(_M_X64)
// Scan the high 32 bits.
if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
// from the MSB.
// Scan the low 32 bits.
if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
return (int)(63 - LeadZeroIdx);
#else
if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
#endif
return 64;
}
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
int InstProfPopcountll(unsigned long long X) {
// This code originates from https://reviews.llvm.org/rG30626254510f.
unsigned long long v = X;
v = v - ((v >> 1) & 0x5555555555555555ULL);
v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
}
#else
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
#endif /* defined(_MSC_VER) && !defined(__clang__) */
/* Map an (observed) memop size value to the representative value of its range.
* For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
InstrProfGetRangeRepValue(uint64_t Value) {
if (Value <= 8)
// The first ranges are individually tracked. Use the value as is.
return Value;
else if (Value >= 513)
// The last range is mapped to its lowest value.
return 513;
else if (InstProfPopcountll(Value) == 1)
// If it's a power of two, use it as is.
return Value;
else
// Otherwise, take to the previous power of two + 1.
return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
}
/* Return true if the range that an (observed) memop size value belongs to has
* only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
* false, 64 -> true, 100 -> false, 513 -> false. */
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
InstrProfIsSingleValRange(uint64_t Value) {
if (Value <= 8)
// The first ranges are individually tracked.
return 1;
else if (InstProfPopcountll(Value) == 1)
// If it's a power of two, there's only one value.
return 1;
else
// Otherwise, there's more than one value in the range.
return 0;
}
#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */