#include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/ModuleSummaryAnalysis.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/TypeMetadataUtils.h"
#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/InitializePasses.h"
#include "llvm/Object/ModuleSymbolTable.h"
#include "llvm/Pass.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/IPO.h"
#include "llvm/Transforms/IPO/FunctionAttrs.h"
#include "llvm/Transforms/IPO/FunctionImport.h"
#include "llvm/Transforms/IPO/LowerTypeTests.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
using namespace llvm;
namespace {
static bool allowPromotionAlias(const std::string &Name) {
for (const char &C : Name) {
if (isAlnum(C) || C == '_' || C == '.')
continue;
return false;
}
return true;
}
void promoteInternals(Module &ExportM, Module &ImportM, StringRef ModuleId,
SetVector<GlobalValue *> &PromoteExtra) {
DenseMap<const Comdat *, Comdat *> RenamedComdats;
for (auto &ExportGV : ExportM.global_values()) {
if (!ExportGV.hasLocalLinkage())
continue;
auto Name = ExportGV.getName();
GlobalValue *ImportGV = nullptr;
if (!PromoteExtra.count(&ExportGV)) {
ImportGV = ImportM.getNamedValue(Name);
if (!ImportGV)
continue;
ImportGV->removeDeadConstantUsers();
if (ImportGV->use_empty()) {
ImportGV->eraseFromParent();
continue;
}
}
std::string OldName = Name.str();
std::string NewName = (Name + ModuleId).str();
if (const auto *C = ExportGV.getComdat())
if (C->getName() == Name)
RenamedComdats.try_emplace(C, ExportM.getOrInsertComdat(NewName));
ExportGV.setName(NewName);
ExportGV.setLinkage(GlobalValue::ExternalLinkage);
ExportGV.setVisibility(GlobalValue::HiddenVisibility);
if (ImportGV) {
ImportGV->setName(NewName);
ImportGV->setVisibility(GlobalValue::HiddenVisibility);
}
if (isa<Function>(&ExportGV) && allowPromotionAlias(OldName)) {
std::string Alias =
".lto_set_conditional " + OldName + "," + NewName + "\n";
ExportM.appendModuleInlineAsm(Alias);
}
}
if (!RenamedComdats.empty())
for (auto &GO : ExportM.global_objects())
if (auto *C = GO.getComdat()) {
auto Replacement = RenamedComdats.find(C);
if (Replacement != RenamedComdats.end())
GO.setComdat(Replacement->second);
}
}
void promoteTypeIds(Module &M, StringRef ModuleId) {
DenseMap<Metadata *, Metadata *> LocalToGlobal;
auto ExternalizeTypeId = [&](CallInst *CI, unsigned ArgNo) {
Metadata *MD =
cast<MetadataAsValue>(CI->getArgOperand(ArgNo))->getMetadata();
if (isa<MDNode>(MD) && cast<MDNode>(MD)->isDistinct()) {
Metadata *&GlobalMD = LocalToGlobal[MD];
if (!GlobalMD) {
std::string NewName = (Twine(LocalToGlobal.size()) + ModuleId).str();
GlobalMD = MDString::get(M.getContext(), NewName);
}
CI->setArgOperand(ArgNo,
MetadataAsValue::get(M.getContext(), GlobalMD));
}
};
if (Function *TypeTestFunc =
M.getFunction(Intrinsic::getName(Intrinsic::type_test))) {
for (const Use &U : TypeTestFunc->uses()) {
auto CI = cast<CallInst>(U.getUser());
ExternalizeTypeId(CI, 1);
}
}
if (Function *PublicTypeTestFunc =
M.getFunction(Intrinsic::getName(Intrinsic::public_type_test))) {
for (const Use &U : PublicTypeTestFunc->uses()) {
auto CI = cast<CallInst>(U.getUser());
ExternalizeTypeId(CI, 1);
}
}
if (Function *TypeCheckedLoadFunc =
M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load))) {
for (const Use &U : TypeCheckedLoadFunc->uses()) {
auto CI = cast<CallInst>(U.getUser());
ExternalizeTypeId(CI, 2);
}
}
for (GlobalObject &GO : M.global_objects()) {
SmallVector<MDNode *, 1> MDs;
GO.getMetadata(LLVMContext::MD_type, MDs);
GO.eraseMetadata(LLVMContext::MD_type);
for (auto MD : MDs) {
auto I = LocalToGlobal.find(MD->getOperand(1));
if (I == LocalToGlobal.end()) {
GO.addMetadata(LLVMContext::MD_type, *MD);
continue;
}
GO.addMetadata(
LLVMContext::MD_type,
*MDNode::get(M.getContext(), {MD->getOperand(0), I->second}));
}
}
}
void simplifyExternals(Module &M) {
FunctionType *EmptyFT =
FunctionType::get(Type::getVoidTy(M.getContext()), false);
for (Function &F : llvm::make_early_inc_range(M)) {
if (F.isDeclaration() && F.use_empty()) {
F.eraseFromParent();
continue;
}
if (!F.isDeclaration() || F.getFunctionType() == EmptyFT ||
F.getName().startswith("llvm."))
continue;
Function *NewF =
Function::Create(EmptyFT, GlobalValue::ExternalLinkage,
F.getAddressSpace(), "", &M);
NewF->copyAttributesFrom(&F);
NewF->setAttributes(AttributeList::get(M.getContext(),
AttributeList::FunctionIndex,
F.getAttributes().getFnAttrs()));
NewF->takeName(&F);
F.replaceAllUsesWith(ConstantExpr::getBitCast(NewF, F.getType()));
F.eraseFromParent();
}
for (GlobalVariable &GV : llvm::make_early_inc_range(M.globals())) {
if (GV.isDeclaration() && GV.use_empty()) {
GV.eraseFromParent();
continue;
}
}
}
static void
filterModule(Module *M,
function_ref<bool(const GlobalValue *)> ShouldKeepDefinition) {
std::vector<GlobalValue *> V;
for (GlobalValue &GV : M->global_values())
if (!ShouldKeepDefinition(&GV))
V.push_back(&GV);
for (GlobalValue *GV : V)
if (!convertToDeclaration(*GV))
GV->eraseFromParent();
}
void forEachVirtualFunction(Constant *C, function_ref<void(Function *)> Fn) {
if (auto *F = dyn_cast<Function>(C))
return Fn(F);
if (isa<GlobalValue>(C))
return;
for (Value *Op : C->operands())
forEachVirtualFunction(cast<Constant>(Op), Fn);
}
static void cloneUsedGlobalVariables(const Module &SrcM, Module &DestM,
bool CompilerUsed) {
SmallVector<GlobalValue *, 4> Used, NewUsed;
collectUsedGlobalVariables(SrcM, Used, CompilerUsed);
for (auto *V : Used) {
auto *GV = DestM.getNamedValue(V->getName());
if (GV && !GV->isDeclaration())
NewUsed.push_back(GV);
}
if (CompilerUsed)
appendToCompilerUsed(DestM, NewUsed);
else
appendToUsed(DestM, NewUsed);
}
void splitAndWriteThinLTOBitcode(
raw_ostream &OS, raw_ostream *ThinLinkOS,
function_ref<AAResults &(Function &)> AARGetter, Module &M) {
std::string ModuleId = getUniqueModuleId(&M);
if (ModuleId.empty()) {
ProfileSummaryInfo PSI(M);
M.addModuleFlag(Module::Error, "ThinLTO", uint32_t(0));
ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, &PSI);
WriteBitcodeToFile(M, OS, false, &Index);
if (ThinLinkOS)
WriteBitcodeToFile(M, *ThinLinkOS, false,
&Index);
return;
}
promoteTypeIds(M, ModuleId);
auto HasTypeMetadata = [](const GlobalObject *GO) {
if (MDNode *MD = GO->getMetadata(LLVMContext::MD_associated))
if (auto *AssocVM = dyn_cast_or_null<ValueAsMetadata>(MD->getOperand(0)))
if (auto *AssocGO = dyn_cast<GlobalObject>(AssocVM->getValue()))
if (AssocGO->hasMetadata(LLVMContext::MD_type))
return true;
return GO->hasMetadata(LLVMContext::MD_type);
};
DenseSet<const Function *> EligibleVirtualFns;
DenseSet<const Comdat *> MergedMComdats;
for (GlobalVariable &GV : M.globals())
if (HasTypeMetadata(&GV)) {
if (const auto *C = GV.getComdat())
MergedMComdats.insert(C);
forEachVirtualFunction(GV.getInitializer(), [&](Function *F) {
auto *RT = dyn_cast<IntegerType>(F->getReturnType());
if (!RT || RT->getBitWidth() > 64 || F->arg_empty() ||
!F->arg_begin()->use_empty())
return;
for (auto &Arg : drop_begin(F->args())) {
auto *ArgT = dyn_cast<IntegerType>(Arg.getType());
if (!ArgT || ArgT->getBitWidth() > 64)
return;
}
if (!F->isDeclaration() &&
computeFunctionBodyMemoryAccess(*F, AARGetter(*F)) ==
FMRB_DoesNotAccessMemory)
EligibleVirtualFns.insert(F);
});
}
ValueToValueMapTy VMap;
std::unique_ptr<Module> MergedM(
CloneModule(M, VMap, [&](const GlobalValue *GV) -> bool {
if (const auto *C = GV->getComdat())
if (MergedMComdats.count(C))
return true;
if (auto *F = dyn_cast<Function>(GV))
return EligibleVirtualFns.count(F);
if (auto *GVar =
dyn_cast_or_null<GlobalVariable>(GV->getAliaseeObject()))
return HasTypeMetadata(GVar);
return false;
}));
StripDebugInfo(*MergedM);
MergedM->setModuleInlineAsm("");
cloneUsedGlobalVariables(M, *MergedM, false);
cloneUsedGlobalVariables(M, *MergedM, true);
for (Function &F : *MergedM)
if (!F.isDeclaration()) {
F.setLinkage(GlobalValue::AvailableExternallyLinkage);
F.setComdat(nullptr);
}
SetVector<GlobalValue *> CfiFunctions;
for (auto &F : M)
if ((!F.hasLocalLinkage() || F.hasAddressTaken()) && HasTypeMetadata(&F))
CfiFunctions.insert(&F);
filterModule(&M, [&](const GlobalValue *GV) {
if (auto *GVar = dyn_cast_or_null<GlobalVariable>(GV->getAliaseeObject()))
if (HasTypeMetadata(GVar))
return false;
if (const auto *C = GV->getComdat())
if (MergedMComdats.count(C))
return false;
return true;
});
promoteInternals(*MergedM, M, ModuleId, CfiFunctions);
promoteInternals(M, *MergedM, ModuleId, CfiFunctions);
auto &Ctx = MergedM->getContext();
SmallVector<MDNode *, 8> CfiFunctionMDs;
for (auto V : CfiFunctions) {
Function &F = *cast<Function>(V);
SmallVector<MDNode *, 2> Types;
F.getMetadata(LLVMContext::MD_type, Types);
SmallVector<Metadata *, 4> Elts;
Elts.push_back(MDString::get(Ctx, F.getName()));
CfiFunctionLinkage Linkage;
if (lowertypetests::isJumpTableCanonical(&F))
Linkage = CFL_Definition;
else if (F.hasExternalWeakLinkage())
Linkage = CFL_WeakDeclaration;
else
Linkage = CFL_Declaration;
Elts.push_back(ConstantAsMetadata::get(
llvm::ConstantInt::get(Type::getInt8Ty(Ctx), Linkage)));
append_range(Elts, Types);
CfiFunctionMDs.push_back(MDTuple::get(Ctx, Elts));
}
if(!CfiFunctionMDs.empty()) {
NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("cfi.functions");
for (auto MD : CfiFunctionMDs)
NMD->addOperand(MD);
}
SmallVector<MDNode *, 8> FunctionAliases;
for (auto &A : M.aliases()) {
if (!isa<Function>(A.getAliasee()))
continue;
auto *F = cast<Function>(A.getAliasee());
Metadata *Elts[] = {
MDString::get(Ctx, A.getName()),
MDString::get(Ctx, F->getName()),
ConstantAsMetadata::get(
ConstantInt::get(Type::getInt8Ty(Ctx), A.getVisibility())),
ConstantAsMetadata::get(
ConstantInt::get(Type::getInt8Ty(Ctx), A.isWeakForLinker())),
};
FunctionAliases.push_back(MDTuple::get(Ctx, Elts));
}
if (!FunctionAliases.empty()) {
NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("aliases");
for (auto MD : FunctionAliases)
NMD->addOperand(MD);
}
SmallVector<MDNode *, 8> Symvers;
ModuleSymbolTable::CollectAsmSymvers(M, [&](StringRef Name, StringRef Alias) {
Function *F = M.getFunction(Name);
if (!F || F->use_empty())
return;
Symvers.push_back(MDTuple::get(
Ctx, {MDString::get(Ctx, Name), MDString::get(Ctx, Alias)}));
});
if (!Symvers.empty()) {
NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("symvers");
for (auto MD : Symvers)
NMD->addOperand(MD);
}
simplifyExternals(*MergedM);
ProfileSummaryInfo PSI(M);
ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, &PSI);
MergedM->addModuleFlag(Module::Error, "ThinLTO", uint32_t(0));
ModuleSummaryIndex MergedMIndex =
buildModuleSummaryIndex(*MergedM, nullptr, &PSI);
SmallVector<char, 0> Buffer;
BitcodeWriter W(Buffer);
ModuleHash ModHash = {{0}};
W.writeModule(M, false, &Index,
true, &ModHash);
W.writeModule(*MergedM, false, &MergedMIndex);
W.writeSymtab();
W.writeStrtab();
OS << Buffer;
if (ThinLinkOS) {
Buffer.clear();
BitcodeWriter W2(Buffer);
StripDebugInfo(M);
W2.writeThinLinkBitcode(M, Index, ModHash);
W2.writeModule(*MergedM, false,
&MergedMIndex);
W2.writeSymtab();
W2.writeStrtab();
*ThinLinkOS << Buffer;
}
}
bool enableSplitLTOUnit(Module &M) {
bool EnableSplitLTOUnit = false;
if (auto *MD = mdconst::extract_or_null<ConstantInt>(
M.getModuleFlag("EnableSplitLTOUnit")))
EnableSplitLTOUnit = MD->getZExtValue();
return EnableSplitLTOUnit;
}
bool hasTypeMetadata(Module &M) {
for (auto &GO : M.global_objects()) {
if (GO.hasMetadata(LLVMContext::MD_type))
return true;
}
return false;
}
void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS,
function_ref<AAResults &(Function &)> AARGetter,
Module &M, const ModuleSummaryIndex *Index) {
std::unique_ptr<ModuleSummaryIndex> NewIndex = nullptr;
if (hasTypeMetadata(M)) {
if (enableSplitLTOUnit(M))
return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M);
std::string ModuleId = getUniqueModuleId(&M);
if (!ModuleId.empty()) {
promoteTypeIds(M, ModuleId);
ProfileSummaryInfo PSI(M);
NewIndex = std::make_unique<ModuleSummaryIndex>(
buildModuleSummaryIndex(M, nullptr, &PSI));
Index = NewIndex.get();
}
}
ModuleHash ModHash = {{0}};
WriteBitcodeToFile(M, OS, false, Index,
true, &ModHash);
if (ThinLinkOS && Index)
writeThinLinkBitcodeToFile(M, *ThinLinkOS, *Index, ModHash);
}
class WriteThinLTOBitcode : public ModulePass {
raw_ostream &OS; raw_ostream *ThinLinkOS = nullptr;
public:
static char ID; WriteThinLTOBitcode() : ModulePass(ID), OS(dbgs()) {
initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry());
}
explicit WriteThinLTOBitcode(raw_ostream &o, raw_ostream *ThinLinkOS)
: ModulePass(ID), OS(o), ThinLinkOS(ThinLinkOS) {
initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry());
}
StringRef getPassName() const override { return "ThinLTO Bitcode Writer"; }
bool runOnModule(Module &M) override {
const ModuleSummaryIndex *Index =
&(getAnalysis<ModuleSummaryIndexWrapperPass>().getIndex());
writeThinLTOBitcode(OS, ThinLinkOS, LegacyAARGetter(*this), M, Index);
return true;
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesAll();
AU.addRequired<AssumptionCacheTracker>();
AU.addRequired<ModuleSummaryIndexWrapperPass>();
AU.addRequired<TargetLibraryInfoWrapperPass>();
}
};
}
char WriteThinLTOBitcode::ID = 0;
INITIALIZE_PASS_BEGIN(WriteThinLTOBitcode, "write-thinlto-bitcode",
"Write ThinLTO Bitcode", false, true)
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
INITIALIZE_PASS_DEPENDENCY(ModuleSummaryIndexWrapperPass)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
INITIALIZE_PASS_END(WriteThinLTOBitcode, "write-thinlto-bitcode",
"Write ThinLTO Bitcode", false, true)
ModulePass *llvm::createWriteThinLTOBitcodePass(raw_ostream &Str,
raw_ostream *ThinLinkOS) {
return new WriteThinLTOBitcode(Str, ThinLinkOS);
}
PreservedAnalyses
llvm::ThinLTOBitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) {
FunctionAnalysisManager &FAM =
AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
writeThinLTOBitcode(OS, ThinLinkOS,
[&FAM](Function &F) -> AAResults & {
return FAM.getResult<AAManager>(F);
},
M, &AM.getResult<ModuleSummaryIndexAnalysis>(M));
return PreservedAnalyses::all();
}