#include "clang/Support/RISCVVIntrinsicUtils.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
#include "llvm/TableGen/Error.h"
#include "llvm/TableGen/Record.h"
#include <numeric>
using namespace llvm;
using namespace clang::RISCV;
namespace {
struct SemaRecord {
std::string Name;
std::string OverloadedName;
unsigned TypeRangeMask;
unsigned Log2LMULMask;
unsigned RequiredExtensions;
SmallVector<PrototypeDescriptor> Prototype;
SmallVector<PrototypeDescriptor> Suffix;
SmallVector<PrototypeDescriptor> OverloadedSuffix;
unsigned NF;
bool HasMasked :1;
bool HasVL :1;
bool HasMaskedOffOperand :1;
};
class SemaSignatureTable {
private:
std::vector<PrototypeDescriptor> SignatureTable;
void insert(ArrayRef<PrototypeDescriptor> Signature);
public:
static constexpr unsigned INVALID_INDEX = ~0U;
void init(ArrayRef<SemaRecord> SemaRecords);
unsigned getIndex(ArrayRef<PrototypeDescriptor> Signature);
void print(raw_ostream &OS);
};
class RVVEmitter {
private:
RecordKeeper &Records;
public:
RVVEmitter(RecordKeeper &R) : Records(R) {}
void createHeader(raw_ostream &o);
void createBuiltins(raw_ostream &o);
void createCodeGen(raw_ostream &o);
void createSema(raw_ostream &o);
private:
void createRVVIntrinsics(std::vector<std::unique_ptr<RVVIntrinsic>> &Out,
std::vector<SemaRecord> *SemaRecords = nullptr);
void createRVVIntrinsicRecords(std::vector<RVVIntrinsicRecord> &Out,
SemaSignatureTable &SST,
ArrayRef<SemaRecord> SemaRecords);
void printHeaderCode(raw_ostream &OS);
};
}
static BasicType ParseBasicType(char c) {
switch (c) {
case 'c':
return BasicType::Int8;
break;
case 's':
return BasicType::Int16;
break;
case 'i':
return BasicType::Int32;
break;
case 'l':
return BasicType::Int64;
break;
case 'x':
return BasicType::Float16;
break;
case 'f':
return BasicType::Float32;
break;
case 'd':
return BasicType::Float64;
break;
default:
return BasicType::Unknown;
}
}
void emitCodeGenSwitchBody(const RVVIntrinsic *RVVI, raw_ostream &OS) {
if (!RVVI->getIRName().empty())
OS << " ID = Intrinsic::riscv_" + RVVI->getIRName() + ";\n";
if (RVVI->getNF() >= 2)
OS << " NF = " + utostr(RVVI->getNF()) + ";\n";
if (RVVI->hasManualCodegen()) {
OS << RVVI->getManualCodegen();
OS << "break;\n";
return;
}
for (const auto &I : enumerate(RVVI->getInputTypes())) {
if (I.value()->isPointer()) {
assert(RVVI->getIntrinsicTypes().front() == -1 &&
"RVVI should be vector load intrinsic.");
OS << " Ops[" << I.index() << "] = Builder.CreateBitCast(Ops[";
OS << I.index() << "], ResultType->getPointerTo());\n";
}
}
if (RVVI->isMasked()) {
if (RVVI->hasVL()) {
OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);\n";
if (RVVI->hasPolicyOperand())
OS << " Ops.push_back(ConstantInt::get(Ops.back()->getType(),"
" TAIL_UNDISTURBED));\n";
} else {
OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());\n";
}
} else {
if (RVVI->hasPolicyOperand())
OS << " Ops.push_back(ConstantInt::get(Ops.back()->getType(), "
"TAIL_UNDISTURBED));\n";
else if (RVVI->hasPassthruOperand()) {
OS << " Ops.push_back(llvm::UndefValue::get(ResultType));\n";
OS << " std::rotate(Ops.rbegin(), Ops.rbegin() + 1, Ops.rend());\n";
}
}
OS << " IntrinsicTypes = {";
ListSeparator LS;
for (const auto &Idx : RVVI->getIntrinsicTypes()) {
if (Idx == -1)
OS << LS << "ResultType";
else
OS << LS << "Ops[" << Idx << "]->getType()";
}
if (RVVI->hasVL())
OS << ", Ops.back()->getType()";
OS << "};\n";
OS << " break;\n";
}
void SemaSignatureTable::init(ArrayRef<SemaRecord> SemaRecords) {
struct Compare {
bool operator()(const SmallVector<PrototypeDescriptor> &A,
const SmallVector<PrototypeDescriptor> &B) const {
if (A.size() != B.size())
return A.size() > B.size();
size_t Len = A.size();
for (size_t i = 0; i < Len; ++i) {
if (A[i] != B[i])
return A[i] < B[i];
}
return false;
}
};
std::set<SmallVector<PrototypeDescriptor>, Compare> Signatures;
auto InsertToSignatureSet =
[&](const SmallVector<PrototypeDescriptor> &Signature) {
if (Signature.empty())
return;
Signatures.insert(Signature);
};
assert(!SemaRecords.empty());
llvm::for_each(SemaRecords, [&](const SemaRecord &SR) {
InsertToSignatureSet(SR.Prototype);
InsertToSignatureSet(SR.Suffix);
InsertToSignatureSet(SR.OverloadedSuffix);
});
llvm::for_each(Signatures, [this](auto &Sig) { insert(Sig); });
}
void SemaSignatureTable::insert(ArrayRef<PrototypeDescriptor> Signature) {
if (getIndex(Signature) != INVALID_INDEX)
return;
SignatureTable.insert(SignatureTable.begin(), Signature.begin(),
Signature.end());
}
unsigned SemaSignatureTable::getIndex(ArrayRef<PrototypeDescriptor> Signature) {
if (Signature.empty())
return 0;
if (Signature.size() < SignatureTable.size()) {
size_t Bound = SignatureTable.size() - Signature.size() + 1;
for (size_t Index = 0; Index < Bound; ++Index) {
if (equal(Signature.begin(), Signature.end(),
SignatureTable.begin() + Index))
return Index;
}
}
return INVALID_INDEX;
}
void SemaSignatureTable::print(raw_ostream &OS) {
for (const auto &Sig : SignatureTable)
OS << "PrototypeDescriptor(" << static_cast<int>(Sig.PT) << ", "
<< static_cast<int>(Sig.VTM) << ", " << static_cast<int>(Sig.TM)
<< "),\n";
}
void RVVEmitter::createHeader(raw_ostream &OS) {
OS << "/*===---- riscv_vector.h - RISC-V V-extension RVVIntrinsics "
"-------------------===\n"
" *\n"
" *\n"
" * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
"Exceptions.\n"
" * See https://llvm.org/LICENSE.txt for license information.\n"
" * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
" *\n"
" *===-----------------------------------------------------------------"
"------===\n"
" */\n\n";
OS << "#ifndef __RISCV_VECTOR_H\n";
OS << "#define __RISCV_VECTOR_H\n\n";
OS << "#include <stdint.h>\n";
OS << "#include <stddef.h>\n\n";
OS << "#ifndef __riscv_vector\n";
OS << "#error \"Vector intrinsics require the vector extension.\"\n";
OS << "#endif\n\n";
OS << "#ifdef __cplusplus\n";
OS << "extern \"C\" {\n";
OS << "#endif\n\n";
OS << "#pragma clang riscv intrinsic vector\n\n";
printHeaderCode(OS);
auto printType = [&](auto T) {
OS << "typedef " << T->getClangBuiltinStr() << " " << T->getTypeStr()
<< ";\n";
};
constexpr int Log2LMULs[] = {-3, -2, -1, 0, 1, 2, 3};
for (int Log2LMUL : Log2LMULs) {
auto T = RVVType::computeType(BasicType::Int8, Log2LMUL,
PrototypeDescriptor::Mask);
if (T)
printType(T.value());
}
for (char I : StringRef("csil")) {
BasicType BT = ParseBasicType(I);
for (int Log2LMUL : Log2LMULs) {
auto T = RVVType::computeType(BT, Log2LMUL, PrototypeDescriptor::Vector);
if (T) {
printType(T.value());
auto UT = RVVType::computeType(
BT, Log2LMUL,
PrototypeDescriptor(BaseTypeModifier::Vector,
VectorTypeModifier::NoModifier,
TypeModifier::UnsignedInteger));
printType(UT.value());
}
}
}
OS << "#if defined(__riscv_zvfh)\n";
for (int Log2LMUL : Log2LMULs) {
auto T = RVVType::computeType(BasicType::Float16, Log2LMUL,
PrototypeDescriptor::Vector);
if (T)
printType(T.value());
}
OS << "#endif\n";
OS << "#if (__riscv_v_elen_fp >= 32)\n";
for (int Log2LMUL : Log2LMULs) {
auto T = RVVType::computeType(BasicType::Float32, Log2LMUL,
PrototypeDescriptor::Vector);
if (T)
printType(T.value());
}
OS << "#endif\n";
OS << "#if (__riscv_v_elen_fp >= 64)\n";
for (int Log2LMUL : Log2LMULs) {
auto T = RVVType::computeType(BasicType::Float64, Log2LMUL,
PrototypeDescriptor::Vector);
if (T)
printType(T.value());
}
OS << "#endif\n\n";
OS << "#define __riscv_v_intrinsic_overloading 1\n";
OS << "\n#ifdef __cplusplus\n";
OS << "}\n";
OS << "#endif // __cplusplus\n";
OS << "#endif // __RISCV_VECTOR_H\n";
}
void RVVEmitter::createBuiltins(raw_ostream &OS) {
std::vector<std::unique_ptr<RVVIntrinsic>> Defs;
createRVVIntrinsics(Defs);
StringMap<RVVIntrinsic *> BuiltinMap;
OS << "#if defined(TARGET_BUILTIN) && !defined(RISCVV_BUILTIN)\n";
OS << "#define RISCVV_BUILTIN(ID, TYPE, ATTRS) TARGET_BUILTIN(ID, TYPE, "
"ATTRS, \"zve32x\")\n";
OS << "#endif\n";
for (auto &Def : Defs) {
auto P =
BuiltinMap.insert(std::make_pair(Def->getBuiltinName(), Def.get()));
if (!P.second) {
if (P.first->second->hasBuiltinAlias() != Def->hasBuiltinAlias())
PrintFatalError("Builtin with same name has different hasAutoDef");
else if (!Def->hasBuiltinAlias() &&
P.first->second->getBuiltinTypeStr() != Def->getBuiltinTypeStr())
PrintFatalError("Builtin with same name has different type string");
continue;
}
OS << "RISCVV_BUILTIN(__builtin_rvv_" << Def->getBuiltinName() << ",\"";
if (!Def->hasBuiltinAlias())
OS << Def->getBuiltinTypeStr();
OS << "\", \"n\")\n";
}
OS << "#undef RISCVV_BUILTIN\n";
}
void RVVEmitter::createCodeGen(raw_ostream &OS) {
std::vector<std::unique_ptr<RVVIntrinsic>> Defs;
createRVVIntrinsics(Defs);
llvm::stable_sort(Defs, [](const std::unique_ptr<RVVIntrinsic> &A,
const std::unique_ptr<RVVIntrinsic> &B) {
return A->getIRName() < B->getIRName();
});
StringMap<RVVIntrinsic *> BuiltinMap;
RVVIntrinsic *PrevDef = Defs.begin()->get();
for (auto &Def : Defs) {
StringRef CurIRName = Def->getIRName();
if (CurIRName != PrevDef->getIRName() ||
(Def->getManualCodegen() != PrevDef->getManualCodegen())) {
emitCodeGenSwitchBody(PrevDef, OS);
}
PrevDef = Def.get();
auto P =
BuiltinMap.insert(std::make_pair(Def->getBuiltinName(), Def.get()));
if (P.second) {
OS << "case RISCVVector::BI__builtin_rvv_" << Def->getBuiltinName()
<< ":\n";
continue;
}
if (P.first->second->getIRName() != Def->getIRName())
PrintFatalError("Builtin with same name has different IRName");
else if (P.first->second->getManualCodegen() != Def->getManualCodegen())
PrintFatalError("Builtin with same name has different ManualCodegen");
else if (P.first->second->getNF() != Def->getNF())
PrintFatalError("Builtin with same name has different NF");
else if (P.first->second->isMasked() != Def->isMasked())
PrintFatalError("Builtin with same name has different isMasked");
else if (P.first->second->hasVL() != Def->hasVL())
PrintFatalError("Builtin with same name has different hasVL");
else if (P.first->second->getPolicyScheme() != Def->getPolicyScheme())
PrintFatalError("Builtin with same name has different getPolicyScheme");
else if (P.first->second->getIntrinsicTypes() != Def->getIntrinsicTypes())
PrintFatalError("Builtin with same name has different IntrinsicTypes");
}
emitCodeGenSwitchBody(Defs.back().get(), OS);
OS << "\n";
}
void RVVEmitter::createRVVIntrinsics(
std::vector<std::unique_ptr<RVVIntrinsic>> &Out,
std::vector<SemaRecord> *SemaRecords) {
std::vector<Record *> RV = Records.getAllDerivedDefinitions("RVVBuiltin");
for (auto *R : RV) {
StringRef Name = R->getValueAsString("Name");
StringRef SuffixProto = R->getValueAsString("Suffix");
StringRef OverloadedName = R->getValueAsString("OverloadedName");
StringRef OverloadedSuffixProto = R->getValueAsString("OverloadedSuffix");
StringRef Prototypes = R->getValueAsString("Prototype");
StringRef TypeRange = R->getValueAsString("TypeRange");
bool HasMasked = R->getValueAsBit("HasMasked");
bool HasMaskedOffOperand = R->getValueAsBit("HasMaskedOffOperand");
bool HasVL = R->getValueAsBit("HasVL");
Record *MPSRecord = R->getValueAsDef("MaskedPolicyScheme");
auto MaskedPolicyScheme =
static_cast<PolicyScheme>(MPSRecord->getValueAsInt("Value"));
Record *UMPSRecord = R->getValueAsDef("UnMaskedPolicyScheme");
auto UnMaskedPolicyScheme =
static_cast<PolicyScheme>(UMPSRecord->getValueAsInt("Value"));
bool HasUnMaskedOverloaded = R->getValueAsBit("HasUnMaskedOverloaded");
std::vector<int64_t> Log2LMULList = R->getValueAsListOfInts("Log2LMUL");
bool HasBuiltinAlias = R->getValueAsBit("HasBuiltinAlias");
StringRef ManualCodegen = R->getValueAsString("ManualCodegen");
StringRef MaskedManualCodegen = R->getValueAsString("MaskedManualCodegen");
std::vector<int64_t> IntrinsicTypes =
R->getValueAsListOfInts("IntrinsicTypes");
std::vector<StringRef> RequiredFeatures =
R->getValueAsListOfStrings("RequiredFeatures");
StringRef IRName = R->getValueAsString("IRName");
StringRef MaskedIRName = R->getValueAsString("MaskedIRName");
unsigned NF = R->getValueAsInt("NF");
SmallVector<PrototypeDescriptor> BasicPrototype =
parsePrototypes(Prototypes);
SmallVector<PrototypeDescriptor> SuffixDesc = parsePrototypes(SuffixProto);
SmallVector<PrototypeDescriptor> OverloadedSuffixDesc =
parsePrototypes(OverloadedSuffixProto);
auto Prototype = RVVIntrinsic::computeBuiltinTypes(
BasicPrototype, false, false,
HasVL, NF);
auto MaskedPrototype = RVVIntrinsic::computeBuiltinTypes(
BasicPrototype, true, HasMaskedOffOperand, HasVL, NF);
for (char I : TypeRange) {
for (int Log2LMUL : Log2LMULList) {
BasicType BT = ParseBasicType(I);
Optional<RVVTypes> Types =
RVVType::computeTypes(BT, Log2LMUL, NF, Prototype);
if (!Types)
continue;
auto SuffixStr = RVVIntrinsic::getSuffixStr(BT, Log2LMUL, SuffixDesc);
auto OverloadedSuffixStr =
RVVIntrinsic::getSuffixStr(BT, Log2LMUL, OverloadedSuffixDesc);
Out.push_back(std::make_unique<RVVIntrinsic>(
Name, SuffixStr, OverloadedName, OverloadedSuffixStr, IRName,
false, false, HasVL,
UnMaskedPolicyScheme, HasUnMaskedOverloaded, HasBuiltinAlias,
ManualCodegen, *Types, IntrinsicTypes, RequiredFeatures, NF));
if (HasMasked) {
Optional<RVVTypes> MaskTypes =
RVVType::computeTypes(BT, Log2LMUL, NF, MaskedPrototype);
Out.push_back(std::make_unique<RVVIntrinsic>(
Name, SuffixStr, OverloadedName, OverloadedSuffixStr,
MaskedIRName,
true, HasMaskedOffOperand, HasVL, MaskedPolicyScheme,
HasUnMaskedOverloaded, HasBuiltinAlias, MaskedManualCodegen,
*MaskTypes, IntrinsicTypes, RequiredFeatures, NF));
}
} }
if (Name == "vsetvli" || Name == "vsetvlimax")
continue;
if (!SemaRecords)
continue;
SemaRecord SR;
SR.Name = Name.str();
SR.OverloadedName = OverloadedName.str();
BasicType TypeRangeMask = BasicType::Unknown;
for (char I : TypeRange)
TypeRangeMask |= ParseBasicType(I);
SR.TypeRangeMask = static_cast<unsigned>(TypeRangeMask);
unsigned Log2LMULMask = 0;
for (int Log2LMUL : Log2LMULList)
Log2LMULMask |= 1 << (Log2LMUL + 3);
SR.Log2LMULMask = Log2LMULMask;
SR.RequiredExtensions = 0;
for (auto RequiredFeature : RequiredFeatures) {
RVVRequire RequireExt = StringSwitch<RVVRequire>(RequiredFeature)
.Case("RV64", RVV_REQ_RV64)
.Case("FullMultiply", RVV_REQ_FullMultiply)
.Default(RVV_REQ_None);
assert(RequireExt != RVV_REQ_None && "Unrecognized required feature?");
SR.RequiredExtensions |= RequireExt;
}
SR.NF = NF;
SR.HasMasked = HasMasked;
SR.HasVL = HasVL;
SR.HasMaskedOffOperand = HasMaskedOffOperand;
SR.Prototype = std::move(BasicPrototype);
SR.Suffix = parsePrototypes(SuffixProto);
SR.OverloadedSuffix = parsePrototypes(OverloadedSuffixProto);
SemaRecords->push_back(SR);
}
}
void RVVEmitter::printHeaderCode(raw_ostream &OS) {
std::vector<Record *> RVVHeaders =
Records.getAllDerivedDefinitions("RVVHeader");
for (auto *R : RVVHeaders) {
StringRef HeaderCodeStr = R->getValueAsString("HeaderCode");
OS << HeaderCodeStr.str();
}
}
void RVVEmitter::createRVVIntrinsicRecords(std::vector<RVVIntrinsicRecord> &Out,
SemaSignatureTable &SST,
ArrayRef<SemaRecord> SemaRecords) {
SST.init(SemaRecords);
for (const auto &SR : SemaRecords) {
Out.emplace_back(RVVIntrinsicRecord());
RVVIntrinsicRecord &R = Out.back();
R.Name = SR.Name.c_str();
R.OverloadedName = SR.OverloadedName.c_str();
R.PrototypeIndex = SST.getIndex(SR.Prototype);
R.SuffixIndex = SST.getIndex(SR.Suffix);
R.OverloadedSuffixIndex = SST.getIndex(SR.OverloadedSuffix);
R.PrototypeLength = SR.Prototype.size();
R.SuffixLength = SR.Suffix.size();
R.OverloadedSuffixSize = SR.OverloadedSuffix.size();
R.RequiredExtensions = SR.RequiredExtensions;
R.TypeRangeMask = SR.TypeRangeMask;
R.Log2LMULMask = SR.Log2LMULMask;
R.NF = SR.NF;
R.HasMasked = SR.HasMasked;
R.HasVL = SR.HasVL;
R.HasMaskedOffOperand = SR.HasMaskedOffOperand;
assert(R.PrototypeIndex !=
static_cast<uint16_t>(SemaSignatureTable::INVALID_INDEX));
assert(R.SuffixIndex !=
static_cast<uint16_t>(SemaSignatureTable::INVALID_INDEX));
assert(R.OverloadedSuffixIndex !=
static_cast<uint16_t>(SemaSignatureTable::INVALID_INDEX));
}
}
void RVVEmitter::createSema(raw_ostream &OS) {
std::vector<std::unique_ptr<RVVIntrinsic>> Defs;
std::vector<RVVIntrinsicRecord> RVVIntrinsicRecords;
SemaSignatureTable SST;
std::vector<SemaRecord> SemaRecords;
createRVVIntrinsics(Defs, &SemaRecords);
createRVVIntrinsicRecords(RVVIntrinsicRecords, SST, SemaRecords);
OS << "#ifdef DECL_SIGNATURE_TABLE\n";
SST.print(OS);
OS << "#endif\n";
OS << "#ifdef DECL_INTRINSIC_RECORDS\n";
for (const RVVIntrinsicRecord &Record : RVVIntrinsicRecords)
OS << Record;
OS << "#endif\n";
}
namespace clang {
void EmitRVVHeader(RecordKeeper &Records, raw_ostream &OS) {
RVVEmitter(Records).createHeader(OS);
}
void EmitRVVBuiltins(RecordKeeper &Records, raw_ostream &OS) {
RVVEmitter(Records).createBuiltins(OS);
}
void EmitRVVBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
RVVEmitter(Records).createCodeGen(OS);
}
void EmitRVVBuiltinSema(RecordKeeper &Records, raw_ostream &OS) {
RVVEmitter(Records).createSema(OS);
}
}