// Mask immediates for MMA instructions (2, 4 and 8 bits).
def Msk2Imm : ImmLeaf<i32, [{ return isUInt<2>(Imm); }]>;
def Msk4Imm : ImmLeaf<i32, [{ return isUInt<4>(Imm); }]>;
def Msk8Imm : ImmLeaf<i32, [{ return isUInt<8>(Imm); }]>;
def MMA : Predicate<"Subtarget->hasMMA()">;
// Multiclass definitions for MMA accumulator instructions.
// ----------------------------------------------------------------------------
// Defines 2 unmasked instructions where the xo field for acc/non-acc version
// is even/odd.
multiclass ACC_UM_XOEO<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
string asmstr> {
let Predicates = [MMA] in {
def NAME :
XX3Form_AT3_XAB6<opcode, !or(xo, 0x01), (outs acc:$AT), IOL,
!strconcat(asmbase#" ", asmstr), IIC_VecFP, []>,
RegConstraint<"@earlyclobber $AT">;
def PP :
XX3Form_AT3_XAB6<opcode, xo, (outs acc:$AT), !con((ins acc:$ATi), IOL),
!strconcat(asmbase#"pp ", asmstr), IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
}
}
// Defines 4 instructions, masked/unmasked with masks 8, 4, 4 bits.
// The XO field for acc/non-acc version is even/odd.
multiclass ACC_UM_M844_XOEO<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
string asmstr> {
defm NAME : ACC_UM_XOEO<opcode, xo, IOL, asmbase, asmstr>;
let Predicates = [MMA, PrefixInstrs] in {
def PM#NAME :
MMIRR_XX3Form_XY4P8_XAB6<
opcode, !or(xo, 0x01), (outs acc:$AT),
!con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u8imm:$PMSK)),
!strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK, $PMSK"),
IIC_VecFP, []>,
RegConstraint<"@earlyclobber $AT">;
def PM#NAME#PP :
MMIRR_XX3Form_XY4P8_XAB6<
opcode, xo, (outs acc:$AT),
!con((ins acc:$ATi),
!con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u8imm:$PMSK))),
!strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK, $PMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
}
}
// Defines 4 instructions, masked/unmasked with masks 4, 4, 4 bits.
// The XO field for acc/non-acc version is even/odd.
multiclass ACC_UM_M444_XOEO<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
string asmstr> {
defm NAME : ACC_UM_XOEO<opcode, xo, IOL, asmbase, asmstr>;
let Predicates = [MMA, PrefixInstrs] in {
def PM#NAME :
MMIRR_XX3Form_XYP4_XAB6<
opcode, !or(xo, 0x01), (outs acc:$AT),
!con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u4imm:$PMSK)),
!strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK, $PMSK"),
IIC_VecFP, []>,
RegConstraint<"@earlyclobber $AT">;
def PM#NAME#PP :
MMIRR_XX3Form_XYP4_XAB6<
opcode, xo, (outs acc:$AT),
!con((ins acc:$ATi),
!con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u4imm:$PMSK))),
!strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK, $PMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
}
}
// Defines 4 instructions, masked/unmasked with masks 2, 4, 4 bits.
// The XO field for acc/non-acc version is even/odd.
multiclass ACC_UM_M244_XOEO<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
string asmstr> {
defm NAME : ACC_UM_XOEO<opcode, xo, IOL, asmbase, asmstr>;
let Predicates = [MMA, PrefixInstrs] in {
def PM#NAME :
MMIRR_XX3Form_XY4P2_XAB6<
opcode, !or(xo, 0x01), (outs acc:$AT),
!con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK)),
!strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK, $PMSK"),
IIC_VecFP, []>,
RegConstraint<"@earlyclobber $AT">;
def PM#NAME#PP :
MMIRR_XX3Form_XY4P2_XAB6<
opcode, xo, (outs acc:$AT),
!con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
!strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK, $PMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
}
}
// Defines 4 instructions, masked/unmasked with masks 2, 4, 4 bits.
// Upper nibble of XO field for acc/non-acc version is 0x4/0x6.
multiclass ACC_UM_M244_XO46<bits<6> opcode, bits<8> xo, dag IOL, string asmbase,
string asmstr> {
let Predicates = [MMA] in {
def NAME :
XX3Form_AT3_XAB6<opcode, xo, (outs acc:$AT), IOL,
!strconcat(asmbase#" ", asmstr), IIC_VecFP, []>,
RegConstraint<"@earlyclobber $AT">;
def PP :
XX3Form_AT3_XAB6<
opcode, !or(xo, 0x20), (outs acc:$AT), !con((ins acc:$ATi), IOL),
!strconcat(asmbase#"pp ", asmstr), IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
}
let Predicates = [MMA, PrefixInstrs] in {
def PM#NAME :
MMIRR_XX3Form_XY4P2_XAB6<
opcode, xo, (outs acc:$AT),
!con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK)),
!strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK, $PMSK"),
IIC_VecFP, []>,
RegConstraint<"@earlyclobber $AT">;
def PM#NAME#PP :
MMIRR_XX3Form_XY4P2_XAB6<
opcode, !or(xo, 0x20), (outs acc:$AT),
!con((ins acc:$ATi),
!con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
!strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK, $PMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
}
}
// Defines 10 instructions, operand negating, unmasked, masked with 2, 4, 4
// bits. Upper nibble are masked with 0x8, 0x4, 0xC for negating operands.
multiclass ACC_NEG_UM_M244_XOM84C<bits<6> opcode, bits<8> xo, dag IOL,
string asmbase, string asmstr> {
defm NAME : ACC_UM_M244_XOEO<opcode, xo, IOL, asmbase, asmstr>;
let Predicates = [MMA] in {
def PN : XX3Form_AT3_XAB6<
opcode, !or(xo, 0x80), (outs acc:$AT), !con((ins acc:$ATi), IOL),
!strconcat(asmbase#"pn ", asmstr), IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def NP : XX3Form_AT3_XAB6<
opcode, !or(xo, 0x40), (outs acc:$AT), !con((ins acc:$ATi), IOL),
!strconcat(asmbase#"np ", asmstr), IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def NN : XX3Form_AT3_XAB6<
opcode, !or(xo, 0xC0), (outs acc:$AT), !con((ins acc:$ATi), IOL),
!strconcat(asmbase#"nn ", asmstr), IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
}
let Predicates = [MMA, PrefixInstrs] in {
def PM#NAME#PN :
MMIRR_XX3Form_XY4P2_XAB6<
opcode, !or(xo, 0x80), (outs acc:$AT),
!con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
!strconcat("pm"#asmbase#"pn ", asmstr#", $XMSK, $YMSK, $PMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def PM#NAME#NP :
MMIRR_XX3Form_XY4P2_XAB6<
opcode, !or(xo, 0x40), (outs acc:$AT),
!con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
!strconcat("pm"#asmbase#"np ", asmstr#", $XMSK, $YMSK, $PMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def PM#NAME#NN :
MMIRR_XX3Form_XY4P2_XAB6<
opcode, !or(xo, 0xC0), (outs acc:$AT),
!con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK, u2imm:$PMSK))),
!strconcat("pm"#asmbase#"nn ", asmstr#", $XMSK, $YMSK, $PMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
}
}
// Defines 5 instructions, unmasked, operand negating.
// Upper nibble are masked with 0x8, 0x4, 0xC for negating operands.
multiclass ACC_NEG_UM_XOM84C<bits<6> opcode, bits<8> xo, dag IOL,
string asmbase, string asmstr> {
defm NAME : ACC_UM_XOEO<opcode, xo, IOL, asmbase, asmstr>;
let Predicates = [MMA] in {
def PN : XX3Form_AT3_XAB6<opcode, !or(xo, 0x80), (outs acc:$AT),
!con((ins acc:$ATi), IOL),
!strconcat(asmbase#"pn ", asmstr), IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def NP : XX3Form_AT3_XAB6<opcode, !or(xo, 0x40), (outs acc:$AT),
!con((ins acc:$ATi), IOL),
!strconcat(asmbase#"np ", asmstr), IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def NN : XX3Form_AT3_XAB6<opcode, !or(xo, 0xC0), (outs acc:$AT),
!con((ins acc:$ATi), IOL),
!strconcat(asmbase#"nn ", asmstr), IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
}
}
// Defines 10 instructions, operand negating, unmasked, masked with 4, 4 bits.
// Upper nibble are masked with 0x8, 0x4, 0xC for negating operands.
multiclass ACC_NEG_UM_M44_XOM84C<bits<6> opcode, bits<8> xo, dag IOL,
string asmbase, string asmstr> {
defm NAME : ACC_NEG_UM_XOM84C<opcode, xo, IOL, asmbase, asmstr>;
let Predicates = [MMA, PrefixInstrs] in {
def PM#NAME :
MMIRR_XX3Form_XY4_XAB6<
opcode, !or(xo, 0x01), (outs acc:$AT),
!con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK)),
!strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK"),
IIC_VecFP, []>,
RegConstraint<"@earlyclobber $AT">;
def PM#NAME#PP :
MMIRR_XX3Form_XY4_XAB6<
opcode, xo, (outs acc:$AT),
!con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK))),
!strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def PM#NAME#PN :
MMIRR_XX3Form_XY4_XAB6<
opcode, !or(xo, 0x80), (outs acc:$AT),
!con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK))),
!strconcat("pm"#asmbase#"pn ", asmstr#", $XMSK, $YMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def PM#NAME#NP :
MMIRR_XX3Form_XY4_XAB6<
opcode, !or(xo, 0x40), (outs acc:$AT),
!con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK))),
!strconcat("pm"#asmbase#"np ", asmstr#", $XMSK, $YMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def PM#NAME#NN :
MMIRR_XX3Form_XY4_XAB6<
opcode, !or(xo, 0xC0), (outs acc:$AT),
!con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u4imm:$YMSK))),
!strconcat("pm"#asmbase#"nn ", asmstr#", $XMSK, $YMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
}
}
// Defines 10 instructions, operand negating, unmasked, masked with 4, 2 bits.
// Upper nibble are masked with 0x8, 0x4, 0xC for negating operands.
multiclass ACC_NEG_UM_M42_XOM84C<bits<6> opcode, bits<8> xo, dag IOL,
string asmbase, string asmstr> {
defm NAME : ACC_NEG_UM_XOM84C<opcode, xo, IOL, asmbase, asmstr>;
let Predicates = [MMA, PrefixInstrs] in {
def PM#NAME :
MMIRR_XX3Form_X4Y2_XAB6<
opcode, !or(xo, 0x01), (outs acc:$AT),
!con(IOL, (ins u4imm:$XMSK, u2imm:$YMSK)),
!strconcat("pm"#asmbase#" ", asmstr#", $XMSK, $YMSK"),
IIC_VecFP, []>,
RegConstraint<"@earlyclobber $AT">;
def PM#NAME#PP :
MMIRR_XX3Form_X4Y2_XAB6<
opcode, xo, (outs acc:$AT),
!con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u2imm:$YMSK))),
!strconcat("pm"#asmbase#"pp ", asmstr#", $XMSK, $YMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def PM#NAME#PN :
MMIRR_XX3Form_X4Y2_XAB6<
opcode, !or(xo, 0x80), (outs acc:$AT),
!con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u2imm:$YMSK))),
!strconcat("pm"#asmbase#"pn ", asmstr#", $XMSK, $YMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def PM#NAME#NP :
MMIRR_XX3Form_X4Y2_XAB6<
opcode, !or(xo, 0x40), (outs acc:$AT),
!con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u2imm:$YMSK))),
!strconcat("pm"#asmbase#"np ", asmstr#", $XMSK, $YMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def PM#NAME#NN :
MMIRR_XX3Form_X4Y2_XAB6<
opcode, !or(xo, 0xC0), (outs acc:$AT),
!con((ins acc:$ATi), !con(IOL, (ins u4imm:$XMSK, u2imm:$YMSK))),
!strconcat("pm"#asmbase#"nn ", asmstr#", $XMSK, $YMSK"),
IIC_VecFP, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
}
}
// End of class definitions.
//-----------------------------------------------------------------------------
let Predicates = [MMA] in {
def XXMFACC :
XForm_AT3<31, 0, 177, (outs acc:$ASo), (ins acc:$AS), "xxmfacc $AS",
IIC_VecGeneral,
[(set v512i1:$ASo, (int_ppc_mma_xxmfacc v512i1:$AS))]>,
RegConstraint<"$ASo = $AS">, NoEncode<"$ASo">;
def XXMTACC :
XForm_AT3<31, 1, 177, (outs acc:$AT), (ins acc:$ATi), "xxmtacc $AT",
IIC_VecGeneral,
[(set v512i1:$AT, (int_ppc_mma_xxmtacc v512i1:$ATi))]>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def KILL_PAIR : PPCPostRAExpPseudo<(outs vsrprc:$XTp), (ins vsrprc:$XSp),
"#KILL_PAIR", []>,
RegConstraint<"$XTp = $XSp">;
def BUILD_UACC : PPCPostRAExpPseudo<(outs acc:$AT), (ins uacc:$AS),
"#BUILD_UACC $AT, $AS", []>;
// We define XXSETACCZ as rematerializable to undo CSE of that intrinsic in
// the backend. We avoid CSE here because it generates a copy of the acc
// register and this copy is more expensive than calling the intrinsic again.
let isAsCheapAsAMove = 1, isReMaterializable = 1 in {
def XXSETACCZ :
XForm_AT3<31, 3, 177, (outs acc:$AT), (ins), "xxsetaccz $AT", IIC_VecGeneral,
[(set v512i1:$AT, (int_ppc_mma_xxsetaccz))]>;
}
def XVI8GER4SPP :
XX3Form_AT3_XAB6<59, 99, (outs acc:$AT), (ins acc:$ATi, vsrc:$XA, vsrc:$XB),
"xvi8ger4spp $AT, $XA, $XB", IIC_VecGeneral, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
let mayStore = 1 in {
def SPILL_ACC: PPCEmitTimePseudo<(outs), (ins acc:$AT, memrix16:$dst),
"#SPILL_ACC", []>;
def SPILL_UACC: PPCEmitTimePseudo<(outs), (ins uacc:$AT, memrix16:$dst),
"#SPILL_UACC", []>;
}
let mayLoad = 1, hasSideEffects = 0 in {
def RESTORE_ACC: PPCEmitTimePseudo<(outs acc:$AT), (ins memrix16:$src),
"#RESTORE_ACC", []>;
def RESTORE_UACC: PPCEmitTimePseudo<(outs uacc:$AT), (ins memrix16:$src),
"#RESTORE_UACC", []>;
}
}
let Predicates = [MMA, PrefixInstrs] in {
def PMXVI8GER4SPP :
MMIRR_XX3Form_XYP4_XAB6<59, 99, (outs acc:$AT),
(ins acc:$ATi, vsrc:$XA,vsrc:$XB, u4imm:$XMSK,
u4imm:$YMSK, u4imm:$PMSK),
"pmxvi8ger4spp $AT, $XA, $XB, $XMSK, $YMSK, $PMSK",
IIC_VecGeneral, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
}
// MMA accumulating/non-accumulating instructions.
//------------------------------------------------------------------------------
// XVBF16GER2, XVBF16GER2PP, XVBF16GER2PN, XVBF16GER2NP, XVBF16GER2NN
// PMXVBF16GER2, PMXVBF16GER2PP, PMXVBF16GER2PN, PMXVBF16GER2NP, PMXVBF16GER2NN
defm XVBF16GER2 : ACC_NEG_UM_M244_XOM84C<59, 50, (ins vsrc:$XA, vsrc:$XB),
"xvbf16ger2", "$AT, $XA, $XB">;
// XVI4GER8, XVI4GER8PP, PMXVI4GER8, PMXVI4GER8PP
defm XVI4GER8 : ACC_UM_M844_XOEO<59, 34, (ins vsrc:$XA, vsrc:$XB),
"xvi4ger8", "$AT, $XA, $XB">;
// XVI8GER4, XVI8GER4PP, PMXVI8GER4, PMXVI8GER4PP
defm XVI8GER4 : ACC_UM_M444_XOEO<59, 2, (ins vsrc:$XA, vsrc:$XB),
"xvi8ger4", "$AT, $XA, $XB">;
// XVI16GER2, XVI16GER2PP, PMXVI16GER2, PMXVI16GER2PP
defm XVI16GER2 : ACC_UM_M244_XO46<59, 75, (ins vsrc:$XA, vsrc:$XB),
"xvi16ger2", "$AT, $XA, $XB">;
// XVI16GER2S, XVI16GER2SPP, PMXVI16GER2S, PMXVI16GER2SPP
defm XVI16GER2S : ACC_UM_M244_XOEO<59, 42, (ins vsrc:$XA, vsrc:$XB),
"xvi16ger2s", "$AT, $XA, $XB">;
// XVF16GER2, XVF16GER2PP, XVF16GER2PN, XVF16GER2NP, XVF16GER2NN
// PMXVF16GER2, PMXVF16GER2PP, PMXVF16GER2PN, PMXVF16GER2NP, PMXVF16GER2NN
defm XVF16GER2 : ACC_NEG_UM_M244_XOM84C<59, 18, (ins vsrc:$XA, vsrc:$XB),
"xvf16ger2", "$AT, $XA, $XB">;
// XVF32GER, XVF32GERPP, XVF32GERPN, XVF32GERNP, XVF32GERPP
// PMXVF32GER, PMXVF32GERPP, PMXVF32GERPN, PMXVF32GERNP, PMXVF32GERPP
defm XVF32GER : ACC_NEG_UM_M44_XOM84C<59, 26, (ins vsrc:$XA, vsrc:$XB),
"xvf32ger", "$AT, $XA, $XB">;
// XVF64GER, XVF64GERPP, XVF64GERPN, XVF64GERNP, XVF64GERNN
// PMXVF64GER, PMXVF64GERPP, PMXVF64GERPN, PMXVF64GERNP, PMXVF64GERNN
defm XVF64GER : ACC_NEG_UM_M42_XOM84C<59, 58, (ins vsrpevenrc:$XA, vsrc:$XB),
"xvf64ger", "$AT, $XA, $XB">;
//------------------------------------------------------------------------------
// MMA Intrinsics
let Predicates = [MMA] in {
def : Pat<(v512i1 (int_ppc_mma_xvi4ger8 v16i8:$XA, v16i8:$XB)),
(XVI4GER8 RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvi4ger8pp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVI4GER8PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvi8ger4 v16i8:$XA, v16i8:$XB)),
(XVI8GER4 RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvi8ger4pp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVI8GER4PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvi16ger2s v16i8:$XA, v16i8:$XB)),
(XVI16GER2S RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvi16ger2spp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVI16GER2SPP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf16ger2 v16i8:$XA, v16i8:$XB)),
(XVF16GER2 RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf16ger2pp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVF16GER2PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf16ger2pn v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVF16GER2PN $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf16ger2np v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVF16GER2NP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf16ger2nn v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVF16GER2NN $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf32ger v16i8:$XA, v16i8:$XB)),
(XVF32GER RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf32gerpp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVF32GERPP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf32gerpn v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVF32GERPN $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf32gernp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVF32GERNP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf32gernn v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVF32GERNN $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf64ger v256i1:$XA, v16i8:$XB)),
(XVF64GER $XA, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf64gerpp v512i1:$ATi, v256i1:$XA, v16i8:$XB)),
(XVF64GERPP $ATi, $XA, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf64gerpn v512i1:$ATi, v256i1:$XA, v16i8:$XB)),
(XVF64GERPN $ATi, $XA, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf64gernp v512i1:$ATi, v256i1:$XA, v16i8:$XB)),
(XVF64GERNP $ATi, $XA, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvf64gernn v512i1:$ATi, v256i1:$XA, v16i8:$XB)),
(XVF64GERNN $ATi, $XA, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvbf16ger2 v16i8:$XA, v16i8:$XB)),
(XVBF16GER2 RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvbf16ger2pp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVBF16GER2PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvbf16ger2pn v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVBF16GER2PN $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvbf16ger2np v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVBF16GER2NP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvbf16ger2nn v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVBF16GER2NN $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvi16ger2 v16i8:$XA, v16i8:$XB)),
(XVI16GER2 RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvi16ger2pp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVI16GER2PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
def : Pat<(v512i1 (int_ppc_mma_xvi8ger4spp v512i1:$ATi, v16i8:$XA, v16i8:$XB)),
(XVI8GER4SPP $ATi, RCCp.AToVSRC, RCCp.BToVSRC)>;
}
// MMA Intrinsics
let Predicates = [MMA, PrefixInstrs] in {
def : Pat<(v512i1 (int_ppc_mma_pmxvi4ger8 v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk8Imm:$PMSK)),
(PMXVI4GER8 RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk8Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvi4ger8pp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk8Imm:$PMSK)),
(PMXVI4GER8PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk8Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvi8ger4 v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk4Imm:$PMSK)),
(PMXVI8GER4 RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk4Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvi8ger4pp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk4Imm:$PMSK)),
(PMXVI8GER4PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk4Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvi16ger2s v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)),
(PMXVI16GER2S RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvi16ger2spp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk2Imm:$PMSK)),
(PMXVI16GER2SPP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf16ger2 v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)),
(PMXVF16GER2 RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf16ger2pp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk2Imm:$PMSK)),
(PMXVF16GER2PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf16ger2pn v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk2Imm:$PMSK)),
(PMXVF16GER2PN $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf16ger2np v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk2Imm:$PMSK)),
(PMXVF16GER2NP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf16ger2nn v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk2Imm:$PMSK)),
(PMXVF16GER2NN $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf32ger v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
Msk4Imm:$YMSK)),
(PMXVF32GER RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf32gerpp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK)),
(PMXVF32GERPP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf32gerpn v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK)),
(PMXVF32GERPN $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf32gernp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK)),
(PMXVF32GERNP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf32gernn v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK)),
(PMXVF32GERNN $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf64ger v256i1:$XA, v16i8:$XB, Msk4Imm:$XMSK,
Msk2Imm:$YMSK)),
(PMXVF64GER $XA, RCCp.BToVSRC, Msk4Imm:$XMSK, Msk2Imm:$YMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf64gerpp v512i1:$ATi, v256i1:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk2Imm:$YMSK)),
(PMXVF64GERPP $ATi, $XA, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk2Imm:$YMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf64gerpn v512i1:$ATi, v256i1:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk2Imm:$YMSK)),
(PMXVF64GERPN $ATi, $XA, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk2Imm:$YMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf64gernp v512i1:$ATi, v256i1:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk2Imm:$YMSK)),
(PMXVF64GERNP $ATi, $XA, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk2Imm:$YMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvf64gernn v512i1:$ATi, v256i1:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk2Imm:$YMSK)),
(PMXVF64GERNN $ATi, $XA, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk2Imm:$YMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvbf16ger2 v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)),
(PMXVBF16GER2 RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvbf16ger2pp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk2Imm:$PMSK)),
(PMXVBF16GER2PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvbf16ger2pn v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk2Imm:$PMSK)),
(PMXVBF16GER2PN $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvbf16ger2np v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk2Imm:$PMSK)),
(PMXVBF16GER2NP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvbf16ger2nn v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk2Imm:$PMSK)),
(PMXVBF16GER2NN $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvi16ger2 v16i8:$XA, v16i8:$XB, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)),
(PMXVI16GER2 RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvi8ger4spp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk2Imm:$PMSK)),
(PMXVI8GER4SPP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
def : Pat<(v512i1 (int_ppc_mma_pmxvi16ger2pp v512i1:$ATi, v16i8:$XA, v16i8:$XB,
Msk4Imm:$XMSK, Msk4Imm:$YMSK,
Msk2Imm:$PMSK)),
(PMXVI16GER2PP $ATi, RCCp.AToVSRC, RCCp.BToVSRC, Msk4Imm:$XMSK,
Msk4Imm:$YMSK, Msk2Imm:$PMSK)>;
}
def ConcatsMMA {
dag VecsToVecPair0 =
(v256i1 (INSERT_SUBREG
(INSERT_SUBREG (IMPLICIT_DEF), $vs0, sub_vsx1),
$vs1, sub_vsx0));
dag VecsToVecPair1 =
(v256i1 (INSERT_SUBREG
(INSERT_SUBREG (IMPLICIT_DEF), $vs2, sub_vsx1),
$vs3, sub_vsx0));
dag VecsToVecQuad =
(BUILD_UACC (INSERT_SUBREG
(INSERT_SUBREG (v512i1 (IMPLICIT_DEF)),
(KILL_PAIR VecsToVecPair0), sub_pair0),
(KILL_PAIR VecsToVecPair1), sub_pair1));
}
def Extracts {
dag Pair0 = (v256i1 (EXTRACT_SUBREG $v, sub_pair0));
dag Pair1 = (v256i1 (EXTRACT_SUBREG $v, sub_pair1));
dag Vec0 = (v4i32 (EXTRACT_SUBREG Pair0, sub_vsx0));
dag Vec1 = (v4i32 (EXTRACT_SUBREG Pair0, sub_vsx1));
dag Vec2 = (v4i32 (EXTRACT_SUBREG Pair1, sub_vsx0));
dag Vec3 = (v4i32 (EXTRACT_SUBREG Pair1, sub_vsx1));
}
let Predicates = [MMA] in {
def : Pat<(v512i1 (PPCAccBuild v4i32:$vs1, v4i32:$vs0, v4i32:$vs3, v4i32:$vs2)),
(XXMTACC ConcatsMMA.VecsToVecQuad)>;
def : Pat<(v512i1 (int_ppc_mma_assemble_acc v16i8:$vs1, v16i8:$vs0,
v16i8:$vs3, v16i8:$vs2)),
(XXMTACC ConcatsMMA.VecsToVecQuad)>;
def : Pat<(v512i1 (PPCxxmfacc v512i1:$AS)), (XXMFACC acc:$AS)>;
def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 0)),
Extracts.Vec0>;
def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 1)),
Extracts.Vec1>;
def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 2)),
Extracts.Vec2>;
def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, 3)),
Extracts.Vec3>;
}