[SLP]Support copyables in the fmuladd multiplicand
Allow a non-fmuladd lane V to be modeled as fmuladd(V, 1.0, -0.0), which equals V for every V (V * 1.0 is exact, and V + -0.0 preserves signed zeros). The addend modeling is still preferred; the multiplicand is used as a fallback when the addend operand does not allow vectorization. The copyable position is kept as an operand index in InstructionsState, so other operands can be modeled later. Reviewers: hiraditya, RKSimon Pull Request: https://github.com/llvm/llvm-project/pull/212808
This commit is contained in:
Alexey Bataev
GitHub
parent
25aec49d40
commit
afd6d6eeb0
3 changed files with 67 additions and 36 deletions
|
|
@ -11338,9 +11338,12 @@ class InstructionsCompatibilityAnalysis {
|
|||
return {V, V};
|
||||
if (!S.isCopyableElement(V))
|
||||
return convertTo(cast<Instruction>(V), S).second;
|
||||
// fmuladd(0.0, -0.0, V) == V.
|
||||
if (RecurrenceDescriptor::isFMulAddIntrinsic(MainOp)) {
|
||||
Type *Ty = MainOp->getType();
|
||||
// fmuladd(V, 1.0, -0.0) == V.
|
||||
if (S.getCopyableOpIdx() == 0)
|
||||
return {V, ConstantFP::get(Ty, 1.0), ConstantFP::getNegativeZero(Ty)};
|
||||
// fmuladd(0.0, -0.0, V) == V.
|
||||
return {ConstantFP::getZero(Ty), ConstantFP::getNegativeZero(Ty), V};
|
||||
}
|
||||
assert(isSupportedMainOp(MainOp) && "Unsupported opcode");
|
||||
|
|
@ -11863,7 +11866,7 @@ public:
|
|||
return OrigS;
|
||||
return S;
|
||||
}
|
||||
// fmuladd is the only 3-operand copyable; the value sits in the addend.
|
||||
// fmuladd is the only 3-operand copyable.
|
||||
assert((Operands.size() == 2 ||
|
||||
(Operands.size() == 3 &&
|
||||
RecurrenceDescriptor::isFMulAddIntrinsic(MainOp))) &&
|
||||
|
|
@ -11916,8 +11919,9 @@ public:
|
|||
}
|
||||
}
|
||||
}
|
||||
// 2. Check, if operands can be vectorized. Skip for fmuladd; its addend
|
||||
// is checked below and may legitimately hold many instructions.
|
||||
// 2. Check, if operands can be vectorized. Skip for fmuladd; the
|
||||
// copyable operand is checked below and may legitimately hold many
|
||||
// instructions.
|
||||
if (Operands.size() == 2 &&
|
||||
count_if(Operands.back(), IsaPred<Instruction>) > 1)
|
||||
return OrigS;
|
||||
|
|
@ -11954,10 +11958,16 @@ public:
|
|||
count_if(Ops, [&](Value *V) { return OpS.isCopyableElement(V); });
|
||||
return CopyableNum <= VL.size() / 2;
|
||||
};
|
||||
// Check the addend for fmuladd, first operand otherwise.
|
||||
if (!CheckOperand(Operands.size() == 2 ? Operands.front()
|
||||
: Operands.back()))
|
||||
return OrigS;
|
||||
// Check the operand holding the copyable values.
|
||||
if (!CheckOperand(Operands[S.getCopyableOpIdx()])) {
|
||||
if (Operands.size() == 2)
|
||||
return OrigS;
|
||||
// Retry with the copyable modeled as the first multiplicand.
|
||||
S.setCopyableOpIdx(0);
|
||||
Operands = buildOperands(S, VL);
|
||||
if (!CheckOperand(Operands[S.getCopyableOpIdx()]))
|
||||
return OrigS;
|
||||
}
|
||||
|
||||
return S;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
#include "llvm/ADT/BitmaskEnum.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include "llvm/Analysis/IVDescriptors.h"
|
||||
#include "llvm/IR/Instruction.h"
|
||||
|
||||
#include <cstdint>
|
||||
|
|
@ -154,6 +155,9 @@ class InstructionsState {
|
|||
Instruction *AltOp = nullptr;
|
||||
/// Whether the instruction state represents copyable instructions.
|
||||
bool HasCopyables = false;
|
||||
/// Index of the operand modeling the copyable values: the addend for
|
||||
/// fmuladd (retried with a multiplicand), the first operand otherwise.
|
||||
unsigned CopyableOpIdx = 0;
|
||||
|
||||
public:
|
||||
Instruction *getMainOp() const {
|
||||
|
|
@ -218,7 +222,10 @@ public:
|
|||
InstructionsState() = delete;
|
||||
InstructionsState(Instruction *MainOp, Instruction *AltOp,
|
||||
bool HasCopyables = false)
|
||||
: MainOp(MainOp), AltOp(AltOp), HasCopyables(HasCopyables) {}
|
||||
: MainOp(MainOp), AltOp(AltOp), HasCopyables(HasCopyables),
|
||||
CopyableOpIdx(MainOp && RecurrenceDescriptor::isFMulAddIntrinsic(MainOp)
|
||||
? 2
|
||||
: 0) {}
|
||||
static InstructionsState invalid() { return {nullptr, nullptr}; }
|
||||
|
||||
/// Checks if the value is a copyable element.
|
||||
|
|
@ -240,6 +247,18 @@ public:
|
|||
assert(valid() && "InstructionsState is invalid.");
|
||||
return HasCopyables;
|
||||
}
|
||||
|
||||
/// Returns the index of the operand the copyable value is modeled in.
|
||||
unsigned getCopyableOpIdx() const {
|
||||
assert(valid() && "InstructionsState is invalid.");
|
||||
return CopyableOpIdx;
|
||||
}
|
||||
|
||||
/// Sets the index of the operand the copyable value is modeled in.
|
||||
void setCopyableOpIdx(unsigned Idx) {
|
||||
assert((Idx == 0 || Idx == 2) && "Unexpected copyable operand index.");
|
||||
CopyableOpIdx = Idx;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace llvm::slpvectorizer
|
||||
|
|
|
|||
|
|
@ -8,18 +8,17 @@ define void @test_mul_copyable(ptr %dst, ptr %srcB, float %p, float %q, float %r
|
|||
; ENABLED-LABEL: define void @test_mul_copyable(
|
||||
; ENABLED-SAME: ptr [[DST:%.*]], ptr [[SRCB:%.*]], float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[T:%.*]], float [[U:%.*]], float [[V:%.*]], float [[W:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]]) {
|
||||
; ENABLED-NEXT: [[ENTRY:.*:]]
|
||||
; ENABLED-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0
|
||||
; ENABLED-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[R]], i64 1
|
||||
; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0
|
||||
; ENABLED-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1
|
||||
; ENABLED-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]]
|
||||
; ENABLED-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[T]], i64 0
|
||||
; ENABLED-NEXT: [[TMP6:%.*]] = insertelement <2 x float> [[TMP5]], float [[V]], i64 1
|
||||
; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <2 x float> poison, float [[U]], i64 0
|
||||
; ENABLED-NEXT: [[TMP8:%.*]] = insertelement <2 x float> [[TMP7]], float [[W]], i64 1
|
||||
; ENABLED-NEXT: [[TMP9:%.*]] = fmul <2 x float> [[TMP6]], [[TMP8]]
|
||||
; ENABLED-NEXT: [[TMP10:%.*]] = fdiv <2 x float> [[TMP6]], [[TMP8]]
|
||||
; ENABLED-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
|
||||
; ENABLED-NEXT: [[TMP0:%.*]] = insertelement <4 x float> poison, float [[P]], i64 0
|
||||
; ENABLED-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[R]], i64 1
|
||||
; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[T]], i64 2
|
||||
; ENABLED-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[V]], i64 3
|
||||
; ENABLED-NEXT: [[TMP4:%.*]] = insertelement <4 x float> poison, float [[Q]], i64 0
|
||||
; ENABLED-NEXT: [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[S]], i64 1
|
||||
; ENABLED-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[U]], i64 2
|
||||
; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[W]], i64 3
|
||||
; ENABLED-NEXT: [[TMP8:%.*]] = fmul <4 x float> [[TMP3]], [[TMP7]]
|
||||
; ENABLED-NEXT: [[TMP9:%.*]] = fdiv <4 x float> [[TMP3]], [[TMP7]]
|
||||
; ENABLED-NEXT: [[TMP10:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
|
||||
; ENABLED-NEXT: [[TMP12:%.*]] = insertelement <2 x float> poison, float [[E]], i64 0
|
||||
; ENABLED-NEXT: [[TMP13:%.*]] = insertelement <2 x float> [[TMP12]], float [[G]], i64 1
|
||||
; ENABLED-NEXT: [[TMP14:%.*]] = insertelement <2 x float> poison, float [[F]], i64 0
|
||||
|
|
@ -28,10 +27,12 @@ define void @test_mul_copyable(ptr %dst, ptr %srcB, float %p, float %q, float %r
|
|||
; ENABLED-NEXT: [[TMP17:%.*]] = fsub <2 x float> [[TMP13]], [[TMP15]]
|
||||
; ENABLED-NEXT: [[TMP18:%.*]] = shufflevector <2 x float> [[TMP16]], <2 x float> [[TMP17]], <2 x i32> <i32 0, i32 3>
|
||||
; ENABLED-NEXT: [[TMP19:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
|
||||
; ENABLED-NEXT: [[TMP20:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP4]], <2 x float> [[TMP19]], <2 x float> [[TMP18]])
|
||||
; ENABLED-NEXT: store <2 x float> [[TMP20]], ptr [[DST]], align 4
|
||||
; ENABLED-NEXT: [[D2:%.*]] = getelementptr float, ptr [[DST]], i32 2
|
||||
; ENABLED-NEXT: store <2 x float> [[TMP11]], ptr [[D2]], align 4
|
||||
; ENABLED-NEXT: [[TMP24:%.*]] = shufflevector <2 x float> [[TMP19]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
|
||||
; ENABLED-NEXT: [[TMP20:%.*]] = shufflevector <4 x float> <float poison, float poison, float 1.000000e+00, float 1.000000e+00>, <4 x float> [[TMP24]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
|
||||
; ENABLED-NEXT: [[TMP21:%.*]] = shufflevector <2 x float> [[TMP18]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
|
||||
; ENABLED-NEXT: [[TMP22:%.*]] = shufflevector <4 x float> <float poison, float poison, float -0.000000e+00, float -0.000000e+00>, <4 x float> [[TMP21]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
|
||||
; ENABLED-NEXT: [[TMP23:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP10]], <4 x float> [[TMP20]], <4 x float> [[TMP22]])
|
||||
; ENABLED-NEXT: store <4 x float> [[TMP23]], ptr [[DST]], align 4
|
||||
; ENABLED-NEXT: ret void
|
||||
;
|
||||
; DISABLED-LABEL: define void @test_mul_copyable(
|
||||
|
|
@ -89,11 +90,8 @@ define void @test_mul_copyable_args(ptr %dst, ptr %srcB, float %p, float %q, flo
|
|||
; ENABLED-LABEL: define void @test_mul_copyable_args(
|
||||
; ENABLED-SAME: ptr [[DST:%.*]], ptr [[SRCB:%.*]], float [[P:%.*]], float [[Q:%.*]], float [[R:%.*]], float [[S:%.*]], float [[X:%.*]], float [[Y:%.*]], float [[E:%.*]], float [[F:%.*]], float [[G:%.*]], float [[H:%.*]]) {
|
||||
; ENABLED-NEXT: [[ENTRY:.*:]]
|
||||
; ENABLED-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[P]], i64 0
|
||||
; ENABLED-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[R]], i64 1
|
||||
; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[Q]], i64 0
|
||||
; ENABLED-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[S]], i64 1
|
||||
; ENABLED-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]]
|
||||
; ENABLED-NEXT: [[A0:%.*]] = fmul float [[P]], [[Q]]
|
||||
; ENABLED-NEXT: [[A1:%.*]] = fmul float [[R]], [[S]]
|
||||
; ENABLED-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[E]], i64 0
|
||||
; ENABLED-NEXT: [[TMP6:%.*]] = insertelement <2 x float> [[TMP5]], float [[G]], i64 1
|
||||
; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <2 x float> poison, float [[F]], i64 0
|
||||
|
|
@ -102,12 +100,16 @@ define void @test_mul_copyable_args(ptr %dst, ptr %srcB, float %p, float %q, flo
|
|||
; ENABLED-NEXT: [[TMP10:%.*]] = fsub <2 x float> [[TMP6]], [[TMP8]]
|
||||
; ENABLED-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
|
||||
; ENABLED-NEXT: [[TMP12:%.*]] = load <2 x float>, ptr [[SRCB]], align 4
|
||||
; ENABLED-NEXT: [[TMP13:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP4]], <2 x float> [[TMP12]], <2 x float> [[TMP11]])
|
||||
; ENABLED-NEXT: store <2 x float> [[TMP13]], ptr [[DST]], align 4
|
||||
; ENABLED-NEXT: [[D2:%.*]] = getelementptr float, ptr [[DST]], i32 2
|
||||
; ENABLED-NEXT: store float [[X]], ptr [[D2]], align 4
|
||||
; ENABLED-NEXT: [[D3:%.*]] = getelementptr float, ptr [[DST]], i32 3
|
||||
; ENABLED-NEXT: store float [[Y]], ptr [[D3]], align 4
|
||||
; ENABLED-NEXT: [[TMP17:%.*]] = insertelement <4 x float> poison, float [[X]], i64 2
|
||||
; ENABLED-NEXT: [[TMP18:%.*]] = insertelement <4 x float> [[TMP17]], float [[Y]], i64 3
|
||||
; ENABLED-NEXT: [[TMP19:%.*]] = insertelement <4 x float> [[TMP18]], float [[A0]], i64 0
|
||||
; ENABLED-NEXT: [[TMP20:%.*]] = insertelement <4 x float> [[TMP19]], float [[A1]], i64 1
|
||||
; ENABLED-NEXT: [[TMP21:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
|
||||
; ENABLED-NEXT: [[TMP13:%.*]] = shufflevector <4 x float> <float poison, float poison, float 1.000000e+00, float 1.000000e+00>, <4 x float> [[TMP21]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
|
||||
; ENABLED-NEXT: [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
|
||||
; ENABLED-NEXT: [[TMP15:%.*]] = shufflevector <4 x float> <float poison, float poison, float -0.000000e+00, float -0.000000e+00>, <4 x float> [[TMP14]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
|
||||
; ENABLED-NEXT: [[TMP16:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP20]], <4 x float> [[TMP13]], <4 x float> [[TMP15]])
|
||||
; ENABLED-NEXT: store <4 x float> [[TMP16]], ptr [[DST]], align 4
|
||||
; ENABLED-NEXT: ret void
|
||||
;
|
||||
; DISABLED-LABEL: define void @test_mul_copyable_args(
|
||||
|
|
|
|||
Loading…
Reference in a new issue