[IA][RISCV] Support gap mask for loads that are de-interleaved through intrinsics (#197062)
In the context of (de)interleaved loads and stores, a gap mask is a mask that effectively skips the entire component / field. Starting from #151612 , the InterleavedAccessPass gained support to recognize masks of this kind and pass it to the TLI hook. RISC-V originally only supported gap mask on fixed vectors, this patch adds support for recognizing gap masks on loads that are de-interleaved through the `llvm.vector.deinterleaveN` intrinsics, with both scalable vectors and fixed vectors.
This commit is contained in:
Min-Yih Hsu
GitHub
parent
131446ad2a
commit
c998fcb0d1
8 changed files with 237 additions and 61 deletions
|
|
@ -3342,8 +3342,11 @@ public:
|
|||
/// \p Load is the accompanying load instruction. Can be either a plain load
|
||||
/// instruction or a vp.load intrinsic.
|
||||
/// \p DI represents the deinterleaveN intrinsic.
|
||||
/// \p GapMask is a mask with zeros for components / fields that may not be
|
||||
/// accessed.
|
||||
virtual bool lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask,
|
||||
IntrinsicInst *DI) const {
|
||||
IntrinsicInst *DI,
|
||||
const APInt &GapMask) const {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -698,6 +698,7 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
|
|||
assert(Factor && "unexpected deinterleave intrinsic");
|
||||
|
||||
Value *Mask = nullptr;
|
||||
auto GapMask = APInt::getAllOnes(Factor);
|
||||
if (LI) {
|
||||
if (!LI->isSimple())
|
||||
return false;
|
||||
|
|
@ -711,24 +712,20 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
|
|||
return false;
|
||||
|
||||
// Check mask operand. Handle both all-true/false and interleaved mask.
|
||||
APInt GapMask(Factor, 0);
|
||||
std::tie(Mask, GapMask) =
|
||||
getMask(getMaskOperand(II), Factor, getDeinterleavedVectorType(DI));
|
||||
if (!Mask)
|
||||
return false;
|
||||
// We haven't supported gap mask if it's deinterleaving using intrinsics.
|
||||
// Yet it is possible that we already changed the IR, hence returning true
|
||||
// here.
|
||||
if (GapMask.popcount() != Factor)
|
||||
return true;
|
||||
|
||||
LLVM_DEBUG(dbgs() << "IA: Found a vp.load or masked.load with deinterleave"
|
||||
<< " intrinsic " << *DI << " and factor = "
|
||||
<< Factor << "\n");
|
||||
LLVM_DEBUG(dbgs() << "IA: With nominal factor " << Factor
|
||||
<< " and actual factor " << GapMask.popcount() << "\n");
|
||||
}
|
||||
|
||||
// Try and match this with target specific intrinsics.
|
||||
if (!TLI->lowerDeinterleaveIntrinsicToLoad(LoadedVal, Mask, DI))
|
||||
if (!TLI->lowerDeinterleaveIntrinsicToLoad(LoadedVal, Mask, DI, GapMask))
|
||||
return false;
|
||||
|
||||
DeadInsts.insert(DI);
|
||||
|
|
|
|||
|
|
@ -18906,8 +18906,10 @@ bool AArch64TargetLowering::lowerInterleavedStore(Instruction *Store,
|
|||
}
|
||||
|
||||
bool AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(
|
||||
Instruction *Load, Value *Mask, IntrinsicInst *DI) const {
|
||||
Instruction *Load, Value *Mask, IntrinsicInst *DI,
|
||||
const APInt &GapMask) const {
|
||||
const unsigned Factor = getDeinterleaveIntrinsicFactor(DI->getIntrinsicID());
|
||||
assert(GapMask.getBitWidth() == Factor);
|
||||
if (Factor != 2 && Factor != 3 && Factor != 4) {
|
||||
LLVM_DEBUG(dbgs() << "Matching ld2, ld3 and ld4 patterns failed\n");
|
||||
return false;
|
||||
|
|
@ -18917,6 +18919,10 @@ bool AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(
|
|||
return false;
|
||||
assert(!Mask && "Unexpected mask on a load\n");
|
||||
|
||||
// Gap mask is currently not supported.
|
||||
if (!GapMask.isAllOnes())
|
||||
return false;
|
||||
|
||||
VectorType *VTy = getDeinterleavedVectorType(DI);
|
||||
|
||||
const DataLayout &DL = LI->getModule()->getDataLayout();
|
||||
|
|
|
|||
|
|
@ -232,7 +232,8 @@ public:
|
|||
const APInt &GapMask) const override;
|
||||
|
||||
bool lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask,
|
||||
IntrinsicInst *DI) const override;
|
||||
IntrinsicInst *DI,
|
||||
const APInt &GapMask) const override;
|
||||
|
||||
bool lowerInterleaveIntrinsicToStore(
|
||||
Instruction *Store, Value *Mask,
|
||||
|
|
|
|||
|
|
@ -438,7 +438,8 @@ public:
|
|||
const APInt &GapMask) const override;
|
||||
|
||||
bool lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask,
|
||||
IntrinsicInst *DI) const override;
|
||||
IntrinsicInst *DI,
|
||||
const APInt &GapMask) const override;
|
||||
|
||||
bool lowerInterleaveIntrinsicToStore(
|
||||
Instruction *Store, Value *Mask,
|
||||
|
|
|
|||
|
|
@ -69,6 +69,12 @@ static const Intrinsic::ID FixedVlssegIntrIds[] = {
|
|||
Intrinsic::riscv_sseg6_load_mask, Intrinsic::riscv_sseg7_load_mask,
|
||||
Intrinsic::riscv_sseg8_load_mask};
|
||||
|
||||
static const Intrinsic::ID ScalableVlssegIntrIds[] = {
|
||||
Intrinsic::riscv_vlsseg2_mask, Intrinsic::riscv_vlsseg3_mask,
|
||||
Intrinsic::riscv_vlsseg4_mask, Intrinsic::riscv_vlsseg5_mask,
|
||||
Intrinsic::riscv_vlsseg6_mask, Intrinsic::riscv_vlsseg7_mask,
|
||||
Intrinsic::riscv_vlsseg8_mask};
|
||||
|
||||
static const Intrinsic::ID ScalableVlsegIntrIds[] = {
|
||||
Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
|
||||
Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
|
||||
|
|
@ -349,15 +355,25 @@ bool RISCVTargetLowering::lowerInterleavedStore(Instruction *Store,
|
|||
}
|
||||
|
||||
bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
|
||||
Instruction *Load, Value *Mask, IntrinsicInst *DI) const {
|
||||
Instruction *Load, Value *Mask, IntrinsicInst *DI,
|
||||
const APInt &GapMask) const {
|
||||
const unsigned Factor = getDeinterleaveIntrinsicFactor(DI->getIntrinsicID());
|
||||
assert(GapMask.getBitWidth() == Factor);
|
||||
if (Factor > 8)
|
||||
return false;
|
||||
|
||||
// We only support cases where the skipped fields are the trailing ones.
|
||||
if (!GapMask.isMask())
|
||||
return false;
|
||||
IRBuilder<> Builder(Load);
|
||||
|
||||
VectorType *ResVTy = getDeinterleavedVectorType(DI);
|
||||
|
||||
unsigned MaskFactor = GapMask.getActiveBits();
|
||||
// For MaskFactor of 1, we still want to lower it with segmented load
|
||||
// (of the original Factor), because the sole field extraction will eventually
|
||||
// turn it into a strided load.
|
||||
bool UseStridedSeg = MaskFactor < Factor && MaskFactor > 1;
|
||||
const DataLayout &DL = Load->getDataLayout();
|
||||
auto *XLenTy = Builder.getIntNTy(Subtarget.getXLen());
|
||||
|
||||
|
|
@ -371,22 +387,53 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
|
|||
if (!isLegalInterleavedAccessType(ResVTy, Factor, Alignment, AS, DL))
|
||||
return false;
|
||||
|
||||
unsigned ElementSizeInBytes = DL.getTypeStoreSize(ResVTy->getElementType());
|
||||
Value *Return;
|
||||
if (isa<FixedVectorType>(ResVTy)) {
|
||||
Return = Builder.CreateIntrinsic(FixedVlsegIntrIds[Factor - 2],
|
||||
{ResVTy, PtrTy, XLenTy}, {Ptr, Mask, VL});
|
||||
Value *SegLoad;
|
||||
if (UseStridedSeg) {
|
||||
// Lower to strided segmented load.
|
||||
Value *Stride = ConstantInt::get(XLenTy, Factor * ElementSizeInBytes);
|
||||
SegLoad = Builder.CreateIntrinsic(FixedVlssegIntrIds[MaskFactor - 2],
|
||||
{ResVTy, PtrTy, XLenTy, XLenTy},
|
||||
{Ptr, Stride, Mask, VL});
|
||||
} else {
|
||||
SegLoad =
|
||||
Builder.CreateIntrinsic(FixedVlsegIntrIds[Factor - 2],
|
||||
{ResVTy, PtrTy, XLenTy}, {Ptr, Mask, VL});
|
||||
}
|
||||
|
||||
if (MaskFactor != Factor) {
|
||||
// Replace masked-off factors with poisons.
|
||||
SmallVector<Type *, 8> AggrTypes{Factor, ResVTy};
|
||||
Return = PoisonValue::get(StructType::get(Load->getContext(), AggrTypes));
|
||||
for (unsigned I = 0; I < MaskFactor; ++I) {
|
||||
Value *SubVec = Builder.CreateExtractValue(SegLoad, I);
|
||||
Return = Builder.CreateInsertValue(Return, SubVec, I);
|
||||
}
|
||||
} else {
|
||||
Return = SegLoad;
|
||||
}
|
||||
} else {
|
||||
unsigned SEW = DL.getTypeSizeInBits(ResVTy->getElementType());
|
||||
unsigned NumElts = ResVTy->getElementCount().getKnownMinValue();
|
||||
Type *VecTupTy = TargetExtType::get(
|
||||
Load->getContext(), "riscv.vector.tuple",
|
||||
ScalableVectorType::get(Builder.getInt8Ty(), NumElts * SEW / 8),
|
||||
Factor);
|
||||
Function *VlsegNFunc = Intrinsic::getOrInsertDeclaration(
|
||||
Load->getModule(), ScalableVlsegIntrIds[Factor - 2],
|
||||
{VecTupTy, PtrTy, Mask->getType(), VL->getType()});
|
||||
UseStridedSeg ? MaskFactor : Factor);
|
||||
Function *SegLoadFunc;
|
||||
if (UseStridedSeg) {
|
||||
// Lower to strided segmented load.
|
||||
SegLoadFunc = Intrinsic::getOrInsertDeclaration(
|
||||
Load->getModule(), ScalableVlssegIntrIds[MaskFactor - 2],
|
||||
{VecTupTy, PtrTy, XLenTy, Mask->getType()});
|
||||
} else {
|
||||
SegLoadFunc = Intrinsic::getOrInsertDeclaration(
|
||||
Load->getModule(), ScalableVlsegIntrIds[Factor - 2],
|
||||
{VecTupTy, PtrTy, Mask->getType(), VL->getType()});
|
||||
}
|
||||
|
||||
Value *Operands[] = {
|
||||
SmallVector<Value *, 8> Operands = {
|
||||
PoisonValue::get(VecTupTy),
|
||||
Ptr,
|
||||
Mask,
|
||||
|
|
@ -394,12 +441,16 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
|
|||
ConstantInt::get(XLenTy,
|
||||
RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC),
|
||||
ConstantInt::get(XLenTy, Log2_64(SEW))};
|
||||
if (UseStridedSeg) {
|
||||
Value *Stride = ConstantInt::get(XLenTy, Factor * ElementSizeInBytes);
|
||||
Operands.insert(std::next(Operands.begin(), 2), Stride);
|
||||
}
|
||||
|
||||
CallInst *Vlseg = Builder.CreateCall(VlsegNFunc, Operands);
|
||||
CallInst *Vlseg = Builder.CreateCall(SegLoadFunc, Operands);
|
||||
|
||||
SmallVector<Type *, 2> AggrTypes{Factor, ResVTy};
|
||||
SmallVector<Type *, 8> AggrTypes{Factor, ResVTy};
|
||||
Return = PoisonValue::get(StructType::get(Load->getContext(), AggrTypes));
|
||||
for (unsigned i = 0; i < Factor; ++i) {
|
||||
for (unsigned i = 0; i < MaskFactor; ++i) {
|
||||
Value *VecExtract = Builder.CreateIntrinsic(
|
||||
Intrinsic::riscv_tuple_extract, {ResVTy, VecTupTy},
|
||||
{Vlseg, Builder.getInt32(i)});
|
||||
|
|
|
|||
|
|
@ -431,6 +431,45 @@ define {<2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>} @vpload_factor4_intrinsics(p
|
|||
ret { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res3
|
||||
}
|
||||
|
||||
; mask = 1010, skip the last two fields
|
||||
define {<4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>} @gap_mask_vpload_factor4_intrinsics(ptr %ptr) {
|
||||
; CHECK-LABEL: gap_mask_vpload_factor4_intrinsics:
|
||||
; CHECK: # %bb.0:
|
||||
; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
|
||||
; CHECK-NEXT: vmv.v.i v0, 5
|
||||
; CHECK-NEXT: li a1, 16
|
||||
; CHECK-NEXT: vlsseg2e32.v v8, (a0), a1, v0.t
|
||||
; CHECK-NEXT: ret
|
||||
%wide.masked.load = call <16 x i32> @llvm.vp.load(ptr %ptr, <16 x i1> <i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0>, i32 16)
|
||||
%d = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4(<16 x i32> %wide.masked.load)
|
||||
%t0 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %d, 0
|
||||
%t1 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %d, 1
|
||||
|
||||
%res0 = insertvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } poison, <4 x i32> %t0, 0
|
||||
%res1 = insertvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %res0, <4 x i32> %t1, 1
|
||||
ret { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %res1
|
||||
}
|
||||
|
||||
; mask = 1010, skip the last three fields. We should not apply the gap-mask optimization here but
|
||||
; we can extract only the first field so that it can be turned into a strided load.
|
||||
define {<4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>} @gap_mask_single_field_vpload_factor4_intrinsics(ptr %ptr) {
|
||||
; CHECK-LABEL: gap_mask_single_field_vpload_factor4_intrinsics:
|
||||
; CHECK: # %bb.0:
|
||||
; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
|
||||
; CHECK-NEXT: vmv.v.i v0, 5
|
||||
; CHECK-NEXT: li a1, 16
|
||||
; CHECK-NEXT: vlse32.v v8, (a0), a1, v0.t
|
||||
; CHECK-NEXT: ret
|
||||
%wide.masked.load = call <16 x i32> @llvm.vp.load(ptr %ptr, <16 x i1> <i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0>, i32 16)
|
||||
%d = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.vector.deinterleave4(<16 x i32> %wide.masked.load)
|
||||
%t0 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %d, 0
|
||||
%t1 = extractvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %d, 1
|
||||
|
||||
%res0 = insertvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } poison, <4 x i32> %t0, 0
|
||||
%res1 = insertvalue { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %res0, <4 x i32> %t1, 1
|
||||
ret { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } %res1
|
||||
}
|
||||
|
||||
define {<4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>} @vpload_factor5(ptr %ptr) {
|
||||
; CHECK-LABEL: vpload_factor5:
|
||||
; CHECK: # %bb.0:
|
||||
|
|
@ -615,8 +654,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
|
|||
; RV32-NEXT: add a1, sp, a1
|
||||
; RV32-NEXT: addi a1, a1, 16
|
||||
; RV32-NEXT: vs4r.v v28, (a1) # vscale x 32-byte Folded Spill
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI27_0)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI27_0)
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI29_0)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI29_0)
|
||||
; RV32-NEXT: lui a6, 49164
|
||||
; RV32-NEXT: lui t1, 3
|
||||
; RV32-NEXT: lui t0, 196656
|
||||
|
|
@ -777,8 +816,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
|
|||
; RV32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
|
||||
; RV32-NEXT: vsetvli zero, zero, e64, m8, ta, ma
|
||||
; RV32-NEXT: vrgatherei16.vv v24, v8, v2
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI27_1)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI27_1)
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI29_1)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI29_1)
|
||||
; RV32-NEXT: lui a3, 3073
|
||||
; RV32-NEXT: addi a3, a3, -1024
|
||||
; RV32-NEXT: vmv.s.x v0, a3
|
||||
|
|
@ -803,8 +842,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
|
|||
; RV32-NEXT: add a1, sp, a1
|
||||
; RV32-NEXT: addi a1, a1, 16
|
||||
; RV32-NEXT: vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI27_3)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI27_3)
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI29_3)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI29_3)
|
||||
; RV32-NEXT: vsetivli zero, 16, e64, m8, ta, ma
|
||||
; RV32-NEXT: vle16.v v28, (a1)
|
||||
; RV32-NEXT: csrr a1, vlenb
|
||||
|
|
@ -814,8 +853,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
|
|||
; RV32-NEXT: addi a1, a1, 16
|
||||
; RV32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
|
||||
; RV32-NEXT: vrgatherei16.vv v16, v8, v30
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI27_2)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI27_2)
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI29_2)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI29_2)
|
||||
; RV32-NEXT: vsetivli zero, 8, e16, m1, ta, ma
|
||||
; RV32-NEXT: vle16.v v20, (a1)
|
||||
; RV32-NEXT: csrr a1, vlenb
|
||||
|
|
@ -863,16 +902,16 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
|
|||
; RV32-NEXT: vrgatherei16.vv v24, v12, v20
|
||||
; RV32-NEXT: vsetivli zero, 10, e32, m4, tu, ma
|
||||
; RV32-NEXT: vmv.v.v v24, v8
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI27_4)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI27_4)
|
||||
; RV32-NEXT: lui a2, %hi(.LCPI27_5)
|
||||
; RV32-NEXT: addi a2, a2, %lo(.LCPI27_5)
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI29_4)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI29_4)
|
||||
; RV32-NEXT: lui a2, %hi(.LCPI29_5)
|
||||
; RV32-NEXT: addi a2, a2, %lo(.LCPI29_5)
|
||||
; RV32-NEXT: vsetivli zero, 16, e16, m2, ta, ma
|
||||
; RV32-NEXT: vle16.v v28, (a2)
|
||||
; RV32-NEXT: vsetivli zero, 8, e16, m1, ta, ma
|
||||
; RV32-NEXT: vle16.v v1, (a1)
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI27_7)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI27_7)
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI29_7)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI29_7)
|
||||
; RV32-NEXT: vsetivli zero, 16, e64, m8, ta, ma
|
||||
; RV32-NEXT: vle16.v v2, (a1)
|
||||
; RV32-NEXT: csrr a1, vlenb
|
||||
|
|
@ -895,14 +934,14 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
|
|||
; RV32-NEXT: vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
|
||||
; RV32-NEXT: vsetivli zero, 16, e64, m8, ta, ma
|
||||
; RV32-NEXT: vrgatherei16.vv v16, v8, v2
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI27_6)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI27_6)
|
||||
; RV32-NEXT: lui a2, %hi(.LCPI27_8)
|
||||
; RV32-NEXT: addi a2, a2, %lo(.LCPI27_8)
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI29_6)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI29_6)
|
||||
; RV32-NEXT: lui a2, %hi(.LCPI29_8)
|
||||
; RV32-NEXT: addi a2, a2, %lo(.LCPI29_8)
|
||||
; RV32-NEXT: vsetivli zero, 8, e16, m1, ta, ma
|
||||
; RV32-NEXT: vle16.v v8, (a1)
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI27_9)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI27_9)
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI29_9)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI29_9)
|
||||
; RV32-NEXT: vsetivli zero, 16, e16, m2, ta, ma
|
||||
; RV32-NEXT: vle16.v v10, (a1)
|
||||
; RV32-NEXT: csrr a1, vlenb
|
||||
|
|
@ -1210,8 +1249,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
|
|||
; RV64-NEXT: add a2, sp, a2
|
||||
; RV64-NEXT: addi a2, a2, 16
|
||||
; RV64-NEXT: vs4r.v v12, (a2) # vscale x 32-byte Folded Spill
|
||||
; RV64-NEXT: lui a2, %hi(.LCPI27_0)
|
||||
; RV64-NEXT: addi a2, a2, %lo(.LCPI27_0)
|
||||
; RV64-NEXT: lui a2, %hi(.LCPI29_0)
|
||||
; RV64-NEXT: addi a2, a2, %lo(.LCPI29_0)
|
||||
; RV64-NEXT: li a3, 1040
|
||||
; RV64-NEXT: vmv.s.x v0, a3
|
||||
; RV64-NEXT: addi a1, a1, -2016
|
||||
|
|
@ -1252,8 +1291,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
|
|||
; RV64-NEXT: add a1, sp, a1
|
||||
; RV64-NEXT: addi a1, a1, 16
|
||||
; RV64-NEXT: vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
|
||||
; RV64-NEXT: lui a1, %hi(.LCPI27_1)
|
||||
; RV64-NEXT: addi a1, a1, %lo(.LCPI27_1)
|
||||
; RV64-NEXT: lui a1, %hi(.LCPI29_1)
|
||||
; RV64-NEXT: addi a1, a1, %lo(.LCPI29_1)
|
||||
; RV64-NEXT: vle16.v v24, (a1)
|
||||
; RV64-NEXT: csrr a1, vlenb
|
||||
; RV64-NEXT: li a2, 37
|
||||
|
|
@ -1268,8 +1307,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
|
|||
; RV64-NEXT: add a1, sp, a1
|
||||
; RV64-NEXT: addi a1, a1, 16
|
||||
; RV64-NEXT: vs8r.v v0, (a1) # vscale x 64-byte Folded Spill
|
||||
; RV64-NEXT: lui a1, %hi(.LCPI27_2)
|
||||
; RV64-NEXT: addi a1, a1, %lo(.LCPI27_2)
|
||||
; RV64-NEXT: lui a1, %hi(.LCPI29_2)
|
||||
; RV64-NEXT: addi a1, a1, %lo(.LCPI29_2)
|
||||
; RV64-NEXT: vle16.v v12, (a1)
|
||||
; RV64-NEXT: csrr a1, vlenb
|
||||
; RV64-NEXT: li a2, 21
|
||||
|
|
@ -1335,12 +1374,12 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
|
|||
; RV64-NEXT: add a1, sp, a1
|
||||
; RV64-NEXT: addi a1, a1, 16
|
||||
; RV64-NEXT: vs4r.v v8, (a1) # vscale x 32-byte Folded Spill
|
||||
; RV64-NEXT: lui a1, %hi(.LCPI27_3)
|
||||
; RV64-NEXT: addi a1, a1, %lo(.LCPI27_3)
|
||||
; RV64-NEXT: lui a1, %hi(.LCPI29_3)
|
||||
; RV64-NEXT: addi a1, a1, %lo(.LCPI29_3)
|
||||
; RV64-NEXT: vsetivli zero, 16, e16, m2, ta, ma
|
||||
; RV64-NEXT: vle16.v v8, (a1)
|
||||
; RV64-NEXT: lui a1, %hi(.LCPI27_4)
|
||||
; RV64-NEXT: addi a1, a1, %lo(.LCPI27_4)
|
||||
; RV64-NEXT: lui a1, %hi(.LCPI29_4)
|
||||
; RV64-NEXT: addi a1, a1, %lo(.LCPI29_4)
|
||||
; RV64-NEXT: vle16.v v10, (a1)
|
||||
; RV64-NEXT: csrr a1, vlenb
|
||||
; RV64-NEXT: li a2, 77
|
||||
|
|
@ -1388,8 +1427,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
|
|||
; RV64-NEXT: vl2r.v v16, (a1) # vscale x 16-byte Folded Reload
|
||||
; RV64-NEXT: vsetivli zero, 16, e64, m8, ta, ma
|
||||
; RV64-NEXT: vrgatherei16.vv v0, v8, v16
|
||||
; RV64-NEXT: lui a1, %hi(.LCPI27_5)
|
||||
; RV64-NEXT: addi a1, a1, %lo(.LCPI27_5)
|
||||
; RV64-NEXT: lui a1, %hi(.LCPI29_5)
|
||||
; RV64-NEXT: addi a1, a1, %lo(.LCPI29_5)
|
||||
; RV64-NEXT: vle16.v v12, (a1)
|
||||
; RV64-NEXT: csrr a1, vlenb
|
||||
; RV64-NEXT: li a2, 61
|
||||
|
|
@ -2037,8 +2076,8 @@ define {<4 x i32>, <4 x i32>, <4 x i32>} @invalid_vp_mask(ptr %ptr) {
|
|||
; RV32-NEXT: vle32.v v12, (a0), v0.t
|
||||
; RV32-NEXT: li a0, 36
|
||||
; RV32-NEXT: vmv.s.x v20, a1
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI65_0)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI65_0)
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI67_0)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI67_0)
|
||||
; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma
|
||||
; RV32-NEXT: vle16.v v21, (a1)
|
||||
; RV32-NEXT: vcompress.vm v8, v12, v11
|
||||
|
|
@ -2113,8 +2152,8 @@ define {<4 x i32>, <4 x i32>, <4 x i32>} @invalid_vp_evl(ptr %ptr) {
|
|||
; RV32-NEXT: vmv.s.x v10, a0
|
||||
; RV32-NEXT: li a0, 146
|
||||
; RV32-NEXT: vmv.s.x v11, a0
|
||||
; RV32-NEXT: lui a0, %hi(.LCPI66_0)
|
||||
; RV32-NEXT: addi a0, a0, %lo(.LCPI66_0)
|
||||
; RV32-NEXT: lui a0, %hi(.LCPI68_0)
|
||||
; RV32-NEXT: addi a0, a0, %lo(.LCPI68_0)
|
||||
; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma
|
||||
; RV32-NEXT: vle16.v v20, (a0)
|
||||
; RV32-NEXT: li a0, 36
|
||||
|
|
@ -2380,8 +2419,8 @@ define {<4 x i32>, <4 x i32>, <4 x i32>} @maskedload_factor3_invalid_skip_field(
|
|||
; RV32-NEXT: vle32.v v12, (a0), v0.t
|
||||
; RV32-NEXT: li a0, 36
|
||||
; RV32-NEXT: vmv.s.x v20, a1
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI78_0)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI78_0)
|
||||
; RV32-NEXT: lui a1, %hi(.LCPI80_0)
|
||||
; RV32-NEXT: addi a1, a1, %lo(.LCPI80_0)
|
||||
; RV32-NEXT: vsetivli zero, 8, e32, m2, ta, ma
|
||||
; RV32-NEXT: vle16.v v21, (a1)
|
||||
; RV32-NEXT: vcompress.vm v8, v12, v11
|
||||
|
|
|
|||
|
|
@ -494,6 +494,85 @@ define <vscale x 2 x i32> @load_factor5_oneactive(ptr %ptr, i32 zeroext %evl) {
|
|||
ret <vscale x 2 x i32> %t3
|
||||
}
|
||||
|
||||
; mask = all ones, skip the last 2 fields
|
||||
define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>} @gap_mask_load_factor4(ptr %ptr, i32 zeroext %evl) {
|
||||
; CHECK-LABEL: gap_mask_load_factor4:
|
||||
; CHECK: # %bb.0:
|
||||
; CHECK-NEXT: li a2, 16
|
||||
; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
|
||||
; CHECK-NEXT: vlsseg2e32.v v8, (a0), a2
|
||||
; CHECK-NEXT: ret
|
||||
%rvl = mul nuw i32 %evl, 4
|
||||
%gap.mask = call <vscale x 8 x i1> @llvm.vector.interleave4(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> splat (i1 false), <vscale x 2 x i1> splat (i1 false))
|
||||
%combined.mask = and <vscale x 8 x i1> %gap.mask, splat (i1 true)
|
||||
|
||||
%wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> %combined.mask, i32 %rvl)
|
||||
%d = call { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave4.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
|
||||
%t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
|
||||
%t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
|
||||
%t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 2
|
||||
%t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 3
|
||||
|
||||
%res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
|
||||
%res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
|
||||
%res2 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res1, <vscale x 2 x i32> %t2, 2
|
||||
%res3 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res2, <vscale x 2 x i32> %t3, 3
|
||||
ret { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res3
|
||||
}
|
||||
|
||||
; mask = %m, skip the last field
|
||||
define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>} @gap_mask_load_factor4_mask(ptr %ptr, <vscale x 2 x i1> %m, i32 zeroext %evl) {
|
||||
; CHECK-LABEL: gap_mask_load_factor4_mask:
|
||||
; CHECK: # %bb.0:
|
||||
; CHECK-NEXT: li a2, 16
|
||||
; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
|
||||
; CHECK-NEXT: vlsseg3e32.v v8, (a0), a2, v0.t
|
||||
; CHECK-NEXT: ret
|
||||
%rvl = mul nuw i32 %evl, 4
|
||||
%actual.mask = call <vscale x 8 x i1> @llvm.vector.interleave4(<vscale x 2 x i1> %m, <vscale x 2 x i1> %m, <vscale x 2 x i1> %m, <vscale x 2 x i1> %m)
|
||||
%gap.mask = call <vscale x 8 x i1> @llvm.vector.interleave4(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> splat (i1 false))
|
||||
%combined.mask = and <vscale x 8 x i1> %gap.mask, %actual.mask
|
||||
|
||||
%wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> %combined.mask, i32 %rvl)
|
||||
%d = call { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave4.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
|
||||
%t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
|
||||
%t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
|
||||
%t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 2
|
||||
%t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 3
|
||||
|
||||
%res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
|
||||
%res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
|
||||
%res2 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res1, <vscale x 2 x i32> %t2, 2
|
||||
%res3 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res2, <vscale x 2 x i32> %t3, 3
|
||||
ret { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res3
|
||||
}
|
||||
|
||||
; mask = %m, skip the last 3 fields. We should not apply the gap-mask optimization here but we can extract only the first field so that it can be turned into a strided load.
|
||||
define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>} @gap_mask_single_field_load_factor4_mask(ptr %ptr, <vscale x 2 x i1> %m, i32 zeroext %evl) {
|
||||
; CHECK-LABEL: gap_mask_single_field_load_factor4_mask:
|
||||
; CHECK: # %bb.0:
|
||||
; CHECK-NEXT: li a2, 16
|
||||
; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
|
||||
; CHECK-NEXT: vlse32.v v8, (a0), a2, v0.t
|
||||
; CHECK-NEXT: ret
|
||||
%rvl = mul nuw i32 %evl, 4
|
||||
%actual.mask = call <vscale x 8 x i1> @llvm.vector.interleave4(<vscale x 2 x i1> %m, <vscale x 2 x i1> %m, <vscale x 2 x i1> %m, <vscale x 2 x i1> %m)
|
||||
%gap.mask = call <vscale x 8 x i1> @llvm.vector.interleave4(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i1> splat (i1 false), <vscale x 2 x i1> splat (i1 false), <vscale x 2 x i1> splat (i1 false))
|
||||
%combined.mask = and <vscale x 8 x i1> %gap.mask, %actual.mask
|
||||
|
||||
%wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> %combined.mask, i32 %rvl)
|
||||
%d = call { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave4.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
|
||||
%t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
|
||||
%t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
|
||||
%t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 2
|
||||
%t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 3
|
||||
|
||||
%res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
|
||||
%res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
|
||||
%res2 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res1, <vscale x 2 x i32> %t2, 2
|
||||
%res3 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res2, <vscale x 2 x i32> %t3, 3
|
||||
ret { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res3
|
||||
}
|
||||
|
||||
; Negative tests
|
||||
|
||||
|
|
@ -615,4 +694,3 @@ define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2
|
|||
%res3 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res2, <vscale x 2 x i32> %t3, 3
|
||||
ret { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res3
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue